Kafka Docker Compose

kafka docker-compose

카프카를 클라우드 플랫폼에 설치를 하면 과금이 많이 됩니다. 또는 개발 버전을 테스트 시에 실제 물리적인 개발 서버에 테스트하기에는 부담스러울 경우가 많습니다. 이런 경우 docker-compose 로 zookeeper, broker, kafka-ui 등을 띄워서 동작을 확인하는 편입니다.

이번 문서에서는 카프카 브로커 클러스터를 두가지 버전으로 구성합니다. 첫번째 버전은 간소화된 버전으로 브로커 3기, 주키퍼, kafka-ui 를 띄운 버전이고 두번째 버전은 주키퍼, schema-registry, kafka connector, 카프카 브로커 3기, kafka-ui 이렇게 구성합니다.

첫번째 버전 : kafka broker (3EA), zookeeper, kafka-ui
두번째 버전 : schema-registry, kafka-connector, zookeeper, kafka broker(3EA), kafka-ui

첫번째 버전 : kafka broker (3EA), zookeeper, kafka-ui

간소화된 버전입니다. 설명은 생략하고 docker-compose 파일만 남겨둡니다.

docker-compose.yml

version: '3.8'
services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:7.2.6
    ports:
      - '32181:32181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_TICK_TIME: 2000
  kafka-1:
    image: confluentinc/cp-kafka:7.2.6
    ports:
      - '9092:9092'
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:32181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://localhost:9092
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_NUM_PARTITIONS: 3
  kafka-2:
    image: confluentinc/cp-kafka:7.2.6
    ports:
      - '9093:9093'
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:32181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29093,EXTERNAL://localhost:9093
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_NUM_PARTITIONS: 3
  kafka-3:
    image: confluentinc/cp-kafka:7.2.6
    ports:
      - '9094:9094'
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:32181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29094,EXTERNAL://localhost:9094
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_NUM_PARTITIONS: 3
  kafka-ui:
    image: provectuslabs/kafka-ui
    container_name: kafka-ui
    ports:
      - "9000:8080"
    restart: always
    environment:
      - KAFKA_CLUSTERS_0_NAME=local
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka-1:29092,kafka-2:29093,kafka-3:29094
      - KAFKA_CLUSTERS_0_ZOOKEEPER=zookeeper-1:22181

두번째 버전 : schema-registry, kafka-connector, zookeeper, kafka broker(3EA), kafka-ui

이번에도 자세한 설명은 생략하겠습니다.

.env

KAFKA_VERSION=7.0.1
GLOBAL_NETWORK=kafka-local-all-hello-world
DEBEZIUM_VERSION=2.1

docker-compose.yml

version: '3.7'
services:
  schema-registry:
    image: confluentinc/cp-schema-registry:${KAFKA_VERSION}
    container_name: kafka-schema-registry
    hostname: schema-registry
    depends_on:
      - kafka-broker-1
      - kafka-broker-2
      - kafka-broker-3
    ports:
      - "9397:9397"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
      SCHEMA_REGISTRY_LISTENERS: http://schema-registry:9397
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-broker-2:9092,LISTENER_LOCAL://kafka-broker-2:29092
      SCHEMA_REGISTRY_DEBUG: 'true'
    networks:
      - kafka-local-all
  kafka-connector:
    image: confluentinc/cp-kafka-connect:${KAFKA_VERSION}
    container_name: kafka-connector
    ports:
      - 9398:9398
    environment:
      CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-broker-2:9092,LISTENER_LOCAL://kafka-broker-2:29092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "quickstart-avro"
      CONNECT_CONFIG_STORAGE_TOPIC: "quickstart-avro-config"
      CONNECT_OFFSET_STORAGE_TOPIC: "quickstart-avro-offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "quickstart-avro-status"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connector"
      CONNECT_LOG4J_ROOT_LOGLEVEL: DEBUG
      CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars"
    depends_on:
      - kafka-broker-1
      - kafka-broker-2
      - kafka-broker-3
      - schema-registry
    volumes:
      - "./kafka-connect/connectors:/etc/kafka-connect/jars"
    networks:
      - kafka-local-all
  kafka-broker-1:
    image: confluentinc/cp-kafka:${KAFKA_VERSION}
    container_name: kafka-broker-1
    hostname: kafka-broker-1
    ports:
      - "19092:19092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker-1:9092,LISTENER_LOCAL://kafka-broker-1:19092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_COMPRESSION_TYPE: producer
    volumes:
      - "./volumes/kafka/broker-1:/var/lib/kafka/data"
    networks:
      - kafka-local-all
  kafka-broker-2:
    image: confluentinc/cp-kafka:${KAFKA_VERSION}
    container_name: kafka-broker-2
    hostname: kafka-broker-2
    ports:
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker-2:9092,LISTENER_LOCAL://kafka-broker-2:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_COMPRESSION_TYPE: producer
    volumes:
      - "./volumes/kafka/broker-2:/var/lib/kafka/data"
    networks:
      - kafka-local-all
  kafka-broker-3:
    image: confluentinc/cp-kafka:${KAFKA_VERSION}
    container_name: kafka-broker-3
    hostname: kafka-broker-3
    ports:
      - "39092:39092"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker-3:9092,LISTENER_LOCAL://kafka-broker-3:39092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_COMPRESSION_TYPE: producer
    volumes:
      - "./volumes/kafka/broker-3:/var/lib/kafka/data"
    networks:
      - kafka-local-all
  kafka-manager:
    image: hlebalbau/kafka-manager:stable
    restart: always
    ports:
      - "9090:9090"
    environment:
      ZK_HOSTS: "zookeeper:2181"
    networks:
      - kafka-local-all
networks:
  kafka-local-all:
    driver: bridge
#    external: true # network 가 이미 있을 경우에 삭제 안하고 할 경우에 사용

Kafka 의 주요 개념들 Spring Cloud Stream