docker-compose多服务器部署kafka集群

• Kafka 是一个开源的分布式事件流平台，依赖Zookeeper或者KRaft，本文基于Zookeeper。

服务器IP配置

本文使用三个服务器来做集群搭建，IP如下：

nodeName	IP
node1	10.10.210.96
node2	10.10.210.97
node3	10.10.210.98

部署zookeeper

• 工作目录为/home/zookeeper

node1配置

目录结构

- zookeeper
  - config
    - zoo.cfg
  - docker-compose.yml

zoo.cfg

dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
clientPort:2181
server.1=127.0.0.1:2888:3888
server.2=10.10.210.97:2888:3888
server.3=10.10.210.98:2888:3888

docker-compose.yml

version: '3'
services:
  zookeeper:
    image: zookeeper:3.7.0
    restart: always
    hostname: zookeeper-node-1
    container_name: zookeeper
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    - 8080:8080
    volumes:
    - ./data:/data
    - ./datalog:/datalog
    - ./config/zoo.cfg:/conf/zoo.cfg
    environment:
      ZOO_MY_ID: 1

node2配置

目录结构

- zookeeper
  - config
    - zoo.cfg
  - docker-compose.yml

zoo.cfg

dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
clientPort:2181
server.1=10.10.210.96:2888:3888
server.2=127.0.0.1:2888:3888
server.3=10.10.210.98:2888:3888

docker-compose.yml

version: '3'
services:
  zookeeper:
    image: zookeeper:3.7.0
    restart: always
    hostname: zookeeper-node-2
    container_name: zookeeper
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    - 8080:8080
    volumes:
    - ./data:/data
    - ./datalog:/datalog
    - ./config/zoo.cfg:/conf/zoo.cfg
    environment:
      ZOO_MY_ID: 2

node3配置

目录结构

- zookeeper
  - config
    - zoo.cfg
  - docker-compose.yml

zoo.cfg

dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
clientPort:2181
server.1=10.10.210.96:2888:3888
server.2=10.10.210.97:2888:3888
server.3=127.0.0.1:2888:3888

docker-compose.yml

version: '3'
services:
  zookeeper:
    image: zookeeper:3.7.0
    restart: always
    hostname: zookeeper-node-3
    container_name: zookeeper
    ports:
    - 2181:2181
    - 2888:2888
    - 3888:3888
    - 8080:8080
    volumes:
    - ./data:/data
    - ./datalog:/datalog
    - ./config/zoo.cfg:/conf/zoo.cfg
    environment:
      ZOO_MY_ID: 3

• 在对应服务器的/home/zookeeper执行 docker-compose up -d 启动三个Zookeeper服务,通过docker-compose logs -f观察启动日志
• ZOO_MY_ID 对应zookeeper的id,多台服务器需设置不同,对应zoo.cfg的server.1,其中.1 就是对应的ZOO_MY_ID
• zoo.cfg配置信息具体可参考 Zookeeper部署和管理指南

部署kafka

• 工作目录为/home/kafka

node1配置

目录结构

- kafka
  - docker-compose.yml
  - config/server.properties

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node-1
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data
    - ./config/server.properties:/opt/bitnami/kafka/config/server.properties

server.properties

broker.id=1
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.10.210.96:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/bitnami/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
max.partition.fetch.bytes=1048576
max.request.size=1048576
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
sasl.mechanism.inter.broker.protocol=

node2配置

目录结构

- kafka
  - docker-compose.yml
  - config/server.properties

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node-2
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data
    - ./config/server.properties:/opt/bitnami/kafka/config/server.properties

server.properties

broker.id=2
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.10.210.97:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/bitnami/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
max.partition.fetch.bytes=1048576
max.request.size=1048576
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
sasl.mechanism.inter.broker.protocol=

node3配置

目录结构

- kafka
  - docker-compose.yml
  - config/server.properties

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node-3
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data
    - ./config/server.properties:/opt/bitnami/kafka/config/server.properties

server.properties

broker.id=3
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://10.10.210.98:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/bitnami/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
max.partition.fetch.bytes=1048576
max.request.size=1048576
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
sasl.mechanism.inter.broker.protocol=

• 在对应服务器的/home/kafka执行 docker-compose up -d 启动三个Kafka服务,通过docker-compose logs -f观察启动日志
• server.properties配置信息具体可参考 Kafka Broker Configs

kafka测试使用

• 通过offset explorer测试连接kafka是否可用。

后记

• 如果想要简单配置的情况下，可以通过environment的方式启动kafka，参考如下：

docker-compose.yml

version: '3'
services:
  kafka:
    image: bitnami/kafka:3.0.0
    restart: always
    hostname: kafka-node
    container_name: kafka
    ports:
    - 9092:9092
    - 9999:9999
    environment:
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.10.210.96:9092
      - KAFKA_ADVERTISED_HOST_NAME=10.10.210.96
      - KAFKA_ADVERTISED_PORT=9092
      - KAFKA_ZOOKEEPER_CONNECT=10.10.210.96:2181,10.10.210.97:2181,10.10.210.98:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
      - JMX_PORT=9999 
    volumes:
    - ./logs:/opt/bitnami/kafka/logs
    - ./data:/bitnami/kafka/data