【docker-compose 跨节点部署 kafka-kraft SASL用户加密集群】全网最新！

一、概述

文本主要讲解使用Docker-compose在三个节点上部署Kafka3.5.1(现阶段最新版本)-kraft模式，加密使用了用户名密码加密的SASL_PLAINTEXT+PLAIN方式。SSL加密在我的docker-compose.yml文件基础上微调一下就好。所有的配置都通过环境变量注入，仅将加密文件进行了挂载，其他配置未挂载出容器。

二、硬件信息

前置需要做集群免密和时间同步操作。

三、前置配置

1. 生成JKS文件
   对于生成密钥，bitnami/kafka镜像官方介绍也给了kafka-generate-ssl.sh脚本用于生成JSK文件。这个脚本可以多次运行，第一次运行遇到提示“Do you need to generate a trust store and associated private key?”，选“y”，完成1和2环节；其他时候运行，选“n”，完成2环节。
   第一次运行成功后查看结果：

$ ls
truststore/    keystore/    kafka-generate-ssl.sh

$ ls truststore
ca-key    kafka.truststore.jks

$ ls keystore
kafka.keystore.jks
1
2
3
4
5
6
7
8

2. 将JKS文件放到需要挂载进去的目录
   我三个节点用的JKS文件是同一个JKS加密文件。

四、docker-compose配置文件

1. node1 配置文件

version: '3'
services:
  kafka-1:
    #环境变量的含义可以去dockerHub查看该镜像的介绍
    image: bitnami/kafka:3.5.1
    hostname: kafka-1
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      - KAFKA_CFG_PROCESS_ROLES=broker,controller #声明角色
      - BITNAMI_DEBUG=true #控制台打印日志
      - ALLOW_PLAINTEXT_LISTENER=no #生产环境选择no
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_NUM_PARTITIONS=6 #默认分区数
      - KAFKA_CFG_LISTENERS=INTERNAL://:9094,CLIENT://:9095,CONTROLLER://:9093, EXTERNAL://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-1:9094,CLIENT://:9095,EXTERNAL://node1:9092 #外部连入方式,暴露出去的端口需要指定宿主机,controller不用申明
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:SASL_PLAINTEXT,CLIENT:SASL_PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT #指定加密方式,我内部传输是明文,按需修改
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM= #不验证域名
      - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
      - KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN
      - KAFKA_CERTIFICATE_PASSWORD=AZ2023 
      - KAFKA_TLS_TYPE=JKS
      - KAFKA_CLIENT_USERS=az
      - KAFKA_CLIENT_PASSWORDS=AZ2023
      - KAFKA_INTER_BROKER_USER=az
      - KAFKA_INTER_BROKER_PASSWORD=AZ2023
      - KAFKA_CFG_NODE_ID=0
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@node1:9093,1@node2:9093,2@node3:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv #集群唯一id
    volumes:
      - "/etc/hosts:/etc/hosts"
      - "./kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro"
      - "./kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

2. node2配置

version: '3'
services:
  kafka-2:
    image: bitnami/kafka:3.5.1
    hostname: kafka-2
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - BITNAMI_DEBUG=false
      - ALLOW_PLAINTEXT_LISTENER=no
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=INTERNAL://:9094,CLIENT://:9095,CONTROLLER://:9093, EXTERNAL://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-1:9094,CLIENT://:9095,EXTERNAL://node2:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:SASL_PLAINTEXT,CLIENT:SASL_PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=
      - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
      - KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN
      - KAFKA_CERTIFICATE_PASSWORD=AZ2023
      - KAFKA_TLS_TYPE=JKS
      - KAFKA_CLIENT_USERS=az
      - KAFKA_CLIENT_PASSWORDS=AZ2023
      - KAFKA_INTER_BROKER_USER=az
      - KAFKA_INTER_BROKER_PASSWORD=AZ2023
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@node1:9093,1@node2:9093,2@node3:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
    volumes:
      - "/etc/hosts:/etc/hosts"
      - "./kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro"
      - "./kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro"
  #这里开启了一个kafka-ui组件等之后验证下集群状态
  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:master
    volumes:
      - /etc/hosts:/etc/hosts
    ports:
      - 9888:8080
    environment:
      DYNAMIC_CONFIG_ENABLED: true

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

3. node3配置

version: '3'
services:
  kafka-3:
    image: bitnami/kafka:3.5.1
    hostname: kafka-3
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - BITNAMI_DEBUG=false
      - ALLOW_PLAINTEXT_LISTENER=no
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=INTERNAL://:9094,CLIENT://:9095,CONTROLLER://:9093, EXTERNAL://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-1:9094,CLIENT://:9095,EXTERNAL://node3:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:SASL_PLAINTEXT,CLIENT:SASL_PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=
      - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
      - KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN
      - KAFKA_CERTIFICATE_PASSWORD=AZ2023
      - KAFKA_TLS_TYPE=JKS
      - KAFKA_CLIENT_USERS=az
      - KAFKA_CLIENT_PASSWORDS=AZ2023
      - KAFKA_INTER_BROKER_USER=az
      - KAFKA_INTER_BROKER_PASSWORD=AZ2023
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@node1:9093,1@node2:9093,2@node3:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
    volumes:
      - "/etc/hosts:/etc/hosts"
      - "./kafka/kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro"
      - "./kafka/kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

五、集群验证

1. 通过kafka-ui的可视化页面验证

相关参考文章:
https://zhuanlan.zhihu.com/p/586005021
https://hub.docker.com/r/bitnami/kafka