kafka3.6.0集群部署

环境准备

• 机器环境

  系统	主机名	IP地址
  centos7.9	kafka01	192.168.200.51
  centos7.9	kafka02	192.168.200.52
  centos7.9	kafka03	192.168.200.53

• 所需软件

  jdk-8u171-linux-x64.tar.gz
  
  apache-zookeeper-3.8.3-bin.tar.gz
  https://dlcdn.apache.org/zookeeper/zookeeper-3.8.3/apache-zookeeper-3.8.3-bin.tar.gz
  
  kafka_2.13-3.6.0.tgz
  https://downloads.apache.org/kafka/3.6.0/kafka_2.13-3.6.0.tgz
  1
  2
  3
  4
  5
  6
  7

• hosts设置

  192.168.200.51 kafka01
  192.168.200.52 kafka02
  192.168.200.53 kafka03
  1
  2
  3

• java环境设置

  mkdir /data/java
  ln -s /data/java/ /opt/
  cd /opt/java
  tar -zxvf jdk-8u171-linux-x64.tar.gz -C /opt/java/
  1
  2
  3
  4

  cat /etc/profile.d/java.sh 
  
  #java
  export JAVA_HOME=/opt/java/jdk1.8.0_171
  export JRE_HOME=$JAVA_HOME/jre
  export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
  export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
  1
  2
  3
  4
  5
  6
  7

  source /etc/profile
  java -version
  1
  2

zookeeper安装部署

• 创建软件安装目录

  mkdir /data/server
  1

• 解压安装

  tar -zxvf apache-zookeeper-3.8.3-bin.tar.gz -C /data/server/
  ln -s /data/server/apache-zookeeper-3.8.3-bin/ /data/server/zookeeper
  
  # 创建zookeeper数据存储目录
  mkdir /data/server/zookeeper/zkData
  
  # 创建zookeeper标识文件
  echo '1' > /data/server/zookeeper/zkData/myid
  1
  2
  3
  4
  5
  6
  7
  8

• 修改配置

  cp /data/server/zookeeper/zoo_sample.cfg /data/server/zookeeper/zoo.cfg
  1

  vim /data/server/zookeeper/zoo.cfg
  # 修改数据存储目录
  dataDir=/data/server/zookeeper/zkData
  # 添加下列集群配置
  server.1=kafka01:2888:3888
  server.2=kafka02:2888:3888
  server.3=kafka03:2888:3888
  1
  2
  3
  4
  5
  6
  7

• 分发软件

  # 分发软件至kafka02
  scp -r /data/server/apache-zookeeper-3.8.3-bin/ kafka02:/data/server/
  
  # 分发软件至kafka03
  scp -r /data/server/apache-zookeeper-3.8.3-bin/ kafka03:/data/server/
  1
  2
  3
  4
  5

• kafka02与kafka03软链接

  ln -s /data/server/apache-zookeeper-3.8.3-bin/ /data/server/zookeeper
  1

• kafka02与kafka03修改myid

  # kafka02修改myid
  echo '2' > /data/server/zookeeper/zkData/myid
  
  # kafka03修改myid
  echo '3' > /data/server/zookeeper/zkData/myid
  1
  2
  3
  4
  5

• 防火墙放行端口

  firewall-cmd --zone=public --permanent --add-port={2181,2888,3888}/tcp
  firewall-cmd --reload
  1
  2

• 设置开机自启

  vim /etc/systemd/system/zookeeper.service
  
  [Unit]
  Description=zookeeper service
  After=network.target
   
  [Service]
  User=root
  Group=root 
  Type=forking
  Environment="JAVA_HOME=/opt/java/jdk1.8.0_171"
  ExecStart=/data/server/zookeeper/bin/zkServer.sh start
  ExecStop=/data/server/zookeeper/bin/zkServer.sh stop
  PrivateTmp=false
  Restart=always
   
  [Install]
  WantedBy=multi-user.target
  1
  2
  3
  4
  5
  6
  7
  8
  9
  10
  11
  12
  13
  14
  15
  16
  17
  18

  systemctl daemon-reload
  systemctl start zookeeper
  systemctl enable zookeeper
  1
  2
  3

• zookeeper状态查看

  • kafka01

    [root@kafka01 ~]# jps
    4049 Jps
    3899 QuorumPeerMain
    
    
    [root@kafka01 ~]# /data/server/zookeeper/bin/zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /data/server/zookeeper/bin/../conf/zoo.cfg
    Client port found: 2181. Client address: localhost. Client SSL: false.
    Mode: follower
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10

  • kafka02

    [root@kafka02 zookeeper]# jps
    2705 QuorumPeerMain
    2920 Jps
    
    
    [root@kafka02 zookeeper]# /data/server/zookeeper/bin/zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /data/server/zookeeper/bin/../conf/zoo.cfg
    Client port found: 2181. Client address: localhost. Client SSL: false.
    Mode: follower
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10

  • kafka03

    [root@kafka03 zookeeper]# jps
    3187 Jps
    3030 QuorumPeerMain
    
    
    [root@kafka03 ~]# /data/server/zookeeper/bin/zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /data/server/zookeeper/bin/../conf/zoo.cfg
    Client port found: 2181. Client address: localhost. Client SSL: false.
    Mode: leader
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10

kafka安装部署

• 解压安装

  tar -zxvf kafka_2.13-3.6.0.tgz -C /data/server/
  ln -s /data/server/kafka_2.13-3.6.0/ /data/server/kafka
  1
  2

• 修改配置

  vim /data/server/kafka/config/server.properties
  broker.id=0
  # 日志目录
  log.dirs=/data/server/kafka/kafka-logs
  
  # zookeeper连接地址及目录
  zookeeper.connect=kafka01:2181,kafka02:2181,kafka03:2181/kafka
  1
  2
  3
  4
  5
  6
  7

• 分发软件

  # 分发软件至kafka02
  scp -r /data/server/kafka_2.13-3.6.0/ kafka02:/data/server/
  
  分发软件至kafka03
  scp -r /data/server/kafka_2.13-3.6.0/ kafka03:/data/server/
  1
  2
  3
  4
  5

• kafka02与kafka03软链接

  ln -s /data/server/kafka_2.13-3.6.0/ /data/server/kafka
  1

• 修改kafka02和kafka03的broker.id

  vim /data/server/kafka/config/server.properties
  
  # kafka02
  broker.id=1
  
  # kafka03
  broker.id=2
  1
  2
  3
  4
  5
  6
  7

• 防火墙放行端口

  firewall-cmd --zone=public --permanent --add-port=9092/tcp
  firewall-cmd --reload
  1
  2

• 开机自启

  vim /etc/systemd/system/kafka.service
  1

  [Unit]
  Description=kafka service
  After=network.target
   
  [Service]
  User=root
  Group=root 
  Type=forking
  Environment="JAVA_HOME=/opt/java/jdk1.8.0_171"
  ExecStart=/data/server/kafka/bin/kafka-server-start.sh -daemon /data/server/kafka/config/server.properties
  ExecStop=/data/server/kafka/bin/kafka-server-stop.sh 
  PrivateTmp=false
  Restart=always
   
  [Install]
  WantedBy=multi-user.target
  1
  2
  3
  4
  5
  6
  7
  8
  9
  10
  11
  12
  13
  14
  15
  16

  systemctl daemon-reload
  systemctl start kafka
  systemctl enable kafka
  1
  2
  3

• 最终效果

  • kafka01

    [root@kafka01 ~]# jps
    5444 Kafka
    8661 Jps
    3899 QuorumPeerMain
    
    1
    2
    3
    4
    5

  • kafka02

    [root@kafka02 ~]# jps
    2705 QuorumPeerMain
    3538 Jps
    3373 Kafka
    1
    2
    3
    4

  • kafka03

    [root@kafka03 ~]# jps
    3668 Kafka
    3030 QuorumPeerMain
    3896 Jps
    
    1
    2
    3
    4
    5