Linux——Kafka集群搭建

思索

发布于 2024-08-16 13:37:11

2K0

简介

这里不做过多的原理的讲解，直接讲部署搭建，具体原理性的东西，大家可以到kafka的官网去查看，原汁原味更带劲。

内容

请记住,你是能让世界洒满阳光的人!

安装包准备

JDK1.8 : JDK1.8 kafka_2.12-2.6.0 : kafka_2.12-2.6.0 zookeeper-3.5.8 : zookeeper-3.5.8

kafka_2.12-2.6.0配套的zookeeper就是3.5.8,所以我们直接使用原配进行搭建zookeeper集群

创建目录

mkdir -p /app/data/{zk-data,zk-logs,kafka-logs}

配置JAVA环境

tar -zxvf jdk-8u221-linux-x64.tar.gz -C /usr/local/
echo "export JAVA_HOME=/usr/local/jdk1.8.0_221/" >> /etc/profile
echo "export PATH=/usr/local/jdk1.8.0_221//bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin" >> /etc/profile
echo "export CLASSPATH=.:/usr/local/jdk1.8.0_221//lib/dt.jar:/usr/local/jdk1.8.0_221//lib/tools.jar" >> /etc/profile
source /etc/profile
java -version

三个节点都需要配置java环境

搭建zookeeper集群

节点一

##部署zookeeper服务
wget https://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
tar -zxvf apache-zookeeper-3.5.8-bin.tar.gz -C /usr/local/

##配置zookeeper集群
cd /usr/local/apache-zookeeper-3.5.8-bin/conf
cp -rvf zoo_sample.cfg zoo.cfg
#以下配置需修改为自己的实际配置
sed -i "/^dataDir=/c dataDir=/app/data/zk-data" zoo.cfg
sed -i "/^dataDir=/a dataLogDir=/app/data/zk-logs" zoo.cfg
echo "server.1=192.168.47.41:2888:3888" >> zoo.cfg
echo "server.2=192.168.47.42:2888:3888" >> zoo.cfg
echo "server.3=192.168.47.43:2888:3888" >> zoo.cfg
#myid每个节点不同,此为第一个节点
echo "1" >> /app/data/zk-data/myid

##启动zookeeper服务
cd /app/local/apache-zookeeper-3.5.8-bin/bin/
./zkServer.sh start

节点二

##部署zookeeper服务
wget https://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
tar -zxvf apache-zookeeper-3.5.8-bin.tar.gz -C /usr/local/

##配置zookeeper集群
cd /usr/local/apache-zookeeper-3.5.8-bin/conf
cp -rvf zoo_sample.cfg zoo.cfg
#以下配置需修改为自己的实际配置
sed -i "/^dataDir=/c dataDir=/app/data/zk-data" zoo.cfg
sed -i "/^dataDir=/a dataLogDir=/app/data/zk-logs" zoo.cfg
echo "server.1=192.168.47.41:2888:3888" >> zoo.cfg
echo "server.2=192.168.47.42:2888:3888" >> zoo.cfg
echo "server.3=192.168.47.43:2888:3888" >> zoo.cfg
#myid每个节点不同,此为第二个节点
echo "2" >> /app/data/zk-data/myid

##启动zookeeper服务
cd /app/local/apache-zookeeper-3.5.8-bin/bin/
./zkServer.sh start

节点三

##部署zookeeper服务
wget https://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
tar -zxvf apache-zookeeper-3.5.8-bin.tar.gz -C /usr/local/

##配置zookeeper集群
cd /usr/local/apache-zookeeper-3.5.8-bin/conf
cp -rvf zoo_sample.cfg zoo.cfg
#以下配置需修改为自己的实际配置
sed -i "/^dataDir=/c dataDir=/app/data/zk-data" zoo.cfg
sed -i "/^dataDir=/a dataLogDir=/app/data/zk-logs" zoo.cfg
echo "server.1=192.168.47.41:2888:3888" >> zoo.cfg
echo "server.2=192.168.47.42:2888:3888" >> zoo.cfg
echo "server.3=192.168.47.43:2888:3888" >> zoo.cfg
#myid每个节点不同,此为第三个节点
echo "3" >> /app/data/zk-data/myid

##启动zookeeper服务
cd /app/local/apache-zookeeper-3.5.8-bin/bin/
./zkServer.sh start

查看zookeeper状态

./zkServer.sh status

搭建Kafka集群

节点一

##部署kafka服务
wget https://mirror.bit.edu.cn/apache/kafka/2.6.0/kafka_2.12-2.6.0.tgz
tar -zxvf kafka_2.12-2.6.0.tgz -C /usr/local/

##配置kafka集群
cd /usr/local/kafka_2.12-2.6.0/config/
#以下配置需修改为自己的实际配置
#broker.id每个节点不同,此为第一个节点
sed -i "/^broker.id=0/c   broker.id=1" server.properties
ip=`ip a | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}'| awk -F "/" '{print $1}'`
sed -i "/#listeners=PLAINTEXT:/c  listeners = PLAINTEXT://${ip}:9092" server.properties
sed -i "/^log.dirs=/c    log.dirs=/app/data/kafka-logs" server.properties
sed -i "/^zookeeper.connect=/c  zookeeper.connect=192.168.47.41:2181,192.168.47.42:2181,192.168.47.43:2181" server.properties
echo "delete.topic.enable=true" >> server.properties

##启动kafka服务并查看状态
cd /usr/local/kafka_2.12-2.6.0/bin/
./kafka-server-start.sh -daemon /usr/local/kafka_2.12-2.6.0/config/server.properties
jps

节点二

##部署kafka服务
wget https://mirror.bit.edu.cn/apache/kafka/2.6.0/kafka_2.12-2.6.0.tgz
tar -zxvf kafka_2.12-2.6.0.tgz -C /usr/local/

##配置kafka集群
cd /usr/local/kafka_2.12-2.6.0/config/
#以下配置需修改为自己的实际配置
#broker.id每个节点不同,此为第二个节点
sed -i "/^broker.id=0/c   broker.id=2" server.properties
ip=`ip a | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}'| awk -F "/" '{print $1}'`
sed -i "/#listeners=PLAINTEXT:/c  listeners = PLAINTEXT://${ip}:9092" server.properties
sed -i "/^log.dirs=/c    log.dirs=/app/data/kafka-logs" server.properties
sed -i "/^zookeeper.connect=/c  zookeeper.connect=192.168.47.41:2181,192.168.47.42:2181,192.168.47.43:2181" server.properties
echo "delete.topic.enable=true" >> server.properties

##启动kafka服务并查看状态
cd /usr/local/kafka_2.12-2.6.0/bin/
./kafka-server-start.sh -daemon /usr/local/kafka_2.12-2.6.0/config/server.properties
jps

节点三

##部署kafka服务
wget https://mirror.bit.edu.cn/apache/kafka/2.6.0/kafka_2.12-2.6.0.tgz
tar -zxvf kafka_2.12-2.6.0.tgz -C /usr/local/

##配置kafka集群
cd /usr/local/kafka_2.12-2.6.0/config/
#以下配置需修改为自己的实际配置
#broker.id每个节点不同,此为第三个节点
sed -i "/^broker.id=0/c   broker.id=3" server.properties
ip=`ip a | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}'| awk -F "/" '{print $1}'`
sed -i "/#listeners=PLAINTEXT:/c  listeners = PLAINTEXT://${ip}:9092" server.properties
sed -i "/^log.dirs=/c    log.dirs=/app/data/kafka-logs" server.properties
sed -i "/^zookeeper.connect=/c  zookeeper.connect=192.168.47.41:2181,192.168.47.42:2181,192.168.47.43:2181" server.properties
echo "delete.topic.enable=true" >> server.properties

##启动kafka服务并查看状态
cd /usr/local/kafka_2.12-2.6.0/bin/
./kafka-server-start.sh -daemon /usr/local/kafka_2.12-2.6.0/config/server.properties
jps

topic

只需在一个节点操作即可

## 创建
./kafka-topics.sh --create --topic wechat --partitions 10 --replication-factor 1 --zookeeper 192.168.47.41:2181,192.168.47.42:2181,192.168.47.43:2181

## 查看
./kafka-topics.sh  --topic wechat --describe --zookeeper 192.168.47.41:2181,192.168.47.42:2181,192.168.47.43:2181

## 删除
./kafka-topics.sh --delete --topic wechat --zookeeper 192.168.47.41:2181,192.168.47.42:2181,192.168.47.43:2181