Kafka常用命令

旧集群kafka地址

10.26.241.54:2181,10.45.137.30:2181,10.45.150.101:2181

新集群broker-list

10.31.146.66:9092,10.81.133.75:9092,10.30.94.117:9092

kafka启动停止

1
2
sudo /opt/kafka/bin/kafka-server-stop.sh  
sudo /opt/kafka/bin/kafka-server-start.sh -daemon /opt/kafka/config/server.properties

zookeeper启动停止

1
2
sudo /opt/zookeeper-3.4.8/bin/zkServer.sh start  
sudo /opt/zookeeper-3.4.8/bin/zkServer.sh stop

生产者生产数据

1
./kafka-console-producer.sh --broker-list kafka01:9092,kafka02:9092,kafka03:9092 --topic test

消费者消费数据

1
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --topic aclog --from-beginning

使用mirrormaker迁移topic

1
./kafka-mirror-maker.sh --consumer.config ../config/consumer.properties  --num.streams 10 --producer.config ../config/producer.properties  --whitelist "aclog,sudlog,sdklog,sflog,arlog,fzbss"

查看同步状态offset

1
./kafka-consumer-offset-checker.sh --group online --zookeeper 10.26.241.54:2181,10.45.137.30:2181,10.45.150.101:2181   --topic aclog

查看offset最小值

1
./kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list kafka01:9092,kafka02:9092,kafka03:9092 —topic aclog --time -1

查看offset最大值

1
./kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list kafka01:9092,kafka02:9092,kafka03:9092 —topic aclog --time -2

查看topic情况

1
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --describe

手动分配副本

1
./kafka-reassign-partitions.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --reassignment-json-file /tmp/increase-replication-factor.json --execute

创建topic

1
2
3
4
5
6
7
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic aclog  
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic sudlog
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic sdklog
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic sflog
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic arlog
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic fzbss
./kafka-topics.sh --create --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --replication-factor 2 --partitions 10 --topic acevent

删除topic

1
2
3
4
5
6
7
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "aclog"  
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "arlog"
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "sdklog"
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "fzbss"
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "sflog"
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "sudlog"
./kafka-topics.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --delete --topic "acevent"

删除分区

1
2
sudo rm -rf `find /data01/ -type d | grep -vw "/data01/"`  
sudo rm -rf `find /data02/ -type d | grep -vw "/data02/"`

删除zookeeper

1
2
3
4
5
6
7
8
9
/opt/zookeeper-3.4.8/bin/zkCli.sh
rmr /cluster
rmr /controller
rmr /controller_epoch
rmr /brokers
rmr /admin
rmr /isr_change_notification
rmr /consumers
rmr /config

cluster, controller, controller_epoch, brokers, zookeeper, admin, isr_change_notification, consumers, config
指定组ID消费
debug

1
2
3
4
5
6
7
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic aclog  
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic sudlog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic sdklog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic sflog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic arlog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic fzbss
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.debug --topic acevent

gp_prod

1
2
3
4
5
6
7
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic aclog  
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic sudlog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic sdklog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic sflog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic arlog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic fzbss
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp_prod --topic acevent

gp5_prod

1
2
3
4
5
6
7
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic aclog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic sudlog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic sdklog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic sflog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic arlog
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic fzbss
./kafka-console-consumer.sh --zookeeper kafka01:2181,kafka02:2181,kafka03:2181 --consumer.config /tmp/consumer.properties.gp5_prod --topic acevent

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
{"version":1001,  
"partitions":[
{"topic":"aclog","partition":0,"replicas":[1001,1002]},
{"topic":"aclog","partition":1,"replicas":[1003,1001]},
{"topic":"aclog","partition":2,"replicas":[1002,1003]},
{"topic":"aclog","partition":3,"replicas":[1001,1002]},
{"topic":"aclog","partition":4,"replicas":[1003,1001]},
{"topic":"aclog","partition":5,"replicas":[1002,1003]},
{"topic":"aclog","partition":6,"replicas":[1001,1002]},
{"topic":"aclog","partition":7,"replicas":[1003,1001]},
{"topic":"aclog","partition":8,"replicas":[1002,1003]},
{"topic":"aclog","partition":9,"replicas":[1001,1002]},
{"topic":"sdklog","partition":0,"replicas":[1003,1001]},
{"topic":"sdklog","partition":1,"replicas":[1002,1003]},
{"topic":"sdklog","partition":2,"replicas":[1001,1002]},
{"topic":"sdklog","partition":3,"replicas":[1003,1001]},
{"topic":"sdklog","partition":4,"replicas":[1002,1003]},
{"topic":"sdklog","partition":5,"replicas":[1001,1002]},
{"topic":"sdklog","partition":6,"replicas":[1003,1001]},
{"topic":"sdklog","partition":7,"replicas":[1002,1003]},
{"topic":"sdklog","partition":8,"replicas":[1001,1002]},
{"topic":"sdklog","partition":9,"replicas":[1003,1001]},
{"topic":"sudlog","partition":0,"replicas":[1002,1003]},
{"topic":"sudlog","partition":1,"replicas":[1001,1002]},
{"topic":"sudlog","partition":2,"replicas":[1003,1001]},
{"topic":"sudlog","partition":3,"replicas":[1002,1003]},
{"topic":"sudlog","partition":4,"replicas":[1001,1002]},
{"topic":"sudlog","partition":5,"replicas":[1003,1001]},
{"topic":"sudlog","partition":6,"replicas":[1002,1003]},
{"topic":"sudlog","partition":7,"replicas":[1001,1002]},
{"topic":"sudlog","partition":8,"replicas":[1003,1001]},
{"topic":"sudlog","partition":9,"replicas":[1002,1003]},
{"topic":"arlog","partition":0,"replicas":[1001,1002]},
{"topic":"arlog","partition":1,"replicas":[1003,1001]},
{"topic":"arlog","partition":2,"replicas":[1001,1002]},
{"topic":"arlog","partition":3,"replicas":[1003,1001]},
{"topic":"arlog","partition":4,"replicas":[1002,1003]},
{"topic":"arlog","partition":5,"replicas":[1001,1002]},
{"topic":"arlog","partition":6,"replicas":[1003,1001]},
{"topic":"arlog","partition":7,"replicas":[1003,1001]},
{"topic":"arlog","partition":8,"replicas":[1002,1003]},
{"topic":"arlog","partition":9,"replicas":[1001,1002]},
{"topic":"sflog","partition":0,"replicas":[1003,1001]},
{"topic":"sflog","partition":1,"replicas":[1002,1003]},
{"topic":"sflog","partition":2,"replicas":[1001,1002]},
{"topic":"sflog","partition":3,"replicas":[1003,1001]},
{"topic":"sflog","partition":4,"replicas":[1002,1003]},
{"topic":"sflog","partition":5,"replicas":[1001,1002]},
{"topic":"sflog","partition":6,"replicas":[1003,1001]},
{"topic":"sflog","partition":7,"replicas":[1002,1003]},
{"topic":"sflog","partition":8,"replicas":[1001,1002]},
{"topic":"sflog","partition":9,"replicas":[1003,1001]},
{"topic":"fzbss","partition":0,"replicas":[1002,1003]},
{"topic":"fzbss","partition":1,"replicas":[1001,1002]},
{"topic":"fzbss","partition":2,"replicas":[1003,1001]},
{"topic":"fzbss","partition":3,"replicas":[1002,1003]},
{"topic":"fzbss","partition":4,"replicas":[1001,1002]},
{"topic":"fzbss","partition":5,"replicas":[1003,1001]},
{"topic":"fzbss","partition":6,"replicas":[1002,1003]},
{"topic":"fzbss","partition":7,"replicas":[1001,1002]},
{"topic":"fzbss","partition":8,"replicas":[1003,1001]},
{"topic":"fzbss","partition":9,"replicas":[1002,1003]}
]}