kafka性能测试代码

 bin/kafka-producer-perf-test.sh --num-records 5000000 --record-size 5000 \
 --topic kafkatopic2   \
  --broker-list 
 
bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance kafkatopic2 50000000 100 -1 acks=1 \
 bootstrap.servers=bdnodedn03.smpv.com:6667,bdnodedn08.smpv.com:6667,bdnodedn09.smpv.com:6667 \
  buffer.memory=67108864 batch.size=8196
 
Producer
Setup
bin/kafka-topics.sh --zookeeper bdnodedn01.smpv.com:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1
bin/kafka-topics.sh --zookeeper bdnodedn01.smpv.com:2181 --create --topic test-rep-two --partitions 6 --replication-factor 3
 
Single thread, no replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-one \
--num-records 6000000 --throughput 100000 \
--record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 \
buffer.memory=67108864 batch.size=8196
 
Single-thread, async 3x replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two \
--num-records 6000000 --throughput 100000 --record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 \
acks=1 buffer.memory=67108864 batch.size=8196
 
Single-thread, sync 3x replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two \
--num-records 6000000 --throughput 100000 --record-size 100 \
--producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=-1 buffer.memory=67108864 batch.size=8196
 
Effect of message size
 
for i in 10 100 1000 10000 100000;
do
echo ""
echo $i
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics \
--topic test-rep-two --num-records  $((1000*1024*1024/$i)) --throughput 100000 \
--record-size $i --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=1 buffer.memory=67108864 batch.size=8196
done;
 
Consumer
 
Consumer throughput
bin/kafka-run-class.sh kafka.tools.ConsumerPerformance  --broker-list bdnodedn03.smpv.com:6667 \
--messages 6000000 --threads 1 \
--topic test-rep-two --print-metrics
 
3 Consumers
 
On three servers, run:
bin/kafka-run-class.sh kafka.tools.ConsumerPerformance  --broker-list bdnodedn03.smpv.com:6667 \
--messages 6000000 --threads 1 \
--topic test-rep-two --print-metrics