kafka性能测试代码

 bin/kafka-producer-perf-test.sh --num-records 5000000 --record-size 5000  --topic kafkatopic2     --broker-list 


bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance kafkatopic2 50000000 100 -1 acks=1  bootstrap.servers=bdnodedn03.smpv.com:6667,bdnodedn08.smpv.com:6667,bdnodedn09.smpv.com:6667   buffer.memory=67108864 batch.size=8196


Producer
Setup
bin/kafka-topics.sh --zookeeper bdnodedn01.smpv.com:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1
bin/kafka-topics.sh --zookeeper bdnodedn01.smpv.com:2181 --create --topic test-rep-two --partitions 6 --replication-factor 3

Single thread, no replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-one --num-records 6000000 --throughput 100000 --record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 buffer.memory=67108864 batch.size=8196

Single-thread, async 3x replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two --num-records 6000000 --throughput 100000 --record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=1 buffer.memory=67108864 batch.size=8196

Single-thread, sync 3x replication
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two --num-records 6000000 --throughput 100000 --record-size 100 --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=-1 buffer.memory=67108864 batch.size=8196

Effect of message size

for i in 10 100 1000 10000 100000;
do
echo ""
echo $i
bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance --print-metrics --topic test-rep-two --num-records  $((1000*1024*1024/$i)) --throughput 100000 --record-size $i --producer-props bootstrap.servers=bdnodedn03.smpv.com:6667 acks=1 buffer.memory=67108864 batch.size=8196
done;


Consumer

Consumer throughput
bin/kafka-run-class.sh kafka.tools.ConsumerPerformance  --broker-list bdnodedn03.smpv.com:6667 --messages 6000000 --threads 1 --topic test-rep-two --print-metrics

3 Consumers

On three servers, run:
bin/kafka-run-class.sh kafka.tools.ConsumerPerformance  --broker-list bdnodedn03.smpv.com:6667 --messages 6000000 --threads 1 --topic test-rep-two --print-metrics