Kafka Java示例

    由于项目原因需要升级kafka，需要重新申请kafka服务，用新的Java API来编写producer和consumer，顺便接触一下Kafka。准备工作就是安装zookeeper和kafka，由于关于安装方面的文章很多，在此就不细说了。

一、一些问题

1.1 kafka错误

错误： WARN Error while fetching metadata with correlation id 0 : {test-topic=LEADER_NOT_AVAILABLE} (org.apache.kafka.clients.NetworkClient)
解决方案：https://github.com/wurstmeister/kafka-docker/issues/85https://github.com/wurstmeister/kafka-docker/issues/85

config/server.properties：

>set advertised.host.name = ip。

1.2 kafka命令行
>创建topic：sh kafka-topics.sh --create --zookeeper 10.95.177.100:2181,10.95.97.175:2181,10.95.176.44:2181  --replication-factor 1 --partitions 1 --topic  topic-test
>查看topic：sh kafka-topics.sh --list --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181
>删除topic：
sh kafka-topics.sh --delete --topic topic-test --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181 
>生产消息：sh kafka-console-producer.sh --broker-list 10.95.177.192:9092,10.95.97.175:9092 --topic topic-test
>消费消息：sh kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic-test --from-beginning  --zookeeper 10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181 
>启动kafka服务：
./kafka-server-start.sh -daemon ../config/server.properties
>停止kafka服务：
./kafka-server-stop

1.3 Java API版本
一种：kafka.javaapi.*：

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
另一种：org.apache.kafka.*:

import org.apache.kafka.clients.producer.KafkaProducer
在Kafka 0.8.2之前，kafka.javaapi.producer.Producer是为唯一官方用Scala实现的Java Client。
在Kafka 0.8.2之后，有新的Java Producer API，org.apache.kafka.clients.producer.KafkaProducer,完全用Java实现的。
二、Java Kafka示例

pom.xml:

   
        org.apache.kafka
        kafka-clients
        0.10.2.0
   
   
        org.apache.kafka
        kafka_2.11
        0.10.2.0
   

2.1 老版本Producer：
KafkaProducerOld.java:

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
 
import java.util.Properties;
 
/**
 * kafka.javaapi.producer.Producer
 * */
public class KafkaProducerOld {
 
    private Producer producer;
 
    public final static String TOPIC = "didi-topic-test";
 
    private KafkaProducerOld() {
        Properties props = new Properties();
        props.put("metadata.broker.list", "10.95.177.192:9092,10.95.97.175:9092");
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        ProducerConfig config = new ProducerConfig(props);
        producer = new Producer(config);
    }
 
    public void produce() {
        int messageNo = 0;
        final int COUNT = 10;
 
        while(messageNo < COUNT) {
            String data = String.format("hello kafka.javaapi.producer message %d", messageNo);
 
            KeyedMessage msg = new KeyedMessage(TOPIC, data);
            
            try {
                producer.send(msg);
            } catch (Exception e) {
                e.printStackTrace();
            }
 
            messageNo++;
        }
 
        producer.close();
    }
 
    public static void main(String[] args) {
        new KafkaProducerOld().produce();
    }
 
}
2.2 老版本Consumer
KafkaConsumerOld.java:

import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.serializer.StringDecoder;
import kafka.utils.VerifiableProperties;
 
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
 
/**
* Consumer: kafka.javaapi.consumer.Consumer
* */
public class KafkaConsumerOld {
 
    private final ConsumerConnector consumerConnector;
 
    public final static String TOPIC = "didi-topic-test";
 
    private KafkaConsumerOld() {
        Properties props = new Properties();
        props.put("zookeeper.connect", "10.95.177.192:2181,10.95.97.175:2181,10.95.176.44:2181");
        props.put("group.id", "group-1");
 
        //zk连接超时
        props.put("zookeeper.session.timeout.ms", "400");
        props.put("zookeeper.sync.time.ms", "200");
        props.put("auto.commit.interval.ms", "1000");
        props.put("auto.offset.reset", "smallest");
        //序列化类
        props.put("serializer.class", "kafka.serializer.StringDecoder");
 
        ConsumerConfig config = new ConsumerConfig(props);
 
        consumerConnector = kafka.consumer.Consumer.createJavaConsumerConnector(config);
    }
 
    public void consume() {
        Map topicCountMap = new HashMap();
        topicCountMap.put(KafkaProducerNew.TOPIC, new Integer(1));
 
        StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
        StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
 
        Map>> consumerMap = consumerConnector.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
 
        List> streams = consumerMap.get(TOPIC);
        for (final KafkaStream stream : streams) {
            ConsumerIterator it = stream.iterator();
            while (it.hasNext()) {
                System.out.println("this is kafka consumer : " + new String( it.next().message().toString()) );
            }
        }
    }
 
    public static void main(String[] args) {
        new KafkaConsumerOld().consume();
    }
}
2.3 新版本Producer：
KafkaProducerNew.java:

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
 
import java.util.Properties;
 
/**
* producer: org.apache.kafka.clients.producer.KafkaProducer;
 * */
 
public class KafkaProducerNew {
 
    private final KafkaProducer producer;
 
    public final static String TOPIC = "didi-topic-test";
 
    private KafkaProducerNew() {
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.95.177.192:9092,10.95.97.175:9092");
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
 
        producer = new KafkaProducer(props);
    }
 
    public void produce() {
        int messageNo = 1;
        final int COUNT = 10;
 
        while(messageNo < COUNT) {
            String key = String.valueOf(messageNo);
            String data = String.format("hello KafkaProducer message %s", key);
            
            try {
                producer.send(new ProducerRecord(TOPIC, data));
            } catch (Exception e) {
                e.printStackTrace();
            }
 
            messageNo++;
        }
        
        producer.close();
    }
 
    public static void main(String[] args) {
        new KafkaProducerNew().produce();
    }
 
}
2.4 新版本Consumer：
KafkaConsumerNew.java:

import org.apache.kafka.clients.consumer.*;
 
import java.util.Arrays;
import java.util.Properties;
 
/**
* consumer: org.apache.kafka.clients.consumer.Consumer
* */
public class KafkaConsumerNew {
 
    private Consumer consumer;
 
    private static String group = "group-1";
 
    private static String TOPIC = "didi-topic-test";
 
    private KafkaConsumerNew() {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.95.177.192:9092,10.95.97.175:9092");
        props.put(ConsumerConfig.GROUP_ID_CONFIG, group);
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); // 自动commit
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); // 自动commit的间隔
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        consumer = new KafkaConsumer(props);
    }
 
    private void consume() {
        consumer.subscribe(Arrays.asList(TOPIC)); // 可消费多个topic,组成一个list
 
        while (true) {
            ConsumerRecords records = consumer.poll(1000);
            for (ConsumerRecord record : records) {
                System.out.printf("offset = %d, key = %s, value = %s \n", record.offset(), record.key(), record.value());
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
    }
 
    public static void main(String[] args) {
        new KafkaConsumerNew().consume();
    }
}

Author：忆之独秀

Email：[email protected]

注明出处：http://blog.csdn.net/lavorange/article/details/78970977
 

原文地址：https://blog.csdn.net/lavorange/article/details/78970977