Java Kafka simple example

Introduction to the

Java Kafka simple code example

Maven dependency Configuration

<! -- kafka --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> <version>0.11. 0. 0</version>
</dependency>
Copy the code

Kakfa production and consumer generation

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.*;

/ * * *@author lw1243925457
 */
public class KafkaUtil {

    public static KafkaConsumer<String, String> createConsumer(String servers, String topic) {
        Properties properties = new Properties();
        properties.put("bootstrap.servers", servers);
        properties.put("group.id"."group-1");
        properties.put("enable.auto.commit"."false");
        properties.put("auto.commit.interval.ms"."1000");
        properties.put("auto.offset.reset"."earliest");
        properties.put("session.timeout.ms"."30000");
        properties.put("key.deserializer"."org.apache.kafka.common.serialization.StringDeserializer");
        properties.put("value.deserializer"."org.apache.kafka.common.serialization.StringDeserializer");

        KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(properties);
        kafkaConsumer.subscribe(Arrays.asList(topic));
        return kafkaConsumer;
    }

    public static void readMessage(KafkaConsumer<String, String> kafkaConsumer, int timeout) {
        while (true) {
            ConsumerRecords<String, String> records = kafkaConsumer.poll(timeout);
            for(ConsumerRecord<String, String> record : records) { String value = record.value(); kafkaConsumer.commitAsync(); System.out.println(value); }}}public static KafkaProducer<String, String> createProducer(String servers) {
        Properties properties = new Properties();
        properties.put("bootstrap.servers", servers);
        properties.put("acks"."all");
        properties.put("retries".0);
        properties.put("batch.size".16384);
        properties.put("linger.ms".1);
        properties.put("buffer.memory".33554432);
        properties.put("key.serializer"."org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer"."org.apache.kafka.common.serialization.StringSerializer");
        return new KafkaProducer<String, String>(properties);
    }

    public static void send(KafkaProducer<String, String> producer, String topic, String message) {
        producer.send(newProducerRecord<String, String>(topic, message)); }}Copy the code

run

public class Main {

    public static void main(String[] args) {
        String servers = "localhost:9092,localhost:9093,localhost:9094";
        String topic = "TestTopic";
        String message = "test";

        KafkaProducer<String, String> producer = KafkaUtil.createProducer(servers);
        KafkaUtil.send(producer, topic, message);

        KafkaConsumer<String, String> consumer = KafkaUtil.createConsumer(servers, topic);
        KafkaUtil.readMessage(consumer, 100); }}Copy the code

Use notes

Always read the oldest message

It may be a group-ID problem. Create a new group-ID name

• Earliest: When there is a submitted offset under each partition, the money will be consumed from the submitted offset; If there is no submitted offset, the consumption starts from scratch
• Latest: Consumes the submitted offset when each partition has an offset. If there is no committed offset, the newly generated data under the partition is consumed
• None: Consumption starts after offset when all partitions in topic have committed offsets. An exception is thrown whenever a partition does not have a committed offset

Refer to the link

• Java implements Kafka message producers and consumers
• Kafka (3) – Java code examples and configuration instructions for Kafka
• Kafka – Offset commit
• Kafka consumer: Read data from Kafka
• Kafka auto. Offset. Reset Specifies the value