spring boot+kafka整合

springboot版本是2.0.4

首先,在maven中引入spring-kafka的jar包

<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
</dependency> 

在application-dev.properties配置生产者

#=============== producer  =======================
spring.kafka.producer.bootstrap-servers=127.0.0.1:9092,127.0.0.1:9093
spring.kafka.producer.retries=1
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432
spring.kafka.producer.properties.max.requst.size=2097152
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

生产者向kafka发送消息

@Component
@Slf4j
public class KafkaSender {

    @Autowired
    private KafkaTemplate<String, Object> kafkaTemplate;

    // 发送消息方法
    public void send(String body) {

        kafkaTemplate.send("testTopic", body);
        log.info("发送消息完成,内容 为:" + body);
    }

}

配置消费者

#=============== consumer  =======================
spring.kafka.consumer.bootstrap-servers=127.0.0.1:9092,127.0.0.1:9093
spring.kafka.consumer.group-id=0
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=100
#=======set comsumer max fetch.byte 2*1024*1024=============
spring.kafka.consumer.properties.max.partition.fetch.bytes=2097152

消费者监听topic=testTopic的消息

@KafkaListener(topics = { "testTopic" })
public void listen(ConsumerRecord<?, ?> record) {
　　log.info("接收消息为:" + record.value());

}

在整合过程中,spring boot帮我们把kafka的大部分属性直接带出来了,但是有些不常用的属性,需要通过

spring.kafka.consumer.properties.*

来设置,例如max.partition.fetch.bytes,一次fetch请求,从一个partition中取得的records最大值.

在application.properties中添加kafka扩展属性,

#设置一次fetch记录的最大值2M(2*1024*1024),默认值为1M
spring.kafka.consumer.properties.max.partition.fetch.bytes=2097152

更多配置请参考kafka属性大全

 # APACHE KAFKA (KafkaProperties)
 spring.kafka.admin.client-id= # ID to pass to the server when making requests. Used for server-side logging.
 spring.kafka.admin.fail-fast=false # Whether to fail fast if the broker is not available on startup.
 spring.kafka.admin.properties.*= # Additional admin-specific properties used to configure the client.
 spring.kafka.admin.ssl.key-password= # Password of the private key in the key store file.
 spring.kafka.admin.ssl.keystore-location= # Location of the key store file.
 spring.kafka.admin.ssl.keystore-password= # Store password for the key store file.
 spring.kafka.admin.ssl.keystore-type= # Type of the key store.
 spring.kafka.admin.ssl.protocol= # SSL protocol to use.
 spring.kafka.admin.ssl.truststore-location= # Location of the trust store file.
 spring.kafka.admin.ssl.truststore-password= # Store password for the trust store file.
 spring.kafka.admin.ssl.truststore-type= # Type of the trust store.
 spring.kafka.bootstrap-servers= # Comma-delimited list of host:port pairs to use for establishing the initial connection to the Kafka cluster.
 spring.kafka.client-id= # ID to pass to the server when making requests. Used for server-side logging.
 spring.kafka.consumer.auto-commit-interval= # Frequency with which the consumer offsets are auto-committed to Kafka if 'enable.auto.commit' is set to true.
 spring.kafka.consumer.auto-offset-reset= # What to do when there is no initial offset in Kafka or if the current offset no longer exists on the server.
 spring.kafka.consumer.bootstrap-servers= # Comma-delimited list of host:port pairs to use for establishing the initial connection to the Kafka cluster.
 spring.kafka.consumer.client-id= # ID to pass to the server when making requests. Used for server-side logging.
 spring.kafka.consumer.enable-auto-commit= # Whether the consumer's offset is periodically committed in the background.
 spring.kafka.consumer.fetch-max-wait= # Maximum amount of time the server blocks before answering the fetch request if there isn't sufficient data to immediately satisfy the requirement given by "fetch.min.bytes".
 spring.kafka.consumer.fetch-min-size= # Minimum amount of data, in bytes, the server should return for a fetch request.
 spring.kafka.consumer.group-id= # Unique string that identifies the consumer group to which this consumer belongs.
 spring.kafka.consumer.heartbeat-interval= # Expected time between heartbeats to the consumer coordinator.
 spring.kafka.consumer.key-deserializer= # Deserializer class for keys.
 spring.kafka.consumer.max-poll-records= # Maximum number of records returned in a single call to poll().
 spring.kafka.consumer.properties.*= # Additional consumer-specific properties used to configure the client.
 spring.kafka.consumer.ssl.key-password= # Password of the private key in the key store file.
 spring.kafka.consumer.ssl.keystore-location= # Location of the key store file.
 spring.kafka.consumer.ssl.keystore-password= # Store password for the key store file.
 spring.kafka.consumer.ssl.keystore-type= # Type of the key store.
 spring.kafka.consumer.ssl.protocol= # SSL protocol to use.
 spring.kafka.consumer.ssl.truststore-location= # Location of the trust store file.
 spring.kafka.consumer.ssl.truststore-password= # Store password for the trust store file.
 spring.kafka.consumer.ssl.truststore-type= # Type of the trust store.
 spring.kafka.consumer.value-deserializer= # Deserializer class for values.
 spring.kafka.jaas.control-flag=required # Control flag for login configuration.
 spring.kafka.jaas.enabled=false # Whether to enable JAAS configuration.
 spring.kafka.jaas.login-module=com.sun.security.auth.module.Krb5LoginModule # Login module.
 spring.kafka.jaas.options= # Additional JAAS options.
 spring.kafka.listener.ack-count= # Number of records between offset commits when ackMode is "COUNT" or "COUNT_TIME".
 spring.kafka.listener.ack-mode= # Listener AckMode. See the spring-kafka documentation.
 spring.kafka.listener.ack-time= # Time between offset commits when ackMode is "TIME" or "COUNT_TIME".
 spring.kafka.listener.client-id= # Prefix for the listener's consumer client.id property.
 spring.kafka.listener.concurrency= # Number of threads to run in the listener containers.
 spring.kafka.listener.idle-event-interval= # Time between publishing idle consumer events (no data received).
 spring.kafka.listener.log-container-config= # Whether to log the container configuration during initialization (INFO level).
 spring.kafka.listener.monitor-interval= # Time between checks for non-responsive consumers. If a duration suffix is not specified, seconds will be used.
 spring.kafka.listener.no-poll-threshold= # Multiplier applied to "pollTimeout" to determine if a consumer is non-responsive.
 spring.kafka.listener.poll-timeout= # Timeout to use when polling the consumer.
 spring.kafka.listener.type=single # Listener type.
 spring.kafka.producer.acks= # Number of acknowledgments the producer requires the leader to have received before considering a request complete.
 spring.kafka.producer.batch-size= # Default batch size in bytes.
 spring.kafka.producer.bootstrap-servers= # Comma-delimited list of host:port pairs to use for establishing the initial connection to the Kafka cluster.
 spring.kafka.producer.buffer-memory= # Total bytes of memory the producer can use to buffer records waiting to be sent to the server.
 spring.kafka.producer.client-id= # ID to pass to the server when making requests. Used for server-side logging.
 spring.kafka.producer.compression-type= # Compression type for all data generated by the producer.
 spring.kafka.producer.key-serializer= # Serializer class for keys.
 spring.kafka.producer.properties.*= # Additional producer-specific properties used to configure the client.
 spring.kafka.producer.retries= # When greater than zero, enables retrying of failed sends.
 spring.kafka.producer.ssl.key-password= # Password of the private key in the key store file.
 spring.kafka.producer.ssl.keystore-location= # Location of the key store file.
 spring.kafka.producer.ssl.keystore-password= # Store password for the key store file.
 spring.kafka.producer.ssl.keystore-type= # Type of the key store.
 spring.kafka.producer.ssl.protocol= # SSL protocol to use.
 spring.kafka.producer.ssl.truststore-location= # Location of the trust store file.
 spring.kafka.producer.ssl.truststore-password= # Store password for the trust store file.
 spring.kafka.producer.ssl.truststore-type= # Type of the trust store.
 spring.kafka.producer.transaction-id-prefix= # When non empty, enables transaction support for producer.
 spring.kafka.producer.value-serializer= # Serializer class for values.
 spring.kafka.properties.*= # Additional properties, common to producers and consumers, used to configure the client.
 spring.kafka.ssl.key-password= # Password of the private key in the key store file.
 spring.kafka.ssl.keystore-location= # Location of the key store file.
 spring.kafka.ssl.keystore-password= # Store password for the key store file.
 spring.kafka.ssl.keystore-type= # Type of the key store.
 spring.kafka.ssl.protocol= # SSL protocol to use.
 spring.kafka.ssl.truststore-location= # Location of the trust store file.
 spring.kafka.ssl.truststore-password= # Store password for the trust store file.
 spring.kafka.ssl.truststore-type= # Type of the trust store.
 spring.kafka.template.default-topic= # Default topic to which messages are sent.

官方文档地址:

https://docs.spring.io/spring-boot/docs/2.0.4.RELEASE/reference/htmlsingle/#common-application-properties