java获取kafka consumer lag、endOffsets、beginningOffsets以及 KafkaConsumer总结
一、java获取kafka consumer lag、endOffsets、beginningOffsets
maven依赖：
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.10.1.0</version>
</dependency>
kafka-clients版本需要0.10.1.0以上，因为调用了新增接口endOffsets，lag=logsize-offset，logsize通过consumer的endOffsets接口获得；offset通过consumer的committed接口获得；
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties; import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition; public class KafkaConsumeLagMonitor {
public static Properties getConsumeProperties(String groupID, String bootstrap_server) {
Properties props = new Properties();
props.put("group.id", groupID);
props.put("bootstrap.servers", bootstrap_server);
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); return props;
} public static void main(String[] args) {
String bootstrap_server = args[0];
String groupID = args[1];
String topic = args[2];
Map<Integer, Long> endOffsetMap = new HashMap<Integer, Long>();
Map<Integer, Long> commitOffsetMap = new HashMap<Integer, Long>(); Properties consumeProps = getConsumeProperties(groupID, bootstrap_server);
System.out.println("consumer properties:" + consumeProps);
//查询topic partitions
KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(consumeProps);
List<TopicPartition> topicPartitions = new ArrayList<TopicPartition>();
List<PartitionInfo> partitionsFor = consumer.partitionsFor(topic);
for (PartitionInfo partitionInfo : partitionsFor) {
TopicPartition topicPartition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
topicPartitions.add(topicPartition);
} //查询log size
Map<TopicPartition, Long> endOffsets = consumer.endOffsets(topicPartitions);
for (TopicPartition partitionInfo : endOffsets.keySet()) {
endOffsetMap.put(partitionInfo.partition(), endOffsets.get(partitionInfo));
}
for (Integer partitionId : endOffsetMap.keySet()) {
System.out.println(String.format("at %s, topic:%s, partition:%s, logSize:%s", System.currentTimeMillis(), topic, partitionId, endOffsetMap.get(partitionId)));
} //查询消费offset
for (TopicPartition topicAndPartition : topicPartitions) {
OffsetAndMetadata committed = consumer.committed(topicAndPartition);
commitOffsetMap.put(topicAndPartition.partition(), committed.offset());
} //累加lag
long lagSum = 0l;
if (endOffsetMap.size() == commitOffsetMap.size()) {
for (Integer partition : endOffsetMap.keySet()) {
long endOffSet = endOffsetMap.get(partition);
long commitOffSet = commitOffsetMap.get(partition);
long diffOffset = endOffSet - commitOffSet;
lagSum += diffOffset;
System.out.println("Topic:" + topic + ", groupID:" + groupID + ", partition:" + partition + ", endOffset:" + endOffSet + ", commitOffset:" + commitOffSet + ", diffOffset:" + diffOffset);
}
System.out.println("Topic:" + topic + ", groupID:" + groupID + ", LAG:" + lagSum);
} else {
System.out.println("this topic partitions lost");
} consumer.close();
}
}
另外还可以通过AdminClient（官方），更多详情可以参考 https://kafka.apache.org/documentation
package kafka.admin
class AdminClient(val time : org.apache.kafka.common.utils.Time, val requestTimeoutMs : scala.Int, val retryBackoffMs : scala.Long, val client : org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient, val bootstrapBrokers : scala.List[org.apache.kafka.common.Node]) extends scala.AnyRef with kafka.utils.Logging {
@scala.volatile
var running : scala.Boolean = { /* compiled code */ }
val pendingFutures : java.util.concurrent.ConcurrentLinkedQueue[org.apache.kafka.clients.consumer.internals.RequestFuture[org.apache.kafka.clients.ClientResponse]] = { /* compiled code */ }
val networkThread : org.apache.kafka.common.utils.KafkaThread = { /* compiled code */ }
def findCoordinator(groupId : scala.Predef.String, timeoutMs : scala.Long = { /* compiled code */ }) : org.apache.kafka.common.Node = { /* compiled code */ }
def listGroups(node : org.apache.kafka.common.Node) : scala.List[kafka.coordinator.group.GroupOverview] = { /* compiled code */ }
def getApiVersions(node : org.apache.kafka.common.Node) : scala.List[org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion] = { /* compiled code */ }
def awaitBrokers() : scala.Unit = { /* compiled code */ }
def findAllBrokers() : scala.List[org.apache.kafka.common.Node] = { /* compiled code */ }
def listAllGroups() : scala.Predef.Map[org.apache.kafka.common.Node, scala.List[kafka.coordinator.group.GroupOverview]] = { /* compiled code */ }
def listAllConsumerGroups() : scala.Predef.Map[org.apache.kafka.common.Node, scala.List[kafka.coordinator.group.GroupOverview]] = { /* compiled code */ }
def listAllGroupsFlattened() : scala.List[kafka.coordinator.group.GroupOverview] = { /* compiled code */ }
def listAllConsumerGroupsFlattened() : scala.List[kafka.coordinator.group.GroupOverview] = { /* compiled code */ }
def listGroupOffsets(groupId : scala.Predef.String) : scala.Predef.Map[org.apache.kafka.common.TopicPartition, scala.Long] = { /* compiled code */ }
def listAllBrokerVersionInfo() : scala.Predef.Map[org.apache.kafka.common.Node, scala.util.Try[org.apache.kafka.clients.NodeApiVersions]] = { /* compiled code */ }
def deleteRecordsBefore(offsets : scala.Predef.Map[org.apache.kafka.common.TopicPartition, scala.Long]) : java.util.concurrent.Future[scala.Predef.Map[org.apache.kafka.common.TopicPartition, kafka.admin.AdminClient.DeleteRecordsResult]] = { /* compiled code */ }
case class ConsumerSummary(val consumerId : scala.Predef.String, val clientId : scala.Predef.String, val host : scala.Predef.String, val assignment : scala.List[org.apache.kafka.common.TopicPartition]) extends scala.AnyRef with scala.Product with scala.Serializable {
}
case class ConsumerGroupSummary(val state : scala.Predef.String, val assignmentStrategy : scala.Predef.String, val consumers : scala.Option[scala.List[AdminClient.this.ConsumerSummary]], val coordinator : org.apache.kafka.common.Node) extends scala.AnyRef with scala.Product with scala.Serializable {
}
def describeConsumerGroupHandler(coordinator : org.apache.kafka.common.Node, groupId : scala.Predef.String) : org.apache.kafka.common.requests.DescribeGroupsResponse.GroupMetadata = { /* compiled code */ }
def describeConsumerGroup(groupId : scala.Predef.String, timeoutMs : scala.Long = { /* compiled code */ }) : AdminClient.this.ConsumerGroupSummary = { /* compiled code */ }
def close() : scala.Unit = { /* compiled code */ }
}
object AdminClient extends scala.AnyRef {
val DefaultConnectionMaxIdleMs : scala.Int = { /* compiled code */ }
val DefaultRequestTimeoutMs : scala.Int = { /* compiled code */ }
val DefaultMaxInFlightRequestsPerConnection : scala.Int = { /* compiled code */ }
val DefaultReconnectBackoffMs : scala.Int = { /* compiled code */ }
val DefaultReconnectBackoffMax : scala.Int = { /* compiled code */ }
val DefaultSendBufferBytes : scala.Int = { /* compiled code */ }
val DefaultReceiveBufferBytes : scala.Int = { /* compiled code */ }
val DefaultRetryBackoffMs : scala.Int = { /* compiled code */ }
val AdminClientIdSequence : java.util.concurrent.atomic.AtomicInteger = { /* compiled code */ }
val AdminConfigDef : org.apache.kafka.common.config.ConfigDef = { /* compiled code */ }
case class DeleteRecordsResult(val lowWatermark : scala.Long, val error : scala.Exception) extends scala.AnyRef with scala.Product with scala.Serializable {
}
class AdminConfig(originals : scala.Predef.Map[_, _]) extends org.apache.kafka.common.config.AbstractConfig {
}
def createSimplePlaintext(brokerUrl : scala.Predef.String) : kafka.admin.AdminClient = { /* compiled code */ }
def create(props : java.util.Properties) : kafka.admin.AdminClient = { /* compiled code */ }
def create(props : scala.Predef.Map[scala.Predef.String, _]) : kafka.admin.AdminClient = { /* compiled code */ }
def create(config : kafka.admin.AdminClient.AdminConfig) : kafka.admin.AdminClient = { /* compiled code */ }
}
二、 org.apache.kafka.clients.consumer Class KafkaConsumer<K,V> 总结
Package org.apache.kafka.clients.consumer (http://kafka.apache.org/20/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html)
Class KafkaConsumer<K,V>
- java.lang.Object
- org.apache.kafka.clients.consumer.KafkaConsumer<K,V>
- All Implemented Interfaces:
java.io.Closeable,java.lang.AutoCloseable,Consumer<K,V>-
public class KafkaConsumer<K,V>
extends java.lang.Object
implements Consumer<K,V>
A client that consumes records from a Kafka cluster.
This client transparently handles the failure of Kafka brokers, and transparently adapts as topic partitions it fetches migrate within the cluster. This client also interacts with the broker to allow groups of consumers to load balance consumption using consumer groups. The consumer maintains TCP connections to the necessary brokers to fetch data. Failure to close the consumer after use will leak these connections. The consumer is not thread-safe. See Multi-threaded Processing for more details. Cross-Version Compatibility
This client can communicate with brokers that are version 0.10.0 or newer. Older or newer brokers may not support certain features. For example, 0.10.0 brokers do not support offsetsForTimes, because this feature was added in version 0.10.1. You will receive an UnsupportedVersionException when invoking an API that is not available on the running broker version.
Offsets and Consumer Position
Kafka maintains a numerical offset for each record in a partition. This offset acts as a unique identifier of a record within that partition, and also denotes the position of the consumer in the partition. For example, a consumer which is at position 5 has consumed records with offsets 0 through 4 and will next receive the record with offset 5. There are actually two notions of position relevant to the user of the consumer:
The position of the consumer gives the offset of the next record that will be given out. It will be one larger than the highest offset the consumer has seen in that partition. It automatically advances every time the consumer receives messages in a call to poll(Duration). The committed position is the last offset that has been stored securely. Should the process fail and restart, this is the offset that the consumer will recover to. The consumer can either automatically commit offsets periodically; or it can choose to control this committed position manually by calling one of the commit APIs (e.g. commitSync and commitAsync). This distinction gives the consumer control over when a record is considered consumed. It is discussed in further detail below. Consumer Groups and Topic Subscriptions
Kafka uses the concept of consumer groups to allow a pool of processes to divide the work of consuming and processing records. These processes can either be running on the same machine or they can be distributed over many machines to provide scalability and fault tolerance for processing. All consumer instances sharing the same group.id will be part of the same consumer group.
Each consumer in a group can dynamically set the list of topics it wants to subscribe to through one of the subscribe APIs. Kafka will deliver each message in the subscribed topics to one process in each consumer group. This is achieved by balancing the partitions between all members in the consumer group so that each partition is assigned to exactly one consumer in the group. So if there is a topic with four partitions, and a consumer group with two processes, each process would consume from two partitions. Membership in a consumer group is maintained dynamically: if a process fails, the partitions assigned to it will be reassigned to other consumers in the same group. Similarly, if a new consumer joins the group, partitions will be moved from existing consumers to the new one. This is known as rebalancing the group and is discussed in more detail below. Group rebalancing is also used when new partitions are added to one of the subscribed topics or when a new topic matching a subscribed regex is created. The group will automatically detect the new partitions through periodic metadata refreshes and assign them to members of the group. Conceptually you can think of a consumer group as being a single logical subscriber that happens to be made up of multiple processes. As a multi-subscriber system, Kafka naturally supports having any number of consumer groups for a given topic without duplicating data (additional consumers are actually quite cheap). This is a slight generalization of the functionality that is common in messaging systems. To get semantics similar to a queue in a traditional messaging system all processes would be part of a single consumer group and hence record delivery would be balanced over the group like with a queue. Unlike a traditional messaging system, though, you can have multiple such groups. To get semantics similar to pub-sub in a traditional messaging system each process would have its own consumer group, so each process would subscribe to all the records published to the topic. In addition, when group reassignment happens automatically, consumers can be notified through a ConsumerRebalanceListener, which allows them to finish necessary application-level logic such as state cleanup, manual offset commits, etc. See Storing Offsets Outside Kafka for more details. It is also possible for the consumer to manually assign specific partitions (similar to the older "simple" consumer) using assign(Collection). In this case, dynamic partition assignment and consumer group coordination will be disabled. Detecting Consumer Failures
After subscribing to a set of topics, the consumer will automatically join the group when poll(Duration) is invoked. The poll API is designed to ensure consumer liveness. As long as you continue to call poll, the consumer will stay in the group and continue to receive messages from the partitions it was assigned. Underneath the covers, the consumer sends periodic heartbeats to the server. If the consumer crashes or is unable to send heartbeats for a duration of session.timeout.ms, then the consumer will be considered dead and its partitions will be reassigned.
It is also possible that the consumer could encounter a "livelock" situation where it is continuing to send heartbeats, but no progress is being made. To prevent the consumer from holding onto its partitions indefinitely in this case, we provide a liveness detection mechanism using the max.poll.interval.ms setting. Basically if you don't call poll at least as frequently as the configured max interval, then the client will proactively leave the group so that another consumer can take over its partitions. When this happens, you may see an offset commit failure (as indicated by a CommitFailedException thrown from a call to commitSync()). This is a safety mechanism which guarantees that only active members of the group are able to commit offsets. So to stay in the group, you must continue to call poll. The consumer provides two configuration settings to control the behavior of the poll loop: max.poll.interval.ms: By increasing the interval between expected polls, you can give the consumer more time to handle a batch of records returned from poll(Duration). The drawback is that increasing this value may delay a group rebalance since the consumer will only join the rebalance inside the call to poll. You can use this setting to bound the time to finish a rebalance, but you risk slower progress if the consumer cannot actually call poll often enough.
max.poll.records: Use this setting to limit the total records returned from a single call to poll. This can make it easier to predict the maximum that must be handled within each poll interval. By tuning this value, you may be able to reduce the poll interval, which will reduce the impact of group rebalancing.
For use cases where message processing time varies unpredictably, neither of these options may be sufficient. The recommended way to handle these cases is to move message processing to another thread, which allows the consumer to continue calling poll while the processor is still working. Some care must be taken to ensure that committed offsets do not get ahead of the actual position. Typically, you must disable automatic commits and manually commit processed offsets for records only after the thread has finished handling them (depending on the delivery semantics you need). Note also that you will need to pause the partition so that no new records are received from poll until after thread has finished handling those previously returned. Usage Examples
The consumer APIs offer flexibility to cover a variety of consumption use cases. Here are some examples to demonstrate how to use them.
Automatic Offset Committing
This example demonstrates a simple usage of Kafka's consumer api that relies on automatic offset committing.
Properties props = new Properties();
props.put("bootstrap.servers", "localhost:9092");
props.put("group.id", "test");
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
consumer.subscribe(Arrays.asList("foo", "bar"));
while (true) {
ConsumerRecords<String, String> records = consumer.poll(100);
for (ConsumerRecord<String, String> record : records)
System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
} The connection to the cluster is bootstrapped by specifying a list of one or more brokers to contact using the configuration >bootstrap.servers. This list is just used to discover the rest of the brokers in the cluster and need not be an exhaustive list of servers in the cluster (though you may want to specify more than one in case there are servers down when the client is connecting).
Setting enable.auto.commit means that offsets are committed automatically with a frequency controlled by the config auto.commit.interval.ms. In this example the consumer is subscribing to the topics foo and bar as part of a group of consumers called test as configured with group.id. The deserializer settings specify how to turn bytes into objects. For example, by specifying string deserializers, we are saying that our record's key and value will just be simple strings. Manual Offset Control
Instead of relying on the consumer to periodically commit consumed offsets, users can also control when records should be considered as consumed and hence commit their offsets. This is useful when the consumption of the messages is coupled with some processing logic and hence a message should not be considered as consumed until it is completed processing.
Properties props = new Properties();
props.put("bootstrap.servers", "localhost:9092");
props.put("group.id", "test");
props.put("enable.auto.commit", "false");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
consumer.subscribe(Arrays.asList("foo", "bar"));
final int minBatchSize = 200;
List<ConsumerRecord<String, String>> buffer = new ArrayList<>();
while (true) {
ConsumerRecords<String, String> records = consumer.poll(100);
for (ConsumerRecord<String, String> record : records) {
buffer.add(record);
}
if (buffer.size() >= minBatchSize) {
insertIntoDb(buffer);
consumer.commitSync();
buffer.clear();
}
} In this example we will consume a batch of records and batch them up in memory. When we have enough records batched, we will insert them into a database. If we allowed offsets to auto commit as in the previous example, records would be considered consumed after they were returned to the user in poll. It would then be possible for our process to fail after batching the records, but before they had been inserted into the database.
To avoid this, we will manually commit the offsets only after the corresponding records have been inserted into the database. This gives us exact control of when a record is considered consumed. This raises the opposite possibility: the process could fail in the interval after the insert into the database but before the commit (even though this would likely just be a few milliseconds, it is a possibility). In this case the process that took over consumption would consume from last committed offset and would repeat the insert of the last batch of data. Used in this way Kafka provides what is often called "at-least-once" delivery guarantees, as each record will likely be delivered one time but in failure cases could be duplicated. Note: Using automatic offset commits can also give you "at-least-once" delivery, but the requirement is that you must consume all data returned from each call to poll(Duration) before any subsequent calls, or before closing the consumer. If you fail to do either of these, it is possible for the committed offset to get ahead of the consumed position, which results in missing records. The advantage of using manual offset control is that you have direct control over when a record is considered "consumed." The above example uses commitSync to mark all received records as committed. In some cases you may wish to have even finer control over which records have been committed by specifying an offset explicitly. In the example below we commit offset after we finish handling the records in each partition. try {
while(running) {
ConsumerRecords<String, String> records = consumer.poll(Long.MAX_VALUE);
for (TopicPartition partition : records.partitions()) {
List<ConsumerRecord<String, String>> partitionRecords = records.records(partition);
for (ConsumerRecord<String, String> record : partitionRecords) {
System.out.println(record.offset() + ": " + record.value());
}
long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset();
consumer.commitSync(Collections.singletonMap(partition, new OffsetAndMetadata(lastOffset + 1)));
}
}
} finally {
consumer.close();
} Note: The committed offset should always be the offset of the next message that your application will read. Thus, when calling commitSync(offsets) you should add one to the offset of the last message processed.
Manual Partition Assignment
In the previous examples, we subscribed to the topics we were interested in and let Kafka dynamically assign a fair share of the partitions for those topics based on the active consumers in the group. However, in some cases you may need finer control over the specific partitions that are assigned. For example:
If the process is maintaining some kind of local state associated with that partition (like a local on-disk key-value store), then it should only get records for the partition it is maintaining on disk.
If the process itself is highly available and will be restarted if it fails (perhaps using a cluster management framework like YARN, Mesos, or AWS facilities, or as part of a stream processing framework). In this case there is no need for Kafka to detect the failure and reassign the partition since the consuming process will be restarted on another machine.
To use this mode, instead of subscribing to the topic using subscribe, you just call assign(Collection) with the full list of partitions that you want to consume. String topic = "foo";
TopicPartition partition0 = new TopicPartition(topic, 0);
TopicPartition partition1 = new TopicPartition(topic, 1);
consumer.assign(Arrays.asList(partition0, partition1)); Once assigned, you can call poll in a loop, just as in the preceding examples to consume records. The group that the consumer specifies is still used for committing offsets, but now the set of partitions will only change with another call to assign. Manual partition assignment does not use group coordination, so consumer failures will not cause assigned partitions to be rebalanced. Each consumer acts independently even if it shares a groupId with another consumer. To avoid offset commit conflicts, you should usually ensure that the groupId is unique for each consumer instance.
Note that it isn't possible to mix manual partition assignment (i.e. using assign) with dynamic partition assignment through topic subscription (i.e. using subscribe). Storing Offsets Outside Kafka
The consumer application need not use Kafka's built-in offset storage, it can store offsets in a store of its own choosing. The primary use case for this is allowing the application to store both the offset and the results of the consumption in the same system in a way that both the results and offsets are stored atomically. This is not always possible, but when it is it will make the consumption fully atomic and give "exactly once" semantics that are stronger than the default "at-least once" semantics you get with Kafka's offset commit functionality.
Here are a couple of examples of this type of usage: If the results of the consumption are being stored in a relational database, storing the offset in the database as well can allow committing both the results and offset in a single transaction. Thus either the transaction will succeed and the offset will be updated based on what was consumed or the result will not be stored and the offset won't be updated.
If the results are being stored in a local store it may be possible to store the offset there as well. For example a search index could be built by subscribing to a particular partition and storing both the offset and the indexed data together. If this is done in a way that is atomic, it is often possible to have it be the case that even if a crash occurs that causes unsync'd data to be lost, whatever is left has the corresponding offset stored as well. This means that in this case the indexing process that comes back having lost recent updates just resumes indexing from what it has ensuring that no updates are lost.
Each record comes with its own offset, so to manage your own offset you just need to do the following: Configure enable.auto.commit=false
Use the offset provided with each ConsumerRecord to save your position.
On restart restore the position of the consumer using seek(TopicPartition, long).
This type of usage is simplest when the partition assignment is also done manually (this would be likely in the search index use case described above). If the partition assignment is done automatically special care is needed to handle the case where partition assignments change. This can be done by providing a ConsumerRebalanceListener instance in the call to subscribe(Collection, ConsumerRebalanceListener) and subscribe(Pattern, ConsumerRebalanceListener). For example, when partitions are taken from a consumer the consumer will want to commit its offset for those partitions by implementing ConsumerRebalanceListener.onPartitionsRevoked(Collection). When partitions are assigned to a consumer, the consumer will want to look up the offset for those new partitions and correctly initialize the consumer to that position by implementing ConsumerRebalanceListener.onPartitionsAssigned(Collection). Another common use for ConsumerRebalanceListener is to flush any caches the application maintains for partitions that are moved elsewhere. Controlling The Consumer's Position
In most use cases the consumer will simply consume records from beginning to end, periodically committing its position (either automatically or manually). However Kafka allows the consumer to manually control its position, moving forward or backwards in a partition at will. This means a consumer can re-consume older records, or skip to the most recent records without actually consuming the intermediate records.
There are several instances where manually controlling the consumer's position can be useful. One case is for time-sensitive record processing it may make sense for a consumer that falls far enough behind to not attempt to catch up processing all records, but rather just skip to the most recent records. Another use case is for a system that maintains local state as described in the previous section. In such a system the consumer will want to initialize its position on start-up to whatever is contained in the local store. Likewise if the local state is destroyed (say because the disk is lost) the state may be recreated on a new machine by re-consuming all the data and recreating the state (assuming that Kafka is retaining sufficient history). Kafka allows specifying the position using seek(TopicPartition, long) to specify the new position. Special methods for seeking to the earliest and latest offset the server maintains are also available ( seekToBeginning(Collection) and seekToEnd(Collection) respectively). Consumption Flow Control
If a consumer is assigned multiple partitions to fetch data from, it will try to consume from all of them at the same time, effectively giving these partitions the same priority for consumption. However in some cases consumers may want to first focus on fetching from some subset of the assigned partitions at full speed, and only start fetching other partitions when these partitions have few or no data to consume.
One of such cases is stream processing, where processor fetches from two topics and performs the join on these two streams. When one of the topics is long lagging behind the other, the processor would like to pause fetching from the ahead topic in order to get the lagging stream to catch up. Another example is bootstraping upon consumer starting up where there are a lot of history data to catch up, the applications usually want to get the latest data on some of the topics before consider fetching other topics. Kafka supports dynamic controlling of consumption flows by using pause(Collection) and resume(Collection) to pause the consumption on the specified assigned partitions and resume the consumption on the specified paused partitions respectively in the future poll(Duration) calls. Reading Transactional Messages
Transactions were introduced in Kafka 0.11.0 wherein applications can write to multiple topics and partitions atomically. In order for this to work, consumers reading from these partitions should be configured to only read committed data. This can be achieved by setting the isolation.level=read_committed in the consumer's configuration. In read_committed mode, the consumer will read only those transactional messages which have been successfully committed. It will continue to read non-transactional messages as before. There is no client-side buffering in read_committed mode. Instead, the end offset of a partition for a read_committed consumer would be the offset of the first message in the partition belonging to an open transaction. This offset is known as the 'Last Stable Offset'(LSO). A read_committed consumer will only read up to the LSO and filter out any transactional messages which have been aborted. The LSO also affects the behavior of seekToEnd(Collection) and endOffsets(Collection) for read_committed consumers, details of which are in each method's documentation. Finally, the fetch lag metrics are also adjusted to be relative to the LSO for read_committed consumers. Partitions with transactional messages will include commit or abort markers which indicate the result of a transaction. There markers are not returned to applications, yet have an offset in the log. As a result, applications reading from topics with transactional messages will see gaps in the consumed offsets. These missing messages would be the transaction markers, and they are filtered out for consumers in both isolation levels. Additionally, applications using read_committed consumers may also see gaps due to aborted transactions, since those messages would not be returned by the consumer and yet would have valid offsets. Multi-threaded Processing
The Kafka consumer is NOT thread-safe. All network I/O happens in the thread of the application making the call. It is the responsibility of the user to ensure that multi-threaded access is properly synchronized. Un-synchronized access will result in ConcurrentModificationException.
The only exception to this rule is wakeup(), which can safely be used from an external thread to interrupt an active operation. In this case, a WakeupException will be thrown from the thread blocking on the operation. This can be used to shutdown the consumer from another thread. The following snippet shows the typical pattern: public class KafkaConsumerRunner implements Runnable {
private final AtomicBoolean closed = new AtomicBoolean(false);
private final KafkaConsumer consumer; public void run() {
try {
consumer.subscribe(Arrays.asList("topic"));
while (!closed.get()) {
ConsumerRecords records = consumer.poll(Duration.ofMillis(10000));
// Handle new records
}
} catch (WakeupException e) {
// Ignore exception if closing
if (!closed.get()) throw e;
} finally {
consumer.close();
}
} // Shutdown hook which can be called from a separate thread
public void shutdown() {
closed.set(true);
consumer.wakeup();
}
} Then in a separate thread, the consumer can be shutdown by setting the closed flag and waking up the consumer.
closed.set(true);
consumer.wakeup(); Note that while it is possible to use thread interrupts instead of wakeup() to abort a blocking operation (in which case, InterruptException will be raised), we discourage their use since they may cause a clean shutdown of the consumer to be aborted. Interrupts are mainly supported for those cases where using wakeup() is impossible, e.g. when a consumer thread is managed by code that is unaware of the Kafka client. We have intentionally avoided implementing a particular threading model for processing. This leaves several options for implementing multi-threaded processing of records. 1. One Consumer Per Thread
A simple option is to give each thread its own consumer instance. Here are the pros and cons of this approach:
PRO: It is the easiest to implement
PRO: It is often the fastest as no inter-thread co-ordination is needed
PRO: It makes in-order processing on a per-partition basis very easy to implement (each thread just processes messages in the order it receives them).
CON: More consumers means more TCP connections to the cluster (one per thread). In general Kafka handles connections very efficiently so this is generally a small cost.
CON: Multiple consumers means more requests being sent to the server and slightly less batching of data which can cause some drop in I/O throughput.
CON: The number of total threads across all processes will be limited by the total number of partitions.
2. Decouple Consumption and Processing
Another alternative is to have one or more consumer threads that do all data consumption and hands off ConsumerRecords instances to a blocking queue consumed by a pool of processor threads that actually handle the record processing. This option likewise has pros and cons:
PRO: This option allows independently scaling the number of consumers and processors. This makes it possible to have a single consumer that feeds many processor threads, avoiding any limitation on partitions.
CON: Guaranteeing order across the processors requires particular care as the threads will execute independently an earlier chunk of data may actually be processed after a later chunk of data just due to the luck of thread execution timing. For processing that has no ordering requirements this is not a problem.
CON: Manually committing the position becomes harder as it requires that all threads co-ordinate to ensure that processing is complete for that partition.
There are many possible variations on this approach. For example each processor thread can have its own queue, and the consumer threads can hash into these queues using the TopicPartition to ensure in-order consumption and simplify commit.
Constructor Summary
Constructors
Constructor Description
KafkaConsumer(java.util.Map<java.lang.String,java.lang.Object> configs)
A consumer is instantiated by providing a set of key-value pairs as configuration.
KafkaConsumer(java.util.Map<java.lang.String,java.lang.Object> configs, Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer)
A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value Deserializer.
KafkaConsumer(java.util.Properties properties)
A consumer is instantiated by providing a Properties object as configuration.
KafkaConsumer(java.util.Properties properties, Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer)
A consumer is instantiated by providing a Properties object as configuration, and a key and a value Deserializer.
Method Summary
All MethodsInstance MethodsConcrete MethodsDeprecated Methods
Modifier and Type Method Description
void assign(java.util.Collection<TopicPartition> partitions)
Manually assign a list of partitions to this consumer.
java.util.Set<TopicPartition> assignment()
Get the set of partitions currently assigned to this consumer.
java.util.Map<TopicPartition,java.lang.Long> beginningOffsets(java.util.Collection<TopicPartition> partitions)
Get the first offset for the given partitions.
java.util.Map<TopicPartition,java.lang.Long> beginningOffsets(java.util.Collection<TopicPartition> partitions, java.time.Duration timeout)
Get the first offset for the given partitions.
void close()
Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup.
void close(long timeout, java.util.concurrent.TimeUnit timeUnit)
Deprecated.
Since 2.0. Use close(Duration) or close().
void close(java.time.Duration timeout)
Tries to close the consumer cleanly within the specified timeout.
void commitAsync()
Commit offsets returned on the last poll(Duration) for all the subscribed list of topics and partition.
void commitAsync(java.util.Map<TopicPartition,OffsetAndMetadata> offsets, OffsetCommitCallback callback)
Commit the specified offsets for the specified list of topics and partitions to Kafka.
void commitAsync(OffsetCommitCallback callback)
Commit offsets returned on the last poll() for the subscribed list of topics and partitions.
void commitSync()
Commit offsets returned on the last poll() for all the subscribed list of topics and partitions.
void commitSync(java.time.Duration timeout)
Commit offsets returned on the last poll() for all the subscribed list of topics and partitions.
void commitSync(java.util.Map<TopicPartition,OffsetAndMetadata> offsets)
Commit the specified offsets for the specified list of topics and partitions.
void commitSync(java.util.Map<TopicPartition,OffsetAndMetadata> offsets, java.time.Duration timeout)
Commit the specified offsets for the specified list of topics and partitions.
OffsetAndMetadata committed(TopicPartition partition)
Get the last committed offset for the given partition (whether the commit happened by this process or another).
OffsetAndMetadata committed(TopicPartition partition, java.time.Duration timeout)
Get the last committed offset for the given partition (whether the commit happened by this process or another).
java.util.Map<TopicPartition,java.lang.Long> endOffsets(java.util.Collection<TopicPartition> partitions)
Get the end offsets for the given partitions.
java.util.Map<TopicPartition,java.lang.Long> endOffsets(java.util.Collection<TopicPartition> partitions, java.time.Duration timeout)
Get the end offsets for the given partitions.
java.util.Map<java.lang.String,java.util.List<PartitionInfo>> listTopics()
Get metadata about partitions for all topics that the user is authorized to view.
java.util.Map<java.lang.String,java.util.List<PartitionInfo>> listTopics(java.time.Duration timeout)
Get metadata about partitions for all topics that the user is authorized to view.
java.util.Map<MetricName,? extends Metric> metrics()
Get the metrics kept by the consumer
java.util.Map<TopicPartition,OffsetAndTimestamp> offsetsForTimes(java.util.Map<TopicPartition,java.lang.Long> timestampsToSearch)
Look up the offsets for the given partitions by timestamp.
java.util.Map<TopicPartition,OffsetAndTimestamp> offsetsForTimes(java.util.Map<TopicPartition,java.lang.Long> timestampsToSearch, java.time.Duration timeout)
Look up the offsets for the given partitions by timestamp.
java.util.List<PartitionInfo> partitionsFor(java.lang.String topic)
Get metadata about the partitions for a given topic.
java.util.List<PartitionInfo> partitionsFor(java.lang.String topic, java.time.Duration timeout)
Get metadata about the partitions for a given topic.
void pause(java.util.Collection<TopicPartition> partitions)
Suspend fetching from the requested partitions.
java.util.Set<TopicPartition> paused()
Get the set of partitions that were previously paused by a call to pause(Collection).
ConsumerRecords<K,V> poll(long timeout)
Deprecated.
Since 2.0. Use poll(Duration), which does not block beyond the timeout awaiting partition assignment. See KIP-266 for more information.
ConsumerRecords<K,V> poll(java.time.Duration timeout)
Fetch data for the topics or partitions specified using one of the subscribe/assign APIs.
long position(TopicPartition partition)
Get the offset of the next record that will be fetched (if a record with that offset exists).
long position(TopicPartition partition, java.time.Duration timeout)
Get the offset of the next record that will be fetched (if a record with that offset exists).
void resume(java.util.Collection<TopicPartition> partitions)
Resume specified partitions which have been paused with pause(Collection).
void seek(TopicPartition partition, long offset)
Overrides the fetch offsets that the consumer will use on the next poll(timeout).
void seekToBeginning(java.util.Collection<TopicPartition> partitions)
Seek to the first offset for each of the given partitions.
void seekToEnd(java.util.Collection<TopicPartition> partitions)
Seek to the last offset for each of the given partitions.
void subscribe(java.util.Collection<java.lang.String> topics)
Subscribe to the given list of topics to get dynamically assigned partitions.
void subscribe(java.util.Collection<java.lang.String> topics, ConsumerRebalanceListener listener)
Subscribe to the given list of topics to get dynamically assigned partitions.
void subscribe(java.util.regex.Pattern pattern)
Subscribe to all topics matching specified pattern to get dynamically assigned partitions.
void subscribe(java.util.regex.Pattern pattern, ConsumerRebalanceListener listener)
Subscribe to all topics matching specified pattern to get dynamically assigned partitions.
java.util.Set<java.lang.String> subscription()
Get the current subscription.
void unsubscribe()
Unsubscribe from topics currently subscribed with subscribe(Collection) or subscribe(Pattern).
void wakeup()
Wakeup the consumer.
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
Constructor Detail
KafkaConsumer
public KafkaConsumer(java.util.Map<java.lang.String,java.lang.Object> configs)
A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings are documented here. Values can be either strings or objects of the appropriate type (for example a numeric configuration would accept either the string "42" or the integer 42).
Valid configuration strings are documented at ConsumerConfig. Note: after creating a KafkaConsumer you must always close() it to avoid resource leaks. Parameters:
configs - The consumer configs
KafkaConsumer
public KafkaConsumer(java.util.Map<java.lang.String,java.lang.Object> configs,
Deserializer<K> keyDeserializer,
Deserializer<V> valueDeserializer)
A consumer is instantiated by providing a set of key-value pairs as configuration, and a key and a value Deserializer.
Valid configuration strings are documented at ConsumerConfig. Note: after creating a KafkaConsumer you must always close() it to avoid resource leaks. Parameters:
configs - The consumer configs
keyDeserializer - The deserializer for key that implements Deserializer. The configure() method won't be called in the consumer when the deserializer is passed in directly.
valueDeserializer - The deserializer for value that implements Deserializer. The configure() method won't be called in the consumer when the deserializer is passed in directly.
KafkaConsumer
public KafkaConsumer(java.util.Properties properties)
A consumer is instantiated by providing a Properties object as configuration.
Valid configuration strings are documented at ConsumerConfig. Note: after creating a KafkaConsumer you must always close() it to avoid resource leaks. Parameters:
properties - The consumer configuration properties
KafkaConsumer
public KafkaConsumer(java.util.Properties properties,
Deserializer<K> keyDeserializer,
Deserializer<V> valueDeserializer)
A consumer is instantiated by providing a Properties object as configuration, and a key and a value Deserializer.
Valid configuration strings are documented at ConsumerConfig. Note: after creating a KafkaConsumer you must always close() it to avoid resource leaks. Parameters:
properties - The consumer configuration properties
keyDeserializer - The deserializer for key that implements Deserializer. The configure() method won't be called in the consumer when the deserializer is passed in directly.
valueDeserializer - The deserializer for value that implements Deserializer. The configure() method won't be called in the consumer when the deserializer is passed in directly.
Method Detail
assignment
public java.util.Set<TopicPartition> assignment()
Get the set of partitions currently assigned to this consumer. If subscription happened by directly assigning partitions using assign(Collection) then this will simply return the same partitions that were assigned. If topic subscription was used, then this will give the set of topic partitions currently assigned to the consumer (which may be none if the assignment hasn't happened yet, or the partitions are in the process of getting reassigned).
Specified by:
assignment in interface Consumer<K,V>
Returns:
The set of partitions currently assigned to this consumer
See Also:
assignment()
subscription
public java.util.Set<java.lang.String> subscription()
Get the current subscription. Will return the same topics used in the most recent call to subscribe(Collection, ConsumerRebalanceListener), or an empty set if no such call has been made.
Specified by:
subscription in interface Consumer<K,V>
Returns:
The set of topics currently subscribed to
See Also:
subscription()
subscribe
public void subscribe(java.util.Collection<java.lang.String> topics,
ConsumerRebalanceListener listener)
Subscribe to the given list of topics to get dynamically assigned partitions. Topic subscriptions are not incremental. This list will replace the current assignment (if there is one). Note that it is not possible to combine topic subscription with group management with manual partition assignment through assign(Collection). If the given list of topics is empty, it is treated the same as unsubscribe().
As part of group management, the consumer will keep track of the list of consumers that belong to a particular group and will trigger a rebalance operation if any one of the following events are triggered: Number of partitions change for any of the subscribed topics
A subscribed topic is created or deleted
An existing member of the consumer group is shutdown or fails
A new member is added to the consumer group
When any of these events are triggered, the provided listener will be invoked first to indicate that the consumer's assignment has been revoked, and then again when the new assignment has been received. Note that rebalances will only occur during an active call to poll(Duration), so callbacks will also only be invoked during that time. The provided listener will immediately override any listener set in a previous call to subscribe. It is guaranteed, however, that the partitions revoked/assigned through this interface are from topics subscribed in this call. See ConsumerRebalanceListener for more details. Specified by:
subscribe in interface Consumer<K,V>
Parameters:
topics - The list of topics to subscribe to
listener - Non-null listener instance to get notifications on partition assignment/revocation for the subscribed topics
Throws:
java.lang.IllegalArgumentException - If topics is null or contains null or empty elements, or if listener is null
java.lang.IllegalStateException - If subscribe() is called previously with pattern, or assign is called previously (without a subsequent call to unsubscribe()), or if not configured at-least one partition assignment strategy
See Also:
subscribe(Collection, ConsumerRebalanceListener)
subscribe
public void subscribe(java.util.Collection<java.lang.String> topics)
Subscribe to the given list of topics to get dynamically assigned partitions. Topic subscriptions are not incremental. This list will replace the current assignment (if there is one). It is not possible to combine topic subscription with group management with manual partition assignment through assign(Collection). If the given list of topics is empty, it is treated the same as unsubscribe().
This is a short-hand for subscribe(Collection, ConsumerRebalanceListener), which uses a no-op listener. If you need the ability to seek to particular offsets, you should prefer subscribe(Collection, ConsumerRebalanceListener), since group rebalances will cause partition offsets to be reset. You should also provide your own listener if you are doing your own offset management since the listener gives you an opportunity to commit offsets before a rebalance finishes. Specified by:
subscribe in interface Consumer<K,V>
Parameters:
topics - The list of topics to subscribe to
Throws:
java.lang.IllegalArgumentException - If topics is null or contains null or empty elements
java.lang.IllegalStateException - If subscribe() is called previously with pattern, or assign is called previously (without a subsequent call to unsubscribe()), or if not configured at-least one partition assignment strategy
See Also:
subscribe(Collection)
subscribe
public void subscribe(java.util.regex.Pattern pattern,
ConsumerRebalanceListener listener)
Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against all topics existing at the time of check. This can be controlled through the metadata.max.age.ms configuration: by lowering the max metadata age, the consumer will refresh metadata more often and check for matching topics.
See subscribe(Collection, ConsumerRebalanceListener) for details on the use of the ConsumerRebalanceListener. Generally rebalances are triggered when there is a change to the topics matching the provided pattern and when consumer group membership changes. Group rebalances only take place during an active call to poll(Duration). Specified by:
subscribe in interface Consumer<K,V>
Parameters:
pattern - Pattern to subscribe to
listener - Non-null listener instance to get notifications on partition assignment/revocation for the subscribed topics
Throws:
java.lang.IllegalArgumentException - If pattern or listener is null
java.lang.IllegalStateException - If subscribe() is called previously with topics, or assign is called previously (without a subsequent call to unsubscribe()), or if not configured at-least one partition assignment strategy
See Also:
subscribe(Pattern, ConsumerRebalanceListener)
subscribe
public void subscribe(java.util.regex.Pattern pattern)
Subscribe to all topics matching specified pattern to get dynamically assigned partitions. The pattern matching will be done periodically against topics existing at the time of check.
This is a short-hand for subscribe(Pattern, ConsumerRebalanceListener), which uses a no-op listener. If you need the ability to seek to particular offsets, you should prefer subscribe(Pattern, ConsumerRebalanceListener), since group rebalances will cause partition offsets to be reset. You should also provide your own listener if you are doing your own offset management since the listener gives you an opportunity to commit offsets before a rebalance finishes. Specified by:
subscribe in interface Consumer<K,V>
Parameters:
pattern - Pattern to subscribe to
Throws:
java.lang.IllegalArgumentException - If pattern is null
java.lang.IllegalStateException - If subscribe() is called previously with topics, or assign is called previously (without a subsequent call to unsubscribe()), or if not configured at-least one partition assignment strategy
See Also:
subscribe(Pattern)
unsubscribe
public void unsubscribe()
Unsubscribe from topics currently subscribed with subscribe(Collection) or subscribe(Pattern). This also clears any partitions directly assigned through assign(Collection).
Specified by:
unsubscribe in interface Consumer<K,V>
See Also:
unsubscribe()
assign
public void assign(java.util.Collection<TopicPartition> partitions)
Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment and will replace the previous assignment (if there is one).
If the given list of topic partitions is empty, it is treated the same as unsubscribe(). Manual topic assignment through this method does not use the consumer's group management functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic metadata change. Note that it is not possible to use both manual partition assignment with assign(Collection) and group assignment with subscribe(Collection, ConsumerRebalanceListener). If auto-commit is enabled, an async commit (based on the old assignment) will be triggered before the new assignment replaces the old one. Specified by:
assign in interface Consumer<K,V>
Parameters:
partitions - The list of partitions to assign this consumer
Throws:
java.lang.IllegalArgumentException - If partitions is null or contains null or empty topics
java.lang.IllegalStateException - If subscribe() is called previously with topics or pattern (without a subsequent call to unsubscribe())
See Also:
assign(Collection)
poll
@Deprecated
public ConsumerRecords<K,V> poll(long timeout)
Deprecated. Since 2.0. Use poll(Duration), which does not block beyond the timeout awaiting partition assignment. See KIP-266 for more information.
Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have subscribed to any topics or partitions before polling for data.
On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last consumed offset can be manually set through seek(TopicPartition, long) or automatically set as the last committed offset for the subscribed list of partitions Specified by:
poll in interface Consumer<K,V>
Parameters:
timeout - The time, in milliseconds, spent waiting in poll if data is not available in the buffer. If 0, returns immediately with any records that are available currently in the buffer, else returns empty. Must not be negative.
Returns:
map of topic to records since the last fetch for the subscribed list of topics and partitions
Throws:
InvalidOffsetException - if the offset for a partition or set of partitions is undefined or out of range and no offset reset policy has been configured
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if caller lacks Read access to any of the subscribed topics or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors (e.g. invalid groupId or session timeout, errors deserializing key/value pairs, or any new error cases in future versions)
java.lang.IllegalArgumentException - if the timeout value is negative
java.lang.IllegalStateException - if the consumer is not subscribed to any topics or manually assigned any partitions to consume from
See Also:
poll(long)
poll
public ConsumerRecords<K,V> poll(java.time.Duration timeout)
Fetch data for the topics or partitions specified using one of the subscribe/assign APIs. It is an error to not have subscribed to any topics or partitions before polling for data.
On each poll, consumer will try to use the last consumed offset as the starting offset and fetch sequentially. The last consumed offset can be manually set through seek(TopicPartition, long) or automatically set as the last committed offset for the subscribed list of partitions This method returns immediately if there are records available. Otherwise, it will await the passed timeout. If the timeout expires, an empty record set will be returned. Note that this method may block beyond the timeout in order to execute custom ConsumerRebalanceListener callbacks. Specified by:
poll in interface Consumer<K,V>
Parameters:
timeout - The maximum time to block (must not be greater than Long.MAX_VALUE milliseconds)
Returns:
map of topic to records since the last fetch for the subscribed list of topics and partitions
Throws:
InvalidOffsetException - if the offset for a partition or set of partitions is undefined or out of range and no offset reset policy has been configured
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if caller lacks Read access to any of the subscribed topics or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors (e.g. invalid groupId or session timeout, errors deserializing key/value pairs, or any new error cases in future versions)
java.lang.IllegalArgumentException - if the timeout value is negative
java.lang.IllegalStateException - if the consumer is not subscribed to any topics or manually assigned any partitions to consume from
java.lang.ArithmeticException - if the timeout is greater than Long.MAX_VALUE milliseconds.
See Also:
poll(Duration)
commitSync
public void commitSync()
Commit offsets returned on the last poll() for all the subscribed list of topics and partitions.
This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. This is a synchronous commit and will block until either the commit succeeds, an unrecoverable error is encountered (in which case it is thrown to the caller), or the timeout specified by default.api.timeout.ms expires (in which case a TimeoutException is thrown to the caller). Note that asynchronous offset commits sent previously with the commitAsync(OffsetCommitCallback) (or similar) are guaranteed to have their callbacks invoked prior to completion of this method. Specified by:
commitSync in interface Consumer<K,V>
Throws:
CommitFailedException - if the commit failed and cannot be retried. This can only occur if you are using automatic group management with subscribe(Collection), or if there is an active group with the same groupId which is using group management.
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors (e.g. if offset metadata is too large or if the topic does not exist).
TimeoutException - if the timeout specified by default.api.timeout.ms expires before successful completion of the offset commit
See Also:
commitSync()
commitSync
public void commitSync(java.time.Duration timeout)
Commit offsets returned on the last poll() for all the subscribed list of topics and partitions.
This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. This is a synchronous commits and will block until either the commit succeeds, an unrecoverable error is encountered (in which case it is thrown to the caller), or the passed timeout expires. Note that asynchronous offset commits sent previously with the commitAsync(OffsetCommitCallback) (or similar) are guaranteed to have their callbacks invoked prior to completion of this method. Specified by:
commitSync in interface Consumer<K,V>
Throws:
CommitFailedException - if the commit failed and cannot be retried. This can only occur if you are using automatic group management with subscribe(Collection), or if there is an active group with the same groupId which is using group management.
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors (e.g. if offset metadata is too large or if the topic does not exist).
TimeoutException - if the timeout expires before successful completion of the offset commit
See Also:
commitSync(Duration)
commitSync
public void commitSync(java.util.Map<TopicPartition,OffsetAndMetadata> offsets)
Commit the specified offsets for the specified list of topics and partitions.
This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. The committed offset should be the next message your application will consume, i.e. lastProcessedMessageOffset + 1. This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is encountered (in which case it is thrown to the caller), or the timeout specified by default.api.timeout.ms expires (in which case a TimeoutException is thrown to the caller). Note that asynchronous offset commits sent previously with the commitAsync(OffsetCommitCallback) (or similar) are guaranteed to have their callbacks invoked prior to completion of this method. Specified by:
commitSync in interface Consumer<K,V>
Parameters:
offsets - A map of offsets by partition with associated metadata
Throws:
CommitFailedException - if the commit failed and cannot be retried. This can only occur if you are using automatic group management with subscribe(Collection), or if there is an active group with the same groupId which is using group management.
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
java.lang.IllegalArgumentException - if the committed offset is negative
KafkaException - for any other unrecoverable errors (e.g. if offset metadata is too large or if the topic does not exist).
TimeoutException - if the timeout expires before successful completion of the offset commit
See Also:
commitSync(Map)
commitSync
public void commitSync(java.util.Map<TopicPartition,OffsetAndMetadata> offsets,
java.time.Duration timeout)
Commit the specified offsets for the specified list of topics and partitions.
This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. The committed offset should be the next message your application will consume, i.e. lastProcessedMessageOffset + 1. This is a synchronous commits and will block until either the commit succeeds, an unrecoverable error is encountered (in which case it is thrown to the caller), or the timeout expires. Note that asynchronous offset commits sent previously with the commitAsync(OffsetCommitCallback) (or similar) are guaranteed to have their callbacks invoked prior to completion of this method. Specified by:
commitSync in interface Consumer<K,V>
Parameters:
offsets - A map of offsets by partition with associated metadata
timeout - The maximum amount of time to await completion of the offset commit
Throws:
CommitFailedException - if the commit failed and cannot be retried. This can only occur if you are using automatic group management with subscribe(Collection), or if there is an active group with the same groupId which is using group management.
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
java.lang.IllegalArgumentException - if the committed offset is negative
KafkaException - for any other unrecoverable errors (e.g. if offset metadata is too large or if the topic does not exist).
TimeoutException - if the timeout expires before successful completion of the offset commit
See Also:
commitSync(Map, Duration)
commitAsync
public void commitAsync()
Commit offsets returned on the last poll(Duration) for all the subscribed list of topics and partition. Same as commitAsync(null)
Specified by:
commitAsync in interface Consumer<K,V>
See Also:
commitAsync()
commitAsync
public void commitAsync(OffsetCommitCallback callback)
Commit offsets returned on the last poll() for the subscribed list of topics and partitions.
This commits offsets only to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. This is an asynchronous call and will not block. Any errors encountered are either passed to the callback (if provided) or discarded. Offsets committed through multiple calls to this API are guaranteed to be sent in the same order as the invocations. Corresponding commit callbacks are also invoked in the same order. Additionally note that offsets committed through this API are guaranteed to complete before a subsequent call to commitSync() (and variants) returns. Specified by:
commitAsync in interface Consumer<K,V>
Parameters:
callback - Callback to invoke when the commit completes
See Also:
commitAsync(OffsetCommitCallback)
commitAsync
public void commitAsync(java.util.Map<TopicPartition,OffsetAndMetadata> offsets,
OffsetCommitCallback callback)
Commit the specified offsets for the specified list of topics and partitions to Kafka.
This commits offsets to Kafka. The offsets committed using this API will be used on the first fetch after every rebalance and also on startup. As such, if you need to store offsets in anything other than Kafka, this API should not be used. The committed offset should be the next message your application will consume, i.e. lastProcessedMessageOffset + 1. This is an asynchronous call and will not block. Any errors encountered are either passed to the callback (if provided) or discarded. Offsets committed through multiple calls to this API are guaranteed to be sent in the same order as the invocations. Corresponding commit callbacks are also invoked in the same order. Additionally note that offsets committed through this API are guaranteed to complete before a subsequent call to commitSync() (and variants) returns. Specified by:
commitAsync in interface Consumer<K,V>
Parameters:
offsets - A map of offsets by partition with associate metadata. This map will be copied internally, so it is safe to mutate the map after returning.
callback - Callback to invoke when the commit completes
See Also:
commitAsync(Map, OffsetCommitCallback)
seek
public void seek(TopicPartition partition,
long offset)
Overrides the fetch offsets that the consumer will use on the next poll(timeout). If this API is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets
Specified by:
seek in interface Consumer<K,V>
Throws:
java.lang.IllegalArgumentException - if the provided offset is negative
java.lang.IllegalStateException - if the provided TopicPartition is not assigned to this consumer
See Also:
seek(TopicPartition, long)
seekToBeginning
public void seekToBeginning(java.util.Collection<TopicPartition> partitions)
Seek to the first offset for each of the given partitions. This function evaluates lazily, seeking to the first offset in all partitions only when poll(Duration) or position(TopicPartition) are called. If no partitions are provided, seek to the first offset for all of the currently assigned partitions.
Specified by:
seekToBeginning in interface Consumer<K,V>
Throws:
java.lang.IllegalArgumentException - if partitions is null
java.lang.IllegalStateException - if any of the provided partitions are not currently assigned to this consumer
See Also:
seekToBeginning(Collection)
seekToEnd
public void seekToEnd(java.util.Collection<TopicPartition> partitions)
Seek to the last offset for each of the given partitions. This function evaluates lazily, seeking to the final offset in all partitions only when poll(Duration) or position(TopicPartition) are called. If no partitions are provided, seek to the final offset for all of the currently assigned partitions.
If isolation.level=read_committed, the end offset will be the Last Stable Offset, i.e., the offset of the first message with an open transaction. Specified by:
seekToEnd in interface Consumer<K,V>
Throws:
java.lang.IllegalArgumentException - if partitions is null
java.lang.IllegalStateException - if any of the provided partitions are not currently assigned to this consumer
See Also:
seekToEnd(Collection)
position
public long position(TopicPartition partition)
Get the offset of the next record that will be fetched (if a record with that offset exists). This method may issue a remote call to the server if there is no current position for the given partition.
This call will block until either the position could be determined or an unrecoverable error is encountered (in which case it is thrown to the caller), or the timeout specified by default.api.timeout.ms expires (in which case a TimeoutException is thrown to the caller). Specified by:
position in interface Consumer<K,V>
Parameters:
partition - The partition to get the position for
Returns:
The current position of the consumer (that is, the offset of the next record to be fetched)
Throws:
java.lang.IllegalStateException - if the provided TopicPartition is not assigned to this consumer
InvalidOffsetException - if no offset is currently defined for the partition
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors
TimeoutException - if the position cannot be determined before the timeout specified by default.api.timeout.ms expires
See Also:
position(TopicPartition)
position
public long position(TopicPartition partition,
java.time.Duration timeout)
Get the offset of the next record that will be fetched (if a record with that offset exists). This method may issue a remote call to the server if there is no current position for the given partition.
This call will block until the position can be determined, an unrecoverable error is encountered (in which case it is thrown to the caller), or the timeout expires. Specified by:
position in interface Consumer<K,V>
Parameters:
partition - The partition to get the position for
timeout - The maximum amount of time to await determination of the current position
Returns:
The current position of the consumer (that is, the offset of the next record to be fetched)
Throws:
java.lang.IllegalStateException - if the provided TopicPartition is not assigned to this consumer
InvalidOffsetException - if no offset is currently defined for the partition
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
TimeoutException - if the position cannot be determined before the passed timeout expires
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors
See Also:
position(TopicPartition, Duration)
committed
public OffsetAndMetadata committed(TopicPartition partition)
Get the last committed offset for the given partition (whether the commit happened by this process or another). This offset will be used as the position for the consumer in the event of a failure.
This call will do a remote call to get the latest committed offset from the server, and will block until the committed offset is gotten successfully, an unrecoverable error is encountered (in which case it is thrown to the caller), or the timeout specified by default.api.timeout.ms expires (in which case a TimeoutException is thrown to the caller). Specified by:
committed in interface Consumer<K,V>
Parameters:
partition - The partition to check
Returns:
The last committed offset and metadata or null if there was no prior commit
Throws:
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors
TimeoutException - if the committed offset cannot be found before the timeout specified by default.api.timeout.ms expires.
See Also:
committed(TopicPartition)
committed
public OffsetAndMetadata committed(TopicPartition partition,
java.time.Duration timeout)
Get the last committed offset for the given partition (whether the commit happened by this process or another). This offset will be used as the position for the consumer in the event of a failure.
This call will block to do a remote call to get the latest committed offsets from the server. Specified by:
committed in interface Consumer<K,V>
Parameters:
partition - The partition to check
timeout - The maximum amount of time to await the current committed offset
Returns:
The last committed offset and metadata or null if there was no prior commit
Throws:
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic or to the configured groupId. See the exception for more details
KafkaException - for any other unrecoverable errors
TimeoutException - if the committed offset cannot be found before expiration of the timeout
See Also:
committed(TopicPartition, Duration)
metrics
public java.util.Map<MetricName,? extends Metric> metrics()
Get the metrics kept by the consumer
Specified by:
metrics in interface Consumer<K,V>
See Also:
metrics()
partitionsFor
public java.util.List<PartitionInfo> partitionsFor(java.lang.String topic)
Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it does not already have any metadata about the given topic.
Specified by:
partitionsFor in interface Consumer<K,V>
Parameters:
topic - The topic to get partition metadata for
Returns:
The list of partitions
Throws:
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the specified topic. See the exception for more details
KafkaException - for any other unrecoverable errors
TimeoutException - if the offset metadata could not be fetched before the amount of time allocated by default.api.timeout.ms expires.
See Also:
partitionsFor(String)
partitionsFor
public java.util.List<PartitionInfo> partitionsFor(java.lang.String topic,
java.time.Duration timeout)
Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it does not already have any metadata about the given topic.
Specified by:
partitionsFor in interface Consumer<K,V>
Parameters:
topic - The topic to get partition metadata for
timeout - The maximum of time to await topic metadata
Returns:
The list of partitions
Throws:
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the specified topic. See the exception for more details
TimeoutException - if topic metadata cannot be fetched before expiration of the passed timeout
KafkaException - for any other unrecoverable errors
See Also:
partitionsFor(String, Duration)
listTopics
public java.util.Map<java.lang.String,java.util.List<PartitionInfo>> listTopics()
Get metadata about partitions for all topics that the user is authorized to view. This method will issue a remote call to the server.
Specified by:
listTopics in interface Consumer<K,V>
Returns:
The map of topics and its partitions
Throws:
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
KafkaException - for any other unrecoverable errors
TimeoutException - if the offset metadata could not be fetched before the amount of time allocated by default.api.timeout.ms expires.
See Also:
listTopics()
listTopics
public java.util.Map<java.lang.String,java.util.List<PartitionInfo>> listTopics(java.time.Duration timeout)
Get metadata about partitions for all topics that the user is authorized to view. This method will issue a remote call to the server.
Specified by:
listTopics in interface Consumer<K,V>
Parameters:
timeout - The maximum time this operation will block to fetch topic metadata
Returns:
The map of topics and its partitions
Throws:
WakeupException - if wakeup() is called before or while this function is called
InterruptException - if the calling thread is interrupted before or while this function is called
TimeoutException - if the topic metadata could not be fetched before expiration of the passed timeout
KafkaException - for any other unrecoverable errors
See Also:
listTopics(Duration)
pause
public void pause(java.util.Collection<TopicPartition> partitions)
Suspend fetching from the requested partitions. Future calls to poll(Duration) will not return any records from these partitions until they have been resumed using resume(Collection). Note that this method does not affect partition subscription. In particular, it does not cause a group rebalance when automatic assignment is used.
Specified by:
pause in interface Consumer<K,V>
Parameters:
partitions - The partitions which should be paused
Throws:
java.lang.IllegalStateException - if any of the provided partitions are not currently assigned to this consumer
See Also:
pause(Collection)
resume
public void resume(java.util.Collection<TopicPartition> partitions)
Resume specified partitions which have been paused with pause(Collection). New calls to poll(Duration) will return records from these partitions if there are any to be fetched. If the partitions were not previously paused, this method is a no-op.
Specified by:
resume in interface Consumer<K,V>
Parameters:
partitions - The partitions which should be resumed
Throws:
java.lang.IllegalStateException - if any of the provided partitions are not currently assigned to this consumer
See Also:
resume(Collection)
paused
public java.util.Set<TopicPartition> paused()
Get the set of partitions that were previously paused by a call to pause(Collection).
Specified by:
paused in interface Consumer<K,V>
Returns:
The set of paused partitions
See Also:
paused()
offsetsForTimes
public java.util.Map<TopicPartition,OffsetAndTimestamp> offsetsForTimes(java.util.Map<TopicPartition,java.lang.Long> timestampsToSearch)
Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. This is a blocking call. The consumer does not have to be assigned the partitions. If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null will be returned for that partition.
Specified by:
offsetsForTimes in interface Consumer<K,V>
Parameters:
timestampsToSearch - the mapping from partition to the timestamp to look up.
Returns:
a mapping from partition to the timestamp and offset of the first message with timestamp greater than or equal to the target timestamp. null will be returned for the partition if there is no such message.
Throws:
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic(s). See the exception for more details
java.lang.IllegalArgumentException - if the target timestamp is negative
TimeoutException - if the offset metadata could not be fetched before the amount of time allocated by default.api.timeout.ms expires.
UnsupportedVersionException - if the broker does not support looking up the offsets by timestamp
See Also:
offsetsForTimes(Map)
offsetsForTimes
public java.util.Map<TopicPartition,OffsetAndTimestamp> offsetsForTimes(java.util.Map<TopicPartition,java.lang.Long> timestampsToSearch,
java.time.Duration timeout)
Look up the offsets for the given partitions by timestamp. The returned offset for each partition is the earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding partition. This is a blocking call. The consumer does not have to be assigned the partitions. If the message format version in a partition is before 0.10.0, i.e. the messages do not have timestamps, null will be returned for that partition.
Specified by:
offsetsForTimes in interface Consumer<K,V>
Parameters:
timestampsToSearch - the mapping from partition to the timestamp to look up.
timeout - The maximum amount of time to await retrieval of the offsets
Returns:
a mapping from partition to the timestamp and offset of the first message with timestamp greater than or equal to the target timestamp. null will be returned for the partition if there is no such message.
Throws:
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic(s). See the exception for more details
java.lang.IllegalArgumentException - if the target timestamp is negative
TimeoutException - if the offset metadata could not be fetched before expiration of the passed timeout
UnsupportedVersionException - if the broker does not support looking up the offsets by timestamp
See Also:
offsetsForTimes(Map, Duration)
beginningOffsets
public java.util.Map<TopicPartition,java.lang.Long> beginningOffsets(java.util.Collection<TopicPartition> partitions)
Get the first offset for the given partitions.
This method does not change the current consumer position of the partitions. Specified by:
beginningOffsets in interface Consumer<K,V>
Parameters:
partitions - the partitions to get the earliest offsets.
Returns:
The earliest available offsets for the given partitions
Throws:
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic(s). See the exception for more details
TimeoutException - if the offset metadata could not be fetched before expiration of the configured default.api.timeout.ms
See Also:
seekToBeginning(Collection)
beginningOffsets
public java.util.Map<TopicPartition,java.lang.Long> beginningOffsets(java.util.Collection<TopicPartition> partitions,
java.time.Duration timeout)
Get the first offset for the given partitions.
This method does not change the current consumer position of the partitions. Specified by:
beginningOffsets in interface Consumer<K,V>
Parameters:
partitions - the partitions to get the earliest offsets
timeout - The maximum amount of time to await retrieval of the beginning offsets
Returns:
The earliest available offsets for the given partitions
Throws:
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic(s). See the exception for more details
TimeoutException - if the offset metadata could not be fetched before expiration of the passed timeout
See Also:
seekToBeginning(Collection)
endOffsets
public java.util.Map<TopicPartition,java.lang.Long> endOffsets(java.util.Collection<TopicPartition> partitions)
Get the end offsets for the given partitions. In the default read_uncommitted isolation level, the end offset is the high watermark (that is, the offset of the last successfully replicated message plus one). For read_committed consumers, the end offset is the last stable offset (LSO), which is the minimum of the high watermark and the smallest offset of any open transaction. Finally, if the partition has never been written to, the end offset is 0.
This method does not change the current consumer position of the partitions. Specified by:
endOffsets in interface Consumer<K,V>
Parameters:
partitions - the partitions to get the end offsets.
Returns:
The end offsets for the given partitions.
Throws:
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic(s). See the exception for more details
TimeoutException - if the offset metadata could not be fetched before the amount of time allocated by request.timeout.ms expires
See Also:
seekToEnd(Collection)
endOffsets
public java.util.Map<TopicPartition,java.lang.Long> endOffsets(java.util.Collection<TopicPartition> partitions,
java.time.Duration timeout)
Get the end offsets for the given partitions. In the default read_uncommitted isolation level, the end offset is the high watermark (that is, the offset of the last successfully replicated message plus one). For read_committed consumers, the end offset is the last stable offset (LSO), which is the minimum of the high watermark and the smallest offset of any open transaction. Finally, if the partition has never been written to, the end offset is 0.
This method does not change the current consumer position of the partitions. Specified by:
endOffsets in interface Consumer<K,V>
Parameters:
partitions - the partitions to get the end offsets.
timeout - The maximum amount of time to await retrieval of the end offsets
Returns:
The end offsets for the given partitions.
Throws:
AuthenticationException - if authentication fails. See the exception for more details
AuthorizationException - if not authorized to the topic(s). See the exception for more details
TimeoutException - if the offsets could not be fetched before expiration of the passed timeout
See Also:
seekToEnd(Collection)
close
public void close()
Close the consumer, waiting for up to the default timeout of 30 seconds for any needed cleanup. If auto-commit is enabled, this will commit the current offsets if possible within the default timeout. See close(Duration) for details. Note that wakeup() cannot be used to interrupt close.
Specified by:
close in interface java.lang.AutoCloseable
Specified by:
close in interface java.io.Closeable
Specified by:
close in interface Consumer<K,V>
Throws:
InterruptException - if the calling thread is interrupted before or while this function is called
KafkaException - for any other error during close
See Also:
close()
close
@Deprecated
public void close(long timeout,
java.util.concurrent.TimeUnit timeUnit)
Deprecated. Since 2.0. Use close(Duration) or close().
Tries to close the consumer cleanly within the specified timeout. This method waits up to timeout for the consumer to complete pending commits and leave the group. If auto-commit is enabled, this will commit the current offsets if possible within the timeout. If the consumer is unable to complete offset commits and gracefully leave the group before the timeout expires, the consumer is force closed. Note that wakeup() cannot be used to interrupt close.
Specified by:
close in interface Consumer<K,V>
Parameters:
timeout - The maximum time to wait for consumer to close gracefully. The value must be non-negative. Specifying a timeout of zero means do not wait for pending requests to complete.
timeUnit - The time unit for the timeout
Throws:
java.lang.IllegalArgumentException - If the timeout is negative.
InterruptException - If the thread is interrupted before or while this function is called
KafkaException - for any other error during close
See Also:
close(long, TimeUnit)
close
public void close(java.time.Duration timeout)
Tries to close the consumer cleanly within the specified timeout. This method waits up to timeout for the consumer to complete pending commits and leave the group. If auto-commit is enabled, this will commit the current offsets if possible within the timeout. If the consumer is unable to complete offset commits and gracefully leave the group before the timeout expires, the consumer is force closed. Note that wakeup() cannot be used to interrupt close.
Specified by:
close in interface Consumer<K,V>
Parameters:
timeout - The maximum time to wait for consumer to close gracefully. The value must be non-negative. Specifying a timeout of zero means do not wait for pending requests to complete.
Throws:
java.lang.IllegalArgumentException - If the timeout is negative.
InterruptException - If the thread is interrupted before or while this function is called
KafkaException - for any other error during close
See Also:
close(Duration)
wakeup
public void wakeup()
Wakeup the consumer. This method is thread-safe and is useful in particular to abort a long poll. The thread which is blocking in an operation will throw WakeupException. If no thread is blocking in a method which can throw WakeupException, the next call to such a method will raise it instead.
Specified by:
wakeup in interface Consumer<K,V>
See Also:
wakeup()
java获取kafka consumer lag、endOffsets、beginningOffsets以及 KafkaConsumer总结的更多相关文章
- Kafka Consumer Lag Monitoring
Sematext Monitoring 是最全面的Kafka监视解决方案之一,可捕获约200个Kafka指标,包括Kafka Broker,Producer和Consumer指标.尽管其中许多指标很 ...
- 获取kafka的lag, offset, logsize的shell和python脚本
python脚本 #!/usr/bin/env python import os import re import sys group_id=sys.argv[1] pn=sys.argv[2] ka ...
- Understanding Kafka Consumer Groups and Consumer Lag
In this post, we will dive into the consumer side of this application ecosystem, which means looking ...
- Java curator操作zookeeper获取kafka
Java curator操作zookeeper获取kafka Curator是Netflix公司开源的一个Zookeeper客户端,与Zookeeper提供的原生客户端相比,Curator的抽象层次更 ...
- 初始 Kafka Consumer 消费者
温馨提示:整个 Kafka 专栏基于 kafka-2.2.1 版本. 1.KafkaConsumer 概述 根据 KafkaConsumer 类上的注释上来看 KafkaConsumer 具有如下特征 ...
- 读Kafka Consumer源码
最近一直在关注阿里的一个开源项目:OpenMessaging OpenMessaging, which includes the establishment of industry guideline ...
- Kafka Consumer接口
对于kafka的consumer接口,提供两种版本, high-level 一种high-level版本,比较简单不用关心offset, 会自动的读zookeeper中该Consumer grou ...
- 【原创】美团二面：聊聊你对 Kafka Consumer 的架构设计
在上一篇中我们详细聊了关于 Kafka Producer 内部的底层原理设计思想和细节, 本篇我们主要来聊聊 Kafka Consumer 即消费者的内部底层原理设计思想. 1.Consumer之总体 ...
- 【原创】Kafka Consumer多线程实例
Kafka 0.9版本开始推出了Java版本的consumer,优化了coordinator的设计以及摆脱了对zookeeper的依赖.社区最近也在探讨正式用这套consumer API替换Scala ...
- 【原创】kafka consumer源代码分析
顾名思义,就是kafka的consumer api包. 一.ConsumerConfig.scala Kafka consumer的配置类,除了一些默认值常量及验证参数的方法之外,就是consumer ...
随机推荐
- vue3中watch监听不是你想的那样简单
vue3 中watch监听数组,数组变化后未触发回调 今天发生了一个很神奇的现象,就是我使用watch监听数组时. 被监听的数组已经发生了变化.但是没有触发回调操作. 当时的我感到很疑惑? 不应该呀? ...
- vm-storage在全部都是旧metric情况下的写入性能测试
作者:张富春(ahfuzhang),转载时请注明作者和引用链接,谢谢! cnblogs博客 zhihu Github 公众号:一本正经的瞎扯 接上篇:测试所有metric都是存在过的metric的情况 ...
- 不同版本的Unity要求的NDK版本和两者对应关系表(Unity NDK Version Match)
IL2CPP需要NDK Unity使用IL2CPP模式出安卓包时,需要用到NDK,如果没有安装则无法导出Android Studio工程或直接生成APK,本篇记录一下我下载NDK不同版本的填坑过程. ...
- C++ Boost库 操作日期与时间
Boost库中默认针对日期与时间的操作库分为,timer,progress_timer,date_time这几类,如下是一些常用的使用方法总结. timer库 #include <iostrea ...
- 深入浅出Java多线程(三)：线程与线程组
「引言」 大家好,我是你们的老伙计秀才!今天带来的是[深入浅出Java多线程]系列的第三篇内容:线程与线程组.大家觉得有用请点赞,喜欢请关注!秀才在此谢过大家了!!! 在现代软件开发中,多线程编程已成 ...
- 错误：基于tensorflow识别mnist数据集出现ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[10000,32,28,28] and type float on
错误:最近,在尝试运行我以前博客代码的时候出现了如下错误 2020-04-03 10:53:22.982491: W tensorflow/core/common_runtime/bfc_alloca ...
- .NET Core开发实战（第4课：Startup：掌握ASP.NET Core的启动过程）--学习笔记
04 | Startup:掌握ASP.NET Core的启动过程 新建一个 ASP.NET Core Web 应用程序 选择 API public class Program { public sta ...
- 点亮.NET的文字云艺术之光——Sdcb.WordCloud 2.0
点亮.NET的文字云艺术之光--Sdcb.WordCloud 2.0 作为一名.NET开发者,你是否渴望拥有一个强大且易用的库,用以在你的应用程序中创造美轮美奂的文字云?我在经过一轮农历新年前的码力全 ...
- NC24141 [USACO 2011 Dec G]Grass Planting
题目链接 题目 题目描述 Farmer John has N barren pastures (2 <= N <= 100,000) connected by N-1 bidirectio ...
- NC50500 凸多边形的划分
题目链接 题目 题目描述 给定一个具有N个顶点的凸多边形,将顶点从1至N标号,每个顶点的权值都是一个正整数.将这个凸多边形划分成N-2个互不相交的三角形,试求这些三角形顶点的权值乘积和至少为多少. 输 ...