kafka 消费者拉取消息
本文只跟踪消费者拉取消息的流程。对于 java 客户端, kafka 的生产者和消费者复用同一个网络 io 类 NetworkClient。
入口在 KafkaConsumer#pollOnce 中,抽出主要步骤:
// 构造 FetchRequest 请求,将请求对象放入 unsent 集合,等待发送
fetcher.sendFetches(); // 取出 unsent 中的请求,调用 NetworkClient#send,NetworkClinet#poll
client.poll(pollTimeout, nowMs, new PollCondition() {
@Override
public boolean shouldBlock() {
// since a fetch might be completed by the background thread, we need this poll condition
// to ensure that we do not block unnecessarily in poll()
return !fetcher.hasCompletedFetches();
}
}); // 返回数据给用户
return fetcher.fetchedRecords();
Fetcher#sendFetches
public synchronized int sendFetches() {
// 构造拉取消息请求。从哪个节点,哪个分区,什么位置拉取消息
Map<Node, FetchSessionHandler.FetchRequestData> fetchRequestMap = prepareFetchRequests();
for (Map.Entry<Node, FetchSessionHandler.FetchRequestData> entry : fetchRequestMap.entrySet()) {
final Node fetchTarget = entry.getKey();
final FetchSessionHandler.FetchRequestData data = entry.getValue();
//1. 借助 Builder 构造 FetchRequest 对象
final FetchRequest.Builder request = FetchRequest.Builder
.forConsumer(this.maxWaitMs, this.minBytes, data.toSend())
.isolationLevel(isolationLevel)
.setMaxBytes(this.maxBytes)
.metadata(data.metadata())
.toForget(data.toForget());
if (log.isDebugEnabled()) {
log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget);
} client.send(fetchTarget, request)
//4. 给 RequestFutureCompletionHandler.future 添加 RequestFutureListener
.addListener(new RequestFutureListener<ClientResponse>() {
@Override
public void onSuccess(ClientResponse resp) {
synchronized (Fetcher.this) {
FetchResponse response = (FetchResponse) resp.responseBody();
FetchSessionHandler handler = sessionHandler(fetchTarget.id());
if (handler == null) {
log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.",
fetchTarget.id());
return;
}
if (!handler.handleResponse(response)) {
return;
} Set<TopicPartition> partitions = new HashSet<>(response.responseData().keySet());
FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions); for (Map.Entry<TopicPartition, FetchResponse.PartitionData> entry : response.responseData().entrySet()) {
TopicPartition partition = entry.getKey();
long fetchOffset = data.sessionPartitions().get(partition).fetchOffset;
FetchResponse.PartitionData fetchData = entry.getValue(); log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",
isolationLevel, fetchOffset, partition, fetchData);
// 10. 把数据放入 completedFetches,最终返回给用户
completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator,
resp.requestHeader().apiVersion()));
} sensors.fetchLatency.record(resp.requestLatencyMs());
}
} @Override
public void onFailure(RuntimeException e) {
synchronized (Fetcher.this) {
FetchSessionHandler handler = sessionHandler(fetchTarget.id());
if (handler != null) {
handler.handleError(e);
}
}
}
});
}
return fetchRequestMap.size();
}
ConsumerNetworkClient#send
public RequestFuture<ClientResponse> send(Node node, AbstractRequest.Builder<?> requestBuilder) {
long now = time.milliseconds();
//2. 使用 RequestFutureCompletionHandler 作为回调函数
RequestFutureCompletionHandler completionHandler = new RequestFutureCompletionHandler();
ClientRequest clientRequest = client.newClientRequest(node.idString(), requestBuilder, now, true,
completionHandler);
//3. 请求放入 unsent 集合
unsent.put(node, clientRequest); // wakeup the client in case it is blocking in poll so that we can send the queued request
client.wakeup();
return completionHandler.future;
}
ConsumerNetworkClient#poll
// 5. 发送 unsent 中的请求,并没有产生网络 io
trySend(now); // 真实的网络数据写和读
// 6. 发送请求
// 7. 接收响应
// 8. 触发 RequestFutureCompletionHandler 回调
client.poll(0, now); // 9. 触发 RequestFutureListener 中的回调
firePendingCompletedRequests();
NetworkClient#handleCompletedReceives
private void handleCompletedReceives(List<ClientResponse> responses, long now) {
for (NetworkReceive receive : this.selector.completedReceives()) {
String source = receive.source();
InFlightRequest req = inFlightRequests.completeNext(source);
Struct responseStruct = parseStructMaybeUpdateThrottleTimeMetrics(receive.payload(), req.header,
throttleTimeSensor, now);
if (log.isTraceEnabled()) {
log.trace("Completed receive from node {} for {} with correlation id {}, received {}", req.destination,
req.header.apiKey(), req.header.correlationId(), responseStruct);
}
AbstractResponse body = AbstractResponse.parseResponse(req.header.apiKey(), responseStruct);
if (req.isInternalRequest && body instanceof MetadataResponse)
metadataUpdater.handleCompletedMetadataResponse(req.header, now, (MetadataResponse) body);
else if (req.isInternalRequest && body instanceof ApiVersionsResponse)
handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) body);
else
// 此处给 responses 添加元素
// return new ClientResponse(header, callback, destination, createdTimeMs, timeMs, false, null, response);
// 直接把请求的 callback 赋值给响应
// 生产者发送消息的 callback,是用户通过参数传入的
// 消费者拉取消息的 callback,是在 ConsumerNetworkClient#send 指定的,是 RequestFutureCompletionHandler
responses.add(req.completed(body, now));
}
}
NetworkClient#completeResponses
private void completeResponses(List<ClientResponse> responses) {
for (ClientResponse response : responses) {
try {
// callback.onComplete(this);
response.onComplete();
} catch (Exception e) {
log.error("Uncaught error in request completion:", e);
}
}
}
RequestFutureCompletionHandler#onComplete
public void onComplete(ClientResponse response) {
this.response = response;
pendingCompletion.add(this);
}
ConsumerNetworkClient#firePendingCompletedRequests
private void firePendingCompletedRequests() {
boolean completedRequestsFired = false;
for (;;) {
RequestFutureCompletionHandler completionHandler = pendingCompletion.poll();
if (completionHandler == null)
break; completionHandler.fireCompletion();
completedRequestsFired = true;
} // wakeup the client in case it is blocking in poll for this future's completion
if (completedRequestsFired)
client.wakeup();
}
ConsumerNetworkClient.RequestFutureCompletionHandler#fireCompletion
public void fireCompletion() {
if (e != null) {
future.raise(e);
} else if (response.wasDisconnected()) {
RequestHeader requestHeader = response.requestHeader();
int correlation = requestHeader.correlationId();
log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected",
requestHeader.apiKey(), requestHeader, correlation, response.destination());
future.raise(DisconnectException.INSTANCE);
} else if (response.versionMismatch() != null) {
future.raise(response.versionMismatch());
} else {
future.complete(response);
}
}
RequestFuture#complete
public void complete(T value) {
try {
if (value instanceof RuntimeException)
throw new IllegalArgumentException("The argument to complete can not be an instance of RuntimeException"); if (!result.compareAndSet(INCOMPLETE_SENTINEL, value))
throw new IllegalStateException("Invalid attempt to complete a request future which is already complete");
fireSuccess();
} finally {
completedLatch.countDown();
}
} private void fireSuccess() {
T value = value();
while (true) {
RequestFutureListener<T> listener = listeners.poll();
if (listener == null)
break;
// 终于调到 RequestFutureListener
listener.onSuccess(value);
}
}
如果不考虑心跳线程,consumer 第一次 poll 是不会有数据的,此时请求才发出去,响应还没回来,必须在第二次 poll 时,才能同时处理网络读写事件。
跟完之后,个人觉得调用链还是挺长的。一点感觉,全程只有一个线程,但是每次走的分支都不一样,给人的启发就是,单线程只要不等待,速度也很快。
kafka 消费者拉取消息的更多相关文章
- Kafka消费者拉取数据异常Unexpected error code 2 while fetching data
Kafka消费程序间歇性报同一个错: 上网没查到相关资料,只好自己分析.通过进一步分析日志发现,只有在拉取某一个特定的topic的数据时报错,如果拉取其他topic的数据则不会报错.而从这个异常信息来 ...
- Kafka消费者手动提交消息偏移
生产者每次调用poll()方法时,它总是返回由生产者写入Kafka但还没有消费的消息,如果消费者一致处于运行状态,那么分区消息偏移量就没什么用处,但是如果消费者发生崩溃或者有新的消费者加入群组,就会触 ...
- 【mq】从零开始实现 mq-09-消费者拉取消息 pull message
前景回顾 [mq]从零开始实现 mq-01-生产者.消费者启动 [mq]从零开始实现 mq-02-如何实现生产者调用消费者? [mq]从零开始实现 mq-03-引入 broker 中间人 [mq]从零 ...
- 【mq】从零开始实现 mq-10-消费者拉取消息回执 pull message ack
前景回顾 [mq]从零开始实现 mq-01-生产者.消费者启动 [mq]从零开始实现 mq-02-如何实现生产者调用消费者? [mq]从零开始实现 mq-03-引入 broker 中间人 [mq]从零 ...
- RocketMQ入门(3)拉取消息
转自:http://www.changeself.net/archives/rocketmq入门(3)拉取消息.html RocketMQ入门(3)拉取消息 RocketMQ不止可以直接推送消息,在消 ...
- RocketMQ 拉取消息-文件获取
看完了上一篇的<RocketMQ 拉取消息-通信模块>,请求进入PullMessageProcessor中,接着 PullMessageProcessor.processRequest(f ...
- RocketMQ 拉取消息-通信模块
首先看server端:class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer 下面这个实现了 ...
- .net MVC 微信公众号 点击菜单拉取消息时的事件推送
官方文档:https://mp.weixin.qq.com/wiki?t=resource/res_main&id=mp1421141016&token=&lang=zh_CN ...
- JAVA封装消息中间件调用二(kafka消费者篇)
上一遍我简单介绍了kafka的生成者使用,调用方式比较简单,今天我给大家分享下封装kafka消费者,作为中间件,我们做的就是最大程度的解耦,使业务方接入我们依赖程度降到最低. 第一步,我们先配置一个消 ...
随机推荐
- js数组与对象的区别
数组和对象两者都可以用来表示数据的集合,曾一度搞不清楚”数组”(array)和”对象”(object)的根本区别在哪里. 有一个数组a=[1,2,3,4],还有一个对象a={0:1,1:2,2:3,3 ...
- 3、sql 表的连接
摘自: https://blog.csdn.net/holly2008/article/details/25704471 表连接分为:CROSS JOIN.INNERT JOIN.OUTER JOIN ...
- python中的正则表达式的使用
一.正则表达式简介 正则表达式的官方文档:https://www.runoob.com/regexp/regexp-tutorial.html 正则表达式:又称正规表示式.正规表示法.正规表达式.规则 ...
- 使用Python的pandas-datareader包下载雅虎财经股价数据
0 准备工作 首先,使用pip方法安装pandas和pandas-datareader两个功能包. 安装的方法十分简单,以管理员身份运行cmd. 输入以下命令. $ pip install panda ...
- html 常用小技巧
style = "cursor:pointer;" 变小手 a{ text-decoration:none; } 或者把这个属性分别加到a标签下, a:link{ text-dec ...
- MapReduce TopN(自主复习)
1.MyTopN 主程序 package com.littlepage.topn; import org.apache.hadoop.conf.Configuration; import org.a ...
- Spring注解配置、Spring aop、整合Junit——Spring学习 day2
注解配置: 1.为主配置文件引入新的命名空间(约束) preference中引入文件 2.开启使用注解代理配置文件 <?xml version="1.0" encoding= ...
- Spring mvc数据转换 格式化 校验(转载)
原文地址:http://www.cnblogs.com/linyueshan/p/5908490.html 数据绑定流程 1. Spring MVC 主框架将 ServletRequest 对象及目标 ...
- Java基本的程序结构设计 控制流程
控制流程 java的控制流程和C和C++基本一致,只是不能使用goto语句,不过break语句可以带标签,实现从内层循环跳出的目的.标签可以放在for或者while前面.如下: package com ...
- 跨域 (2) cors
html <!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF- ...