RocketMQ长轮询机制
长轮询机制简单解释
RocketMQ 的长轮询机制（Long Polling）主要用于消费者拉取消息时的一种优化方式，特别是对于PushConsumer 实际上是 Pull 实现的场景。在 RocketMQ 中，长轮询机制提高了消费实时性和系统资源利用率。
长轮询机制简单来说，就是当Broker接收到Consumer的Pull请求时，判断如果没有对应的消息，不用直接给Consumer响应(给响应也是个空的，没意义)，而是就将这个Pull请求给缓存起来。当Producer发送消息过来时，增加一个步骤去检查是否有对应的已缓存的Pull请求，如果有，就及时将请求从缓存中拉取出来，并将消息通知给Consumer。
处理器PullMessageProcessor的processRequest()方法
public class PullMessageProcessor implements NettyRequestProcessor {
...
private PullMessageResultHandler pullMessageResultHandler;
...
public PullMessageProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
this.pullMessageResultHandler = new DefaultPullMessageResultHandler(brokerController);
}
...
private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend,
boolean brokerAllowFlowCtrSuspend)
throws RemotingCommandException {
...
messageStore.getMessageAsync(group, topic, queueId, requestHeader.getQueueOffset(),
requestHeader.getMaxMsgNums(), messageFilter)
.thenApply(result -> {
if (null == result) {
finalResponse.setCode(ResponseCode.SYSTEM_ERROR);
finalResponse.setRemark("store getMessage return null");
return finalResponse;
}
brokerController.getColdDataCgCtrService().coldAcc(requestHeader.getConsumerGroup(), result.getColdDataSum());
return pullMessageResultHandler.handle(//DefaultPullMessageResultHandler处理
result,
request,
requestHeader,
channel,
finalSubscriptionData,
subscriptionGroupConfig,
brokerAllowSuspend,
messageFilter,
finalResponse,
mappingContext,
beginTimeMills
);
})
.thenAccept(result -> NettyRemotingAbstract.writeResponse(channel, request, result));
...
}
}
DefaultPullMessageResultHandler 关键代码
public class DefaultPullMessageResultHandler implements PullMessageResultHandler {
...
@Override
public RemotingCommand handle(final GetMessageResult getMessageResult,
final RemotingCommand request,
final PullMessageRequestHeader requestHeader,
final Channel channel,
final SubscriptionData subscriptionData,
final SubscriptionGroupConfig subscriptionGroupConfig,
final boolean brokerAllowSuspend,
final MessageFilter messageFilter,
RemotingCommand response,
TopicQueueMappingContext mappingContext,
long beginTimeMills) {
switch (response.getCode()) {
case ResponseCode.SUCCESS://成功找到消息返回数据
...
case ResponseCode.PULL_NOT_FOUND://没有找到消息
final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag());
final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0;
if (brokerAllowSuspend && hasSuspendFlag) {
long pollingTimeMills = suspendTimeoutMillisLong;
if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) {
pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();
}
String topic = requestHeader.getTopic();
long offset = requestHeader.getQueueOffset();
int queueId = requestHeader.getQueueId();
PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);//复制保存本次请求
this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);//等待服务暂停拉取请求，把请求保存到pullRequestTable，PullRequestHoldService在BrokerController.startBasicService()中已经start
return null;
}
}
...
}
PullRequestHoldService关键代码
public class PullRequestHoldService extends ServiceThread {
...
protected ConcurrentMap<String/* topic@queueId */, ManyPullRequest> pullRequestTable =
new ConcurrentHashMap<>(1024);
...
public void suspendPullRequest(final String topic, final int queueId, final PullRequest pullRequest) {
String key = this.buildKey(topic, queueId);
ManyPullRequest mpr = this.pullRequestTable.get(key);
if (null == mpr) {
mpr = new ManyPullRequest();
ManyPullRequest prev = this.pullRequestTable.putIfAbsent(key, mpr);//把请求保存到pullRequestTable
if (prev != null) {
mpr = prev;
}
}
pullRequest.getRequestCommand().setSuspended(true);
mpr.addPullRequest(pullRequest);
}
...
@Override
public void run() {
log.info("{} service started", this.getServiceName());
while (!this.isStopped()) {//保活
try {
if (this.brokerController.getBrokerConfig().isLongPollingEnable()) {
this.waitForRunning(5 * 1000);//休眠一段时间
} else {
this.waitForRunning(this.brokerController.getBrokerConfig().getShortPollingTimeMills());//休眠一段时间
}
long beginLockTimestamp = this.systemClock.now();
this.checkHoldRequest();//主动检查等待的请求有没有对应的消息可返回
long costTime = this.systemClock.now() - beginLockTimestamp;
if (costTime > 5 * 1000) {
log.warn("PullRequestHoldService: check hold pull request cost {}ms", costTime);
}
} catch (Throwable e) {
log.warn(this.getServiceName() + " service has exception. ", e);
}
}
log.info("{} service end", this.getServiceName());
}
...
protected void checkHoldRequest() {//主动检查等待的请求有没有对应的消息可返回
for (String key : this.pullRequestTable.keySet()) {
String[] kArray = key.split(TOPIC_QUEUEID_SEPARATOR);
if (2 == kArray.length) {
String topic = kArray[0];
int queueId = Integer.parseInt(kArray[1]);
try {
final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
log.error(
"PullRequestHoldService: failed to check hold request failed, topic={}, queueId={}", topic,
queueId, e);
}
}
}
}
...
//通知消息到达
public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode,
long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
String key = this.buildKey(topic, queueId);
ManyPullRequest mpr = this.pullRequestTable.get(key);
if (mpr != null) {
List<PullRequest> requestList = mpr.cloneListAndClear();
if (requestList != null) {
List<PullRequest> replayList = new ArrayList<>();
for (PullRequest request : requestList) {
long newestOffset = maxOffset;
if (newestOffset <= request.getPullFromThisOffset()) {
try {
newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
} catch (ConsumeQueueException e) {
log.error("Failed tp get max offset in queue", e);
continue;
}
}
if (newestOffset > request.getPullFromThisOffset()) {
boolean match = request.getMessageFilter().isMatchedByConsumeQueue(tagsCode,
new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));
// match by bit map, need eval again when properties is not null.
if (match && properties != null) {
match = request.getMessageFilter().isMatchedByCommitLog(null, properties);
}
if (match) {
try {
this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
request.getRequestCommand());
} catch (Throwable e) {
log.error(
"PullRequestHoldService#notifyMessageArriving: failed to execute request when "
+ "message matched, topic={}, queueId={}", topic, queueId, e);
}
continue;
}
}
if (System.currentTimeMillis() >= (request.getSuspendTimestamp() + request.getTimeoutMillis())) {
try {
this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
request.getRequestCommand());
} catch (Throwable e) {
log.error(
"PullRequestHoldService#notifyMessageArriving: failed to execute request when time's "
+ "up, topic={}, queueId={}", topic, queueId, e);
}
continue;
}
replayList.add(request);
}
if (!replayList.isEmpty()) {
mpr.addPullRequest(replayList);
}
}
}
}
...
}
broker在DefaultMessageStore/RocksDBMessageStore注册了NotifyMessageArrivingListener，ReputMessageService(该服务会获取commitLog的新数据)服务在doReput会调用NotifyMessageArrivingListener.arriving()，该方法又调用了pullRequestHoldService.notifyMessageArriving。主动+被动最快速度获取数据
