rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/README.en.md +428 -0
  2. package/README.md +424 -2
  3. package/dist/client/admin.d.ts +229 -0
  4. package/dist/client/admin.js +2462 -0
  5. package/dist/client/allocate.d.ts +48 -0
  6. package/dist/client/allocate.js +365 -0
  7. package/dist/client/backpressure.d.ts +23 -0
  8. package/dist/client/backpressure.js +95 -0
  9. package/dist/client/consumer.d.ts +294 -0
  10. package/dist/client/consumer.js +3095 -0
  11. package/dist/client/consumer_result.d.ts +129 -0
  12. package/dist/client/consumer_result.js +138 -0
  13. package/dist/client/consumer_stats.d.ts +69 -0
  14. package/dist/client/consumer_stats.js +258 -0
  15. package/dist/client/exception.d.ts +39 -0
  16. package/dist/client/exception.js +36 -0
  17. package/dist/client/hook.d.ts +145 -0
  18. package/dist/client/hook.js +171 -0
  19. package/dist/client/latency.d.ts +61 -0
  20. package/dist/client/latency.js +239 -0
  21. package/dist/client/lite_pull_consumer.d.ts +151 -0
  22. package/dist/client/lite_pull_consumer.js +1096 -0
  23. package/dist/client/mq_client.d.ts +188 -0
  24. package/dist/client/mq_client.js +1678 -0
  25. package/dist/client/mq_client_manager.d.ts +21 -0
  26. package/dist/client/mq_client_manager.js +129 -0
  27. package/dist/client/offset_store.d.ts +45 -0
  28. package/dist/client/offset_store.js +389 -0
  29. package/dist/client/open_tracing_hook.d.ts +52 -0
  30. package/dist/client/open_tracing_hook.js +241 -0
  31. package/dist/client/pop_api.d.ts +28 -0
  32. package/dist/client/pop_api.js +224 -0
  33. package/dist/client/pop_consumer.d.ts +23 -0
  34. package/dist/client/pop_consumer.js +719 -0
  35. package/dist/client/pop_process_queue.d.ts +15 -0
  36. package/dist/client/pop_process_queue.js +51 -0
  37. package/dist/client/process_queue.d.ts +49 -0
  38. package/dist/client/process_queue.js +261 -0
  39. package/dist/client/produce_accumulator.d.ts +75 -0
  40. package/dist/client/produce_accumulator.js +288 -0
  41. package/dist/client/producer.d.ts +231 -0
  42. package/dist/client/producer.js +1774 -0
  43. package/dist/client/pull_api.d.ts +26 -0
  44. package/dist/client/pull_api.js +264 -0
  45. package/dist/client/pull_consumer.d.ts +119 -0
  46. package/dist/client/pull_consumer.js +876 -0
  47. package/dist/client/pull_schedule_service.d.ts +71 -0
  48. package/dist/client/pull_schedule_service.js +424 -0
  49. package/dist/client/request_reply.d.ts +61 -0
  50. package/dist/client/request_reply.js +232 -0
  51. package/dist/client/retry_message.d.ts +2 -0
  52. package/dist/client/retry_message.js +27 -0
  53. package/dist/client/send_result.d.ts +57 -0
  54. package/dist/client/send_result.js +70 -0
  55. package/dist/client/top_addressing.d.ts +20 -0
  56. package/dist/client/top_addressing.js +75 -0
  57. package/dist/client/trace_context.d.ts +61 -0
  58. package/dist/client/trace_context.js +373 -0
  59. package/dist/client/trace_dispatcher.d.ts +31 -0
  60. package/dist/client/trace_dispatcher.js +236 -0
  61. package/dist/client/trace_hook.d.ts +23 -0
  62. package/dist/client/trace_hook.js +228 -0
  63. package/dist/client/traceparent.d.ts +9 -0
  64. package/dist/client/traceparent.js +86 -0
  65. package/dist/common/boundary_type.d.ts +7 -0
  66. package/dist/common/boundary_type.js +16 -0
  67. package/dist/common/compress.d.ts +28 -0
  68. package/dist/common/compress.js +512 -0
  69. package/dist/common/message.d.ts +129 -0
  70. package/dist/common/message.js +214 -0
  71. package/dist/common/messageClientIdSetter.d.ts +7 -0
  72. package/dist/common/messageClientIdSetter.js +48 -0
  73. package/dist/common/messageConst.d.ts +68 -0
  74. package/dist/common/messageConst.js +68 -0
  75. package/dist/common/messageDecoder.d.ts +40 -0
  76. package/dist/common/messageDecoder.js +504 -0
  77. package/dist/common/messageType.d.ts +11 -0
  78. package/dist/common/messageType.js +11 -0
  79. package/dist/common/message_accessor.d.ts +39 -0
  80. package/dist/common/message_accessor.js +135 -0
  81. package/dist/common/mixAll.d.ts +106 -0
  82. package/dist/common/mixAll.js +247 -0
  83. package/dist/common/recall_message_handle.d.ts +25 -0
  84. package/dist/common/recall_message_handle.js +98 -0
  85. package/dist/common/subscriptionData.d.ts +36 -0
  86. package/dist/common/subscriptionData.js +64 -0
  87. package/dist/common/sysflag.d.ts +129 -0
  88. package/dist/common/sysflag.js +96 -0
  89. package/dist/common/topic_config.d.ts +66 -0
  90. package/dist/common/topic_config.js +106 -0
  91. package/dist/common/topic_validator.d.ts +49 -0
  92. package/dist/common/topic_validator.js +115 -0
  93. package/dist/common/utilAll.d.ts +39 -0
  94. package/dist/common/utilAll.js +154 -0
  95. package/dist/common/validators.d.ts +25 -0
  96. package/dist/common/validators.js +80 -0
  97. package/dist/index.d.ts +24 -0
  98. package/dist/index.js +36 -0
  99. package/dist/logging.d.ts +24 -0
  100. package/dist/logging.js +246 -0
  101. package/dist/remoting/acl.d.ts +16 -0
  102. package/dist/remoting/acl.js +39 -0
  103. package/dist/remoting/admin_body.d.ts +257 -0
  104. package/dist/remoting/admin_body.js +669 -0
  105. package/dist/remoting/bodies.d.ts +475 -0
  106. package/dist/remoting/bodies.js +1131 -0
  107. package/dist/remoting/client.d.ts +73 -0
  108. package/dist/remoting/client.js +591 -0
  109. package/dist/remoting/codes.d.ts +296 -0
  110. package/dist/remoting/codes.js +301 -0
  111. package/dist/remoting/exception.d.ts +45 -0
  112. package/dist/remoting/exception.js +58 -0
  113. package/dist/remoting/extra_info.d.ts +61 -0
  114. package/dist/remoting/extra_info.js +220 -0
  115. package/dist/remoting/headers.d.ts +758 -0
  116. package/dist/remoting/headers.js +1307 -0
  117. package/dist/remoting/heartbeat.d.ts +91 -0
  118. package/dist/remoting/heartbeat.js +227 -0
  119. package/dist/remoting/namespace.d.ts +14 -0
  120. package/dist/remoting/namespace.js +85 -0
  121. package/dist/remoting/pop_bodies.d.ts +58 -0
  122. package/dist/remoting/pop_bodies.js +214 -0
  123. package/dist/remoting/remotingCommand.d.ts +39 -0
  124. package/dist/remoting/remotingCommand.js +225 -0
  125. package/dist/remoting/route.d.ts +61 -0
  126. package/dist/remoting/route.js +288 -0
  127. package/dist/remoting/rpc_hooks.d.ts +31 -0
  128. package/dist/remoting/rpc_hooks.js +73 -0
  129. package/dist/remoting/serialize.d.ts +33 -0
  130. package/dist/remoting/serialize.js +372 -0
  131. package/dist/remoting/socks_proxy.d.ts +29 -0
  132. package/dist/remoting/socks_proxy.js +379 -0
  133. package/dist/remoting/static_topic.d.ts +49 -0
  134. package/dist/remoting/static_topic.js +244 -0
  135. package/dist/remoting/subscription.d.ts +65 -0
  136. package/dist/remoting/subscription.js +172 -0
  137. package/package.json +38 -4
@@ -0,0 +1,719 @@
1
+ import { MessageQueue } from "../common/message.js";
2
+ import { MessageConst } from "../common/messageConst.js";
3
+ import { MixAll } from "../common/mixAll.js";
4
+ import { mqKey } from "./offset_store.js";
5
+ import { PopProcessQueue } from "./pop_process_queue.js";
6
+ import { PopStatus, POP_DELAY_LEVEL, popDelayLevelSeconds, popDelayLevelForElapsed, popBatchTimedOut, popInvisibleTimeOf, ckParse, getRealTopicFromCk, } from "./pop_api.js";
7
+ import { AckMessageRequestHeader, ChangeInvisibleTimeRequestHeader, PopMessageRequestHeader } from "../remoting/headers.js";
8
+ import { MESSAGE_REQUEST_MODE_POP } from "../remoting/pop_bodies.js";
9
+ import { ConsumeFromWhere } from "../remoting/heartbeat.js";
10
+ import { ResponseCode } from "../remoting/codes.js";
11
+ import { isLogicalQueueMockBrokerName } from "../remoting/static_topic.js";
12
+ import { MQBrokerException, RemotingTimeoutException } from "../remoting/exception.js";
13
+ import { ConsumeConcurrentlyContext, ConsumeConcurrentlyStatus } from "./consumer_result.js";
14
+ import { ConsumeMessageContext, FilterMessageContext } from "./hook.js";
15
+ import { clientSideTagFilter } from "./pull_api.js";
16
+ import { getLogger } from "../logging.js";
17
+ const logger = getLogger('pop');
18
+ // POP timing constants.
19
+ export const POP_BROKER_SUSPEND_MAX_TIME_MILLIS = 1000 * 15; // long-poll budget sent as pollTime
20
+ export const POP_ASYNC_TIMEOUT_MILLIS = 3000; // ACK and CHANGE_INVISIBLETIME
21
+ export const POP_REQUEST_EXTRA_NETWORK_MILLIS = 10 * 1000; // "+10s" on the long-poll timeout
22
+ export const POP_MIN_INVISIBLE_TIME = 5000;
23
+ export const POP_MAX_INVISIBLE_TIME = 300000;
24
+ // Fallback applied at request time when the configured value is out of range —
25
+ // NOT an error.
26
+ export const POP_DEFAULT_INVISIBLE_TIME = 60000;
27
+ // The two start positions a POP request can name for a group that has never
28
+ // popped this queue: MIN(0) replays from the queue's first message, MAX(1)
29
+ // starts at its tail.
30
+ export const POP_INIT_MODE_MIN = 0;
31
+ export const POP_INIT_MODE_MAX = 1;
32
+ // consumeInitMode translates the consumer's start policy into that position.
33
+ // CONSUME_FROM_FIRST_OFFSET is the ONLY value that means MIN: every other
34
+ // value, the default CONSUME_FROM_LAST_OFFSET included, means MAX. A POP group
35
+ // configured to start at the tail therefore must not send MIN — the symptom of
36
+ // that mistake is a fresh group replaying the whole backlog.
37
+ export function consumeInitMode(consumeFromWhere) {
38
+ return consumeFromWhere === ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET
39
+ ? POP_INIT_MODE_MIN : POP_INIT_MODE_MAX;
40
+ }
41
+ // The pause a POP round takes when this client's own gate stopped it. Same
42
+ // number as the pull path's cache wait, and it is the duration named in the
43
+ // shared flow-control WARN.
44
+ export const POP_DELAY_WHEN_CACHE_FLOW_CONTROL = 50;
45
+ export const POP_DELAY_WHEN_BROKER_FLOW = 20;
46
+ export const POP_DELAY_WHEN_SUSPEND = 1000;
47
+ // There is deliberately NO POP-specific error-backoff constant here: a failed POP
48
+ // round, and an answer this client does not know how to read (POLLING_FULL
49
+ // included), wait the consumer's single generic error backoff
50
+ // (DefaultMQPushConsumer.pullRetryDelayMillis, default 1000 ms), the same knob the
51
+ // pull path uses, so one setting moves both paths.
52
+ // ---------------------------------------------------------------------------
53
+ // pop loop
54
+ // ---------------------------------------------------------------------------
55
+ // queuePopLoop pops one queue until it is retired or the consumer stops.
56
+ export async function queuePopLoop(c, mq, stopFlag) {
57
+ const key = mqKey(mq);
58
+ // The generic error backoff is taken ONCE, when this queue's loop is set up:
59
+ // a later setPullRetryDelayMillis() reaches the loops the next rebalance
60
+ // builds, never a round already in flight.
61
+ const retryDelayMillis = c.pullRetryDelayMillis;
62
+ while (!c._isStopped(stopFlag)) {
63
+ const pq = c.popQueueTable.get(key);
64
+ if (!pq || pq.isDropped())
65
+ return;
66
+ pq.setLastPopTimestamp(Date.now());
67
+ // POP reads the SAME consumer-wide pause flag as the pull loop, and the
68
+ // gate sits after the stamp for the same reason (a suspended-but-alive pop
69
+ // loop must not be judged expired, or rebalance retires the queue and
70
+ // in-flight batches get abandoned to the broker's revive).
71
+ if (c.isPaused()) {
72
+ if (await c._sleepOrStop(POP_DELAY_WHEN_SUSPEND, stopFlag))
73
+ return;
74
+ continue;
75
+ }
76
+ // No subscription for this topic: there is no next round to schedule, so the
77
+ // loop exits and rebalance owns the queue; the error backoff waits for a
78
+ // request this loop will never re-issue.
79
+ const sub = c.subscription.get(mq.getTopic());
80
+ if (!sub)
81
+ return;
82
+ // Flow control on this queue's outstanding-ACK debt, against
83
+ // popThresholdForQueue. The gate lives on the consumer because it feeds the
84
+ // SAME flowControlTriggered counter (and the SAME WARN cadence) as the pull
85
+ // path, so one flowControlTimes number covers both paths.
86
+ if (c._popFlowControlHit(key, pq)) {
87
+ if (await c._sleepOrStop(POP_DELAY_WHEN_CACHE_FLOW_CONTROL, stopFlag))
88
+ return;
89
+ continue;
90
+ }
91
+ // beginTimestamp is stamped immediately before the POP request, so the
92
+ // POP round trip includes the broker's poll hold.
93
+ const begin = Date.now();
94
+ let result;
95
+ try {
96
+ result = await popOnce(c, mq, sub);
97
+ }
98
+ catch (e) {
99
+ // The code lives on MQBrokerException.responseCode (the field the remoting
100
+ // layer fills in), so reading anything else here silently disables the
101
+ // branch below and every throttled queue falls back to the generic backoff.
102
+ const code = e instanceof MQBrokerException ? e.responseCode : undefined;
103
+ if (code === ResponseCode.FLOW_CONTROL) {
104
+ // Broker-side flow control: back off briefly, do not rescan.
105
+ if (await c._sleepOrStop(POP_DELAY_WHEN_BROKER_FLOW, stopFlag))
106
+ return;
107
+ continue;
108
+ }
109
+ if (e instanceof RemotingTimeoutException) {
110
+ // A long poll timing out while idle is normal (the broker clamps the
111
+ // suspend time); keep it at debug so the run log stays clean.
112
+ logger.debug('pop long-poll timeout for %s (benign, will retry): %s', key, e.message);
113
+ }
114
+ else {
115
+ logger.debug('pop error for %s: %s', key, e.message);
116
+ if (await c._sleepOrStop(retryDelayMillis, stopFlag))
117
+ return;
118
+ }
119
+ continue;
120
+ }
121
+ if (c._isStopped(stopFlag))
122
+ return;
123
+ switch (result.popStatus) {
124
+ case PopStatus.FOUND: {
125
+ // The pre-delivery filter runs first, so what it removes never reaches
126
+ // the pull stats, never enters the queue's answer debt and never
127
+ // reaches the listener; the rows it removes are ACKed there and then.
128
+ processPopResult(c, mq, sub, result);
129
+ const msgs = result.msgFoundList;
130
+ // The RT counter is bumped for every FOUND answer — including one whose
131
+ // list is empty, whether the broker sent none or the filter removed
132
+ // them all. Only the TPS counter follows the delivered count.
133
+ c._incPullRT(mq.getTopic(), Date.now() - begin);
134
+ if (msgs.length === 0) {
135
+ // Nothing to hand over: re-pop immediately, and count no TPS for the
136
+ // round.
137
+ continue;
138
+ }
139
+ c._incPullTPS(mq.getTopic(), msgs.length);
140
+ // The debt is fed with the DELIVERED count only: every one of these owes
141
+ // an answer, and the rows the filter already ACKed are not counted, so
142
+ // the counter returns to zero once the listener has settled this batch.
143
+ pq.incFoundMsg(msgs.length);
144
+ submitPopConsume(c, mq, msgs, pq);
145
+ if (c.pullInterval > 0 && await c._sleepOrStop(c.pullInterval, stopFlag))
146
+ return;
147
+ break;
148
+ }
149
+ case PopStatus.NO_NEW_MSG:
150
+ case PopStatus.POLLING_NOT_FOUND:
151
+ // Retry immediately: the broker already held the poll open.
152
+ continue;
153
+ default:
154
+ // POLLING_FULL and anything else: back off with the generic error delay.
155
+ if (await c._sleepOrStop(retryDelayMillis, stopFlag))
156
+ return;
157
+ break;
158
+ }
159
+ }
160
+ }
161
+ // popOnce issues one POP_MESSAGE and returns the processed result.
162
+ async function popOnce(c, mq, sub) {
163
+ const client = c.mqClient;
164
+ if (!client)
165
+ throw new Error('consumer not started');
166
+ // An out-of-range configured value is replaced by 60s at REQUEST time;
167
+ // checkConfig has already rejected the startup, so this only bites when the
168
+ // setter runs mid-flight.
169
+ let invisibleTime = c.popInvisibleTime;
170
+ if (invisibleTime < POP_MIN_INVISIBLE_TIME || invisibleTime > POP_MAX_INVISIBLE_TIME) {
171
+ invisibleTime = POP_DEFAULT_INVISIBLE_TIME;
172
+ }
173
+ let resolved = c._findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);
174
+ if (!resolved) {
175
+ await client.updateTopicRouteInfoFromNameServer(mq.getTopic(), false).catch(() => { });
176
+ resolved = c._findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);
177
+ if (!resolved) {
178
+ throw new Error('The broker[' + mq.getBrokerName() + '] not exist');
179
+ }
180
+ }
181
+ const header = new PopMessageRequestHeader();
182
+ header.bname = mq.getBrokerName();
183
+ header.consumerGroup = c.consumerGroup;
184
+ header.topic = mq.getTopic();
185
+ header.queueId = mq.getQueueId();
186
+ header.maxMsgNums = c.popBatchNums;
187
+ header.invisibleTime = invisibleTime;
188
+ header.pollTime = POP_BROKER_SUSPEND_MAX_TIME_MILLIS;
189
+ header.bornTime = Date.now();
190
+ // The consumer's own start policy, translated: a tail-starting group must not
191
+ // ask the broker to replay the queue from its first message.
192
+ header.initMode = consumeInitMode(c.consumeFromWhere);
193
+ // order stays false: this port does not do orderly POP.
194
+ header.order = false;
195
+ if (sub) {
196
+ header.expType = sub.expressionType;
197
+ header.exp = sub.subString;
198
+ }
199
+ // Wire rule: poll=true always here, and the network timeout gets +10s on
200
+ // top of the poll window we ask for.
201
+ const timeout = POP_BROKER_SUSPEND_MAX_TIME_MILLIS + POP_REQUEST_EXTRA_NETWORK_MILLIS;
202
+ return client.popMessage(mq.getBrokerName(), resolved.addr, header, c.namespace, timeout);
203
+ }
204
+ // ---------------------------------------------------------------------------
205
+ // pre-delivery filter
206
+ // ---------------------------------------------------------------------------
207
+ // processPopResult filters a FOUND pop answer before the round is booked or its
208
+ // rows are handed over. Four steps, in this order:
209
+ //
210
+ // (a) the client-side string tag check the pull result also gets. A
211
+ // subscription with a non-empty tag set drops a row whose TAGS property is
212
+ // absent or unknown to that set; an empty tag set (SUB_ALL) or a
213
+ // class/SQL92 expression switches the step off entirely and keeps
214
+ // everything, because then the broker's hash code set is the only filter.
215
+ // (b) the FilterMessageHook list, run over the mutable kept list. A hook that
216
+ // throws is logged and skipped: it must not cost the round its messages.
217
+ // (c) the retry budget, as an EARLY guard: a row whose reconsume count is
218
+ // STRICTLY past the effective budget (the knob, or 16 while the knob is
219
+ // -1) is dropped here instead of being handed to the listener. A row that
220
+ // has exactly reached the budget still goes through, and what the listener
221
+ // does not ACK stays governed by the post-listener budget handling in
222
+ // processPopConsumeResult.
223
+ // (d) every row (a)-(c) removed is ACKed immediately, with the same ACK the
224
+ // success path sends. An un-ACKed POP row just comes back when its
225
+ // invisibility window closes, so "filtered out but not acked" would replay
226
+ // the same rows round after round. The filter runs BEFORE the queue's
227
+ // answer debt is fed, so a dropped row never enters it: feeding the
228
+ // accumulator with the remaining count alone is what settles its
229
+ // bookkeeping.
230
+ //
231
+ // Nothing else about the round changes: the result's list is replaced by the
232
+ // remaining rows and everything else sees only those.
233
+ export function processPopResult(c, mq, sub, result) {
234
+ if (result.popStatus !== PopStatus.FOUND)
235
+ return result;
236
+ const found = result.msgFoundList;
237
+ if (found.length === 0)
238
+ return result;
239
+ let kept = clientSideTagFilter(sub, found);
240
+ const hooks = c.filterMessageHookList;
241
+ if (hooks.length > 0) {
242
+ const ctx = new FilterMessageContext();
243
+ ctx.consumerGroup = c.consumerGroup;
244
+ ctx.msgList = kept;
245
+ ctx.mq = mq;
246
+ ctx.unitMode = c.unitMode;
247
+ ctx.accessChannel = c.accessChannel;
248
+ for (const hook of hooks.slice()) {
249
+ try {
250
+ hook.filterMessage(ctx);
251
+ }
252
+ catch (e) {
253
+ logger.error('execute filter hook error, hookName=%s: %s', hook.getHookName(), e.message);
254
+ }
255
+ }
256
+ // A hook that replaced the list hands back its own; one that left it alone
257
+ // (or threw) leaves the list this step was given.
258
+ kept = Array.isArray(ctx.msgList) ? ctx.msgList : kept;
259
+ }
260
+ const maxReconsumeTimes = popMaxReconsumeTimes(c);
261
+ const deliverable = kept.filter((msg) => {
262
+ if (msg.getReconsumeTimes() > maxReconsumeTimes) {
263
+ logger.info('Reconsume times has reached %d, so ack msg=%s', msg.getReconsumeTimes(), msg.getMsgId());
264
+ return false;
265
+ }
266
+ return true;
267
+ });
268
+ if (deliverable.length !== found.length) {
269
+ for (const msg of found) {
270
+ if (!deliverable.includes(msg))
271
+ void ackMessagePop(c, msg);
272
+ }
273
+ }
274
+ result.msgFoundList = deliverable;
275
+ return result;
276
+ }
277
+ // ---------------------------------------------------------------------------
278
+ // consume
279
+ // ---------------------------------------------------------------------------
280
+ // submitPopConsume splits one pop batch into consumeMessageBatchMaxSize chunks
281
+ // and dispatches them; mirrors the broker-facing submit semantics.
282
+ function submitPopConsume(c, mq, msgs, pq) {
283
+ const batchSize = Math.max(1, c.consumeMessageBatchMaxSize);
284
+ for (let total = 0; total < msgs.length;) {
285
+ const end = Math.min(total + batchSize, msgs.length);
286
+ const chunk = msgs.slice(total, end);
287
+ total = end;
288
+ if (!c._beginInFlight()) {
289
+ // Shutdown froze new work: hand the debt back so nothing is left
290
+ // waiting for an ACK that will never come.
291
+ pq.decFoundMsg(-chunk.length);
292
+ return;
293
+ }
294
+ void consumePopBatch(c, mq, chunk, pq)
295
+ .catch((e) => {
296
+ logger.error('pop consume batch error for %s: %s', mqKey(mq), e.message);
297
+ pq.decFoundMsg(-chunk.length);
298
+ })
299
+ .finally(() => c._endInFlight());
300
+ }
301
+ }
302
+ // consumePopBatch runs the listener on one pop batch and settles the result.
303
+ async function consumePopBatch(c, mq, batch, pq) {
304
+ if (batch.length === 0)
305
+ return;
306
+ const key = mqKey(mq);
307
+ if (pq.isDropped()) {
308
+ logger.debug('pop batch dropped for %s (queue retired)', key);
309
+ return;
310
+ }
311
+ // The window can already have closed while this batch sat in the dispatch
312
+ // queue. Abort here without acking — the broker has revived the messages and
313
+ // another consumer may hold them.
314
+ if (popBatchTimedOut(batch)) {
315
+ logger.debug('the pop message time out so abort consume, mq=%s', key);
316
+ pq.decFoundMsg(-batch.length);
317
+ return;
318
+ }
319
+ // POP retry rows arrive under the retry topic the broker hid them on; the
320
+ // listener is handed the business topic they were subscribed to instead. The
321
+ // plain %RETRY%<group> restore and the namespace strip follow right after.
322
+ c._resetPopRetryTopic(batch);
323
+ c._resetRetryTopicAndNamespace(batch);
324
+ const ctx = new ConsumeConcurrentlyContext(mq);
325
+ const hooks = c.consumeMessageHookList;
326
+ let hookCtx = null;
327
+ if (hooks.length > 0) {
328
+ hookCtx = new ConsumeMessageContext(c.consumerGroup, batch, mq.getTopic());
329
+ hookCtx.mq = mq;
330
+ hookCtx.success = false;
331
+ hookCtx.props = {};
332
+ for (const hook of hooks.slice()) {
333
+ try {
334
+ hook.consumeMessageBefore(hookCtx);
335
+ }
336
+ catch (e) { /* hook errors don't stop delivery */ }
337
+ }
338
+ }
339
+ const begin = Date.now();
340
+ // Same CONSUME_START_TIME stamping as the pull path.
341
+ for (const msg of batch) {
342
+ msg.putProperty('CONSUME_START_TIME', String(Date.now()));
343
+ }
344
+ const { status: rawStatus, threw } = c._listenerCall(batch, ctx);
345
+ let status = rawStatus;
346
+ if (threw) {
347
+ logger.debug('pop listener error, treat as RECONSUME_LATER: mq=%s', key);
348
+ status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
349
+ }
350
+ const consumeRT = Date.now() - begin;
351
+ if (status !== ConsumeConcurrentlyStatus.CONSUME_SUCCESS
352
+ && status !== ConsumeConcurrentlyStatus.RECONSUME_LATER) {
353
+ logger.warning('consumeMessage return unknown status, Group: %s Msgs: %d MQ: %s', c.consumerGroup, batch.length, key);
354
+ status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
355
+ }
356
+ const invisibleTime = popInvisibleTimeOf(batch);
357
+ if (hookCtx) {
358
+ // The POP flavour of the consume-hook epilogue. It cannot reuse the pull
359
+ // path's rule: the pull path declares TIME_OUT when the listener took
360
+ // longer than consumeTimeout MINUTES, while the POP path declares it when
361
+ // the listener took longer than the batch's INVISIBILITY window — which is
362
+ // seconds, not minutes. Sharing the rule would make a POP trace report
363
+ // SUCCESS for a listener that overran its window.
364
+ const unknown = rawStatus !== ConsumeConcurrentlyStatus.CONSUME_SUCCESS
365
+ && rawStatus !== ConsumeConcurrentlyStatus.RECONSUME_LATER;
366
+ let type;
367
+ if (unknown) {
368
+ type = threw ? 'EXCEPTION' : 'RETURNNULL';
369
+ }
370
+ else if (consumeRT >= invisibleTime * 1000) {
371
+ type = 'TIME_OUT';
372
+ }
373
+ else if (status === ConsumeConcurrentlyStatus.RECONSUME_LATER) {
374
+ type = 'FAILED';
375
+ }
376
+ else {
377
+ type = 'SUCCESS';
378
+ }
379
+ hookCtx.props['ConsumeContextType'] = type;
380
+ hookCtx.success = status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
381
+ hookCtx.status = status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS
382
+ ? 'CONSUME_SUCCESS' : 'RECONSUME_LATER';
383
+ for (const hook of hooks.slice()) {
384
+ try {
385
+ hook.consumeMessageAfter(hookCtx);
386
+ }
387
+ catch (e) { /* hooks never break delivery */ }
388
+ }
389
+ }
390
+ // CONSUME_RT is bumped after the hook and before the validity re-check, so
391
+ // a batch that expired while the listener ran still contributes its RT (the
392
+ // listener DID run).
393
+ c._incConsumeRT(mq.getTopic(), consumeRT);
394
+ // Check the window again after the listener: a slow listener must not ACK
395
+ // messages the broker already owns.
396
+ if (pq.isDropped() || popBatchTimedOut(batch)) {
397
+ logger.warning('processQueue invalid or popTimeout, mq=%s', key);
398
+ pq.decFoundMsg(-batch.length);
399
+ return;
400
+ }
401
+ processPopConsumeResult(c, mq, batch, pq, status, ctx);
402
+ }
403
+ // processPopConsumeResult settles one batch:
404
+ //
405
+ // Two loops, and EVERY message decrements the debt exactly once, so the
406
+ // counter returns to zero whatever the outcome:
407
+ //
408
+ // - [0, ackIndex] are ACKed;
409
+ // - (ackIndex, size) are re-hidden: either with a longer invisibility window
410
+ // (changePopInvisibleTime) or — once the message has exhausted its retry
411
+ // budget — by checkNeedAckOrDelay, which gives up and ACKs when the message
412
+ // is older than twice the longest delay.
413
+ function processPopConsumeResult(c, mq, batch, pq, status, ctx) {
414
+ let ackIndex = -1;
415
+ if (status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS) {
416
+ ackIndex = ctx.ackIndex;
417
+ if (ackIndex >= batch.length)
418
+ ackIndex = batch.length - 1;
419
+ }
420
+ // The same OK/Failed accounting as the pull path, from the same clamped
421
+ // ackIndex the ACK loop below uses, and likewise before the loop.
422
+ if (status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS) {
423
+ c._incConsumeOKTPS(mq.getTopic(), ackIndex + 1);
424
+ if (batch.length - ackIndex - 1 > 0) {
425
+ c._incConsumeFailedTPS(mq.getTopic(), batch.length - ackIndex - 1);
426
+ }
427
+ }
428
+ else {
429
+ c._incConsumeFailedTPS(mq.getTopic(), batch.length);
430
+ }
431
+ for (let i = 0; i < batch.length; i++) {
432
+ const msg = batch[i];
433
+ if (i <= ackIndex) {
434
+ ackMessagePop(c, msg);
435
+ }
436
+ else if (msg.getReconsumeTimes() >= popMaxReconsumeTimes(c)) {
437
+ // Retry budget exhausted. checkNeedAckOrDelay either ACKs the message
438
+ // for good or extends its invisibility by the next notch.
439
+ checkNeedAckOrDelay(c, msg);
440
+ }
441
+ else {
442
+ const delayLevel = ctx.delayLevelWhenNextConsume;
443
+ changePopInvisibleTime(c, msg, delayLevel);
444
+ }
445
+ pq.ack();
446
+ }
447
+ }
448
+ // popMaxReconsumeTimes returns the retry budget for the POP path: -1 means 16
449
+ // (the concurrent rule), NOT Number.MAX_VALUE.
450
+ function popMaxReconsumeTimes(c) {
451
+ return c.maxReconsumeTimes === -1 ? 16 : c.maxReconsumeTimes;
452
+ }
453
+ // ---------------------------------------------------------------------------
454
+ // ack / extend
455
+ // ---------------------------------------------------------------------------
456
+ // popAckAddress resolves the broker address an ACK/CHANGE_INVISIBLETIME
457
+ // belongs to: the broker name from checkpoint segment 5, looked up against
458
+ // the master, refreshing the route once.
459
+ async function popAckAddress(c, brokerName, topic, queueId) {
460
+ const client = c.mqClient;
461
+ if (!client)
462
+ throw new Error('consumer not started');
463
+ const resolveAddr = (name) => c._findBrokerAddressInSubscribe(name, MixAll.MASTER_ID, true);
464
+ const desBrokerName = isLogicalQueueMockBrokerName(brokerName)
465
+ ? client.getBrokerNameFromMessageQueue(new MessageQueue(topic, brokerName, queueId))
466
+ : brokerName;
467
+ let resolved = resolveAddr(desBrokerName);
468
+ if (!resolved) {
469
+ await client.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
470
+ const retryName = isLogicalQueueMockBrokerName(brokerName)
471
+ ? client.getBrokerNameFromMessageQueue(new MessageQueue(topic, brokerName, queueId))
472
+ : brokerName;
473
+ resolved = resolveAddr(retryName);
474
+ }
475
+ if (!resolved) {
476
+ throw new Error('The broker[' + desBrokerName + '] master node does not exist');
477
+ }
478
+ return resolved.addr;
479
+ }
480
+ // ackMessagePop sends ACK_MESSAGE(200051) for one popped message.
481
+ //
482
+ // The request fields are read back out of the checkpoint, with two exceptions:
483
+ // the topic is re-derived through getRealTopic (so a retried message ACKs on
484
+ // the pop-retry topic) and the offset is segment 7 — the message's own queue
485
+ // offset, never the batch start.
486
+ export async function ackMessagePop(c, msg) {
487
+ const extraInfo = msg.getProperty(MessageConst.PROPERTY_POP_CK);
488
+ if (!extraInfo) {
489
+ logger.warning('ack skipped: message %s carries no POP_CK', msg.getMsgId());
490
+ return;
491
+ }
492
+ const parsed = ckParse(extraInfo);
493
+ if (!parsed) {
494
+ logger.warning('ack failed to parse POP_CK %q', extraInfo);
495
+ return;
496
+ }
497
+ const { brokerName, queueId, queueOffset } = parsed;
498
+ // The topic is resolved through the checkpoint's retry marker because the
499
+ // message's own topic has been rewritten to the request topic.
500
+ const topic = getRealTopicFromCk(extraInfo, msg.getTopic(), c.consumerGroup);
501
+ let addr;
502
+ try {
503
+ addr = await popAckAddress(c, brokerName, msg.getTopic(), queueId);
504
+ }
505
+ catch (e) {
506
+ logger.warning('ack failed, broker %s not resolvable: %s', brokerName, e.message);
507
+ return;
508
+ }
509
+ const header = new AckMessageRequestHeader();
510
+ header.bname = brokerName;
511
+ header.consumerGroup = c.consumerGroup;
512
+ header.topic = topic;
513
+ header.queueId = queueId;
514
+ header.extraInfo = extraInfo;
515
+ header.offset = queueOffset;
516
+ try {
517
+ const code = await c.mqClient.ackMessage(addr, header, POP_ASYNC_TIMEOUT_MILLIS);
518
+ if (code !== ResponseCode.SUCCESS) {
519
+ // Java maps any non-SUCCESS to AckStatus.NO_EXIST and only warns
520
+ // (DefaultMQPushConsumerImpl:842) — the checkpoint is already gone.
521
+ logger.warning('Ack message fail. extraInfo: %s AckStatus.NO_EXIST code=%s', extraInfo, code);
522
+ }
523
+ }
524
+ catch (e) {
525
+ logger.warning('Ack message fail. extraInfo: %s error: %s', extraInfo, e.message);
526
+ }
527
+ }
528
+ // changePopInvisibleTime extends a message's invisibility window.
529
+ //
530
+ // delayLevel 0 means "not chosen": it is replaced by the message's own
531
+ // reconsume count, which is what makes the back-off grow with each retry. The
532
+ // value is then looked up in the SECONDS table and sent as MILLISECONDS — the
533
+ // wire unit is milliseconds, so sending seconds here would hold messages for
534
+ // only milliseconds.
535
+ export function changePopInvisibleTime(c, msg, delayLevel) {
536
+ if (delayLevel === 0) {
537
+ delayLevel = msg.getReconsumeTimes();
538
+ }
539
+ changePopInvisibleTimeAtLevel(c, msg, delayLevel);
540
+ }
541
+ // changePopInvisibleTimeAtLevel is the shared body with the level ALREADY
542
+ // resolved: the "0 means unset" rule must not be applied a second time here,
543
+ // or a deliberate level-0 (the 10s back-off) would be re-read as "use the
544
+ // reconsume count" and every give-up path would silently wait much longer.
545
+ function changePopInvisibleTimeAtLevel(c, msg, delayLevel) {
546
+ const delaySecond = popDelayLevelSeconds(delayLevel);
547
+ const extraInfo = msg.getProperty(MessageConst.PROPERTY_POP_CK);
548
+ if (!extraInfo) {
549
+ logger.warning('changePopInvisibleTime skipped: message %s carries no POP_CK', msg.getMsgId());
550
+ return;
551
+ }
552
+ const parsed = ckParse(extraInfo);
553
+ if (!parsed) {
554
+ logger.warning('changePopInvisibleTime failed to parse POP_CK %q', extraInfo);
555
+ return;
556
+ }
557
+ const { brokerName, queueId, queueOffset } = parsed;
558
+ const topic = getRealTopicFromCk(extraInfo, msg.getTopic(), c.consumerGroup);
559
+ void (async () => {
560
+ let addr;
561
+ try {
562
+ addr = await popAckAddress(c, brokerName, msg.getTopic(), queueId);
563
+ }
564
+ catch (e) {
565
+ logger.warning('changePopInvisibleTime failed, broker %s not resolvable: %s', brokerName, e.message);
566
+ return;
567
+ }
568
+ const header = new ChangeInvisibleTimeRequestHeader();
569
+ header.bname = brokerName;
570
+ header.consumerGroup = c.consumerGroup;
571
+ header.topic = topic;
572
+ header.queueId = queueId;
573
+ header.extraInfo = extraInfo;
574
+ header.offset = queueOffset;
575
+ header.invisibleTime = delaySecond * 1000;
576
+ header.suspend = false;
577
+ try {
578
+ const outcome = await c.mqClient.changeInvisibleTime(addr, header, POP_ASYNC_TIMEOUT_MILLIS);
579
+ if (outcome.code !== ResponseCode.SUCCESS) {
580
+ // AckStatus.NO_EXIST (Java changeInvisibleTimeAsync:1030): the
581
+ // checkpoint was already revived/evicted — report it, nothing to rebuild.
582
+ logger.error('changePopInvisibleTimeAsync fail, group:%s msg:%s AckStatus.NO_EXIST code=%s remark=%s', c.consumerGroup, msg.getMsgId(), outcome.code, outcome.remark);
583
+ }
584
+ }
585
+ catch (e) {
586
+ logger.error('changePopInvisibleTimeAsync fail, group:%s msg:%s error:%s', c.consumerGroup, msg.getMsgId(), e.message);
587
+ }
588
+ })();
589
+ }
590
+ // checkNeedAckOrDelay: called once the retry budget is spent. If the message
591
+ // has been bouncing for longer than twice the longest back-off there is no
592
+ // point extending again — ACK it and let it go. Otherwise extend by the next
593
+ // notch above the elapsed time.
594
+ //
595
+ // The elapsed-to-notch lookup can return -1 (elapsed below the first notch,
596
+ // 10s). This port clamps to the first notch instead: the same "wait at least
597
+ // 10s" intent, without a negative index.
598
+ //
599
+ // The clamp goes through changePopInvisibleTimeAtLevel rather than
600
+ // changePopInvisibleTime on purpose: the latter treats 0 as "the caller did
601
+ // not choose a level, use the reconsume count", which would turn this clamp
602
+ // into a back-off of table[reconsumeTimes] seconds. 10s is the honest reading
603
+ // of the clamp.
604
+ function checkNeedAckOrDelay(c, msg) {
605
+ const last = POP_DELAY_LEVEL[POP_DELAY_LEVEL.length - 1];
606
+ const elapsed = Date.now() - msg.getBornTimestamp();
607
+ if (elapsed > last * 1000 * 2) {
608
+ logger.warning('Consume too many times, ack message async. message %s', msg.getMsgId());
609
+ void ackMessagePop(c, msg);
610
+ return;
611
+ }
612
+ let delayLevel = popDelayLevelForElapsed(elapsed);
613
+ if (delayLevel < 0)
614
+ delayLevel = 0;
615
+ changePopInvisibleTimeAtLevel(c, msg, delayLevel);
616
+ logger.warning('Consume too many times, but delay time %d not enough. changePopInvisibleTime to delayLevel %d . message key:%s', elapsed, delayLevel, msg.getProperty(MessageConst.PROPERTY_KEYS) || '');
617
+ }
618
+ // ---------------------------------------------------------------------------
619
+ // rebalance integration
620
+ // ---------------------------------------------------------------------------
621
+ // syncPopLoops is syncPullLoops' POP twin: retire the queues we no longer own,
622
+ // then start a pop loop for each newly assigned one.
623
+ //
624
+ // Order matters for the same reason as the pull path (retire before add), and
625
+ // there is no offset to settle: POP keeps no client cursor, so retiring only
626
+ // has to stop the loop and mark the queue dropped so an in-flight batch aborts
627
+ // instead of ACKing a batch the broker has taken back.
628
+ export function syncPopLoops(c) {
629
+ const current = new Set(c.assigned.map(mqKey));
630
+ for (const key of Array.from(c.popQueueTable.keys())) {
631
+ if (!current.has(key)) {
632
+ retirePopQueue(c, key);
633
+ }
634
+ }
635
+ for (const mq of c.assigned) {
636
+ const key = mqKey(mq);
637
+ if (c.popQueueTable.has(key))
638
+ continue;
639
+ const pq = new PopProcessQueue();
640
+ c.popQueueTable.set(key, pq);
641
+ const stopFlag = c._newStopFlag();
642
+ c.popQueueStopFlags.set(key, stopFlag);
643
+ void queuePopLoop(c, mq, stopFlag).catch((e) => logger.error('pop loop error for %s: %s', key, e.message || e));
644
+ }
645
+ // The broker beat and the application's queue listener are NOT spent here: POP
646
+ // syncs the loops it owns, and a round that only rebuilt one does not change what
647
+ // the broker holds. doRebalance pays both once, for the topics whose set moved.
648
+ }
649
+ // retirePopQueue drops one queue's POP state.
650
+ export function retirePopQueue(c, key) {
651
+ const stopFlag = c.popQueueStopFlags.get(key);
652
+ if (stopFlag) {
653
+ c._stopFlags.delete(stopFlag); // signal the pop loop to exit
654
+ c.popQueueStopFlags.delete(key);
655
+ }
656
+ const pq = c.popQueueTable.get(key);
657
+ if (pq)
658
+ pq.setDropped(true);
659
+ c.popQueueTable.delete(key);
660
+ }
661
+ // ---------------------------------------------------------------------------
662
+ // broker mode switch
663
+ // ---------------------------------------------------------------------------
664
+ // setMessageRequestModeOnBroker sends SET_MESSAGE_REQUEST_MODE(401) for every
665
+ // subscribed business topic, telling each broker to serve this (group, topic)
666
+ // pair in POP mode.
667
+ //
668
+ // This is an ADDITION to the client surface this SDK speaks: the classic
669
+ // remoting protocol expects the mode to be set by the operator (mqadmin /
670
+ // the console) and clients only read it back from the assignment. Without
671
+ // the call a client-side setPopMode(true) would pop a broker that still
672
+ // answers PULL requests, which looks like "POP returns nothing" rather than
673
+ // an error.
674
+ //
675
+ // The %RETRY% topics are skipped: the POP retry topic is
676
+ // %RETRY%<group>_<topic>, created by the broker, and its request mode follows
677
+ // the business topic.
678
+ export async function setMessageRequestModeOnBroker(c, timeoutMillis) {
679
+ const client = c.mqClient;
680
+ if (!client)
681
+ return new Error('consumer not started');
682
+ const topics = [];
683
+ for (const topic of c.subscription.keys()) {
684
+ if (MixAll.isRetryTopic(topic) || MixAll.isDlqTopic(topic))
685
+ continue;
686
+ topics.push(topic);
687
+ }
688
+ topics.sort();
689
+ let firstErr = null;
690
+ for (const topic of topics) {
691
+ // Resolve the master address by the topic's route.
692
+ const route = client.getTopicRouteData(topic);
693
+ let masterAddr = null;
694
+ if (route) {
695
+ for (const bd of route.brokerDatas || []) {
696
+ const master = bd.brokerAddrs && bd.brokerAddrs[MixAll.MASTER_ID];
697
+ if (master != null) {
698
+ masterAddr = master;
699
+ break;
700
+ }
701
+ }
702
+ }
703
+ if (!masterAddr) {
704
+ logger.warning('set message request mode POP: no broker for %s', topic);
705
+ if (!firstErr)
706
+ firstErr = new Error('no broker for topic ' + topic);
707
+ continue;
708
+ }
709
+ try {
710
+ await client.setMessageRequestMode(masterAddr, topic, c.consumerGroup, MESSAGE_REQUEST_MODE_POP, c.popShareQueueNum, timeoutMillis);
711
+ }
712
+ catch (e) {
713
+ logger.warning('set message request mode POP failed for %s/%s: %s', c.consumerGroup, topic, e.message);
714
+ if (!firstErr)
715
+ firstErr = e;
716
+ }
717
+ }
718
+ return firstErr;
719
+ }