rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/README.en.md +428 -0
  2. package/README.md +424 -2
  3. package/dist/client/admin.d.ts +229 -0
  4. package/dist/client/admin.js +2462 -0
  5. package/dist/client/allocate.d.ts +48 -0
  6. package/dist/client/allocate.js +365 -0
  7. package/dist/client/backpressure.d.ts +23 -0
  8. package/dist/client/backpressure.js +95 -0
  9. package/dist/client/consumer.d.ts +294 -0
  10. package/dist/client/consumer.js +3095 -0
  11. package/dist/client/consumer_result.d.ts +129 -0
  12. package/dist/client/consumer_result.js +138 -0
  13. package/dist/client/consumer_stats.d.ts +69 -0
  14. package/dist/client/consumer_stats.js +258 -0
  15. package/dist/client/exception.d.ts +39 -0
  16. package/dist/client/exception.js +36 -0
  17. package/dist/client/hook.d.ts +145 -0
  18. package/dist/client/hook.js +171 -0
  19. package/dist/client/latency.d.ts +61 -0
  20. package/dist/client/latency.js +239 -0
  21. package/dist/client/lite_pull_consumer.d.ts +151 -0
  22. package/dist/client/lite_pull_consumer.js +1096 -0
  23. package/dist/client/mq_client.d.ts +188 -0
  24. package/dist/client/mq_client.js +1678 -0
  25. package/dist/client/mq_client_manager.d.ts +21 -0
  26. package/dist/client/mq_client_manager.js +129 -0
  27. package/dist/client/offset_store.d.ts +45 -0
  28. package/dist/client/offset_store.js +389 -0
  29. package/dist/client/open_tracing_hook.d.ts +52 -0
  30. package/dist/client/open_tracing_hook.js +241 -0
  31. package/dist/client/pop_api.d.ts +28 -0
  32. package/dist/client/pop_api.js +224 -0
  33. package/dist/client/pop_consumer.d.ts +23 -0
  34. package/dist/client/pop_consumer.js +719 -0
  35. package/dist/client/pop_process_queue.d.ts +15 -0
  36. package/dist/client/pop_process_queue.js +51 -0
  37. package/dist/client/process_queue.d.ts +49 -0
  38. package/dist/client/process_queue.js +261 -0
  39. package/dist/client/produce_accumulator.d.ts +75 -0
  40. package/dist/client/produce_accumulator.js +288 -0
  41. package/dist/client/producer.d.ts +231 -0
  42. package/dist/client/producer.js +1774 -0
  43. package/dist/client/pull_api.d.ts +26 -0
  44. package/dist/client/pull_api.js +264 -0
  45. package/dist/client/pull_consumer.d.ts +119 -0
  46. package/dist/client/pull_consumer.js +876 -0
  47. package/dist/client/pull_schedule_service.d.ts +71 -0
  48. package/dist/client/pull_schedule_service.js +424 -0
  49. package/dist/client/request_reply.d.ts +61 -0
  50. package/dist/client/request_reply.js +232 -0
  51. package/dist/client/retry_message.d.ts +2 -0
  52. package/dist/client/retry_message.js +27 -0
  53. package/dist/client/send_result.d.ts +57 -0
  54. package/dist/client/send_result.js +70 -0
  55. package/dist/client/top_addressing.d.ts +20 -0
  56. package/dist/client/top_addressing.js +75 -0
  57. package/dist/client/trace_context.d.ts +61 -0
  58. package/dist/client/trace_context.js +373 -0
  59. package/dist/client/trace_dispatcher.d.ts +31 -0
  60. package/dist/client/trace_dispatcher.js +236 -0
  61. package/dist/client/trace_hook.d.ts +23 -0
  62. package/dist/client/trace_hook.js +228 -0
  63. package/dist/client/traceparent.d.ts +9 -0
  64. package/dist/client/traceparent.js +86 -0
  65. package/dist/common/boundary_type.d.ts +7 -0
  66. package/dist/common/boundary_type.js +16 -0
  67. package/dist/common/compress.d.ts +28 -0
  68. package/dist/common/compress.js +512 -0
  69. package/dist/common/message.d.ts +129 -0
  70. package/dist/common/message.js +214 -0
  71. package/dist/common/messageClientIdSetter.d.ts +7 -0
  72. package/dist/common/messageClientIdSetter.js +48 -0
  73. package/dist/common/messageConst.d.ts +68 -0
  74. package/dist/common/messageConst.js +68 -0
  75. package/dist/common/messageDecoder.d.ts +40 -0
  76. package/dist/common/messageDecoder.js +504 -0
  77. package/dist/common/messageType.d.ts +11 -0
  78. package/dist/common/messageType.js +11 -0
  79. package/dist/common/message_accessor.d.ts +39 -0
  80. package/dist/common/message_accessor.js +135 -0
  81. package/dist/common/mixAll.d.ts +106 -0
  82. package/dist/common/mixAll.js +247 -0
  83. package/dist/common/recall_message_handle.d.ts +25 -0
  84. package/dist/common/recall_message_handle.js +98 -0
  85. package/dist/common/subscriptionData.d.ts +36 -0
  86. package/dist/common/subscriptionData.js +64 -0
  87. package/dist/common/sysflag.d.ts +129 -0
  88. package/dist/common/sysflag.js +96 -0
  89. package/dist/common/topic_config.d.ts +66 -0
  90. package/dist/common/topic_config.js +106 -0
  91. package/dist/common/topic_validator.d.ts +49 -0
  92. package/dist/common/topic_validator.js +115 -0
  93. package/dist/common/utilAll.d.ts +39 -0
  94. package/dist/common/utilAll.js +154 -0
  95. package/dist/common/validators.d.ts +25 -0
  96. package/dist/common/validators.js +80 -0
  97. package/dist/index.d.ts +24 -0
  98. package/dist/index.js +36 -0
  99. package/dist/logging.d.ts +24 -0
  100. package/dist/logging.js +246 -0
  101. package/dist/remoting/acl.d.ts +16 -0
  102. package/dist/remoting/acl.js +39 -0
  103. package/dist/remoting/admin_body.d.ts +257 -0
  104. package/dist/remoting/admin_body.js +669 -0
  105. package/dist/remoting/bodies.d.ts +475 -0
  106. package/dist/remoting/bodies.js +1131 -0
  107. package/dist/remoting/client.d.ts +73 -0
  108. package/dist/remoting/client.js +591 -0
  109. package/dist/remoting/codes.d.ts +296 -0
  110. package/dist/remoting/codes.js +301 -0
  111. package/dist/remoting/exception.d.ts +45 -0
  112. package/dist/remoting/exception.js +58 -0
  113. package/dist/remoting/extra_info.d.ts +61 -0
  114. package/dist/remoting/extra_info.js +220 -0
  115. package/dist/remoting/headers.d.ts +758 -0
  116. package/dist/remoting/headers.js +1307 -0
  117. package/dist/remoting/heartbeat.d.ts +91 -0
  118. package/dist/remoting/heartbeat.js +227 -0
  119. package/dist/remoting/namespace.d.ts +14 -0
  120. package/dist/remoting/namespace.js +85 -0
  121. package/dist/remoting/pop_bodies.d.ts +58 -0
  122. package/dist/remoting/pop_bodies.js +214 -0
  123. package/dist/remoting/remotingCommand.d.ts +39 -0
  124. package/dist/remoting/remotingCommand.js +225 -0
  125. package/dist/remoting/route.d.ts +61 -0
  126. package/dist/remoting/route.js +288 -0
  127. package/dist/remoting/rpc_hooks.d.ts +31 -0
  128. package/dist/remoting/rpc_hooks.js +73 -0
  129. package/dist/remoting/serialize.d.ts +33 -0
  130. package/dist/remoting/serialize.js +372 -0
  131. package/dist/remoting/socks_proxy.d.ts +29 -0
  132. package/dist/remoting/socks_proxy.js +379 -0
  133. package/dist/remoting/static_topic.d.ts +49 -0
  134. package/dist/remoting/static_topic.js +244 -0
  135. package/dist/remoting/subscription.d.ts +65 -0
  136. package/dist/remoting/subscription.js +172 -0
  137. package/package.json +38 -4
@@ -0,0 +1,3095 @@
1
+ // DefaultMQPushConsumer — the push consumer: subscription and rebalance entry
2
+ // points plus the per-queue pull loops, in one class.
3
+ //
4
+ // Shape of the thing, per client instance:
5
+ //
6
+ // rebalance loop (20s, or 2s while starting up with nothing assigned)
7
+ // -> assigns queues, then syncPullLoops() retires/adds
8
+ // per-queue pull async-loop (long poll, suspend=true)
9
+ // -> buffers messages into that queue's processQueue
10
+ // one dispatch async-loop
11
+ // -> hands batches to concurrent listeners (bounded by corePoolSize),
12
+ // or consumes orderly batches inline (serialised)
13
+ // instance-scheduled tasks
14
+ // -> heartbeat (30s, MQClient), offset persist (5s), clean-expire, lock
15
+ //
16
+ // Ordering rules that are load-bearing (the same rationale applies throughout):
17
+ // - The instance's heartbeat puts this group into the broker's
18
+ // ConsumerManager; without it GET_CONSUMER_LIST_BY_GROUP answers empty and
19
+ // rebalance KEEPS the previous assignment (never "I own every queue").
20
+ // - Retiring a queue persists its CONSUMED offset BEFORE the new owner starts.
21
+ // - The initial offset is resolved at ASSIGNMENT time, not lazily at the
22
+ // first pull.
23
+ // - Messages pulled from %RETRY%<group> get msg.topic restored to the
24
+ // business topic BEFORE the listener (resetRetryTopicAndNamespace).
25
+ import { Buffer } from 'node:buffer';
26
+ import { MQClient, DEFAULT_CONCURRENT_HEARTBEAT_THREAD_POOL_SIZE } from "./mq_client.js";
27
+ import { MQClientManager } from "./mq_client_manager.js";
28
+ import { ProcessQueue } from "./process_queue.js";
29
+ import { syncPopLoops, retirePopQueue, setMessageRequestModeOnBroker, POP_DELAY_WHEN_CACHE_FLOW_CONTROL, } from "./pop_consumer.js";
30
+ import { LocalFileOffsetStore, RemoteBrokerOffsetStore, ReadOffsetMode, mqKey, messageQueueKeyJson, } from "./offset_store.js";
31
+ import { AllocateMessageQueueAveragely, } from "./allocate.js";
32
+ import { PullStatus, PullResult, ConsumeConcurrentlyStatus, ConsumeOrderlyStatus, ConsumeConcurrentlyContext, ConsumeOrderlyContext, MessageSelector, } from "./consumer_result.js";
33
+ import { ConsumeMessageContext, FilterMessageContext } from "./hook.js";
34
+ import { AccessChannelLocal, AccessChannelCloud } from "./trace_context.js";
35
+ import { PullMessageRequestHeader } from "../remoting/headers.js";
36
+ import { ConsumerSendMsgBackRequestHeader, LockBatchMqRequestHeader, UnlockBatchMqRequestHeader, GetConsumerRunningInfoRequestHeader, } from "../remoting/headers.js";
37
+ import { RemotingCommand } from "../remoting/remotingCommand.js";
38
+ import { RequestCode, ResponseCode } from "../remoting/codes.js";
39
+ import { RemotingSerializable } from "../remoting/serialize.js";
40
+ import { ConsumeType, MessageModel, ConsumeFromWhere, } from "../remoting/heartbeat.js";
41
+ import { GetConsumerStatusBody } from "../remoting/bodies.js";
42
+ import { PullSysFlag } from "../common/sysflag.js";
43
+ import { MessageQueue } from "../common/message.js";
44
+ import { MessageConst } from "../common/messageConst.js";
45
+ import { clientMsgId } from "../common/messageClientIdSetter.js";
46
+ import { MixAll } from "../common/mixAll.js";
47
+ import { defaultConsumeTimestamp, parseConsumeTimestamp, currentTimeMillis } from "../common/utilAll.js";
48
+ import { FilterAPI, ExpressionType } from "../common/subscriptionData.js";
49
+ import { decodeMessages, decodeInnerBatchMessages } from "../common/messageDecoder.js";
50
+ import { Validators } from "../common/validators.js";
51
+ import { buildRetryMessage } from "./retry_message.js";
52
+ import { registerRpcHooks } from "../remoting/rpc_hooks.js";
53
+ import { getLogger } from "../logging.js";
54
+ import { MQBrokerException, MQClientException, RemotingTimeoutException } from "../remoting/exception.js";
55
+ const logger = getLogger('client.consumer');
56
+ // Push-consumer defaults.
57
+ const DEFAULT_PULL_BATCH_SIZE = 32;
58
+ const DEFAULT_CONSUME_MESSAGE_BATCH_MAX_SIZE = 1;
59
+ const DEFAULT_CONSUME_THREAD_MIN = 20;
60
+ const DEFAULT_CONSUME_THREAD_MAX = 20;
61
+ const DEFAULT_ADJUST_THREAD_POOL_NUMS_THRESH = 100000;
62
+ // Java's default is 0 (do not wait); this port waits, see the field's note. The value is also
63
+ // the historic hard-coded drain budget shutdown() used before the knob existed.
64
+ const DEFAULT_AWAIT_TERMINATION_MILLIS_WHEN_SHUTDOWN = 30000;
65
+ const DEFAULT_CONSUME_CONCURRENTLY_MAX_SPAN = 2000;
66
+ const DEFAULT_PULL_THRESHOLD_FOR_QUEUE = 1000;
67
+ const DEFAULT_PULL_THRESHOLD_SIZE_FOR_QUEUE = 100;
68
+ const DEFAULT_PULL_THRESHOLD_FOR_TOPIC = -1;
69
+ const DEFAULT_PULL_THRESHOLD_SIZE_FOR_TOPIC = -1;
70
+ const DEFAULT_PULL_INTERVAL = 0;
71
+ const DEFAULT_PULL_TIMEOUT_MILLIS = 30000;
72
+ const DEFAULT_PULL_SUSPEND_TIMEOUT_MILLIS = 20000;
73
+ const DEFAULT_PULL_BATCH_SIZE_IN_BYTES = 256 * 1024;
74
+ const DEFAULT_MAX_RECONSUME_TIMES = -1;
75
+ const DEFAULT_SUSPEND_CURRENT_QUEUE_TIME_MS = 1000;
76
+ const DEFAULT_CONSUME_TIMEOUT = 15;
77
+ // The rebalance period is 20s. While the consumer is young
78
+ // (60s) and has NOTHING assigned, retry every 2s so a consumer that starts
79
+ // before the topic exists is not stalled for a long time.
80
+ const REBALANCE_INTERVAL = 20 * 1000;
81
+ const REBALANCE_INTERVAL_DURING_STARTUP = 2 * 1000;
82
+ const REBALANCE_STARTUP_WINDOW = 60 * 1000;
83
+ const PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL = 50;
84
+ // The BROKER refusing a pull because it is flow-controlling its own queues (215)
85
+ // is a different event from this client's own cache threshold: nothing landed in
86
+ // the local buffer and the request already round-tripped, so the queue is
87
+ // readable again as soon as the broker's pressure drops — a shorter wait.
88
+ export const PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL = 20;
89
+ // DEFAULT_PULL_RETRY_DELAY_MILLIS is the default of the ONE generic error
90
+ // backoff: the wait a queue's loop takes before asking again after a round that
91
+ // failed for any reason other than the broker's own flow control (a system error,
92
+ // a busy answer, a dead connection, a broker the route cannot resolve, a queue
93
+ // with no cursor yet). It is a per-client knob (pullRetryDelayMillis) rather than
94
+ // a fixed constant: retrying hot at a sick broker adds pressure to whatever is
95
+ // already wrong, so the patience this client shows is the operator's call.
96
+ export const DEFAULT_PULL_RETRY_DELAY_MILLIS = 1000;
97
+ // A throttled queue trips a cache gate on every pull round, so warning every round buries the
98
+ // signal in noise: only every Nth trip warns (the first, then the 1001st, ...).
99
+ const FLOW_CONTROL_WARN_EVERY = 1000;
100
+ // PULL_TIME_DELAY_MILLS_WHEN_SUSPEND (1000ms): while suspend() is in effect the
101
+ // pull/pop loops do not exit — they wait this long and re-check the gate.
102
+ const PULL_TIME_DELAY_MILLS_WHEN_SUSPEND = 1000;
103
+ const INT32_MAX = 2147483647;
104
+ const DEFAULT_CONSUMER_GROUP = 'DEFAULT_CONSUMER';
105
+ export { MessageSelector };
106
+ // isFlowControlWarnTrip reports whether the n-th (1-based) flow-control trip logs a WARN.
107
+ export function isFlowControlWarnTrip(times) {
108
+ return (times - 1) % FLOW_CONTROL_WARN_EVERY === 0;
109
+ }
110
+ // pullRetryDelayMillisFor maps a FAILED pull round to how long this queue's loop
111
+ // waits before asking again:
112
+ // * the broker's own flow-control answer (215): the short pause — nothing landed
113
+ // in the local buffer and the queue is readable again as soon as the broker's
114
+ // pressure drops, so this round triggers no rebalance and no threshold rescan;
115
+ // * anything else (a system error, a busy answer, a dead connection, or no code
116
+ // at all because the failure never reached the wire): the generic pause this
117
+ // client was configured with.
118
+ //
119
+ // `pullRetryDelayMillis` is the loop's OWN copy of the knob, captured when the
120
+ // queue's loop was set up: a caller who changes the knob afterwards does not
121
+ // alter a round that is already scheduled. It defaults to the knob's own default,
122
+ // so a caller that maps a code without a consumer in hand still gets 1000 ms.
123
+ //
124
+ // This client's OWN backlog threshold is not here: _flowControlHit stops the pull
125
+ // before the request goes out and sleeps PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL,
126
+ // so the two waits never meet.
127
+ export function pullRetryDelayMillisFor(responseCode, pullRetryDelayMillis = DEFAULT_PULL_RETRY_DELAY_MILLIS) {
128
+ return responseCode === ResponseCode.FLOW_CONTROL
129
+ ? PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL
130
+ : pullRetryDelayMillis;
131
+ }
132
+ // NO_MESSAGE_QUEUE_OFFSET stands in for "this round produced no row" wherever the
133
+ // first row's queue offset is compared against a cursor: no cursor a round asked
134
+ // with can sit above it, so an empty round can only trip the next-offset leg.
135
+ const NO_MESSAGE_QUEUE_OFFSET = Number.MAX_SAFE_INTEGER;
136
+ // pullOffsetsLookWrong is the offset sanity check of a FOUND pull round. The round
137
+ // asked with `prevRequestOffset`; the answer either points the cursor backwards
138
+ // (`nextBeginOffset`) or hands over a first row (`firstMsgOffset`) from before that
139
+ // ask. Either way the cursor this client keeps and the queue it reads disagree, and
140
+ // that is worth a line in the log — nothing else: the check changes no offset, no
141
+ // wait and no counter, because the round's own values are all still correct to use.
142
+ export function pullOffsetsLookWrong(nextBeginOffset, firstMsgOffset, prevRequestOffset) {
143
+ return nextBeginOffset < prevRequestOffset || firstMsgOffset < prevRequestOffset;
144
+ }
145
+ // ConsumeReturnType — the ORDINAL is a wire-format value: a SubAfter trace
146
+ // record stores it as contextCode. Do not reorder.
147
+ export const ConsumeReturnType = {
148
+ SUCCESS: 0, TIME_OUT: 1, EXCEPTION: 2, RETURNNULL: 3, FAILED: 4,
149
+ };
150
+ const CONSUME_RETURN_TYPE_NAMES = ['SUCCESS', 'TIME_OUT', 'EXCEPTION', 'RETURNNULL', 'FAILED'];
151
+ function consumeReturnTypeName(t) {
152
+ return CONSUME_RETURN_TYPE_NAMES[t] || 'SUCCESS';
153
+ }
154
+ export class DefaultMQPushConsumer {
155
+ consumerGroup;
156
+ namespace = '';
157
+ // Server-side namespaceV2 — the `nsd`/`ns` extFields stamped by
158
+ // NamespaceRpcHook, independent from `namespace`.
159
+ namespaceV2 = null;
160
+ // User RPCHook (e.g. AclRPCHook for broker-side ACL signing), installed on
161
+ // the shared MQClient instance's remoting client at creation, exactly like the
162
+ // producer's `rpcHook`. Registered AFTER the namespace hook so the ACL
163
+ // signature covers every extField the namespace hook stamps.
164
+ rpcHook = null;
165
+ // Java ClientConfig#accessChannel (default LOCAL): CLOUD routes trace records to
166
+ // rmq_sys_TRACE_DATA_<regionId> and is stamped onto every consume hook context.
167
+ accessChannel = AccessChannelLocal;
168
+ instanceName = 'DEFAULT';
169
+ clientID = '';
170
+ unitMode = false;
171
+ tlsEnable = null;
172
+ // TLS details (see setTlsOptions); null = test-mode.
173
+ tlsOptions = null;
174
+ socksProxyConfig = null;
175
+ messageModel = MessageModel.CLUSTERING;
176
+ consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET;
177
+ consumeTimestamp = defaultConsumeTimestamp();
178
+ consumeThreadMin = DEFAULT_CONSUME_THREAD_MIN;
179
+ consumeThreadMax = DEFAULT_CONSUME_THREAD_MAX;
180
+ corePoolSize = DEFAULT_CONSUME_THREAD_MIN;
181
+ adjustThreadPoolNumsThreshold = DEFAULT_ADJUST_THREAD_POOL_NUMS_THRESH;
182
+ // awaitTerminationMillisWhenShutdown (Java DefaultMQPushConsumer#setAwaitTerminationMillisWhenShutdown):
183
+ // how long shutdown() waits for in-flight listener calls before detaching them.
184
+ // DEVIATION: Java defaults this to 0 ("do not wait"). This port keeps its historic 30000
185
+ // because an immediate process exit can cut a CONSUMER_SEND_MSG_BACK(36) short and the
186
+ // message then never reaches %RETRY%/%DLQ% — see the drain in shutdown(). Both 0 and the
187
+ // 30s window are reachable through the setter, so Java-configured code keeps its meaning.
188
+ awaitTerminationMillisWhenShutdown = DEFAULT_AWAIT_TERMINATION_MILLIS_WHEN_SHUTDOWN;
189
+ consumeConcurrentlyMaxSpan = DEFAULT_CONSUME_CONCURRENTLY_MAX_SPAN;
190
+ pullThresholdForQueue = DEFAULT_PULL_THRESHOLD_FOR_QUEUE;
191
+ pullThresholdSizeForQueue = DEFAULT_PULL_THRESHOLD_SIZE_FOR_QUEUE;
192
+ pullThresholdForTopic = DEFAULT_PULL_THRESHOLD_FOR_TOPIC;
193
+ pullThresholdSizeForTopic = DEFAULT_PULL_THRESHOLD_SIZE_FOR_TOPIC;
194
+ pullInterval = DEFAULT_PULL_INTERVAL;
195
+ // The generic error backoff, in ms: the wait before re-issuing a pull (or POP)
196
+ // request after a round that failed for any reason other than the broker's own
197
+ // flow control. Both loops take this value when a queue's loop is set up, so
198
+ // changing it after start() does not alter a loop already running.
199
+ pullRetryDelayMillis = DEFAULT_PULL_RETRY_DELAY_MILLIS;
200
+ pullTimeoutMillis = DEFAULT_PULL_TIMEOUT_MILLIS;
201
+ pullSuspendTimeoutMillis = DEFAULT_PULL_SUSPEND_TIMEOUT_MILLIS;
202
+ consumeMessageBatchMaxSize = DEFAULT_CONSUME_MESSAGE_BATCH_MAX_SIZE;
203
+ pullBatchSize = DEFAULT_PULL_BATCH_SIZE;
204
+ pullBatchSizeInBytes = DEFAULT_PULL_BATCH_SIZE_IN_BYTES;
205
+ postSubscriptionWhenPull = false;
206
+ maxReconsumeTimes = DEFAULT_MAX_RECONSUME_TIMES;
207
+ suspendCurrentQueueTimeMillis = DEFAULT_SUSPEND_CURRENT_QUEUE_TIME_MS;
208
+ consumeTimeout = DEFAULT_CONSUME_TIMEOUT;
209
+ // Offset flush period (default 5000 ms): applied to
210
+ // the shared MQClient instance when THIS facade creates it (first creator's
211
+ // config wins, because the instance is built once and then shared); it drives
212
+ // the instance-level persistAllConsumerOffset sweep.
213
+ persistConsumerOffsetInterval = 5000;
214
+ // Heartbeat V2: off by default, so heartbeats keep the V1 shape unless enabled.
215
+ useHeartbeatV2 = false;
216
+ // Materialise received message bodies during batch decode (default true).
217
+ decodeReadBody = true;
218
+ // Internal-API timeout (heartbeat, route fetch, check client, unregister and
219
+ // consumer-list RPCs), in milliseconds. Default 3000.
220
+ mqClientApiTimeout = 3000;
221
+ // Concurrent heartbeat round (ignored while heartbeat V2 is on).
222
+ enableConcurrentHeartbeat = false;
223
+ concurrentHeartbeatThreadPoolSize = DEFAULT_CONCURRENT_HEARTBEAT_THREAD_POOL_SIZE;
224
+ // Page size of the paged metadata admin calls (maxTopicNum / maxGroupNum).
225
+ maxPageSizeInGetMetadata = 2000;
226
+ allocateStrategy = new AllocateMessageQueueAveragely();
227
+ messageQueueListener = null;
228
+ nameServerAddr = '';
229
+ // wire fields read by MQClient.buildHeartbeatData:
230
+ consumeType = ConsumeType.CONSUME_PASSIVELY;
231
+ unitModeFlag = false;
232
+ get unitModeValue() { return this.unitMode; }
233
+ subscription = new Map();
234
+ get subscriptionDataSet() { return this.subscriptions(); }
235
+ listener = null;
236
+ orderly = false;
237
+ consumeMessageHookList = [];
238
+ filterMessageHookList = [];
239
+ enableTrace = false;
240
+ traceMsgBatchNum = 10;
241
+ traceTopic = '';
242
+ traceDispatcher = null;
243
+ mqClient = null;
244
+ // Reverse-request processors this facade registers on the shared instance
245
+ // (code -> handler); removed on shutdown so an exiting consumer stops
246
+ // consuming the multicast fan-out.
247
+ _facadeHandlers = new Map();
248
+ pullFromWhichNode = new Map(); // mqKey -> brokerId
249
+ offsetStore = null;
250
+ started = false;
251
+ // `pause` is a CONSUMER-WIDE switch flipped by suspend()/resume(). Distinct
252
+ // from the lite-pull consumer's per-queue paused set.
253
+ _paused = false;
254
+ _stopFlags = new Set();
255
+ _startTime = 0;
256
+ // processQueueTable: the queues this instance currently owns. Membership is
257
+ // also "which queues may be pulled".
258
+ processQueueTable = new Map();
259
+ _queueStopFlags = new Map();
260
+ assigned = [];
261
+ // offsetTable is the pull cursor (the pull request's nextOffset), distinct
262
+ // from the consumed offset the store holds.
263
+ offsetTable = new Map();
264
+ _frozenOffsets = new Set();
265
+ _queueEpoch = new Map();
266
+ _msgAccCnt = new Map();
267
+ _dispatchSem = 0;
268
+ _draining = false;
269
+ _inFlight = 0;
270
+ _inFlightWaiters = [];
271
+ _timers = [];
272
+ _rebalanceNow = false;
273
+ _producer = null;
274
+ flowControlTriggered = 0;
275
+ // ---- POP mode. When popMode is on the broker serves this
276
+ // (group, topic) pair in POP mode: no client cursor, ACK per message.
277
+ popMode = false;
278
+ popInvisibleTime = 60000;
279
+ popBatchNums = 32;
280
+ popShareQueueNum = 0;
281
+ popThresholdForQueue = 96;
282
+ // popQueueTable is the POP counterpart of processQueueTable. Only one of the
283
+ // two is populated at a time; POP has no offset cursor, so the only state is
284
+ // the outstanding-ACK debt per queue.
285
+ popQueueTable = new Map();
286
+ popQueueStopFlags = new Map();
287
+ // Consumer-side stats (ConsumerStatsManager, shared on the MQClient
288
+ // instance; assigned in start()). Differential-window model — see
289
+ // consumer_stats.ts.
290
+ _statsManager = null;
291
+ constructor(consumerGroup = 'DEFAULT_CONSUMER') {
292
+ this.consumerGroup = consumerGroup || 'DEFAULT_CONSUMER';
293
+ }
294
+ // ------------------------------------------------------------- config
295
+ setNamesrvAddr(addr) { this.nameServerAddr = addr; return this; }
296
+ setNamespace(ns) { this.namespace = ns; return this; }
297
+ // Server-side namespace accessor, read live per request by NamespaceRpcHook.
298
+ setNamespaceV2(ns) { this.namespaceV2 = ns; return this; }
299
+ getNamespaceV2() { return this.namespaceV2; }
300
+ // Java ClientConfig#setAccessChannel: the trace dispatcher and every consume
301
+ // hook context read this. Set it before start(); LOCAL is the default.
302
+ setAccessChannel(channel) {
303
+ if (channel !== AccessChannelLocal && channel !== AccessChannelCloud) {
304
+ throw new Error(`invalid accessChannel: ${channel}`);
305
+ }
306
+ this.accessChannel = channel;
307
+ return this;
308
+ }
309
+ getAccessChannel() { return this.accessChannel; }
310
+ setInstanceName(name) { this.instanceName = name; return this; }
311
+ setUnitMode(mode) { this.unitMode = mode; return this; }
312
+ setTlsEnable(enable) { this.tlsEnable = enable; return this; }
313
+ // TLS details (caCert=strict CA verification; clientCert/clientKey=mTLS; serverName=hostname override).
314
+ setTlsOptions(opts) {
315
+ this.tlsOptions = opts;
316
+ return this;
317
+ }
318
+ setSocksProxyConfig(config) { this.socksProxyConfig = config; return this; }
319
+ setMessageModel(model) { this.messageModel = model; return this; }
320
+ setConsumeFromWhere(where) { this.consumeFromWhere = where; return this; }
321
+ setConsumeTimestamp(ts) { this.consumeTimestamp = ts; return this; }
322
+ setConsumeThreadMin(n) { this.consumeThreadMin = n; this.corePoolSize = n; return this; }
323
+ setConsumeThreadMax(n) { this.consumeThreadMax = n; return this; }
324
+ setConsumeThreadNums(n) {
325
+ this.consumeThreadMin = n;
326
+ this.consumeThreadMax = n;
327
+ this.corePoolSize = n;
328
+ return this;
329
+ }
330
+ // Java DefaultMQPushConsumer#setAwaitTerminationMillisWhenShutdown: the shutdown() drain
331
+ // budget for in-flight listener calls. 0 = do not wait (detach what is running).
332
+ setAwaitTerminationMillisWhenShutdown(ms) {
333
+ this.awaitTerminationMillisWhenShutdown = Math.max(0, Math.floor(ms));
334
+ return this;
335
+ }
336
+ getAwaitTerminationMillisWhenShutdown() { return this.awaitTerminationMillisWhenShutdown; }
337
+ setPullBatchSize(n) { this.pullBatchSize = n; return this; }
338
+ setPullBatchSizeInBytes(n) { this.pullBatchSizeInBytes = n; return this; }
339
+ setPullInterval(ms) { this.pullInterval = ms; return this; }
340
+ // SetPullRetryDelayMillis sets the generic error backoff (ms): how long a
341
+ // queue's loop waits before re-issuing a pull/POP request after a round that
342
+ // failed for any reason other than the broker's own flow control. A queue's
343
+ // loop keeps the value it was set up with.
344
+ setPullRetryDelayMillis(ms) { this.pullRetryDelayMillis = ms; return this; }
345
+ getPullRetryDelayMillis() { return this.pullRetryDelayMillis; }
346
+ setPullTimeoutMillis(ms) { this.pullTimeoutMillis = ms; return this; }
347
+ setPullSuspendTimeoutMillis(ms) { this.pullSuspendTimeoutMillis = ms; return this; }
348
+ setConsumeMessageBatchMaxSize(n) { this.consumeMessageBatchMaxSize = n; return this; }
349
+ setPostSubscriptionWhenPull(enable) { this.postSubscriptionWhenPull = enable; return this; }
350
+ setMaxReconsumeTimes(n) { this.maxReconsumeTimes = n; return this; }
351
+ setSuspendCurrentQueueTimeMillis(ms) { this.suspendCurrentQueueTimeMillis = ms; return this; }
352
+ setConsumeTimeout(minutes) { this.consumeTimeout = minutes; return this; }
353
+ setConsumeConcurrentlyMaxSpan(v) { this.consumeConcurrentlyMaxSpan = v; return this; }
354
+ // SetPopMode switches this consumer to POP mode (the broker-side
355
+ // MessageRequestMode). A POP consumer must not be orderly: the POP protocol
356
+ // has no queue lock to serialise on, so an orderly POP consumer would
357
+ // silently lose ordering — checkConfig rejects the combination at start().
358
+ setPopMode(enable) { this.popMode = enable; return this; }
359
+ get isPopMode() { return this.popMode; }
360
+ // SetPopInvisibleTime sets how long a popped batch stays invisible (ms).
361
+ setPopInvisibleTime(ms) { this.popInvisibleTime = ms; return this; }
362
+ setPopBatchNums(n) { this.popBatchNums = n; return this; }
363
+ setPopShareQueueNum(n) { this.popShareQueueNum = n; return this; }
364
+ setPopThresholdForQueue(n) { this.popThresholdForQueue = n; return this; }
365
+ setPullThresholdForQueue(v) { this.pullThresholdForQueue = v; return this; }
366
+ setPullThresholdSizeForQueue(v) { this.pullThresholdSizeForQueue = v; return this; }
367
+ setPullThresholdForTopic(v) { this.pullThresholdForTopic = v; return this; }
368
+ setPullThresholdSizeForTopic(v) { this.pullThresholdSizeForTopic = v; return this; }
369
+ setAllocateMessageQueueStrategy(s) { this.allocateStrategy = s; return this; }
370
+ setMessageQueueListener(l) { this.messageQueueListener = l; return this; }
371
+ registerConsumeMessageHook(hook) {
372
+ if (hook)
373
+ this.consumeMessageHookList.push(hook);
374
+ return this;
375
+ }
376
+ registerFilterMessageHook(hook) {
377
+ if (hook)
378
+ this.filterMessageHookList.push(hook);
379
+ return this;
380
+ }
381
+ setEnableTrace(enable) { this.enableTrace = enable; return this; }
382
+ setTraceTopic(topic) { this.traceTopic = topic; return this; }
383
+ // Flush period for consumed offsets, in milliseconds; must be greater than 0.
384
+ // When already started the change is pushed through to the shared instance's
385
+ // sweep timer immediately.
386
+ setPersistConsumerOffsetInterval(ms) {
387
+ if (!Number.isFinite(ms) || ms <= 0)
388
+ throw new Error(`persistConsumerOffsetInterval must be > 0, got ${ms}`);
389
+ this.persistConsumerOffsetInterval = Math.floor(ms);
390
+ if (this.mqClient)
391
+ this.mqClient.setPersistConsumerOffsetInterval(this.persistConsumerOffsetInterval);
392
+ return this;
393
+ }
394
+ getPersistConsumerOffsetInterval() { return this.persistConsumerOffsetInterval; }
395
+ // Heartbeat V2 switch. Only a broker that
396
+ // answers the V2 ext fields on HEART_BEAT is ever sent a without-subscription
397
+ // body, so turning this on against an old broker is harmless.
398
+ setUseHeartbeatV2(enable) {
399
+ this.useHeartbeatV2 = enable === true;
400
+ if (this.mqClient)
401
+ this.mqClient.setUseHeartbeatV2(this.useHeartbeatV2);
402
+ return this;
403
+ }
404
+ getUseHeartbeatV2() { return this.useHeartbeatV2; }
405
+ // Received-body decode switch, mirrored onto the shared instance.
406
+ setDecodeReadBody(enable) {
407
+ this.decodeReadBody = enable === true;
408
+ if (this.mqClient)
409
+ this.mqClient.setDecodeReadBody(this.decodeReadBody);
410
+ return this;
411
+ }
412
+ getDecodeReadBody() { return this.decodeReadBody; }
413
+ // Internal-API timeout, in milliseconds. An explicit per-call timeout passed
414
+ // by a caller still wins.
415
+ setMqClientApiTimeout(ms) {
416
+ if (!Number.isFinite(ms) || ms <= 0)
417
+ throw new Error(`mqClientApiTimeout must be > 0, got ${ms}`);
418
+ this.mqClientApiTimeout = Math.floor(ms);
419
+ if (this.mqClient)
420
+ this.mqClient.setMqClientApiTimeout(this.mqClientApiTimeout);
421
+ return this;
422
+ }
423
+ getMqClientApiTimeout() { return this.mqClientApiTimeout; }
424
+ // Concurrent heartbeat round; heartbeat V2 still wins when it is enabled.
425
+ setEnableConcurrentHeartbeat(enable) {
426
+ this.enableConcurrentHeartbeat = enable === true;
427
+ if (this.mqClient)
428
+ this.mqClient.setEnableConcurrentHeartbeat(this.enableConcurrentHeartbeat);
429
+ return this;
430
+ }
431
+ getEnableConcurrentHeartbeat() { return this.enableConcurrentHeartbeat; }
432
+ setConcurrentHeartbeatThreadPoolSize(size) {
433
+ if (!Number.isInteger(size) || size <= 0) {
434
+ throw new Error(`concurrentHeartbeatThreadPoolSize must be a positive integer, got ${size}`);
435
+ }
436
+ this.concurrentHeartbeatThreadPoolSize = size;
437
+ if (this.mqClient)
438
+ this.mqClient.setConcurrentHeartbeatThreadPoolSize(size);
439
+ return this;
440
+ }
441
+ getConcurrentHeartbeatThreadPoolSize() { return this.concurrentHeartbeatThreadPoolSize; }
442
+ setMaxPageSizeInGetMetadata(size) {
443
+ if (!Number.isInteger(size) || size <= 0) {
444
+ throw new Error(`maxPageSizeInGetMetadata must be a positive integer, got ${size}`);
445
+ }
446
+ this.maxPageSizeInGetMetadata = size;
447
+ if (this.mqClient)
448
+ this.mqClient.setMaxPageSizeInGetMetadata(size);
449
+ return this;
450
+ }
451
+ getMaxPageSizeInGetMetadata() { return this.maxPageSizeInGetMetadata; }
452
+ isOrderly() { return this.orderly; }
453
+ isStarted() { return this.started; }
454
+ getConsumerGroup() { return this.consumerGroup; }
455
+ getClientId() { return this.clientID; }
456
+ // Subscribe registers a tag subscription. An empty expression defaults to
457
+ // "*". A selector object {type, expression} selects the expression type.
458
+ subscribe(topic, subExpression = '*') {
459
+ if (!topic || !topic.trim())
460
+ throw new Error('subscription topic is empty');
461
+ // With a namespace set the broker-side topic is the prefixed name; every
462
+ // downstream step (route, pull request, heartbeat subscription) reads the
463
+ // subscription key, and delivery strips the prefix again.
464
+ topic = MixAll.wrapNamespace(this.namespace, topic);
465
+ let expr, exprType;
466
+ if (typeof subExpression === 'string') {
467
+ expr = subExpression || '*';
468
+ exprType = ExpressionType.TAG;
469
+ }
470
+ else {
471
+ expr = subExpression.expression || '*';
472
+ exprType = subExpression.type || ExpressionType.TAG;
473
+ }
474
+ // FilterAPI.build, NOT buildSubscriptionData: a SQL92/CLASS_FILTER
475
+ // subscription must go out with EMPTY tagsSet/codeSet (a separate build
476
+ // branch). Filling them makes the broker intersect the pull against the
477
+ // hash of the expression text and deliver nothing.
478
+ const sub = FilterAPI.build(topic, expr, exprType);
479
+ const existed = this.subscription.has(topic);
480
+ this.subscription.set(topic, sub);
481
+ if (existed && this.started) {
482
+ // Subscription change: re-run rebalance immediately so a changed
483
+ // expression takes effect without waiting out the 20s timer.
484
+ this.rebalanceImmediately();
485
+ }
486
+ return this;
487
+ }
488
+ unsubscribe(topic) {
489
+ const existed = this.subscription.delete(MixAll.wrapNamespace(this.namespace, topic));
490
+ if (existed && this.started)
491
+ this.rebalanceImmediately();
492
+ }
493
+ subscriptions() {
494
+ return Array.from(this.subscription.values()).sort((a, b) => a.topic.localeCompare(b.topic));
495
+ }
496
+ // SetMessageListener installs the listener: a function (msgs, ctx) => status
497
+ // or an object with a consumeMessage method. Default is CONCURRENT; use
498
+ // setOrderly(true) / the second arg for orderly mode.
499
+ registerMessageListener(listener, orderly = false) {
500
+ this.listener = listener;
501
+ this.orderly = !!orderly;
502
+ return this;
503
+ }
504
+ registerMessageListenerConcurrently(listener) { return this.registerMessageListener(listener, false); }
505
+ registerMessageListenerOrderly(listener) { return this.registerMessageListener(listener, true); }
506
+ _listenerCall(msgs, ctx) {
507
+ const fn = typeof this.listener === 'function'
508
+ ? this.listener
509
+ : (this.listener && typeof this.listener.consumeMessage === 'function' ? this.listener.consumeMessage.bind(this.listener) : null);
510
+ if (!fn)
511
+ throw new Error('listener is not callable');
512
+ try {
513
+ return { status: fn(msgs, ctx), threw: false };
514
+ }
515
+ catch (e) {
516
+ // Listener exception rule: a throw is RECONSUME_LATER (concurrent) or
517
+ // suspend-and-retry-in-place (orderly) — never a crash.
518
+ logger.debug('listener error: %s', e.message);
519
+ return { status: -1, threw: true };
520
+ }
521
+ }
522
+ // ------------------------------------------------- broker-initiated requests
523
+ // RebalanceImmediately is the NOTIFY_CONSUMER_IDS_CHANGED(40) fan-out target.
524
+ // It must only set a flag — this runs on the remoting read path, and any
525
+ // synchronous request here would deadlock the connection.
526
+ rebalanceImmediately() {
527
+ this._rebalanceNow = true;
528
+ }
529
+ // ---------------------------------------------------------------- lifecycle
530
+ // Start brings the consumer up in a fixed order: validate config, make the
531
+ // instance name unique, register hooks, pre-flight check, first heartbeat,
532
+ // then the first rebalance.
533
+ async start() {
534
+ if (this.started)
535
+ return;
536
+ if (this.namespace) {
537
+ // Must run BEFORE the retry topic is derived: the retry topic is
538
+ // %RETRY% + the WRAPPED group name.
539
+ this.consumerGroup = MixAll.wrapNamespace(this.namespace, this.consumerGroup);
540
+ }
541
+ Validators.checkGroup(this.consumerGroup);
542
+ if (this.consumerGroup === DEFAULT_CONSUMER_GROUP) {
543
+ throw new Error(`consumerGroup can not equal ${DEFAULT_CONSUMER_GROUP}, please specify another one.`);
544
+ }
545
+ if (!this.nameServerAddr)
546
+ throw new Error('name server address is not set');
547
+ if (this.subscription.size === 0) {
548
+ throw new Error('subscription is not set, call subscribe() first');
549
+ }
550
+ if (this.listener == null)
551
+ throw new Error('message listener is not set');
552
+ parseConsumeTimestamp(this.consumeTimestamp); // hard fail on bad format
553
+ this._checkConfigRanges();
554
+ if (this.messageModel === MessageModel.CLUSTERING) {
555
+ // Only CLUSTERING rewrites the DEFAULT instance name, and
556
+ // changeInstanceNameToPid makes it `<pid>#<nanotime>`,
557
+ // i.e. unique PER INSTANCE. A pid-only name is shared by every push
558
+ // consumer in this process, so the broker sees one member and
559
+ // allocate() hands both instances the same slice — duplicate consumption.
560
+ this.instanceName = MixAll.changeInstanceNameToPid(this.instanceName);
561
+ }
562
+ // buildMQClientId: `<ip>@<instanceName>[@<unitName>][@STREAM]`
563
+ // (the push consumer leaves the stream flag off).
564
+ this.clientID = MixAll.clientIdFor(this.instanceName);
565
+ const manager = MQClientManager.getInstance();
566
+ const { instance: client, created } = manager.getOrCreateMQClientInstance(this.clientID, () => new MQClient(this.clientID, this.nameServerAddr));
567
+ if (created) {
568
+ if (this.tlsEnable != null)
569
+ client.remotingClient.tlsEnable = this.tlsEnable;
570
+ if (this.tlsOptions != null)
571
+ client.remotingClient.tlsOptions = this.tlsOptions;
572
+ if (this.socksProxyConfig != null)
573
+ client.remotingClient.socksProxyConfig = this.socksProxyConfig;
574
+ client.setPersistConsumerOffsetInterval(this.persistConsumerOffsetInterval);
575
+ if (this.useHeartbeatV2)
576
+ client.setUseHeartbeatV2(true);
577
+ client.setDecodeReadBody(this.decodeReadBody);
578
+ client.setMqClientApiTimeout(this.mqClientApiTimeout);
579
+ client.setEnableConcurrentHeartbeat(this.enableConcurrentHeartbeat);
580
+ client.setConcurrentHeartbeatThreadPoolSize(this.concurrentHeartbeatThreadPoolSize);
581
+ client.setMaxPageSizeInGetMetadata(this.maxPageSizeInGetMetadata);
582
+ registerRpcHooks(client.remotingClient, {
583
+ namespaceV2: () => this.namespaceV2,
584
+ userHook: this.rpcHook,
585
+ });
586
+ }
587
+ manager.registerClient(this.clientID, client);
588
+ this.mqClient = client;
589
+ // The consumer stats manager is instance-level shared on the MQClient.
590
+ this._statsManager = client.consumerStatsManager;
591
+ client.registerConsumer(this.consumerGroup, this);
592
+ client.start();
593
+ // Clustering subscribes its retry topic automatically; the broker
594
+ // redelivers into it. SUB_ALL keeps tagsSet EMPTY (see FilterAPI).
595
+ if (this.messageModel !== MessageModel.BROADCASTING) {
596
+ const retryTopic = MixAll.getRetryTopic(this.consumerGroup);
597
+ if (!this.subscription.has(retryTopic)) {
598
+ this.subscription.set(retryTopic, FilterAPI.buildSubscriptionData(retryTopic, '*'));
599
+ }
600
+ }
601
+ if (this.messageModel === MessageModel.BROADCASTING) {
602
+ this.offsetStore = new LocalFileOffsetStore(this.clientID, this.consumerGroup);
603
+ }
604
+ else {
605
+ this.offsetStore = new RemoteBrokerOffsetStore(client, this.consumerGroup);
606
+ }
607
+ this.corePoolSize = this.consumeThreadMin;
608
+ this._dispatchSem = 0;
609
+ this._draining = false;
610
+ this._inFlight = 0;
611
+ this._stopFlags.clear();
612
+ this._queueStopFlags.clear();
613
+ this._startTime = Date.now();
614
+ this.started = true;
615
+ await this.offsetStore.load().catch((e) => {
616
+ logger.warning('load offset store failed for %s: %s', this.consumerGroup, e.message);
617
+ });
618
+ for (const topic of this.subscription.keys()) {
619
+ await client.updateTopicRouteInfoFromNameServer(topic, false).catch((e) => {
620
+ logger.debug('initial route refresh failed for %s: %s', topic, e.message);
621
+ });
622
+ }
623
+ // Register broker-initiated request processors BEFORE the first heartbeat.
624
+ this._registerProcessors();
625
+ // Pre-flight: subscription routes
626
+ // first, then the 46 check, then the first heartbeat. A failure
627
+ // here is fatal: log, shut the consumer down and rethrow.
628
+ try {
629
+ await client.checkClientInBroker();
630
+ }
631
+ catch (e) {
632
+ logger.warning('Start the consumer %s fail: %s', this.consumerGroup, e.message);
633
+ await this.shutdown().catch((e2) => { });
634
+ throw e;
635
+ }
636
+ // Heartbeat must precede rebalance: rebalance asks the broker for the
637
+ // group's client list.
638
+ await this.sendHeartbeatNow();
639
+ // The first assignment is computed synchronously; otherwise the pull loops
640
+ // would spin on an empty assignment until the first timer tick.
641
+ await this.doRebalance();
642
+ // POP mode: tell the broker to serve this (group, topic) pair as POP — the
643
+ // broker normally does this out of band (mqadmin / console); doing it here
644
+ // keeps the client
645
+ // self-contained. A failure is logged, not fatal — a broker that already
646
+ // has the group in POP mode answers SUCCESS anyway, and an old broker
647
+ // without POP support must not take the consumer down.
648
+ if (this.popMode) {
649
+ const err = await setMessageRequestModeOnBroker(this, 3000);
650
+ if (err)
651
+ logger.warning('enable POP on broker failed for group %s: %s', this.consumerGroup, err.message);
652
+ }
653
+ this._goLoop(() => this._rebalanceLoop());
654
+ this._goLoop(() => this._dispatchLoop());
655
+ if (!this.orderly && !this.popMode)
656
+ this._goLoop(() => this._cleanExpireLoop());
657
+ if (this.orderly && this.messageModel !== MessageModel.BROADCASTING) {
658
+ this._goLoop(() => this._lockLoop());
659
+ }
660
+ this._goLoop(() => this._persistOffsetLoop());
661
+ this._goLoop(() => this._heartbeatLoop());
662
+ if (this.enableTrace) {
663
+ this._startTraceDispatcher();
664
+ }
665
+ }
666
+ // Shutdown unwinds in a fixed order: persist offsets (while started is still
667
+ // true), unlock if orderly, unregister at every broker, then stop the loops.
668
+ async shutdown() {
669
+ if (!this.started)
670
+ return;
671
+ this._draining = true;
672
+ const client = this.mqClient;
673
+ try {
674
+ await this.persistConsumerOffset();
675
+ }
676
+ catch (e) { /* best effort */ }
677
+ if (this.orderly && this.messageModel !== MessageModel.BROADCASTING) {
678
+ await this.unlockAssigned();
679
+ }
680
+ if (client) {
681
+ await client.unregisterClientAllBrokers('', this.consumerGroup).catch(() => { });
682
+ client.unregisterConsumer(this.consumerGroup);
683
+ }
684
+ // Stop all loops.
685
+ for (const flag of Array.from(this._stopFlags)) {
686
+ this._stopFlags.delete(flag); // delete = stop signal for sleepers
687
+ }
688
+ this._stopFlags.add('__stopped');
689
+ // POP loops ride the same stop-flag set; drop their queue state so an
690
+ // in-flight batch is marked dropped and never ACKs a batch the broker has
691
+ // taken back.
692
+ for (const key of Array.from(this.popQueueTable.keys())) {
693
+ retirePopQueue(this, key);
694
+ }
695
+ for (const t of this._timers)
696
+ clearTimeout(t);
697
+ this._timers = [];
698
+ // Wait (bounded) for in-flight batches — the listener call and its
699
+ // send-back. Without this window an immediate process exit can cut a
700
+ // CONSUMER_SEND_MSG_BACK(36) short and the message never reaches
701
+ // %RETRY%/%DLQ%. The budget is awaitTerminationMillisWhenShutdown (Java
702
+ // DefaultMQPushConsumer#setAwaitTerminationMillisWhenShutdown); 0 means do
703
+ // not wait at all, which detaches whatever is running.
704
+ await this._waitInFlightDrain(this.awaitTerminationMillisWhenShutdown);
705
+ // Persist a second time at the end, mirroring the MQClient instance's
706
+ // flush on shutdown: offsets the
707
+ // drain just advanced must land too.
708
+ try {
709
+ await this.persistConsumerOffset();
710
+ }
711
+ catch (e) { /* best effort */ }
712
+ this.started = false;
713
+ if (this._producer) {
714
+ try {
715
+ this._producer.shutdown();
716
+ }
717
+ catch (e) { /* ignore */ }
718
+ this._producer = null;
719
+ }
720
+ // Remove this facade's reverse-request processors, then hand the instance
721
+ // back to the manager: decrement the count; the REAL shutdown (stop
722
+ // heartbeat/route timers, drop remoting) happens only when it hits ZERO.
723
+ // Other facades sharing the same clientId — and another _statsManager —
724
+ // keep running.
725
+ this._unregisterProcessors();
726
+ if (client) {
727
+ MQClientManager.getInstance().unregisterClient(client.clientId, client);
728
+ }
729
+ this._statsManager = null; // when the refcount hits zero the manager shuts the instance down and the shared sampling timer stops
730
+ if (this.traceDispatcher) {
731
+ try {
732
+ this.traceDispatcher.shutdown();
733
+ }
734
+ catch (e) { /* ignore */ }
735
+ this.traceDispatcher = null;
736
+ }
737
+ }
738
+ // suspend() flips ONE boolean: the pull/pop loops keep running, keep stamping
739
+ // their lastPull/lastPop timestamp, and just stop STARTING new requests,
740
+ // backing off PULL_TIME_DELAY_MILLS_WHEN_SUSPEND each round.
741
+ //
742
+ // The gate must sit AFTER the stamp (stamp first, check pause second).
743
+ // Gate first, stamp second, and a merely-suspended consumer goes silent for
744
+ // more than PULL_MAX_IDLE_TIME (120s), the stall detector calls it a dead loop,
745
+ // and rebalance tears the assignment down — i.e. pausing an operational
746
+ // consumer for two minutes silently re-delivers its whole backlog on resume.
747
+ suspend() {
748
+ this._paused = true;
749
+ logger.info(`suspend this consumer, ${this.consumerGroup}`);
750
+ }
751
+ // resume() clears the flag and rebalances at once,
752
+ // so queues that appeared while suspended start being pulled immediately
753
+ // instead of waiting out the 20s rebalance period.
754
+ resume() {
755
+ this._paused = false;
756
+ this.rebalanceImmediately();
757
+ logger.info(`resume this consumer, ${this.consumerGroup}`);
758
+ }
759
+ // isPaused() reports whether the consumer-wide suspend flag is set.
760
+ isPaused() { return this._paused; }
761
+ _startTraceDispatcher() {
762
+ // Trace-stack failures only log — a broken
763
+ // trace stack must never take the consumer down.
764
+ try {
765
+ // Lazy wiring; the trace dispatcher lives in trace_dispatcher.ts.
766
+ // Imported lazily to avoid a hard dependency cycle for consumers that
767
+ // never enable trace.
768
+ import("./trace_dispatcher.js").then((m) => {
769
+ if (!this.started || this.traceDispatcher)
770
+ return;
771
+ const dispatcher = new m.AsyncTraceDispatcher(this.consumerGroup, 'CONSUME', this.traceMsgBatchNum, this.traceTopic || undefined);
772
+ dispatcher.setHostConsumer(this);
773
+ // Propagate the namespace to the
774
+ // internal trace producer, so trace rows land in the same namespace.
775
+ dispatcher.setNamespaceV2(this.namespaceV2);
776
+ // The internal trace producer must carry the host's ACL hook (Java builds it
777
+ // with `new DefaultMQProducer(rpcHook)`), and the dispatcher must know the
778
+ // facade's channel to pick the region topic in CLOUD mode.
779
+ dispatcher.rpcHook = this.rpcHook;
780
+ dispatcher.accessChannel = this.accessChannel;
781
+ this.traceDispatcher = dispatcher;
782
+ if (this.consumerGroup) {
783
+ import("./trace_hook.js").then((th) => {
784
+ this.registerConsumeMessageHook(new th.ConsumeMessageTraceHookImpl(dispatcher));
785
+ }).catch((e) => logger.warning('trace hook load failed: %s', e.message));
786
+ }
787
+ dispatcher.start(this.nameServerAddr, this.accessChannel).catch((e) => logger.warning('trace dispatcher start failed: %s', e.message));
788
+ }).catch((e) => logger.warning('trace dispatcher load failed: %s', e.message));
789
+ }
790
+ catch (e) {
791
+ logger.warning('trace dispatcher init failed: %s', e.message);
792
+ }
793
+ }
794
+ // ---------------------------------------------------------------- loops
795
+ _goLoop(fn) {
796
+ fn().catch((e) => logger.error('consumer loop error: %s', e.message || e));
797
+ }
798
+ _newStopFlag() {
799
+ const flag = `sf-${Math.random().toString(36).slice(2)}-${Date.now()}`;
800
+ this._stopFlags.add(flag);
801
+ return flag;
802
+ }
803
+ _isStopped(flag) {
804
+ if (this._stopFlags.has('__stopped'))
805
+ return true;
806
+ if (flag && !this._stopFlags.has(flag))
807
+ return true; // retired queue
808
+ return false;
809
+ }
810
+ // sleepOrStop waits, and reports whether it was interrupted.
811
+ _sleepOrStop(ms, flag) {
812
+ if (ms <= 0)
813
+ return Promise.resolve(this._isStopped(flag));
814
+ return new Promise((resolve) => {
815
+ const timer = setTimeout(() => {
816
+ clearInterval(poll);
817
+ resolve(false);
818
+ }, ms);
819
+ const poll = setInterval(() => {
820
+ if (this._isStopped(flag)) {
821
+ clearTimeout(timer);
822
+ clearInterval(poll);
823
+ resolve(true);
824
+ }
825
+ }, 20);
826
+ if (poll && typeof poll.unref === 'function')
827
+ poll.unref();
828
+ if (typeof timer.unref === 'function')
829
+ timer.unref();
830
+ });
831
+ }
832
+ _beginInFlight() {
833
+ if (this._draining)
834
+ return false;
835
+ this._inFlight++;
836
+ return true;
837
+ }
838
+ _endInFlight() {
839
+ this._inFlight = Math.max(0, this._inFlight - 1);
840
+ if (this._inFlight === 0 && this._inFlightWaiters.length) {
841
+ const waiters = this._inFlightWaiters;
842
+ this._inFlightWaiters = [];
843
+ for (const w of waiters)
844
+ w();
845
+ }
846
+ }
847
+ _waitInFlightDrain(budgetMs) {
848
+ if (this._inFlight === 0)
849
+ return Promise.resolve();
850
+ return new Promise((resolve) => {
851
+ const timer = setTimeout(() => {
852
+ logger.warning('consumer shutdown drain timed out after %d ms, group=%s; in-flight batches detached', budgetMs, this.consumerGroup);
853
+ resolve();
854
+ }, budgetMs);
855
+ this._inFlightWaiters.push(() => {
856
+ clearTimeout(timer);
857
+ resolve();
858
+ });
859
+ });
860
+ }
861
+ // Immediate heartbeat round over every broker of the route table. The push
862
+ // facade calls this at start() — before rebalance asks the broker for the
863
+ // group's member list — and from its own 30s loop.
864
+ async sendHeartbeatNow() {
865
+ const client = this.mqClient;
866
+ if (client == null)
867
+ return;
868
+ await client.sendHeartbeatToAllBrokers();
869
+ }
870
+ async _heartbeatLoop() {
871
+ const flag = this._newStopFlag();
872
+ while (!this._isStopped(flag)) {
873
+ // The beat period is the SHARED instance's configured
874
+ // heartbeatIntervalMillis (default 30s): one knob drives every beat of
875
+ // this clientId, set once at facade registration.
876
+ const interval = this.mqClient != null ? this.mqClient.heartbeatIntervalMillis : 30 * 1000;
877
+ if (!(await this._sleepOrStop(interval, flag))) {
878
+ try {
879
+ await this.sendHeartbeatNow();
880
+ }
881
+ catch (e) { /* best effort */ }
882
+ }
883
+ }
884
+ }
885
+ async _persistOffsetLoop() {
886
+ const flag = this._newStopFlag();
887
+ while (!this._isStopped(flag)) {
888
+ // The facade-level loop honors this facade's persistConsumerOffsetInterval;
889
+ // the instance-level sweep in MQClient covers the
890
+ // pull / lite-pull facades that have no loop of their own.
891
+ if (!(await this._sleepOrStop(this.persistConsumerOffsetInterval, flag))) {
892
+ try {
893
+ await this.persistConsumerOffset();
894
+ }
895
+ catch (e) { /* best effort */ }
896
+ }
897
+ }
898
+ }
899
+ async _rebalanceLoop() {
900
+ const flag = this._newStopFlag();
901
+ while (!this._isStopped(flag)) {
902
+ const startingUp = (Date.now() - this._startTime) < REBALANCE_STARTUP_WINDOW;
903
+ const interval = (startingUp && this.assigned.length === 0)
904
+ ? REBALANCE_INTERVAL_DURING_STARTUP : REBALANCE_INTERVAL;
905
+ const stopped = await this._sleepOrStop(interval, flag);
906
+ if (stopped || !this.started)
907
+ return;
908
+ // NOTIFY_CONSUMER_IDS_CHANGED path: the flag was set on the read thread
909
+ // and is consumed here, on the rebalance loop.
910
+ this._rebalanceNow = false;
911
+ try {
912
+ await this.doRebalance();
913
+ }
914
+ catch (e) {
915
+ logger.debug('rebalance error: %s', e.message);
916
+ }
917
+ }
918
+ }
919
+ // ------------------------------------------------------------- rebalance
920
+ // doRebalance resolves the queue set for every subscribed
921
+ // topic, then syncPullLoops.
922
+ //
923
+ // BROADCASTING: every queue is ours, no broker coordination.
924
+ // CLUSTERING: ask the broker for the group's client list -> sort cidAll and
925
+ // mqAll -> run the strategy -> take our slice.
926
+ //
927
+ // A missing consumer list KEEPS the current assignment (warn). Never
928
+ // degrade to "I own everything": co-instances would duplicate every message.
929
+ //
930
+ // POP takes its queues from THIS same client-side rebalance (one POP loop per
931
+ // assigned queue + ack). The alternative broker-side path — asking the broker
932
+ // with clientRebalance off via QUERY_ASSIGNMENT=400 (MessageQueueAssignment
933
+ // with mode=POP) — is a deliberate divergence: this port resolves queues
934
+ // client-side only. The semantics are equivalent; only WHO picks the queue
935
+ // set differs.
936
+ async doRebalance() {
937
+ if (!this.mqClient || !this.started)
938
+ return;
939
+ const client = this.mqClient;
940
+ const wasByTopic = queuesByTopicKeySet(this.assigned);
941
+ const topics = Array.from(this.subscription.keys()).sort();
942
+ const assigned = [];
943
+ for (const topic of topics) {
944
+ const route = client.getTopicRouteData(topic);
945
+ const info = route ? route.getAllSubscribeMessageQueue(topic) : [];
946
+ if (info.length === 0) {
947
+ logger.debug('rebalance: no subscribe info for topic %s', topic);
948
+ }
949
+ if (this.messageModel === MessageModel.BROADCASTING) {
950
+ assigned.push(...info);
951
+ continue;
952
+ }
953
+ const mqAll = [...info].sort((a, b) => a.compareTo(b));
954
+ if (mqAll.length === 0)
955
+ continue;
956
+ const { cidAll, answered } = await this._getConsumerIdListByGroup(topic);
957
+ if (!answered || cidAll.length === 0) {
958
+ logger.debug('rebalance: no consumer id list for %s/%s, keep current', this.consumerGroup, topic);
959
+ for (const mq of this.assigned) {
960
+ if (mq.getTopic() === topic)
961
+ assigned.push(mq);
962
+ }
963
+ continue;
964
+ }
965
+ cidAll.sort();
966
+ const strategy = this.allocateStrategy || new AllocateMessageQueueAveragely();
967
+ const got = strategy.allocate(this.consumerGroup, this.clientID, mqAll, cidAll);
968
+ if (!got || got.length === 0) {
969
+ logger.warning('allocate message queue returned nothing, strategy=%s group=%s', strategy.name(), this.consumerGroup);
970
+ }
971
+ assigned.push(...got);
972
+ }
973
+ assigned.sort((a, b) => a.compareTo(b));
974
+ // Resolve the initial offset for freshly assigned queues NOW, not lazily
975
+ // on the first pull: CONSUME_FROM_LAST_OFFSET means "the newest offset as
976
+ // of the moment the queue was assigned". A queue whose offset cannot be
977
+ // resolved is dropped from this round's assignment instead of starting at
978
+ // the tail, which would skip its backlog. POP names no offset — the start
979
+ // position rides initMode on the request — so a POP assignment never reads
980
+ // one and never loses a queue to an unreadable offset table.
981
+ const skipped = [];
982
+ if (!this.popMode) {
983
+ for (const mq of assigned) {
984
+ // The gate is "does this queue still have a live cursor", NOT "was it
985
+ // assigned before". A queue that stayed assigned can still lose its
986
+ // offsetTable entry: syncPullLoops' self-heal sweep and the OFFSET_ILLEGAL
987
+ // fix both route through _retireQueue, which deletes it (Java: the same
988
+ // removeOffset on revoke). Gating on the previous assignment skipped the
989
+ // recompute for exactly those queues, so the rebuilt pull loop read
990
+ // offset === undefined, spun silently in its backoff branch, and the
991
+ // queue never pulled again (2026-11-12: orderly/transaction live test).
992
+ if (this.offsetTable.has(mqKey(mq)))
993
+ continue;
994
+ if (!this.subscription.has(mq.getTopic()))
995
+ continue;
996
+ try {
997
+ const next = await this.computePullFromWhere(mq);
998
+ if (next < 0) {
999
+ // Same fail-visible semantics as the catch below: an owned queue
1000
+ // with no cursor means the rebuilt pull loop spins on
1001
+ // offset === undefined forever. Drop it from this round; the next
1002
+ // round retries the compute.
1003
+ skipped.push(mq);
1004
+ continue;
1005
+ }
1006
+ if (!this.offsetTable.has(mqKey(mq)))
1007
+ this.offsetTable.set(mqKey(mq), next);
1008
+ if (this.offsetStore)
1009
+ this.offsetStore.updateOffset(mq, next, false);
1010
+ }
1011
+ catch (e) {
1012
+ logger.warning('doRebalance, %s, compute offset failed, skip %s this round: %s', this.consumerGroup, mqKey(mq), e.message);
1013
+ skipped.push(mq);
1014
+ }
1015
+ }
1016
+ }
1017
+ if (skipped.length > 0) {
1018
+ const skip = new Set(skipped.map(mqKey));
1019
+ const kept = assigned.filter((mq) => !skip.has(mqKey(mq)));
1020
+ assigned.length = 0;
1021
+ assigned.push(...kept);
1022
+ }
1023
+ this.assigned = assigned;
1024
+ // The topics whose queue SET moved, so a swap that keeps the COUNT (one queue
1025
+ // lost, another gained) still counts: which buffers exist changed, the topic
1026
+ // budgets have to be re-divided over the new set, and the broker has to be
1027
+ // told about the subscriptions of the queues this client just picked up.
1028
+ const moved = changedTopics(wasByTopic, queuesByTopicKeySet(assigned));
1029
+ if (moved.length > 0) {
1030
+ logger.info('rebalance result changed, group=%s clientId=%s assigned=%d topics=%s', this.consumerGroup, this.clientID, assigned.length, moved.join(','));
1031
+ }
1032
+ this.syncPullLoops();
1033
+ // ORDER MATTERS, and the four steps are one package that only a moved
1034
+ // assignment pays: the topics that gained or lost a queue re-share their
1035
+ // subscription version, the topic cache budgets are divided over the queues
1036
+ // this round NOW owns (a shrink from 4 queues to 2 must divide by 2, not by 4),
1037
+ // every broker gets the immediate beat that carries those versions, and last
1038
+ // the application's queue listener hears about the topics that moved.
1039
+ this._afterAssignmentChange(moved);
1040
+ }
1041
+ // _afterAssignmentChange is the whole of what a rebalance round whose queue set
1042
+ // moved does, in one order that cannot drift.
1043
+ //
1044
+ // The beat is NOT conditioned on the heartbeat mode: with V2 on it is the beat
1045
+ // that drops the cached fingerprints, with V2 off — the default — the same call
1046
+ // is the plain beat. Waiting for the 30s periodic round instead leaves the broker
1047
+ // holding the old slice of this group's queue set in the meantime.
1048
+ _afterAssignmentChange(moved) {
1049
+ if (moved.length === 0)
1050
+ return;
1051
+ this._restampSubscriptionVersions(moved);
1052
+ this._divideTopicThresholds();
1053
+ // The beat reads the subscription table when the call starts, so the versions
1054
+ // stamped above are what it carries.
1055
+ void this.mqClient.sendHeartbeatToAllBrokers(true);
1056
+ this._notifyQueueChanged(moved);
1057
+ }
1058
+ // _restampSubscriptionVersions re-shares the subscription version of every topic
1059
+ // whose assignment moved. The beat that follows carries it, the heartbeat's
1060
+ // subscription equality covers the version, and the pull request sends it too —
1061
+ // so a broker that had suppressed this client's unchanged subscriptions now takes
1062
+ // them again, which is what stops a freshly picked-up queue from being served
1063
+ // under a stale filter.
1064
+ _restampSubscriptionVersions(topics) {
1065
+ if (topics.length === 0)
1066
+ return;
1067
+ const newVersion = currentTimeMillis();
1068
+ for (const topic of topics) {
1069
+ const sub = this.subscription.get(topic);
1070
+ if (!sub)
1071
+ continue;
1072
+ logger.info('%s rebalance changed, also update version: %d, %d', topic, sub.subVersion, newVersion);
1073
+ // The map holds the same objects the heartbeat builder and every pull read,
1074
+ // so the new version is what the next beat and the next pull carry.
1075
+ sub.subVersion = newVersion;
1076
+ }
1077
+ }
1078
+ // _divideTopicThresholds turns each topic-level cache budget into a
1079
+ // per-queue one for the queues this consumer owns right now: every enabled
1080
+ // knob (-1 means off) is divided by that queue count — ALL owned queues,
1081
+ // across EVERY topic, not the queues of one topic — and the quotient is
1082
+ // written into the matching per-queue knob. The divisor is the live
1083
+ // per-queue state table of the current consumption mode: processQueueTable
1084
+ // on the pull path, popQueueTable under POP.
1085
+ //
1086
+ // The dividend is ALWAYS the topic knob, never the current per-queue value,
1087
+ // so re-running the division with an unchanged queue count writes the same
1088
+ // number again (idempotent). The user's per-queue value is deliberately
1089
+ // overwritten while a topic knob is enabled — reading pullThresholdForQueue
1090
+ // afterwards shows the divided number, not what the user set.
1091
+ //
1092
+ // Owning no queues makes the division meaningless: both per-queue knobs
1093
+ // keep the values the user set (no division by zero).
1094
+ _divideTopicThresholds() {
1095
+ const owned = this.popMode ? this.popQueueTable.size : this.processQueueTable.size;
1096
+ if (owned === 0)
1097
+ return;
1098
+ if (this.pullThresholdForTopic !== -1) {
1099
+ const newVal = Math.max(1, Math.floor(this.pullThresholdForTopic / owned));
1100
+ if (newVal !== this.pullThresholdForQueue) {
1101
+ logger.info('The pullThresholdForQueue is changed from %d to %d', this.pullThresholdForQueue, newVal);
1102
+ }
1103
+ this.pullThresholdForQueue = newVal;
1104
+ }
1105
+ if (this.pullThresholdSizeForTopic !== -1) {
1106
+ const newVal = Math.max(1, Math.floor(this.pullThresholdSizeForTopic / owned));
1107
+ if (newVal !== this.pullThresholdSizeForQueue) {
1108
+ logger.info('The pullThresholdSizeForQueue is changed from %d to %d', this.pullThresholdSizeForQueue, newVal);
1109
+ }
1110
+ this.pullThresholdSizeForQueue = newVal;
1111
+ }
1112
+ }
1113
+ // _getConsumerIdListByGroup asks the broker for the group's consumer id list:
1114
+ // one broker of the topic's route. Every client heartbeats to every broker,
1115
+ // so any one broker holds the COMPLETE list for the group.
1116
+ //
1117
+ // Broker address selection prefers MASTER, else
1118
+ // the lowest remaining id (a slave). Master-only resolution breaks a
1119
+ // slave-only topology — the route then has no id=0 entry, the member list
1120
+ // never resolves, and a freshly started consumer is assigned no queues at
1121
+ // all (it silently consumes nothing).
1122
+ async _getConsumerIdListByGroup(topic) {
1123
+ if (!this.mqClient)
1124
+ return { cidAll: [], answered: false };
1125
+ const client = this.mqClient;
1126
+ const route = client.getTopicRouteData(topic);
1127
+ if (!route)
1128
+ return { cidAll: [], answered: false };
1129
+ let addr = null;
1130
+ for (const bd of route.brokerDatas || []) {
1131
+ const m = bd.brokerAddrs || {};
1132
+ if (MixAll.MASTER_ID in m) {
1133
+ addr = m[MixAll.MASTER_ID];
1134
+ break;
1135
+ }
1136
+ // selectBrokerAddr fallback: lowest id (JS integer-key objects iterate
1137
+ // ascending), i.e. the primary slave when the master is gone.
1138
+ const ids = Object.keys(m).map(Number).sort((a, b) => a - b);
1139
+ if (ids.length > 0) {
1140
+ addr = m[String(ids[0])];
1141
+ break;
1142
+ }
1143
+ }
1144
+ if (!addr)
1145
+ return { cidAll: [], answered: false };
1146
+ try {
1147
+ const response = await client.getConsumerListByGroup(addr, this.consumerGroup);
1148
+ if (response.code !== ResponseCode.SUCCESS)
1149
+ return { cidAll: [], answered: false };
1150
+ const body = response.body && response.body.length
1151
+ ? RemotingSerializable.decode(response.body) : null;
1152
+ const ids = Array.isArray(body) ? body
1153
+ : (body && Array.isArray(body['consumerIdList'])) ? body['consumerIdList'] : [];
1154
+ return { cidAll: ids.map(String), answered: true };
1155
+ }
1156
+ catch (e) {
1157
+ logger.debug('getConsumerIdListByGroup failed, %s %s: %s', addr, this.consumerGroup, e.message);
1158
+ return { cidAll: [], answered: false };
1159
+ }
1160
+ }
1161
+ // syncPullLoops updates the process-queue table after a rebalance:
1162
+ // retire what we no longer own, then start a pull loop per new queue.
1163
+ //
1164
+ // Revocation has three mandatory steps — persist the CONSUMED offset, drop
1165
+ // the ProcessQueue, and UNLOCK_BATCH_MQ for an orderly clustering consumer.
1166
+ // Order matters: retire -> settle -> add. Reversed, the new loop would start
1167
+ // from a stale offset and write the smaller value back.
1168
+ syncPullLoops() {
1169
+ // POP mode branches here too: the POP twin retires and
1170
+ // adds pop loops instead of pull loops — no offset to settle, only the
1171
+ // dropped marker so in-flight batches abort instead of ACKing.
1172
+ if (this.popMode) {
1173
+ syncPopLoops(this);
1174
+ return;
1175
+ }
1176
+ const current = new Set(this.assigned.map(mqKey));
1177
+ const revoked = [];
1178
+ for (const [key, pq] of Array.from(this.processQueueTable.entries())) {
1179
+ if (!current.has(key)) {
1180
+ revoked.push(this._retireQueue(key));
1181
+ continue;
1182
+ }
1183
+ if (this.started && pq.isPullExpired()) {
1184
+ logger.error('[BUG]doRebalance, %s, try remove unnecessary mq, %s, because pull is pause, so try to fixed it', this.consumerGroup, key);
1185
+ revoked.push(this._retireQueue(key));
1186
+ }
1187
+ }
1188
+ if (revoked.length > 0) {
1189
+ this._persistRevoked(revoked);
1190
+ }
1191
+ for (const mq of this.assigned) {
1192
+ const key = mqKey(mq);
1193
+ if (this.processQueueTable.has(key))
1194
+ continue;
1195
+ const pq = new ProcessQueue(this.orderly);
1196
+ this.processQueueTable.set(key, pq);
1197
+ // A fresh ProcessQueue lifts the freeze (as removeProcessQueue's
1198
+ // removeOffset does): the rebuilt queue advances from the corrected offset.
1199
+ this._frozenOffsets.delete(key);
1200
+ pq.touchPull();
1201
+ // A rebuilt queue starts over from the master instead of the slave the
1202
+ // previous incarnation had drifted to.
1203
+ this.pullFromWhichNode.delete(key);
1204
+ const stopFlag = this._newStopFlag();
1205
+ this._queueStopFlags.set(key, stopFlag);
1206
+ this._goLoop(() => this._queuePullLoop(mq, stopFlag));
1207
+ }
1208
+ if (revoked.length > 0) {
1209
+ logger.info('queues revoked, group=%s count=%d', this.consumerGroup, revoked.length);
1210
+ }
1211
+ // The broker beat and the user's queue listener are NOT spent here: a round that
1212
+ // only self-heals a stalled loop (the queue stays assigned, so the set did not
1213
+ // move) changes nothing the broker knows about. doRebalance hands the change to
1214
+ // _afterAssignmentChange once it knows which topics moved.
1215
+ }
1216
+ // _retireQueue drops one queue's local state.
1217
+ //
1218
+ // The epoch bump invalidates every in-flight batch's ack (the queue is marked
1219
+ // dropped); the freeze marker is KEPT until the queue is rebuilt, so
1220
+ // a corrected offset cannot be overwritten by an old ack.
1221
+ _retireQueue(key) {
1222
+ const stopFlag = this._queueStopFlags.get(key);
1223
+ if (stopFlag) {
1224
+ this._stopFlags.delete(stopFlag); // signal the pull loop to exit
1225
+ this._queueStopFlags.delete(key);
1226
+ }
1227
+ const pq = this.processQueueTable.get(key);
1228
+ const [topic, brokerName, queueId] = parseMqKey(key);
1229
+ const mq = new MessageQueue(topic, brokerName, queueId);
1230
+ let offset = 0;
1231
+ let hasOffset = false;
1232
+ if (this.offsetStore && typeof this.offsetStore.tableSnapshot === 'function') {
1233
+ const snap = this.offsetStore.tableSnapshot();
1234
+ const v = snap.get(key);
1235
+ if (v !== undefined && v >= 0) {
1236
+ offset = v;
1237
+ hasOffset = true;
1238
+ }
1239
+ }
1240
+ if (pq) {
1241
+ pq.setDropped();
1242
+ pq.clear();
1243
+ }
1244
+ this.processQueueTable.delete(key);
1245
+ this.offsetTable.delete(key);
1246
+ this._msgAccCnt.delete(key);
1247
+ this.pullFromWhichNode.delete(key);
1248
+ this._queueEpoch.set(key, (this._queueEpoch.get(key) || 0) + 1);
1249
+ return { mq, offset, hasOffset };
1250
+ }
1251
+ // _persistRevoked persists the CONSUMED offset of queues we no longer own —
1252
+ // removeUnnecessaryMessageQueue for revoked subscriptions.
1253
+ _persistRevoked(revoked) {
1254
+ if (!this.offsetStore)
1255
+ return;
1256
+ if (this.messageModel === MessageModel.BROADCASTING) {
1257
+ // Broadcast offsets live in a local file. Persist BEFORE removeOffset so
1258
+ // the value stays on disk and a rebuilt queue can continue from it. Each
1259
+ // revoked queue is flushed individually: an empty flush list is a no-op
1260
+ // and would leave the last consumed offsets unsaved.
1261
+ for (const q of revoked) {
1262
+ if (!q.hasOffset)
1263
+ continue;
1264
+ this.offsetStore.updateOffset(q.mq, q.offset, false);
1265
+ this.offsetStore.persist(q.mq).catch((e) => logger.debug('persist local offset on revoke failed for %s: %s', mqKey(q.mq), e.message));
1266
+ }
1267
+ return;
1268
+ }
1269
+ for (const q of revoked) {
1270
+ if (q.hasOffset) {
1271
+ // Write the value now, do not wait for the 5s
1272
+ // periodic flush — the new owner may start pulling any moment.
1273
+ this.offsetStore.updateOffset(q.mq, q.offset, false);
1274
+ this.offsetStore.persist(q.mq).catch((e) => logger.debug('persist offset on revoke failed for %s: %s', mqKey(q.mq), e.message));
1275
+ }
1276
+ if (this.orderly && this.mqClient) {
1277
+ // Orderly: release the broker queue lock so the new owner can start.
1278
+ this._unlockBatchMQ([q.mq]).catch(() => { });
1279
+ }
1280
+ }
1281
+ }
1282
+ // _notifyQueueChanged fires the application's messageQueueChanged callback for the
1283
+ // topics listed, i.e. ONLY for the topics whose assignment moved. A round that
1284
+ // re-pulled the same queues tells the listener nothing new, and the callback is
1285
+ // documented as a change notification, not a heartbeat.
1286
+ _notifyQueueChanged(topics) {
1287
+ if (topics.length === 0)
1288
+ return;
1289
+ const listener = this.messageQueueListener;
1290
+ const client = this.mqClient;
1291
+ const assigned = [...this.assigned];
1292
+ if (!listener || !client)
1293
+ return;
1294
+ const byTopic = new Map();
1295
+ for (const mq of assigned) {
1296
+ const list = byTopic.get(mq.getTopic()) || [];
1297
+ list.push(mq);
1298
+ byTopic.set(mq.getTopic(), list);
1299
+ }
1300
+ for (const topic of topics) {
1301
+ const route = client.getTopicRouteData(topic);
1302
+ listener(topic, route ? route.getAllSubscribeMessageQueue(topic) : [], byTopic.get(topic) || []);
1303
+ }
1304
+ }
1305
+ // computePullFromWhere decides where a newly assigned queue starts pulling.
1306
+ //
1307
+ // READ_FROM_STORE first, then the per-policy fallback. Two items are
1308
+ // deliberate: CONSUME_FROM_FIRST_OFFSET returns 0 WITHOUT querying minOffset
1309
+ // (the offset gets fixed by the OFFSET_ILLEGAL path); and a %RETRY% topic
1310
+ // with no committed offset starts at 0 rather than maxOffset, because
1311
+ // retried messages must all be retried.
1312
+ async computePullFromWhere(mq) {
1313
+ const store = this.offsetStore;
1314
+ const client = this.mqClient;
1315
+ if (!store || !client)
1316
+ return -1;
1317
+ const last = await store.readOffset(mq, ReadOffsetMode.READ_FROM_STORE);
1318
+ if (last >= 0)
1319
+ return last;
1320
+ if (last !== -1) {
1321
+ // -1 means "no committed offset". Anything else is a failed read: falling
1322
+ // through to the tail policy would silently skip the whole backlog, so the
1323
+ // queue is left unowned for this round instead.
1324
+ throw new MQClientException(`Failed to query consume offset from offset store for ${mqKey(mq)}`);
1325
+ }
1326
+ const where = this.consumeFromWhere;
1327
+ let result = -1;
1328
+ if (where === ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET) {
1329
+ // The offset gets fixed by the OFFSET_ILLEGAL path.
1330
+ result = 0;
1331
+ }
1332
+ else if (where === ConsumeFromWhere.CONSUME_FROM_TIMESTAMP) {
1333
+ if (MixAll.isRetryTopic(mq.getTopic())) {
1334
+ result = await this._getMaxOffsetForMq(mq);
1335
+ }
1336
+ else {
1337
+ const ts = parseConsumeTimestamp(this.consumeTimestamp);
1338
+ result = await this._searchOffsetForMq(mq, ts);
1339
+ }
1340
+ }
1341
+ else if (MixAll.isRetryTopic(mq.getTopic())) {
1342
+ // CONSUME_FROM_LAST_OFFSET (and the two deprecated variants) on a retry
1343
+ // topic: retried messages must all be retried, so start at 0.
1344
+ result = 0;
1345
+ }
1346
+ else {
1347
+ result = await this._getMaxOffsetForMq(mq);
1348
+ }
1349
+ if (result < 0) {
1350
+ // A failed min/max/search query is not "start at the tail": the caller
1351
+ // skips the queue for this round.
1352
+ throw new MQClientException(`Found unexpected result ${result} for ${mqKey(mq)}`);
1353
+ }
1354
+ return result;
1355
+ }
1356
+ async _getMaxOffsetForMq(mq) {
1357
+ if (!this.mqClient)
1358
+ return -1;
1359
+ const addr = this.mqClient.findBrokerAddrForQueue(mq);
1360
+ if (!addr)
1361
+ return -1;
1362
+ try {
1363
+ return await this.mqClient.getMaxOffset(addr, mq.getTopic(), mq.getQueueId());
1364
+ }
1365
+ catch (e) {
1366
+ return -1;
1367
+ }
1368
+ }
1369
+ async _searchOffsetForMq(mq, timestamp) {
1370
+ if (!this.mqClient)
1371
+ return -1;
1372
+ const addr = this.mqClient.findBrokerAddrForQueue(mq);
1373
+ if (!addr)
1374
+ return -1;
1375
+ try {
1376
+ return await this.mqClient.searchOffsetByTimestamp(addr, mq.getTopic(), mq.getQueueId(), timestamp);
1377
+ }
1378
+ catch (e) {
1379
+ return -1;
1380
+ }
1381
+ }
1382
+ // ------------------------------------------------------------- pull loop
1383
+ // _queuePullLoop pulls one queue until it is retired or the consumer stops.
1384
+ //
1385
+ // Why one loop PER QUEUE rather than one shared pull dispatcher
1386
+ // thread: the push consumer long-polls (suspend=true), so an idle queue
1387
+ // parks its own request for the whole suspend window. A shared loop would
1388
+ // serialise those parks and starve every other queue.
1389
+ async _queuePullLoop(mq, stopFlag) {
1390
+ const key = mqKey(mq);
1391
+ // The generic error backoff is taken ONCE, when this queue's loop is set up:
1392
+ // a later setPullRetryDelayMillis() reaches the loops the next rebalance
1393
+ // builds, never a round already in flight.
1394
+ const retryDelayMillis = this.pullRetryDelayMillis;
1395
+ while (!this._isStopped(stopFlag)) {
1396
+ const pq = this.processQueueTable.get(key);
1397
+ if (!pq || pq.isDropped())
1398
+ return;
1399
+ // The timestamp is stamped when a pull is STARTED, before the
1400
+ // flow-control/lock decisions: the stall detector asks "is this loop alive".
1401
+ pq.touchPull();
1402
+ // The suspend gate, deliberately AFTER the
1403
+ // stamp above (see suspend() for why that order is the whole point).
1404
+ if (this._paused) {
1405
+ if (await this._sleepOrStop(PULL_TIME_DELAY_MILLS_WHEN_SUSPEND, stopFlag))
1406
+ return;
1407
+ continue;
1408
+ }
1409
+ // No subscription for this topic: there is no next round to schedule, so the
1410
+ // loop exits and rebalance owns the queue. The error backoff does not apply
1411
+ // here — it waits for a request this loop will never re-issue.
1412
+ const sub = this.subscription.get(mq.getTopic());
1413
+ if (!sub)
1414
+ return;
1415
+ if (this.orderly && this.messageModel !== MessageModel.BROADCASTING) {
1416
+ // Orderly in CLUSTERING: do not pull before LOCK_BATCH_MQ granted this
1417
+ // queue, or we would consume what another instance is about to lock. A
1418
+ // grant older than the live time counts as unlocked — the periodic
1419
+ // sweep renews it. BROADCASTING owns every queue locally and never
1420
+ // sweeps, so it must bypass this gate entirely.
1421
+ if (!pq.isLocked() || pq.isLockExpired()) {
1422
+ pq.incTryUnlockTimes();
1423
+ logger.warning('orderly queue %s %s, wait for the lock sweep', key, pq.isLocked() ? 'lock expired' : 'not locked');
1424
+ if (await this._sleepOrStop(200, stopFlag))
1425
+ return;
1426
+ continue;
1427
+ }
1428
+ }
1429
+ if (this._flowControlHit(mq, pq)) {
1430
+ if (await this._sleepOrStop(PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, stopFlag))
1431
+ return;
1432
+ continue;
1433
+ }
1434
+ let offset = this.offsetTable.get(key);
1435
+ if (offset === undefined) {
1436
+ // No cursor for this queue yet, so the round never reached the wire:
1437
+ // a failed round like any other, so it waits the generic backoff.
1438
+ // Logged at debug so a queue stuck here is diagnosable — this branch
1439
+ // spinning forever was the 2026-11-12 permanent-stall defect.
1440
+ logger.debug('no cursor yet for %s, wait for the next rebalance round', key);
1441
+ if (await this._sleepOrStop(retryDelayMillis, stopFlag))
1442
+ return;
1443
+ continue;
1444
+ }
1445
+ // The cursor THIS round asks with. The answer cannot be judged without it:
1446
+ // by the time the round is handled, the cursor has already moved on.
1447
+ const prevRequestOffset = offset;
1448
+ // The subscription expression goes on the wire only when
1449
+ // postSubscriptionWhenPull is on AND the subscription is not a class
1450
+ // filter. Off by default: the broker then does not filter and the
1451
+ // client's second-stage tag check covers it.
1452
+ const withExpr = this.postSubscriptionWhenPull && !sub.classFilterMode;
1453
+ let sysFlag = PullSysFlag.buildSysFlag(false, true, withExpr, false);
1454
+ let commitOffsetValue = 0;
1455
+ const client = this.mqClient;
1456
+ if (!client)
1457
+ return;
1458
+ // Resolve the broker address by the pullFromWhichNode id (rule #22):
1459
+ // every response rewrites the table, and a MISSING field means master(0)
1460
+ // — not "keep the previous value".
1461
+ let brokerId = this.pullFromWhichNode.get(key);
1462
+ if (brokerId === undefined)
1463
+ brokerId = MixAll.MASTER_ID;
1464
+ // Static topic: address resolution must go through the endpoint rewrite
1465
+ // (resolve the real brokerName from the MessageQueue first, then look up
1466
+ // the subscribe address); the brokerName in the wire header keeps the
1467
+ // logical (mock) name, and the broker-side TopicQueueMappingDetail
1468
+ // performs the logical->physical translation. The test project's
1469
+ // duck-typed mock client may not have this method — for non-static
1470
+ // topics the rewrite yields the original broker name, so the fallback is
1471
+ // safe.
1472
+ const realBrokerName = typeof client.getBrokerNameFromMessageQueue === 'function'
1473
+ ? client.getBrokerNameFromMessageQueue(mq) : mq.getBrokerName();
1474
+ let resolved = this._findBrokerAddressInSubscribe(realBrokerName, brokerId, false);
1475
+ if (!resolved) {
1476
+ await client.updateTopicRouteInfoFromNameServer(mq.getTopic(), false).catch(() => { });
1477
+ resolved = this._findBrokerAddressInSubscribe(realBrokerName, brokerId, false);
1478
+ if (!resolved) {
1479
+ logger.debug('The broker[%s] not exist', realBrokerName);
1480
+ if (await this._sleepOrStop(retryDelayMillis, stopFlag))
1481
+ return;
1482
+ continue;
1483
+ }
1484
+ }
1485
+ let sysFlagFinal = sysFlag;
1486
+ if (resolved.isSlave) {
1487
+ // A slave does not accept offset commits, so the commit-offset flag
1488
+ // must be cleared before the pull goes out.
1489
+ sysFlagFinal = PullSysFlag.clearCommitOffsetFlag(sysFlag);
1490
+ }
1491
+ // The pull RT is measured from just before the request, so it covers the
1492
+ // long-poll hold too.
1493
+ const begin = Date.now();
1494
+ let result = null;
1495
+ try {
1496
+ result = await this._pullKernel(mq, offset, sub, sysFlagFinal, commitOffsetValue, this.pullBatchSize, this.pullBatchSizeInBytes, this.pullSuspendTimeoutMillis, this.pullTimeoutMillis, resolved.addr);
1497
+ }
1498
+ catch (e) {
1499
+ if (e instanceof RemotingTimeoutException) {
1500
+ // A long poll timing out while suspended is NORMAL: an idle queue
1501
+ // answers on its own schedule, so our own request timeout can expire
1502
+ // before the reply does and idle queues time out periodically.
1503
+ // Debug level, so the run log keeps ERROR=0.
1504
+ logger.debug('pull long-poll timeout for %s (benign, will retry): %s', key, e.message);
1505
+ }
1506
+ else {
1507
+ logger.debug('pull error for %s: %s', key, e.message);
1508
+ const code = e instanceof MQBrokerException ? e.responseCode : undefined;
1509
+ if (await this._sleepOrStop(pullRetryDelayMillisFor(code, retryDelayMillis), stopFlag))
1510
+ return;
1511
+ }
1512
+ continue;
1513
+ }
1514
+ result = this._processPullResult(mq, result, sub);
1515
+ if (!this.started)
1516
+ return;
1517
+ const current = this.processQueueTable.get(key);
1518
+ if (!current || current !== pq || pq.isDropped()) {
1519
+ // Revoked mid-pull: discard the batch — do not consume it and do not
1520
+ // advance the offset. The new owner redelivers it from the last
1521
+ // offset we persisted.
1522
+ logger.debug('queue %s revoked during pull, discard %d fetched messages', key, result.msgFoundList.length);
1523
+ return;
1524
+ }
1525
+ if (result.status === PullStatus.FOUND && result.msgFoundList.length > 0) {
1526
+ pq.putMessage(result.msgFoundList);
1527
+ this._msgAccCnt.set(key, pq.msgAccCntValue());
1528
+ }
1529
+ this.offsetTable.set(key, result.nextBeginOffset);
1530
+ if (result.status === PullStatus.OFFSET_ILLEGAL) {
1531
+ // On OFFSET_ILLEGAL freeze the corrected offset
1532
+ // here and do the revoke/persist outside the loop.
1533
+ this._frozenOffsets.add(key);
1534
+ if (this.offsetStore)
1535
+ this.offsetStore.updateOffset(mq, result.nextBeginOffset, false);
1536
+ logger.warning('the pull request offset illegal, fix it, queue=%s', key);
1537
+ const q = this._retireQueue(key);
1538
+ this._persistRevoked([q]);
1539
+ this.rebalanceImmediately();
1540
+ return;
1541
+ }
1542
+ this._correctTagsOffset(mq, result.status, result.nextBeginOffset);
1543
+ // Both counters fire inside the FOUND arm: the RT even when the list
1544
+ // came back empty, the TPS only for a non-empty one.
1545
+ if (result.status === PullStatus.FOUND) {
1546
+ this._incPullRT(mq.getTopic(), Date.now() - begin);
1547
+ // The sanity check closes the FOUND round: the cursor already advanced and
1548
+ // the batch is already handed over, so a warning here can only describe the
1549
+ // round, never change it. An emptied list has no first row to point at, so
1550
+ // it stands in with the sentinel and only the next-offset leg can fire.
1551
+ const firstMsgOffset = result.msgFoundList.length > 0
1552
+ ? result.msgFoundList[0].getQueueOffset()
1553
+ : NO_MESSAGE_QUEUE_OFFSET;
1554
+ if (pullOffsetsLookWrong(result.nextBeginOffset, firstMsgOffset, prevRequestOffset)) {
1555
+ logger.warn('the FOUND pull round does not line up with the offset it asked with,'
1556
+ + ' queue=%s nextBeginOffset=%d firstMsgOffset=%d prevRequestOffset=%d', key, result.nextBeginOffset, firstMsgOffset, prevRequestOffset);
1557
+ }
1558
+ if (result.msgFoundList.length > 0) {
1559
+ this._incPullTPS(mq.getTopic(), result.msgFoundList.length);
1560
+ // Round pacing: ONLY a round that handed rows over waits pullInterval,
1561
+ // and only when the knob is above 0. The cadence is a pacing rule for what
1562
+ // was delivered, so a round that delivered nothing — an empty FOUND answer,
1563
+ // one the filter emptied, NO_NEW_MSG, NO_MATCHED_MSG — asks again at once:
1564
+ // nothing but a delivered round may be held back, and a local wait on an idle
1565
+ // one would only delay the next fetch. The error arms keep their own waits
1566
+ // and are not touched by this.
1567
+ if (this.pullInterval > 0 && await this._sleepOrStop(this.pullInterval, stopFlag))
1568
+ return;
1569
+ }
1570
+ }
1571
+ }
1572
+ }
1573
+ // _findBrokerAddressInSubscribe resolves a broker address for a subscription
1574
+ // target. Returns (addr, isSlave) or null.
1575
+ //
1576
+ // The isSlave flag comes from the id that was ACTUALLY matched, which is why
1577
+ // the fallback branch recomputes it: a request for slave id 3 that falls
1578
+ // back to the master must NOT be reported as a slave request — otherwise the
1579
+ // COMMIT_OFFSET bit gets cleared for nothing.
1580
+ _findBrokerAddressInSubscribe(brokerName, brokerId, onlyThisBroker) {
1581
+ const client = this.mqClient;
1582
+ if (!client || !brokerName)
1583
+ return null;
1584
+ const addrs = client.getBrokerAddrTable().get(brokerName);
1585
+ if (!addrs || Object.keys(addrs).length === 0)
1586
+ return null;
1587
+ const snapshot = new Map();
1588
+ for (const [id, addr] of Object.entries(addrs))
1589
+ snapshot.set(parseInt(id, 10), addr);
1590
+ const hit = snapshot.get(brokerId);
1591
+ if (hit != null)
1592
+ return { addr: hit, isSlave: brokerId !== MixAll.MASTER_ID };
1593
+ if (brokerId !== MixAll.MASTER_ID) {
1594
+ // Broker convention: a slave registers as <slaveId>, so +1 is the
1595
+ // "primary slave" of the same group.
1596
+ const next = snapshot.get(brokerId + 1);
1597
+ if (next != null)
1598
+ return { addr: next, isSlave: true };
1599
+ }
1600
+ if (!onlyThisBroker) {
1601
+ // Rather than taking an arbitrary map-order entry, resolve
1602
+ // deterministically: the smallest id (the master when it
1603
+ // exists).
1604
+ let minID = 0;
1605
+ let first = true;
1606
+ for (const id of snapshot.keys()) {
1607
+ if (first || id < minID) {
1608
+ minID = id;
1609
+ first = false;
1610
+ }
1611
+ }
1612
+ return { addr: snapshot.get(minID), isSlave: minID !== MixAll.MASTER_ID };
1613
+ }
1614
+ return null;
1615
+ }
1616
+ // _pullKernel issues the pull at the wire layer: PULL_MESSAGE(11).
1617
+ async _pullKernel(mq, offset, sub, sysFlag, commitOffset, maxMsgNums, maxMsgBytes, suspendTimeoutMillis, timeoutMillis, addr) {
1618
+ const client = this.mqClient;
1619
+ logger.debug('pull request %s offset=%d addr=%s timeout=%d suspend=%d', mqKey(mq), offset, addr, timeoutMillis, suspendTimeoutMillis);
1620
+ const header = new PullMessageRequestHeader();
1621
+ header.consumerGroup = this.consumerGroup;
1622
+ header.topic = mq.getTopic();
1623
+ header.queueId = mq.getQueueId();
1624
+ header.queueOffset = offset;
1625
+ header.maxMsgNums = maxMsgNums;
1626
+ header.sysFlag = sysFlag;
1627
+ header.commitOffset = commitOffset;
1628
+ header.suspendTimeoutMillis = suspendTimeoutMillis;
1629
+ header.subVersion = sub.subVersion;
1630
+ header.expressionType = sub.expressionType || ExpressionType.TAG;
1631
+ header.maxMsgBytes = maxMsgBytes;
1632
+ header.brokerName = mq.getBrokerName();
1633
+ // Rule: the expression rides the wire only when the SUBSCRIPTION bit is
1634
+ // on. The key must not be present at all when the bit is off.
1635
+ if (PullSysFlag.hasSubscriptionFlag(sysFlag)) {
1636
+ header.subscription = sub.subString;
1637
+ }
1638
+ const request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, header);
1639
+ const response = await client.remotingClient.invokeSync(addr, request, timeoutMillis);
1640
+ let status;
1641
+ if (response.code === ResponseCode.SUCCESS)
1642
+ status = PullStatus.FOUND;
1643
+ else if (response.code === ResponseCode.PULL_NOT_FOUND)
1644
+ status = PullStatus.NO_NEW_MSG;
1645
+ else if (response.code === ResponseCode.PULL_OFFSET_MOVED)
1646
+ status = PullStatus.OFFSET_ILLEGAL;
1647
+ else if (response.code === ResponseCode.PULL_RETRY_IMMEDIATELY)
1648
+ status = PullStatus.NO_MATCHED_MSG;
1649
+ // The code rides on the exception: the loop's backoff branches on it (215 is
1650
+ // the broker's own throttle and gets the short pause). A bare Error would
1651
+ // leave the number only in the message text and silently disable that.
1652
+ else
1653
+ throw new MQBrokerException(response.code, `pull failed: CODE ${response.code} ${response.remark || ''}`, addr);
1654
+ const ext = response.extFields || {};
1655
+ const num = (k, d) => (ext[k] != null ? parseInt(ext[k], 10) : d);
1656
+ let found = [];
1657
+ if (response.body && response.body.length > 0) {
1658
+ found = decodeMessages(response.body, client.decodeReadBody);
1659
+ for (const msg of found) {
1660
+ msg.setBrokerName(mq.getBrokerName());
1661
+ msg.setQueueId(mq.getQueueId());
1662
+ }
1663
+ }
1664
+ // suggestWhichBrokerId may be ABSENT — that means master(0), not "keep the
1665
+ // previous value" (rule #22).
1666
+ const suggest = ext['suggestWhichBrokerId'] != null ? parseInt(ext['suggestWhichBrokerId'], 10) : null;
1667
+ return new PullResult(status, num('nextBeginOffset', 0), num('minOffset', 0), num('maxOffset', 0), found, suggest);
1668
+ }
1669
+ // _processPullResult post-processes a pull reply. It MUST
1670
+ // run for every response, not only FOUND ones — the node table is fed by
1671
+ // every reply, and absent means master.
1672
+ _processPullResult(mq, result, sub) {
1673
+ const brokerId = result.suggestWhichBrokerId != null ? result.suggestWhichBrokerId : MixAll.MASTER_ID;
1674
+ this.pullFromWhichNode.set(mqKey(mq), brokerId);
1675
+ if (result.status !== PullStatus.FOUND || result.msgFoundList.length === 0)
1676
+ return result;
1677
+ // Unwrap inner batches before filtering so the tag check and the hooks see
1678
+ // the individual messages.
1679
+ let msgs = decodeInnerBatchMessages(result.msgFoundList);
1680
+ msgs = clientSideTagFilter(sub, msgs);
1681
+ if (this.filterMessageHookList.length > 0) {
1682
+ // FilterMessageHook MUST swallow exceptions and continue (rule #7): the
1683
+ // pull path silently skips filtered messages.
1684
+ const ctx = new FilterMessageContext();
1685
+ ctx.consumerGroup = this.consumerGroup;
1686
+ ctx.msgList = msgs;
1687
+ ctx.mq = mq;
1688
+ ctx.unitMode = this.unitMode;
1689
+ ctx.accessChannel = this.accessChannel;
1690
+ for (const hook of this.filterMessageHookList.slice()) {
1691
+ try {
1692
+ hook.filterMessage(ctx);
1693
+ }
1694
+ catch (e) { /* swallowed by contract */ }
1695
+ }
1696
+ msgs = ctx.msgList;
1697
+ }
1698
+ for (const msg of msgs) {
1699
+ // A half message carries its transaction id in UNIQ_KEY; lift it
1700
+ // so the listener can see it as MessageExt.getTransactionId().
1701
+ if (msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED) === 'true') {
1702
+ const uniq = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
1703
+ if (uniq)
1704
+ msg.setTransactionId(uniq);
1705
+ }
1706
+ msg.putProperty(MessageConst.PROPERTY_MIN_OFFSET, String(result.minOffset));
1707
+ msg.putProperty(MessageConst.PROPERTY_MAX_OFFSET, String(result.maxOffset));
1708
+ msg.setBrokerName(mq.getBrokerName());
1709
+ }
1710
+ result.msgFoundList = msgs;
1711
+ return result;
1712
+ }
1713
+ // _flowControlHit applies the ProcessQueue put/pull threshold checks: ONE
1714
+ // queue's own holdings are judged against exactly three per-queue gates
1715
+ // (count / size / span). There is NO pull-time topic aggregation: the
1716
+ // topic-level budgets were divided into the per-queue knobs at rebalance
1717
+ // (see _divideTopicThresholds), so the sibling queues of a topic never add
1718
+ // up here. Every cache gate compares STRICTLY GREATER THAN the threshold as it
1719
+ // stands — no floor of 1, no "0 means disabled" escape, because keeping the
1720
+ // knob inside its legal range is _checkConfigRanges' job, not the gate's. A
1721
+ // buffer that has reached the threshold exactly is still allowed to pull, so a
1722
+ // threshold reads as "how many may be cached", not "how many stop us", and a
1723
+ // threshold of 0 simply means "cache nothing".
1724
+ //
1725
+ // The three terms all read the SAME held set (everything the queue still holds,
1726
+ // listener-held batches included). The one per-term difference is that the SPAN
1727
+ // gate does not run at all for ordered consumption: an orderly queue has no
1728
+ // run-ahead to guard against, because its pull cursor only moves once the batch
1729
+ // in the listener's hands has settled, while count and size bind either way.
1730
+ _flowControlHit(mq, pq) {
1731
+ const reason = this._cacheGateReason(pq);
1732
+ if (!reason)
1733
+ return false;
1734
+ this.flowControlTriggered++;
1735
+ logger.debug('flow control: queue %s %s, pause pull', mqKey(mq), reason);
1736
+ if (isFlowControlWarnTrip(this.flowControlTriggered)) {
1737
+ logger.warn('flow control: queue %s %s exceeds its cache threshold, pause pull for %d ms'
1738
+ + ' (flowControlTimes=%d)', mqKey(mq), reason, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, this.flowControlTriggered);
1739
+ }
1740
+ return true;
1741
+ }
1742
+ // _cacheGateReason names the gate this queue's holdings are over — "count=N",
1743
+ // "size=NMB" or "span=N" — or '' when the queue may pull. The three checks run in
1744
+ // one fixed order (count, then size, then span) and the first one that matches is
1745
+ // the reason reported, so a buffer over every budget says which gate the operator
1746
+ // has to raise. It judges ONE queue and reads no sibling queue.
1747
+ _cacheGateReason(pq) {
1748
+ const [count, sizeMiB, span] = pq.heldStats();
1749
+ if (count > this.pullThresholdForQueue)
1750
+ return `count=${count}`;
1751
+ if (sizeMiB > this.pullThresholdSizeForQueue)
1752
+ return `size=${sizeMiB}MB`;
1753
+ if (!pq.consumeOrderly && span > this.consumeConcurrentlyMaxSpan)
1754
+ return `span=${span}`;
1755
+ return '';
1756
+ }
1757
+ // _popFlowControlHit is the POP twin of _flowControlHit and it feeds the SAME
1758
+ // counter, so flowControlTimes stays one client-wide number that spans the
1759
+ // pull path and the POP path and the WARN cadence throttles both gates
1760
+ // together.
1761
+ //
1762
+ // What it measures is not a buffer but this queue's OUTSTANDING-ACK DEBT: the
1763
+ // messages the broker made invisible for this queue and whose answer (ACK or
1764
+ // CHANGE_INVISIBLETIME) is still owed. Like the pull gate, it judges ONE
1765
+ // queue's own debt strictly GREATER THAN popThresholdForQueue and never sums
1766
+ // the debts of sibling queues: a queue that owes nothing keeps popping while
1767
+ // its hot sibling waits, so one runaway queue cannot stall the group.
1768
+ // Reaching the threshold exactly still pops, so the knob reads as "how many
1769
+ // answers may be outstanding", not "how many stop us".
1770
+ _popFlowControlHit(key, pq) {
1771
+ const waitAck = pq.waitAckMsgCount();
1772
+ if (waitAck <= this.popThresholdForQueue)
1773
+ return false;
1774
+ this.flowControlTriggered++;
1775
+ logger.debug('pop flow control: queue %s waiting-ack=%d exceeds pop threshold %d', key, waitAck, this.popThresholdForQueue);
1776
+ if (isFlowControlWarnTrip(this.flowControlTriggered)) {
1777
+ logger.warn('pop flow control: queue %s waiting-ack=%d exceeds pop threshold %d,'
1778
+ + ' pause pop for %d ms (flowControlTimes=%d)', key, waitAck, this.popThresholdForQueue, POP_DELAY_WHEN_CACHE_FLOW_CONTROL, this.flowControlTriggered);
1779
+ }
1780
+ return true;
1781
+ }
1782
+ // ---------------------------------------------------------------- dispatch
1783
+ // _dispatchLoop takes batches out of the buffers and hands them to the
1784
+ // listener.
1785
+ //
1786
+ // Concurrent mode: an async batch, bounded by the core pool size. Orderly
1787
+ // mode: consumed INLINE, so a queue's ordering is preserved.
1788
+ async _dispatchLoop() {
1789
+ const stopFlag = this._newStopFlag();
1790
+ while (!this._isStopped(stopFlag)) {
1791
+ let progressed = false;
1792
+ const batchMax = this.consumeMessageBatchMaxSize;
1793
+ for (const [key, pq] of Array.from(this.processQueueTable.entries())) {
1794
+ if (this._isStopped(stopFlag))
1795
+ return;
1796
+ if (pq.isDropped())
1797
+ continue;
1798
+ const batch = pq.takeBatch(Math.max(1, batchMax));
1799
+ if (batch.length === 0)
1800
+ continue;
1801
+ const epoch = this._queueEpoch.get(key) || 0;
1802
+ progressed = true;
1803
+ if (!this._beginInFlight()) {
1804
+ // Shutdown froze new work: hand the batch back instead of consuming
1805
+ // it. Its offset was never advanced, so the broker redelivers it on
1806
+ // the next start.
1807
+ pq.requeueBatch(batch);
1808
+ return;
1809
+ }
1810
+ const [topic, brokerName, queueId] = parseMqKey(key);
1811
+ const mq = new MessageQueue(topic, brokerName, queueId);
1812
+ if (this.orderly) {
1813
+ try {
1814
+ await this._consumeBatch(mq, batch, epoch);
1815
+ }
1816
+ finally {
1817
+ this._endInFlight();
1818
+ }
1819
+ continue;
1820
+ }
1821
+ if (this._dispatchSem >= Math.max(1, this.corePoolSize)) {
1822
+ // Pool full: wait a tick rather than unbounded queueing (real
1823
+ // concurrency is core-size bound too).
1824
+ pq.requeueBatch(batch);
1825
+ this._endInFlight();
1826
+ await this._sleepOrStop(5, stopFlag);
1827
+ continue;
1828
+ }
1829
+ this._dispatchSem++;
1830
+ void this._consumeBatch(mq, batch, epoch)
1831
+ .catch((e) => {
1832
+ logger.error('dispatch batch error for %s (requeued): %s', key, e.message);
1833
+ const cur = this.processQueueTable.get(key);
1834
+ if (cur)
1835
+ cur.requeueBatch(batch);
1836
+ })
1837
+ .finally(() => {
1838
+ this._dispatchSem = Math.max(0, this._dispatchSem - 1);
1839
+ this._endInFlight();
1840
+ });
1841
+ }
1842
+ if (!progressed) {
1843
+ if (await this._sleepOrStop(50, stopFlag))
1844
+ return;
1845
+ }
1846
+ }
1847
+ }
1848
+ // _consumeBatch dispatches one batch and handles redelivery/suspension.
1849
+ async _consumeBatch(mq, batch, epoch) {
1850
+ if (batch.length === 0)
1851
+ return;
1852
+ const key = mqKey(mq);
1853
+ if (epoch !== (this._queueEpoch.get(key) || 0)) {
1854
+ logger.warning('the message queue not be able to consume, because it\'s dropped. group=%s mq=%s msgs=%d', this.consumerGroup, key, batch.length);
1855
+ return;
1856
+ }
1857
+ // The POP retry naming is normalised at the CONCURRENT delivery entry —
1858
+ // pull batches included, where it matches nothing — and the orderly
1859
+ // delivery path never runs it.
1860
+ if (!this.orderly)
1861
+ this._resetPopRetryTopic(batch);
1862
+ this._resetRetryTopicAndNamespace(batch);
1863
+ if (this.orderly)
1864
+ await this._consumeOrderlyBatch(mq, batch, epoch);
1865
+ else
1866
+ await this._consumeConcurrentlyBatch(mq, batch, epoch);
1867
+ }
1868
+ // _resetPopRetryTopic rewrites a POP retry topic back to the business topic
1869
+ // it retries, so the listener sees the topic it subscribed to.
1870
+ //
1871
+ // A retried POP row is hidden under %RETRY%<group>_<topic>, or — under the
1872
+ // newer retry naming — under %RETRY%<group>+<topic>: the prefix and the
1873
+ // separator are broker-side bookkeeping, so everything after the separator is
1874
+ // the real topic. Exactly %RETRY%<group> carries no separator and is left to
1875
+ // _resetRetryTopicAndNamespace to restore from the RETRY_TOPIC property, and
1876
+ // an ordinary topic matches neither prefix, which is what makes this step a
1877
+ // no-op for a pull batch.
1878
+ //
1879
+ // With a namespace set, the name can carry the local prefix on BOTH sides of
1880
+ // the separator, so the prefix is stripped first, the name is normalised, and
1881
+ // the prefix is stripped from the result. The group inside the retry topic is
1882
+ // the plain one: start() may have wrapped the namespace into consumerGroup,
1883
+ // so the prefix is built from the unwrapped name.
1884
+ _resetPopRetryTopic(msgs) {
1885
+ if (msgs.length === 0)
1886
+ return;
1887
+ const group = this.namespace ? MixAll.withoutNamespace(this.consumerGroup, this.namespace)
1888
+ : this.consumerGroup;
1889
+ const v1Prefix = MixAll.RETRY_GROUP_TOPIC_PREFIX + group + '_';
1890
+ const v2Prefix = MixAll.RETRY_GROUP_TOPIC_PREFIX + group + '+';
1891
+ for (const msg of msgs) {
1892
+ let topic = msg.getTopic();
1893
+ if (this.namespace)
1894
+ topic = MixAll.withoutNamespace(topic, this.namespace);
1895
+ let normal = null;
1896
+ if (topic.startsWith(v2Prefix))
1897
+ normal = topic.slice(v2Prefix.length);
1898
+ else if (topic.startsWith(v1Prefix))
1899
+ normal = topic.slice(v1Prefix.length);
1900
+ // Nothing to rewrite: an empty name after the separator is not a topic,
1901
+ // and a name that matched neither prefix keeps its original topic.
1902
+ if (!normal)
1903
+ continue;
1904
+ if (this.namespace)
1905
+ normal = MixAll.withoutNamespace(normal, this.namespace);
1906
+ msg.setTopic(normal);
1907
+ }
1908
+ }
1909
+ // _resetRetryTopicAndNamespace rewrites the %RETRY% topic back to the origin
1910
+ // topic and strips the namespace prefix, called BEFORE the
1911
+ // listener.
1912
+ //
1913
+ // A retried message physically lives under %RETRY%<group>; the broker writes
1914
+ // the business topic into the RETRY_TOPIC property. Restoring it here is
1915
+ // what lets the listener branch on the topic it subscribed to.
1916
+ _resetRetryTopicAndNamespace(msgs) {
1917
+ const groupTopic = MixAll.getRetryTopic(this.consumerGroup);
1918
+ for (const msg of msgs) {
1919
+ const retryTopic = msg.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
1920
+ if (retryTopic && msg.getTopic() === groupTopic) {
1921
+ msg.setTopic(retryTopic);
1922
+ }
1923
+ if (this.namespace) {
1924
+ msg.setTopic(MixAll.withoutNamespace(msg.getTopic(), this.namespace));
1925
+ }
1926
+ }
1927
+ }
1928
+ // ------------------------------------------------------------ concurrent
1929
+ async _consumeConcurrentlyBatch(mq, batch, epoch) {
1930
+ const ctx = new ConsumeConcurrentlyContext(mq);
1931
+ // AckIndex defaults to "the whole batch" — INT32_MAX clamped to
1932
+ // len(batch)-1 on success. Getting it wrong in the other direction
1933
+ // silently acks only one message or none at all.
1934
+ ctx.ackIndex = 2147483647;
1935
+ const hooks = this.consumeMessageHookList;
1936
+ let hookCtx = null;
1937
+ if (hooks.length > 0) {
1938
+ hookCtx = new ConsumeMessageContext();
1939
+ hookCtx.consumerGroup = this.consumerGroup;
1940
+ hookCtx.msgList = batch;
1941
+ hookCtx.mq = mq;
1942
+ hookCtx.success = false;
1943
+ hookCtx.props = {};
1944
+ // Java ConsumeMessageConcurrentlyService:410 stamps the facade's channel.
1945
+ hookCtx.accessChannel = this.accessChannel;
1946
+ for (const hook of hooks.slice()) {
1947
+ try {
1948
+ hook.consumeMessageBefore(hookCtx);
1949
+ }
1950
+ catch (e) { /* hook errors don't stop delivery */ }
1951
+ }
1952
+ }
1953
+ const begin = Date.now();
1954
+ // CONSUME_START_TIME is stamped
1955
+ // on every delivery BEFORE the listener; cleanExpiredMsg's escape hatch
1956
+ // reads it.
1957
+ for (const msg of batch) {
1958
+ msg.putProperty('CONSUME_START_TIME', String(Date.now()));
1959
+ }
1960
+ const { status: rawStatus, threw } = this._listenerCall(batch, ctx);
1961
+ let status = rawStatus;
1962
+ if (threw) {
1963
+ logger.debug('listener error, treat as RECONSUME_LATER: mq=%s', mqKey(mq));
1964
+ status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
1965
+ }
1966
+ // The RT is taken ONCE, immediately after the listener returns.
1967
+ const consumeRT = Date.now() - begin;
1968
+ if (status !== ConsumeConcurrentlyStatus.CONSUME_SUCCESS && status !== ConsumeConcurrentlyStatus.RECONSUME_LATER) {
1969
+ // An out-of-range return behaves as RECONSUME_LATER instead of silently
1970
+ // falling into SUCCESS and acking an unhandled batch.
1971
+ logger.warning('consumeMessage return unknown status, Group: %s Msgs: %d MQ: %s', this.consumerGroup, batch.length, mqKey(mq));
1972
+ status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
1973
+ }
1974
+ let ackIndex = ctx.ackIndex;
1975
+ if (status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS) {
1976
+ if (ackIndex >= batch.length)
1977
+ ackIndex = batch.length - 1;
1978
+ }
1979
+ else {
1980
+ ackIndex = -1;
1981
+ }
1982
+ if (hookCtx) {
1983
+ const unknown = rawStatus !== ConsumeConcurrentlyStatus.CONSUME_SUCCESS
1984
+ && rawStatus !== ConsumeConcurrentlyStatus.RECONSUME_LATER;
1985
+ hookCtx.props['ConsumeContextType'] = consumeReturnTypeName(unknown ? (threw ? ConsumeReturnType.EXCEPTION : ConsumeReturnType.RETURNNULL)
1986
+ : (consumeRT >= this.consumeTimeout * 60 * 1000 ? ConsumeReturnType.TIME_OUT
1987
+ : (status === ConsumeConcurrentlyStatus.RECONSUME_LATER ? ConsumeReturnType.FAILED
1988
+ : ConsumeReturnType.SUCCESS)));
1989
+ hookCtx.success = status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
1990
+ hookCtx.status = status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS ? 'CONSUME_SUCCESS' : 'RECONSUME_LATER';
1991
+ for (const hook of hooks.slice()) {
1992
+ try {
1993
+ hook.consumeMessageAfter(hookCtx);
1994
+ }
1995
+ catch (e) { /* hooks never break delivery */ }
1996
+ }
1997
+ }
1998
+ this._incConsumeRT(mq.getTopic(), consumeRT);
1999
+ if (status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS) {
2000
+ this._incConsumeOKTPS(mq.getTopic(), ackIndex + 1);
2001
+ if (batch.length - ackIndex - 1 > 0) {
2002
+ this._incConsumeFailedTPS(mq.getTopic(), batch.length - ackIndex - 1);
2003
+ }
2004
+ }
2005
+ else {
2006
+ this._incConsumeFailedTPS(mq.getTopic(), batch.length);
2007
+ }
2008
+ if (this.messageModel === MessageModel.BROADCASTING) {
2009
+ // Broadcasting never sends back: the unacked tail is dropped with a
2010
+ // warning and the whole batch advances.
2011
+ const dropped = batch.length - ackIndex - 1;
2012
+ if (dropped > 0) {
2013
+ logger.warning('BROADCASTING, the message consume failed, drop it: %d msgs in %s', dropped, mqKey(mq));
2014
+ }
2015
+ this._commitConcurrently(mq, batch, epoch);
2016
+ return;
2017
+ }
2018
+ if (ackIndex + 1 >= batch.length) {
2019
+ this._commitConcurrently(mq, batch, epoch);
2020
+ return;
2021
+ }
2022
+ // Cluster mode: redeliver the unacked tail one by one to %RETRY%<group>
2023
+ // (delay 3+reconsumeTimes; the broker moves it to %DLQ% once
2024
+ // maxReconsumeTimes is exceeded).
2025
+ const failed = await this._sendBackBatch(mq, batch.slice(ackIndex + 1), ctx);
2026
+ if (failed.length > 0) {
2027
+ // The ones whose send-back failed are re-submitted later: they go back
2028
+ // to the front of the buffer.
2029
+ const pq = this.processQueueTable.get(mqKey(mq));
2030
+ if (pq)
2031
+ pq.requeueBatch(failed);
2032
+ await this._sleepOrStop(200);
2033
+ }
2034
+ const failedOffsets = new Set(failed.map((m) => m.getQueueOffset()));
2035
+ const acked = batch.filter((m) => !failedOffsets.has(m.getQueueOffset()));
2036
+ // The commit is what removeMessage(msgs) answers, and `msgs` is the ACKED
2037
+ // list (the batch minus msgBackFailed).
2038
+ this._commitConcurrently(mq, acked, epoch);
2039
+ }
2040
+ // _commitConcurrently removes acked messages from the ProcessQueue and
2041
+ // updates the consume offset for the concurrent paths.
2042
+ //
2043
+ // The target is the queue's high-water mark + 1 — NOT the end of this batch.
2044
+ // Batches of one queue are consumed in parallel, so the batch that finishes
2045
+ // second is not necessarily the one with the higher offsets.
2046
+ _commitConcurrently(mq, acked, epoch) {
2047
+ if (acked.length === 0)
2048
+ return;
2049
+ let drained = batchEnd(acked);
2050
+ let floor = -1;
2051
+ const pq = this.processQueueTable.get(mqKey(mq));
2052
+ if (pq) {
2053
+ drained = pq.queueOffsetMaxValue() + 1;
2054
+ floor = pq.minRemainingExcept(acked);
2055
+ }
2056
+ this._advanceConsumeOffset(mq, acked, drained, floor, epoch);
2057
+ }
2058
+ // _sendBackBatch redelivers the unacked entries one by one and returns the
2059
+ // ones that FAILED.
2060
+ //
2061
+ // A failed entry gets reconsumeTimes+1 locally — the broker never recorded
2062
+ // the attempt, and without the local bump the message can never reach the
2063
+ // dead-letter queue.
2064
+ async _sendBackBatch(mq, batch, ctx) {
2065
+ const failed = [];
2066
+ const pq = this.processQueueTable.get(mqKey(mq));
2067
+ for (const msg of batch) {
2068
+ if (pq && !pq.contains(msg.getQueueOffset())) {
2069
+ // An entry already swept (or whose queue was revoked) is skipped: it
2070
+ // is on its way back to the broker, and sending it again would
2071
+ // duplicate it.
2072
+ logger.info('Message is not found in its process queue; skip send-back-procedure, topic=%s, brokerName=%s, queueId=%d, queueOffset=%d', msg.getTopic(), msg.getBrokerName(), msg.getQueueId(), msg.getQueueOffset());
2073
+ continue;
2074
+ }
2075
+ let delayLevel = ctx.delayLevelWhenNextConsume || 0;
2076
+ if (delayLevel === 0)
2077
+ delayLevel = 3 + msg.getReconsumeTimes();
2078
+ try {
2079
+ await this.sendMessageBack(msg, delayLevel);
2080
+ }
2081
+ catch (e) {
2082
+ logger.debug('send message back failed for msg %s: %s', msg.getMsgId(), e.message);
2083
+ msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
2084
+ failed.push(msg);
2085
+ }
2086
+ }
2087
+ return failed;
2088
+ }
2089
+ // sendMessageBack asks the broker for a per-message
2090
+ // redelivery through CONSUMER_SEND_MSG_BACK(36).
2091
+ //
2092
+ // `offset` is the message's commitLogOffset, NOT its queueOffset — the
2093
+ // broker looks the record up by physical offset.
2094
+ //
2095
+ // A refused bounce — a missing master, a transport error or a non-success
2096
+ // answer — degrades to re-sending the message as a normal message to
2097
+ // %RETRY%<group> through the internal producer, so the redelivery is not
2098
+ // lost; only when that send fails too does the original error reach the
2099
+ // caller.
2100
+ async sendMessageBack(msg, delayLevel) {
2101
+ const client = this.mqClient;
2102
+ if (!client)
2103
+ throw new Error('consumer not started');
2104
+ const maxReconsume = this.maxReconsumeTimes === -1 ? 16 : this.maxReconsumeTimes;
2105
+ try {
2106
+ const addr = client.findBrokerAddressInPublish(msg.getBrokerName());
2107
+ if (!addr)
2108
+ throw new Error(`Broker[${msg.getBrokerName()}] master node does not exist`);
2109
+ const h = new ConsumerSendMsgBackRequestHeader();
2110
+ h.offset = msg.getCommitLogOffset();
2111
+ h.group = this.consumerGroup;
2112
+ h.delayLevel = delayLevel;
2113
+ h.originMsgId = clientMsgId(msg);
2114
+ // Delivery hands the listener the bare topic; the broker-side record —
2115
+ // the one this bounce references — lives under the prefixed name.
2116
+ h.originTopic = MixAll.wrapNamespace(this.namespace, msg.getTopic());
2117
+ h.unitMode = this.unitMode;
2118
+ h.maxReconsumeTimes = maxReconsume;
2119
+ const request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, h);
2120
+ const response = await client.remotingClient.invokeSync(client.vipChannel(addr), request, 5000);
2121
+ if (response.code !== ResponseCode.SUCCESS) {
2122
+ throw new Error(`sendMessageBack failed: CODE ${response.code} ${response.remark || ''}`);
2123
+ }
2124
+ }
2125
+ catch (e) {
2126
+ logger.error('Failed to send message back, consumerGroup=%s, brokerName=%s, msg=%s: %s; fallback to normal send', this.consumerGroup, msg.getBrokerName(), msg.getMsgId(), e.message);
2127
+ try {
2128
+ await this._sendRetryMessage(msg, maxReconsume);
2129
+ }
2130
+ catch (fallbackError) {
2131
+ logger.error('Fallback normal send failed, consumerGroup=%s, msg=%s: %s', this.consumerGroup, msg.getMsgId(), fallbackError.message);
2132
+ throw e;
2133
+ }
2134
+ }
2135
+ }
2136
+ // ------------------------------------------------------------- orderly
2137
+ async _consumeOrderlyBatch(mq, batch, epoch) {
2138
+ const ctx = new ConsumeOrderlyContext(mq);
2139
+ ctx.autoCommit = true;
2140
+ ctx.suspendCurrentQueueTimeMillis = -1;
2141
+ const hooks = this.consumeMessageHookList;
2142
+ let hookCtx = null;
2143
+ if (hooks.length > 0) {
2144
+ hookCtx = new ConsumeMessageContext();
2145
+ hookCtx.consumerGroup = this.consumerGroup;
2146
+ hookCtx.msgList = batch;
2147
+ hookCtx.mq = mq;
2148
+ hookCtx.success = false;
2149
+ hookCtx.props = {};
2150
+ // Java ConsumeMessageConcurrentlyService:410 stamps the facade's channel.
2151
+ hookCtx.accessChannel = this.accessChannel;
2152
+ for (const hook of hooks.slice()) {
2153
+ try {
2154
+ hook.consumeMessageBefore(hookCtx);
2155
+ }
2156
+ catch (e) { /* ignore */ }
2157
+ }
2158
+ }
2159
+ const begin = Date.now();
2160
+ const { status: rawStatus } = this._listenerCall(batch, ctx);
2161
+ let status = rawStatus;
2162
+ if (status < 0) {
2163
+ logger.debug('orderly listener error (retry in place): mq=%s', mqKey(mq));
2164
+ }
2165
+ const consumeRT = Date.now() - begin;
2166
+ if (status !== ConsumeOrderlyStatus.SUCCESS && status !== ConsumeOrderlyStatus.ROLLBACK
2167
+ && status !== ConsumeOrderlyStatus.COMMIT && status !== ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT) {
2168
+ // A null/stray status is normalised to "suspend" BEFORE the hook.
2169
+ // Without this a stray value would fall into the SUCCESS branch and
2170
+ // silently ack messages that were never consumed.
2171
+ status = ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
2172
+ }
2173
+ if (hookCtx) {
2174
+ const unknown = rawStatus !== ConsumeOrderlyStatus.SUCCESS && rawStatus !== ConsumeOrderlyStatus.ROLLBACK
2175
+ && rawStatus !== ConsumeOrderlyStatus.COMMIT && rawStatus !== ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
2176
+ hookCtx.props['ConsumeContextType'] = consumeReturnTypeName(unknown ? (rawStatus < 0 ? ConsumeReturnType.EXCEPTION : ConsumeReturnType.RETURNNULL)
2177
+ : (consumeRT >= this.consumeTimeout * 60 * 1000 ? ConsumeReturnType.TIME_OUT
2178
+ : (status === ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT ? ConsumeReturnType.FAILED
2179
+ : ConsumeReturnType.SUCCESS)));
2180
+ // The hook sees the NORMALISED status, while the returnType is derived
2181
+ // from the raw one; success is SUCCESS||COMMIT.
2182
+ hookCtx.success = status === ConsumeOrderlyStatus.SUCCESS || status === ConsumeOrderlyStatus.COMMIT;
2183
+ hookCtx.status = status === ConsumeOrderlyStatus.SUCCESS ? 'SUCCESS'
2184
+ : status === ConsumeOrderlyStatus.ROLLBACK ? 'ROLLBACK'
2185
+ : status === ConsumeOrderlyStatus.COMMIT ? 'COMMIT' : 'SUSPEND_CURRENT_QUEUE_A_MOMENT';
2186
+ for (const hook of hooks.slice()) {
2187
+ try {
2188
+ hook.consumeMessageAfter(hookCtx);
2189
+ }
2190
+ catch (e) { /* ignore */ }
2191
+ }
2192
+ }
2193
+ this._incConsumeRT(mq.getTopic(), consumeRT);
2194
+ const autoCommit = ctx.autoCommit !== false;
2195
+ if (autoCommit) {
2196
+ if (status === ConsumeOrderlyStatus.COMMIT || status === ConsumeOrderlyStatus.ROLLBACK) {
2197
+ // With autoCommit on, COMMIT/ROLLBACK are illegal (they belong to the
2198
+ // binlog consumer). Warn and do NOT break: the messages are
2199
+ // acked.
2200
+ logger.warning('the message queue consume result is illegal, we think you want to ack these message %s', mqKey(mq));
2201
+ status = ConsumeOrderlyStatus.SUCCESS;
2202
+ }
2203
+ if (status === ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT) {
2204
+ // SUSPEND counts the WHOLE batch as failed, before the retry decision.
2205
+ this._incConsumeFailedTPS(mq.getTopic(), batch.length);
2206
+ // checkReconsumeTimes runs first: only "still within the retry budget,
2207
+ // or the send-back failed" suspends in place; once the message has
2208
+ // been handed to the broker the offset moves on, otherwise a poison
2209
+ // message parks the queue forever.
2210
+ if (await this._checkOrderlyReconsumeTimes(batch)) {
2211
+ const pq = this.processQueueTable.get(mqKey(mq));
2212
+ if (pq)
2213
+ pq.requeueBatch(batch);
2214
+ await this._sleepOrStop(this._sleepOrderlyMillis(ctx));
2215
+ return;
2216
+ }
2217
+ }
2218
+ else {
2219
+ this._incConsumeOKTPS(mq.getTopic(), batch.length);
2220
+ }
2221
+ this._advanceConsumeOffset(mq, batch, batchEnd(batch), -1, epoch);
2222
+ return;
2223
+ }
2224
+ // autoCommit == false (the binlog path).
2225
+ switch (status) {
2226
+ case ConsumeOrderlyStatus.COMMIT:
2227
+ this._advanceConsumeOffset(mq, batch, batchEnd(batch), -1, epoch);
2228
+ return;
2229
+ case ConsumeOrderlyStatus.ROLLBACK: {
2230
+ const pq = this.processQueueTable.get(mqKey(mq));
2231
+ if (pq)
2232
+ pq.requeueBatch(batch);
2233
+ await this._sleepOrStop(this._sleepOrderlyMillis(ctx));
2234
+ return;
2235
+ }
2236
+ case ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT: {
2237
+ this._incConsumeFailedTPS(mq.getTopic(), batch.length);
2238
+ if (await this._checkOrderlyReconsumeTimes(batch)) {
2239
+ const pq = this.processQueueTable.get(mqKey(mq));
2240
+ if (pq)
2241
+ pq.requeueBatch(batch);
2242
+ await this._sleepOrStop(this._sleepOrderlyMillis(ctx));
2243
+ }
2244
+ // Unlike the autoCommit branch the offset is NOT committed; whether to
2245
+ // advance is the binlog consumer's call.
2246
+ return;
2247
+ }
2248
+ default: {
2249
+ // SUCCESS with autoCommit off: the offset stays put, nothing is lost,
2250
+ // and the loop does not spin.
2251
+ this._incConsumeOKTPS(mq.getTopic(), batch.length);
2252
+ const pq = this.processQueueTable.get(mqKey(mq));
2253
+ if (pq)
2254
+ pq.requeueBatch(batch);
2255
+ await this._sleepOrStop(this._sleepOrderlyMillis(ctx));
2256
+ return;
2257
+ }
2258
+ }
2259
+ }
2260
+ // _sleepOrderlyMillis resolves the orderly suspend delay: -1 means "not set"
2261
+ // and falls back to the consumer's configuration, then the result is clamped
2262
+ // to [10, 30000]. The clamp is not decoration: a listener returning 0 would
2263
+ // otherwise turn the consume loop into a busy loop.
2264
+ _sleepOrderlyMillis(ctx) {
2265
+ let ms = ctx.suspendCurrentQueueTimeMillis;
2266
+ if (ms === -1 || ms == null)
2267
+ ms = this.suspendCurrentQueueTimeMillis;
2268
+ if (ms < 10)
2269
+ return 10;
2270
+ if (ms > 30000)
2271
+ return 30000;
2272
+ return ms;
2273
+ }
2274
+ // _orderlyMaxReconsumeTimes: -1 means INT32_MAX
2275
+ // here (orderly retries in place, the broker never counts).
2276
+ _orderlyMaxReconsumeTimes() {
2277
+ return this.maxReconsumeTimes === -1 ? INT32_MAX : this.maxReconsumeTimes;
2278
+ }
2279
+ // _checkOrderlyReconsumeTimes compares the message's reconsume count against
2280
+ // the orderly max. Returns whether the
2281
+ // batch still has to be suspended in place.
2282
+ async _checkOrderlyReconsumeTimes(msgs) {
2283
+ let suspend = false;
2284
+ const maxTimes = this._orderlyMaxReconsumeTimes();
2285
+ for (const msg of msgs) {
2286
+ if (msg.getReconsumeTimes() >= maxTimes) {
2287
+ msg.putProperty(MessageConst.PROPERTY_RECONSUME_TIME, String(msg.getReconsumeTimes()));
2288
+ if (!(await this._orderlySendMessageBackSync(msg))) {
2289
+ suspend = true;
2290
+ msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
2291
+ }
2292
+ }
2293
+ else {
2294
+ suspend = true;
2295
+ msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
2296
+ }
2297
+ }
2298
+ return suspend;
2299
+ }
2300
+ // _orderlySendMessageBackSync sends back for the orderly path — a PLAIN send
2301
+ // to %RETRY%<group>, not a CONSUMER_SEND_MSG_BACK. It waits for the send and
2302
+ // returns the REAL outcome: only a confirmed send lets the offset move past
2303
+ // the message.
2304
+ async _orderlySendMessageBackSync(msg) {
2305
+ try {
2306
+ return await this._doOrderlySendMessageBack(msg);
2307
+ }
2308
+ catch (e) {
2309
+ return false;
2310
+ }
2311
+ }
2312
+ async _doOrderlySendMessageBack(msg) {
2313
+ try {
2314
+ await this._sendRetryMessage(msg, this._orderlyMaxReconsumeTimes());
2315
+ return true;
2316
+ }
2317
+ catch (e) {
2318
+ logger.debug('orderly send message back failed, group=%s msg=%s: %s', this.consumerGroup, msg.getMsgId(), e.message);
2319
+ return false;
2320
+ }
2321
+ }
2322
+ // _sendRetryMessage sends a failed delivery as a normal message to
2323
+ // %RETRY%<group> through the internal producer.
2324
+ async _sendRetryMessage(msg, maxReconsume) {
2325
+ const producer = await this._innerProducer();
2326
+ if (!producer)
2327
+ throw new Error('inner producer is not available');
2328
+ await producer.sendBySelector(buildRetryMessage(this.consumerGroup, msg, maxReconsume), undefined);
2329
+ }
2330
+ // _innerProducer lazily builds the CLIENT_INNER_PRODUCER used by the orderly
2331
+ // send-back path.
2332
+ async _innerProducer() {
2333
+ if (this._producer)
2334
+ return this._producer;
2335
+ try {
2336
+ const { DefaultMQProducer: P } = await import("./producer.js");
2337
+ const producer = new P(MixAll.CLIENT_INNER_PRODUCER_GROUP || 'CLIENT_INNER_PRODUCER');
2338
+ producer.setNamesrvAddr(this.nameServerAddr);
2339
+ await producer.start();
2340
+ this._producer = producer;
2341
+ return producer;
2342
+ }
2343
+ catch (e) {
2344
+ logger.debug('inner producer start failed: %s', e.message);
2345
+ return null;
2346
+ }
2347
+ }
2348
+ // ---------------------------------------------------------------- offsets
2349
+ // _advanceConsumeOffset advances the consumed offset: it writes `drained` —
2350
+ // the offset to commit once the buffer holds nothing else — and then lowers it
2351
+ // to `floor`, the smallest offset that WILL still be buffered, whenever that
2352
+ // is lower.
2353
+ //
2354
+ // epoch is the process-queue generation captured with the batch; a mismatch
2355
+ // means the queue was revoked or rebuilt and the ack is void. A frozen
2356
+ // offset (OFFSET_ILLEGAL recovery) must not move either.
2357
+ _advanceConsumeOffset(mq, batch, drained, floor, epoch) {
2358
+ if (batch.length === 0)
2359
+ return;
2360
+ const key = mqKey(mq);
2361
+ let nextOffset = drained;
2362
+ if (floor >= 0 && floor < nextOffset)
2363
+ nextOffset = floor;
2364
+ if (epoch !== (this._queueEpoch.get(key) || 0)) {
2365
+ logger.debug('drop ack for %s: process queue was dropped (epoch %s -> %s)', key, epoch, this._queueEpoch.get(key) || 0);
2366
+ return;
2367
+ }
2368
+ if (this._frozenOffsets.has(key))
2369
+ return;
2370
+ if (this.offsetStore)
2371
+ this.offsetStore.updateOffset(mq, nextOffset, true);
2372
+ const pq = this.processQueueTable.get(key);
2373
+ if (pq)
2374
+ pq.completeBatch(batch);
2375
+ }
2376
+ // _correctTagsOffset catches up a consumed offset that would otherwise stall.
2377
+ //
2378
+ // When a pull answers NO_NEW_MSG or NO_MATCHED_MSG the CONSUMED offset must
2379
+ // follow the pull cursor, otherwise it parks forever. Gate it on an empty
2380
+ // buffer (msgCount == 0): a concurrent in-flight batch is still
2381
+ // in there until the listener returns — raising the offset before that would
2382
+ // silently skip the batch on a crash.
2383
+ _correctTagsOffset(mq, status, nextOffset) {
2384
+ if (status !== PullStatus.NO_NEW_MSG && status !== PullStatus.NO_MATCHED_MSG)
2385
+ return;
2386
+ const key = mqKey(mq);
2387
+ if (this._frozenOffsets.has(key))
2388
+ return;
2389
+ const pq = this.processQueueTable.get(key);
2390
+ if (pq && pq.msgCount() !== 0)
2391
+ return;
2392
+ if (!this.offsetStore)
2393
+ return;
2394
+ this.offsetStore.readOffset(mq, ReadOffsetMode.READ_FROM_MEMORY).then((cur) => {
2395
+ if (cur >= nextOffset)
2396
+ return;
2397
+ this.offsetStore.updateOffset(mq, nextOffset, true);
2398
+ }).catch(() => { });
2399
+ }
2400
+ // persistConsumerOffset is the periodic flush of every assigned
2401
+ // queue's consumed offset.
2402
+ async persistConsumerOffset() {
2403
+ const store = this.offsetStore;
2404
+ if (!store || !this.started)
2405
+ return;
2406
+ const mqs = [...this.processQueueTable.keys()]
2407
+ .map((key) => {
2408
+ const [topic, brokerName, queueId] = parseMqKey(key);
2409
+ return new MessageQueue(topic, brokerName, queueId);
2410
+ })
2411
+ .sort((a, b) => a.compareTo(b));
2412
+ await store.persistAll(mqs);
2413
+ }
2414
+ // getConsumerStatus answers GET_CONSUMER_STATUS_FROM_CLIENT(221): the
2415
+ // CONSUMED offsets (a snapshot of the offset table for the topic).
2416
+ getConsumerStatus(topic) {
2417
+ const out = [];
2418
+ if (!this.offsetStore)
2419
+ return out;
2420
+ const snap = this.offsetStore.tableSnapshot
2421
+ ? this.offsetStore.tableSnapshot() : new Map();
2422
+ const keys = Array.from(snap.keys()).sort();
2423
+ for (const key of keys) {
2424
+ const [t, brokerName, queueId] = parseMqKey(key);
2425
+ if (topic && t !== topic)
2426
+ continue;
2427
+ out.push({ mq: new MessageQueue(t, brokerName, queueId), offset: snap.get(key) });
2428
+ }
2429
+ return out;
2430
+ }
2431
+ // resetOffset handles RESET_CONSUMER_CLIENT_OFFSET(220).
2432
+ //
2433
+ // Four steps: drop and clear the ProcessQueue -> wait for concurrent
2434
+ // consumption to finish -> update the consumed offset and persist it now ->
2435
+ // drop the queue so rebalance rebuilds it at the new offset.
2436
+ async resetOffset(topic, table) {
2437
+ if (!table || table.length === 0)
2438
+ return;
2439
+ const todo = [];
2440
+ for (const entry of table) {
2441
+ if (topic && entry.mq.getTopic() !== topic)
2442
+ continue;
2443
+ if (!this.processQueueTable.has(mqKey(entry.mq)))
2444
+ continue;
2445
+ if (this.offsetStore)
2446
+ this.offsetStore.updateOffset(entry.mq, entry.offset, false);
2447
+ todo.push(entry);
2448
+ }
2449
+ const revoked = [];
2450
+ for (const p of todo) {
2451
+ const q = this._retireQueue(mqKey(p.mq));
2452
+ q.offset = p.offset;
2453
+ q.hasOffset = true;
2454
+ revoked.push(q);
2455
+ }
2456
+ if (revoked.length === 0)
2457
+ return;
2458
+ await this._sleepOrStop(200);
2459
+ this._persistRevoked(revoked);
2460
+ this.rebalanceImmediately();
2461
+ void this.doRebalance();
2462
+ logger.info('reset offset applied, group=%s topic=%s queues=%d', this.consumerGroup, topic, revoked.length);
2463
+ }
2464
+ // ------------------------------------------------------------- sweeps
2465
+ // _cleanExpireLoop runs the expire sweep on a fixed schedule: initialDelay
2466
+ // and period are the SAME, so the first sweep also waits a full period.
2467
+ async _cleanExpireLoop() {
2468
+ const flag = this._newStopFlag();
2469
+ const period = Math.max(1, this.consumeTimeout) * 60 * 1000;
2470
+ while (!this._isStopped(flag)) {
2471
+ if (await this._sleepOrStop(period, flag))
2472
+ return;
2473
+ if (!this.started)
2474
+ return;
2475
+ if (!this._beginInFlight())
2476
+ return; // shutdown froze new work
2477
+ try {
2478
+ this._cleanExpiredMsgOnce();
2479
+ }
2480
+ catch (e) {
2481
+ logger.error('scheduleAtFixedRate cleanExpireMsg exception: %s', e.message);
2482
+ }
2483
+ finally {
2484
+ this._endInFlight();
2485
+ }
2486
+ }
2487
+ }
2488
+ // _cleanExpiredMsgOnce walks the queues currently held and sweeps each.
2489
+ _cleanExpiredMsgOnce() {
2490
+ for (const mq of [...this.assigned]) {
2491
+ this._cleanExpiredQueue(mq);
2492
+ }
2493
+ }
2494
+ // _cleanExpiredQueue sweeps one queue's expired messages.
2495
+ // Three rules: only ever look at the HEAD (smallest offset), the message
2496
+ // must be STRICTLY older than consumeTimeout, and at most 16 per round.
2497
+ _cleanExpiredQueue(mq) {
2498
+ if (this.orderly)
2499
+ return; // orderly has no such path
2500
+ const pq = this.processQueueTable.get(mqKey(mq));
2501
+ if (!pq)
2502
+ return;
2503
+ const timeoutMillis = this.consumeTimeout * 60 * 1000;
2504
+ const sweep = (i) => {
2505
+ if (i >= 16)
2506
+ return;
2507
+ const head = pq.firstMessage();
2508
+ if (!head)
2509
+ return;
2510
+ const stamp = head.getProperty('CONSUME_START_TIME');
2511
+ if (!stamp)
2512
+ return; // never handed to the listener -> not expired
2513
+ const began = parseInt(stamp, 10);
2514
+ if (Number.isNaN(began))
2515
+ return;
2516
+ if (Date.now() - began <= timeoutMillis)
2517
+ return;
2518
+ this.sendMessageBack(head, 3).then(() => {
2519
+ logger.info('send expire msg back. topic=%s, msgId=%s, storeHost=%s, queueId=%d, queueOffset=%d', head.getTopic(), head.getMsgId(), head.getStoreHostString(), head.getQueueId(), head.getQueueOffset());
2520
+ // Remove only if it is STILL the head: a racing normal completion wins
2521
+ // and we must not steal its message.
2522
+ const nowHead = pq.firstMessage();
2523
+ if (nowHead && nowHead.getQueueOffset() === head.getQueueOffset()) {
2524
+ pq.removeMessage([head]);
2525
+ }
2526
+ sweep(i + 1);
2527
+ }).catch((e) => {
2528
+ // A failed send-back is logged only: the message stays where it is and
2529
+ // the next round retries. Removing it would lose it for good.
2530
+ logger.error('send expired msg exception: %s', e.message);
2531
+ });
2532
+ };
2533
+ sweep(0);
2534
+ }
2535
+ // ------------------------------------------------------------- lock sweep
2536
+ // _lockMQOnce issues LOCK_BATCH_MQ(41)
2537
+ // for every assigned queue.
2538
+ async _lockMQOnce() {
2539
+ const mqs = [...this.assigned];
2540
+ if (mqs.length === 0)
2541
+ return;
2542
+ const locked = await this._lockBatchMQ(mqs);
2543
+ const lockedSet = new Set(locked.map(mqKey));
2544
+ const now = Date.now();
2545
+ for (const [key, pq] of this.processQueueTable) {
2546
+ const granted = lockedSet.has(key);
2547
+ pq.setLocked(granted);
2548
+ // Stamp every grant: an old timestamp is what the pull gate reads as
2549
+ // "lock expired".
2550
+ if (granted)
2551
+ pq.setLastLockTimestamp(now);
2552
+ }
2553
+ logger.debug('lock_batch_mq: %d/%d queues locked', lockedSet.size, mqs.length);
2554
+ }
2555
+ async _lockLoop() {
2556
+ const flag = this._newStopFlag();
2557
+ while (!this._isStopped(flag)) {
2558
+ try {
2559
+ await this._lockMQOnce();
2560
+ }
2561
+ catch (e) {
2562
+ logger.debug('lock mq error: %s', e.message);
2563
+ }
2564
+ if (await this._sleepOrStop(20000, flag))
2565
+ return;
2566
+ }
2567
+ }
2568
+ async _lockBatchMQ(mqs) {
2569
+ const client = this.mqClient;
2570
+ if (!client)
2571
+ return [];
2572
+ const byBroker = new Map();
2573
+ for (const mq of mqs) {
2574
+ const list = byBroker.get(mq.getBrokerName()) || [];
2575
+ list.push(mq);
2576
+ byBroker.set(mq.getBrokerName(), list);
2577
+ }
2578
+ const locked = [];
2579
+ for (const [brokerName, group] of byBroker) {
2580
+ // Master only, and deliberately NO route refresh: an unlocatable master
2581
+ // means "this round cannot lock", and the next round retries.
2582
+ const addr = client.findBrokerAddressInPublish(brokerName);
2583
+ if (!addr)
2584
+ continue;
2585
+ const body = {
2586
+ consumerGroup: this.consumerGroup,
2587
+ clientId: this.clientID,
2588
+ mqSet: group.map((mq) => ({
2589
+ topic: mq.getTopic(), brokerName: mq.getBrokerName(), queueId: mq.getQueueId(),
2590
+ })),
2591
+ };
2592
+ const h = new LockBatchMqRequestHeader();
2593
+ h.consumerGroup = this.consumerGroup;
2594
+ h.clientId = this.clientID;
2595
+ const request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, h);
2596
+ request.body = Buffer.from(JSON.stringify(body), 'utf-8');
2597
+ try {
2598
+ const response = await client.remotingClient.invokeSync(client.vipChannel(addr), request, 1000);
2599
+ if (response.code !== ResponseCode.SUCCESS) {
2600
+ logger.warning('batch lock rejected by broker %s: CODE %s %s', brokerName, response.code, response.remark || '');
2601
+ continue;
2602
+ }
2603
+ const decoded = response.body && response.body.length
2604
+ ? RemotingSerializable.decode(response.body) : null;
2605
+ const okList = decoded && (decoded['lockOKMQSet'] || decoded['lockOKMqSet']);
2606
+ if (Array.isArray(okList)) {
2607
+ for (const q of okList) {
2608
+ locked.push(new MessageQueue(String(q['topic'] || ''), String(q['brokerName'] || ''), Number(q['queueId'] || 0)));
2609
+ }
2610
+ }
2611
+ }
2612
+ catch (e) {
2613
+ logger.warning('batch lock failed for broker %s: %s', brokerName, e.message);
2614
+ }
2615
+ }
2616
+ return locked;
2617
+ }
2618
+ async _unlockBatchMQ(mqs) {
2619
+ const client = this.mqClient;
2620
+ if (!client)
2621
+ return;
2622
+ const byBroker = new Map();
2623
+ for (const mq of mqs) {
2624
+ const list = byBroker.get(mq.getBrokerName()) || [];
2625
+ list.push(mq);
2626
+ byBroker.set(mq.getBrokerName(), list);
2627
+ }
2628
+ for (const [brokerName, group] of byBroker) {
2629
+ const addr = client.findBrokerAddressInPublish(brokerName);
2630
+ if (!addr)
2631
+ continue;
2632
+ const body = {
2633
+ consumerGroup: this.consumerGroup,
2634
+ clientId: this.clientID,
2635
+ mqSet: group.map((mq) => ({
2636
+ topic: mq.getTopic(), brokerName: mq.getBrokerName(), queueId: mq.getQueueId(),
2637
+ })),
2638
+ };
2639
+ const h = new UnlockBatchMqRequestHeader();
2640
+ h.consumerGroup = this.consumerGroup;
2641
+ h.clientId = this.clientID;
2642
+ const request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, h);
2643
+ request.body = Buffer.from(JSON.stringify(body), 'utf-8');
2644
+ try {
2645
+ await client.remotingClient.invokeSync(client.vipChannel(addr), request, 1000);
2646
+ }
2647
+ catch (e) {
2648
+ logger.debug('batch unlock failed for broker %s: %s', brokerName, e.message);
2649
+ }
2650
+ }
2651
+ }
2652
+ async unlockAssigned() {
2653
+ await this._unlockBatchMQ([...this.assigned]);
2654
+ }
2655
+ // ------------------------------------------------- broker-initiated RPCs
2656
+ // _registerProcessors wires the broker-initiated requests. Handlers run on
2657
+ // the remoting read path: set flags / schedule work, NEVER send a synchronous
2658
+ // request inline (self-deadlock).
2659
+ _registerProcessors() {
2660
+ const client = this.mqClient;
2661
+ if (!client)
2662
+ return;
2663
+ const groupOf = (cmd) => {
2664
+ const ext = cmd.extFields || {};
2665
+ return String(ext['consumerGroup'] != null ? ext['consumerGroup'] : (ext['group'] != null ? ext['group'] : ''));
2666
+ };
2667
+ const mine = (cmd) => {
2668
+ const g = groupOf(cmd);
2669
+ return g === '' || g === this.consumerGroup;
2670
+ };
2671
+ const handlers = new Map();
2672
+ handlers.set(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, (cmd) => {
2673
+ if (!mine(cmd))
2674
+ return null;
2675
+ this.rebalanceImmediately();
2676
+ return null;
2677
+ });
2678
+ handlers.set(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT, (cmd) => {
2679
+ // This one answers by group exactly: the offset table is looked up per
2680
+ // group, so a request naming another (or no) group is not this consumer's
2681
+ // to answer. The instance replies with an empty table when nobody owns it.
2682
+ const ext = cmd.extFields || {};
2683
+ if (String(ext['group'] || '') !== this.consumerGroup)
2684
+ return null;
2685
+ const topic = ext['topic'] || undefined;
2686
+ const response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
2687
+ // The body is a GetConsumerStatusBody OBJECT: the side that asked decodes it
2688
+ // into that type and reads `messageQueueTable`, so a bare list of entries —
2689
+ // or a `mq`/`offset` shape — decodes into nothing and the query silently
2690
+ // returns empty. The keys are the rendered MessageQueue objects.
2691
+ const body = new GetConsumerStatusBody();
2692
+ for (const entry of this.getConsumerStatus(topic)) {
2693
+ body.messageQueueTable.set(entry.mq, entry.offset);
2694
+ }
2695
+ response.body = body.encode();
2696
+ return response;
2697
+ });
2698
+ handlers.set(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, (cmd) => {
2699
+ if (!mine(cmd))
2700
+ return null;
2701
+ try {
2702
+ const body = cmd.body && cmd.body.length ? RemotingSerializable.decode(cmd.body) : null;
2703
+ const table = Array.isArray(body) ? body
2704
+ : (body && Array.isArray(body['offsetTable'])) ? body['offsetTable'] : [];
2705
+ const entries = table.map((e) => new MessageQueue(String(e['topic'] || e['queue']?.['topic'] || ''), String(e['brokerName'] || e['queue']?.['brokerName'] || ''), Number(e['queueId'] ?? e['queue']?.['queueId'] ?? 0))).filter((mq) => mq.getTopic()).map((mq, i) => ({
2706
+ mq, offset: Number(table[i]['offset'] ?? 0),
2707
+ }));
2708
+ const topic = (cmd.extFields || {})['topic'] || '';
2709
+ void this.resetOffset(topic, entries);
2710
+ }
2711
+ catch (e) {
2712
+ logger.warning('reset offset request failed: %s', e.message);
2713
+ }
2714
+ return RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
2715
+ });
2716
+ handlers.set(RequestCode.GET_CONSUMER_RUNNING_INFO, (cmd) => {
2717
+ if (!mine(cmd))
2718
+ return null;
2719
+ const h = new GetConsumerRunningInfoRequestHeader();
2720
+ try {
2721
+ h.fromExtFields(cmd.extFields || {});
2722
+ }
2723
+ catch (e) { /* defaults */ }
2724
+ const response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
2725
+ response.body = Buffer.from(encodeRunningInfoBody(this.buildConsumerRunningInfo(h.jstack === true)), 'utf-8');
2726
+ return response;
2727
+ });
2728
+ handlers.set(RequestCode.CONSUME_MESSAGE_DIRECTLY, (cmd) => {
2729
+ if (!mine(cmd))
2730
+ return null;
2731
+ // Consume one message directly and report the result. The message body
2732
+ // rides the request body (MessageExt wire) when the broker pushes it;
2733
+ // otherwise report a failure remark.
2734
+ const response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
2735
+ try {
2736
+ const msgs = cmd.body && cmd.body.length ? decodeMessages(cmd.body) : [];
2737
+ if (msgs.length === 0) {
2738
+ response.remark = 'no message to consume directly';
2739
+ return response;
2740
+ }
2741
+ this._resetRetryTopicAndNamespace(msgs);
2742
+ const ctx = new ConsumeConcurrentlyContext(new MessageQueue(msgs[0].getTopic(), msgs[0].getBrokerName(), msgs[0].getQueueId()));
2743
+ const { status } = this._listenerCall(msgs, ctx);
2744
+ response.body = Buffer.from(JSON.stringify({
2745
+ consumeResult: status === ConsumeConcurrentlyStatus.CONSUME_SUCCESS ? 'CONSUME_SUCCESS' : 'CONSUME_FAIL',
2746
+ remark: '',
2747
+ }), 'utf-8');
2748
+ }
2749
+ catch (e) {
2750
+ response.body = Buffer.from(JSON.stringify({
2751
+ consumeResult: 'CONSUME_FAIL', remark: e.message,
2752
+ }), 'utf-8');
2753
+ }
2754
+ return response;
2755
+ });
2756
+ for (const [code, handler] of handlers)
2757
+ client.registerFacadeProcessor(code, handler);
2758
+ this._facadeHandlers = handlers;
2759
+ }
2760
+ // Remove the reverse-request processors this facade registered on the shared instance (used by shutdown).
2761
+ _unregisterProcessors() {
2762
+ const client = this.mqClient;
2763
+ if (client != null) {
2764
+ for (const [code, handler] of this._facadeHandlers)
2765
+ client.unregisterFacadeProcessor(code, handler);
2766
+ }
2767
+ this._facadeHandlers = new Map();
2768
+ }
2769
+ // buildConsumerRunningInfo assembles the 307 payload: properties,
2770
+ // subscriptionSet, mqTable, statusTable, userConsumerInfo {}.
2771
+ buildConsumerRunningInfo(jstack = false) {
2772
+ void jstack;
2773
+ const mqTable = {};
2774
+ for (const [key, pq] of this.processQueueTable) {
2775
+ const [topic, brokerName, queueId] = parseMqKey(key);
2776
+ const mq = new MessageQueue(topic, brokerName, queueId);
2777
+ const [first, last, count] = pq.bufferedOffsetSpan();
2778
+ const [txMin, txMax, txCount] = pq.consumingStats();
2779
+ mqTable[messageQueueKeyJson(mq)] = {
2780
+ commitOffset: this.offsetTable.get(key) ?? -1,
2781
+ cachedMsgMinOffset: count > 0 ? first : 0,
2782
+ cachedMsgMaxOffset: count > 0 ? last : 0,
2783
+ cachedMsgCount: count,
2784
+ cachedMsgSizeInMiB: Math.floor(pq.msgSizeMB()),
2785
+ transactionMsgMinOffset: txCount > 0 ? txMin : 0,
2786
+ transactionMsgMaxOffset: txCount > 0 ? txMax : 0,
2787
+ transactionMsgCount: txCount,
2788
+ locked: pq.isLocked(),
2789
+ tryUnlockTimes: pq.tryUnlockTimes,
2790
+ lastLockTimestamp: pq.lastLockTimestampValue(),
2791
+ droped: pq.isDropped(),
2792
+ lastPullTimestamp: pq.lastPullTimestampValue(),
2793
+ lastConsumeTimestamp: pq.lastConsumeTimestamp || 0,
2794
+ };
2795
+ }
2796
+ const statusTable = this._buildStatusTable();
2797
+ const subscriptionSet = this.subscriptions().map((s) => s.toDict ? s.toDict() : s);
2798
+ return {
2799
+ properties: {
2800
+ 'PROP_CONSUME_TYPE': this.consumeType,
2801
+ 'PROP_START_TIMESTAMP': String(this._startTime),
2802
+ 'PROP_CONSUMEORDERLY': String(this.orderly),
2803
+ // PROP_THREADPOOL_CORE_SIZE reads the SERVICE's getCorePoolSize(),
2804
+ // not the config field — so the runtime value an operator pushed via
2805
+ // updateCorePoolSize() is what shows up in 307.
2806
+ 'PROP_THREADPOOL_CORE_SIZE': String(this.getCorePoolSize()),
2807
+ 'PROP_CONSUMER_START_TIMESTAMP': String(this._startTime),
2808
+ 'PROP_CONSUME_SUSPEND': String(false),
2809
+ 'PROP_CONSUMEFLOWCONTROL': String(this.pullThresholdForQueue),
2810
+ },
2811
+ subscriptionSet,
2812
+ mqTable,
2813
+ mqPopTable: this._buildPopTable(),
2814
+ statusTable,
2815
+ userConsumerInfo: {},
2816
+ };
2817
+ }
2818
+ // _buildPopTable is the 307 mqPopTable, one entry per POP process queue.
2819
+ // All three fields are written
2820
+ // unconditionally — unlike mqTable there is no "only when non-empty" branch,
2821
+ // so an idle POP queue still reports its debt and its last pop time.
2822
+ _buildPopTable() {
2823
+ const out = {};
2824
+ for (const [key, pq] of this.popQueueTable) {
2825
+ out[key] = {
2826
+ waitAckCount: pq.waitAckMsgCount(),
2827
+ droped: pq.isDropped(),
2828
+ lastPopTimestamp: pq.getLastPopTimestamp(),
2829
+ };
2830
+ }
2831
+ return out;
2832
+ }
2833
+ // _buildStatusTable is the 307 statusTable: per-topic ConsumeStatus from the
2834
+ // consumer stats manager (consumeStatus(group, topic), MINUTE snapshots;
2835
+ // consumeFailedMsgs comes from the HOUR window).
2836
+ // NOTE: real-clock TPS between two in-process samples is ~0; tests inject
2837
+ // timestamps via StatsItem sampling.
2838
+ _buildStatusTable() {
2839
+ const table = {};
2840
+ for (const s of this.subscriptions()) {
2841
+ if (this._statsManager != null) {
2842
+ table[s.topic] = this._statsManager.consumeStatus(this.consumerGroup, s.topic).toDict();
2843
+ }
2844
+ else {
2845
+ table[s.topic] = { pullRT: 0, pullTPS: 0, consumeRT: 0, consumeOKTPS: 0, consumeFailedTPS: 0, consumeFailedMsgs: 0 };
2846
+ }
2847
+ }
2848
+ return table;
2849
+ }
2850
+ // ------------------------------------------------------------- stats
2851
+ // Differential-window recording via the stats manager's same-named
2852
+ // methods. No-op before start(): the manager lives on the MQClient.
2853
+ _incPullRT(topic, rt) {
2854
+ if (this._statsManager != null)
2855
+ this._statsManager.incPullRT(this.consumerGroup, topic, rt);
2856
+ }
2857
+ _incPullTPS(topic, n) {
2858
+ if (this._statsManager != null)
2859
+ this._statsManager.incPullTPS(this.consumerGroup, topic, n);
2860
+ }
2861
+ _incConsumeRT(topic, rt) {
2862
+ if (this._statsManager != null)
2863
+ this._statsManager.incConsumeRT(this.consumerGroup, topic, rt);
2864
+ }
2865
+ _incConsumeOKTPS(topic, n) {
2866
+ if (this._statsManager != null)
2867
+ this._statsManager.incConsumeOKTPS(this.consumerGroup, topic, n);
2868
+ }
2869
+ _incConsumeFailedTPS(topic, n) {
2870
+ if (this._statsManager != null)
2871
+ this._statsManager.incConsumeFailedTPS(this.consumerGroup, topic, n);
2872
+ }
2873
+ // ------------------------------------------------------------- misc
2874
+ assignedQueues() { return [...this.assigned]; }
2875
+ assignedQueueCount() { return this.assigned.length; }
2876
+ processQueueCount() { return this.processQueueTable.size; }
2877
+ // ------------------------------------------------- thread elasticity
2878
+ // computeAccumulationTotal: sum of every
2879
+ // ProcessQueue's msgAccCnt (the backlog estimate adjustThreadPool runs on).
2880
+ computeAccumulationTotal() {
2881
+ let total = 0;
2882
+ for (const v of this._msgAccCnt.values())
2883
+ total += v;
2884
+ return total;
2885
+ }
2886
+ // updateCorePoolSize guards a rejected value strictly:
2887
+ // 0 < n <= Short.MAX_VALUE (32767) &&
2888
+ // n < consumeThreadMax; a rejected value is SILENTLY ignored.
2889
+ // The boolean exists only so tests can assert whether it took effect.
2890
+ //
2891
+ // It is not cosmetic here: _dispatchLoop caps in-flight batches at corePoolSize,
2892
+ // so this is the live concurrency knob.
2893
+ updateCorePoolSize(corePoolSize) {
2894
+ if (!Number.isFinite(corePoolSize))
2895
+ return false;
2896
+ if (corePoolSize <= 0 || corePoolSize > 32767)
2897
+ return false; // Short.MAX_VALUE
2898
+ if (corePoolSize >= this.consumeThreadMax)
2899
+ return false;
2900
+ this.corePoolSize = corePoolSize;
2901
+ return true;
2902
+ }
2903
+ // getCorePoolSize returns the live core size. There is no user-injected
2904
+ // executor in this port, so it never answers -1.
2905
+ getCorePoolSize() { return this.corePoolSize; }
2906
+ // adjustThreadPool is ⚠ a NO-OP on this port: the pool grow/shrink hooks it
2907
+ // would call have empty bodies, so no auto-resize ever takes effect. Keep it
2908
+ // a no-op; do not "fix" it. The effective knob is the explicit
2909
+ // updateCorePoolSize(). Thresholds are computed and logged so msgAccCnt and
2910
+ // the config stay observable.
2911
+ adjustThreadPool() {
2912
+ const accTotal = this.computeAccumulationTotal();
2913
+ const threshold = this.adjustThreadPoolNumsThreshold;
2914
+ if (accTotal >= threshold) {
2915
+ logger.debug('adjustThreadPool: acc=%d >= incThreshold=%d (inc is a no-op upstream)', accTotal, threshold);
2916
+ }
2917
+ if (accTotal < threshold * 0.8) {
2918
+ logger.debug('adjustThreadPool: acc=%d < decThreshold=%d (dec is a no-op upstream)', accTotal, Math.trunc(threshold * 0.8));
2919
+ }
2920
+ }
2921
+ // _checkConfigRanges validates the numeric consumer config — a fixed order,
2922
+ // fixed bounds, and the exact error strings the broker and callers match.
2923
+ _checkConfigRanges() {
2924
+ if (this.consumeThreadMin < 1 || this.consumeThreadMin > 1000) {
2925
+ throw new Error('consumeThreadMin Out of range [1, 1000]');
2926
+ }
2927
+ if (this.consumeThreadMax < 1 || this.consumeThreadMax > 1000) {
2928
+ throw new Error('consumeThreadMax Out of range [1, 1000]');
2929
+ }
2930
+ if (this.consumeThreadMin > this.consumeThreadMax) {
2931
+ throw new Error(`consumeThreadMin (${this.consumeThreadMin}) is larger than consumeThreadMax (${this.consumeThreadMax})`);
2932
+ }
2933
+ if (this.consumeConcurrentlyMaxSpan < 1 || this.consumeConcurrentlyMaxSpan > 65535) {
2934
+ throw new Error('consumeConcurrentlyMaxSpan Out of range [1, 65535]');
2935
+ }
2936
+ if (this.pullThresholdForQueue < 1 || this.pullThresholdForQueue > 65535) {
2937
+ throw new Error('pullThresholdForQueue Out of range [1, 65535]');
2938
+ }
2939
+ if (this.pullThresholdForTopic !== -1 && (this.pullThresholdForTopic < 1 || this.pullThresholdForTopic > 6553500)) {
2940
+ throw new Error('pullThresholdForTopic Out of range [1, 6553500]');
2941
+ }
2942
+ if (this.pullThresholdSizeForQueue < 1 || this.pullThresholdSizeForQueue > 1024) {
2943
+ throw new Error('pullThresholdSizeForQueue Out of range [1, 1024]');
2944
+ }
2945
+ if (this.pullThresholdSizeForTopic !== -1
2946
+ && (this.pullThresholdSizeForTopic < 1 || this.pullThresholdSizeForTopic > 102400)) {
2947
+ throw new Error('pullThresholdSizeForTopic Out of range [1, 102400]');
2948
+ }
2949
+ if (this.pullInterval < 0 || this.pullInterval > 65535) {
2950
+ throw new Error('pullInterval Out of range [0, 65535]');
2951
+ }
2952
+ if (!Number.isFinite(this.pullRetryDelayMillis) || this.pullRetryDelayMillis < 0) {
2953
+ throw new Error('pullRetryDelayMillis must be a finite number >= 0');
2954
+ }
2955
+ if (this.consumeMessageBatchMaxSize < 1 || this.consumeMessageBatchMaxSize > 1024) {
2956
+ throw new Error('consumeMessageBatchMaxSize Out of range [1, 1024]');
2957
+ }
2958
+ if (this.pullBatchSize < 1 || this.pullBatchSize > 1024) {
2959
+ throw new Error('pullBatchSize Out of range [1, 1024]');
2960
+ }
2961
+ if (this.popInvisibleTime < 5000 || this.popInvisibleTime > 300000) {
2962
+ throw new Error('popInvisibleTime Out of range [5000, 300000]');
2963
+ }
2964
+ if (this.popBatchNums <= 0 || this.popBatchNums > 32) {
2965
+ throw new Error('popBatchNums Out of range [1, 32]');
2966
+ }
2967
+ if (this.popMode && this.orderly) {
2968
+ // The POP protocol has no queue lock to serialise on, so an orderly
2969
+ // POP consumer would silently lose ordering — refuse instead.
2970
+ throw new Error('pop mode does not support orderly consumption');
2971
+ }
2972
+ }
2973
+ // ------------------------------------------------------------- pop surface
2974
+ popProcessQueueCount() { return this.popQueueTable.size; }
2975
+ // popWaitAckCount totals the outstanding ACK debt across every queue.
2976
+ popWaitAckCount() {
2977
+ let total = 0;
2978
+ for (const pq of this.popQueueTable.values())
2979
+ total += pq.waitAckMsgCount();
2980
+ return total;
2981
+ }
2982
+ // viewMessage is Java DefaultMQPushConsumerImpl#viewMessage
2983
+ // (DefaultMQPushConsumerImpl.java:1329) parity: the MQAdminImpl single-leg
2984
+ // path — offset msgId only, no uniq-key fallback at this layer.
2985
+ async viewMessage(topic, msgId) {
2986
+ const client = this.mqClient;
2987
+ if (!client)
2988
+ throw new Error('consumer not started');
2989
+ return client.viewMessage(topic, msgId);
2990
+ }
2991
+ }
2992
+ // ---------------------------------------------------------------- helpers
2993
+ // batchEnd is "the largest offset in the batch + 1" — the orderly path's
2994
+ // commit target, and the concurrent path's fallback when there is no buffer.
2995
+ function batchEnd(batch) {
2996
+ let next = -1;
2997
+ for (const msg of batch) {
2998
+ if (msg.getQueueOffset() > next)
2999
+ next = msg.getQueueOffset();
3000
+ }
3001
+ return next + 1;
3002
+ }
3003
+ // clientSideTagFilter is the second, string-level tag check applied to every
3004
+ // pull result. The broker filters by the hash of the tag
3005
+ // (codeSet), so a collision can let a non-matching message through; the client
3006
+ // re-checks by exact string.
3007
+ //
3008
+ // The guard `!tagsSet.isEmpty() && !isClassFilterMode` is why FilterAPI's
3009
+ // SUB_ALL path must keep tagsSet EMPTY: subscribing to "*" turns the filter
3010
+ // off entirely rather than filtering everything out.
3011
+ function clientSideTagFilter(sub, msgs) {
3012
+ // tagsSet is a Set in this port — use .size, NOT .length.
3013
+ // A `.length === 0` guard on a Set is always false, which silently turns
3014
+ // the '*' subscription (empty tagsSet → filter OFF) into "filter everything".
3015
+ const tagsSet = sub ? sub.tagsSet : null;
3016
+ const empty = tagsSet == null ||
3017
+ (typeof tagsSet.size === 'number' ? tagsSet.size === 0 : tagsSet.length === 0);
3018
+ if (msgs.length === 0 || !sub || empty || sub.classFilterMode) {
3019
+ return msgs;
3020
+ }
3021
+ const accepted = typeof tagsSet.has === 'function'
3022
+ ? tagsSet : new Set(tagsSet);
3023
+ const out = [];
3024
+ for (const msg of msgs) {
3025
+ const tags = msg.getProperty(MessageConst.PROPERTY_TAGS);
3026
+ if (!tags)
3027
+ continue;
3028
+ if (accepted.has(tags))
3029
+ out.push(msg);
3030
+ }
3031
+ return out;
3032
+ }
3033
+ // encodeRunningInfoBody renders the 307 payload. mqTable is keyed by inline
3034
+ // JSON MessageQueue objects, which JSON.stringify cannot produce (it would
3035
+ // quote and escape the key text), so that one table is spliced in raw exactly
3036
+ // like the local offset file's table.
3037
+ export function encodeRunningInfoBody(info) {
3038
+ const head = { ...info };
3039
+ const table = head['mqTable'] || {};
3040
+ delete head['mqTable'];
3041
+ const entries = [];
3042
+ for (const [key, value] of Object.entries(table)) {
3043
+ entries.push(`${key}:${JSON.stringify(value)}`);
3044
+ }
3045
+ const base = JSON.stringify(head);
3046
+ const body = base.slice(0, base.length - 1);
3047
+ const sep = body.length > 1 ? ',' : '';
3048
+ return `${body}${sep}"mqTable":{${entries.join(',')}}}`;
3049
+ }
3050
+ // parseMqKey reverses the canonical "topic@brokerName@queueId" key.
3051
+ export function parseMqKey(key) {
3052
+ const i1 = key.indexOf('@');
3053
+ const i2 = key.lastIndexOf('@');
3054
+ if (i1 < 0 || i2 <= i1)
3055
+ return ['', '', 0];
3056
+ return [key.slice(0, i1), key.slice(i1 + 1, i2), parseInt(key.slice(i2 + 1), 10) || 0];
3057
+ }
3058
+ // queuesByTopicKeySet groups one assignment by topic as a SET of queue keys.
3059
+ function queuesByTopicKeySet(mqs) {
3060
+ const out = new Map();
3061
+ for (const mq of mqs) {
3062
+ let set = out.get(mq.getTopic());
3063
+ if (!set) {
3064
+ set = new Set();
3065
+ out.set(mq.getTopic(), set);
3066
+ }
3067
+ set.add(mqKey(mq));
3068
+ }
3069
+ return out;
3070
+ }
3071
+ // changedTopics lists, sorted, the topics whose SET of assigned queues differs
3072
+ // between two rebalance rounds. Comparing sizes per topic is not enough: losing one
3073
+ // queue and gaining another keeps the count while changing who owns what. A topic
3074
+ // that appeared or vanished entirely counts too, because its queues moved.
3075
+ function changedTopics(before, after) {
3076
+ const moved = [];
3077
+ for (const [topic, nowQueues] of after) {
3078
+ const wasQueues = before.get(topic);
3079
+ if (!wasQueues || wasQueues.size !== nowQueues.size || !isSubsetOf(wasQueues, nowQueues)) {
3080
+ moved.push(topic);
3081
+ }
3082
+ }
3083
+ for (const [topic, wasQueues] of before) {
3084
+ if (!after.has(topic) && wasQueues.size > 0)
3085
+ moved.push(topic);
3086
+ }
3087
+ return moved.sort();
3088
+ }
3089
+ function isSubsetOf(a, b) {
3090
+ for (const v of a) {
3091
+ if (!b.has(v))
3092
+ return false;
3093
+ }
3094
+ return true;
3095
+ }