rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/README.en.md +428 -0
  2. package/README.md +424 -2
  3. package/dist/client/admin.d.ts +229 -0
  4. package/dist/client/admin.js +2462 -0
  5. package/dist/client/allocate.d.ts +48 -0
  6. package/dist/client/allocate.js +365 -0
  7. package/dist/client/backpressure.d.ts +23 -0
  8. package/dist/client/backpressure.js +95 -0
  9. package/dist/client/consumer.d.ts +294 -0
  10. package/dist/client/consumer.js +3095 -0
  11. package/dist/client/consumer_result.d.ts +129 -0
  12. package/dist/client/consumer_result.js +138 -0
  13. package/dist/client/consumer_stats.d.ts +69 -0
  14. package/dist/client/consumer_stats.js +258 -0
  15. package/dist/client/exception.d.ts +39 -0
  16. package/dist/client/exception.js +36 -0
  17. package/dist/client/hook.d.ts +145 -0
  18. package/dist/client/hook.js +171 -0
  19. package/dist/client/latency.d.ts +61 -0
  20. package/dist/client/latency.js +239 -0
  21. package/dist/client/lite_pull_consumer.d.ts +151 -0
  22. package/dist/client/lite_pull_consumer.js +1096 -0
  23. package/dist/client/mq_client.d.ts +188 -0
  24. package/dist/client/mq_client.js +1678 -0
  25. package/dist/client/mq_client_manager.d.ts +21 -0
  26. package/dist/client/mq_client_manager.js +129 -0
  27. package/dist/client/offset_store.d.ts +45 -0
  28. package/dist/client/offset_store.js +389 -0
  29. package/dist/client/open_tracing_hook.d.ts +52 -0
  30. package/dist/client/open_tracing_hook.js +241 -0
  31. package/dist/client/pop_api.d.ts +28 -0
  32. package/dist/client/pop_api.js +224 -0
  33. package/dist/client/pop_consumer.d.ts +23 -0
  34. package/dist/client/pop_consumer.js +719 -0
  35. package/dist/client/pop_process_queue.d.ts +15 -0
  36. package/dist/client/pop_process_queue.js +51 -0
  37. package/dist/client/process_queue.d.ts +49 -0
  38. package/dist/client/process_queue.js +261 -0
  39. package/dist/client/produce_accumulator.d.ts +75 -0
  40. package/dist/client/produce_accumulator.js +288 -0
  41. package/dist/client/producer.d.ts +231 -0
  42. package/dist/client/producer.js +1774 -0
  43. package/dist/client/pull_api.d.ts +26 -0
  44. package/dist/client/pull_api.js +264 -0
  45. package/dist/client/pull_consumer.d.ts +119 -0
  46. package/dist/client/pull_consumer.js +876 -0
  47. package/dist/client/pull_schedule_service.d.ts +71 -0
  48. package/dist/client/pull_schedule_service.js +424 -0
  49. package/dist/client/request_reply.d.ts +61 -0
  50. package/dist/client/request_reply.js +232 -0
  51. package/dist/client/retry_message.d.ts +2 -0
  52. package/dist/client/retry_message.js +27 -0
  53. package/dist/client/send_result.d.ts +57 -0
  54. package/dist/client/send_result.js +70 -0
  55. package/dist/client/top_addressing.d.ts +20 -0
  56. package/dist/client/top_addressing.js +75 -0
  57. package/dist/client/trace_context.d.ts +61 -0
  58. package/dist/client/trace_context.js +373 -0
  59. package/dist/client/trace_dispatcher.d.ts +31 -0
  60. package/dist/client/trace_dispatcher.js +236 -0
  61. package/dist/client/trace_hook.d.ts +23 -0
  62. package/dist/client/trace_hook.js +228 -0
  63. package/dist/client/traceparent.d.ts +9 -0
  64. package/dist/client/traceparent.js +86 -0
  65. package/dist/common/boundary_type.d.ts +7 -0
  66. package/dist/common/boundary_type.js +16 -0
  67. package/dist/common/compress.d.ts +28 -0
  68. package/dist/common/compress.js +512 -0
  69. package/dist/common/message.d.ts +129 -0
  70. package/dist/common/message.js +214 -0
  71. package/dist/common/messageClientIdSetter.d.ts +7 -0
  72. package/dist/common/messageClientIdSetter.js +48 -0
  73. package/dist/common/messageConst.d.ts +68 -0
  74. package/dist/common/messageConst.js +68 -0
  75. package/dist/common/messageDecoder.d.ts +40 -0
  76. package/dist/common/messageDecoder.js +504 -0
  77. package/dist/common/messageType.d.ts +11 -0
  78. package/dist/common/messageType.js +11 -0
  79. package/dist/common/message_accessor.d.ts +39 -0
  80. package/dist/common/message_accessor.js +135 -0
  81. package/dist/common/mixAll.d.ts +106 -0
  82. package/dist/common/mixAll.js +247 -0
  83. package/dist/common/recall_message_handle.d.ts +25 -0
  84. package/dist/common/recall_message_handle.js +98 -0
  85. package/dist/common/subscriptionData.d.ts +36 -0
  86. package/dist/common/subscriptionData.js +64 -0
  87. package/dist/common/sysflag.d.ts +129 -0
  88. package/dist/common/sysflag.js +96 -0
  89. package/dist/common/topic_config.d.ts +66 -0
  90. package/dist/common/topic_config.js +106 -0
  91. package/dist/common/topic_validator.d.ts +49 -0
  92. package/dist/common/topic_validator.js +115 -0
  93. package/dist/common/utilAll.d.ts +39 -0
  94. package/dist/common/utilAll.js +154 -0
  95. package/dist/common/validators.d.ts +25 -0
  96. package/dist/common/validators.js +80 -0
  97. package/dist/index.d.ts +24 -0
  98. package/dist/index.js +36 -0
  99. package/dist/logging.d.ts +24 -0
  100. package/dist/logging.js +246 -0
  101. package/dist/remoting/acl.d.ts +16 -0
  102. package/dist/remoting/acl.js +39 -0
  103. package/dist/remoting/admin_body.d.ts +257 -0
  104. package/dist/remoting/admin_body.js +669 -0
  105. package/dist/remoting/bodies.d.ts +475 -0
  106. package/dist/remoting/bodies.js +1131 -0
  107. package/dist/remoting/client.d.ts +73 -0
  108. package/dist/remoting/client.js +591 -0
  109. package/dist/remoting/codes.d.ts +296 -0
  110. package/dist/remoting/codes.js +301 -0
  111. package/dist/remoting/exception.d.ts +45 -0
  112. package/dist/remoting/exception.js +58 -0
  113. package/dist/remoting/extra_info.d.ts +61 -0
  114. package/dist/remoting/extra_info.js +220 -0
  115. package/dist/remoting/headers.d.ts +758 -0
  116. package/dist/remoting/headers.js +1307 -0
  117. package/dist/remoting/heartbeat.d.ts +91 -0
  118. package/dist/remoting/heartbeat.js +227 -0
  119. package/dist/remoting/namespace.d.ts +14 -0
  120. package/dist/remoting/namespace.js +85 -0
  121. package/dist/remoting/pop_bodies.d.ts +58 -0
  122. package/dist/remoting/pop_bodies.js +214 -0
  123. package/dist/remoting/remotingCommand.d.ts +39 -0
  124. package/dist/remoting/remotingCommand.js +225 -0
  125. package/dist/remoting/route.d.ts +61 -0
  126. package/dist/remoting/route.js +288 -0
  127. package/dist/remoting/rpc_hooks.d.ts +31 -0
  128. package/dist/remoting/rpc_hooks.js +73 -0
  129. package/dist/remoting/serialize.d.ts +33 -0
  130. package/dist/remoting/serialize.js +372 -0
  131. package/dist/remoting/socks_proxy.d.ts +29 -0
  132. package/dist/remoting/socks_proxy.js +379 -0
  133. package/dist/remoting/static_topic.d.ts +49 -0
  134. package/dist/remoting/static_topic.js +244 -0
  135. package/dist/remoting/subscription.d.ts +65 -0
  136. package/dist/remoting/subscription.js +172 -0
  137. package/package.json +38 -4
@@ -0,0 +1,1096 @@
1
+ // DefaultLitePullConsumer — the lite pull consumer.
2
+ //
3
+ // Two assignment modes:
4
+ // - subscribe(topic, expr): the assignment is rebalance-driven (AVG over the
5
+ // group's consumer list).
6
+ // - assign(mqs): fully manual, no broker coordination.
7
+ //
8
+ // The pull is always a SHORT poll via LITE_PULL_MESSAGE(361) with no inline
9
+ // offset commit (the consumer owns its own commit path), and the expression is
10
+ // ALWAYS sent (this port never drops the subscription field on lite pull).
11
+ import { MQClient, DEFAULT_CONCURRENT_HEARTBEAT_THREAD_POOL_SIZE } from "./mq_client.js";
12
+ import { MQClientManager } from "./mq_client_manager.js";
13
+ import { PullAPI } from "./pull_api.js";
14
+ import { registerRpcHooks } from "../remoting/rpc_hooks.js";
15
+ import { LocalFileOffsetStore, RemoteBrokerOffsetStore, ReadOffsetMode, mqKey, } from "./offset_store.js";
16
+ import { PullStatus } from "./consumer_result.js";
17
+ import { MessageModel, ConsumeType, ConsumeFromWhere } from "../remoting/heartbeat.js";
18
+ import { MessageQueue } from "../common/message.js";
19
+ import { AllocateMessageQueueAveragely } from "./allocate.js";
20
+ import { MixAll } from "../common/mixAll.js";
21
+ import { FilterAPI } from "../common/subscriptionData.js";
22
+ import { PullSysFlag } from "../common/sysflag.js";
23
+ import { ResponseCode } from "../remoting/codes.js";
24
+ import { MQBrokerException } from "../remoting/exception.js";
25
+ import { PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL } from "./consumer.js";
26
+ import { Validators } from "../common/validators.js";
27
+ import { defaultConsumeTimestamp, parseConsumeTimestamp } from "../common/utilAll.js";
28
+ import { getLogger } from "../logging.js";
29
+ const logger = getLogger('client.lite_pull_consumer');
30
+ const DEFAULT_PULL_BATCH_SIZE = 10;
31
+ const DEFAULT_POLL_TIMEOUT_MILLIS = 5000;
32
+ // Shortest auto-commit interval setAutoCommitIntervalMillis accepts; below it a setting is ignored.
33
+ const MIN_AUTO_COMMIT_INTERVAL_MILLIS = 1000;
34
+ // Single queue, nothing found: the wait between two rounds of the same poll()
35
+ // call, so an idle queue does not hot-loop against the broker.
36
+ const IDLE_POLL_RETRY_DELAY_MILLIS = 200;
37
+ // Default value of topicMetadataCheckIntervalMillis: the watched topic's queue
38
+ // set is compared once every 30s.
39
+ const DEFAULT_TOPIC_METADATA_CHECK_INTERVAL_MILLIS = 30 * 1000;
40
+ // The period is clamped to a 1s floor: a timer with a 0 period makes the
41
+ // background scheduler fail outright.
42
+ const MIN_TOPIC_METADATA_CHECK_INTERVAL_MILLIS = 1000;
43
+ // The first comparison only runs 10s after start(), not immediately.
44
+ const METADATA_CHECK_FIRST_DELAY_MILLIS = 1000 * 10;
45
+ // Queue-set equality: "never seen this topic before" (no snapshot) is not equal
46
+ // to any set — fetchMessageQueues always returns a non-empty list, so the first
47
+ // round after registering a listener always fires once.
48
+ function isQueueSetEqual(oldQueues, newQueues) {
49
+ if (oldQueues === undefined)
50
+ return false;
51
+ if (oldQueues.length !== newQueues.length)
52
+ return false;
53
+ const oldSet = new Set(oldQueues.map(mqKey));
54
+ return newQueues.every((mq) => oldSet.has(mqKey(mq)));
55
+ }
56
+ // litePullParkMillisFor maps a FAILED lite pull round to how long THIS queue is
57
+ // parked before poll() asks it again:
58
+ // * the broker's own flow-control answer (215): the short pause — nothing was
59
+ // handed to the application and the queue is readable again as soon as the
60
+ // broker's pressure drops;
61
+ // * every other code (and a failure that never reached the wire, which carries
62
+ // no code at all): 0, i.e. no park — that is today's behaviour and it stays
63
+ // whatever the port already does for a failed lite pull.
64
+ //
65
+ // The park is per queue: a throttled queue is stepped over while its siblings are
66
+ // still served, exactly like a queue the application paused.
67
+ export function litePullParkMillisFor(responseCode) {
68
+ return responseCode === ResponseCode.FLOW_CONTROL
69
+ ? PULL_TIME_DELAY_MILLS_WHEN_BROKER_FLOW_CONTROL : 0;
70
+ }
71
+ export class DefaultLitePullConsumer {
72
+ consumerGroup;
73
+ // namespace: the client-side prefix (`NS%Topic`). The group, every topic and
74
+ // every queue this consumer talks to carry it on the wire; delivered messages
75
+ // and queue-change callbacks hand the bare names back.
76
+ namespace = '';
77
+ // namespaceV2: the server-side namespace; when non-empty NamespaceRpcHook tags
78
+ // every request with nsd=true / ns=<value>.
79
+ namespaceV2 = null;
80
+ nameServerAddr = '';
81
+ messageModel = MessageModel.CLUSTERING;
82
+ // wire fields read by MQClient.buildHeartbeatData:
83
+ consumeType = ConsumeType.CONSUME_ACTIVELY;
84
+ // The topics assigned in ASSIGN mode are not a topic-wide subscription, so they
85
+ // register with the broker only when the application opts in through
86
+ // buildSubscriptionsForHeartbeat(); subscribe() mode's topics always ride the
87
+ // heartbeat, and a topic both cover keeps the rebalance expression.
88
+ get subscriptionDataSet() {
89
+ const subs = [...this.subscription.values()];
90
+ for (const [topic, sub] of this.heartbeatSubscriptions) {
91
+ if (!this.subscription.has(topic))
92
+ subs.push(sub);
93
+ }
94
+ return subs;
95
+ }
96
+ // The extra set announced on top of the rebalance subscriptions.
97
+ heartbeatSubscriptions = new Map();
98
+ unitMode = false;
99
+ // instanceName / unitName / enableStreamRequestType: lite pull turns the stream
100
+ // marker on by default — `@STREAM` is appended to the clientId and every
101
+ // request carries ReqT=0.
102
+ instanceName = 'DEFAULT';
103
+ unitName = '';
104
+ enableStreamRequestType = true;
105
+ clientID = '';
106
+ mqClient = null;
107
+ pullAPI = null;
108
+ offsetStore = null;
109
+ pullBatchSize = DEFAULT_PULL_BATCH_SIZE;
110
+ pollTimeoutMillis = DEFAULT_POLL_TIMEOUT_MILLIS;
111
+ autoCommit = true;
112
+ autoCommitIntervalMillis = 5000;
113
+ // Default CONSUME_FROM_LAST_OFFSET: a queue with no committed offset starts at
114
+ // the broker's maxOffset.
115
+ consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET;
116
+ // The start instant of CONSUME_FROM_TIMESTAMP, in "yyyyMMddHHmmss". The default is boot time minus
117
+ // 30 minutes: an empty value would fail at start(), and silently falling back to "now" would turn
118
+ // the policy into "skip the backlog".
119
+ consumeTimestamp = defaultConsumeTimestamp();
120
+ // The period at which a background task compares the queue set of every
121
+ // watched topic.
122
+ topicMetadataCheckIntervalMillis = DEFAULT_TOPIC_METADATA_CHECK_INTERVAL_MILLIS;
123
+ // Offset flush period (default 5000 ms): applied to
124
+ // the shared MQClient instance when THIS facade creates it; drives the
125
+ // instance-level persistAllConsumerOffset sweep (which calls this facade's
126
+ // persistConsumerOffset -> commitSync).
127
+ persistConsumerOffsetInterval = 5000;
128
+ // Heartbeat V2: off by default, so heartbeats keep the V1 shape unless enabled.
129
+ useHeartbeatV2 = false;
130
+ // Materialise received message bodies during batch decode (default true).
131
+ decodeReadBody = true;
132
+ // Internal-API timeout (heartbeat, route fetch, check client, unregister and
133
+ // consumer-list RPCs), in milliseconds. Default 3000.
134
+ mqClientApiTimeout = 3000;
135
+ // Concurrent heartbeat round (ignored while heartbeat V2 is on).
136
+ enableConcurrentHeartbeat = false;
137
+ concurrentHeartbeatThreadPoolSize = DEFAULT_CONCURRENT_HEARTBEAT_THREAD_POOL_SIZE;
138
+ // Page size of the paged metadata admin calls (maxTopicNum / maxGroupNum).
139
+ maxPageSizeInGetMetadata = 2000;
140
+ // Queue allocation used by subscribe() mode.
141
+ allocateMessageQueueStrategy = new AllocateMessageQueueAveragely();
142
+ // Long-polling bounds. They are validated at start(); a consumer configured
143
+ // to time out before the broker's suspend window would give up on every
144
+ // parked pull.
145
+ brokerSuspendMaxTimeMillis = 20000;
146
+ consumerTimeoutMillisWhenSuspend = 30000;
147
+ // Pull-from-which-node pin (Java DefaultLitePullConsumer#setConnectBrokerByUser /
148
+ // #setDefaultBrokerId, backed by PullAPIWrapper). With the switch on, every pull round
149
+ // asks defaultBrokerId and ignores the broker's suggestWhichBrokerId -- the only way to
150
+ // read from a slave. Held here rather than only on PullAPI because the setters are legal
151
+ // before start(), and PullAPI is built inside start().
152
+ connectBrokerByUser = false;
153
+ defaultBrokerId = MixAll.MASTER_ID;
154
+ setConsumeFromWhere(where) { this.consumeFromWhere = where; return this; }
155
+ // The CONSUME_FROM_TIMESTAMP start instant ("yyyyMMddHHmmss"); the format is checked at start().
156
+ setConsumeTimestamp(ts) { this.consumeTimestamp = ts; return this; }
157
+ getConsumeTimestamp() { return this.consumeTimestamp; }
158
+ subscription = new Map();
159
+ // Which of subscribe()/assign() this consumer claimed. Whichever came first
160
+ // holds; the other is refused, so an assignment is never silently switched out
161
+ // from under the queues a caller already pinned.
162
+ mode = 'none';
163
+ assignedManual = null; // assign() mode
164
+ assignedAuto = []; // subscribe() mode
165
+ // ASSIGN mode's per-topic expression. The caller names it with
166
+ // setSubExpressionForAssign(); assign() seeds '*' for a topic nobody named yet,
167
+ // which is the default the pull task builds when no expression was set.
168
+ assignSubscription = new Map();
169
+ paused = new Set();
170
+ pullCursor = new Map(); // mqKey -> next pull offset
171
+ // mqKey -> the instant (epoch ms) until which poll() skips this queue, armed by
172
+ // the broker's own flow-control answer (215). Keyed exactly like the other
173
+ // per-queue state, and read by every round of poll().
174
+ pullParkUntil = new Map();
175
+ // Each topic's listener plus the queue set last seen.
176
+ // The key is the **bare** topic, consistent with this implementation's other
177
+ // lite pull state (subscribe / route cache / rebalance) — namespaceV2 takes
178
+ // effect at the wire layer via NamespaceRpcHook rather than by prefixing topic
179
+ // names.
180
+ topicChangeListeners = new Map();
181
+ messageQueuesForTopic = new Map();
182
+ started = false;
183
+ // Start state machine: a consumer can only be started from CREATE_JUST.
184
+ // Every other state refuses, so a second start() can never run a half-built
185
+ // second incarnation.
186
+ _serviceState = 'CREATE_JUST';
187
+ _rebalanceTimer = null;
188
+ _autoCommitTimer = null;
189
+ _metadataTimer = null;
190
+ _metadataInitialTimer = null;
191
+ constructor(consumerGroup = 'DEFAULT_CONSUMER') {
192
+ this.consumerGroup = consumerGroup || 'DEFAULT_CONSUMER';
193
+ }
194
+ setNamesrvAddr(addr) { this.nameServerAddr = addr; return this; }
195
+ // Sets/reads the client-side namespace prefix.
196
+ setNamespace(ns) { this.namespace = ns || ''; return this; }
197
+ getNamespace() { return this.namespace; }
198
+ // Sets/reads namespaceV2 (read live on every request).
199
+ setNamespaceV2(ns) { this.namespaceV2 = ns; return this; }
200
+ getNamespaceV2() { return this.namespaceV2; }
201
+ setMessageModel(model) { this.messageModel = model; return this; }
202
+ setInstanceName(name) { this.instanceName = name; return this; }
203
+ setUnitName(name) { this.unitName = name; return this; }
204
+ setPullBatchSize(n) { this.pullBatchSize = n; return this; }
205
+ setPollTimeoutMillis(ms) { this.pollTimeoutMillis = ms; return this; }
206
+ setAutoCommit(enable) { this.autoCommit = enable; return this; }
207
+ // Set the auto-commit interval. A value below the floor is ignored outright, so the
208
+ // interval that was already running keeps running instead of being pulled down to the floor.
209
+ setAutoCommitIntervalMillis(ms) {
210
+ if (Number.isFinite(ms) && ms >= MIN_AUTO_COMMIT_INTERVAL_MILLIS)
211
+ this.autoCommitIntervalMillis = ms;
212
+ return this;
213
+ }
214
+ // Flush period for consumed offsets, in milliseconds; must be greater than 0.
215
+ // When already started the change is pushed through to the shared instance's
216
+ // sweep timer immediately.
217
+ setPersistConsumerOffsetInterval(ms) {
218
+ if (!Number.isFinite(ms) || ms <= 0)
219
+ throw new Error(`persistConsumerOffsetInterval must be > 0, got ${ms}`);
220
+ this.persistConsumerOffsetInterval = Math.floor(ms);
221
+ if (this.mqClient)
222
+ this.mqClient.setPersistConsumerOffsetInterval(this.persistConsumerOffsetInterval);
223
+ return this;
224
+ }
225
+ getPersistConsumerOffsetInterval() { return this.persistConsumerOffsetInterval; }
226
+ // Heartbeat V2 switch; pushed through to
227
+ // the shared instance when this facade already owns one.
228
+ setUseHeartbeatV2(enable) {
229
+ this.useHeartbeatV2 = enable === true;
230
+ if (this.mqClient)
231
+ this.mqClient.setUseHeartbeatV2(this.useHeartbeatV2);
232
+ return this;
233
+ }
234
+ getUseHeartbeatV2() { return this.useHeartbeatV2; }
235
+ // Received-body decode switch, mirrored onto the shared instance.
236
+ setDecodeReadBody(enable) {
237
+ this.decodeReadBody = enable === true;
238
+ if (this.mqClient)
239
+ this.mqClient.setDecodeReadBody(this.decodeReadBody);
240
+ return this;
241
+ }
242
+ getDecodeReadBody() { return this.decodeReadBody; }
243
+ // Internal-API timeout, in milliseconds. An explicit per-call timeout passed
244
+ // by a caller still wins.
245
+ setMqClientApiTimeout(ms) {
246
+ if (!Number.isFinite(ms) || ms <= 0)
247
+ throw new Error(`mqClientApiTimeout must be > 0, got ${ms}`);
248
+ this.mqClientApiTimeout = Math.floor(ms);
249
+ if (this.mqClient)
250
+ this.mqClient.setMqClientApiTimeout(this.mqClientApiTimeout);
251
+ return this;
252
+ }
253
+ getMqClientApiTimeout() { return this.mqClientApiTimeout; }
254
+ // Concurrent heartbeat round; heartbeat V2 still wins when it is enabled.
255
+ setEnableConcurrentHeartbeat(enable) {
256
+ this.enableConcurrentHeartbeat = enable === true;
257
+ if (this.mqClient)
258
+ this.mqClient.setEnableConcurrentHeartbeat(this.enableConcurrentHeartbeat);
259
+ return this;
260
+ }
261
+ getEnableConcurrentHeartbeat() { return this.enableConcurrentHeartbeat; }
262
+ setConcurrentHeartbeatThreadPoolSize(size) {
263
+ if (!Number.isInteger(size) || size <= 0) {
264
+ throw new Error(`concurrentHeartbeatThreadPoolSize must be a positive integer, got ${size}`);
265
+ }
266
+ this.concurrentHeartbeatThreadPoolSize = size;
267
+ if (this.mqClient)
268
+ this.mqClient.setConcurrentHeartbeatThreadPoolSize(size);
269
+ return this;
270
+ }
271
+ getConcurrentHeartbeatThreadPoolSize() { return this.concurrentHeartbeatThreadPoolSize; }
272
+ setMaxPageSizeInGetMetadata(size) {
273
+ if (!Number.isInteger(size) || size <= 0) {
274
+ throw new Error(`maxPageSizeInGetMetadata must be a positive integer, got ${size}`);
275
+ }
276
+ this.maxPageSizeInGetMetadata = size;
277
+ if (this.mqClient)
278
+ this.mqClient.setMaxPageSizeInGetMetadata(size);
279
+ return this;
280
+ }
281
+ getMaxPageSizeInGetMetadata() { return this.maxPageSizeInGetMetadata; }
282
+ getConsumerGroup() { return this.consumerGroup; }
283
+ isStarted() { return this.started; }
284
+ getServiceState() { return this._serviceState; }
285
+ setAllocateMessageQueueStrategy(strategy) {
286
+ this.allocateMessageQueueStrategy = strategy;
287
+ return this;
288
+ }
289
+ getAllocateMessageQueueStrategy() {
290
+ return this.allocateMessageQueueStrategy;
291
+ }
292
+ setBrokerSuspendMaxTimeMillis(ms) { this.brokerSuspendMaxTimeMillis = ms; return this; }
293
+ getBrokerSuspendMaxTimeMillis() { return this.brokerSuspendMaxTimeMillis; }
294
+ setConsumerTimeoutMillisWhenSuspend(ms) {
295
+ this.consumerTimeoutMillisWhenSuspend = ms;
296
+ return this;
297
+ }
298
+ getConsumerTimeoutMillisWhenSuspend() { return this.consumerTimeoutMillisWhenSuspend; }
299
+ // Java DefaultLitePullConsumer#setConnectBrokerByUser / #isConnectBrokerByUser: pin every
300
+ // pull round to defaultBrokerId instead of the broker-suggested node. Applies on the next
301
+ // round; when start() has not run yet the value is carried into the PullAPI it builds.
302
+ setConnectBrokerByUser(on) {
303
+ this.connectBrokerByUser = on;
304
+ if (this.pullAPI)
305
+ this.pullAPI.connectBrokerByUser = on;
306
+ return this;
307
+ }
308
+ isConnectBrokerByUser() {
309
+ return this.pullAPI ? this.pullAPI.connectBrokerByUser : this.connectBrokerByUser;
310
+ }
311
+ setDefaultBrokerId(id) {
312
+ this.defaultBrokerId = id;
313
+ if (this.pullAPI)
314
+ this.pullAPI.defaultBrokerId = id;
315
+ return this;
316
+ }
317
+ getDefaultBrokerId() {
318
+ return this.pullAPI ? this.pullAPI.defaultBrokerId : this.defaultBrokerId;
319
+ }
320
+ // Sets the topic metadata comparison period (anything below the floor or
321
+ // otherwise illegal is clamped to the floor).
322
+ setTopicMetadataCheckIntervalMillis(ms) {
323
+ this.topicMetadataCheckIntervalMillis =
324
+ Number.isFinite(ms) && ms >= MIN_TOPIC_METADATA_CHECK_INTERVAL_MILLIS
325
+ ? Math.floor(ms) : MIN_TOPIC_METADATA_CHECK_INTERVAL_MILLIS;
326
+ return this;
327
+ }
328
+ getTopicMetadataCheckIntervalMillis() { return this.topicMetadataCheckIntervalMillis; }
329
+ // subscribe() and assign() are mutually exclusive: the first claim holds, and the
330
+ // other side is refused with the same message however it is reached.
331
+ claimMode(mode) {
332
+ if (this.mode === 'none') {
333
+ this.mode = mode;
334
+ return;
335
+ }
336
+ if (this.mode !== mode)
337
+ throw new Error('Subscribe and assign are mutually exclusive.');
338
+ }
339
+ // ---- namespace (client-side prefix) ----
340
+ _withNamespace(resource) {
341
+ return MixAll.wrapNamespace(this.namespace, resource);
342
+ }
343
+ _queueWithNamespace(mq) {
344
+ if (!this.namespace)
345
+ return mq;
346
+ const topic = this._withNamespace(mq.getTopic());
347
+ return topic === mq.getTopic() ? mq : new MessageQueue(topic, mq.getBrokerName(), mq.getQueueId());
348
+ }
349
+ _bareQueue(mq) {
350
+ if (!this.namespace)
351
+ return mq;
352
+ const topic = MixAll.withoutNamespace(mq.getTopic(), this.namespace);
353
+ return topic === mq.getTopic() ? mq : new MessageQueue(topic, mq.getBrokerName(), mq.getQueueId());
354
+ }
355
+ // The caller hands in and gets back bare topics; the broker only knows prefixed
356
+ // ones, so messages are reset on the way out.
357
+ _resetMessagesTopic(msgs) {
358
+ if (!this.namespace)
359
+ return;
360
+ for (const msg of msgs)
361
+ msg.setTopic(MixAll.withoutNamespace(msg.getTopic(), this.namespace));
362
+ }
363
+ subscribe(topic, subExpression = '*') {
364
+ if (!topic || !topic.trim())
365
+ throw new Error('Topic can not be null or empty.');
366
+ this.claimMode('subscribe');
367
+ const full = this._withNamespace(topic);
368
+ const expr = subExpression || '*';
369
+ this.subscription.set(full, FilterAPI.buildSubscriptionData(full, expr));
370
+ // A subscription added while RUNNING needs its route and a heartbeat now:
371
+ // the rebalance loop only reads the cached route, and the broker learns the
372
+ // expression only from a fresh heartbeat.
373
+ const client = this.mqClient;
374
+ if (this.started && client) {
375
+ void client.updateTopicRouteInfoFromNameServer(full, false).catch((e) => logger.debug('lite subscribe: route refresh for %s failed: %s', full, e.message));
376
+ void client.sendHeartbeatToAllBrokers().catch(() => { });
377
+ }
378
+ return this;
379
+ }
380
+ unsubscribe(topic) {
381
+ const full = this._withNamespace(topic);
382
+ this.subscription.delete(full);
383
+ // The topic's queues leave with it, without waiting for the next rebalance,
384
+ // and their per-queue state goes with them — the same revocation path a
385
+ // rebalance that no longer covers them takes.
386
+ const keep = (mq) => mq.getTopic() !== full;
387
+ for (const mq of this._assignedQueues().filter((mq) => !keep(mq)))
388
+ this.dropQueueState(mq);
389
+ this.assignedAuto = this.assignedAuto.filter(keep);
390
+ if (this.assignedManual != null)
391
+ this.assignedManual = this.assignedManual.filter(keep);
392
+ }
393
+ // setSubExpressionForAssign names the tag/SQL expression one assigned topic pulls
394
+ // with. It is a pre-start setting: a running consumer has already told the broker
395
+ // what filter this queue uses.
396
+ setSubExpressionForAssign(topic, subExpression) {
397
+ if (!subExpression || !subExpression.trim()) {
398
+ throw new Error('subExpression can not be null or empty.');
399
+ }
400
+ if (this.started)
401
+ throw new Error('setSubExpressionForAssign only can be called before start.');
402
+ if (!topic || !topic.trim())
403
+ throw new Error('Topic can not be null or empty.');
404
+ this.claimMode('assign');
405
+ const full = this._withNamespace(topic);
406
+ this.assignSubscription.set(full, FilterAPI.buildSubscriptionData(full, subExpression));
407
+ return this;
408
+ }
409
+ // buildSubscriptionsForHeartbeat replaces the topics this consumer announces in
410
+ // its heartbeat beyond the rebalance subscriptions. An assign-mode consumer has
411
+ // no topic-wide subscription of its own, yet the broker filters the group's
412
+ // pulls by the expression it was last told, so this is how such an application
413
+ // announces the filters its queues need.
414
+ //
415
+ // Every call replaces the whole set (a topic left out stops being announced),
416
+ // a nil or empty map clears it, a topic the rebalance subscriptions cover keeps
417
+ // their expression, and a rejected selector leaves the previous set intact.
418
+ buildSubscriptionsForHeartbeat(selectors) {
419
+ const built = new Map();
420
+ for (const [topic, selector] of selectors ?? new Map()) {
421
+ if (!topic || !topic.trim())
422
+ throw new Error('Topic can not be null or empty.');
423
+ const full = this._withNamespace(topic);
424
+ built.set(full, FilterAPI.build(full, selector.getExpression(), selector.getType()));
425
+ }
426
+ this.heartbeatSubscriptions = built;
427
+ return this;
428
+ }
429
+ // getSubscriptionsForHeartbeat returns the built set, sorted by topic; a
430
+ // snapshot, not a live view.
431
+ getSubscriptionsForHeartbeat() {
432
+ return [...this.heartbeatSubscriptions.keys()].sort()
433
+ .map((topic) => this.heartbeatSubscriptions.get(topic));
434
+ }
435
+ // assign switches to MANUAL mode: the given queues are polled directly.
436
+ // An empty list is refused: taken literally it would drop every queue held so far,
437
+ // which is exactly what a route lookup that came back empty does by accident.
438
+ // The list is a set: a queue named twice is held once.
439
+ assign(mqs) {
440
+ if (!mqs || mqs.length === 0) {
441
+ throw new Error('Message queues can not be null or empty.');
442
+ }
443
+ this.claimMode('assign');
444
+ const seen = new Set();
445
+ const kept = [];
446
+ for (const raw of mqs) {
447
+ const mq = this._queueWithNamespace(raw);
448
+ const key = mqKey(mq);
449
+ if (seen.has(key))
450
+ continue;
451
+ seen.add(key);
452
+ kept.push(mq);
453
+ if (!this.assignSubscription.has(mq.getTopic())) {
454
+ this.assignSubscription.set(mq.getTopic(), FilterAPI.buildSubscriptionData(mq.getTopic(), '*'));
455
+ }
456
+ }
457
+ // A queue this call no longer names loses its cursors, its armed park and its
458
+ // paused flag with the assignment; nothing stale may hold up a later round.
459
+ for (const mq of this.assignedManual ?? []) {
460
+ if (!seen.has(mqKey(mq)))
461
+ this.dropQueueState(mq);
462
+ }
463
+ this.assignedManual = kept;
464
+ return this;
465
+ }
466
+ // Drop every piece of per-queue state this consumer keeps: the pull cursor, the
467
+ // armed 215 park and the paused flag.
468
+ dropQueueState(mq) {
469
+ const key = mqKey(mq);
470
+ this.pullCursor.delete(key);
471
+ this.pullParkUntil.delete(key);
472
+ this.paused.delete(key);
473
+ }
474
+ // _checkConfig validates the pre-start configuration. Subscription is NOT a
475
+ // precondition here: queues can also be named after start() via assign().
476
+ _checkConfig() {
477
+ Validators.checkGroup(this.consumerGroup);
478
+ if (this.consumerGroup === MixAll.DEFAULT_CONSUMER_GROUP) {
479
+ throw new Error(`consumerGroup can not equal ${MixAll.DEFAULT_CONSUMER_GROUP}`);
480
+ }
481
+ if (!this.messageModel)
482
+ throw new Error('messageModel can not be null');
483
+ if (!this.allocateMessageQueueStrategy)
484
+ throw new Error('allocateMessageQueueStrategy can not be null');
485
+ if (this.consumerTimeoutMillisWhenSuspend < this.brokerSuspendMaxTimeMillis) {
486
+ throw new Error('Long polling mode, the consumer consumerTimeoutMillisWhenSuspend '
487
+ + 'must greater than brokerSuspendMaxTimeMillis');
488
+ }
489
+ }
490
+ async start() {
491
+ // The group is a resource too: on the wire it carries the namespace prefix.
492
+ // wrapNamespace is idempotent, so a start after a failed start does not stack
493
+ // prefixes.
494
+ this.consumerGroup = this._withNamespace(this.consumerGroup);
495
+ switch (this._serviceState) {
496
+ case 'CREATE_JUST':
497
+ // Leave CREATE_JUST before anything can fail, so a failed start is not
498
+ // retried with half of the state already built.
499
+ this._serviceState = 'START_FAILED';
500
+ this._checkConfig();
501
+ break;
502
+ case 'RUNNING':
503
+ case 'START_FAILED':
504
+ case 'SHUTDOWN_ALREADY':
505
+ throw new Error(`The PullConsumer service state not OK, maybe started once, ${this._serviceState}`);
506
+ default:
507
+ break;
508
+ }
509
+ if (!this.nameServerAddr)
510
+ throw new Error('name server address is not set');
511
+ // The start instant is checked whatever the policy is: a malformed value must fail here rather
512
+ // than surface mid-consumption as a failed offset resolution on one queue.
513
+ parseConsumeTimestamp(this.consumeTimestamp);
514
+ // CLUSTERING rewrites the default instanceName, after which the clientId looks
515
+ // like `<ip>@<instanceName>[@<unitName>][@STREAM]`. ⚠ If only the pid is used
516
+ // as the instance name, every lite pull consumer in one process shares a
517
+ // clientId, allocate() hands the same set of queues to two instances, and
518
+ // both read exactly the same messages.
519
+ if (this.messageModel === MessageModel.CLUSTERING) {
520
+ this.instanceName = MixAll.changeInstanceNameToPid(this.instanceName);
521
+ }
522
+ this.clientID = MixAll.clientIdFor(this.instanceName, this.unitName || null, this.enableStreamRequestType);
523
+ const manager = MQClientManager.getInstance();
524
+ const { instance: client, created } = manager.getOrCreateMQClientInstance(this.clientID, () => new MQClient(this.clientID, this.nameServerAddr));
525
+ if (created) {
526
+ client.setPersistConsumerOffsetInterval(this.persistConsumerOffsetInterval);
527
+ if (this.useHeartbeatV2)
528
+ client.setUseHeartbeatV2(true);
529
+ client.setDecodeReadBody(this.decodeReadBody);
530
+ client.setMqClientApiTimeout(this.mqClientApiTimeout);
531
+ client.setEnableConcurrentHeartbeat(this.enableConcurrentHeartbeat);
532
+ client.setConcurrentHeartbeatThreadPoolSize(this.concurrentHeartbeatThreadPoolSize);
533
+ client.setMaxPageSizeInGetMetadata(this.maxPageSizeInGetMetadata);
534
+ registerRpcHooks(client.remotingClient, {
535
+ namespaceV2: () => this.namespaceV2,
536
+ enableStreamRequestType: this.enableStreamRequestType,
537
+ });
538
+ }
539
+ manager.registerClient(this.clientID, client);
540
+ this.mqClient = client;
541
+ client.registerConsumer(this.consumerGroup, this);
542
+ client.start();
543
+ if (this.messageModel === MessageModel.BROADCASTING) {
544
+ this.offsetStore = new LocalFileOffsetStore(this.clientID, this.consumerGroup);
545
+ }
546
+ else {
547
+ this.offsetStore = new RemoteBrokerOffsetStore(client, this.consumerGroup);
548
+ }
549
+ this.pullAPI = new PullAPI(client, this.consumerGroup);
550
+ // Carry the pre-start pin onto the PullAPI that start() just built (the setters above
551
+ // only reach it once it exists).
552
+ this.pullAPI.connectBrokerByUser = this.connectBrokerByUser;
553
+ this.pullAPI.defaultBrokerId = this.defaultBrokerId;
554
+ await this.offsetStore.load().catch((e) => logger.warning('load offset store failed: %s', e.message));
555
+ // Warm the route cache for every topic this consumer will pull: the
556
+ // subscribed topics in subscribe mode, the assigned queues' topics in assign
557
+ // mode (which advertises no subscription at all).
558
+ const routeTopics = this.mode === 'assign'
559
+ ? [...new Set((this.assignedManual ?? []).map((mq) => mq.getTopic()))]
560
+ : [...this.subscription.keys()];
561
+ for (const topic of routeTopics) {
562
+ await client.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
563
+ }
564
+ // Once the queues are resolved but before the group is registered at the broker,
565
+ // send one CHECK_CLIENT_IN_BROKER(46) pre-check.
566
+ try {
567
+ await client.checkClientInBroker();
568
+ }
569
+ catch (e) {
570
+ logger.warning('Start the lite pull consumer %s fail: %s', this.consumerGroup, e.message);
571
+ // started is still false here, so shutdown() would no-op — unwind the
572
+ // registration this start() already did by hand (the balancing return of
573
+ // registerClient: unregisterClient counts -1, and only hitting zero really
574
+ // shuts down).
575
+ client.unregisterConsumer(this.consumerGroup);
576
+ manager.unregisterClient(client.clientId, client);
577
+ throw e;
578
+ }
579
+ // Register the consumer group at the broker right away: the first beat of the
580
+ // client's heartbeat loop may already have run before any route was cached,
581
+ // and waiting for the 30s period would delay group registration — and hence
582
+ // the first rebalance — by half a minute.
583
+ // So start() proactively sends one extra heartbeat.
584
+ await client.sendHeartbeatToAllBrokers().catch(() => { });
585
+ this.started = true;
586
+ this._serviceState = 'RUNNING';
587
+ // The metadata comparison loop starts unconditionally, whether or not any
588
+ // listener was registered.
589
+ this._startMetadataLoop();
590
+ if (this.mode === 'subscribe') {
591
+ await this._doRebalance();
592
+ this._rebalanceTimer = setInterval(() => {
593
+ this._doRebalance().catch((e) => logger.debug('lite pull rebalance error: %s', e.message));
594
+ }, 20000);
595
+ if (typeof this._rebalanceTimer.unref === 'function')
596
+ this._rebalanceTimer.unref();
597
+ }
598
+ if (this.autoCommit) {
599
+ this._autoCommitTimer = setInterval(() => {
600
+ this.commitSync().catch((e) => logger.debug('lite pull auto commit error: %s', e.message));
601
+ }, this.autoCommitIntervalMillis);
602
+ if (typeof this._autoCommitTimer.unref === 'function')
603
+ this._autoCommitTimer.unref();
604
+ }
605
+ }
606
+ async shutdown() {
607
+ if (!this.started)
608
+ return;
609
+ if (this._rebalanceTimer) {
610
+ clearInterval(this._rebalanceTimer);
611
+ this._rebalanceTimer = null;
612
+ }
613
+ if (this._autoCommitTimer) {
614
+ clearInterval(this._autoCommitTimer);
615
+ this._autoCommitTimer = null;
616
+ }
617
+ if (this._metadataTimer) {
618
+ clearInterval(this._metadataTimer);
619
+ this._metadataTimer = null;
620
+ }
621
+ if (this._metadataInitialTimer) {
622
+ clearTimeout(this._metadataInitialTimer);
623
+ this._metadataInitialTimer = null;
624
+ }
625
+ try {
626
+ await this.commitSync();
627
+ }
628
+ catch (e) { /* best effort */ }
629
+ if (this.mqClient) {
630
+ await this.mqClient.unregisterClientAllBrokers('', this.consumerGroup).catch(() => { });
631
+ this.mqClient.unregisterConsumer(this.consumerGroup);
632
+ // Handed back to the manager: the count goes -1, and only at zero does it really
633
+ // shut down.
634
+ MQClientManager.getInstance().unregisterClient(this.mqClient.clientId, this.mqClient);
635
+ }
636
+ this.started = false;
637
+ this._serviceState = 'SHUTDOWN_ALREADY';
638
+ }
639
+ // _doRebalance assigns queues only in subscribe() mode.
640
+ async _doRebalance() {
641
+ if (this.mode !== 'subscribe' || !this.mqClient)
642
+ return;
643
+ const client = this.mqClient;
644
+ const assigned = [];
645
+ for (const topic of this.subscription.keys()) {
646
+ await client.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
647
+ const route = client.getTopicRouteData(topic);
648
+ const mqAll = route ? route.getAllSubscribeMessageQueue(topic) : [];
649
+ if (!mqAll.length)
650
+ continue;
651
+ if (this.messageModel === MessageModel.BROADCASTING) {
652
+ assigned.push(...mqAll);
653
+ continue;
654
+ }
655
+ // Ask one master for the group's consumer id list; on failure KEEP the
656
+ // current assignment (never "I own everything").
657
+ let addr = null;
658
+ for (const bd of route.brokerDatas || []) {
659
+ const m = bd.brokerAddrs || {};
660
+ if (MixAll.MASTER_ID in m) {
661
+ addr = m[MixAll.MASTER_ID];
662
+ break;
663
+ }
664
+ }
665
+ if (!addr)
666
+ continue;
667
+ try {
668
+ const response = await client.getConsumerListByGroup(addr, this.consumerGroup);
669
+ if (response.code !== 0)
670
+ continue;
671
+ const { RemotingSerializable } = await import("../remoting/serialize.js");
672
+ const body = response.body && response.body.length
673
+ ? RemotingSerializable.decode(response.body) : null;
674
+ const ids = Array.isArray(body) ? body
675
+ : (body && Array.isArray(body['consumerIdList'])) ? body['consumerIdList'] : [];
676
+ if (!ids.length)
677
+ continue;
678
+ const sorted = [...mqAll].sort((a, b) => a.compareTo(b));
679
+ assigned.push(...this.allocateMessageQueueStrategy.allocate(this.consumerGroup, this.clientID, sorted, ids.map(String).sort()));
680
+ }
681
+ catch (e) {
682
+ logger.debug('lite pull rebalance: no consumer id list for %s: %s', topic, e.message);
683
+ }
684
+ }
685
+ assigned.sort((a, b) => a.compareTo(b));
686
+ // A queue the new allocation no longer covers is revoked: its cursor, its
687
+ // armed park and its paused flag go with it, so a later round cannot keep
688
+ // serving a queue this consumer no longer owns.
689
+ const next = new Set(assigned.map(mqKey));
690
+ for (const mq of this.assignedAuto) {
691
+ if (!next.has(mqKey(mq)))
692
+ this.dropQueueState(mq);
693
+ }
694
+ this.assignedAuto = assigned;
695
+ }
696
+ // fetchMessageQueues returns the SUBSCRIBE view (read perm, readQueueNums, NO
697
+ // master filter). An unroutable topic throws: handing back [] would read as
698
+ // "this topic has zero queues", so a nameserver blip would look like a
699
+ // scale-in and the queue-change listener would happily report it as one.
700
+ async fetchMessageQueues(topic) {
701
+ if (!this.mqClient)
702
+ throw new Error('The consumer not running, please start it first.');
703
+ const full = this._withNamespace(topic);
704
+ await this.mqClient.updateTopicRouteInfoFromNameServer(full, false).catch(() => { });
705
+ const route = this.mqClient.getTopicRouteData(full);
706
+ if (!route)
707
+ throw new Error(`Can not find MessageQueue for topic: ${topic}`);
708
+ const queues = route.getAllSubscribeMessageQueue(full);
709
+ if (queues.length === 0) {
710
+ throw new Error(`Can not find Message Queue for this topic, ${topic} Namesrv return empty`);
711
+ }
712
+ // The caller works with the bare names: prefixing happens inside this class.
713
+ return queues.map((mq) => this._bareQueue(mq));
714
+ }
715
+ fetchSubscribeMessageQueues(topic) {
716
+ return this.fetchMessageQueues(topic);
717
+ }
718
+ // Constraints, one by one:
719
+ // * empty topic / missing listener -> report "Topic or listener is null";
720
+ // * re-registering the same topic logs a WARN and overwrites the old listener;
721
+ // * only a consumer in the RUNNING state takes the snapshot immediately —
722
+ // otherwise the first round would report the set that "was already there"
723
+ // as a change.
724
+ // A snapshot is one route RPC, hence the await here; registering before start()
725
+ // is fine too, because the comparison loop starts taking snapshots at
726
+ // start().
727
+ async registerTopicMessageQueueChangeListener(topic, listener) {
728
+ if (!topic || !topic.trim() || listener == null) {
729
+ throw new Error('Topic or listener is null');
730
+ }
731
+ // Watched under the broker-side name; the snapshot is taken with the same
732
+ // lookup every later comparison uses.
733
+ const full = this._withNamespace(topic);
734
+ if (this.topicChangeListeners.has(full)) {
735
+ logger.warning('Topic %s had been registered, new listener will overwrite the old one', full);
736
+ }
737
+ this.topicChangeListeners.set(full, listener);
738
+ if (!this.started)
739
+ return;
740
+ try {
741
+ this.messageQueuesForTopic.set(full, await this.fetchMessageQueues(full));
742
+ }
743
+ catch (e) {
744
+ logger.debug('register listener: fetch queues for %s failed: %s', full, e.message);
745
+ }
746
+ }
747
+ // For each watched topic: compare the route's current queue set against the one
748
+ // seen last time; onChanged fires only on a real change, and only for that
749
+ // topic — one topic's route lookup failing must not abort the whole round (the
750
+ // catch sits inside each topic's loop and only logs).
751
+ // Returns the number of listeners notified (0 when nothing changed).
752
+ async fetchTopicMessageQueuesAndCompare() {
753
+ let fired = 0;
754
+ // Snapshot the entries: an onChanged implementation may register or drop
755
+ // listeners, and JS Map iteration would then see the mutation.
756
+ for (const [topic, listener] of [...this.topicChangeListeners.entries()]) {
757
+ try {
758
+ const newQueues = await this.fetchMessageQueues(topic);
759
+ if (isQueueSetEqual(this.messageQueuesForTopic.get(topic), newQueues))
760
+ continue;
761
+ this.messageQueuesForTopic.set(topic, newQueues);
762
+ await listener.onChanged(topic, newQueues);
763
+ fired++;
764
+ }
765
+ catch (e) {
766
+ logger.error('ScheduledTask fetchMessageQueuesAndCompare exception: %s', e.message);
767
+ }
768
+ }
769
+ return fired;
770
+ }
771
+ // Fixed-rate scheduling: the first round is delayed 10s, then one beat per
772
+ // interval. The period is read once at start() and never changes afterwards;
773
+ // the 10s first-round delay keeps a just-started consumer from answering route
774
+ // lookups before the broker has registered the queues.
775
+ _startMetadataLoop(firstDelayMillis = METADATA_CHECK_FIRST_DELAY_MILLIS, periodMillis = this.topicMetadataCheckIntervalMillis) {
776
+ if (this._metadataTimer != null)
777
+ return;
778
+ const beat = () => {
779
+ this.fetchTopicMessageQueuesAndCompare().catch((e) => logger.debug('lite pull topic queue change check error: %s', e.message));
780
+ };
781
+ const initial = setTimeout(beat, firstDelayMillis);
782
+ if (typeof initial.unref === 'function')
783
+ initial.unref();
784
+ this._metadataInitialTimer = initial;
785
+ this._metadataTimer = setInterval(beat, periodMillis);
786
+ if (typeof this._metadataTimer.unref === 'function')
787
+ this._metadataTimer.unref();
788
+ }
789
+ // poll fetches the next batch from any assigned, non-paused queue.
790
+ //
791
+ // The cursor starts at the committed offset (or 0 for a %RETRY% topic) and
792
+ // advances by nextBeginOffset after every round.
793
+ //
794
+ // A queue the broker throttled (215) is skipped until its short park expires;
795
+ // the round steps over it and keeps serving its siblings, and a round where
796
+ // NOTHING is pullable waits out the shortest remaining park rather than burning
797
+ // the caller's deadline on a spin.
798
+ async poll(timeoutMillis = this.pollTimeoutMillis) {
799
+ if (!this.started || !this.pullAPI)
800
+ throw new Error('The consumer not running, please start it first.');
801
+ if (timeoutMillis < 0)
802
+ throw new Error('Timeout must not be negative');
803
+ const deadline = Date.now() + Math.max(1, timeoutMillis);
804
+ const queues = this._activeQueues();
805
+ if (queues.length === 0)
806
+ return [];
807
+ const out = [];
808
+ let idx = 0;
809
+ while (Date.now() < deadline && out.length < this.pullBatchSize) {
810
+ // Round-robin across queues so no single queue starves the rest.
811
+ let pos = idx % queues.length;
812
+ let steppedOver = 0;
813
+ while (steppedOver < queues.length && this._parkMillisLeft(queues[pos]) > 0) {
814
+ pos = (pos + 1) % queues.length;
815
+ steppedOver++;
816
+ }
817
+ if (steppedOver >= queues.length) {
818
+ // Every queue is parked: the shortest remaining park is the soonest any
819
+ // answer can arrive, so waiting it out respects both the broker's pause
820
+ // and the caller's deadline.
821
+ let shortest = this._parkMillisLeft(queues[0]);
822
+ for (const mq of queues)
823
+ shortest = Math.min(shortest, this._parkMillisLeft(mq));
824
+ await this._sleepPollRound(Math.min(shortest, Math.max(1, deadline - Date.now())));
825
+ continue;
826
+ }
827
+ idx = pos + 1;
828
+ const mq = queues[pos];
829
+ const key = mqKey(mq);
830
+ const sub = this._subscriptionFor(mq.getTopic());
831
+ if (!sub)
832
+ continue;
833
+ let offset = this.pullCursor.get(key);
834
+ if (offset === undefined) {
835
+ const committed = await this.offsetStore.readOffset(mq, ReadOffsetMode.READ_FROM_MEMORY_THEN_STORE);
836
+ // A stored offset always wins; only a group that has never committed one
837
+ // is placed by the consumeFromWhere policy.
838
+ if (committed >= 0)
839
+ offset = committed;
840
+ else if (this.consumeFromWhere === ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET)
841
+ offset = 0;
842
+ else if (this.consumeFromWhere === ConsumeFromWhere.CONSUME_FROM_TIMESTAMP) {
843
+ // The policy comes before the retry-topic rule: a retry queue is created by the first
844
+ // redelivery, long after the configured instant, so searching there answers nothing and
845
+ // its replay starts at the tail.
846
+ offset = MixAll.isRetryTopic(mq.getTopic())
847
+ ? await this._maxOffsetFor(mq)
848
+ : await this._searchOffsetFor(mq, parseConsumeTimestamp(this.consumeTimestamp));
849
+ }
850
+ else {
851
+ // CONSUME_FROM_LAST_OFFSET (and the deprecated variants): a retry queue replays from 0
852
+ // because retried messages must all be retried; a business queue skips the backlog.
853
+ offset = MixAll.isRetryTopic(mq.getTopic()) ? 0 : await this._maxOffsetFor(mq);
854
+ }
855
+ if (offset < 0)
856
+ continue; // unknown offset; nothing to pull from
857
+ this.pullCursor.set(key, offset);
858
+ }
859
+ try {
860
+ // LITE_PULL_MESSAGE(361), SHORT poll, expression ALWAYS sent, and the
861
+ // lite-pull bit set in sysFlag.
862
+ const sysFlag = PullSysFlag.buildSysFlag(false, false, true, false, true);
863
+ let result = await this.pullAPI.pullKernelLite(mq, offset, sub, sysFlag, this.pullBatchSize - out.length, this.brokerSuspendMaxTimeMillis, Math.max(1, deadline - Date.now()));
864
+ result = this.pullAPI.processPullResult(mq, result, sub);
865
+ this.pullCursor.set(key, result.nextBeginOffset);
866
+ if (result.status === PullStatus.FOUND && result.msgFoundList.length > 0) {
867
+ out.push(...result.msgFoundList);
868
+ if (this.offsetStore)
869
+ this.offsetStore.updateOffset(mq, result.nextBeginOffset, true);
870
+ if (out.length >= this.pullBatchSize)
871
+ break;
872
+ }
873
+ else if (result.status === PullStatus.OFFSET_ILLEGAL) {
874
+ // The broker corrected the offset: adopt it.
875
+ this.pullCursor.set(key, result.nextBeginOffset);
876
+ if (this.offsetStore)
877
+ this.offsetStore.updateOffset(mq, result.nextBeginOffset, false);
878
+ }
879
+ }
880
+ catch (e) {
881
+ logger.debug('lite pull error for %s: %s', key, e.message);
882
+ // A short poll error: fall through so the remaining queues still get
883
+ // their turn within the deadline. The ONE exception is the broker's own
884
+ // throttle answer, which parks just this queue for the short 20ms pause —
885
+ // retrying it hot is exactly what the broker is asking for not to happen.
886
+ const parkMillis = litePullParkMillisFor(e instanceof MQBrokerException ? e.responseCode : undefined);
887
+ if (parkMillis > 0)
888
+ this.setPullParkUntil(mq, Date.now() + parkMillis);
889
+ }
890
+ if (queues.length === 1 && out.length === 0 && this._parkMillisLeft(mq) <= 0) {
891
+ // Single queue and nothing found: don't hot-loop against the broker. A
892
+ // queue this round parked on 215 waits its own short pause above instead,
893
+ // so the idle guard must not swallow it.
894
+ await this._sleepPollRound(IDLE_POLL_RETRY_DELAY_MILLIS);
895
+ }
896
+ }
897
+ this._resetMessagesTopic(out);
898
+ return out;
899
+ }
900
+ // commitSync persists the offsets of every assigned queue.
901
+ async commitSync() {
902
+ if (!this.offsetStore)
903
+ return;
904
+ const queues = this._activeQueues();
905
+ await this.offsetStore.persistAll(queues);
906
+ }
907
+ // persistConsumerOffset — the MQConsumerInner contract name the MQClient's
908
+ // instance-level persistAllConsumerOffset sweep calls every
909
+ // persistConsumerOffsetInterval; it is only a thin wrapper
910
+ // over commitSync.
911
+ async persistConsumerOffset() {
912
+ return this.commitSync();
913
+ }
914
+ // seek repositions one queue: the next poll of it starts at `offset`.
915
+ //
916
+ // The queue must be one this consumer holds — writing a cursor for a stranger
917
+ // would leave an entry nothing pulls, so the caller would watch an offset
918
+ // change that no round ever honours — and the offset is held to the queue's own
919
+ // [minOffset, maxOffset] range. Reading those two bounds is also where the
920
+ // running-state check happens: they are broker RPCs, and a stopped consumer has
921
+ // no broker to ask.
922
+ async seek(mq, offset) {
923
+ const fullMq = this._queueWithNamespace(mq);
924
+ const held = this._assignedQueues().some((q) => mqKey(q) === mqKey(fullMq));
925
+ if (!held) {
926
+ throw new Error(this.mode === 'assign'
927
+ ? `The message queue is not in assigned list, message queue: ${mq}`
928
+ : `The message queue is not in assigned list, may be rebalancing, message queue: ${mq}`);
929
+ }
930
+ if (!this.started || !this.mqClient) {
931
+ throw new Error('The consumer not running, please start it first.');
932
+ }
933
+ // A bound the broker could not answer must not become a cursor: a negative
934
+ // cursor would only come back as OFFSET_ILLEGAL on the next poll.
935
+ const minOffset = await this._boundaryOffset(fullMq, 'min');
936
+ if (minOffset < 0)
937
+ throw new Error(`Can not get min offset of queue ${mqKey(mq)}`);
938
+ const maxOffset = await this._boundaryOffset(fullMq, 'max');
939
+ if (maxOffset < 0)
940
+ throw new Error(`Can not get max offset of queue ${mqKey(mq)}`);
941
+ if (offset < minOffset || offset > maxOffset) {
942
+ throw new Error(`Seek offset illegal, seek offset = ${offset}, min offset = ${minOffset}, max offset = ${maxOffset}`);
943
+ }
944
+ const key = mqKey(fullMq);
945
+ this.pullCursor.set(key, offset);
946
+ // An explicit repositioning lifts a leftover throttle park: it belonged to
947
+ // the position that is being left behind.
948
+ this.pullParkUntil.delete(key);
949
+ if (this.offsetStore)
950
+ this.offsetStore.updateOffset(fullMq, offset, false);
951
+ }
952
+ // assignment(): returns the queues currently assigned.
953
+ assignment() {
954
+ return [...(this.assignedManual != null ? this.assignedManual : this.assignedAuto)];
955
+ }
956
+ // The min/max offset on the **master** broker that owns the queue (returns -1
957
+ // when the route or broker is unknown — the caller skips the queue this round
958
+ // instead of seeking to a fake cursor). Management queries only accept the
959
+ // master address.
960
+ async _boundaryOffset(mq, kind, timestamp = 0) {
961
+ if (!this.mqClient)
962
+ return -1;
963
+ await this.mqClient.updateTopicRouteInfoFromNameServer(mq.getTopic(), false).catch(() => { });
964
+ const route = this.mqClient.getTopicRouteData(mq.getTopic());
965
+ if (!route)
966
+ return -1;
967
+ for (const bd of route.brokerDatas || []) {
968
+ const addrs = bd.brokerAddrs || {};
969
+ const addr = addrs['0'] != null ? addrs['0'] : Object.values(addrs)[0];
970
+ if (addr != null && bd.brokerName === mq.getBrokerName()) {
971
+ try {
972
+ if (kind === 'min') {
973
+ return await this.mqClient.getMinOffset(addr, mq.getTopic(), mq.getQueueId());
974
+ }
975
+ if (kind === 'search') {
976
+ return await this.mqClient.searchOffsetByTimestamp(addr, mq.getTopic(), mq.getQueueId(), timestamp);
977
+ }
978
+ return await this.mqClient.getMaxOffset(addr, mq.getTopic(), mq.getQueueId());
979
+ }
980
+ catch (e) {
981
+ return -1;
982
+ }
983
+ }
984
+ }
985
+ return -1;
986
+ }
987
+ _maxOffsetFor(mq) {
988
+ return this._boundaryOffset(mq, 'max');
989
+ }
990
+ _searchOffsetFor(mq, timestamp) {
991
+ return this._boundaryOffset(mq, 'search', timestamp);
992
+ }
993
+ // seekToBegin: read minOffset first, then seek; seekToEnd: read maxOffset first,
994
+ // then seek.
995
+ // When the broker cannot supply the boundary value it throws instead of seeking
996
+ // to -1: a negative cursor would come back as OFFSET_ILLEGAL on the next poll,
997
+ // which amounts to a silent error.
998
+ async seekToBegin(mq) {
999
+ if (!this.started)
1000
+ throw new Error('The consumer not running, please start it first.');
1001
+ const fullMq = this._queueWithNamespace(mq);
1002
+ const begin = await this._boundaryOffset(fullMq, 'min');
1003
+ if (begin < 0)
1004
+ throw new Error(`seekToBegin: minOffset unavailable for ${mqKey(mq)}`);
1005
+ await this.seek(mq, begin);
1006
+ }
1007
+ async seekToEnd(mq) {
1008
+ if (!this.started)
1009
+ throw new Error('The consumer not running, please start it first.');
1010
+ const fullMq = this._queueWithNamespace(mq);
1011
+ const end = await this._boundaryOffset(fullMq, 'max');
1012
+ if (end < 0)
1013
+ throw new Error(`seekToEnd: maxOffset unavailable for ${mqKey(mq)}`);
1014
+ await this.seek(mq, end);
1015
+ }
1016
+ // offsetForTimestamp: the queue's first offset at or after `timestamp` on the
1017
+ // master broker; -1 when the route or broker is unknown.
1018
+ async offsetForTimestamp(mq, timestamp) {
1019
+ if (!this.started)
1020
+ throw new Error('The consumer not running, please start it first.');
1021
+ return this._boundaryOffset(this._queueWithNamespace(mq), 'search', timestamp);
1022
+ }
1023
+ // committed(mq): the offset last committed for this queue — the in-memory
1024
+ // commit table first, then the broker — with -1 for "nobody has committed".
1025
+ // A broker read that FAILED is not "nobody committed": it throws, so a broken
1026
+ // read cannot be mistaken for a fresh consumer.
1027
+ async committed(mq) {
1028
+ if (!this.started || !this.offsetStore) {
1029
+ throw new Error('The consumer not running, please start it first.');
1030
+ }
1031
+ const offset = await this.offsetStore.readOffset(this._queueWithNamespace(mq), ReadOffsetMode.READ_FROM_MEMORY_THEN_STORE);
1032
+ if (offset === -2)
1033
+ throw new Error('Fetch consume offset from broker exception');
1034
+ return offset;
1035
+ }
1036
+ pause(mqs) {
1037
+ for (const mq of mqs)
1038
+ this.paused.add(mqKey(this._queueWithNamespace(mq)));
1039
+ }
1040
+ resume(mqs) {
1041
+ for (const mq of mqs)
1042
+ this.paused.delete(mqKey(this._queueWithNamespace(mq)));
1043
+ }
1044
+ // setPullParkUntil makes this queue's next pull wait until `untilMillis` (epoch
1045
+ // ms) — the knob behind the broker 215 short pause, exposed so an application (or
1046
+ // a test) can hold ONE queue back without pausing its siblings. A value not in
1047
+ // the future is dropped, and clearing an unknown queue is a no-op.
1048
+ setPullParkUntil(mq, untilMillis) {
1049
+ const key = mqKey(this._queueWithNamespace(mq));
1050
+ if (!Number.isFinite(untilMillis) || untilMillis <= Date.now()) {
1051
+ this.pullParkUntil.delete(key);
1052
+ return;
1053
+ }
1054
+ this.pullParkUntil.set(key, Math.floor(untilMillis));
1055
+ }
1056
+ // getPullParkUntil reads back the instant one queue is parked until, or 0 when
1057
+ // nothing holds it back. An already-expired park reads as 0 and is pruned here,
1058
+ // so the map cannot drift away from the queues it can actually hold back.
1059
+ getPullParkUntil(mq) {
1060
+ const key = mqKey(this._queueWithNamespace(mq));
1061
+ const until = this.pullParkUntil.get(key);
1062
+ if (until === undefined)
1063
+ return 0;
1064
+ if (until <= Date.now()) {
1065
+ this.pullParkUntil.delete(key);
1066
+ return 0;
1067
+ }
1068
+ return until;
1069
+ }
1070
+ // Remaining park for one queue, in ms (0 when it is pullable now).
1071
+ _parkMillisLeft(mq) {
1072
+ return this.getPullParkUntil(mq) - Date.now();
1073
+ }
1074
+ // _sleepPollRound is the one place a poll() round waits between two asks. It is
1075
+ // a method so the pause a round asked for can be RECORDED instead of slept by
1076
+ // the offline tests — the same handle the push path's _sleepOrStop gives its
1077
+ // gate tests. A real wait can only be late, never early, so the short park keeps
1078
+ // its lower bound.
1079
+ async _sleepPollRound(millis) {
1080
+ await new Promise((r) => setTimeout(r, Math.max(1, Math.floor(millis))));
1081
+ }
1082
+ // _subscriptionFor picks the expression a queue pulls with: the assigned-topic
1083
+ // expression in assign mode (subscribe() is refused there), the subscription
1084
+ // table in subscribe mode.
1085
+ _subscriptionFor(topic) {
1086
+ return this.mode === 'assign'
1087
+ ? this.assignSubscription.get(topic)
1088
+ : this.subscription.get(topic);
1089
+ }
1090
+ _assignedQueues() {
1091
+ return this.assignedManual != null ? this.assignedManual : this.assignedAuto;
1092
+ }
1093
+ _activeQueues() {
1094
+ return this._assignedQueues().filter((mq) => !this.paused.has(mqKey(mq)));
1095
+ }
1096
+ }