rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +428 -0
- package/README.md +424 -2
- package/dist/client/admin.d.ts +229 -0
- package/dist/client/admin.js +2462 -0
- package/dist/client/allocate.d.ts +48 -0
- package/dist/client/allocate.js +365 -0
- package/dist/client/backpressure.d.ts +23 -0
- package/dist/client/backpressure.js +95 -0
- package/dist/client/consumer.d.ts +294 -0
- package/dist/client/consumer.js +3095 -0
- package/dist/client/consumer_result.d.ts +129 -0
- package/dist/client/consumer_result.js +138 -0
- package/dist/client/consumer_stats.d.ts +69 -0
- package/dist/client/consumer_stats.js +258 -0
- package/dist/client/exception.d.ts +39 -0
- package/dist/client/exception.js +36 -0
- package/dist/client/hook.d.ts +145 -0
- package/dist/client/hook.js +171 -0
- package/dist/client/latency.d.ts +61 -0
- package/dist/client/latency.js +239 -0
- package/dist/client/lite_pull_consumer.d.ts +151 -0
- package/dist/client/lite_pull_consumer.js +1096 -0
- package/dist/client/mq_client.d.ts +188 -0
- package/dist/client/mq_client.js +1678 -0
- package/dist/client/mq_client_manager.d.ts +21 -0
- package/dist/client/mq_client_manager.js +129 -0
- package/dist/client/offset_store.d.ts +45 -0
- package/dist/client/offset_store.js +389 -0
- package/dist/client/open_tracing_hook.d.ts +52 -0
- package/dist/client/open_tracing_hook.js +241 -0
- package/dist/client/pop_api.d.ts +28 -0
- package/dist/client/pop_api.js +224 -0
- package/dist/client/pop_consumer.d.ts +23 -0
- package/dist/client/pop_consumer.js +719 -0
- package/dist/client/pop_process_queue.d.ts +15 -0
- package/dist/client/pop_process_queue.js +51 -0
- package/dist/client/process_queue.d.ts +49 -0
- package/dist/client/process_queue.js +261 -0
- package/dist/client/produce_accumulator.d.ts +75 -0
- package/dist/client/produce_accumulator.js +288 -0
- package/dist/client/producer.d.ts +231 -0
- package/dist/client/producer.js +1774 -0
- package/dist/client/pull_api.d.ts +26 -0
- package/dist/client/pull_api.js +264 -0
- package/dist/client/pull_consumer.d.ts +119 -0
- package/dist/client/pull_consumer.js +876 -0
- package/dist/client/pull_schedule_service.d.ts +71 -0
- package/dist/client/pull_schedule_service.js +424 -0
- package/dist/client/request_reply.d.ts +61 -0
- package/dist/client/request_reply.js +232 -0
- package/dist/client/retry_message.d.ts +2 -0
- package/dist/client/retry_message.js +27 -0
- package/dist/client/send_result.d.ts +57 -0
- package/dist/client/send_result.js +70 -0
- package/dist/client/top_addressing.d.ts +20 -0
- package/dist/client/top_addressing.js +75 -0
- package/dist/client/trace_context.d.ts +61 -0
- package/dist/client/trace_context.js +373 -0
- package/dist/client/trace_dispatcher.d.ts +31 -0
- package/dist/client/trace_dispatcher.js +236 -0
- package/dist/client/trace_hook.d.ts +23 -0
- package/dist/client/trace_hook.js +228 -0
- package/dist/client/traceparent.d.ts +9 -0
- package/dist/client/traceparent.js +86 -0
- package/dist/common/boundary_type.d.ts +7 -0
- package/dist/common/boundary_type.js +16 -0
- package/dist/common/compress.d.ts +28 -0
- package/dist/common/compress.js +512 -0
- package/dist/common/message.d.ts +129 -0
- package/dist/common/message.js +214 -0
- package/dist/common/messageClientIdSetter.d.ts +7 -0
- package/dist/common/messageClientIdSetter.js +48 -0
- package/dist/common/messageConst.d.ts +68 -0
- package/dist/common/messageConst.js +68 -0
- package/dist/common/messageDecoder.d.ts +40 -0
- package/dist/common/messageDecoder.js +504 -0
- package/dist/common/messageType.d.ts +11 -0
- package/dist/common/messageType.js +11 -0
- package/dist/common/message_accessor.d.ts +39 -0
- package/dist/common/message_accessor.js +135 -0
- package/dist/common/mixAll.d.ts +106 -0
- package/dist/common/mixAll.js +247 -0
- package/dist/common/recall_message_handle.d.ts +25 -0
- package/dist/common/recall_message_handle.js +98 -0
- package/dist/common/subscriptionData.d.ts +36 -0
- package/dist/common/subscriptionData.js +64 -0
- package/dist/common/sysflag.d.ts +129 -0
- package/dist/common/sysflag.js +96 -0
- package/dist/common/topic_config.d.ts +66 -0
- package/dist/common/topic_config.js +106 -0
- package/dist/common/topic_validator.d.ts +49 -0
- package/dist/common/topic_validator.js +115 -0
- package/dist/common/utilAll.d.ts +39 -0
- package/dist/common/utilAll.js +154 -0
- package/dist/common/validators.d.ts +25 -0
- package/dist/common/validators.js +80 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +36 -0
- package/dist/logging.d.ts +24 -0
- package/dist/logging.js +246 -0
- package/dist/remoting/acl.d.ts +16 -0
- package/dist/remoting/acl.js +39 -0
- package/dist/remoting/admin_body.d.ts +257 -0
- package/dist/remoting/admin_body.js +669 -0
- package/dist/remoting/bodies.d.ts +475 -0
- package/dist/remoting/bodies.js +1131 -0
- package/dist/remoting/client.d.ts +73 -0
- package/dist/remoting/client.js +591 -0
- package/dist/remoting/codes.d.ts +296 -0
- package/dist/remoting/codes.js +301 -0
- package/dist/remoting/exception.d.ts +45 -0
- package/dist/remoting/exception.js +58 -0
- package/dist/remoting/extra_info.d.ts +61 -0
- package/dist/remoting/extra_info.js +220 -0
- package/dist/remoting/headers.d.ts +758 -0
- package/dist/remoting/headers.js +1307 -0
- package/dist/remoting/heartbeat.d.ts +91 -0
- package/dist/remoting/heartbeat.js +227 -0
- package/dist/remoting/namespace.d.ts +14 -0
- package/dist/remoting/namespace.js +85 -0
- package/dist/remoting/pop_bodies.d.ts +58 -0
- package/dist/remoting/pop_bodies.js +214 -0
- package/dist/remoting/remotingCommand.d.ts +39 -0
- package/dist/remoting/remotingCommand.js +225 -0
- package/dist/remoting/route.d.ts +61 -0
- package/dist/remoting/route.js +288 -0
- package/dist/remoting/rpc_hooks.d.ts +31 -0
- package/dist/remoting/rpc_hooks.js +73 -0
- package/dist/remoting/serialize.d.ts +33 -0
- package/dist/remoting/serialize.js +372 -0
- package/dist/remoting/socks_proxy.d.ts +29 -0
- package/dist/remoting/socks_proxy.js +379 -0
- package/dist/remoting/static_topic.d.ts +49 -0
- package/dist/remoting/static_topic.js +244 -0
- package/dist/remoting/subscription.d.ts +65 -0
- package/dist/remoting/subscription.js +172 -0
- package/package.json +38 -4
|
@@ -0,0 +1,876 @@
|
|
|
1
|
+
// DefaultMQPullConsumer — the pull consumer.
|
|
2
|
+
//
|
|
3
|
+
// Key constraint: `pull()` is a SHORT pull — suspend=false in sysFlag and
|
|
4
|
+
// suspendTimeoutMillis=0 on the wire. Only `pullBlockIfNotFound` sets
|
|
5
|
+
// suspend=true, which makes the request wait for the window it carries instead
|
|
6
|
+
// of answering at once). Unconditionally writing suspend=true manifests as a 5s timeout
|
|
7
|
+
// on every idle pull.
|
|
8
|
+
import { PullAPI } from "./pull_api.js";
|
|
9
|
+
import { MQClient, DEFAULT_CONCURRENT_HEARTBEAT_THREAD_POOL_SIZE } from "./mq_client.js";
|
|
10
|
+
import { MQClientManager } from "./mq_client_manager.js";
|
|
11
|
+
import { registerRpcHooks } from "../remoting/rpc_hooks.js";
|
|
12
|
+
import { LocalFileOffsetStore, RemoteBrokerOffsetStore, ReadOffsetMode, mqKey, } from "./offset_store.js";
|
|
13
|
+
import { AllocateMessageQueueAveragely } from "./allocate.js";
|
|
14
|
+
import { MessageModel, ConsumeType } from "../remoting/heartbeat.js";
|
|
15
|
+
import { MessageQueue } from "../common/message.js";
|
|
16
|
+
import { clientMsgId } from "../common/messageClientIdSetter.js";
|
|
17
|
+
import { MixAll } from "../common/mixAll.js";
|
|
18
|
+
import { FilterAPI } from "../common/subscriptionData.js";
|
|
19
|
+
import { PullSysFlag } from "../common/sysflag.js";
|
|
20
|
+
import { RequestCode, ResponseCode } from "../remoting/codes.js";
|
|
21
|
+
import { RemotingCommand } from "../remoting/remotingCommand.js";
|
|
22
|
+
import { RemotingSerializable } from "../remoting/serialize.js";
|
|
23
|
+
import { ConsumerSendMsgBackRequestHeader } from "../remoting/headers.js";
|
|
24
|
+
import { Validators } from "../common/validators.js";
|
|
25
|
+
import { MQBrokerException, MQClientException } from "../remoting/exception.js";
|
|
26
|
+
import { buildRetryMessage } from "./retry_message.js";
|
|
27
|
+
import { getLogger } from "../logging.js";
|
|
28
|
+
const logger = getLogger('client.pull_consumer');
|
|
29
|
+
const DEFAULT_PULL_BATCH_SIZE = 32;
|
|
30
|
+
const DEFAULT_PULL_TIMEOUT_MILLIS = 20000;
|
|
31
|
+
export class DefaultMQPullConsumer {
|
|
32
|
+
consumerGroup;
|
|
33
|
+
namespace = '';
|
|
34
|
+
// namespaceV2: the server-side namespace (NamespaceRpcHook stamps the
|
|
35
|
+
// nsd/ns ext fields), independent of the client-side `namespace`.
|
|
36
|
+
namespaceV2 = null;
|
|
37
|
+
nameServerAddr = '';
|
|
38
|
+
messageModel = MessageModel.CLUSTERING;
|
|
39
|
+
unitMode = false;
|
|
40
|
+
// instanceName / unitName / enableStreamRequestType: pull consumers enable
|
|
41
|
+
// the stream flag by default — first so clientId gets `@STREAM` appended,
|
|
42
|
+
// preventing a pull consumer and a push consumer in the same process from
|
|
43
|
+
// silently sharing one client instance; second so every request carries
|
|
44
|
+
// ReqT=0 in the ACL signature.
|
|
45
|
+
instanceName = 'DEFAULT';
|
|
46
|
+
unitName = '';
|
|
47
|
+
enableStreamRequestType = true;
|
|
48
|
+
// wire fields read by MQClient.buildHeartbeatData:
|
|
49
|
+
consumeType = ConsumeType.CONSUME_ACTIVELY;
|
|
50
|
+
get subscriptionDataSet() { return [...this.subscription.values()]; }
|
|
51
|
+
clientID = '';
|
|
52
|
+
mqClient = null;
|
|
53
|
+
pullAPI = null;
|
|
54
|
+
offsetStore = null;
|
|
55
|
+
pullBatchSize = DEFAULT_PULL_BATCH_SIZE;
|
|
56
|
+
pullTimeoutMillis = DEFAULT_PULL_TIMEOUT_MILLIS;
|
|
57
|
+
// Retry ceiling reported with message redelivery: pull mode's own default is
|
|
58
|
+
// 16, and -1 ("unlimited") is reported as 16 as well.
|
|
59
|
+
maxReconsumeTimes = 16;
|
|
60
|
+
// Offset flush period (default 5000 ms): applied to
|
|
61
|
+
// the shared MQClient instance when THIS facade creates it (first creator's
|
|
62
|
+
// config wins). The instance-level persistAllConsumerOffset sweep is what
|
|
63
|
+
// flushes this facade's offsets between the explicit shutdown/revoke flushes.
|
|
64
|
+
persistConsumerOffsetInterval = 5000;
|
|
65
|
+
// Heartbeat V2: off by default, so heartbeats keep the V1 shape unless enabled.
|
|
66
|
+
useHeartbeatV2 = false;
|
|
67
|
+
// Materialise received message bodies during batch decode (default true).
|
|
68
|
+
decodeReadBody = true;
|
|
69
|
+
// Internal-API timeout (heartbeat, route fetch, check client, unregister and
|
|
70
|
+
// consumer-list RPCs), in milliseconds. Default 3000.
|
|
71
|
+
mqClientApiTimeout = 3000;
|
|
72
|
+
// Concurrent heartbeat round (ignored while heartbeat V2 is on).
|
|
73
|
+
enableConcurrentHeartbeat = false;
|
|
74
|
+
concurrentHeartbeatThreadPoolSize = DEFAULT_CONCURRENT_HEARTBEAT_THREAD_POOL_SIZE;
|
|
75
|
+
// Page size of the paged metadata admin calls (maxTopicNum / maxGroupNum).
|
|
76
|
+
maxPageSizeInGetMetadata = 2000;
|
|
77
|
+
// ---- pull-side rebalance (Java DefaultMQPullConsumer.java:97 / impl :388-406) ----
|
|
78
|
+
// Java's pull consumer DOES rebalance: the shared RebalanceService thread calls
|
|
79
|
+
// `mqClientFactory.doRebalance()` every `waitInterval` (20s, system property
|
|
80
|
+
// `rocketmq.client.rebalance.waitInterval`), which walks the registered consumers
|
|
81
|
+
// and calls `tryRebalance()`; that gates on `enableRebalance` (default true) and
|
|
82
|
+
// then runs `rebalanceImpl.doRebalance(false)` -> allocate -> `messageQueueChanged`.
|
|
83
|
+
//
|
|
84
|
+
// Deviation (documented): Java runs ONE RebalanceService per client instance, this
|
|
85
|
+
// port gives each facade its own timer. The observable behaviour -- which queues
|
|
86
|
+
// this instance owns and when the listener fires -- is the same.
|
|
87
|
+
enableRebalance = true;
|
|
88
|
+
rebalanceWaitMillis = 20000;
|
|
89
|
+
rebalanceMinIntervalMillis = 1000;
|
|
90
|
+
// The allocation strategy: used by the background rebalance and by
|
|
91
|
+
// fetchMessageQueuesInBalance's on-the-spot computation, so both always agree on
|
|
92
|
+
// which queues are mine (diverging would have two instances claim one queue).
|
|
93
|
+
allocateMessageQueueStrategy = new AllocateMessageQueueAveragely();
|
|
94
|
+
// messageQueueChanged callback, in this port's function shape (the push consumer
|
|
95
|
+
// uses the same one). Java passes two Sets; arrays with stable order are used here.
|
|
96
|
+
messageQueueListener = null;
|
|
97
|
+
// The product of the background rebalance: topic -> the queues allocated to this
|
|
98
|
+
// instance. Java reads the keys of the rebalance's process-queue table; here it is
|
|
99
|
+
// a plain table, and fetchMessageQueuesInBalance prefers it when it is warm.
|
|
100
|
+
rebalanceTable = new Map();
|
|
101
|
+
// Topics registered for rebalance. Java's registerMessageQueueListener(topic,
|
|
102
|
+
// listener) fills this; subscribe() fills `subscription`. The rebalance walks the
|
|
103
|
+
// union, so either registration path gets an allocation.
|
|
104
|
+
registerTopics = new Set();
|
|
105
|
+
_rebalanceTimer = null;
|
|
106
|
+
_rebalanceActive = false;
|
|
107
|
+
subscription = new Map();
|
|
108
|
+
started = false;
|
|
109
|
+
// The internal producer the send-back fallback sends through; built lazily on
|
|
110
|
+
// the first refused bounce and torn down with the consumer.
|
|
111
|
+
_producer = null;
|
|
112
|
+
constructor(consumerGroup = 'DEFAULT_CONSUMER') {
|
|
113
|
+
this.consumerGroup = consumerGroup || 'DEFAULT_CONSUMER';
|
|
114
|
+
}
|
|
115
|
+
setNamesrvAddr(addr) { this.nameServerAddr = addr; return this; }
|
|
116
|
+
setNamespace(ns) { this.namespace = ns; return this; }
|
|
117
|
+
// Set/read namespaceV2 (read live per request).
|
|
118
|
+
setNamespaceV2(ns) { this.namespaceV2 = ns; return this; }
|
|
119
|
+
getNamespaceV2() { return this.namespaceV2; }
|
|
120
|
+
setMessageModel(model) { this.messageModel = model; return this; }
|
|
121
|
+
setInstanceName(name) { this.instanceName = name; return this; }
|
|
122
|
+
setUnitName(name) { this.unitName = name; return this; }
|
|
123
|
+
setPullBatchSize(n) { this.pullBatchSize = n; return this; }
|
|
124
|
+
setPullTimeoutMillis(ms) { this.pullTimeoutMillis = ms; return this; }
|
|
125
|
+
setMaxReconsumeTimes(n) { this.maxReconsumeTimes = n; return this; }
|
|
126
|
+
// Flush period for consumed offsets, in milliseconds; must be greater than 0.
|
|
127
|
+
// When already started the change is pushed through to the shared instance's
|
|
128
|
+
// sweep timer immediately.
|
|
129
|
+
setPersistConsumerOffsetInterval(ms) {
|
|
130
|
+
if (!Number.isFinite(ms) || ms <= 0)
|
|
131
|
+
throw new Error(`persistConsumerOffsetInterval must be > 0, got ${ms}`);
|
|
132
|
+
this.persistConsumerOffsetInterval = Math.floor(ms);
|
|
133
|
+
if (this.mqClient)
|
|
134
|
+
this.mqClient.setPersistConsumerOffsetInterval(this.persistConsumerOffsetInterval);
|
|
135
|
+
return this;
|
|
136
|
+
}
|
|
137
|
+
getPersistConsumerOffsetInterval() { return this.persistConsumerOffsetInterval; }
|
|
138
|
+
// Heartbeat V2 switch; pushed through to
|
|
139
|
+
// the shared instance when this facade already owns one.
|
|
140
|
+
setUseHeartbeatV2(enable) {
|
|
141
|
+
this.useHeartbeatV2 = enable === true;
|
|
142
|
+
if (this.mqClient)
|
|
143
|
+
this.mqClient.setUseHeartbeatV2(this.useHeartbeatV2);
|
|
144
|
+
return this;
|
|
145
|
+
}
|
|
146
|
+
getUseHeartbeatV2() { return this.useHeartbeatV2; }
|
|
147
|
+
// Received-body decode switch, mirrored onto the shared instance.
|
|
148
|
+
setDecodeReadBody(enable) {
|
|
149
|
+
this.decodeReadBody = enable === true;
|
|
150
|
+
if (this.mqClient)
|
|
151
|
+
this.mqClient.setDecodeReadBody(this.decodeReadBody);
|
|
152
|
+
return this;
|
|
153
|
+
}
|
|
154
|
+
getDecodeReadBody() { return this.decodeReadBody; }
|
|
155
|
+
// Internal-API timeout, in milliseconds. An explicit per-call timeout passed
|
|
156
|
+
// by a caller still wins.
|
|
157
|
+
setMqClientApiTimeout(ms) {
|
|
158
|
+
if (!Number.isFinite(ms) || ms <= 0)
|
|
159
|
+
throw new Error(`mqClientApiTimeout must be > 0, got ${ms}`);
|
|
160
|
+
this.mqClientApiTimeout = Math.floor(ms);
|
|
161
|
+
if (this.mqClient)
|
|
162
|
+
this.mqClient.setMqClientApiTimeout(this.mqClientApiTimeout);
|
|
163
|
+
return this;
|
|
164
|
+
}
|
|
165
|
+
getMqClientApiTimeout() { return this.mqClientApiTimeout; }
|
|
166
|
+
// Concurrent heartbeat round; heartbeat V2 still wins when it is enabled.
|
|
167
|
+
setEnableConcurrentHeartbeat(enable) {
|
|
168
|
+
this.enableConcurrentHeartbeat = enable === true;
|
|
169
|
+
if (this.mqClient)
|
|
170
|
+
this.mqClient.setEnableConcurrentHeartbeat(this.enableConcurrentHeartbeat);
|
|
171
|
+
return this;
|
|
172
|
+
}
|
|
173
|
+
getEnableConcurrentHeartbeat() { return this.enableConcurrentHeartbeat; }
|
|
174
|
+
setConcurrentHeartbeatThreadPoolSize(size) {
|
|
175
|
+
if (!Number.isInteger(size) || size <= 0) {
|
|
176
|
+
throw new Error(`concurrentHeartbeatThreadPoolSize must be a positive integer, got ${size}`);
|
|
177
|
+
}
|
|
178
|
+
this.concurrentHeartbeatThreadPoolSize = size;
|
|
179
|
+
if (this.mqClient)
|
|
180
|
+
this.mqClient.setConcurrentHeartbeatThreadPoolSize(size);
|
|
181
|
+
return this;
|
|
182
|
+
}
|
|
183
|
+
getConcurrentHeartbeatThreadPoolSize() { return this.concurrentHeartbeatThreadPoolSize; }
|
|
184
|
+
setMaxPageSizeInGetMetadata(size) {
|
|
185
|
+
if (!Number.isInteger(size) || size <= 0) {
|
|
186
|
+
throw new Error(`maxPageSizeInGetMetadata must be a positive integer, got ${size}`);
|
|
187
|
+
}
|
|
188
|
+
this.maxPageSizeInGetMetadata = size;
|
|
189
|
+
if (this.mqClient)
|
|
190
|
+
this.mqClient.setMaxPageSizeInGetMetadata(size);
|
|
191
|
+
return this;
|
|
192
|
+
}
|
|
193
|
+
getMaxPageSizeInGetMetadata() { return this.maxPageSizeInGetMetadata; }
|
|
194
|
+
// ---- pull-side rebalance configuration ----
|
|
195
|
+
// Java DefaultMQPullConsumer#setEnableRebalance (:507). Off means
|
|
196
|
+
// doRebalance/tryRebalance return without allocating anything, so the caller's
|
|
197
|
+
// own fetchMessageQueuesInBalance keeps computing on the spot.
|
|
198
|
+
setEnableRebalance(enable) { this.enableRebalance = enable === true; return this; }
|
|
199
|
+
isEnableRebalance() { return this.enableRebalance; }
|
|
200
|
+
// Round period: Java system property `rocketmq.client.rebalance.waitInterval` (20000).
|
|
201
|
+
setRebalanceWaitMillis(ms) {
|
|
202
|
+
this.rebalanceWaitMillis = Number.isFinite(ms) ? Math.max(1, Math.floor(ms)) : 1;
|
|
203
|
+
return this;
|
|
204
|
+
}
|
|
205
|
+
// Lower bound between two rounds: Java `rocketmq.client.rebalance.minInterval` (1000).
|
|
206
|
+
setRebalanceMinIntervalMillis(ms) {
|
|
207
|
+
this.rebalanceMinIntervalMillis = Number.isFinite(ms) ? Math.max(1, Math.floor(ms)) : 1;
|
|
208
|
+
return this;
|
|
209
|
+
}
|
|
210
|
+
setAllocateMessageQueueStrategy(strategy) {
|
|
211
|
+
// start()/the rebalance would otherwise fail on every round; like Java, an
|
|
212
|
+
// explicit null is rejected rather than silently replaced.
|
|
213
|
+
if (strategy == null)
|
|
214
|
+
throw new Error('allocateMessageQueueStrategy is null');
|
|
215
|
+
this.allocateMessageQueueStrategy = strategy;
|
|
216
|
+
return this;
|
|
217
|
+
}
|
|
218
|
+
setMessageQueueListener(listener) {
|
|
219
|
+
this.messageQueueListener = listener;
|
|
220
|
+
return this;
|
|
221
|
+
}
|
|
222
|
+
// Java DefaultMQPullConsumer#registerMessageQueueListener (:328-335): registers the
|
|
223
|
+
// topic AND installs the listener in one call. The topic matters because the
|
|
224
|
+
// background rebalance only visits registered topics; a listener installed without
|
|
225
|
+
// one would silently never fire. A null listener keeps the current one.
|
|
226
|
+
registerMessageQueueListener(topic, listener) {
|
|
227
|
+
if (topic == null || !String(topic).trim())
|
|
228
|
+
return this;
|
|
229
|
+
this.registerTopics.add(this._withNamespace(String(topic)));
|
|
230
|
+
if (listener != null)
|
|
231
|
+
this.messageQueueListener = listener;
|
|
232
|
+
return this;
|
|
233
|
+
}
|
|
234
|
+
getRegisterTopics() { return new Set(this.registerTopics); }
|
|
235
|
+
// Snapshot of the background rebalance's product. Callers get fresh arrays, so
|
|
236
|
+
// mutating the answer cannot change which queues this instance owns.
|
|
237
|
+
getRebalanceTable() {
|
|
238
|
+
const out = new Map();
|
|
239
|
+
for (const [topic, mqs] of this.rebalanceTable)
|
|
240
|
+
out.set(topic, [...mqs]);
|
|
241
|
+
return out;
|
|
242
|
+
}
|
|
243
|
+
getConsumerGroup() { return this.consumerGroup; }
|
|
244
|
+
isStarted() { return this.started; }
|
|
245
|
+
// subscribe registers a topic subscription. A string is a tag expression
|
|
246
|
+
// ("*" = all); a MessageSelector selects the expression type — bySql rides the
|
|
247
|
+
// wire as expressionType=SQL92 and is evaluated BY THE BROKER: this
|
|
248
|
+
// client has no SQL92 engine, and FilterAPI.build keeps tagsSet
|
|
249
|
+
// EMPTY so the client-side tag filter stays off.
|
|
250
|
+
subscribe(topic, subExpression = '*') {
|
|
251
|
+
if (!topic || !topic.trim())
|
|
252
|
+
throw new Error('subscription topic is empty');
|
|
253
|
+
topic = this._withNamespace(topic);
|
|
254
|
+
this.subscription.set(topic, this._buildSub(topic, subExpression));
|
|
255
|
+
return this;
|
|
256
|
+
}
|
|
257
|
+
// ---- namespace (client-side prefix) ----
|
|
258
|
+
// With a namespace set the broker-side name of every resource is the prefixed
|
|
259
|
+
// one: subscriptions, the group, route lookups and pull requests all use it,
|
|
260
|
+
// and delivered messages get the bare topic back before the caller sees them.
|
|
261
|
+
_withNamespace(resource) {
|
|
262
|
+
return MixAll.wrapNamespace(this.namespace, resource);
|
|
263
|
+
}
|
|
264
|
+
_queueWithNamespace(mq) {
|
|
265
|
+
if (!this.namespace)
|
|
266
|
+
return mq;
|
|
267
|
+
const topic = this._withNamespace(mq.getTopic());
|
|
268
|
+
return topic === mq.getTopic() ? mq : new MessageQueue(topic, mq.getBrokerName(), mq.getQueueId());
|
|
269
|
+
}
|
|
270
|
+
_bareQueue(mq) {
|
|
271
|
+
if (!this.namespace)
|
|
272
|
+
return mq;
|
|
273
|
+
const topic = MixAll.withoutNamespace(mq.getTopic(), this.namespace);
|
|
274
|
+
return topic === mq.getTopic() ? mq : new MessageQueue(topic, mq.getBrokerName(), mq.getQueueId());
|
|
275
|
+
}
|
|
276
|
+
_buildSub(topic, subExpression) {
|
|
277
|
+
if (typeof subExpression === 'string') {
|
|
278
|
+
return FilterAPI.buildSubscriptionData(topic, subExpression || '*');
|
|
279
|
+
}
|
|
280
|
+
return FilterAPI.build(topic, subExpression.expression, subExpression.type);
|
|
281
|
+
}
|
|
282
|
+
async start() {
|
|
283
|
+
if (this.started)
|
|
284
|
+
return;
|
|
285
|
+
// The broker-side group carries the prefix; wrapNamespace is idempotent, so
|
|
286
|
+
// a restart after shutdown does not stack prefixes.
|
|
287
|
+
this.consumerGroup = this._withNamespace(this.consumerGroup);
|
|
288
|
+
Validators.checkGroup(this.consumerGroup);
|
|
289
|
+
if (!this.nameServerAddr)
|
|
290
|
+
throw new Error('name server address is not set');
|
|
291
|
+
// No subscription is required to start: a pull consumer names the topic in
|
|
292
|
+
// fetchSubscribeMessageQueues/pull and may pass the expression inline, so
|
|
293
|
+
// subscribe() is only one of the ways to get one.
|
|
294
|
+
// Only CLUSTERING rewrites the default instanceName (BROADCASTING keeps
|
|
295
|
+
// "DEFAULT", letting same-process instances share one client instance);
|
|
296
|
+
// clientId has the form `<ip>@<instanceName>[@<unitName>][@STREAM]`. ⚠ Using
|
|
297
|
+
// pid alone as the instance name collides between two pull consumers in the
|
|
298
|
+
// same process — allocate() hands the same queue slice to both, causing
|
|
299
|
+
// duplicate consumption.
|
|
300
|
+
if (this.messageModel === MessageModel.CLUSTERING) {
|
|
301
|
+
this.instanceName = MixAll.changeInstanceNameToPid(this.instanceName);
|
|
302
|
+
}
|
|
303
|
+
this.clientID = MixAll.clientIdFor(this.instanceName, this.unitName || null, this.enableStreamRequestType);
|
|
304
|
+
const manager = MQClientManager.getInstance();
|
|
305
|
+
const { instance: client, created } = manager.getOrCreateMQClientInstance(this.clientID, () => new MQClient(this.clientID, this.nameServerAddr));
|
|
306
|
+
if (created) {
|
|
307
|
+
client.setPersistConsumerOffsetInterval(this.persistConsumerOffsetInterval);
|
|
308
|
+
if (this.useHeartbeatV2)
|
|
309
|
+
client.setUseHeartbeatV2(true);
|
|
310
|
+
client.setDecodeReadBody(this.decodeReadBody);
|
|
311
|
+
client.setMqClientApiTimeout(this.mqClientApiTimeout);
|
|
312
|
+
client.setEnableConcurrentHeartbeat(this.enableConcurrentHeartbeat);
|
|
313
|
+
client.setConcurrentHeartbeatThreadPoolSize(this.concurrentHeartbeatThreadPoolSize);
|
|
314
|
+
client.setMaxPageSizeInGetMetadata(this.maxPageSizeInGetMetadata);
|
|
315
|
+
registerRpcHooks(client.remotingClient, {
|
|
316
|
+
namespaceV2: () => this.namespaceV2,
|
|
317
|
+
enableStreamRequestType: this.enableStreamRequestType,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
manager.registerClient(this.clientID, client);
|
|
321
|
+
this.mqClient = client;
|
|
322
|
+
client.registerConsumer(this.consumerGroup, this);
|
|
323
|
+
client.start();
|
|
324
|
+
if (this.messageModel === MessageModel.BROADCASTING) {
|
|
325
|
+
this.offsetStore = new LocalFileOffsetStore(this.clientID, this.consumerGroup);
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
this.offsetStore = new RemoteBrokerOffsetStore(client, this.consumerGroup);
|
|
329
|
+
}
|
|
330
|
+
this.pullAPI = new PullAPI(client, this.consumerGroup);
|
|
331
|
+
await this.offsetStore.load().catch((e) => logger.warning('load offset store failed: %s', e.message));
|
|
332
|
+
for (const topic of this._rebalanceTopics()) {
|
|
333
|
+
await client.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
|
|
334
|
+
}
|
|
335
|
+
// Register the group at the broker immediately (see lite_pull_consumer:
|
|
336
|
+
// the client heartbeat loop's 30s cadence is too slow for first rebalance).
|
|
337
|
+
await client.sendHeartbeatToAllBrokers().catch(() => { });
|
|
338
|
+
this.started = true;
|
|
339
|
+
// Java registers the pull consumer's rebalance with the shared RebalanceService
|
|
340
|
+
// in start(); here the facade drives its own unref'd timer (see the field block).
|
|
341
|
+
this._rebalanceActive = true;
|
|
342
|
+
this._startRebalanceLoop();
|
|
343
|
+
}
|
|
344
|
+
async shutdown() {
|
|
345
|
+
if (!this.started)
|
|
346
|
+
return;
|
|
347
|
+
// Stop the rebalance loop first: it calls back into the client instance which is
|
|
348
|
+
// about to be handed back to the manager.
|
|
349
|
+
this._rebalanceActive = false;
|
|
350
|
+
if (this._rebalanceTimer) {
|
|
351
|
+
clearTimeout(this._rebalanceTimer);
|
|
352
|
+
this._rebalanceTimer = null;
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
await this.persistConsumeOffset();
|
|
356
|
+
}
|
|
357
|
+
catch (e) { /* best effort */ }
|
|
358
|
+
if (this.mqClient) {
|
|
359
|
+
await this.mqClient.unregisterClientAllBrokers('', this.consumerGroup).catch(() => { });
|
|
360
|
+
this.mqClient.unregisterConsumer(this.consumerGroup);
|
|
361
|
+
// Hand back to the manager: count -1; the real shutdown happens only when
|
|
362
|
+
// it hits zero (when other facades share the same clientId, the instance
|
|
363
|
+
// and its background timers keep running).
|
|
364
|
+
MQClientManager.getInstance().unregisterClient(this.mqClient.clientId, this.mqClient);
|
|
365
|
+
}
|
|
366
|
+
// The lazily built fallback producer owns its own client slot, so it is
|
|
367
|
+
// torn down explicitly rather than left to the manager's refcount.
|
|
368
|
+
if (this._producer) {
|
|
369
|
+
try {
|
|
370
|
+
await this._producer.shutdown();
|
|
371
|
+
}
|
|
372
|
+
catch (e) { /* ignore */ }
|
|
373
|
+
this._producer = null;
|
|
374
|
+
}
|
|
375
|
+
this.started = false;
|
|
376
|
+
}
|
|
377
|
+
// fetchPublishMessageQueues: the PUBLISH view (write perm, writeQueueNums,
|
|
378
|
+
// must have a master). ⚠ This once wrongly returned the SUBSCRIBE view: when
|
|
379
|
+
// readQueueNums != writeQueueNums, or when the route has no master broker,
|
|
380
|
+
// the two lists differ, and a caller sharding by this list would read queues
|
|
381
|
+
// not assigned to it (or miss the ones that were).
|
|
382
|
+
async fetchPublishMessageQueues(topic) {
|
|
383
|
+
const full = this._withNamespace(topic);
|
|
384
|
+
return (await this.routeOf(full, 'publish')).getAllMessageQueue(full);
|
|
385
|
+
}
|
|
386
|
+
// fetchSubscribeMessageQueues: the SUBSCRIBE view (read perm, readQueueNums,
|
|
387
|
+
// no master filtering). This is the list a puller iterates.
|
|
388
|
+
async fetchSubscribeMessageQueues(topic) {
|
|
389
|
+
const full = this._withNamespace(topic);
|
|
390
|
+
const queues = (await this.routeOf(full, 'subscribe')).getAllSubscribeMessageQueue(full);
|
|
391
|
+
// The queue list is the caller's view: hand back the bare topic, as the
|
|
392
|
+
// pull/offset APIs accept it and prefix it again internally.
|
|
393
|
+
return queues.map((mq) => this._bareQueue(mq));
|
|
394
|
+
}
|
|
395
|
+
async routeOf(topic, view) {
|
|
396
|
+
if (!this.mqClient)
|
|
397
|
+
throw new Error('consumer not started');
|
|
398
|
+
await this.mqClient.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
|
|
399
|
+
const route = this.mqClient.getTopicRouteData(topic);
|
|
400
|
+
if (!route) {
|
|
401
|
+
throw new Error(`Can not find MessageQueue for topic: ${topic}`);
|
|
402
|
+
}
|
|
403
|
+
return route;
|
|
404
|
+
}
|
|
405
|
+
// pull is a SHORT poll (rule #10): suspend=false, suspendTimeoutMillis=0.
|
|
406
|
+
async pull(mq, subExpression, offset, maxNums, timeoutMillis = this.pullTimeoutMillis) {
|
|
407
|
+
return this._pullImpl(mq, subExpression, offset, maxNums, false, 0, timeoutMillis);
|
|
408
|
+
}
|
|
409
|
+
// pullAsync -- the callback form of pull(mq, subExpression, offset,
|
|
410
|
+
// maxNums, PullCallback, timeout). Event-loop semantics: the callback fires
|
|
411
|
+
// EXACTLY ONCE (success XOR exception) and NEVER inline -- even an immediate
|
|
412
|
+
// local failure (consumer not started, no subscription) is delivered through
|
|
413
|
+
// setImmediate, so the caller's statement always completes first.
|
|
414
|
+
pullAsync(mq, subExpression, offset, maxNums, callback, timeoutMillis = this.pullTimeoutMillis) {
|
|
415
|
+
if (callback == null || typeof callback.onSuccess !== 'function'
|
|
416
|
+
|| typeof callback.onException !== 'function') {
|
|
417
|
+
throw new Error('pullAsync requires a PullCallback with onSuccess/onException');
|
|
418
|
+
}
|
|
419
|
+
let delivered = false;
|
|
420
|
+
const deliver = (fn) => {
|
|
421
|
+
if (delivered)
|
|
422
|
+
return; // exactly once, even if both settle paths raced
|
|
423
|
+
delivered = true;
|
|
424
|
+
setImmediate(() => {
|
|
425
|
+
try {
|
|
426
|
+
fn();
|
|
427
|
+
}
|
|
428
|
+
catch (e) {
|
|
429
|
+
logger.warning('pullAsync callback raised: %s', e.message);
|
|
430
|
+
}
|
|
431
|
+
});
|
|
432
|
+
};
|
|
433
|
+
this._pullImpl(mq, subExpression, offset, maxNums, false, 0, timeoutMillis).then((r) => deliver(() => callback.onSuccess(r)), (e) => deliver(() => callback.onException(e instanceof Error ? e : new Error(String(e)))));
|
|
434
|
+
}
|
|
435
|
+
// pullBlockIfNotFound is a LONG poll: suspend=true, so the request carries a
|
|
436
|
+
// suspend window (suspendTimeoutMillis) instead of answering at once, and a
|
|
437
|
+
// round that finds nothing reports an idle status — a benign timeout on the
|
|
438
|
+
// client is NORMAL here.
|
|
439
|
+
async pullBlockIfNotFound(mq, subExpression, offset, maxNums, timeoutMillis = this.pullTimeoutMillis, suspendTimeoutMillis = 20000) {
|
|
440
|
+
return this._pullImpl(mq, subExpression, offset, maxNums, true, suspendTimeoutMillis, timeoutMillis);
|
|
441
|
+
}
|
|
442
|
+
async _pullImpl(mq, subExpression, offset, maxNums, suspend, suspendTimeoutMillis, timeoutMillis) {
|
|
443
|
+
// The pull entry checks the service state first: a shutdown facade must report,
|
|
444
|
+
// never keep pulling through the (still open) client instance it shared.
|
|
445
|
+
if (!this.started || !this.pullAPI)
|
|
446
|
+
throw new Error('consumer not started');
|
|
447
|
+
if (offset < 0)
|
|
448
|
+
throw new MQClientException('offset < 0');
|
|
449
|
+
if (maxNums <= 0)
|
|
450
|
+
throw new MQClientException('maxNums <= 0');
|
|
451
|
+
const fullMq = this._queueWithNamespace(mq);
|
|
452
|
+
const topic = fullMq.getTopic();
|
|
453
|
+
let sub;
|
|
454
|
+
if (subExpression == null) {
|
|
455
|
+
const cached = this.subscription.get(topic);
|
|
456
|
+
if (!cached)
|
|
457
|
+
throw new Error(`subscription for topic ${topic} is not set`);
|
|
458
|
+
sub = cached;
|
|
459
|
+
}
|
|
460
|
+
else {
|
|
461
|
+
sub = this._buildSub(topic, subExpression);
|
|
462
|
+
}
|
|
463
|
+
// Pull / lite-pull consumers ALWAYS send the expression (rule #21):
|
|
464
|
+
// subscription bit on.
|
|
465
|
+
const sysFlag = PullSysFlag.buildSysFlag(false, suspend, true, false);
|
|
466
|
+
let result = await this.pullAPI.pullKernel(fullMq, offset, sub, sysFlag, 0, maxNums, suspendTimeoutMillis, timeoutMillis);
|
|
467
|
+
result = this.pullAPI.processPullResult(fullMq, result, sub);
|
|
468
|
+
if (this.namespace) {
|
|
469
|
+
// The caller handed in the bare topic; hand it back on the messages too.
|
|
470
|
+
for (const msg of result.msgFoundList) {
|
|
471
|
+
msg.setTopic(MixAll.withoutNamespace(msg.getTopic(), this.namespace));
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
return result;
|
|
475
|
+
}
|
|
476
|
+
// updateConsumeOffset records the consumed offset (the store persists it).
|
|
477
|
+
updateConsumeOffset(mq, offset) {
|
|
478
|
+
if (!this.offsetStore)
|
|
479
|
+
throw new Error('consumer not started');
|
|
480
|
+
if (offset < 0)
|
|
481
|
+
throw new Error(`offset must be >= 0, got ${offset}`);
|
|
482
|
+
this.offsetStore.updateOffset(this._queueWithNamespace(mq), offset, true);
|
|
483
|
+
}
|
|
484
|
+
// fetchConsumeOffset reads the consumed offset (-1 when unknown).
|
|
485
|
+
async fetchConsumeOffset(mq, fromStore) {
|
|
486
|
+
if (!this.offsetStore)
|
|
487
|
+
throw new Error('consumer not started');
|
|
488
|
+
const fullMq = this._queueWithNamespace(mq);
|
|
489
|
+
if (fromStore)
|
|
490
|
+
await this._warmQueueRoute(fullMq.getTopic());
|
|
491
|
+
return this.offsetStore.readOffset(fullMq, fromStore ? ReadOffsetMode.READ_FROM_STORE : ReadOffsetMode.READ_FROM_MEMORY_THEN_STORE);
|
|
492
|
+
}
|
|
493
|
+
// The offset store resolves a queue's broker address from the cached route
|
|
494
|
+
// table, and a consumer started without subscribe() has nothing cached: a cold
|
|
495
|
+
// store read then silently answers -1 — the caller restarts at zero and
|
|
496
|
+
// re-consumes the whole topic — while a cold persist throws.
|
|
497
|
+
async _warmQueueRoute(topic) {
|
|
498
|
+
if (!this.mqClient || !topic)
|
|
499
|
+
return;
|
|
500
|
+
if (this.mqClient.getTopicRouteData(topic))
|
|
501
|
+
return;
|
|
502
|
+
await this.mqClient.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
|
|
503
|
+
}
|
|
504
|
+
// persistConsumeOffset flushes all recorded offsets.
|
|
505
|
+
async persistConsumeOffset() {
|
|
506
|
+
if (!this.offsetStore)
|
|
507
|
+
return;
|
|
508
|
+
const snap = this.offsetStore.tableSnapshot
|
|
509
|
+
? this.offsetStore.tableSnapshot() : new Map();
|
|
510
|
+
const mqs = [];
|
|
511
|
+
for (const key of snap.keys()) {
|
|
512
|
+
const i1 = key.indexOf('@');
|
|
513
|
+
const i2 = key.lastIndexOf('@');
|
|
514
|
+
if (i1 > 0 && i2 > i1) {
|
|
515
|
+
mqs.push(new MessageQueue(key.slice(0, i1), key.slice(i1 + 1, i2), parseInt(key.slice(i2 + 1), 10) || 0));
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
for (const topic of new Set(mqs.map(mq => mq.getTopic()))) {
|
|
519
|
+
await this._warmQueueRoute(topic);
|
|
520
|
+
}
|
|
521
|
+
await this.offsetStore.persistAll(mqs);
|
|
522
|
+
}
|
|
523
|
+
// persistConsumerOffset — the MQConsumerInner contract name the MQClient's
|
|
524
|
+
// instance-level persistAllConsumerOffset sweep calls every
|
|
525
|
+
// persistConsumerOffsetInterval; delegates to persistConsumeOffset.
|
|
526
|
+
async persistConsumerOffset() {
|
|
527
|
+
return this.persistConsumeOffset();
|
|
528
|
+
}
|
|
529
|
+
// maxOffset -- the queue tail on the MASTER broker, i.e. the highest
|
|
530
|
+
// offset ever written to that queue. Throws when the route/broker is unknown;
|
|
531
|
+
// the pull schedule context treats that as "start from 0".
|
|
532
|
+
async maxOffset(mq) {
|
|
533
|
+
if (!this.mqClient)
|
|
534
|
+
throw new Error('consumer not started');
|
|
535
|
+
const fullMq = this._queueWithNamespace(mq);
|
|
536
|
+
await this._warmQueueRoute(fullMq.getTopic());
|
|
537
|
+
const addrs = this.mqClient.getBrokerAddrTable().get(fullMq.getBrokerName());
|
|
538
|
+
const addr = addrs != null ? addrs[MixAll.MASTER_ID] : null;
|
|
539
|
+
if (addr == null)
|
|
540
|
+
throw new Error(`Broker[${fullMq.getBrokerName()}] master node does not exist`);
|
|
541
|
+
return this.mqClient.getMaxOffset(addr, fullMq.getTopic(), fullMq.getQueueId());
|
|
542
|
+
}
|
|
543
|
+
// fetchMessageQueuesInBalance is Java DefaultMQPullConsumerImpl's pure query
|
|
544
|
+
// (:408-...): when the background rebalance has already produced this topic's
|
|
545
|
+
// share, that table IS the answer. Otherwise (rebalance off, or the table is
|
|
546
|
+
// still cold) the same allocation is computed on the spot. Neither path fires
|
|
547
|
+
// the listener or writes the table — a read must not look like a queue change.
|
|
548
|
+
async fetchMessageQueuesInBalance(topic) {
|
|
549
|
+
const queues = await this._fetchMessageQueuesInBalance(this._withNamespace(topic));
|
|
550
|
+
return queues.map((mq) => this._bareQueue(mq));
|
|
551
|
+
}
|
|
552
|
+
async _fetchMessageQueuesInBalance(topic) {
|
|
553
|
+
if (!this.mqClient)
|
|
554
|
+
throw new Error('consumer not started');
|
|
555
|
+
// A warm table is authoritative. While the switch is off the stale table must
|
|
556
|
+
// be ignored: the caller asked for the on-the-spot allocation.
|
|
557
|
+
if (this.enableRebalance) {
|
|
558
|
+
const cached = this.rebalanceTable.get(topic);
|
|
559
|
+
if (cached !== undefined) {
|
|
560
|
+
return [...cached].sort((a, b) => a.compareTo(b));
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
const keep = (why) => {
|
|
564
|
+
logger.debug(`fetchMessageQueuesInBalance: ${why} for ${this.consumerGroup}/${topic}`
|
|
565
|
+
+ ', keep current assignment');
|
|
566
|
+
const api = this.pullAPI;
|
|
567
|
+
if (!api)
|
|
568
|
+
return [];
|
|
569
|
+
return api.pulledQueues().filter(mq => mq.getTopic() === topic)
|
|
570
|
+
.sort((a, b) => a.compareTo(b));
|
|
571
|
+
};
|
|
572
|
+
let mqAll;
|
|
573
|
+
try {
|
|
574
|
+
mqAll = (await this.routeOf(topic, 'subscribe')).getAllSubscribeMessageQueue(topic);
|
|
575
|
+
}
|
|
576
|
+
catch (e) {
|
|
577
|
+
return keep(`no route (${e.message})`);
|
|
578
|
+
}
|
|
579
|
+
mqAll = [...mqAll].sort((a, b) => a.compareTo(b));
|
|
580
|
+
if (!mqAll.length)
|
|
581
|
+
return keep('no readable queue in the route');
|
|
582
|
+
if (this.messageModel === MessageModel.BROADCASTING)
|
|
583
|
+
return mqAll;
|
|
584
|
+
// The broker learns about a group only from a HEARTBEAT carrying its
|
|
585
|
+
// ConsumerData. start() beats once before any topic route is cached, so a
|
|
586
|
+
// consumer that starts without subscribe() has an empty address table and
|
|
587
|
+
// that beat reaches nobody. This one-shot query beats again; the background
|
|
588
|
+
// rebalance does not (it is the heartbeat loop's job, as in Java).
|
|
589
|
+
const ids = await this._getConsumerIdsFor(topic, true);
|
|
590
|
+
if (!ids)
|
|
591
|
+
return keep('the group is unknown to the broker');
|
|
592
|
+
try {
|
|
593
|
+
const got = this.allocateMessageQueueStrategy.allocate(this.consumerGroup, this.clientID, mqAll, [...ids].sort());
|
|
594
|
+
return (got || []).filter(mq => mq.getTopic() === topic)
|
|
595
|
+
.sort((a, b) => a.compareTo(b));
|
|
596
|
+
}
|
|
597
|
+
catch (e) {
|
|
598
|
+
return keep(`allocation failed (${e.message})`);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
// _getConsumerIdsFor asks one master broker for the group's consumer id list.
|
|
602
|
+
// `null` means "unavailable" (no route, no master, a failed query or an empty
|
|
603
|
+
// list) — both callers keep the existing assignment and never read it as
|
|
604
|
+
// "I am the only one, so I own everything".
|
|
605
|
+
async _getConsumerIdsFor(topic, sendHeartbeat = false) {
|
|
606
|
+
const client = this.mqClient;
|
|
607
|
+
if (!client)
|
|
608
|
+
return null;
|
|
609
|
+
const route = client.getTopicRouteData(topic);
|
|
610
|
+
if (!route)
|
|
611
|
+
return null;
|
|
612
|
+
let addr = null;
|
|
613
|
+
for (const bd of route.brokerDatas || []) {
|
|
614
|
+
const m = bd.brokerAddrs || {};
|
|
615
|
+
if (MixAll.MASTER_ID in m) {
|
|
616
|
+
addr = m[MixAll.MASTER_ID];
|
|
617
|
+
break;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
if (!addr)
|
|
621
|
+
return null;
|
|
622
|
+
if (sendHeartbeat)
|
|
623
|
+
await client.sendHeartbeatToAllBrokers().catch(() => { });
|
|
624
|
+
try {
|
|
625
|
+
const response = await client.getConsumerListByGroup(addr, this.consumerGroup);
|
|
626
|
+
if (response.code !== ResponseCode.SUCCESS)
|
|
627
|
+
return null;
|
|
628
|
+
const body = response.body && response.body.length
|
|
629
|
+
? RemotingSerializable.decode(response.body) : null;
|
|
630
|
+
const raw = Array.isArray(body) ? body
|
|
631
|
+
: (body && Array.isArray(body['consumerIdList'])) ? body['consumerIdList'] : [];
|
|
632
|
+
const ids = raw.map(String);
|
|
633
|
+
return ids.length ? ids : null;
|
|
634
|
+
}
|
|
635
|
+
catch (e) {
|
|
636
|
+
return null;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
// ---------------------------------------------------------------- rebalance
|
|
640
|
+
// _rebalanceTopics is the union of subscribe()'d topics and topics named by
|
|
641
|
+
// registerMessageQueueListener(): Java stores them in two tables and the pull
|
|
642
|
+
// rebalance must serve either registration path.
|
|
643
|
+
_rebalanceTopics() {
|
|
644
|
+
return [...new Set([...this.subscription.keys(), ...this.registerTopics])].sort();
|
|
645
|
+
}
|
|
646
|
+
// _startRebalanceLoop drives one rebalance round per timer tick, unref'd so a
|
|
647
|
+
// live consumer never keeps the Node process alive on its own (shutdown clears
|
|
648
|
+
// the handle explicitly). Deviation from Java's shared RebalanceService is
|
|
649
|
+
// documented on the enableRebalance field.
|
|
650
|
+
_startRebalanceLoop() {
|
|
651
|
+
if (this._rebalanceTimer)
|
|
652
|
+
return;
|
|
653
|
+
const schedule = (delayMillis) => {
|
|
654
|
+
if (!this._rebalanceActive)
|
|
655
|
+
return;
|
|
656
|
+
this._rebalanceTimer = setTimeout(() => {
|
|
657
|
+
this._rebalanceTimer = null;
|
|
658
|
+
if (!this._rebalanceActive)
|
|
659
|
+
return;
|
|
660
|
+
void (async () => {
|
|
661
|
+
let balanced = true;
|
|
662
|
+
try {
|
|
663
|
+
balanced = await this.doRebalance();
|
|
664
|
+
}
|
|
665
|
+
catch (e) {
|
|
666
|
+
logger.debug('rebalance loop error: %s', e.message);
|
|
667
|
+
balanced = false;
|
|
668
|
+
}
|
|
669
|
+
// balanced -> the full period; otherwise retry quickly so a freshly
|
|
670
|
+
// appeared queue is claimed without waiting out 20s.
|
|
671
|
+
schedule(balanced ? this.rebalanceWaitMillis : this.rebalanceMinIntervalMillis);
|
|
672
|
+
})();
|
|
673
|
+
}, delayMillis);
|
|
674
|
+
// unref(): a pending rebalance round must not pin the event loop at exit.
|
|
675
|
+
if (typeof this._rebalanceTimer.unref === 'function')
|
|
676
|
+
this._rebalanceTimer.unref();
|
|
677
|
+
};
|
|
678
|
+
schedule(this.rebalanceWaitMillis);
|
|
679
|
+
}
|
|
680
|
+
// doRebalance is Java DefaultMQPullConsumerImpl#tryRebalance (:398-406).
|
|
681
|
+
// With enableRebalance off Java returns TRUE ("nothing left to retry"); returning
|
|
682
|
+
// false would pin the loop at minInterval and burn a round-trip every second.
|
|
683
|
+
async doRebalance() {
|
|
684
|
+
if (!this.enableRebalance)
|
|
685
|
+
return true;
|
|
686
|
+
if (!this.mqClient)
|
|
687
|
+
return true;
|
|
688
|
+
let balanced = true;
|
|
689
|
+
for (const topic of this._rebalanceTopics()) {
|
|
690
|
+
try {
|
|
691
|
+
if (!(await this.rebalanceByTopic(topic)))
|
|
692
|
+
balanced = false;
|
|
693
|
+
}
|
|
694
|
+
catch (e) {
|
|
695
|
+
logger.debug('rebalance topic %s failed: %s', topic, e.message);
|
|
696
|
+
balanced = false;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return balanced;
|
|
700
|
+
}
|
|
701
|
+
// rebalanceByTopic is Java RebalanceImpl#rebalanceByTopic (pull path) plus
|
|
702
|
+
// RebalancePullImpl#messageQueueChanged. Returns whether this topic is already
|
|
703
|
+
// balanced. The listener fires ONLY when the allocation changed, receiving
|
|
704
|
+
// (topic, mqAll, mqDivided) — the same three arguments RebalancePullImpl
|
|
705
|
+
// (:44-53) passes.
|
|
706
|
+
//
|
|
707
|
+
// Deviation (documented): Java compares allocateResultSet with the freshly
|
|
708
|
+
// updated process-queue table, so a "changed then already in place" round can
|
|
709
|
+
// report balanced immediately. Here balanced means "identical to the previous
|
|
710
|
+
// round", which costs at most one extra minInterval confirmation round.
|
|
711
|
+
async rebalanceByTopic(topic) {
|
|
712
|
+
const client = this.mqClient;
|
|
713
|
+
if (!client)
|
|
714
|
+
return true;
|
|
715
|
+
// Java reads the cached subscribe-info table (the route refresh loop keeps it
|
|
716
|
+
// warm); warm it here too when a listener-only topic has never been routed.
|
|
717
|
+
let route = client.getTopicRouteData(topic);
|
|
718
|
+
if (!route) {
|
|
719
|
+
await client.updateTopicRouteInfoFromNameServer(topic, false).catch(() => { });
|
|
720
|
+
route = client.getTopicRouteData(topic);
|
|
721
|
+
}
|
|
722
|
+
const mqAll = (route ? route.getAllSubscribeMessageQueue(topic) : [])
|
|
723
|
+
.slice().sort((a, b) => a.compareTo(b));
|
|
724
|
+
if (mqAll.length === 0) {
|
|
725
|
+
// Java warns "topic not exist" and notifies with two empty sets, then
|
|
726
|
+
// `break`s — which leaves balanced at TRUE. Returning false here would
|
|
727
|
+
// retry at minInterval forever while the topic is merely absent.
|
|
728
|
+
const previous = this.rebalanceTable.get(topic);
|
|
729
|
+
if (previous && previous.length > 0) {
|
|
730
|
+
this.rebalanceTable.set(topic, []);
|
|
731
|
+
logger.warning('doRebalance, %s, topic not exist: %s', this.consumerGroup, topic);
|
|
732
|
+
this.notifyQueueChanged(topic, [], []);
|
|
733
|
+
}
|
|
734
|
+
return true;
|
|
735
|
+
}
|
|
736
|
+
let divided;
|
|
737
|
+
if (this.messageModel === MessageModel.BROADCASTING) {
|
|
738
|
+
divided = [...mqAll];
|
|
739
|
+
}
|
|
740
|
+
else {
|
|
741
|
+
const ids = await this._getConsumerIdsFor(topic);
|
|
742
|
+
if (!ids) {
|
|
743
|
+
// Java warns "get consumer id list failed" and NOTIFIES NOTHING, leaving
|
|
744
|
+
// balanced at true so the next round waits the full interval.
|
|
745
|
+
logger.warning('doRebalance, %s %s, get consumer id list failed', this.consumerGroup, topic);
|
|
746
|
+
return true;
|
|
747
|
+
}
|
|
748
|
+
let allocated;
|
|
749
|
+
try {
|
|
750
|
+
allocated = this.allocateMessageQueueStrategy.allocate(this.consumerGroup, this.clientID, mqAll, [...ids].sort());
|
|
751
|
+
}
|
|
752
|
+
catch (e) {
|
|
753
|
+
logger.error('allocate message queue exception. strategy name: %s, ex: %s', this.allocateMessageQueueStrategy.name(), e.message);
|
|
754
|
+
return false;
|
|
755
|
+
}
|
|
756
|
+
divided = (allocated || []).filter(mq => mq.getTopic() === topic)
|
|
757
|
+
.sort((a, b) => a.compareTo(b));
|
|
758
|
+
}
|
|
759
|
+
const previous = this.rebalanceTable.get(topic);
|
|
760
|
+
// First sight of the topic: Java's updateProcessQueueTableInRebalance counts only table
|
|
761
|
+
// additions/removals as a change, so "nothing -> nothing" is no change -- an empty
|
|
762
|
+
// opening share stays silent and the round counts as balanced.
|
|
763
|
+
const balanced = previous === undefined
|
|
764
|
+
? divided.length === 0
|
|
765
|
+
: this._sameMqSet(previous, divided);
|
|
766
|
+
this.rebalanceTable.set(topic, [...divided]);
|
|
767
|
+
if (!balanced) {
|
|
768
|
+
logger.info('client rebalanced result changed. Group: %s, topic: %s, '
|
|
769
|
+
+ 'mqAllSize: %d, rebalanceResultSize: %d', this.consumerGroup, topic, mqAll.length, divided.length);
|
|
770
|
+
this.notifyQueueChanged(topic, mqAll, divided);
|
|
771
|
+
}
|
|
772
|
+
return balanced;
|
|
773
|
+
}
|
|
774
|
+
// _sameMqSet compares queue identities, not order: losing one queue and gaining
|
|
775
|
+
// another keeps the count while changing who owns what.
|
|
776
|
+
_sameMqSet(left, right) {
|
|
777
|
+
if (left.length !== right.length)
|
|
778
|
+
return false;
|
|
779
|
+
const seen = new Set(left.map(mqKey));
|
|
780
|
+
for (const mq of right)
|
|
781
|
+
if (!seen.has(mqKey(mq)))
|
|
782
|
+
return false;
|
|
783
|
+
return true;
|
|
784
|
+
}
|
|
785
|
+
// notifyQueueChanged is RebalancePullImpl#messageQueueChanged (:44-53): a null
|
|
786
|
+
// listener is fine, and a throwing one is logged and swallowed — one bad
|
|
787
|
+
// callback must not kill the loop and freeze every allocation.
|
|
788
|
+
notifyQueueChanged(topic, mqAll, mqDivided) {
|
|
789
|
+
const listener = this.messageQueueListener;
|
|
790
|
+
if (!listener)
|
|
791
|
+
return;
|
|
792
|
+
try {
|
|
793
|
+
listener(topic, mqAll, mqDivided);
|
|
794
|
+
}
|
|
795
|
+
catch (e) {
|
|
796
|
+
logger.error('messageQueueChanged exception: %s', e.message);
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
// sendMessageBack redelivers one message via CONSUMER_SEND_MSG_BACK(36).
|
|
800
|
+
// `offset` is the message's commitLogOffset, NOT its queueOffset.
|
|
801
|
+
//
|
|
802
|
+
// A refused bounce — a missing master, a transport error or a non-success
|
|
803
|
+
// answer — degrades to re-sending the message as a normal message to
|
|
804
|
+
// %RETRY%<group> through the internal producer, so the redelivery is not
|
|
805
|
+
// lost; only when that send fails too does the original error reach the
|
|
806
|
+
// caller.
|
|
807
|
+
async sendMessageBack(msg, delayLevel = 0) {
|
|
808
|
+
if (!this.mqClient)
|
|
809
|
+
throw new Error('consumer not started');
|
|
810
|
+
const maxReconsume = this.maxReconsumeTimes === -1 ? 16 : this.maxReconsumeTimes;
|
|
811
|
+
try {
|
|
812
|
+
const addr = this.mqClient.findBrokerAddressInPublish(msg.getBrokerName());
|
|
813
|
+
if (!addr)
|
|
814
|
+
throw new Error(`Broker[${msg.getBrokerName()}] master node does not exist`);
|
|
815
|
+
const h = new ConsumerSendMsgBackRequestHeader();
|
|
816
|
+
h.offset = msg.getCommitLogOffset();
|
|
817
|
+
h.group = this.consumerGroup;
|
|
818
|
+
h.delayLevel = delayLevel;
|
|
819
|
+
h.originMsgId = clientMsgId(msg);
|
|
820
|
+
h.originTopic = this._withNamespace(msg.getTopic());
|
|
821
|
+
h.unitMode = this.unitMode;
|
|
822
|
+
h.maxReconsumeTimes = maxReconsume;
|
|
823
|
+
const request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, h);
|
|
824
|
+
const response = await this.mqClient.remotingClient.invokeSync(this.mqClient.vipChannel(addr), request, 3000);
|
|
825
|
+
if (response.code !== ResponseCode.SUCCESS) {
|
|
826
|
+
// Same rule as the pull kernel: the broker code rides on the typed
|
|
827
|
+
// exception, so a caller can branch on it instead of parsing the text.
|
|
828
|
+
throw new MQBrokerException(response.code, `sendMessageBack failed: CODE ${response.code} ${response.remark || ''}`, addr);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
catch (e) {
|
|
832
|
+
logger.error('Failed to send message back, consumerGroup=%s, brokerName=%s, msg=%s: %s; fallback to normal send', this.consumerGroup, msg.getBrokerName(), msg.getMsgId(), e.message);
|
|
833
|
+
try {
|
|
834
|
+
await this._sendRetryMessage(msg, maxReconsume);
|
|
835
|
+
}
|
|
836
|
+
catch (fallbackError) {
|
|
837
|
+
logger.error('Fallback normal send failed, consumerGroup=%s, msg=%s: %s', this.consumerGroup, msg.getMsgId(), fallbackError.message);
|
|
838
|
+
throw e;
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
// _sendRetryMessage sends a failed delivery as a normal message to
|
|
843
|
+
// %RETRY%<group> through the internal producer.
|
|
844
|
+
async _sendRetryMessage(msg, maxReconsume) {
|
|
845
|
+
const producer = await this._innerProducer();
|
|
846
|
+
if (!producer)
|
|
847
|
+
throw new Error('inner producer is not available');
|
|
848
|
+
await producer.sendBySelector(buildRetryMessage(this.consumerGroup, msg, maxReconsume), undefined);
|
|
849
|
+
}
|
|
850
|
+
// _innerProducer lazily builds the CLIENT_INNER_PRODUCER the send-back
|
|
851
|
+
// fallback sends through.
|
|
852
|
+
async _innerProducer() {
|
|
853
|
+
if (this._producer)
|
|
854
|
+
return this._producer;
|
|
855
|
+
try {
|
|
856
|
+
const { DefaultMQProducer: P } = await import("./producer.js");
|
|
857
|
+
const producer = new P(MixAll.CLIENT_INNER_PRODUCER_GROUP || 'CLIENT_INNER_PRODUCER');
|
|
858
|
+
producer.setNamesrvAddr(this.nameServerAddr);
|
|
859
|
+
await producer.start();
|
|
860
|
+
this._producer = producer;
|
|
861
|
+
return producer;
|
|
862
|
+
}
|
|
863
|
+
catch (e) {
|
|
864
|
+
logger.debug('inner producer start failed: %s', e.message);
|
|
865
|
+
return null;
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
// viewMessage is Java DefaultMQPullConsumerImpl#viewMessage
|
|
869
|
+
// (DefaultMQPullConsumerImpl.java:838) parity: the MQAdminImpl single-leg
|
|
870
|
+
// path — offset msgId only, no uniq-key fallback at this layer.
|
|
871
|
+
async viewMessage(topic, msgId) {
|
|
872
|
+
if (!this.mqClient)
|
|
873
|
+
throw new Error('consumer not started');
|
|
874
|
+
return this.mqClient.viewMessage(topic, msgId);
|
|
875
|
+
}
|
|
876
|
+
}
|