rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +428 -0
- package/README.md +424 -2
- package/dist/client/admin.d.ts +229 -0
- package/dist/client/admin.js +2462 -0
- package/dist/client/allocate.d.ts +48 -0
- package/dist/client/allocate.js +365 -0
- package/dist/client/backpressure.d.ts +23 -0
- package/dist/client/backpressure.js +95 -0
- package/dist/client/consumer.d.ts +294 -0
- package/dist/client/consumer.js +3095 -0
- package/dist/client/consumer_result.d.ts +129 -0
- package/dist/client/consumer_result.js +138 -0
- package/dist/client/consumer_stats.d.ts +69 -0
- package/dist/client/consumer_stats.js +258 -0
- package/dist/client/exception.d.ts +39 -0
- package/dist/client/exception.js +36 -0
- package/dist/client/hook.d.ts +145 -0
- package/dist/client/hook.js +171 -0
- package/dist/client/latency.d.ts +61 -0
- package/dist/client/latency.js +239 -0
- package/dist/client/lite_pull_consumer.d.ts +151 -0
- package/dist/client/lite_pull_consumer.js +1096 -0
- package/dist/client/mq_client.d.ts +188 -0
- package/dist/client/mq_client.js +1678 -0
- package/dist/client/mq_client_manager.d.ts +21 -0
- package/dist/client/mq_client_manager.js +129 -0
- package/dist/client/offset_store.d.ts +45 -0
- package/dist/client/offset_store.js +389 -0
- package/dist/client/open_tracing_hook.d.ts +52 -0
- package/dist/client/open_tracing_hook.js +241 -0
- package/dist/client/pop_api.d.ts +28 -0
- package/dist/client/pop_api.js +224 -0
- package/dist/client/pop_consumer.d.ts +23 -0
- package/dist/client/pop_consumer.js +719 -0
- package/dist/client/pop_process_queue.d.ts +15 -0
- package/dist/client/pop_process_queue.js +51 -0
- package/dist/client/process_queue.d.ts +49 -0
- package/dist/client/process_queue.js +261 -0
- package/dist/client/produce_accumulator.d.ts +75 -0
- package/dist/client/produce_accumulator.js +288 -0
- package/dist/client/producer.d.ts +231 -0
- package/dist/client/producer.js +1774 -0
- package/dist/client/pull_api.d.ts +26 -0
- package/dist/client/pull_api.js +264 -0
- package/dist/client/pull_consumer.d.ts +119 -0
- package/dist/client/pull_consumer.js +876 -0
- package/dist/client/pull_schedule_service.d.ts +71 -0
- package/dist/client/pull_schedule_service.js +424 -0
- package/dist/client/request_reply.d.ts +61 -0
- package/dist/client/request_reply.js +232 -0
- package/dist/client/retry_message.d.ts +2 -0
- package/dist/client/retry_message.js +27 -0
- package/dist/client/send_result.d.ts +57 -0
- package/dist/client/send_result.js +70 -0
- package/dist/client/top_addressing.d.ts +20 -0
- package/dist/client/top_addressing.js +75 -0
- package/dist/client/trace_context.d.ts +61 -0
- package/dist/client/trace_context.js +373 -0
- package/dist/client/trace_dispatcher.d.ts +31 -0
- package/dist/client/trace_dispatcher.js +236 -0
- package/dist/client/trace_hook.d.ts +23 -0
- package/dist/client/trace_hook.js +228 -0
- package/dist/client/traceparent.d.ts +9 -0
- package/dist/client/traceparent.js +86 -0
- package/dist/common/boundary_type.d.ts +7 -0
- package/dist/common/boundary_type.js +16 -0
- package/dist/common/compress.d.ts +28 -0
- package/dist/common/compress.js +512 -0
- package/dist/common/message.d.ts +129 -0
- package/dist/common/message.js +214 -0
- package/dist/common/messageClientIdSetter.d.ts +7 -0
- package/dist/common/messageClientIdSetter.js +48 -0
- package/dist/common/messageConst.d.ts +68 -0
- package/dist/common/messageConst.js +68 -0
- package/dist/common/messageDecoder.d.ts +40 -0
- package/dist/common/messageDecoder.js +504 -0
- package/dist/common/messageType.d.ts +11 -0
- package/dist/common/messageType.js +11 -0
- package/dist/common/message_accessor.d.ts +39 -0
- package/dist/common/message_accessor.js +135 -0
- package/dist/common/mixAll.d.ts +106 -0
- package/dist/common/mixAll.js +247 -0
- package/dist/common/recall_message_handle.d.ts +25 -0
- package/dist/common/recall_message_handle.js +98 -0
- package/dist/common/subscriptionData.d.ts +36 -0
- package/dist/common/subscriptionData.js +64 -0
- package/dist/common/sysflag.d.ts +129 -0
- package/dist/common/sysflag.js +96 -0
- package/dist/common/topic_config.d.ts +66 -0
- package/dist/common/topic_config.js +106 -0
- package/dist/common/topic_validator.d.ts +49 -0
- package/dist/common/topic_validator.js +115 -0
- package/dist/common/utilAll.d.ts +39 -0
- package/dist/common/utilAll.js +154 -0
- package/dist/common/validators.d.ts +25 -0
- package/dist/common/validators.js +80 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +36 -0
- package/dist/logging.d.ts +24 -0
- package/dist/logging.js +246 -0
- package/dist/remoting/acl.d.ts +16 -0
- package/dist/remoting/acl.js +39 -0
- package/dist/remoting/admin_body.d.ts +257 -0
- package/dist/remoting/admin_body.js +669 -0
- package/dist/remoting/bodies.d.ts +475 -0
- package/dist/remoting/bodies.js +1131 -0
- package/dist/remoting/client.d.ts +73 -0
- package/dist/remoting/client.js +591 -0
- package/dist/remoting/codes.d.ts +296 -0
- package/dist/remoting/codes.js +301 -0
- package/dist/remoting/exception.d.ts +45 -0
- package/dist/remoting/exception.js +58 -0
- package/dist/remoting/extra_info.d.ts +61 -0
- package/dist/remoting/extra_info.js +220 -0
- package/dist/remoting/headers.d.ts +758 -0
- package/dist/remoting/headers.js +1307 -0
- package/dist/remoting/heartbeat.d.ts +91 -0
- package/dist/remoting/heartbeat.js +227 -0
- package/dist/remoting/namespace.d.ts +14 -0
- package/dist/remoting/namespace.js +85 -0
- package/dist/remoting/pop_bodies.d.ts +58 -0
- package/dist/remoting/pop_bodies.js +214 -0
- package/dist/remoting/remotingCommand.d.ts +39 -0
- package/dist/remoting/remotingCommand.js +225 -0
- package/dist/remoting/route.d.ts +61 -0
- package/dist/remoting/route.js +288 -0
- package/dist/remoting/rpc_hooks.d.ts +31 -0
- package/dist/remoting/rpc_hooks.js +73 -0
- package/dist/remoting/serialize.d.ts +33 -0
- package/dist/remoting/serialize.js +372 -0
- package/dist/remoting/socks_proxy.d.ts +29 -0
- package/dist/remoting/socks_proxy.js +379 -0
- package/dist/remoting/static_topic.d.ts +49 -0
- package/dist/remoting/static_topic.js +244 -0
- package/dist/remoting/subscription.d.ts +65 -0
- package/dist/remoting/subscription.js +172 -0
- package/package.json +38 -4
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { MessageQueue } from '../common/message.js';
|
|
2
|
+
export interface AllocateMessageQueueStrategy {
|
|
3
|
+
allocate(consumerGroup: string, currentCID: string, mqAll: MessageQueue[], cidAll: string[]): MessageQueue[];
|
|
4
|
+
name(): string;
|
|
5
|
+
}
|
|
6
|
+
export declare class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy {
|
|
7
|
+
name(): string;
|
|
8
|
+
allocate(consumerGroup: string, currentCID: string, mqAllRaw: MessageQueue[], cidAllRaw: string[]): MessageQueue[];
|
|
9
|
+
}
|
|
10
|
+
export declare class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
|
|
11
|
+
name(): string;
|
|
12
|
+
allocate(consumerGroup: string, currentCID: string, mqAllRaw: MessageQueue[], cidAllRaw: string[]): MessageQueue[];
|
|
13
|
+
}
|
|
14
|
+
export declare class AllocateMessageQueueByConfig implements AllocateMessageQueueStrategy {
|
|
15
|
+
messageQueueList: MessageQueue[];
|
|
16
|
+
constructor(mqs: MessageQueue[]);
|
|
17
|
+
name(): string;
|
|
18
|
+
allocate(_consumerGroup: string, _currentCID: string, _mqAll: MessageQueue[], _cidAll: string[]): MessageQueue[];
|
|
19
|
+
}
|
|
20
|
+
export interface HashFunction {
|
|
21
|
+
hash(key: string): number;
|
|
22
|
+
}
|
|
23
|
+
export declare class AllocateMessageQueueConsistentHash implements AllocateMessageQueueStrategy {
|
|
24
|
+
virtualNodeCnt: number;
|
|
25
|
+
customHashFunction?: HashFunction;
|
|
26
|
+
constructor(virtualNodeCnt?: number);
|
|
27
|
+
name(): string;
|
|
28
|
+
allocate(consumerGroup: string, currentCID: string, mqAllRaw: MessageQueue[], cidAllRaw: string[]): MessageQueue[];
|
|
29
|
+
}
|
|
30
|
+
export declare class AllocateMessageQueueByMachineRoom implements AllocateMessageQueueStrategy {
|
|
31
|
+
consumerIDCs: Set<string>;
|
|
32
|
+
constructor(idcs: string[]);
|
|
33
|
+
name(): string;
|
|
34
|
+
allocate(consumerGroup: string, currentCID: string, mqAllRaw: MessageQueue[], cidAllRaw: string[]): MessageQueue[];
|
|
35
|
+
}
|
|
36
|
+
export declare const ALLOCATE_STRATEGIES: Record<string, () => AllocateMessageQueueStrategy>;
|
|
37
|
+
export declare function createAllocateStrategy(name: string): AllocateMessageQueueStrategy;
|
|
38
|
+
export interface MachineRoomResolver {
|
|
39
|
+
brokerDeployIn(messageQueue: MessageQueue): string;
|
|
40
|
+
consumerDeployIn(clientID: string): string;
|
|
41
|
+
}
|
|
42
|
+
export declare class AllocateMachineRoomNearby implements AllocateMessageQueueStrategy {
|
|
43
|
+
inner: AllocateMessageQueueStrategy;
|
|
44
|
+
resolver: MachineRoomResolver;
|
|
45
|
+
constructor(inner: AllocateMessageQueueStrategy, resolver: MachineRoomResolver);
|
|
46
|
+
name(): string;
|
|
47
|
+
allocate(consumerGroup: string, currentCID: string, mqAllRaw: MessageQueue[], cidAllRaw: string[]): MessageQueue[];
|
|
48
|
+
}
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
// Queue allocation strategies (the queue-splitting algorithms inside a consumer
|
|
2
|
+
// group).
|
|
3
|
+
//
|
|
4
|
+
// One behavioural convention holds everywhere: on illegal input (currentCID not
|
|
5
|
+
// in cidAll, empty mqAll/cidAll) nothing is thrown and the strategy returns an
|
|
6
|
+
// empty list — rebalance is a background timer, and one dirty input must not kill
|
|
7
|
+
// the consumer.
|
|
8
|
+
//
|
|
9
|
+
// Determinism is what matters: every instance in the group must sort mqAll and
|
|
10
|
+
// cidAll the same way and run the same index arithmetic, otherwise two instances
|
|
11
|
+
// compute overlapping allocations and every message is consumed twice.
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { getLogger } from "../logging.js";
|
|
14
|
+
const logger = getLogger('client.allocate');
|
|
15
|
+
// checkCid is the shared pre-check for all strategies: currentCID must be in
|
|
16
|
+
// cidAll; when it is not, log a [BUG] line and let the strategy return an empty
|
|
17
|
+
// allocation.
|
|
18
|
+
function checkCid(consumerGroup, currentCID, mqAll, cidAll) {
|
|
19
|
+
if (!currentCID || mqAll.length === 0 || cidAll.length === 0)
|
|
20
|
+
return -1;
|
|
21
|
+
const index = cidAll.indexOf(currentCID);
|
|
22
|
+
if (index < 0) {
|
|
23
|
+
logger.info('[BUG] ConsumerGroup: %s The consumerId: %s not in cidAll: %j', consumerGroup, currentCID, cidAll);
|
|
24
|
+
return -1;
|
|
25
|
+
}
|
|
26
|
+
return index;
|
|
27
|
+
}
|
|
28
|
+
function sortMqAll(mqAll) {
|
|
29
|
+
return [...mqAll].sort((a, b) => a.compareTo(b));
|
|
30
|
+
}
|
|
31
|
+
function sortCidAll(cidAll) {
|
|
32
|
+
return [...cidAll].sort();
|
|
33
|
+
}
|
|
34
|
+
// ------------------------------------------------------------- AVG
|
|
35
|
+
// AVG: hands contiguous ranges of the sorted queues to the consumers of the
|
|
36
|
+
// group, with the remainder going to the first `mod` of them.
|
|
37
|
+
export class AllocateMessageQueueAveragely {
|
|
38
|
+
name() { return 'AVG'; }
|
|
39
|
+
allocate(consumerGroup, currentCID, mqAllRaw, cidAllRaw) {
|
|
40
|
+
const mqAll = sortMqAll(mqAllRaw);
|
|
41
|
+
const cidAll = sortCidAll(cidAllRaw);
|
|
42
|
+
const index = checkCid(consumerGroup, currentCID, mqAll, cidAll);
|
|
43
|
+
if (index < 0)
|
|
44
|
+
return [];
|
|
45
|
+
const mod = mqAll.length % cidAll.length;
|
|
46
|
+
const averageSize = Math.floor(mqAll.length / cidAll.length);
|
|
47
|
+
if (averageSize === 0) {
|
|
48
|
+
if (index < mqAll.length)
|
|
49
|
+
return [mqAll[index]];
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
let start, end;
|
|
53
|
+
if (mod > 0 && index < mod) {
|
|
54
|
+
start = index * (averageSize + 1);
|
|
55
|
+
end = start + averageSize + 1;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
start = mod * (averageSize + 1) + (index - mod) * averageSize;
|
|
59
|
+
end = start + averageSize;
|
|
60
|
+
}
|
|
61
|
+
return mqAll.slice(start, end);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// ------------------------------------------------ AVG_BY_CIRCLE
|
|
65
|
+
// AVG_BY_CIRCLE: takes queues in a round-robin circle over the sorted cidAll.
|
|
66
|
+
export class AllocateMessageQueueAveragelyByCircle {
|
|
67
|
+
name() { return 'AVG_BY_CIRCLE'; }
|
|
68
|
+
allocate(consumerGroup, currentCID, mqAllRaw, cidAllRaw) {
|
|
69
|
+
const mqAll = sortMqAll(mqAllRaw);
|
|
70
|
+
const cidAll = sortCidAll(cidAllRaw);
|
|
71
|
+
const index = checkCid(consumerGroup, currentCID, mqAll, cidAll);
|
|
72
|
+
if (index < 0)
|
|
73
|
+
return [];
|
|
74
|
+
const out = [];
|
|
75
|
+
for (let i = index; i < mqAll.length; i += cidAll.length) {
|
|
76
|
+
out.push(mqAll[i]);
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// ------------------------------------------------------------- CONFIG
|
|
82
|
+
// CONFIG: returns the queue list the caller configured, verbatim. Note that it
|
|
83
|
+
// does **no validation at all** here — an empty group or cidAll still returns
|
|
84
|
+
// the configured list.
|
|
85
|
+
export class AllocateMessageQueueByConfig {
|
|
86
|
+
messageQueueList;
|
|
87
|
+
constructor(mqs) {
|
|
88
|
+
this.messageQueueList = [...mqs];
|
|
89
|
+
}
|
|
90
|
+
name() { return 'CONFIG'; }
|
|
91
|
+
// Every argument is ignored (no validation runs here).
|
|
92
|
+
allocate(_consumerGroup, _currentCID, _mqAll, _cidAll) {
|
|
93
|
+
return [...this.messageQueueList];
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// md5 hash: takes the **first four bytes** of the MD5 digest and assembles them
|
|
97
|
+
// big-endian into a uint32. Using the whole 128-bit value (or a different slice)
|
|
98
|
+
// lands on a different hash ring than existing clients use, and queue ownership
|
|
99
|
+
// no longer lines up.
|
|
100
|
+
class Md5Hash {
|
|
101
|
+
hash(key) {
|
|
102
|
+
const sum = createHash('md5').update(key, 'utf8').digest();
|
|
103
|
+
return sum.readUInt32BE(0) >>> 0;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
class ConsistentHashRouter {
|
|
107
|
+
hashFn;
|
|
108
|
+
ring = new Map(); // key: uint32 hash
|
|
109
|
+
keys = []; // sorted ascending
|
|
110
|
+
constructor(pNodes, vNodeCount, fn) {
|
|
111
|
+
this.hashFn = fn || new Md5Hash();
|
|
112
|
+
for (const node of pNodes)
|
|
113
|
+
this.addNode(node, vNodeCount);
|
|
114
|
+
}
|
|
115
|
+
addNode(pNode, vNodeCount) {
|
|
116
|
+
const existing = this.existingReplicas(pNode);
|
|
117
|
+
for (let i = 0; i < vNodeCount; i++) {
|
|
118
|
+
const v = { key: virtualNodeRingKey(pNode, i + existing), physicalKey: pNode };
|
|
119
|
+
const key = this.hashFn.hash(v.key);
|
|
120
|
+
if (!this.ring.has(key)) {
|
|
121
|
+
this.keys = insertU32(this.keys, key);
|
|
122
|
+
}
|
|
123
|
+
// On a hash collision the old node is overwritten on the same key (the
|
|
124
|
+
// semantics of an ordered-map put).
|
|
125
|
+
this.ring.set(key, v);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
removeNode(pNode) {
|
|
129
|
+
const kept = [];
|
|
130
|
+
for (const key of this.keys) {
|
|
131
|
+
if (this.ring.get(key).physicalKey === pNode) {
|
|
132
|
+
this.ring.delete(key);
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
kept.push(key);
|
|
136
|
+
}
|
|
137
|
+
this.keys = kept;
|
|
138
|
+
}
|
|
139
|
+
// routeNode: takes the first key on the ring that is >= the hash (the wrap-around
|
|
140
|
+
// range is a **closed** interval, endpoint included), wrapping to the first key
|
|
141
|
+
// when out of range.
|
|
142
|
+
routeNode(objectKey) {
|
|
143
|
+
if (this.keys.length === 0)
|
|
144
|
+
return null;
|
|
145
|
+
let idx = lowerBound(this.keys, this.hashFn.hash(objectKey));
|
|
146
|
+
if (idx === this.keys.length)
|
|
147
|
+
idx = 0;
|
|
148
|
+
return this.ring.get(this.keys[idx]).physicalKey;
|
|
149
|
+
}
|
|
150
|
+
existingReplicas(pNode) {
|
|
151
|
+
let n = 0;
|
|
152
|
+
for (const v of this.ring.values()) {
|
|
153
|
+
if (v.physicalKey === pNode)
|
|
154
|
+
n++;
|
|
155
|
+
}
|
|
156
|
+
return n;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
function virtualNodeRingKey(pNode, replicaIndex) {
|
|
160
|
+
return `${pNode}-${replicaIndex}`;
|
|
161
|
+
}
|
|
162
|
+
function insertU32(sorted, v) {
|
|
163
|
+
const i = lowerBound(sorted, v);
|
|
164
|
+
sorted.splice(i, 0, v);
|
|
165
|
+
return sorted;
|
|
166
|
+
}
|
|
167
|
+
function lowerBound(sorted, v) {
|
|
168
|
+
let lo = 0, hi = sorted.length;
|
|
169
|
+
while (lo < hi) {
|
|
170
|
+
const mid = (lo + hi) >> 1;
|
|
171
|
+
if (sorted[mid] < v)
|
|
172
|
+
lo = mid + 1;
|
|
173
|
+
else
|
|
174
|
+
hi = mid;
|
|
175
|
+
}
|
|
176
|
+
return lo;
|
|
177
|
+
}
|
|
178
|
+
// The key string a queue is hashed by on the ring:
|
|
179
|
+
// `MessageQueue [topic=..., brokerName=..., queueId=...]` — it must match the
|
|
180
|
+
// existing shape of the classic remoting protocol character for character.
|
|
181
|
+
function queueRingKeyString(mq) {
|
|
182
|
+
return `MessageQueue [topic=${mq.getTopic()}, brokerName=${mq.getBrokerName()}, queueId=${mq.getQueueId()}]`;
|
|
183
|
+
}
|
|
184
|
+
// ------------------------------------------- CONSISTENT_HASH
|
|
185
|
+
// CONSISTENT_HASH: the consistent-hash strategy. The point is not uniformity but
|
|
186
|
+
// **stability**: when the queue count or the consumer count changes, only the
|
|
187
|
+
// queues on the affected arc change owner, whereas AVG moves everyone's boundary.
|
|
188
|
+
export class AllocateMessageQueueConsistentHash {
|
|
189
|
+
virtualNodeCnt;
|
|
190
|
+
customHashFunction;
|
|
191
|
+
constructor(virtualNodeCnt = 10) {
|
|
192
|
+
this.virtualNodeCnt = virtualNodeCnt;
|
|
193
|
+
}
|
|
194
|
+
name() { return 'CONSISTENT_HASH'; }
|
|
195
|
+
allocate(consumerGroup, currentCID, mqAllRaw, cidAllRaw) {
|
|
196
|
+
const mqAll = sortMqAll(mqAllRaw);
|
|
197
|
+
const cidAll = sortCidAll(cidAllRaw);
|
|
198
|
+
if (checkCid(consumerGroup, currentCID, mqAll, cidAll) < 0)
|
|
199
|
+
return [];
|
|
200
|
+
const router = new ConsistentHashRouter(cidAll, this.virtualNodeCnt, this.customHashFunction);
|
|
201
|
+
const out = [];
|
|
202
|
+
for (const mq of mqAll) {
|
|
203
|
+
const node = router.routeNode(queueRingKeyString(mq));
|
|
204
|
+
if (node !== null && node === currentCID)
|
|
205
|
+
out.push(mq);
|
|
206
|
+
}
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// ------------------------------------------------- MACHINE_ROOM
|
|
211
|
+
// splitTrimmingTrailingEmpty: splits on the separator and drops **trailing**
|
|
212
|
+
// empty segments (the limit=0 split semantics); but with no separator present the
|
|
213
|
+
// whole string is returned as one segment — so "" stays [""], it does not become
|
|
214
|
+
// an empty array.
|
|
215
|
+
function splitTrimmingTrailingEmpty(text, sep) {
|
|
216
|
+
const parts = text.split(sep);
|
|
217
|
+
if (parts.length === 1)
|
|
218
|
+
return parts;
|
|
219
|
+
while (parts.length > 0 && parts[parts.length - 1] === '')
|
|
220
|
+
parts.pop();
|
|
221
|
+
return parts;
|
|
222
|
+
}
|
|
223
|
+
// MACHINE_ROOM: broker names follow the `<idc>@<brokerName>` convention; only
|
|
224
|
+
// queues whose idc prefix is in the ConsumerIDCs set take part in allocation, and
|
|
225
|
+
// they are then split by the AVG-like rule with the remainder going to the first
|
|
226
|
+
// `mod` consumers (the comparison is `rem > index`, not `>=`).
|
|
227
|
+
export class AllocateMessageQueueByMachineRoom {
|
|
228
|
+
consumerIDCs;
|
|
229
|
+
constructor(idcs) {
|
|
230
|
+
this.consumerIDCs = new Set(idcs);
|
|
231
|
+
}
|
|
232
|
+
name() { return 'MACHINE_ROOM'; }
|
|
233
|
+
allocate(consumerGroup, currentCID, mqAllRaw, cidAllRaw) {
|
|
234
|
+
const mqAll = sortMqAll(mqAllRaw);
|
|
235
|
+
const cidAll = sortCidAll(cidAllRaw);
|
|
236
|
+
const index = checkCid(consumerGroup, currentCID, mqAll, cidAll);
|
|
237
|
+
if (index < 0)
|
|
238
|
+
return [];
|
|
239
|
+
// consumerIDCs has no built-in default (omitting it would blow up idc
|
|
240
|
+
// matching); the default here is the empty set, i.e. "assign nothing".
|
|
241
|
+
const candidates = [];
|
|
242
|
+
for (const mq of mqAll) {
|
|
243
|
+
const parts = splitTrimmingTrailingEmpty(mq.getBrokerName(), '@');
|
|
244
|
+
if (parts.length !== 2)
|
|
245
|
+
continue;
|
|
246
|
+
if (!this.consumerIDCs.has(parts[0]))
|
|
247
|
+
continue;
|
|
248
|
+
if (parts[1] === '')
|
|
249
|
+
continue;
|
|
250
|
+
candidates.push(mq);
|
|
251
|
+
}
|
|
252
|
+
const rem = candidates.length % cidAll.length;
|
|
253
|
+
const size = Math.floor(candidates.length / cidAll.length);
|
|
254
|
+
let start = index * size;
|
|
255
|
+
if (rem > index)
|
|
256
|
+
start += index;
|
|
257
|
+
else
|
|
258
|
+
start += rem;
|
|
259
|
+
let end = start + size;
|
|
260
|
+
if (rem > index)
|
|
261
|
+
end++;
|
|
262
|
+
const out = [];
|
|
263
|
+
for (let i = start; i < end; i++)
|
|
264
|
+
out.push(candidates[i]);
|
|
265
|
+
return out;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// Strategy factory: name -> instance (covering the built-in strategies above).
|
|
269
|
+
export const ALLOCATE_STRATEGIES = {
|
|
270
|
+
AVG: () => new AllocateMessageQueueAveragely(),
|
|
271
|
+
AVG_BY_CIRCLE: () => new AllocateMessageQueueAveragelyByCircle(),
|
|
272
|
+
CONFIG: () => new AllocateMessageQueueByConfig([]),
|
|
273
|
+
MACHINE_ROOM: () => new AllocateMessageQueueByMachineRoom([]),
|
|
274
|
+
CONSISTENT_HASH: () => new AllocateMessageQueueConsistentHash(),
|
|
275
|
+
// MACHINE_ROOM_NEARBY is a PROXY: it needs an inner strategy plus a
|
|
276
|
+
// MachineRoomResolver, neither of which the zero-arg factory signature can
|
|
277
|
+
// supply. It is registered with the same defaults a caller would pass
|
|
278
|
+
// explicitly (AVG inside, brokerName as the room) so a name-based lookup
|
|
279
|
+
// still resolves instead of silently degrading to AVG — which would
|
|
280
|
+
// distribute queues across machine rooms and defeat the whole point.
|
|
281
|
+
MACHINE_ROOM_NEARBY: () => new AllocateMachineRoomNearby(new AllocateMessageQueueAveragely(), {
|
|
282
|
+
brokerDeployIn: (mq) => mq.getBrokerName(),
|
|
283
|
+
consumerDeployIn: (cid) => cid,
|
|
284
|
+
}),
|
|
285
|
+
};
|
|
286
|
+
export function createAllocateStrategy(name) {
|
|
287
|
+
const f = ALLOCATE_STRATEGIES[name];
|
|
288
|
+
return f ? f() : new AllocateMessageQueueAveragely();
|
|
289
|
+
}
|
|
290
|
+
// AllocateMachineRoomNearby: "nearby machine room" allocation as a proxy in
|
|
291
|
+
// front of another strategy. Queues and consumers are first grouped by machine
|
|
292
|
+
// room, then
|
|
293
|
+
//
|
|
294
|
+
// 1. the queues of this consumer's room go only to the consumers of that room
|
|
295
|
+
// (handled by the inner strategy);
|
|
296
|
+
// 2. the queues of any room with no live consumer are shared by **all**
|
|
297
|
+
// consumers (also via the inner strategy) — otherwise those queues would
|
|
298
|
+
// have nobody consuming them.
|
|
299
|
+
//
|
|
300
|
+
// name() is "MACHINE_ROOM_NEARBY" + "-" + <inner name>, so logs reveal which
|
|
301
|
+
// algorithm actually does the splitting.
|
|
302
|
+
//
|
|
303
|
+
// The constructor throws on empty arguments, and so does a resolver that returns
|
|
304
|
+
// an empty machine room. Failures during allocate propagate straight out of
|
|
305
|
+
// allocate() — the rebalance loop catches them and keeps the existing allocation
|
|
306
|
+
// (aborting the round).
|
|
307
|
+
export class AllocateMachineRoomNearby {
|
|
308
|
+
inner;
|
|
309
|
+
resolver;
|
|
310
|
+
constructor(inner, resolver) {
|
|
311
|
+
if (!inner)
|
|
312
|
+
throw new Error('allocateMessageQueueStrategy is null');
|
|
313
|
+
if (!resolver)
|
|
314
|
+
throw new Error('machineRoomResolver is null');
|
|
315
|
+
this.inner = inner;
|
|
316
|
+
this.resolver = resolver;
|
|
317
|
+
}
|
|
318
|
+
name() { return `MACHINE_ROOM_NEARBY-${this.inner.name()}`; }
|
|
319
|
+
allocate(consumerGroup, currentCID, mqAllRaw, cidAllRaw) {
|
|
320
|
+
const mqAll = sortMqAll(mqAllRaw);
|
|
321
|
+
const cidAll = sortCidAll(cidAllRaw);
|
|
322
|
+
if (checkCid(consumerGroup, currentCID, mqAll, cidAll) < 0)
|
|
323
|
+
return [];
|
|
324
|
+
// Group queues by machine room: the rooms are processed in lexicographic order —
|
|
325
|
+
// determinism comes from sorting the input plus Map insertion order.
|
|
326
|
+
const mr2Mq = new Map();
|
|
327
|
+
for (const mq of mqAll) {
|
|
328
|
+
const room = this.resolver.brokerDeployIn(mq);
|
|
329
|
+
if (!room) {
|
|
330
|
+
throw new Error(`Machine room is null for mq ${queueRingKeyString(mq)}`);
|
|
331
|
+
}
|
|
332
|
+
let list = mr2Mq.get(room);
|
|
333
|
+
if (!list)
|
|
334
|
+
mr2Mq.set(room, (list = []));
|
|
335
|
+
list.push(mq);
|
|
336
|
+
}
|
|
337
|
+
// Group consumers by machine room (same non-empty rule).
|
|
338
|
+
const mr2C = new Map();
|
|
339
|
+
for (const cid of cidAll) {
|
|
340
|
+
const room = this.resolver.consumerDeployIn(cid);
|
|
341
|
+
if (!room) {
|
|
342
|
+
throw new Error(`Machine room is null for consumer id ${cid}`);
|
|
343
|
+
}
|
|
344
|
+
let list = mr2C.get(room);
|
|
345
|
+
if (!list)
|
|
346
|
+
mr2C.set(room, (list = []));
|
|
347
|
+
list.push(cid);
|
|
348
|
+
}
|
|
349
|
+
const out = [];
|
|
350
|
+
// 1. This consumer's room: queues split among same-room consumers only.
|
|
351
|
+
const currentRoom = this.resolver.consumerDeployIn(currentCID);
|
|
352
|
+
const mqHere = mr2Mq.get(currentRoom);
|
|
353
|
+
mr2Mq.delete(currentRoom);
|
|
354
|
+
if (mqHere && mqHere.length > 0) {
|
|
355
|
+
out.push(...this.inner.allocate(consumerGroup, currentCID, mqHere, mr2C.get(currentRoom) ?? []));
|
|
356
|
+
}
|
|
357
|
+
// 2. Rooms with no alive consumer: every consumer shares their queues.
|
|
358
|
+
for (const [room, roomMqs] of mr2Mq) {
|
|
359
|
+
if (!mr2C.has(room)) {
|
|
360
|
+
out.push(...this.inner.allocate(consumerGroup, currentCID, roomMqs, cidAll));
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return out;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export declare const MIN_ASYNC_SEND_NUM = 10;
|
|
2
|
+
export declare const MIN_ASYNC_SEND_SIZE: number;
|
|
3
|
+
export declare class FairSemaphore {
|
|
4
|
+
totalPermits: number;
|
|
5
|
+
availablePermits: number;
|
|
6
|
+
private readonly _waiters;
|
|
7
|
+
constructor(totalPermits?: number);
|
|
8
|
+
tryAcquire(permits?: number, timeoutMillis?: number): Promise<boolean>;
|
|
9
|
+
release(permits?: number): void;
|
|
10
|
+
availablePermitsCount(): number;
|
|
11
|
+
totalPermitsCount(): number;
|
|
12
|
+
waitingCount(): number;
|
|
13
|
+
setTotalPermits(permits: number): void;
|
|
14
|
+
private _dispatch;
|
|
15
|
+
private _grant;
|
|
16
|
+
private _drop;
|
|
17
|
+
}
|
|
18
|
+
declare const _default: {
|
|
19
|
+
FairSemaphore: typeof FairSemaphore;
|
|
20
|
+
MIN_ASYNC_SEND_NUM: number;
|
|
21
|
+
MIN_ASYNC_SEND_SIZE: number;
|
|
22
|
+
};
|
|
23
|
+
export default _default;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// The async-send backpressure gate: two fair semaphores bound how many async sends, and how
|
|
2
|
+
// many in-flight bytes, the producer has outstanding at once. They are only consulted while
|
|
3
|
+
// enableBackpressureForAsyncMode is on.
|
|
4
|
+
//
|
|
5
|
+
// A request that finds no permits free waits for them until its own budget runs out, and then
|
|
6
|
+
// reports the refusal to its caller. Only the head of the line may be granted, so a request
|
|
7
|
+
// never starves behind a larger one that keeps jumping the queue.
|
|
8
|
+
//
|
|
9
|
+
// Waiting is only fair if a refusal is final. A wake-up can be delivered while the caller still
|
|
10
|
+
// has budget and taken after that budget is gone, so every pass after the first re-checks the
|
|
11
|
+
// deadline before granting: handing a permit to a caller that had already given up would leave
|
|
12
|
+
// it holding capacity the next waiter could still have used in time.
|
|
13
|
+
//
|
|
14
|
+
// A non-positive budget is a single poll: granted when permits are free right now, refused
|
|
15
|
+
// otherwise -- it never waits.
|
|
16
|
+
export const MIN_ASYNC_SEND_NUM = 10;
|
|
17
|
+
export const MIN_ASYNC_SEND_SIZE = 1024 * 1024;
|
|
18
|
+
export class FairSemaphore {
|
|
19
|
+
totalPermits;
|
|
20
|
+
availablePermits;
|
|
21
|
+
_waiters = [];
|
|
22
|
+
constructor(totalPermits = 1024) {
|
|
23
|
+
this.totalPermits = totalPermits;
|
|
24
|
+
this.availablePermits = totalPermits;
|
|
25
|
+
}
|
|
26
|
+
tryAcquire(permits = 1, timeoutMillis = 0) {
|
|
27
|
+
const deadline = Date.now() + Math.max(timeoutMillis, 0);
|
|
28
|
+
if (this._waiters.length === 0) {
|
|
29
|
+
if (this.availablePermits >= permits) {
|
|
30
|
+
this.availablePermits -= permits;
|
|
31
|
+
return Promise.resolve(true);
|
|
32
|
+
}
|
|
33
|
+
if (timeoutMillis <= 0)
|
|
34
|
+
return Promise.resolve(false);
|
|
35
|
+
}
|
|
36
|
+
else if (timeoutMillis <= 0) {
|
|
37
|
+
return Promise.resolve(false);
|
|
38
|
+
}
|
|
39
|
+
let settle = () => { };
|
|
40
|
+
const answered = new Promise((resolve) => { settle = resolve; });
|
|
41
|
+
const waiter = {
|
|
42
|
+
permits,
|
|
43
|
+
deadline,
|
|
44
|
+
resolve: (granted) => settle(granted),
|
|
45
|
+
timer: setTimeout(() => { this._drop(waiter); this._dispatch(); }, Math.max(deadline - Date.now(), 1)),
|
|
46
|
+
};
|
|
47
|
+
this._waiters.push(waiter);
|
|
48
|
+
return answered;
|
|
49
|
+
}
|
|
50
|
+
release(permits = 1) {
|
|
51
|
+
if (permits <= 0)
|
|
52
|
+
return;
|
|
53
|
+
this.availablePermits += permits;
|
|
54
|
+
this._dispatch();
|
|
55
|
+
}
|
|
56
|
+
availablePermitsCount() { return this.availablePermits; }
|
|
57
|
+
totalPermitsCount() { return this.totalPermits; }
|
|
58
|
+
// Requests still in line. Only a caller that gave up on the spot leaves none behind, so this
|
|
59
|
+
// is the observable half of "a refusal is final".
|
|
60
|
+
waitingCount() { return this._waiters.length; }
|
|
61
|
+
setTotalPermits(permits) {
|
|
62
|
+
// The free counter moves by the delta, so shrinking below what is already in flight gives a
|
|
63
|
+
// negative margin that the gate refuses to grant over until that traffic drains.
|
|
64
|
+
this.availablePermits += permits - this.totalPermits;
|
|
65
|
+
this.totalPermits = permits;
|
|
66
|
+
this._dispatch();
|
|
67
|
+
}
|
|
68
|
+
_dispatch() {
|
|
69
|
+
while (this._waiters.length > 0) {
|
|
70
|
+
const head = this._waiters[0];
|
|
71
|
+
if (head.deadline - Date.now() <= 0) {
|
|
72
|
+
this._drop(head);
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (this.availablePermits < head.permits)
|
|
76
|
+
return;
|
|
77
|
+
this._grant(head);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
_grant(head) {
|
|
81
|
+
this._waiters.shift();
|
|
82
|
+
clearTimeout(head.timer);
|
|
83
|
+
this.availablePermits -= head.permits;
|
|
84
|
+
head.resolve(true);
|
|
85
|
+
}
|
|
86
|
+
_drop(head) {
|
|
87
|
+
const at = this._waiters.indexOf(head);
|
|
88
|
+
if (at < 0)
|
|
89
|
+
return;
|
|
90
|
+
this._waiters.splice(at, 1);
|
|
91
|
+
clearTimeout(head.timer);
|
|
92
|
+
head.resolve(false);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
export default { FairSemaphore, MIN_ASYNC_SEND_NUM, MIN_ASYNC_SEND_SIZE };
|