rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +428 -0
- package/README.md +424 -2
- package/dist/client/admin.d.ts +229 -0
- package/dist/client/admin.js +2462 -0
- package/dist/client/allocate.d.ts +48 -0
- package/dist/client/allocate.js +365 -0
- package/dist/client/backpressure.d.ts +23 -0
- package/dist/client/backpressure.js +95 -0
- package/dist/client/consumer.d.ts +294 -0
- package/dist/client/consumer.js +3095 -0
- package/dist/client/consumer_result.d.ts +129 -0
- package/dist/client/consumer_result.js +138 -0
- package/dist/client/consumer_stats.d.ts +69 -0
- package/dist/client/consumer_stats.js +258 -0
- package/dist/client/exception.d.ts +39 -0
- package/dist/client/exception.js +36 -0
- package/dist/client/hook.d.ts +145 -0
- package/dist/client/hook.js +171 -0
- package/dist/client/latency.d.ts +61 -0
- package/dist/client/latency.js +239 -0
- package/dist/client/lite_pull_consumer.d.ts +151 -0
- package/dist/client/lite_pull_consumer.js +1096 -0
- package/dist/client/mq_client.d.ts +188 -0
- package/dist/client/mq_client.js +1678 -0
- package/dist/client/mq_client_manager.d.ts +21 -0
- package/dist/client/mq_client_manager.js +129 -0
- package/dist/client/offset_store.d.ts +45 -0
- package/dist/client/offset_store.js +389 -0
- package/dist/client/open_tracing_hook.d.ts +52 -0
- package/dist/client/open_tracing_hook.js +241 -0
- package/dist/client/pop_api.d.ts +28 -0
- package/dist/client/pop_api.js +224 -0
- package/dist/client/pop_consumer.d.ts +23 -0
- package/dist/client/pop_consumer.js +719 -0
- package/dist/client/pop_process_queue.d.ts +15 -0
- package/dist/client/pop_process_queue.js +51 -0
- package/dist/client/process_queue.d.ts +49 -0
- package/dist/client/process_queue.js +261 -0
- package/dist/client/produce_accumulator.d.ts +75 -0
- package/dist/client/produce_accumulator.js +288 -0
- package/dist/client/producer.d.ts +231 -0
- package/dist/client/producer.js +1774 -0
- package/dist/client/pull_api.d.ts +26 -0
- package/dist/client/pull_api.js +264 -0
- package/dist/client/pull_consumer.d.ts +119 -0
- package/dist/client/pull_consumer.js +876 -0
- package/dist/client/pull_schedule_service.d.ts +71 -0
- package/dist/client/pull_schedule_service.js +424 -0
- package/dist/client/request_reply.d.ts +61 -0
- package/dist/client/request_reply.js +232 -0
- package/dist/client/retry_message.d.ts +2 -0
- package/dist/client/retry_message.js +27 -0
- package/dist/client/send_result.d.ts +57 -0
- package/dist/client/send_result.js +70 -0
- package/dist/client/top_addressing.d.ts +20 -0
- package/dist/client/top_addressing.js +75 -0
- package/dist/client/trace_context.d.ts +61 -0
- package/dist/client/trace_context.js +373 -0
- package/dist/client/trace_dispatcher.d.ts +31 -0
- package/dist/client/trace_dispatcher.js +236 -0
- package/dist/client/trace_hook.d.ts +23 -0
- package/dist/client/trace_hook.js +228 -0
- package/dist/client/traceparent.d.ts +9 -0
- package/dist/client/traceparent.js +86 -0
- package/dist/common/boundary_type.d.ts +7 -0
- package/dist/common/boundary_type.js +16 -0
- package/dist/common/compress.d.ts +28 -0
- package/dist/common/compress.js +512 -0
- package/dist/common/message.d.ts +129 -0
- package/dist/common/message.js +214 -0
- package/dist/common/messageClientIdSetter.d.ts +7 -0
- package/dist/common/messageClientIdSetter.js +48 -0
- package/dist/common/messageConst.d.ts +68 -0
- package/dist/common/messageConst.js +68 -0
- package/dist/common/messageDecoder.d.ts +40 -0
- package/dist/common/messageDecoder.js +504 -0
- package/dist/common/messageType.d.ts +11 -0
- package/dist/common/messageType.js +11 -0
- package/dist/common/message_accessor.d.ts +39 -0
- package/dist/common/message_accessor.js +135 -0
- package/dist/common/mixAll.d.ts +106 -0
- package/dist/common/mixAll.js +247 -0
- package/dist/common/recall_message_handle.d.ts +25 -0
- package/dist/common/recall_message_handle.js +98 -0
- package/dist/common/subscriptionData.d.ts +36 -0
- package/dist/common/subscriptionData.js +64 -0
- package/dist/common/sysflag.d.ts +129 -0
- package/dist/common/sysflag.js +96 -0
- package/dist/common/topic_config.d.ts +66 -0
- package/dist/common/topic_config.js +106 -0
- package/dist/common/topic_validator.d.ts +49 -0
- package/dist/common/topic_validator.js +115 -0
- package/dist/common/utilAll.d.ts +39 -0
- package/dist/common/utilAll.js +154 -0
- package/dist/common/validators.d.ts +25 -0
- package/dist/common/validators.js +80 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +36 -0
- package/dist/logging.d.ts +24 -0
- package/dist/logging.js +246 -0
- package/dist/remoting/acl.d.ts +16 -0
- package/dist/remoting/acl.js +39 -0
- package/dist/remoting/admin_body.d.ts +257 -0
- package/dist/remoting/admin_body.js +669 -0
- package/dist/remoting/bodies.d.ts +475 -0
- package/dist/remoting/bodies.js +1131 -0
- package/dist/remoting/client.d.ts +73 -0
- package/dist/remoting/client.js +591 -0
- package/dist/remoting/codes.d.ts +296 -0
- package/dist/remoting/codes.js +301 -0
- package/dist/remoting/exception.d.ts +45 -0
- package/dist/remoting/exception.js +58 -0
- package/dist/remoting/extra_info.d.ts +61 -0
- package/dist/remoting/extra_info.js +220 -0
- package/dist/remoting/headers.d.ts +758 -0
- package/dist/remoting/headers.js +1307 -0
- package/dist/remoting/heartbeat.d.ts +91 -0
- package/dist/remoting/heartbeat.js +227 -0
- package/dist/remoting/namespace.d.ts +14 -0
- package/dist/remoting/namespace.js +85 -0
- package/dist/remoting/pop_bodies.d.ts +58 -0
- package/dist/remoting/pop_bodies.js +214 -0
- package/dist/remoting/remotingCommand.d.ts +39 -0
- package/dist/remoting/remotingCommand.js +225 -0
- package/dist/remoting/route.d.ts +61 -0
- package/dist/remoting/route.js +288 -0
- package/dist/remoting/rpc_hooks.d.ts +31 -0
- package/dist/remoting/rpc_hooks.js +73 -0
- package/dist/remoting/serialize.d.ts +33 -0
- package/dist/remoting/serialize.js +372 -0
- package/dist/remoting/socks_proxy.d.ts +29 -0
- package/dist/remoting/socks_proxy.js +379 -0
- package/dist/remoting/static_topic.d.ts +49 -0
- package/dist/remoting/static_topic.js +244 -0
- package/dist/remoting/subscription.d.ts +65 -0
- package/dist/remoting/subscription.js +172 -0
- package/package.json +38 -4
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { MQClient } from './mq_client.js';
|
|
2
|
+
type MQClientFactory = () => MQClient;
|
|
3
|
+
export declare class MQClientManager {
|
|
4
|
+
private static _singleton;
|
|
5
|
+
static getInstance(): MQClientManager;
|
|
6
|
+
private _factoryTable;
|
|
7
|
+
private constructor();
|
|
8
|
+
getOrCreateMQClientInstance(clientId: string, factory: MQClientFactory): {
|
|
9
|
+
instance: MQClient;
|
|
10
|
+
created: boolean;
|
|
11
|
+
};
|
|
12
|
+
registerClient(clientId: string, instance: MQClient): void;
|
|
13
|
+
unregisterClient(clientId: string, instance: MQClient | null): boolean;
|
|
14
|
+
private _shutdownQuietly;
|
|
15
|
+
removeClientFactory(clientId: string, instance?: MQClient | null): void;
|
|
16
|
+
getClientFactory(clientId: string): MQClient | null;
|
|
17
|
+
refcountOf(clientId: string): number;
|
|
18
|
+
instanceCount(): number;
|
|
19
|
+
reset(): void;
|
|
20
|
+
}
|
|
21
|
+
export default MQClientManager;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { getLogger } from "../logging.js";
|
|
2
|
+
const logger = getLogger('mqclient.manager');
|
|
3
|
+
// Cache entry: instance + in-use facade count (reference count).
|
|
4
|
+
class FactoryEntry {
|
|
5
|
+
instance;
|
|
6
|
+
refcount;
|
|
7
|
+
constructor(instance) {
|
|
8
|
+
this.instance = instance;
|
|
9
|
+
this.refcount = 0;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export class MQClientManager {
|
|
13
|
+
static _singleton = null;
|
|
14
|
+
static getInstance() {
|
|
15
|
+
if (MQClientManager._singleton == null) {
|
|
16
|
+
MQClientManager._singleton = new MQClientManager();
|
|
17
|
+
}
|
|
18
|
+
return MQClientManager._singleton;
|
|
19
|
+
}
|
|
20
|
+
_factoryTable;
|
|
21
|
+
constructor() {
|
|
22
|
+
this._factoryTable = new Map();
|
|
23
|
+
}
|
|
24
|
+
getOrCreateMQClientInstance(clientId, factory) {
|
|
25
|
+
if (clientId == null || !String(clientId).trim()) {
|
|
26
|
+
throw new Error('clientId is empty');
|
|
27
|
+
}
|
|
28
|
+
const entry = this._factoryTable.get(clientId);
|
|
29
|
+
if (entry != null) {
|
|
30
|
+
logger.debug('reuse MQClientInstance for clientId: %s', clientId);
|
|
31
|
+
return { instance: entry.instance, created: false };
|
|
32
|
+
}
|
|
33
|
+
const instance = factory();
|
|
34
|
+
// A synchronous event loop guarantees nobody can slip in, but re-check the
|
|
35
|
+
// table with a "only the winner takes effect" rule anyway, to guard
|
|
36
|
+
// against a possible future await-ification refactor.
|
|
37
|
+
const existing = this._factoryTable.get(clientId);
|
|
38
|
+
if (existing != null) {
|
|
39
|
+
return { instance: existing.instance, created: false };
|
|
40
|
+
}
|
|
41
|
+
this._factoryTable.set(clientId, new FactoryEntry(instance));
|
|
42
|
+
logger.info('Created new MQClientInstance for clientId: %s', clientId);
|
|
43
|
+
return { instance, created: true };
|
|
44
|
+
}
|
|
45
|
+
// ---------------- reference counting ----------------
|
|
46
|
+
// Called on facade start(): in-use count +1.
|
|
47
|
+
// Instances that never went through getOrCreateMQClientInstance (test
|
|
48
|
+
// doubles / legacy callers constructing directly) are registered the same
|
|
49
|
+
// way, keeping unregister's count balanced.
|
|
50
|
+
registerClient(clientId, instance) {
|
|
51
|
+
const entry = this._factoryTable.get(clientId);
|
|
52
|
+
if (entry == null || entry.instance !== instance) {
|
|
53
|
+
this._factoryTable.set(clientId, new FactoryEntry(instance));
|
|
54
|
+
const fresh = this._factoryTable.get(clientId);
|
|
55
|
+
fresh.refcount = 1;
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
entry.refcount += 1;
|
|
59
|
+
}
|
|
60
|
+
// Called on facade shutdown(): count -1; the real close happens ONLY when it
|
|
61
|
+
// hits ZERO.
|
|
62
|
+
// On zero, remove from the table first, then shutdown() — after removal,
|
|
63
|
+
// other facades' getOrCreate can only create a new instance.
|
|
64
|
+
// When the instance is not in the table (test double / an old instance
|
|
65
|
+
// already displaced or removed), keep the facade's old shutdown semantics:
|
|
66
|
+
// close it directly (if it has a close method); the caller needs no
|
|
67
|
+
// distinction.
|
|
68
|
+
// Returns true when this call actually shut the instance down.
|
|
69
|
+
unregisterClient(clientId, instance) {
|
|
70
|
+
if (instance == null)
|
|
71
|
+
return false;
|
|
72
|
+
const entry = this._factoryTable.get(clientId);
|
|
73
|
+
if (entry == null || entry.instance !== instance) {
|
|
74
|
+
// Not the instance in this manager's cache: leave any other entry in the
|
|
75
|
+
// cache alone, and just close the instance the caller holds, per the old
|
|
76
|
+
// semantics.
|
|
77
|
+
this._shutdownQuietly(instance);
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
entry.refcount -= 1;
|
|
81
|
+
if (entry.refcount > 0) {
|
|
82
|
+
logger.debug('MQClientInstance %s still held by %d client(s)', clientId, entry.refcount);
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
this._factoryTable.delete(clientId);
|
|
86
|
+
logger.info('the client factory [%s] shutdown OK', clientId);
|
|
87
|
+
this._shutdownQuietly(instance);
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
_shutdownQuietly(instance) {
|
|
91
|
+
try {
|
|
92
|
+
if (instance != null && typeof instance.shutdown === 'function') {
|
|
93
|
+
instance.shutdown();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
catch (e) {
|
|
97
|
+
logger.warn('MQClientInstance shutdown raised: %s', e.message);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
removeClientFactory(clientId, instance) {
|
|
101
|
+
const entry = this._factoryTable.get(clientId);
|
|
102
|
+
if (entry == null)
|
|
103
|
+
return;
|
|
104
|
+
if (instance != null && entry.instance !== instance)
|
|
105
|
+
return;
|
|
106
|
+
this._factoryTable.delete(clientId);
|
|
107
|
+
}
|
|
108
|
+
// ---------------- observability / tests ----------------
|
|
109
|
+
getClientFactory(clientId) {
|
|
110
|
+
const entry = this._factoryTable.get(clientId);
|
|
111
|
+
return entry != null ? entry.instance : null;
|
|
112
|
+
}
|
|
113
|
+
// The current in-use count for this clientId (observability / unit tests).
|
|
114
|
+
refcountOf(clientId) {
|
|
115
|
+
const entry = this._factoryTable.get(clientId);
|
|
116
|
+
return entry != null ? entry.refcount : 0;
|
|
117
|
+
}
|
|
118
|
+
// The number of currently live instances.
|
|
119
|
+
instanceCount() {
|
|
120
|
+
return this._factoryTable.size;
|
|
121
|
+
}
|
|
122
|
+
// Clear the cache (for test isolation). Only removes table entries, no
|
|
123
|
+
// shutdown — matching the old behavior; leaked instances are reclaimed at
|
|
124
|
+
// process exit without introducing new close side effects.
|
|
125
|
+
reset() {
|
|
126
|
+
this._factoryTable.clear();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
export default MQClientManager;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { MessageQueue } from '../common/message.js';
|
|
2
|
+
export declare const ReadOffsetMode: {
|
|
3
|
+
readonly READ_FROM_MEMORY: 0;
|
|
4
|
+
readonly READ_FROM_STORE: 1;
|
|
5
|
+
readonly READ_FROM_MEMORY_THEN_STORE: 2;
|
|
6
|
+
};
|
|
7
|
+
export type ReadOffsetMode = number;
|
|
8
|
+
export declare function mqKey(mq: MessageQueue): string;
|
|
9
|
+
export interface OffsetStore {
|
|
10
|
+
load(): Promise<void>;
|
|
11
|
+
updateOffset(mq: MessageQueue, offset: number, increaseOnly: boolean): void;
|
|
12
|
+
readOffset(mq: MessageQueue, mode: ReadOffsetMode): Promise<number>;
|
|
13
|
+
persistAll(mqs: MessageQueue[]): Promise<void>;
|
|
14
|
+
persist(mq: MessageQueue): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
export declare class RemoteBrokerOffsetStore implements OffsetStore {
|
|
17
|
+
private mqClient;
|
|
18
|
+
private group;
|
|
19
|
+
private table;
|
|
20
|
+
constructor(mqClient: any, group: string);
|
|
21
|
+
load(): Promise<void>;
|
|
22
|
+
updateOffset(mq: MessageQueue, offset: number, increaseOnly: boolean): void;
|
|
23
|
+
readOffset(mq: MessageQueue, mode: ReadOffsetMode): Promise<number>;
|
|
24
|
+
persistAll(mqs: MessageQueue[]): Promise<void>;
|
|
25
|
+
persist(mq: MessageQueue): Promise<void>;
|
|
26
|
+
tableSnapshot(): Map<string, number>;
|
|
27
|
+
private updateConsumeOffsetToBroker;
|
|
28
|
+
}
|
|
29
|
+
export declare class LocalFileOffsetStore implements OffsetStore {
|
|
30
|
+
private clientID;
|
|
31
|
+
private group;
|
|
32
|
+
private table;
|
|
33
|
+
private storePath;
|
|
34
|
+
constructor(clientID: string, group: string, baseDir?: string);
|
|
35
|
+
load(): Promise<void>;
|
|
36
|
+
updateOffset(mq: MessageQueue, offset: number, increaseOnly: boolean): void;
|
|
37
|
+
readOffset(mq: MessageQueue, _mode: ReadOffsetMode): Promise<number>;
|
|
38
|
+
persistAll(mqs: MessageQueue[]): Promise<void>;
|
|
39
|
+
persist(mq: MessageQueue): Promise<void>;
|
|
40
|
+
private readDiskTable;
|
|
41
|
+
private writeTable;
|
|
42
|
+
storePathValue(): string;
|
|
43
|
+
tableSnapshot(): Map<string, number>;
|
|
44
|
+
}
|
|
45
|
+
export declare function messageQueueKeyJson(mq: MessageQueue): string;
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import { MessageQueue } from "../common/message.js";
|
|
5
|
+
import { getLogger } from "../logging.js";
|
|
6
|
+
const logger = getLogger('client.offset_store');
|
|
7
|
+
export const ReadOffsetMode = {
|
|
8
|
+
READ_FROM_MEMORY: 0,
|
|
9
|
+
READ_FROM_STORE: 1,
|
|
10
|
+
READ_FROM_MEMORY_THEN_STORE: 2,
|
|
11
|
+
};
|
|
12
|
+
// Canonical map key for a MessageQueue — the (topic, brokerName, queueId)
|
|
13
|
+
// triple. NEVER key only by queueId: the implicit %RETRY%<group> subscription
|
|
14
|
+
// shares queueIds with the business topic and would produce phantom overlaps.
|
|
15
|
+
export function mqKey(mq) {
|
|
16
|
+
return `${mq.getTopic()}@${mq.getBrokerName()}@${mq.getQueueId()}`;
|
|
17
|
+
}
|
|
18
|
+
// ---------------------------------------------------------------- remote store
|
|
19
|
+
// RemoteBrokerOffsetStore backs CLUSTERING consumers.
|
|
20
|
+
export class RemoteBrokerOffsetStore {
|
|
21
|
+
mqClient;
|
|
22
|
+
group;
|
|
23
|
+
table = new Map();
|
|
24
|
+
constructor(mqClient, group) {
|
|
25
|
+
this.mqClient = mqClient;
|
|
26
|
+
this.group = group;
|
|
27
|
+
}
|
|
28
|
+
// Load is a no-op: remote offsets are fetched on demand.
|
|
29
|
+
async load() { }
|
|
30
|
+
updateOffset(mq, offset, increaseOnly) {
|
|
31
|
+
const key = mqKey(mq);
|
|
32
|
+
const old = this.table.get(key);
|
|
33
|
+
if (old !== undefined) {
|
|
34
|
+
if (increaseOnly && offset < old)
|
|
35
|
+
return;
|
|
36
|
+
this.table.set(key, offset);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
this.table.set(key, offset);
|
|
40
|
+
}
|
|
41
|
+
// ReadOffset answers from the table (memory modes) or the broker.
|
|
42
|
+
// -1 = unknown (no entry / broker says QUERY_NOT_FOUND); -2 = the broker read
|
|
43
|
+
// itself failed, which is NOT "nobody committed" and must stay distinguishable.
|
|
44
|
+
async readOffset(mq, mode) {
|
|
45
|
+
if (mode === ReadOffsetMode.READ_FROM_MEMORY || mode === ReadOffsetMode.READ_FROM_MEMORY_THEN_STORE) {
|
|
46
|
+
const offset = this.table.get(mqKey(mq));
|
|
47
|
+
if (offset !== undefined)
|
|
48
|
+
return offset;
|
|
49
|
+
if (mode === ReadOffsetMode.READ_FROM_MEMORY)
|
|
50
|
+
return -1;
|
|
51
|
+
}
|
|
52
|
+
// READ_FROM_STORE (and MEMORY_FIRST on a table miss): ask the broker.
|
|
53
|
+
const addr = this.mqClient.findBrokerAddrForQueue(mq);
|
|
54
|
+
if (!addr) {
|
|
55
|
+
logger.warning('readOffset: broker[%s] not exist for %s', mq.getBrokerName(), mqKey(mq));
|
|
56
|
+
return -2;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
const { found, offset } = await this.mqClient.queryConsumerOffset(addr, this.group, mq);
|
|
60
|
+
if (!found)
|
|
61
|
+
return -1;
|
|
62
|
+
this.updateOffset(mq, offset, false);
|
|
63
|
+
return offset;
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
logger.warning('readOffset failed for %s: %s', mqKey(mq), e.message);
|
|
67
|
+
return -2;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
async persistAll(mqs) {
|
|
71
|
+
if (mqs.length === 0)
|
|
72
|
+
return;
|
|
73
|
+
// Entries already in the table when the flush started: those not requested
|
|
74
|
+
// are revoked queues and get dropped (Java persistAll parity). Entries added
|
|
75
|
+
// DURING the flush — a concurrent callback committing a just-pulled offset —
|
|
76
|
+
// must survive: dropping them used to lose offsets whenever two
|
|
77
|
+
// callback-driven flushes overlapped (the second flush then saw an empty
|
|
78
|
+
// table and silently persisted nothing).
|
|
79
|
+
const presentAtStart = new Set(this.table.keys());
|
|
80
|
+
let firstErr = null;
|
|
81
|
+
for (const mq of mqs) {
|
|
82
|
+
const offset = this.table.get(mqKey(mq));
|
|
83
|
+
if (offset === undefined || offset < 0)
|
|
84
|
+
continue;
|
|
85
|
+
try {
|
|
86
|
+
await this.updateConsumeOffsetToBroker(mq, offset);
|
|
87
|
+
}
|
|
88
|
+
catch (e) {
|
|
89
|
+
if (!firstErr)
|
|
90
|
+
firstErr = e;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const requested = new Set();
|
|
94
|
+
for (const mq of mqs)
|
|
95
|
+
requested.add(mqKey(mq));
|
|
96
|
+
for (const key of Array.from(this.table.keys())) {
|
|
97
|
+
if (presentAtStart.has(key) && !requested.has(key))
|
|
98
|
+
this.table.delete(key);
|
|
99
|
+
}
|
|
100
|
+
if (firstErr)
|
|
101
|
+
throw firstErr;
|
|
102
|
+
}
|
|
103
|
+
// Persist flushes one queue.
|
|
104
|
+
async persist(mq) {
|
|
105
|
+
const offset = this.table.get(mqKey(mq));
|
|
106
|
+
if (offset === undefined || offset < 0)
|
|
107
|
+
return;
|
|
108
|
+
return this.updateConsumeOffsetToBroker(mq, offset);
|
|
109
|
+
}
|
|
110
|
+
// Table snapshots the cached table (checks and tests).
|
|
111
|
+
tableSnapshot() {
|
|
112
|
+
return new Map(this.table);
|
|
113
|
+
}
|
|
114
|
+
async updateConsumeOffsetToBroker(mq, offset) {
|
|
115
|
+
const addr = this.mqClient.findBrokerAddrForQueue(mq);
|
|
116
|
+
if (!addr)
|
|
117
|
+
throw new Error(`The broker[${mq.getBrokerName()}] not exist`);
|
|
118
|
+
await this.mqClient.updateConsumerOffset(addr, this.group, mq, offset);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// ---------------------------------------------------------------- local store
|
|
122
|
+
// LocalFileOffsetStore backs BROADCASTING consumers with a JSON file:
|
|
123
|
+
// ~/.rocketmq_offsets/<clientId>/<group>/offsets.json.
|
|
124
|
+
export class LocalFileOffsetStore {
|
|
125
|
+
clientID;
|
|
126
|
+
group;
|
|
127
|
+
table = new Map(); // mqKey -> offset
|
|
128
|
+
storePath;
|
|
129
|
+
constructor(clientID, group, baseDir) {
|
|
130
|
+
this.clientID = clientID;
|
|
131
|
+
this.group = group;
|
|
132
|
+
let base = baseDir || process.env['rocketmq.client.localOffsetStoreDir'] || '';
|
|
133
|
+
if (!base)
|
|
134
|
+
base = path.join(os.homedir(), '.rocketmq_offsets');
|
|
135
|
+
this.storePath = path.join(base, clientID, group, 'offsets.json');
|
|
136
|
+
}
|
|
137
|
+
async load() {
|
|
138
|
+
let data = null;
|
|
139
|
+
try {
|
|
140
|
+
data = fs.readFileSync(this.storePath);
|
|
141
|
+
}
|
|
142
|
+
catch (e1) {
|
|
143
|
+
try {
|
|
144
|
+
data = fs.readFileSync(this.storePath + '.bak');
|
|
145
|
+
}
|
|
146
|
+
catch (e2) {
|
|
147
|
+
return; // both missing -> empty table
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
this.table = decodeLocalOffsetTable(data);
|
|
152
|
+
}
|
|
153
|
+
catch (e) {
|
|
154
|
+
logger.warning('load local offset store %s failed: %s', this.storePath, e.message);
|
|
155
|
+
this.table = new Map();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// UpdateOffset records the offset (increaseOnly keeps the max).
|
|
159
|
+
updateOffset(mq, offset, increaseOnly) {
|
|
160
|
+
const key = mqKey(mq);
|
|
161
|
+
const old = this.table.get(key);
|
|
162
|
+
if (old !== undefined) {
|
|
163
|
+
if (increaseOnly && offset < old)
|
|
164
|
+
return;
|
|
165
|
+
this.table.set(key, offset);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
this.table.set(key, offset);
|
|
169
|
+
}
|
|
170
|
+
// ReadOffset answers from memory only; -1 when the queue has no entry.
|
|
171
|
+
async readOffset(mq, _mode) {
|
|
172
|
+
const offset = this.table.get(mqKey(mq));
|
|
173
|
+
return offset === undefined ? -1 : offset;
|
|
174
|
+
}
|
|
175
|
+
// PersistAll merges with the on-disk table: the file still holds offsets for
|
|
176
|
+
// queues this round no longer owns, so read it back and overlay only the
|
|
177
|
+
// flushed queues instead of replacing the file with the in-memory table.
|
|
178
|
+
async persistAll(mqs) {
|
|
179
|
+
if (mqs.length === 0)
|
|
180
|
+
return;
|
|
181
|
+
const merged = this.readDiskTable();
|
|
182
|
+
for (const mq of mqs) {
|
|
183
|
+
const key = mqKey(mq);
|
|
184
|
+
const offset = this.table.get(key);
|
|
185
|
+
if (offset === undefined)
|
|
186
|
+
continue;
|
|
187
|
+
merged.set(key, offset);
|
|
188
|
+
}
|
|
189
|
+
this.writeTable(merged);
|
|
190
|
+
}
|
|
191
|
+
async persist(mq) {
|
|
192
|
+
const key = mqKey(mq);
|
|
193
|
+
const offset = this.table.get(key);
|
|
194
|
+
if (offset === undefined)
|
|
195
|
+
return;
|
|
196
|
+
const merged = this.readDiskTable();
|
|
197
|
+
merged.set(key, offset);
|
|
198
|
+
this.writeTable(merged);
|
|
199
|
+
}
|
|
200
|
+
readDiskTable() {
|
|
201
|
+
let data = null;
|
|
202
|
+
try {
|
|
203
|
+
data = fs.readFileSync(this.storePath);
|
|
204
|
+
}
|
|
205
|
+
catch (e1) {
|
|
206
|
+
try {
|
|
207
|
+
data = fs.readFileSync(this.storePath + '.bak');
|
|
208
|
+
}
|
|
209
|
+
catch (e2) {
|
|
210
|
+
return new Map(); // both missing -> nothing on disk yet
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
try {
|
|
214
|
+
return decodeLocalOffsetTable(data);
|
|
215
|
+
}
|
|
216
|
+
catch (e) {
|
|
217
|
+
logger.warning('read local offset store %s failed: %s', this.storePath, e.message);
|
|
218
|
+
return new Map();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
writeTable(table) {
|
|
222
|
+
const body = encodeLocalOffsetTable(table);
|
|
223
|
+
if (fs.existsSync(this.storePath)) {
|
|
224
|
+
try {
|
|
225
|
+
fs.writeFileSync(this.storePath + '.bak', fs.readFileSync(this.storePath));
|
|
226
|
+
}
|
|
227
|
+
catch (e) { /* best effort */ }
|
|
228
|
+
}
|
|
229
|
+
fs.mkdirSync(path.dirname(this.storePath), { recursive: true });
|
|
230
|
+
fs.writeFileSync(this.storePath, body, 'utf-8');
|
|
231
|
+
}
|
|
232
|
+
storePathValue() { return this.storePath; }
|
|
233
|
+
// Test/diagnostic accessor.
|
|
234
|
+
tableSnapshot() {
|
|
235
|
+
return new Map(this.table);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
// ------------------------------------------------------------- file format
|
|
239
|
+
// Encoder for the inline-object MessageQueue key form used by the local offset
|
|
240
|
+
// file: {"brokerName":"b","queueId":0,"topic":"T"}. Also reused by the running
|
|
241
|
+
// info table.
|
|
242
|
+
export function messageQueueKeyJson(mq) {
|
|
243
|
+
return JSON.stringify({
|
|
244
|
+
brokerName: mq.getBrokerName(),
|
|
245
|
+
queueId: mq.getQueueId(),
|
|
246
|
+
topic: mq.getTopic(),
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
// The table is written with inline JSON object keys, which JSON.stringify
|
|
250
|
+
// cannot produce (it would quote and escape the key text). Serialize the outer
|
|
251
|
+
// document by hand so the keys stay raw objects.
|
|
252
|
+
function encodeLocalOffsetTable(table) {
|
|
253
|
+
const entries = [];
|
|
254
|
+
for (const [key, offset] of table) {
|
|
255
|
+
const mq = parseMqKey(key);
|
|
256
|
+
if (!mq)
|
|
257
|
+
continue;
|
|
258
|
+
entries.push(`${messageQueueKeyJson(mq)}:${offset}`);
|
|
259
|
+
}
|
|
260
|
+
const body = entries.length === 0 ? '' : `\n\t${entries.join(',\n\t')}\n`;
|
|
261
|
+
return `{\n\t"offsetTable":{${body}}\n}\n`;
|
|
262
|
+
}
|
|
263
|
+
// Reads the offset table accepting all three key spellings seen on disk:
|
|
264
|
+
// an inline object key, a quoted-string key holding that same JSON text, and
|
|
265
|
+
// the legacy topic@brokerName@queueId text.
|
|
266
|
+
function decodeLocalOffsetTable(data) {
|
|
267
|
+
const out = new Map();
|
|
268
|
+
const text = data.toString('utf-8');
|
|
269
|
+
const marker = /"offsetTable"\s*:\s*\{/.exec(text);
|
|
270
|
+
if (!marker)
|
|
271
|
+
return out;
|
|
272
|
+
let i = marker.index + marker[0].length;
|
|
273
|
+
while (i < text.length) {
|
|
274
|
+
while (i < text.length && (text[i] === ',' || /\s/.test(text[i])))
|
|
275
|
+
i++;
|
|
276
|
+
if (i >= text.length || text[i] === '}')
|
|
277
|
+
break;
|
|
278
|
+
let keyText;
|
|
279
|
+
if (text[i] === '{') {
|
|
280
|
+
const end = scanBalancedObject(text, i);
|
|
281
|
+
if (end < 0)
|
|
282
|
+
break;
|
|
283
|
+
keyText = text.slice(i, end + 1);
|
|
284
|
+
i = end + 1;
|
|
285
|
+
}
|
|
286
|
+
else if (text[i] === '"') {
|
|
287
|
+
const parsed = readJsonString(text, i);
|
|
288
|
+
if (parsed.next < 0)
|
|
289
|
+
break;
|
|
290
|
+
keyText = parsed.value;
|
|
291
|
+
i = parsed.next;
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
while (i < text.length && /\s/.test(text[i]))
|
|
297
|
+
i++;
|
|
298
|
+
if (text[i] !== ':')
|
|
299
|
+
break;
|
|
300
|
+
i++;
|
|
301
|
+
while (i < text.length && /\s/.test(text[i]))
|
|
302
|
+
i++;
|
|
303
|
+
const numStart = i;
|
|
304
|
+
while (i < text.length && /[-+0-9.eE]/.test(text[i]))
|
|
305
|
+
i++;
|
|
306
|
+
const offset = Number(text.slice(numStart, i));
|
|
307
|
+
if (Number.isNaN(offset))
|
|
308
|
+
continue;
|
|
309
|
+
const mq = parseFileKey(keyText);
|
|
310
|
+
if (mq)
|
|
311
|
+
out.set(mqKey(mq), offset);
|
|
312
|
+
}
|
|
313
|
+
return out;
|
|
314
|
+
}
|
|
315
|
+
function parseFileKey(keyText) {
|
|
316
|
+
const trimmed = keyText.trim();
|
|
317
|
+
if (trimmed.startsWith('{')) {
|
|
318
|
+
try {
|
|
319
|
+
const inner = JSON.parse(trimmed);
|
|
320
|
+
if (!inner || typeof inner !== 'object')
|
|
321
|
+
return null;
|
|
322
|
+
return new MessageQueue(String(inner['topic'] ?? ''), String(inner['brokerName'] ?? ''), Number(inner['queueId'] ?? 0) | 0);
|
|
323
|
+
}
|
|
324
|
+
catch (e) {
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return parseMqKey(trimmed);
|
|
329
|
+
}
|
|
330
|
+
function scanStringEnd(text, start) {
|
|
331
|
+
let i = start + 1;
|
|
332
|
+
while (i < text.length) {
|
|
333
|
+
if (text[i] === '\\') {
|
|
334
|
+
i += 2;
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
if (text[i] === '"')
|
|
338
|
+
return i;
|
|
339
|
+
i++;
|
|
340
|
+
}
|
|
341
|
+
return -1;
|
|
342
|
+
}
|
|
343
|
+
function readJsonString(text, start) {
|
|
344
|
+
const end = scanStringEnd(text, start);
|
|
345
|
+
if (end < 0)
|
|
346
|
+
return { value: '', next: -1 };
|
|
347
|
+
try {
|
|
348
|
+
return { value: String(JSON.parse(text.slice(start, end + 1))), next: end + 1 };
|
|
349
|
+
}
|
|
350
|
+
catch (e) {
|
|
351
|
+
return { value: '', next: -1 };
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
function scanBalancedObject(text, start) {
|
|
355
|
+
let depth = 0;
|
|
356
|
+
let i = start;
|
|
357
|
+
while (i < text.length) {
|
|
358
|
+
const ch = text[i];
|
|
359
|
+
if (ch === '"') {
|
|
360
|
+
const end = scanStringEnd(text, i);
|
|
361
|
+
if (end < 0)
|
|
362
|
+
return -1;
|
|
363
|
+
i = end + 1;
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
366
|
+
if (ch === '{')
|
|
367
|
+
depth++;
|
|
368
|
+
else if (ch === '}') {
|
|
369
|
+
depth--;
|
|
370
|
+
if (depth === 0)
|
|
371
|
+
return i;
|
|
372
|
+
}
|
|
373
|
+
i++;
|
|
374
|
+
}
|
|
375
|
+
return -1;
|
|
376
|
+
}
|
|
377
|
+
// parseMqKey reverses mqKey() (test/file helpers).
|
|
378
|
+
function parseMqKey(key) {
|
|
379
|
+
const idx1 = key.indexOf('@');
|
|
380
|
+
const idx2 = key.lastIndexOf('@');
|
|
381
|
+
if (idx1 < 0 || idx2 <= idx1)
|
|
382
|
+
return null;
|
|
383
|
+
const topic = key.slice(0, idx1);
|
|
384
|
+
const brokerName = key.slice(idx1 + 1, idx2);
|
|
385
|
+
const queueId = parseInt(key.slice(idx2 + 1), 10);
|
|
386
|
+
if (Number.isNaN(queueId))
|
|
387
|
+
return null;
|
|
388
|
+
return new MessageQueue(topic, brokerName, queueId);
|
|
389
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { ConsumeMessageContext, ConsumeMessageHook, EndTransactionContext, EndTransactionHook, SendMessageContext, SendMessageHook } from './hook.js';
|
|
2
|
+
export declare const OT_SPAN_KIND = "span.kind";
|
|
3
|
+
export declare const OT_SPAN_KIND_PRODUCER = "producer";
|
|
4
|
+
export declare const OT_SPAN_KIND_CONSUMER = "consumer";
|
|
5
|
+
export declare const OT_PEER_SERVICE = "peer.service";
|
|
6
|
+
export declare const OT_MESSAGE_BUS_DESTINATION = "message_bus.destination";
|
|
7
|
+
export declare const OT_ROCKETMQ_SERVICE = "rocketmq";
|
|
8
|
+
export declare const OT_ROCKETMQ_SUCCESS = "rocketmq.success";
|
|
9
|
+
export declare const OT_ROCKETMQ_TAGS = "rocketmq.tags";
|
|
10
|
+
export declare const OT_ROCKETMQ_KEYS = "rocketmq.keys";
|
|
11
|
+
export declare const OT_ROCKETMQ_STORE_HOST = "rocketmq.store_host";
|
|
12
|
+
export declare const OT_ROCKETMQ_BODY_LENGTH = "rocketmq.body_length";
|
|
13
|
+
export declare const OT_ROCKETMQ_MSG_ID = "rocketmq.mgs_id";
|
|
14
|
+
export declare const OT_ROCKETMQ_MSG_TYPE = "rocketmq.mgs_type";
|
|
15
|
+
export declare const OT_ROCKETMQ_REGION_ID = "rocketmq.region_id";
|
|
16
|
+
export declare const OT_ROCKETMQ_TRANSACTION_ID = "rocketmq.transaction_id";
|
|
17
|
+
export declare const OT_ROCKETMQ_TRANSACTION_STATE = "rocketmq.transaction_state";
|
|
18
|
+
export declare const OT_ROCKETMQ_IS_FROM_TRANSACTION_CHECK = "rocketmq.is_from_transaction_check";
|
|
19
|
+
export declare const OT_ROCKETMQ_RETRY_TIMERS = "rocketmq.retry_times";
|
|
20
|
+
export interface OTSpanLike {
|
|
21
|
+
setTag(key: string, value: unknown): unknown;
|
|
22
|
+
finish(): unknown;
|
|
23
|
+
}
|
|
24
|
+
export interface OTTracerLike {
|
|
25
|
+
startSpan(operation: string, options?: {
|
|
26
|
+
tags?: Record<string, unknown>;
|
|
27
|
+
refs?: unknown[];
|
|
28
|
+
}): OTSpanLike;
|
|
29
|
+
extract(carrier: Record<string, string>): unknown;
|
|
30
|
+
inject(spanContext: unknown, carrier: Record<string, string>): void;
|
|
31
|
+
}
|
|
32
|
+
export declare class SendMessageOpenTracingHook extends SendMessageHook {
|
|
33
|
+
private tracer;
|
|
34
|
+
constructor(tracer: OTTracerLike);
|
|
35
|
+
getHookName(): string;
|
|
36
|
+
sendMessageBefore(ctx: SendMessageContext): void;
|
|
37
|
+
sendMessageAfter(ctx: SendMessageContext): void;
|
|
38
|
+
}
|
|
39
|
+
export declare class ConsumeMessageOpenTracingHook extends ConsumeMessageHook {
|
|
40
|
+
private tracer;
|
|
41
|
+
constructor(tracer: OTTracerLike);
|
|
42
|
+
getHookName(): string;
|
|
43
|
+
consumeMessageBefore(ctx: ConsumeMessageContext): void;
|
|
44
|
+
consumeMessageAfter(ctx: ConsumeMessageContext): void;
|
|
45
|
+
}
|
|
46
|
+
export declare class EndTransactionOpenTracingHook extends EndTransactionHook {
|
|
47
|
+
private tracer;
|
|
48
|
+
constructor(tracer: OTTracerLike);
|
|
49
|
+
getHookName(): string;
|
|
50
|
+
endTransactionBefore(ctx: EndTransactionContext): void;
|
|
51
|
+
endTransactionAfter(_ctx: EndTransactionContext): void;
|
|
52
|
+
}
|