rocketmq-client-remoting-nodejs 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +428 -0
- package/README.md +424 -2
- package/dist/client/admin.d.ts +229 -0
- package/dist/client/admin.js +2462 -0
- package/dist/client/allocate.d.ts +48 -0
- package/dist/client/allocate.js +365 -0
- package/dist/client/backpressure.d.ts +23 -0
- package/dist/client/backpressure.js +95 -0
- package/dist/client/consumer.d.ts +294 -0
- package/dist/client/consumer.js +3095 -0
- package/dist/client/consumer_result.d.ts +129 -0
- package/dist/client/consumer_result.js +138 -0
- package/dist/client/consumer_stats.d.ts +69 -0
- package/dist/client/consumer_stats.js +258 -0
- package/dist/client/exception.d.ts +39 -0
- package/dist/client/exception.js +36 -0
- package/dist/client/hook.d.ts +145 -0
- package/dist/client/hook.js +171 -0
- package/dist/client/latency.d.ts +61 -0
- package/dist/client/latency.js +239 -0
- package/dist/client/lite_pull_consumer.d.ts +151 -0
- package/dist/client/lite_pull_consumer.js +1096 -0
- package/dist/client/mq_client.d.ts +188 -0
- package/dist/client/mq_client.js +1678 -0
- package/dist/client/mq_client_manager.d.ts +21 -0
- package/dist/client/mq_client_manager.js +129 -0
- package/dist/client/offset_store.d.ts +45 -0
- package/dist/client/offset_store.js +389 -0
- package/dist/client/open_tracing_hook.d.ts +52 -0
- package/dist/client/open_tracing_hook.js +241 -0
- package/dist/client/pop_api.d.ts +28 -0
- package/dist/client/pop_api.js +224 -0
- package/dist/client/pop_consumer.d.ts +23 -0
- package/dist/client/pop_consumer.js +719 -0
- package/dist/client/pop_process_queue.d.ts +15 -0
- package/dist/client/pop_process_queue.js +51 -0
- package/dist/client/process_queue.d.ts +49 -0
- package/dist/client/process_queue.js +261 -0
- package/dist/client/produce_accumulator.d.ts +75 -0
- package/dist/client/produce_accumulator.js +288 -0
- package/dist/client/producer.d.ts +231 -0
- package/dist/client/producer.js +1774 -0
- package/dist/client/pull_api.d.ts +26 -0
- package/dist/client/pull_api.js +264 -0
- package/dist/client/pull_consumer.d.ts +119 -0
- package/dist/client/pull_consumer.js +876 -0
- package/dist/client/pull_schedule_service.d.ts +71 -0
- package/dist/client/pull_schedule_service.js +424 -0
- package/dist/client/request_reply.d.ts +61 -0
- package/dist/client/request_reply.js +232 -0
- package/dist/client/retry_message.d.ts +2 -0
- package/dist/client/retry_message.js +27 -0
- package/dist/client/send_result.d.ts +57 -0
- package/dist/client/send_result.js +70 -0
- package/dist/client/top_addressing.d.ts +20 -0
- package/dist/client/top_addressing.js +75 -0
- package/dist/client/trace_context.d.ts +61 -0
- package/dist/client/trace_context.js +373 -0
- package/dist/client/trace_dispatcher.d.ts +31 -0
- package/dist/client/trace_dispatcher.js +236 -0
- package/dist/client/trace_hook.d.ts +23 -0
- package/dist/client/trace_hook.js +228 -0
- package/dist/client/traceparent.d.ts +9 -0
- package/dist/client/traceparent.js +86 -0
- package/dist/common/boundary_type.d.ts +7 -0
- package/dist/common/boundary_type.js +16 -0
- package/dist/common/compress.d.ts +28 -0
- package/dist/common/compress.js +512 -0
- package/dist/common/message.d.ts +129 -0
- package/dist/common/message.js +214 -0
- package/dist/common/messageClientIdSetter.d.ts +7 -0
- package/dist/common/messageClientIdSetter.js +48 -0
- package/dist/common/messageConst.d.ts +68 -0
- package/dist/common/messageConst.js +68 -0
- package/dist/common/messageDecoder.d.ts +40 -0
- package/dist/common/messageDecoder.js +504 -0
- package/dist/common/messageType.d.ts +11 -0
- package/dist/common/messageType.js +11 -0
- package/dist/common/message_accessor.d.ts +39 -0
- package/dist/common/message_accessor.js +135 -0
- package/dist/common/mixAll.d.ts +106 -0
- package/dist/common/mixAll.js +247 -0
- package/dist/common/recall_message_handle.d.ts +25 -0
- package/dist/common/recall_message_handle.js +98 -0
- package/dist/common/subscriptionData.d.ts +36 -0
- package/dist/common/subscriptionData.js +64 -0
- package/dist/common/sysflag.d.ts +129 -0
- package/dist/common/sysflag.js +96 -0
- package/dist/common/topic_config.d.ts +66 -0
- package/dist/common/topic_config.js +106 -0
- package/dist/common/topic_validator.d.ts +49 -0
- package/dist/common/topic_validator.js +115 -0
- package/dist/common/utilAll.d.ts +39 -0
- package/dist/common/utilAll.js +154 -0
- package/dist/common/validators.d.ts +25 -0
- package/dist/common/validators.js +80 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +36 -0
- package/dist/logging.d.ts +24 -0
- package/dist/logging.js +246 -0
- package/dist/remoting/acl.d.ts +16 -0
- package/dist/remoting/acl.js +39 -0
- package/dist/remoting/admin_body.d.ts +257 -0
- package/dist/remoting/admin_body.js +669 -0
- package/dist/remoting/bodies.d.ts +475 -0
- package/dist/remoting/bodies.js +1131 -0
- package/dist/remoting/client.d.ts +73 -0
- package/dist/remoting/client.js +591 -0
- package/dist/remoting/codes.d.ts +296 -0
- package/dist/remoting/codes.js +301 -0
- package/dist/remoting/exception.d.ts +45 -0
- package/dist/remoting/exception.js +58 -0
- package/dist/remoting/extra_info.d.ts +61 -0
- package/dist/remoting/extra_info.js +220 -0
- package/dist/remoting/headers.d.ts +758 -0
- package/dist/remoting/headers.js +1307 -0
- package/dist/remoting/heartbeat.d.ts +91 -0
- package/dist/remoting/heartbeat.js +227 -0
- package/dist/remoting/namespace.d.ts +14 -0
- package/dist/remoting/namespace.js +85 -0
- package/dist/remoting/pop_bodies.d.ts +58 -0
- package/dist/remoting/pop_bodies.js +214 -0
- package/dist/remoting/remotingCommand.d.ts +39 -0
- package/dist/remoting/remotingCommand.js +225 -0
- package/dist/remoting/route.d.ts +61 -0
- package/dist/remoting/route.js +288 -0
- package/dist/remoting/rpc_hooks.d.ts +31 -0
- package/dist/remoting/rpc_hooks.js +73 -0
- package/dist/remoting/serialize.d.ts +33 -0
- package/dist/remoting/serialize.js +372 -0
- package/dist/remoting/socks_proxy.d.ts +29 -0
- package/dist/remoting/socks_proxy.js +379 -0
- package/dist/remoting/static_topic.d.ts +49 -0
- package/dist/remoting/static_topic.js +244 -0
- package/dist/remoting/subscription.d.ts +65 -0
- package/dist/remoting/subscription.js +172 -0
- package/package.json +38 -4
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
// -*- coding: utf-8 -*-
|
|
2
|
+
// LZ4 block + frame codec and a minimal usable ZSTD frame codec — pure TypeScript,
|
|
3
|
+
// zero third-party dependencies (repository rule).
|
|
4
|
+
//
|
|
5
|
+
// LZ4: the classic remoting protocol transmits **LZ4 Frame format** on the wire (the server persists
|
|
6
|
+
// and dispatches per the Frame spec, and the lz4 CLI uses the same spec), so compression dispatch
|
|
7
|
+
// calls `lz4CompressFrame`/`lz4DecompressFrame`; the block layer below is what each frame block wraps internally.
|
|
8
|
+
// Why a bare block cannot be sent alone is explained later (see the ⚠ note).
|
|
9
|
+
//
|
|
10
|
+
// ZSTD: compression uses the `node:zlib` zstd binding — bundled with Node, still zero third-party deps. This
|
|
11
|
+
// matters: the broker may write **compressed** Compressed blocks, and a decoder that only reads Raw/RLE frames
|
|
12
|
+
// cannot decode messages written by the broker or other producers.
|
|
13
|
+
// On runtimes without that binding (< 23.8) we fall back to the hand-written codec below: ENCODE emits only
|
|
14
|
+
// valid zstd frames made of RAW blocks (long repeated runs use RLE), which any standard decoder accepts; DECODE only
|
|
15
|
+
// understands Raw/RLE frames. The fallback throws a clear error on a Compressed block —
|
|
16
|
+
// **if unsupported, throw; never silently pass through**.
|
|
17
|
+
import zlib from 'node:zlib';
|
|
18
|
+
import { MessageSysFlag } from "./sysflag.js";
|
|
19
|
+
// ---------------------------------------------------------------- LZ4 block
|
|
20
|
+
const LZ4_MIN_MATCH = 4;
|
|
21
|
+
// Do not start a match within the last 5 bytes; encoder stops matching at
|
|
22
|
+
// len-12 (the conservative "last match must start ≥12B before the end" rule).
|
|
23
|
+
const LZ4_MF_LIMIT = 12;
|
|
24
|
+
const LZ4_HASH_LOG = 16;
|
|
25
|
+
function lz4Hash(u32) {
|
|
26
|
+
return Math.imul(u32, 2654435761) >>> (32 - LZ4_HASH_LOG);
|
|
27
|
+
}
|
|
28
|
+
// lz4CompressBlock encodes data into one LZ4 block (without a frame header) —
|
|
29
|
+
// what RocketMQ places on the wire is the bare block, as one block of a frame.
|
|
30
|
+
export function lz4CompressBlock(data) {
|
|
31
|
+
const n = data.length;
|
|
32
|
+
if (n === 0)
|
|
33
|
+
return Buffer.alloc(0);
|
|
34
|
+
const out = [];
|
|
35
|
+
const table = new Int32Array(1 << LZ4_HASH_LOG).fill(-1);
|
|
36
|
+
let anchor = 0;
|
|
37
|
+
let i = 0;
|
|
38
|
+
const readU32 = (p) => data.readUInt32LE(p);
|
|
39
|
+
// emitSequence writes token -> literals -> offset -> match-length extension
|
|
40
|
+
// (the on-wire order). matchLen < 4 means "literals only" (no match).
|
|
41
|
+
const emitSequence = (litFrom, litTo, off = 0, matchLen = 0) => {
|
|
42
|
+
const litLen = litTo - litFrom;
|
|
43
|
+
let token = 0;
|
|
44
|
+
let ll = litLen;
|
|
45
|
+
if (ll >= 15) {
|
|
46
|
+
token |= 0xf0;
|
|
47
|
+
ll -= 15;
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
token |= ll << 4;
|
|
51
|
+
ll = -1;
|
|
52
|
+
}
|
|
53
|
+
let ml = matchLen - LZ4_MIN_MATCH; // -4-... ; <0 when literals-only
|
|
54
|
+
if (ml >= 0) {
|
|
55
|
+
if (ml >= 15) {
|
|
56
|
+
token |= 0x0f;
|
|
57
|
+
ml -= 15;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
token |= ml;
|
|
61
|
+
ml = -1;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
out.push(token);
|
|
65
|
+
if (ll >= 0) {
|
|
66
|
+
while (ll >= 255) {
|
|
67
|
+
out.push(255);
|
|
68
|
+
ll -= 255;
|
|
69
|
+
}
|
|
70
|
+
out.push(ll);
|
|
71
|
+
}
|
|
72
|
+
for (let p = litFrom; p < litTo; p++)
|
|
73
|
+
out.push(data[p]);
|
|
74
|
+
if (matchLen >= LZ4_MIN_MATCH) {
|
|
75
|
+
out.push(off & 0xff);
|
|
76
|
+
out.push((off >> 8) & 0xff);
|
|
77
|
+
if (ml >= 0) {
|
|
78
|
+
while (ml >= 255) {
|
|
79
|
+
out.push(255);
|
|
80
|
+
ml -= 255;
|
|
81
|
+
}
|
|
82
|
+
out.push(ml);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
while (i + LZ4_MIN_MATCH <= n - LZ4_MF_LIMIT) {
|
|
87
|
+
const u = readU32(i);
|
|
88
|
+
const h = lz4Hash(u);
|
|
89
|
+
const ref = table[h];
|
|
90
|
+
table[h] = i;
|
|
91
|
+
if (ref < 0 || ref >= i || (i - ref) > 65535 || readU32(ref) !== u) {
|
|
92
|
+
i++;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
// Extend the match forward. The last 5 bytes of the block must remain
|
|
96
|
+
// literals (LZ4 block-format rule — reference encoders/decoders rely on
|
|
97
|
+
// it), so a match may not run past n-5.
|
|
98
|
+
let matchLen = LZ4_MIN_MATCH;
|
|
99
|
+
while (i + matchLen < n - 5 && data[ref + matchLen] === data[i + matchLen])
|
|
100
|
+
matchLen++;
|
|
101
|
+
emitSequence(anchor, i, i - ref, matchLen);
|
|
102
|
+
i += matchLen;
|
|
103
|
+
anchor = i;
|
|
104
|
+
}
|
|
105
|
+
// Final literals (no match after them).
|
|
106
|
+
emitSequence(anchor, n);
|
|
107
|
+
return Buffer.from(out);
|
|
108
|
+
}
|
|
109
|
+
// lz4DecompressBlock decodes one LZ4 block. Throws on malformed input.
|
|
110
|
+
export function lz4DecompressBlock(data) {
|
|
111
|
+
const n = data.length;
|
|
112
|
+
// Output grows as needed; LZ4 typically expands ≤4x, 64KB slack on top.
|
|
113
|
+
let out = Buffer.allocUnsafe(data.length * 4 + 65536);
|
|
114
|
+
let o = 0;
|
|
115
|
+
let src = 0;
|
|
116
|
+
const ensure = (extra) => {
|
|
117
|
+
if (o + extra <= out.length)
|
|
118
|
+
return;
|
|
119
|
+
let cap = out.length * 2;
|
|
120
|
+
while (o + extra > cap)
|
|
121
|
+
cap *= 2;
|
|
122
|
+
const grown = Buffer.allocUnsafe(cap);
|
|
123
|
+
out.copy(grown, 0, 0, o);
|
|
124
|
+
out = grown;
|
|
125
|
+
};
|
|
126
|
+
const extLen = (len) => {
|
|
127
|
+
if (len !== 15)
|
|
128
|
+
return len;
|
|
129
|
+
let b = 0;
|
|
130
|
+
do {
|
|
131
|
+
if (src >= n)
|
|
132
|
+
throw new Error('lz4 decompress: truncated extended length');
|
|
133
|
+
b = data[src++];
|
|
134
|
+
len += b;
|
|
135
|
+
} while (b === 255);
|
|
136
|
+
return len;
|
|
137
|
+
};
|
|
138
|
+
while (src < n) {
|
|
139
|
+
const token = data[src++];
|
|
140
|
+
const litLen = extLen(token >> 4);
|
|
141
|
+
if (src + litLen > n)
|
|
142
|
+
throw new Error('lz4 decompress: literals overrun input');
|
|
143
|
+
if (litLen > 0) {
|
|
144
|
+
ensure(litLen);
|
|
145
|
+
data.copy(out, o, src, src + litLen);
|
|
146
|
+
o += litLen;
|
|
147
|
+
src += litLen;
|
|
148
|
+
}
|
|
149
|
+
if (src >= n)
|
|
150
|
+
break; // last-literals block ends here
|
|
151
|
+
if (src + 2 > n)
|
|
152
|
+
throw new Error('lz4 decompress: truncated match offset');
|
|
153
|
+
const off = data[src] | (data[src + 1] << 8);
|
|
154
|
+
src += 2;
|
|
155
|
+
if (off === 0)
|
|
156
|
+
throw new Error('lz4 decompress: zero match offset');
|
|
157
|
+
const matchLen = extLen(token & 0x0f) + LZ4_MIN_MATCH;
|
|
158
|
+
const ref = o - off;
|
|
159
|
+
if (ref < 0)
|
|
160
|
+
throw new Error('lz4 decompress: match offset before start');
|
|
161
|
+
ensure(matchLen);
|
|
162
|
+
// Byte-wise copy (the format allows overlapping matches).
|
|
163
|
+
for (let i = 0; i < matchLen; i++)
|
|
164
|
+
out[o + i] = out[ref + i];
|
|
165
|
+
o += matchLen;
|
|
166
|
+
}
|
|
167
|
+
return out.subarray(0, o);
|
|
168
|
+
}
|
|
169
|
+
// ---------------------------------------------------------------- LZ4 frame
|
|
170
|
+
// ⚠ The LZ4 on RocketMQ's wire is the **LZ4 Frame format** (the server reads/writes per the LZ4 Frame spec) —
|
|
171
|
+
// a bare block can only round-trip with itself and cannot decode messages written by the broker or any other writer.
|
|
172
|
+
// Sending a bare block directly was a latent defect (only exposed when compression interoperability was tested).
|
|
173
|
+
// Frame = magic + FLG + BD + [C.Size(8B)] + HC + {BlockSize(4B) + block}*
|
|
174
|
+
// + EndMark(4B) + [C.Checksum(4B)]; a block is the bare block layer described above.
|
|
175
|
+
const LZ4_MAGIC = 0x184d2204;
|
|
176
|
+
const LZ4_FRAME_BLOCK_SIZE = 65536; // BD=0x40 → 64KB, matching the common LZ4 Frame default
|
|
177
|
+
// xxhash32 (used for the Frame's HC and C.Checksum). Math.imul is naturally mod 2^32.
|
|
178
|
+
export function xxh32(data, seed = 0) {
|
|
179
|
+
const P1 = 2654435761, P2 = 2246822519, P3 = 3266489917, P4 = 668265263, P5 = 374761393;
|
|
180
|
+
const n = data.length;
|
|
181
|
+
let i = 0;
|
|
182
|
+
let h;
|
|
183
|
+
if (n >= 16) {
|
|
184
|
+
let v1 = (seed + P1 + P2) >>> 0;
|
|
185
|
+
let v2 = (seed + P2) >>> 0;
|
|
186
|
+
let v3 = seed >>> 0;
|
|
187
|
+
let v4 = (seed - P1) >>> 0;
|
|
188
|
+
const limit = n - 16;
|
|
189
|
+
do {
|
|
190
|
+
v1 = Math.imul(Math.imul(data.readUInt32LE(i), P2) + v1, P1);
|
|
191
|
+
v1 = ((v1 << 13) | (v1 >>> 19)) >>> 0;
|
|
192
|
+
v2 = Math.imul(Math.imul(data.readUInt32LE(i + 4), P2) + v2, P1);
|
|
193
|
+
v2 = ((v2 << 13) | (v2 >>> 19)) >>> 0;
|
|
194
|
+
v3 = Math.imul(Math.imul(data.readUInt32LE(i + 8), P2) + v3, P1);
|
|
195
|
+
v3 = ((v3 << 13) | (v3 >>> 19)) >>> 0;
|
|
196
|
+
v4 = Math.imul(Math.imul(data.readUInt32LE(i + 12), P2) + v4, P1);
|
|
197
|
+
v4 = ((v4 << 13) | (v4 >>> 19)) >>> 0;
|
|
198
|
+
i += 16;
|
|
199
|
+
} while (i <= limit);
|
|
200
|
+
h = (((v1 << 1) | (v1 >>> 31)) + ((v2 << 7) | (v2 >>> 25))
|
|
201
|
+
+ ((v3 << 12) | (v3 >>> 20)) + ((v4 << 18) | (v4 >>> 14))) >>> 0;
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
h = (seed + P5) >>> 0;
|
|
205
|
+
}
|
|
206
|
+
h = (h + n) >>> 0;
|
|
207
|
+
// The trailing 4 bytes cycle through P3/P4 (per the official XXH32_finalize spec; not P5/P1)
|
|
208
|
+
while (i + 4 <= n) {
|
|
209
|
+
h = (h + Math.imul(data.readUInt32LE(i), P3)) >>> 0;
|
|
210
|
+
h = (((h << 17) | (h >>> 15)) >>> 0);
|
|
211
|
+
h = Math.imul(h, P4);
|
|
212
|
+
i += 4;
|
|
213
|
+
}
|
|
214
|
+
while (i < n) {
|
|
215
|
+
h = (h + Math.imul(data[i], P5)) >>> 0;
|
|
216
|
+
h = (((h << 11) | (h >>> 21)) >>> 0);
|
|
217
|
+
h = Math.imul(h, P1);
|
|
218
|
+
i++;
|
|
219
|
+
}
|
|
220
|
+
h ^= h >>> 15;
|
|
221
|
+
h = Math.imul(h, P2);
|
|
222
|
+
h ^= h >>> 13;
|
|
223
|
+
h = Math.imul(h, P3);
|
|
224
|
+
h ^= h >>> 16;
|
|
225
|
+
return h >>> 0;
|
|
226
|
+
}
|
|
227
|
+
// lz4CompressFrame compresses into an LZ4 Frame (the spec the server expects). FLG = version01 | B.Indep |
|
|
228
|
+
// C.Size; no block checksum / content checksum (following the common Frame default); when a block cannot be
|
|
229
|
+
// compressed it is stored as a raw block (bit31 set).
|
|
230
|
+
export function lz4CompressFrame(data) {
|
|
231
|
+
const flg = 0x40 | 0x20 | 0x08; // version=01, B.Indep=1, C.Size=1
|
|
232
|
+
const bd = 0x40; // BlockMaxSize=64KB
|
|
233
|
+
const header = Buffer.from([flg, bd]);
|
|
234
|
+
const cs = Buffer.alloc(8);
|
|
235
|
+
cs.writeBigUInt64LE(BigInt(data.length), 0);
|
|
236
|
+
const parts = [Buffer.from([0x04, 0x22, 0x4d, 0x18]), header, cs];
|
|
237
|
+
// HC = the second byte of xxh32(FLG+BD+C.Size, 0)
|
|
238
|
+
parts.push(Buffer.from([((xxh32(Buffer.concat([header, cs])) >>> 8) & 0xff)]));
|
|
239
|
+
const n = data.length;
|
|
240
|
+
let pos = 0;
|
|
241
|
+
do {
|
|
242
|
+
const chunk = data.subarray(pos, pos + LZ4_FRAME_BLOCK_SIZE);
|
|
243
|
+
pos += chunk.length;
|
|
244
|
+
const block = lz4CompressBlock(chunk);
|
|
245
|
+
const sizeField = Buffer.alloc(4);
|
|
246
|
+
if (block.length === 0 || block.length >= chunk.length) {
|
|
247
|
+
// Cannot compress: store a raw block (bit31 set)
|
|
248
|
+
sizeField.writeUInt32LE((0x80000000 | chunk.length) >>> 0, 0);
|
|
249
|
+
parts.push(sizeField, chunk);
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
sizeField.writeUInt32LE(block.length, 0);
|
|
253
|
+
parts.push(sizeField, block);
|
|
254
|
+
}
|
|
255
|
+
} while (pos < n);
|
|
256
|
+
parts.push(Buffer.from([0, 0, 0, 0])); // EndMark
|
|
257
|
+
return Buffer.concat(parts);
|
|
258
|
+
}
|
|
259
|
+
// lz4DecompressFrame decodes an LZ4 Frame; a bad magic / bad HC / bad block always throws.
|
|
260
|
+
export function lz4DecompressFrame(data) {
|
|
261
|
+
if (data.length < 7)
|
|
262
|
+
throw new Error('lz4 frame decompress: input too short');
|
|
263
|
+
if (data.readUInt32LE(0) !== LZ4_MAGIC) {
|
|
264
|
+
throw new Error(`lz4 frame decompress: bad magic 0x${data.readUInt32LE(0).toString(16)}`);
|
|
265
|
+
}
|
|
266
|
+
let p = 4;
|
|
267
|
+
const flg = data[p++];
|
|
268
|
+
if (flg >>> 6 !== 0x1)
|
|
269
|
+
throw new Error(`lz4 frame decompress: unsupported version ${flg >>> 6}`);
|
|
270
|
+
const blockChecksum = (flg & 0x10) !== 0;
|
|
271
|
+
const contentSizeFlag = (flg & 0x08) !== 0;
|
|
272
|
+
const contentChecksum = (flg & 0x04) !== 0;
|
|
273
|
+
p += 1; // BD (decoding follows the block header size; BlockMaxSize is not needed)
|
|
274
|
+
let contentSize = 0;
|
|
275
|
+
if (contentSizeFlag) {
|
|
276
|
+
if (p + 8 > data.length)
|
|
277
|
+
throw new Error('lz4 frame decompress: truncated content size');
|
|
278
|
+
contentSize = Number(data.readBigUInt64LE(p));
|
|
279
|
+
p += 8;
|
|
280
|
+
}
|
|
281
|
+
// HC = the second byte of xxh32(FLG+BD+[C.Size]) (the header checksum exists in standard frames)
|
|
282
|
+
const headerEnd = p;
|
|
283
|
+
if (p >= data.length)
|
|
284
|
+
throw new Error('lz4 frame decompress: truncated header checksum');
|
|
285
|
+
const hc = data[p++];
|
|
286
|
+
if ((((xxh32(data.subarray(4, headerEnd)) >>> 8) & 0xff)) !== hc) {
|
|
287
|
+
throw new Error('lz4 frame decompress: header checksum mismatch');
|
|
288
|
+
}
|
|
289
|
+
const parts = [];
|
|
290
|
+
for (;;) {
|
|
291
|
+
if (p + 4 > data.length)
|
|
292
|
+
throw new Error('lz4 frame decompress: truncated block size');
|
|
293
|
+
const sizeField = data.readUInt32LE(p);
|
|
294
|
+
p += 4;
|
|
295
|
+
if (sizeField === 0)
|
|
296
|
+
break; // EndMark
|
|
297
|
+
const isRaw = (sizeField & 0x80000000) !== 0;
|
|
298
|
+
const blockLen = sizeField & 0x7fffffff;
|
|
299
|
+
if (p + blockLen > data.length)
|
|
300
|
+
throw new Error('lz4 frame decompress: truncated block data');
|
|
301
|
+
const block = data.subarray(p, p + blockLen);
|
|
302
|
+
p += blockLen;
|
|
303
|
+
parts.push(isRaw ? block : lz4DecompressBlock(block));
|
|
304
|
+
if (blockChecksum)
|
|
305
|
+
p += 4;
|
|
306
|
+
}
|
|
307
|
+
const out = Buffer.concat(parts);
|
|
308
|
+
if (contentChecksum) {
|
|
309
|
+
if (p + 4 > data.length)
|
|
310
|
+
throw new Error('lz4 frame decompress: truncated content checksum');
|
|
311
|
+
if (xxh32(out) !== data.readUInt32LE(p)) {
|
|
312
|
+
throw new Error('lz4 frame decompress: content checksum mismatch');
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (contentSize !== 0 && out.length !== contentSize) {
|
|
316
|
+
throw new Error('lz4 frame decompress: content size mismatch');
|
|
317
|
+
}
|
|
318
|
+
return out;
|
|
319
|
+
}
|
|
320
|
+
// ---------------------------------------------------------------- ZSTD frame
|
|
321
|
+
const ZSTD_MAGIC = 0xfd2fb528;
|
|
322
|
+
const ZSTD_BLOCK_MAX = 128 * 1024;
|
|
323
|
+
// zstdCompressRaw builds a minimal valid ZSTD frame: header (single-segment,
|
|
324
|
+
// 8-byte content size) + RAW/RLE blocks. The broker's zstd decoder accepts it
|
|
325
|
+
// as-is; there is no compression gain, only wire compatibility.
|
|
326
|
+
export function zstdCompressRaw(data) {
|
|
327
|
+
const parts = [];
|
|
328
|
+
const magic = Buffer.alloc(4);
|
|
329
|
+
magic.writeUInt32LE(ZSTD_MAGIC, 0);
|
|
330
|
+
parts.push(magic);
|
|
331
|
+
// Frame header descriptor: FCS_Field_Size=8 (flag 3), Single_Segment=1.
|
|
332
|
+
const desc = Buffer.from([0xe0 | 0x20]);
|
|
333
|
+
parts.push(desc);
|
|
334
|
+
const fcs = Buffer.alloc(8);
|
|
335
|
+
fcs.writeBigUInt64LE(BigInt(data.length), 0);
|
|
336
|
+
parts.push(fcs);
|
|
337
|
+
let pos = 0;
|
|
338
|
+
while (pos < data.length || data.length === 0) {
|
|
339
|
+
const chunk = Math.min(ZSTD_BLOCK_MAX, data.length - pos);
|
|
340
|
+
if (data.length > 0 && chunk > 1 && isRunOfOneByte(data, pos, chunk)) {
|
|
341
|
+
// RLE block: 1 payload byte repeated `chunk` times.
|
|
342
|
+
// Block header: bit0=Last_Block, bits1-2=Block_Type(RLE=1), bits3-23=size.
|
|
343
|
+
const sizeBits = chunk << 3;
|
|
344
|
+
const header = Buffer.alloc(3);
|
|
345
|
+
header[0] = (sizeBits & 0xff) | 0x02; // Block_Type = RLE
|
|
346
|
+
header[1] = (sizeBits >> 8) & 0xff;
|
|
347
|
+
header[2] = (sizeBits >> 16) & 0xff;
|
|
348
|
+
if (pos + chunk >= data.length)
|
|
349
|
+
header[0] |= 0x01; // last block
|
|
350
|
+
parts.push(header, Buffer.from([data[pos]]));
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
const header = Buffer.alloc(3);
|
|
354
|
+
const sizeBits = chunk << 3; // Block_Type=0 (Raw)
|
|
355
|
+
header[0] = sizeBits & 0xff;
|
|
356
|
+
header[1] = (sizeBits >> 8) & 0xff;
|
|
357
|
+
header[2] = (sizeBits >> 16) & 0xff;
|
|
358
|
+
if (pos + chunk >= data.length)
|
|
359
|
+
header[0] |= 0x01; // last block
|
|
360
|
+
parts.push(header, data.subarray(pos, pos + chunk));
|
|
361
|
+
}
|
|
362
|
+
pos += chunk;
|
|
363
|
+
if (data.length === 0)
|
|
364
|
+
break;
|
|
365
|
+
}
|
|
366
|
+
return Buffer.concat(parts);
|
|
367
|
+
}
|
|
368
|
+
function isRunOfOneByte(data, from, len) {
|
|
369
|
+
const b0 = data[from];
|
|
370
|
+
for (let i = 1; i < len; i++) {
|
|
371
|
+
if (data[from + i] !== b0)
|
|
372
|
+
return false;
|
|
373
|
+
}
|
|
374
|
+
return true;
|
|
375
|
+
}
|
|
376
|
+
// zstdDecompressFrame is the binding-independent fallback decoder: it only decodes ZSTD frames made of Raw/RLE
|
|
377
|
+
// blocks (the kind zstdCompressRaw itself writes, and older Node versions wrote). A Compressed block
|
|
378
|
+
// — produced by a real zstd encoder (the kind the server writer uses) — will throw; use zstdDecompress instead.
|
|
379
|
+
export function zstdDecompressFrame(data) {
|
|
380
|
+
let p = 0;
|
|
381
|
+
if (data.length < 4)
|
|
382
|
+
throw new Error('zstd decompress: input too short');
|
|
383
|
+
const magic = data.readUInt32LE(p);
|
|
384
|
+
if (magic !== ZSTD_MAGIC)
|
|
385
|
+
throw new Error(`zstd decompress: bad magic 0x${magic.toString(16)}`);
|
|
386
|
+
p += 4;
|
|
387
|
+
const desc = data[p++];
|
|
388
|
+
const fcsFlag = (desc >> 6) & 0x3;
|
|
389
|
+
const singleSegment = (desc & 0x20) !== 0;
|
|
390
|
+
const checksumFlag = (desc & 0x04) !== 0;
|
|
391
|
+
const dictIdFlag = desc & 0x3;
|
|
392
|
+
if (!singleSegment)
|
|
393
|
+
p += 1; // window descriptor (ignored)
|
|
394
|
+
const dictSizes = [0, 1, 2, 4];
|
|
395
|
+
p += dictSizes[dictIdFlag];
|
|
396
|
+
const fcsSizes = singleSegment ? [1, 2, 4, 8] : [0, 2, 4, 8];
|
|
397
|
+
p += fcsSizes[fcsFlag];
|
|
398
|
+
const parts = [];
|
|
399
|
+
while (p < data.length) {
|
|
400
|
+
if (p + 3 > data.length)
|
|
401
|
+
throw new Error('zstd decompress: truncated block header');
|
|
402
|
+
const h = data[p] | (data[p + 1] << 8) | (data[p + 2] << 16);
|
|
403
|
+
p += 3;
|
|
404
|
+
const last = (h & 1) === 1;
|
|
405
|
+
const type = (h >> 1) & 0x3;
|
|
406
|
+
const size = h >> 3;
|
|
407
|
+
if (type === 0) {
|
|
408
|
+
// Raw
|
|
409
|
+
if (p + size > data.length)
|
|
410
|
+
throw new Error('zstd decompress: truncated raw block');
|
|
411
|
+
parts.push(data.subarray(p, p + size));
|
|
412
|
+
p += size;
|
|
413
|
+
}
|
|
414
|
+
else if (type === 1) {
|
|
415
|
+
// RLE
|
|
416
|
+
if (p >= data.length)
|
|
417
|
+
throw new Error('zstd decompress: truncated rle block');
|
|
418
|
+
parts.push(Buffer.alloc(size, data[p]));
|
|
419
|
+
p += 1;
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
throw new Error(`zstd decompress: unsupported block type ${type} (compressed block — this codec only reads Raw/RLE frames)`);
|
|
423
|
+
}
|
|
424
|
+
if (last)
|
|
425
|
+
break;
|
|
426
|
+
}
|
|
427
|
+
void checksumFlag;
|
|
428
|
+
return Buffer.concat(parts);
|
|
429
|
+
}
|
|
430
|
+
// ---------------------------------------------------------------- dispatch
|
|
431
|
+
class UnsupportedCompressionError extends Error {
|
|
432
|
+
}
|
|
433
|
+
export { UnsupportedCompressionError };
|
|
434
|
+
// node:zlib gained zstd bindings in Node 23.8. Feature-detected instead of
|
|
435
|
+
// version-checked: the fallback path is a working codec, not a crash.
|
|
436
|
+
const nodeZstd = zlib;
|
|
437
|
+
const HAS_NODE_ZSTD = typeof nodeZstd.zstdCompressSync === 'function'
|
|
438
|
+
&& typeof nodeZstd.zstdDecompressSync === 'function';
|
|
439
|
+
// zstdCompress writes a real zstd frame (level 1..22; <=0 uses the library default, as does the broker-side
|
|
440
|
+
// writer). Without the binding it falls back to a stored frame that only writes Raw/RLE — valid on the wire,
|
|
441
|
+
// just with no compression benefit.
|
|
442
|
+
export function zstdCompress(data, level) {
|
|
443
|
+
if (!HAS_NODE_ZSTD)
|
|
444
|
+
return zstdCompressRaw(data);
|
|
445
|
+
return level && level >= 1
|
|
446
|
+
? nodeZstd.zstdCompressSync(data, { level })
|
|
447
|
+
: nodeZstd.zstdCompressSync(data);
|
|
448
|
+
}
|
|
449
|
+
// zstdDecompress reads any standard frame, Compressed blocks included.
|
|
450
|
+
export function zstdDecompress(data) {
|
|
451
|
+
if (!HAS_NODE_ZSTD)
|
|
452
|
+
return zstdDecompressFrame(data);
|
|
453
|
+
try {
|
|
454
|
+
return nodeZstd.zstdDecompressSync(data);
|
|
455
|
+
}
|
|
456
|
+
catch (e) {
|
|
457
|
+
throw new Error(`zstd decompress: ${e.message}`);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
// compressFor encodes `data` for the given MessageSysFlag compression type
|
|
461
|
+
// (the value at bit 8-10 of sysFlag: 1=LZ4, 2=ZSTD, 3=ZLIB).
|
|
462
|
+
//
|
|
463
|
+
// `level` is the compressLevel configured on the producer (default 5, range 0-9). ZLIB uses it directly as the deflate
|
|
464
|
+
// level; ZSTD maps it onto the zstd level (both the 1-9 scale and others are valid). LZ4's block format has no
|
|
465
|
+
// notion of level.
|
|
466
|
+
export function compressFor(data, compressionType, level = 5) {
|
|
467
|
+
switch (compressionType) {
|
|
468
|
+
case MessageSysFlag.ZLIB_TYPE: {
|
|
469
|
+
// zlib accepts 0(-none)..9; anything outside maps to the library default.
|
|
470
|
+
const lv = Number.isFinite(level) && level >= 0 && level <= 9 ? Math.floor(level) : undefined;
|
|
471
|
+
return lv === undefined ? zlib.deflateSync(data) : zlib.deflateSync(data, { level: lv });
|
|
472
|
+
}
|
|
473
|
+
case MessageSysFlag.LZ4_TYPE: return lz4CompressFrame(data);
|
|
474
|
+
case MessageSysFlag.ZSTD_TYPE: return zstdCompress(data, level);
|
|
475
|
+
default:
|
|
476
|
+
throw new UnsupportedCompressionError(`unsupported compression type: ${compressionType}`);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
// decompressFor is the inverse of compressFor.
|
|
480
|
+
export function decompressFor(data, compressionType) {
|
|
481
|
+
switch (compressionType) {
|
|
482
|
+
case MessageSysFlag.ZLIB_TYPE: return zlib.inflateSync(data);
|
|
483
|
+
case MessageSysFlag.LZ4_TYPE: return lz4DecompressFrame(data);
|
|
484
|
+
case MessageSysFlag.ZSTD_TYPE: return zstdDecompress(data);
|
|
485
|
+
default:
|
|
486
|
+
throw new UnsupportedCompressionError(`unsupported compression type: ${compressionType}`);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
// compressionTypeByName maps the producer-facing compression name ("ZLIB"/"LZ4"/"ZSTD", etc.) onto
|
|
490
|
+
// the numeric value in sysFlag. The name is case-insensitive and numeric strings are also accepted.
|
|
491
|
+
export function compressionTypeByName(name) {
|
|
492
|
+
if (typeof name === 'number') {
|
|
493
|
+
if (name === MessageSysFlag.LZ4_TYPE || name === MessageSysFlag.ZSTD_TYPE
|
|
494
|
+
|| name === MessageSysFlag.ZLIB_TYPE)
|
|
495
|
+
return name;
|
|
496
|
+
throw new UnsupportedCompressionError(`unsupported compression type: ${name}`);
|
|
497
|
+
}
|
|
498
|
+
switch (String(name).toUpperCase()) {
|
|
499
|
+
case 'ZLIB':
|
|
500
|
+
case 'DEFLATE': return MessageSysFlag.ZLIB_TYPE;
|
|
501
|
+
case 'LZ4': return MessageSysFlag.LZ4_TYPE;
|
|
502
|
+
case 'ZSTD':
|
|
503
|
+
case 'ZSTANDARD': return MessageSysFlag.ZSTD_TYPE;
|
|
504
|
+
default:
|
|
505
|
+
throw new UnsupportedCompressionError(`unsupported compression type: ${name}`);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
export default {
|
|
509
|
+
lz4CompressBlock, lz4DecompressBlock, zstdCompressRaw, zstdDecompressFrame,
|
|
510
|
+
zstdCompress, zstdDecompress,
|
|
511
|
+
compressFor, decompressFor, compressionTypeByName, UnsupportedCompressionError,
|
|
512
|
+
};
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
export declare class MessageQueue {
|
|
2
|
+
topic: string;
|
|
3
|
+
brokerName: string;
|
|
4
|
+
queueId: number;
|
|
5
|
+
constructor(topic?: string, brokerName?: string, queueId?: number);
|
|
6
|
+
getTopic(): string;
|
|
7
|
+
setTopic(t: string): void;
|
|
8
|
+
getBrokerName(): string;
|
|
9
|
+
setBrokerName(b: string): void;
|
|
10
|
+
getQueueId(): number;
|
|
11
|
+
getQueueIdStr(): string;
|
|
12
|
+
setQueueId(q: number): void;
|
|
13
|
+
hashcode(): number;
|
|
14
|
+
equals(other: any): boolean;
|
|
15
|
+
compareTo(other: MessageQueue): number;
|
|
16
|
+
toString(): string;
|
|
17
|
+
}
|
|
18
|
+
export declare function isWaitStoreMsgOK(msg: any): boolean;
|
|
19
|
+
export declare class Message {
|
|
20
|
+
topic: string;
|
|
21
|
+
flag: number;
|
|
22
|
+
properties: Record<string, string>;
|
|
23
|
+
body: Buffer;
|
|
24
|
+
transactionId: string | null;
|
|
25
|
+
constructor(topic?: string, body?: Buffer | null, tags?: string | null, keys?: string | null, flag?: number);
|
|
26
|
+
setTags(t: string): void;
|
|
27
|
+
getTags(): string;
|
|
28
|
+
setKeys(k: string): void;
|
|
29
|
+
getKeys(): string;
|
|
30
|
+
setDelayTimeLevel(level: number): void;
|
|
31
|
+
getDelayTimeLevel(): string;
|
|
32
|
+
setDelayTimeSec(sec: number): void;
|
|
33
|
+
getDelayTimeSec(): string;
|
|
34
|
+
setDelayTimeMs(ms: number): void;
|
|
35
|
+
getDelayTimeMs(): string;
|
|
36
|
+
setDeliverTimeMs(ms: number): void;
|
|
37
|
+
getDeliverTimeMs(): string;
|
|
38
|
+
setWaitStoreMsgOK(ok: boolean): void;
|
|
39
|
+
getWaitStoreMsgOK(): string;
|
|
40
|
+
setUserProperty(name: string, value: string): void;
|
|
41
|
+
getUserProperty(name: string): string;
|
|
42
|
+
putProperty(name: string, value: string): void;
|
|
43
|
+
getProperty(name: string): string;
|
|
44
|
+
removeProperty(name: string): void;
|
|
45
|
+
clearProperty(): void;
|
|
46
|
+
getTopic(): string;
|
|
47
|
+
setTopic(t: string): void;
|
|
48
|
+
getBody(): Buffer;
|
|
49
|
+
setBody(b: Buffer): void;
|
|
50
|
+
getFlag(): number;
|
|
51
|
+
setFlag(f: number): void;
|
|
52
|
+
getProperties(): Record<string, string>;
|
|
53
|
+
setProperties(p: Record<string, string>): void;
|
|
54
|
+
getTransactionId(): string;
|
|
55
|
+
setTransactionId(id: string): void;
|
|
56
|
+
toString(): string;
|
|
57
|
+
}
|
|
58
|
+
export declare class MessageExt extends Message {
|
|
59
|
+
queueId: number;
|
|
60
|
+
storeSize: number;
|
|
61
|
+
queueOffset: number;
|
|
62
|
+
sysFlag: number;
|
|
63
|
+
bornTimestamp: number;
|
|
64
|
+
bornHost: string | null;
|
|
65
|
+
bornHostPort: number;
|
|
66
|
+
storeTimestamp: number;
|
|
67
|
+
storeHost: string | null;
|
|
68
|
+
storeHostPort: number;
|
|
69
|
+
msgId: string | null;
|
|
70
|
+
commitLogOffset: number;
|
|
71
|
+
bodyCrc: number;
|
|
72
|
+
reconsumeTimes: number;
|
|
73
|
+
preparedTransactionOffset: number;
|
|
74
|
+
brokerName: string | null;
|
|
75
|
+
offsetMsgId: string | null;
|
|
76
|
+
msgType: string | null;
|
|
77
|
+
constructor(topic?: string, body?: Buffer | null, tags?: string | null, keys?: string | null, flag?: number);
|
|
78
|
+
getQueueId(): number;
|
|
79
|
+
setQueueId(q: number): void;
|
|
80
|
+
getStoreSize(): number;
|
|
81
|
+
setStoreSize(s: number): void;
|
|
82
|
+
getQueueOffset(): number;
|
|
83
|
+
setQueueOffset(o: number): void;
|
|
84
|
+
getSysFlag(): number;
|
|
85
|
+
setSysFlag(f: number): void;
|
|
86
|
+
getBornTimestamp(): number;
|
|
87
|
+
setBornTimestamp(t: number): void;
|
|
88
|
+
getBornHost(): string;
|
|
89
|
+
setBornHost(h: string): void;
|
|
90
|
+
getStoreTimestamp(): number;
|
|
91
|
+
setStoreTimestamp(t: number): void;
|
|
92
|
+
getStoreHost(): string;
|
|
93
|
+
setStoreHost(h: string): void;
|
|
94
|
+
getMsgId(): string;
|
|
95
|
+
setMsgId(id: string): void;
|
|
96
|
+
getCommitLogOffset(): number;
|
|
97
|
+
setCommitLogOffset(o: number): void;
|
|
98
|
+
getBodyCrc(): number;
|
|
99
|
+
setBodyCrc(c: number): void;
|
|
100
|
+
getReconsumeTimes(): number;
|
|
101
|
+
setReconsumeTimes(n: number): void;
|
|
102
|
+
getPreparedTransactionOffset(): number;
|
|
103
|
+
setPreparedTransactionOffset(o: number): void;
|
|
104
|
+
setBrokerName(b: string): void;
|
|
105
|
+
getBrokerName(): string;
|
|
106
|
+
getOffsetMsgId(): string;
|
|
107
|
+
setOffsetMsgId(id: string): void;
|
|
108
|
+
getMsgType(): string;
|
|
109
|
+
setMsgType(t: string): void;
|
|
110
|
+
getBornHostString(): string;
|
|
111
|
+
getStoreHostString(): string;
|
|
112
|
+
toString(): string;
|
|
113
|
+
}
|
|
114
|
+
export declare class MessageBatch extends Message {
|
|
115
|
+
messages: Message[];
|
|
116
|
+
constructor(messages?: Message[] | null);
|
|
117
|
+
encode(): Buffer;
|
|
118
|
+
[Symbol.iterator](): ArrayIterator<Message>;
|
|
119
|
+
get length(): number;
|
|
120
|
+
static generateFromList(messages: Message[]): MessageBatch;
|
|
121
|
+
}
|
|
122
|
+
declare const _default: {
|
|
123
|
+
MessageQueue: typeof MessageQueue;
|
|
124
|
+
Message: typeof Message;
|
|
125
|
+
MessageExt: typeof MessageExt;
|
|
126
|
+
MessageBatch: typeof MessageBatch;
|
|
127
|
+
isWaitStoreMsgOK: typeof isWaitStoreMsgOK;
|
|
128
|
+
};
|
|
129
|
+
export default _default;
|