queen-mq 1.0.6 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -4
- package/client-v2/Queen.js +68 -0
- package/client-v2/admin/Admin.js +15 -1
- package/client-v2/buffer/BufferManager.js +18 -6
- package/client-v2/buffer/MessageBuffer.js +17 -1
- package/client-v2/buffer/sinks.js +89 -0
- package/client-v2/builders/QueueBuilder.js +175 -16
- package/client-v2/consumer/ConsumerManager.js +79 -12
- package/client-v2/ephemeral/Ephemeral.js +551 -0
- package/client-v2/index.js +12 -0
- package/client-v2/streams/Stream.js +3 -0
- package/client-v2/streams/runtime/Runner.js +26 -3
- package/client-v2/utils/autopilot.js +168 -0
- package/client-v2/utils/conflation.js +118 -0
- package/client-v2/utils/defaults.js +16 -3
- package/package.json +3 -3
- package/test-v2/autopilot-unit/autopilotWire.test.js +410 -0
- package/test-v2/conflation-unit/conflationWire.test.js +398 -0
- package/test-v2/ephemeral-unit/_planServer.js +73 -0
- package/test-v2/ephemeral-unit/durableSinkPin.test.js +112 -0
- package/test-v2/ephemeral-unit/ephemeralBuffer.test.js +305 -0
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +398 -0
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conflation: wire contract + degrade-loudly (PLAN_CONFLATION.md §3.1, §3.3, §4).
|
|
3
|
+
*
|
|
4
|
+
* Three things are pinned here, and each of them is a defect that ships
|
|
5
|
+
* silently if it is not:
|
|
6
|
+
*
|
|
7
|
+
* 1. `conflation` reaches the wire from BOTH entry points. pop() and
|
|
8
|
+
* consume() build their query strings in two SEPARATE places
|
|
9
|
+
* (QueueBuilder.pop's inline params and ConsumerManager#buildParams) —
|
|
10
|
+
* the standing comment at QueueBuilder.js documents a bug of exactly this
|
|
11
|
+
* shape, where the parameter was added to one builder and the other kept
|
|
12
|
+
* working while the option "did nothing". Both directions are asserted,
|
|
13
|
+
* including the negative: an undeclared conflation must not add the key at
|
|
14
|
+
* all, because the broker's response bytes are pinned byte-for-byte for
|
|
15
|
+
* flag-off deployments.
|
|
16
|
+
*
|
|
17
|
+
* 2. Degrade-loudly (§4). No SDK negotiates a version, so a new client against
|
|
18
|
+
* a pre-1.1.0 broker sends `conflation=true`, the broker ignores the unknown
|
|
19
|
+
* query parameter, and the consumer quietly drains the whole backlog one
|
|
20
|
+
* message at a time. The detection is response-driven: a conflating broker
|
|
21
|
+
* echoes `"conflation":true` on EVERY response, empty ones included (it
|
|
22
|
+
* answers 200-with-body instead of 204 precisely so the echo has somewhere
|
|
23
|
+
* to ride), so the first round trip is enough. An old broker's empty pop is
|
|
24
|
+
* a bodiless 204 => `null` here, which is why null must raise too.
|
|
25
|
+
*
|
|
26
|
+
* 3. Declaration conflict (§3.3). The STORED group policy wins; the consumer
|
|
27
|
+
* that disagreed keeps working and warns exactly ONCE per (queue, group)
|
|
28
|
+
* per process. Rejecting instead would take down the correct half of a
|
|
29
|
+
* rolling deploy (plan Q3), and warning per response would flood.
|
|
30
|
+
*
|
|
31
|
+
* Same style as http-unit/retry429.test.js and kv-unit/kvWire.test.js: a real
|
|
32
|
+
* node:http server playing a canned plan, real fetch, real JSON. No mocking
|
|
33
|
+
* framework, and every assertion is about bytes that actually crossed a socket.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import { describe, it, beforeEach, afterEach } from 'node:test'
|
|
37
|
+
import assert from 'node:assert/strict'
|
|
38
|
+
|
|
39
|
+
import { Queen } from '../../client-v2/index.js'
|
|
40
|
+
import { Runner } from '../../client-v2/streams/runtime/Runner.js'
|
|
41
|
+
import * as logger from '../../client-v2/utils/logger.js'
|
|
42
|
+
import {
|
|
43
|
+
CONFLATION_UNSUPPORTED,
|
|
44
|
+
resetConflationWarnings
|
|
45
|
+
} from '../../client-v2/utils/conflation.js'
|
|
46
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
47
|
+
|
|
48
|
+
const QUEUE = 'test-conflation'
|
|
49
|
+
const GROUP = 'workers'
|
|
50
|
+
|
|
51
|
+
/** One delivered frame, shaped like a real pop response element. */
|
|
52
|
+
function frame(n = 1) {
|
|
53
|
+
return {
|
|
54
|
+
transactionId: `txn-${n}`,
|
|
55
|
+
partitionId: `part-${n}`,
|
|
56
|
+
partition: 'Default',
|
|
57
|
+
payload: { n },
|
|
58
|
+
leaseId: 'lease-1',
|
|
59
|
+
consumerGroup: GROUP
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** A 200 pop body; `extra` carries the conflation echo keys under test. */
|
|
64
|
+
function popBody(extra = {}, frames = [frame()]) {
|
|
65
|
+
return ok({ messages: frames, partitionsClaimed: frames.length, ...extra })
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** A pre-1.1.0 broker's empty pop: bodiless 204 (HttpClient turns it into null). */
|
|
69
|
+
const emptyNoContent = { status: 204 }
|
|
70
|
+
|
|
71
|
+
/** Query parameters of a recorded hit. */
|
|
72
|
+
function query(url) {
|
|
73
|
+
const i = url.indexOf('?')
|
|
74
|
+
return new URLSearchParams(i < 0 ? '' : url.slice(i + 1))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Path (without query) of a recorded hit. */
|
|
78
|
+
function path(url) {
|
|
79
|
+
const i = url.indexOf('?')
|
|
80
|
+
return i < 0 ? url : url.slice(0, i)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function withQueen(plan, defaultResponse, run) {
|
|
84
|
+
await withPlanServer(plan, defaultResponse, async (url, hits) => {
|
|
85
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
86
|
+
try {
|
|
87
|
+
await run(queen, hits)
|
|
88
|
+
} finally {
|
|
89
|
+
await queen.close()
|
|
90
|
+
}
|
|
91
|
+
})
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// The conflict warning is deliberately once-per-process state, so every test
|
|
95
|
+
// starts from a clean registry.
|
|
96
|
+
beforeEach(() => { resetConflationWarnings() })
|
|
97
|
+
afterEach(() => { resetConflationWarnings(); logger.configure(null) })
|
|
98
|
+
|
|
99
|
+
// ---------------------------------------------------------------------------
|
|
100
|
+
// 1. The option reaches the wire — from pop() AND from consume().
|
|
101
|
+
// ---------------------------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
describe('conflation — wire (pop)', () => {
|
|
104
|
+
it('pop() sends conflation=true when the builder declares it', async () => {
|
|
105
|
+
await withQueen([popBody({ conflation: true })], popBody({ conflation: true }), async (queen, hits) => {
|
|
106
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
107
|
+
|
|
108
|
+
assert.equal(messages.length, 1)
|
|
109
|
+
assert.equal(hits.length, 1)
|
|
110
|
+
assert.equal(path(hits[0].url), `/api/v1/pop/queue/${QUEUE}`)
|
|
111
|
+
assert.equal(query(hits[0].url).get('conflation'), 'true')
|
|
112
|
+
assert.equal(query(hits[0].url).get('consumerGroup'), GROUP)
|
|
113
|
+
})
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
it('pop() omits the parameter entirely when conflation is not declared', async () => {
|
|
117
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
118
|
+
await queen.queue(QUEUE).group(GROUP).pop()
|
|
119
|
+
|
|
120
|
+
assert.equal(hits.length, 1)
|
|
121
|
+
assert.equal(
|
|
122
|
+
query(hits[0].url).has('conflation'),
|
|
123
|
+
false,
|
|
124
|
+
'a flag-off pop must be byte-identical to today: no conflation key at all'
|
|
125
|
+
)
|
|
126
|
+
})
|
|
127
|
+
})
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
describe('conflation — wire (consume)', () => {
|
|
131
|
+
it('consume() sends conflation=true through ConsumerManager#buildParams', async () => {
|
|
132
|
+
await withQueen([popBody({ conflation: true })], popBody({ conflation: true }), async (queen, hits) => {
|
|
133
|
+
let handled = 0
|
|
134
|
+
await queen.queue(QUEUE)
|
|
135
|
+
.group(GROUP)
|
|
136
|
+
.conflation(true)
|
|
137
|
+
.wait(false)
|
|
138
|
+
.autoAck(false)
|
|
139
|
+
.limit(1)
|
|
140
|
+
.consume(async () => { handled++ })
|
|
141
|
+
|
|
142
|
+
assert.equal(handled, 1)
|
|
143
|
+
assert.ok(hits.length >= 1)
|
|
144
|
+
assert.equal(path(hits[0].url), `/api/v1/pop/queue/${QUEUE}`)
|
|
145
|
+
assert.equal(
|
|
146
|
+
query(hits[0].url).get('conflation'),
|
|
147
|
+
'true',
|
|
148
|
+
'consume() builds its params in a DIFFERENT place than pop() — both must carry it'
|
|
149
|
+
)
|
|
150
|
+
})
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
it('consume() omits the parameter entirely when conflation is not declared', async () => {
|
|
154
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
155
|
+
await queen.queue(QUEUE)
|
|
156
|
+
.group(GROUP)
|
|
157
|
+
.wait(false)
|
|
158
|
+
.autoAck(false)
|
|
159
|
+
.limit(1)
|
|
160
|
+
.consume(async () => {})
|
|
161
|
+
|
|
162
|
+
assert.ok(hits.length >= 1)
|
|
163
|
+
assert.equal(query(hits[0].url).has('conflation'), false)
|
|
164
|
+
})
|
|
165
|
+
})
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
// 2. Degrade-loudly (§4): an old broker must never be silent.
|
|
170
|
+
// ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
describe('conflation — degrade loudly against a pre-1.1.0 broker', () => {
|
|
173
|
+
it('pop() raises when the response carries messages but no conflation echo', async () => {
|
|
174
|
+
await withQueen([popBody()], popBody(), async (queen) => {
|
|
175
|
+
await assert.rejects(
|
|
176
|
+
() => queen.queue(QUEUE).group(GROUP).conflation(true).pop(),
|
|
177
|
+
(err) => {
|
|
178
|
+
assert.equal(err.code, CONFLATION_UNSUPPORTED)
|
|
179
|
+
assert.match(err.message, /requires broker >= 1\.1\.0/)
|
|
180
|
+
return true
|
|
181
|
+
}
|
|
182
|
+
)
|
|
183
|
+
})
|
|
184
|
+
})
|
|
185
|
+
|
|
186
|
+
it('pop() raises on the FIRST empty pop, which an old broker answers 204', async () => {
|
|
187
|
+
await withQueen([emptyNoContent], emptyNoContent, async (queen, hits) => {
|
|
188
|
+
await assert.rejects(
|
|
189
|
+
() => queen.queue(QUEUE).group(GROUP).conflation(true).pop(),
|
|
190
|
+
(err) => {
|
|
191
|
+
assert.equal(err.code, CONFLATION_UNSUPPORTED)
|
|
192
|
+
return true
|
|
193
|
+
}
|
|
194
|
+
)
|
|
195
|
+
assert.equal(hits.length, 1, 'the error fires on the first round trip, before any message')
|
|
196
|
+
})
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
it('consume() stops the loop and rejects before the handler ever runs', async () => {
|
|
200
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
201
|
+
let handled = 0
|
|
202
|
+
// consume() returns a thenable ConsumeBuilder, not a Promise: await it
|
|
203
|
+
// inside the arrow so assert.rejects sees a real promise.
|
|
204
|
+
await assert.rejects(
|
|
205
|
+
async () => {
|
|
206
|
+
await queen.queue(QUEUE)
|
|
207
|
+
.group(GROUP)
|
|
208
|
+
.conflation(true)
|
|
209
|
+
.wait(false)
|
|
210
|
+
.autoAck(false)
|
|
211
|
+
.consume(async () => { handled++ })
|
|
212
|
+
},
|
|
213
|
+
(err) => {
|
|
214
|
+
assert.equal(err.code, CONFLATION_UNSUPPORTED)
|
|
215
|
+
assert.match(err.message, /requires broker >= 1\.1\.0/)
|
|
216
|
+
return true
|
|
217
|
+
}
|
|
218
|
+
)
|
|
219
|
+
assert.equal(handled, 0, 'not one message may be processed under a silent degrade')
|
|
220
|
+
assert.equal(hits.length, 1, 'the loop stops instead of draining the backlog')
|
|
221
|
+
})
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
it('leaves a conflation-less response alone when conflation was never requested', async () => {
|
|
225
|
+
await withQueen([popBody()], popBody(), async (queen) => {
|
|
226
|
+
const messages = await queen.queue(QUEUE).group(GROUP).pop()
|
|
227
|
+
assert.equal(messages.length, 1, 'today\'s behaviour is untouched when the flag is off')
|
|
228
|
+
})
|
|
229
|
+
})
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
// ---------------------------------------------------------------------------
|
|
233
|
+
// 3. Declaration conflict (§3.3): the stored policy wins, loudly but once.
|
|
234
|
+
// ---------------------------------------------------------------------------
|
|
235
|
+
|
|
236
|
+
/** Collect every warning the SDK emits while `run` executes. */
|
|
237
|
+
async function captureWarnings(run) {
|
|
238
|
+
const warnings = []
|
|
239
|
+
logger.configure({
|
|
240
|
+
info() {},
|
|
241
|
+
warn(entry) { warnings.push(typeof entry === 'string' ? entry : JSON.stringify(entry)) },
|
|
242
|
+
error() {}
|
|
243
|
+
})
|
|
244
|
+
try {
|
|
245
|
+
await run()
|
|
246
|
+
} finally {
|
|
247
|
+
logger.configure(null)
|
|
248
|
+
}
|
|
249
|
+
return warnings
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
describe('conflation — declaration conflict', () => {
|
|
253
|
+
it('warns exactly once per (queue, group) and keeps delivering messages', async () => {
|
|
254
|
+
const conflicted = popBody({ conflationConflict: true })
|
|
255
|
+
await withQueen([conflicted, conflicted, conflicted], conflicted, async (queen, hits) => {
|
|
256
|
+
const warnings = await captureWarnings(async () => {
|
|
257
|
+
for (let i = 0; i < 3; i++) {
|
|
258
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
259
|
+
assert.equal(messages.length, 1, 'a conflict never costs the consumer its messages')
|
|
260
|
+
}
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
assert.equal(hits.length, 3)
|
|
264
|
+
const conflictWarnings = warnings.filter(w => w.includes('Conflation.conflict'))
|
|
265
|
+
assert.equal(
|
|
266
|
+
conflictWarnings.length,
|
|
267
|
+
1,
|
|
268
|
+
`exactly one warning per (queue,group) per process, got: ${JSON.stringify(conflictWarnings)}`
|
|
269
|
+
)
|
|
270
|
+
assert.match(conflictWarnings[0], new RegExp(QUEUE))
|
|
271
|
+
assert.match(conflictWarnings[0], new RegExp(GROUP))
|
|
272
|
+
})
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
it('warns again for a different group on the same queue', async () => {
|
|
276
|
+
const conflicted = popBody({ conflationConflict: true })
|
|
277
|
+
await withQueen([conflicted, conflicted], conflicted, async (queen) => {
|
|
278
|
+
const warnings = await captureWarnings(async () => {
|
|
279
|
+
await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
280
|
+
await queen.queue(QUEUE).group('audit').conflation(true).pop()
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
const conflictWarnings = warnings.filter(w => w.includes('Conflation.conflict'))
|
|
284
|
+
assert.equal(conflictWarnings.length, 2, 'the warn-once key is (queue, group), not queue')
|
|
285
|
+
})
|
|
286
|
+
})
|
|
287
|
+
|
|
288
|
+
it('a conflict is never a degrade error — the broker plainly understood the flag', async () => {
|
|
289
|
+
const conflicted = popBody({ conflationConflict: true })
|
|
290
|
+
await withQueen([conflicted], conflicted, async (queen) => {
|
|
291
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
292
|
+
assert.equal(messages.length, 1)
|
|
293
|
+
})
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
it('an EMPTY conflicting pop is answered 200-with-body and must not raise', async () => {
|
|
297
|
+
// The steady state of a disagreeing consumer, and the case the SDK cannot
|
|
298
|
+
// rescue on its own: this is a consumer's FIRST poll of an idle queue, so
|
|
299
|
+
// the knownConflicts memo below is empty and only the response can say what
|
|
300
|
+
// is going on. The broker keeps the 200 whenever the answer has anything to
|
|
301
|
+
// say about conflation (`pop_status`); if it ever regressed to a bodiless
|
|
302
|
+
// 204 here, every conflicting consumer would exit on its first poll.
|
|
303
|
+
const emptyConflict = ok({ messages: [], partitionsClaimed: 0, conflationConflict: true })
|
|
304
|
+
await withQueen([emptyConflict], emptyConflict, async (queen, hits) => {
|
|
305
|
+
const warnings = await captureWarnings(async () => {
|
|
306
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
307
|
+
assert.equal(messages.length, 0)
|
|
308
|
+
})
|
|
309
|
+
assert.equal(hits.length, 1)
|
|
310
|
+
assert.equal(warnings.filter(w => w.includes('Conflation.conflict')).length, 1)
|
|
311
|
+
})
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
it('pop maintenance is not a version skew', async () => {
|
|
315
|
+
// `{"messages":[],"paused":true}` means an operator turned pops off. The
|
|
316
|
+
// request never reached the claim path, so there is no echo to expect —
|
|
317
|
+
// raising here would stop every conflating consumer in the fleet on a
|
|
318
|
+
// routine operator action.
|
|
319
|
+
const paused = ok({ messages: [], paused: true })
|
|
320
|
+
await withQueen([paused], paused, async (queen) => {
|
|
321
|
+
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
322
|
+
assert.equal(messages.length, 0)
|
|
323
|
+
})
|
|
324
|
+
})
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
// ---------------------------------------------------------------------------
|
|
328
|
+
// 4. The streams runtime pops through the same QueueBuilder — so the option has
|
|
329
|
+
// to be threaded there too, or `run({ conflation: true })` is a no-op.
|
|
330
|
+
// ---------------------------------------------------------------------------
|
|
331
|
+
|
|
332
|
+
/** A QueueBuilder-shaped source that records the chain the Runner builds. */
|
|
333
|
+
function recordingSource(calls) {
|
|
334
|
+
const qb = {
|
|
335
|
+
batch(v) { calls.push(['batch', v]); return qb },
|
|
336
|
+
wait(v) { calls.push(['wait', v]); return qb },
|
|
337
|
+
timeoutMillis(v) { calls.push(['timeoutMillis', v]); return qb },
|
|
338
|
+
group(v) { calls.push(['group', v]); return qb },
|
|
339
|
+
partitions(v) { calls.push(['partitions', v]); return qb },
|
|
340
|
+
subscriptionMode(v) { calls.push(['subscriptionMode', v]); return qb },
|
|
341
|
+
subscriptionFrom(v) { calls.push(['subscriptionFrom', v]); return qb },
|
|
342
|
+
conflation(v) { calls.push(['conflation', v]); return qb },
|
|
343
|
+
async pop() { calls.push(['pop']); return [] }
|
|
344
|
+
}
|
|
345
|
+
return qb
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function runnerWith(opts, calls) {
|
|
349
|
+
return new Runner({
|
|
350
|
+
url: 'http://127.0.0.1:1',
|
|
351
|
+
queryId: 'test-conflation-query',
|
|
352
|
+
stream: { source: recordingSource(calls), stages: {}, config_hash: 'hash' },
|
|
353
|
+
...opts
|
|
354
|
+
})
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
describe('conflation — streams runtime', () => {
|
|
358
|
+
it('Stream.run({ conflation: true }) declares it on the source pop', async () => {
|
|
359
|
+
const calls = []
|
|
360
|
+
await runnerWith({ conflation: true }, calls)._popMessages()
|
|
361
|
+
assert.deepEqual(calls.filter(c => c[0] === 'conflation'), [['conflation', true]])
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
it('leaves the source pop untouched when the option is absent', async () => {
|
|
365
|
+
const calls = []
|
|
366
|
+
await runnerWith({}, calls)._popMessages()
|
|
367
|
+
assert.deepEqual(calls.filter(c => c[0] === 'conflation'), [])
|
|
368
|
+
})
|
|
369
|
+
})
|
|
370
|
+
|
|
371
|
+
// ---------------------------------------------------------------------------
|
|
372
|
+
// 5. Depth: the new fields must survive the admin call (§2.5/§5.3).
|
|
373
|
+
// ---------------------------------------------------------------------------
|
|
374
|
+
|
|
375
|
+
describe('conflation — queue depth', () => {
|
|
376
|
+
it('getQueueDepth surfaces partitionsPending / conflation / effectivePending', async () => {
|
|
377
|
+
const depth = ok({
|
|
378
|
+
queue: QUEUE,
|
|
379
|
+
group: GROUP,
|
|
380
|
+
pending: 4000000,
|
|
381
|
+
partitionsPending: 12,
|
|
382
|
+
conflation: true,
|
|
383
|
+
effectivePending: 12,
|
|
384
|
+
partitions: [{ partition: 'a', pending: 333333 }]
|
|
385
|
+
})
|
|
386
|
+
await withQueen([depth], depth, async (queen, hits) => {
|
|
387
|
+
const got = await queen.admin.getQueueDepth(QUEUE, GROUP)
|
|
388
|
+
|
|
389
|
+
assert.equal(path(hits[0].url), `/api/v1/resources/queues/${QUEUE}/depth`)
|
|
390
|
+
assert.equal(query(hits[0].url).get('group'), GROUP)
|
|
391
|
+
// Log depth vs work depth: 4M positions to retire, 12 handler calls owed.
|
|
392
|
+
assert.equal(got.pending, 4000000)
|
|
393
|
+
assert.equal(got.partitionsPending, 12)
|
|
394
|
+
assert.equal(got.conflation, true)
|
|
395
|
+
assert.equal(got.effectivePending, 12)
|
|
396
|
+
})
|
|
397
|
+
})
|
|
398
|
+
})
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The scripted plan server, plus the ephemeral envelopes these tests answer
|
|
3
|
+
* with.
|
|
4
|
+
*
|
|
5
|
+
* The server itself is kv-unit's -- a real node:http socket driven by a canned
|
|
6
|
+
* list of responses, recording method, url, raw bytes and parsed body for every
|
|
7
|
+
* request. It is IMPORTED rather than copied for the same reason
|
|
8
|
+
* conflation-unit imports it: two plan servers would drift, and the day one of
|
|
9
|
+
* them stops recording `raw` is the day a byte-identity pin silently becomes a
|
|
10
|
+
* shape check.
|
|
11
|
+
*
|
|
12
|
+
* What is local here is the vocabulary: the ephemeral routes have their own
|
|
13
|
+
* envelopes (§3.1), and a test that spells `{queue, messages}` out by hand at
|
|
14
|
+
* every call site hides the wire behind its own noise.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { Queen } from '../../client-v2/index.js'
|
|
18
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
19
|
+
|
|
20
|
+
export { withPlanServer, ok }
|
|
21
|
+
|
|
22
|
+
/** `POST /api/v1/ephemeral/push` -> 201 `{pushed}` (all-or-nothing per request). */
|
|
23
|
+
export function pushed(count) {
|
|
24
|
+
return { status: 201, body: { pushed: count } }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** `GET /api/v1/ephemeral/pop` -> 200 `{queue, messages}`; empty array on timeout. */
|
|
28
|
+
export function popped(queue, messages = []) {
|
|
29
|
+
return ok({ queue, messages })
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One delivered frame, shaped like a real ephemeral pop element (§3.1). */
|
|
33
|
+
export function frame(n = 1, extra = {}) {
|
|
34
|
+
return { id: `e:beef:Default:${n}`, partition: 'Default', payload: { n }, attempts: 0, ...extra }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** `POST /api/v1/ephemeral/ack` -> 200 `{results:[{id, outcome}]}`. */
|
|
38
|
+
export function acked(...results) {
|
|
39
|
+
return ok({ results })
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* What a broker or proxy older than 1.1 answers on every route of this family:
|
|
44
|
+
* the broker because the routes do not exist, the proxy because an unknown API
|
|
45
|
+
* path is `route_blocked` and fails closed.
|
|
46
|
+
*/
|
|
47
|
+
export const OLD_BROKER = { status: 404, body: { error: 'not_found' } }
|
|
48
|
+
export const OLD_PROXY = { status: 404, body: { error: 'route_blocked', code: 'route_blocked' } }
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The OTHER 404, and the reason the mapping has to read the body: a broker that
|
|
52
|
+
* fully supports the family, answering `depth` about a queue that is not there.
|
|
53
|
+
*/
|
|
54
|
+
export const QUEUE_NOT_FOUND = {
|
|
55
|
+
status: 404,
|
|
56
|
+
body: { error: 'ephemeral queue not found', code: 'ephemeral_queue_not_found' }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Run `fn(ephemeral, hits, queen)` against a plan server, closing the client
|
|
61
|
+
* afterwards. `handleSignals:false` because a test process must not have its
|
|
62
|
+
* SIGINT handler replaced by every client it builds.
|
|
63
|
+
*/
|
|
64
|
+
export async function withEphemeral(plan, run, clientOptions = {}) {
|
|
65
|
+
await withPlanServer(plan, ok({ ok: true }), async (url, hits) => {
|
|
66
|
+
const queen = new Queen({ url, handleSignals: false, ...clientOptions })
|
|
67
|
+
try {
|
|
68
|
+
await run(queen.ephemeral, hits, queen)
|
|
69
|
+
} finally {
|
|
70
|
+
await queen.close()
|
|
71
|
+
}
|
|
72
|
+
})
|
|
73
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The durable sink, pinned to the byte.
|
|
3
|
+
*
|
|
4
|
+
* EPHEMERAL_QUEUES.md §4.1 buys the ephemeral buffered push by PARAMETRIZING
|
|
5
|
+
* the drain rather than duplicating it, and §7.3 names the price of that
|
|
6
|
+
* bargain: "a pin that the DURABLE sink's bodies are byte-identical before and
|
|
7
|
+
* after the sink refactor". This file is that pin.
|
|
8
|
+
*
|
|
9
|
+
* It is written against the durable path only. Nothing here mentions an
|
|
10
|
+
* ephemeral queue, and that is deliberate: the question it answers is not
|
|
11
|
+
* "does the new feature work" but "did the refactor that made the new feature
|
|
12
|
+
* possible move a single byte on the path that was already in production". A
|
|
13
|
+
* `{queue, partition, payload, transactionId}` item whose key ORDER changed, an
|
|
14
|
+
* envelope that grew a `queue` field because the ephemeral wire has one, a path
|
|
15
|
+
* that became `/api/v1/push/batch` -- none of those would fail a deepEqual on a
|
|
16
|
+
* parsed body, and all of them are a broken 1.0.6 producer.
|
|
17
|
+
*
|
|
18
|
+
* The literal below is the request the buffered durable push made before sinks
|
|
19
|
+
* existed. It is not derived from the code under test: derive it and the pin
|
|
20
|
+
* pins nothing.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { describe, it } from 'node:test'
|
|
24
|
+
import assert from 'node:assert/strict'
|
|
25
|
+
|
|
26
|
+
import { Queen } from '../../client-v2/index.js'
|
|
27
|
+
import { BufferManager } from '../../client-v2/buffer/BufferManager.js'
|
|
28
|
+
import { MessageBuffer } from '../../client-v2/buffer/MessageBuffer.js'
|
|
29
|
+
import { DURABLE_SINK, DURABLE_DESTINATION, durableAddress } from '../../client-v2/buffer/sinks.js'
|
|
30
|
+
import { withPlanServer, ok } from './_planServer.js'
|
|
31
|
+
|
|
32
|
+
const tick = (ms = 0) => new Promise(resolve => setTimeout(resolve, ms))
|
|
33
|
+
|
|
34
|
+
async function until(predicate, what, timeoutMs = 2000) {
|
|
35
|
+
const deadline = Date.now() + timeoutMs
|
|
36
|
+
while (Date.now() < deadline) {
|
|
37
|
+
if (predicate()) return
|
|
38
|
+
await tick(2)
|
|
39
|
+
}
|
|
40
|
+
assert.fail(`timed out waiting for: ${what}`)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** The exact bytes a buffered durable push of two items has always produced. */
|
|
44
|
+
const PINNED_BODY =
|
|
45
|
+
'{"items":[' +
|
|
46
|
+
'{"queue":"orders","partition":"Default","payload":{"n":1},"transactionId":"fixed-1"},' +
|
|
47
|
+
'{"queue":"orders","partition":"Default","payload":{"n":2},"transactionId":"fixed-2"}' +
|
|
48
|
+
']}'
|
|
49
|
+
|
|
50
|
+
describe('durable sink — byte identity', () => {
|
|
51
|
+
it('drains the buffered durable push to the same path, with the same bytes', async () => {
|
|
52
|
+
await withPlanServer([ok([])], ok([]), async (url, hits) => {
|
|
53
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
54
|
+
try {
|
|
55
|
+
await queen
|
|
56
|
+
.queue('orders')
|
|
57
|
+
.partition('Default')
|
|
58
|
+
.buffer({ messageCount: 2, timeMillis: 60000 })
|
|
59
|
+
.push([
|
|
60
|
+
{ data: { n: 1 }, transactionId: 'fixed-1' },
|
|
61
|
+
{ data: { n: 2 }, transactionId: 'fixed-2' }
|
|
62
|
+
])
|
|
63
|
+
|
|
64
|
+
await until(() => hits.length === 1, 'the count threshold to drain the batch')
|
|
65
|
+
assert.equal(hits[0].method, 'POST')
|
|
66
|
+
assert.equal(hits[0].url, '/api/v1/push')
|
|
67
|
+
assert.equal(hits[0].raw, PINNED_BODY, 'the durable buffered push must be byte-identical to 1.0.6')
|
|
68
|
+
} finally {
|
|
69
|
+
await queen.close()
|
|
70
|
+
}
|
|
71
|
+
})
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
it('formats the envelope from the batch alone — the queue and partition arguments do not leak into it', () => {
|
|
75
|
+
// The durable wire repeats the identity on every ITEM, so the sink is handed
|
|
76
|
+
// a queue and a partition it must ignore. An implementation that hoisted
|
|
77
|
+
// them to the envelope "for symmetry" with the ephemeral sink would produce
|
|
78
|
+
// a body no broker of any version has ever parsed.
|
|
79
|
+
const batch = [{ queue: 'orders', partition: 'Default', payload: { n: 1 }, transactionId: 'fixed-1' }]
|
|
80
|
+
assert.equal(DURABLE_SINK.path, '/api/v1/push')
|
|
81
|
+
assert.equal(JSON.stringify(DURABLE_SINK.format('ignored', 'ignored', batch)), JSON.stringify({ items: batch }))
|
|
82
|
+
assert.equal(JSON.stringify(DURABLE_SINK.format(null, null, batch)), JSON.stringify({ items: batch }))
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
it('is what a buffer gets when nobody names a destination', async () => {
|
|
86
|
+
// Every caller written before sinks existed passes three arguments to
|
|
87
|
+
// addMessage. They must keep draining where they always did, without
|
|
88
|
+
// knowing the parameter exists.
|
|
89
|
+
const posts = []
|
|
90
|
+
const httpClient = { async post(path, body) { posts.push({ path, body }); return [] } }
|
|
91
|
+
const manager = new BufferManager(httpClient)
|
|
92
|
+
|
|
93
|
+
const address = durableAddress('orders', 'Default')
|
|
94
|
+
assert.equal(address, 'orders/Default', 'the durable address is unchanged: no namespace prefix')
|
|
95
|
+
|
|
96
|
+
await manager.addMessage(address, { queue: 'orders', partition: 'Default', payload: { n: 1 }, transactionId: 'fixed-1' }, { messageCount: 1, timeMillis: 60000 })
|
|
97
|
+
await until(() => posts.length === 1, 'the drain')
|
|
98
|
+
|
|
99
|
+
assert.equal(posts[0].path, '/api/v1/push')
|
|
100
|
+
assert.deepEqual(Object.keys(posts[0].body), ['items'], 'the durable envelope is {items} and only {items}')
|
|
101
|
+
assert.deepEqual(posts[0].body.items, [{ queue: 'orders', partition: 'Default', payload: { n: 1 }, transactionId: 'fixed-1' }])
|
|
102
|
+
|
|
103
|
+
manager.cleanup()
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
it('defaults a MessageBuffer built without one to the durable destination', () => {
|
|
107
|
+
const buffer = new MessageBuffer('orders/Default', { messageCount: 1 }, () => {})
|
|
108
|
+
assert.equal(buffer.destination, DURABLE_DESTINATION)
|
|
109
|
+
assert.equal(buffer.destination.sink.path, '/api/v1/push')
|
|
110
|
+
buffer.cleanup()
|
|
111
|
+
})
|
|
112
|
+
})
|