queen-mq 1.3.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -21
- package/client-v2/Queen.js +47 -15
- package/client-v2/README.md +34 -13
- package/client-v2/admin/Admin.js +30 -55
- package/client-v2/builders/QueueBuilder.js +106 -34
- package/client-v2/builders/TimerBuilder.js +2 -2
- package/client-v2/builders/TransactionBuilder.js +35 -9
- package/client-v2/consumer/ConsumerManager.js +79 -53
- package/client-v2/ephemeral/Ephemeral.js +47 -17
- package/client-v2/kv/Kv.js +4 -4
- package/client-v2/kv/expiry.js +1 -1
- package/client-v2/streams/Stream.js +8 -3
- package/client-v2/streams/helpers/rateLimiter.js +4 -4
- package/client-v2/streams/operators/GateOperator.js +9 -2
- package/client-v2/streams/operators/ReduceOperator.js +2 -2
- package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
- package/client-v2/streams/runtime/Runner.js +111 -54
- package/client-v2/streams/runtime/cycle.js +4 -4
- package/client-v2/streams/runtime/register.js +1 -1
- package/client-v2/utils/conflation.js +0 -6
- package/client-v2/utils/consumerGroup.js +54 -0
- package/client-v2/utils/defaults.js +3 -2
- package/package.json +5 -8
- package/test-v2/_kvtimers.js +12 -13
- package/test-v2/ackwindow.js +12 -192
- package/test-v2/admin-unit/removedRoutes.test.js +63 -0
- package/test-v2/bootstrap.js +2 -2
- package/test-v2/conflation-unit/conflationWire.test.js +0 -12
- package/test-v2/consume.js +41 -1
- package/test-v2/consumer-unit/handlerError.test.js +161 -0
- package/test-v2/consumer-unit/nackScope.test.js +88 -0
- package/test-v2/docs.js +5 -4
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +24 -1
- package/test-v2/http-unit/pushStatus.test.js +142 -0
- package/test-v2/http-unit/renew.test.js +96 -0
- package/test-v2/kv-unit/timerWire.test.js +1 -1
- package/test-v2/kv-unit/txnWire.test.js +101 -2
- package/test-v2/kv.js +6 -5
- package/test-v2/pop-unit/popDefaults.test.js +236 -0
- package/test-v2/pop.js +96 -1
- package/test-v2/run.js +50 -114
- package/test-v2/runner-unit/fatalExit.test.js +64 -0
- package/test-v2/semantics.js +20 -34
- package/test-v2/stream/_helpers.js +10 -19
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/gate.js +54 -0
- package/test-v2/stream/index.js +2 -0
- package/test-v2/stream/tumbling.js +3 -3
- package/test-v2/streams-unit/ack.test.js +61 -0
- package/test-v2/streams-unit/cycle.test.js +1 -1
- package/test-v2/streams-unit/e2e.test.js +6 -10
- package/test-v2/streams-unit/gate.test.js +189 -0
- package/test-v2/timers.js +6 -5
- package/test-v2/transaction.js +169 -0
- package/test-v2/watermark.js +38 -176
- package/test-v2/maintenance.js +0 -277
package/test-v2/ackwindow.js
CHANGED
|
@@ -1,33 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Ack-window honesty
|
|
2
|
+
* Ack-window honesty test (2026-07-30).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* the cursor never moved — a silent redelivery livelock, and a nack/dlq in
|
|
12
|
-
* that state could never dead-letter its poison message.
|
|
4
|
+
* An ack whose transactionId the broker cannot resolve to a message of that
|
|
5
|
+
* partition (here: one that was never pushed) is correctly NOT applied: the
|
|
6
|
+
* cursor stays put and the real messages redeliver (redelivery over loss). The
|
|
7
|
+
* BUG this test pins: the broker reported such items as success=true, so the
|
|
8
|
+
* client believed the ack landed while the cursor never moved -- a silent
|
|
9
|
+
* redelivery livelock, and a nack/dlq in that state could never dead-letter
|
|
10
|
+
* its poison message.
|
|
13
11
|
*
|
|
14
12
|
* Expected contract (post-fix): unresolvable items come back success=false
|
|
15
|
-
* with an explicit "unresolvable" error, and the
|
|
16
|
-
*
|
|
17
|
-
* Companion test: emptyPartitionCursorSealsAfterRetention pins the empty-
|
|
18
|
-
* partition cursor seal in log_pop_v1 (the Rust port of the C++
|
|
19
|
-
* pop_unified_batch_v4 starvation fix): a partition whose segments were all
|
|
20
|
-
* removed by retention must stop being "phantom pending" after one pop.
|
|
13
|
+
* with an explicit "unresolvable" error, and the lease survives for the real
|
|
14
|
+
* batch.
|
|
21
15
|
*
|
|
22
16
|
* Run: node run.js ackUnknownTxnMustFail
|
|
23
|
-
* node run.js ackAfterHashPurgeMustFailExplicitly
|
|
24
|
-
* node run.js emptyPartitionCursorSealsAfterRetention
|
|
25
17
|
*/
|
|
26
18
|
|
|
27
|
-
import { dbPool } from './run.js'
|
|
28
|
-
|
|
29
|
-
const TENANT = '00000000-0000-0000-0000-000000000001'
|
|
30
|
-
|
|
31
19
|
function sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
|
|
32
20
|
|
|
33
21
|
async function popRetry(client, queue, { batch = 3, group = null, partition = null, tries = 30, mode = null } = {}) {
|
|
@@ -43,35 +31,9 @@ async function popRetry(client, queue, { batch = 3, group = null, partition = nu
|
|
|
43
31
|
return []
|
|
44
32
|
}
|
|
45
33
|
|
|
46
|
-
async function partitionRow(queue, partition = 'Default') {
|
|
47
|
-
const r = await dbPool.query(
|
|
48
|
-
`SELECT p.id::text AS id, p.last_offset::bigint AS last_offset, p.log_start::bigint AS log_start
|
|
49
|
-
FROM queen.log_partitions p
|
|
50
|
-
JOIN queen.queues lq ON lq.id = p.queue_id
|
|
51
|
-
WHERE lq.name = $1 AND p.name = $2 AND lq.tenant_id = $3::uuid`,
|
|
52
|
-
[queue, partition, TENANT])
|
|
53
|
-
return r.rows[0] || null
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
async function committedOf(partitionId, group) {
|
|
57
|
-
const r = await dbPool.query(
|
|
58
|
-
`SELECT c.committed::bigint AS committed FROM queen.log_consumers c
|
|
59
|
-
WHERE c.partition_id = $1::uuid AND c.consumer_group = $2`,
|
|
60
|
-
[partitionId, group])
|
|
61
|
-
return r.rows.length ? Number(r.rows[0].committed) : null
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
async function segmentCount(partitionId) {
|
|
65
|
-
const r = await dbPool.query(
|
|
66
|
-
`SELECT count(*)::int AS n FROM queen.log_segments WHERE partition_id = $1::uuid`,
|
|
67
|
-
[partitionId])
|
|
68
|
-
return r.rows[0].n
|
|
69
|
-
}
|
|
70
|
-
|
|
71
34
|
/**
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
* answers success=true for both.
|
|
35
|
+
* Wire-only: acking a transactionId that never existed must fail explicitly,
|
|
36
|
+
* for both a completed ack and a failed nack.
|
|
75
37
|
*/
|
|
76
38
|
export async function ackUnknownTxnMustFail(client) {
|
|
77
39
|
const queue = 'test-queue-v2-ackwindow-unknown'
|
|
@@ -121,145 +83,3 @@ export async function ackUnknownTxnMustFail(client) {
|
|
|
121
83
|
|
|
122
84
|
return { success: true, message: 'unknown-txn ack and nack both rejected explicitly; real batch still ackable' }
|
|
123
85
|
}
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
* Test B (with SQL): the real-world shape — messages still delivered (segments
|
|
127
|
-
* intact) but their log_txns hash rows purged. Per-message acks must fail
|
|
128
|
-
* explicitly and the cursor must not move. Pre-fix: every ack reported
|
|
129
|
-
* success=true while committed stayed put (silent livelock).
|
|
130
|
-
*/
|
|
131
|
-
export async function ackAfterHashPurgeMustFailExplicitly(client) {
|
|
132
|
-
const queue = 'test-queue-v2-ackwindow-purged'
|
|
133
|
-
await client.queue(queue).create()
|
|
134
|
-
const txns = [1, 2, 3, 4, 5].map(n => `${queue}-tx-${n}`)
|
|
135
|
-
await client.queue(queue).partition('Default').push(
|
|
136
|
-
txns.map((t, i) => ({ data: { n: i }, transactionId: t })))
|
|
137
|
-
|
|
138
|
-
// Wait until the segments are visible, then purge the hash sidecar —
|
|
139
|
-
// exactly what log_txns_purge_step_v1 does after the window expires
|
|
140
|
-
// (the 900s floor makes the real purge untestable in-suite).
|
|
141
|
-
const part = await partitionRow(queue)
|
|
142
|
-
if (!part) return { success: false, message: 'partition row not found' }
|
|
143
|
-
for (let i = 0; i < 30 && (await segmentCount(part.id)) === 0; i++) await sleep(100)
|
|
144
|
-
|
|
145
|
-
const del = await dbPool.query(
|
|
146
|
-
`DELETE FROM queen.log_txns WHERE partition_id = $1::uuid`, [part.id])
|
|
147
|
-
if (del.rowCount === 0) return { success: false, message: 'no log_txns rows to purge — test setup broken' }
|
|
148
|
-
|
|
149
|
-
// Segments untouched: the messages still DELIVER.
|
|
150
|
-
const msgs = await popRetry(client, queue, { batch: 5 })
|
|
151
|
-
if (msgs.length !== 5) return { success: false, message: `expected 5 delivered messages after purge, got ${msgs.length}` }
|
|
152
|
-
|
|
153
|
-
// Per-message completed ack (the JS per-message consumer shape): the hash
|
|
154
|
-
// cannot resolve, so the honest answer is an explicit failure.
|
|
155
|
-
const r1 = await client.ack([msgs[0]], true)
|
|
156
|
-
const item1 = (r1.results || [])[0] || {}
|
|
157
|
-
if (item1.success !== false || !/unresolv/i.test(item1.error || '')) {
|
|
158
|
-
return { success: false, message: `BUG: ack with purged hash answered success=${item1.success}, error='${item1.error}'` }
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
// Nack of a purged-hash message: same — and pre-fix this also meant the
|
|
162
|
-
// poison could never reach the DLQ.
|
|
163
|
-
const r2 = await client.ack([msgs[1]], false)
|
|
164
|
-
const item2 = (r2.results || [])[0] || {}
|
|
165
|
-
if (item2.success !== false || !/unresolv/i.test(item2.error || '')) {
|
|
166
|
-
return { success: false, message: `BUG: nack with purged hash answered success=${item2.success}, error='${item2.error}'` }
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
// The cursor must not have moved (redelivery over loss — that part of the
|
|
170
|
-
// contract was always right; only the reporting lied).
|
|
171
|
-
const committed = await committedOf(part.id, '__QUEUE_MODE__')
|
|
172
|
-
if (committed !== -1) {
|
|
173
|
-
return { success: false, message: `cursor moved on unresolvable acks: committed=${committed}, expected -1` }
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
return { success: true, message: 'purged-hash ack and nack rejected explicitly, cursor untouched, messages still deliverable' }
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* Test C (bug 2, with SQL): empty-partition cursor seal. A group consumes
|
|
181
|
-
* part of a partition, retention then deletes ALL segments (time rule deletes
|
|
182
|
-
* unconsumed data too): last_offset > committed with zero segments = a
|
|
183
|
-
* phantom-pending row that used to stay a wildcard candidate forever and
|
|
184
|
-
* pinned the empty-scan watermark at epoch. After the fix, one empty pop
|
|
185
|
-
* seals committed to the partition's last_offset. A delayed-processing queue
|
|
186
|
-
* pins the guard: segments EXIST but are deferred → no seal.
|
|
187
|
-
*/
|
|
188
|
-
export async function emptyPartitionCursorSealsAfterRetention(client) {
|
|
189
|
-
const queue = 'test-queue-v2-ackwindow-seal'
|
|
190
|
-
const group = 'g-seal'
|
|
191
|
-
await client.queue(queue).config({ retentionEnabled: true, retentionSeconds: 1, leaseTime: 30 }).create()
|
|
192
|
-
await client.queue(queue).partition('Default').push(
|
|
193
|
-
[0, 1, 2, 3, 4, 5].map(n => ({ data: { n }, transactionId: `${queue}-tx-${n}` })))
|
|
194
|
-
|
|
195
|
-
// Consume + ack only the first two: committed lands at 1, the rest stays.
|
|
196
|
-
// First contact for g-seal, and the 6 messages are already pushed: the
|
|
197
|
-
// group has to ask for the backlog it is about to half-consume.
|
|
198
|
-
const first = await popRetry(client, queue, { batch: 2, group, mode: 'all' })
|
|
199
|
-
if (first.length !== 2) return { success: false, message: `expected 2 messages, got ${first.length}` }
|
|
200
|
-
const ackr = await client.ack(first, true, { group })
|
|
201
|
-
if (ackr.success !== true) return { success: false, message: `ack failed: ${ackr.error}` }
|
|
202
|
-
|
|
203
|
-
const part = await partitionRow(queue)
|
|
204
|
-
if (!part) return { success: false, message: 'partition row not found' }
|
|
205
|
-
|
|
206
|
-
// Retention (interval 2s in the harness, cutoff 1s) must delete ALL
|
|
207
|
-
// segments — the time rule ignores cursors.
|
|
208
|
-
let segs = -1
|
|
209
|
-
for (let i = 0; i < 60; i++) {
|
|
210
|
-
segs = await segmentCount(part.id)
|
|
211
|
-
if (segs === 0) break
|
|
212
|
-
await sleep(500)
|
|
213
|
-
}
|
|
214
|
-
if (segs !== 0) return { success: false, message: `retention never emptied the partition (segments=${segs})` }
|
|
215
|
-
|
|
216
|
-
const before = await committedOf(part.id, group)
|
|
217
|
-
const rowNow = await partitionRow(queue)
|
|
218
|
-
if (before === null) return { success: false, message: 'consumer row missing before the empty pop' }
|
|
219
|
-
if (before >= Number(rowNow.last_offset)) {
|
|
220
|
-
return { success: false, message: `setup broken: no phantom span (committed=${before}, last_offset=${rowNow.last_offset})` }
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
// ONE empty pop on the emptied partition must seal the cursor.
|
|
224
|
-
const empty = await client.queue(queue).partition('Default').group(group).batch(1).wait(false).pop()
|
|
225
|
-
if (empty && empty.length > 0) return { success: false, message: `expected an empty pop, got ${empty.length} messages` }
|
|
226
|
-
|
|
227
|
-
const after = await committedOf(part.id, group)
|
|
228
|
-
if (after !== Number(rowNow.last_offset)) {
|
|
229
|
-
return { success: false, message: `BUG: empty pop did not seal the cursor (committed=${after}, want last_offset=${rowNow.last_offset}) — partition stays phantom-pending forever` }
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
// Guard: delayed messages are NOT sealed over. Segments exist but are
|
|
233
|
-
// deferred; the cursor must stay put so they deliver when eligible.
|
|
234
|
-
const dqueue = 'test-queue-v2-ackwindow-seal-delayed'
|
|
235
|
-
await client.queue(dqueue).config({ delayedProcessing: 30, leaseTime: 30 }).create()
|
|
236
|
-
await client.queue(dqueue).partition('Default').push([
|
|
237
|
-
{ data: { n: 0 }, transactionId: `${dqueue}-tx-0` },
|
|
238
|
-
{ data: { n: 1 }, transactionId: `${dqueue}-tx-1` },
|
|
239
|
-
])
|
|
240
|
-
// Give the fusion flush a moment so the segments exist before the pop.
|
|
241
|
-
const dpart0 = await (async () => {
|
|
242
|
-
for (let i = 0; i < 30; i++) {
|
|
243
|
-
const p = await partitionRow(dqueue)
|
|
244
|
-
if (p && (await segmentCount(p.id)) > 0) return p
|
|
245
|
-
await sleep(100)
|
|
246
|
-
}
|
|
247
|
-
return null
|
|
248
|
-
})()
|
|
249
|
-
if (!dpart0) return { success: false, message: 'delayed queue segments never appeared' }
|
|
250
|
-
|
|
251
|
-
// First contact for g-seal on THIS queue too (metadata is queue-scoped), and
|
|
252
|
-
// the 2 delayed messages are already pushed: under the default 'new' the
|
|
253
|
-
// cursor would seed at last_offset and the guard below could not tell a seed
|
|
254
|
-
// from a seal. 'all' seeds at -1, so a non -1 cursor can only be the seal.
|
|
255
|
-
const dempty = await client.queue(dqueue).partition('Default').group(group).subscriptionMode('all').batch(1).wait(false).pop()
|
|
256
|
-
if (dempty && dempty.length > 0) return { success: false, message: 'delayed message delivered early?' }
|
|
257
|
-
const dcommitted = await committedOf(dpart0.id, group)
|
|
258
|
-
const dsegs = await segmentCount(dpart0.id)
|
|
259
|
-
if (dsegs === 0) return { success: false, message: 'delayed queue segments vanished' }
|
|
260
|
-
if (dcommitted !== -1 && dcommitted !== null) {
|
|
261
|
-
return { success: false, message: `BUG: seal fired over deferred-but-present segments (committed=${dcommitted}) — delayed messages would be skipped` }
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
return { success: true, message: 'empty partition sealed after one pop; deferred segments not sealed over' }
|
|
265
|
-
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Admin methods whose route the 2.x broker does not have.
|
|
3
|
+
*
|
|
4
|
+
* clearQueue() sent DELETE /api/v1/queues/:name/clear and moveMessageToDLQ()
|
|
5
|
+
* sent POST /api/v1/messages/:partitionId/:transactionId/dlq. Neither route is
|
|
6
|
+
* registered (server/src/rsm/facade/real/phase2/reads.rs: the messages family
|
|
7
|
+
* is GET, DELETE and POST .../retry; there is no /api/v1/queues/... family), so
|
|
8
|
+
* a 2.x broker answers both with 404 no_such_route, and the caller saw
|
|
9
|
+
* "not found". Both now throw before any request, naming the way that works.
|
|
10
|
+
*
|
|
11
|
+
* Same style as conflation-unit/conflationWire.test.js: a real node:http
|
|
12
|
+
* server records every request, so "before any request" is asserted on the
|
|
13
|
+
* socket, not on a mock.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it } from 'node:test'
|
|
17
|
+
import assert from 'node:assert/strict'
|
|
18
|
+
|
|
19
|
+
import { Queen } from '../../client-v2/index.js'
|
|
20
|
+
import { withPlanServer } from '../kv-unit/_planServer.js'
|
|
21
|
+
|
|
22
|
+
const notFound = { status: 404, body: { code: 'no_such_route', error: 'not found' } }
|
|
23
|
+
|
|
24
|
+
async function withQueen(run) {
|
|
25
|
+
await withPlanServer([], notFound, async (url, hits) => {
|
|
26
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
27
|
+
try {
|
|
28
|
+
await run(queen, hits)
|
|
29
|
+
} finally {
|
|
30
|
+
await queen.close()
|
|
31
|
+
}
|
|
32
|
+
})
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
describe('Admin — routes the 2.x broker does not have', () => {
|
|
36
|
+
it('moveMessageToDLQ() throws before any request and names the dlq ack', async () => {
|
|
37
|
+
await withQueen(async (queen, hits) => {
|
|
38
|
+
await assert.rejects(
|
|
39
|
+
queen.admin.moveMessageToDLQ('7', 'tx-1'),
|
|
40
|
+
(err) => {
|
|
41
|
+
assert.match(err.message, /no route/)
|
|
42
|
+
assert.match(err.message, /queen\.ack\(message, 'dlq', \{ group \}\)/)
|
|
43
|
+
return true
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
assert.equal(hits.length, 0, 'no request was sent')
|
|
47
|
+
})
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
it('clearQueue() throws before any request and names the seek to the end', async () => {
|
|
51
|
+
await withQueen(async (queen, hits) => {
|
|
52
|
+
await assert.rejects(
|
|
53
|
+
queen.admin.clearQueue('orders', 'p1'),
|
|
54
|
+
(err) => {
|
|
55
|
+
assert.match(err.message, /no route/)
|
|
56
|
+
assert.match(err.message, /seekConsumerGroup\(group, 'orders', \{ toEnd: true \}\)/)
|
|
57
|
+
return true
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
assert.equal(hits.length, 0, 'no request was sent')
|
|
61
|
+
})
|
|
62
|
+
})
|
|
63
|
+
})
|
package/test-v2/bootstrap.js
CHANGED
|
@@ -217,8 +217,8 @@ export async function testCgBootstrapTimestamp(client) {
|
|
|
217
217
|
|
|
218
218
|
// 3. Record cutoff timestamp (after historical, before new messages)
|
|
219
219
|
// Wait before capturing the cutoff to handle clock skew between the
|
|
220
|
-
// client (Date.now) and
|
|
221
|
-
// push batch may have a
|
|
220
|
+
// client (Date.now) and the broker's clock. Without this, the last
|
|
221
|
+
// push batch may have a broker created_at AFTER the client-side cutoff.
|
|
222
222
|
await new Promise(resolve => setTimeout(resolve, 2000))
|
|
223
223
|
const cutoffTimestamp = new Date().toISOString()
|
|
224
224
|
console.log(` Cutoff timestamp: ${cutoffTimestamp}`)
|
|
@@ -310,18 +310,6 @@ describe('conflation — declaration conflict', () => {
|
|
|
310
310
|
assert.equal(warnings.filter(w => w.includes('Conflation.conflict')).length, 1)
|
|
311
311
|
})
|
|
312
312
|
})
|
|
313
|
-
|
|
314
|
-
it('pop maintenance is not a version skew', async () => {
|
|
315
|
-
// `{"messages":[],"paused":true}` means an operator turned pops off. The
|
|
316
|
-
// request never reached the claim path, so there is no echo to expect —
|
|
317
|
-
// raising here would stop every conflating consumer in the fleet on a
|
|
318
|
-
// routine operator action.
|
|
319
|
-
const paused = ok({ messages: [], paused: true })
|
|
320
|
-
await withQueen([paused], paused, async (queen) => {
|
|
321
|
-
const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
|
|
322
|
-
assert.equal(messages.length, 0)
|
|
323
|
-
})
|
|
324
|
-
})
|
|
325
313
|
})
|
|
326
314
|
|
|
327
315
|
// ---------------------------------------------------------------------------
|
package/test-v2/consume.js
CHANGED
|
@@ -708,4 +708,44 @@ export async function testConsumerMultiPartitionGlobalCap(client) {
|
|
|
708
708
|
}
|
|
709
709
|
|
|
710
710
|
return { success: true, message: 'Global batch cap respected across multi-partition consume' }
|
|
711
|
-
}
|
|
711
|
+
}
|
|
712
|
+
// A handler that throws under autoAck(false) (2026-10-02, 2.0.0-beta.6): the
|
|
713
|
+
// consumer stopped at the first throw, consume() rejected, and the message
|
|
714
|
+
// stayed leased until its lease ran out. It now nacks what the handler was
|
|
715
|
+
// given and keeps consuming: the broker redelivers, the handler acks it.
|
|
716
|
+
export async function manualAckHandlerErrorKeepsConsuming(client) {
|
|
717
|
+
const queueName = 'test-queue-v2-manual-ack-throw'
|
|
718
|
+
const group = 'test-manual-ack-throw'
|
|
719
|
+
const queue = await client.queue(queueName).create()
|
|
720
|
+
if (!queue.configured) {
|
|
721
|
+
return { success: false, message: 'Queue not created' }
|
|
722
|
+
}
|
|
723
|
+
await client.queue(queueName).push([{ data: { id: 1 } }, { data: { id: 2 } }])
|
|
724
|
+
|
|
725
|
+
const seen = []
|
|
726
|
+
try {
|
|
727
|
+
await client
|
|
728
|
+
.queue(queueName)
|
|
729
|
+
.group(group)
|
|
730
|
+
.subscriptionMode('all')
|
|
731
|
+
.autoAck(false)
|
|
732
|
+
.each()
|
|
733
|
+
.batch(1)
|
|
734
|
+
.wait(false)
|
|
735
|
+
.limit(3)
|
|
736
|
+
.consume(async (msg) => {
|
|
737
|
+
seen.push(`${msg.data.id}#${msg.deliveryAttempt}`)
|
|
738
|
+
if (seen.length === 1) throw new Error('first delivery fails')
|
|
739
|
+
// No { group }: the ack takes it from the message.
|
|
740
|
+
const res = await client.ack(msg, true)
|
|
741
|
+
if (!res.success) throw new Error(`ack refused: ${res.error}`)
|
|
742
|
+
})
|
|
743
|
+
} catch (error) {
|
|
744
|
+
return { success: false, message: `consume() rejected: ${error.message} (deliveries ${seen.join(', ')})` }
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
const leftover = await client.queue(queueName).group(group).batch(10).wait(false).pop()
|
|
748
|
+
const ids = seen.map(s => s.split('#')[0]).join(',')
|
|
749
|
+
const success = ids === '1,1,2' && leftover.length === 0
|
|
750
|
+
return { success, message: `deliveries ${seen.join(', ')} (expected 1,1,2), ${leftover.length} left unacked` }
|
|
751
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* consume(): what a handler that throws does to the consumer.
|
|
3
|
+
*
|
|
4
|
+
* The rule, the same with and without autoAck: the messages the handler was
|
|
5
|
+
* given are NACKED (the broker redelivers them, and files them in the DLQ
|
|
6
|
+
* once the queue's retryLimit is spent) and the worker KEEPS CONSUMING.
|
|
7
|
+
* autoAck(false) hands the success path to the handler, not the failure path:
|
|
8
|
+
* a handler that threw did not get to settle its messages.
|
|
9
|
+
*
|
|
10
|
+
* Before: under autoAck(false) the error left the worker loop, the consume()
|
|
11
|
+
* promise rejected after one message, and the messages stayed leased until
|
|
12
|
+
* the lease expired (found 2026-10-02 against 2.0.0-beta.6). With
|
|
13
|
+
* concurrency > 1 the other workers kept running behind a promise that had
|
|
14
|
+
* already rejected.
|
|
15
|
+
*
|
|
16
|
+
* `.onError(fn)` is the way to decide for yourself: the error never reaches
|
|
17
|
+
* the consumer, so nothing is nacked on your behalf.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, it } from 'node:test'
|
|
21
|
+
import assert from 'node:assert/strict'
|
|
22
|
+
import { createServer } from 'node:http'
|
|
23
|
+
|
|
24
|
+
import { Queen } from '../../client-v2/index.js'
|
|
25
|
+
|
|
26
|
+
const GROUP = 'workers'
|
|
27
|
+
|
|
28
|
+
const message = (n) => ({
|
|
29
|
+
id: `msg-${n}`,
|
|
30
|
+
transactionId: `tx-${n}`,
|
|
31
|
+
partitionId: '7',
|
|
32
|
+
partition: 'p1',
|
|
33
|
+
leaseId: `lease-${n}`,
|
|
34
|
+
consumerGroup: GROUP,
|
|
35
|
+
data: { n },
|
|
36
|
+
createdAt: '2026-10-02T10:00:00.000Z'
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* A fake broker routed by path: each pop answers the next batch of `pops`
|
|
41
|
+
* (an empty 204 once they run out), each ack or batch ack is accepted. Every
|
|
42
|
+
* request is recorded.
|
|
43
|
+
*/
|
|
44
|
+
async function withBroker(pops, run) {
|
|
45
|
+
const queue = [...pops]
|
|
46
|
+
const requests = []
|
|
47
|
+
const server = createServer((req, res) => {
|
|
48
|
+
let raw = ''
|
|
49
|
+
req.on('data', chunk => { raw += chunk })
|
|
50
|
+
req.on('end', () => {
|
|
51
|
+
const body = raw ? JSON.parse(raw) : null
|
|
52
|
+
const path = req.url.split('?')[0]
|
|
53
|
+
requests.push({ method: req.method, path, body })
|
|
54
|
+
if (req.method === 'GET' && path.startsWith('/api/v1/pop')) {
|
|
55
|
+
const batch = queue.shift()
|
|
56
|
+
if (!batch) { res.writeHead(204); res.end(); return }
|
|
57
|
+
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
58
|
+
res.end(JSON.stringify({ success: true, consumerGroup: GROUP, messages: batch }))
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
if (req.method === 'POST' && (path === '/api/v1/ack' || path === '/api/v1/ack/batch')) {
|
|
62
|
+
const acks = body.acknowledgments || [body]
|
|
63
|
+
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
64
|
+
res.end(JSON.stringify(acks.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null }))))
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
res.writeHead(404, { 'Content-Type': 'application/json' })
|
|
68
|
+
res.end('{"error":"not found"}')
|
|
69
|
+
})
|
|
70
|
+
})
|
|
71
|
+
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
|
|
72
|
+
const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, handleSignals: false })
|
|
73
|
+
try {
|
|
74
|
+
await run(queen, requests)
|
|
75
|
+
} finally {
|
|
76
|
+
await queen.close()
|
|
77
|
+
await new Promise(resolve => server.close(resolve))
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const acksIn = (requests) => requests.filter(r => r.method === 'POST' && r.path.startsWith('/api/v1/ack'))
|
|
82
|
+
const statusesOf = (ackRequest) => (ackRequest.body.acknowledgments || [ackRequest.body]).map(a => a.status)
|
|
83
|
+
|
|
84
|
+
describe('consume() — a handler that throws is nacked, and the consumer keeps going', () => {
|
|
85
|
+
it('autoAck(false), one message at a time: nack, redelivery, done', async () => {
|
|
86
|
+
// Delivery 1 of tx-1 fails, delivery 2 succeeds and the handler acks it.
|
|
87
|
+
await withBroker([[message(1)], [message(1)]], async (queen, requests) => {
|
|
88
|
+
const seen = []
|
|
89
|
+
await queen.queue('orders').group(GROUP).wait(false).autoAck(false).each().limit(2)
|
|
90
|
+
.consume(async (msg) => {
|
|
91
|
+
seen.push(msg.transactionId)
|
|
92
|
+
if (seen.length === 1) throw new Error('boom')
|
|
93
|
+
await queen.ack(msg, true)
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
assert.deepEqual(seen, ['tx-1', 'tx-1'], 'the consumer survived the throw and got the redelivery')
|
|
97
|
+
const acks = acksIn(requests)
|
|
98
|
+
assert.equal(acks.length, 2)
|
|
99
|
+
assert.deepEqual(statusesOf(acks[0]), ['failed'], 'the throw nacked the message')
|
|
100
|
+
assert.equal(acks[0].body.consumerGroup, GROUP)
|
|
101
|
+
assert.equal(acks[0].body.leaseId, 'lease-1', 'the nack names the lease it releases')
|
|
102
|
+
assert.equal(acks[0].body.error, 'boom', 'the handler error travels with the nack')
|
|
103
|
+
assert.deepEqual(statusesOf(acks[1]), ['completed'], 'the handler\'s own ack of the redelivery')
|
|
104
|
+
})
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
it('autoAck(false), batch handler: the whole batch is nacked in one call', async () => {
|
|
108
|
+
await withBroker([[message(1), message(2)], [message(1), message(2)]], async (queen, requests) => {
|
|
109
|
+
let calls = 0
|
|
110
|
+
await queen.queue('orders').group(GROUP).wait(false).autoAck(false).limit(3)
|
|
111
|
+
.consume(async (msgs) => {
|
|
112
|
+
calls++
|
|
113
|
+
if (calls === 1) throw new Error('batch boom')
|
|
114
|
+
await queen.ack(msgs, true)
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
assert.equal(calls, 2)
|
|
118
|
+
const acks = acksIn(requests)
|
|
119
|
+
assert.equal(acks[0].path, '/api/v1/ack/batch')
|
|
120
|
+
assert.deepEqual(statusesOf(acks[0]), ['failed', 'failed'])
|
|
121
|
+
assert.equal(acks[0].body.consumerGroup, GROUP)
|
|
122
|
+
assert.deepEqual(statusesOf(acks[1]), ['completed', 'completed'])
|
|
123
|
+
})
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
it('autoAck(false), .each(): the rest of the popped batch is abandoned after a nack', async () => {
|
|
127
|
+
// The nack released the lease and clamps the cursor at tx-1, so tx-2 comes
|
|
128
|
+
// back with it: handling it now would only produce a duplicate.
|
|
129
|
+
await withBroker([[message(1), message(2)], [message(1), message(2)]], async (queen) => {
|
|
130
|
+
const seen = []
|
|
131
|
+
await queen.queue('orders').group(GROUP).wait(false).autoAck(false).each().limit(3)
|
|
132
|
+
.consume(async (msg) => {
|
|
133
|
+
seen.push(msg.transactionId)
|
|
134
|
+
if (seen.length === 1) throw new Error('boom')
|
|
135
|
+
})
|
|
136
|
+
assert.deepEqual(seen, ['tx-1', 'tx-1', 'tx-2'])
|
|
137
|
+
})
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
it('autoAck(true) behaves the same way: nack and keep going', async () => {
|
|
141
|
+
await withBroker([[message(1)], [message(1)]], async (queen, requests) => {
|
|
142
|
+
let calls = 0
|
|
143
|
+
await queen.queue('orders').group(GROUP).wait(false).each().limit(2)
|
|
144
|
+
.consume(async () => { if (++calls === 1) throw new Error('boom') })
|
|
145
|
+
assert.equal(calls, 2)
|
|
146
|
+
assert.deepEqual(acksIn(requests).map(statusesOf), [['failed'], ['completed']])
|
|
147
|
+
})
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
it('with .onError() the handler decides: nothing is nacked on its behalf', async () => {
|
|
151
|
+
await withBroker([[message(1)], [message(2)]], async (queen, requests) => {
|
|
152
|
+
const failures = []
|
|
153
|
+
await queen.queue('orders').group(GROUP).wait(false).autoAck(false).each().limit(2)
|
|
154
|
+
.consume(async (msg) => { if (msg.transactionId === 'tx-1') throw new Error('boom') })
|
|
155
|
+
.onError(async (msg, err) => { failures.push([msg.transactionId, err.message]) })
|
|
156
|
+
|
|
157
|
+
assert.deepEqual(failures, [['tx-1', 'boom']])
|
|
158
|
+
assert.equal(acksIn(requests).length, 0, 'autoAck(false) + onError: the consumer sent no ack and no nack')
|
|
159
|
+
})
|
|
160
|
+
})
|
|
161
|
+
})
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* each(): a nack releases ONE partition. A multi-partition pop claims several
|
|
3
|
+
* partitions under one lease; when the handler fails a message, the nack
|
|
4
|
+
* releases that message's partition and clamps its cursor, so the later
|
|
5
|
+
* messages of THAT partition come back on the next pop. The other partitions
|
|
6
|
+
* are still leased to this worker: their messages must be handled now.
|
|
7
|
+
*
|
|
8
|
+
* Before: the loop abandoned the whole popped batch after a nack. The other
|
|
9
|
+
* partitions' messages stayed leased and came back only when the lease
|
|
10
|
+
* expired (found live 2026-10-06 against 2.0.1: B1 and B2 waited the whole
|
|
11
|
+
* 6 s lease after A1 failed).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, it } from 'node:test'
|
|
15
|
+
import assert from 'node:assert/strict'
|
|
16
|
+
import { createServer } from 'node:http'
|
|
17
|
+
|
|
18
|
+
import { Queen } from '../../client-v2/index.js'
|
|
19
|
+
|
|
20
|
+
const GROUP = 'workers'
|
|
21
|
+
|
|
22
|
+
const message = (partition, n) => ({
|
|
23
|
+
id: `msg-${partition}${n}`,
|
|
24
|
+
transactionId: `tx-${partition}${n}`,
|
|
25
|
+
partitionId: `pid-${partition}`,
|
|
26
|
+
partition,
|
|
27
|
+
leaseId: 'lease-1',
|
|
28
|
+
consumerGroup: GROUP,
|
|
29
|
+
data: { tag: `${partition}${n}` },
|
|
30
|
+
createdAt: '2026-10-06T10:00:00.000Z'
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
async function withBroker(pops, run) {
|
|
34
|
+
const queue = [...pops]
|
|
35
|
+
const requests = []
|
|
36
|
+
const server = createServer((req, res) => {
|
|
37
|
+
let raw = ''
|
|
38
|
+
req.on('data', chunk => { raw += chunk })
|
|
39
|
+
req.on('end', () => {
|
|
40
|
+
const body = raw ? JSON.parse(raw) : null
|
|
41
|
+
const path = req.url.split('?')[0]
|
|
42
|
+
requests.push({ method: req.method, path, body })
|
|
43
|
+
if (req.method === 'GET' && path.startsWith('/api/v1/pop')) {
|
|
44
|
+
const batch = queue.shift()
|
|
45
|
+
if (!batch) { res.writeHead(204); res.end(); return }
|
|
46
|
+
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
47
|
+
res.end(JSON.stringify({ success: true, consumerGroup: GROUP, messages: batch }))
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
if (req.method === 'POST' && (path === '/api/v1/ack' || path === '/api/v1/ack/batch')) {
|
|
51
|
+
const acks = body.acknowledgments || [body]
|
|
52
|
+
res.writeHead(200, { 'Content-Type': 'application/json' })
|
|
53
|
+
res.end(JSON.stringify(acks.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null }))))
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
res.writeHead(404, { 'Content-Type': 'application/json' })
|
|
57
|
+
res.end('{"error":"not found"}')
|
|
58
|
+
})
|
|
59
|
+
})
|
|
60
|
+
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
|
|
61
|
+
const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, handleSignals: false })
|
|
62
|
+
try {
|
|
63
|
+
await run(queen, requests)
|
|
64
|
+
} finally {
|
|
65
|
+
await queen.close()
|
|
66
|
+
await new Promise(resolve => server.close(resolve))
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
describe('each(): a nack skips only its own partition', () => {
|
|
71
|
+
it('handles the other partitions of the pop after a failure', async () => {
|
|
72
|
+
const pop = [message('A', 1), message('A', 2), message('B', 1), message('B', 2)]
|
|
73
|
+
await withBroker([pop], async (queen, requests) => {
|
|
74
|
+
const handled = []
|
|
75
|
+
await queen.queue('orders').group(GROUP).each().batch(4).limit(3).idleMillis(500)
|
|
76
|
+
.consume(async (m) => {
|
|
77
|
+
handled.push(m.data.tag)
|
|
78
|
+
if (m.data.tag === 'A1') throw new Error('A1 fails')
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
assert.deepEqual(handled, ['A1', 'B1', 'B2'], 'A2 is skipped (it comes back after the nack), B is handled now')
|
|
82
|
+
const settled = requests.filter(r => r.path.startsWith('/api/v1/ack'))
|
|
83
|
+
.flatMap(r => r.body.acknowledgments || [r.body])
|
|
84
|
+
.map(a => `${a.transactionId}:${a.status}`)
|
|
85
|
+
assert.deepEqual(settled, ['tx-A1:failed', 'tx-B1:completed', 'tx-B2:completed'])
|
|
86
|
+
})
|
|
87
|
+
})
|
|
88
|
+
})
|
package/test-v2/docs.js
CHANGED
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
*
|
|
11
11
|
* After editing a marked region, regenerate the partials with
|
|
12
12
|
* `pnpm --dir webdoc gen` or the docs CI check fails on drift. The queues used
|
|
13
|
-
* here (orders, payments, invoices)
|
|
14
|
-
*
|
|
13
|
+
* here (orders, payments, invoices) start empty because every suite run gets a
|
|
14
|
+
* fresh broker (test/run.sh), exactly like the test-% queues.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
export async function docsProduceAndConsume(client) {
|
|
@@ -65,8 +65,9 @@ export async function docsProduceAndConsume(client) {
|
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
export async function docsDeduplication(client) {
|
|
68
|
-
// The fixed transactionId below survives reruns because
|
|
69
|
-
//
|
|
68
|
+
// The fixed transactionId below survives reruns because every run gets a
|
|
69
|
+
// fresh broker (test/run.sh creates its data volume empty and destroys it
|
|
70
|
+
// with `down -v`), so no earlier run's dedup entry is ever there.
|
|
70
71
|
// docs:start(js-push-dedup)
|
|
71
72
|
const first = await client
|
|
72
73
|
.queue('payments')
|