queen-mq 1.3.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -21
- package/client-v2/Queen.js +47 -15
- package/client-v2/README.md +34 -13
- package/client-v2/admin/Admin.js +30 -55
- package/client-v2/builders/QueueBuilder.js +106 -34
- package/client-v2/builders/TimerBuilder.js +2 -2
- package/client-v2/builders/TransactionBuilder.js +35 -9
- package/client-v2/consumer/ConsumerManager.js +79 -53
- package/client-v2/ephemeral/Ephemeral.js +47 -17
- package/client-v2/kv/Kv.js +4 -4
- package/client-v2/kv/expiry.js +1 -1
- package/client-v2/streams/Stream.js +8 -3
- package/client-v2/streams/helpers/rateLimiter.js +4 -4
- package/client-v2/streams/operators/GateOperator.js +9 -2
- package/client-v2/streams/operators/ReduceOperator.js +2 -2
- package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
- package/client-v2/streams/runtime/Runner.js +111 -54
- package/client-v2/streams/runtime/cycle.js +4 -4
- package/client-v2/streams/runtime/register.js +1 -1
- package/client-v2/utils/conflation.js +0 -6
- package/client-v2/utils/consumerGroup.js +54 -0
- package/client-v2/utils/defaults.js +3 -2
- package/package.json +5 -8
- package/test-v2/_kvtimers.js +12 -13
- package/test-v2/ackwindow.js +12 -192
- package/test-v2/admin-unit/removedRoutes.test.js +63 -0
- package/test-v2/bootstrap.js +2 -2
- package/test-v2/conflation-unit/conflationWire.test.js +0 -12
- package/test-v2/consume.js +41 -1
- package/test-v2/consumer-unit/handlerError.test.js +161 -0
- package/test-v2/consumer-unit/nackScope.test.js +88 -0
- package/test-v2/docs.js +5 -4
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +24 -1
- package/test-v2/http-unit/pushStatus.test.js +142 -0
- package/test-v2/http-unit/renew.test.js +96 -0
- package/test-v2/kv-unit/timerWire.test.js +1 -1
- package/test-v2/kv-unit/txnWire.test.js +101 -2
- package/test-v2/kv.js +6 -5
- package/test-v2/pop-unit/popDefaults.test.js +236 -0
- package/test-v2/pop.js +96 -1
- package/test-v2/run.js +50 -114
- package/test-v2/runner-unit/fatalExit.test.js +64 -0
- package/test-v2/semantics.js +20 -34
- package/test-v2/stream/_helpers.js +10 -19
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/gate.js +54 -0
- package/test-v2/stream/index.js +2 -0
- package/test-v2/stream/tumbling.js +3 -3
- package/test-v2/streams-unit/ack.test.js +61 -0
- package/test-v2/streams-unit/cycle.test.js +1 -1
- package/test-v2/streams-unit/e2e.test.js +6 -10
- package/test-v2/streams-unit/gate.test.js +189 -0
- package/test-v2/timers.js +6 -5
- package/test-v2/transaction.js +169 -0
- package/test-v2/watermark.js +38 -176
- package/test-v2/maintenance.js +0 -277
|
@@ -201,7 +201,7 @@ describe('ephemeral wire — pop', () => {
|
|
|
201
201
|
|
|
202
202
|
it('pop puts every declared parameter on the query string, in order', async () => {
|
|
203
203
|
await withEphemeral([popped(QUEUE, [])], async (eph, hits) => {
|
|
204
|
-
await eph.pop(QUEUE, { partition: 'room-7', batch: 10, wait: true, timeout: 2000, group: 'workers',
|
|
204
|
+
await eph.pop(QUEUE, { partition: 'room-7', batch: 10, wait: true, timeout: 2000, group: 'workers', commitOnDelivery: true })
|
|
205
205
|
assert.equal(
|
|
206
206
|
hits[0].url,
|
|
207
207
|
`/api/v1/ephemeral/pop?queue=${QUEUE}&partition=room-7&batch=10&wait=true&timeout=2000&group=workers&autoAck=true`
|
|
@@ -209,6 +209,29 @@ describe('ephemeral wire — pop', () => {
|
|
|
209
209
|
})
|
|
210
210
|
})
|
|
211
211
|
|
|
212
|
+
it('pop sends `commitOnDelivery` as autoAck=true, and nothing for false', async () => {
|
|
213
|
+
await withEphemeral([popped(QUEUE, []), popped(QUEUE, [])], async (eph, hits) => {
|
|
214
|
+
await eph.pop(QUEUE, { commitOnDelivery: true })
|
|
215
|
+
assert.equal(hits[0].url, `/api/v1/ephemeral/pop?queue=${QUEUE}&autoAck=true`)
|
|
216
|
+
|
|
217
|
+
await eph.pop(QUEUE, { commitOnDelivery: false })
|
|
218
|
+
assert.equal(hits[1].url, `/api/v1/ephemeral/pop?queue=${QUEUE}`)
|
|
219
|
+
})
|
|
220
|
+
})
|
|
221
|
+
|
|
222
|
+
it('pop still reads the deprecated `autoAck` as `commitOnDelivery`, and refuses both spellings', async () => {
|
|
223
|
+
await withEphemeral([popped(QUEUE, [])], async (eph, hits) => {
|
|
224
|
+
await eph.pop(QUEUE, { group: 'workers', autoAck: true })
|
|
225
|
+
assert.equal(hits[0].url, `/api/v1/ephemeral/pop?queue=${QUEUE}&group=workers&autoAck=true`)
|
|
226
|
+
|
|
227
|
+
await assert.rejects(
|
|
228
|
+
() => eph.pop(QUEUE, { commitOnDelivery: true, autoAck: true }),
|
|
229
|
+
/pass either `commitOnDelivery` or its deprecated alias `autoAck`, not both/
|
|
230
|
+
)
|
|
231
|
+
assert.equal(hits.length, 1)
|
|
232
|
+
})
|
|
233
|
+
})
|
|
234
|
+
|
|
212
235
|
it('pop sends an explicit timeout whenever it waits, and none when it does not', async () => {
|
|
213
236
|
// The HTTP deadline is set PAST the server's, so the broker's long poll
|
|
214
237
|
// always ends the request first. Letting the two defaults drift apart is
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Push per-item statuses.
|
|
3
|
+
*
|
|
4
|
+
* POST /api/v1/push answers 201 with one result per item, in input order, and
|
|
5
|
+
* a refused item fails INSIDE a successful request, so the per-item status is
|
|
6
|
+
* the only signal that a message was not stored. The statuses brokers send:
|
|
7
|
+
*
|
|
8
|
+
* queued stored (every broker)
|
|
9
|
+
* duplicate its transactionId is already stored (every broker)
|
|
10
|
+
* error refused, no per-item message (2.x)
|
|
11
|
+
* failed refused, with `error` (1.x)
|
|
12
|
+
* buffered written to the broker's failover file, replayed (1.x)
|
|
13
|
+
*
|
|
14
|
+
* The client used to look for `failed` only, so a 2.x refusal (`error`) was
|
|
15
|
+
* neither reported nor thrown: push() resolved as if the message were stored.
|
|
16
|
+
* Found 2026-10-02 against 2.0.0-beta.6, which answers `error` when a
|
|
17
|
+
* partition's planned append exceeds QUEEN_RAFT_ENTRY_MAX_BYTES.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, it } from 'node:test'
|
|
21
|
+
import assert from 'node:assert/strict'
|
|
22
|
+
|
|
23
|
+
import { Queen } from '../../client-v2/index.js'
|
|
24
|
+
import { withPlanServer } from '../kv-unit/_planServer.js'
|
|
25
|
+
|
|
26
|
+
const QUEUE = 'test-push-status'
|
|
27
|
+
|
|
28
|
+
const item = (index, status, extra = {}) => ({
|
|
29
|
+
index,
|
|
30
|
+
message_id: `0199aaaa-0000-7000-8000-00000000000${index}`,
|
|
31
|
+
transaction_id: `tx-${index}`,
|
|
32
|
+
queueName: QUEUE,
|
|
33
|
+
status,
|
|
34
|
+
...extra
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
// The exact shape 2.0.0-beta.6 answered for an item it refused (no offset,
|
|
38
|
+
// no error text).
|
|
39
|
+
const created = (results) => ({ status: 201, body: results })
|
|
40
|
+
|
|
41
|
+
async function withQueen(response, run) {
|
|
42
|
+
await withPlanServer([response], response, async (url, hits) => {
|
|
43
|
+
const queen = new Queen({ url, handleSignals: false, retryAttempts: 1 })
|
|
44
|
+
try {
|
|
45
|
+
await run(queen, hits)
|
|
46
|
+
} finally {
|
|
47
|
+
await queen.close()
|
|
48
|
+
}
|
|
49
|
+
})
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('push — a refused item is reported, whatever the broker calls it', () => {
|
|
53
|
+
it('a 2.x `error` item makes push() reject, carrying the per-item results', async () => {
|
|
54
|
+
await withQueen(created([item(0, 'error')]), async (queen) => {
|
|
55
|
+
await assert.rejects(
|
|
56
|
+
async () => { await queen.queue(QUEUE).push([{ data: { n: 1 } }]) },
|
|
57
|
+
(err) => {
|
|
58
|
+
assert.match(err.message, /rejected by the broker/)
|
|
59
|
+
assert.match(err.message, /"error"/, 'the status the broker answered is named')
|
|
60
|
+
assert.deepEqual(err.results.map(r => r.status), ['error'])
|
|
61
|
+
return true
|
|
62
|
+
}
|
|
63
|
+
)
|
|
64
|
+
})
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
it('a 2.x `error` item goes to onError, and the stored ones still go to onSuccess', async () => {
|
|
68
|
+
await withQueen(created([item(0, 'queued', { offset: 0 }), item(1, 'error')]), async (queen) => {
|
|
69
|
+
let failedItems = null
|
|
70
|
+
let failure = null
|
|
71
|
+
let succeeded = null
|
|
72
|
+
const res = await queen.queue(QUEUE)
|
|
73
|
+
.push([{ data: { n: 1 } }, { data: { n: 2 } }])
|
|
74
|
+
.onError(async (items, err) => { failedItems = items; failure = err })
|
|
75
|
+
.onSuccess(async (items) => { succeeded = items })
|
|
76
|
+
|
|
77
|
+
assert.ok(Array.isArray(res), 'with onError the call resolves with the broker results')
|
|
78
|
+
assert.equal(failedItems.length, 1)
|
|
79
|
+
assert.deepEqual(failedItems[0].payload, { n: 2 })
|
|
80
|
+
assert.equal(failedItems[0].result.status, 'error')
|
|
81
|
+
assert.match(failure.message, /rejected by the broker/)
|
|
82
|
+
assert.equal(succeeded.length, 1)
|
|
83
|
+
assert.deepEqual(succeeded[0].payload, { n: 1 })
|
|
84
|
+
})
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('a 1.x `failed` item is reported with the broker\'s own text', async () => {
|
|
88
|
+
await withQueen(created([item(0, 'failed', { error: 'File buffer write failed' })]), async (queen) => {
|
|
89
|
+
await assert.rejects(
|
|
90
|
+
async () => { await queen.queue(QUEUE).push([{ data: { n: 1 } }]) },
|
|
91
|
+
(err) => { assert.equal(err.message, 'File buffer write failed'); return true }
|
|
92
|
+
)
|
|
93
|
+
})
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
it('a status this client does not know is a failure, not a silent success', async () => {
|
|
97
|
+
await withQueen(created([item(0, 'refused-by-something-new')]), async (queen) => {
|
|
98
|
+
await assert.rejects(
|
|
99
|
+
async () => { await queen.queue(QUEUE).push([{ data: { n: 1 } }]) },
|
|
100
|
+
(err) => { assert.match(err.message, /refused-by-something-new/); return true }
|
|
101
|
+
)
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
it('several refused items: one error for the call, counting them', async () => {
|
|
106
|
+
await withQueen(created([item(0, 'error'), item(1, 'queued', { offset: 0 }), item(2, 'error')]), async (queen) => {
|
|
107
|
+
await assert.rejects(
|
|
108
|
+
async () => { await queen.queue(QUEUE).push([{ data: 1 }, { data: 2 }, { data: 3 }]) },
|
|
109
|
+
(err) => { assert.match(err.message, /^2 of 3 pushed items rejected/); return true }
|
|
110
|
+
)
|
|
111
|
+
})
|
|
112
|
+
})
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
describe('push — accepted statuses stay accepted', () => {
|
|
116
|
+
it('`queued` resolves with the broker results and reaches onSuccess', async () => {
|
|
117
|
+
await withQueen(created([item(0, 'queued', { offset: 7 })]), async (queen) => {
|
|
118
|
+
let succeeded = null
|
|
119
|
+
const res = await queen.queue(QUEUE).push([{ data: { n: 1 } }]).onSuccess(async (items) => { succeeded = items })
|
|
120
|
+
assert.equal(res[0].status, 'queued')
|
|
121
|
+
assert.equal(succeeded.length, 1)
|
|
122
|
+
})
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
it('a 1.x `buffered` item is accepted: the broker owns its delivery', async () => {
|
|
126
|
+
await withQueen(created([item(0, 'buffered')]), async (queen) => {
|
|
127
|
+
let succeeded = null
|
|
128
|
+
const res = await queen.queue(QUEUE).push([{ data: { n: 1 } }]).onSuccess(async (items) => { succeeded = items })
|
|
129
|
+
assert.equal(res[0].status, 'buffered')
|
|
130
|
+
assert.equal(succeeded.length, 1)
|
|
131
|
+
})
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
it('`duplicate` goes to onDuplicate and is not a failure', async () => {
|
|
135
|
+
await withQueen(created([item(0, 'duplicate', { offset: 3 })]), async (queen) => {
|
|
136
|
+
let dups = null
|
|
137
|
+
const res = await queen.queue(QUEUE).push([{ data: { n: 1 } }]).onDuplicate(async (items) => { dups = items })
|
|
138
|
+
assert.equal(res[0].status, 'duplicate')
|
|
139
|
+
assert.equal(dups.length, 1)
|
|
140
|
+
})
|
|
141
|
+
})
|
|
142
|
+
})
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queen.renew() outcome.
|
|
3
|
+
*
|
|
4
|
+
* POST /api/v1/lease/:leaseId/extend answers HTTP 200 whether or not anything
|
|
5
|
+
* was renewed. `success` in the body is the only signal, and the client used
|
|
6
|
+
* to ignore it: renewing a lease that had expired, or that an ack had already
|
|
7
|
+
* released, reported `success: true` with `newExpiresAt: null` (found
|
|
8
|
+
* 2026-10-02 against 2.0.0-beta.6). The bodies below are what brokers send:
|
|
9
|
+
*
|
|
10
|
+
* 1.x and 2.x: {leaseId, success, renewed, newExpiresAt, expiresAt, lease_expires_at}
|
|
11
|
+
* C++: [{index, leaseId, success, error, expiresAt}]
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, it } from 'node:test'
|
|
15
|
+
import assert from 'node:assert/strict'
|
|
16
|
+
|
|
17
|
+
import { Queen } from '../../client-v2/index.js'
|
|
18
|
+
import { withPlanServer } from '../kv-unit/_planServer.js'
|
|
19
|
+
|
|
20
|
+
const LEASE = '01a0fcf2-1a28-7001-8810-d74deda1fd42'
|
|
21
|
+
const EXPIRES = '2026-10-02T14:09:40.273Z'
|
|
22
|
+
|
|
23
|
+
// What 2.0.0-beta.6 answered for a live lease covering two partitions, and
|
|
24
|
+
// for a lease that no longer exists.
|
|
25
|
+
const renewed = (leaseId = LEASE) => ({
|
|
26
|
+
leaseId, success: true, renewed: 2, newExpiresAt: EXPIRES, expiresAt: EXPIRES, lease_expires_at: EXPIRES
|
|
27
|
+
})
|
|
28
|
+
const notRenewed = (leaseId = LEASE) => ({
|
|
29
|
+
leaseId, success: false, renewed: 0, newExpiresAt: null, expiresAt: null, lease_expires_at: null
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
async function withQueen(plan, run) {
|
|
33
|
+
await withPlanServer(plan, { status: 200, body: renewed() }, async (url, hits) => {
|
|
34
|
+
const queen = new Queen({ url, handleSignals: false, retryAttempts: 1 })
|
|
35
|
+
try {
|
|
36
|
+
await run(queen, hits)
|
|
37
|
+
} finally {
|
|
38
|
+
await queen.close()
|
|
39
|
+
}
|
|
40
|
+
})
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
describe('Queen.renew — the broker\'s verdict, not the HTTP status', () => {
|
|
44
|
+
it('a renewed lease reports success, its new expiry, and how many partitions it covers', async () => {
|
|
45
|
+
await withQueen([{ body: renewed() }], async (queen, hits) => {
|
|
46
|
+
const res = await queen.renew(LEASE)
|
|
47
|
+
assert.deepEqual(res, { leaseId: LEASE, success: true, newExpiresAt: EXPIRES, renewed: 2 })
|
|
48
|
+
assert.equal(hits[0].url, `/api/v1/lease/${LEASE}/extend`)
|
|
49
|
+
})
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
it('a lease that is gone reports failure, with a reason', async () => {
|
|
53
|
+
await withQueen([{ body: notRenewed() }], async (queen) => {
|
|
54
|
+
const res = await queen.renew({ leaseId: LEASE, transactionId: 'tx-1' })
|
|
55
|
+
assert.equal(res.success, false)
|
|
56
|
+
assert.equal(res.renewed, 0)
|
|
57
|
+
assert.equal(res.newExpiresAt, null)
|
|
58
|
+
assert.match(res.error, /not renewed/)
|
|
59
|
+
})
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
it('an array of messages answers one outcome per distinct lease', async () => {
|
|
63
|
+
const other = '01a0fcf2-0000-7000-8000-000000000000'
|
|
64
|
+
await withQueen([{ body: renewed() }, { body: notRenewed(other) }], async (queen, hits) => {
|
|
65
|
+
const res = await queen.renew([
|
|
66
|
+
{ leaseId: LEASE, transactionId: 'tx-1' },
|
|
67
|
+
{ leaseId: LEASE, transactionId: 'tx-2' },
|
|
68
|
+
{ leaseId: other, transactionId: 'tx-3' }
|
|
69
|
+
])
|
|
70
|
+
assert.equal(hits.length, 2, 'one call per lease, not per message')
|
|
71
|
+
assert.deepEqual(res.map(r => [r.leaseId, r.success]), [[LEASE, true], [other, false]])
|
|
72
|
+
})
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
it('reads the C++ broker\'s array, including its error text', async () => {
|
|
76
|
+
await withQueen([
|
|
77
|
+
{ body: [{ index: 0, leaseId: LEASE, success: true, error: null, expiresAt: EXPIRES }] },
|
|
78
|
+
{ body: [{ index: 0, leaseId: LEASE, success: false, error: 'Lease already expired', expiresAt: null }] }
|
|
79
|
+
], async (queen) => {
|
|
80
|
+
const ok = await queen.renew(LEASE)
|
|
81
|
+
assert.equal(ok.success, true)
|
|
82
|
+
assert.equal(ok.newExpiresAt, EXPIRES)
|
|
83
|
+
const gone = await queen.renew(LEASE)
|
|
84
|
+
assert.equal(gone.success, false)
|
|
85
|
+
assert.equal(gone.error, 'Lease already expired')
|
|
86
|
+
})
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
it('an HTTP failure is still a failed renewal, not a throw', async () => {
|
|
90
|
+
await withQueen([{ status: 500, body: { error: 'boom' } }], async (queen) => {
|
|
91
|
+
const res = await queen.renew(LEASE)
|
|
92
|
+
assert.equal(res.success, false)
|
|
93
|
+
assert.ok(res.error)
|
|
94
|
+
})
|
|
95
|
+
})
|
|
96
|
+
})
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* a tenant that is over quota -- while the fire never stops by itself, so
|
|
11
11
|
* that tenant keeps producing messages it cannot stop.
|
|
12
12
|
* 2. delayMs, never delaySeconds and never an absolute instant (§4.2,
|
|
13
|
-
* §20.6). One clock,
|
|
13
|
+
* §20.6). One clock, the broker's. `deliverAt` in an op is a 22023.
|
|
14
14
|
* 3. The payload is base64 (§6.2), and the SDK encodes it, so a caller never
|
|
15
15
|
* has to know.
|
|
16
16
|
* 4. `txn` is mandatory on a schedule and is echoed on `absent`, which is
|
|
@@ -181,11 +181,19 @@ describe('Transaction wire — the verdict that must not throw', () => {
|
|
|
181
181
|
})
|
|
182
182
|
|
|
183
183
|
it('commit() still THROWS on every other failure', async () => {
|
|
184
|
-
|
|
184
|
+
// The body a 2.0 broker answers for an ack whose lease is gone (a 1.x
|
|
185
|
+
// broker spelled the reason ack_rejected).
|
|
186
|
+
const rejected = ok({
|
|
187
|
+
transactionId: 'txn-1',
|
|
188
|
+
success: false,
|
|
189
|
+
reason: 'rejected_ack',
|
|
190
|
+
error: 'QTXN 1 acked message(s) on partition 3 are not leased by this worker or were already acked; the transaction rolled back',
|
|
191
|
+
results: []
|
|
192
|
+
})
|
|
185
193
|
await withQueen([rejected], async (queen) => {
|
|
186
194
|
await assert.rejects(() => queen.transaction().ack(MSG).commit(), (e) => {
|
|
187
195
|
assert.match(e.message, /QTXN/)
|
|
188
|
-
assert.equal(e.reason, '
|
|
196
|
+
assert.equal(e.reason, 'rejected_ack', 'the broker\'s reason travels on the error, so nobody matches the message')
|
|
189
197
|
return true
|
|
190
198
|
})
|
|
191
199
|
})
|
|
@@ -220,3 +228,94 @@ describe('Transaction wire — the verdict that must not throw', () => {
|
|
|
220
228
|
})
|
|
221
229
|
})
|
|
222
230
|
})
|
|
231
|
+
|
|
232
|
+
// ---------------------------------------------------------------------------
|
|
233
|
+
// The consumer group of an ack op. The lease an ack has to match belongs to
|
|
234
|
+
// the group the message was popped under, and an op that names no group is
|
|
235
|
+
// judged in queue mode: a message popped by a group and acked without one
|
|
236
|
+
// rolled the whole bundle back with `rejected_ack` (found 2026-10-02 against
|
|
237
|
+
// 2.0.0-beta.6). Every pop answers each message with its `consumerGroup`.
|
|
238
|
+
// ---------------------------------------------------------------------------
|
|
239
|
+
|
|
240
|
+
describe('Transaction wire — the ack op carries the group the message was popped under', () => {
|
|
241
|
+
const fromGroup = (group, n = 1) => ({
|
|
242
|
+
...MSG,
|
|
243
|
+
transactionId: `aaaaaaaa-aaaa-7aaa-8aaa-00000000000${n}`,
|
|
244
|
+
consumerGroup: group
|
|
245
|
+
})
|
|
246
|
+
|
|
247
|
+
it('takes consumerGroup from the message when the caller names none', async () => {
|
|
248
|
+
await withQueen([success()], async (queen, hits) => {
|
|
249
|
+
await queen.transaction().ack(fromGroup('workers')).commit()
|
|
250
|
+
assert.equal(hits[0].body.operations[0].consumerGroup, 'workers')
|
|
251
|
+
})
|
|
252
|
+
})
|
|
253
|
+
|
|
254
|
+
it('a queue-mode message adds no consumerGroup key, as before', async () => {
|
|
255
|
+
await withQueen([success(), success()], async (queen, hits) => {
|
|
256
|
+
await queen.transaction().ack(fromGroup('__QUEUE_MODE__')).commit()
|
|
257
|
+
await queen.transaction().ack(MSG).commit() // hand-built, no group at all
|
|
258
|
+
for (const hit of hits) {
|
|
259
|
+
assert.deepEqual(Object.keys(hit.body.operations[0]).sort(), ['leaseId', 'partitionId', 'status', 'transactionId', 'type'])
|
|
260
|
+
}
|
|
261
|
+
})
|
|
262
|
+
})
|
|
263
|
+
|
|
264
|
+
it('an explicit group wins, as { consumerGroup } or as { group }', async () => {
|
|
265
|
+
await withQueen([success(), success()], async (queen, hits) => {
|
|
266
|
+
await queen.transaction().ack(fromGroup('workers'), 'completed', { consumerGroup: 'auditors' }).commit()
|
|
267
|
+
await queen.transaction().ack(fromGroup('workers'), 'completed', { group: 'auditors' }).commit()
|
|
268
|
+
assert.equal(hits[0].body.operations[0].consumerGroup, 'auditors')
|
|
269
|
+
assert.equal(hits[1].body.operations[0].consumerGroup, 'auditors')
|
|
270
|
+
})
|
|
271
|
+
})
|
|
272
|
+
|
|
273
|
+
it('messages from different groups each keep their own', async () => {
|
|
274
|
+
await withQueen([success()], async (queen, hits) => {
|
|
275
|
+
await queen.transaction().ack([fromGroup('a', 1), fromGroup('b', 2)]).commit()
|
|
276
|
+
assert.deepEqual(hits[0].body.operations.map(op => op.consumerGroup), ['a', 'b'])
|
|
277
|
+
})
|
|
278
|
+
})
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
// The lease of an ack op. A Queen 2 broker fences each ack with the leaseId
|
|
282
|
+
// the operation carries, and lends it one from requiredLeases only when every
|
|
283
|
+
// lease in the bundle is the same lease. An ack whose lease travelled in
|
|
284
|
+
// requiredLeases alone was applied unfenced in a bundle of two leases, even
|
|
285
|
+
// after that lease expired and another consumer took the message (found
|
|
286
|
+
// 2026-10-03 against 2.0.0-beta.6).
|
|
287
|
+
// ---------------------------------------------------------------------------
|
|
288
|
+
|
|
289
|
+
describe('Transaction wire — each ack op carries its own lease', () => {
|
|
290
|
+
const leased = (n, leaseId) => ({
|
|
291
|
+
transactionId: `aaaaaaaa-aaaa-7aaa-8aaa-00000000000${n}`,
|
|
292
|
+
partitionId: `pppppppp-pppp-7ppp-8ppp-00000000000${n}`,
|
|
293
|
+
...(leaseId === undefined ? {} : { leaseId })
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
it('a bundle spanning two leases names each lease on its own ack', async () => {
|
|
297
|
+
await withQueen([success()], async (queen, hits) => {
|
|
298
|
+
await queen.transaction()
|
|
299
|
+
.ack(leased(1, 'lease-1'))
|
|
300
|
+
.ack(leased(2, 'lease-2'))
|
|
301
|
+
.ack(leased(3))
|
|
302
|
+
.commit()
|
|
303
|
+
|
|
304
|
+
const [first, second, third] = hits[0].body.operations
|
|
305
|
+
assert.equal(first.leaseId, 'lease-1')
|
|
306
|
+
assert.equal(second.leaseId, 'lease-2')
|
|
307
|
+
assert.ok(!('leaseId' in third), 'a message without a lease adds no leaseId key')
|
|
308
|
+
assert.deepEqual(hits[0].body.requiredLeases, ['lease-1', 'lease-2'])
|
|
309
|
+
})
|
|
310
|
+
})
|
|
311
|
+
|
|
312
|
+
it('a null or empty lease adds no leaseId key either', async () => {
|
|
313
|
+
await withQueen([success()], async (queen, hits) => {
|
|
314
|
+
await queen.transaction().ack([leased(1, null), leased(2, '')]).commit()
|
|
315
|
+
for (const op of hits[0].body.operations) {
|
|
316
|
+
assert.ok(!('leaseId' in op))
|
|
317
|
+
}
|
|
318
|
+
assert.deepEqual(hits[0].body.requiredLeases, [])
|
|
319
|
+
})
|
|
320
|
+
})
|
|
321
|
+
})
|
package/test-v2/kv.js
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* KV integration suite (PLAN_KV_TIMERS.md §5, §8.3).
|
|
3
3
|
*
|
|
4
|
-
* Every test here is repeatable ONLY because run
|
|
5
|
-
*
|
|
4
|
+
* Every test here is repeatable ONLY because every run starts on a fresh broker
|
|
5
|
+
* (test/run.sh creates the lane's data volume empty and destroys it with
|
|
6
|
+
* `down -v`): against a broker that kept the previous run's keys, a
|
|
6
7
|
* `putIfAbsent` test is green on its first execution and red forever after,
|
|
7
|
-
* and an `incr` test accumulates between runs.
|
|
8
|
-
*
|
|
8
|
+
* and an `incr` test accumulates between runs. Keep the namespaces under
|
|
9
|
+
* `test-`, like every other fixture of the suite.
|
|
9
10
|
*
|
|
10
11
|
* And `forever` is BANNED in this file. A test that goes wrong with a TTL
|
|
11
12
|
* leaves state that expires by itself; a test that goes wrong with `forever`
|
|
12
|
-
* leaves immortal state in a shared
|
|
13
|
+
* leaves immortal state in a shared broker.
|
|
13
14
|
*/
|
|
14
15
|
|
|
15
16
|
import { KV_NS, sleep } from './_kvtimers.js'
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pop() and consume() share one builder but not their defaults.
|
|
3
|
+
*
|
|
4
|
+
* autoAck commitOnDelivery wait
|
|
5
|
+
* pop() never sent off; on sends autoAck true (POP_DEFAULTS)
|
|
6
|
+
* consume true, client-side refused true (CONSUME_DEFAULTS)
|
|
7
|
+
*
|
|
8
|
+
* autoAck() is consume()'s ack after the handler and never reaches the wire.
|
|
9
|
+
* The broker's at-most-once commit at delivery is the pop's own option,
|
|
10
|
+
* commitOnDelivery(), which sends autoAck=true; consume() always leases, so it
|
|
11
|
+
* refuses that option before any request.
|
|
12
|
+
* POP_DEFAULTS.wait said false while every pop long-polled; the long poll is
|
|
13
|
+
* what callers rely on, so it stays, and POP_DEFAULTS and the guide say so.
|
|
14
|
+
*
|
|
15
|
+
* The broker contract (server/src/handlers/data.rs, PopParams): autoAck=true
|
|
16
|
+
* commits the messages at delivery, with an empty leaseId; an absent autoAck
|
|
17
|
+
* is false. An absent wait is false too, but this SDK always sends wait.
|
|
18
|
+
*
|
|
19
|
+
* Same style as conflation-unit/conflationWire.test.js: a real node:http
|
|
20
|
+
* server playing a canned plan, real fetch, and every assertion is about the
|
|
21
|
+
* bytes that crossed the socket.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { describe, it } from 'node:test'
|
|
25
|
+
import assert from 'node:assert/strict'
|
|
26
|
+
|
|
27
|
+
import { Queen } from '../../client-v2/index.js'
|
|
28
|
+
import { withPlanServer, ok } from '../kv-unit/_planServer.js'
|
|
29
|
+
|
|
30
|
+
const QUEUE = 'q'
|
|
31
|
+
const GROUP = 'g'
|
|
32
|
+
|
|
33
|
+
const frame = (n = 1) => ({
|
|
34
|
+
transactionId: `txn-${n}`,
|
|
35
|
+
partitionId: `part-${n}`,
|
|
36
|
+
partition: 'Default',
|
|
37
|
+
payload: { n },
|
|
38
|
+
leaseId: 'lease-1',
|
|
39
|
+
consumerGroup: GROUP
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
const popBody = (frames = [frame()]) => ok({ messages: frames, partitionsClaimed: frames.length })
|
|
43
|
+
|
|
44
|
+
// Acks answer with one result per acknowledgment; the plan only needs enough
|
|
45
|
+
// of them for the consume() cases below.
|
|
46
|
+
const ackBody = ok([{ index: 0, transactionId: 'txn-1', success: true, error: null }])
|
|
47
|
+
|
|
48
|
+
function query(url) {
|
|
49
|
+
const i = url.indexOf('?')
|
|
50
|
+
return new URLSearchParams(i < 0 ? '' : url.slice(i + 1))
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const pops = (hits) => hits.filter(h => h.method === 'GET' && h.url.startsWith('/api/v1/pop'))
|
|
54
|
+
const acks = (hits) => hits.filter(h => h.method === 'POST' && h.url.startsWith('/api/v1/ack'))
|
|
55
|
+
const statusesOf = (hit) => (hit.body.acknowledgments || [hit.body]).map(a => a.status)
|
|
56
|
+
|
|
57
|
+
async function withQueen(plan, defaultResponse, run) {
|
|
58
|
+
await withPlanServer(plan, defaultResponse, async (url, hits) => {
|
|
59
|
+
const queen = new Queen({ url, handleSignals: false })
|
|
60
|
+
try {
|
|
61
|
+
await run(queen, hits)
|
|
62
|
+
} finally {
|
|
63
|
+
await queen.close()
|
|
64
|
+
}
|
|
65
|
+
})
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
describe('pop() — autoAck', () => {
|
|
69
|
+
// autoAck() is consume()'s ack after the handler and never reaches the
|
|
70
|
+
// wire: a pop comes back leased whatever autoAck() said. The broker's
|
|
71
|
+
// at-most-once autoAck is commitOnDelivery(), below.
|
|
72
|
+
for (const [label, build] of [
|
|
73
|
+
['autoAck() never called', (q) => q.group(GROUP)],
|
|
74
|
+
['autoAck(true)', (q) => q.group(GROUP).autoAck(true)],
|
|
75
|
+
['autoAck(false)', (q) => q.group(GROUP).autoAck(false)],
|
|
76
|
+
]) {
|
|
77
|
+
it(`sends no autoAck after ${label}`, async () => {
|
|
78
|
+
await withQueen([popBody(), popBody()], popBody(), async (queen, hits) => {
|
|
79
|
+
await build(queen.queue(QUEUE)).pop()
|
|
80
|
+
await build(queen.queue(QUEUE)).popResult()
|
|
81
|
+
|
|
82
|
+
for (const hit of pops(hits)) {
|
|
83
|
+
assert.equal(query(hit.url).has('autoAck'), false)
|
|
84
|
+
}
|
|
85
|
+
})
|
|
86
|
+
})
|
|
87
|
+
}
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
describe('pop() — commitOnDelivery', () => {
|
|
91
|
+
// The broker's commit at delivery, on the wire name every 2.x broker reads:
|
|
92
|
+
// the pop carries autoAck=true and nothing else changes.
|
|
93
|
+
it('pop() and popResult() send autoAck=true and nothing else', async () => {
|
|
94
|
+
await withQueen([popBody(), popBody(), popBody()], popBody(), async (queen, hits) => {
|
|
95
|
+
await queen.queue(QUEUE).group(GROUP).pop()
|
|
96
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery().pop()
|
|
97
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery(true).popResult()
|
|
98
|
+
|
|
99
|
+
const [plain, ...committed] = pops(hits)
|
|
100
|
+
assert.equal(committed.length, 2)
|
|
101
|
+
for (const hit of committed) {
|
|
102
|
+
const q = query(hit.url)
|
|
103
|
+
assert.equal(q.get('autoAck'), 'true')
|
|
104
|
+
q.delete('autoAck')
|
|
105
|
+
assert.deepEqual(Object.fromEntries(q), Object.fromEntries(query(plain.url)))
|
|
106
|
+
assert.equal(hit.url.split('?')[0], plain.url.split('?')[0])
|
|
107
|
+
}
|
|
108
|
+
})
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
it('sends no autoAck after commitOnDelivery(false)', async () => {
|
|
112
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
113
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery(true).commitOnDelivery(false).pop()
|
|
114
|
+
|
|
115
|
+
assert.equal(query(pops(hits)[0].url).has('autoAck'), false)
|
|
116
|
+
})
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
it('does not depend on autoAck(), which stays consume()\'s', async () => {
|
|
120
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
121
|
+
await queen.queue(QUEUE).group(GROUP).autoAck(false).commitOnDelivery().pop()
|
|
122
|
+
|
|
123
|
+
assert.equal(query(pops(hits)[0].url).get('autoAck'), 'true')
|
|
124
|
+
})
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
it('with conflation(), sends both and raises the broker\'s 400 instead of returning []', async () => {
|
|
128
|
+
// The broker refuses conflation together with a commit at delivery
|
|
129
|
+
// (server/src/handlers/data.rs, conflation_refusal).
|
|
130
|
+
const refusal = {
|
|
131
|
+
status: 400,
|
|
132
|
+
body: { success: false, error: 'conflation cannot be combined with autoAck', messages: [] }
|
|
133
|
+
}
|
|
134
|
+
await withQueen([refusal], refusal, async (queen, hits) => {
|
|
135
|
+
await assert.rejects(
|
|
136
|
+
() => queen.queue(QUEUE).group(GROUP).conflation().commitOnDelivery().pop(),
|
|
137
|
+
(err) => err.status === 400
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
const q = query(pops(hits)[0].url)
|
|
141
|
+
assert.equal(q.get('autoAck'), 'true')
|
|
142
|
+
assert.equal(q.get('conflation'), 'true')
|
|
143
|
+
})
|
|
144
|
+
})
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
describe('consume() refuses commitOnDelivery()', () => {
|
|
148
|
+
it('throws before any request', async () => {
|
|
149
|
+
await withQueen([], popBody(), async (queen, hits) => {
|
|
150
|
+
assert.throws(
|
|
151
|
+
() => queen.queue(QUEUE).group(GROUP).commitOnDelivery().consume(async () => {}),
|
|
152
|
+
{
|
|
153
|
+
message: 'commitOnDelivery() is a pop() option; consume() always leases its messages'
|
|
154
|
+
}
|
|
155
|
+
)
|
|
156
|
+
assert.equal(hits.length, 0)
|
|
157
|
+
})
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
it('consumes as before after commitOnDelivery(false)', async () => {
|
|
161
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
162
|
+
await queen.queue(QUEUE).group(GROUP).commitOnDelivery(false).limit(1).consume(async () => {})
|
|
163
|
+
|
|
164
|
+
assert.equal(query(pops(hits)[0].url).has('autoAck'), false)
|
|
165
|
+
assert.equal(acks(hits).length, 1)
|
|
166
|
+
})
|
|
167
|
+
})
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
describe('pop() — wait', () => {
|
|
171
|
+
it('long-polls when wait() was never called', async () => {
|
|
172
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
173
|
+
await queen.queue(QUEUE).group(GROUP).pop()
|
|
174
|
+
|
|
175
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'true')
|
|
176
|
+
})
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
it('long-polls for timeoutMillis() when only that was called', async () => {
|
|
180
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
181
|
+
await queen.queue(QUEUE).timeoutMillis(2000).pop()
|
|
182
|
+
|
|
183
|
+
const q = query(pops(hits)[0].url)
|
|
184
|
+
assert.equal(q.get('wait'), 'true')
|
|
185
|
+
assert.equal(q.get('timeout'), '2000')
|
|
186
|
+
})
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('sends wait=false after wait(false)', async () => {
|
|
190
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
191
|
+
await queen.queue(QUEUE).wait(false).pop()
|
|
192
|
+
|
|
193
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'false')
|
|
194
|
+
})
|
|
195
|
+
})
|
|
196
|
+
})
|
|
197
|
+
|
|
198
|
+
describe('consume() keeps its own defaults', () => {
|
|
199
|
+
it('long-polls and never sends autoAck, also after autoAck(true): it acks after the handler', async () => {
|
|
200
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
201
|
+
await queen.queue(QUEUE).group(GROUP).autoAck(true).limit(1)
|
|
202
|
+
.consume(async () => {})
|
|
203
|
+
|
|
204
|
+
const q = query(pops(hits)[0].url)
|
|
205
|
+
assert.equal(q.get('wait'), 'true')
|
|
206
|
+
assert.equal(q.has('autoAck'), false)
|
|
207
|
+
assert.equal(acks(hits).length, 1, 'the consumer acked after the handler returned')
|
|
208
|
+
assert.deepEqual(statusesOf(acks(hits)[0]), ['completed'])
|
|
209
|
+
})
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
it('acks after the handler when autoAck() was never called', async () => {
|
|
213
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
214
|
+
await queen.queue(QUEUE).group(GROUP).limit(1).consume(async () => {})
|
|
215
|
+
|
|
216
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'true')
|
|
217
|
+
assert.equal(acks(hits).length, 1)
|
|
218
|
+
})
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
it('sends no ack after autoAck(false)', async () => {
|
|
222
|
+
await withQueen([popBody()], popBody(), async (queen, hits) => {
|
|
223
|
+
await queen.queue(QUEUE).group(GROUP).autoAck(false).limit(1).consume(async () => {})
|
|
224
|
+
|
|
225
|
+
assert.equal(acks(hits).length, 0)
|
|
226
|
+
})
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
it('sends wait=false after wait(false)', async () => {
|
|
230
|
+
await withQueen([popBody(), ackBody], ackBody, async (queen, hits) => {
|
|
231
|
+
await queen.queue(QUEUE).group(GROUP).wait(false).limit(1).consume(async () => {})
|
|
232
|
+
|
|
233
|
+
assert.equal(query(pops(hits)[0].url).get('wait'), 'false')
|
|
234
|
+
})
|
|
235
|
+
})
|
|
236
|
+
})
|