queen-mq 1.3.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -21
- package/client-v2/Queen.js +47 -15
- package/client-v2/README.md +34 -13
- package/client-v2/admin/Admin.js +30 -55
- package/client-v2/builders/QueueBuilder.js +106 -34
- package/client-v2/builders/TimerBuilder.js +2 -2
- package/client-v2/builders/TransactionBuilder.js +35 -9
- package/client-v2/consumer/ConsumerManager.js +79 -53
- package/client-v2/ephemeral/Ephemeral.js +47 -17
- package/client-v2/kv/Kv.js +4 -4
- package/client-v2/kv/expiry.js +1 -1
- package/client-v2/streams/Stream.js +8 -3
- package/client-v2/streams/helpers/rateLimiter.js +4 -4
- package/client-v2/streams/operators/GateOperator.js +9 -2
- package/client-v2/streams/operators/ReduceOperator.js +2 -2
- package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
- package/client-v2/streams/runtime/Runner.js +111 -54
- package/client-v2/streams/runtime/cycle.js +4 -4
- package/client-v2/streams/runtime/register.js +1 -1
- package/client-v2/utils/conflation.js +0 -6
- package/client-v2/utils/consumerGroup.js +54 -0
- package/client-v2/utils/defaults.js +3 -2
- package/package.json +5 -8
- package/test-v2/_kvtimers.js +12 -13
- package/test-v2/ackwindow.js +12 -192
- package/test-v2/admin-unit/removedRoutes.test.js +63 -0
- package/test-v2/bootstrap.js +2 -2
- package/test-v2/conflation-unit/conflationWire.test.js +0 -12
- package/test-v2/consume.js +41 -1
- package/test-v2/consumer-unit/handlerError.test.js +161 -0
- package/test-v2/consumer-unit/nackScope.test.js +88 -0
- package/test-v2/docs.js +5 -4
- package/test-v2/ephemeral-unit/ephemeralWire.test.js +24 -1
- package/test-v2/http-unit/pushStatus.test.js +142 -0
- package/test-v2/http-unit/renew.test.js +96 -0
- package/test-v2/kv-unit/timerWire.test.js +1 -1
- package/test-v2/kv-unit/txnWire.test.js +101 -2
- package/test-v2/kv.js +6 -5
- package/test-v2/pop-unit/popDefaults.test.js +236 -0
- package/test-v2/pop.js +96 -1
- package/test-v2/run.js +50 -114
- package/test-v2/runner-unit/fatalExit.test.js +64 -0
- package/test-v2/semantics.js +20 -34
- package/test-v2/stream/_helpers.js +10 -19
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/gate.js +54 -0
- package/test-v2/stream/index.js +2 -0
- package/test-v2/stream/tumbling.js +3 -3
- package/test-v2/streams-unit/ack.test.js +61 -0
- package/test-v2/streams-unit/cycle.test.js +1 -1
- package/test-v2/streams-unit/e2e.test.js +6 -10
- package/test-v2/streams-unit/gate.test.js +189 -0
- package/test-v2/timers.js +6 -5
- package/test-v2/transaction.js +169 -0
- package/test-v2/watermark.js +38 -176
- package/test-v2/maintenance.js +0 -277
package/test-v2/stream/index.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* recovery.js — config_hash mismatch + reset + mid-stream resume
|
|
13
13
|
* throughput.js — multi-partition / many-window throughput
|
|
14
14
|
* combined.js — full pipelines + concurrent streams
|
|
15
|
+
* gate.js — .gate() partial acks against the broker
|
|
15
16
|
*/
|
|
16
17
|
|
|
17
18
|
export * from './operators.js'
|
|
@@ -23,3 +24,4 @@ export * from './eventTime.js'
|
|
|
23
24
|
export * from './recovery.js'
|
|
24
25
|
export * from './throughput.js'
|
|
25
26
|
export * from './combined.js'
|
|
27
|
+
export * from './gate.js'
|
|
@@ -294,9 +294,9 @@ export async function tumblingForeachCtxHasWindowAndPartition(client) {
|
|
|
294
294
|
const queryId = mkName('tumblingForeachCtxHasWindowAndPartition', 'q')
|
|
295
295
|
|
|
296
296
|
await client.queue(src).create()
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
}
|
|
297
|
+
// One push, so all three get the same broker createdAt: three pushes can
|
|
298
|
+
// straddle a second and split across two 1 s windows (count 1, then 2).
|
|
299
|
+
await client.queue(src).partition('myKey').push([0, 1, 2].map(v => ({ data: { v } })))
|
|
300
300
|
|
|
301
301
|
const captured = []
|
|
302
302
|
const handle = await Stream
|
|
@@ -201,3 +201,64 @@ describe('Queen.ack batch (/api/v1/ack/batch)', () => {
|
|
|
201
201
|
)
|
|
202
202
|
})
|
|
203
203
|
})
|
|
204
|
+
|
|
205
|
+
// The consumer group of an ack. /ack and /ack/batch judge an ack that names no
|
|
206
|
+
// group in queue mode, so a message popped by a group and acked without one
|
|
207
|
+
// came back `invalid or expired lease` (found 2026-10-02 against
|
|
208
|
+
// 2.0.0-beta.6). Every pop answers each message with its `consumerGroup`.
|
|
209
|
+
describe('Queen.ack — the group the message was popped under', () => {
|
|
210
|
+
const okFor = (body) => (body.acknowledgments || [body]).map((a, i) => ackResultItem(i, a.transactionId, true))
|
|
211
|
+
const popped = (n, group) => ({ transactionId: `tx-${n}`, partitionId: 'p-1', leaseId: 'lease-1', consumerGroup: group })
|
|
212
|
+
|
|
213
|
+
it('single: takes consumerGroup from the message when the caller names none', async () => {
|
|
214
|
+
await withAckServer(okFor, async (client, requests) => {
|
|
215
|
+
await client.ack(popped(1, 'workers'), true)
|
|
216
|
+
assert.equal(requests[0].body.consumerGroup, 'workers')
|
|
217
|
+
})
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
it('single: a queue-mode or hand-built message still sends consumerGroup null', async () => {
|
|
221
|
+
await withAckServer(okFor, async (client, requests) => {
|
|
222
|
+
await client.ack(popped(1, '__QUEUE_MODE__'), true)
|
|
223
|
+
await client.ack(MSG, true)
|
|
224
|
+
assert.equal(requests[0].body.consumerGroup, null)
|
|
225
|
+
assert.equal(requests[1].body.consumerGroup, null)
|
|
226
|
+
})
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
it('single: an explicit { group } wins', async () => {
|
|
230
|
+
await withAckServer(okFor, async (client, requests) => {
|
|
231
|
+
await client.ack(popped(1, 'workers'), false, { group: 'auditors' })
|
|
232
|
+
assert.equal(requests[0].body.consumerGroup, 'auditors')
|
|
233
|
+
})
|
|
234
|
+
})
|
|
235
|
+
|
|
236
|
+
it('batch: the group the messages share', async () => {
|
|
237
|
+
await withAckServer(okFor, async (client, requests) => {
|
|
238
|
+
const result = await client.ack([popped(1, 'workers'), popped(2, 'workers')], true)
|
|
239
|
+
assert.equal(result.success, true)
|
|
240
|
+
assert.equal(requests[0].path, '/api/v1/ack/batch')
|
|
241
|
+
assert.equal(requests[0].body.consumerGroup, 'workers')
|
|
242
|
+
})
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
it('batch: queue-mode messages still send consumerGroup null', async () => {
|
|
246
|
+
await withAckServer(okFor, async (client, requests) => {
|
|
247
|
+
await client.ack([popped(1, '__QUEUE_MODE__'), popped(2, '__QUEUE_MODE__')], true)
|
|
248
|
+
assert.equal(requests[0].body.consumerGroup, null)
|
|
249
|
+
})
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
it('batch: messages from two groups are refused before anything is sent', async () => {
|
|
253
|
+
await withAckServer(okFor, async (client, requests) => {
|
|
254
|
+
await assert.rejects(
|
|
255
|
+
() => client.ack([popped(1, 'a'), popped(2, 'b')], true),
|
|
256
|
+
/different consumer groups in one call \(a, b\)/
|
|
257
|
+
)
|
|
258
|
+
assert.equal(requests.length, 0)
|
|
259
|
+
// ... unless the caller names the group, which is then theirs to get right.
|
|
260
|
+
await client.ack([popped(1, 'a'), popped(2, 'b')], true, { group: 'a' })
|
|
261
|
+
assert.equal(requests[0].body.consumerGroup, 'a')
|
|
262
|
+
})
|
|
263
|
+
})
|
|
264
|
+
})
|
|
@@ -100,7 +100,7 @@ describe('Runner — windowed aggregation', () => {
|
|
|
100
100
|
// The 10:00 window is closed (created and closed in the same batch
|
|
101
101
|
// since the third message crosses the minute boundary). The 10:01
|
|
102
102
|
// window stays open. Because the 10:00 window was never persisted to
|
|
103
|
-
//
|
|
103
|
+
// the broker (created+closed in one cycle), no delete state op is emitted —
|
|
104
104
|
// only the closed-window push and the open-window upsert.
|
|
105
105
|
assert.equal(c.push_items.length, 1, 'one closed-window emit')
|
|
106
106
|
assert.deepEqual(c.push_items[0].payload, { count: 2, sum: 30 })
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* End-to-end test for queen-streams against a live Queen server
|
|
2
|
+
* End-to-end test for queen-streams against a live Queen server.
|
|
3
3
|
*
|
|
4
4
|
* What it covers (per the plan's Definition of Done):
|
|
5
5
|
* 1. 10,000 messages across 50 partitions
|
|
@@ -8,14 +8,10 @@
|
|
|
8
8
|
* 4. Exactly-once on the sink queue
|
|
9
9
|
* 5. Per-partition state isolation
|
|
10
10
|
* 6. Downstream consumer of sink wakes up immediately
|
|
11
|
-
* (validates inline update_partition_lookup_v1 in streams_cycle_v1)
|
|
12
11
|
*
|
|
13
12
|
* Prerequisites:
|
|
14
13
|
* - Queen running on QUEEN_URL (default http://localhost:6632)
|
|
15
|
-
* -
|
|
16
|
-
* 002_streams_schema, 007_log_streams, 008_streams_register_query_v1,
|
|
17
|
-
* 009_streams_state_get_v1 (the retired rows-era streams cycle's role
|
|
18
|
-
* now lives in 007_log_streams' log_streams_cycle_v1)
|
|
14
|
+
* - A broker serving the /streams/v1 routes (register, state/get, cycle)
|
|
19
15
|
*
|
|
20
16
|
* Run:
|
|
21
17
|
* QUEEN_URL=http://localhost:6632 node --test test/e2e.test.js
|
|
@@ -129,7 +125,7 @@ describe('queen-streams e2e (live Queen)', { skip: !SHOULD_RUN }, () => {
|
|
|
129
125
|
|
|
130
126
|
// 3. Mid-stream kill: after 1.5s, stop the runner abruptly to simulate
|
|
131
127
|
// a worker crash, then restart it. Cursor + state should reconcile
|
|
132
|
-
// via Queen's lease/retry +
|
|
128
|
+
// via Queen's lease/retry + the per-partition streams state key.
|
|
133
129
|
setTimeout(async () => {
|
|
134
130
|
if (!stopped) {
|
|
135
131
|
await handle.stop()
|
|
@@ -205,8 +201,8 @@ describe('queen-streams e2e (live Queen)', { skip: !SHOULD_RUN }, () => {
|
|
|
205
201
|
const winKey = m.data && m.data.__winKey
|
|
206
202
|
// The aggregate value doesn't carry windowKey by default; we check for
|
|
207
203
|
// duplicates by (partition + transactionId) instead, which Queen
|
|
208
|
-
// guarantees unique across a partition. The
|
|
209
|
-
//
|
|
204
|
+
// guarantees unique across a partition. The streams cycle's sink
|
|
205
|
+
// pushes go through the SAME transactionId dedup as any push.
|
|
210
206
|
const dedupKey = `${m.partition}|${m.transactionId}`
|
|
211
207
|
assert.ok(!seen.has(dedupKey), 'duplicate sink message for ' + dedupKey)
|
|
212
208
|
seen.add(dedupKey)
|
|
@@ -214,7 +210,7 @@ describe('queen-streams e2e (live Queen)', { skip: !SHOULD_RUN }, () => {
|
|
|
214
210
|
|
|
215
211
|
// 6. Per-partition state isolation: every cycle the runner committed
|
|
216
212
|
// must touch state rows ONLY for its own partition. We can't easily
|
|
217
|
-
// observe this from the client side (
|
|
213
|
+
// observe this from the client side (the stored state is private), but the
|
|
218
214
|
// aggregate-shape correctness in (5) above implies it (a leaked
|
|
219
215
|
// cross-partition write would produce wrong sums).
|
|
220
216
|
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* .gate() settles SOURCE MESSAGES, not envelopes.
|
|
3
|
+
*
|
|
4
|
+
* The gate's partial ack is an offset commit: a 2.x broker advances the
|
|
5
|
+
* cursor `ack.count` messages from the head of the leased batch and keeps the
|
|
6
|
+
* lease (release_lease=false). A pre-stage breaks the one-to-one map between
|
|
7
|
+
* messages and the envelopes the gate sees -- `.filter()` leaves a message
|
|
8
|
+
* with none, `.flatMap()` with several -- and the runner counted envelopes:
|
|
9
|
+
*
|
|
10
|
+
* flatMap x2, message 2 denied count 2 = messages 1 AND 2 acked; the
|
|
11
|
+
* denied message was never redelivered
|
|
12
|
+
* (reproduced live on 2.0.0-beta.6: 4 of 6
|
|
13
|
+
* sink items, message 2 lost)
|
|
14
|
+
* filter drops every message nothing committed; the batch came back on
|
|
15
|
+
* every lease expiry, forever
|
|
16
|
+
*
|
|
17
|
+
* Same fix as the Rust client (clients/client-rust/src/streams/runner.rs,
|
|
18
|
+
* gate_cycle): a message is settled only when every envelope it produced is
|
|
19
|
+
* allowed, and what a denied message did to the state is rolled back.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { describe, it, before, after } from 'node:test'
|
|
23
|
+
import assert from 'node:assert/strict'
|
|
24
|
+
|
|
25
|
+
import { Stream } from '../../client-v2/streams/Stream.js'
|
|
26
|
+
import { createFakeStreamsServer, createFakeSource, fakeMessage } from './fakeServer.js'
|
|
27
|
+
|
|
28
|
+
const PART = '00000000-0000-0000-0000-00000000ga7e'
|
|
29
|
+
|
|
30
|
+
const batch = (...ns) => ns.map((n, i) => fakeMessage({
|
|
31
|
+
partitionId: PART,
|
|
32
|
+
partitionName: 'tenant-1',
|
|
33
|
+
data: { n },
|
|
34
|
+
createdAt: `2026-10-02T10:00:0${i}.000Z`
|
|
35
|
+
}))
|
|
36
|
+
|
|
37
|
+
const quiet = { info() {}, warn() {}, error() {}, debug() {} }
|
|
38
|
+
|
|
39
|
+
// Run until `cycles` cycles were committed (or the time is up), then stop.
|
|
40
|
+
async function runFor(server, stream, { cycles = 1, timeoutMs = 1500, queryId }) {
|
|
41
|
+
const baseline = server.recorded.cycles.length
|
|
42
|
+
const handle = await stream.run({ queryId, url: server.url, logger: quiet })
|
|
43
|
+
const start = Date.now()
|
|
44
|
+
while (server.recorded.cycles.length - baseline < cycles && Date.now() - start < timeoutMs) {
|
|
45
|
+
await new Promise(r => setTimeout(r, 20))
|
|
46
|
+
}
|
|
47
|
+
await handle.stop()
|
|
48
|
+
return { handle, cycles: server.recorded.cycles.slice(baseline) }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
describe('.gate() — the partial ack counts source messages', () => {
|
|
52
|
+
let server
|
|
53
|
+
before(async () => { server = await createFakeStreamsServer() })
|
|
54
|
+
after(async () => { await server.close() })
|
|
55
|
+
|
|
56
|
+
it('flatMap before the gate: a denied message is not acked with its predecessor', async () => {
|
|
57
|
+
const msgs = batch(1, 2, 3)
|
|
58
|
+
const stream = Stream.from(createFakeSource('src', [msgs]))
|
|
59
|
+
.flatMap(m => [m.data, m.data])
|
|
60
|
+
.gate(v => v.n !== 2)
|
|
61
|
+
.to({ _queueName: 'sink' })
|
|
62
|
+
|
|
63
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.flatmap' })
|
|
64
|
+
assert.equal(cycles.length, 1)
|
|
65
|
+
const c = cycles[0]
|
|
66
|
+
assert.equal(c.ack.count, 1, 'one MESSAGE settled; two envelopes acked messages 1 and 2')
|
|
67
|
+
assert.equal(c.ack.transactionId, msgs[0].transactionId)
|
|
68
|
+
assert.equal(c.release_lease, false, 'the denied tail stays leased, in order')
|
|
69
|
+
assert.deepEqual(c.push_items.map(p => p.payload.n), [1, 1], 'only the settled message reaches the sink')
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('flatMap before the gate: a message half allowed is not settled at all', async () => {
|
|
73
|
+
let seen = 0
|
|
74
|
+
const stream = Stream.from(createFakeSource('src', [batch(1, 2)]))
|
|
75
|
+
.flatMap(m => [m.data, m.data])
|
|
76
|
+
.gate(() => ++seen === 1) // the first value of message 1 passes, its second does not
|
|
77
|
+
|
|
78
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.half', timeoutMs: 400 })
|
|
79
|
+
assert.equal(cycles.length, 0, 'nothing settled, nothing committed: the lease brings both back')
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
it('filter before the gate: dropped messages are settled with the prefix', async () => {
|
|
83
|
+
const msgs = batch(1, 2, 3)
|
|
84
|
+
const stream = Stream.from(createFakeSource('src', [msgs]))
|
|
85
|
+
.filter(m => m.data.n === 3)
|
|
86
|
+
.gate(() => true)
|
|
87
|
+
.to({ _queueName: 'sink' })
|
|
88
|
+
|
|
89
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.filter' })
|
|
90
|
+
assert.equal(cycles.length, 1)
|
|
91
|
+
assert.equal(cycles[0].ack.count, 3, 'all three messages are done; one envelope is not one message')
|
|
92
|
+
assert.equal(cycles[0].ack.transactionId, msgs[2].transactionId)
|
|
93
|
+
assert.equal(cycles[0].release_lease, true)
|
|
94
|
+
assert.deepEqual(cycles[0].push_items.map(p => p.payload.n), [3])
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
it('filter that drops the whole batch: the batch is acked, not held forever', async () => {
|
|
98
|
+
const msgs = batch(1, 2)
|
|
99
|
+
const stream = Stream.from(createFakeSource('src', [msgs]))
|
|
100
|
+
.filter(() => false)
|
|
101
|
+
.gate(() => true)
|
|
102
|
+
.to({ _queueName: 'sink' })
|
|
103
|
+
|
|
104
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.allfiltered' })
|
|
105
|
+
assert.equal(cycles.length, 1, 'a cycle is committed even though the gate saw nothing')
|
|
106
|
+
assert.equal(cycles[0].ack.count, 2)
|
|
107
|
+
assert.equal(cycles[0].release_lease, true)
|
|
108
|
+
assert.equal(cycles[0].push_items.length, 0)
|
|
109
|
+
assert.equal(cycles[0].state_ops.length, 0)
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
it('filter, then a deny: the count stops before the denied message', async () => {
|
|
113
|
+
const msgs = batch(1, 2, 3, 4)
|
|
114
|
+
const stream = Stream.from(createFakeSource('src', [msgs]))
|
|
115
|
+
.filter(m => m.data.n !== 2) // message 2 produces nothing
|
|
116
|
+
.gate(v => v.n !== 3) // message 3 is denied
|
|
117
|
+
|
|
118
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.filterdeny' })
|
|
119
|
+
assert.equal(cycles.length, 1)
|
|
120
|
+
assert.equal(cycles[0].ack.count, 2, 'messages 1 and 2 settled; the gate allowed one envelope')
|
|
121
|
+
assert.equal(cycles[0].ack.transactionId, msgs[1].transactionId)
|
|
122
|
+
assert.equal(cycles[0].release_lease, false)
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
it('a denied first message still commits nothing', async () => {
|
|
126
|
+
const stream = Stream.from(createFakeSource('src', [batch(1, 2)]))
|
|
127
|
+
.gate(() => false)
|
|
128
|
+
const { cycles, handle } = await runFor(server, stream, { queryId: 'gate.firstdeny', timeoutMs: 400 })
|
|
129
|
+
assert.equal(cycles.length, 0)
|
|
130
|
+
assert.equal(handle.metrics().gateDenialsTotal, 2, 'denials are counted in messages')
|
|
131
|
+
})
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
describe('.gate() — a denied message did not happen', () => {
|
|
135
|
+
let server
|
|
136
|
+
before(async () => { server = await createFakeStreamsServer() })
|
|
137
|
+
after(async () => { await server.close() })
|
|
138
|
+
|
|
139
|
+
it('what a denied message did to the state is not persisted', async () => {
|
|
140
|
+
// A counter the gate bumps BEFORE deciding: message 1 is allowed at 1,
|
|
141
|
+
// message 2 bumps it to 2 and is denied. The state that commits is 1.
|
|
142
|
+
const stream = Stream.from(createFakeSource('src', [batch(1, 2)]))
|
|
143
|
+
.gate((v, ctx) => {
|
|
144
|
+
ctx.state.count = (ctx.state.count || 0) + 1
|
|
145
|
+
return v.n === 1
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.rollback' })
|
|
149
|
+
assert.equal(cycles.length, 1)
|
|
150
|
+
assert.equal(cycles[0].ack.count, 1)
|
|
151
|
+
assert.deepEqual(cycles[0].state_ops, [{ type: 'upsert', key: PART, value: { count: 1 } }])
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
it('a key only a denied message touched is not written at all', async () => {
|
|
155
|
+
const stream = Stream.from(createFakeSource('src', [batch(1, 2)]))
|
|
156
|
+
.keyBy(m => `k${m.data.n}`)
|
|
157
|
+
.gate((v, ctx) => {
|
|
158
|
+
ctx.state.seen = true
|
|
159
|
+
return v.n === 1
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.untouched' })
|
|
163
|
+
assert.equal(cycles.length, 1)
|
|
164
|
+
assert.deepEqual(cycles[0].state_ops.map(op => op.key), ['k1'])
|
|
165
|
+
})
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
describe('.gate() — the prefix is the broker\'s, in offset order', () => {
|
|
169
|
+
let server
|
|
170
|
+
before(async () => { server = await createFakeStreamsServer() })
|
|
171
|
+
after(async () => { await server.close() })
|
|
172
|
+
|
|
173
|
+
it('orders by offset when the broker sends one, not by (createdAt, id)', async () => {
|
|
174
|
+
// Two producers pushed in the same millisecond: same createdAt, and the
|
|
175
|
+
// message the broker stored FIRST carries the higher id. Sorting by
|
|
176
|
+
// (createdAt, id) put it second, so a deny on it acked it anyway.
|
|
177
|
+
const createdAt = '2026-10-02T10:00:00.000Z'
|
|
178
|
+
const first = { ...fakeMessage({ partitionId: PART, data: { n: 'first' }, createdAt }), id: 'msg-z', offset: 10 }
|
|
179
|
+
const second = { ...fakeMessage({ partitionId: PART, data: { n: 'second' }, createdAt }), id: 'msg-a', offset: 11 }
|
|
180
|
+
const stream = Stream.from(createFakeSource('src', [[second, first]]))
|
|
181
|
+
.gate(v => v.n === 'first')
|
|
182
|
+
|
|
183
|
+
const { cycles } = await runFor(server, stream, { queryId: 'gate.offsetorder' })
|
|
184
|
+
assert.equal(cycles.length, 1)
|
|
185
|
+
assert.equal(cycles[0].ack.count, 1)
|
|
186
|
+
assert.equal(cycles[0].ack.transactionId, first.transactionId, 'the settled prefix is offset 10, the one the broker acks')
|
|
187
|
+
assert.equal(cycles[0].release_lease, false)
|
|
188
|
+
})
|
|
189
|
+
})
|
package/test-v2/timers.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Timer integration suite (PLAN_KV_TIMERS.md §4, §9.6, §20.2).
|
|
3
3
|
*
|
|
4
|
-
* Repeatable only because run
|
|
5
|
-
*
|
|
4
|
+
* Repeatable only because every run starts on a fresh broker (test/run.sh
|
|
5
|
+
* creates the lane's data volume empty and destroys it with `down -v`): on a
|
|
6
|
+
* broker that kept it, a pending timer left behind by a previous run fires
|
|
6
7
|
* into the queue of the next one and shows up as a phantom message in an
|
|
7
8
|
* unrelated test.
|
|
8
9
|
*
|
|
@@ -21,7 +22,7 @@ export async function timerSchedulesAndFires(client) {
|
|
|
21
22
|
// A per-run marker, so a message left in this queue by an earlier run can
|
|
22
23
|
// never be mistaken for this run's delivery. It is the one place in these
|
|
23
24
|
// suites where per-run uniqueness is right: the assertion is about THIS
|
|
24
|
-
// timer's identity, and
|
|
25
|
+
// timer's identity, and a clean start cannot be relied on to prove identity.
|
|
25
26
|
const run = `fire-${Date.now()}`
|
|
26
27
|
const res = await client.timer(queueName)
|
|
27
28
|
.key('fire-1')
|
|
@@ -34,8 +35,8 @@ export async function timerSchedulesAndFires(client) {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
// deliverAt is "not before", never "exactly at", and the visibility of a
|
|
37
|
-
// fired timer to a consumer is bounded by the
|
|
38
|
-
// delay -- see popUntil
|
|
38
|
+
// fired timer to a consumer is bounded by the broker's fire step, not by the
|
|
39
|
+
// delay -- see popUntil. The assertion is arrival.
|
|
39
40
|
const mine = await popUntil(client, queueName, m => m.data && m.data.run === run)
|
|
40
41
|
|
|
41
42
|
if (mine.length !== 1) {
|
package/test-v2/transaction.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { TEST_CONFIG } from './run.js'
|
|
2
|
+
|
|
1
3
|
export async function transactionBasicPushAck(client) {
|
|
2
4
|
const queueA = await client.queue('test-queue-v2-txn-basic-a').create()
|
|
3
5
|
const queueB = await client.queue('test-queue-v2-txn-basic-b').create()
|
|
@@ -572,3 +574,170 @@ export async function transactionAckWithConsumerGroup(client) {
|
|
|
572
574
|
}
|
|
573
575
|
}
|
|
574
576
|
|
|
577
|
+
|
|
578
|
+
// An ack takes the consumer group the message was popped under (2026-10-02,
|
|
579
|
+
// 2.0.0-beta.6). Without an explicit { consumerGroup } the op was judged in
|
|
580
|
+
// queue mode and the bundle rolled back with rejected_ack; queen.ack() came
|
|
581
|
+
// back "invalid or expired lease" the same way.
|
|
582
|
+
export async function transactionAckTakesGroupFromMessage(client) {
|
|
583
|
+
const source = 'test-queue-v2-txn-group-from-msg'
|
|
584
|
+
const sink = 'test-queue-v2-txn-group-from-msg-out'
|
|
585
|
+
const group = 'test-txn-group-from-msg'
|
|
586
|
+
const qa = await client.queue(source).create()
|
|
587
|
+
const qb = await client.queue(sink).create()
|
|
588
|
+
if (!qa.configured || !qb.configured) {
|
|
589
|
+
return { success: false, message: 'Queues not created' }
|
|
590
|
+
}
|
|
591
|
+
await client.queue(source).push([{ data: { value: 1 } }, { data: { value: 2 } }])
|
|
592
|
+
|
|
593
|
+
const messages = await client.queue(source).group(group).subscriptionMode('all').batch(2).wait(false).pop()
|
|
594
|
+
if (messages.length !== 2) {
|
|
595
|
+
return { success: false, message: `Expected 2 messages, got ${messages.length}` }
|
|
596
|
+
}
|
|
597
|
+
if (messages[0].consumerGroup !== group) {
|
|
598
|
+
return { success: false, message: `pop answered consumerGroup=${messages[0].consumerGroup}, expected ${group}` }
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
let committed
|
|
602
|
+
try {
|
|
603
|
+
committed = await client.transaction()
|
|
604
|
+
.ack(messages[0])
|
|
605
|
+
.queue(sink).push([{ data: { from: messages[0].data.value } }])
|
|
606
|
+
.commit()
|
|
607
|
+
} catch (error) {
|
|
608
|
+
return { success: false, message: `commit threw (${error.reason}): ${error.message}` }
|
|
609
|
+
}
|
|
610
|
+
const direct = await client.ack(messages[1], true)
|
|
611
|
+
const after = await client.queue(source).group(group).batch(2).wait(false).pop()
|
|
612
|
+
const forwarded = await client.queue(sink).batch(1).wait(false).pop()
|
|
613
|
+
|
|
614
|
+
const success = committed.success === true && direct.success === true && after.length === 0 && forwarded.length === 1
|
|
615
|
+
return {
|
|
616
|
+
success,
|
|
617
|
+
message: success
|
|
618
|
+
? 'transaction().ack and queen.ack took the group from the message'
|
|
619
|
+
: `txn=${committed.success} direct=${direct.success} (${direct.error}) after=${after.length} forwarded=${forwarded.length}`
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// ============================================================================
|
|
624
|
+
// A bundle that acks messages of two leases (2026-10-03, 2.0.0-beta.6)
|
|
625
|
+
// ============================================================================
|
|
626
|
+
//
|
|
627
|
+
// A Queen 2 broker fences each ack with the leaseId the operation carries, and
|
|
628
|
+
// lends requiredLeases to an ack without one only when the bundle names a
|
|
629
|
+
// single lease. An ack that left its lease in requiredLeases alone was applied
|
|
630
|
+
// in a bundle of two leases even after its lease expired and another consumer
|
|
631
|
+
// took the message. The builder has no per-pop lease knob, so these pops go
|
|
632
|
+
// through fetch, as in semantics.js.
|
|
633
|
+
|
|
634
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
|
|
635
|
+
const fenceQueue = name => `test-queue-v2-txn-fence-${name}-${Date.now()}`
|
|
636
|
+
|
|
637
|
+
/** One message of a queue-mode pop leased for `leaseSeconds`, or null. */
|
|
638
|
+
async function popLeased(queue, leaseSeconds, tries = 20) {
|
|
639
|
+
const qs = new URLSearchParams({
|
|
640
|
+
batch: '1',
|
|
641
|
+
wait: 'false',
|
|
642
|
+
leaseSeconds: String(leaseSeconds),
|
|
643
|
+
subscriptionMode: 'all'
|
|
644
|
+
})
|
|
645
|
+
// Lazy: run.js imports this module before TEST_CONFIG is initialized.
|
|
646
|
+
const url = `${TEST_CONFIG.baseUrls[0]}/api/v1/pop/queue/${queue}?${qs}`
|
|
647
|
+
for (let i = 0; i < tries; i++) {
|
|
648
|
+
const res = await fetch(url)
|
|
649
|
+
if (res.status !== 204) {
|
|
650
|
+
if (!res.ok) throw new Error(`pop ${queue}: HTTP ${res.status}`)
|
|
651
|
+
const body = await res.json()
|
|
652
|
+
if (body.messages && body.messages.length > 0) return body.messages[0]
|
|
653
|
+
}
|
|
654
|
+
await sleep(150)
|
|
655
|
+
}
|
|
656
|
+
return null
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/** The outcome of a commit, whether the client threw the refusal or returned it. */
|
|
660
|
+
async function commitOutcome(transaction) {
|
|
661
|
+
try {
|
|
662
|
+
const result = await transaction.commit()
|
|
663
|
+
return { committed: result.success === true, error: result.error || null }
|
|
664
|
+
} catch (error) {
|
|
665
|
+
return { committed: false, error: error.message }
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
export async function transactionTwoLeaseBundleExpiredLeaseRefused(client) {
|
|
670
|
+
const queueA = fenceQueue('stale-a')
|
|
671
|
+
const queueB = fenceQueue('stale-b')
|
|
672
|
+
const qa = await client.queue(queueA).create()
|
|
673
|
+
const qb = await client.queue(queueB).create()
|
|
674
|
+
if (!qa.configured || !qb.configured) {
|
|
675
|
+
return { success: false, message: 'Queues not created' }
|
|
676
|
+
}
|
|
677
|
+
await client.queue(queueA).push([{ data: { name: 'a' } }])
|
|
678
|
+
await client.queue(queueB).push([{ data: { name: 'b' } }])
|
|
679
|
+
|
|
680
|
+
const a = await popLeased(queueA, 1)
|
|
681
|
+
const b = await popLeased(queueB, 60)
|
|
682
|
+
if (!a?.leaseId || !b?.leaseId || a.leaseId === b.leaseId) {
|
|
683
|
+
return { success: false, message: `Expected two messages under two leases, got a=${a?.leaseId} b=${b?.leaseId}` }
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// Lease A expires, and another consumer takes the message.
|
|
687
|
+
const taken = await popLeased(queueA, 60, 100)
|
|
688
|
+
if (!taken || taken.transactionId !== a.transactionId || taken.leaseId === a.leaseId) {
|
|
689
|
+
return { success: false, message: 'The message of the expired lease was not delivered again under a new lease' }
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
const stale = await commitOutcome(client.transaction().ack(a).ack(b))
|
|
693
|
+
if (stale.committed) {
|
|
694
|
+
return { success: false, message: 'The bundle committed: the ack under the expired lease was applied unfenced' }
|
|
695
|
+
}
|
|
696
|
+
if (!/rolled back/.test(stale.error || '')) {
|
|
697
|
+
return { success: false, message: `The bundle was refused, but not as a rollback: ${stale.error}` }
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// Nothing of the bundle happened: each holder still settles its own message.
|
|
701
|
+
const takenAck = await commitOutcome(client.transaction().ack(taken))
|
|
702
|
+
const bAck = await commitOutcome(client.transaction().ack(b))
|
|
703
|
+
const leftA = await client.queue(queueA).batch(10).wait(false).pop()
|
|
704
|
+
const leftB = await client.queue(queueB).batch(10).wait(false).pop()
|
|
705
|
+
|
|
706
|
+
const success = takenAck.committed && bAck.committed && leftA.length === 0 && leftB.length === 0
|
|
707
|
+
return {
|
|
708
|
+
success,
|
|
709
|
+
message: success
|
|
710
|
+
? 'A bundle with an ack under an expired lease was refused whole; each holder then settled its own message'
|
|
711
|
+
: `new holder ack=${takenAck.committed} (${takenAck.error}), B ack=${bAck.committed} (${bAck.error}), left A=${leftA.length} B=${leftB.length}`
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
export async function transactionTwoLeaseBundleLiveLeasesCommit(client) {
|
|
716
|
+
const queueA = fenceQueue('live-a')
|
|
717
|
+
const queueB = fenceQueue('live-b')
|
|
718
|
+
const qa = await client.queue(queueA).create()
|
|
719
|
+
const qb = await client.queue(queueB).create()
|
|
720
|
+
if (!qa.configured || !qb.configured) {
|
|
721
|
+
return { success: false, message: 'Queues not created' }
|
|
722
|
+
}
|
|
723
|
+
await client.queue(queueA).push([{ data: { name: 'a' } }])
|
|
724
|
+
await client.queue(queueB).push([{ data: { name: 'b' } }])
|
|
725
|
+
|
|
726
|
+
const a = await popLeased(queueA, 60)
|
|
727
|
+
const b = await popLeased(queueB, 60)
|
|
728
|
+
if (!a?.leaseId || !b?.leaseId || a.leaseId === b.leaseId) {
|
|
729
|
+
return { success: false, message: `Expected two messages under two leases, got a=${a?.leaseId} b=${b?.leaseId}` }
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
const outcome = await commitOutcome(client.transaction().ack(a).ack(b))
|
|
733
|
+
const leftA = await client.queue(queueA).batch(10).wait(false).pop()
|
|
734
|
+
const leftB = await client.queue(queueB).batch(10).wait(false).pop()
|
|
735
|
+
|
|
736
|
+
const success = outcome.committed && leftA.length === 0 && leftB.length === 0
|
|
737
|
+
return {
|
|
738
|
+
success,
|
|
739
|
+
message: success
|
|
740
|
+
? 'A bundle of two live leases completed both messages'
|
|
741
|
+
: `commit=${outcome.committed} (${outcome.error}), left A=${leftA.length} B=${leftB.length}`
|
|
742
|
+
}
|
|
743
|
+
}
|