queen-mq 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -0
- package/client-v2/Queen.js +54 -0
- package/client-v2/README.md +32 -0
- package/client-v2/builders/QueueBuilder.js +8 -16
- package/client-v2/builders/TimerBuilder.js +262 -0
- package/client-v2/builders/TransactionBuilder.js +185 -8
- package/client-v2/kv/Kv.js +432 -0
- package/client-v2/kv/expiry.js +148 -0
- package/package.json +11 -3
- package/test-v2/_kvtimers.js +71 -0
- package/test-v2/ackwindow.js +10 -3
- package/test-v2/consume.js +0 -2
- package/test-v2/docs.js +204 -0
- package/test-v2/http-unit/retry429.test.js +6 -1
- package/test-v2/kv-unit/_planServer.js +65 -0
- package/test-v2/kv-unit/kvWire.test.js +377 -0
- package/test-v2/kv-unit/timerWire.test.js +177 -0
- package/test-v2/kv-unit/txnWire.test.js +222 -0
- package/test-v2/kv.js +273 -0
- package/test-v2/pop.js +0 -2
- package/test-v2/push.js +0 -4
- package/test-v2/run.js +34 -4
- package/test-v2/semantics.js +16 -7
- package/test-v2/stream/_helpers.js +7 -0
- package/test-v2/stream/combined.js +4 -3
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/eventTime.js +8 -5
- package/test-v2/stream/operators.js +5 -5
- package/test-v2/stream/recovery.js +4 -1
- package/test-v2/stream/session.js +3 -3
- package/test-v2/stream/sliding.js +1 -1
- package/test-v2/stream/throughput.js +3 -2
- package/test-v2/stream/tumbling.js +6 -6
- package/test-v2/timers.js +209 -0
- package/test-v2/transaction.js +4 -3
|
@@ -84,6 +84,11 @@ export async function pushSpread(client, queueName, items) {
|
|
|
84
84
|
* immediately, return the concatenated list of messages. Stops when a
|
|
85
85
|
* pop returns 0 messages OR cumulative timeout is hit.
|
|
86
86
|
*
|
|
87
|
+
* NOTE on subscription mode: the drain group is created AFTER the stream has
|
|
88
|
+
* already emitted onto the sink, so it must ask for 'all' explicitly — with
|
|
89
|
+
* the broker default of 'new' its cursor would be seeded at the sink's tail
|
|
90
|
+
* and the emits under test would be invisible.
|
|
91
|
+
*
|
|
87
92
|
* NOTE on batch size: we use batch=1 so that each ack triggers
|
|
88
93
|
* `acked_count >= batch_size` in queen.partition_consumers and the lease
|
|
89
94
|
* is released between pops. With a larger batch the lease would stay
|
|
@@ -99,6 +104,7 @@ export async function drainSink(client, queueName, { timeoutMs = 5000, group } =
|
|
|
99
104
|
const popped = await client
|
|
100
105
|
.queue(queueName)
|
|
101
106
|
.group(cg)
|
|
107
|
+
.subscriptionMode('all')
|
|
102
108
|
.batch(1)
|
|
103
109
|
.wait(false)
|
|
104
110
|
.pop()
|
|
@@ -127,6 +133,7 @@ export async function drainUntil(client, queueName, { until, timeoutMs = 15_000,
|
|
|
127
133
|
const popped = await client
|
|
128
134
|
.queue(queueName)
|
|
129
135
|
.group(cg)
|
|
136
|
+
.subscriptionMode('all')
|
|
130
137
|
.batch(1)
|
|
131
138
|
.wait(true)
|
|
132
139
|
.timeoutMillis(500)
|
|
@@ -58,7 +58,8 @@ export async function combinedFullPipeline(client) {
|
|
|
58
58
|
url: STREAMS_URL,
|
|
59
59
|
batchSize: 100,
|
|
60
60
|
maxPartitions: 4,
|
|
61
|
-
reset: true
|
|
61
|
+
reset: true,
|
|
62
|
+
subscriptionMode: 'all'
|
|
62
63
|
})
|
|
63
64
|
|
|
64
65
|
// Wait long enough for windowEnd to pass + idle flush to close all 3
|
|
@@ -117,8 +118,8 @@ export async function combinedTwoConcurrentStreams(client) {
|
|
|
117
118
|
.to(client.queue(sinkB))
|
|
118
119
|
|
|
119
120
|
const [handleA, handleB] = await Promise.all([
|
|
120
|
-
streamA.run({ queryId: qA, url: STREAMS_URL, batchSize: 50, reset: true }),
|
|
121
|
-
streamB.run({ queryId: qB, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
121
|
+
streamA.run({ queryId: qA, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' }),
|
|
122
|
+
streamB.run({ queryId: qB, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
122
123
|
])
|
|
123
124
|
|
|
124
125
|
// Wait until the FULL totals are present on each sink. Each stream sees
|
package/test-v2/stream/cron.js
CHANGED
|
@@ -81,7 +81,7 @@ export async function cronEverySecondLive(client) {
|
|
|
81
81
|
.windowCron({ every: 'second', idleFlushMs: 500 })
|
|
82
82
|
.aggregate({ count: () => 1 })
|
|
83
83
|
.to(client.queue(sink))
|
|
84
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
84
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
85
85
|
|
|
86
86
|
await pushPromise
|
|
87
87
|
const emits = await drainUntil(client, sink, {
|
|
@@ -47,7 +47,7 @@ export async function eventTimeBucketsByExtractor(client) {
|
|
|
47
47
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
48
48
|
.map((agg, ctx) => ({ windowKey: ctx.windowKey, ...agg }))
|
|
49
49
|
.to(client.queue(sink))
|
|
50
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
50
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
51
51
|
|
|
52
52
|
// The 10:00 window closes when the 10:01:05 event advances the
|
|
53
53
|
// watermark past its end. The 10:01 window stays open until either
|
|
@@ -85,7 +85,10 @@ export async function eventTimeLateDropExcludesEvent(client) {
|
|
|
85
85
|
await client.queue(src).create()
|
|
86
86
|
await client.queue(sink).create()
|
|
87
87
|
|
|
88
|
-
// Run 1: advance the watermark by emitting a fresh event.
|
|
88
|
+
// Run 1: advance the watermark by emitting a fresh event. The push happens
|
|
89
|
+
// before the runner's consumer group exists, so run 1 asks for 'all' — with
|
|
90
|
+
// the broker default of 'new' the watermark never moves and the "late"
|
|
91
|
+
// event in run 2 is not late at all.
|
|
89
92
|
await client.queue(src).partition('p').push([
|
|
90
93
|
{ data: { v: 100, eventTs: '2026-01-01T11:00:00Z' } }
|
|
91
94
|
])
|
|
@@ -103,7 +106,7 @@ export async function eventTimeLateDropExcludesEvent(client) {
|
|
|
103
106
|
.to(client.queue(sink))
|
|
104
107
|
.run(handler)
|
|
105
108
|
|
|
106
|
-
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
109
|
+
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
107
110
|
await sleep(2000)
|
|
108
111
|
await handle1.stop()
|
|
109
112
|
|
|
@@ -150,7 +153,7 @@ export async function eventTimeAllowedLatenessIncluded(client) {
|
|
|
150
153
|
.to(client.queue(sink))
|
|
151
154
|
.run(handler)
|
|
152
155
|
|
|
153
|
-
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
156
|
+
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
154
157
|
await sleep(2000)
|
|
155
158
|
await handle1.stop()
|
|
156
159
|
|
|
@@ -202,7 +205,7 @@ export async function eventTimeLateIncludePolicy(client) {
|
|
|
202
205
|
.to(client.queue(sink))
|
|
203
206
|
.run(handler)
|
|
204
207
|
|
|
205
|
-
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
208
|
+
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
206
209
|
await sleep(2000)
|
|
207
210
|
await handle1.stop()
|
|
208
211
|
|
|
@@ -35,7 +35,7 @@ export async function streamMapFilterSink(client) {
|
|
|
35
35
|
.filter(m => m.data.type !== 'heartbeat')
|
|
36
36
|
.map(m => ({ ...m.data, marked: true }))
|
|
37
37
|
.to(client.queue(sink))
|
|
38
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true })
|
|
38
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true, subscriptionMode: 'all' })
|
|
39
39
|
|
|
40
40
|
// 20 messages should land on the sink (10 orders + 10 logs).
|
|
41
41
|
const drained = await drainUntil(client, sink, {
|
|
@@ -81,7 +81,7 @@ export async function streamFlatMapFanout(client) {
|
|
|
81
81
|
{ id: m.data.id, copy: 4 }
|
|
82
82
|
])
|
|
83
83
|
.to(client.queue(sink))
|
|
84
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true })
|
|
84
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true, subscriptionMode: 'all' })
|
|
85
85
|
|
|
86
86
|
const drained = await drainUntil(client, sink, {
|
|
87
87
|
until: out => out.length >= 20,
|
|
@@ -114,7 +114,7 @@ export async function streamForeachCapturesAll(client) {
|
|
|
114
114
|
const handle = await Stream
|
|
115
115
|
.from(client.queue(src))
|
|
116
116
|
.foreach((value) => { captured.push(value) })
|
|
117
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
117
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
118
118
|
|
|
119
119
|
// Wait until at-least-once semantics catch up to 12 messages.
|
|
120
120
|
const start = Date.now()
|
|
@@ -152,7 +152,7 @@ export async function streamKeyByDoesNotCrash(client) {
|
|
|
152
152
|
.keyBy(m => m.data.region)
|
|
153
153
|
.map(m => m.data)
|
|
154
154
|
.to(client.queue(sink))
|
|
155
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true })
|
|
155
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true, subscriptionMode: 'all' })
|
|
156
156
|
|
|
157
157
|
const drained = await drainUntil(client, sink, {
|
|
158
158
|
until: out => out.length >= 6,
|
|
@@ -192,7 +192,7 @@ export async function streamAsyncMap(client) {
|
|
|
192
192
|
return { ...m.data, asyncEnriched: true }
|
|
193
193
|
})
|
|
194
194
|
.to(client.queue(sink))
|
|
195
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true })
|
|
195
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 100, reset: true, subscriptionMode: 'all' })
|
|
196
196
|
|
|
197
197
|
const drained = await drainUntil(client, sink, {
|
|
198
198
|
until: out => out.length >= 5,
|
|
@@ -113,7 +113,10 @@ export async function recoveryMidStreamResume(client) {
|
|
|
113
113
|
for (let i = 0; i < 5; i++) {
|
|
114
114
|
await client.queue(src).partition('p').push([{ data: { v: 1 } }])
|
|
115
115
|
}
|
|
116
|
-
|
|
116
|
+
// Run 1 creates the source consumer group AFTER the pushes, so it has to
|
|
117
|
+
// ask for the backlog. Run 2 reuses the same group and resumes from its
|
|
118
|
+
// stored cursor — the mode is only consulted on a group's first contact.
|
|
119
|
+
const handle1 = await buildStream({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
117
120
|
// Wait for at least one closed-window emit.
|
|
118
121
|
await drainUntil(client, sink, { until: out => out.length >= 1, timeoutMs: 5000 })
|
|
119
122
|
await handle1.stop()
|
|
@@ -38,7 +38,7 @@ export async function sessionGapClosesAndStartsNew(client) {
|
|
|
38
38
|
.windowSession({ gap: 2, idleFlushMs: 500 })
|
|
39
39
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
40
40
|
.to(client.queue(sink))
|
|
41
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
41
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
42
42
|
|
|
43
43
|
await pushPromise
|
|
44
44
|
// Wait for idle flush to close the second (still-open) session.
|
|
@@ -80,7 +80,7 @@ export async function sessionIdleFlushClosesQuietSession(client) {
|
|
|
80
80
|
.windowSession({ gap: 1, idleFlushMs: 500 })
|
|
81
81
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
82
82
|
.to(client.queue(sink))
|
|
83
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
83
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
84
84
|
|
|
85
85
|
// Wait for gap + flush to elapse.
|
|
86
86
|
const emits = await drainUntil(client, sink, {
|
|
@@ -125,7 +125,7 @@ export async function sessionMultipleKeysIndependent(client) {
|
|
|
125
125
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
126
126
|
.map((v, ctx) => ({ user: ctx.key, ...v }))
|
|
127
127
|
.to(client.queue(sink))
|
|
128
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
128
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
129
129
|
|
|
130
130
|
const emits = await drainUntil(client, sink, {
|
|
131
131
|
until: out => out.length >= 2, timeoutMs: 8000
|
|
@@ -55,7 +55,7 @@ export async function slidingEventsAppearInOverlappingWindows(client) {
|
|
|
55
55
|
.windowSliding({ size: 4, slide: 2, idleFlushMs: 1000 })
|
|
56
56
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
57
57
|
.to(client.queue(sink))
|
|
58
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
58
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
59
59
|
|
|
60
60
|
await itemPushPromise
|
|
61
61
|
// Allow idle flush to close trailing windows.
|
|
@@ -58,7 +58,8 @@ export async function throughput5kAcross20Partitions(client) {
|
|
|
58
58
|
url: STREAMS_URL,
|
|
59
59
|
batchSize: 200,
|
|
60
60
|
maxPartitions: 8,
|
|
61
|
-
reset: true
|
|
61
|
+
reset: true,
|
|
62
|
+
subscriptionMode: 'all'
|
|
62
63
|
})
|
|
63
64
|
|
|
64
65
|
// Wait until we have at least 20 partition emits — possibly more if
|
|
@@ -130,7 +131,7 @@ export async function throughputManyWindowsSinglePartition(client) {
|
|
|
130
131
|
.windowTumbling({ seconds: 1, idleFlushMs: 600 })
|
|
131
132
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
132
133
|
.to(client.queue(sink))
|
|
133
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
134
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
134
135
|
|
|
135
136
|
// Wait for all 200 messages to be reflected in emits.
|
|
136
137
|
const emits = await drainUntil(client, sink, {
|
|
@@ -39,7 +39,7 @@ export async function tumblingBasicWindowSum(client) {
|
|
|
39
39
|
.windowTumbling({ seconds: 2, idleFlushMs: 800 })
|
|
40
40
|
.aggregate({ count: () => 1, sum: m => m.amount })
|
|
41
41
|
.to(client.queue(sink))
|
|
42
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
42
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
43
43
|
|
|
44
44
|
await itemPushPromise
|
|
45
45
|
// Wait until the total count across all emits = 8 (idle flush must
|
|
@@ -94,7 +94,7 @@ export async function tumblingPerPartitionIsolation(client) {
|
|
|
94
94
|
.aggregate({ count: () => 1, sum: m => m.amount })
|
|
95
95
|
.map((agg, ctx) => ({ partition: ctx.partition, ...agg }))
|
|
96
96
|
.to(client.queue(sink))
|
|
97
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, maxPartitions: 4, reset: true })
|
|
97
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, maxPartitions: 4, reset: true, subscriptionMode: 'all' })
|
|
98
98
|
|
|
99
99
|
await itemPushPromise
|
|
100
100
|
// Wait until total count across all emits = 18 (3 customers × 6 events).
|
|
@@ -168,7 +168,7 @@ export async function tumblingAggregateAllStats(client) {
|
|
|
168
168
|
avg: m => m.v
|
|
169
169
|
})
|
|
170
170
|
.to(client.queue(sink))
|
|
171
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
171
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
172
172
|
|
|
173
173
|
// Wait for idle flush to fire and emit the closed window (3s + grace + slack).
|
|
174
174
|
const emits = await drainUntil(client, sink, {
|
|
@@ -216,7 +216,7 @@ export async function tumblingGracePeriodDelaysClose(client) {
|
|
|
216
216
|
.windowTumbling({ seconds: 2, gracePeriod: 4, idleFlushMs: 500 })
|
|
217
217
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
218
218
|
.to(client.queue(sink))
|
|
219
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
219
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
220
220
|
|
|
221
221
|
// After ~3s the window's end is past, but grace=4s means it should NOT
|
|
222
222
|
// be closed yet (close at ~ windowStart + 2 + 4 = 6s after first event).
|
|
@@ -265,7 +265,7 @@ export async function tumblingIdleFlushClosesQuietPartitions(client) {
|
|
|
265
265
|
.windowTumbling({ seconds: 1, idleFlushMs: 700 })
|
|
266
266
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
267
267
|
.to(client.queue(sink))
|
|
268
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
268
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
269
269
|
|
|
270
270
|
// Wait for idle flush to fire.
|
|
271
271
|
const emits = await drainUntil(client, sink, {
|
|
@@ -300,7 +300,7 @@ export async function tumblingForeachCtxHasWindowAndPartition(client) {
|
|
|
300
300
|
.windowTumbling({ seconds: 1, idleFlushMs: 700 })
|
|
301
301
|
.aggregate({ count: () => 1, sum: m => m.v })
|
|
302
302
|
.foreach((value, ctx) => { captured.push({ value, ctx }) })
|
|
303
|
-
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true })
|
|
303
|
+
.run({ queryId, url: STREAMS_URL, batchSize: 50, reset: true, subscriptionMode: 'all' })
|
|
304
304
|
|
|
305
305
|
const start = Date.now()
|
|
306
306
|
while (captured.length < 1 && Date.now() - start < 8000) await sleep(100)
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timer integration suite (PLAN_KV_TIMERS.md §4, §9.6, §20.2).
|
|
3
|
+
*
|
|
4
|
+
* Repeatable only because run.js purges `queen.log_timers` for the `test-%`
|
|
5
|
+
* queues before the run: a pending timer left behind by a previous run fires
|
|
6
|
+
* into the queue of the next one and shows up as a phantom message in an
|
|
7
|
+
* unrelated test.
|
|
8
|
+
*
|
|
9
|
+
* Every delay here is either short (it is meant to fire) or minutes long (it is
|
|
10
|
+
* meant not to). Nothing in this file schedules anything that outlives the
|
|
11
|
+
* suite.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { TIMER_QUEUE, popUntil } from './_kvtimers.js'
|
|
15
|
+
|
|
16
|
+
export async function timerSchedulesAndFires(client) {
|
|
17
|
+
const queueName = `${TIMER_QUEUE}-fire`
|
|
18
|
+
const queue = await client.queue(queueName).create()
|
|
19
|
+
if (!queue.configured) return { success: false, message: 'Queue not created' }
|
|
20
|
+
|
|
21
|
+
// A per-run marker, so a message left in this queue by an earlier run can
|
|
22
|
+
// never be mistaken for this run's delivery. It is the one place in these
|
|
23
|
+
// suites where per-run uniqueness is right: the assertion is about THIS
|
|
24
|
+
// timer's identity, and the purge cannot be relied on to prove identity.
|
|
25
|
+
const run = `fire-${Date.now()}`
|
|
26
|
+
const res = await client.timer(queueName)
|
|
27
|
+
.key('fire-1')
|
|
28
|
+
.delay('500ms')
|
|
29
|
+
.payload({ kind: 'reminder', run })
|
|
30
|
+
.schedule()
|
|
31
|
+
|
|
32
|
+
if (res.status !== 'scheduled' || !res.messageId || !res.deliverAt) {
|
|
33
|
+
return { success: false, message: `schedule answered ${JSON.stringify(res)}` }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// deliverAt is "not before", never "exactly at", and the visibility of a
|
|
37
|
+
// fired timer to a consumer is bounded by the hot-list reseed, not by the
|
|
38
|
+
// delay -- see popUntil, which measured it. The assertion is arrival.
|
|
39
|
+
const mine = await popUntil(client, queueName, m => m.data && m.data.run === run)
|
|
40
|
+
|
|
41
|
+
if (mine.length !== 1) {
|
|
42
|
+
return { success: false, message: 'the scheduled timer was never delivered into the queue' }
|
|
43
|
+
}
|
|
44
|
+
// The two identifiers promised at schedule time are the two the delivered
|
|
45
|
+
// frame carries -- which is what lets a client correlate without a second API.
|
|
46
|
+
if (mine[0].transactionId !== res.txn) {
|
|
47
|
+
return { success: false, message: `delivered txn ${mine[0].transactionId} != promised ${res.txn}` }
|
|
48
|
+
}
|
|
49
|
+
if (mine[0].id !== res.messageId) {
|
|
50
|
+
return { success: false, message: `delivered messageId ${mine[0].id} != promised ${res.messageId}` }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
await client.ack(mine[0])
|
|
54
|
+
return { success: true, message: 'a scheduled timer was delivered carrying the txn and messageId promised at schedule time' }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export async function timerCancelBeforeFire(client) {
|
|
58
|
+
const queueName = `${TIMER_QUEUE}-cancel`
|
|
59
|
+
const queue = await client.queue(queueName).create()
|
|
60
|
+
if (!queue.configured) return { success: false, message: 'Queue not created' }
|
|
61
|
+
|
|
62
|
+
await client.timer(queueName).key('cancel-1').delay('30m').payload({ n: 1 }).schedule()
|
|
63
|
+
|
|
64
|
+
const pending = await client.timer(queueName).key('cancel-1').peek()
|
|
65
|
+
if (pending.found !== true) {
|
|
66
|
+
return { success: false, message: `peek did not find the pending timer: ${JSON.stringify(pending)}` }
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const cancelled = await client.timer(queueName).key('cancel-1').cancel()
|
|
70
|
+
const gone = await client.timer(queueName).key('cancel-1').peek()
|
|
71
|
+
|
|
72
|
+
const ok = cancelled.ok === true && cancelled.status === 'cancelled' && gone.found === false
|
|
73
|
+
return {
|
|
74
|
+
success: ok,
|
|
75
|
+
message: ok
|
|
76
|
+
? 'a pending timer was peeked, cancelled and is gone'
|
|
77
|
+
: `cancel=${JSON.stringify(cancelled)} peek=${JSON.stringify(gone)}`
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* §4.4, the point where a user gets hurt: there is no tombstone. A cancel on
|
|
83
|
+
* something that is not pending answers `absent` with **ok:false**, and echoes
|
|
84
|
+
* the txn so the log -- the only authority on "did it already go out?" -- can
|
|
85
|
+
* be consulted without a second API.
|
|
86
|
+
*/
|
|
87
|
+
export async function timerCancelAbsentCarriesOkFalseAndTheTxn(client) {
|
|
88
|
+
const queueName = `${TIMER_QUEUE}-absent`
|
|
89
|
+
const expected = 'txn-that-may-already-have-been-delivered'
|
|
90
|
+
const res = await client.timer(queueName).key('never-scheduled').txn(expected).cancel()
|
|
91
|
+
|
|
92
|
+
const ok = res.ok === false && res.status === 'absent' && res.txn === expected
|
|
93
|
+
return {
|
|
94
|
+
success: ok,
|
|
95
|
+
message: ok
|
|
96
|
+
? 'absent answers ok:false and hands back the txn to check against the log'
|
|
97
|
+
: JSON.stringify(res)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* §20.2: a reschedule is an UPSERT that mints a new txn, because a rescheduled
|
|
103
|
+
* timer is a new message. The corollary belongs in the docs, not in an
|
|
104
|
+
* assertion: there is no dedup net on the fire, so rescheduling one that has
|
|
105
|
+
* already gone out produces a second message.
|
|
106
|
+
*/
|
|
107
|
+
export async function timerRescheduleOverwritesTheTxn(client) {
|
|
108
|
+
const queueName = `${TIMER_QUEUE}-resched`
|
|
109
|
+
const queue = await client.queue(queueName).create()
|
|
110
|
+
if (!queue.configured) return { success: false, message: 'Queue not created' }
|
|
111
|
+
|
|
112
|
+
const first = await client.timer(queueName).key('resched-1').delay('30m').payload({ v: 1 }).schedule()
|
|
113
|
+
const second = await client.timer(queueName).key('resched-1').delay('45m').payload({ v: 2 }).schedule()
|
|
114
|
+
const pending = await client.timer(queueName).key('resched-1').peek()
|
|
115
|
+
|
|
116
|
+
const ok = first.status === 'scheduled'
|
|
117
|
+
&& second.status === 'rescheduled'
|
|
118
|
+
&& second.txn !== first.txn
|
|
119
|
+
&& pending.found === true
|
|
120
|
+
&& pending.txn === second.txn
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
success: ok,
|
|
124
|
+
message: ok
|
|
125
|
+
? 'the second schedule upserted the same key, reported "rescheduled" and replaced the txn'
|
|
126
|
+
: `first=${JSON.stringify(first)} second=${JSON.stringify(second)} peek=${JSON.stringify(pending)}`
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export async function timerListIsKeysetOverOneQueue(client) {
|
|
131
|
+
const queueName = `${TIMER_QUEUE}-list`
|
|
132
|
+
const queue = await client.queue(queueName).create()
|
|
133
|
+
if (!queue.configured) return { success: false, message: 'Queue not created' }
|
|
134
|
+
|
|
135
|
+
for (const k of ['list-a', 'list-b', 'list-c']) {
|
|
136
|
+
await client.timer(queueName).key(k).delay('30m').payload({ k }).schedule()
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const page1 = await client.timer(queueName).list({ limit: 2 })
|
|
140
|
+
if (page1.rows.length !== 2 || page1.truncated !== true || !page1.nextAfter) {
|
|
141
|
+
return { success: false, message: `first page: ${JSON.stringify(page1)}` }
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const page2 = await client.timer(queueName).list({ after: page1.nextAfter, limit: 2 })
|
|
145
|
+
const keys = [...page1.rows, ...page2.rows].map(r => r.timerKey)
|
|
146
|
+
|
|
147
|
+
const ok = keys.length === 3 && keys.join(',') === 'list-a,list-b,list-c'
|
|
148
|
+
return {
|
|
149
|
+
success: ok,
|
|
150
|
+
message: ok
|
|
151
|
+
? 'list pages over one queue on an exclusive keyset cursor'
|
|
152
|
+
: `saw ${JSON.stringify(keys)}`
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* The timer as a rider of a transaction: it commits with the push, in the same
|
|
158
|
+
* transaction, or not at all. This is the shape a saga uses -- schedule the
|
|
159
|
+
* compensation together with the work that may need compensating.
|
|
160
|
+
*/
|
|
161
|
+
export async function timerRidesTheTransaction(client) {
|
|
162
|
+
const queueName = `${TIMER_QUEUE}-txn`
|
|
163
|
+
const queue = await client.queue(queueName).create()
|
|
164
|
+
if (!queue.configured) return { success: false, message: 'Queue not created' }
|
|
165
|
+
|
|
166
|
+
const res = await client.transaction()
|
|
167
|
+
.queue(queueName)
|
|
168
|
+
.push([{ data: { step: 'reserved' } }])
|
|
169
|
+
.timer(queueName).key('compensate-1').delay('30m').payload({ step: 'compensate' }).schedule()
|
|
170
|
+
.commit()
|
|
171
|
+
|
|
172
|
+
if (res.success !== true) {
|
|
173
|
+
return { success: false, message: `the bundle should commit: ${JSON.stringify(res)}` }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const pending = await client.timer(queueName).key('compensate-1').peek()
|
|
177
|
+
const cancelled = await client.timer(queueName).key('compensate-1').cancel()
|
|
178
|
+
|
|
179
|
+
const ok = pending.found === true && cancelled.ok === true
|
|
180
|
+
return {
|
|
181
|
+
success: ok,
|
|
182
|
+
message: ok
|
|
183
|
+
? 'a timer scheduled inside a transaction is pending after the commit'
|
|
184
|
+
: `peek=${JSON.stringify(pending)} cancel=${JSON.stringify(cancelled)}`
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* A delay in the past is LEGAL and fires on the first cycle (§4.2). It is also
|
|
190
|
+
* the cheapest end-to-end proof that the sweeper is running at all.
|
|
191
|
+
*/
|
|
192
|
+
export async function timerWithAPastDelayFiresImmediately(client) {
|
|
193
|
+
const queueName = `${TIMER_QUEUE}-past`
|
|
194
|
+
const queue = await client.queue(queueName).create()
|
|
195
|
+
if (!queue.configured) return { success: false, message: 'Queue not created' }
|
|
196
|
+
|
|
197
|
+
const run = `past-${Date.now()}`
|
|
198
|
+
await client.timer(queueName).key('past-1').delayMs(-5000).payload({ late: true, run }).schedule()
|
|
199
|
+
|
|
200
|
+
const mine = await popUntil(client, queueName, m => m.data && m.data.run === run)
|
|
201
|
+
if (mine.length === 1) await client.ack(mine[0])
|
|
202
|
+
|
|
203
|
+
return {
|
|
204
|
+
success: mine.length === 1,
|
|
205
|
+
message: mine.length === 1
|
|
206
|
+
? 'a delayMs in the past fired on the first sweep cycle'
|
|
207
|
+
: 'the late timer was never delivered'
|
|
208
|
+
}
|
|
209
|
+
}
|
package/test-v2/transaction.js
CHANGED
|
@@ -14,14 +14,12 @@ export async function transactionBasicPushAck(client) {
|
|
|
14
14
|
return { success: false, message: 'No message to consume' }
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
// docs:start(js-transaction)
|
|
18
17
|
await client
|
|
19
18
|
.transaction()
|
|
20
19
|
.queue('test-queue-v2-txn-basic-b')
|
|
21
20
|
.push([{ data: { value: messages[0].data.value + 1 } }])
|
|
22
21
|
.ack(messages[0])
|
|
23
22
|
.commit()
|
|
24
|
-
// docs:end
|
|
25
23
|
|
|
26
24
|
// Verify message moved to B
|
|
27
25
|
const resultB = await client.queue('test-queue-v2-txn-basic-b').batch(1).wait(false).pop()
|
|
@@ -520,9 +518,11 @@ export async function transactionAckWithConsumerGroup(client) {
|
|
|
520
518
|
{ data: { value: 2 } }
|
|
521
519
|
])
|
|
522
520
|
|
|
523
|
-
// Pop with consumer group
|
|
521
|
+
// Pop with consumer group. Both groups here are created after the push,
|
|
522
|
+
// so they must ask for 'all' to see the backlog under test.
|
|
524
523
|
const messages = await client.queue('test-queue-v2-txn-cg-a')
|
|
525
524
|
.group(consumerGroup)
|
|
525
|
+
.subscriptionMode('all')
|
|
526
526
|
.batch(2)
|
|
527
527
|
.wait(false)
|
|
528
528
|
.pop()
|
|
@@ -554,6 +554,7 @@ export async function transactionAckWithConsumerGroup(client) {
|
|
|
554
554
|
// Verify: Using a different consumer group should still see the messages
|
|
555
555
|
const messagesOtherGroup = await client.queue('test-queue-v2-txn-cg-a')
|
|
556
556
|
.group('other-consumer-group')
|
|
557
|
+
.subscriptionMode('all')
|
|
557
558
|
.batch(2)
|
|
558
559
|
.wait(false)
|
|
559
560
|
.pop()
|