queen-mq 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -18
- package/client-v2/Queen.js +45 -13
- package/client-v2/README.md +26 -11
- package/client-v2/admin/Admin.js +0 -47
- package/client-v2/builders/QueueBuilder.js +33 -10
- package/client-v2/builders/TimerBuilder.js +2 -2
- package/client-v2/builders/TransactionBuilder.js +35 -9
- package/client-v2/consumer/ConsumerManager.js +67 -46
- package/client-v2/ephemeral/Ephemeral.js +7 -9
- package/client-v2/kv/Kv.js +4 -4
- package/client-v2/kv/expiry.js +1 -1
- package/client-v2/streams/Stream.js +8 -3
- package/client-v2/streams/helpers/rateLimiter.js +4 -4
- package/client-v2/streams/operators/GateOperator.js +9 -2
- package/client-v2/streams/operators/ReduceOperator.js +2 -2
- package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
- package/client-v2/streams/runtime/Runner.js +111 -54
- package/client-v2/streams/runtime/cycle.js +4 -4
- package/client-v2/streams/runtime/register.js +1 -1
- package/client-v2/utils/conflation.js +0 -6
- package/client-v2/utils/consumerGroup.js +54 -0
- package/package.json +5 -8
- package/test-v2/_kvtimers.js +12 -13
- package/test-v2/ackwindow.js +12 -192
- package/test-v2/bootstrap.js +2 -2
- package/test-v2/conflation-unit/conflationWire.test.js +0 -12
- package/test-v2/consume.js +41 -1
- package/test-v2/consumer-unit/handlerError.test.js +161 -0
- package/test-v2/docs.js +5 -4
- package/test-v2/http-unit/pushStatus.test.js +142 -0
- package/test-v2/http-unit/renew.test.js +96 -0
- package/test-v2/kv-unit/timerWire.test.js +1 -1
- package/test-v2/kv-unit/txnWire.test.js +101 -2
- package/test-v2/kv.js +6 -5
- package/test-v2/pop.js +28 -1
- package/test-v2/run.js +50 -114
- package/test-v2/runner-unit/fatalExit.test.js +64 -0
- package/test-v2/semantics.js +20 -34
- package/test-v2/stream/_helpers.js +10 -19
- package/test-v2/stream/cron.js +1 -1
- package/test-v2/stream/gate.js +54 -0
- package/test-v2/stream/index.js +2 -0
- package/test-v2/stream/tumbling.js +3 -3
- package/test-v2/streams-unit/ack.test.js +61 -0
- package/test-v2/streams-unit/cycle.test.js +1 -1
- package/test-v2/streams-unit/e2e.test.js +6 -10
- package/test-v2/streams-unit/gate.test.js +189 -0
- package/test-v2/timers.js +6 -5
- package/test-v2/transaction.js +169 -0
- package/test-v2/watermark.js +38 -176
- package/test-v2/maintenance.js +0 -277
package/test-v2/timers.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Timer integration suite (PLAN_KV_TIMERS.md §4, §9.6, §20.2).
|
|
3
3
|
*
|
|
4
|
-
* Repeatable only because run
|
|
5
|
-
*
|
|
4
|
+
* Repeatable only because every run starts on a fresh broker (test/run.sh
|
|
5
|
+
* creates the lane's data volume empty and destroys it with `down -v`): on a
|
|
6
|
+
* broker that kept it, a pending timer left behind by a previous run fires
|
|
6
7
|
* into the queue of the next one and shows up as a phantom message in an
|
|
7
8
|
* unrelated test.
|
|
8
9
|
*
|
|
@@ -21,7 +22,7 @@ export async function timerSchedulesAndFires(client) {
|
|
|
21
22
|
// A per-run marker, so a message left in this queue by an earlier run can
|
|
22
23
|
// never be mistaken for this run's delivery. It is the one place in these
|
|
23
24
|
// suites where per-run uniqueness is right: the assertion is about THIS
|
|
24
|
-
// timer's identity, and
|
|
25
|
+
// timer's identity, and a clean start cannot be relied on to prove identity.
|
|
25
26
|
const run = `fire-${Date.now()}`
|
|
26
27
|
const res = await client.timer(queueName)
|
|
27
28
|
.key('fire-1')
|
|
@@ -34,8 +35,8 @@ export async function timerSchedulesAndFires(client) {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
// deliverAt is "not before", never "exactly at", and the visibility of a
|
|
37
|
-
// fired timer to a consumer is bounded by the
|
|
38
|
-
// delay -- see popUntil
|
|
38
|
+
// fired timer to a consumer is bounded by the broker's fire step, not by the
|
|
39
|
+
// delay -- see popUntil. The assertion is arrival.
|
|
39
40
|
const mine = await popUntil(client, queueName, m => m.data && m.data.run === run)
|
|
40
41
|
|
|
41
42
|
if (mine.length !== 1) {
|
package/test-v2/transaction.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { TEST_CONFIG } from './run.js'
|
|
2
|
+
|
|
1
3
|
export async function transactionBasicPushAck(client) {
|
|
2
4
|
const queueA = await client.queue('test-queue-v2-txn-basic-a').create()
|
|
3
5
|
const queueB = await client.queue('test-queue-v2-txn-basic-b').create()
|
|
@@ -572,3 +574,170 @@ export async function transactionAckWithConsumerGroup(client) {
|
|
|
572
574
|
}
|
|
573
575
|
}
|
|
574
576
|
|
|
577
|
+
|
|
578
|
+
// An ack takes the consumer group the message was popped under (2026-10-02,
|
|
579
|
+
// 2.0.0-beta.6). Without an explicit { consumerGroup } the op was judged in
|
|
580
|
+
// queue mode and the bundle rolled back with rejected_ack; queen.ack() came
|
|
581
|
+
// back "invalid or expired lease" the same way.
|
|
582
|
+
export async function transactionAckTakesGroupFromMessage(client) {
|
|
583
|
+
const source = 'test-queue-v2-txn-group-from-msg'
|
|
584
|
+
const sink = 'test-queue-v2-txn-group-from-msg-out'
|
|
585
|
+
const group = 'test-txn-group-from-msg'
|
|
586
|
+
const qa = await client.queue(source).create()
|
|
587
|
+
const qb = await client.queue(sink).create()
|
|
588
|
+
if (!qa.configured || !qb.configured) {
|
|
589
|
+
return { success: false, message: 'Queues not created' }
|
|
590
|
+
}
|
|
591
|
+
await client.queue(source).push([{ data: { value: 1 } }, { data: { value: 2 } }])
|
|
592
|
+
|
|
593
|
+
const messages = await client.queue(source).group(group).subscriptionMode('all').batch(2).wait(false).pop()
|
|
594
|
+
if (messages.length !== 2) {
|
|
595
|
+
return { success: false, message: `Expected 2 messages, got ${messages.length}` }
|
|
596
|
+
}
|
|
597
|
+
if (messages[0].consumerGroup !== group) {
|
|
598
|
+
return { success: false, message: `pop answered consumerGroup=${messages[0].consumerGroup}, expected ${group}` }
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
let committed
|
|
602
|
+
try {
|
|
603
|
+
committed = await client.transaction()
|
|
604
|
+
.ack(messages[0])
|
|
605
|
+
.queue(sink).push([{ data: { from: messages[0].data.value } }])
|
|
606
|
+
.commit()
|
|
607
|
+
} catch (error) {
|
|
608
|
+
return { success: false, message: `commit threw (${error.reason}): ${error.message}` }
|
|
609
|
+
}
|
|
610
|
+
const direct = await client.ack(messages[1], true)
|
|
611
|
+
const after = await client.queue(source).group(group).batch(2).wait(false).pop()
|
|
612
|
+
const forwarded = await client.queue(sink).batch(1).wait(false).pop()
|
|
613
|
+
|
|
614
|
+
const success = committed.success === true && direct.success === true && after.length === 0 && forwarded.length === 1
|
|
615
|
+
return {
|
|
616
|
+
success,
|
|
617
|
+
message: success
|
|
618
|
+
? 'transaction().ack and queen.ack took the group from the message'
|
|
619
|
+
: `txn=${committed.success} direct=${direct.success} (${direct.error}) after=${after.length} forwarded=${forwarded.length}`
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// ============================================================================
|
|
624
|
+
// A bundle that acks messages of two leases (2026-10-03, 2.0.0-beta.6)
|
|
625
|
+
// ============================================================================
|
|
626
|
+
//
|
|
627
|
+
// A Queen 2 broker fences each ack with the leaseId the operation carries, and
|
|
628
|
+
// lends requiredLeases to an ack without one only when the bundle names a
|
|
629
|
+
// single lease. An ack that left its lease in requiredLeases alone was applied
|
|
630
|
+
// in a bundle of two leases even after its lease expired and another consumer
|
|
631
|
+
// took the message. The builder has no per-pop lease knob, so these pops go
|
|
632
|
+
// through fetch, as in semantics.js.
|
|
633
|
+
|
|
634
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
|
|
635
|
+
const fenceQueue = name => `test-queue-v2-txn-fence-${name}-${Date.now()}`
|
|
636
|
+
|
|
637
|
+
/** One message of a queue-mode pop leased for `leaseSeconds`, or null. */
|
|
638
|
+
async function popLeased(queue, leaseSeconds, tries = 20) {
|
|
639
|
+
const qs = new URLSearchParams({
|
|
640
|
+
batch: '1',
|
|
641
|
+
wait: 'false',
|
|
642
|
+
leaseSeconds: String(leaseSeconds),
|
|
643
|
+
subscriptionMode: 'all'
|
|
644
|
+
})
|
|
645
|
+
// Lazy: run.js imports this module before TEST_CONFIG is initialized.
|
|
646
|
+
const url = `${TEST_CONFIG.baseUrls[0]}/api/v1/pop/queue/${queue}?${qs}`
|
|
647
|
+
for (let i = 0; i < tries; i++) {
|
|
648
|
+
const res = await fetch(url)
|
|
649
|
+
if (res.status !== 204) {
|
|
650
|
+
if (!res.ok) throw new Error(`pop ${queue}: HTTP ${res.status}`)
|
|
651
|
+
const body = await res.json()
|
|
652
|
+
if (body.messages && body.messages.length > 0) return body.messages[0]
|
|
653
|
+
}
|
|
654
|
+
await sleep(150)
|
|
655
|
+
}
|
|
656
|
+
return null
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/** The outcome of a commit, whether the client threw the refusal or returned it. */
|
|
660
|
+
async function commitOutcome(transaction) {
|
|
661
|
+
try {
|
|
662
|
+
const result = await transaction.commit()
|
|
663
|
+
return { committed: result.success === true, error: result.error || null }
|
|
664
|
+
} catch (error) {
|
|
665
|
+
return { committed: false, error: error.message }
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
export async function transactionTwoLeaseBundleExpiredLeaseRefused(client) {
|
|
670
|
+
const queueA = fenceQueue('stale-a')
|
|
671
|
+
const queueB = fenceQueue('stale-b')
|
|
672
|
+
const qa = await client.queue(queueA).create()
|
|
673
|
+
const qb = await client.queue(queueB).create()
|
|
674
|
+
if (!qa.configured || !qb.configured) {
|
|
675
|
+
return { success: false, message: 'Queues not created' }
|
|
676
|
+
}
|
|
677
|
+
await client.queue(queueA).push([{ data: { name: 'a' } }])
|
|
678
|
+
await client.queue(queueB).push([{ data: { name: 'b' } }])
|
|
679
|
+
|
|
680
|
+
const a = await popLeased(queueA, 1)
|
|
681
|
+
const b = await popLeased(queueB, 60)
|
|
682
|
+
if (!a?.leaseId || !b?.leaseId || a.leaseId === b.leaseId) {
|
|
683
|
+
return { success: false, message: `Expected two messages under two leases, got a=${a?.leaseId} b=${b?.leaseId}` }
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// Lease A expires, and another consumer takes the message.
|
|
687
|
+
const taken = await popLeased(queueA, 60, 100)
|
|
688
|
+
if (!taken || taken.transactionId !== a.transactionId || taken.leaseId === a.leaseId) {
|
|
689
|
+
return { success: false, message: 'The message of the expired lease was not delivered again under a new lease' }
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
const stale = await commitOutcome(client.transaction().ack(a).ack(b))
|
|
693
|
+
if (stale.committed) {
|
|
694
|
+
return { success: false, message: 'The bundle committed: the ack under the expired lease was applied unfenced' }
|
|
695
|
+
}
|
|
696
|
+
if (!/rolled back/.test(stale.error || '')) {
|
|
697
|
+
return { success: false, message: `The bundle was refused, but not as a rollback: ${stale.error}` }
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// Nothing of the bundle happened: each holder still settles its own message.
|
|
701
|
+
const takenAck = await commitOutcome(client.transaction().ack(taken))
|
|
702
|
+
const bAck = await commitOutcome(client.transaction().ack(b))
|
|
703
|
+
const leftA = await client.queue(queueA).batch(10).wait(false).pop()
|
|
704
|
+
const leftB = await client.queue(queueB).batch(10).wait(false).pop()
|
|
705
|
+
|
|
706
|
+
const success = takenAck.committed && bAck.committed && leftA.length === 0 && leftB.length === 0
|
|
707
|
+
return {
|
|
708
|
+
success,
|
|
709
|
+
message: success
|
|
710
|
+
? 'A bundle with an ack under an expired lease was refused whole; each holder then settled its own message'
|
|
711
|
+
: `new holder ack=${takenAck.committed} (${takenAck.error}), B ack=${bAck.committed} (${bAck.error}), left A=${leftA.length} B=${leftB.length}`
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
export async function transactionTwoLeaseBundleLiveLeasesCommit(client) {
|
|
716
|
+
const queueA = fenceQueue('live-a')
|
|
717
|
+
const queueB = fenceQueue('live-b')
|
|
718
|
+
const qa = await client.queue(queueA).create()
|
|
719
|
+
const qb = await client.queue(queueB).create()
|
|
720
|
+
if (!qa.configured || !qb.configured) {
|
|
721
|
+
return { success: false, message: 'Queues not created' }
|
|
722
|
+
}
|
|
723
|
+
await client.queue(queueA).push([{ data: { name: 'a' } }])
|
|
724
|
+
await client.queue(queueB).push([{ data: { name: 'b' } }])
|
|
725
|
+
|
|
726
|
+
const a = await popLeased(queueA, 60)
|
|
727
|
+
const b = await popLeased(queueB, 60)
|
|
728
|
+
if (!a?.leaseId || !b?.leaseId || a.leaseId === b.leaseId) {
|
|
729
|
+
return { success: false, message: `Expected two messages under two leases, got a=${a?.leaseId} b=${b?.leaseId}` }
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
const outcome = await commitOutcome(client.transaction().ack(a).ack(b))
|
|
733
|
+
const leftA = await client.queue(queueA).batch(10).wait(false).pop()
|
|
734
|
+
const leftB = await client.queue(queueB).batch(10).wait(false).pop()
|
|
735
|
+
|
|
736
|
+
const success = outcome.committed && leftA.length === 0 && leftB.length === 0
|
|
737
|
+
return {
|
|
738
|
+
success,
|
|
739
|
+
message: success
|
|
740
|
+
? 'A bundle of two live leases completed both messages'
|
|
741
|
+
: `commit=${outcome.committed} (${outcome.error}), left A=${leftA.length} B=${leftB.length}`
|
|
742
|
+
}
|
|
743
|
+
}
|
package/test-v2/watermark.js
CHANGED
|
@@ -1,90 +1,36 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Seek / delete re-consumption tests
|
|
2
|
+
* Seek / delete re-consumption tests.
|
|
3
3
|
*
|
|
4
|
-
* These
|
|
4
|
+
* These pin the BEHAVIOUR a user cares about, and nothing else:
|
|
5
5
|
* 1. Seeking a consumer group backwards re-exposes the old messages, and the
|
|
6
6
|
* group re-consumes them from the start within a short bound.
|
|
7
7
|
* 2. Deleting a consumer group (all queues, or for one queue) lets a group of
|
|
8
8
|
* the same name consume every message again from the start.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* it to exercise the 2-minute buffer:
|
|
16
|
-
* pl.updated_at >= (v_watermark - interval '2 minutes')
|
|
17
|
-
*
|
|
18
|
-
* With the wildcard hot-list ON (QUEEN_HOTLIST=1, the default) empty polls are
|
|
19
|
-
* answered from the in-memory candidate ring and that bookkeeping row is NEVER
|
|
20
|
-
* written — by design, not a bug: the ring, not the watermark, is the discovery
|
|
21
|
-
* source. So asserting the row exists fails on the hot-list path even though
|
|
22
|
-
* re-consumption works perfectly.
|
|
23
|
-
*
|
|
24
|
-
* The behaviour is identical on both engines, so these tests now assert the
|
|
25
|
-
* behaviour directly and only exercise the watermark-age scenario on the LEGACY
|
|
26
|
-
* path — detected at runtime by whether the empty poll actually wrote the row
|
|
27
|
-
* (no env flag needed). On the hot-list path the broker's seek/delete handlers
|
|
28
|
-
* re-seed / invalidate the ring so the reconsume is immediate; the idle-bounded
|
|
29
|
-
* consume below (idleMillis 3000) is what proves "within a short bound".
|
|
10
|
+
* Each test first drains the queue and polls it empty, so the group has seen
|
|
11
|
+
* "nothing to deliver" before the seek or the delete. "Within a short bound"
|
|
12
|
+
* is the idle-bounded consume (idleMillis 3000) that follows: a seek or a
|
|
13
|
+
* delete that left stale per-group state behind makes that consume go idle
|
|
14
|
+
* before every message came back, and the count comes up short.
|
|
30
15
|
*
|
|
31
16
|
* Run: node run.js seekBackwardsAllowsReconsume
|
|
32
17
|
* node run.js deleteConsumerGroupAllowsReconsume
|
|
33
18
|
* node run.js deleteConsumerGroupForQueueAllowsReconsume
|
|
34
19
|
*/
|
|
35
20
|
|
|
36
|
-
import { dbPool } from './run.js'
|
|
37
|
-
|
|
38
21
|
const QUEUE_NAME_SEEK = 'test-queue-v2-watermark-seek'
|
|
39
22
|
const QUEUE_NAME_DELETE = 'test-queue-v2-watermark-delete'
|
|
40
23
|
|
|
41
|
-
/**
|
|
42
|
-
* Helper: Move the watermark FORWARD in time to simulate that partitions
|
|
43
|
-
* are "old" relative to the watermark.
|
|
44
|
-
*
|
|
45
|
-
* The filter is: pl.updated_at >= (v_watermark - interval '2 minutes')
|
|
46
|
-
*
|
|
47
|
-
* If watermark = NOW + 10 minutes, then:
|
|
48
|
-
* filter = pl.updated_at >= (NOW + 10 - 2) = pl.updated_at >= NOW + 8
|
|
49
|
-
*
|
|
50
|
-
* Partitions updated at NOW will fail this filter (NOW < NOW + 8)
|
|
51
|
-
*/
|
|
52
|
-
async function advanceWatermark(queueName, consumerGroup, minutesForward = 10) {
|
|
53
|
-
const result = await dbPool.query(`
|
|
54
|
-
UPDATE queen.consumer_watermarks w
|
|
55
|
-
SET last_empty_scan_at = NOW() + interval '${minutesForward} minutes',
|
|
56
|
-
updated_at = NOW() + interval '${minutesForward} minutes'
|
|
57
|
-
FROM queen.queues q
|
|
58
|
-
WHERE w.queue_id = q.id AND q.name = $1 AND w.consumer_group = $2
|
|
59
|
-
RETURNING w.*
|
|
60
|
-
`, [queueName, consumerGroup])
|
|
61
|
-
return result.rows[0]
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* Helper: Check if watermark exists for a queue/consumer_group
|
|
66
|
-
*/
|
|
67
|
-
async function getWatermark(queueName, consumerGroup) {
|
|
68
|
-
const result = await dbPool.query(`
|
|
69
|
-
SELECT w.* FROM queen.consumer_watermarks w
|
|
70
|
-
JOIN queen.queues q ON q.id = w.queue_id
|
|
71
|
-
WHERE q.name = $1 AND w.consumer_group = $2
|
|
72
|
-
`, [queueName, consumerGroup])
|
|
73
|
-
return result.rows[0]
|
|
74
|
-
}
|
|
75
|
-
|
|
76
24
|
/**
|
|
77
25
|
* Test: Seeking backwards should allow re-consumption of messages
|
|
78
26
|
*
|
|
79
27
|
* Steps:
|
|
80
28
|
* 1. Push messages to queue
|
|
81
|
-
* 2. Consume all messages with a consumer group
|
|
82
|
-
* 3.
|
|
83
|
-
* 4.
|
|
84
|
-
* 5. Try to consume again - should get all messages
|
|
29
|
+
* 2. Consume all messages with a consumer group, then poll it empty
|
|
30
|
+
* 3. Seek the consumer group backwards to before the messages
|
|
31
|
+
* 4. Try to consume again - should get all messages
|
|
85
32
|
*
|
|
86
|
-
* Expected: All messages are re-consumed after seek
|
|
87
|
-
* Actual (bug): Watermark blocks re-consumption, 0 messages returned
|
|
33
|
+
* Expected: All messages are re-consumed after seek, within the idle window
|
|
88
34
|
*/
|
|
89
35
|
export async function seekBackwardsAllowsReconsume(client) {
|
|
90
36
|
const consumerGroup = 'watermark-seek-test-cg'
|
|
@@ -92,11 +38,6 @@ export async function seekBackwardsAllowsReconsume(client) {
|
|
|
92
38
|
// Cleanup
|
|
93
39
|
try {
|
|
94
40
|
await client.deleteConsumerGroup(consumerGroup, true)
|
|
95
|
-
// Also clean up any orphaned watermark from previous runs
|
|
96
|
-
await dbPool.query(
|
|
97
|
-
'DELETE FROM queen.consumer_watermarks WHERE consumer_group = $1',
|
|
98
|
-
[consumerGroup]
|
|
99
|
-
)
|
|
100
41
|
} catch (e) {
|
|
101
42
|
// Ignore
|
|
102
43
|
}
|
|
@@ -147,7 +88,7 @@ export async function seekBackwardsAllowsReconsume(client) {
|
|
|
147
88
|
}
|
|
148
89
|
}
|
|
149
90
|
|
|
150
|
-
// 5. Verify queue is empty for this consumer (
|
|
91
|
+
// 5. Verify queue is empty for this consumer (the group has now polled empty)
|
|
151
92
|
const emptyCheck = await client
|
|
152
93
|
.queue(QUEUE_NAME_SEEK)
|
|
153
94
|
.group(consumerGroup)
|
|
@@ -162,24 +103,9 @@ export async function seekBackwardsAllowsReconsume(client) {
|
|
|
162
103
|
}
|
|
163
104
|
}
|
|
164
105
|
|
|
165
|
-
console.log(` Verified queue is empty
|
|
166
|
-
|
|
167
|
-
// 6. Detect which discovery path we are on. On the LEGACY path the empty poll
|
|
168
|
-
// wrote a consumer_watermarks row; on the hot-list path it did not (empty
|
|
169
|
-
// polls are served from the ring — by design). We do NOT assert the row: it
|
|
170
|
-
// is a path-specific bookkeeping detail, not the behaviour under test.
|
|
171
|
-
const watermarkBefore = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
|
|
172
|
-
if (watermarkBefore) {
|
|
173
|
-
// Legacy path: age the watermark 10 minutes into the future so the old
|
|
174
|
-
// partitions fall outside the 2-minute buffer — the original repro that
|
|
175
|
-
// proves seek must reset the watermark.
|
|
176
|
-
const advancedWatermark = await advanceWatermark(QUEUE_NAME_SEEK, consumerGroup, 10)
|
|
177
|
-
console.log(` [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
|
|
178
|
-
} else {
|
|
179
|
-
console.log(` [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
|
|
180
|
-
}
|
|
106
|
+
console.log(` Verified queue is empty`)
|
|
181
107
|
|
|
182
|
-
//
|
|
108
|
+
// 6. Seek backwards to BEFORE the messages were pushed
|
|
183
109
|
console.log(` Seeking to timestamp: ${timestampBeforeMessages}`)
|
|
184
110
|
|
|
185
111
|
const seekResult = await client.admin.seekConsumerGroup(
|
|
@@ -190,18 +116,10 @@ export async function seekBackwardsAllowsReconsume(client) {
|
|
|
190
116
|
|
|
191
117
|
console.log(` Seek result: ${JSON.stringify(seekResult)}`)
|
|
192
118
|
|
|
193
|
-
// 9. Check if watermark was reset (it should be, but currently isn't - that's the bug)
|
|
194
|
-
const watermarkAfterSeek = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
|
|
195
|
-
if (watermarkAfterSeek) {
|
|
196
|
-
console.log(` WARNING: Watermark still exists after seek: ${watermarkAfterSeek.last_empty_scan_at}`)
|
|
197
|
-
} else {
|
|
198
|
-
console.log(` Watermark was properly deleted after seek`)
|
|
199
|
-
}
|
|
200
|
-
|
|
201
119
|
// Small delay after seek
|
|
202
120
|
await new Promise(resolve => setTimeout(resolve, 500))
|
|
203
121
|
|
|
204
|
-
//
|
|
122
|
+
// 7. Try to consume again - should get all messages back
|
|
205
123
|
let secondConsumeCount = 0
|
|
206
124
|
await client
|
|
207
125
|
.queue(QUEUE_NAME_SEEK)
|
|
@@ -216,17 +134,13 @@ export async function seekBackwardsAllowsReconsume(client) {
|
|
|
216
134
|
|
|
217
135
|
console.log(` Second consume (after seek): ${secondConsumeCount} messages`)
|
|
218
136
|
|
|
219
|
-
//
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
// (still at ${watermarkAfterSeek?.last_empty_scan_at}); on the hot-list path
|
|
223
|
-
// the ring was not re-seeded, so reconsume would only resume at the ≤30s
|
|
224
|
-
// periodic floor instead of immediately.
|
|
137
|
+
// 8. Verify we got all messages back within the idle-bounded consume above.
|
|
138
|
+
// A shortfall means the backward seek did not re-expose the messages
|
|
139
|
+
// promptly.
|
|
225
140
|
if (secondConsumeCount !== messageCount) {
|
|
226
141
|
return {
|
|
227
142
|
success: false,
|
|
228
|
-
message: `After seek backwards, should re-consume ${messageCount} messages within the idle window, but got ${secondConsumeCount}
|
|
229
|
-
`(legacy watermark after seek: ${watermarkAfterSeek?.last_empty_scan_at})`
|
|
143
|
+
message: `After seek backwards, should re-consume ${messageCount} messages within the idle window, but got ${secondConsumeCount}`
|
|
230
144
|
}
|
|
231
145
|
}
|
|
232
146
|
|
|
@@ -241,14 +155,12 @@ export async function seekBackwardsAllowsReconsume(client) {
|
|
|
241
155
|
*
|
|
242
156
|
* Steps:
|
|
243
157
|
* 1. Push messages to queue
|
|
244
|
-
* 2. Consume all messages with a consumer group
|
|
245
|
-
* 3.
|
|
246
|
-
* 4.
|
|
247
|
-
* 5.
|
|
248
|
-
* 6. Try to consume - should get all messages
|
|
158
|
+
* 2. Consume all messages with a consumer group, then poll it empty
|
|
159
|
+
* 3. Delete the consumer group
|
|
160
|
+
* 4. Create a new consumer group with the SAME name
|
|
161
|
+
* 5. Try to consume - should get all messages
|
|
249
162
|
*
|
|
250
163
|
* Expected: All messages are consumed by the "new" consumer group
|
|
251
|
-
* Actual (bug): Orphaned watermark blocks consumption, 0 messages returned
|
|
252
164
|
*/
|
|
253
165
|
export async function deleteConsumerGroupAllowsReconsume(client) {
|
|
254
166
|
const consumerGroup = 'watermark-delete-test-cg'
|
|
@@ -256,11 +168,6 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
|
|
|
256
168
|
// Cleanup
|
|
257
169
|
try {
|
|
258
170
|
await client.deleteConsumerGroup(consumerGroup, true)
|
|
259
|
-
// Also clean up any orphaned watermark from previous runs
|
|
260
|
-
await dbPool.query(
|
|
261
|
-
'DELETE FROM queen.consumer_watermarks WHERE consumer_group = $1',
|
|
262
|
-
[consumerGroup]
|
|
263
|
-
)
|
|
264
171
|
} catch (e) {
|
|
265
172
|
// Ignore
|
|
266
173
|
}
|
|
@@ -305,7 +212,7 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
|
|
|
305
212
|
}
|
|
306
213
|
}
|
|
307
214
|
|
|
308
|
-
// 4. Verify queue is empty for this consumer (
|
|
215
|
+
// 4. Verify queue is empty for this consumer (the group has now polled empty)
|
|
309
216
|
const emptyCheck = await client
|
|
310
217
|
.queue(QUEUE_NAME_DELETE)
|
|
311
218
|
.group(consumerGroup)
|
|
@@ -320,40 +227,20 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
|
|
|
320
227
|
}
|
|
321
228
|
}
|
|
322
229
|
|
|
323
|
-
console.log(` Verified queue is empty
|
|
324
|
-
|
|
325
|
-
// 5. Detect the discovery path (see seekBackwardsAllowsReconsume). The
|
|
326
|
-
// watermark row is a legacy-path artefact, not the behaviour under test, so
|
|
327
|
-
// we do not assert it — we only exercise the watermark-age scenario when the
|
|
328
|
-
// row was actually written (legacy path).
|
|
329
|
-
const watermarkBefore = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
|
|
330
|
-
if (watermarkBefore) {
|
|
331
|
-
const advancedWatermark = await advanceWatermark(QUEUE_NAME_DELETE, consumerGroup, 10)
|
|
332
|
-
console.log(` [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
|
|
333
|
-
} else {
|
|
334
|
-
console.log(` [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
|
|
335
|
-
}
|
|
230
|
+
console.log(` Verified queue is empty`)
|
|
336
231
|
|
|
337
|
-
//
|
|
232
|
+
// 5. Delete the consumer group
|
|
338
233
|
console.log(` Deleting consumer group: ${consumerGroup}`)
|
|
339
234
|
|
|
340
235
|
const deleteResult = await client.deleteConsumerGroup(consumerGroup, true)
|
|
341
236
|
console.log(` Delete result: ${JSON.stringify(deleteResult)}`)
|
|
342
237
|
|
|
343
|
-
// 8. Check if watermark was deleted (it should be, but currently isn't - that's the bug)
|
|
344
|
-
const watermarkAfterDelete = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
|
|
345
|
-
if (watermarkAfterDelete) {
|
|
346
|
-
console.log(` WARNING: Orphaned watermark still exists: ${watermarkAfterDelete.last_empty_scan_at}`)
|
|
347
|
-
} else {
|
|
348
|
-
console.log(` Watermark was properly deleted`)
|
|
349
|
-
}
|
|
350
|
-
|
|
351
238
|
// Small delay after delete
|
|
352
239
|
await new Promise(resolve => setTimeout(resolve, 500))
|
|
353
240
|
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
//
|
|
241
|
+
// 6. Create a "new" consumer group with the same name and try to consume.
|
|
242
|
+
// The delete removed the group's cursors and metadata, so it must start
|
|
243
|
+
// fresh; any per-group state that survived would skip the messages.
|
|
357
244
|
let secondConsumeCount = 0
|
|
358
245
|
await client
|
|
359
246
|
.queue(QUEUE_NAME_DELETE)
|
|
@@ -369,16 +256,14 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
|
|
|
369
256
|
|
|
370
257
|
console.log(` Second consume (after delete + recreate): ${secondConsumeCount} messages`)
|
|
371
258
|
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
// hot-list path, a surviving group ring / stale seeded-groups cache that
|
|
376
|
-
// suppressed the from-the-start reconsume.
|
|
259
|
+
// 7. Verify we got all messages within the idle-bounded consume above. A
|
|
260
|
+
// shortfall means the delete left stale per-group state behind that
|
|
261
|
+
// suppressed the from-the-start reconsume.
|
|
377
262
|
if (secondConsumeCount !== messageCount) {
|
|
378
263
|
return {
|
|
379
264
|
success: false,
|
|
380
265
|
message: `After deleting CG, new CG with same name should consume ${messageCount} messages within the idle window, ` +
|
|
381
|
-
`but got ${secondConsumeCount}
|
|
266
|
+
`but got ${secondConsumeCount}`
|
|
382
267
|
}
|
|
383
268
|
}
|
|
384
269
|
|
|
@@ -400,11 +285,6 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
|
|
|
400
285
|
// Cleanup
|
|
401
286
|
try {
|
|
402
287
|
await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
|
|
403
|
-
// Also clean up any orphaned watermark from previous runs
|
|
404
|
-
await dbPool.query(
|
|
405
|
-
'DELETE FROM queen.consumer_watermarks w USING queen.queues q WHERE w.queue_id = q.id AND q.name = $1 AND w.consumer_group = $2',
|
|
406
|
-
[queueName, consumerGroup]
|
|
407
|
-
)
|
|
408
288
|
} catch (e) {
|
|
409
289
|
// Ignore
|
|
410
290
|
}
|
|
@@ -449,7 +329,7 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
|
|
|
449
329
|
}
|
|
450
330
|
}
|
|
451
331
|
|
|
452
|
-
// 4. Verify queue is empty (
|
|
332
|
+
// 4. Verify queue is empty (the group has now polled empty)
|
|
453
333
|
const emptyCheck = await client
|
|
454
334
|
.queue(queueName)
|
|
455
335
|
.group(consumerGroup)
|
|
@@ -464,35 +344,17 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
|
|
|
464
344
|
}
|
|
465
345
|
}
|
|
466
346
|
|
|
467
|
-
console.log(` Verified queue is empty
|
|
468
|
-
|
|
469
|
-
// 5. Detect the discovery path (see seekBackwardsAllowsReconsume). The
|
|
470
|
-
// watermark row is a legacy-path artefact, not the behaviour under test.
|
|
471
|
-
const watermarkBefore = await getWatermark(queueName, consumerGroup)
|
|
472
|
-
if (watermarkBefore) {
|
|
473
|
-
const advancedWatermark = await advanceWatermark(queueName, consumerGroup, 10)
|
|
474
|
-
console.log(` [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
|
|
475
|
-
} else {
|
|
476
|
-
console.log(` [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
|
|
477
|
-
}
|
|
347
|
+
console.log(` Verified queue is empty`)
|
|
478
348
|
|
|
479
|
-
//
|
|
349
|
+
// 5. Delete consumer group for this specific queue
|
|
480
350
|
console.log(` Deleting consumer group for queue: ${consumerGroup} / ${queueName}`)
|
|
481
351
|
|
|
482
352
|
const deleteResult = await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
|
|
483
353
|
console.log(` Delete result: ${JSON.stringify(deleteResult)}`)
|
|
484
354
|
|
|
485
|
-
// 8. Check if watermark was deleted
|
|
486
|
-
const watermarkAfterDelete = await getWatermark(queueName, consumerGroup)
|
|
487
|
-
if (watermarkAfterDelete) {
|
|
488
|
-
console.log(` WARNING: Orphaned watermark still exists: ${watermarkAfterDelete.last_empty_scan_at}`)
|
|
489
|
-
} else {
|
|
490
|
-
console.log(` Watermark was properly deleted`)
|
|
491
|
-
}
|
|
492
|
-
|
|
493
355
|
await new Promise(resolve => setTimeout(resolve, 500))
|
|
494
356
|
|
|
495
|
-
//
|
|
357
|
+
// 6. Try to consume again
|
|
496
358
|
let secondConsumeCount = 0
|
|
497
359
|
await client
|
|
498
360
|
.queue(queueName)
|
|
@@ -512,7 +374,7 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
|
|
|
512
374
|
return {
|
|
513
375
|
success: false,
|
|
514
376
|
message: `After deleting CG for queue, should consume ${messageCount} messages within the idle window, ` +
|
|
515
|
-
`but got ${secondConsumeCount}
|
|
377
|
+
`but got ${secondConsumeCount}`
|
|
516
378
|
}
|
|
517
379
|
}
|
|
518
380
|
|