queen-mq 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +71 -18
  2. package/client-v2/Queen.js +45 -13
  3. package/client-v2/README.md +26 -11
  4. package/client-v2/admin/Admin.js +0 -47
  5. package/client-v2/builders/QueueBuilder.js +33 -10
  6. package/client-v2/builders/TimerBuilder.js +2 -2
  7. package/client-v2/builders/TransactionBuilder.js +35 -9
  8. package/client-v2/consumer/ConsumerManager.js +67 -46
  9. package/client-v2/ephemeral/Ephemeral.js +7 -9
  10. package/client-v2/kv/Kv.js +4 -4
  11. package/client-v2/kv/expiry.js +1 -1
  12. package/client-v2/streams/Stream.js +8 -3
  13. package/client-v2/streams/helpers/rateLimiter.js +4 -4
  14. package/client-v2/streams/operators/GateOperator.js +9 -2
  15. package/client-v2/streams/operators/ReduceOperator.js +2 -2
  16. package/client-v2/streams/operators/WindowSessionOperator.js +3 -3
  17. package/client-v2/streams/runtime/Runner.js +111 -54
  18. package/client-v2/streams/runtime/cycle.js +4 -4
  19. package/client-v2/streams/runtime/register.js +1 -1
  20. package/client-v2/utils/conflation.js +0 -6
  21. package/client-v2/utils/consumerGroup.js +54 -0
  22. package/package.json +5 -8
  23. package/test-v2/_kvtimers.js +12 -13
  24. package/test-v2/ackwindow.js +12 -192
  25. package/test-v2/bootstrap.js +2 -2
  26. package/test-v2/conflation-unit/conflationWire.test.js +0 -12
  27. package/test-v2/consume.js +41 -1
  28. package/test-v2/consumer-unit/handlerError.test.js +161 -0
  29. package/test-v2/docs.js +5 -4
  30. package/test-v2/http-unit/pushStatus.test.js +142 -0
  31. package/test-v2/http-unit/renew.test.js +96 -0
  32. package/test-v2/kv-unit/timerWire.test.js +1 -1
  33. package/test-v2/kv-unit/txnWire.test.js +101 -2
  34. package/test-v2/kv.js +6 -5
  35. package/test-v2/pop.js +28 -1
  36. package/test-v2/run.js +50 -114
  37. package/test-v2/runner-unit/fatalExit.test.js +64 -0
  38. package/test-v2/semantics.js +20 -34
  39. package/test-v2/stream/_helpers.js +10 -19
  40. package/test-v2/stream/cron.js +1 -1
  41. package/test-v2/stream/gate.js +54 -0
  42. package/test-v2/stream/index.js +2 -0
  43. package/test-v2/stream/tumbling.js +3 -3
  44. package/test-v2/streams-unit/ack.test.js +61 -0
  45. package/test-v2/streams-unit/cycle.test.js +1 -1
  46. package/test-v2/streams-unit/e2e.test.js +6 -10
  47. package/test-v2/streams-unit/gate.test.js +189 -0
  48. package/test-v2/timers.js +6 -5
  49. package/test-v2/transaction.js +169 -0
  50. package/test-v2/watermark.js +38 -176
  51. package/test-v2/maintenance.js +0 -277
@@ -0,0 +1,54 @@
1
+ /**
2
+ * The consumer group a popped message belongs to.
3
+ *
4
+ * Every pop answers each message with the group it was claimed under
5
+ * (`consumerGroup`), and the lease an ack has to match belongs to that group.
6
+ * An ack that names no group is judged in queue mode, so a message popped by a
7
+ * group and acked without one is refused: `rejected_ack` from a transaction,
8
+ * `invalid or expired lease` from /ack. Reading the group off the message is
9
+ * what lets `ack(message)` work without the caller repeating `.group(...)`.
10
+ */
11
+
12
+ /** The broker's name for "no consumer group" (queue mode). */
13
+ export const QUEUE_MODE_GROUP = '__QUEUE_MODE__'
14
+
15
+ /**
16
+ * The group to send with an ack of `message`, or null for queue mode.
17
+ *
18
+ * Null for queue mode on purpose: an ack with no group IS a queue-mode ack, so
19
+ * leaving the key out keeps those requests byte-identical to what this client
20
+ * always sent. Also null for anything that is not a popped message (a bare
21
+ * transactionId string, a hand-built `{transactionId, partitionId}`).
22
+ */
23
+ export function consumerGroupOf(message) {
24
+ if (message === null || typeof message !== 'object') return null
25
+ const group = message.consumerGroup
26
+ if (typeof group !== 'string' || group.length === 0 || group === QUEUE_MODE_GROUP) return null
27
+ return group
28
+ }
29
+
30
+ /**
31
+ * The one group to send with a batch ack of `messages`, or null for queue mode.
32
+ *
33
+ * /api/v1/ack/batch carries ONE consumer group for the whole request, so a
34
+ * batch whose messages were popped under different groups cannot be expressed
35
+ * as one call: it throws rather than ack some of them under the wrong group.
36
+ * Messages that state no group (hand-built ones) take whatever the others say.
37
+ */
38
+ export function sharedConsumerGroupOf(messages) {
39
+ const named = new Set()
40
+ for (const message of messages) {
41
+ if (message !== null && typeof message === 'object' &&
42
+ typeof message.consumerGroup === 'string' && message.consumerGroup.length > 0) {
43
+ named.add(message.consumerGroup)
44
+ }
45
+ }
46
+ if (named.size > 1) {
47
+ throw new Error(
48
+ `Cannot ack messages from different consumer groups in one call (${[...named].join(', ')}): ` +
49
+ 'a batch ack carries one consumer group. Ack each group separately, or pass { group }.'
50
+ )
51
+ }
52
+ const [group] = named
53
+ return group && group !== QUEUE_MODE_GROUP ? group : null
54
+ }
package/package.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "queen-mq",
3
- "version": "1.3.0",
3
+ "version": "2.0.0",
4
4
  "type": "module",
5
- "description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
5
+ "description": "Partitioned message queue on a replicated broker log — broker client + fluent streaming SDK (windows, joins, gates) in one package",
6
6
  "main": "client-v2/index.js",
7
7
  "scripts": {
8
- "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js && node test-v2/run.js human",
9
- "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js",
8
+ "test": "npm run test:unit && node test-v2/run.js human",
9
+ "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js",
10
10
  "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
11
11
  "test:integration": "node test-v2/run.js human",
12
12
  "test:streams": "node test-v2/run.js stream",
@@ -19,12 +19,11 @@
19
19
  ],
20
20
  "dependencies": {
21
21
  "axios": "^1.12.2",
22
- "pg": "^8.16.3",
23
22
  "undici": "^6.21.0",
24
23
  "uuid": "^13.0.0"
25
24
  },
26
25
  "engines": {
27
- "node": ">=22.0.0"
26
+ "node": ">=24.0.0"
28
27
  },
29
28
  "author": "Smartness",
30
29
  "license": "Apache-2.0",
@@ -39,8 +38,6 @@
39
38
  "keywords": [
40
39
  "message-queue",
41
40
  "queue",
42
- "postgres",
43
- "postgresql",
44
41
  "broker",
45
42
  "message-queue-system",
46
43
  "fifo",
@@ -19,8 +19,9 @@
19
19
  * says nothing, and that was only tolerable while the 404 was legitimate.
20
20
  *
21
21
  * What still exists is the operator's RUNTIME kill switch (`kv_enabled`,
22
- * `timers_schedule_enabled`, `timers_fire_enabled` in `queen.system_state`) --
23
- * the maintenance-mode lever, pulled live during an incident. It answers 503
22
+ * `timers_schedule_enabled`, `timers_fire_enabled`, set through
23
+ * `POST /api/v1/system/kv-timers`) --
24
+ * a lever pulled live during an incident. It answers 503
24
25
  * with `Retry-After` on the kv/timer routes and 403 on a `kv`/`timers` rider
25
26
  * inside a transaction, never 404. If a run here ever goes red against that,
26
27
  * the switch is down on the rig and somebody pulled it; it is not something
@@ -35,20 +36,18 @@ export const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
35
36
  /**
36
37
  * Poll a queue until `match` accepts a message, or the deadline passes.
37
38
  *
38
- * WHY THIS IS A POLL AND NOT A LONG POLL, MEASURED ON A REAL RIG. A timer's
39
- * delivery is committed by the sweeper INSIDE PostgreSQL
40
- * (`log_timers_fire_v1` = DELETE + push in one transaction), so it does not
41
- * pass through the broker's push handler and the destination partition is not
42
- * marked ready in that broker's hot list. A consumer therefore sees the
43
- * message only at the next hot-list reseed: on a default broker
44
- * (QUEEN_HOTLIST_RESEED_MS = 30000) a timer scheduled for +300 ms was measured
45
- * arriving at +30.7 s, and a 20 s long poll timed out with the message already
46
- * committed in the log.
39
+ * WHY THIS IS A POLL AND NOT A LONG POLL. A timer's delivery is not a client
40
+ * push: the broker's own fire step appends it (the leader plans due timers in
41
+ * a step of its own loop, each fire's message append and timer removal in one
42
+ * entry), so it never passes through the push handler, and when it becomes
43
+ * visible is bounded by that step, not by the delay under test. A short poll
44
+ * with a deadline asserts the arrival without depending on whether, or when,
45
+ * a parked long poll is woken by a fire.
47
46
  *
48
47
  * That is a broker property, not a client one, and it is why these tests
49
48
  * assert ARRIVAL and never latency: `deliverAt` is "not before", never
50
- * "exactly at". The deadline here is generous on purpose -- it is sized
51
- * against the reseed period, not against the delay under test.
49
+ * "exactly at". The deadline here is generous on purpose -- it is sized for a
50
+ * slow rig, not against the delay under test.
52
51
  */
53
52
  export async function popUntil(client, queueName, match, timeoutMs = 60000) {
54
53
  const deadline = Date.now() + timeoutMs
@@ -1,33 +1,21 @@
1
1
  /**
2
- * Ack-window honesty tests (2026-07-30).
2
+ * Ack-window honesty test (2026-07-30).
3
3
  *
4
- * queen.log_ack_by_hash_v1 resolves txn hashes through the queen.log_txns
5
- * sidecar, which is purged on its own clock (GREATEST(dedup_window,
6
- * completed_retention, 900s)). A hash that cannot be resolved — purged row,
7
- * or a transactionId that never existed — is correctly NOT acked (the cursor
8
- * stops, frames redeliver: redelivery over loss). The BUG these tests pin:
9
- * the broker reported those items as success=true (they appear in neither
10
- * noopHashes nor staleHashes), so the client believed the ack landed while
11
- * the cursor never moved — a silent redelivery livelock, and a nack/dlq in
12
- * that state could never dead-letter its poison message.
4
+ * An ack whose transactionId the broker cannot resolve to a message of that
5
+ * partition (here: one that was never pushed) is correctly NOT applied: the
6
+ * cursor stays put and the real messages redeliver (redelivery over loss). The
7
+ * BUG this test pins: the broker reported such items as success=true, so the
8
+ * client believed the ack landed while the cursor never moved -- a silent
9
+ * redelivery livelock, and a nack/dlq in that state could never dead-letter
10
+ * its poison message.
13
11
  *
14
12
  * Expected contract (post-fix): unresolvable items come back success=false
15
- * with an explicit "unresolvable" error, and the cursor is untouched.
16
- *
17
- * Companion test: emptyPartitionCursorSealsAfterRetention pins the empty-
18
- * partition cursor seal in log_pop_v1 (the Rust port of the C++
19
- * pop_unified_batch_v4 starvation fix): a partition whose segments were all
20
- * removed by retention must stop being "phantom pending" after one pop.
13
+ * with an explicit "unresolvable" error, and the lease survives for the real
14
+ * batch.
21
15
  *
22
16
  * Run: node run.js ackUnknownTxnMustFail
23
- * node run.js ackAfterHashPurgeMustFailExplicitly
24
- * node run.js emptyPartitionCursorSealsAfterRetention
25
17
  */
26
18
 
27
- import { dbPool } from './run.js'
28
-
29
- const TENANT = '00000000-0000-0000-0000-000000000001'
30
-
31
19
  function sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
32
20
 
33
21
  async function popRetry(client, queue, { batch = 3, group = null, partition = null, tries = 30, mode = null } = {}) {
@@ -43,35 +31,9 @@ async function popRetry(client, queue, { batch = 3, group = null, partition = nu
43
31
  return []
44
32
  }
45
33
 
46
- async function partitionRow(queue, partition = 'Default') {
47
- const r = await dbPool.query(
48
- `SELECT p.id::text AS id, p.last_offset::bigint AS last_offset, p.log_start::bigint AS log_start
49
- FROM queen.log_partitions p
50
- JOIN queen.queues lq ON lq.id = p.queue_id
51
- WHERE lq.name = $1 AND p.name = $2 AND lq.tenant_id = $3::uuid`,
52
- [queue, partition, TENANT])
53
- return r.rows[0] || null
54
- }
55
-
56
- async function committedOf(partitionId, group) {
57
- const r = await dbPool.query(
58
- `SELECT c.committed::bigint AS committed FROM queen.log_consumers c
59
- WHERE c.partition_id = $1::uuid AND c.consumer_group = $2`,
60
- [partitionId, group])
61
- return r.rows.length ? Number(r.rows[0].committed) : null
62
- }
63
-
64
- async function segmentCount(partitionId) {
65
- const r = await dbPool.query(
66
- `SELECT count(*)::int AS n FROM queen.log_segments WHERE partition_id = $1::uuid`,
67
- [partitionId])
68
- return r.rows[0].n
69
- }
70
-
71
34
  /**
72
- * Test A (wire-only): acking a transactionId that never existed must fail
73
- * explicitly, for both a completed ack and a failed nack. Today the broker
74
- * answers success=true for both.
35
+ * Wire-only: acking a transactionId that never existed must fail explicitly,
36
+ * for both a completed ack and a failed nack.
75
37
  */
76
38
  export async function ackUnknownTxnMustFail(client) {
77
39
  const queue = 'test-queue-v2-ackwindow-unknown'
@@ -121,145 +83,3 @@ export async function ackUnknownTxnMustFail(client) {
121
83
 
122
84
  return { success: true, message: 'unknown-txn ack and nack both rejected explicitly; real batch still ackable' }
123
85
  }
124
-
125
- /**
126
- * Test B (with SQL): the real-world shape — messages still delivered (segments
127
- * intact) but their log_txns hash rows purged. Per-message acks must fail
128
- * explicitly and the cursor must not move. Pre-fix: every ack reported
129
- * success=true while committed stayed put (silent livelock).
130
- */
131
- export async function ackAfterHashPurgeMustFailExplicitly(client) {
132
- const queue = 'test-queue-v2-ackwindow-purged'
133
- await client.queue(queue).create()
134
- const txns = [1, 2, 3, 4, 5].map(n => `${queue}-tx-${n}`)
135
- await client.queue(queue).partition('Default').push(
136
- txns.map((t, i) => ({ data: { n: i }, transactionId: t })))
137
-
138
- // Wait until the segments are visible, then purge the hash sidecar —
139
- // exactly what log_txns_purge_step_v1 does after the window expires
140
- // (the 900s floor makes the real purge untestable in-suite).
141
- const part = await partitionRow(queue)
142
- if (!part) return { success: false, message: 'partition row not found' }
143
- for (let i = 0; i < 30 && (await segmentCount(part.id)) === 0; i++) await sleep(100)
144
-
145
- const del = await dbPool.query(
146
- `DELETE FROM queen.log_txns WHERE partition_id = $1::uuid`, [part.id])
147
- if (del.rowCount === 0) return { success: false, message: 'no log_txns rows to purge — test setup broken' }
148
-
149
- // Segments untouched: the messages still DELIVER.
150
- const msgs = await popRetry(client, queue, { batch: 5 })
151
- if (msgs.length !== 5) return { success: false, message: `expected 5 delivered messages after purge, got ${msgs.length}` }
152
-
153
- // Per-message completed ack (the JS per-message consumer shape): the hash
154
- // cannot resolve, so the honest answer is an explicit failure.
155
- const r1 = await client.ack([msgs[0]], true)
156
- const item1 = (r1.results || [])[0] || {}
157
- if (item1.success !== false || !/unresolv/i.test(item1.error || '')) {
158
- return { success: false, message: `BUG: ack with purged hash answered success=${item1.success}, error='${item1.error}'` }
159
- }
160
-
161
- // Nack of a purged-hash message: same — and pre-fix this also meant the
162
- // poison could never reach the DLQ.
163
- const r2 = await client.ack([msgs[1]], false)
164
- const item2 = (r2.results || [])[0] || {}
165
- if (item2.success !== false || !/unresolv/i.test(item2.error || '')) {
166
- return { success: false, message: `BUG: nack with purged hash answered success=${item2.success}, error='${item2.error}'` }
167
- }
168
-
169
- // The cursor must not have moved (redelivery over loss — that part of the
170
- // contract was always right; only the reporting lied).
171
- const committed = await committedOf(part.id, '__QUEUE_MODE__')
172
- if (committed !== -1) {
173
- return { success: false, message: `cursor moved on unresolvable acks: committed=${committed}, expected -1` }
174
- }
175
-
176
- return { success: true, message: 'purged-hash ack and nack rejected explicitly, cursor untouched, messages still deliverable' }
177
- }
178
-
179
- /**
180
- * Test C (bug 2, with SQL): empty-partition cursor seal. A group consumes
181
- * part of a partition, retention then deletes ALL segments (time rule deletes
182
- * unconsumed data too): last_offset > committed with zero segments = a
183
- * phantom-pending row that used to stay a wildcard candidate forever and
184
- * pinned the empty-scan watermark at epoch. After the fix, one empty pop
185
- * seals committed to the partition's last_offset. A delayed-processing queue
186
- * pins the guard: segments EXIST but are deferred → no seal.
187
- */
188
- export async function emptyPartitionCursorSealsAfterRetention(client) {
189
- const queue = 'test-queue-v2-ackwindow-seal'
190
- const group = 'g-seal'
191
- await client.queue(queue).config({ retentionEnabled: true, retentionSeconds: 1, leaseTime: 30 }).create()
192
- await client.queue(queue).partition('Default').push(
193
- [0, 1, 2, 3, 4, 5].map(n => ({ data: { n }, transactionId: `${queue}-tx-${n}` })))
194
-
195
- // Consume + ack only the first two: committed lands at 1, the rest stays.
196
- // First contact for g-seal, and the 6 messages are already pushed: the
197
- // group has to ask for the backlog it is about to half-consume.
198
- const first = await popRetry(client, queue, { batch: 2, group, mode: 'all' })
199
- if (first.length !== 2) return { success: false, message: `expected 2 messages, got ${first.length}` }
200
- const ackr = await client.ack(first, true, { group })
201
- if (ackr.success !== true) return { success: false, message: `ack failed: ${ackr.error}` }
202
-
203
- const part = await partitionRow(queue)
204
- if (!part) return { success: false, message: 'partition row not found' }
205
-
206
- // Retention (interval 2s in the harness, cutoff 1s) must delete ALL
207
- // segments — the time rule ignores cursors.
208
- let segs = -1
209
- for (let i = 0; i < 60; i++) {
210
- segs = await segmentCount(part.id)
211
- if (segs === 0) break
212
- await sleep(500)
213
- }
214
- if (segs !== 0) return { success: false, message: `retention never emptied the partition (segments=${segs})` }
215
-
216
- const before = await committedOf(part.id, group)
217
- const rowNow = await partitionRow(queue)
218
- if (before === null) return { success: false, message: 'consumer row missing before the empty pop' }
219
- if (before >= Number(rowNow.last_offset)) {
220
- return { success: false, message: `setup broken: no phantom span (committed=${before}, last_offset=${rowNow.last_offset})` }
221
- }
222
-
223
- // ONE empty pop on the emptied partition must seal the cursor.
224
- const empty = await client.queue(queue).partition('Default').group(group).batch(1).wait(false).pop()
225
- if (empty && empty.length > 0) return { success: false, message: `expected an empty pop, got ${empty.length} messages` }
226
-
227
- const after = await committedOf(part.id, group)
228
- if (after !== Number(rowNow.last_offset)) {
229
- return { success: false, message: `BUG: empty pop did not seal the cursor (committed=${after}, want last_offset=${rowNow.last_offset}) — partition stays phantom-pending forever` }
230
- }
231
-
232
- // Guard: delayed messages are NOT sealed over. Segments exist but are
233
- // deferred; the cursor must stay put so they deliver when eligible.
234
- const dqueue = 'test-queue-v2-ackwindow-seal-delayed'
235
- await client.queue(dqueue).config({ delayedProcessing: 30, leaseTime: 30 }).create()
236
- await client.queue(dqueue).partition('Default').push([
237
- { data: { n: 0 }, transactionId: `${dqueue}-tx-0` },
238
- { data: { n: 1 }, transactionId: `${dqueue}-tx-1` },
239
- ])
240
- // Give the fusion flush a moment so the segments exist before the pop.
241
- const dpart0 = await (async () => {
242
- for (let i = 0; i < 30; i++) {
243
- const p = await partitionRow(dqueue)
244
- if (p && (await segmentCount(p.id)) > 0) return p
245
- await sleep(100)
246
- }
247
- return null
248
- })()
249
- if (!dpart0) return { success: false, message: 'delayed queue segments never appeared' }
250
-
251
- // First contact for g-seal on THIS queue too (metadata is queue-scoped), and
252
- // the 2 delayed messages are already pushed: under the default 'new' the
253
- // cursor would seed at last_offset and the guard below could not tell a seed
254
- // from a seal. 'all' seeds at -1, so a non -1 cursor can only be the seal.
255
- const dempty = await client.queue(dqueue).partition('Default').group(group).subscriptionMode('all').batch(1).wait(false).pop()
256
- if (dempty && dempty.length > 0) return { success: false, message: 'delayed message delivered early?' }
257
- const dcommitted = await committedOf(dpart0.id, group)
258
- const dsegs = await segmentCount(dpart0.id)
259
- if (dsegs === 0) return { success: false, message: 'delayed queue segments vanished' }
260
- if (dcommitted !== -1 && dcommitted !== null) {
261
- return { success: false, message: `BUG: seal fired over deferred-but-present segments (committed=${dcommitted}) — delayed messages would be skipped` }
262
- }
263
-
264
- return { success: true, message: 'empty partition sealed after one pop; deferred segments not sealed over' }
265
- }
@@ -217,8 +217,8 @@ export async function testCgBootstrapTimestamp(client) {
217
217
 
218
218
  // 3. Record cutoff timestamp (after historical, before new messages)
219
219
  // Wait before capturing the cutoff to handle clock skew between the
220
- // client (Date.now) and PostgreSQL (NOW()). Without this, the last
221
- // push batch may have a PG created_at AFTER the client-side cutoff.
220
+ // client (Date.now) and the broker's clock. Without this, the last
221
+ // push batch may have a broker created_at AFTER the client-side cutoff.
222
222
  await new Promise(resolve => setTimeout(resolve, 2000))
223
223
  const cutoffTimestamp = new Date().toISOString()
224
224
  console.log(` Cutoff timestamp: ${cutoffTimestamp}`)
@@ -310,18 +310,6 @@ describe('conflation — declaration conflict', () => {
310
310
  assert.equal(warnings.filter(w => w.includes('Conflation.conflict')).length, 1)
311
311
  })
312
312
  })
313
-
314
- it('pop maintenance is not a version skew', async () => {
315
- // `{"messages":[],"paused":true}` means an operator turned pops off. The
316
- // request never reached the claim path, so there is no echo to expect —
317
- // raising here would stop every conflating consumer in the fleet on a
318
- // routine operator action.
319
- const paused = ok({ messages: [], paused: true })
320
- await withQueen([paused], paused, async (queen) => {
321
- const messages = await queen.queue(QUEUE).group(GROUP).conflation(true).pop()
322
- assert.equal(messages.length, 0)
323
- })
324
- })
325
313
  })
326
314
 
327
315
  // ---------------------------------------------------------------------------
@@ -708,4 +708,44 @@ export async function testConsumerMultiPartitionGlobalCap(client) {
708
708
  }
709
709
 
710
710
  return { success: true, message: 'Global batch cap respected across multi-partition consume' }
711
- }
711
+ }
712
+ // A handler that throws under autoAck(false) (2026-10-02, 2.0.0-beta.6): the
713
+ // consumer stopped at the first throw, consume() rejected, and the message
714
+ // stayed leased until its lease ran out. It now nacks what the handler was
715
+ // given and keeps consuming: the broker redelivers, the handler acks it.
716
+ export async function manualAckHandlerErrorKeepsConsuming(client) {
717
+ const queueName = 'test-queue-v2-manual-ack-throw'
718
+ const group = 'test-manual-ack-throw'
719
+ const queue = await client.queue(queueName).create()
720
+ if (!queue.configured) {
721
+ return { success: false, message: 'Queue not created' }
722
+ }
723
+ await client.queue(queueName).push([{ data: { id: 1 } }, { data: { id: 2 } }])
724
+
725
+ const seen = []
726
+ try {
727
+ await client
728
+ .queue(queueName)
729
+ .group(group)
730
+ .subscriptionMode('all')
731
+ .autoAck(false)
732
+ .each()
733
+ .batch(1)
734
+ .wait(false)
735
+ .limit(3)
736
+ .consume(async (msg) => {
737
+ seen.push(`${msg.data.id}#${msg.deliveryAttempt}`)
738
+ if (seen.length === 1) throw new Error('first delivery fails')
739
+ // No { group }: the ack takes it from the message.
740
+ const res = await client.ack(msg, true)
741
+ if (!res.success) throw new Error(`ack refused: ${res.error}`)
742
+ })
743
+ } catch (error) {
744
+ return { success: false, message: `consume() rejected: ${error.message} (deliveries ${seen.join(', ')})` }
745
+ }
746
+
747
+ const leftover = await client.queue(queueName).group(group).batch(10).wait(false).pop()
748
+ const ids = seen.map(s => s.split('#')[0]).join(',')
749
+ const success = ids === '1,1,2' && leftover.length === 0
750
+ return { success, message: `deliveries ${seen.join(', ')} (expected 1,1,2), ${leftover.length} left unacked` }
751
+ }
@@ -0,0 +1,161 @@
1
+ /**
2
+ * consume(): what a handler that throws does to the consumer.
3
+ *
4
+ * The rule, the same with and without autoAck: the messages the handler was
5
+ * given are NACKED (the broker redelivers them, and files them in the DLQ
6
+ * once the queue's retryLimit is spent) and the worker KEEPS CONSUMING.
7
+ * autoAck(false) hands the success path to the handler, not the failure path:
8
+ * a handler that threw did not get to settle its messages.
9
+ *
10
+ * Before: under autoAck(false) the error left the worker loop, the consume()
11
+ * promise rejected after one message, and the messages stayed leased until
12
+ * the lease expired (found 2026-10-02 against 2.0.0-beta.6). With
13
+ * concurrency > 1 the other workers kept running behind a promise that had
14
+ * already rejected.
15
+ *
16
+ * `.onError(fn)` is the way to decide for yourself: the error never reaches
17
+ * the consumer, so nothing is nacked on your behalf.
18
+ */
19
+
20
+ import { describe, it } from 'node:test'
21
+ import assert from 'node:assert/strict'
22
+ import { createServer } from 'node:http'
23
+
24
+ import { Queen } from '../../client-v2/index.js'
25
+
26
+ const GROUP = 'workers'
27
+
28
+ const message = (n) => ({
29
+ id: `msg-${n}`,
30
+ transactionId: `tx-${n}`,
31
+ partitionId: '7',
32
+ partition: 'p1',
33
+ leaseId: `lease-${n}`,
34
+ consumerGroup: GROUP,
35
+ data: { n },
36
+ createdAt: '2026-10-02T10:00:00.000Z'
37
+ })
38
+
39
+ /**
40
+ * A fake broker routed by path: each pop answers the next batch of `pops`
41
+ * (an empty 204 once they run out), each ack or batch ack is accepted. Every
42
+ * request is recorded.
43
+ */
44
+ async function withBroker(pops, run) {
45
+ const queue = [...pops]
46
+ const requests = []
47
+ const server = createServer((req, res) => {
48
+ let raw = ''
49
+ req.on('data', chunk => { raw += chunk })
50
+ req.on('end', () => {
51
+ const body = raw ? JSON.parse(raw) : null
52
+ const path = req.url.split('?')[0]
53
+ requests.push({ method: req.method, path, body })
54
+ if (req.method === 'GET' && path.startsWith('/api/v1/pop')) {
55
+ const batch = queue.shift()
56
+ if (!batch) { res.writeHead(204); res.end(); return }
57
+ res.writeHead(200, { 'Content-Type': 'application/json' })
58
+ res.end(JSON.stringify({ success: true, consumerGroup: GROUP, messages: batch }))
59
+ return
60
+ }
61
+ if (req.method === 'POST' && (path === '/api/v1/ack' || path === '/api/v1/ack/batch')) {
62
+ const acks = body.acknowledgments || [body]
63
+ res.writeHead(200, { 'Content-Type': 'application/json' })
64
+ res.end(JSON.stringify(acks.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null }))))
65
+ return
66
+ }
67
+ res.writeHead(404, { 'Content-Type': 'application/json' })
68
+ res.end('{"error":"not found"}')
69
+ })
70
+ })
71
+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
72
+ const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, handleSignals: false })
73
+ try {
74
+ await run(queen, requests)
75
+ } finally {
76
+ await queen.close()
77
+ await new Promise(resolve => server.close(resolve))
78
+ }
79
+ }
80
+
81
+ const acksIn = (requests) => requests.filter(r => r.method === 'POST' && r.path.startsWith('/api/v1/ack'))
82
+ const statusesOf = (ackRequest) => (ackRequest.body.acknowledgments || [ackRequest.body]).map(a => a.status)
83
+
84
+ describe('consume() — a handler that throws is nacked, and the consumer keeps going', () => {
85
+ it('autoAck(false), one message at a time: nack, redelivery, done', async () => {
86
+ // Delivery 1 of tx-1 fails, delivery 2 succeeds and the handler acks it.
87
+ await withBroker([[message(1)], [message(1)]], async (queen, requests) => {
88
+ const seen = []
89
+ await queen.queue('orders').group(GROUP).wait(false).autoAck(false).each().limit(2)
90
+ .consume(async (msg) => {
91
+ seen.push(msg.transactionId)
92
+ if (seen.length === 1) throw new Error('boom')
93
+ await queen.ack(msg, true)
94
+ })
95
+
96
+ assert.deepEqual(seen, ['tx-1', 'tx-1'], 'the consumer survived the throw and got the redelivery')
97
+ const acks = acksIn(requests)
98
+ assert.equal(acks.length, 2)
99
+ assert.deepEqual(statusesOf(acks[0]), ['failed'], 'the throw nacked the message')
100
+ assert.equal(acks[0].body.consumerGroup, GROUP)
101
+ assert.equal(acks[0].body.leaseId, 'lease-1', 'the nack names the lease it releases')
102
+ assert.equal(acks[0].body.error, 'boom', 'the handler error travels with the nack')
103
+ assert.deepEqual(statusesOf(acks[1]), ['completed'], 'the handler\'s own ack of the redelivery')
104
+ })
105
+ })
106
+
107
+ it('autoAck(false), batch handler: the whole batch is nacked in one call', async () => {
108
+ await withBroker([[message(1), message(2)], [message(1), message(2)]], async (queen, requests) => {
109
+ let calls = 0
110
+ await queen.queue('orders').group(GROUP).wait(false).autoAck(false).limit(3)
111
+ .consume(async (msgs) => {
112
+ calls++
113
+ if (calls === 1) throw new Error('batch boom')
114
+ await queen.ack(msgs, true)
115
+ })
116
+
117
+ assert.equal(calls, 2)
118
+ const acks = acksIn(requests)
119
+ assert.equal(acks[0].path, '/api/v1/ack/batch')
120
+ assert.deepEqual(statusesOf(acks[0]), ['failed', 'failed'])
121
+ assert.equal(acks[0].body.consumerGroup, GROUP)
122
+ assert.deepEqual(statusesOf(acks[1]), ['completed', 'completed'])
123
+ })
124
+ })
125
+
126
+ it('autoAck(false), .each(): the rest of the popped batch is abandoned after a nack', async () => {
127
+ // The nack released the lease and clamps the cursor at tx-1, so tx-2 comes
128
+ // back with it: handling it now would only produce a duplicate.
129
+ await withBroker([[message(1), message(2)], [message(1), message(2)]], async (queen) => {
130
+ const seen = []
131
+ await queen.queue('orders').group(GROUP).wait(false).autoAck(false).each().limit(3)
132
+ .consume(async (msg) => {
133
+ seen.push(msg.transactionId)
134
+ if (seen.length === 1) throw new Error('boom')
135
+ })
136
+ assert.deepEqual(seen, ['tx-1', 'tx-1', 'tx-2'])
137
+ })
138
+ })
139
+
140
+ it('autoAck(true) behaves the same way: nack and keep going', async () => {
141
+ await withBroker([[message(1)], [message(1)]], async (queen, requests) => {
142
+ let calls = 0
143
+ await queen.queue('orders').group(GROUP).wait(false).each().limit(2)
144
+ .consume(async () => { if (++calls === 1) throw new Error('boom') })
145
+ assert.equal(calls, 2)
146
+ assert.deepEqual(acksIn(requests).map(statusesOf), [['failed'], ['completed']])
147
+ })
148
+ })
149
+
150
+ it('with .onError() the handler decides: nothing is nacked on its behalf', async () => {
151
+ await withBroker([[message(1)], [message(2)]], async (queen, requests) => {
152
+ const failures = []
153
+ await queen.queue('orders').group(GROUP).wait(false).autoAck(false).each().limit(2)
154
+ .consume(async (msg) => { if (msg.transactionId === 'tx-1') throw new Error('boom') })
155
+ .onError(async (msg, err) => { failures.push([msg.transactionId, err.message]) })
156
+
157
+ assert.deepEqual(failures, [['tx-1', 'boom']])
158
+ assert.equal(acksIn(requests).length, 0, 'autoAck(false) + onError: the consumer sent no ack and no nack')
159
+ })
160
+ })
161
+ })
package/test-v2/docs.js CHANGED
@@ -10,8 +10,8 @@
10
10
  *
11
11
  * After editing a marked region, regenerate the partials with
12
12
  * `pnpm --dir webdoc gen` or the docs CI check fails on drift. The queues used
13
- * here (orders, payments, invoices) are wiped by cleanupTestData in run.js
14
- * before every suite run, exactly like the test-% queues.
13
+ * here (orders, payments, invoices) start empty because every suite run gets a
14
+ * fresh broker (test/run.sh), exactly like the test-% queues.
15
15
  */
16
16
 
17
17
  export async function docsProduceAndConsume(client) {
@@ -65,8 +65,9 @@ export async function docsProduceAndConsume(client) {
65
65
  }
66
66
 
67
67
  export async function docsDeduplication(client) {
68
- // The fixed transactionId below survives reruns because cleanupTestData
69
- // purges log_txns for these queues before the suite starts.
68
+ // The fixed transactionId below survives reruns because every run gets a
69
+ // fresh broker (test/run.sh creates its data volume empty and destroys it
70
+ // with `down -v`), so no earlier run's dedup entry is ever there.
70
71
  // docs:start(js-push-dedup)
71
72
  const first = await client
72
73
  .queue('payments')