queen-mq 0.16.0 → 1.0.0

package/test-v2/stream/_helpers.js

@@ -35,7 +35,7 @@ export function mkName(testName, suffix = '') {
   return `test-stream-${slug}-${stamp}-${_nameCounter}${tail}`
 }
 
-export const STREAMS_URL = process.env.QUEEN_URL || 'http://localhost:6632'
+export const STREAMS_URL = process.env.QUEEN_URL || process.env.QUEEN_SERVER_URL || 'http://localhost:6632'
 
 /**
  * Sleep helper.

package/test-v2/stream/tumbling.js

@@ -249,12 +249,16 @@ export async function tumblingIdleFlushClosesQuietPartitions(client) {
   await client.queue(src).create()
   await client.queue(sink).create()
 
-  // Push 3 messages then go silent — without idle flush, the window would
-  // stay open forever. With a 700ms flush cadence and 1-second windows,
-  // the flush should close the window within ~2 seconds of the last push.
-  for (let i = 0; i < 3; i++) {
-    await client.queue(src).partition('quiet').push([{ data: { v: 7 } }])
-  }
+  // Push 3 messages IN ONE BATCH then go silent — without idle flush, the
+  // window would stay open forever. One push call = one segment = one
+  // timestamp, so the three messages cannot straddle a 1-second window
+  // boundary. Three sequential pushes could land at x.98s / x+1.01s, splitting
+  // the window: emits[0] then reads {count:2, sum:14} and the FIRST window is
+  // closed by the cycle path instead of the idle flush (flushCyclesTotal 0) —
+  // observed live, ~1 run in 5 under load.
+  await client.queue(src).partition('quiet').push([
+    { data: { v: 7 } }, { data: { v: 7 } }, { data: { v: 7 } }
+  ])
 
   const handle = await Stream
     .from(client.queue(src))

package/test-v2/streams-unit/ack.test.js

@@ -0,0 +1,203 @@
+/**
+ * Queen.ack() wire-format tests.
+ *
+ * The server (routes/ack.cpp + queen.ack_messages_v2) responds to both
+ * /api/v1/ack and /api/v1/ack/batch with a top-level JSON array, one item per
+ * acknowledgment in request order:
+ *
+ *   [{"index":0,"transactionId":"...","success":false,"error":"Invalid or expired lease",
+ *     "queueName":"q","partitionName":"Default","leaseReleased":false,"dlq":false}]
+ *
+ * These tests pin that wire format so a rejected ack/nack (e.g. "Invalid or
+ * expired lease", "Message not found") is never reported as success: true.
+ * Mirror of clients/client-go/ack_test.go.
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { createServer } from 'node:http'
+
+import { Queen } from '../../client-v2/index.js'
+
+function ackResultItem(index, transactionId, success, error = null) {
+  return {
+    index,
+    transactionId,
+    success,
+    error,
+    queueName: 'test-queue',
+    partitionName: 'Default',
+    leaseReleased: success,
+    dlq: false
+  }
+}
+
+/**
+ * Spin up a real HTTP server, point a Queen client at it, run the test body,
+ * then tear both down. respond(body) returns the JSON payload to send back.
+ * Requests are recorded as { path, body } for assertions.
+ */
+async function withAckServer(respond, run) {
+  const requests = []
+  const server = createServer((req, res) => {
+    let raw = ''
+    req.on('data', chunk => { raw += chunk })
+    req.on('end', () => {
+      const body = raw ? JSON.parse(raw) : {}
+      requests.push({ path: req.url, body })
+      res.writeHead(200, { 'Content-Type': 'application/json' })
+      res.end(JSON.stringify(respond(body)))
+    })
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  const { port } = server.address()
+
+  const client = new Queen({
+    url: `http://127.0.0.1:${port}`,
+    handleSignals: false,
+    retryAttempts: 1
+  })
+
+  try {
+    await run(client, requests)
+  } finally {
+    await client.close()
+    await new Promise(resolve => server.close(resolve))
+  }
+}
+
+const MSG = { transactionId: 'tx-1', partitionId: 'p-1', leaseId: 'lease-1' }
+
+describe('Queen.ack single (/api/v1/ack)', () => {
+  it('reports success for an accepted ack', async () => {
+    await withAckServer(
+      () => [ackResultItem(0, 'tx-1', true)],
+      async (client, requests) => {
+        const result = await client.ack(MSG, true)
+        assert.equal(result.success, true)
+        assert.equal(result.error, null)
+        assert.equal(result.transactionId, 'tx-1')
+        assert.equal(requests[0].path, '/api/v1/ack')
+        assert.equal(requests[0].body.status, 'completed')
+      }
+    )
+  })
+
+  it('reports failure when the ack is rejected for an expired lease', async () => {
+    await withAckServer(
+      () => [ackResultItem(0, 'tx-1', false, 'Invalid or expired lease')],
+      async (client) => {
+        const result = await client.ack(MSG, true)
+        assert.equal(result.success, false)
+        assert.equal(result.error, 'Invalid or expired lease')
+      }
+    )
+  })
+
+  it('reports failure when a nack is rejected for a missing message', async () => {
+    await withAckServer(
+      () => [ackResultItem(0, 'tx-1', false, 'Message not found')],
+      async (client, requests) => {
+        const result = await client.ack(MSG, false)
+        assert.equal(result.success, false)
+        assert.equal(result.error, 'Message not found')
+        assert.equal(requests[0].body.status, 'failed')
+      }
+    )
+  })
+
+  it('still fails when rejected without an error string', async () => {
+    await withAckServer(
+      () => [ackResultItem(0, 'tx-1', false, null)],
+      async (client) => {
+        const result = await client.ack(MSG, true)
+        assert.equal(result.success, false)
+        assert.equal(result.error, 'Acknowledgment rejected by server')
+      }
+    )
+  })
+
+  it('propagates a top-level error envelope', async () => {
+    await withAckServer(
+      () => ({ error: 'internal error' }),
+      async (client) => {
+        const result = await client.ack(MSG, true)
+        assert.equal(result.success, false)
+        assert.equal(result.error, 'internal error')
+      }
+    )
+  })
+
+  it('fails on a response that is neither a result array nor an error envelope', async () => {
+    await withAckServer(
+      () => ({ ok: true }),
+      async (client) => {
+        const result = await client.ack(MSG, true)
+        assert.equal(result.success, false)
+        assert.match(result.error, /Unexpected ack response format/)
+      }
+    )
+  })
+})
+
+describe('Queen.ack batch (/api/v1/ack/batch)', () => {
+  const MSGS = [
+    { transactionId: 'tx-1', partitionId: 'p-1', leaseId: 'lease-1' },
+    { transactionId: 'tx-2', partitionId: 'p-1', leaseId: 'lease-1' }
+  ]
+
+  it('reports success when every acknowledgment is accepted', async () => {
+    await withAckServer(
+      () => [ackResultItem(0, 'tx-1', true), ackResultItem(1, 'tx-2', true)],
+      async (client, requests) => {
+        const result = await client.ack(MSGS, true)
+        assert.equal(result.success, true)
+        assert.equal(result.processed, 2)
+        assert.equal(result.results.length, 2)
+        assert.equal(requests[0].path, '/api/v1/ack/batch')
+        assert.equal(requests[0].body.acknowledgments.length, 2)
+      }
+    )
+  })
+
+  it('reports failure with per-item results on mixed outcomes', async () => {
+    await withAckServer(
+      () => [
+        ackResultItem(0, 'tx-1', true),
+        ackResultItem(1, 'tx-2', false, 'Invalid or expired lease')
+      ],
+      async (client) => {
+        const result = await client.ack(MSGS, true)
+        assert.equal(result.success, false)
+        assert.equal(result.error, 'Invalid or expired lease')
+        assert.equal(result.results[0].success, true)
+        assert.equal(result.results[1].success, false)
+        assert.equal(result.results[1].error, 'Invalid or expired lease')
+      }
+    )
+  })
+
+  it('fails when the server returns fewer results than acknowledgments sent', async () => {
+    await withAckServer(
+      () => [ackResultItem(0, 'tx-1', true)],
+      async (client) => {
+        const result = await client.ack(MSGS, true)
+        assert.equal(result.success, false)
+        assert.match(result.error, /1 results, expected 2/)
+      }
+    )
+  })
+
+  it('propagates a top-level error envelope to every acknowledgment', async () => {
+    await withAckServer(
+      () => ({ error: 'internal error' }),
+      async (client) => {
+        const result = await client.ack(MSGS, true)
+        assert.equal(result.success, false)
+        assert.equal(result.error, '2 of 2 acknowledgments rejected: internal error')
+        assert.equal(result.results.length, 2)
+        assert.equal(result.results[0].error, 'internal error')
+      }
+    )
+  })
+})

package/test-v2/streams-unit/e2e.test.js

@@ -12,7 +12,10 @@
  *
  * Prerequisites:
  *   - Queen running on QUEEN_URL (default http://localhost:6632)
- *   - The schema procedures (019..022) applied (auto-loaded on Queen boot)
+ *   - The streams procedures applied (auto-loaded on Queen boot):
+ *     002_streams_schema, 007_log_streams, 008_streams_register_query_v1,
+ *     009_streams_state_get_v1 (the retired rows-era streams cycle's role
+ *     now lives in 007_log_streams' log_streams_cycle_v1)
  *
  * Run:
  *   QUEEN_URL=http://localhost:6632 node --test test/e2e.test.js

package/test-v2/transaction.js

@@ -14,12 +14,14 @@ export async function transactionBasicPushAck(client) {
         return { success: false, message: 'No message to consume' }
     }
 
+    // docs:start(js-transaction)
     await client
         .transaction()
         .queue('test-queue-v2-txn-basic-b')
         .push([{ data: { value: messages[0].data.value + 1 } }])
         .ack(messages[0])
         .commit()
+    // docs:end
 
     // Verify message moved to B
     const resultB = await client.queue('test-queue-v2-txn-basic-b').batch(1).wait(false).pop()

package/test-v2/watermark.js

@@ -1,25 +1,36 @@
 /**
- * Watermark Bug Tests
+ * Seek / delete re-consumption tests (behaviour, not bookkeeping)
  *
- * These tests verify that consumer_watermarks are properly reset when:
- * 1. Seeking a consumer group backwards to a timestamp
- * 2. Deleting a consumer group (watermark should be cleaned up)
+ * These tests verify the BEHAVIOUR a user cares about:
+ *   1. Seeking a consumer group backwards re-exposes the old messages, and the
+ *      group re-consumes them from the start within a short bound.
+ *   2. Deleting a consumer group (all queues, or for one queue) lets a group of
+ *      the same name consume every message again from the start.
  *
- * The consumer_watermarks table optimizes POP by tracking when a consumer
- * last found the queue empty. However, if the watermark isn't reset when
- * the cursor moves backwards, old messages become invisible.
+ * WHY THIS WAS REWRITTEN (2026-07-24)
+ * -----------------------------------
+ * The consumer_watermarks table is a LEGACY-path optimisation: it records when a
+ * consumer last found a queue empty so an indexed empty-scan can be skipped. The
+ * original tests asserted the *row* was created after an empty poll and then aged
+ * it to exercise the 2-minute buffer:
+ *     pl.updated_at >= (v_watermark - interval '2 minutes')
  *
- * BUG: Currently, seek and delete operations don't touch consumer_watermarks,
- * causing re-consumption to fail silently.
+ * With the wildcard hot-list ON (QUEEN_HOTLIST=1, the default) empty polls are
+ * answered from the in-memory candidate ring and that bookkeeping row is NEVER
+ * written — by design, not a bug: the ring, not the watermark, is the discovery
+ * source. So asserting the row exists fails on the hot-list path even though
+ * re-consumption works perfectly.
  *
- * IMPORTANT: The watermark filter has a 2-minute buffer:
- *   pl.updated_at >= (v_watermark - interval '2 minutes')
- *
- * So the bug only manifests after 2+ minutes have passed. To test this without
- * waiting, we directly manipulate the watermark timestamp in the database.
+ * The behaviour is identical on both engines, so these tests now assert the
+ * behaviour directly and only exercise the watermark-age scenario on the LEGACY
+ * path — detected at runtime by whether the empty poll actually wrote the row
+ * (no env flag needed). On the hot-list path the broker's seek/delete handlers
+ * re-seed / invalidate the ring so the reconsume is immediate; the idle-bounded
+ * consume below (idleMillis 3000) is what proves "within a short bound".
  *
  * Run: node run.js seekBackwardsAllowsReconsume
  *      node run.js deleteConsumerGroupAllowsReconsume
+ *      node run.js deleteConsumerGroupForQueueAllowsReconsume
  */
 
 import { dbPool } from './run.js'
@@ -40,11 +51,12 @@ const QUEUE_NAME_DELETE = 'test-queue-v2-watermark-delete'
  */
 async function advanceWatermark(queueName, consumerGroup, minutesForward = 10) {
     const result = await dbPool.query(`
-        UPDATE queen.consumer_watermarks
+        UPDATE queen.consumer_watermarks w
         SET last_empty_scan_at = NOW() + interval '${minutesForward} minutes',
             updated_at = NOW() + interval '${minutesForward} minutes'
-        WHERE queue_name = $1 AND consumer_group = $2
-        RETURNING *
+        FROM queen.queues q
+        WHERE w.queue_id = q.id AND q.name = $1 AND w.consumer_group = $2
+        RETURNING w.*
     `, [queueName, consumerGroup])
     return result.rows[0]
 }
@@ -54,8 +66,9 @@ async function advanceWatermark(queueName, consumerGroup, minutesForward = 10) {
  */
 async function getWatermark(queueName, consumerGroup) {
     const result = await dbPool.query(`
-        SELECT * FROM queen.consumer_watermarks
-        WHERE queue_name = $1 AND consumer_group = $2
+        SELECT w.* FROM queen.consumer_watermarks w
+        JOIN queen.queues q ON q.id = w.queue_id
+        WHERE q.name = $1 AND w.consumer_group = $2
     `, [queueName, consumerGroup])
     return result.rows[0]
 }
@@ -151,20 +164,20 @@ export async function seekBackwardsAllowsReconsume(client) {
 
     console.log(`  Verified queue is empty (watermark should be set)`)
 
-    // 6. Check watermark was created
+    // 6. Detect which discovery path we are on. On the LEGACY path the empty poll
+    //    wrote a consumer_watermarks row; on the hot-list path it did not (empty
+    //    polls are served from the ring — by design). We do NOT assert the row: it
+    //    is a path-specific bookkeeping detail, not the behaviour under test.
     const watermarkBefore = await getWatermark(QUEUE_NAME_SEEK, consumerGroup)
-    if (!watermarkBefore) {
-        return {
-            success: false,
-            message: 'Watermark was not created after empty queue check'
-        }
+    if (watermarkBefore) {
+        // Legacy path: age the watermark 10 minutes into the future so the old
+        // partitions fall outside the 2-minute buffer — the original repro that
+        // proves seek must reset the watermark.
+        const advancedWatermark = await advanceWatermark(QUEUE_NAME_SEEK, consumerGroup, 10)
+        console.log(`  [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
+    } else {
+        console.log(`  [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
     }
-    console.log(`  Watermark exists: last_empty_scan_at = ${watermarkBefore.last_empty_scan_at}`)
-
-    // 7. CRITICAL: Advance the watermark forward to simulate that partitions are "old"
-    //    relative to the watermark. This triggers the bug where old partitions are filtered out.
-    const advancedWatermark = await advanceWatermark(QUEUE_NAME_SEEK, consumerGroup, 10)
-    console.log(`  Advanced watermark to: ${advancedWatermark.last_empty_scan_at} (10 minutes in future)`)
 
     // 8. Seek backwards to BEFORE the messages were pushed
     console.log(`  Seeking to timestamp: ${timestampBeforeMessages}`)
@@ -203,12 +216,17 @@ export async function seekBackwardsAllowsReconsume(client) {
 
     console.log(`  Second consume (after seek): ${secondConsumeCount} messages`)
 
-    // 11. Verify we got all messages back
+    // 11. Verify we got all messages back within the idle-bounded consume above.
+    //     A shortfall means the backward seek did not re-expose the messages
+    //     promptly: on the legacy path the consumer_watermarks row was not reset
+    //     (still at ${watermarkAfterSeek?.last_empty_scan_at}); on the hot-list path
+    //     the ring was not re-seeded, so reconsume would only resume at the ≤30s
+    //     periodic floor instead of immediately.
     if (secondConsumeCount !== messageCount) {
         return {
             success: false,
-            message: `After seek backwards, should re-consume ${messageCount} messages, but got ${secondConsumeCount}. ` +
-                     `This is because consumer_watermarks was not reset on seek (watermark still at ${watermarkAfterSeek?.last_empty_scan_at}).`
+            message: `After seek backwards, should re-consume ${messageCount} messages within the idle window, but got ${secondConsumeCount}. ` +
+                     `(legacy watermark after seek: ${watermarkAfterSeek?.last_empty_scan_at})`
         }
     }
 
@@ -304,19 +322,17 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
 
     console.log(`  Verified queue is empty (watermark should be set)`)
 
-    // 5. Check watermark was created
+    // 5. Detect the discovery path (see seekBackwardsAllowsReconsume). The
+    //    watermark row is a legacy-path artefact, not the behaviour under test, so
+    //    we do not assert it — we only exercise the watermark-age scenario when the
+    //    row was actually written (legacy path).
     const watermarkBefore = await getWatermark(QUEUE_NAME_DELETE, consumerGroup)
-    if (!watermarkBefore) {
-        return {
-            success: false,
-            message: 'Watermark was not created after empty queue check'
-        }
+    if (watermarkBefore) {
+        const advancedWatermark = await advanceWatermark(QUEUE_NAME_DELETE, consumerGroup, 10)
+        console.log(`  [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
+    } else {
+        console.log(`  [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
     }
-    console.log(`  Watermark exists: last_empty_scan_at = ${watermarkBefore.last_empty_scan_at}`)
-
-    // 6. CRITICAL: Advance the watermark forward to simulate that partitions are "old"
-    const advancedWatermark = await advanceWatermark(QUEUE_NAME_DELETE, consumerGroup, 10)
-    console.log(`  Advanced watermark to: ${advancedWatermark.last_empty_scan_at} (10 minutes in future)`)
 
     // 7. Delete the consumer group
     console.log(`  Deleting consumer group: ${consumerGroup}`)
@@ -353,12 +369,16 @@ export async function deleteConsumerGroupAllowsReconsume(client) {
 
     console.log(`  Second consume (after delete + recreate): ${secondConsumeCount} messages`)
 
-    // 10. Verify we got all messages
+    // 10. Verify we got all messages within the idle-bounded consume above. A
+    //     shortfall means the delete left stale discovery state: a legacy orphaned
+    //     watermark (at ${watermarkAfterDelete?.last_empty_scan_at}), or, on the
+    //     hot-list path, a surviving group ring / stale seeded-groups cache that
+    //     suppressed the from-the-start reconsume.
     if (secondConsumeCount !== messageCount) {
         return {
             success: false,
-            message: `After deleting CG, new CG with same name should consume ${messageCount} messages, ` +
-                     `but got ${secondConsumeCount}. Orphaned watermark at ${watermarkAfterDelete?.last_empty_scan_at}.`
+            message: `After deleting CG, new CG with same name should consume ${messageCount} messages within the idle window, ` +
+                     `but got ${secondConsumeCount}. (legacy watermark after delete: ${watermarkAfterDelete?.last_empty_scan_at})`
         }
     }
 
@@ -382,7 +402,7 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
         await client.admin.deleteConsumerGroupForQueue(consumerGroup, queueName, true)
         // Also clean up any orphaned watermark from previous runs
         await dbPool.query(
-            'DELETE FROM queen.consumer_watermarks WHERE queue_name = $1 AND consumer_group = $2',
+            'DELETE FROM queen.consumer_watermarks w USING queen.queues q WHERE w.queue_id = q.id AND q.name = $1 AND w.consumer_group = $2',
             [queueName, consumerGroup]
         )
     } catch (e) {
@@ -446,19 +466,15 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
 
     console.log(`  Verified queue is empty (watermark should be set)`)
 
-    // 5. Check watermark was created
+    // 5. Detect the discovery path (see seekBackwardsAllowsReconsume). The
+    //    watermark row is a legacy-path artefact, not the behaviour under test.
     const watermarkBefore = await getWatermark(queueName, consumerGroup)
-    if (!watermarkBefore) {
-        return {
-            success: false,
-            message: 'Watermark was not created after empty queue check'
-        }
+    if (watermarkBefore) {
+        const advancedWatermark = await advanceWatermark(queueName, consumerGroup, 10)
+        console.log(`  [legacy] Watermark row exists; advanced to ${advancedWatermark.last_empty_scan_at} (10m future)`)
+    } else {
+        console.log(`  [hotlist] No watermark row (empty polls served from the ring) — skipping the watermark-age step`)
     }
-    console.log(`  Watermark exists: last_empty_scan_at = ${watermarkBefore.last_empty_scan_at}`)
-
-    // 6. CRITICAL: Advance the watermark forward
-    const advancedWatermark = await advanceWatermark(queueName, consumerGroup, 10)
-    console.log(`  Advanced watermark to: ${advancedWatermark.last_empty_scan_at} (10 minutes in future)`)
 
     // 7. Delete consumer group for this specific queue
     console.log(`  Deleting consumer group for queue: ${consumerGroup} / ${queueName}`)
@@ -495,8 +511,8 @@ export async function deleteConsumerGroupForQueueAllowsReconsume(client) {
     if (secondConsumeCount !== messageCount) {
         return {
             success: false,
-            message: `After deleting CG for queue, should consume ${messageCount} messages, ` +
-                     `but got ${secondConsumeCount}. Orphaned watermark at ${watermarkAfterDelete?.last_empty_scan_at}.`
+            message: `After deleting CG for queue, should consume ${messageCount} messages within the idle window, ` +
+                     `but got ${secondConsumeCount}. (legacy watermark after delete: ${watermarkAfterDelete?.last_empty_scan_at})`
         }
     }