queen-mq 2.0.0 → 2.0.3

package/client-v2/http/HttpClient.js

@@ -4,6 +4,47 @@
 
 import * as logger from '../utils/logger.js'
 
+/**
+ * The error a request ends with when its caller aborted it (a consumer being
+ * stopped). It says nothing about the backend: it is never retried, never
+ * fails over to another node, and never marks a node unhealthy.
+ */
+function callerAborted(method, url) {
+  const error = new Error(`${method} ${url} aborted by the caller`)
+  error.name = 'AbortError'
+  error.aborted = true
+  return error
+}
+
+/**
+ * Wait `delay` ms before another attempt, unless the caller aborts first: then
+ * reject at once with the caller-aborted error instead of sitting out the wait.
+ */
+function waitBeforeRetry(delay, signal, method, url) {
+  if (signal?.aborted) return Promise.reject(callerAborted(method, url))
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      clearTimeout(timer)
+      reject(callerAborted(method, url))
+    }
+    const timer = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort)
+      resolve()
+    }, delay)
+    signal?.addEventListener('abort', onAbort, { once: true })
+  })
+}
+
+/** A pop answer carries leases: once it is arriving, it is read to the end. */
+function isPop(method, url) {
+  if (method !== 'GET') return false
+  try {
+    return new URL(url).pathname.startsWith('/api/v1/pop')
+  } catch {
+    return false
+  }
+}
+
 // --------------------------------------------------------------------------
 // Host-routed proxy support (queen_proxy selects the tenant cluster from the
 // first DNS label of the Host header -- proxy/src/cache.rs
@@ -211,14 +252,15 @@ export class HttpClient {
    * only status this layer treats as retryable; 5xx/network retry and
    * cross-backend failover are handled by the caller.
    */
-  async #executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind) {
+  async #executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind, signal = null) {
     const { maxAttempts, baseMs, capMs } = this.#retry429PolicyFor(retryKind)
     let tries = 0
     // eslint-disable-next-line no-constant-condition
     while (true) {
+      if (signal?.aborted) throw callerAborted(method, url)
       tries++
       try {
-        return await this.#executeRequest(url, method, body, requestTimeoutMillis)
+        return await this.#executeRequest(url, method, body, requestTimeoutMillis, signal)
       } catch (error) {
         if (error.status !== 429) throw error
 
@@ -229,7 +271,7 @@ export class HttpClient {
 
         const delay = this.#computeRetry429DelayMs(tries - 1, error.retryAfterSeconds, baseMs, capMs)
         logger.warn('HttpClient.retry429', { method, url, attempt: tries, retryKind: retryKind || 'default', nextDelayMs: delay, retryAfterSeconds: error.retryAfterSeconds ?? null, code: error.code ?? null })
-        await new Promise(resolve => setTimeout(resolve, delay))
+        await waitBeforeRetry(delay, signal, method, url)
       }
     }
   }
@@ -323,12 +365,17 @@ export class HttpClient {
     }
   }
 
-  async #executeRequest(url, method, body = null, requestTimeoutMillis = null) {
+  async #executeRequest(url, method, body = null, requestTimeoutMillis = null, signal = null) {
+    if (signal?.aborted) throw callerAborted(method, url)
     const effectiveTimeout = requestTimeoutMillis || this.#timeoutMillis
     logger.log('HttpClient.request', { method, url, hasBody: !!body, timeout: effectiveTimeout, host: this.#hostOverride ? this.#hostOverride.authority : undefined })
 
     const controller = new AbortController()
     const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout)
+    // The caller's signal closes the request too: a long poll its consumer no
+    // longer wants must not stay open for the broker to hand it a message.
+    const abortFromCaller = () => controller.abort()
+    signal?.addEventListener('abort', abortFromCaller, { once: true })
 
     try {
       const headers = { 'Content-Type': 'application/json' }
@@ -367,6 +414,14 @@ export class HttpClient {
 
       const response = await fetch(requestUrl, options)
 
+      // The broker granted the leases of a pop when it sent these headers. From
+      // here the caller's abort no longer cuts the read: the body says which
+      // messages this consumer holds, and it needs them to give them back. The
+      // timeout still bounds the read.
+      if (response.ok && isPop(method, url)) {
+        signal?.removeEventListener('abort', abortFromCaller)
+      }
+
       logger.log('HttpClient.response', { method, url, status: response.status })
 
       // Handle 204 No Content
@@ -421,9 +476,16 @@ export class HttpClient {
         }
       }
 
-      return response.json()
+      // Awaited here, not returned: the body is still being read, and both the
+      // timeout (cleared in `finally`) and the abort classification below must
+      // cover that read too.
+      return await response.json()
 
     } catch (error) {
+      if (signal?.aborted) {
+        logger.log('HttpClient.request', { method, url, status: 'aborted-by-caller' })
+        throw callerAborted(method, url)
+      }
       if (error.name === 'AbortError') {
         const timeoutError = new Error(`Request timeout after ${effectiveTimeout}ms`)
         timeoutError.name = 'AbortError'
@@ -441,18 +503,20 @@ export class HttpClient {
       logger.error('HttpClient.request', { method, url, error: error.message })
       throw error
     } finally {
+      signal?.removeEventListener('abort', abortFromCaller)
       clearTimeout(timeoutId)
     }
   }
 
-  async #requestWithRetry(method, path, body = null, requestTimeoutMillis = null, retryKind = null) {
+  async #requestWithRetry(method, path, body = null, requestTimeoutMillis = null, retryKind = null, signal = null) {
     let lastError = null
 
     for (let attempt = 0; attempt < this.#retryAttempts; attempt++) {
       try {
         const url = this.#getUrl() + path
-        return await this.#executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind)
+        return await this.#executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind, signal)
       } catch (error) {
+        if (error.aborted) throw error
         lastError = error
 
         // Don't retry on client errors (4xx)
@@ -464,7 +528,7 @@ export class HttpClient {
         if (attempt < this.#retryAttempts - 1) {
           const delay = this.#retryDelayMillis * Math.pow(2, attempt)
           logger.warn('HttpClient.retry', { method, path, attempt: attempt + 1, delay, error: error.message })
-          await new Promise(resolve => setTimeout(resolve, delay))
+          await waitBeforeRetry(delay, signal, method, path)
         }
       }
     }
@@ -473,9 +537,9 @@ export class HttpClient {
     throw lastError
   }
 
-  async #requestWithFailover(method, path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {
+  async #requestWithFailover(method, path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
     if (!this.#loadBalancer || !this.#enableFailover) {
-      return this.#requestWithRetry(method, path, body, requestTimeoutMillis, retryKind)
+      return this.#requestWithRetry(method, path, body, requestTimeoutMillis, retryKind, signal)
     }
 
     const urls = this.#loadBalancer.getAllUrls()
@@ -498,13 +562,16 @@ export class HttpClient {
         // 429s are retried in place (same backend, backoff-paced) inside
         // #executeWithRetry429 -- they are not a backend-health signal, so
         // they must not trigger failover to a different server.
-        const result = await this.#executeWithRetry429(url + path, method, body, requestTimeoutMillis, retryKind)
+        const result = await this.#executeWithRetry429(url + path, method, body, requestTimeoutMillis, retryKind, signal)
 
         // Mark backend as healthy on success
         this.#loadBalancer.markHealthy(url)
 
         return result
       } catch (error) {
+        // Stopped by its caller: this node did nothing wrong, and another
+        // node must not get the request instead.
+        if (error.aborted) throw error
         lastError = error
 
         // Mark backend as unhealthy on failure (5xx or network errors)
@@ -538,8 +605,10 @@ export class HttpClient {
   // `retryKind`: pass 'pop' for long-poll (wait=true) pop requests to get the
   // unbounded-with-backoff 429 policy; omit for everything else (push, admin
   // calls, non-waiting pop), which get the bounded default (10 attempts).
-  async get(path, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {
-    return this.#requestWithFailover('GET', path, null, requestTimeoutMillis, affinityKey, retryKind)
+  // `signal`: aborting it closes the request, which then rejects with an
+  // error carrying `aborted: true` -- never retried and never failed over.
+  async get(path, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
+    return this.#requestWithFailover('GET', path, null, requestTimeoutMillis, affinityKey, retryKind, signal)
   }
 
   async post(path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {

package/client-v2/utils/defaults.js

@@ -81,9 +81,10 @@ export const CONSUME_DEFAULTS = {
 // As in CONSUME_DEFAULTS, batch is the autopilot-OFF default.
 export const POP_DEFAULTS = {
   batch: 1,                            // One message (autopilot off only)
-  wait: false,                         // No long polling (immediate return)
+  wait: true,                          // Long polling, as every pop has done; .wait(false) returns at once
   timeoutMillis: 30000,                // 30 seconds if wait=true
-  autoAck: false                       // Server-side auto-ack (false = manual ack required)
+  autoAck: false,                      // Never sent: autoAck() is consume()'s ack after the handler
+  commitOnDelivery: false              // Leased; true sends autoAck=true, the broker's at-most-once commit at delivery
 }
 
 export const BUFFER_DEFAULTS = {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "2.0.0",
+  "version": "2.0.3",
   "type": "module",
   "description": "Partitioned message queue on a replicated broker log — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
     "test": "npm run test:unit && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/consumer-unit/stopOnAbort.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",

package/test-v2/admin-unit/removedRoutes.test.js

@@ -0,0 +1,63 @@
+/**
+ * Admin methods whose route the 2.x broker does not have.
+ *
+ * clearQueue() sent DELETE /api/v1/queues/:name/clear and moveMessageToDLQ()
+ * sent POST /api/v1/messages/:partitionId/:transactionId/dlq. Neither route is
+ * registered (server/src/rsm/facade/real/phase2/reads.rs: the messages family
+ * is GET, DELETE and POST .../retry; there is no /api/v1/queues/... family), so
+ * a 2.x broker answers both with 404 no_such_route, and the caller saw
+ * "not found". Both now throw before any request, naming the way that works.
+ *
+ * Same style as conflation-unit/conflationWire.test.js: a real node:http
+ * server records every request, so "before any request" is asserted on the
+ * socket, not on a mock.
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { Queen } from '../../client-v2/index.js'
+import { withPlanServer } from '../kv-unit/_planServer.js'
+
+const notFound = { status: 404, body: { code: 'no_such_route', error: 'not found' } }
+
+async function withQueen(run) {
+  await withPlanServer([], notFound, async (url, hits) => {
+    const queen = new Queen({ url, handleSignals: false })
+    try {
+      await run(queen, hits)
+    } finally {
+      await queen.close()
+    }
+  })
+}
+
+describe('Admin — routes the 2.x broker does not have', () => {
+  it('moveMessageToDLQ() throws before any request and names the dlq ack', async () => {
+    await withQueen(async (queen, hits) => {
+      await assert.rejects(
+        queen.admin.moveMessageToDLQ('7', 'tx-1'),
+        (err) => {
+          assert.match(err.message, /no route/)
+          assert.match(err.message, /queen\.ack\(message, 'dlq', \{ group \}\)/)
+          return true
+        }
+      )
+      assert.equal(hits.length, 0, 'no request was sent')
+    })
+  })
+
+  it('clearQueue() throws before any request and names the seek to the end', async () => {
+    await withQueen(async (queen, hits) => {
+      await assert.rejects(
+        queen.admin.clearQueue('orders', 'p1'),
+        (err) => {
+          assert.match(err.message, /no route/)
+          assert.match(err.message, /seekConsumerGroup\(group, 'orders', \{ toEnd: true \}\)/)
+          return true
+        }
+      )
+      assert.equal(hits.length, 0, 'no request was sent')
+    })
+  })
+})

package/test-v2/consumer-unit/nackScope.test.js

@@ -0,0 +1,88 @@
+/**
+ * each(): a nack releases ONE partition. A multi-partition pop claims several
+ * partitions under one lease; when the handler fails a message, the nack
+ * releases that message's partition and clamps its cursor, so the later
+ * messages of THAT partition come back on the next pop. The other partitions
+ * are still leased to this worker: their messages must be handled now.
+ *
+ * Before: the loop abandoned the whole popped batch after a nack. The other
+ * partitions' messages stayed leased and came back only when the lease
+ * expired (found live 2026-10-06 against 2.0.1: B1 and B2 waited the whole
+ * 6 s lease after A1 failed).
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { createServer } from 'node:http'
+
+import { Queen } from '../../client-v2/index.js'
+
+const GROUP = 'workers'
+
+const message = (partition, n) => ({
+  id: `msg-${partition}${n}`,
+  transactionId: `tx-${partition}${n}`,
+  partitionId: `pid-${partition}`,
+  partition,
+  leaseId: 'lease-1',
+  consumerGroup: GROUP,
+  data: { tag: `${partition}${n}` },
+  createdAt: '2026-10-06T10:00:00.000Z'
+})
+
+async function withBroker(pops, run) {
+  const queue = [...pops]
+  const requests = []
+  const server = createServer((req, res) => {
+    let raw = ''
+    req.on('data', chunk => { raw += chunk })
+    req.on('end', () => {
+      const body = raw ? JSON.parse(raw) : null
+      const path = req.url.split('?')[0]
+      requests.push({ method: req.method, path, body })
+      if (req.method === 'GET' && path.startsWith('/api/v1/pop')) {
+        const batch = queue.shift()
+        if (!batch) { res.writeHead(204); res.end(); return }
+        res.writeHead(200, { 'Content-Type': 'application/json' })
+        res.end(JSON.stringify({ success: true, consumerGroup: GROUP, messages: batch }))
+        return
+      }
+      if (req.method === 'POST' && (path === '/api/v1/ack' || path === '/api/v1/ack/batch')) {
+        const acks = body.acknowledgments || [body]
+        res.writeHead(200, { 'Content-Type': 'application/json' })
+        res.end(JSON.stringify(acks.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null }))))
+        return
+      }
+      res.writeHead(404, { 'Content-Type': 'application/json' })
+      res.end('{"error":"not found"}')
+    })
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  const queen = new Queen({ url: `http://127.0.0.1:${server.address().port}`, handleSignals: false })
+  try {
+    await run(queen, requests)
+  } finally {
+    await queen.close()
+    await new Promise(resolve => server.close(resolve))
+  }
+}
+
+describe('each(): a nack skips only its own partition', () => {
+  it('handles the other partitions of the pop after a failure', async () => {
+    const pop = [message('A', 1), message('A', 2), message('B', 1), message('B', 2)]
+    await withBroker([pop], async (queen, requests) => {
+      const handled = []
+      await queen.queue('orders').group(GROUP).each().batch(4).limit(3).idleMillis(500)
+        .consume(async (m) => {
+          handled.push(m.data.tag)
+          if (m.data.tag === 'A1') throw new Error('A1 fails')
+        })
+
+      assert.deepEqual(handled, ['A1', 'B1', 'B2'], 'A2 is skipped (it comes back after the nack), B is handled now')
+      const settled = requests.filter(r => r.path.startsWith('/api/v1/ack'))
+        .flatMap(r => r.body.acknowledgments || [r.body])
+        .map(a => `${a.transactionId}:${a.status}`)
+      assert.deepEqual(settled, ['tx-A1:failed', 'tx-B1:completed', 'tx-B2:completed'])
+    })
+  })
+})