queen-mq 2.0.2 → 2.0.3

package/README.md

@@ -325,6 +325,36 @@ await queen.queue('tasks')
 Earlier versions stopped the consumer on a throw under `.autoAck(false)`: `consume()` rejected after
 the first failure, and the message stayed leased until its lease expired.
 
+**Stopping a consumer** (a graceful shutdown) is aborting its `signal`. The handler call in progress
+finishes, with its ack, and `consume()` resolves. Nothing is left leased:
+
+- The long poll in flight is closed at once. The broker hands nothing to a poll whose caller is gone,
+  so a message that arrives during the shutdown goes to another consumer straight away.
+- A pop answer that has already started arriving is read to the end: the broker leased its messages
+  when it sent it, and the body says which ones they are.
+- With `.each()`, messages already popped but not yet handed to the handler go back with a `retry`
+  ack. The broker releases their lease and redelivers them first, in order, without charging a
+  retry. The same happens to messages popped beyond `.limit()`.
+- A wait between attempts (a 429 backoff, a retry after a 5xx or a network error) ends at once.
+
+```javascript
+const stop = new AbortController()
+// consume() starts when awaited: Promise.resolve() starts it now and keeps the
+// promise that settles once the consumer has stopped.
+const consuming = Promise.resolve(queen.queue('tasks').group('workers').each()
+  .consume(async (message) => { await processTask(message.data) }, { signal: stop.signal }))
+
+process.once('SIGTERM', async () => {
+  stop.abort()
+  await consuming      // the message in the handler is finished and acked
+  await queen.close()
+})
+```
+
+Earlier versions checked the signal only between polls. A poll open at the abort stayed open for up
+to its timeout, and `.each()` dropped what it brought back without settling it, so its partition
+waited out the whole lease.
+
 ### Pop Messages (On-Demand Processing)
 
 ```javascript

package/client-v2/consumer/ConsumerManager.js

@@ -7,6 +7,20 @@ import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/confla
 import { popSizing, parseAutopilotDecision, emptyPollDelayMillis } from '../utils/autopilot.js'
 import { CONSUME_DEFAULTS } from '../utils/defaults.js'
 
+/** Wait `ms`, or less if the consumer is stopped meanwhile: the loop checks the signal next. */
+function pause(ms, signal) {
+  if (signal?.aborted) return Promise.resolve()
+  return new Promise(resolve => {
+    const done = () => {
+      clearTimeout(timer)
+      signal?.removeEventListener('abort', done)
+      resolve()
+    }
+    const timer = setTimeout(done, ms)
+    signal?.addEventListener('abort', done, { once: true })
+  })
+}
+
 export class ConsumerManager {
   #httpClient
   #queen
@@ -160,7 +174,10 @@ export class ConsumerManager {
         // a long-poll: mark it 'pop' so a 429 backs off and keeps waiting
         // instead of giving up after the bounded push-like attempt budget.
         const clientTimeout = wait ? timeoutMillis + 5000 : timeoutMillis
-        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey, wait ? 'pop' : null)
+        // The signal closes the poll as well: a broker hands nothing to a poll
+        // whose caller is gone, so a stopped consumer is never given a message
+        // it would only sit on until the lease expires.
+        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey, wait ? 'pop' : null, signal)
 
         // Degrade-loudly (PLAN_CONFLATION §4). Checked BEFORE the empty-response
         // branch on purpose: a pre-1.1.0 broker answers an empty pop with a
@@ -181,7 +198,7 @@ export class ConsumerManager {
             // pop engaged autopilot and the broker had an opinion (it knows the
             // arrival rate on this queue and this client does not), otherwise
             // the historical 100ms.
-            await new Promise(resolve => setTimeout(resolve, emptyPollDelayMillis(parseAutopilotDecision(result))))
+            await pause(emptyPollDelayMillis(parseAutopilotDecision(result)), signal)
             continue
           }
         }
@@ -218,8 +235,17 @@ export class ConsumerManager {
             // partitions of a multi-partition pop are still leased to this
             // worker, so their messages are handled now, not after the lease.
             const nackedPartitions = new Set()
-            for (const message of messages) {
-              if (signal && signal.aborted) break
+            for (const [i, message] of messages.entries()) {
+              // Stopped, or the limit reached, with messages still in hand:
+              // give them back rather than leave them leased. Those of a
+              // nacked partition were already given back by the nack.
+              if ((signal && signal.aborted) || (limit && processedCount >= limit)) {
+                const unstarted = messages.slice(i).filter(m => !nackedPartitions.has(m.partitionId ?? m.partition))
+                if (unstarted.length > 0) {
+                  await this.#releaseUnstarted(unstarted, group, signal && signal.aborted ? 'aborted' : 'limit-reached')
+                }
+                break
+              }
 
               const partition = message.partitionId ?? message.partition
               if (nackedPartitions.has(partition)) continue
@@ -231,8 +257,6 @@ export class ConsumerManager {
                 nackedPartitions.add(partition)
                 logger.warn('ConsumerManager.worker', { workerId, status: 'partition-abandoned-after-nack', partition })
               }
-
-              if (limit && processedCount >= limit) break
             }
           } else {
             // Process as batch
@@ -249,6 +273,13 @@ export class ConsumerManager {
         }
 
       } catch (error) {
+        // Stopped while a poll was open: the poll was closed, nothing was
+        // taken, the worker is done. Not an error, whatever the wait mode.
+        if (error.aborted || (signal && signal.aborted)) {
+          logger.log('ConsumerManager.worker', { workerId, status: 'aborted', processedCount })
+          break
+        }
+
         // Conflation faults are terminal and are classified FIRST, ahead of the
         // message-substring heuristics below: a consumer that asked for
         // last-value delivery and is not getting it must stop, not retry
@@ -278,7 +309,7 @@ export class ConsumerManager {
             ? error.retryAfterSeconds * 1000
             : 1000
           logger.warn('ConsumerManager.worker', { workerId, status: 'rate-limited', code: error.code, retryAfterMs })
-          await new Promise(resolve => setTimeout(resolve, retryAfterMs))
+          await pause(retryAfterMs, signal)
           continue
         }
 
@@ -290,7 +321,7 @@ export class ConsumerManager {
         if (isNetworkError) {
           logger.warn('ConsumerManager.worker', { workerId, error: 'network', message: error.message })
           // Wait before retry
-          await new Promise(resolve => setTimeout(resolve, 1000))
+          await pause(1000, signal)
           continue
         }
 
@@ -339,6 +370,26 @@ export class ConsumerManager {
     return true
   }
 
+  /**
+   * Hand back messages this worker holds but will not process (it was stopped,
+   * or reached its limit): a `retry` ack releases their lease, the broker
+   * redelivers them first, in order, and charges no retry. Best effort -- a
+   * release that fails leaves the message to its lease, as before.
+   */
+  async #releaseUnstarted(messages, group, reason) {
+    const subject = { count: messages.length, transactionIds: messages.map(m => m.transactionId) }
+    try {
+      const res = await this.#queen.ack(messages, 'retry', group ? { group } : {})
+      if (res && res.success === false) {
+        logger.warn('ConsumerManager.release', { ...subject, reason, status: 'release-rejected', error: res.error })
+      } else {
+        logger.log('ConsumerManager.release', { ...subject, reason, status: 'released' })
+      }
+    } catch (error) {
+      logger.warn('ConsumerManager.release', { ...subject, reason, status: 'release-failed', error: error.message })
+    }
+  }
+
   async #processBatch(messages, handler, autoAck, group) {
     try {
       await handler(messages)

package/client-v2/http/HttpClient.js

@@ -4,6 +4,47 @@
 
 import * as logger from '../utils/logger.js'
 
+/**
+ * The error a request ends with when its caller aborted it (a consumer being
+ * stopped). It says nothing about the backend: it is never retried, never
+ * fails over to another node, and never marks a node unhealthy.
+ */
+function callerAborted(method, url) {
+  const error = new Error(`${method} ${url} aborted by the caller`)
+  error.name = 'AbortError'
+  error.aborted = true
+  return error
+}
+
+/**
+ * Wait `delay` ms before another attempt, unless the caller aborts first: then
+ * reject at once with the caller-aborted error instead of sitting out the wait.
+ */
+function waitBeforeRetry(delay, signal, method, url) {
+  if (signal?.aborted) return Promise.reject(callerAborted(method, url))
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      clearTimeout(timer)
+      reject(callerAborted(method, url))
+    }
+    const timer = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort)
+      resolve()
+    }, delay)
+    signal?.addEventListener('abort', onAbort, { once: true })
+  })
+}
+
+/** A pop answer carries leases: once it is arriving, it is read to the end. */
+function isPop(method, url) {
+  if (method !== 'GET') return false
+  try {
+    return new URL(url).pathname.startsWith('/api/v1/pop')
+  } catch {
+    return false
+  }
+}
+
 // --------------------------------------------------------------------------
 // Host-routed proxy support (queen_proxy selects the tenant cluster from the
 // first DNS label of the Host header -- proxy/src/cache.rs
@@ -211,14 +252,15 @@ export class HttpClient {
    * only status this layer treats as retryable; 5xx/network retry and
    * cross-backend failover are handled by the caller.
    */
-  async #executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind) {
+  async #executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind, signal = null) {
     const { maxAttempts, baseMs, capMs } = this.#retry429PolicyFor(retryKind)
     let tries = 0
     // eslint-disable-next-line no-constant-condition
     while (true) {
+      if (signal?.aborted) throw callerAborted(method, url)
       tries++
       try {
-        return await this.#executeRequest(url, method, body, requestTimeoutMillis)
+        return await this.#executeRequest(url, method, body, requestTimeoutMillis, signal)
       } catch (error) {
         if (error.status !== 429) throw error
 
@@ -229,7 +271,7 @@ export class HttpClient {
 
         const delay = this.#computeRetry429DelayMs(tries - 1, error.retryAfterSeconds, baseMs, capMs)
         logger.warn('HttpClient.retry429', { method, url, attempt: tries, retryKind: retryKind || 'default', nextDelayMs: delay, retryAfterSeconds: error.retryAfterSeconds ?? null, code: error.code ?? null })
-        await new Promise(resolve => setTimeout(resolve, delay))
+        await waitBeforeRetry(delay, signal, method, url)
       }
     }
   }
@@ -323,12 +365,17 @@ export class HttpClient {
     }
   }
 
-  async #executeRequest(url, method, body = null, requestTimeoutMillis = null) {
+  async #executeRequest(url, method, body = null, requestTimeoutMillis = null, signal = null) {
+    if (signal?.aborted) throw callerAborted(method, url)
     const effectiveTimeout = requestTimeoutMillis || this.#timeoutMillis
     logger.log('HttpClient.request', { method, url, hasBody: !!body, timeout: effectiveTimeout, host: this.#hostOverride ? this.#hostOverride.authority : undefined })
 
     const controller = new AbortController()
     const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout)
+    // The caller's signal closes the request too: a long poll its consumer no
+    // longer wants must not stay open for the broker to hand it a message.
+    const abortFromCaller = () => controller.abort()
+    signal?.addEventListener('abort', abortFromCaller, { once: true })
 
     try {
       const headers = { 'Content-Type': 'application/json' }
@@ -367,6 +414,14 @@ export class HttpClient {
 
       const response = await fetch(requestUrl, options)
 
+      // The broker granted the leases of a pop when it sent these headers. From
+      // here the caller's abort no longer cuts the read: the body says which
+      // messages this consumer holds, and it needs them to give them back. The
+      // timeout still bounds the read.
+      if (response.ok && isPop(method, url)) {
+        signal?.removeEventListener('abort', abortFromCaller)
+      }
+
       logger.log('HttpClient.response', { method, url, status: response.status })
 
       // Handle 204 No Content
@@ -421,9 +476,16 @@ export class HttpClient {
         }
       }
 
-      return response.json()
+      // Awaited here, not returned: the body is still being read, and both the
+      // timeout (cleared in `finally`) and the abort classification below must
+      // cover that read too.
+      return await response.json()
 
     } catch (error) {
+      if (signal?.aborted) {
+        logger.log('HttpClient.request', { method, url, status: 'aborted-by-caller' })
+        throw callerAborted(method, url)
+      }
       if (error.name === 'AbortError') {
         const timeoutError = new Error(`Request timeout after ${effectiveTimeout}ms`)
         timeoutError.name = 'AbortError'
@@ -441,18 +503,20 @@ export class HttpClient {
       logger.error('HttpClient.request', { method, url, error: error.message })
       throw error
     } finally {
+      signal?.removeEventListener('abort', abortFromCaller)
       clearTimeout(timeoutId)
     }
   }
 
-  async #requestWithRetry(method, path, body = null, requestTimeoutMillis = null, retryKind = null) {
+  async #requestWithRetry(method, path, body = null, requestTimeoutMillis = null, retryKind = null, signal = null) {
     let lastError = null
 
     for (let attempt = 0; attempt < this.#retryAttempts; attempt++) {
       try {
         const url = this.#getUrl() + path
-        return await this.#executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind)
+        return await this.#executeWithRetry429(url, method, body, requestTimeoutMillis, retryKind, signal)
       } catch (error) {
+        if (error.aborted) throw error
         lastError = error
 
         // Don't retry on client errors (4xx)
@@ -464,7 +528,7 @@ export class HttpClient {
         if (attempt < this.#retryAttempts - 1) {
           const delay = this.#retryDelayMillis * Math.pow(2, attempt)
           logger.warn('HttpClient.retry', { method, path, attempt: attempt + 1, delay, error: error.message })
-          await new Promise(resolve => setTimeout(resolve, delay))
+          await waitBeforeRetry(delay, signal, method, path)
         }
       }
     }
@@ -473,9 +537,9 @@ export class HttpClient {
     throw lastError
   }
 
-  async #requestWithFailover(method, path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {
+  async #requestWithFailover(method, path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
     if (!this.#loadBalancer || !this.#enableFailover) {
-      return this.#requestWithRetry(method, path, body, requestTimeoutMillis, retryKind)
+      return this.#requestWithRetry(method, path, body, requestTimeoutMillis, retryKind, signal)
     }
 
     const urls = this.#loadBalancer.getAllUrls()
@@ -498,13 +562,16 @@ export class HttpClient {
         // 429s are retried in place (same backend, backoff-paced) inside
         // #executeWithRetry429 -- they are not a backend-health signal, so
         // they must not trigger failover to a different server.
-        const result = await this.#executeWithRetry429(url + path, method, body, requestTimeoutMillis, retryKind)
+        const result = await this.#executeWithRetry429(url + path, method, body, requestTimeoutMillis, retryKind, signal)
 
         // Mark backend as healthy on success
         this.#loadBalancer.markHealthy(url)
 
         return result
       } catch (error) {
+        // Stopped by its caller: this node did nothing wrong, and another
+        // node must not get the request instead.
+        if (error.aborted) throw error
         lastError = error
 
         // Mark backend as unhealthy on failure (5xx or network errors)
@@ -538,8 +605,10 @@ export class HttpClient {
   // `retryKind`: pass 'pop' for long-poll (wait=true) pop requests to get the
   // unbounded-with-backoff 429 policy; omit for everything else (push, admin
   // calls, non-waiting pop), which get the bounded default (10 attempts).
-  async get(path, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {
-    return this.#requestWithFailover('GET', path, null, requestTimeoutMillis, affinityKey, retryKind)
+  // `signal`: aborting it closes the request, which then rejects with an
+  // error carrying `aborted: true` -- never retried and never failed over.
+  async get(path, requestTimeoutMillis = null, affinityKey = null, retryKind = null, signal = null) {
+    return this.#requestWithFailover('GET', path, null, requestTimeoutMillis, affinityKey, retryKind, signal)
   }
 
   async post(path, body = null, requestTimeoutMillis = null, affinityKey = null, retryKind = null) {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "type": "module",
   "description": "Partitioned message queue on a replicated broker log — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
     "test": "npm run test:unit && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/streams-unit/gate.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/http-unit/pushStatus.test.js test-v2/http-unit/renew.test.js test-v2/consumer-unit/handlerError.test.js test-v2/consumer-unit/nackScope.test.js test-v2/consumer-unit/stopOnAbort.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js test-v2/runner-unit/fatalExit.test.js test-v2/pop-unit/popDefaults.test.js test-v2/admin-unit/removedRoutes.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",

package/test-v2/consumer-unit/stopOnAbort.test.js

@@ -0,0 +1,537 @@
+/**
+ * consume() — what aborting its signal does.
+ *
+ * The rule: stopping a consumer never strands a message. The long poll in
+ * flight is closed, so the broker stops handing this consumer messages (a
+ * broker hands nothing to a poll whose caller is gone, and releases what a
+ * forwarded pop claimed for one). A message the client already holds but has
+ * not handed to the handler goes back with a `retry` ack: the lease is
+ * released, the message is redelivered first, and no retry is charged.
+ *
+ * Before: the signal was only checked between polls. A poll open at the
+ * abort stayed open for up to its timeout, the broker could still hand it a
+ * message, and each() then dropped that message without settling it. The
+ * partition stayed blocked until the lease expired (measured against 2.0.1:
+ * the full lease, 60-180 s on a rolling restart, every restart).
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { createServer } from 'node:http'
+import { readFile } from 'node:fs/promises'
+import vm from 'node:vm'
+
+import { Queen } from '../../client-v2/index.js'
+import { HttpClient } from '../../client-v2/http/HttpClient.js'
+import { LoadBalancer } from '../../client-v2/http/LoadBalancer.js'
+
+const GROUP = 'workers'
+
+const message = (n) => ({
+  id: `msg-${n}`,
+  transactionId: `tx-${n}`,
+  partitionId: '7',
+  partition: 'p1',
+  leaseId: 'lease-1',
+  consumerGroup: GROUP,
+  data: { n },
+  createdAt: '2026-10-06T10:00:00.000Z'
+})
+
+/**
+ * A fake broker. A pop answers the next batch of `pops`; once they run out it
+ * is held open, like a long poll on an idle queue, and recorded as `closed`
+ * when the client goes away. `ackStatus` answers acks (default: accepted).
+ */
+async function startBroker(pops = [], { ackStatus = 200 } = {}) {
+  const queue = [...pops]
+  const requests = []
+  const held = []
+  const server = createServer((req, res) => {
+    let raw = ''
+    req.on('data', chunk => { raw += chunk })
+    req.on('end', () => {
+      const body = raw ? JSON.parse(raw) : null
+      const path = req.url.split('?')[0]
+      const record = { method: req.method, path, body, closed: false }
+      requests.push(record)
+      if (req.method === 'GET' && path.startsWith('/api/v1/pop')) {
+        const batch = queue.shift()
+        if (batch) {
+          res.writeHead(200, { 'Content-Type': 'application/json' })
+          res.end(JSON.stringify({ success: true, consumerGroup: GROUP, messages: batch }))
+          return
+        }
+        res.on('close', () => { if (!res.writableEnded) record.closed = true })
+        held.push(res)
+        return
+      }
+      if (req.method === 'POST' && (path === '/api/v1/ack' || path === '/api/v1/ack/batch')) {
+        const acks = body.acknowledgments || [body]
+        res.writeHead(ackStatus, { 'Content-Type': 'application/json' })
+        res.end(ackStatus === 200
+          ? JSON.stringify(acks.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null, leaseReleased: true })))
+          : '{"error":"unavailable"}')
+        return
+      }
+      res.writeHead(404, { 'Content-Type': 'application/json' })
+      res.end('{"error":"not found"}')
+    })
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  return {
+    url: `http://127.0.0.1:${server.address().port}`,
+    requests,
+    pops: () => requests.filter(r => r.method === 'GET' && r.path.startsWith('/api/v1/pop')),
+    acks: () => requests.filter(r => r.method === 'POST' && r.path.startsWith('/api/v1/ack')),
+    async stop() {
+      for (const res of held) if (!res.writableEnded && !res.destroyed) { res.writeHead(204); res.end() }
+      server.closeAllConnections()
+      await new Promise(resolve => server.close(resolve))
+    }
+  }
+}
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+/** Resolves with 'resolved' / 'rejected: …', or 'pending' if it took longer than `ms`. */
+const settleWithin = (promise, ms) => Promise.race([
+  Promise.resolve(promise).then(() => 'resolved', e => `rejected: ${e.message}`),
+  sleep(ms).then(() => 'pending')
+])
+
+const settledStatuses = (acks) => acks.flatMap(a => (a.body.acknowledgments || [a.body]).map(x => `${x.transactionId}:${x.status}`))
+
+describe('consume() — aborting the signal never strands a message', () => {
+  it('closes the long poll in flight, and the consumer ends at once', async () => {
+    const broker = await startBroker()
+    const queen = new Queen({ url: broker.url, handleSignals: false })
+    try {
+      const ac = new AbortController()
+      const running = queen.queue('orders').group(GROUP).timeoutMillis(30000)
+        .consume(async () => {}, { signal: ac.signal })
+      const outcome = settleWithin(running, 2000)
+      await sleep(150)
+      assert.equal(broker.pops().length, 1, 'one long poll is open')
+
+      ac.abort()
+
+      assert.equal(await outcome, 'resolved', 'consume() resolves without waiting out the 30 s poll')
+      await sleep(50)
+      assert.equal(broker.pops()[0].closed, true, 'the client closed the poll it had open')
+      assert.equal(broker.pops().length, 1, 'and opened no other')
+    } finally {
+      await queen.close()
+      await broker.stop()
+    }
+  })
+
+  it('is not a backend failure: the poll does not fail over to another node', async () => {
+    const a = await startBroker()
+    const b = await startBroker()
+    const queen = new Queen({ urls: [a.url, b.url], enableFailover: true, loadBalancingStrategy: 'affinity', handleSignals: false })
+    try {
+      const ac = new AbortController()
+      const running = queen.queue('orders').group(GROUP).timeoutMillis(30000)
+        .consume(async () => {}, { signal: ac.signal })
+      const outcome = settleWithin(running, 2000)
+      await sleep(150)
+
+      ac.abort()
+
+      assert.equal(await outcome, 'resolved')
+      await sleep(50)
+      assert.equal(a.pops().length + b.pops().length, 1, 'the abort did not fail over to the other node')
+    } finally {
+      await queen.close()
+      await a.stop()
+      await b.stop()
+    }
+  })
+
+  it('HttpClient: an aborted request rejects as aborted, and leaves every node healthy', async () => {
+    const a = await startBroker()
+    const b = await startBroker()
+    const lb = new LoadBalancer([a.url, b.url], 'affinity')
+    const http = new HttpClient({ loadBalancer: lb, enableFailover: true, retryAttempts: 3 })
+    try {
+      const ac = new AbortController()
+      const request = http.get('/api/v1/pop/queue/orders?wait=true&timeout=30000', 35000, 'orders:*:workers', 'pop', ac.signal)
+      const settled = request.then(() => null, e => e)
+      await sleep(150)
+
+      ac.abort()
+
+      const error = await settled
+      assert.equal(error?.aborted, true, 'the rejection says the caller aborted it')
+      assert.equal(a.pops().length + b.pops().length, 1, 'no other node was tried')
+      for (const [url, status] of lb.getHealthStatus()) {
+        assert.equal(status.healthy, true, `${url} is still healthy`)
+      }
+    } finally {
+      await a.stop()
+      await b.stop()
+    }
+  })
+
+  it('wait(false): an abort while a pop is in flight ends the consumer without an error', async () => {
+    const broker = await startBroker()
+    const queen = new Queen({ url: broker.url, handleSignals: false })
+    try {
+      const ac = new AbortController()
+      const running = queen.queue('orders').group(GROUP).wait(false).timeoutMillis(30000)
+        .consume(async () => {}, { signal: ac.signal })
+      const outcome = settleWithin(running, 2000)
+      await sleep(150)
+
+      ac.abort()
+
+      assert.equal(await outcome, 'resolved', 'consume() resolves, it does not reject')
+    } finally {
+      await queen.close()
+      await broker.stop()
+    }
+  })
+
+  it('each(): the messages not yet handed to the handler go back with a retry ack', async () => {
+    const broker = await startBroker([[message(1), message(2), message(3)]])
+    const queen = new Queen({ url: broker.url, handleSignals: false })
+    try {
+      const ac = new AbortController()
+      const seen = []
+      const running = queen.queue('orders').group(GROUP).batch(3).each()
+        .consume(async (msg) => { seen.push(msg.transactionId); ac.abort() }, { signal: ac.signal })
+
+      assert.equal(await settleWithin(running, 2000), 'resolved')
+      assert.deepEqual(seen, ['tx-1'], 'nothing is handed to the handler after the stop')
+      assert.deepEqual(settledStatuses(broker.acks()), ['tx-1:completed', 'tx-2:retry', 'tx-3:retry'],
+        'the one in the handler finishes; the other two are released, not dropped')
+      const release = broker.acks().at(-1).body
+      for (const a of release.acknowledgments || [release]) {
+        assert.equal(a.leaseId ?? release.leaseId, 'lease-1', 'the release names the lease it gives back')
+      }
+    } finally {
+      await queen.close()
+      await broker.stop()
+    }
+  })
+
+  it('each() with a limit: the messages past the limit go back too, instead of staying leased', async () => {
+    const broker = await startBroker([[message(1), message(2), message(3)]])
+    const queen = new Queen({ url: broker.url, handleSignals: false })
+    try {
+      const seen = []
+      const running = queen.queue('orders').group(GROUP).batch(3).each().limit(1)
+        .consume(async (msg) => { seen.push(msg.transactionId) })
+
+      assert.equal(await settleWithin(running, 2000), 'resolved')
+      assert.deepEqual(seen, ['tx-1'])
+      assert.deepEqual(settledStatuses(broker.acks()), ['tx-1:completed', 'tx-2:retry', 'tx-3:retry'])
+    } finally {
+      await queen.close()
+      await broker.stop()
+    }
+  })
+
+  it('a release that cannot be delivered does not fail the consumer: the lease is the fallback', async () => {
+    const broker = await startBroker([[message(1), message(2)]], { ackStatus: 503 })
+    const queen = new Queen({ url: broker.url, handleSignals: false, retryAttempts: 1 })
+    try {
+      const ac = new AbortController()
+      const running = queen.queue('orders').group(GROUP).batch(2).each().autoAck(false)
+        .consume(async () => { ac.abort() }, { signal: ac.signal })
+
+      assert.equal(await settleWithin(running, 3000), 'resolved', 'consume() still resolves')
+      assert.ok(settledStatuses(broker.acks()).includes('tx-2:retry'), 'the release was attempted')
+    } finally {
+      await queen.close()
+      await broker.stop()
+    }
+  })
+})
+
+/**
+ * A server that sends the headers and half a JSON body, then stalls. Each
+ * request is counted; `closeAll` ends the stalled bodies.
+ */
+async function startStallingServer() {
+  let hits = 0
+  const open = []
+  const server = createServer((req, res) => {
+    hits++
+    res.writeHead(200, { 'Content-Type': 'application/json' })
+    res.write('{"value":')
+    open.push(res)
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  return {
+    url: `http://127.0.0.1:${server.address().port}`,
+    hits: () => hits,
+    async stop() {
+      for (const res of open) if (!res.destroyed) res.destroy()
+      server.closeAllConnections()
+      await new Promise(resolve => server.close(resolve))
+    }
+  }
+}
+
+/** Resolves once fetch has handed back the response headers for a URL with `prefix`. */
+function onResponseHeaders(prefix) {
+  const original = globalThis.fetch
+  let ready
+  const promise = new Promise(resolve => { ready = resolve })
+  globalThis.fetch = async (...args) => {
+    const response = await original(...args)
+    if (String(args[0]).startsWith(prefix)) setImmediate(ready)
+    return response
+  }
+  return { ready: promise, restore() { globalThis.fetch = original } }
+}
+
+describe('HttpClient — a response whose body is still being read', () => {
+  it('an abort during the body read is a caller abort: no node is marked unhealthy', async () => {
+    const a = await startStallingServer()
+    const b = await startStallingServer()
+    const lb = new LoadBalancer([a.url, b.url], 'affinity')
+    const http = new HttpClient({ loadBalancer: lb, enableFailover: true, retryAttempts: 1 })
+    const headers = onResponseHeaders('http://127.0.0.1:')
+    try {
+      const ac = new AbortController()
+      const settled = http.get('/status', 10000, 'key', null, ac.signal).then(() => null, e => e)
+      await headers.ready
+
+      ac.abort()
+
+      const error = await settled
+      assert.equal(error?.aborted, true, 'the rejection says the caller aborted it')
+      assert.equal(a.hits() + b.hits(), 1, 'no other node was tried')
+      for (const [url, status] of lb.getHealthStatus()) {
+        assert.equal(status.healthy, true, `${url} is still healthy`)
+      }
+    } finally {
+      headers.restore()
+      await http.destroy()
+      await a.stop()
+      await b.stop()
+    }
+  })
+
+  it('the request timeout also bounds the body read', async () => {
+    const server = await startStallingServer()
+    const http = new HttpClient({ baseUrl: server.url, retryAttempts: 1 })
+    try {
+      const started = Date.now()
+      const outcome = await settleWithin(http.get('/status', 300), 3000)
+      assert.match(outcome, /^rejected: Request timeout/, 'a stalled body ends as a timeout, not a hang')
+      assert.ok(Date.now() - started < 2000)
+    } finally {
+      await http.destroy()
+      await server.stop()
+    }
+  })
+})
+
+describe('consume() — a pop response that has started arriving', () => {
+  it('is read to the end after a stop, and its message goes back with a retry ack', async () => {
+    // The broker granted the lease when it sent the headers: from then on the
+    // message is this consumer's to give back, which needs the body.
+    const message1 = message(1)
+    const body = JSON.stringify({ success: true, consumerGroup: GROUP, messages: [message1] })
+    let finish
+    const finished = new Promise(resolve => { finish = resolve })
+    let pops = 0
+    const acks = []
+    const server = createServer((req, res) => {
+      let raw = ''
+      req.on('data', chunk => { raw += chunk })
+      req.on('end', () => {
+        if (req.method === 'GET') {
+          pops++
+          res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) })
+          res.write(body.slice(0, body.length / 2))
+          finished.then(() => { if (!res.destroyed) res.end(body.slice(body.length / 2)) })
+          return
+        }
+        const parsed = JSON.parse(raw)
+        const items = parsed.acknowledgments || [parsed]
+        acks.push(...items)
+        res.writeHead(200, { 'Content-Type': 'application/json' })
+        res.end(JSON.stringify(items.map((a, i) => ({ index: i, transactionId: a.transactionId, success: true, error: null, leaseReleased: true }))))
+      })
+    })
+    await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+    const url = `http://127.0.0.1:${server.address().port}`
+    const queen = new Queen({ url, handleSignals: false, retryAttempts: 1 })
+    const headers = onResponseHeaders(url)
+    try {
+      const stop = new AbortController()
+      const seen = []
+      const running = Promise.resolve(queen.queue('orders').group(GROUP).batch(1).each()
+        .consume(async (m) => { seen.push(m.transactionId) }, { signal: stop.signal }))
+      await headers.ready
+
+      stop.abort()
+      finish()
+
+      assert.equal(await settleWithin(running, 2000), 'resolved')
+      assert.deepEqual(seen, [], 'nothing is handed to the handler after the stop')
+      assert.equal(pops, 1)
+      assert.deepEqual(acks.map(a => `${a.transactionId}:${a.status}`), ['tx-1:retry'],
+        'the message in the half-read response is given back, not left to its lease')
+    } finally {
+      finish()
+      headers.restore()
+      await queen.close()
+      server.closeAllConnections()
+      await new Promise(resolve => server.close(resolve))
+    }
+  })
+})
+
+/** A server that answers every request with `status` (and Retry-After), counting them. */
+async function startRefusingServer(status, retryAfterSeconds = null) {
+  let hits = 0
+  let firstHit
+  const hit = new Promise(resolve => { firstHit = resolve })
+  const server = createServer((req, res) => {
+    hits++
+    const headers = { 'Content-Type': 'application/json' }
+    if (retryAfterSeconds != null) headers['Retry-After'] = String(retryAfterSeconds)
+    res.writeHead(status, headers)
+    res.end('{"error":"refused"}', () => firstHit())
+  })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  return {
+    url: `http://127.0.0.1:${server.address().port}`,
+    hits: () => hits,
+    firstHit: hit,
+    async stop() {
+      server.closeAllConnections()
+      await new Promise(resolve => server.close(resolve))
+    }
+  }
+}
+
+/** A server that drops every connection without answering. */
+async function startDroppingServer() {
+  let hits = 0
+  let firstHit
+  const hit = new Promise(resolve => { firstHit = resolve })
+  const server = createServer((req) => { hits++; req.socket.destroy(); firstHit() })
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
+  return {
+    url: `http://127.0.0.1:${server.address().port}`,
+    hits: () => hits,
+    firstHit: hit,
+    async stop() {
+      server.closeAllConnections()
+      await new Promise(resolve => server.close(resolve))
+    }
+  }
+}
+
+/** Aborts once the client is inside its wait, and measures how long stopping takes. */
+async function abortDuringWait(server, running, ac) {
+  await server.firstHit
+  await sleep(100) // the client has read the answer and is now waiting
+  const started = performance.now()
+  ac.abort()
+  const outcome = await settleWithin(running, 3000)
+  return { outcome, elapsed: performance.now() - started }
+}
+
+describe('a stop during a wait between attempts', () => {
+  it('HttpClient: a 429 backoff ends at once', async () => {
+    const server = await startRefusingServer(429, 2)
+    const http = new HttpClient({ baseUrl: server.url, retryAttempts: 1 })
+    try {
+      const ac = new AbortController()
+      const running = http.get('/api/v1/pop/queue/orders?wait=true', 10000, null, 'pop', ac.signal).then(() => null, e => { throw e })
+      const errorP = running.catch(e => e)
+      const { elapsed } = await abortDuringWait(server, running, ac)
+      const error = await errorP
+      assert.equal(error?.aborted, true)
+      assert.equal(server.hits(), 1, 'no attempt after the stop')
+      assert.ok(elapsed < 250, `stopping took ${Math.round(elapsed)} ms, not the 2 s Retry-After`)
+    } finally {
+      await http.destroy()
+      await server.stop()
+    }
+  })
+
+  it('HttpClient: the backoff before retrying a 5xx ends at once', async () => {
+    const server = await startRefusingServer(503)
+    const http = new HttpClient({ baseUrl: server.url, retryAttempts: 3, retryDelayMillis: 2000 })
+    try {
+      const ac = new AbortController()
+      const running = http.get('/status', 10000, null, null, ac.signal)
+      const errorP = running.catch(e => e)
+      const { elapsed } = await abortDuringWait(server, running, ac)
+      const error = await errorP
+      assert.equal(error?.aborted, true)
+      assert.equal(server.hits(), 1, 'no attempt after the stop')
+      assert.ok(elapsed < 250, `stopping took ${Math.round(elapsed)} ms, not the 2 s backoff`)
+    } finally {
+      await http.destroy()
+      await server.stop()
+    }
+  })
+
+  it('consume(): the pause after a 429 the client gave up retrying ends at once', async () => {
+    const server = await startRefusingServer(429, 2)
+    const queen = new Queen({ url: server.url, handleSignals: false, retryAttempts: 1, retry429: { maxAttempts: 1 } })
+    try {
+      const ac = new AbortController()
+      const running = Promise.resolve(queen.queue('orders').group(GROUP).consume(async () => {}, { signal: ac.signal }))
+      const { outcome, elapsed } = await abortDuringWait(server, running, ac)
+      assert.equal(outcome, 'resolved')
+      assert.ok(elapsed < 250, `stopping took ${Math.round(elapsed)} ms, not the 2 s Retry-After`)
+    } finally {
+      await queen.close()
+      await server.stop()
+    }
+  })
+
+  it('consume(): the pause after a network error ends at once', async () => {
+    const server = await startDroppingServer()
+    const queen = new Queen({ url: server.url, handleSignals: false, retryAttempts: 1 })
+    try {
+      const ac = new AbortController()
+      const running = Promise.resolve(queen.queue('orders').group(GROUP).consume(async () => {}, { signal: ac.signal }))
+      const { outcome, elapsed } = await abortDuringWait(server, running, ac)
+      assert.equal(outcome, 'resolved')
+      assert.ok(elapsed < 250, `stopping took ${Math.round(elapsed)} ms, not the 1 s pause`)
+    } finally {
+      await queen.close()
+      await server.stop()
+    }
+  })
+})
+
+describe('README — the "Stopping a consumer" example', () => {
+  it('starts consuming before the stop, not after it', async () => {
+    const broker = await startBroker()
+    const queen = new Queen({ url: broker.url, handleSignals: false })
+    const readme = await readFile(new URL('../../README.md', import.meta.url), 'utf8')
+    const section = readme.slice(readme.indexOf('**Stopping a consumer**'))
+    const snippet = section.match(/```javascript\n([\s\S]*?)```/)[1]
+    let shutdown
+    const context = vm.createContext({
+      queen, AbortController, Promise,
+      processTask: async () => {},
+      process: { once(event, fn) { assert.equal(event, 'SIGTERM'); shutdown = fn } }
+    })
+    try {
+      vm.runInContext(snippet, context)
+      await sleep(150)
+      const popsBeforeStop = broker.pops().length
+
+      await shutdown()
+
+      assert.equal(popsBeforeStop, 1, 'the example is consuming before SIGTERM')
+    } finally {
+      await queen.close().catch(() => {})
+      await broker.stop()
+    }
+  })
+})