queen-mq 0.16.0 → 1.0.3

package/client-v2/builders/TimerBuilder.js

@@ -0,0 +1,262 @@
+/**
+ * Timers (PLAN_KV_TIMERS.md §4, §8.1, §9.6).
+ *
+ * A timer is a message you promise now and the broker delivers later, into a
+ * real queue, through the real log. Four terminals, all EXPLICIT -- nothing on
+ * this builder does anything until one of them is called:
+ *
+ *     await queen.timer('orders').key('order-9f1').delay('30s')
+ *                .payload({ orderId: '9f1' }).schedule()
+ *     await queen.timer('orders').key('order-9f1').cancel()
+ *     await queen.timer('orders').key('order-9f1').peek()
+ *     await queen.timer('orders').list({ limit: 50 })
+ *
+ * THE FOUR THINGS THIS FILE EXISTS TO GET RIGHT
+ *
+ * 1. CANCEL USES ITS OWN ROUTE, AND THAT IS NOT A DETAIL (§9.6).
+ *    `DELETE /api/v1/timers/:queue/*timerKey` is the one route a proxy is
+ *    forbidden to block; a cancel sent inside `POST /api/v1/timers` inherits
+ *    the schedule's authorization instead. Since the FIRE never switches
+ *    itself off, a tenant that can no longer cancel keeps producing messages it
+ *    cannot stop -- a block there produces the exact opposite of its purpose.
+ *    So `cancel()` on this builder always uses the DELETE route. (Inside a
+ *    transaction it necessarily rides the bundle's array and inherits the
+ *    bundle's fate; see TransactionBuilder, which says so where it happens.)
+ *
+ * 2. ONLY RELATIVE DURATIONS, IN MILLISECONDS (§4.2, §20.6). `delayMs`, never
+ *    `delaySeconds` and never an absolute instant: `deliver_at` is computed in
+ *    Postgres, so there is ONE clock and no broker's skew can enter. The rule
+ *    of the product is "durations that can be sub-second are in milliseconds,
+ *    the ones that cannot are in seconds" -- a 250 ms retry backoff is a real
+ *    and central use of timers, which is why this wire is the millisecond one.
+ *    A delay in the past is LEGAL and fires on the first cycle.
+ *
+ * 3. `deliverAt` IS "NOT BEFORE", NEVER "EXACTLY AT". The floor on this stack
+ *    is a single hop (p50 ~10 ms, fsync ~4 ms) plus one sweep cycle.
+ *
+ * 4. THERE IS NO TOMBSTONE (§4.4). Once a timer has fired its row is gone, so a
+ *    later cancel answers `absent` -- **which MAY mean it was already
+ *    delivered**. `absent` carries `ok:false` for exactly that reason, and the
+ *    response echoes the `txn` so the authority (the log, in the destination
+ *    queue) can be consulted without a second API. Any saga that cancels a
+ *    compensation timer must have the compensating consumer check the saga's
+ *    KV state before compensating: otherwise "the timer went out 5 ms before
+ *    the cancel" unwinds a booking that was already shipped.
+ */
+
+import * as logger from '../utils/logger.js'
+import { generateUUID } from './QueueBuilder.js'
+import { parseDurationMs } from '../kv/expiry.js'
+
+/** JSON payloads are encoded for the caller; raw bytes are passed through. */
+function encodePayload(payload) {
+  if (payload === undefined) return undefined
+  if (typeof Buffer !== 'undefined' && Buffer.isBuffer(payload)) {
+    return payload.toString('base64')
+  }
+  if (payload instanceof Uint8Array) {
+    return Buffer.from(payload).toString('base64')
+  }
+  return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64')
+}
+
+export class TimerBuilder {
+  #httpClient
+  #queue
+  #sink
+  #timerKey = null
+  #partition = null
+  #delayMs = null
+  #payloadB64 = undefined
+  #txn = null
+
+  /**
+   * `sink`, when present, receives the finished op instead of the network:
+   * that is how the same builder serves `queen.timer(...)` and
+   * `tx.timer(...)`, so the wire shape of a timer op is written once.
+   */
+  constructor(httpClient, queue, sink = null) {
+    if (typeof queue !== 'string' || queue.length === 0) {
+      throw new Error('timer: a queue name is required — a timer without a queue has nowhere to be delivered')
+    }
+    this.#httpClient = httpClient
+    this.#queue = queue
+    this.#sink = sink
+  }
+
+  /** The timer's identity inside its queue. `(queue, timerKey)` is the primary key. */
+  key(timerKey) {
+    if (typeof timerKey !== 'string' || timerKey.length === 0) {
+      throw new Error('timer: timerKey must be a non-empty string')
+    }
+    this.#timerKey = timerKey
+    return this
+  }
+
+  /** Destination partition of the delivered message. Defaults to 'Default'. */
+  partition(name) {
+    this.#partition = name
+    return this
+  }
+
+  /** Fire no earlier than this many milliseconds from now. Negative is legal: it fires on the first cycle. */
+  delayMs(ms) {
+    if (typeof ms !== 'number' || !Number.isFinite(ms)) {
+      throw new Error(`timer: delayMs must be a finite number of milliseconds, got ${JSON.stringify(ms)}`)
+    }
+    this.#delayMs = Math.round(ms)
+    return this
+  }
+
+  /** The same delay as a duration string: '250ms', '30s', '2h'. Converted to delayMs here. */
+  delay(duration) {
+    this.#delayMs = Math.round(parseDurationMs(duration))
+    return this
+  }
+
+  /** The message body. Any JSON value, or a Buffer/Uint8Array for raw bytes. */
+  payload(payload) {
+    this.#payloadB64 = encodePayload(payload)
+    return this
+  }
+
+  /**
+   * The transaction id the delivered message will carry. Minted when absent.
+   *
+   * §20.2: every reschedule overwrites it, because a rescheduled timer is a
+   * NEW message -- and the corollary that must travel with it is that there is
+   * no dedup net on the fire at all, so rescheduling or republishing a timer
+   * that has already gone out produces a second message in the log and nothing
+   * stops it.
+   */
+  txn(transactionId) {
+    this.#txn = transactionId
+    return this
+  }
+
+  // ------------------------------------------------------------ op shapes
+
+  #scheduleOp() {
+    if (!this.#timerKey) throw new Error('timer: key(...) is required to schedule')
+    if (this.#delayMs === null) {
+      throw new Error('timer: delayMs(...) or delay(...) is required — an absolute instant is not expressible on this wire')
+    }
+    if (this.#payloadB64 === undefined) throw new Error('timer: payload(...) is required to schedule')
+    const op = {
+      op: 'schedule',
+      queue: this.#queue,
+      timerKey: this.#timerKey
+    }
+    if (this.#partition !== null && this.#partition !== undefined) op.partition = this.#partition
+    op.delayMs = this.#delayMs
+    op.txn = this.#txn || generateUUID()
+    op.payload = this.#payloadB64
+    return op
+  }
+
+  #cancelOp() {
+    if (!this.#timerKey) throw new Error('timer: key(...) is required to cancel')
+    const op = { op: 'cancel', queue: this.#queue, timerKey: this.#timerKey }
+    if (this.#txn) op.txn = this.#txn
+    return op
+  }
+
+  #path(suffix = '') {
+    return `/api/v1/timers/${encodeURIComponent(this.#queue)}${suffix}`
+  }
+
+  #keyPath() {
+    return this.#path(`/${encodeURIComponent(this.#timerKey)}`)
+  }
+
+  #single(body, what) {
+    if (!body || typeof body !== 'object' || !Array.isArray(body.results)) {
+      throw new Error(`timer: unexpected ${what} response envelope — expected {"results":[...]}`)
+    }
+    if (body.results.length !== 1) {
+      throw new Error(`timer: got ${body.results.length} results for 1 operation`)
+    }
+    return body.results[0]
+  }
+
+  // ------------------------------------------------------------ terminals
+
+  /**
+   * Schedule (or reschedule) this timer. An UPSERT on `(queue, timerKey)`, so
+   * a retry after a client crash is safe by construction; the answer says
+   * which it was, `status: 'scheduled' | 'rescheduled'`.
+   *
+   * A reschedule resets `attempts` and clears `last_error`: a rescheduled
+   * timer is a new timer under an old name, and a freshly corrected payload
+   * must not inherit the budget consumed by the one that was poisoning it.
+   *
+   * `too_late` (with `ok:false`) means the timer is already claimed by a
+   * broker that is about to commit it. Bounded by the sweeper lease. The
+   * remedy is a new key, or waiting for the delivery and acting on the message.
+   */
+  // NOT `async`, deliberately: with a sink this terminal is the transaction's
+  // own chaining call and must hand back the TransactionBuilder itself, not a
+  // promise of it, or `.timer(q)...schedule().commit()` stops being a chain.
+  // The network path returns the promise, so `await ...schedule()` is
+  // unchanged for the standalone caller.
+  schedule() {
+    if (this.#sink) return this.#sink(this.#scheduleOp())
+    return this.#scheduleRemote()
+  }
+
+  // The op is built INSIDE the async function so that a shape error on the
+  // network path is a rejected promise like every other failure of this
+  // client, and not a synchronous throw that `.catch()` would miss. On the
+  // transaction path it necessarily stays synchronous: there is no promise
+  // there to reject.
+  async #scheduleRemote() {
+    const op = this.#scheduleOp()
+    logger.log('TimerBuilder.schedule', { queue: this.#queue, timerKey: op.timerKey, delayMs: op.delayMs })
+    const body = await this.#httpClient.post('/api/v1/timers', { operations: [op] })
+    return this.#single(body, 'schedule')
+  }
+
+  /**
+   * Cancel this timer, through the route that is never blockable (§9.6).
+   *
+   * Idempotent: `absent` answers `ok:false` and MAY MEAN ALREADY DELIVERED --
+   * there is no tombstone. Pass `.txn(...)` and it comes back in the answer,
+   * so the log can settle the question.
+   */
+  cancel() {
+    if (this.#sink) return this.#sink(this.#cancelOp())
+    return this.#cancelRemote()
+  }
+
+  async #cancelRemote() {
+    const op = this.#cancelOp()
+    logger.log('TimerBuilder.cancel', { queue: this.#queue, timerKey: op.timerKey })
+    const query = op.txn ? `?txn=${encodeURIComponent(op.txn)}` : ''
+    return this.#httpClient.delete(`${this.#keyPath()}${query}`)
+  }
+
+  /** Read one pending timer, payload included. A miss is `{found:false}` with HTTP 200, never a 404. */
+  async peek() {
+    if (this.#sink) throw new Error('timer: peek is a read, not a transaction operation — use queen.timer(...).peek()')
+    if (!this.#timerKey) throw new Error('timer: key(...) is required to peek')
+    logger.log('TimerBuilder.peek', { queue: this.#queue, timerKey: this.#timerKey })
+    return this.#httpClient.get(this.#keyPath())
+  }
+
+  /**
+   * List the pending timers of this queue: `{rows, truncated, nextAfter}`.
+   *
+   * The queue is mandatory and that is why it is a path segment rather than a
+   * filter: a tenant-wide list would be a scan that an end user of the
+   * customer could trigger. `after` is an exclusive keyset cursor.
+   */
+  async list(opts = {}) {
+    if (this.#sink) throw new Error('timer: list is a read, not a transaction operation — use queen.timer(...).list()')
+    const params = new URLSearchParams()
+    if (opts.after !== undefined && opts.after !== null && opts.after !== '') params.append('after', opts.after)
+    if (opts.limit !== undefined && opts.limit !== null) params.append('limit', String(opts.limit))
+    const query = params.toString()
+    logger.log('TimerBuilder.list', { queue: this.#queue, after: opts.after, limit: opts.limit })
+    return this.#httpClient.get(this.#path(query ? `?${query}` : ''))
+  }
+}

package/client-v2/builders/TransactionBuilder.js

@@ -1,13 +1,46 @@
 /**
  * Transaction builder for atomic operations
+ *
+ * KV AND TIMER RIDERS (PLAN_KV_TIMERS.md §6.3, §8.2, §10.4)
+ *
+ * `kv` and `timers` are TOP-LEVEL fields of the request body, beside
+ * `operations` and NEVER elements of it. That is a wire decision, not a style
+ * one, and the reason is a silent failure in another language that this shape
+ * has to respect because one broker parses one wire for all seven clients: two
+ * Go struct fields carrying the same JSON key at the same level are BOTH
+ * DROPPED by encoding/json, with no error and no warning. A bundle would go
+ * out with zero kv operations, the broker would commit a transaction with no
+ * gate, and the `putIfAbsent` the bundle existed for would never have
+ * happened. An op that still arrives as `{"type":"kv"}` inside `operations`
+ * gets a named 400 from the broker, which is the best failure available.
+ *
+ * A bundle that carries neither array produces exactly the body it produces
+ * today -- no `kv: []`, no `timers: null` -- so nothing changes for anyone who
+ * does not use the feature, including brokers that predate it.
+ *
+ * WHY THE RIDERS ARE WORTH THE PARAGRAPH: the transaction is the PRIMARY
+ * fence, and `expect` only the secondary assertion. A KV write that shares the
+ * transaction with the ack is undone when an expired lease makes the ack fail
+ * -- something a CAS cannot do, because an `expect` on a still-matching
+ * version succeeds even from a zombie worker.
  */
 
 import * as logger from '../utils/logger.js'
+import { generateUUID } from './QueueBuilder.js'
+import { isValidUUID } from '../utils/validation.js'
+import { kvOp, materializeKvOp } from '../kv/Kv.js'
+import { TimerBuilder } from './TimerBuilder.js'
 
 export class TransactionBuilder {
   #httpClient
   #operations = []
   #requiredLeases = []
+  // Unresolved on purpose: `until` is an instant, and freezing it when the op
+  // was queued would ship a TTL already stale by however long the bundle took
+  // to assemble. Materialized in commit(), which is send time.
+  #kvEntries = []
+  #timerOps = []
+  #kvApi = null
 
   constructor(httpClient) {
     this.#httpClient = httpClient
@@ -82,16 +115,26 @@ export class TransactionBuilder {
               payloadValue = item
             }
 
+            // Same contract as QueueBuilder.push: the caller's transactionId is
+            // what makes a retried transaction idempotent inside the dedup
+            // window, so it has to reach the wire. Absent, mint one here rather
+            // than leaving the broker to do it, so the id is knowable client
+            // side either way.
             const result = {
               queue: queueName,
-              payload: payloadValue
+              payload: payloadValue,
+              transactionId: item.transactionId || generateUUID()
             }
-            
+
             // Add partition if set
             if (partition !== null) {
               result.partition = partition
             }
 
+            if (item.traceId && isValidUUID(item.traceId)) {
+              result.traceId = item.traceId
+            }
+
             return result
           })
         })
@@ -103,23 +146,169 @@ export class TransactionBuilder {
     return subBuilder
   }
 
+  // ===========================
+  // KV rider (§5, §8.2)
+  // ===========================
+
+  /**
+   * KV operations that commit with this transaction.
+   *
+   *     await queen.transaction()
+   *       .ack(msg)
+   *       .kv.put('saga', sagaId, { step: 'charged' }, { ttl: '24h' })
+   *       .commit()
+   *
+   * Every method returns the TRANSACTION, so the chain keeps reading like one.
+   * Same op shapes as `queen.kv`, built by the same code, so the two surfaces
+   * cannot drift.
+   *
+   * `getPrefix` is deliberately absent from this wire and throws here rather
+   * than at the broker: it is unbounded read work inside the transaction that
+   * holds the outermost lock space and, downstream, the partition ones. `get`
+   * and `getMany` are allowed because the caller fixes their cost -- the
+   * boundary is COST, not the kind of operation.
+   */
+  get kv() {
+    if (this.#kvApi) return this.#kvApi
+    const add = (entry) => {
+      this.#kvEntries.push(entry)
+      return this
+    }
+    this.#kvApi = {
+      get: (ns, key) => add(kvOp.get(ns, key)),
+      getMany: (ns, keys) => add(kvOp.getMany(ns, keys)),
+      put: (ns, key, value, opts = {}) => add(kvOp.put(ns, key, value, opts)),
+      putIfAbsent: (ns, key, value, opts = {}) => add(kvOp.putIfAbsent(ns, key, value, opts)),
+      delete: (ns, key, opts = {}) => add(kvOp.delete(ns, key, opts)),
+      incr: (ns, key, delta = 1, opts = {}) => add(kvOp.incr(ns, key, delta, opts)),
+      getPrefix: () => {
+        throw new Error(
+          'kv: getPrefix is not available inside a transaction — unbounded read work under the outermost ' +
+          'lock space. Use POST /api/v1/kv (queen.kv.getPrefix / queen.kv.listAll) outside the bundle.'
+        )
+      }
+    }
+    return this.#kvApi
+  }
+
+  /**
+   * The gate, which is the reason this feature exists: do the bundle at most
+   * once.
+   *
+   *     const res = await queen.transaction()
+   *       .ack(msg)
+   *       .queue('emails').push([{ data: mail }])
+   *       .once('test-idem', orderId, { ttl: '24h' })
+   *       .commit()
+   *     if (res.success === false) return    // a redelivery: already done
+   *
+   * `putIfAbsent` with `required:true`, so a marker that already exists ABORTS
+   * the transaction: the push and the ack roll back together with it. That is
+   * what makes "the email is sent exactly once" a property of the database
+   * rather than a hope about redelivery.
+   *
+   * The verdict is RETURNED by `commit()`, not thrown -- see there.
+   */
+  once(ns, key, opts = {}) {
+    const value = opts.value !== undefined ? opts.value : true
+    const required = opts.required === false ? {} : { required: true }
+    return this.kv.putIfAbsent(ns, key, value, { ...opts, ...required })
+  }
+
+  // ===========================
+  // Timer rider (§4, §9.6)
+  // ===========================
+
+  /**
+   * Schedule or cancel a timer as part of this transaction.
+   *
+   *     await queen.transaction()
+   *       .ack(msg)
+   *       .timer('reminders').key(orderId).delay('24h').payload({ orderId }).schedule()
+   *       .commit()
+   *
+   * `.schedule()` and `.cancel()` are the terminals; they add the op and hand
+   * the transaction back. `peek` and `list` are reads and are not transaction
+   * operations.
+   *
+   * ONE ASYMMETRY TO KNOW ABOUT (§9.6): a cancel sent this way rides the
+   * bundle and shares its fate, including its authorization. The cancel that
+   * is guaranteed never to be blocked is the standalone
+   * `queen.timer(q).key(k).cancel()`, on its own DELETE route. Cancel inside a
+   * bundle when you need atomicity with an ack; cancel outside it when you
+   * need the cancel to land no matter what.
+   */
+  timer(queueName) {
+    return new TimerBuilder(null, queueName, (op) => {
+      this.#timerOps.push(op)
+      return this
+    })
+  }
+
   async commit() {
-    if (this.#operations.length === 0) {
+    const riderCount = this.#kvEntries.length + this.#timerOps.length
+    if (this.#operations.length === 0 && riderCount === 0) {
       logger.error('TransactionBuilder.commit', 'No operations to commit')
       throw new Error('Transaction has no operations to commit')
     }
 
-    logger.log('TransactionBuilder.commit', { operationCount: this.#operations.length, requiredLeases: this.#requiredLeases.length })
+    logger.log('TransactionBuilder.commit', {
+      operationCount: this.#operations.length,
+      requiredLeases: this.#requiredLeases.length,
+      kv: this.#kvEntries.length,
+      timers: this.#timerOps.length
+    })
+
+    // Byte-identity when the riders are absent (§6.3): the keys are added only
+    // when there is something in them, so a bundle that uses neither feature
+    // produces exactly the body it produced before this feature existed.
+    const body = {
+      operations: this.#operations,
+      requiredLeases: [...new Set(this.#requiredLeases)] // Unique leases
+    }
+    if (this.#kvEntries.length > 0) {
+      const now = Date.now()
+      body.kv = this.#kvEntries.map(e => materializeKvOp(e, now))
+    }
+    if (this.#timerOps.length > 0) {
+      body.timers = this.#timerOps
+    }
 
     try {
-      const result = await this.#httpClient.post('/api/v1/transaction', {
-        operations: this.#operations,
-        requiredLeases: [...new Set(this.#requiredLeases)] // Unique leases
-      })
+      const result = await this.#httpClient.post('/api/v1/transaction', body)
 
       if (!result.success) {
-        logger.error('TransactionBuilder.commit', { error: result.error })
-        throw new Error(result.error || 'Transaction failed')
+        // THE ONE OUTCOME THAT RETURNS INSTEAD OF THROWING (§8.3).
+        //
+        // A lost `required` KV precondition is not a failure: it is the
+        // EXPECTED outcome of every legitimate redelivery -- the idempotency
+        // marker doing its job -- and it arrives as HTTP 200 with
+        // `success:false, reason:'kv_precondition'` precisely so that it stays
+        // out of retry policies and error metrics. Throwing it would put the
+        // single most frequent outcome of this product inside every caller's
+        // catch block, where the natural reflex is to retry, which is the one
+        // thing that must not happen.
+        //
+        // The body carries `failedIndex` (in the FLAT index space of
+        // `results[]`), `kvReason`, `version` and `value`, so the caller can
+        // see who won without a second round trip.
+        if (result.reason === 'kv_precondition') {
+          logger.log('TransactionBuilder.commit', {
+            status: 'kv_precondition',
+            failedIndex: result.failedIndex,
+            kvReason: result.kvReason
+          })
+          return result
+        }
+
+        logger.error('TransactionBuilder.commit', { error: result.error, reason: result.reason })
+        const error = new Error(result.error || 'Transaction failed')
+        // The closed-taxonomy code travels ON the error, so no caller ever has
+        // to match the message: bad_request | duplicate | ack_rejected |
+        // timer_horizon_exceeded | payload_too_large | misaligned | db_error.
+        if (result.reason) error.reason = result.reason
+        error.result = result
+        throw error
       }
 
       logger.log('TransactionBuilder.commit', { status: 'success' })

package/client-v2/consumer/ConsumerManager.js

@@ -144,9 +144,11 @@ export class ConsumerManager {
       }
 
       try {
-        // Pop messages with affinity key for consistent routing
+        // Pop messages with affinity key for consistent routing. wait=true is
+        // a long-poll: mark it 'pop' so a 429 backs off and keeps waiting
+        // instead of giving up after the bounded push-like attempt budget.
         const clientTimeout = wait ? timeoutMillis + 5000 : timeoutMillis
-        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey)
+        const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey, wait ? 'pop' : null)
 
         // Handle empty response
         if (!result || !result.messages || result.messages.length === 0) {
@@ -188,9 +190,18 @@ export class ConsumerManager {
             for (const message of messages) {
               if (signal && signal.aborted) break
 
-              await this.#processMessage(message, handler, autoAck, group)
+              const ok = await this.#processMessage(message, handler, autoAck, group)
               processedCount++
 
+              // A nack releases the lease and clamps the server cursor at the
+              // failed message: everything after it in this popped batch WILL
+              // be redelivered. Processing it now would only produce duplicates
+              // and rejected acks — abandon the rest of the batch.
+              if (autoAck && !ok) {
+                logger.warn('ConsumerManager.worker', { workerId, status: 'batch-abandoned-after-nack', remaining: messages.length - messages.indexOf(message) - 1 })
+                break
+              }
+
               if (limit && processedCount >= limit) break
             }
           } else {
@@ -216,6 +227,20 @@ export class ConsumerManager {
           continue // Retry on timeout
         }
 
+        // 429 (rate limited): HttpClient already retries this internally
+        // with backoff (unbounded for wait=true pop, per retry429 policy) --
+        // this branch is a defensive fallback for the case where an explicit
+        // retry429.maxAttempts override got exhausted. Back off and keep
+        // polling instead of hot-looping or rethrowing/dying.
+        if (error.status === 429) {
+          const retryAfterMs = typeof error.retryAfterSeconds === 'number' && error.retryAfterSeconds >= 0
+            ? error.retryAfterSeconds * 1000
+            : 1000
+          logger.warn('ConsumerManager.worker', { workerId, status: 'rate-limited', code: error.code, retryAfterMs })
+          await new Promise(resolve => setTimeout(resolve, retryAfterMs))
+          continue
+        }
+
         // Check if network error
         const isNetworkError = error.message?.includes('fetch failed') ||
                               error.message?.includes('ECONNREFUSED') ||
@@ -228,8 +253,18 @@ export class ConsumerManager {
           continue
         }
 
+        // 403 (forbidden): terminal. cluster_suspended in particular can
+        // never resolve itself, and none of the other proxy codes
+        // (storage_quota_exceeded / feature_gated / forbidden) are worth
+        // hot-looping either -- stop this worker and surface the error
+        // (with .code) to the caller instead of retrying.
+        if (error.status === 403) {
+          logger.error('ConsumerManager.worker', { workerId, status: 'forbidden', code: error.code, error: error.message })
+          throw error
+        }
+
         // Other errors - rethrow
-        logger.error('ConsumerManager.worker', { workerId, error: error.message })
+        logger.error('ConsumerManager.worker', { workerId, error: error.message, code: error.code })
         throw error
       }
     }
@@ -237,6 +272,11 @@ export class ConsumerManager {
     logger.log('ConsumerManager.worker', { workerId, status: 'stopped', processedCount })
   }
 
+  /**
+   * Returns true when the message was handled (and acked) successfully,
+   * false when it was nacked — the caller must abandon the rest of the
+   * popped batch (the nack released the lease server-side).
+   */
   async #processMessage(message, handler, autoAck, group) {
     try {
       await handler(message)
@@ -244,18 +284,26 @@ export class ConsumerManager {
       // Auto-ack on success if enabled
       if (autoAck) {
         const context = group ? { group } : {}
-        await this.#queen.ack(message, true, context)
-        logger.log('ConsumerManager.processMessage', { transactionId: message.transactionId, status: 'acked' })
+        const res = await this.#queen.ack(message, true, context)
+        if (res && res.success === false) {
+          logger.error('ConsumerManager.processMessage', { transactionId: message.transactionId, status: 'ack-rejected', error: res.error })
+        } else {
+          logger.log('ConsumerManager.processMessage', { transactionId: message.transactionId, status: 'acked' })
+        }
       }
+      return true
     } catch (error) {
       // Auto-nack on error if enabled
       if (autoAck) {
-        const context = group ? { group } : {}
-        await this.#queen.ack(message, false, context)
+        const context = group ? { group, error: error.message } : { error: error.message }
+        const res = await this.#queen.ack(message, false, context)
+        if (res && res.success === false) {
+          logger.error('ConsumerManager.processMessage', { transactionId: message.transactionId, status: 'nack-rejected', error: res.error })
+        }
         logger.error('ConsumerManager.processMessage', { transactionId: message.transactionId, error: error.message, status: 'nacked' })
         // Don't rethrow when autoAck is enabled - NACK was already sent
         // This allows the consumer to continue and retry
-        return
+        return false
       }
       logger.error('ConsumerManager.processMessage', { transactionId: message.transactionId, error: error.message })
       throw error
@@ -269,13 +317,17 @@ export class ConsumerManager {
       // Auto-ack on success if enabled
       if (autoAck) {
         const context = group ? { group } : {}
-        await this.#queen.ack(messages, true, context)
-        logger.log('ConsumerManager.processBatch', { count: messages.length, status: 'acked' })
+        const res = await this.#queen.ack(messages, true, context)
+        if (res && res.success === false) {
+          logger.error('ConsumerManager.processBatch', { count: messages.length, status: 'ack-rejected', error: res.error })
+        } else {
+          logger.log('ConsumerManager.processBatch', { count: messages.length, status: 'acked' })
+        }
       }
     } catch (error) {
       // Auto-nack on error if enabled
       if (autoAck) {
-        const context = group ? { group } : {}
+        const context = group ? { group, error: error.message } : { error: error.message }
         await this.#queen.ack(messages, false, context)
         logger.error('ConsumerManager.processBatch', { count: messages.length, error: error.message, status: 'nacked' })
         // Don't rethrow when autoAck is enabled - NACK was already sent