queen-mq 1.0.5 → 1.0.6

package/README.md

@@ -324,6 +324,21 @@ await queen.flushAllBuffers()
 // Result: 10x-100x faster than individual pushes
 ```
 
+The buffer is **bounded and lossless**, and both properties are why `push()` must
+be awaited:
+
+| Option | Default | Meaning |
+| --- | --- | --- |
+| `messageCount` | `100` | Flush once this many messages are waiting |
+| `timeMillis` | `1000` | Or this long after the first message arrives |
+| `maxSize` | `4 x messageCount` | Backpressure bound: past this many buffered messages, `push()` WAITS for the flusher instead of growing the heap. There is no unbounded setting |
+| `retryDelayMillis` | `250` | Delay before retrying a batch whose POST failed. Failed batches go back to the front of the buffer, in order, and are retried — never dropped |
+
+A producer that outruns the flush pipeline is therefore paced down to the drain
+rate, and a broker outage shows up as slow pushes with bounded memory rather
+than as messages that quietly disappeared. `close()` flushes with a 30 second
+deadline and logs how many messages were left unsent if it expires.
+
 ### Dead Letter Queue
 
 ```javascript

package/client-v2/Queen.js

@@ -17,6 +17,12 @@ import { CLIENT_DEFAULTS } from './utils/defaults.js'
 import { validateUrl, validateUrls } from './utils/validation.js'
 import * as logger from './utils/logger.js'
 
+// How long close() keeps retrying a push batch the broker will not take before
+// it gives up, logs how many messages were never sent, and lets the process
+// exit. Matches CLIENT_DEFAULTS.timeoutMillis and the usual 30s SIGTERM grace:
+// long enough to ride out a broker restart, short enough that shutdown ends.
+const CLOSE_FLUSH_DEADLINE_MILLIS = 30000
+
 // Both /api/v1/ack and /api/v1/ack/batch respond with a top-level JSON array,
 // one item per acknowledgment in request order:
 //   [{index, transactionId, success, error, queueName, partitionName, leaseReleased, dlq}]
@@ -596,9 +602,13 @@ export class Queen {
   async close() {
     logger.log('Queen.close', 'Starting shutdown')
 
-    // Flush all buffers
+    // Flush all buffers, with a deadline. The flusher retries a failed batch
+    // forever rather than dropping it, which is right while the process is
+    // running and wrong on the way out: a SIGTERM grace period is finite, so
+    // shutdown stops retrying after CLOSE_FLUSH_DEADLINE_MILLIS and reports
+    // what is left instead of hanging until the runtime is killed.
     try {
-      await this.#bufferManager.flushAllBuffers()
+      await this.#bufferManager.flushAllBuffers({ deadlineMillis: CLOSE_FLUSH_DEADLINE_MILLIS })
       logger.log('Queen.close', 'All buffers flushed')
     } catch (error) {
       logger.error('Queen.close', { error: error.message, phase: 'buffer-flush' })

package/client-v2/buffer/BufferManager.js

@@ -1,176 +1,220 @@
 /**
- * Buffer manager for client-side message buffering across queues
+ * Buffer manager for client-side message buffering across queues.
+ *
+ * One MessageBuffer per `queue/partition` address (the granularity the broker
+ * fuses writes on), and exactly ONE drain loop per buffer. The drain is the
+ * only thing that sends: it takes `messageCount`-sized batches off the front,
+ * POSTs them, and wakes producers parked on the buffer's maxSize bound after
+ * each batch that is definitively gone.
+ *
+ * A batch whose POST fails goes straight back to the front of the buffer, in
+ * order, and is retried after `retryDelayMillis` -- indefinitely, until it
+ * lands or the buffer is stopped. That is the half of the 2026-08-20 fix that
+ * removes loss on flush error; MessageBuffer's docs carry the other half
+ * (blocking backpressure) and the measurements behind both.
+ *
+ * Deadlines: an explicit flush (`flushBuffer`, `flushAllBuffers`) may pass
+ * `deadlineMillis` to bound how long it is willing to keep retrying, because
+ * "retry forever" is right for a background flusher and wrong for a shutdown
+ * path that has a SIGTERM grace period to respect. When the deadline expires
+ * the messages are still in the buffer -- the error says how many -- so the
+ * failure is loud rather than silent.
  */
 
 import { MessageBuffer } from './MessageBuffer.js'
-import { BUFFER_DEFAULTS } from '../utils/defaults.js'
 import * as logger from '../utils/logger.js'
 
 export class BufferManager {
   #httpClient
   #buffers = new Map() // queueAddress -> MessageBuffer
-  #pendingFlushes = new Set() // Track in-flight flush promises
+  #drains = new Map()  // queueAddress -> { promise, ctl } for the in-flight drain
   #flushCount = 0
+  #stopped = false
 
   constructor(httpClient) {
     this.#httpClient = httpClient
   }
 
-  addMessage(queueAddress, formattedMessage, bufferOptions) {
-    const options = { ...BUFFER_DEFAULTS, ...bufferOptions }
+  /**
+   * Buffer one message, waiting for room if the buffer is at its bound.
+   *
+   * Returns a promise: the add path is where backpressure is applied, so
+   * callers MUST await it. PushBuilder does; anything that forgets would be
+   * back to the unbounded behaviour this replaced.
+   *
+   * @param {string} queueAddress
+   * @param {object} formattedMessage
+   * @param {object} bufferOptions
+   * @param {{ signal?: AbortSignal }} [opts]
+   */
+  async addMessage(queueAddress, formattedMessage, bufferOptions, { signal } = {}) {
+    // A push after cleanup() would otherwise create a fresh buffer that nothing
+    // will ever flush -- messages accepted into a client that is already shut
+    // down, which is the same false success the bound exists to remove.
+    if (this.#stopped) {
+      throw new Error(`Queen client is closed: message not buffered for ${queueAddress}`)
+    }
 
     if (!this.#buffers.has(queueAddress)) {
-      logger.log('BufferManager.createBuffer', { queueAddress, options })
-      this.#buffers.set(queueAddress, new MessageBuffer(
-        queueAddress,
-        options,
-        (addr) => this.#flushBuffer(addr)
-      ))
+      // The raw options go to the buffer, which fills in the defaults itself:
+      // maxSize is derived from the messageCount this caller asked for, and
+      // merging defaults here first would hide the difference between "not set"
+      // and "set to the default".
+      const created = new MessageBuffer(queueAddress, bufferOptions, (addr) => { this.#startDrain(addr) })
+      logger.log('BufferManager.createBuffer', { queueAddress, options: created.options })
+      this.#buffers.set(queueAddress, created)
     }
 
     const buffer = this.#buffers.get(queueAddress)
-    buffer.add(formattedMessage)
+    await buffer.add(formattedMessage, { signal })
     logger.log('BufferManager.addMessage', { queueAddress, messageCount: buffer.messageCount })
   }
 
-  async #flushBuffer(queueAddress) {
-    const buffer = this.#buffers.get(queueAddress)
-    if (!buffer || buffer.messageCount === 0) {
-      logger.debug('BufferManager.flushBuffer', { queueAddress, status: 'empty' })
-      return
+  /**
+   * Start the drain loop for an address, or join the one already running.
+   *
+   * Joining rather than starting a second loop is what keeps batches in order:
+   * two concurrent senders on the same partition would interleave their POSTs.
+   * Returns null when there is nothing to send.
+   *
+   * @param {string} queueAddress
+   * @param {number|null} deadlineMillis - how long a caller is willing to keep
+   *   retrying a failing batch; null (the default, and what background flushes
+   *   use) means "until it lands or the buffer stops".
+   */
+  #startDrain(queueAddress, deadlineMillis = null) {
+    const deadline = deadlineMillis === null || deadlineMillis === undefined
+      ? Number.POSITIVE_INFINITY
+      : Date.now() + deadlineMillis
+
+    const running = this.#drains.get(queueAddress)
+    if (running) {
+      // A caller with a deadline joining a background drain tightens it: the
+      // shortest patience wins, otherwise a shutdown could be held open by a
+      // retry loop that was started with none.
+      running.ctl.deadline = Math.min(running.ctl.deadline, deadline)
+      return running.promise
     }
 
-    logger.log('BufferManager.flushBuffer', { queueAddress, messageCount: buffer.messageCount })
-    buffer.setFlushing(true)
-
-    // Create a promise for this flush and track it
-    const flushPromise = (async () => {
-      try {
-        const messages = buffer.extractMessages()
-
-        logger.debug('BufferManager.flushBuffer', { queueAddress, extracted: messages.length })
-
-        if (messages.length === 0) return
-
-        // Send to server
-        const result = await this.#httpClient.post('/api/v1/push', { items: messages })
-        logger.debug('BufferManager.flushBuffer', { queueAddress, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
-
-        this.#flushCount++
-        logger.log('BufferManager.flushBuffer', { queueAddress, status: 'success', messagesSent: messages.length })
-
-        // Remove empty buffer
-        this.#buffers.delete(queueAddress)
-
-      } catch (error) {
-        logger.error('BufferManager.flushBuffer', { queueAddress, error: error.message })
-        buffer.setFlushing(false)
-        throw error
-      } finally {
-        // Remove from pending flushes
-        this.#pendingFlushes.delete(flushPromise)
-      }
-    })()
-
-    // Track this flush
-    this.#pendingFlushes.add(flushPromise)
-
-    return flushPromise
-  }
-
-  async #flushBufferBatch(queueAddress, batchSize) {
     const buffer = this.#buffers.get(queueAddress)
-    if (!buffer || buffer.messageCount === 0) {
-      return
-    }
-
-    buffer.setFlushing(true)
-
-    // Create a promise for this flush and track it
-    const flushPromise = (async () => {
-      try {
-        const messages = buffer.extractMessages(batchSize)
-
-        logger.debug('BufferManager.flushBufferBatch', { queueAddress, extracted: messages.length })
-
-        if (messages.length === 0) return
-
-        // Send to server
-        const result = await this.#httpClient.post('/api/v1/push', { items: messages })
-        logger.debug('BufferManager.flushBufferBatch', { queueAddress, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
-
-        this.#flushCount++
+    if (!buffer || !buffer.beginFlush()) return null
+
+    const ctl = { deadline }
+    const promise = this.#drain(queueAddress, buffer, ctl)
+    this.#drains.set(queueAddress, { promise, ctl })
+    // The count-threshold and timer triggers do not await this promise, so give
+    // it a handler of its own: a deadline tightened by a concurrent explicit
+    // flush would otherwise surface as an unhandled rejection.
+    promise.catch(() => {})
+    return promise
+  }
 
-        // Remove empty buffer if no more messages
-        if (buffer.messageCount === 0) {
-          this.#buffers.delete(queueAddress)
-        } else {
-          buffer.setFlushing(false)
+  async #drain(queueAddress, buffer, ctl) {
+    const { messageCount, retryDelayMillis } = buffer.options
+
+    try {
+      while (buffer.messageCount > 0 && !buffer.isStopped) {
+        const batch = buffer.takeBatch(messageCount)
+        if (batch.length === 0) break
+
+        try {
+          const result = await this.#httpClient.post('/api/v1/push', { items: batch })
+          logger.debug('BufferManager.drain', { queueAddress, sent: batch.length, serverResponse: result ? `${result.length || 'N/A'} items` : 'null' })
+
+          this.#flushCount++
+          // Capacity freed. Woken here rather than at takeBatch, because a
+          // batch that fails to send goes straight back: waking on extraction
+          // would let producers refill against room that never freed.
+          buffer.wakeWaiters()
+        } catch (error) {
+          // NOT dropped. The batch goes back at the front, in order, and this
+          // loop retries it. Before 2026-08-20 this branch logged and moved on,
+          // losing up to messageCount messages per failed POST.
+          buffer.restoreBatch(batch)
+          logger.error('BufferManager.drain', { queueAddress, error: error.message, requeued: batch.length })
+
+          const remaining = ctl.deadline - Date.now()
+          if (remaining <= 0) {
+            error.queenUnflushedCount = buffer.messageCount
+            error.message = `${error.message} (${buffer.messageCount} message(s) still buffered for ${queueAddress}, not sent)`
+            throw error
+          }
+
+          await buffer.sleepUnlessStopped(Math.min(retryDelayMillis, remaining))
         }
-
-      } catch (error) {
-        logger.error('BufferManager.flushBufferBatch', { queueAddress, error: error.message })
-        buffer.setFlushing(false)
-        throw error
-      } finally {
-        // Remove from pending flushes
-        this.#pendingFlushes.delete(flushPromise)
       }
-    })()
-
-    // Track this flush
-    this.#pendingFlushes.add(flushPromise)
-
-    return flushPromise
+    } finally {
+      buffer.endFlush()
+      this.#drains.delete(queueAddress)
+      // Drop the entry only when nothing can still be pointed at it: a parked
+      // add holds this exact object, and deleting it here would leave that add
+      // appending into an orphan no drain would ever visit.
+      if (buffer.messageCount === 0 && !buffer.hasParkedAdds && !buffer.isStopped) {
+        this.#buffers.delete(queueAddress)
+      }
+      buffer.wakeWaiters()
+    }
   }
 
-  async flushBuffer(queueAddress) {
-    logger.log('BufferManager.flushBuffer', { queueAddress, activeBuffers: this.#buffers.size, pendingFlushes: this.#pendingFlushes.size })
-    
+  /**
+   * Send everything buffered for one address.
+   *
+   * @param {string} queueAddress
+   * @param {{ deadlineMillis?: number }} [opts] - stop retrying a failing batch
+   *   after this long and throw (the messages stay in the buffer). Omit to
+   *   retry until the batch lands.
+   */
+  async flushBuffer(queueAddress, { deadlineMillis = null } = {}) {
+    logger.log('BufferManager.flushBuffer', { queueAddress, activeBuffers: this.#buffers.size, pendingFlushes: this.#drains.size })
+
     const buffer = this.#buffers.get(queueAddress)
     if (!buffer) {
       logger.debug('BufferManager.flushBuffer', { queueAddress, status: 'not-found' })
-      await this.#waitForPendingFlushes()
+      await this.#waitForDrains()
       return
     }
 
-    // Cancel timer to prevent time-based flush
+    // Cancel the timer to prevent a time-based flush racing this one.
     buffer.cancelTimer()
-    
-    // Get the batch size from buffer options
-    const batchSize = buffer.options.messageCount
-    
-    // Flush all messages in batches
-    while (buffer.messageCount > 0) {
-      logger.debug('BufferManager.flushBuffer', { queueAddress, batchSize, remaining: buffer.messageCount })
-      await this.#flushBufferBatch(queueAddress, batchSize)
-    }
-    
-    // Wait for all pending flushes to complete
-    await this.#waitForPendingFlushes()
-    
+
+    const drain = this.#startDrain(queueAddress, deadlineMillis)
+    if (drain) await drain
+
     logger.debug('BufferManager.flushBuffer', { queueAddress, status: 'completed' })
   }
 
-  async flushAllBuffers() {
-    // Get all queue addresses that have buffers
-    const queueAddresses = Array.from(this.#buffers.keys())
-    logger.log('BufferManager.flushAllBuffers', { bufferCount: queueAddresses.length, pendingFlushes: this.#pendingFlushes.size })
-    
-    // Flush each buffer in batches
+  /**
+   * Send everything buffered, for every address.
+   *
+   * Drains run concurrently across addresses (they are independent buffers, and
+   * a shutdown should not pay for them one at a time) and every one is awaited
+   * before the first error is rethrown: an unreachable queue must not strand
+   * the others' messages.
+   */
+  async flushAllBuffers({ deadlineMillis = null } = {}) {
+    const queueAddresses = new Set([...this.#buffers.keys(), ...this.#drains.keys()])
+    logger.log('BufferManager.flushAllBuffers', { bufferCount: queueAddresses.size, pendingFlushes: this.#drains.size })
+
+    const drains = []
     for (const queueAddress of queueAddresses) {
-      await this.flushBuffer(queueAddress)
+      const buffer = this.#buffers.get(queueAddress)
+      if (buffer) buffer.cancelTimer()
+      const drain = this.#startDrain(queueAddress, deadlineMillis)
+      if (drain) drains.push(drain)
     }
-    
-    logger.log('BufferManager.flushAllBuffers', { status: 'completed' })
+
+    const outcomes = await Promise.allSettled(drains)
+    const failure = outcomes.find(outcome => outcome.status === 'rejected')
+
+    logger.log('BufferManager.flushAllBuffers', { status: failure ? 'failed' : 'completed' })
+    if (failure) throw failure.reason
   }
 
-  async #waitForPendingFlushes() {
-    if (this.#pendingFlushes.size === 0) return
-    
-    logger.debug('BufferManager.waitForPendingFlushes', { count: this.#pendingFlushes.size })
-    await Promise.all(Array.from(this.#pendingFlushes))
-    logger.debug('BufferManager.waitForPendingFlushes', { status: 'completed' })
+  async #waitForDrains() {
+    if (this.#drains.size === 0) return
+    logger.debug('BufferManager.waitForDrains', { count: this.#drains.size })
+    await Promise.allSettled([...this.#drains.values()].map(entry => entry.promise))
+    logger.debug('BufferManager.waitForDrains', { status: 'completed' })
   }
 
   getStats() {
@@ -189,12 +233,25 @@ export class BufferManager {
       oldestBufferAge,
       flushesPerformed: this.#flushCount
     }
-    
+
     logger.log('BufferManager.getStats', stats)
     return stats
   }
 
+  /**
+   * Stop every buffer and discard what is left.
+   *
+   * Stopping wakes parked adds (they reject: their message was never buffered)
+   * and ends any retry loop, so this also unhangs a drain that was waiting out
+   * a broker outage. Anything still buffered here is lost -- which is why
+   * Queen.close() flushes with a deadline first and logs what remains.
+   */
   cleanup() {
+    this.#stopped = true
+    const unflushed = this.getStats().totalBufferedMessages
+    if (unflushed > 0) {
+      logger.error('BufferManager.cleanup', { unflushedMessages: unflushed, status: 'discarded' })
+    }
     logger.log('BufferManager.cleanup', { bufferCount: this.#buffers.size })
     for (const buffer of this.#buffers.values()) {
       buffer.cleanup()

package/client-v2/buffer/MessageBuffer.js

@@ -1,7 +1,43 @@
 /**
- * Message buffer for a single queue
+ * Message buffer for a single queue/partition.
+ *
+ * This is the client-side linger: single pushes accumulate here and leave as
+ * one request once `messageCount` messages are waiting, or `timeMillis` has
+ * passed since the first one arrived. Two properties beyond that batching are
+ * load-bearing, and neither existed before 2026-08-20:
+ *
+ *  - `maxSize` is a BLOCKING bound, not a hint. `add()` returns a promise that
+ *    does not resolve while the buffer is full, so a producer that outruns the
+ *    flush pipeline is paced down to the drain rate instead of growing the
+ *    heap. Measured on the Go client, whose buffer had exactly this shape:
+ *    filling at 1.46M msg/s against a 1.0M msg/s flush pipeline accumulated
+ *    20.9M messages (11.7 GB of RSS) in 45 seconds and lost every one of them
+ *    at process exit, with ZERO client-side errors reported anywhere. The
+ *    bounded version sustained 881,148 msg/s with exact send/receive parity
+ *    (39,655,787 = 39,655,787) and 71 MB of RSS.
+ *
+ *  - A batch that fails to send is put BACK at the front of the buffer, in
+ *    order, and retried after `retryDelayMillis`. It is never dropped. Before
+ *    this, the flusher took the batch out before the POST and only logged the
+ *    failure, so up to `messageCount` messages vanished per failed request.
+ *
+ * Together those two turn a broker outage into blocked producers with bounded
+ * memory, instead of silent loss.
+ *
+ * BLOCKING IDIOM: JavaScript is single-threaded, so "block" cannot mean
+ * "occupy the thread" -- that would starve the very flush that frees the
+ * capacity being waited for. It means an awaitable gate: parked adds hold a
+ * promise that the flusher resolves after each drained batch (and `stop()`
+ * rejects). No spin loop, no setInterval poll: the event loop is free to run
+ * the flush while producers wait.
+ *
+ * Buffered messages still live only in this process's memory. A crash, or a
+ * `process.exit()` that skips `close()`, loses them -- buffering belongs on
+ * telemetry-shaped traffic, not on anything that must not be lost.
  */
 
+import { BUFFER_DEFAULTS } from '../utils/defaults.js'
+
 export class MessageBuffer {
   #queueAddress
   #messages = []
@@ -10,15 +46,93 @@ export class MessageBuffer {
   #timer = null
   #firstMessageTime = null
   #flushing = false
+  #stopped = false
+  // Adds parked on the maxSize bound. Each entry can be woken (capacity freed)
+  // or failed (stop, or the caller's AbortSignal).
+  #waiters = []
+  // Counts adds that have been woken but have not resumed yet. JS resumes an
+  // awaiting function on a later microtask, so the waiter list is already empty
+  // while those adds are still in flight; without this counter BufferManager
+  // could drop the buffer entry out of its map in that window and the resumed
+  // add would append to an orphan nobody ever flushes.
+  #parked = 0
+  #stopWaiters = []
 
   constructor(queueAddress, options, flushCallback) {
     this.#queueAddress = queueAddress
-    this.#options = options
+    this.#options = MessageBuffer.normalizeOptions(options)
     this.#flushCallback = flushCallback
   }
 
-  add(formattedMessage) {
-    // Set first message time if this is the first message
+  /**
+   * Fill in defaults and enforce the bound's invariants.
+   *
+   * `maxSize: 0` (or absent) means the DEFAULT bound, never "unbounded":
+   * unbounded is the defect this knob exists to close, so opting out of
+   * backpressure is deliberately not expressible. The floor keeps the bound
+   * sane when a caller sets a `messageCount` larger than their `maxSize` --
+   * a buffer that must block before it can even assemble one batch would
+   * deadlock against its own flush threshold.
+   */
+  static normalizeOptions(options) {
+    // The caller's raw options, NOT BUFFER_DEFAULTS spread over them: the bound
+    // is derived from whatever messageCount this buffer ended up with, so
+    // `buffer({ messageCount: 10 })` gets a bound of 40, not the 400 that suits
+    // the default batch of 100. Unknown keys are carried through untouched.
+    const provided = options || {}
+
+    const messageCount = provided.messageCount > 0 ? provided.messageCount : BUFFER_DEFAULTS.messageCount
+    const timeMillis = provided.timeMillis > 0 ? provided.timeMillis : BUFFER_DEFAULTS.timeMillis
+
+    let maxSize = provided.maxSize > 0 ? provided.maxSize : 4 * messageCount
+    if (maxSize < messageCount) maxSize = messageCount
+
+    const retryDelayMillis = provided.retryDelayMillis > 0
+      ? provided.retryDelayMillis
+      : BUFFER_DEFAULTS.retryDelayMillis
+
+    return { ...provided, messageCount, timeMillis, maxSize, retryDelayMillis }
+  }
+
+  /**
+   * Append one message, waiting for room if the buffer is at its bound.
+   *
+   * Resolves once the message is in the buffer. Rejects if the buffer is
+   * stopped while parked, or if `signal` aborts -- an add that could not be
+   * buffered must never look like a successful push.
+   *
+   * @param {object} formattedMessage
+   * @param {{ signal?: AbortSignal }} [opts]
+   */
+  async add(formattedMessage, { signal } = {}) {
+    if (this.#stopped) {
+      throw new Error(`Queen buffer ${this.#queueAddress} is stopped: message not buffered`)
+    }
+    if (signal?.aborted) throw abortReason(signal)
+
+    // BACKPRESSURE. Re-checked in a loop, not once: a broadcast wakes every
+    // parked add, and the first ones to resume can fill the room that was
+    // freed, so the rest have to park again.
+    while (this.#messages.length >= this.#options.maxSize && !this.#stopped) {
+      // Being at the bound means producers outran the flusher. Make sure one is
+      // actually running before parking -- the time-based flush may be a full
+      // `timeMillis` away, and nothing else will start it.
+      this.#triggerFlush()
+      // The counter is held across the resumption itself, not just the wait:
+      // see #parked for why the window between "woken" and "resumed" matters.
+      this.#parked++
+      try {
+        await this.#waitForCapacity(signal)
+      } finally {
+        this.#parked--
+      }
+      if (signal?.aborted) throw abortReason(signal)
+    }
+
+    if (this.#stopped) {
+      throw new Error(`Queen buffer ${this.#queueAddress} stopped while waiting for capacity: message not buffered`)
+    }
+
     if (this.#messages.length === 0) {
       this.#firstMessageTime = Date.now()
       this.#startTimer()
@@ -26,77 +140,146 @@ export class MessageBuffer {
 
     this.#messages.push(formattedMessage)
 
-    // Check if we should flush based on size
     if (this.#messages.length >= this.#options.messageCount) {
       this.#triggerFlush()
     }
   }
 
+  #waitForCapacity(signal) {
+    return new Promise((resolve, reject) => {
+      const waiter = { resolve, reject, signal, onAbort: null }
+
+      waiter.settle = (fn, arg) => {
+        const index = this.#waiters.indexOf(waiter)
+        if (index !== -1) this.#waiters.splice(index, 1)
+        if (waiter.onAbort) signal.removeEventListener('abort', waiter.onAbort)
+        fn(arg)
+      }
+
+      if (signal) {
+        waiter.onAbort = () => waiter.settle(reject, abortReason(signal))
+        signal.addEventListener('abort', waiter.onAbort, { once: true })
+      }
+
+      this.#waiters.push(waiter)
+    })
+  }
+
+  /**
+   * Wake every parked add. Called by the flusher after a batch is definitively
+   * gone (the POST succeeded), not when the batch is merely taken out of the
+   * buffer: a batch that fails goes straight back, and waking on extraction
+   * would let producers refill against room that never actually freed.
+   */
+  wakeWaiters() {
+    const waiters = this.#waiters.slice()
+    for (const waiter of waiters) waiter.settle(waiter.resolve)
+  }
+
   #startTimer() {
     if (this.#timer) return // Timer already running
 
+    // Deliberately NOT unref()'d: a short script that pushes and returns must
+    // stay alive long enough for the time-based flush to fire.
     this.#timer = setTimeout(() => {
+      this.#timer = null
       this.#triggerFlush()
     }, this.#options.timeMillis)
   }
 
   #triggerFlush() {
-    if (this.#flushing || this.#messages.length === 0) return
-
-    // Clear timer
-    if (this.#timer) {
-      clearTimeout(this.#timer)
-      this.#timer = null
-    }
-
-    // Trigger flush via callback
+    if (this.#flushing || this.#stopped || this.#messages.length === 0) return
     this.#flushCallback(this.#queueAddress)
   }
 
-  extractMessages(batchSize = null) {
-    // If no batch size specified, extract all messages
-    if (batchSize === null || batchSize >= this.#messages.length) {
-      const messages = [...this.#messages]
-      this.#messages = []
-      this.#firstMessageTime = null
-      this.#flushing = false
-
-      if (this.#timer) {
-        clearTimeout(this.#timer)
-        this.#timer = null
-      }
+  /**
+   * Claim the right to flush this buffer. Returns false when a flush is already
+   * running: one drain loop per buffer, so a burst of adds past the threshold
+   * cannot start a second sender that would interleave batches out of order.
+   */
+  beginFlush() {
+    if (this.#flushing || this.#stopped || this.#messages.length === 0) return false
+    this.#flushing = true
+    this.cancelTimer()
+    return true
+  }
 
-      return messages
-    }
+  /**
+   * Release the flush claim. Anything still buffered (a batch put back by a
+   * failed send, or messages added while the drain was stopping) gets a fresh
+   * timer, so it cannot sit there unnoticed until the next add.
+   */
+  endFlush() {
+    this.#flushing = false
+    if (this.#messages.length > 0 && !this.#stopped) this.#startTimer()
+  }
 
-    // Extract a batch of messages
+  /**
+   * Take up to `batchSize` messages off the front.
+   *
+   * `splice` returns a fresh array, so the batch does not alias the buffer's
+   * storage and putting it back cannot corrupt what is left behind. (The Go
+   * reference has to copy explicitly there -- its slices do alias.)
+   */
+  takeBatch(batchSize) {
     const messages = this.#messages.splice(0, batchSize)
-    
-    // If buffer is now empty, reset state
-    if (this.#messages.length === 0) {
-      this.#firstMessageTime = null
-      this.#flushing = false
-      
-      if (this.#timer) {
-        clearTimeout(this.#timer)
-        this.#timer = null
-      }
-    }
-
+    if (this.#messages.length === 0) this.#firstMessageTime = null
     return messages
   }
 
-  setFlushing(value) {
-    this.#flushing = value
+  /**
+   * Put a failed batch back at the FRONT, preserving order: these messages were
+   * queued before everything still in the buffer, and a retry must not reorder
+   * a partition's lane. Occupancy can overshoot maxSize by this one batch --
+   * documented on BUFFER_DEFAULTS.maxSize.
+   */
+  restoreBatch(messages) {
+    if (messages.length === 0) return
+    this.#messages.unshift(...messages)
+    if (this.#firstMessageTime === null) this.#firstMessageTime = Date.now()
   }
 
-  forceFlush() {
-    // Immediately trigger flush, ignoring timers
-    if (this.#timer) {
-      clearTimeout(this.#timer)
-      this.#timer = null
+  /**
+   * Sleep, but return early if the buffer is stopped. Used for the retry delay
+   * between attempts at a failed batch: a shutdown must not wait out a delay
+   * that only exists to pace a broker that is not answering.
+   */
+  sleepUnlessStopped(millis) {
+    if (this.#stopped || millis <= 0) return Promise.resolve()
+    return new Promise(resolve => {
+      let timer = null
+      const wake = () => {
+        if (timer) clearTimeout(timer)
+        const index = this.#stopWaiters.indexOf(wake)
+        if (index !== -1) this.#stopWaiters.splice(index, 1)
+        resolve()
+      }
+      timer = setTimeout(wake, millis)
+      this.#stopWaiters.push(wake)
+    })
+  }
+
+  /**
+   * Stop accepting messages and wake everything parked, so shutdown cannot
+   * hang. Parked adds REJECT rather than resolve: their message was never
+   * buffered, and reporting success for a message that was dropped on the floor
+   * is the failure mode this whole change exists to remove.
+   */
+  stop() {
+    if (this.#stopped) return
+    this.#stopped = true
+    this.cancelTimer()
+
+    const waiters = this.#waiters.slice()
+    for (const waiter of waiters) {
+      waiter.settle(
+        waiter.reject,
+        new Error(`Queen buffer ${this.#queueAddress} stopped while waiting for capacity: message not buffered`)
+      )
     }
-    this.#triggerFlush()
+
+    const sleepers = this.#stopWaiters.splice(0, this.#stopWaiters.length)
+    for (const wake of sleepers) wake()
   }
 
   cancelTimer() {
@@ -115,18 +298,39 @@ export class MessageBuffer {
     return this.#options
   }
 
+  get isFlushing() {
+    return this.#flushing
+  }
+
+  get isStopped() {
+    return this.#stopped
+  }
+
+  /** True while any add is parked on the bound, or woken but not yet resumed. */
+  get hasParkedAdds() {
+    return this.#waiters.length > 0 || this.#parked > 0
+  }
+
   get firstMessageAge() {
     return this.#firstMessageTime ? Date.now() - this.#firstMessageTime : 0
   }
 
   cleanup() {
-    if (this.#timer) {
-      clearTimeout(this.#timer)
-      this.#timer = null
-    }
+    this.stop()
     this.#messages = []
     this.#firstMessageTime = null
     this.#flushing = false
   }
 }
 
+/**
+ * The reason an AbortSignal carries, or a plain AbortError when the runtime
+ * (or the caller's controller) did not set one.
+ */
+function abortReason(signal) {
+  if (signal.reason instanceof Error) return signal.reason
+  const error = new Error('Queen buffered push aborted while waiting for buffer capacity')
+  error.name = 'AbortError'
+  if (signal.reason !== undefined) error.cause = signal.reason
+  return error
+}

package/client-v2/builders/QueueBuilder.js

@@ -138,6 +138,21 @@ export class QueueBuilder {
     return this
   }
 
+  /**
+   * Batch pushes client-side instead of sending each one.
+   *
+   * @param {object} options
+   * @param {number} [options.messageCount=100] - flush once this many messages are waiting
+   * @param {number} [options.timeMillis=1000] - flush this long after the first message arrives
+   * @param {number} [options.maxSize] - backpressure bound: past this many buffered
+   *   messages `push()` WAITS for the flusher instead of growing the heap.
+   *   Defaults to 4 x messageCount, floored at messageCount. There is no
+   *   unbounded setting: unbounded is what lost 20.9M messages in the
+   *   2026-08-20 measurement.
+   * @param {number} [options.retryDelayMillis=250] - delay before retrying a batch
+   *   whose POST failed. Failed batches are re-queued at the front of the
+   *   buffer and retried, never dropped.
+   */
   buffer(options) {
     this.#bufferOptions = options
     return this
@@ -622,20 +637,42 @@ class PushBuilder {
   async #execute() {
     logger.log('PushBuilder.execute', { queue: this.#queueName, partition: this.#partition, count: this.#formattedItems.length, buffered: !!this.#bufferOptions })
     
-    // Client-side buffering
+    // Client-side buffering. Awaited, one message at a time: addMessage is
+    // where the maxSize backpressure bound is applied, so a buffered push
+    // resolves only once every item is actually IN the buffer. Firing these
+    // off without awaiting would report success for messages the buffer never
+    // accepted -- the exact failure this bound exists to remove.
     if (this.#bufferOptions) {
-      for (const item of this.#formattedItems) {
-        const queueAddress = `${this.#queueName}/${this.#partition}`
-        this.#bufferManager.addMessage(queueAddress, item, this.#bufferOptions)
+      const queueAddress = `${this.#queueName}/${this.#partition}`
+      const accepted = []
+
+      try {
+        for (const item of this.#formattedItems) {
+          await this.#bufferManager.addMessage(queueAddress, item, this.#bufferOptions)
+          accepted.push(item)
+        }
+      } catch (error) {
+        // The buffer refused this message (the client is closing, or the wait
+        // for capacity was aborted). Report the items that did NOT make it, the
+        // same way the immediate push below reports rejected items -- counting
+        // them as buffered would be the false success this bound removes.
+        const unbuffered = this.#formattedItems.slice(accepted.length)
+        logger.error('PushBuilder.execute', { status: 'not-buffered', count: unbuffered.length, error: error.message })
+        if (this.#onErrorCallback) {
+          await this.#onErrorCallback(unbuffered, error)
+          return null
+        }
+        throw error
       }
-      const result = { buffered: true, count: this.#formattedItems.length }
-      
-      logger.log('PushBuilder.execute', { status: 'buffered', count: this.#formattedItems.length })
-      
+
+      const result = { buffered: true, count: accepted.length }
+
+      logger.log('PushBuilder.execute', { status: 'buffered', count: accepted.length })
+
       if (this.#onSuccessCallback) {
-        await this.#onSuccessCallback(this.#formattedItems)
+        await this.#onSuccessCallback(accepted)
       }
-      
+
       return result
     }
 

package/client-v2/utils/defaults.js

@@ -75,6 +75,20 @@ export const POP_DEFAULTS = {
 
 export const BUFFER_DEFAULTS = {
   messageCount: 100,                   // Flush after 100 messages
-  timeMillis: 1000                     // Or flush after 1 second
+  timeMillis: 1000,                    // Or flush after 1 second
+  // Backpressure bound: once this many messages are waiting, a buffered push
+  // WAITS for the flusher to drain below it instead of growing the heap. 0 (or
+  // absent) means 4 x messageCount -- "unbounded" is deliberately not
+  // expressible, because unbounded was the defect. Measured motivation
+  // (2026-08-20): without a bound, a producer filling at 1.46M msg/s against a
+  // 1.0M msg/s flush pipeline accumulated 20.9M messages (11.7 GB) in 45
+  // seconds and lost every one of them at process exit, with zero client-side
+  // errors. The bound is approximate: a batch that fails to send is put back,
+  // so occupancy can briefly overshoot by up to one messageCount.
+  maxSize: 400,
+  // How long the flusher waits before retrying a batch whose POST failed. The
+  // batch is re-queued at the front of the buffer and retried until it lands
+  // (or the buffer is stopped) -- never dropped. 0 means 250.
+  retryDelayMillis: 250
 }
 

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "type": "module",
   "description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
-    "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js",
+    "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js && node test-v2/run.js human",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",

package/test-v2/buffer-unit/buffer.test.js

@@ -0,0 +1,404 @@
+/**
+ * Client-side push buffer: backpressure and loss-free retry.
+ *
+ * These pin the 2026-08-20 fix, ported from the Go reference
+ * (clients/client-go/message_buffer.go). Two defects were measured there and
+ * were present in this SDK too:
+ *
+ *   1. add() appended to an unbounded array and returned. A producer filling at
+ *      1.46M msg/s against a 1.0M msg/s flush pipeline accumulated 20.9M
+ *      messages (11.7 GB of RSS) in 45 seconds and lost every one at process
+ *      exit, with zero client-side errors reported.
+ *   2. The flusher took a batch out of the buffer before the POST and dropped
+ *      it on error, losing up to messageCount messages per failed request.
+ *
+ * No broker and no network: the sink is a fake object with a post() the test
+ * drives, which is the whole point -- the buffer's contract is about ordering,
+ * occupancy and retries, none of which need a server to observe.
+ */
+
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { QueueBuilder } from '../../client-v2/builders/QueueBuilder.js'
+import { BufferManager } from '../../client-v2/buffer/BufferManager.js'
+import { MessageBuffer } from '../../client-v2/buffer/MessageBuffer.js'
+import { BUFFER_DEFAULTS } from '../../client-v2/utils/defaults.js'
+
+const ADDRESS = 'orders/Default'
+
+const item = n => ({ queue: 'orders', partition: 'Default', payload: { n }, transactionId: `t-${n}` })
+const payloadNumbers = items => items.map(i => i.payload.n)
+
+/** A sink whose every POST hangs until the test releases or fails it. */
+function controlledSink() {
+  const sent = []
+  const attempts = []
+  const inflight = []
+
+  return {
+    sent,
+    attempts,
+    get inflightCount() { return inflight.length },
+    post(_path, body) {
+      const items = body.items
+      attempts.push(items)
+      return new Promise((resolve, reject) => {
+        inflight.push({ items, resolve, reject })
+      })
+    },
+    /** Complete the oldest in-flight POST successfully. */
+    release() {
+      const call = inflight.shift()
+      assert.ok(call, 'no POST in flight to release')
+      sent.push(...call.items)
+      call.resolve([])
+    },
+    /** Fail the oldest in-flight POST. */
+    reject(message = 'connect ECONNREFUSED') {
+      const call = inflight.shift()
+      assert.ok(call, 'no POST in flight to reject')
+      call.reject(new Error(message))
+    }
+  }
+}
+
+/**
+ * A sink that answers immediately, failing whenever `shouldFail(attemptIndex)`
+ * says so. Used for the parity runs, where the interesting question is what
+ * comes out the far end after N failures, not when each one happened.
+ */
+function flakySink(shouldFail) {
+  const sent = []
+  let attempt = 0
+  return {
+    sent,
+    get attempts() { return attempt },
+    async post(_path, body) {
+      const index = attempt++
+      if (shouldFail(index)) throw new Error(`sink refused attempt ${index}`)
+      sent.push(...body.items)
+      return []
+    }
+  }
+}
+
+const tick = (ms = 0) => new Promise(resolve => setTimeout(resolve, ms))
+
+/** Poll until `predicate()` holds, or fail the test. Cheaper than a fixed sleep
+ *  on an idle machine and less flaky than one on a loaded machine. */
+async function until(predicate, what, timeoutMs = 2000) {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    if (predicate()) return
+    await tick(2)
+  }
+  assert.fail(`timed out waiting for: ${what}`)
+}
+
+/** Track whether a promise has settled, without awaiting it. */
+function watch(promise) {
+  const state = { settled: false, value: undefined, error: undefined }
+  promise.then(
+    value => { state.settled = true; state.value = value },
+    error => { state.settled = true; state.error = error }
+  )
+  return state
+}
+
+describe('buffer options', () => {
+  it('resolves an absent or zero maxSize to a BOUND, never to unbounded', () => {
+    // Unbounded is the defect this knob closes, so it is deliberately not
+    // expressible: 0 means the default bound, not infinity.
+    assert.equal(MessageBuffer.normalizeOptions({ messageCount: 10 }).maxSize, 40)
+    assert.equal(MessageBuffer.normalizeOptions({ messageCount: 10, maxSize: 0 }).maxSize, 40)
+    assert.equal(MessageBuffer.normalizeOptions({ messageCount: 10, maxSize: -1 }).maxSize, 40)
+    assert.equal(MessageBuffer.normalizeOptions({}).maxSize, BUFFER_DEFAULTS.maxSize)
+  })
+
+  it('floors maxSize at messageCount', () => {
+    // A bound below the flush threshold would block the producer before the
+    // buffer could assemble the batch that unblocks it.
+    assert.equal(MessageBuffer.normalizeOptions({ messageCount: 100, maxSize: 10 }).maxSize, 100)
+  })
+
+  it('defaults retryDelayMillis rather than retrying in a hot loop', () => {
+    assert.equal(MessageBuffer.normalizeOptions({ messageCount: 10 }).retryDelayMillis, 250)
+    assert.equal(MessageBuffer.normalizeOptions({ retryDelayMillis: 0 }).retryDelayMillis, 250)
+    assert.equal(MessageBuffer.normalizeOptions({ retryDelayMillis: 5 }).retryDelayMillis, 5)
+  })
+
+  it('ships a bounded default', () => {
+    assert.equal(BUFFER_DEFAULTS.maxSize, 4 * BUFFER_DEFAULTS.messageCount)
+  })
+})
+
+describe('backpressure', () => {
+  it('parks an add at the bound and resumes it when the flusher drains', async () => {
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 2, timeMillis: 60000, maxSize: 4, retryDelayMillis: 5 }
+
+    // The first two adds trip the count threshold and start the drain, which
+    // takes them and then hangs on the sink. The next four pile up behind it.
+    for (let n = 0; n < 6; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await until(() => sink.inflightCount === 1, 'the first batch to reach the sink')
+    assert.equal(manager.getStats().totalBufferedMessages, 4, 'the buffer should be sitting exactly at its bound')
+
+    // The seventh add has nowhere to go: it must WAIT, not grow the buffer.
+    const parked = watch(manager.addMessage(ADDRESS, item(6), options))
+    await tick(20)
+    assert.equal(parked.settled, false, 'add returned while the buffer was full: that is the unbounded defect')
+    assert.equal(manager.getStats().totalBufferedMessages, 4, 'buffer grew past maxSize while an add was parked')
+
+    // Draining a batch frees room, which is what wakes the parked add.
+    sink.release()
+    await until(() => parked.settled, 'the parked add to resume once capacity freed')
+    assert.equal(parked.error, undefined, 'the resumed add must succeed, not error')
+    assert.deepEqual(payloadNumbers(sink.sent), [0, 1], 'the drained batch went out in order')
+
+    manager.cleanup()
+  })
+
+  it('wakes parked adds on cleanup, and tells them the message was NOT buffered', async () => {
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 2, timeMillis: 60000, maxSize: 2, retryDelayMillis: 5 }
+
+    for (let n = 0; n < 4; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await until(() => manager.getStats().totalBufferedMessages === 2, 'the buffer to reach its bound')
+
+    const parked = watch(manager.addMessage(ADDRESS, item(99), options))
+    await tick(20)
+    assert.equal(parked.settled, false, 'precondition: the add is parked')
+
+    // Shutdown must not leave a producer waiting forever...
+    manager.cleanup()
+    await until(() => parked.settled, 'cleanup() to wake the parked add')
+
+    // ...and must not tell it the message went somewhere. It did not.
+    assert.ok(parked.error instanceof Error, 'a parked add woken by cleanup must reject, not resolve')
+    assert.match(parked.error.message, /not buffered/)
+  })
+
+  it('fails a parked add when its AbortSignal fires, instead of reporting success', async () => {
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 2, timeMillis: 60000, maxSize: 2, retryDelayMillis: 5 }
+
+    for (let n = 0; n < 4; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await until(() => manager.getStats().totalBufferedMessages === 2, 'the buffer to reach its bound')
+
+    const controller = new AbortController()
+    const parked = watch(manager.addMessage(ADDRESS, item(99), options, { signal: controller.signal }))
+    await tick(20)
+    assert.equal(parked.settled, false, 'precondition: the add is parked')
+
+    controller.abort()
+    await until(() => parked.settled, 'the abort to release the parked add')
+    assert.ok(parked.error instanceof Error, 'an aborted add must reject')
+    assert.equal(parked.error.name, 'AbortError')
+    assert.equal(manager.getStats().totalBufferedMessages, 2, 'an aborted add must not leave its message behind')
+
+    manager.cleanup()
+  })
+})
+
+describe('failed flushes', () => {
+  it('puts a failed batch back at the FRONT, in order, and retries it after retryDelayMillis', async () => {
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 3, timeMillis: 60000, maxSize: 12, retryDelayMillis: 40 }
+
+    for (let n = 0; n < 3; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await until(() => sink.inflightCount === 1, 'the first attempt')
+
+    const failedAt = Date.now()
+    sink.reject()
+
+    // The batch is back in the buffer -- not logged and forgotten.
+    await until(() => manager.getStats().totalBufferedMessages === 3, 'the failed batch to be re-queued')
+    assert.equal(manager.getStats().flushesPerformed, 0, 'a POST that never landed must not count as a flush')
+
+    await until(() => sink.inflightCount === 1, 'the retry')
+    assert.ok(Date.now() - failedAt >= 35, 'the retry must wait out retryDelayMillis, not spin')
+    assert.deepEqual(payloadNumbers(sink.attempts[1]), [0, 1, 2], 'the retry must resend the same batch, in the same order')
+
+    sink.release()
+    await until(() => manager.getStats().totalBufferedMessages === 0, 'the retry to drain the buffer')
+    assert.deepEqual(payloadNumbers(sink.sent), [0, 1, 2])
+    assert.equal(manager.getStats().flushesPerformed, 1)
+
+    manager.cleanup()
+  })
+
+  it('keeps a re-queued batch ahead of messages added while it was failing', async () => {
+    // Ordering within a partition is the product's headline promise: a retry
+    // that landed behind newer messages would reorder the lane.
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 2, timeMillis: 60000, maxSize: 20, retryDelayMillis: 10 }
+
+    for (let n = 0; n < 2; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await until(() => sink.inflightCount === 1, 'the first attempt')
+    sink.reject()
+    await until(() => manager.getStats().totalBufferedMessages === 2, 're-queue')
+
+    for (let n = 2; n < 6; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+
+    for (let batch = 0; batch < 3; batch++) {
+      await until(() => sink.inflightCount === 1, `attempt for batch ${batch}`)
+      sink.release()
+    }
+    await until(() => manager.getStats().totalBufferedMessages === 0, 'the buffer to drain')
+    assert.deepEqual(payloadNumbers(sink.sent), [0, 1, 2, 3, 4, 5])
+
+    manager.cleanup()
+  })
+
+  it('loses nothing across a run with intermittent failures', async () => {
+    // The whole point, stated as count parity: every message a caller was told
+    // was buffered comes out of the sink exactly once, in order, however many
+    // POSTs failed on the way.
+    const total = 500
+    const sink = flakySink(attempt => attempt % 3 === 1)
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 7, timeMillis: 50, maxSize: 21, retryDelayMillis: 1 }
+
+    for (let n = 0; n < total; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await manager.flushAllBuffers()
+
+    assert.equal(sink.sent.length, total, `sent ${sink.sent.length} of ${total}: messages were dropped`)
+    assert.deepEqual(payloadNumbers(sink.sent), Array.from({ length: total }, (_, n) => n), 'order was not preserved')
+    assert.equal(manager.getStats().totalBufferedMessages, 0)
+    assert.ok(sink.attempts > total / options.messageCount, 'precondition: the sink actually failed some attempts')
+
+    manager.cleanup()
+  })
+
+  it('bounds an explicit flush by its deadline and says how much is still buffered', async () => {
+    // Background flushes retry forever rather than drop; a shutdown cannot.
+    const sink = flakySink(() => true)
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 2, timeMillis: 60000, maxSize: 8, retryDelayMillis: 5 }
+
+    for (let n = 0; n < 4; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+
+    await assert.rejects(
+      () => manager.flushAllBuffers({ deadlineMillis: 30 }),
+      error => {
+        assert.equal(error.queenUnflushedCount, 4, 'the error must report what was left unsent')
+        assert.match(error.message, /still buffered/)
+        return true
+      }
+    )
+    assert.equal(manager.getStats().totalBufferedMessages, 4, 'a flush that gave up must leave the messages in the buffer, not drop them')
+
+    manager.cleanup()
+  })
+})
+
+describe('drain loop', () => {
+  it('runs one drain per buffer, no matter how many adds trip the threshold', async () => {
+    // Two senders on one partition would interleave batches and reorder the
+    // lane; the flushing flag is what prevents that.
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 2, timeMillis: 60000, maxSize: 100, retryDelayMillis: 5 }
+
+    for (let n = 0; n < 10; n++) {
+      await manager.addMessage(ADDRESS, item(n), options)
+    }
+    await until(() => sink.inflightCount === 1, 'the first batch')
+    await tick(20)
+    assert.equal(sink.inflightCount, 1, 'a second drain started: batches can now interleave')
+    assert.equal(sink.attempts.length, 1)
+
+    manager.cleanup()
+  })
+
+  it('keeps separate buffers per queue/partition address', async () => {
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 100, timeMillis: 60000, maxSize: 400, retryDelayMillis: 5 }
+
+    await manager.addMessage('orders/eu', item(1), options)
+    await manager.addMessage('orders/us', item(2), options)
+    await manager.addMessage('other/eu', item(3), options)
+
+    const stats = manager.getStats()
+    assert.equal(stats.activeBuffers, 3)
+    assert.equal(stats.totalBufferedMessages, 3)
+
+    manager.cleanup()
+  })
+
+  it('keeps the public push API shape now that the add path can wait', async () => {
+    // addMessage became awaitable so backpressure could exist at all. The
+    // builder is what most callers actually touch, so pin its shape: a buffered
+    // push still resolves to { buffered, count } and still lands in the buffer.
+    // The builder is driven over the fake sink rather than a Queen, so the
+    // test owns the teardown: a real client's pending flush timer would keep
+    // the runner alive for the full timeMillis.
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const builder = new QueueBuilder(null, sink, manager, 'orders')
+
+    const result = await builder
+      .partition('eu')
+      .buffer({ messageCount: 1000, timeMillis: 60000 })
+      .push([{ data: { n: 1 } }, { data: { n: 2 } }])
+
+    assert.deepEqual(result, { buffered: true, count: 2 })
+    const stats = manager.getStats()
+    assert.equal(stats.totalBufferedMessages, 2)
+    assert.equal(stats.activeBuffers, 1, 'one buffer per queue/partition, not one per push')
+
+    manager.cleanup()
+  })
+
+  it('reports the items a stopped buffer refused instead of counting them as pushed', async () => {
+    const sink = controlledSink()
+    const manager = new BufferManager(sink)
+    const builder = new QueueBuilder(null, sink, manager, 'orders')
+    manager.cleanup() // client already closing
+
+    // push() returns a thenable builder, not a Promise, so await it inside.
+    await assert.rejects(
+      async () => { await builder.buffer({ messageCount: 1000, timeMillis: 60000 }).push([{ data: { n: 1 } }]) },
+      /not buffered/
+    )
+    assert.equal(manager.getStats().totalBufferedMessages, 0, 'a closed client must not accumulate messages nothing will flush')
+  })
+
+  it('flushes a buffer that never reaches its count, on the timer', async () => {
+    // The branch every other test here avoids with a 60s timer, and the one a
+    // low-volume producer actually lives on.
+    const sink = flakySink(() => false)
+    const manager = new BufferManager(sink)
+    const options = { messageCount: 1000, timeMillis: 30, maxSize: 4000, retryDelayMillis: 5 }
+
+    await manager.addMessage(ADDRESS, item(1), options)
+    await manager.addMessage(ADDRESS, item(2), options)
+    assert.equal(manager.getStats().totalBufferedMessages, 2, 'two messages are nowhere near the threshold')
+
+    await until(() => sink.sent.length === 2, 'the time-based flush to fire')
+    manager.cleanup()
+  })
+})