queen-mq 2.6.0 → 2.7.0

package/client-v2/consumer/ConsumerManager.js

@@ -22,6 +22,59 @@ function pause(ms, signal) {
   })
 }
 
+// After a pop that failed for a reason waiting can fix, the next one waits:
+// the broker's Retry-After when it sent one (1 s on its 503s), else 1 s
+// doubling with each failure in a row. Never more than 30 s, so a consumer
+// resumes within 30 s of the broker answering again; ±20 % so a fleet of
+// workers does not come back in step.
+const TRANSIENT_BASE_MILLIS = 1000
+const TRANSIENT_CAP_MILLIS = 30000
+
+/**
+ * Whether a failed pop is worth polling again, because the broker or the way
+ * to it could not answer for a while and waiting changes that:
+ *
+ * - HTTP 408, 429 or any 5xx. The broker's own retryable answers are all a 503
+ *   with Retry-After: no leader (`no_leader`, `retry`), an elapsed deadline
+ *   (`timeout`, "the request deadline elapsed"), a leader change mid-commit
+ *   (`outcome_unknown`), a standby cluster (`standby`). A 500 on a pop comes
+ *   from a node shutting down or a roll between versions; a 502 or 504 from a
+ *   gateway in front of a node that is restarting.
+ * - No HTTP answer at all: this client's request timeout, or a connection that
+ *   could not be made or was cut before or while the answer was read.
+ *
+ * Not worth it, so the consumer stops as before: a 503
+ * `raft_phase1_unsupported` (the broker cannot serve the route at all), every
+ * other 4xx (a request or a credential refused the same way next time), a stop
+ * by the caller, and an error without an HTTP status that is not a transport
+ * fault (a bug, a `hostHeader` this runtime cannot send).
+ */
+export function isTransientPopError(error) {
+  if (!error || error.aborted) return false
+  if (typeof error.status === 'number') {
+    if (error.code === 'raft_phase1_unsupported') return false
+    return error.status === 408 || error.status === 429 || error.status >= 500
+  }
+  const message = typeof error.message === 'string' ? error.message : ''
+  return error.name === 'AbortError' ||   // this client's request timeout
+    message.includes('fetch failed') ||   // no connection, or it dropped before the answer
+    message === 'terminated' ||           // it dropped while the answer was read
+    message.includes('ECONNREFUSED') || error.code === 'ECONNREFUSED'
+}
+
+/**
+ * The wait before the next pop, after `failures` transient failures in a row.
+ * `random` stands in for Math.random in tests.
+ */
+export function transientDelayMillis(failures, retryAfterSeconds, random = Math.random) {
+  const hasRetryAfter = typeof retryAfterSeconds === 'number' && Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0
+  const base = hasRetryAfter
+    ? retryAfterSeconds * 1000
+    : TRANSIENT_BASE_MILLIS * 2 ** Math.max(0, failures - 1)
+  const jittered = Math.min(base, TRANSIENT_CAP_MILLIS) * (0.8 + 0.4 * random())
+  return Math.round(Math.min(jittered, TRANSIENT_CAP_MILLIS))
+}
+
 export class ConsumerManager {
   #httpClient
   #queen
@@ -154,6 +207,9 @@ export class ConsumerManager {
     
     let processedCount = 0
     let lastMessageTime = idleMillis ? Date.now() : null
+    // Pops that failed in a row for a reason waiting can fix, and since when.
+    let transientFailures = 0
+    let failingSince = null
 
     while (true) {
       // Check abort signal
@@ -187,6 +243,12 @@ export class ConsumerManager {
         // it would only sit on until the lease expires.
         const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey, wait ? 'pop' : null, signal)
 
+        if (transientFailures > 0) {
+          logger.warn('ConsumerManager.worker', { workerId, status: 'pop-recovered', failedPops: transientFailures, failingMillis: Date.now() - failingSince })
+          transientFailures = 0
+          failingSince = null
+        }
+
         // Degrade-loudly (PLAN_CONFLATION §4). Checked BEFORE the empty-response
         // branch on purpose: a pre-1.1.0 broker answers an empty pop with a
         // bodiless 204 (result === null), which is the first thing a consumer
@@ -299,37 +361,35 @@ export class ConsumerManager {
           throw error
         }
 
-        // Check if this is a timeout error (expected for long polling)
-        const isTimeoutError = error.name === 'AbortError' ||
-                              error.message?.includes('timeout')
-
-        if (isTimeoutError && wait) {
-          continue // Retry on timeout
-        }
-
-        // 429 (rate limited): HttpClient already retries this internally
-        // with backoff (unbounded for wait=true pop, per retry429 policy) --
-        // this branch is a defensive fallback for the case where an explicit
-        // retry429.maxAttempts override got exhausted. Back off and keep
-        // polling instead of hot-looping or rethrowing/dying.
-        if (error.status === 429) {
-          const retryAfterMs = typeof error.retryAfterSeconds === 'number' && error.retryAfterSeconds >= 0
-            ? error.retryAfterSeconds * 1000
-            : 1000
-          logger.warn('ConsumerManager.worker', { workerId, status: 'rate-limited', code: error.code, retryAfterMs })
-          await pause(retryAfterMs, signal)
+        // This client's own timeout on a long poll: the poll already waited
+        // its whole time, so poll again at once. Checked on the error's kind,
+        // not its text: a 4xx whose message says "timeout" is a refusal.
+        if (wait && error.name === 'AbortError' && error.status === undefined) {
           continue
         }
 
-        // Check if network error
-        const isNetworkError = error.message?.includes('fetch failed') ||
-                              error.message?.includes('ECONNREFUSED') ||
-                              error.code === 'ECONNREFUSED'
-
-        if (isNetworkError) {
-          logger.warn('ConsumerManager.worker', { workerId, error: 'network', message: error.message })
-          // Wait before retry
-          await pause(1000, signal)
+        // The broker, or the way to it, could not answer for a while: a
+        // leader election, a node restarting, a gateway timing out, a lost
+        // connection, a 429 the HttpClient stopped retrying. Wait, then poll
+        // again, for as long as it takes (see isTransientPopError). Before
+        // this only a 429 and a refused connection were retried here: on
+        // 2026-10-29 the 503s of a leader election stopped every consumer of
+        // a service whose process stayed up, and nothing was consumed until
+        // it was restarted by hand.
+        if (isTransientPopError(error)) {
+          transientFailures++
+          if (failingSince === null) failingSince = Date.now()
+          const delay = transientDelayMillis(transientFailures, error.retryAfterSeconds)
+          logger.warn('ConsumerManager.worker', {
+            workerId,
+            status: 'pop-failed-retrying',
+            httpStatus: error.status,
+            code: error.code,
+            error: error.message,
+            failedPops: transientFailures,
+            retryInMillis: delay
+          })
+          await pause(delay, signal)
           continue
         }
 
@@ -343,8 +403,10 @@ export class ConsumerManager {
           throw error
         }
 
-        // Other errors - rethrow
-        logger.error('ConsumerManager.worker', { workerId, error: error.message, code: error.code })
+        // Anything else stops this worker and rejects consume(): another 4xx
+        // (refused the same way next time), a 503 raft_phase1_unsupported, or
+        // an error that is not a transport fault.
+        logger.error('ConsumerManager.worker', { workerId, error: error.message, httpStatus: error.status, code: error.code })
         throw error
       }
     }

package/client-v2/http/HttpClient.js

@@ -468,11 +468,14 @@ export class HttpClient {
         // 'storage_quota_exceeded' | 'feature_gated' | 'forbidden' }. `.code`
         // is attached whenever the body has one so callers can branch without
         // string-matching `.message`.
-        if (response.status === 429) {
-          const retryAfterHeader = response.headers.get('retry-after')
-          const retryAfterSeconds = retryAfterHeader != null ? Number(retryAfterHeader) : NaN
-          error.retryAfterSeconds = Number.isFinite(retryAfterSeconds) ? retryAfterSeconds : null
-        }
+        //
+        // Retry-After is read on every error, not only a 429: the broker also
+        // sends it (1 s) with each 503 it means to be retried (no leader, a
+        // leader change, an elapsed deadline), and consume() waits that long
+        // before its next pop. null when absent or not a number of seconds.
+        const retryAfterHeader = response.headers.get('retry-after')
+        const retryAfterSeconds = retryAfterHeader != null ? Number(retryAfterHeader) : NaN
+        error.retryAfterSeconds = Number.isFinite(retryAfterSeconds) ? retryAfterSeconds : null
 
         try {
           const text = await response.text()

package/client-v2/streams/operators/ReduceOperator.js

@@ -38,9 +38,16 @@ export class ReduceOperator {
    * }} ctx
    * @returns {Promise<{
    *   stateOps: Array<{type:'upsert'|'delete',key:string,value?:any}>,
-   *   emits: Array<{key:string, windowStart:number, windowKey:string, value:any}>
+   *   emits: Array<{key:string, windowStart:number, windowKey:string, value:any}>,
+   *   nextCloseMs: number|null
    * }>}
    *
+   * `nextCloseMs` is when the earliest window still open closes
+   * (`windowEnd + gracePeriodMs`, on the clock the windows are kept in),
+   * counting the stored windows this batch did not touch; null when nothing
+   * stays open. The Runner claims the partition then to close it, even if
+   * no more messages come.
+   *
    * Window-close semantics
    * ----------------------
    * A window is closed when the partition's stream time has advanced past
@@ -121,8 +128,13 @@ export class ReduceOperator {
 
     const stateOps = []
     const emits = []
+    let nextCloseMs = null
     for (const [stateKey, entry] of accumulators.entries()) {
-      const closed = (entry.windowEnd + gracePeriodMs) <= effectiveStreamTime
+      const closesAt = entry.windowEnd + gracePeriodMs
+      const closed = closesAt <= effectiveStreamTime
+      if (!closed && (nextCloseMs === null || closesAt < nextCloseMs)) {
+        nextCloseMs = closesAt
+      }
       if (closed) {
         emits.push({
           key: entry.key,
@@ -142,7 +154,7 @@ export class ReduceOperator {
         })
       }
     }
-    return { stateOps, emits }
+    return { stateOps, emits, nextCloseMs }
   }
 
   _initial() {

package/client-v2/streams/operators/WindowSessionOperator.js

@@ -187,10 +187,17 @@ export class WindowSessionOperator {
       }
     }
 
-    // Build state ops.
+    // Build state ops, and note when the earliest session still open closes
+    // (`lastEventTime + gap + grace`): the Runner claims the partition then
+    // to close it, even if no more messages come. null: nothing is open.
     const stateOps = []
+    let nextCloseMs = null
     for (const [userKey, s] of sessions.entries()) {
       const stateKey = this.openSessionStateKey(userKey)
+      if (!s.closed && s.lastEventTime !== null) {
+        const closesAt = s.lastEventTime + this.gapMs + this.gracePeriodMs
+        if (nextCloseMs === null || closesAt < nextCloseMs) nextCloseMs = closesAt
+      }
       if (s.closed) {
         if (s.seeded) stateOps.push({ type: 'delete', key: stateKey })
         // Newly-opened sessions that closed in the same batch were never
@@ -208,7 +215,7 @@ export class WindowSessionOperator {
       }
     }
 
-    return { stateOps, emits: closedEmits }
+    return { stateOps, emits: closedEmits, nextCloseMs }
   }
 
   _buildEmit(userKey, s) {

package/client-v2/streams/operators/WindowTumblingOperator.js

@@ -14,7 +14,13 @@
  *   seconds         (req) window size in seconds
  *   gracePeriod     (sec) accept events for windowEnd + grace before closing (default 0)
  *   idleFlushMs     (ms)  Runner closes ripe windows on quiet partitions every
- *                         this many ms. Default 5000. Pass 0 to disable.
+ *                         this many ms. Default 5000. Pass 0 to disable. On a
+ *                         broker with POST /streams/v1/claim (2.7) a
+ *                         processing-time window closes within this interval
+ *                         of its end, under a lease on its partition, also
+ *                         after a restart and with several runners; on an
+ *                         older broker, only on the partitions this process
+ *                         read in the last five minutes.
  *   eventTime       (fn)  msg => epoch_ms; if set, switches to event-time mode
  *   allowedLateness (sec) drop events older than wm - allowedLateness (default 0)
  *   onLate          (str) 'drop' (default) | 'include' — what to do with
@@ -29,7 +35,8 @@
  *   processing-time: windowEnd + grace <= max(createdAt) seen in batch
  *                                          OR latest streamTime fed by Runner
  *   event-time:      windowEnd + grace <= partition watermark
- *   idle flush:      windowEnd + grace <= Date.now() (per Runner timer)
+ *   idle flush:      windowEnd + grace <= the broker's clock at the claim
+ *                    (Date.now() on a broker without claims)
  */
 
 import { extractEventTime, normaliseLateOpts, MS_PER_SEC } from './_windowCommon.js'