queen-mq 1.0.6 → 1.2.0

package/client-v2/builders/QueueBuilder.js

@@ -10,7 +10,10 @@ export const generateUUID = () => {
 
 //import { generateUUID } from '../../utils/uuid.js'
 import { isValidUUID } from '../utils/validation.js'
+import { durableAddress } from '../buffer/sinks.js'
 import { QUEUE_DEFAULTS, CONSUME_DEFAULTS, POP_DEFAULTS } from '../utils/defaults.js'
+import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
+import { popSizing, parseAutopilotDecision } from '../utils/autopilot.js'
 import * as logger from '../utils/logger.js'
 
 export class QueueBuilder {
@@ -26,7 +29,12 @@ export class QueueBuilder {
 
   // Consume options
   #concurrency = CONSUME_DEFAULTS.concurrency
-  #batch = CONSUME_DEFAULTS.batch
+  // batch / maxPartitions hold the USER's value, and null means the setter was
+  // never called -- which is the dimension pop autopilot gets to choose. The
+  // client-side defaults are applied at emission time (utils/autopilot.js), not
+  // here, because filling them in here would erase the difference between
+  // "never called batch()" and "called batch(1)".
+  #batch = null
   #limit = CONSUME_DEFAULTS.limit
   #idleMillis = CONSUME_DEFAULTS.idleMillis
   #autoAck = CONSUME_DEFAULTS.autoAck
@@ -36,8 +44,12 @@ export class QueueBuilder {
   #renewLeaseIntervalMillis = CONSUME_DEFAULTS.renewLeaseIntervalMillis
   #subscriptionMode = CONSUME_DEFAULTS.subscriptionMode
   #subscriptionFrom = CONSUME_DEFAULTS.subscriptionFrom
+  #conflation = CONSUME_DEFAULTS.conflation
   #each = false
-  #maxPartitions = 1
+  #maxPartitions = null
+  // Per-call override for pop autopilot: null = the client default (on unless
+  // QUEEN_SDK_POP_AUTOPILOT turned it off).
+  #autopilot = null
 
   // Buffer options
   #bufferOptions = null
@@ -211,8 +223,15 @@ export class QueueBuilder {
     return this
   }
 
+  /**
+   * Pin the message budget for one pop. Leave it unset and the broker sizes it
+   * (see `autopilot`), where it used to mean the client-side default of 1.
+   *
+   * `batch(0)` is not "a batch of zero" and never was: it is the absence of an
+   * opinion, so it reads as unset.
+   */
   batch(size) {
-    this.#batch = Math.max(1, size)
+    this.#batch = size > 0 ? size : null
     return this
   }
 
@@ -225,13 +244,45 @@ export class QueueBuilder {
    * partitions, in a single network round-trip. All N share one leaseId
    * (renewing once extends them all).
    *
-   * Default 1 = legacy single-partition behavior.
+   * Leave it unset and the broker chooses the sweep width (see `autopilot`);
+   * `partitions(1)` pins the legacy single-partition behaviour, which is a
+   * decision the broker is told about and never overrides.
    */
   partitions(n) {
-    this.#maxPartitions = Math.max(1, n)
+    this.#maxPartitions = n > 0 ? n : null
+    return this
+  }
+
+  /**
+   * Turn broker-side pop sizing on or off for this builder.
+   *
+   * On (the default) the broker chooses `batch` and `partitions` for the pops
+   * of this builder. Even then, a `batch` or `partitions` set explicitly
+   * travels on the wire as it always did and is never second-guessed: autopilot
+   * only ever fills the knobs left unset.
+   *
+   * `autopilot(false)` restores this SDK's pre-1.2 behaviour byte for byte: the
+   * client-side defaults come back (batch 1, partitions 1) and no autopilot
+   * parameter is sent. QUEEN_SDK_POP_AUTOPILOT=off does the same for a whole
+   * process; an explicit call here outranks the environment in both directions.
+   *
+   * Setting BOTH batch and partitions leaves autopilot nothing to decide, so no
+   * autopilot parameter is sent in that case either, whatever this flag says.
+   */
+  autopilot(enabled = true) {
+    this.#autopilot = !!enabled
     return this
   }
 
+  /**
+   * This builder's resolved autopilot decision: its own flag when set,
+   * otherwise the client-wide default settled in the Queen constructor.
+   */
+  #autopilotEnabled() {
+    if (this.#autopilot !== null) return this.#autopilot
+    return !this.#queen || !this.#queen.autopilotOff
+  }
+
   limit(count) {
     this.#limit = count
     return this
@@ -265,6 +316,34 @@ export class QueueBuilder {
     return this
   }
 
+  /**
+   * Last-value delivery for this consumer group (PLAN_CONFLATION §1.1).
+   *
+   * A pop of a partition delivers exactly ONE message — the newest visible one
+   * — and commits past everything it skipped. For command-style queues where
+   * one partition is one logical task key ("recompute entity X"), a consumer
+   * behind a backlog then does the work once with the latest input instead of
+   * replaying every stale intermediate.
+   *
+   * It is a property of the GROUP, not of the call: it is persisted when the
+   * group first registers on the queue, and from then on the stored value wins
+   * for every consumer of that group. Declaring the opposite later does not
+   * flip it — the SDK warns once and keeps working. Default off; a group
+   * created without it behaves exactly as before.
+   *
+   * Requires broker >= 1.1.0. An older broker ignores the parameter and would
+   * quietly deliver the whole backlog, so the SDK raises on the first response
+   * that does not echo the flag rather than draining it silently.
+   *
+   * Refused by the broker (400) when combined with queue mode (no consumer
+   * group) or with autoAck, which commits at delivery and would turn the
+   * "the newest state is definitely processed" guarantee into at-most-once.
+   */
+  conflation(enabled = true) {
+    this.#conflation = !!enabled
+    return this
+  }
+
   each() {
     this.#each = true
     return this
@@ -292,8 +371,14 @@ export class QueueBuilder {
       renewLeaseIntervalMillis: this.#renewLeaseIntervalMillis,
       subscriptionMode: this.#subscriptionMode,
       subscriptionFrom: this.#subscriptionFrom,
+      conflation: this.#conflation,
       each: this.#each,
       maxPartitions: this.#maxPartitions,
+      // Resolved here so ConsumerManager sees a decision and not a null. batch
+      // and maxPartitions keep their null when autopilot is on, and that null
+      // has to survive all the way to #buildParams: it is the ONLY record that
+      // the user said nothing about that dimension.
+      autopilot: this.#autopilotEnabled(),
       signal: options.signal
     }
 
@@ -323,7 +408,27 @@ export class QueueBuilder {
     return this
   }
 
+  /**
+   * Claim messages and report what the broker chose for this pop.
+   *
+   * Same call as `pop()` — this is the shape that also carries the additive
+   * `autopilot` echo, which is null when this pop did not engage autopilot or
+   * the broker is older than 1.2.
+   *
+   *   const { messages, autopilot } = await client.queue('events').group('w').popResult()
+   *   if (autopilot) console.log(autopilot.partitions, autopilot.batch, autopilot.waitMillis)
+   *
+   * @returns {Promise<{messages: object[], autopilot: {partitions: number, batch: number, waitMillis: number}|null}>}
+   */
+  async popResult() {
+    return this.#popWithDecision()
+  }
+
   async pop() {
+    return (await this.#popWithDecision()).messages
+  }
+
+  async #popWithDecision() {
     logger.log('QueueBuilder.pop', { queue: this.#queueName, partition: this.#partition, namespace: this.#namespace, task: this.#task, batch: this.#batch, wait: this.#wait, group: this.#group })
     
     try {
@@ -333,20 +438,37 @@ export class QueueBuilder {
       // Override autoAck to false unless explicitly set
       const effectiveAutoAck = this.#autoAck !== CONSUME_DEFAULTS.autoAck ? this.#autoAck : POP_DEFAULTS.autoAck
       
-      // Build params with correct autoAck for pop
-      const params = new URLSearchParams({
-        batch: this.#batch.toString(),
-        wait: this.#wait.toString(),
-        timeout: this.#timeoutMillis.toString()
+      // Batch, partitions and with them the autopilot flag. The RULE for which
+      // of the three travel lives in one place (utils/autopilot.js) because
+      // consume() builds its query string separately; only the PLACEMENT is
+      // here, and it is the pre-autopilot placement so an autopilot-off request
+      // is byte-identical to the one this SDK used to send.
+      const sizing = popSizing({
+        batch: this.#batch,
+        maxPartitions: this.#maxPartitions,
+        fallbackBatch: POP_DEFAULTS.batch,
+        autopilot: this.#autopilotEnabled()
       })
 
+      // Build params with correct autoAck for pop
+      const params = new URLSearchParams()
+      if (sizing.autopilot) params.append('autopilot', 'true')
+      if (sizing.batch !== null) params.append('batch', sizing.batch)
+      params.append('wait', this.#wait.toString())
+      params.append('timeout', this.#timeoutMillis.toString())
+
       if (this.#group) params.append('consumerGroup', this.#group)
       if (this.#namespace) params.append('namespace', this.#namespace)
       if (this.#task) params.append('task', this.#task)
       if (effectiveAutoAck) params.append('autoAck', 'true')
       if (this.#subscriptionMode) params.append('subscriptionMode', this.#subscriptionMode)
       if (this.#subscriptionFrom) params.append('subscriptionFrom', this.#subscriptionFrom)
-      if (this.#maxPartitions > 1) params.append('partitions', this.#maxPartitions.toString())
+      if (sizing.partitions !== null) params.append('partitions', sizing.partitions)
+      // Conflation (PLAN_CONFLATION §3.1): sent ONLY when true, so an
+      // undeclared pop is byte-identical to today. NOTE: this is the pop
+      // builder; consume() builds its params in ConsumerManager#buildParams —
+      // see the comment below #buildPopPath about exactly this hazard.
+      if (this.#conflation) params.append('conflation', 'true')
 
       // Generate affinity key for consistent routing to same backend
       const affinityKey = this.#getAffinityKey()
@@ -355,22 +477,51 @@ export class QueueBuilder {
       // rather than give up after a handful of tries (retryKind: 'pop').
       const result = await this.#httpClient.get(`${path}?${params}`, this.#timeoutMillis + 5000, affinityKey, this.#wait ? 'pop' : null)
 
+      // Degrade-loudly (PLAN_CONFLATION §4), BEFORE the empty-response return:
+      // an old broker's empty pop is a bodiless 204 (result === null), and that
+      // is precisely the first thing a consumer on an idle queue sees. Also
+      // where a declaration conflict is warned about, exactly once.
+      if (this.#conflation) {
+        checkConflationResponse(result, {
+          queue: this.#queueName,
+          namespace: this.#namespace,
+          task: this.#task,
+          group: this.#group
+        })
+      }
+
+      // The broker's own account of how it sized this pop, when the request
+      // engaged autopilot and the answer had a body to carry it (a bodiless 204
+      // cannot, so an empty short pop reports null).
+      const autopilot = parseAutopilotDecision(result)
+
       if (!result || !result.messages) {
         logger.log('QueueBuilder.pop', { status: 'no-messages' })
-        return []
+        return { messages: [], autopilot }
       }
 
       const messages = result.messages.filter(msg => msg != null)
       logger.log('QueueBuilder.pop', { status: 'success', count: messages.length })
-      return messages
+      return { messages, autopilot }
     } catch (error) {
+      // Conflation is the one thing this method does NOT swallow. The
+      // swallow-to-[] contract exists for transport faults, where [] means "no
+      // messages right now"; for a declared conflation it would mean "your
+      // last-value policy is not in force and you will never be told", which is
+      // the silent failure the feature is not allowed to have (§4). Both the
+      // missing-echo error and the broker's 400 refusals (queue mode / autoAck)
+      // are permanent config faults, so they raise.
+      if (error.code === CONFLATION_UNSUPPORTED || (this.#conflation && error.status === 400)) {
+        logger.error('QueueBuilder.pop', { error: error.message, status: error.status, code: error.code, conflation: true })
+        throw error
+      }
       // Return empty array on error instead of throwing. This also covers a
       // 429 whose retry429 policy was exhausted (bounded pop, or an explicit
       // maxAttempts override) and a terminal 403 (e.g. cluster_suspended) --
       // both are logged with their `.code` rather than raising, matching this
       // method's existing swallow-to-[] contract.
       logger.error('QueueBuilder.pop', { error: error.message, status: error.status, code: error.code })
-      return []
+      return { messages: [], autopilot: null }
     }
   }
 
@@ -397,6 +548,11 @@ export class QueueBuilder {
   // copy nobody calls: the pop would keep working and the parameter would
   // simply never arrive, which reads as a server-side mystery and not as a
   // client bug.
+  //
+  // The pair that is still live and MUST be kept in sync is pop()'s inline
+  // params above and ConsumerManager#buildParams: every pop query parameter
+  // (subscriptionMode, subscriptionFrom, partitions, conflation, ...) has to be
+  // appended in BOTH, because pop() and consume() share no builder.
 
   // ===========================
   // Buffer Management Methods
@@ -406,7 +562,7 @@ export class QueueBuilder {
     if (!this.#queueName) {
       throw new Error('Queue name is required for buffer flush')
     }
-    const queueAddress = `${this.#queueName}/${this.#partition}`
+    const queueAddress = durableAddress(this.#queueName, this.#partition)
     logger.log('QueueBuilder.flushBuffer', { queueAddress })
     await this.#bufferManager.flushBuffer(queueAddress)
   }
@@ -643,7 +799,10 @@ class PushBuilder {
     // off without awaiting would report success for messages the buffer never
     // accepted -- the exact failure this bound exists to remove.
     if (this.#bufferOptions) {
-      const queueAddress = `${this.#queueName}/${this.#partition}`
+      // No destination: the durable push is the default sink, so this address's
+      // buffer drains to POST /api/v1/push with a `{items}` body exactly as it
+      // did before sinks existed (buffer/sinks.js).
+      const queueAddress = durableAddress(this.#queueName, this.#partition)
       const accepted = []
 
       try {

package/client-v2/consumer/ConsumerManager.js

@@ -3,6 +3,9 @@
  */
 
 import * as logger from '../utils/logger.js'
+import { checkConflationResponse, CONFLATION_UNSUPPORTED } from '../utils/conflation.js'
+import { popSizing, parseAutopilotDecision, emptyPollDelayMillis } from '../utils/autopilot.js'
+import { CONSUME_DEFAULTS } from '../utils/defaults.js'
 
 export class ConsumerManager {
   #httpClient
@@ -47,8 +50,10 @@ export class ConsumerManager {
       renewLeaseIntervalMillis,
       subscriptionMode,
       subscriptionFrom,
+      conflation,
       each,
       maxPartitions,
+      autopilot,
       signal
     } = options
 
@@ -68,7 +73,7 @@ export class ConsumerManager {
 
     // Build the path and params for pop requests
     const path = this.#buildPath(queue, partition, namespace, task)
-    const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions)
+    const baseParams = this.#buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation, this.#autopilotEnabled(autopilot))
     
     // Generate affinity key for consistent routing to same backend
     const affinityKey = this.#getAffinityKey(queue, partition, namespace, task, group)
@@ -88,7 +93,12 @@ export class ConsumerManager {
         each,
         signal,
         group,  // Pass consumer group to workers
-        affinityKey  // Pass affinity key to workers
+        affinityKey,  // Pass affinity key to workers
+        // Conflation was REQUESTED by this consumer: the worker has to check
+        // every response for the broker's echo (PLAN_CONFLATION §4) and needs
+        // the pop target to key the once-per-(queue,group) conflict warning.
+        conflation,
+        conflationCtx: { queue, namespace, task, group }
       }))
     }
 
@@ -113,7 +123,9 @@ export class ConsumerManager {
       each,
       signal,
       group,
-      affinityKey
+      affinityKey,
+      conflation,
+      conflationCtx
     } = options
 
     logger.log('ConsumerManager.worker', { workerId, status: 'started', limit, idleMillis })
@@ -150,13 +162,26 @@ export class ConsumerManager {
         const clientTimeout = wait ? timeoutMillis + 5000 : timeoutMillis
         const result = await this.#httpClient.get(`${path}?${baseParams}`, clientTimeout, affinityKey, wait ? 'pop' : null)
 
+        // Degrade-loudly (PLAN_CONFLATION §4). Checked BEFORE the empty-response
+        // branch on purpose: a pre-1.1.0 broker answers an empty pop with a
+        // bodiless 204 (result === null), which is the first thing a consumer
+        // on an idle queue sees — and the whole point is to raise before a
+        // single message of a backlog is processed one-by-one. Throwing here
+        // leaves the loop through the catch below, which stops this worker.
+        if (conflation) {
+          checkConflationResponse(result, conflationCtx)
+        }
+
         // Handle empty response
         if (!result || !result.messages || result.messages.length === 0) {
           if (wait) {
             continue // Long polling timeout, retry
           } else {
-            // Short delay before retry
-            await new Promise(resolve => setTimeout(resolve, 100))
+            // Short delay before retry -- the broker's advised pacing when this
+            // pop engaged autopilot and the broker had an opinion (it knows the
+            // arrival rate on this queue and this client does not), otherwise
+            // the historical 100ms.
+            await new Promise(resolve => setTimeout(resolve, emptyPollDelayMillis(parseAutopilotDecision(result))))
             continue
           }
         }
@@ -219,6 +244,17 @@ export class ConsumerManager {
         }
 
       } catch (error) {
+        // Conflation faults are terminal and are classified FIRST, ahead of the
+        // message-substring heuristics below: a consumer that asked for
+        // last-value delivery and is not getting it must stop, not retry
+        // (PLAN_CONFLATION §4). The broker's 400 refusals (queue mode /
+        // autoAck) are permanent config faults and stop the loop for the same
+        // reason — retrying them forever would be the silent version.
+        if (error.code === CONFLATION_UNSUPPORTED || (conflation && error.status === 400)) {
+          logger.error('ConsumerManager.worker', { workerId, status: 'conflation-unavailable', code: error.code, httpStatus: error.status, error: error.message })
+          throw error
+        }
+
         // Check if this is a timeout error (expected for long polling)
         const isTimeoutError = error.name === 'AbortError' ||
                               error.message?.includes('timeout')
@@ -442,20 +478,51 @@ export class ConsumerManager {
     throw new Error('Must specify queue, namespace, or task')
   }
 
-  #buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions) {
-    const params = new URLSearchParams({
-      batch: batch.toString(),
-      wait: wait.toString(),
-      timeout: timeoutMillis.toString()  // Server expects 'timeout', not 'timeoutMillis'
+  /**
+   * The autopilot decision for one consume: the caller's explicit option if
+   * there is one, otherwise the client-wide default settled in the Queen
+   * constructor. The builder path has already resolved it; the undefined case
+   * is for callers that drive ConsumerManager with options of their own.
+   */
+  #autopilotEnabled(autopilot) {
+    if (typeof autopilot === 'boolean') return autopilot
+    return !this.#queen || !this.#queen.autopilotOff
+  }
+
+  #buildParams(batch, wait, timeoutMillis, group, subscriptionMode, subscriptionFrom, namespace, task, autoAck, maxPartitions, conflation, autopilot = true) {
+    // Batch, partitions and with them the autopilot flag. null/0 means the user
+    // set nothing (QueueBuilder leaves it that way on purpose), which is the
+    // dimension the broker gets to choose. THE RULE lives in one place
+    // (utils/autopilot.js) precisely because this is the SECOND parameter
+    // builder; only the placement of the keys is here, and it is the
+    // pre-autopilot placement so an autopilot-off request is byte-identical.
+    const sizing = popSizing({
+      batch,
+      maxPartitions,
+      fallbackBatch: CONSUME_DEFAULTS.batch,
+      autopilot
     })
 
+    const params = new URLSearchParams()
+    if (sizing.autopilot) params.append('autopilot', 'true')
+    if (sizing.batch !== null) params.append('batch', sizing.batch)
+    params.append('wait', wait.toString())
+    params.append('timeout', timeoutMillis.toString())  // Server expects 'timeout', not 'timeoutMillis'
+
     if (group) params.append('consumerGroup', group)
     if (subscriptionMode) params.append('subscriptionMode', subscriptionMode)
     if (subscriptionFrom) params.append('subscriptionFrom', subscriptionFrom)
     if (namespace) params.append('namespace', namespace)
     if (task) params.append('task', task)
-    // v4 multi-partition pop: drain up to N sparse partitions per call.
-    if (maxPartitions && maxPartitions > 1) params.append('partitions', maxPartitions.toString())
+    // v4 multi-partition pop: drain up to N sparse partitions per call. Under
+    // autopilot a pinned width travels even when it is 1, because 1 is then a
+    // decision and not the absence of one.
+    if (sizing.partitions !== null) params.append('partitions', sizing.partitions)
+    // Conflation (PLAN_CONFLATION §3.1): last-value delivery for this group.
+    // Sent ONLY when true, so a consumer that never declares it puts no new
+    // bytes on the wire. THIS IS THE SECOND PARAMETER BUILDER — the pop() one
+    // lives inline in QueueBuilder.pop and must gain every parameter too.
+    if (conflation) params.append('conflation', 'true')
     // NEVER send autoAck for consume - client always manages acking
     // autoAck is only for pop() where server auto-acks immediately