queen-mq 1.0.6 → 1.2.0

package/client-v2/utils/autopilot.js

@@ -0,0 +1,168 @@
+/**
+ * Pop autopilot, client side.
+ *
+ * The broker owns a controller that sizes a pop from state this client cannot
+ * see: how many partitions of the (queue, group) are ready, how old their
+ * oldest ready message is, at what rate messages are arriving. Two knobs are
+ * under its control — `partitions` (the sweep width) and `batch` (the message
+ * budget for the sweep).
+ *
+ * THE RULE, and it is the only one: an explicit user value is sacred. Autopilot
+ * applies ONLY to the knobs the user left unset, and it applies to them one by
+ * one. A consumer that pins `partitions(1)` and says nothing about batch keeps
+ * its single-partition claim forever and lets the broker size the batch; the
+ * pinned dimension is never "adjusted", not even towards a value the controller
+ * would consider better.
+ *
+ * The wire shape follows the conflation precedent (see conflation.js): a client
+ * that is not engaging autopilot sends the byte-identical request it sent
+ * before this feature existed.
+ *
+ *   autopilot=true      emitted ONLY when at least one of the two knobs is
+ *                       being left to the broker. Never as autopilot=false.
+ *   partitions / batch  OMITTED for the dimensions the broker is choosing,
+ *                       sent exactly as before for the ones the user set.
+ *
+ * WHAT AN OLD BROKER DOES, and why there is no capability check here. A broker
+ * older than 1.2 ignores unknown query params: the request succeeds, and the
+ * two omitted knobs fall back to the SERVER-side defaults (batch 200,
+ * partitions 1) instead of the old client-side ones. That is a sizing
+ * difference, not a correctness one — nothing is lost, misordered or delivered
+ * twice — so unlike conflation (which silently hands a last-value consumer a
+ * whole backlog, hence CONFLATION_UNSUPPORTED) this degrades quietly and on
+ * purpose. Callers who need the old numbers against an old broker set them
+ * explicitly, or turn autopilot off.
+ */
+
+/**
+ * The environment variable that disables pop autopilot for a whole process:
+ * QUEEN_SDK_POP_AUTOPILOT=off restores the client-side defaults this SDK
+ * applied before autopilot existed, byte for byte. It is read once, in the
+ * Queen constructor, so a single deployment can be rolled back without touching
+ * code.
+ *
+ * "off", "false", "0", "no" and "disabled" all disable it (case-insensitive,
+ * surrounding space ignored). Every other value, including the empty one,
+ * leaves autopilot on.
+ */
+export const ENV_POP_AUTOPILOT = 'QUEEN_SDK_POP_AUTOPILOT'
+
+const DISABLING_VALUES = new Set(['off', 'false', '0', 'no', 'disabled'])
+
+/** Whether ENV_POP_AUTOPILOT asks for the pre-autopilot behavior. */
+export function popAutopilotDisabledByEnv() {
+  // `process` is absent in a browser bundle; there the variable cannot be set
+  // and autopilot is simply on.
+  const raw = typeof process !== 'undefined' && process.env ? process.env[ENV_POP_AUTOPILOT] : undefined
+  return DISABLING_VALUES.has(String(raw ?? '').trim().toLowerCase())
+}
+
+/**
+ * The batch/partitions/autopilot decision for one pop — the values that travel
+ * and the ones that do not.
+ *
+ * IT EXISTS SO THERE IS EXACTLY ONE COPY OF THE EMISSION RULE. pop() and
+ * consume() build their query strings separately (QueueBuilder.pop's inline
+ * params vs ConsumerManager#buildParams) and the two have drifted before — the
+ * standing comment in QueueBuilder.js is there because of it. A rule with three
+ * branches and a per-dimension carve-out is precisely the kind that gets copied
+ * wrong, so both builders call this and then only PLACE what it returns; where
+ * each key sits in the query string stays with the builder, because the
+ * pre-autopilot key order is part of what "byte-identical" means here.
+ *
+ * @param {object} opts
+ * @param {number|null} opts.batch - the USER's batch. null/0/undefined means
+ *   unset: the dimension the broker gets to choose. Neither builder may
+ *   substitute a default before calling this.
+ * @param {number|null} opts.maxPartitions - the USER's sweep width, same
+ *   convention.
+ * @param {number} opts.fallbackBatch - client-side default applied to an unset
+ *   batch when autopilot is NOT engaged.
+ * @param {boolean} opts.autopilot - the resolved decision for this call.
+ * @returns {{autopilot: boolean, batch: string|null, partitions: string|null}}
+ *   Strings are ready to append; null means "this key does not travel".
+ */
+export function popSizing({ batch, maxPartitions, fallbackBatch, autopilot }) {
+  const batchSet = typeof batch === 'number' && batch > 0
+  const partitionsSet = typeof maxPartitions === 'number' && maxPartitions > 0
+
+  // Note the case that looks like an omission and is not: when the user set
+  // BOTH knobs there is nothing left for the controller to decide, so
+  // autopilot=true is NOT emitted and the request is byte-identical to the one
+  // this SDK sent before autopilot existed. Sending the flag anyway would be
+  // harmless on the broker and dishonest in a packet capture.
+  if (autopilot && !(batchSet && partitionsSet)) {
+    return {
+      autopilot: true,
+      batch: batchSet ? String(batch) : null,
+      partitions: partitionsSet ? String(maxPartitions) : null
+    }
+  }
+
+  return {
+    autopilot: false,
+    batch: String(batchSet ? batch : fallbackBatch),
+    // The legacy gate: partitions travels only above 1, because 1 IS the
+    // server-side default and a v4-era client never sent it.
+    partitions: partitionsSet && maxPartitions > 1 ? String(maxPartitions) : null
+  }
+}
+
+/**
+ * What the broker chose for one pop, echoed back in the response under
+ * "autopilot" when the request engaged autopilot. It is additive: a broker that
+ * does not send it, or a pop that never asked, yields null everywhere it is
+ * exposed.
+ *
+ * Reading it is optional — the messages are already sized by it — but it is the
+ * only way to see the controller working from the client side, and the only
+ * input to the empty-poll pacing below.
+ *
+ * Unknown keys inside it are ignored, and an unknown-shaped value is treated as
+ * absent rather than as an error: this field is the broker telling the client
+ * what it did, and a client that refuses to run because a newer broker grew a
+ * fourth number would be a self-inflicted outage.
+ *
+ * @param {object|null} result - parsed pop response (null for a 204)
+ * @returns {{partitions: number, batch: number, waitMillis: number}|null}
+ */
+export function parseAutopilotDecision(result) {
+  const raw = result && typeof result === 'object' ? result.autopilot : null
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null
+
+  const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : 0)
+  return {
+    /** Sweep width the broker used for this pop. */
+    partitions: num(raw.partitions),
+    /** Message budget the broker used for this pop. */
+    batch: num(raw.batch),
+    /**
+     * The broker's advice on how long to wait before polling again (wire name:
+     * waitMs). Present only when the broker has an opinion, and it is advice,
+     * not a lease: the consume loop honors it for the sleep it was already
+     * taking between empty non-waiting pops, nothing more. 0 = no advice.
+     */
+    waitMillis: num(raw.waitMs)
+  }
+}
+
+/**
+ * The sleep the consume loop has always taken between two empty pops that are
+ * NOT long-polling. A waiting pop already blocks on the broker, so it never
+ * reaches here.
+ */
+export const EMPTY_POLL_BACKOFF_MILLIS = 100
+
+/**
+ * How long to wait after an empty pop: the broker's advice when it gave one,
+ * the historical constant otherwise.
+ *
+ * The advice is honored as given, without a ceiling of this client's invention.
+ * The sleep it feeds is raced against the caller's abort signal by the loop, so
+ * even an absurd value cannot outlive a cancellation — which is the only
+ * property that has to hold locally.
+ */
+export function emptyPollDelayMillis(decision) {
+  if (decision && decision.waitMillis > 0) return decision.waitMillis
+  return EMPTY_POLL_BACKOFF_MILLIS
+}

package/client-v2/utils/conflation.js

@@ -0,0 +1,118 @@
+/**
+ * Conflation response contract (PLAN_CONFLATION.md §3.3, §4).
+ *
+ * Conflation — last-value delivery for a consumer group — is declared as a pop
+ * query parameter and answered in the pop RESPONSE. Two response keys carry the
+ * whole client-side contract, and both are handled here so that pop() and the
+ * consume loop cannot drift apart:
+ *
+ *   "conflation": true          the broker actually applied last-value delivery.
+ *                               Rides EVERY conflating response, empty ones
+ *                               included — a conflating pop answers 200 with a
+ *                               body instead of a bodiless 204 precisely so the
+ *                               echo has somewhere to sit.
+ *   "conflationConflict": true  this consumer declared a policy the group does
+ *                               not have. The STORED group setting wins (§3.3);
+ *                               the consumer keeps working and warns once.
+ *
+ * DEGRADE LOUDLY (§4). No SDK negotiates a version with the broker. A 1.1.0
+ * client against an older broker sends `conflation=true`, the broker ignores the
+ * unknown query parameter, and the consumer silently processes the entire
+ * backlog message by message — the exact silent failure the feature must not
+ * have. So: requested conflation + no echo => raise, on the FIRST response,
+ * before a single message is handled. `null` (an old broker's 204 on an empty
+ * pop) is the most important case to raise on, not the least: it is what an idle
+ * queue looks like, and it is the first thing a fresh consumer sees.
+ */
+
+import * as logger from './logger.js'
+
+/** `error.code` on the degrade-loudly error, so callers can branch without string-matching. */
+export const CONFLATION_UNSUPPORTED = 'conflation_unsupported'
+
+/** The message the plan fixes for every SDK (§4). Keep it identical across clients. */
+export const CONFLATION_UNSUPPORTED_MESSAGE =
+  'conflation was requested but this broker did not apply it — requires broker >= 1.1.0'
+
+// Once per (queue, group) PER PROCESS, per §3.3: a mismatched fleet during a
+// rolling deploy would otherwise emit one warning per pop, forever.
+const warnedConflicts = new Set()
+
+// A (queue, group) whose conflict the broker has ALREADY told us about. Proof
+// that this broker understands the parameter, so a later response without the
+// echo for that same pair is the same known conflict and not an old broker.
+//
+// It is belt-and-braces now rather than the load-bearing part it once was: a
+// 1.1.0 broker keeps the 200 and the body whenever the answer has anything to
+// say about conflation (`pop_status`), so an EMPTY conflicting pop carries
+// `conflationConflict` too and the check above catches it on the first round
+// trip. Keeping the memo costs nothing and still covers a first contact whose
+// answer somehow arrives without either key.
+const knownConflicts = new Set()
+
+/**
+ * Identity of the thing a policy hangs on: the queue (or namespace/task pop
+ * target) and the consumer group. Deliberately NOT the affinity key, which
+ * includes the partition — the policy is per (queue, group).
+ */
+export function conflationKey({ queue, namespace, task, group }) {
+  const target = queue || `${namespace || '*'}/${task || '*'}`
+  return `${target}|${group || '__QUEUE_MODE__'}`
+}
+
+/**
+ * Inspect one pop response on behalf of a consumer that ASKED for conflation.
+ * Call it for every response, including empty ones — that is where the old
+ * broker shows itself first.
+ *
+ * @param {object|null} result - parsed pop response (null for a 204)
+ * @param {object} ctx - { queue, namespace, task, group }
+ * @throws {Error} with `.code === CONFLATION_UNSUPPORTED` when the broker did
+ *   not apply conflation and did not explain why.
+ */
+export function checkConflationResponse(result, ctx) {
+  const key = conflationKey(ctx)
+
+  if (result && result.conflationConflict === true) {
+    knownConflicts.add(key)
+    if (!warnedConflicts.has(key)) {
+      warnedConflicts.add(key)
+      logger.warn('Conflation.conflict', {
+        queue: ctx.queue || null,
+        namespace: ctx.namespace || null,
+        task: ctx.task || null,
+        group: ctx.group || null,
+        message:
+          'this consumer declared conflation but the consumer group is registered without it — ' +
+          'the STORED group setting wins and this consumer is receiving normal batches. ' +
+          'Delete/recreate the group to change its policy.'
+      })
+    }
+    return
+  }
+
+  if (result && result.conflation === true) return
+
+  // Pop maintenance: the broker refused the pop before it reached the claim
+  // path, so there is no policy to echo and nothing to conclude from the
+  // absence of one. Reading it as "old broker" would stop every conflating
+  // consumer in the fleet the moment an operator pauses pops.
+  if (result && result.paused === true) return
+
+  // The group is known to disagree with us; this response is that same
+  // conflict, seen through a body that could not carry the flag.
+  if (knownConflicts.has(key)) return
+
+  const error = new Error(CONFLATION_UNSUPPORTED_MESSAGE)
+  error.code = CONFLATION_UNSUPPORTED
+  throw error
+}
+
+/**
+ * Test hook: the warn-once and known-conflict registries are process-wide by
+ * design, which makes them sticky across tests in one runner process.
+ */
+export function resetConflationWarnings() {
+  warnedConflicts.clear()
+  knownConflicts.clear()
+}

package/client-v2/utils/defaults.js

@@ -51,9 +51,14 @@ export const QUEUE_DEFAULTS = {
   encryptionEnabled: false             // No encryption by default
 }
 
+// batch and maxPartitions here are the AUTOPILOT-OFF defaults. With autopilot on
+// (the default) a knob the caller never set is not defaulted at all -- it is
+// omitted from the pop so the broker sizes it (see utils/autopilot.js). These
+// values are what comes back with QueueBuilder.autopilot(false), or with
+// QUEEN_SDK_POP_AUTOPILOT=off for a whole process.
 export const CONSUME_DEFAULTS = {
   concurrency: 1,                      // Single worker
-  batch: 1,                            // One message at a time
+  batch: 1,                            // One message at a time (autopilot off only)
   autoAck: true,                       // Client-side auto-ack (NOT sent to server)
   wait: true,                          // Long polling enabled
   timeoutMillis: 30000,                // 30 seconds long poll timeout
@@ -63,11 +68,19 @@ export const CONSUME_DEFAULTS = {
   renewLeaseIntervalMillis: null,      // Auto-renewal interval when enabled
   subscriptionMode: null,              // No subscription mode (standard queue mode)
   subscriptionFrom: null,              // No subscription start point
-  maxPartitions: 1                     // v4 multi-partition pop cap (1 = legacy single-partition)
+  maxPartitions: 1,                    // v4 multi-partition pop cap (autopilot off only)
+  // Last-value delivery for this consumer group (PLAN_CONFLATION §1.1): a pop
+  // of a partition delivers only the NEWEST visible message and commits past
+  // the ones it skipped. Off by default, and only ever SENT when true, so a
+  // client that never touches it is byte-identical on the wire. Requires
+  // broker >= 1.1.0 — an older one ignores the parameter, which the SDK
+  // detects from the missing response echo and raises on (§4).
+  conflation: false
 }
 
+// As in CONSUME_DEFAULTS, batch is the autopilot-OFF default.
 export const POP_DEFAULTS = {
-  batch: 1,                            // One message
+  batch: 1,                            // One message (autopilot off only)
   wait: false,                         // No long polling (immediate return)
   timeoutMillis: 30000,                // 30 seconds if wait=true
   autoAck: false                       // Server-side auto-ack (false = manual ack required)

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "queen-mq",
-  "version": "1.0.6",
+  "version": "1.2.0",
   "type": "module",
   "description": "Partitioned message queue on PostgreSQL — broker client + fluent streaming SDK (windows, joins, gates) in one package",
   "main": "client-v2/index.js",
   "scripts": {
-    "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js && node test-v2/run.js human",
-    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js",
+    "test": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js && node test-v2/run.js human",
+    "test:unit": "node --test test-v2/streams-unit/configHash.test.js test-v2/streams-unit/operators.test.js test-v2/streams-unit/cycle.test.js test-v2/streams-unit/eventTime.test.js test-v2/streams-unit/ack.test.js test-v2/http-unit/retry429.test.js test-v2/http-unit/hostHeader.test.js test-v2/kv-unit/kvWire.test.js test-v2/kv-unit/timerWire.test.js test-v2/kv-unit/txnWire.test.js test-v2/buffer-unit/buffer.test.js test-v2/conflation-unit/conflationWire.test.js test-v2/autopilot-unit/autopilotWire.test.js test-v2/ephemeral-unit/ephemeralWire.test.js test-v2/ephemeral-unit/ephemeralBuffer.test.js test-v2/ephemeral-unit/durableSinkPin.test.js",
     "test:unit:e2e": "node --test test-v2/streams-unit/e2e.test.js",
     "test:integration": "node test-v2/run.js human",
     "test:streams": "node test-v2/run.js stream",