queen-mq 0.14.0 → 0.15.0

package/client-v2/streams/operators/FlatMapOperator.js

@@ -0,0 +1,21 @@
+/**
+ * FlatMapOperator — emit zero or more values per input record.
+ *
+ * The user fn returns an array of values; the operator wraps each in an
+ * envelope sharing the source message's metadata (so subsequent operators
+ * still see the original partition_id etc.).
+ */
+export class FlatMapOperator {
+  constructor(fn) {
+    this.kind = 'flatMap'
+    this.fn = fn
+  }
+
+  async apply(envelope) {
+    const out = await this.fn(envelope.msg, envelope.ctx)
+    if (!Array.isArray(out)) {
+      throw new Error('flatMap fn must return an array (got ' + typeof out + ')')
+    }
+    return out.map(value => ({ ...envelope, value }))
+  }
+}

package/client-v2/streams/operators/ForeachOperator.js

@@ -0,0 +1,29 @@
+/**
+ * ForeachOperator — terminal at-least-once side-effect.
+ *
+ * The runtime invokes the user fn for each emitted record after state has
+ * been computed but BEFORE the cycle commits the source ack. If `fn`
+ * resolves successfully for all records, the cycle commits as normal. If
+ * `fn` throws, the cycle is aborted and the messages will be redelivered
+ * by Queen's lease/retry mechanism.
+ *
+ * User-fn signature: `(value, ctx?) => void|Promise<void>`
+ *   value: the post-reducer value (e.g. an aggregate { count, sum, ... })
+ *   ctx:   { partition, partitionId, key, windowKey, windowStart, windowEnd }
+ *
+ * For exactly-once external effects, use `.to(queue)` and have a separate
+ * worker drain the sink queue with a transactional ack.
+ */
+export class ForeachOperator {
+  constructor(fn) {
+    this.kind = 'foreach'
+    this.fn = fn
+    this.config = { kind: 'foreach' }
+  }
+
+  async runEffects(entries) {
+    for (const entry of entries) {
+      await this.fn(entry.value, entry.ctx)
+    }
+  }
+}

package/client-v2/streams/operators/GateOperator.js

@@ -0,0 +1,85 @@
+/**
+ * GateOperator — per-message ALLOW/DENY decision with persistent per-key state.
+ *
+ * Used to build rate limiters, throttlers, fairness gates, circuit breakers,
+ * and any other "should this message proceed RIGHT NOW?" pattern.
+ *
+ * Semantics
+ * ---------
+ * The runtime processes a popped batch sequentially in source order. For each
+ * message, it builds a `ctx` object with mutable `state` (loaded from
+ * `queen_streams.state` for the message's key) and invokes the user fn.
+ *
+ *   .gate((msg, ctx) => boolean)
+ *
+ * Returning `true` (or `{allow:true}`):
+ *   - The mutated `ctx.state` is persisted as part of the cycle.
+ *   - Downstream operators (sink, foreach) receive this message.
+ *   - The message counts toward the cycle's ack count.
+ *
+ * Returning `false` (or `{allow:false}`):
+ *   - The mutated state is DISCARDED for this message (it didn't "happen").
+ *   - The runner stops processing the rest of the batch.
+ *   - The cycle commits with `release_lease=false` and `ack.count = K`
+ *     (the count of allowed messages BEFORE this one).
+ *   - The message and its successors stay leased until natural lease
+ *     expiry; the broker then redelivers them to a runner in their
+ *     original partition order.
+ *
+ * If K === 0 (the very first message is denied), the runner skips the
+ * cycle entirely — no state, no push, no ack — and the lease times out
+ * naturally.
+ *
+ * Ordering
+ * --------
+ * FIFO per-partition is preserved by construction: the same partition lease
+ * is held continuously across the deny → expiry → redeliver cycle, so the
+ * denied messages never overtake messages that arrived after them.
+ *
+ * State key
+ * ---------
+ * State is keyed by the envelope's `key` (typically the source partition_id
+ * unless `.keyBy()` overrides it). For a rate limiter, the natural setup is:
+ *   - source queue partitioned by the rate-limit key (e.g. tenantId)
+ *   - no .keyBy() needed (key defaults to partitionId)
+ *
+ * If the key DIVERGES from the partition (cross-partition keying), the same
+ * cross-worker contention warning that applies to .reduce() applies here too:
+ * two workers may hold leases for different partitions but write the same
+ * (query_id, partition_id, key) row. For rate limiting this is almost always
+ * a configuration mistake — keep partition == limit key.
+ *
+ * Why a dedicated operator
+ * ------------------------
+ * The "stop the batch + don't release the lease" semantics has no analog in
+ * the existing operators (.map / .filter / .reduce all process the full batch).
+ * Modelling it as a return-value side effect of a regular .map would be
+ * surprising; a dedicated `.gate()` makes the intent and the cycle behaviour
+ * explicit at the chain site.
+ */
+export class GateOperator {
+  constructor(fn) {
+    if (typeof fn !== 'function') {
+      throw new Error('.gate(fn) requires a function')
+    }
+    this.kind = 'gate'
+    this.fn = fn
+  }
+
+  /**
+   * Evaluate the gate for a single envelope. Called by the runtime, not by
+   * other operators. Returns `{ allow: boolean }`.
+   *
+   * The user fn receives `(msg, ctx)` where ctx.state is mutable and ctx
+   * also carries `streamTimeMs` (system clock) and `partitionId` for
+   * observability / time-based bucket math.
+   */
+  async evaluate(envelope, ctx) {
+    const out = await this.fn(envelope.value, ctx)
+    if (out === true || out === false) return { allow: out }
+    if (out && typeof out === 'object' && typeof out.allow === 'boolean') {
+      return { allow: out.allow }
+    }
+    throw new Error('.gate(fn) must return boolean or { allow: boolean }')
+  }
+}

package/client-v2/streams/operators/KeyByOperator.js

@@ -0,0 +1,23 @@
+/**
+ * KeyByOperator — override the implicit partition-key with a user-derived
+ * key for downstream stateful operators (windowTumbling/reduce/aggregate).
+ *
+ * The default key is the source partition_id, which gives per-partition
+ * state isolation with zero cross-worker contention. .keyBy(...) overrides
+ * to a different value per record. The Runner will warn at start time if
+ * the chain's logical key consistently differs from the partition key,
+ * because that means multiple workers may write the same logical key from
+ * different partition leases — i.e. cross-worker contention on
+ * (query_id, partition_id, key) state rows.
+ */
+export class KeyByOperator {
+  constructor(fn) {
+    this.kind = 'keyBy'
+    this.fn = fn
+  }
+
+  async apply(envelope) {
+    const key = String(await this.fn(envelope.msg))
+    return [{ ...envelope, key }]
+  }
+}

package/client-v2/streams/operators/MapOperator.js

@@ -0,0 +1,27 @@
+/**
+ * MapOperator — transform each input record. Sync or async.
+ *
+ * Operates on the runtime "envelope" object:
+ *   { msg, key, value, ctx? }
+ *
+ * User-fn signature: `(msg, ctx?) => newValue`
+ *
+ * Pre-reducer:
+ *   msg = the original Queen message (with .data, .partitionId, ...)
+ *   ctx = undefined (Queen msg already carries the metadata)
+ *
+ * Post-reducer (after .windowTumbling + .reduce/.aggregate):
+ *   msg = the aggregated value (e.g. { count, sum, min, max })
+ *   ctx = { partition, partitionId, key, windowKey, windowStart, windowEnd }
+ */
+export class MapOperator {
+  constructor(fn) {
+    this.kind = 'map'
+    this.fn = fn
+  }
+
+  async apply(envelope) {
+    const value = await this.fn(envelope.msg, envelope.ctx)
+    return [{ ...envelope, value }]
+  }
+}

package/client-v2/streams/operators/ReduceOperator.js

@@ -0,0 +1,196 @@
+/**
+ * ReduceOperator — fold envelopes within a window into a single value.
+ *
+ * Operates over a *batch* of envelopes (called once per cycle, after
+ * stateless ops + window annotation). Returns:
+ *   - state mutations (upserts for still-open windows, deletes for closed)
+ *   - emit values (closed windows' final accumulator) for sink/foreach
+ *
+ * The reducer expects each envelope to carry `key`, `windowStart`,
+ * `windowEnd`, and `windowKey` (from upstream KeyBy + WindowTumbling).
+ *
+ * Window-close semantics: a window is treated as closed when its
+ * windowEnd <= maxWindowEnd in this cycle's batch. Any window equal to
+ * maxWindowEnd is still open (more events may arrive on the next cycle).
+ */
+export class ReduceOperator {
+  constructor(fn, initial) {
+    this.kind = 'reduce'
+    this.fn = fn
+    this.initial = initial
+    this.config = { hasInitial: initial !== undefined }
+  }
+
+  /**
+   * Run the reducer over a batch of envelopes that all share the same
+   * source partition_id (the cycle's scope).
+   *
+   * @param {{
+   *   envelopes: Array<{
+   *     key:string, windowStart:number, windowEnd:number, windowKey:string,
+   *     value:any, operatorTag?:string, gracePeriodMs?:number
+   *   }>,
+   *   loadedState: Map<string, any>,        // state-key -> stored accumulator
+   *   streamTimeMs?: number,                 // partition clock (max createdAt of batch)
+   *   operatorTag?: string,                  // window operator's structural tag
+   *   gracePeriodMs?: number,                // accept events for windowEnd + grace
+   *   ignoreUnowned?: boolean                // skip seeded rows whose tag doesn't match
+   * }} ctx
+   * @returns {Promise<{
+   *   stateOps: Array<{type:'upsert'|'delete',key:string,value?:any}>,
+   *   emits: Array<{key:string, windowStart:number, windowKey:string, value:any}>
+   * }>}
+   *
+   * Window-close semantics
+   * ----------------------
+   * A window is closed when the partition's stream time has advanced past
+   * its windowEnd plus the grace period:
+   *
+   *   closed = (windowEnd + gracePeriodMs) <= effectiveStreamTime
+   *
+   * The caller seeds `loadedState` with ALL open windows for this partition
+   * (Runner loads via /streams/v1/state/get with keys=[]) so windows which
+   * received no traffic in this batch can still be detected as closed and
+   * flushed.
+   *
+   * State key shape (v0.2+): `${operatorTag}\u001f${windowKey}\u001f${userKey}`.
+   * Seeded state rows whose operatorTag doesn't match this reducer's are
+   * left untouched (they belong to a different window operator in the same
+   * partition shard, or are reserved internal keys like `__wm__`).
+   */
+  async run({ envelopes, loadedState, streamTimeMs, operatorTag = '', gracePeriodMs = 0, ignoreUnowned = true }) {
+    // accumulators[stateKey] = { key, windowStart, windowEnd, windowKey, acc, touched, seeded }
+    const accumulators = new Map()
+
+    // Seed accumulators with every loaded state row owned by THIS operator.
+    // Rows owned by other operators (or internal keys like __wm__) are
+    // skipped — they're not ours to close or upsert.
+    for (const [stateKey, value] of loadedState.entries()) {
+      const parts = parseStateKey(stateKey)
+      if (!parts) continue
+      // Skip reserved internal keys.
+      if (parts.operatorTag.startsWith('__')) continue
+      // Skip rows owned by a different operator.
+      if (ignoreUnowned && operatorTag && parts.operatorTag !== operatorTag) continue
+
+      const ws = value && typeof value.windowStart === 'number'
+        ? value.windowStart
+        : Date.parse(parts.windowKey) || 0
+      const we = value && typeof value.windowEnd === 'number'
+        ? value.windowEnd
+        : ws
+      accumulators.set(stateKey, {
+        key: parts.userKey,
+        windowStart: ws,
+        windowEnd: we,
+        windowKey: parts.windowKey,
+        acc: value && value.acc !== undefined ? value.acc : value,
+        touched: false,
+        seeded: true
+      })
+    }
+
+    // Apply this batch's envelopes onto matching accumulators.
+    let batchMaxWindowStart = -Infinity
+    for (const env of envelopes) {
+      const tag = env.operatorTag || operatorTag || ''
+      const stateKey = stateKeyFor(tag, env.windowKey, env.key)
+      let entry = accumulators.get(stateKey)
+      if (!entry) {
+        entry = {
+          key: env.key,
+          windowStart: env.windowStart,
+          windowEnd: env.windowEnd,
+          windowKey: env.windowKey,
+          acc: this._initial(),
+          touched: false,
+          seeded: false
+        }
+        accumulators.set(stateKey, entry)
+      }
+      entry.acc = await this.fn(entry.acc, env.value !== undefined ? env.value : env.msg)
+      entry.touched = true
+      if (env.windowStart > batchMaxWindowStart) batchMaxWindowStart = env.windowStart
+    }
+
+    // Stream time = caller-provided clock OR the latest batch window's
+    // start as a conservative fallback (keeps the latest window open).
+    const effectiveStreamTime = typeof streamTimeMs === 'number' && streamTimeMs > -Infinity
+      ? streamTimeMs
+      : batchMaxWindowStart
+
+    const stateOps = []
+    const emits = []
+    for (const [stateKey, entry] of accumulators.entries()) {
+      const closed = (entry.windowEnd + gracePeriodMs) <= effectiveStreamTime
+      if (closed) {
+        emits.push({
+          key: entry.key,
+          windowStart: entry.windowStart,
+          windowEnd: entry.windowEnd,
+          windowKey: entry.windowKey,
+          value: entry.acc
+        })
+        if (entry.seeded) {
+          stateOps.push({ type: 'delete', key: stateKey })
+        }
+      } else if (entry.touched) {
+        stateOps.push({
+          type: 'upsert',
+          key: stateKey,
+          value: { acc: entry.acc, windowStart: entry.windowStart, windowEnd: entry.windowEnd }
+        })
+      }
+    }
+    return { stateOps, emits }
+  }
+
+  _initial() {
+    if (this.initial === undefined) return undefined
+    // Deep clone so each window starts fresh
+    if (this.initial === null || typeof this.initial !== 'object') return this.initial
+    return JSON.parse(JSON.stringify(this.initial))
+  }
+}
+
+/**
+ * Build a state key for a windowed reducer.
+ *
+ * Shape (v0.2+):
+ *   `${operatorTag}\u001f${windowKey}\u001f${userKey}`
+ *
+ * The `operatorTag` (e.g. "tumb:60", "slide:60:10", "sess:30", "cron:minute")
+ * lets multiple window operators within one query coexist in the same
+ * `queen_streams.state` table without colliding, and lets the idle-flush
+ * scan filter to a single operator's keys via prefix LIKE matching.
+ *
+ * Reserved key prefix: state keys starting with "__" are internal (e.g.
+ * "__wm__" for per-partition watermarks). User code should never write
+ * keys starting with "__".
+ */
+export function stateKeyFor(operatorTag, windowKey, userKey) {
+  // Backwards-compat shim: if called with two arguments (the v0.1 shape),
+  // route through with a synthetic "legacy" tag so existing tests keep
+  // working until they explicitly pass an operatorTag.
+  if (userKey === undefined) {
+    return `${operatorTag}\u001f${windowKey}`
+  }
+  return `${operatorTag}\u001f${windowKey}\u001f${userKey}`
+}
+
+/**
+ * Decompose a state key into its parts. Returns null if the key doesn't
+ * match the expected shape.
+ */
+export function parseStateKey(stateKey) {
+  if (typeof stateKey !== 'string') return null
+  const parts = stateKey.split('\u001f')
+  if (parts.length === 3) {
+    return { operatorTag: parts[0], windowKey: parts[1], userKey: parts[2] }
+  }
+  if (parts.length === 2) {
+    // Legacy v0.1 shape: ${windowKey}\u001f${userKey}
+    return { operatorTag: '', windowKey: parts[0], userKey: parts[1] }
+  }
+  return null
+}

package/client-v2/streams/operators/SinkOperator.js

@@ -0,0 +1,54 @@
+/**
+ * SinkOperator — terminal `.to(queue)`. Records flowing into the sink are
+ * pushed onto the destination Queen queue as part of the cycle commit.
+ *
+ * The runtime is responsible for shaping each sink push item:
+ *   { queue, partition?, payload }
+ *
+ * If the sink has a partition function (opts.partition), it's called with
+ * the value to derive the partition name; otherwise the source partition
+ * name is reused (preserves entity-level co-partitioning by default).
+ */
+export class SinkOperator {
+  constructor(queueBuilder, opts = {}) {
+    this.kind = 'sink'
+    this.queueBuilder = queueBuilder
+    this.queueName = extractQueueName(queueBuilder)
+    this.partitionResolver = opts.partition
+    this.config = { kind: 'sink', queue: this.queueName }
+  }
+
+  /**
+   * Build sink push items from a list of {key, value, sourceMsg} entries.
+   * Called by the Runner after upstream operators have produced final values.
+   */
+  buildPushItems(entries, sourcePartitionName) {
+    return entries.map(({ value, sourceMsg }) => ({
+      queue: this.queueName,
+      partition: this._resolvePartition(value, sourcePartitionName),
+      payload: value === null || typeof value !== 'object' ? { value } : value
+    }))
+  }
+
+  _resolvePartition(value, sourcePartitionName) {
+    if (typeof this.partitionResolver === 'function') {
+      return String(this.partitionResolver(value))
+    }
+    if (typeof this.partitionResolver === 'string') {
+      return this.partitionResolver
+    }
+    return sourcePartitionName || 'Default'
+  }
+}
+
+function extractQueueName(qb) {
+  // QueueBuilder.name (getter) is the canonical accessor since v0.14.x.
+  // Test fakes and raw objects fall back to _queueName / queueName fields.
+  if (qb && typeof qb.name === 'string' && qb.name.length > 0) return qb.name
+  if (qb && typeof qb._queueName === 'string') return qb._queueName
+  if (qb && typeof qb.queueName === 'string') return qb.queueName
+  throw new Error(
+    'Could not extract queue name from sink target. ' +
+    'Pass either a queen.queue("name") builder or { queueName: "name" }.'
+  )
+}

package/client-v2/streams/operators/WindowCronOperator.js

@@ -0,0 +1,83 @@
+/**
+ * WindowCronOperator — wall-clock-aligned tumbling windows.
+ *
+ * Bucket events by absolute UTC boundaries (minute-on-the-minute,
+ * hour-on-the-hour, day at 00:00 UTC, week starting Mon 00:00 UTC).
+ * Equivalent to WindowTumblingOperator with a fixed size, but the boundary
+ * is anchored to wall-clock instead of relative-to-epoch arithmetic.
+ *
+ * v0.2 ships only the `every:` shorthand; full cron expressions
+ * (`'*\/5 * * * *'`) are reserved for v0.3 and would add a dep.
+ *
+ * Config:
+ *   every           (str) 'second'|'minute'|'hour'|'day'|'week' (required)
+ *   gracePeriod     (sec) accept events for windowEnd + grace before closing
+ *   idleFlushMs     (ms)  ripe-window flush cadence; default 30_000
+ *   eventTime, allowedLateness, onLate — same as the other window operators
+ */
+
+import { extractEventTime, normaliseLateOpts, everyToMs, MS_PER_SEC } from './_windowCommon.js'
+
+const WEEK_EPOCH_OFFSET_MS = 4 * 86_400 * 1000  // Jan 1 1970 UTC was Thursday;
+// shift so weeks align to Monday 00:00 UTC
+
+export class WindowCronOperator {
+  constructor({
+    every,
+    gracePeriod   = 0,
+    idleFlushMs   = 30_000,
+    eventTime,
+    allowedLateness = 0,
+    onLate
+  } = {}) {
+    if (typeof every !== 'string') {
+      throw new Error("windowCron requires { every: 'second'|'minute'|'hour'|'day'|'week' }")
+    }
+    this.kind             = 'window'
+    this.windowKind       = 'cron'
+    this.every            = every
+    this.windowMs         = everyToMs(every)
+    this.gracePeriodMs    = Math.max(0, Math.floor(gracePeriod * MS_PER_SEC))
+    this.idleFlushMs      = Math.max(0, Math.floor(idleFlushMs))
+    this.eventTimeFn      = typeof eventTime === 'function' ? eventTime : null
+    this.allowedLatenessMs = Math.max(0, Math.floor(allowedLateness * MS_PER_SEC))
+    this.onLatePolicy     = normaliseLateOpts(onLate)
+    this.operatorTag      = `cron:${every}`
+    this.config           = {
+      kind: 'window-cron',
+      every,
+      gracePeriod,
+      idleFlushMs:      this.idleFlushMs,
+      eventTime:        !!this.eventTimeFn,
+      allowedLateness,
+      onLate:           this.onLatePolicy
+    }
+  }
+
+  async apply(envelope) {
+    const ts = extractEventTime(envelope.msg, this.eventTimeFn)
+    if (ts === null) {
+      throw new Error('windowCron could not determine timestamp for message')
+    }
+    const windowStart = this._floorToBoundary(ts)
+    return [{
+      ...envelope,
+      eventTimeMs:    ts,
+      windowStart,
+      windowEnd:      windowStart + this.windowMs,
+      windowKey:      new Date(windowStart).toISOString(),
+      operatorTag:    this.operatorTag,
+      gracePeriodMs:  this.gracePeriodMs
+    }]
+  }
+
+  _floorToBoundary(ts) {
+    if (this.every === 'week') {
+      // Align to Monday 00:00 UTC.
+      const shifted = ts - WEEK_EPOCH_OFFSET_MS
+      const week    = Math.floor(shifted / this.windowMs)
+      return week * this.windowMs + WEEK_EPOCH_OFFSET_MS
+    }
+    return Math.floor(ts / this.windowMs) * this.windowMs
+  }
+}