queen-mq 0.14.0 → 0.15.0

package/client-v2/builders/QueueBuilder.js

@@ -49,6 +49,16 @@ export class QueueBuilder {
     this.#queueName = queueName
   }
 
+  /**
+   * Public read-only accessor for the queue name. Useful for tooling that
+   * holds a QueueBuilder reference and needs to know the underlying name —
+   * e.g. @queenmq/streams resolves sink-queue names from the QueueBuilder
+   * passed to .to(...).
+   */
+  get name() {
+    return this.#queueName
+  }
+
   // ===========================
   // Affinity Key Generation
   // ===========================

package/client-v2/index.js

@@ -1,8 +1,23 @@
 /**
  * Queen Message Queue Client - Entry Point
+ *
+ * One npm package for everything:
+ *   import { Queen, Stream, tokenBucketGate, slidingWindowGate } from 'queen-mq'
+ *
+ * The broker client (`Queen`, `Admin`) and the fluent streaming SDK
+ * (`Stream`, `tokenBucketGate`, `slidingWindowGate`, plus all operator
+ * classes) ship in the same package. ESM named exports + tree-shaking
+ * mean bundlers (Vite, Rollup, webpack with `sideEffects:false`) can
+ * drop the streaming code from your output if you only import broker
+ * symbols.
+ *
+ * Streaming source lives under `./streams/` of this package.
  */
 
 export { Queen } from './Queen.js'
 export { Admin } from './admin/Admin.js'
 export { CLIENT_DEFAULTS, QUEUE_DEFAULTS, CONSUME_DEFAULTS, POP_DEFAULTS, BUFFER_DEFAULTS } from './utils/defaults.js'
 
+// Streaming SDK — full source under ./streams/.
+export { Stream } from './streams/Stream.js'
+export { tokenBucketGate, slidingWindowGate } from './streams/helpers/rateLimiter.js'

package/client-v2/streams/Stream.js

@@ -0,0 +1,392 @@
+/**
+ * Stream — fluent builder for streaming pipelines.
+ *
+ * A Stream is an immutable chain of Operators that gets compiled and run
+ * by a Runner. Each combinator (`.map`, `.filter`, ...) returns a NEW Stream
+ * with the operator appended; the chain is finalized by `.run({ queryId })`.
+ *
+ * The chain shape (operator types + their stateful config) is fingerprinted
+ * via configHash so that re-deploying with a different chain under the same
+ * queryId is rejected at registration unless `{ reset: true }` is passed.
+ */
+
+import { MapOperator } from './operators/MapOperator.js'
+import { FilterOperator } from './operators/FilterOperator.js'
+import { FlatMapOperator } from './operators/FlatMapOperator.js'
+import { KeyByOperator } from './operators/KeyByOperator.js'
+import { WindowTumblingOperator } from './operators/WindowTumblingOperator.js'
+import { WindowSlidingOperator } from './operators/WindowSlidingOperator.js'
+import { WindowSessionOperator } from './operators/WindowSessionOperator.js'
+import { WindowCronOperator } from './operators/WindowCronOperator.js'
+import { ReduceOperator } from './operators/ReduceOperator.js'
+import { AggregateOperator } from './operators/AggregateOperator.js'
+import { GateOperator } from './operators/GateOperator.js'
+import { SinkOperator } from './operators/SinkOperator.js'
+import { ForeachOperator } from './operators/ForeachOperator.js'
+import { Runner } from './runtime/Runner.js'
+import { configHashOf } from './util/configHash.js'
+
+export class Stream {
+  /**
+   * Build a Stream sourced from a Queen QueueBuilder.
+   *
+   * @param {object} queueBuilder - the result of queen.queue('name') — must
+   *   expose .pop() / .ack() / .push() and the queue name.
+   * @param {object} [options]
+   * @returns {Stream}
+   */
+  static from(queueBuilder, options = {}) {
+    return new Stream({
+      source: queueBuilder,
+      operators: [],
+      sourceOptions: options
+    })
+  }
+
+  /**
+   * @param {object} args
+   * @param {object} args.source - source QueueBuilder
+   * @param {object} [args.sourceOptions]
+   * @param {Array} args.operators - operator chain
+   */
+  constructor({ source, operators, sourceOptions = {} }) {
+    this._source = source
+    this._sourceOptions = sourceOptions
+    this._operators = operators
+  }
+
+  // ---------------------------------------------------------------- Stateless
+
+  /** @param {(msg:object)=>any|Promise<any>} fn */
+  map(fn) {
+    return this._extend(new MapOperator(fn))
+  }
+
+  /** @param {(msg:object)=>boolean|Promise<boolean>} predicate */
+  filter(predicate) {
+    return this._extend(new FilterOperator(predicate))
+  }
+
+  /** @param {(msg:object)=>(any[]|Promise<any[]>)} fn */
+  flatMap(fn) {
+    return this._extend(new FlatMapOperator(fn))
+  }
+
+  // -------------------------------------------------------------------- Keying
+
+  /**
+   * Override the implicit partition-key for downstream stateful operators.
+   * If omitted, stateful operators use the source partition_id as the key,
+   * which is the natural fit for queues partitioned by entity.
+   *
+   * Warning: when the key differs from the partition, multiple workers
+   * holding different partition leases may write the same logical key,
+   * causing cross-worker contention on (query_id, partition_id, key) state
+   * rows. The repartition pattern (push to a co-keyed intermediate queue,
+   * then process) avoids this.
+   *
+   * @param {(msg:object)=>string} fn
+   */
+  keyBy(fn) {
+    return this._extend(new KeyByOperator(fn))
+  }
+
+  // -------------------------------------------------------------------- Windows
+
+  /**
+   * Fixed-size, non-overlapping windows.
+   * @param {{
+   *   seconds: number,
+   *   gracePeriod?: number,    // accept events for windowEnd + grace before closing
+   *   idleFlushMs?: number,    // close ripe windows on quiet partitions every N ms (default 5000)
+   *   eventTime?: (msg:any)=>(number|Date|string),  // event-time mode if set
+   *   allowedLateness?: number, // event-time only: drop events older than wm - allowedLateness
+   *   onLate?: 'drop' | 'include'
+   * }} opts
+   */
+  windowTumbling(opts) {
+    return this._extend(new WindowTumblingOperator(opts))
+  }
+
+  /**
+   * Overlapping fixed-size windows that hop every `slide` seconds.
+   * Each event creates `size/slide` state rows per key.
+   * @param {{size:number, slide:number, gracePeriod?:number, idleFlushMs?:number,
+   *          eventTime?:(msg:any)=>any, allowedLateness?:number, onLate?:string}} opts
+   */
+  windowSliding(opts) {
+    return this._extend(new WindowSlidingOperator(opts))
+  }
+
+  /**
+   * Per-key activity-based windows. A session for a key extends as long
+   * as events keep arriving within `gap` seconds of each other.
+   * @param {{gap:number, gracePeriod?:number, idleFlushMs?:number,
+   *          eventTime?:(msg:any)=>any, allowedLateness?:number, onLate?:string}} opts
+   */
+  windowSession(opts) {
+    return this._extend(new WindowSessionOperator(opts))
+  }
+
+  /**
+   * Wall-clock-aligned windows. Currently supports the `every:` shorthand
+   * (`'second'|'minute'|'hour'|'day'|'week'`); full cron syntax is reserved
+   * for v0.3.
+   * @param {{every:string, gracePeriod?:number, idleFlushMs?:number,
+   *          eventTime?:(msg:any)=>any, allowedLateness?:number, onLate?:string}} opts
+   */
+  windowCron(opts) {
+    return this._extend(new WindowCronOperator(opts))
+  }
+
+  // -------------------------------------------------------------------- Reduce
+
+  /**
+   * @template T
+   * @param {(acc:T, msg:object)=>T|Promise<T>} fn
+   * @param {T} initial
+   */
+  reduce(fn, initial) {
+    return this._extend(new ReduceOperator(fn, initial))
+  }
+
+  /**
+   * Sugar over reduce. Each provided extractor produces one named field on
+   * the aggregate value.
+   *
+   * Example:
+   *   .aggregate({ count: () => 1, sum: m => m.data.amount })
+   *
+   * Output value shape: { count: number, sum: number, ... }
+   *
+   * @param {Record<string,(msg:object)=>number>} extractors
+   */
+  aggregate(extractors) {
+    return this._extend(new AggregateOperator(extractors))
+  }
+
+  // ---------------------------------------------------------------------- Gate
+
+  /**
+   * Per-message ALLOW/DENY decision with persistent per-key state.
+   *
+   * Used to build rate limiters, throttlers, fairness gates, circuit
+   * breakers — anything where the semantics is "should this message
+   * proceed RIGHT NOW given the recent history of this key?".
+   *
+   * The user fn receives `(value, ctx)` where:
+   *   - value: the message payload (post any pre-stage map/filter)
+   *   - ctx.state: mutable per-key state (loaded from queen_streams.state,
+   *                persisted only if you return ALLOW for this message)
+   *   - ctx.streamTimeMs: system clock for the cycle (use for refill math)
+   *   - ctx.partitionId: source partition_id (= state shard)
+   *
+   * Return `true` (or `{allow:true}`) to let the message through.
+   * Return `false` (or `{allow:false}`) to halt the batch here. The runner
+   * commits an ack for the prefix that was allowed and DOES NOT release the
+   * source lease, so the denied message and its successors get redelivered
+   * in their original order when the lease expires. FIFO per partition is
+   * preserved without any deferred queue.
+   *
+   * Example (token bucket rate limiter):
+   *
+   *   .gate((req, ctx) => {
+   *     const cfg = { capacity: 10, refillPerSec: 5 }
+   *     const now = ctx.streamTimeMs
+   *     ctx.state.tokens       ??= cfg.capacity
+   *     ctx.state.lastRefillAt ??= now
+   *     const elapsedSec = (now - ctx.state.lastRefillAt) / 1000
+   *     ctx.state.tokens = Math.min(
+   *       cfg.capacity,
+   *       ctx.state.tokens + elapsedSec * cfg.refillPerSec
+   *     )
+   *     ctx.state.lastRefillAt = now
+   *     if (ctx.state.tokens >= 1) {
+   *       ctx.state.tokens -= 1
+   *       return true
+   *     }
+   *     return false   // halt batch, lease expires, redelivered in order
+   *   })
+   *
+   * Constraints
+   *  - At most one .gate() per stream (multiple gates would compose
+   *    awkwardly with the partial-ack semantics; chain serially via two
+   *    streams if needed).
+   *  - .gate() is incompatible with windowing/reducing in the same stream.
+   *    The window+reducer model assumes the FULL batch is consumed atomically;
+   *    gating breaks that. Run them as two separate streams.
+   *
+   * @param {(value:any, ctx:{state:object, streamTimeMs:number, partitionId:string}) => boolean | {allow:boolean} | Promise<boolean | {allow:boolean}>} fn
+   */
+  gate(fn) {
+    return this._extend(new GateOperator(fn))
+  }
+
+  // ---------------------------------------------------------------------- Sink
+
+  /**
+   * Sink to a Queen queue. The cycle commits state + push + ack atomically.
+   *
+   * @param {object} sinkQueueBuilder - `queen.queue('sink-name')`
+   * @param {{partition?: string|((value:any)=>string)}} [opts]
+   */
+  to(sinkQueueBuilder, opts = {}) {
+    return this._extend(new SinkOperator(sinkQueueBuilder, opts))
+  }
+
+  /**
+   * Terminal at-least-once side-effect. The cycle acks the source only after
+   * `fn` has resolved successfully, so a crash mid-`fn` will redeliver. If
+   * you need exactly-once external effects, write to a sink queue and have
+   * a dedicated worker consume it.
+   *
+   * @param {(value:any)=>void|Promise<void>} fn
+   */
+  foreach(fn) {
+    return this._extend(new ForeachOperator(fn))
+  }
+
+  // ----------------------------------------------------------------- Compile/run
+
+  /**
+   * Start the streaming runner. Returns a handle exposing:
+   *   - stop(): graceful drain
+   *   - metrics(): cycle/throughput/lag stats
+   *
+   * @param {object} runOptions
+   * @param {string} runOptions.queryId - durable identity of this query
+   * @param {number} [runOptions.batchSize=200] - messages per cycle
+   * @param {number} [runOptions.maxPartitions=4] - lease up to N partitions/cycle
+   * @param {number} [runOptions.maxWaitMillis=1000] - long-poll wait for source pop
+   * @param {string} [runOptions.subscriptionMode] - 'all' (default) | 'new'
+   * @param {string} [runOptions.subscriptionFrom] - ISO timestamp or 'now'
+   * @param {boolean} [runOptions.reset=false] - wipe state on config_hash mismatch
+   * @param {(err:Error, ctx:object)=>void} [runOptions.onError] - cycle error hook
+   * @param {AbortSignal} [runOptions.abortSignal] - external cancellation
+   */
+  async run(runOptions) {
+    if (!runOptions || !runOptions.queryId) {
+      throw new Error('run({ queryId }) is required')
+    }
+    const compiled = this._compile()
+    const runner = new Runner({
+      ...runOptions,
+      stream: compiled
+    })
+    await runner.start()
+    return runner
+  }
+
+  // ===== internals =========================================================
+
+  /** @returns {Stream} */
+  _extend(operator) {
+    return new Stream({
+      source: this._source,
+      sourceOptions: this._sourceOptions,
+      operators: [...this._operators, operator]
+    })
+  }
+
+  /**
+   * Compile the operator chain into a runnable description: stages, sink,
+   * and the configHash. Exposed for the Runner.
+   */
+  _compile() {
+    const sink = this._operators.find(op => op.kind === 'sink' || op.kind === 'foreach')
+    const sinkIdx = sink ? this._operators.indexOf(sink) : this._operators.length
+
+    // Verify the sink (if present) is the last operator.
+    if (sink && sinkIdx !== this._operators.length - 1) {
+      throw new Error(
+        'sink operators (.to / .foreach) must be the last in the chain; ' +
+        `found ${sink.kind} at position ${sinkIdx} of ${this._operators.length}`
+      )
+    }
+
+    const upstream = sink ? this._operators.slice(0, sinkIdx) : this._operators
+
+    // Decompose into stages:
+    //   pre:    [stateless...] applied to each source record
+    //   keyBy:  optional KeyByOperator (operates on source record)
+    //   window: optional WindowTumblingOperator (annotates envelopes)
+    //   reducer: optional ReduceOperator/AggregateOperator
+    //   post:   [stateless...] applied to each emit value (post-reducer)
+    //   sink:   optional SinkOperator/ForeachOperator (terminal)
+    //
+    // "phase" walks from `pre` -> `keyed` -> `window` -> `reducer` -> `post`.
+    // Stateless operators belong to whichever side of the reducer they're
+    // declared on. A reducer is required to switch to `post`; without one,
+    // all stateless ops are pre-stage.
+    const stages = {
+      pre: [],
+      keyBy: null,
+      window: null,
+      reducer: null,
+      gate: null,
+      post: [],
+      sink
+    }
+    let phase = 'pre'
+    for (const op of upstream) {
+      if (op.kind === 'map' || op.kind === 'filter' || op.kind === 'flatMap') {
+        if (phase === 'pre' || phase === 'keyed') {
+          stages.pre.push(op)
+        } else if (phase === 'window') {
+          // Stateless ops between window and reducer are unusual; treat as pre
+          // (they see the windowed envelope but have no reducer state to act on).
+          stages.pre.push(op)
+        } else {
+          // post-reducer or post-gate: operates on each emit value.
+          stages.post.push(op)
+        }
+      } else if (op.kind === 'keyBy') {
+        if (stages.keyBy) throw new Error('only one .keyBy() per stream')
+        if (phase === 'reducer' || phase === 'gate') {
+          throw new Error('.keyBy() must come before window/reduce/gate')
+        }
+        stages.keyBy = op
+        if (phase === 'pre') phase = 'keyed'
+      } else if (op.kind === 'window') {
+        if (stages.window) throw new Error('only one window operator per stream')
+        if (phase === 'reducer') throw new Error('window must come before reduce/aggregate')
+        if (stages.gate) throw new Error('window+reduce is incompatible with .gate() in the same stream')
+        stages.window = op
+        phase = 'window'
+      } else if (op.kind === 'reduce' || op.kind === 'aggregate') {
+        if (stages.reducer) throw new Error('only one reduce/aggregate per stream')
+        if (!stages.window) {
+          throw new Error('reduce/aggregate requires a preceding window operator')
+        }
+        if (stages.gate) throw new Error('reduce/aggregate is incompatible with .gate() in the same stream')
+        stages.reducer = op
+        phase = 'reducer'
+      } else if (op.kind === 'gate') {
+        if (stages.gate) throw new Error('only one .gate() per stream')
+        if (stages.window || stages.reducer) {
+          throw new Error('.gate() is incompatible with windowing/reduce in the same stream')
+        }
+        stages.gate = op
+        phase = 'gate'
+      } else {
+        throw new Error(`unsupported operator: ${op.kind}`)
+      }
+    }
+    // Backwards-compat: keep the old name `stateless` aliasing `pre` so any
+    // existing code paths that referenced stages.stateless keep working.
+    stages.stateless = stages.pre
+
+    // Compute config_hash from the chain shape (operator kinds + their
+    // structural config; user functions are intentionally not hashed since
+    // we cannot stably serialise closures).
+    const config_hash = configHashOf(this._operators)
+
+    return {
+      source: this._source,
+      sourceOptions: this._sourceOptions,
+      stages,
+      operators: this._operators,
+      config_hash
+    }
+  }
+}

package/client-v2/streams/helpers/rateLimiter.js

@@ -0,0 +1,138 @@
+/**
+ * Rate-limiter helpers — composable token-bucket / sliding-window gate
+ * factories that return a function suitable for `.gate(fn)` on a Stream.
+ *
+ * The point of these helpers is purely ergonomic: the user shouldn't have to
+ * re-write the refill math every time. The semantics (per-key state in
+ * queen_streams.state, partial-ack on deny, FIFO order on lease expiry) all
+ * come from the underlying Stream `.gate()` runtime — these factories only
+ * decide HOW each request consumes from the bucket.
+ *
+ * All four common rate-limit shapes are expressible by varying `costFn`:
+ *
+ *   A) "100 req/sec, batches arbitrary":    costFn = () => 1            on a queue of REQUESTS
+ *   B) "100 msg/sec, in any number of req": costFn = () => 1            on a queue of MESSAGES
+ *   C) "100 req/sec, exactly 1 msg/req":    costFn = () => 1            same as B (degenerate)
+ *   D) "100 weight/sec, cost varies":       costFn = req => req.weight  on a queue of REQUESTS
+ *
+ * Two gates can be chained on the same pipeline (req/s + msg/s simultaneously
+ * required by some OTAs) by running them as two consecutive Stream queries
+ * with an intermediate sink queue.
+ *
+ * Sizing rule (also enforced by tokenBucketGate at start time, and warned
+ * by the assertion below): the maximum sustainable rate the bucket can
+ * deliver is `min(refillPerSec, capacity / leaseSec)`. Pick capacity and
+ * leaseSec so the second term doesn't artificially cap you below
+ * refillPerSec — usually `capacity ≈ refillPerSec × leaseSec`.
+ */
+
+/**
+ * Token-bucket gate factory.
+ *
+ * Returns a `(msg, ctx) => boolean` function suitable for `.gate(fn)`.
+ * The bucket state lives in `ctx.state` (which the runtime persists per-key
+ * in queen_streams.state on every ALLOWED message).
+ *
+ * @param {object} opts
+ * @param {number}                 opts.capacity        Max tokens in the bucket (= max burst).
+ * @param {number}                 opts.refillPerSec    Steady-state refill rate (tokens / sec).
+ * @param {(msg:any)=>number}     [opts.costFn]         Maps a message to its cost in tokens.
+ *                                                       Defaults to `() => 1` (one token per message).
+ * @param {boolean}               [opts.allowZeroCost]  If true, costFn() may return 0 to mean
+ *                                                       "always allow, no token consumed". Defaults true.
+ * @returns {(msg:any, ctx:object)=>boolean}
+ */
+export function tokenBucketGate({
+  capacity,
+  refillPerSec,
+  costFn = () => 1,
+  allowZeroCost = true
+} = {}) {
+  if (!Number.isFinite(capacity) || capacity <= 0) {
+    throw new Error('tokenBucketGate: capacity must be a positive number')
+  }
+  if (!Number.isFinite(refillPerSec) || refillPerSec <= 0) {
+    throw new Error('tokenBucketGate: refillPerSec must be a positive number')
+  }
+  if (typeof costFn !== 'function') {
+    throw new Error('tokenBucketGate: costFn must be a function')
+  }
+  return function tokenBucket(msg, ctx) {
+    const now = ctx.streamTimeMs
+    if (typeof ctx.state.tokens !== 'number') ctx.state.tokens = capacity
+    if (typeof ctx.state.lastRefillAt !== 'number') ctx.state.lastRefillAt = now
+    // Refill based on elapsed wall-clock time. Capped at capacity so we
+    // never exceed the configured burst.
+    const elapsedSec = Math.max(0, (now - ctx.state.lastRefillAt) / 1000)
+    ctx.state.tokens = Math.min(capacity, ctx.state.tokens + elapsedSec * refillPerSec)
+    ctx.state.lastRefillAt = now
+
+    let cost = costFn(msg)
+    if (!Number.isFinite(cost) || cost < 0) cost = 1
+    if (cost === 0) return allowZeroCost   // never consume, never block
+
+    if (ctx.state.tokens >= cost) {
+      ctx.state.tokens -= cost
+      // Lightweight observability: the SDK persists ctx.state on ALLOW only,
+      // so these counters automatically reflect "what the bucket actually let
+      // through" and can be inspected via SQL on queen_streams.state.
+      ctx.state.allowedTotal = (ctx.state.allowedTotal || 0) + 1
+      ctx.state.consumedTotal = (ctx.state.consumedTotal || 0) + cost
+      return true
+    }
+    // On deny, ctx.state mutations are discarded by the runtime — so
+    // deniedSeen is "lost" and would only be useful if the runtime
+    // persisted on every eval. We leave it OUT here so callers don't
+    // misinterpret the persisted counters.
+    return false
+  }
+}
+
+/**
+ * Sliding-window approximation gate (for "max N events in last W seconds"
+ * style limits — e.g. SendGrid daily quotas, OTA hourly quotas).
+ *
+ * IMPORTANT: this is NOT a precise sliding window. It's a 2-bucket
+ * approximation (current + previous) that's accurate within ±1× rate at
+ * the boundary. For exact sliding window, you'd need to keep per-event
+ * timestamps in state — usually overkill for rate-limiting purposes.
+ *
+ * @param {object} opts
+ * @param {number}                opts.limit          Max events per window.
+ * @param {number}                opts.windowSec      Window length in seconds.
+ * @param {(msg:any)=>number}    [opts.costFn]        Cost per event (default 1).
+ */
+export function slidingWindowGate({ limit, windowSec, costFn = () => 1 } = {}) {
+  if (!Number.isFinite(limit) || limit <= 0) {
+    throw new Error('slidingWindowGate: limit must be a positive number')
+  }
+  if (!Number.isFinite(windowSec) || windowSec <= 0) {
+    throw new Error('slidingWindowGate: windowSec must be a positive number')
+  }
+  const windowMs = windowSec * 1000
+  return function slidingWindow(msg, ctx) {
+    const now = ctx.streamTimeMs
+    const currentWindow = Math.floor(now / windowMs)
+    const elapsedInWindow = (now % windowMs) / windowMs   // 0..1
+
+    if (ctx.state.window !== currentWindow) {
+      // Roll: previous window becomes "previous", current resets.
+      ctx.state.previousCount = ctx.state.window === currentWindow - 1
+        ? (ctx.state.currentCount || 0)
+        : 0
+      ctx.state.currentCount = 0
+      ctx.state.window = currentWindow
+    }
+    // 2-bucket sliding estimate.
+    const estimated = (ctx.state.previousCount || 0) * (1 - elapsedInWindow)
+                    + (ctx.state.currentCount  || 0)
+    let cost = costFn(msg)
+    if (!Number.isFinite(cost) || cost < 0) cost = 1
+
+    if (estimated + cost <= limit) {
+      ctx.state.currentCount = (ctx.state.currentCount || 0) + cost
+      return true
+    }
+    return false
+  }
+}

package/client-v2/streams/operators/AggregateOperator.js

@@ -0,0 +1,87 @@
+/**
+ * AggregateOperator — sugar over reduce. Each named extractor produces one
+ * field of the per-window aggregate value.
+ *
+ * Supported extractor names:
+ *   count: () => 1                         // scalar count contribution
+ *   sum:   m => m.data.amount              // running sum
+ *   min:   m => m.data.value               // running min (ignores nulls)
+ *   max:   m => m.data.value               // running max (ignores nulls)
+ *   avg:   m => m.data.value               // running average (sum + count internally)
+ *
+ * Output shape per window:
+ *   { count?: number, sum?: number, min?: number, max?: number, avg?: number, ... }
+ *
+ * Custom keys (any name not in the list above) are also supported and
+ * treated as `sum`-like (numeric running total). Use plain `.reduce(...)`
+ * if you need non-additive aggregates.
+ */
+
+import { ReduceOperator } from './ReduceOperator.js'
+
+const RESERVED = new Set(['count', 'sum', 'min', 'max', 'avg'])
+
+export class AggregateOperator extends ReduceOperator {
+  constructor(extractors) {
+    if (!extractors || typeof extractors !== 'object') {
+      throw new Error('aggregate requires an object of named extractors')
+    }
+    const fields = Object.keys(extractors).map(name => {
+      const fn = extractors[name]
+      if (typeof fn !== 'function') {
+        throw new Error(`aggregate.${name} must be a function`)
+      }
+      return { name, fn }
+    })
+    if (fields.length === 0) {
+      throw new Error('aggregate requires at least one extractor')
+    }
+
+    const initial = {}
+    for (const { name } of fields) {
+      if (name === 'min') initial[name] = null
+      else if (name === 'max') initial[name] = null
+      else if (name === 'avg') { initial.__avg_sum = 0; initial.__avg_count = 0; initial.avg = 0 }
+      else initial[name] = 0
+    }
+
+    const fn = async (acc, msg) => {
+      const next = { ...acc }
+      for (const { name, fn: ex } of fields) {
+        const v = await ex(msg)
+        if (name === 'count') {
+          next.count = (next.count || 0) + (typeof v === 'number' ? v : 1)
+        } else if (name === 'sum') {
+          next.sum = (next.sum || 0) + (typeof v === 'number' ? v : 0)
+        } else if (name === 'min') {
+          if (typeof v === 'number') {
+            next.min = next.min === null || next.min === undefined ? v : Math.min(next.min, v)
+          }
+        } else if (name === 'max') {
+          if (typeof v === 'number') {
+            next.max = next.max === null || next.max === undefined ? v : Math.max(next.max, v)
+          }
+        } else if (name === 'avg') {
+          if (typeof v === 'number') {
+            next.__avg_sum = (next.__avg_sum || 0) + v
+            next.__avg_count = (next.__avg_count || 0) + 1
+            next.avg = next.__avg_count > 0 ? next.__avg_sum / next.__avg_count : 0
+          }
+        } else {
+          // Custom user-named field; treat as numeric running total.
+          next[name] = (next[name] || 0) + (typeof v === 'number' ? v : 0)
+        }
+      }
+      return next
+    }
+
+    super(fn, initial)
+    this.kind = 'aggregate'
+    this.fields = fields.map(f => f.name)
+    // For configHash: expose the extractor field names + presence of reserved keys.
+    this.config = { fields: fields.map(f => f.name) }
+  }
+}
+
+// Re-export the constant in case downstream wants the supported list.
+export const RESERVED_AGGREGATE_FIELDS = Array.from(RESERVED)

package/client-v2/streams/operators/FilterOperator.js

@@ -0,0 +1,18 @@
+/**
+ * FilterOperator — drop envelopes where predicate returns false.
+ *
+ * The predicate sees the original Queen `msg` (msg.data, msg.partitionId, ...)
+ * to keep filtering decisions tied to the source record, not to a possibly
+ * already-mapped value.
+ */
+export class FilterOperator {
+  constructor(predicate) {
+    this.kind = 'filter'
+    this.fn = predicate
+  }
+
+  async apply(envelope) {
+    const keep = await this.fn(envelope.msg, envelope.ctx)
+    return keep ? [envelope] : []
+  }
+}