wicked-bus 1.1.0 → 2.0.0

package/lib/archive.js

@@ -0,0 +1,198 @@
+/**
+ * Archive bucket lifecycle for the v2 tiered-storage warm tier.
+ *
+ * Each warm bucket is an independent SQLite file at
+ * `<data_dir>/archive/bus-YYYY-MM[suffix].db`. Files are independent —
+ * corrupting one bucket affects only the events in that month/split.
+ *
+ * Each bucket carries a `_meta` table (NOT a fictitious PRAGMA) holding
+ * `min_event_id`, `max_event_id`, `created_at`, and `sealed_at`. The query
+ * resolver reads these via normal SELECT to decide which buckets to ATTACH
+ * during a warm-spill (see lib/query.js and DESIGN-v2.md §5.4).
+ *
+ * Buckets without a `sealed_at` value are still being filled by sweep and
+ * are conservatively treated as covering up to MAX_SAFE_INTEGER until sealed.
+ *
+ * @module lib/archive
+ */
+
+import path from 'node:path';
+import fs from 'node:fs';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+const BUCKET_FILE_RE = /^bus-\d{4}-\d{2}[a-z]?\.db$/;
+
+/**
+ * Build the absolute path to the archive directory for a given data_dir.
+ * @param {string} dataDir
+ * @returns {string}
+ */
+export function archiveDir(dataDir) {
+  return path.join(dataDir, 'archive');
+}
+
+/**
+ * Ensure the archive directory exists (idempotent).
+ * @param {string} archDir
+ */
+export function ensureArchiveDir(archDir) {
+  fs.mkdirSync(archDir, { recursive: true });
+}
+
+/**
+ * Create a new archive bucket file with the v2 `events` schema mirrored
+ * from the live tier, plus the `_meta` table. Returns the open Database
+ * handle. Caller is responsible for closing it.
+ *
+ * @param {string} filepath - absolute path to the bucket .db file
+ * @param {{ minEventId?: number, maxEventId?: number }} [meta]
+ * @returns {import('better-sqlite3').Database}
+ */
+export function createBucket(filepath, meta = {}) {
+  ensureArchiveDir(path.dirname(filepath));
+  const Database = require('better-sqlite3');
+  const db = new Database(filepath);
+
+  db.pragma('journal_mode = WAL');
+  db.pragma('synchronous = NORMAL');
+  db.pragma('busy_timeout = 5000');
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS events (
+      event_id                 INTEGER PRIMARY KEY,
+      event_type               TEXT    NOT NULL,
+      domain                   TEXT    NOT NULL,
+      subdomain                TEXT    NOT NULL DEFAULT '',
+      payload                  TEXT    NOT NULL,
+      schema_version           TEXT    NOT NULL DEFAULT '1.0.0',
+      idempotency_key          TEXT    NOT NULL,
+      emitted_at               INTEGER NOT NULL,
+      expires_at               INTEGER NOT NULL,
+      dedup_expires_at         INTEGER NOT NULL,
+      metadata                 TEXT,
+      parent_event_id          INTEGER,
+      session_id               TEXT,
+      correlation_id           TEXT,
+      producer_id              TEXT,
+      origin_node_id           TEXT,
+      registry_schema_version  INTEGER,
+      payload_cas_sha          TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_events_event_type      ON events(event_type);
+    CREATE INDEX IF NOT EXISTS idx_events_domain          ON events(domain);
+    CREATE INDEX IF NOT EXISTS idx_events_correlation_id  ON events(correlation_id);
+
+    CREATE TABLE IF NOT EXISTS _meta (
+      k TEXT PRIMARY KEY,
+      v TEXT NOT NULL
+    );
+  `);
+
+  setMeta(db, 'created_at', String(Date.now()));
+  if (meta.minEventId != null) setMeta(db, 'min_event_id', String(meta.minEventId));
+  if (meta.maxEventId != null) setMeta(db, 'max_event_id', String(meta.maxEventId));
+
+  return db;
+}
+
+/**
+ * Read the `_meta` key/value pairs from a bucket file.
+ * Opens read-only so a sealed bucket can be inspected without taking a write lock.
+ *
+ * @param {string} filepath
+ * @returns {Record<string,string>}
+ */
+export function getBucketMeta(filepath) {
+  const Database = require('better-sqlite3');
+  const db = new Database(filepath, { readonly: true });
+  try {
+    const rows = db.prepare('SELECT k, v FROM _meta').all();
+    const meta = {};
+    for (const r of rows) meta[r.k] = r.v;
+    return meta;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Mark a bucket sealed: write `sealed_at` and (optionally) finalize `max_event_id`.
+ * Once sealed, a bucket's covered range is fixed and the resolver can use
+ * the [min, max] interval to skip it when out of range.
+ *
+ * @param {string} filepath
+ * @param {{ maxEventId?: number }} [opts]
+ */
+export function sealBucket(filepath, opts = {}) {
+  const Database = require('better-sqlite3');
+  const db = new Database(filepath);
+  try {
+    setMeta(db, 'sealed_at', String(Date.now()));
+    if (opts.maxEventId != null) {
+      setMeta(db, 'max_event_id', String(opts.maxEventId));
+    }
+  } finally {
+    db.close();
+  }
+}
+
+function setMeta(db, k, v) {
+  db.prepare('INSERT OR REPLACE INTO _meta(k, v) VALUES (?, ?)').run(k, v);
+}
+
+/**
+ * List all bucket files in the archive directory, sorted lexicographically.
+ * Suffix letters (`a`, `b`, …) preserve creation order within a month after
+ * auto-split, so lexical sort matches creation order.
+ *
+ * @param {string} archDir
+ * @returns {string[]} absolute paths
+ */
+export function listBuckets(archDir) {
+  if (!fs.existsSync(archDir)) return [];
+  return fs.readdirSync(archDir)
+    .filter(f => BUCKET_FILE_RE.test(f))
+    .sort()
+    .map(f => path.join(archDir, f));
+}
+
+/**
+ * Find buckets that cover (any part of) the event_id range [gapStart, gapEnd].
+ * Returns descriptors with absolute path + parsed range + unsealed flag.
+ * Unsealed buckets are conservatively included whenever their `min_event_id`
+ * is at or below `gapEnd`.
+ *
+ * @param {string} archDir
+ * @param {number} gapStart - inclusive
+ * @param {number} gapEnd   - inclusive
+ * @returns {Array<{ filepath: string, min: number, max: number, unsealed: boolean }>}
+ */
+export function bucketsCoveringRange(archDir, gapStart, gapEnd) {
+  const all = listBuckets(archDir);
+  const covering = [];
+  for (const filepath of all) {
+    let meta;
+    try {
+      meta = getBucketMeta(filepath);
+    } catch (_e) {
+      // Unreadable bucket — surface to caller via spill, not enumeration.
+      // Caller's ATTACH attempt will throw WB-013 if needed.
+      covering.push({ filepath, min: -Infinity, max: Infinity, unsealed: true });
+      continue;
+    }
+    const min = meta.min_event_id != null ? Number(meta.min_event_id) : null;
+    const sealed = meta.sealed_at != null;
+    const max = sealed && meta.max_event_id != null
+      ? Number(meta.max_event_id)
+      : Number.MAX_SAFE_INTEGER;
+
+    if (min === null) continue; // empty bucket — nothing to cover
+
+    if (max >= gapStart && min <= gapEnd) {
+      covering.push({ filepath, min, max, unsealed: !sealed });
+    }
+  }
+  return covering;
+}

package/lib/cas.js

@@ -0,0 +1,310 @@
+/**
+ * Content-addressable store (CAS) for large event payloads.
+ *
+ * Per DESIGN-v2.md §6:
+ *   - Storage: `<dataDir>/cas/<sha[0:2]>/<sha>` (200-bit sharding)
+ *   - SHA-256 of uncompressed content; filename is full hex SHA
+ *   - O_EXCL writes; collision = no-op (immutable, content-addressed)
+ *   - Object size cap (default 256 MB) — `WB-008 PAYLOAD_TOO_LARGE` above
+ *   - Reference-tracked via `events.payload_cas_sha`; GC walks live + warm
+ *     buckets and deletes orphans past a configurable grace window
+ *   - GC aborts with `WB-010 CAS_GC_INCOMPLETE_BUCKET_SET` if any bucket is
+ *     unreadable (offline-bucket safety, round-1 council fix)
+ *
+ * The functions are synchronous because better-sqlite3 is synchronous and
+ * fs.* sync APIs are fine for the CAS sizes we target (≤256 MB by default).
+ *
+ * @module lib/cas
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { createRequire } from 'node:module';
+import { WBError } from './errors.js';
+import { archiveDir, listBuckets } from './archive.js';
+
+const require = createRequire(import.meta.url);
+
+export const DEFAULT_OBJECT_MAX_BYTES   = 256 * 1024 * 1024;
+export const DEFAULT_GC_GRACE_DAYS      = 7;
+
+/**
+ * Standard CAS root for a given dataDir.
+ * @param {string} dataDir
+ * @returns {string}
+ */
+export function casDir(dataDir) {
+  return path.join(dataDir, 'cas');
+}
+
+function shardedPathFor(dataDir, sha) {
+  return path.join(casDir(dataDir), sha.slice(0, 2), sha);
+}
+
+function sha256(buf) {
+  return createHash('sha256').update(buf).digest('hex');
+}
+
+function ensureBuffer(content) {
+  if (Buffer.isBuffer(content)) return content;
+  if (typeof content === 'string') return Buffer.from(content, 'utf8');
+  throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+    message: 'cas.put requires Buffer or string content',
+    received: typeof content,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// put / get / exists / stats
+// ---------------------------------------------------------------------------
+
+/**
+ * Store content in the CAS. Returns the SHA-256 hex digest.
+ * Writes are O_EXCL — duplicate content (same SHA) is a no-op.
+ *
+ * @param {string} dataDir
+ * @param {Buffer|string} content
+ * @param {object} [opts]
+ * @param {number} [opts.max_bytes]   override DEFAULT_OBJECT_MAX_BYTES
+ * @returns {string}                  SHA-256 hex digest
+ */
+export function put(dataDir, content, opts = {}) {
+  const buf = ensureBuffer(content);
+  const cap = opts.max_bytes ?? DEFAULT_OBJECT_MAX_BYTES;
+
+  if (buf.length > cap) {
+    throw new WBError('WB-008', 'PAYLOAD_TOO_LARGE', {
+      message: `CAS object exceeds ${cap}-byte cap`,
+      size: buf.length,
+      cap,
+    });
+  }
+
+  const sha = sha256(buf);
+  const target = shardedPathFor(dataDir, sha);
+
+  if (fs.existsSync(target)) return sha;   // immutable; collision = no-op
+
+  fs.mkdirSync(path.dirname(target), { recursive: true });
+
+  // Write to a temp file in the same directory then rename; this gives us
+  // atomic visibility and avoids partial reads during a crash.
+  const tmp = `${target}.tmp.${process.pid}.${Date.now()}`;
+  let fd;
+  try {
+    fd = fs.openSync(tmp, 'wx', 0o600);
+    fs.writeSync(fd, buf);
+    fs.fsyncSync(fd);
+  } finally {
+    if (fd !== undefined) {
+      try { fs.closeSync(fd); } catch (_e) { /* ignore */ }
+    }
+  }
+
+  try {
+    fs.renameSync(tmp, target);
+  } catch (e) {
+    // Race: another writer just moved their copy into place. Same SHA →
+    // identical content → safe to drop ours.
+    try { fs.unlinkSync(tmp); } catch (_e) { /* ignore */ }
+    if (e.code !== 'EEXIST') throw e;
+  }
+
+  return sha;
+}
+
+/**
+ * Read content for a given SHA. Returns a Buffer, or null if not found.
+ */
+export function get(dataDir, sha) {
+  const target = shardedPathFor(dataDir, sha);
+  try {
+    return fs.readFileSync(target);
+  } catch (e) {
+    if (e.code === 'ENOENT') return null;
+    throw e;
+  }
+}
+
+export function exists(dataDir, sha) {
+  return fs.existsSync(shardedPathFor(dataDir, sha));
+}
+
+/**
+ * Aggregate stats: total objects, total bytes, root path.
+ */
+export function stats(dataDir) {
+  const root = casDir(dataDir);
+  if (!fs.existsSync(root)) {
+    return { root, object_count: 0, total_bytes: 0 };
+  }
+
+  let count = 0;
+  let bytes = 0;
+  for (const file of walkCasFiles(root)) {
+    count++;
+    try { bytes += fs.statSync(file).size; }
+    catch (_e) { /* removed concurrently */ }
+  }
+  return { root, object_count: count, total_bytes: bytes };
+}
+
+function* walkCasFiles(root) {
+  for (const shard of safeReaddir(root)) {
+    const shardDir = path.join(root, shard);
+    let stat;
+    try { stat = fs.statSync(shardDir); }
+    catch (_e) { continue; }
+    if (!stat.isDirectory()) continue;
+    for (const f of safeReaddir(shardDir)) {
+      yield path.join(shardDir, f);
+    }
+  }
+}
+
+function safeReaddir(p) {
+  try { return fs.readdirSync(p); } catch (_e) { return []; }
+}
+
+// ---------------------------------------------------------------------------
+// GC — offline-bucket-safe (round-1 council fix)
+// ---------------------------------------------------------------------------
+
+/**
+ * Walk the live tier and all warm buckets to compute the live SHA set, then
+ * remove CAS entries that are NOT referenced and were last modified before
+ * `now - graceMs`. ABORTS with WB-010 if any bucket is unreadable so we
+ * never silently shrink the live set on incomplete data.
+ *
+ * @param {object} opts
+ * @param {string} opts.dataDir
+ * @param {import('better-sqlite3').Database} opts.liveDb
+ * @param {number} [opts.grace_days]
+ * @param {number} [opts.now]                              test-injectable timestamp
+ * @param {string[]} [opts.allow_missing_buckets]          basenames (or globs) the
+ *                                                         operator has agreed are
+ *                                                         intentionally absent (e.g.,
+ *                                                         exported to cold and removed)
+ * @param {boolean} [opts.dry_run=false]
+ * @returns {{
+ *   live_shas: number,
+ *   considered: number,
+ *   deleted: number,
+ *   skipped_in_grace: number,
+ *   bytes_freed: number,
+ * }}
+ */
+export function gc(opts) {
+  const dataDir   = opts.dataDir;
+  const liveDb    = opts.liveDb;
+  const graceDays = opts.grace_days ?? DEFAULT_GC_GRACE_DAYS;
+  const graceMs   = graceDays * 86400_000;
+  const now       = opts.now ?? Date.now();
+  const allow     = new Set(opts.allow_missing_buckets ?? []);
+  const dryRun    = !!opts.dry_run;
+
+  if (!dataDir) throw new Error('gc requires opts.dataDir');
+  if (!liveDb)  throw new Error('gc requires opts.liveDb');
+
+  const liveShas = collectLiveShasOrAbort(dataDir, liveDb, allow);
+
+  const root = casDir(dataDir);
+  if (!fs.existsSync(root)) {
+    return { live_shas: liveShas.size, considered: 0, deleted: 0, skipped_in_grace: 0, bytes_freed: 0 };
+  }
+
+  let considered = 0;
+  let deleted = 0;
+  let skippedInGrace = 0;
+  let bytesFreed = 0;
+
+  for (const file of walkCasFiles(root)) {
+    considered++;
+    const sha = path.basename(file);
+    if (liveShas.has(sha)) continue;
+
+    let stat;
+    try { stat = fs.statSync(file); }
+    catch (_e) { continue; }                         // removed concurrently
+
+    if (now - stat.mtimeMs < graceMs) {
+      skippedInGrace++;
+      continue;
+    }
+
+    if (!dryRun) {
+      try {
+        fs.unlinkSync(file);
+        deleted++;
+        bytesFreed += stat.size;
+      } catch (_e) { /* concurrent delete or permission — skip */ }
+    } else {
+      deleted++;
+      bytesFreed += stat.size;
+    }
+  }
+
+  return {
+    live_shas: liveShas.size,
+    considered,
+    deleted,
+    skipped_in_grace: skippedInGrace,
+    bytes_freed: bytesFreed,
+  };
+}
+
+/**
+ * Build the set of payload_cas_sha values referenced by ANY event in live or
+ * any warm bucket. Aborts with WB-010 if any bucket is unreadable.
+ */
+function collectLiveShasOrAbort(dataDir, liveDb, allowSet) {
+  const shas = new Set();
+
+  // Live tier
+  for (const row of liveDb.prepare(
+    'SELECT payload_cas_sha FROM events WHERE payload_cas_sha IS NOT NULL'
+  ).all()) {
+    if (row.payload_cas_sha) shas.add(row.payload_cas_sha);
+  }
+
+  // Warm tier — every expected bucket must be readable
+  const archDir = archiveDir(dataDir);
+  for (const filepath of listBuckets(archDir)) {
+    const basename = path.basename(filepath);
+    if (allowSet.has(basename)) continue;            // operator-acknowledged absence
+
+    try { fs.accessSync(filepath, fs.constants.R_OK); }
+    catch (e) {
+      throw new WBError('WB-010', 'CAS_GC_INCOMPLETE_BUCKET_SET', {
+        message: `CAS GC requires every warm bucket to be readable; ${basename} is not`,
+        bucket: filepath,
+        cause: e.code,
+      });
+    }
+
+    let bucketDb;
+    try {
+      const Database = require('better-sqlite3');
+      bucketDb = new Database(filepath, { readonly: true });
+    } catch (e) {
+      throw new WBError('WB-010', 'CAS_GC_INCOMPLETE_BUCKET_SET', {
+        message: `failed to open warm bucket: ${basename}`,
+        bucket: filepath,
+        cause: e.message,
+      });
+    }
+
+    try {
+      for (const row of bucketDb.prepare(
+        'SELECT payload_cas_sha FROM events WHERE payload_cas_sha IS NOT NULL'
+      ).all()) {
+        if (row.payload_cas_sha) shas.add(row.payload_cas_sha);
+      }
+    } finally {
+      bucketDb.close();
+    }
+  }
+
+  return shas;
+}

package/lib/causality.js

@@ -0,0 +1,129 @@
+/**
+ * Cross-process causality propagation — DESIGN-v2.md §8.
+ *
+ * In-process: callers wrap a body in `withContext({ correlation_id, ... }, fn)`;
+ * any `emit()` calls inside the body inherit the context, and successive
+ * emits chain via parent_event_id.
+ *
+ * Cross-process: spawned subprocesses inherit four env vars and the same
+ * `withContext` runs at the entry point of the child to re-enter the trace.
+ *   - WICKED_BUS_CORRELATION_ID
+ *   - WICKED_BUS_SESSION_ID
+ *   - WICKED_BUS_PARENT_EVENT_ID
+ *   - WICKED_BUS_PRODUCER_ID
+ *
+ * The `currentContext()` accessor returns the active context, drawing from
+ * the AsyncLocalStorage frame if one exists, otherwise from the env vars.
+ *
+ * @module lib/causality
+ */
+
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+const ENV_KEYS = Object.freeze({
+  correlation_id:   'WICKED_BUS_CORRELATION_ID',
+  session_id:       'WICKED_BUS_SESSION_ID',
+  parent_event_id:  'WICKED_BUS_PARENT_EVENT_ID',
+  producer_id:      'WICKED_BUS_PRODUCER_ID',
+});
+
+const als = new AsyncLocalStorage();
+
+/**
+ * Run `fn` with the given context fields attached. Nested withContext()
+ * calls inherit-and-override (last write wins per field).
+ *
+ * @param {object} ctx
+ * @param {string} [ctx.correlation_id]
+ * @param {string} [ctx.session_id]
+ * @param {number} [ctx.parent_event_id]
+ * @param {string} [ctx.producer_id]
+ * @param {Function} fn  - sync or async; awaited if it returns a Promise
+ * @returns {*}
+ */
+export function withContext(ctx, fn) {
+  const inherited = currentContext();
+  const merged = { ...inherited, ...sanitize(ctx) };
+  return als.run(merged, fn);
+}
+
+/**
+ * Read the active context. Returns an object with the four fields (any of
+ * which may be `null`/`undefined`). The result is a snapshot copy — mutating
+ * it does not affect the running context.
+ */
+export function currentContext() {
+  const frame = als.getStore();
+  if (frame) return { ...frame };
+
+  // Fallback: read env vars. Useful at the entry point of a spawned child
+  // before any explicit withContext() wrap.
+  const fromEnv = {
+    correlation_id:  process.env[ENV_KEYS.correlation_id]  || null,
+    session_id:      process.env[ENV_KEYS.session_id]      || null,
+    parent_event_id: parseIntOrNull(process.env[ENV_KEYS.parent_event_id]),
+    producer_id:     process.env[ENV_KEYS.producer_id]     || null,
+  };
+  // Return undefined-y context only when truly nothing is set — keeps the
+  // happy path's "no causality at all" comparison cheap.
+  if (
+    !fromEnv.correlation_id && !fromEnv.session_id &&
+    !fromEnv.parent_event_id && !fromEnv.producer_id
+  ) return {};
+  return fromEnv;
+}
+
+/**
+ * Set the parent_event_id for subsequent emits inside the active context.
+ * Used by emit() so the next event in the same withContext() block chains
+ * to the most recently emitted event_id.
+ *
+ * @param {number} eventId
+ */
+export function recordEmit(eventId) {
+  const frame = als.getStore();
+  if (frame) frame.parent_event_id = eventId;
+}
+
+/**
+ * Build an env object suitable for passing to spawn()/exec() so the child
+ * inherits the active causality context. Caller usually does:
+ *
+ *     spawn(cmd, args, { env: { ...process.env, ...causalityEnv() } });
+ *
+ * If no context is active, returns {} so the caller's spawn is unaffected.
+ */
+export function causalityEnv() {
+  const ctx = currentContext();
+  const out = {};
+  if (ctx.correlation_id)  out[ENV_KEYS.correlation_id]  = String(ctx.correlation_id);
+  if (ctx.session_id)      out[ENV_KEYS.session_id]      = String(ctx.session_id);
+  if (ctx.parent_event_id != null) {
+    out[ENV_KEYS.parent_event_id] = String(ctx.parent_event_id);
+  }
+  if (ctx.producer_id)     out[ENV_KEYS.producer_id]     = String(ctx.producer_id);
+  return out;
+}
+
+/** Constant export for callers wanting to manipulate env vars directly. */
+export const CAUSALITY_ENV_KEYS = ENV_KEYS;
+
+// ---------------------------------------------------------------------------
+
+function sanitize(obj) {
+  const out = {};
+  if (obj.correlation_id)  out.correlation_id  = String(obj.correlation_id);
+  if (obj.session_id)      out.session_id      = String(obj.session_id);
+  if (obj.parent_event_id != null) {
+    const n = Number(obj.parent_event_id);
+    if (Number.isInteger(n) && n > 0) out.parent_event_id = n;
+  }
+  if (obj.producer_id)     out.producer_id     = String(obj.producer_id);
+  return out;
+}
+
+function parseIntOrNull(v) {
+  if (v == null) return null;
+  const n = parseInt(v, 10);
+  return Number.isInteger(n) ? n : null;
+}