wicked-bus 1.1.1 → 2.0.0

package/commands/cli.js

@@ -33,7 +33,7 @@ function printUsage() {
     commands: [
       'init', 'emit', 'subscribe', 'status', 'replay',
       'cleanup', 'register', 'deregister', 'list', 'ack',
-      'dlq',
+      'dlq', 'daemon', 'ui',
     ],
     global_flags: ['--db-path <path>', '--json', '--log-level <level>'],
   };
@@ -128,6 +128,16 @@ async function main() {
         await cmdDlq(args, globals, args._positional || []);
         break;
       }
+      case 'daemon': {
+        const { cmdDaemon } = await import('./cmd-daemon.js');
+        await cmdDaemon(args, globals, args._positional || []);
+        break;
+      }
+      case 'ui': {
+        const { cmdUi } = await import('./cmd-ui.js');
+        await cmdUi(args, globals);
+        break;
+      }
       default:
         printUsage();
         process.exit(command ? 1 : 0);

package/commands/cmd-daemon.js

@@ -0,0 +1,238 @@
+/**
+ * `wicked-bus daemon <subcommand>` — start, stop, status.
+ *
+ * - `start [--detached]`   spawn the daemon in this process or detach a child
+ * - `stop`                 send SIGTERM to the running daemon (clean shutdown)
+ * - `status`               JSON snapshot: socket path, pid, subscribers
+ *
+ * Lifecycle uses lib/daemon-singleton.js for the PID lock and
+ * lib/daemon.js for the socket server. Subscribers connect via the
+ * library / wrapper APIs; this CLI is operator-facing only.
+ *
+ * @module commands/cmd-daemon
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { resolveDataDir, ensureDataDir } from '../lib/paths.js';
+import {
+  startDaemon,
+  socketPath,
+} from '../lib/daemon.js';
+import {
+  acquireDaemonLock,
+} from '../lib/daemon-singleton.js';
+import { probeDaemon } from '../lib/daemon-client.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname  = path.dirname(__filename);
+
+export async function cmdDaemon(args, globals, positional) {
+  const sub = (positional && positional[0]) || null;
+  if (!sub) {
+    return emitJson({
+      error: 'usage',
+      message: 'daemon requires a subcommand: start | stop | status',
+    });
+  }
+
+  switch (sub) {
+    case 'start':  return cmdDaemonStart(args, globals);
+    case 'stop':   return cmdDaemonStop(args, globals);
+    case 'status': return cmdDaemonStatus(args, globals);
+    default:
+      return emitJson({
+        error: 'usage',
+        message: `unknown daemon subcommand: ${sub}`,
+      });
+  }
+}
+
+// ---------------------------------------------------------------------------
+// start
+// ---------------------------------------------------------------------------
+
+async function cmdDaemonStart(args /*, globals */) {
+  const dataDir = resolveDataDir();
+  ensureDataDir();
+
+  // Detached: fork ourselves with `--no-detach` so the child runs the
+  // foreground branch. Parent waits for the socket to come up, prints
+  // status, exits 0. Stderr is captured to a log file in dataDir so the
+  // child's startup errors are diagnosable even when stdio is detached.
+  if (args.detached !== false && (args.detached === true || args['detach'] === true)) {
+    const logPath = path.join(dataDir, 'daemon.log');
+    const logFd = fs.openSync(logPath, 'a');
+    const child = spawn(
+      process.execPath,
+      [path.resolve(__dirname, 'cli.js'), 'daemon', 'start', '--no-detach'],
+      {
+        detached: true,
+        stdio: ['ignore', logFd, logFd],
+        env: process.env,
+        cwd: dataDir,
+      },
+    );
+    fs.closeSync(logFd);
+    child.unref();
+
+    const ok = await waitForSocket(dataDir, 5000);
+    if (!ok) {
+      let logTail = '';
+      try { logTail = fs.readFileSync(logPath, 'utf8').slice(-2000); }
+      catch (_e) { /* ignore */ }
+      return emitJsonExit({
+        error: 'daemon-start-timeout',
+        message: 'spawned daemon did not start listening within 5s',
+        socket: socketPath(dataDir),
+        log_tail: logTail,
+      }, 1);
+    }
+    return emitJson({
+      ok: true,
+      mode: 'detached',
+      pid: child.pid,
+      socket: socketPath(dataDir),
+    });
+  }
+
+  // Foreground: hold the lock, run the daemon, hand off to a SIGTERM/SIGINT
+  // handler for clean shutdown. This branch is what `--detached` re-enters.
+  let lock;
+  try {
+    lock = acquireDaemonLock(dataDir);
+  } catch (e) {
+    if (e.code === 'EALREADY_RUNNING') {
+      return emitJsonExit({
+        error: 'EALREADY_RUNNING',
+        message: e.message,
+        prior_pid: e.priorPid,
+        socket: socketPath(dataDir),
+      }, 1);
+    }
+    throw e;
+  }
+
+  const daemon = await startDaemon({
+    dataDir,
+    inline_payload_max_bytes: numFlag(args, 'inline-payload-max-bytes'),
+    subscriber_queue_max:     numFlag(args, 'subscriber-queue-max'),
+  });
+
+  const shutdown = async () => {
+    try { await daemon.stop(); } catch (_e) { /* ignore */ }
+    try { lock.release(); } catch (_e) { /* ignore */ }
+    process.exit(0);
+  };
+  process.on('SIGTERM', shutdown);
+  process.on('SIGINT',  shutdown);
+
+  emitJson({
+    ok: true,
+    mode: 'foreground',
+    pid: process.pid,
+    socket: daemon.socketPath,
+  });
+
+  // Stay alive until a signal fires shutdown(). The Node event loop keeps
+  // the process running because the server holds an active handle.
+}
+
+// ---------------------------------------------------------------------------
+// stop
+// ---------------------------------------------------------------------------
+
+async function cmdDaemonStop(/* args, globals */) {
+  const dataDir = resolveDataDir();
+  const lockPath = path.join(dataDir, 'daemon.lock');
+
+  let pid;
+  try {
+    pid = parseInt(fs.readFileSync(lockPath, 'utf8'), 10) || null;
+  } catch (_e) {
+    pid = null;
+  }
+
+  if (!pid) {
+    return emitJson({ ok: false, reason: 'no-lock-file', socket: socketPath(dataDir) });
+  }
+
+  try {
+    process.kill(pid, 'SIGTERM');
+  } catch (e) {
+    return emitJsonExit({
+      error: 'kill-failed',
+      pid,
+      message: e.message,
+    }, 1);
+  }
+
+  // Best-effort wait for the daemon to release the lock.
+  const released = await waitForLockRelease(lockPath, 5000);
+  return emitJson({
+    ok: true,
+    pid,
+    released,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// status
+// ---------------------------------------------------------------------------
+
+async function cmdDaemonStatus(/* args, globals */) {
+  const dataDir = resolveDataDir();
+  const lockPath = path.join(dataDir, 'daemon.lock');
+  const sockPath = socketPath(dataDir);
+
+  let pid = null;
+  try { pid = parseInt(fs.readFileSync(lockPath, 'utf8'), 10) || null; }
+  catch (_e) { /* no lock */ }
+
+  const reachable = await probeDaemon(dataDir, 200);
+
+  return emitJson({
+    socket: sockPath,
+    lock_path: lockPath,
+    pid,
+    reachable,
+  });
+}
+
+// ---------------------------------------------------------------------------
+
+function emitJson(obj) {
+  process.stdout.write(JSON.stringify(obj) + '\n');
+}
+
+function emitJsonExit(obj, code) {
+  process.stderr.write(JSON.stringify(obj) + '\n');
+  process.exit(code);
+}
+
+function numFlag(args, name) {
+  const v = args[name];
+  if (v == null || v === true) return undefined;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : undefined;
+}
+
+async function waitForSocket(dataDir, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await probeDaemon(dataDir, 100)) return true;
+    await new Promise(r => setTimeout(r, 50));
+  }
+  return false;
+}
+
+async function waitForLockRelease(lockPath, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (!fs.existsSync(lockPath)) return true;
+    await new Promise(r => setTimeout(r, 50));
+  }
+  return false;
+}

package/commands/cmd-ui.js

@@ -0,0 +1,156 @@
+/**
+ * `wicked-bus ui` — operator-facing CLI for the read-only UI server.
+ *
+ *   wicked-bus ui                 start in foreground, JSON to stdout
+ *   wicked-bus ui --detached      fork a child, print connection info, exit 0
+ *   wicked-bus ui --rotate-token  regenerate the bearer token
+ *   wicked-bus ui --host 0.0.0.0  bind non-loopback (warning printed)
+ *   wicked-bus ui --port 7842     bind a specific port (default 0 = ephemeral
+ *                                 in foreground; default 7842 in detached
+ *                                 mode so a known port is reachable)
+ *
+ * Foreground sets up SIGTERM/SIGINT handlers for clean shutdown. Detached
+ * redirects child stdio to `<dataDir>/ui.log` so startup failures are
+ * diagnosable (same lesson as the daemon CLI: stdio:'ignore' on a long-
+ * running spawn hides everything).
+ *
+ * @module commands/cmd-ui
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { resolveDataDir, ensureDataDir } from '../lib/paths.js';
+import { openDb } from '../lib/db.js';
+import { startUiServer, DEFAULT_HOST, DEFAULT_PORT } from '../lib/ui-server.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname  = path.dirname(__filename);
+
+export async function cmdUi(args /*, globals */) {
+  const dataDir = resolveDataDir();
+  ensureDataDir();
+
+  // --no-detach is the inner branch; --detached is the outer fork.
+  if (args.detached === true || args['detach'] === true) {
+    return startDetached(args, dataDir);
+  }
+
+  return startForeground(args, dataDir);
+}
+
+// ---------------------------------------------------------------------------
+
+async function startDetached(args, dataDir) {
+  const logPath = path.join(dataDir, 'ui.log');
+  const logFd = fs.openSync(logPath, 'a');
+
+  // Re-invoke ourselves with --no-detach + a known port so the child can
+  // start cleanly and the parent can probe.
+  const port = args.port ? String(args.port) : String(DEFAULT_PORT);
+  const host = args.host ? String(args.host) : DEFAULT_HOST;
+
+  const childArgs = [
+    path.resolve(__dirname, 'cli.js'),
+    'ui', '--no-detach',
+    '--port', port,
+    '--host', host,
+  ];
+  if (args['rotate-token'] === true || args.rotate_token === true) {
+    childArgs.push('--rotate-token');
+  }
+
+  const child = spawn(process.execPath, childArgs, {
+    detached: true,
+    stdio: ['ignore', logFd, logFd],
+    env: process.env,
+    cwd: dataDir,
+  });
+  fs.closeSync(logFd);
+  child.unref();
+
+  const reachable = await waitForUi(host, port, 5000);
+  if (!reachable) {
+    let logTail = '';
+    try { logTail = fs.readFileSync(logPath, 'utf8').slice(-2000); }
+    catch (_e) { /* ignore */ }
+    return emitJsonExit({
+      error: 'ui-start-timeout',
+      message: 'spawned UI did not start listening within 5s',
+      log_tail: logTail,
+    }, 1);
+  }
+
+  return emitJson({
+    ok: true,
+    mode: 'detached',
+    pid: child.pid,
+    host,
+    port: Number(port),
+  });
+}
+
+// ---------------------------------------------------------------------------
+
+async function startForeground(args, dataDir) {
+  const liveDb = openDb();
+  const ui = await startUiServer({
+    dataDir,
+    liveDb,
+    host: args.host ?? DEFAULT_HOST,
+    port: args.port != null ? Number(args.port) : DEFAULT_PORT,
+    rotate_token: args['rotate-token'] === true || args.rotate_token === true,
+  });
+
+  const shutdown = async () => {
+    try { await ui.stop(); } catch (_e) { /* ignore */ }
+    try { liveDb.close(); } catch (_e) { /* ignore */ }
+    process.exit(0);
+  };
+  process.on('SIGTERM', shutdown);
+  process.on('SIGINT',  shutdown);
+
+  emitJson({
+    ok: true,
+    mode: 'foreground',
+    pid: process.pid,
+    host: ui.host,
+    port: ui.port,
+    token_path: ui.token_path,
+  });
+}
+
+// ---------------------------------------------------------------------------
+
+async function waitForUi(host, port, timeoutMs) {
+  const http = await import('node:http');
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await ping(http.default, host, port)) return true;
+    await new Promise(r => setTimeout(r, 50));
+  }
+  return false;
+}
+
+function ping(http, host, port) {
+  return new Promise((resolve) => {
+    const req = http.request({ host, port, path: '/healthz', method: 'GET', timeout: 200 }, (res) => {
+      // Drain so the connection releases.
+      res.resume();
+      resolve(res.statusCode === 200);
+    });
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => { req.destroy(); resolve(false); });
+    req.end();
+  });
+}
+
+function emitJson(obj) {
+  process.stdout.write(JSON.stringify(obj) + '\n');
+}
+
+function emitJsonExit(obj, code) {
+  process.stderr.write(JSON.stringify(obj) + '\n');
+  process.exit(code);
+}

package/lib/archive.js

@@ -0,0 +1,198 @@
+/**
+ * Archive bucket lifecycle for the v2 tiered-storage warm tier.
+ *
+ * Each warm bucket is an independent SQLite file at
+ * `<data_dir>/archive/bus-YYYY-MM[suffix].db`. Files are independent —
+ * corrupting one bucket affects only the events in that month/split.
+ *
+ * Each bucket carries a `_meta` table (NOT a fictitious PRAGMA) holding
+ * `min_event_id`, `max_event_id`, `created_at`, and `sealed_at`. The query
+ * resolver reads these via normal SELECT to decide which buckets to ATTACH
+ * during a warm-spill (see lib/query.js and DESIGN-v2.md §5.4).
+ *
+ * Buckets without a `sealed_at` value are still being filled by sweep and
+ * are conservatively treated as covering up to MAX_SAFE_INTEGER until sealed.
+ *
+ * @module lib/archive
+ */
+
+import path from 'node:path';
+import fs from 'node:fs';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+const BUCKET_FILE_RE = /^bus-\d{4}-\d{2}[a-z]?\.db$/;
+
+/**
+ * Build the absolute path to the archive directory for a given data_dir.
+ * @param {string} dataDir
+ * @returns {string}
+ */
+export function archiveDir(dataDir) {
+  return path.join(dataDir, 'archive');
+}
+
+/**
+ * Ensure the archive directory exists (idempotent).
+ * @param {string} archDir
+ */
+export function ensureArchiveDir(archDir) {
+  fs.mkdirSync(archDir, { recursive: true });
+}
+
+/**
+ * Create a new archive bucket file with the v2 `events` schema mirrored
+ * from the live tier, plus the `_meta` table. Returns the open Database
+ * handle. Caller is responsible for closing it.
+ *
+ * @param {string} filepath - absolute path to the bucket .db file
+ * @param {{ minEventId?: number, maxEventId?: number }} [meta]
+ * @returns {import('better-sqlite3').Database}
+ */
+export function createBucket(filepath, meta = {}) {
+  ensureArchiveDir(path.dirname(filepath));
+  const Database = require('better-sqlite3');
+  const db = new Database(filepath);
+
+  db.pragma('journal_mode = WAL');
+  db.pragma('synchronous = NORMAL');
+  db.pragma('busy_timeout = 5000');
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS events (
+      event_id                 INTEGER PRIMARY KEY,
+      event_type               TEXT    NOT NULL,
+      domain                   TEXT    NOT NULL,
+      subdomain                TEXT    NOT NULL DEFAULT '',
+      payload                  TEXT    NOT NULL,
+      schema_version           TEXT    NOT NULL DEFAULT '1.0.0',
+      idempotency_key          TEXT    NOT NULL,
+      emitted_at               INTEGER NOT NULL,
+      expires_at               INTEGER NOT NULL,
+      dedup_expires_at         INTEGER NOT NULL,
+      metadata                 TEXT,
+      parent_event_id          INTEGER,
+      session_id               TEXT,
+      correlation_id           TEXT,
+      producer_id              TEXT,
+      origin_node_id           TEXT,
+      registry_schema_version  INTEGER,
+      payload_cas_sha          TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_events_event_type      ON events(event_type);
+    CREATE INDEX IF NOT EXISTS idx_events_domain          ON events(domain);
+    CREATE INDEX IF NOT EXISTS idx_events_correlation_id  ON events(correlation_id);
+
+    CREATE TABLE IF NOT EXISTS _meta (
+      k TEXT PRIMARY KEY,
+      v TEXT NOT NULL
+    );
+  `);
+
+  setMeta(db, 'created_at', String(Date.now()));
+  if (meta.minEventId != null) setMeta(db, 'min_event_id', String(meta.minEventId));
+  if (meta.maxEventId != null) setMeta(db, 'max_event_id', String(meta.maxEventId));
+
+  return db;
+}
+
+/**
+ * Read the `_meta` key/value pairs from a bucket file.
+ * Opens read-only so a sealed bucket can be inspected without taking a write lock.
+ *
+ * @param {string} filepath
+ * @returns {Record<string,string>}
+ */
+export function getBucketMeta(filepath) {
+  const Database = require('better-sqlite3');
+  const db = new Database(filepath, { readonly: true });
+  try {
+    const rows = db.prepare('SELECT k, v FROM _meta').all();
+    const meta = {};
+    for (const r of rows) meta[r.k] = r.v;
+    return meta;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Mark a bucket sealed: write `sealed_at` and (optionally) finalize `max_event_id`.
+ * Once sealed, a bucket's covered range is fixed and the resolver can use
+ * the [min, max] interval to skip it when out of range.
+ *
+ * @param {string} filepath
+ * @param {{ maxEventId?: number }} [opts]
+ */
+export function sealBucket(filepath, opts = {}) {
+  const Database = require('better-sqlite3');
+  const db = new Database(filepath);
+  try {
+    setMeta(db, 'sealed_at', String(Date.now()));
+    if (opts.maxEventId != null) {
+      setMeta(db, 'max_event_id', String(opts.maxEventId));
+    }
+  } finally {
+    db.close();
+  }
+}
+
+function setMeta(db, k, v) {
+  db.prepare('INSERT OR REPLACE INTO _meta(k, v) VALUES (?, ?)').run(k, v);
+}
+
+/**
+ * List all bucket files in the archive directory, sorted lexicographically.
+ * Suffix letters (`a`, `b`, …) preserve creation order within a month after
+ * auto-split, so lexical sort matches creation order.
+ *
+ * @param {string} archDir
+ * @returns {string[]} absolute paths
+ */
+export function listBuckets(archDir) {
+  if (!fs.existsSync(archDir)) return [];
+  return fs.readdirSync(archDir)
+    .filter(f => BUCKET_FILE_RE.test(f))
+    .sort()
+    .map(f => path.join(archDir, f));
+}
+
+/**
+ * Find buckets that cover (any part of) the event_id range [gapStart, gapEnd].
+ * Returns descriptors with absolute path + parsed range + unsealed flag.
+ * Unsealed buckets are conservatively included whenever their `min_event_id`
+ * is at or below `gapEnd`.
+ *
+ * @param {string} archDir
+ * @param {number} gapStart - inclusive
+ * @param {number} gapEnd   - inclusive
+ * @returns {Array<{ filepath: string, min: number, max: number, unsealed: boolean }>}
+ */
+export function bucketsCoveringRange(archDir, gapStart, gapEnd) {
+  const all = listBuckets(archDir);
+  const covering = [];
+  for (const filepath of all) {
+    let meta;
+    try {
+      meta = getBucketMeta(filepath);
+    } catch (_e) {
+      // Unreadable bucket — surface to caller via spill, not enumeration.
+      // Caller's ATTACH attempt will throw WB-013 if needed.
+      covering.push({ filepath, min: -Infinity, max: Infinity, unsealed: true });
+      continue;
+    }
+    const min = meta.min_event_id != null ? Number(meta.min_event_id) : null;
+    const sealed = meta.sealed_at != null;
+    const max = sealed && meta.max_event_id != null
+      ? Number(meta.max_event_id)
+      : Number.MAX_SAFE_INTEGER;
+
+    if (min === null) continue; // empty bucket — nothing to cover
+
+    if (max >= gapStart && min <= gapEnd) {
+      covering.push({ filepath, min, max, unsealed: !sealed });
+    }
+  }
+  return covering;
+}