wicked-bus 1.0.0

package/lib/poll.js

@@ -0,0 +1,212 @@
+/**
+ * Event polling and acknowledgment.
+ * @module lib/poll
+ */
+
+import { WBError } from './errors.js';
+
+/**
+ * Match an event against a filter string.
+ * @param {string} eventType - The event_type to test
+ * @param {string} domain - The domain of the event
+ * @param {string} filterStr - Filter pattern, e.g. 'wicked.test.run.*@wicked-testing'
+ * @returns {boolean}
+ */
+export function matchesFilter(eventType, domain, filterStr) {
+  let typePattern, domainFilter;
+  const atIdx = filterStr.indexOf('@');
+  if (atIdx !== -1) {
+    typePattern = filterStr.slice(0, atIdx);
+    domainFilter = filterStr.slice(atIdx + 1);
+  } else {
+    typePattern = filterStr;
+    domainFilter = null;
+  }
+
+  // Domain check
+  if (domainFilter && domain !== domainFilter) return false;
+
+  // Catch-all (*@domain)
+  if (typePattern === '*') return true;
+
+  // Exact match
+  if (typePattern === eventType) return true;
+
+  // Single-level wildcard (prefix.*)
+  if (typePattern.endsWith('.*')) {
+    const prefix = typePattern.slice(0, -2);
+    if (eventType.startsWith(prefix + '.')) {
+      const remainder = eventType.slice(prefix.length + 1);
+      return !remainder.includes('.'); // single-level only
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Build SQL WHERE clauses from a filter string for optimized queries.
+ * @param {string} filterStr
+ * @returns {{ where: string, params: object }}
+ */
+function buildFilterSql(filterStr) {
+  let typePattern, domainFilter;
+  const atIdx = filterStr.indexOf('@');
+  if (atIdx !== -1) {
+    typePattern = filterStr.slice(0, atIdx);
+    domainFilter = filterStr.slice(atIdx + 1);
+  } else {
+    typePattern = filterStr;
+    domainFilter = null;
+  }
+
+  const conditions = [];
+  const params = {};
+
+  // Domain filter
+  if (domainFilter) {
+    conditions.push('domain = :domain_filter');
+    params.domain_filter = domainFilter;
+  }
+
+  // Type filter
+  if (typePattern === '*') {
+    // Catch-all: no type filter
+  } else if (typePattern.endsWith('.*')) {
+    const prefix = typePattern.slice(0, -2);
+    conditions.push("event_type LIKE :prefix_like");
+    conditions.push("event_type NOT LIKE :prefix_multi");
+    params.prefix_like = prefix + '.%';
+    params.prefix_multi = prefix + '.%.%';
+  } else {
+    // Exact match
+    conditions.push('event_type = :exact_type');
+    params.exact_type = typePattern;
+  }
+
+  return {
+    where: conditions.length > 0 ? conditions.join(' AND ') : '1=1',
+    params,
+  };
+}
+
+/**
+ * Poll for events from the given cursor position.
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} cursorId
+ * @param {object} [options]
+ * @param {number} [options.batchSize=100]
+ * @returns {object[]} Array of event rows
+ */
+export function poll(db, cursorId, options = {}) {
+  const batchSize = options.batchSize || 100;
+
+  // Load cursor
+  const cursor = db.prepare(
+    'SELECT * FROM cursors WHERE cursor_id = ?'
+  ).get(cursorId);
+
+  if (!cursor || cursor.deregistered_at != null) {
+    throw new WBError('WB-006', 'CURSOR_NOT_FOUND', {
+      message: 'Cursor not found or deregistered',
+      cursor_id: cursorId,
+      reason: 'cursor not found or deregistered',
+    });
+  }
+
+  // WB-003 check: cursor behind oldest available row
+  const oldest = db.prepare('SELECT MIN(event_id) as min_id FROM events').get();
+  if (oldest && oldest.min_id != null) {
+    if (cursor.last_event_id < oldest.min_id - 1) {
+      throw new WBError('WB-003', 'CURSOR_BEHIND_TTL_WINDOW', {
+        message: 'Cursor is behind the TTL window; events have been swept',
+        cursor_last_event_id: cursor.last_event_id,
+        oldest_available_event_id: oldest.min_id,
+      });
+    }
+  }
+
+  // Load subscription for filter
+  const sub = db.prepare(
+    'SELECT * FROM subscriptions WHERE subscription_id = ?'
+  ).get(cursor.subscription_id);
+
+  if (!sub) {
+    throw new WBError('WB-006', 'CURSOR_NOT_FOUND', {
+      message: 'Subscription not found for cursor',
+      cursor_id: cursorId,
+      reason: 'subscription not found',
+    });
+  }
+
+  const filter = sub.event_type_filter;
+  const { where, params } = buildFilterSql(filter);
+  const now = Date.now();
+
+  const sql = `
+    SELECT * FROM events
+    WHERE event_id > :last_event_id
+      AND expires_at > :now
+      AND ${where}
+    ORDER BY event_id ASC
+    LIMIT :batch_size
+  `;
+
+  const allParams = {
+    ...params,
+    last_event_id: cursor.last_event_id,
+    now,
+    batch_size: batchSize,
+  };
+
+  return db.prepare(sql).all(allParams);
+}
+
+/**
+ * Acknowledge events up to the given event_id for a cursor.
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} cursorId
+ * @param {number} lastEventId
+ * @returns {{ acked: boolean, cursor_id: string, last_event_id: number }}
+ */
+export function ack(db, cursorId, lastEventId) {
+  const now = Date.now();
+
+  // Check cursor exists and is active
+  const cursor = db.prepare(
+    'SELECT * FROM cursors WHERE cursor_id = ?'
+  ).get(cursorId);
+
+  if (!cursor || cursor.deregistered_at != null) {
+    throw new WBError('WB-006', 'CURSOR_NOT_FOUND', {
+      message: 'Cursor not found or deregistered',
+      cursor_id: cursorId,
+      reason: 'cursor not found or deregistered',
+    });
+  }
+
+  const update = db.prepare(`
+    UPDATE cursors
+    SET last_event_id = ?, acked_at = ?
+    WHERE cursor_id = ? AND deregistered_at IS NULL
+  `);
+
+  const txn = db.transaction(() => {
+    const result = update.run(lastEventId, now, cursorId);
+    if (result.changes === 0) {
+      throw new WBError('WB-006', 'CURSOR_NOT_FOUND', {
+        message: 'Cursor not found or deregistered',
+        cursor_id: cursorId,
+        reason: 'cursor not found or deregistered',
+      });
+    }
+  });
+
+  txn();
+
+  return {
+    acked: true,
+    cursor_id: cursorId,
+    last_event_id: lastEventId,
+  };
+}

package/lib/register.js

@@ -0,0 +1,164 @@
+/**
+ * Registration and deregistration of providers/subscribers.
+ * @module lib/register
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+import { join } from 'node:path';
+import { mkdirSync, writeFileSync, unlinkSync } from 'node:fs';
+import { resolveDataDir } from './paths.js';
+import { WBError } from './errors.js';
+
+/**
+ * Register a provider or subscriber.
+ * @param {import('better-sqlite3').Database} db
+ * @param {object} opts
+ * @param {string} opts.plugin - Plugin name
+ * @param {'provider'|'subscriber'} opts.role
+ * @param {string} opts.filter - Event type filter (for subscribers) or comma-separated event types (for providers)
+ * @param {string} [opts.schema_version] - Schema version (providers)
+ * @param {'oldest'|'latest'} [opts.cursor_init='latest'] - Cursor initialization (subscribers)
+ * @returns {object}
+ */
+export function register(db, opts) {
+  const subscriptionId = uuidv4();
+  const now = Date.now();
+
+  const insertSub = db.prepare(`
+    INSERT INTO subscriptions (
+      subscription_id, plugin, role, event_type_filter,
+      schema_version, registered_at
+    ) VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  insertSub.run(
+    subscriptionId,
+    opts.plugin,
+    opts.role,
+    opts.filter,
+    opts.schema_version || null,
+    now
+  );
+
+  const result = {
+    subscription_id: subscriptionId,
+    plugin: opts.plugin,
+    role: opts.role,
+    registered_at: now,
+  };
+
+  if (opts.role === 'provider') {
+    // Write sidecar JSON
+    writeSidecar(opts.plugin, {
+      subscription_id: subscriptionId,
+      plugin: opts.plugin,
+      role: 'provider',
+      event_types: opts.filter.split(',').map(s => s.trim()),
+      schema_version: opts.schema_version || null,
+      registered_at: new Date(now).toISOString(),
+      registered_at_ms: now,
+    });
+    result.filter = opts.filter;
+  }
+
+  if (opts.role === 'subscriber') {
+    // Create cursor
+    const cursorId = uuidv4();
+    let lastEventId = 0;
+
+    if (opts.cursor_init === 'latest') {
+      const row = db.prepare('SELECT MAX(event_id) as max_id FROM events').get();
+      lastEventId = row && row.max_id != null ? row.max_id : 0;
+    }
+
+    db.prepare(`
+      INSERT INTO cursors (
+        cursor_id, subscription_id, last_event_id, created_at
+      ) VALUES (?, ?, ?, ?)
+    `).run(cursorId, subscriptionId, lastEventId, now);
+
+    result.cursor_id = cursorId;
+    result.filter = opts.filter;
+    result.cursor_init = opts.cursor_init || 'latest';
+    result.last_event_id = lastEventId;
+  }
+
+  return result;
+}
+
+/**
+ * Deregister a subscription (soft delete).
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} subscriptionId
+ * @returns {object}
+ */
+export function deregister(db, subscriptionId) {
+  const now = Date.now();
+
+  const sub = db.prepare(
+    'SELECT * FROM subscriptions WHERE subscription_id = ?'
+  ).get(subscriptionId);
+
+  if (!sub) {
+    throw new WBError('WB-006', 'CURSOR_NOT_FOUND', {
+      message: `Subscription not found: ${subscriptionId}`,
+      subscription_id: subscriptionId,
+      reason: 'subscription not found',
+    });
+  }
+
+  const txn = db.transaction(() => {
+    // Soft-delete the subscription
+    db.prepare(
+      'UPDATE subscriptions SET deregistered_at = ? WHERE subscription_id = ?'
+    ).run(now, subscriptionId);
+
+    // Soft-delete associated cursors (for subscribers)
+    if (sub.role === 'subscriber') {
+      db.prepare(
+        'UPDATE cursors SET deregistered_at = ? WHERE subscription_id = ? AND deregistered_at IS NULL'
+      ).run(now, subscriptionId);
+    }
+
+    // Remove provider sidecar
+    if (sub.role === 'provider') {
+      removeSidecar(sub.plugin);
+    }
+  });
+
+  txn();
+
+  return {
+    deregistered: true,
+    subscription_id: subscriptionId,
+    deregistered_at: now,
+  };
+}
+
+/**
+ * Write a provider sidecar JSON file.
+ */
+function writeSidecar(plugin, data) {
+  try {
+    const dataDir = resolveDataDir();
+    const providersDir = join(dataDir, 'providers');
+    mkdirSync(providersDir, { recursive: true });
+    const sidecarPath = join(providersDir, `${plugin}.json`);
+    writeFileSync(sidecarPath, JSON.stringify(data, null, 2) + '\n', 'utf8');
+  } catch (_) {
+    // Sidecar write failure is non-fatal
+  }
+}
+
+/**
+ * Remove a provider sidecar JSON file.
+ */
+function removeSidecar(plugin) {
+  try {
+    const dataDir = resolveDataDir();
+    const sidecarPath = join(dataDir, 'providers', `${plugin}.json`);
+    unlinkSync(sidecarPath);
+  } catch (_) {
+    // Sidecar removal failure is non-fatal
+  }
+}

package/lib/schema.sql

@@ -0,0 +1,68 @@
+PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+PRAGMA foreign_keys = ON;
+PRAGMA busy_timeout = 5000;
+
+-- events
+CREATE TABLE IF NOT EXISTS events (
+    event_id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_type        TEXT    NOT NULL CHECK(length(event_type) <= 128),
+    domain            TEXT    NOT NULL CHECK(length(domain) <= 64),
+    subdomain         TEXT    NOT NULL DEFAULT '' CHECK(length(subdomain) <= 64),
+    payload           TEXT    NOT NULL,
+    schema_version    TEXT    NOT NULL DEFAULT '1.0.0',
+    idempotency_key   TEXT    NOT NULL UNIQUE,
+    emitted_at        INTEGER NOT NULL,
+    expires_at        INTEGER NOT NULL,
+    dedup_expires_at  INTEGER NOT NULL,
+    metadata          TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_events_event_type       ON events(event_type);
+CREATE INDEX IF NOT EXISTS idx_events_domain           ON events(domain);
+CREATE INDEX IF NOT EXISTS idx_events_subdomain        ON events(subdomain);
+CREATE INDEX IF NOT EXISTS idx_events_type_domain      ON events(event_type, domain);
+CREATE INDEX IF NOT EXISTS idx_events_emitted_at       ON events(emitted_at);
+CREATE INDEX IF NOT EXISTS idx_events_expires_at       ON events(expires_at);
+CREATE INDEX IF NOT EXISTS idx_events_dedup_expires_at ON events(dedup_expires_at);
+
+-- subscriptions
+CREATE TABLE IF NOT EXISTS subscriptions (
+    subscription_id          TEXT    PRIMARY KEY,
+    plugin                   TEXT    NOT NULL,
+    role                     TEXT    NOT NULL CHECK(role IN ('provider','subscriber')),
+    event_type_filter        TEXT    NOT NULL,
+    schema_version           TEXT,
+    registered_at            INTEGER NOT NULL,
+    deregistered_at          INTEGER,
+    health_check_interval_ms INTEGER DEFAULT 60000
+);
+
+CREATE INDEX IF NOT EXISTS idx_subscriptions_plugin ON subscriptions(plugin);
+CREATE INDEX IF NOT EXISTS idx_subscriptions_active
+    ON subscriptions(plugin, role) WHERE deregistered_at IS NULL;
+
+-- cursors
+CREATE TABLE IF NOT EXISTS cursors (
+    cursor_id         TEXT    PRIMARY KEY,
+    subscription_id   TEXT    NOT NULL
+                        REFERENCES subscriptions(subscription_id) ON DELETE RESTRICT,
+    last_event_id     INTEGER NOT NULL DEFAULT 0,
+    acked_at          INTEGER,
+    created_at        INTEGER NOT NULL,
+    deregistered_at   INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS idx_cursors_subscription_id ON cursors(subscription_id);
+CREATE INDEX IF NOT EXISTS idx_cursors_active
+    ON cursors(subscription_id) WHERE deregistered_at IS NULL;
+
+-- schema_migrations
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version     INTEGER PRIMARY KEY,
+    applied_at  INTEGER NOT NULL,
+    description TEXT
+);
+
+INSERT OR IGNORE INTO schema_migrations(version, applied_at, description)
+VALUES (1, unixepoch() * 1000, 'initial schema');

package/lib/sweep.js

@@ -0,0 +1,71 @@
+/**
+ * TTL sweep -- deletes expired events, optionally archiving them first.
+ * @module lib/sweep
+ */
+
+/**
+ * Run a single sweep pass.
+ * @param {import('better-sqlite3').Database} db
+ * @param {object} config
+ * @returns {{ events_deleted: number }}
+ */
+export function runSweep(db, config) {
+  const now = Date.now();
+
+  const txn = db.transaction(() => {
+    if (config.archive_mode) {
+      // Create archive table if not exists
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS events_archive (
+            event_id          INTEGER PRIMARY KEY,
+            event_type        TEXT    NOT NULL,
+            domain            TEXT    NOT NULL,
+            subdomain         TEXT    NOT NULL DEFAULT '',
+            payload           TEXT    NOT NULL,
+            schema_version    TEXT    NOT NULL DEFAULT '1.0.0',
+            idempotency_key   TEXT    NOT NULL,
+            emitted_at        INTEGER NOT NULL,
+            expires_at        INTEGER NOT NULL,
+            dedup_expires_at  INTEGER NOT NULL,
+            metadata          TEXT
+        );
+      `);
+
+      // Copy to archive before deletion
+      db.prepare(`
+        INSERT OR IGNORE INTO events_archive
+        SELECT * FROM events WHERE dedup_expires_at < ?
+      `).run(now);
+    }
+
+    // Delete expired events
+    const result = db.prepare(
+      'DELETE FROM events WHERE dedup_expires_at < ?'
+    ).run(now);
+
+    return { events_deleted: result.changes };
+  });
+
+  return txn();
+}
+
+/**
+ * Start a background sweep interval.
+ * @param {import('better-sqlite3').Database} db
+ * @param {object} config
+ * @returns {NodeJS.Timeout|null} The interval handle, or null if sweep is disabled.
+ */
+export function startSweep(db, config) {
+  if (!config.sweep_interval_minutes || config.sweep_interval_minutes === 0) {
+    return null;
+  }
+
+  const intervalMs = config.sweep_interval_minutes * 60_000;
+  return setInterval(() => {
+    try {
+      runSweep(db, config);
+    } catch (_) {
+      // Sweep errors are non-fatal
+    }
+  }, intervalMs);
+}

package/lib/validate.js

@@ -0,0 +1,156 @@
+/**
+ * Event validation.
+ * @module lib/validate
+ */
+
+import { WBError } from './errors.js';
+
+// Event type: wicked.<segments> where segments are lowercase alphanum/underscore separated by dots
+const EVENT_TYPE_REGEX = /^wicked\.[a-z0-9_]+(\.[a-z0-9_]+)*$/;
+const SEMVER_REGEX = /^\d+\.\d+\.\d+$/;
+
+/**
+ * Validate an event object before writing.
+ * @param {object} event - { event_type, domain, payload, schema_version?, subdomain?, metadata? }
+ * @param {object} config - Merged config with max_payload_bytes
+ * @throws {WBError} WB-001 or WB-005
+ */
+export function validateEvent(event, config) {
+  const missing = [];
+  const received = Object.keys(event || {});
+
+  if (!event || typeof event !== 'object') {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: 'Event must be an object',
+      received_fields: [],
+      missing_fields: ['event_type', 'domain', 'payload'],
+    });
+  }
+
+  // Check required fields
+  if (!event.event_type) missing.push('event_type');
+  if (!event.domain) missing.push('domain');
+  if (event.payload === undefined || event.payload === null) missing.push('payload');
+
+  if (missing.length > 0) {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: `Missing required fields: ${missing.join(', ')}`,
+      received_fields: received,
+      missing_fields: missing,
+    });
+  }
+
+  // event_type validation
+  if (typeof event.event_type !== 'string') {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: 'event_type must be a string',
+      received_fields: received,
+      violation: 'event_type must be a string',
+    });
+  }
+  if (event.event_type.length > 128) {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: `event_type exceeds 128 chars (got ${event.event_type.length})`,
+      received_fields: received,
+      violation: 'event_type exceeds 128 chars',
+    });
+  }
+  if (!EVENT_TYPE_REGEX.test(event.event_type)) {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: `event_type does not match pattern: ${EVENT_TYPE_REGEX}`,
+      received_fields: received,
+      violation: `event_type must match ${EVENT_TYPE_REGEX}`,
+    });
+  }
+
+  // domain validation
+  if (typeof event.domain !== 'string') {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: 'domain must be a string',
+      received_fields: received,
+      violation: 'domain must be a string',
+    });
+  }
+  if (event.domain.length > 64) {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: `domain exceeds 64 chars (got ${event.domain.length})`,
+      received_fields: received,
+      violation: 'domain exceeds 64 chars',
+    });
+  }
+
+  // subdomain validation (optional, defaults to '')
+  if (event.subdomain !== undefined && event.subdomain !== null) {
+    if (typeof event.subdomain !== 'string') {
+      throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+        message: 'subdomain must be a string',
+        received_fields: received,
+        violation: 'subdomain must be a string',
+      });
+    }
+    if (event.subdomain.length > 64) {
+      throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+        message: `subdomain exceeds 64 chars (got ${event.subdomain.length})`,
+        received_fields: received,
+        violation: 'subdomain exceeds 64 chars',
+      });
+    }
+  }
+
+  // payload validation
+  if (typeof event.payload !== 'object' || event.payload === null || Array.isArray(event.payload)) {
+    // If it's a string, try to parse it
+    if (typeof event.payload === 'string') {
+      try {
+        const parsed = JSON.parse(event.payload);
+        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+          throw new Error('not an object');
+        }
+      } catch (_) {
+        throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+          message: 'payload must be a valid JSON object',
+          received_fields: received,
+          violation: 'payload must be a valid JSON object',
+        });
+      }
+    } else {
+      throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+        message: 'payload must be a valid JSON object',
+        received_fields: received,
+        violation: 'payload must be a valid JSON object',
+      });
+    }
+  }
+
+  // Payload size check
+  const payloadStr = typeof event.payload === 'string'
+    ? event.payload
+    : JSON.stringify(event.payload);
+  const payloadBytes = Buffer.byteLength(payloadStr, 'utf8');
+  if (payloadBytes > config.max_payload_bytes) {
+    throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+      message: `Payload size ${payloadBytes} bytes exceeds max_payload_bytes (${config.max_payload_bytes})`,
+      received_fields: received,
+      violation: 'payload exceeds max_payload_bytes',
+    });
+  }
+
+  // schema_version validation (if present)
+  if (event.schema_version !== undefined && event.schema_version !== null) {
+    if (typeof event.schema_version !== 'string' || !SEMVER_REGEX.test(event.schema_version)) {
+      throw new WBError('WB-001', 'INVALID_EVENT_SCHEMA', {
+        message: 'schema_version must be a valid semver string (e.g. 1.0.0)',
+        received_fields: received,
+        violation: 'invalid schema_version format',
+      });
+    }
+    const major = parseInt(event.schema_version.split('.')[0], 10);
+    if (major > 1) {
+      throw new WBError('WB-005', 'SCHEMA_VERSION_UNSUPPORTED', {
+        message: `schema_version ${event.schema_version} is not supported (max 1.x)`,
+        declared: event.schema_version,
+        max_supported: '1.x',
+      });
+    }
+  }
+}