station-kit 1.0.8 → 1.1.0

package/src/server/log-store.ts

@@ -1,56 +1,207 @@
-import Database from "better-sqlite3";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { appendFile } from "node:fs/promises";
+import { dirname } from "node:path";
 import type { LogEntry } from "./log-buffer.js";
 
-export class LogStore {
-  private db: Database.Database;
-  private insertStmt: Database.Statement;
-  private selectStmt: Database.Statement;
-
-  constructor(dbPath: string) {
-    this.db = new Database(dbPath);
-    this.db.pragma("journal_mode = WAL");
-    this.db.exec(`
-      CREATE TABLE IF NOT EXISTS logs (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        run_id TEXT NOT NULL,
-        signal_name TEXT NOT NULL,
-        level TEXT NOT NULL,
-        message TEXT NOT NULL,
-        timestamp TEXT NOT NULL
-      )
-    `);
-    this.db.exec(`CREATE INDEX IF NOT EXISTS idx_logs_run_id ON logs(run_id)`);
-
-    this.insertStmt = this.db.prepare(
-      `INSERT INTO logs (run_id, signal_name, level, message, timestamp) VALUES (?, ?, ?, ?, ?)`,
-    );
-    this.selectStmt = this.db.prepare(
-      `SELECT run_id, signal_name, level, message, timestamp FROM logs WHERE run_id = ? ORDER BY id`,
+/**
+ * Pluggable storage backend for run logs. Implementations only persist
+ * and query records — bounded in-memory buffering for live UI streams
+ * lives in `LogBuffer`. May be sync or async; the LogStore wrapper
+ * normalizes both.
+ *
+ * Contract for implementers:
+ *
+ * - **`add(entry)`** is treated as fire-and-forget at the LogStore
+ *   boundary. Signal runners never block on log writes. Adapters that
+ *   need durability guarantees (queues, retries, batching) should
+ *   implement that internally; thrown errors and rejected promises are
+ *   caught and surfaced via the LogStore's `onError` hook (if set) but
+ *   never rethrown to the caller.
+ * - **`get(runId)`** must return entries for that run in append order
+ *   (oldest first). Routes that aggregate across runs may re-sort by
+ *   timestamp, but per-run ordering is the adapter's responsibility.
+ * - **`close?()`** is called once on graceful shutdown. Use it to flush
+ *   any in-flight buffers. It is NOT called on `SIGKILL` / OOM kill —
+ *   adapters that must guarantee durability per write should not rely
+ *   on it.
+ *
+ * Single-process semantics: the built-in `FileLogStorage` is safe for a
+ * single Node process. Running multiple processes against the same file
+ * path WILL produce interleaved bytes and lost entries — use a real
+ * database adapter (Postgres, MySQL, Redis, etc.) for multi-process or
+ * distributed deployments.
+ */
+export interface LogStorageAdapter {
+  add(entry: LogEntry): Promise<void> | void;
+  get(runId: string): Promise<LogEntry[]> | LogEntry[];
+  close?(): Promise<void> | void;
+}
+
+// ─── File-backed default ────────────────────────────────────────────
+
+export interface FileLogStorageOptions {
+  filePath: string;
+  /**
+   * Called when a background write to the underlying file fails. Use
+   * this to surface persistence problems (disk full, permission denied,
+   * etc.) to your monitoring system. If unset, write failures are
+   * silently dropped — acceptable for local dev, NOT for production.
+   */
+  onError?: (err: unknown) => void;
+}
+
+/**
+ * File-backed log storage using append-only JSONL framing. Each line is
+ * a JSON-serialized `LogEntry`; existing entries are loaded into memory
+ * on construction; appends are serialized through an async write queue
+ * so concurrent writers can't interleave bytes within one process.
+ *
+ * No native dependencies — works on any Node 18+ install.
+ *
+ * **Production caveats** (in order of severity):
+ *
+ * 1. **Single-process only.** Two Node processes appending to the same
+ *    file WILL interleave bytes once individual JSON lines exceed the
+ *    OS pipe buffer (4 KB on Linux), corrupting the file.
+ * 2. **Best-effort durability.** Writes are queued and flushed via
+ *    `fs.appendFile`; on `SIGKILL` / OOM kill, in-flight writes are lost.
+ *    Set `onError` to surface fs failures.
+ * 3. **Unbounded memory on replay.** The whole file is loaded into a
+ *    Map on startup. For high-volume deployments (gigabytes of logs)
+ *    use a database-backed adapter instead.
+ *
+ * For multi-process, distributed, or high-durability deployments,
+ * implement `LogStorageAdapter` against Postgres / MySQL / Redis / S3.
+ */
+export class FileLogStorage implements LogStorageAdapter {
+  private path: string;
+  private byRunId = new Map<string, LogEntry[]>();
+  private writeQueue: Promise<void> = Promise.resolve();
+  private onError: (err: unknown) => void;
+
+  constructor(options: FileLogStorageOptions) {
+    // Backwards compat: callers used to pass `.db` paths for the sqlite store.
+    // Transparently swap to `.jsonl` so existing config files keep working.
+    this.path = options.filePath.endsWith(".db")
+      ? options.filePath.replace(/\.db$/, ".jsonl")
+      : options.filePath;
+    this.onError = options.onError ?? (() => {});
+    mkdirSync(dirname(this.path), { recursive: true, mode: 0o700 });
+    this.replay();
+  }
+
+  private replay(): void {
+    if (!existsSync(this.path)) return;
+    let content: string;
+    try {
+      content = readFileSync(this.path, "utf8");
+    } catch (err) {
+      this.onError(err);
+      return;
+    }
+    const lines = content.split("\n");
+    for (const line of lines) {
+      if (!line) continue;
+      try {
+        const entry = JSON.parse(line) as LogEntry;
+        this.indexEntry(entry);
+      } catch {
+        // Skip malformed line; a partial write may have left a truncated tail.
+      }
+    }
+  }
+
+  private indexEntry(entry: LogEntry): void {
+    let entries = this.byRunId.get(entry.runId);
+    if (!entries) {
+      entries = [];
+      this.byRunId.set(entry.runId, entries);
+    }
+    entries.push(entry);
+  }
+
+  add(entry: LogEntry): void {
+    this.indexEntry(entry);
+    const line = JSON.stringify(entry) + "\n";
+    this.writeQueue = this.writeQueue.then(
+      () => appendFile(this.path, line, { mode: 0o600 }).catch((err) => {
+        this.onError(err);
+      }),
     );
   }
 
+  get(runId: string): LogEntry[] {
+    return this.byRunId.get(runId) ?? [];
+  }
+
+  async close(): Promise<void> {
+    await this.writeQueue;
+  }
+}
+
+// ─── In-memory storage for tests / ephemeral deployments ────────────
+
+export class MemoryLogStorage implements LogStorageAdapter {
+  private byRunId = new Map<string, LogEntry[]>();
+
   add(entry: LogEntry): void {
-    this.insertStmt.run(entry.runId, entry.signalName, entry.level, entry.message, entry.timestamp);
+    let entries = this.byRunId.get(entry.runId);
+    if (!entries) {
+      entries = [];
+      this.byRunId.set(entry.runId, entries);
+    }
+    entries.push(entry);
   }
 
   get(runId: string): LogEntry[] {
-    const rows = this.selectStmt.all(runId) as Array<{
-      run_id: string;
-      signal_name: string;
-      level: string;
-      message: string;
-      timestamp: string;
-    }>;
-    return rows.map((row) => ({
-      runId: row.run_id,
-      signalName: row.signal_name,
-      level: row.level as "stdout" | "stderr",
-      message: row.message,
-      timestamp: row.timestamp,
-    }));
-  }
-
-  close(): void {
-    this.db.close();
+    return this.byRunId.get(runId) ?? [];
+  }
+}
+
+// ─── LogStore — thin wrapper that delegates to an adapter ───────────
+
+/**
+ * LogStore is the consumer-facing handle that wraps a `LogStorageAdapter`.
+ * It exists so signal runners and route handlers can interact with a
+ * single concrete type, while the underlying persistence is swappable.
+ *
+ * `add` is fire-and-forget — adapter promises are caught at this boundary
+ * so a slow or failing log backend can never block (or crash) a signal
+ * runner. `get` always returns a Promise so callers can transparently
+ * support async backends (Postgres, Redis, etc.).
+ */
+export class LogStore {
+  private storage: LogStorageAdapter;
+
+  /**
+   * Pass a `LogStorageAdapter` for any backend. The string overload is
+   * a shortcut for `new FileLogStorage({ filePath })` — useful for
+   * local dev and the default Station data directory.
+   */
+  constructor(storageOrPath: LogStorageAdapter | string) {
+    if (typeof storageOrPath === "string") {
+      this.storage = new FileLogStorage({ filePath: storageOrPath });
+    } else {
+      this.storage = storageOrPath;
+    }
+  }
+
+  add(entry: LogEntry): void {
+    try {
+      const result = this.storage.add(entry);
+      if (result && typeof (result as Promise<void>).catch === "function") {
+        (result as Promise<void>).catch(() => {});
+      }
+    } catch {
+      // Swallow sync throws; a broken log adapter must not crash signal runs.
+    }
+  }
+
+  async get(runId: string): Promise<LogEntry[]> {
+    return await this.storage.get(runId);
+  }
+
+  async close(): Promise<void> {
+    if (this.storage.close) await this.storage.close();
   }
 }

package/src/server/middleware/auth.ts

@@ -15,7 +15,7 @@ export function authResolver(deps: AuthDeps) {
     if (authHeader?.startsWith("Bearer ") && deps.keyStore) {
       const token = authHeader.slice(7);
       if (token.startsWith("sk_")) {
-        const key = deps.keyStore.verify(token);
+        const key = await deps.keyStore.verify(token);
         if (key) {
           c.set("authType", "api-key");
           c.set("apiKeyId", key.id);

package/src/server/routes/broadcasts.ts

@@ -123,7 +123,9 @@ export function broadcastRoutes(deps: BroadcastDeps) {
     const allLogs: Array<{ runId: string; signalName: string; level: string; message: string; timestamp: string; nodeName: string }> = [];
     for (const nr of nodes) {
       if (nr.signalRunId) {
-        const logs = deps.logStore?.get(nr.signalRunId) ?? deps.logBuffer?.get(nr.signalRunId) ?? [];
+        const logs = deps.logStore
+          ? await deps.logStore.get(nr.signalRunId)
+          : deps.logBuffer?.get(nr.signalRunId) ?? [];
         for (const log of logs) {
           allLogs.push({ ...log, nodeName: nr.nodeName });
         }

package/src/server/routes/runs.ts

@@ -120,7 +120,7 @@ export function runRoutes(deps: RunDeps) {
 
   app.get("/runs/:id/logs", async (c) => {
     const id = c.req.param("id");
-    const logs = deps.logStore?.get(id) ?? deps.logBuffer.get(id);
+    const logs = deps.logStore ? await deps.logStore.get(id) : deps.logBuffer.get(id);
     return c.json({ data: logs });
   });
 

package/src/server/routes/v1/definitions.ts

@@ -0,0 +1,164 @@
+import { Hono } from "hono";
+import type {
+  BroadcastRunner,
+  BroadcastQueueAdapter,
+  DynamicBroadcastSpec,
+} from "station-broadcast";
+import { validateDynamicSpec, type DynamicValidationContext } from "station-broadcast";
+import type { SignalRunner } from "station-signal";
+import type { SchemaField } from "station-expressions";
+import type { StationSignalSubscriber } from "../../subscriber.js";
+
+export interface V1DefinitionDeps {
+  broadcastRunner?: BroadcastRunner;
+  broadcastAdapter?: BroadcastQueueAdapter;
+  signalRunner?: SignalRunner;
+  signalSubscriber?: StationSignalSubscriber;
+}
+
+/**
+ * Read-scope routes for dynamic broadcast definitions: list, get, version
+ * history, get-by-version, plus the read-only `validate` endpoint.
+ */
+export function v1DefinitionReadRoutes(deps: V1DefinitionDeps) {
+  const app = new Hono();
+
+  app.get("/broadcast-definitions", async (c) => {
+    if (!deps.broadcastAdapter?.listDefinitions) return c.json({ data: [] });
+    const list = await deps.broadcastAdapter.listDefinitions();
+    return c.json({ data: list.map(serializeSpec) });
+  });
+
+  app.get("/broadcast-definitions/:name", async (c) => {
+    if (!deps.broadcastAdapter?.getDefinition) return c.json({ error: "unavailable" }, 503);
+    const spec = await deps.broadcastAdapter.getDefinition(c.req.param("name"));
+    if (!spec) return c.json({ error: "not_found" }, 404);
+    return c.json({ data: serializeSpec(spec) });
+  });
+
+  app.get("/broadcast-definitions/:name/versions", async (c) => {
+    if (!deps.broadcastAdapter?.listDefinitionVersions) return c.json({ data: [] });
+    const versions = await deps.broadcastAdapter.listDefinitionVersions(c.req.param("name"));
+    return c.json({ data: versions.map(serializeSpec) });
+  });
+
+  app.get("/broadcast-definitions/:name/versions/:n", async (c) => {
+    if (!deps.broadcastAdapter?.getDefinition) return c.json({ error: "unavailable" }, 503);
+    const version = parseInt(c.req.param("n"), 10);
+    if (Number.isNaN(version)) {
+      return c.json({ error: "bad_request", message: "Version must be a number." }, 400);
+    }
+    const spec = await deps.broadcastAdapter.getDefinition(c.req.param("name"), version);
+    if (!spec) return c.json({ error: "not_found" }, 404);
+    return c.json({ data: serializeSpec(spec) });
+  });
+
+  app.post("/broadcast-definitions/validate", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const spec = body as DynamicBroadcastSpec;
+    if (!spec || typeof spec !== "object" || typeof spec.name !== "string") {
+      return c.json({ error: "bad_request", message: "Spec is missing required fields." }, 400);
+    }
+    const ctx = buildValidationContext(deps);
+    const result = validateDynamicSpec(spec, ctx);
+    return c.json({ data: result });
+  });
+
+  return app;
+}
+
+/**
+ * Admin-scope routes for mutating dynamic broadcast definitions. GETs are
+ * mounted under read scope in server/index.ts.
+ */
+export function v1DefinitionRoutes(deps: V1DefinitionDeps) {
+  const app = new Hono();
+
+  app.post("/broadcast-definitions", async (c) => {
+    if (!deps.broadcastAdapter?.saveDefinition) {
+      return c.json({ error: "unavailable", message: "Broadcast adapter does not support dynamic definitions." }, 503);
+    }
+    const body = await c.req.json().catch(() => ({}));
+    const incoming = body as DynamicBroadcastSpec;
+    if (!incoming?.name || !Array.isArray(incoming.nodes)) {
+      return c.json({ error: "bad_request", message: "Spec is missing required fields." }, 400);
+    }
+
+    const ctx = buildValidationContext(deps);
+    const validation = validateDynamicSpec(incoming, ctx);
+    if (!validation.ok) {
+      return c.json({ error: "validation_failed", data: validation }, 422);
+    }
+
+    const apiKeyId = c.get("apiKeyId" as never) as string | undefined;
+    const now = new Date();
+    const toSave: DynamicBroadcastSpec = {
+      ...incoming,
+      version: 0,
+      createdAt: incoming.createdAt ?? now,
+      updatedAt: now,
+      createdBy: apiKeyId,
+      failurePolicy: incoming.failurePolicy ?? "fail-fast",
+    };
+
+    const saved = await deps.broadcastAdapter.saveDefinition(toSave);
+
+    // Trigger an eager reconciliation so the new version is live immediately.
+    if (deps.broadcastRunner) {
+      void deps.broadcastRunner.reconcileDynamicDefinitions().catch(() => {});
+    }
+
+    return c.json({ data: serializeSpec(saved) }, 201);
+  });
+
+  // GET routes for definitions live in the read-scope mount in server/index.ts;
+  // this router only exposes mutating endpoints under admin scope.
+
+  app.delete("/broadcast-definitions/:name", async (c) => {
+    if (!deps.broadcastAdapter?.deleteDefinition) {
+      return c.json({ error: "unavailable" }, 503);
+    }
+    const success = await deps.broadcastAdapter.deleteDefinition(c.req.param("name"));
+    if (!success) return c.json({ error: "not_found" }, 404);
+    if (deps.broadcastRunner) {
+      void deps.broadcastRunner.reconcileDynamicDefinitions().catch(() => {});
+    }
+    return c.json({ data: { deleted: true } });
+  });
+
+  return app;
+}
+
+function serializeSpec(spec: DynamicBroadcastSpec): Record<string, unknown> {
+  return {
+    ...spec,
+    createdAt: spec.createdAt?.toISOString?.() ?? spec.createdAt,
+    updatedAt: spec.updatedAt?.toISOString?.() ?? spec.updatedAt,
+    deletedAt: spec.deletedAt?.toISOString?.() ?? spec.deletedAt,
+  };
+}
+
+function buildValidationContext(deps: V1DefinitionDeps): DynamicValidationContext {
+  const signalSchemas = new Map<string, { inputSchema: SchemaField; outputSchema: SchemaField }>();
+  // Best-effort schema reflection: we don't know the precise SchemaField for
+  // each signal here without traversing Zod schemas. Use `any` for now — the
+  // structural checks (signal exists, deps exist, no cycles, expression
+  // wellformedness) still run. Zod input validation runs at trigger time.
+  const sigs = deps.signalRunner?.getAllSignals();
+  if (sigs) {
+    for (const name of sigs.keys()) {
+      signalSchemas.set(name, {
+        inputSchema: { type: "any" },
+        outputSchema: { type: "any" },
+      });
+    }
+  } else if (deps.signalSubscriber) {
+    for (const meta of deps.signalSubscriber.getAllSignalMeta()) {
+      signalSchemas.set(meta.name, {
+        inputSchema: { type: "any" },
+        outputSchema: { type: "any" },
+      });
+    }
+  }
+  return { signalSchemas };
+}

package/src/server/routes/v1/expressions.ts

@@ -0,0 +1,76 @@
+import { Hono } from "hono";
+import {
+  evaluate,
+  validate,
+  parse,
+  ExpressionEvalError,
+  ExpressionParseError,
+  type ExprNode,
+  type SchemaField,
+} from "station-expressions";
+
+export function v1ExpressionRoutes() {
+  const app = new Hono();
+
+  app.post("/expressions/evaluate", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const { node, context } = body as {
+      node?: ExprNode;
+      context?: { input?: unknown; upstream?: Record<string, unknown> };
+    };
+    if (!node) {
+      return c.json({ error: "bad_request", message: "Missing `node`." }, 400);
+    }
+    try {
+      const result = evaluate(node, {
+        input: context?.input,
+        upstream: context?.upstream ?? {},
+      });
+      return c.json({ data: { value: result } });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      const status = err instanceof ExpressionEvalError ? 400 : 500;
+      return c.json({ error: "evaluation_failed", message }, status);
+    }
+  });
+
+  app.post("/expressions/validate", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const { node, schemaContext } = body as {
+      node?: ExprNode;
+      schemaContext?: {
+        inputSchema?: SchemaField;
+        upstreamSchemas?: Record<string, SchemaField>;
+        expectedSchema?: SchemaField;
+      };
+    };
+    if (!node) {
+      return c.json({ error: "bad_request", message: "Missing `node`." }, 400);
+    }
+    const result = validate(node, {
+      inputSchema: schemaContext?.inputSchema ?? { type: "any" },
+      upstreamSchemas: schemaContext?.upstreamSchemas ?? {},
+      expectedSchema: schemaContext?.expectedSchema,
+    });
+    return c.json({ data: result });
+  });
+
+  app.post("/expressions/parse", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const { source } = body as { source?: string };
+    if (typeof source !== "string") {
+      return c.json({ error: "bad_request", message: "Missing `source` string." }, 400);
+    }
+    try {
+      const node = parse(source);
+      return c.json({ data: { node } });
+    } catch (err) {
+      if (err instanceof ExpressionParseError) {
+        return c.json({ error: "parse_error", message: err.message, position: err.position }, 400);
+      }
+      return c.json({ error: "parse_error", message: err instanceof Error ? err.message : String(err) }, 400);
+    }
+  });
+
+  return app;
+}

package/src/server/routes/v1/keys.ts

@@ -17,7 +17,7 @@ export function v1KeyRoutes(deps: V1KeyDeps) {
     const name = body.name || "Unnamed key";
     const scopes = Array.isArray(body.scopes) ? body.scopes : ["trigger", "read"];
 
-    const { key, record } = deps.keyStore.create(name, scopes);
+    const { key, record } = await deps.keyStore.create(name, scopes);
     return c.json(
       {
         data: {
@@ -37,7 +37,7 @@ export function v1KeyRoutes(deps: V1KeyDeps) {
     if (!deps.keyStore) {
       return c.json({ error: "unavailable", message: "Auth not configured." }, 503);
     }
-    const keys = deps.keyStore.list();
+    const keys = await deps.keyStore.list();
     return c.json({ data: keys });
   });
 
@@ -46,7 +46,7 @@ export function v1KeyRoutes(deps: V1KeyDeps) {
       return c.json({ error: "unavailable", message: "Auth not configured." }, 503);
     }
     const id = c.req.param("id");
-    const success = deps.keyStore.revoke(id);
+    const success = await deps.keyStore.revoke(id);
     if (!success) {
       return c.json({ error: "not_found", message: "Key not found." }, 404);
     }

package/src/server/routes/v1/runs.ts

@@ -78,7 +78,7 @@ export function v1RunRoutes(deps: V1RunDeps) {
 
   app.get("/runs/:id/logs", async (c) => {
     const id = c.req.param("id");
-    const logs = deps.logStore?.get(id) ?? deps.logBuffer.get(id);
+    const logs = deps.logStore ? await deps.logStore.get(id) : deps.logBuffer.get(id);
     return c.json({ data: logs });
   });