trajectories 0.0.0 → 0.1.1

package/legacy/scripts/verify-runtime-rollout.ts

@@ -0,0 +1,151 @@
+#!/usr/bin/env bun
+import { Database } from "bun:sqlite";
+import { createHash } from "node:crypto";
+import { createReadStream, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+// Private artifacts only: fixtures may contain short fragments of user prompts.
+const [mode, dbArg, outArg] = process.argv.slice(2);
+if (!["snapshot", "freeze", "queries"].includes(mode ?? "") || !dbArg || !outArg) {
+  throw new Error("Usage: bun legacy/scripts/verify-runtime-rollout.ts snapshot|freeze|queries DB NEW_OUTPUT");
+}
+const dbPath = resolve(dbArg);
+const out = resolve(outArg);
+if (!existsSync(dbPath) || existsSync(out)) throw new Error("Database must exist and output must be new");
+process.umask(0o077);
+mkdirSync(dirname(out), { recursive: true, mode: 0o700 });
+const started = new Date().toISOString();
+const start = performance.now();
+const hash = (text: string) => createHash("sha256").update(text).digest("hex");
+const save = (value: unknown) => writeFileSync(out, JSON.stringify(value, null, 2) + "\n", { mode: 0o600, flag: "wx" });
+function open() {
+  const db = new Database(dbPath, { readonly: true });
+  db.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=1000");
+  return db;
+}
+
+if (mode === "snapshot") {
+  const db = open();
+  const tables: Record<string, { count: number; sha256: string }> = {};
+  // These projections exist in both the deployed v6 and the candidate v10.
+  const projections: Record<string, string> = {
+    sessions: "id,source,source_path,content_hash,title,cwd,model,started_at,ended_at,record_count,created_at,updated_at,parent_session_id,parent_tool_use_id,agent_nickname,thread_role",
+    records: "*",
+    records_embeddings: "*",
+    ingest_cursors: "source,listing_id,source_path,session_id,mtime,size,bytes_ingested,record_count,status,updated_at,file_identity,prefix_hash",
+    ingest_staged_records: "*",
+  };
+  const stagedColumns = db.query<{ name: string }, []>("PRAGMA table_info(ingest_staged_sessions)").all()
+    .map(({ name }) => name).filter((name) => name !== "diagnostics_json");
+  projections.ingest_staged_sessions = stagedColumns.map((name) => `"${name.replaceAll('"', '""')}"`).join(",");
+  db.exec("BEGIN");
+  try {
+    for (const [table, projection] of Object.entries(projections)) {
+      const digest = createHash("sha256");
+      let count = 0;
+      const order = table === "ingest_cursors" ? "source,listing_id,source_path" : "rowid";
+      for (const row of db.query(`SELECT ${projection} FROM ${table} ORDER BY ${order}`).iterate()) {
+        digest.update(JSON.stringify(row) + "\n");
+        count++;
+      }
+      tables[table] = { count, sha256: digest.digest("hex") };
+    }
+    const result = {
+      started_at: started,
+      schema: db.query("PRAGMA user_version").get(),
+      tables,
+      coverage: db.query("SELECT source,count(*) AS sessions,sum(record_count) AS records FROM sessions GROUP BY source ORDER BY source").all(),
+      cursors: db.query("SELECT source,status,count(*) AS count FROM ingest_cursors GROUP BY source,status ORDER BY source,status").all(),
+      quick_check: db.query("PRAGMA quick_check").all(),
+      foreign_key_violations: db.query("PRAGMA foreign_key_check").all().length,
+      elapsed_ms: performance.now() - start,
+      resource_usage: process.resourceUsage(),
+    };
+    save(result);
+    console.log(JSON.stringify({ mode, schema: result.schema, records: tables.records.count, elapsed_ms: result.elapsed_ms }));
+  } finally {
+    db.exec("ROLLBACK");
+    db.close();
+  }
+} else if (mode === "freeze") {
+  const seedPath = process.argv[5];
+  if (!seedPath) throw new Error("freeze also requires a private seed-case JSON file with a cases array");
+  const seeds = JSON.parse(readFileSync(seedPath, "utf8"));
+  if (!Array.isArray(seeds.cases) || seeds.cases.some((item: any) =>
+    !item || typeof item.name !== "string" || typeof item.query !== "string" ||
+    typeof item.required !== "boolean" || (!item.negative && typeof item.session !== "string"))) {
+    throw new Error("Invalid private seed cases");
+  }
+  const db = open();
+  const cases: Array<{ name: string; query: string; source?: string; session?: string; required: boolean; negative?: boolean }> = [...seeds.cases];
+  const sources = db.query<{ source: string }, []>("SELECT DISTINCT source FROM sessions ORDER BY source").all();
+  const missingSeeds: string[] = [];
+  for (const { source } of sources) {
+    const rows = db.query<{ session_id: string; content: string }, [string]>(
+      "SELECT r.session_id,r.content FROM sessions s JOIN records r ON r.session_id=s.id WHERE s.source=? AND r.record_type='user' AND length(r.content) BETWEEN 80 AND 800 LIMIT 200",
+    ).all(source);
+    const row = rows.find(({ content }) =>
+      !/[A-Za-z0-9_+\/-]{32,}|api.?key|token|password|secret|cookie|authorization|BEGIN.*KEY/i.test(content) &&
+      (content.match(/[\p{L}\p{N}_]+/gu)?.length ?? 0) >= 16,
+    );
+    if (!row) { missingSeeds.push(source); continue; }
+    const words = row.content.match(/[\p{L}\p{N}_]+/gu)!;
+    const middle = Math.floor((words.length - 12) / 2);
+    cases.push({ name: `source-retrieval-${source}`, source, session: row.session_id, query: `"${words.slice(middle, middle + 12).join(" ")}"`, required: true });
+  }
+  db.close();
+  save({ started_at: started, limit: 50, cases, missing_seeds: missingSeeds });
+  console.log(JSON.stringify({ mode, cases: cases.length, missing_seeds: missingSeeds }));
+} else {
+  const fixturePath = process.argv[5];
+  if (!fixturePath) throw new Error("queries also requires a frozen fixture JSON path");
+  const fixtureText = readFileSync(fixturePath, "utf8");
+  const fixture = JSON.parse(fixtureText);
+  const cli = join(import.meta.dir, "../bin/ledger.ts");
+  async function footprint() {
+    const result: Record<string, string | null> = {};
+    for (const suffix of ["", "-wal", "-write.lock"]) {
+      if (!existsSync(dbPath + suffix)) { result[suffix || "db"] = null; continue; }
+      const digest = createHash("sha256");
+      for await (const chunk of createReadStream(dbPath + suffix)) digest.update(chunk);
+      result[suffix || "db"] = digest.digest("hex");
+    }
+    return result;
+  }
+  const before = await footprint();
+  const attempts = [];
+  for (const item of fixture.cases) {
+    const args = [process.execPath, cli, "search", item.query, "--db", dbPath, "--limit", String(fixture.limit), "--robot", "--json", "--output-version", "2"];
+    if (item.source) args.push("--source", item.source);
+    const began = performance.now();
+    const child = Bun.spawn(args, { stdout: "pipe", stderr: "pipe", env: { ...process.env, LEDGER_WIDEN_MAX_RECORDS: "10000" } });
+    let timedOut = false;
+    const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, 30000);
+    const stdoutPromise = new Response(child.stdout).text();
+    const stderrPromise = new Response(child.stderr).text();
+    const code = await child.exited;
+    clearTimeout(timer);
+    const stdout = await stdoutPromise;
+    const stderr = await stderrPromise;
+    let payload: any = null;
+    try { payload = JSON.parse(stdout); } catch { /* Retain a sanitized failure below. */ }
+    const hits = Array.isArray(payload?.hits) ? payload.hits : [];
+    const rank = hits.findIndex((hit: any) => hit.session_id === item.session) + 1;
+    const shapeOk = payload?.search_meta?.type === "search_meta" && Array.isArray(payload.hits);
+    const passed = code === 0 && !timedOut && shapeOk && (item.negative ? hits.length === 0 : rank > 0) && (!item.source || hits.every((hit: any) => hit.source === item.source));
+    // Bun exposes these as non-enumerable getters, and CPU counters may be bigint.
+    const usage = child.resourceUsage();
+    const resources = usage ? {
+      cpu_us: { user: Number(usage.cpuTime.user), system: Number(usage.cpuTime.system), total: Number(usage.cpuTime.total) },
+      max_rss_bytes: usage.maxRSS,
+      io_operations: { input: usage.ops.in, output: usage.ops.out },
+    } : null;
+    attempts.push({ name: item.name, required: item.required, passed, exit_code: code, timed_out: timedOut, query_sha256: hash(item.query), expected_session_sha256: item.session ? hash(item.session) : null, target_rank: rank || null, returned_hits: hits.length, search_meta: payload?.search_meta ?? null, error_code: payload?.error?.code ?? null, stdout_sha256: hash(stdout), stderr_sha256: hash(stderr), elapsed_ms: performance.now() - began, resource_usage: resources });
+  }
+  const after = await footprint();
+  const unchanged = JSON.stringify(before) === JSON.stringify(after);
+  const passed = unchanged && fixture.missing_seeds.length === 0 && attempts.filter((a) => a.required).every((a) => a.passed);
+  save({ started_at: started, fixture_sha256: hash(fixtureText), attempts, read_only_unchanged: unchanged, footprint_before: before, footprint_after: after, passed, elapsed_ms: performance.now() - start });
+  console.log(JSON.stringify({ mode, passed, read_only_unchanged: unchanged, attempts: attempts.map(({ name, passed, required, target_rank, elapsed_ms }) => ({ name, passed, required, target_rank, elapsed_ms })) }));
+  if (!passed) process.exitCode = 1;
+}

package/legacy/src/canonical.ts

@@ -0,0 +1,108 @@
+import { mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { homedir } from "node:os";
+import { runCli } from "./cli.js";
+import { ensurePrivateDirectory } from "./private-storage.js";
+
+/**
+ * Database resolution for the canonical `ledger` executable:
+ * explicit --db PATH (or --db=PATH) > TRAJECTORIES_DB > LEDGER_DB >
+ * ${XDG_DATA_HOME:-$HOME/.local/share}/ledger/ledger.sqlite.
+ *
+ * Unlike the legacy entrypoints, the canonical executable never falls back to
+ * a database in the current working directory.
+ */
+export function resolveCanonicalDbPath(
+  env: { [key: string]: string | undefined } = process.env,
+): string {
+  const envDb = env.TRAJECTORIES_DB || env.LEDGER_DB;
+  if (typeof envDb === "string" && envDb.length > 0) {
+    return envDb;
+  }
+  const dataHome =
+    typeof env.XDG_DATA_HOME === "string" && env.XDG_DATA_HOME.length > 0
+      ? env.XDG_DATA_HOME
+      : join(env.HOME || homedir(), ".local", "share");
+  return join(dataHome, "ledger", "ledger.sqlite");
+}
+
+/** Commands that (may) write the store; only these may create the default data dir. */
+const WRITER_COMMANDS = new Set(["sync", "build", "watch"]);
+
+function emitUsageError(argv: string[], message: string): never {
+  if (argv.includes("--robot")) {
+    console.log(JSON.stringify({ error: { code: "INVALID_ARGUMENT", message } }));
+  } else {
+    console.error(`Error: ${message}`);
+  }
+  process.exit(1);
+}
+
+function hasExplicitDb(argv: string[]): boolean {
+  let found = false;
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--db") {
+      const value = argv[i + 1];
+      if (!value || value.startsWith("-")) {
+        emitUsageError(argv, "--db requires a path value.");
+      }
+      found = true;
+      i++;
+    }
+    if (arg === "--db=") {
+      emitUsageError(argv, "--db= requires a path value.");
+    }
+    if (arg.startsWith("--db=")) {
+      found = true;
+    }
+  }
+  return found;
+}
+
+/**
+ * Return argv with a resolved `--db` appended when the caller did not name a
+ * database explicitly. Reads stay pure; writer commands additionally ensure
+ * the default data directory exists so a first `ledger sync` does not fail on
+ * a missing parent directory.
+ */
+export function canonicalArgv(argv: string[], env: NodeJS.ProcessEnv = process.env): string[] {
+  // Help/version must remain observational even for writer subcommands.
+  if (argv.some(arg => ["--help", "-h", "--version", "-v"].includes(arg))) return argv;
+  if (hasExplicitDb(argv)) return argv;
+
+  const head = argv[0];
+  if (
+    argv.length === 0 ||
+    head === "--help" || head === "-h" ||
+    head === "--version" || head === "-v"
+  ) {
+    return argv;
+  }
+
+  const dbPath = resolveCanonicalDbPath(env);
+  if (WRITER_COMMANDS.has(head)) {
+    // Tighten only our default directory, never a custom database's parent.
+    if (!env.TRAJECTORIES_DB && !env.LEDGER_DB) ensurePrivateDirectory(dirname(dbPath));
+    else mkdirSync(dirname(dbPath), { recursive: true, mode: 0o700 });
+  }
+  return [...argv, "--db", dbPath];
+}
+
+/** Entry point for bin/ledger.ts: the canonical Ledger CLI. */
+export async function runCanonicalCli(argv: string[]): Promise<void> {
+  const dbPath = resolveCanonicalDbPath();
+  let patched: string[];
+  try { patched = canonicalArgv(argv); }
+  catch (error) {
+    if (argv.includes("--robot")) {
+      console.log(JSON.stringify({ error: { code: "UNSAFE_STORE_PATH", message: error instanceof Error ? error.message : String(error) } }));
+      process.exit(1);
+    }
+    throw error;
+  }
+  await runCli(patched, {
+    programName: "trajectories",
+    dbHelpNote: ` (default: ${dbPath})`,
+  });
+}