trajectories 0.0.0 → 0.1.2

package/legacy/src/native/sqlite-ro.ts

@@ -0,0 +1,281 @@
+import { Database } from "bun:sqlite";
+import { createHash } from "node:crypto";
+import { chmodSync, copyFileSync, existsSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+/**
+ * Read-only access to another application's SQLite store, with a private-copy
+ * fallback for stores a direct read cannot reach.
+ *
+ * Gradient ported from Wake@d9f702b6
+ * (`crates/wake-core/src/adapters/sqlite_ro.rs#L20-L46`): read-only open plus a
+ * probe query → copy the database and its sidecars into a private temp
+ * directory and read the copy → give up and surface the original error. A
+ * store held by its own writer (rollback-journal `BEGIN EXCLUSIVE`, WAL with
+ * `locking_mode = EXCLUSIVE`) answers `SQLITE_BUSY` to a direct reader but
+ * copies cleanly, because the copy is a different file with no lock holder.
+ *
+ * The source store is never written, and `immutable=1` is never used: an
+ * immutable open ignores the `-wal`, which is exactly where a live store keeps
+ * its newest sessions (`sqlite_ro.rs#L16-L19`).
+ */
+
+/** Total bytes of database plus sidecars we will copy to reach a locked store. */
+export const MAX_SQLITE_COPY_BYTES = 512 * 1024 * 1024;
+
+/** Root of the private copies; one directory per (database, process). */
+export const NATIVE_COPY_ROOT = join(tmpdir(), "ledger-native-db");
+
+const COPY_FILE = "db.sqlite";
+// Sidecars in the order they are copied. `-shm` precedes `-wal` so the copied
+// index can never claim frames the copied `-wal` does not contain: an older
+// consistent snapshot is the safe direction, a short frame read is not.
+// `-journal` is a Ledger addition to the pinned copy set — a rollback-journal
+// writer can leave uncommitted pages in the database file, and the copy is
+// opened writable so SQLite rolls them back (see `openCopy`) instead of
+// reporting them as committed.
+const SIDECARS = ["-shm", "-wal", "-journal"] as const;
+const PROBE = "SELECT count(*) AS tables FROM sqlite_master";
+
+interface PrivateCopy {
+  /** `nativeDbStamp` of the source when the copy was taken. */
+  stamp: number;
+  dir: string;
+  path: string;
+}
+
+const copies = new Map<string, PrivateCopy>();
+let cleanupRegistered = false;
+
+/**
+ * Invalidation key for a copied store: the newest mtime of the database and
+ * its `-wal`, ported from Wake's `db_cache_stamp` (`sqlite_ro.rs#L68-L76`).
+ * WAL writers leave the database file untouched until a checkpoint, so a stamp
+ * that watched only the database would pin a stale copy while new sessions
+ * pile up in the `-wal`. `-shm` is deliberately excluded: it changes on every
+ * reader's read-mark and would force a copy per call.
+ */
+export function nativeDbStamp(dbPath: string): number {
+  const mtime = (path: string): number => {
+    try { return statSync(path).mtimeMs; } catch { return 0; }
+  };
+  return Math.max(mtime(dbPath), mtime(`${dbPath}-wal`));
+}
+
+/**
+ * Path of the private copy currently serving `dbPath`, or null when the store
+ * is read directly. A copy whose stamp no longer matches the source is
+ * discarded: the caller must not read a snapshot the source has moved past.
+ */
+export function cachedCopyPath(dbPath: string): string | null {
+  const target = resolve(dbPath);
+  const copy = copies.get(target);
+  if (!copy) return null;
+  if (copy.stamp === nativeDbStamp(target) && existsSync(copy.path)) return copy.path;
+  discard(target);
+  return null;
+}
+
+/**
+ * Runs `read` against a consistent snapshot of `dbPath`, in a deferred
+ * transaction so every query sees the same state. Never writes to the store.
+ */
+export function withReadOnlyDb<T>(dbPath: string, read: (db: Database) => T): T {
+  const target = resolve(dbPath);
+  const cached = cachedCopyPath(target);
+  const warm = cached ? openCopy(cached) : null;
+  // A private copy that cannot be opened is stale furniture, not a verdict:
+  // drop it and let the source answer.
+  if (cached && !warm) discard(target);
+  if (warm) {
+    let probed = false;
+    try {
+      return runInSnapshot(warm, read, () => { probed = true; });
+    } catch (error) {
+      // A copy that cannot even be probed is not a verdict on the store; a
+      // read that ran on a readable copy fails for its own reasons and stands.
+      if (probed) throw error;
+      discard(target);
+    } finally { warm.close(); }
+  }
+  let probed = false;
+  try {
+    const db = new Database(target, { readonly: true });
+    try {
+      return runInSnapshot(db, read, () => { probed = true; });
+    } finally { db.close(); }
+  } catch (error) {
+    // The store answered the probe and `read` failed: that verdict stands.
+    if (probed) throw error;
+    // The copy answers *concurrency*, not unreachable bytes: a store whose
+    // `-wal` was deleted out-of-band is not revived by copying it, and reading
+    // it at its last checkpoint would report partial evidence as the whole
+    // session. Everything but lock contention keeps the direct diagnosis.
+    if (!isLockContention(error)) throw error;
+    const copy = acquireCopy(target, nativeDbStamp(target));
+    const db = copy ? openCopy(copy) : null;
+    if (!db) throw error;
+    let copyProbed = false;
+    try {
+      return runInSnapshot(db, read, () => { copyProbed = true; });
+    } catch (copyError) {
+      // Copy unavailable is a copy problem, not a store verdict: the caller
+      // keeps the direct diagnosis, the same one this store produced before
+      // the fallback existed.
+      throw copyProbed ? copyError : error;
+    } finally { db.close(); }
+  }
+}
+
+/** Another connection holding the store surfaces as `SQLITE_BUSY`/`SQLITE_LOCKED`. */
+function isLockContention(error: unknown): boolean {
+  if (typeof error !== "object" || error === null || !("code" in error)) return false;
+  const { code } = error;
+  return typeof code === "string" && /^SQLITE_(?:BUSY|LOCKED)/.test(code);
+}
+
+/**
+ * The copy is ours: written only by this process, mode 0600, under a 0700
+ * directory, removed on exit. Opening it writable is what lets SQLite recover
+ * it — roll back a copied hot journal, rebuild a `-shm` the source never had —
+ * so a store in recovery is readable rather than a second `read_error`.
+ */
+function openCopy(path: string): Database | null {
+  try { return new Database(path, { readwrite: true }); } catch { return null; }
+}
+
+function runInSnapshot<T>(db: Database, read: (db: Database) => T, probed?: () => void): T {
+  db.exec("PRAGMA busy_timeout = 1000");
+  return db.transaction(() => {
+    db.query(PROBE).get();
+    probed?.();
+    return read(db);
+  }).deferred();
+}
+
+/**
+ * Copies `dbPath` and its sidecars into this process's private directory and
+ * returns the copied database path, or null when the store cannot be copied
+ * within budget — the caller then surfaces the direct error unchanged.
+ */
+function acquireCopy(target: string, stamp: number): string | null {
+  const dir = copyDir(target);
+  const path = join(dir, COPY_FILE);
+  try {
+    if (!privateRoot()) return null;
+    sweepOrphans();
+    if (setBytes(target) > MAX_SQLITE_COPY_BYTES) return null;
+    // Two attempts: a database that moves under the copy is rejected and read
+    // once more, because the move is usually a checkpoint that has finished by
+    // then.
+    for (let attempt = 0; attempt < 2; attempt++) {
+      // Recreate rather than overwrite: a sidecar the source no longer has (a
+      // checkpointed `-wal`) must not survive into the next snapshot.
+      rmSync(dir, { recursive: true, force: true });
+      mkdirSync(dir, { recursive: true, mode: 0o700 });
+      const base = fileIdentity(target);
+      for (const source of [target, ...SIDECARS.map((suffix) => `${target}${suffix}`)]) {
+        if (!existsSync(source)) continue;
+        const destination = source === target ? path : `${path}${source.slice(target.length)}`;
+        // copyFileSync preserves the source mode; a read-only store would
+        // leave a copy SQLite cannot open for recovery.
+        copyFileSync(source, destination);
+        chmodSync(destination, 0o600);
+      }
+      // A checkpoint backfills pages into the database *and* restarts the wal
+      // generation. Frames and index then stay individually valid while the
+      // database base is stale, so the copy would silently lose every page the
+      // checkpoint moved — the reviewer's `t1=['A'] → t1=[]` case. The database
+      // file is the only witness: ordinary WAL commits never touch it, so its
+      // identity is stable for every copy that is actually consistent.
+      if (base !== null && base === fileIdentity(target)) {
+        // A store growing during the copy can outrun the pre-copy check; the
+        // overshoot is discarded rather than read.
+        if (setBytes(path) <= MAX_SQLITE_COPY_BYTES) {
+          copies.set(target, { stamp, dir, path });
+          registerCleanup();
+          return path;
+        }
+        break;
+      }
+    }
+    rmSync(dir, { recursive: true, force: true });
+    return null;
+  } catch {
+    rmSync(dir, { recursive: true, force: true });
+    return null;
+  }
+}
+
+/** `size:mtime` of a file, or null when it cannot be read — the copy base identity. */
+function fileIdentity(path: string): string | null {
+  try {
+    const stat = statSync(path);
+    return `${stat.size}:${stat.mtimeMs}`;
+  } catch { return null; }
+}
+
+/**
+ * The copy root must be ours and private. A pre-existing world-readable root —
+ * another user on a shared TMPDIR, or a planted directory — would expose
+ * transcript copies until each file is chmodded and would let foreign entries
+ * steer the orphan sweep, so the fallback refuses to use it at all.
+ */
+function privateRoot(): boolean {
+  mkdirSync(NATIVE_COPY_ROOT, { recursive: true, mode: 0o700 });
+  const root = statSync(NATIVE_COPY_ROOT);
+  const uid = typeof process.getuid === "function" ? process.getuid() : root.uid;
+  return root.isDirectory() && root.uid === uid && (root.mode & 0o777) === 0o700;
+}
+
+/** Bytes of a database plus whatever sidecars exist beside it. */
+function setBytes(dbPath: string): number {
+  let bytes = 0;
+  for (const path of [dbPath, ...SIDECARS.map((suffix) => `${dbPath}${suffix}`)]) {
+    try {
+      const stat = statSync(path);
+      if (stat.isFile()) bytes += stat.size;
+    } catch { /* missing sidecar */ }
+  }
+  return bytes;
+}
+
+function copyDir(target: string): string {
+  const key = createHash("sha256").update(target).digest("hex").slice(0, 16);
+  return join(NATIVE_COPY_ROOT, `copy-${key}-${process.pid}`);
+}
+
+function discard(target: string): void {
+  const copy = copies.get(target);
+  if (!copy) return;
+  copies.delete(target);
+  rmSync(copy.dir, { recursive: true, force: true });
+}
+
+/**
+ * Copies are private agent history; they do not outlive the process that took
+ * them. A crash skips this hook, so `sweepOrphans` removes the directories of
+ * processes that no longer exist before the next copy.
+ */
+function registerCleanup(): void {
+  if (cleanupRegistered) return;
+  cleanupRegistered = true;
+  process.once("exit", () => {
+    for (const copy of copies.values()) rmSync(copy.dir, { recursive: true, force: true });
+  });
+}
+
+function sweepOrphans(): void {
+  for (const entry of readdirSync(NATIVE_COPY_ROOT, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const pid = /^copy-[0-9a-f]{16}-(\d+)$/.exec(entry.name)?.[1];
+    if (pid === undefined || Number(pid) === process.pid || isLive(Number(pid))) continue;
+    rmSync(join(NATIVE_COPY_ROOT, entry.name), { recursive: true, force: true });
+  }
+}
+
+function isLive(pid: number): boolean {
+  try { process.kill(pid, 0); return true; }
+  catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; }
+}

package/legacy/src/native.ts

@@ -0,0 +1,97 @@
+import { createHash } from "node:crypto";
+import { readdirSync, statSync } from "node:fs";
+import { dirname, join } from "node:path";
+import type { DiscoveredSession } from "./types.js";
+import { readCursorSession } from "./native/cursor.js";
+import { grokSessionFiles, readGrokSession } from "./native/grok.js";
+import { kimiSessionFiles, readKimiSession } from "./native/kimi.js";
+import { openCodeSessionFiles, readOpenCodeSession } from "./native/opencode.js";
+import { MAX_NATIVE_RECORDS, type NativeReadOptions, type NativeTranscript } from "./native/common.js";
+
+interface NativeIdentity {
+  nativeVersion: 2;
+  snapshot: string;
+  failureMaxBytes?: number;
+}
+
+export function nativeIdentity(value?: string | null): NativeIdentity | null {
+  if (!value) return null;
+  try {
+    const identity = JSON.parse(value);
+    return identity?.nativeVersion === 2 && typeof identity.snapshot === "string" ? identity : null;
+  } catch { return null; }
+}
+
+export function readNativeSession(item: DiscoveredSession, options: NativeReadOptions = {}): NativeTranscript {
+  if (item.native?.kind === "fx-directory") {
+    throw new Error("FX directory sessions require path-backed SDK normalization; readNativeSession cannot decode only session.json.");
+  }
+  return item.native?.kind === "cursor-sqlite" || item.native?.kind === "cursor-blobstore"
+    ? readCursorSession(item, options)
+    : item.native?.kind === "grok-updates"
+      ? readGrokSession(item, options)
+      : item.native?.kind === "kimi-wire"
+        ? readKimiSession(item, options)
+        : readOpenCodeSession(item, options);
+}
+
+/** Database sidecars and legacy part files can change while the primary file does not. */
+export function nativeSourceFingerprint(item: DiscoveredSession): {
+  mtime: string; size: number; file_identity: string;
+} | null {
+  if (!item.native) return null;
+  try {
+    if (!statSync(item.path).isFile()) return null;
+    const sqlite = item.native.kind === "cursor-sqlite" || item.native.kind === "cursor-blobstore"
+      || item.native.kind === "opencode-sqlite";
+    const paths = item.native.kind === "fx-directory"
+      ? fxSessionFiles(item.path)
+      : sqlite
+      ? [item.path, `${item.path}-wal`, `${item.path}-journal`]
+      : item.native.kind === "grok-updates"
+        ? grokSessionFiles(item)
+        : item.native.kind === "kimi-wire" ? kimiSessionFiles(item) : openCodeSessionFiles(item);
+    if (paths.length > MAX_NATIVE_RECORDS) return null;
+    const hash = createHash("sha256").update(JSON.stringify([item.native, item.id]));
+    let bytes = 0;
+    let modified = 0;
+    for (const path of [...new Set(paths)].sort()) {
+      hash.update(JSON.stringify(path));
+      try {
+        const stat = statSync(path, { bigint: true });
+        // SQLite read-only opens may chmod the WAL to match database permissions,
+        // changing ctime without changing its bytes. Keep content-bearing mtime,
+        // size and generation identity; ordinary native files still include ctime.
+        hash.update(JSON.stringify([stat.dev, stat.ino, stat.birthtimeNs, stat.mtimeNs,
+          ...(sqlite ? [] : [stat.ctimeNs]), stat.size].map(String)));
+        if (stat.isFile()) bytes += Number(stat.size);
+        modified = Math.max(modified, Number(stat.mtimeMs));
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+        hash.update("missing");
+      }
+    }
+    return {
+      mtime: new Date(modified).toISOString(),
+      // A shared database's full byte size is not each session's backlog.
+      size: sqlite ? item.sizeBytes ?? -1 : bytes,
+      file_identity: JSON.stringify({ nativeVersion: 2, snapshot: hash.digest("hex") } satisfies NativeIdentity),
+    };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+}
+
+function fxSessionFiles(sessionPath: string): string[] {
+  const directory = dirname(sessionPath);
+  const paths = [sessionPath, join(directory, "events.jsonl")];
+  try {
+    for (const entry of readdirSync(join(directory, "tool-results"), { withFileTypes: true })) {
+      if (entry.isFile() || entry.isSymbolicLink()) paths.push(join(directory, "tool-results", entry.name));
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+  }
+  return paths;
+}