trajectories 0.0.0 → 0.1.1

package/legacy/src/semantic-top-k.ts

@@ -0,0 +1,71 @@
+/** Narrow scoring state: no transcript bodies or stored vector blobs. */
+export interface SemanticCandidate {
+  record_id: string;
+  session_id: string;
+  score: number;
+  ordinal: number;
+}
+
+function better(a: SemanticCandidate, b: SemanticCandidate): boolean {
+  return a.score > b.score || (a.score === b.score && a.ordinal < b.ordinal);
+}
+
+/**
+ * Exact top-K distinct sessions using a worst-first heap. Once a session is
+ * evicted, its earlier best score cannot beat the monotonically improving
+ * cutoff; forgetting that session is safe. Space is O(K), update O(log K).
+ */
+export class SemanticTopK {
+  private heap: SemanticCandidate[] = [];
+  private positions = new Map<string, number>();
+  constructor(readonly limit: number) {
+    if (!Number.isSafeInteger(limit) || limit < 1) throw new RangeError("Semantic limit must be positive.");
+  }
+  get size(): number { return this.heap.length; }
+  private swap(a: number, b: number): void {
+    [this.heap[a], this.heap[b]] = [this.heap[b], this.heap[a]];
+    this.positions.set(this.heap[a].session_id, a);
+    this.positions.set(this.heap[b].session_id, b);
+  }
+  private down(index: number): void {
+    for (;;) {
+      const left = index * 2 + 1;
+      if (left >= this.heap.length) return;
+      const right = left + 1;
+      const worst = right < this.heap.length && better(this.heap[left], this.heap[right]) ? right : left;
+      if (!better(this.heap[index], this.heap[worst])) return;
+      this.swap(index, worst);
+      index = worst;
+    }
+  }
+  offer(candidate: SemanticCandidate): void {
+    if (!Number.isFinite(candidate.score)) return;
+    const existing = this.positions.get(candidate.session_id);
+    if (existing !== undefined) {
+      if (better(candidate, this.heap[existing])) {
+        this.heap[existing] = candidate;
+        this.down(existing);
+      }
+      return;
+    }
+    if (this.heap.length < this.limit) {
+      let index = this.heap.length;
+      this.heap.push(candidate);
+      this.positions.set(candidate.session_id, index);
+      while (index > 0) {
+        const parent = Math.floor((index - 1) / 2);
+        if (!better(this.heap[parent], this.heap[index])) break;
+        this.swap(parent, index);
+        index = parent;
+      }
+    } else if (better(candidate, this.heap[0])) {
+      this.positions.delete(this.heap[0].session_id);
+      this.heap[0] = candidate;
+      this.positions.set(candidate.session_id, 0);
+      this.down(0);
+    }
+  }
+  sorted(): SemanticCandidate[] {
+    return [...this.heap].sort((a, b) => b.score - a.score || a.ordinal - b.ordinal);
+  }
+}

package/legacy/src/session-hash.ts

@@ -0,0 +1,49 @@
+import { createHash } from "node:crypto";
+import { nativeId } from "./identity.js";
+import type { CanonicalDbRecord } from "./types.js";
+
+/**
+ * Version 2 is a commutative cryptographic accumulator rather than one opaque
+ * streaming digest. Each record leaf includes its durable order and identity,
+ * while modular addition makes the session state extendable even when a later
+ * transcript line canonically sorts beside an earlier tool call. Version 1
+ * denotes the pre-incident whole-session SHA-256.
+ */
+export const SESSION_HASH_VERSION = 2;
+const DIGEST_MODULUS = 1n << 256n;
+
+type SessionHashRecord = Pick<
+  CanonicalDbRecord,
+  "id" | "record_type" | "source_order_id" | "component_index" | "content_hash"
+>;
+
+function sha256(parts: Array<string | Buffer>): string {
+  const hash = createHash("sha256");
+  for (const part of parts) hash.update(part);
+  return hash.digest("hex");
+}
+
+export function initialSessionHash(source: string, sessionId: string): string {
+  return sha256([`ledger-session-hash:v${SESSION_HASH_VERSION}:${source}:${nativeId(source, sessionId)}\n`]);
+}
+
+export function extendSessionHash(previous: string, records: Iterable<SessionHashRecord>): string {
+  if (!/^[a-f0-9]{64}$/.test(previous)) throw new Error("Invalid prior session hash state");
+  let state = BigInt(`0x${previous}`);
+  for (const record of records) {
+    const leaf = sha256([
+      `ledger-session-record:v${SESSION_HASH_VERSION}\n`,
+      `${record.record_type}:${record.source_order_id}:${record.component_index}:${record.id}:${record.content_hash}\n`,
+    ]);
+    state = (state + BigInt(`0x${leaf}`)) % DIGEST_MODULUS;
+  }
+  return state.toString(16).padStart(64, "0");
+}
+
+export function hashSessionRecords(
+  source: string,
+  sessionId: string,
+  records: Iterable<SessionHashRecord>,
+): string {
+  return extendSessionHash(initialSessionHash(source, sessionId), records);
+}

package/legacy/src/store-status.ts

@@ -0,0 +1,218 @@
+import type { DiscoverySummary } from "./discovery.js";
+import { readDiagnosticsSummary, readSourceCoverage, readDiscoverySummary } from "./db.js";
+import { echoCount } from "./filter.js";
+import { FTS_FORMAT, SCHEMA_VERSION } from "./schema.js";
+import { classifyReadStore, formatMismatchMessage, incompleteStoreMessage, type ReadStoreRefusal } from "./schema-gate.js";
+import type { DiagnosticsSummary, SourceCoverage } from "./types.js";
+
+/**
+ * Read-only snapshot of a Ledger store. `null` fields mean "unknown because
+ * no initialized store exists at this path"; they are never guessed.
+ */
+export interface StoreStatus {
+  path: string;
+  initialized: boolean;
+  schemaVersion: number | null;
+  sessions: number | null;
+  records: number | null;
+  /**
+   * Canonical records the self-referential echo filter keeps out of the index
+   * (filter.ts): the transcripts of agents asking Ledger itself. It is the
+   * derived set the index's own writes maintain, so it describes the index that
+   * is answering reads — not a claim about what a future derivation would do.
+   * Null when that description cannot be made: no store exists at this path,
+   * the store predates the filter's table, the index was derived under other
+   * rules (`fts_format`), or the derivation this build stamps at open is still
+   * pending (`fts_reindex` — deferred under critical pressure, or interrupted)
+   * — unknown, not zero.
+   */
+  echoFiltered: number | null;
+  /** Null when no store exists or the store predates the diagnostics table. */
+  diagnostics: DiagnosticsSummary | null;
+  /**
+   * Sessions deleted by the user through the explicit verb, and the records
+   * those deletions still hold back — the disclosure that makes the two counts
+   * above honest, since they deliberately include the rows a `suppress`
+   * deletion keeps (reader.ts's `countSessions` carries the argument: the store
+   * *holds* them and the audit is why). Both are 0, never null, for a store
+   * that predates the tombstone table: no build could have written a tombstone
+   * into it, so the set is empty rather than unknown.
+   */
+  tombstones: number | null;
+  tombstonedRecords: number | null;
+  /**
+   * Durable per-source scan coverage, in source order. Null when no store
+   * exists or the store predates the coverage table; empty when the table
+   * exists but no pass has scanned yet — unknown is not the same as none.
+   */
+  coverage: SourceCoverage[] | null;
+  /** Absent until a scheduled discovery pass is recorded; never implies full archival retention. */
+  discovery?: DiscoverySummary;
+}
+
+export type StoreStatusErrorCode =
+  | "STORE_UNREADABLE"
+  | "SCHEMA_MISMATCH"
+  | "FORMAT_MISMATCH"
+  | "STORE_SCHEMA_INCOMPLETE";
+
+export class StoreStatusError extends Error {
+  readonly code: StoreStatusErrorCode;
+
+  constructor(code: StoreStatusErrorCode, message: string) {
+    super(message);
+    this.name = "StoreStatusError";
+    this.code = code;
+  }
+}
+
+/**
+ * Inspect a store path without initializing, migrating, or creating anything.
+ * A missing or empty store is normal and returns initialized=false; callers
+ * decide how to render it. Anything unreadable, unsupported, or missing the
+ * read schema raises StoreStatusError — the same typed ladder the read verbs
+ * refuse with (`classifyReadStore`), so the diagnostic verb cannot report a
+ * store the read path would refuse, or vice versa.
+ */
+export function readStoreStatus(dbPath: string): StoreStatus {
+  const store = classifyReadStore(dbPath);
+
+  if (store.state === "not_initialized") {
+    return {
+      path: dbPath,
+      initialized: false,
+      schemaVersion: null,
+      sessions: null,
+      records: null,
+      echoFiltered: null,
+      diagnostics: null,
+      coverage: null,
+      tombstones: null,
+      tombstonedRecords: null,
+    };
+  }
+  if (store.state !== "ok") throw statusRefusal(dbPath, store);
+
+  const db = store.db;
+  try {
+    const sessions =
+      db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM sessions").get()?.count ?? 0;
+    const records =
+      db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM records").get()?.count ?? 0;
+    // A store written by an older Ledger is readable but has no diagnostics
+    // table; report that as unknown coverage rather than as zero findings.
+    const hasDiagnostics = !!db
+      .query<{ name: string }, []>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='ingest_diagnostics' LIMIT 1;",
+      )
+      .get();
+    // Same rule for durable coverage, one schema later: a pre-v12 store has
+    // never recorded a verdict, so its answer is unknown, not "no sources".
+    const hasCoverage = !!db
+      .query<{ name: string }, []>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='source_coverage' LIMIT 1;",
+      )
+      .get();
+    // The echo set is a claim about the index, so it is published only when the
+    // derivation under *this* build's rules completed. A `fts_format` naming
+    // other rules means the served index was re-derived by another build, whose
+    // re-projection cannot maintain the set (it rewrites the index whole); a
+    // pending `fts_reindex` means the derivation this build stamps at open has
+    // not run yet — deferred under critical pressure, or left by a crash — with
+    // the lookups still in the served index. Publishing the empty set as a
+    // count there would read as "this store filters nothing" while searches
+    // echo; unknown is the honest answer, and the next writer pass derives it.
+    const hasMeta = !!db
+      .query<{ name: string }, []>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_meta' LIMIT 1;",
+      )
+      .get();
+    const derivation = hasMeta
+      ? db
+          .query<{ format: string | null; pending: number | null }, []>(
+            `SELECT (SELECT value FROM schema_meta WHERE key = 'fts_format') AS format,
+                    (SELECT 1 FROM schema_meta WHERE key = 'fts_reindex') AS pending;`,
+          )
+          .get()
+      : null;
+    const echoFiltered =
+      derivation?.format === FTS_FORMAT && !derivation.pending ? echoCount(db) : null;
+    // The tombstones themselves, and what they still hold back. A store without
+    // the table predates the verb, so its answer is an empty set rather than
+    // unknown — this is the one field here whose absence is a fact, not a gap.
+    const hasTombstones = !!db
+      .query<{ name: string }, []>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='session_tombstones' LIMIT 1;",
+      )
+      .get();
+    const tombstones = hasTombstones
+      ? db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM session_tombstones").get()?.count ?? 0
+      : 0;
+    const tombstonedRecords = hasTombstones
+      ? db
+          .query<{ count: number }, []>(
+            `SELECT COUNT(*) AS count FROM records r
+             WHERE EXISTS (SELECT 1 FROM session_tombstones t WHERE t.session_id = r.session_id)`,
+          )
+          .get()?.count ?? 0
+      : 0;
+    const discovery = readDiscoverySummary(db);
+    return {
+      ...(discovery ? { discovery } : {}),
+      path: dbPath,
+      initialized: true,
+      schemaVersion: store.schemaVersion,
+      sessions,
+      records,
+      echoFiltered,
+      diagnostics: hasDiagnostics ? readDiagnosticsSummary(db) : null,
+      tombstones,
+      tombstonedRecords,
+      coverage: hasCoverage
+        // Projected field by field rather than spread minus a key: this is a
+        // published payload, so a column added later must be published
+        // deliberately, not by leaking through a rest element.
+        ? readSourceCoverage(db).map((row) => ({
+          source: row.source,
+          root: row.root,
+          status: row.status,
+          discovered: row.discovered,
+          detail: row.detail,
+          updated_at: row.updated_at,
+        }))
+        : null,
+    };
+  } finally {
+    db.close();
+  }
+}
+
+/** A classified refusal, in this verb's own wording. */
+function statusRefusal(dbPath: string, store: ReadStoreRefusal): StoreStatusError {
+  switch (store.state) {
+    case "foreign":
+      return new StoreStatusError(
+        "STORE_UNREADABLE",
+        `Database at ${dbPath} exists but is not an initialized Ledger store (no sessions table).`,
+      );
+    case "unreadable":
+      return new StoreStatusError(
+        "STORE_UNREADABLE",
+        store.phase === "open"
+          ? `Cannot open database at ${dbPath}: ${store.detail}`
+          : `Cannot read database at ${dbPath}: ${store.detail}. The file may be corrupt or not a SQLite database.`,
+      );
+    case "schema_mismatch":
+      return new StoreStatusError(
+        "SCHEMA_MISMATCH",
+        `Database schema version ${store.schemaVersion} at ${dbPath} is newer than supported schema version ${SCHEMA_VERSION}.`,
+      );
+    case "format_mismatch":
+      return new StoreStatusError("FORMAT_MISMATCH", formatMismatchMessage(dbPath, store));
+    case "incomplete":
+      return new StoreStatusError(
+        "STORE_SCHEMA_INCOMPLETE",
+        incompleteStoreMessage(dbPath, store.firstGap, store.schemaVersion),
+      );
+  }
+}

package/legacy/src/sync-preparation.ts

@@ -0,0 +1,45 @@
+import { TrajectoryDb } from "./db.js";
+import { sessionNeedsIngest } from "./ingest.js";
+import { scanSessions, type SessionScanOptions } from "./scanner.js";
+
+export interface SyncPreparationJob {
+  protocol_version: 2;
+  db_path: string;
+  sources?: string[];
+  roots?: Record<string, string>;
+  scan_limit: number;
+  max_record_bytes: number;
+}
+export interface SyncPreparationReply {
+  protocol_version: 2;
+  type: "sync_preparation_result";
+  discovered: number;
+  init_ms: number;
+  scan_ms: number;
+  queue_ms: number;
+}
+
+/**
+ * Initialization, migration, native discovery and queue admission are all
+ * synchronous-I/O-bearing work. Production runs this entire phase in a leaf
+ * child, so a blocked scan or migration cannot wedge the lease-holding parent.
+ * The child inherits write authority only after its caller checks the lease.
+ */
+export async function prepareSyncStore(job: SyncPreparationJob): Promise<SyncPreparationReply> {
+  const started = performance.now();
+  const db = new TrajectoryDb(job.db_path);
+  const init_ms = performance.now() - started;
+  try {
+    const scanStarted = performance.now();
+    const options: SessionScanOptions = {
+      sources: job.sources, roots: job.roots, limit: job.scan_limit,
+      discovery: db.getDiscoveryCheckpoints(),
+    };
+    const result = await scanSessions(options);
+    const scan_ms = performance.now() - scanStarted;
+    const queueStarted = performance.now();
+    db.admitDiscovery(result, item => sessionNeedsIngest(db, item, { maxRecordBytes: job.max_record_bytes }), new Date());
+    return { protocol_version: 2, type: "sync_preparation_result", discovered: result.sessions.length,
+      init_ms, scan_ms, queue_ms: performance.now() - queueStarted };
+  } finally { db.close(); }
+}