trajectories 0.0.0 → 0.1.1

package/legacy/src/filter.ts

@@ -0,0 +1,602 @@
+/**
+ * The self-referential echo filter: the records of an agent *asking Ledger
+ * itself* are excluded from the derived search index, so a store cannot answer
+ * a query with the transcript of the previous identical query.
+ *
+ * The failure this names is concrete and was observed daily: agents that use
+ * Ledger are themselves ingested by Ledger, so every `ledger search` a session
+ * ran is a record that mentions the query words — and comes back as a hit for
+ * them. Wake ships the same exclusion for its own surface (`wake_*` MCP tools,
+ * `wake-cli`/`wake-mcp` shells; `crates/wake-core/src/adapters/mod.rs#L425-L447`,
+ * `#L469-L535`) and its stance is the one taken here:
+ *
+ * - Only the *lookup* is excluded. Dialogue records are never touched, and a
+ *   `git commit -m "…ledger…"`, a `cd` into a checkout, a `grep` for the word,
+ *   a `Read` of a source file, or a `bun test` in this repo stay searchable —
+ *   this repository's own development sessions must survive the filter intact.
+ * - Matching is structural, never a substring. A tool name must *end* with a
+ *   Ledger tool name at a word boundary (client prefixes vary:
+ *   `mcp__ledger__ledger_search`, `mcp_ledger_search`, bare `ledger_search`;
+ *   `awake_search` is not one), and a shell command must *name a Ledger binary
+ *   in command position*: the first word, or the word after a separator
+ *   (`&&`, `||`, `;`, `|`, `&`, newline), a command substitution, or a prefix
+ *   word (`sudo`, `env`, `time`, …). `grep ledger`, `cat ledger.ts`,
+ *   `echo ledger`, and a `/Volumes/dev/workspace/ledger` path in an argument
+ *   match neither rule.
+ * - Quoting is lexical, not textual: a separator inside quotes is text
+ *   (`rg -n "a|ledger|b"` is one command running `rg`), and a heredoc body is
+ *   data, not commands (`cat > notes.md <<'EOF'` writing a line that begins
+ *   `ledger search …` is a document, not a lookup).
+ * - A tool *result* is excluded only when its own call is: the pairing is the
+ *   record's `tool_call_id` against an excluded call in the same session, never
+ *   a guess from the result text.
+ *
+ * The rule is one pure function over the three canonical fields that carry the
+ * decision — `record_type`, `tool_name`, `tool_arguments_json` — so ingest, the
+ * index re-derivation, the parity gate, and the status verb all classify a
+ * record the same way, and a record is a lookup by its canonical content, not
+ * by a flag someone could forget to maintain.
+ *
+ * What is *not* here, deliberately: the thin subclass of shell shapes this rule
+ * cannot see (a JSON shape with no `command`/`cmd`/`input` field, a program
+ * invoked from inside a script or a `subprocess.run` list). Wrapped scripts —
+ * `bash -c "…"`, `bash -lc '…'`, argv arrays — ARE segmented; see
+ * `commandSegments`. The residual shapes cost a noisy hit, never a hidden
+ * record: the rule errs toward indexing.
+ */
+import type { Database } from "bun:sqlite";
+
+/**
+ * The tools a Ledger MCP server exposes. A client namespaces them under its own
+ * prefix, so the name is matched as a word-boundary *suffix* — Wake's rule
+ * (`ends_with_word`, `adapters/mod.rs#L517-L535`) — and not as a substring.
+ */
+const LOOKUP_TOOLS: readonly string[] = [
+  "ledger_search",
+  "ledger_recent",
+  "ledger_status",
+  "ledger_sync",
+  "ledger_scan",
+  "ledger_build",
+  "ledger_watch",
+  "ledger_models",
+];
+
+/**
+ * Claude Code's MCP namespacing (`mcp__<server>__<tool>`). Every tool a Ledger
+ * MCP server exposes is a lookup whatever it is called, so a name carrying the
+ * server segment is excluded without consulting {@link LOOKUP_TOOLS} — the
+ * server's own namespace is the stronger statement.
+ */
+const MCP_SERVER_SEGMENT = "mcp__ledger__";
+
+/**
+ * The binaries that *are* the lookup, by basename (`ledger`, `ledger-core`,
+ * `traj-bridge` and the `.ts`/`.js`/`.exe` forms `bun` runs and Windows
+ * installs). A path is accepted (`/Users/…/.local/bin/ledger`,
+ * `./bin/ledger.ts`) because a tokenizer cannot know PATH.
+ */
+const BINARY_BASENAMES: Record<string, true> = {
+  trajectories: true,
+  "trajectories.ts": true,
+  "trajectories.js": true,
+  "trajectories.exe": true,
+  ledger: true,
+  "ledger.ts": true,
+  "ledger.js": true,
+  "ledger.exe": true,
+  "ledger-core": true,
+  "ledger-core.ts": true,
+  "ledger-core.js": true,
+  "ledger-core.exe": true,
+  "traj-bridge": true,
+  "traj-bridge.ts": true,
+  "traj-bridge.js": true,
+  "traj-bridge.exe": true,
+};
+
+/** Runners whose next word is the program: `bun run ./bin/ledger.ts`. */
+const RUNNERS: Record<string, true> = { bun: true, "bun.exe": true, bunx: true, "bunx.exe": true };
+
+/** Shell tools whose arguments may hide the command inside an input string. */
+const SHELL_TOOL_NAMES: Record<string, true> = {
+  bash: true,
+  exec: true,
+  exec_command: true,
+  shell: true,
+  shell_command: true,
+  terminal: true,
+};
+
+/** A path-shaped token, so `pwd)/ledger` and `$X/ledger` are not read as paths. */
+const PATH_TOKEN = /^[A-Za-z0-9._~+/@%:\\-]+$/;
+
+/** Words that precede a command without being it. */
+const PREFIX_WORDS: Record<string, true> = {
+  sudo: true,
+  command: true,
+  env: true,
+  time: true,
+  nohup: true,
+  nice: true,
+  exec: true,
+  builtin: true,
+  do: true,
+  then: true,
+  else: true,
+  if: true,
+  while: true,
+  until: true,
+};
+
+/** `FOO=bar` assignments may also precede a command. */
+const ENV_ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/;
+
+/** A heredoc redirection: `<<EOF`, `<<'EOF'`, `<<-"EOF"`. */
+const HEREDOC_START = /<<(-?)(?:(["']?)([A-Za-z0-9_]+)\2)/y;
+/** A shell wrapper flag that makes the next word a script: `-c`, `-lc`, `-ic`. */
+const SHELL_WRAPPER_FLAG = /^-[A-Za-z]*c[A-Za-z]*$/;
+/** A shell executable by basename, for the `bash -lc "…"` shape. */
+const SHELL_BASENAMES: Record<string, true> = {
+  bash: true,
+  sh: true,
+  dash: true,
+  zsh: true,
+  ksh: true,
+  fish: true,
+};
+
+/** The basename of a path-shaped token, or null when the token is not one. */
+function basename(token: string): string | null {
+  const bare = token.replace(/^["']|["']$/g, "");
+  if (!PATH_TOKEN.test(bare)) return null;
+  return bare.slice(Math.max(bare.lastIndexOf("/"), bare.lastIndexOf("\\")) + 1);
+}
+
+/** Whether the token names a Ledger binary in path or extension form. */
+function binaryBasename(token: string): string | null {
+  const base = basename(token);
+  return base !== null && BINARY_BASENAMES[base] ? base : null;
+}
+
+/**
+ * A command string with its heredoc bodies removed. Bodies are data the shell
+ * feeds to a program's stdin — a `<<'EOF'` line that begins `ledger search` is
+ * what the program printed or wrote, not what ran.
+ */
+function withoutHeredocs(command: string): string {
+  let out = "";
+  let quote: string | null = null;
+  for (let i = 0; i < command.length; i++) {
+    const char = command[i];
+    if (quote !== null) {
+      out += char;
+      if (char === "\\" && quote === '"') out += command[++i] ?? "";
+      else if (char === quote) quote = null;
+      continue;
+    }
+    if (char === "'" || char === '"') {
+      quote = char;
+      out += char;
+      continue;
+    }
+    if (char === "\\") {
+      out += char + (command[++i] ?? "");
+      continue;
+    }
+    if (char === "<" && command[i + 1] === "<") {
+      HEREDOC_START.lastIndex = i;
+      const start = HEREDOC_START.exec(command);
+      if (start !== null) {
+        const [, dashes, , delimiter] = start;
+        let cursor = command.indexOf("\n", HEREDOC_START.lastIndex);
+        let end = command.length;
+        while (cursor !== -1) {
+          const lineEnd = command.indexOf("\n", cursor + 1);
+          const line = command.slice(cursor + 1, lineEnd === -1 ? command.length : lineEnd);
+          if ((dashes ? line.replace(/^\t+/, "") : line) === delimiter) {
+            end = lineEnd === -1 ? command.length : lineEnd + 1;
+            break;
+          }
+          cursor = lineEnd;
+        }
+        out += "\n";
+        i = end - 1;
+        continue;
+      }
+    }
+    out += char;
+  }
+  return out;
+}
+
+/**
+ * A command string split at its command positions: the start, and every word
+ * after an unquoted separator (`&&`, `||`, `;`, `|`, `&`, newline) or command
+ * substitution (`$(`, backtick). Everything after a separator is judged
+ * separately — a later command in a `&&` chain is caught, an argument that
+ * merely mentions a binary is not.
+ */
+function commandSegments(command: string): string[] {
+  const stripped = withoutHeredocs(command);
+  const segments: string[] = [];
+  let current = "";
+  let quote: string | null = null;
+  for (let i = 0; i < stripped.length; i++) {
+    const char = stripped[i];
+    if (quote !== null) {
+      current += char;
+      if (char === "\\" && quote === '"') current += stripped[++i] ?? "";
+      else if (char === quote) quote = null;
+      continue;
+    }
+    if (char === "'" || char === '"') {
+      quote = char;
+      current += char;
+      continue;
+    }
+    if (char === "\\") {
+      current += char + (stripped[++i] ?? "");
+      continue;
+    }
+    if (char === "\n" || char === ";" || char === "&" || char === "|" || char === "`") {
+      segments.push(current);
+      current = "";
+      if ((char === "&" || char === "|") && stripped[i + 1] === char) i++;
+      continue;
+    }
+    if (char === "$" && stripped[i + 1] === "(") {
+      segments.push(current);
+      current = "";
+      i++;
+      continue;
+    }
+    current += char;
+  }
+  segments.push(current);
+  return segments;
+}
+
+/** One segment's words, split on unquoted whitespace; quotes stay on the word. */
+function segmentWords(segment: string): string[] {
+  const words: string[] = [];
+  let current = "";
+  let quote: string | null = null;
+  for (let i = 0; i < segment.length; i++) {
+    const char = segment[i];
+    if (quote !== null) {
+      current += char;
+      if (char === "\\" && quote === '"') current += segment[++i] ?? "";
+      else if (char === quote) quote = null;
+      continue;
+    }
+    if (char === "'" || char === '"') {
+      quote = char;
+      current += char;
+      continue;
+    }
+    if (/\s/.test(char)) {
+      if (current !== "") words.push(current);
+      current = "";
+      continue;
+    }
+    current += char;
+  }
+  if (current !== "") words.push(current);
+  return words;
+}
+
+/**
+ * Whether one command segment runs a Ledger binary: skip the words that may
+ * precede a command, then require a Ledger binary in command position — or, for
+ * a runner (`bun`), the program it is told to run. A shell wrapper's `-c`
+ * script is one more command string, judged the same way.
+ */
+function segmentNamesLedgerBinary(segment: string): boolean {
+  const words = segmentWords(segment);
+  let i = 0;
+  while (i < words.length && (PREFIX_WORDS[words[i]] || ENV_ASSIGNMENT.test(words[i]))) i++;
+  if (i >= words.length) return false;
+  const base = basename(words[i]);
+  if (base === null) return false;
+  if (RUNNERS[base]) {
+    i++;
+    while (i < words.length && (words[i].startsWith("--") || words[i] === "run" || words[i] === "x")) i++;
+    return i < words.length && binaryBasename(words[i]) !== null;
+  }
+  if (binaryBasename(words[i]) !== null) return true;
+  if (!SHELL_BASENAMES[base] || !SHELL_WRAPPER_FLAG.test(words[i + 1] ?? "")) return false;
+  // The script is one more command string. It is still wearing the quotes the
+  // shell took off (`bash -lc "ledger search x"` reaches `segmentWords` as one
+  // word), so they are stripped and the same segmentation judges it — which
+  // also respects quotes *inside* the script, so an alternation that merely
+  // contains the word (`bash -lc "rg -n 'ard|ledger|review' docs"`) is not a
+  // run, exactly as it is not one outside a wrapper.
+  const script = words
+    .slice(i + 2)
+    .join(" ")
+    .replace(/^["']|["']$/g, "");
+  return commandSegments(script).some(segmentNamesLedgerBinary);
+}
+
+/**
+ * The command a shell wrapper was given: `["bash", "-lc", "ledger search"]` is
+ * `ledger search`; any other argv array is joined as written.
+ */
+function joinedArgv(argv: readonly string[]): string {
+  const [head, ...rest] = argv;
+  if (head !== undefined && rest.length > 0 && SHELL_WRAPPER_FLAG.test(rest[0])) {
+    const base = head.replace(/^["']|["']$/g, "").replace(/^.*[/\\]/, "");
+    if (SHELL_BASENAMES[base]) return rest.slice(1).join(" ");
+  }
+  return argv.join(" ");
+}
+
+/** The Codex-style `exec` shape: a call list with `cmd:"…"` inside it. */
+const EMBEDDED_CMD = /\bcmd\s*:\s*(["'])/g;
+
+function embeddedCommands(input: string): string[] {
+  const commands: string[] = [];
+  for (const match of input.matchAll(EMBEDDED_CMD)) {
+    const quote = match[1];
+    const start = match.index + match[0].length;
+    const end = input.indexOf(quote, start);
+    if (end !== -1) commands.push(input.slice(start, end));
+  }
+  return commands;
+}
+
+/**
+ * Every shell command string a tool call's arguments carry: the structured
+ * `command`/`cmd` fields (string or argv array), and — for shell tools whose
+ * input is a call list — the `cmd:"…"` strings inside that input. The
+ * structured fields are read for *any* tool, because a tool that names a
+ * command names a command; the input scan is for shell tool names only, so a
+ * prompt or message body quoting a `cmd:` line is never read as a run.
+ */
+export function commandStrings(
+  toolName: string | null,
+  toolArgumentsJson: string | null,
+): string[] {
+  if (!toolArgumentsJson) return [];
+  let args: unknown;
+  try {
+    args = JSON.parse(toolArgumentsJson);
+  } catch {
+    return [];
+  }
+  if (args === null || typeof args !== "object" || Array.isArray(args)) return [];
+  const fields = args as Record<string, unknown>;
+  const commands: string[] = [];
+  for (const key of ["command", "cmd"]) {
+    const value = fields[key];
+    if (typeof value === "string") commands.push(value);
+    else if (Array.isArray(value) && value.every((part) => typeof part === "string")) {
+      commands.push(joinedArgv(value));
+    }
+  }
+  const input = fields["input"];
+  if (typeof input === "string" && toolName !== null && SHELL_TOOL_NAMES[toolName.toLowerCase()]) {
+    commands.push(...embeddedCommands(input));
+  }
+  return commands;
+}
+
+/** Whether `name` ends with `word` at a word boundary (Wake's `ends_with_word`). */
+function endsWithWord(name: string, word: string): boolean {
+  if (!name.endsWith(word)) return false;
+  const previous = name[name.length - word.length - 1];
+  return previous === undefined || !/[\p{L}\p{N}]/u.test(previous);
+}
+
+/**
+ * Whether one `assistant-tool-call` record is the agent querying Ledger: a
+ * Ledger MCP tool by name, or a shell command that runs a Ledger binary in
+ * command position. Any other record type is never a lookup — dialogue, meta,
+ * and tool *results* are excluded only through their call.
+ */
+export function isEchoCall(
+  recordType: string,
+  toolName: string | null,
+  toolArgumentsJson: string | null,
+): boolean {
+  if (recordType !== "assistant-tool-call") return false;
+  const name = toolName?.trim();
+  if (name) {
+    if (LOOKUP_TOOLS.some((tool) => endsWithWord(name, tool))) return true;
+    if (name.includes(MCP_SERVER_SEGMENT)) return true;
+  }
+  return commandStrings(toolName, toolArgumentsJson).some((command) =>
+    commandSegments(command).some(segmentNamesLedgerBinary),
+  );
+}
+
+/**
+ * The SQL superset of the shell half of {@link isEchoCall}: rows whose
+ * arguments can possibly name a Ledger binary. The LIKE is deliberately loose —
+ * it is the database doing the cheap part of the scan, so the rule only reads
+ * the candidate rows — and it must only ever *broaden* the set: every token the
+ * rule accepts (`ledger`, `ledger-core`, `traj-bridge`, in any path form)
+ * contains `ledger` or `traj-bridge` as written.
+ */
+export const ECHO_CANDIDATE_SQL =
+  "(tool_name LIKE '%ledger%' OR tool_arguments_json LIKE '%ledger%' OR tool_arguments_json LIKE '%traj-bridge%' OR tool_arguments_json LIKE '%trajectories%')";
+
+/** The canonical fields the filter reads from one record. */
+export interface EchoCandidate {
+  readonly id: string;
+  readonly record_type: string;
+  readonly tool_call_id: string | null;
+  readonly tool_name: string | null;
+  readonly tool_arguments_json: string | null;
+}
+
+/** A candidate row read back from `records`, which carries its session too. */
+interface StoredCall extends EchoCandidate {
+  readonly session_id: string;
+}
+
+/** A tool result the pairing half reads. */
+interface ResultRow {
+  id: string;
+  session_id: string;
+  tool_call_id: string | null;
+}
+
+/**
+ * A session-qualified call key. Pairing never crosses a session: a result's
+ * `tool_call_id` names a call in its own transcript, so the key carries the
+ * session and the same client-side id in two sessions cannot pair across them.
+ */
+function pairingKey(sessionId: string, toolCallId: string): string {
+  return `${sessionId}\u0000${toolCallId}`;
+}
+
+/**
+ * The excluded id and `tool_call_id` of one call, when the rule names it a
+ * lookup. The single place `isEchoCall` meets the `records` shape, so the
+ * whole-store and batch forms below cannot classify a call differently — and
+ * the record's *own* type decides: a tool result that happens to carry a tool
+ * name is not a call, or the batch form would exclude it and the next
+ * re-derivation would index it back, the two forms disagreeing about one row.
+ */
+function echoCallOf(call: EchoCandidate): { id: string; toolCallId: string | null } | null {
+  if (!isEchoCall(call.record_type, call.tool_name, call.tool_arguments_json)) return null;
+  return { id: call.id, toolCallId: call.tool_call_id };
+}
+
+/**
+ * The size of the store's derived echo set, or null when the store has no such
+ * table — every store written before the filter, and any store no writer has
+ * opened since. Null is the honest answer there: *unknown*, not zero, because
+ * the rule has not been derived into the store yet. A writer pass creates the
+ * table (CREATE_TABLES_SQL) and the index re-derivation fills it.
+ */
+export function echoCount(db: Database): number | null {
+  const present = db
+    .query<{ name: string }, []>(
+      "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'records_echo' LIMIT 1;",
+    )
+    .get();
+  if (!present) return null;
+  return db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM records_echo;").get()!.count;
+}
+
+/**
+ * The record ids the store excludes: every echo tool call, and every tool
+ * result paired with one.
+ *
+ * One streaming pass over `records` — a call is judged by {@link isEchoCall}
+ * where it is read, and a tool result is checked against the echo calls already
+ * seen — because this is the form the re-derivation, the parity gate, and the
+ * status verb share, and each of them runs it over the whole store: a second
+ * scan, or a candidate pass that re-reads every argument blob, is measurable on
+ * a large store (one pass over this repo's own 1.1 GiB store reads ~5 s cold,
+ * ~0.35 s warm). Results are held until the call set is complete rather than
+ * relying on source order, so a store whose adapter wrote a result ahead of its
+ * call still derives the same index as one that did not.
+ *
+ * Pairing never crosses a session: a result's `tool_call_id` names a call in
+ * its own transcript, so the key carries the session and the same client-side
+ * id in two sessions cannot pair across them.
+ */
+export function echoExcludedIds(db: Database): Set<string> {
+  const excluded = new Set<string>();
+  const echoCallKeys = new Set<string>();
+  const results: ResultRow[] = [];
+  const rows = db
+    .query<StoredCall, []>(
+      `SELECT id, session_id, record_type, tool_call_id, tool_name, tool_arguments_json FROM records
+       WHERE record_type = 'tool'
+          OR (record_type = 'assistant-tool-call' AND ${ECHO_CANDIDATE_SQL})`,
+    )
+    .iterate();
+  for (const row of rows) {
+    if (row.record_type === "tool") {
+      if (row.tool_call_id !== null) results.push({ id: row.id, session_id: row.session_id, tool_call_id: row.tool_call_id });
+      continue;
+    }
+    const echo = echoCallOf(row);
+    if (echo === null) continue;
+    excluded.add(echo.id);
+    if (echo.toolCallId !== null) echoCallKeys.add(pairingKey(row.session_id, echo.toolCallId));
+  }
+  if (echoCallKeys.size === 0) return excluded;
+  for (const result of results) {
+    if (echoCallKeys.has(pairingKey(result.session_id, result.tool_call_id as string))) {
+      excluded.add(result.id);
+    }
+  }
+  return excluded;
+}
+
+/**
+ * One ingest batch's share of the echo filter: the record ids this batch must
+ * keep out of the index, and the `tool_call_id`s of the echo calls behind them.
+ *
+ * The ids are the batch's own rows. The call ids are what the writer needs to
+ * repair rows an *earlier* batch committed: an adapter may write a result
+ * before its call, and the batch that carried the result had nothing to pair it
+ * with — so the call arriving now has to take that result's index row back.
+ * The earlier batch is past rewriting; this is the same derivation one batch
+ * late, and it lands in the call's own transaction so the index and the derived
+ * set cannot disagree about the pair.
+ *
+ * The batch's own rows are classified directly; a result whose call sits in an
+ * earlier batch is resolved through a lookup by `tool_call_id` in the same
+ * session, which is what makes a call and its result in different batches
+ * derive to the same index state as a re-derivation from scratch.
+ */
+export interface BatchEchoes {
+  readonly excluded: ReadonlySet<string>;
+  readonly callIds: ReadonlySet<string>;
+}
+
+export function echoExcludedBatchIds(
+  db: Database,
+  sessionId: string,
+  records: readonly EchoCandidate[],
+): BatchEchoes {
+  const excluded = new Set<string>();
+  const keys = new Set<string>();
+  const callIds = new Set<string>();
+  const pending = new Set<string>();
+  for (const record of records) {
+    const echo = echoCallOf(record);
+    if (echo !== null) {
+      excluded.add(echo.id);
+      if (echo.toolCallId !== null) {
+        keys.add(pairingKey(sessionId, echo.toolCallId));
+        callIds.add(echo.toolCallId);
+      }
+    } else if (record.record_type === "tool" && record.tool_call_id !== null) {
+      pending.add(record.tool_call_id);
+    }
+  }
+  for (const toolCallId of pending) {
+    if (keys.has(pairingKey(sessionId, toolCallId))) pending.delete(toolCallId);
+  }
+  if (pending.size > 0) {
+    // The call may be a row this batch cannot see: read it back from the session
+    // and let the same rule decide, rather than trusting the result's text.
+    const calls = db
+      .query<StoredCall, (string | number)[]>(
+        `SELECT id, session_id, record_type, tool_call_id, tool_name, tool_arguments_json FROM records
+         WHERE session_id = ? AND record_type = 'assistant-tool-call'
+           AND tool_call_id IN (SELECT value FROM json_each(?))`,
+      )
+      .all(sessionId, JSON.stringify([...pending]));
+    for (const call of calls) {
+      const echo = echoCallOf(call);
+      if (echo !== null && echo.toolCallId !== null) {
+        keys.add(pairingKey(sessionId, echo.toolCallId));
+        callIds.add(echo.toolCallId);
+      }
+    }
+  }
+  for (const record of records) {
+    if (record.record_type !== "tool" || record.tool_call_id === null) continue;
+    if (keys.has(pairingKey(sessionId, record.tool_call_id))) excluded.add(record.id);
+  }
+  return { excluded, callIds };
+}