switchroom 0.21.17 → 0.21.18

package/telegram-plugin/uat/flip/recall-log.ts

@@ -0,0 +1,178 @@
+/**
+ * recall_log.jsonl reader for the Memory v2 M3 directive-flip UAT gate.
+ *
+ * The recall hook appends one JSON row per turn to
+ * `<agent>/.claude/plugins/data/hindsight-memory-inline/state/recall_log.jsonl`
+ * (see `vendor/hindsight-memory/scripts/recall.py` `_write_recall_log`). Three
+ * fields on each row make the flip measurable WITHOUT a model:
+ *
+ *   - `directive_count`     — how many active directives were injected into
+ *                             the `<active_directives>` block that turn.
+ *   - `directives_omitted`  — how many the MAX_DIRECTIVES cap dropped.
+ *   - `directive_ids`       — the exact ids injected, priority-descending.
+ *
+ * The flip's whole point is that AFTER `memory.inject_directives:false` no
+ * directives are injected — so `directive_count` collapses to 0 and
+ * `directive_ids` empties. This reader + {@link directiveInjectionDelta} turn
+ * that into a deterministic before/after assertion the gate can fail on.
+ *
+ * HONEST SCOPE NOTE (real-source finding): recall_log rows carry directive
+ * COUNTS and IDS, not rendered token bytes — the byte/token size of the
+ * residue lives in the M2 residue harness (`src/memory/directive-residue.ts`)
+ * and the Tier-1 rules-block budget, not here. So the "token delta" the gate
+ * consumes from this file is a directive-injection-VOLUME delta (count of
+ * directives no longer injected), which is the recall_log's real signal;
+ * pairing it with the residue harness's byte number is the caller's job.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+/** One recall_log.jsonl row. Only the fields this gate reads are typed; the
+ *  row carries many more (see recall.py) and they pass through untouched. */
+export interface RecallLogRow {
+  /** ISO-8601 UTC timestamp (`YYYY-MM-DDTHH:MM:SSZ`). */
+  ts?: string;
+  directive_count?: number | null;
+  directives_omitted?: number | null;
+  directive_ids?: string[] | null;
+  [k: string]: unknown;
+}
+
+/** Absolute path to an agent's recall_log.jsonl. */
+export function recallLogPath(agentsDir: string, agent: string): string {
+  return join(
+    agentsDir,
+    agent,
+    ".claude",
+    "plugins",
+    "data",
+    "hindsight-memory-inline",
+    "state",
+    "recall_log.jsonl",
+  );
+}
+
+export interface ReadRecallLogOptions {
+  /** Root agents dir. Defaults to `~/.switchroom/agents`. */
+  agentsDir?: string;
+  /** Return only the last N rows (the "tail"). Omit for all rows. */
+  tail?: number;
+}
+
+/**
+ * Read + parse an agent's recall_log.jsonl. Tolerant: a blank or malformed
+ * line is skipped, not thrown on (the log is append-only and the last line can
+ * be a partial write). Missing file ⇒ empty array. Rows come back in file
+ * order (oldest first); `tail` slices the most-recent N.
+ */
+export function readRecallLog(agent: string, opts: ReadRecallLogOptions = {}): RecallLogRow[] {
+  const agentsDir = opts.agentsDir ?? join(homedir(), ".switchroom", "agents");
+  const path = recallLogPath(agentsDir, agent);
+  if (!existsSync(path)) return [];
+  const rows: RecallLogRow[] = [];
+  for (const line of readFileSync(path, "utf8").split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) continue;
+    try {
+      rows.push(JSON.parse(trimmed) as RecallLogRow);
+    } catch {
+      // partial / corrupt line — skip.
+    }
+  }
+  return typeof opts.tail === "number" ? rows.slice(-opts.tail) : rows;
+}
+
+function num(v: number | null | undefined): number {
+  return typeof v === "number" && Number.isFinite(v) ? v : 0;
+}
+
+export interface InjectionSummary {
+  rowCount: number;
+  /** Peak `directive_count` seen across the window. */
+  maxDirectiveCount: number;
+  /** `directive_count` on the most-recent row (null when no rows). */
+  lastDirectiveCount: number | null;
+  /** Union of every id that appeared in any row's `directive_ids`. */
+  everInjectedIds: string[];
+  /** Peak `directives_omitted` — >0 means the cap was dropping real rules. */
+  maxDirectivesOmitted: number;
+}
+
+/** Summarize the directive-injection signal over a window of rows. */
+export function summarizeInjection(rows: readonly RecallLogRow[]): InjectionSummary {
+  const ids = new Set<string>();
+  let maxCount = 0;
+  let maxOmitted = 0;
+  for (const r of rows) {
+    maxCount = Math.max(maxCount, num(r.directive_count));
+    maxOmitted = Math.max(maxOmitted, num(r.directives_omitted));
+    for (const id of r.directive_ids ?? []) ids.add(id);
+  }
+  const last = rows.length > 0 ? rows[rows.length - 1] : null;
+  return {
+    rowCount: rows.length,
+    maxDirectiveCount: maxCount,
+    lastDirectiveCount: last ? num(last.directive_count) : null,
+    everInjectedIds: [...ids],
+    maxDirectivesOmitted: maxOmitted,
+  };
+}
+
+export interface DirectiveInjectionDelta {
+  baseline: InjectionSummary;
+  postflip: InjectionSummary;
+  /** Drop in peak injected-directive volume (baseline − postflip). Positive is
+   *  the expected direction: the flip stopped injecting directives. */
+  volumeDelta: number;
+  /** True when the postflip window injected ZERO directives — the flip's
+   *  success condition on the recall_log side. */
+  postflipFullySuppressed: boolean;
+  /** Ids still injected postflip (should be empty after a real flip). */
+  residualIds: string[];
+}
+
+/**
+ * Compare a baseline window (before the flip) against a postflip window and
+ * report the directive-injection-volume delta. `postflipFullySuppressed` is
+ * the gate's success condition: after `inject_directives:false`, no directive
+ * is injected, so postflip `maxDirectiveCount` is 0 and `residualIds` empty.
+ *
+ * This is DETERMINISTIC and pure — the caller supplies the two windows
+ * (typically `readRecallLog(...).slice()` around the flip timestamp); this does
+ * no IO of its own.
+ */
+export function directiveInjectionDelta(
+  baselineRows: readonly RecallLogRow[],
+  postflipRows: readonly RecallLogRow[],
+): DirectiveInjectionDelta {
+  const baseline = summarizeInjection(baselineRows);
+  const postflip = summarizeInjection(postflipRows);
+  return {
+    baseline,
+    postflip,
+    volumeDelta: baseline.maxDirectiveCount - postflip.maxDirectiveCount,
+    postflipFullySuppressed: postflip.maxDirectiveCount === 0 && postflip.everInjectedIds.length === 0,
+    residualIds: postflip.everInjectedIds,
+  };
+}
+
+/** Split a single row stream into baseline/postflip windows at a flip
+ *  timestamp (ISO string). Rows with `ts < flipTs` are baseline; `ts >=
+ *  flipTs` are postflip. A row missing `ts` is treated as baseline (it
+ *  predates the instrumented flip). Convenience over hand-slicing. */
+export function partitionByFlip(
+  rows: readonly RecallLogRow[],
+  flipTs: string,
+): { baseline: RecallLogRow[]; postflip: RecallLogRow[] } {
+  const flipMs = Date.parse(flipTs);
+  const baseline: RecallLogRow[] = [];
+  const postflip: RecallLogRow[] = [];
+  for (const r of rows) {
+    const t = r.ts ? Date.parse(r.ts) : NaN;
+    if (Number.isNaN(t) || Number.isNaN(flipMs) || t < flipMs) baseline.push(r);
+    else postflip.push(r);
+  }
+  return { baseline, postflip };
+}

package/telegram-plugin/uat/flip/report.ts

@@ -0,0 +1,95 @@
+/**
+ * Markdown report renderer for the M3 directive-flip UAT gate.
+ *
+ * Same shape as the agent-self-sufficiency runner's report
+ * (`telegram-plugin/uat/runners/report.ts`): a headline verdict, a per-agent ×
+ * per-check pass/fail matrix an operator reads in one glance, then a verbatim
+ * triage list of every failing check so a PR reviewer can diff without
+ * re-running. Pure — takes the gate verdicts, returns a string.
+ */
+
+import type { GateRun, GateVerdict, GateCheck } from "./gate.js";
+
+export interface FlipReportOptions {
+  startedAt?: Date;
+  durationSeconds?: number;
+}
+
+function mark(c: GateCheck): string {
+  if (c.skipped) return "—";
+  return c.pass ? "✅" : "❌";
+}
+
+export function renderFlipReport(run: GateRun, opts: FlipReportOptions = {}): string {
+  const { verdicts } = run;
+  const passed = verdicts.filter((v) => v.pass).length;
+  const lines: string[] = [];
+
+  lines.push("# M3 directive-flip UAT gate report");
+  lines.push("");
+  if (opts.startedAt) lines.push(`- **Run start:** ${opts.startedAt.toISOString()}`);
+  if (typeof opts.durationSeconds === "number")
+    lines.push(`- **Duration:** ${opts.durationSeconds.toFixed(1)}s`);
+  lines.push(`- **Agents:** ${verdicts.map((v) => v.agent).join(", ") || "(none)"}`);
+  lines.push(`- **Verdict:** ${run.exitCode === 0 ? "PASS" : "FAIL"} (${passed}/${verdicts.length} agents green)`);
+  lines.push("");
+
+  // Per-agent × per-check matrix. Check names are stable across agents, so use
+  // the first verdict's check order as the column set.
+  const checkNames = verdicts[0]?.checks.map((c) => c.name) ?? [];
+  if (checkNames.length > 0 && verdicts.length > 0) {
+    lines.push("## Check matrix");
+    lines.push("");
+    lines.push(`| Agent | Verdict | ${checkNames.map((n) => shortName(n)).join(" | ")} |`);
+    lines.push(`|---|---|${checkNames.map(() => "---").join("|")}|`);
+    for (const v of verdicts) {
+      const byName = new Map(v.checks.map((c) => [c.name, c]));
+      const cells = checkNames.map((n) => {
+        const c = byName.get(n);
+        return c ? mark(c) : "?";
+      });
+      lines.push(`| \`${v.agent}\` | ${v.pass ? "PASS" : "FAIL"} | ${cells.join(" | ")} |`);
+    }
+    lines.push("");
+    lines.push("_Legend: ✅ pass · ❌ fail · — skipped (input not supplied)._");
+    lines.push("");
+  }
+
+  // Triage — every failing check, verbatim detail.
+  const failing: Array<{ agent: string; check: GateCheck }> = [];
+  for (const v of verdicts) {
+    for (const c of v.checks) {
+      if (!c.pass && !c.skipped) failing.push({ agent: v.agent, check: c });
+    }
+  }
+  lines.push("## Triage — failing checks");
+  lines.push("");
+  if (failing.length === 0) {
+    lines.push("No failing checks. Every ran check passed.");
+  } else {
+    lines.push("| Agent | Check | Detail |");
+    lines.push("|---|---|---|");
+    for (const f of failing) {
+      lines.push(`| \`${f.agent}\` | ${escapeCell(f.check.name)} | ${escapeCell(f.check.detail)} |`);
+    }
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+/** Render just one agent's verdict as a compact block (for a per-agent log). */
+export function renderVerdictLine(v: GateVerdict): string {
+  const parts = v.checks.map((c) => `${mark(c)} ${shortName(c.name)}`);
+  return `${v.pass ? "PASS" : "FAIL"} ${v.agent}: ${parts.join(" · ")}`;
+}
+
+/** Drop the `tierN: ` / `recall_log: ` prefix for a compact column header. */
+function shortName(name: string): string {
+  const idx = name.indexOf(": ");
+  return idx === -1 ? name : name.slice(idx + 2);
+}
+
+function escapeCell(s: string): string {
+  return s.replace(/\|/g, "\\|").replace(/\n/g, " ").replace(/`/g, "ʼ");
+}