pgroast 0.0.0-stage → 0.1.0

package/src/eval/judge.ts

@@ -0,0 +1,136 @@
+// (b) judge: one model call per case, through the roast LLM backend (critic
+// model, no tools). It compares the case's decisive objections with the
+// objections the roast produced. Buckets are NOT shown to the judge (so a
+// "refuted" label cannot bias it); code looks the bucket up afterwards.
+
+import type { JsonSchema, Roast } from "../roast/schema.ts";
+import type { LlmBackend } from "../roast/llm.ts";
+import type { EvalCase } from "./cases.ts";
+import { bucketedObjections, type JudgeItem } from "./score.ts";
+
+export const JUDGE_SCHEMA: JsonSchema = {
+  type: "object",
+  properties: {
+    items: {
+      type: "array",
+      minItems: 1,
+      items: {
+        type: "object",
+        properties: {
+          decisive_index: { type: "integer", minimum: 1 },
+          result: { type: "string", enum: ["found", "partially", "not_found"] },
+          objection_id: { type: "string", nullable: true },
+          reason: { type: "string" },
+        },
+        required: ["decisive_index", "result", "objection_id", "reason"],
+      },
+    },
+  },
+  required: ["items"],
+};
+
+export const JUDGE_SYSTEM = `You grade an automated reviewer of PostgreSQL proposals.
+
+You get (1) the objection(s) that actually decided a historical proposal (who raised it, when, a quote), and (2) the list of objections the reviewer produced without seeing that discussion.
+
+For EACH decisive objection, answer:
+- "found": one reviewer objection states the same core reason: the same mechanism, risk or concern, even in different words or with a different example.
+- "partially": a reviewer objection is in the same area but weaker or more generic (e.g. "maintenance burden" where the real reason was a specific maintenance problem), or covers only part of the real reason.
+- "not_found": no reviewer objection matches.
+
+Give the id of the single best-matching reviewer objection (null for not_found) and one short sentence explaining the match or the miss. Judge only by the text given; do not use what you may remember about the real thread. Be strict: topical overlap is not a match. Reply with JSON only.`;
+
+const clip = (s: string, n: number) => (s.length > n ? s.slice(0, n) + "…" : s);
+
+/** The decisive objections as shown to the classifier and the judge. */
+export function decisiveList(c: EvalCase): string {
+  return c.decisive_objections.map((d, i) => {
+    const src = d.message_url ? "" : d.source ? `\n    (source: ${clip(String(d.source), 200)})` : "";
+    return `D${i + 1}. ${d.who ?? "unknown"}${d.date ? `, ${d.date}` : ""}${d.category ? ` [${d.category}]` : ""}\n    "${clip(String(d.quote ?? ""), 1500)}"${src}`;
+  }).join("\n");
+}
+
+export function judgePrompt(c: EvalCase, r: Roast): string {
+  const dec = decisiveList(c);
+  const objs = bucketedObjections(r).map((o) =>
+    `- id: ${o.id}\n  title: ${o.title}\n  severity: ${o.severity}; category: ${o.category}; level: ${o.level}\n  failure scenario: ${clip(o.failure_scenario, 900)}\n  argument: ${clip(o.argument, 900)}`).join("\n");
+  return `Proposal: ${c.title}
+Outcome: ${c.outcome}
+
+Decisive objection(s) from the real discussion:
+${dec}
+
+Reviewer objections (${bucketedObjections(r).length}):
+${objs || "(none)"}
+
+Return {"items": [...]} with one item per decisive objection; decisive_index is the number after D.`;
+}
+
+export async function runJudge(c: EvalCase, r: Roast, backend: LlmBackend, model: string): Promise<JudgeItem[]> {
+  const out = await backend.complete<{ items: JudgeItem[] }>({
+    system: JUDGE_SYSTEM, prompt: judgePrompt(c, r), model, schema: JUDGE_SCHEMA, label: "judge",
+  });
+  return out.items.filter((i) => i.decisive_index >= 1 && i.decisive_index <= c.decisive_objections.length);
+}
+
+// ---------------------------------------------------------------- level classification
+// A separate call that sees only the case (never the roast), so the label is
+// independent of what the roast said. Levels from prompts/committer-paranoia.md.
+
+export type DecisiveLevel = "idea" | "design" | "implementation";
+
+export interface ClassifyItem {
+  decisive_index: number;
+  level: DecisiveLevel;
+  reason: string;
+}
+
+export const CLASSIFY_SCHEMA: JsonSchema = {
+  type: "object",
+  properties: {
+    items: {
+      type: "array",
+      minItems: 1,
+      items: {
+        type: "object",
+        properties: {
+          decisive_index: { type: "integer", minimum: 1 },
+          level: { type: "string", enum: ["idea", "design", "implementation"] },
+          reason: { type: "string" },
+        },
+        required: ["decisive_index", "level", "reason"],
+      },
+    },
+  },
+  required: ["items"],
+};
+
+export const CLASSIFY_SYSTEM = `You classify objections raised against PostgreSQL proposals by the level of the review pyramid they belong to (Robert Haas's patch review hierarchy, as used by the PG Roast rubric):
+
+- "idea" (levels 1a and 1b): understandability and desirability. Is the problem clear, is it worth solving at all, does it belong in core, is there demand, is it a non-standard or unwanted feature, was it rejected before as an idea.
+- "design" (level 2): architecture and design in broad strokes. Given the goal, is the approach right: interfaces, user-visible behaviour, compatibility, on-disk/WAL/protocol/catalog consequences, security model, interaction with other subsystems, maintenance cost of the chosen approach.
+- "implementation" (levels 3-5): code structure, specific bugs, crashes, test or buildfarm failures, performance regressions of the particular code, missing docs/tests, style. An objection that needs the code (or running it) to be seen.
+
+Classify by what the objection is about, not by who raised it or how the proposal ended. Reply with JSON only.`;
+
+export function classifyPrompt(c: EvalCase): string {
+  return `Proposal: ${c.title}
+
+Decisive objection(s):
+${decisiveList(c)}
+
+Return {"items": [...]} with one item per decisive objection (decisive_index is the number after D): level and one short sentence of reason.`;
+}
+
+export async function runClassify(c: EvalCase, backend: LlmBackend, model: string): Promise<ClassifyItem[]> {
+  const out = await backend.complete<{ items: ClassifyItem[] }>({
+    system: CLASSIFY_SYSTEM, prompt: classifyPrompt(c), model, schema: CLASSIFY_SCHEMA, label: "classify",
+  });
+  return out.items.filter((i) => i.decisive_index >= 1 && i.decisive_index <= c.decisive_objections.length);
+}
+
+/** Case label: the most fundamental level among its decisive objections. */
+export function caseLevel(items: ClassifyItem[]): DecisiveLevel | null {
+  for (const l of ["idea", "design", "implementation"] as const) if (items.some((i) => i.level === l)) return l;
+  return null;
+}

package/src/eval/report.ts

@@ -0,0 +1,268 @@
+// Score a run and write eval/results/<run-id>/report.{md,json}.
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { connect } from "../corpus/db.ts";
+import { flatUrl } from "../corpus/util.ts";
+import { CliBackend, DryRunBackend, modelsFromEnv, pickBackend, type BackendName, type LlmBackend } from "../roast/llm.ts";
+import type { Roast } from "../roast/schema.ts";
+import { msgidOf, type EvalCase } from "./cases.ts";
+import { runClassify, runJudge, type ClassifyItem } from "./judge.ts";
+import { RESULTS_DIR, type CaseMeta } from "./run.ts";
+import { headline, scoreCase, type CaseScore, type CorpusThreadRow, type Headline, type JudgeItem } from "./score.ts";
+
+const readJson = <T>(p: string): T | null => (existsSync(p) ? (JSON.parse(readFileSync(p, "utf8")) as T) : null);
+
+/** Corpus rows holding any of the given thread message-ids. */
+export async function corpusRowsFor(msgids: string[]): Promise<CorpusThreadRow[]> {
+  if (!msgids.length) return [];
+  const sql = connect();
+  try {
+    const urls = JSON.stringify([...new Set(msgids)].map(flatUrl));
+    const rows = await sql`
+      select id, to_char(date, 'YYYY-MM-DD') as date, to_char(first_date, 'YYYY-MM-DD') as first_date,
+             array_to_json(thread_urls)::text as thread_urls
+      from precedent
+      where thread_urls && array(select jsonb_array_elements_text((${urls}::text)::jsonb))`;
+    return rows.map((r: { id: string; date: string | null; first_date: string | null; thread_urls: string }) =>
+      ({ id: r.id, date: r.date, first_date: r.first_date, thread_urls: JSON.parse(r.thread_urls) as string[] }));
+  } finally {
+    await sql.close();
+  }
+}
+
+export interface ScoreOptions {
+  runId: string;
+  cases: EvalCase[];
+  judge?: boolean;
+  backend?: BackendName;
+  log?: (m: string) => void;
+  /** injected for tests */
+  judgeBackend?: (dir: string, dryRun: boolean) => LlmBackend;
+  corpusRows?: CorpusThreadRow[];
+}
+
+export interface Report {
+  run_id: string;
+  scored_at: string;
+  /** true when any roast in the run came from the DryRunBackend */
+  dry_run: boolean;
+  rules: string;
+  cases_in_file: number;
+  cases_scored: number;
+  repeats: number;
+  headline: Headline;
+  by_post_training_cutoff: { true: Headline; false: Headline; unknown: Headline };
+  by_outcome: Record<string, Headline>;
+  /** by the most fundamental level of the decisive objection (idea / design / implementation / unclassified) */
+  by_decisive_level: Record<string, Headline>;
+  by_repeat: Headline[] | null;
+  agreement: { case_id: string; verdicts: (string | null)[]; modal_share: number }[] | null;
+  scores: CaseScore[];
+  totals: { roast_calls: number; judge_calls: number; cost_usd: number | null; wall_ms: number };
+}
+
+function repeatDirs(runId: string, caseId: string): { k: number; dir: string }[] {
+  const base = join(RESULTS_DIR, runId, caseId);
+  if (!existsSync(base)) return [];
+  const rs = readdirSync(base).filter((d) => /^r\d+$/.test(d)).map((d) => ({ k: Number(d.slice(1)), dir: join(base, d) }));
+  return rs.length ? rs.sort((a, b) => a.k - b.k) : [{ k: 1, dir: base }];
+}
+
+export async function scoreRun(o: ScoreOptions): Promise<Report> {
+  const log = o.log ?? ((m: string) => console.error(m));
+  const runDir = join(RESULTS_DIR, o.runId);
+  if (!existsSync(runDir)) throw new Error(`no such run: ${runDir}`);
+  const wanted = o.cases.flatMap((c) => c.earlier_attempts.map((e) => msgidOf(e.thread_url)).filter((x): x is string => !!x));
+  const corpusRows = o.corpusRows ?? (await corpusRowsFor(wanted));
+  const scores: CaseScore[] = [];
+  let repeats = 1;
+  let dryRun = false;
+  for (const c of o.cases) {
+    const dirs = repeatDirs(o.runId, c.id);
+    if (!dirs.length) continue; // case not part of this run
+    repeats = Math.max(repeats, dirs.length);
+    for (const { k, dir } of dirs) {
+      const r = readJson<Roast>(join(dir, "roast.json"));
+      if (r?.backend === "dry-run") dryRun = true;
+      const meta = readJson<CaseMeta>(join(dir, "case-meta.json"));
+      const err = readJson<{ error: string }>(join(dir, "error.json"));
+      let judge: JudgeItem[] | null = null;
+      let judgeCalls = 0;
+      let judgeCost: number | null = null;
+      let classify: ClassifyItem[] | null = null;
+      if (r && c.decisive_objections.length) {
+        const dry = r.backend === "dry-run";
+        const mk = () => o.judgeBackend?.(dir, dry) ?? (dry
+          ? new DryRunBackend({ transcriptDir: join(dir, "judge-llm"), print: () => undefined })
+          : new CliBackend({ backend: pickBackend(o.backend), transcriptDir: join(dir, "judge-llm"), log }));
+        const model = modelsFromEnv(process.env, dry ? "claude" : pickBackend(o.backend)).critic;
+        // 1. level classification: sees the case only, never the roast; runs first
+        const cc = readJson<{ items: ClassifyItem[]; calls: number; cost_usd: number | null }>(join(dir, "classify.json"));
+        if (cc) {
+          classify = cc.items;
+          judgeCalls += cc.calls;
+          judgeCost = cc.cost_usd;
+        } else if (o.judge !== false) {
+          const b = mk();
+          try {
+            classify = await runClassify(c, b, model);
+            writeFileSync(join(dir, "classify.json"), JSON.stringify({ items: classify, calls: b.stats().calls, cost_usd: b.stats().costUsd, model, dry_run: dry }, null, 2) + "\n");
+          } catch (e) {
+            log(`[${c.id}] classify failed: ${(e as Error).message}`);
+          }
+          judgeCalls += b.stats().calls;
+          judgeCost = b.stats().costUsd;
+        }
+        // 2. judge: roast objections vs decisive objections
+        const cached = readJson<{ items: JudgeItem[]; calls: number; cost_usd: number | null }>(join(dir, "judge.json"));
+        const addCost = (x: number | null) => { judgeCost = judgeCost === null && x === null ? null : (judgeCost ?? 0) + (x ?? 0); };
+        if (cached) {
+          judge = cached.items;
+          judgeCalls += cached.calls;
+          addCost(cached.cost_usd);
+        } else if (o.judge !== false) {
+          const b = mk();
+          try {
+            judge = await runJudge(c, r, b, model);
+            writeFileSync(join(dir, "judge.json"), JSON.stringify({ items: judge, calls: b.stats().calls, cost_usd: b.stats().costUsd, model, dry_run: dry }, null, 2) + "\n");
+            log(`[${c.id}${dirs.length > 1 ? `/r${k}` : ""}] judged`);
+          } catch (e) {
+            log(`[${c.id}] judge failed: ${(e as Error).message}`);
+          }
+          judgeCalls += b.stats().calls;
+          addCost(b.stats().costUsd);
+        }
+      }
+      scores.push(scoreCase(c, r, judge, corpusRows, {
+        repeat: k, error: r ? null : (err?.error?.split("\n")[0] ?? "no roast.json (not run yet)"),
+        judgeCalls, judgeCost, wallMs: meta?.wall_ms ?? null, classify,
+        lastShown: meta?.options?.last_shown_date ?? null,
+        precedentCutoff: meta?.options?.precedent_cutoff ?? null,
+      }));
+    }
+  }
+  const pick = (v: boolean | null) => scores.filter((s) => s.post_training_cutoff === v);
+  const byRepeat = repeats > 1
+    ? Array.from({ length: repeats }, (_, i) => headline(scores.filter((s) => s.repeat === i + 1)))
+    : null;
+  const agreement = repeats > 1
+    ? [...new Set(scores.map((s) => s.case_id))].map((id) => {
+        const vs = scores.filter((s) => s.case_id === id).map((s) => s.verdict);
+        const counts = new Map<string | null, number>();
+        for (const v of vs) counts.set(v, (counts.get(v) ?? 0) + 1);
+        return { case_id: id, verdicts: vs, modal_share: Math.max(...counts.values()) / vs.length };
+      })
+    : null;
+  const costs = scores.map((s) => s.cost.cost_usd);
+  return {
+    run_id: o.runId,
+    scored_at: new Date().toISOString(),
+    dry_run: dryRun,
+    rules: "docs/eval.md",
+    cases_in_file: o.cases.length,
+    cases_scored: new Set(scores.map((s) => s.case_id)).size,
+    repeats,
+    headline: headline(scores),
+    by_post_training_cutoff: { true: headline(pick(true)), false: headline(pick(false)), unknown: headline(pick(null)) },
+    by_outcome: Object.fromEntries(["rejected", "accepted", "reverted"].map((k) => [k, headline(scores.filter((s) => s.outcome === k))])),
+    by_decisive_level: Object.fromEntries(["idea", "design", "implementation", "unclassified"].map((k) =>
+      [k, headline(scores.filter((s) => (s.decisive_level ?? "unclassified") === k))])),
+    by_repeat: byRepeat,
+    agreement,
+    scores,
+    totals: {
+      roast_calls: scores.reduce((a, s) => a + s.cost.roast_calls, 0),
+      judge_calls: scores.reduce((a, s) => a + s.cost.judge_calls, 0),
+      cost_usd: costs.every((c) => c === null) ? null : Math.round(costs.reduce<number>((a, c) => a + (c ?? 0), 0) * 100) / 100,
+      wall_ms: scores.reduce((a, s) => a + (s.cost.wall_ms ?? 0), 0),
+    },
+  };
+}
+
+// ---------------------------------------------------------------- markdown
+
+const pct = (r: { num: number; den: number; pct: number | null }) => (r.den ? `${r.num}/${r.den} (${r.pct}%)` : "n/a");
+const yn = (b: boolean | null) => (b === null ? "–" : b ? "yes" : "no");
+
+function headlineTable(rows: [string, Headline][]): string {
+  const h = "| | cases | (a) verdict lenient | (a) strict | (b) decisive found | (b) found or partial | (c) earlier found | (d) false alarms |\n|---|---:|---|---|---|---|---|---|\n";
+  return h + rows.map(([n, x]) =>
+    `| ${n} | ${x.cases}${x.errors ? ` (${x.errors} err)` : ""} | ${pct(x.verdict_lenient)} | ${pct(x.verdict_strict)} | ${pct(x.decisive_found)} | ${pct(x.decisive_found_or_partial)} | ${pct(x.earlier_found)} | ${pct(x.false_alarm_rate)} |`).join("\n") + "\n";
+}
+
+export function renderReport(rep: Report, cases: EvalCase[]): string {
+  const byId = new Map(cases.map((c) => [c.id, c]));
+  const H = rep.headline;
+  let md = `# PG Roast eval: run \`${rep.run_id}\`\n\n`;
+  md += `Scored ${rep.scored_at}. Rules: [docs/eval.md](../../../docs/eval.md) (fixed before any result). `;
+  md += `${rep.cases_scored} of ${rep.cases_in_file} cases in this run${rep.repeats > 1 ? `, ${rep.repeats} repeats each` : ""}.\n\n`;
+  if (rep.dry_run) md += `> **Dry run**: placeholder verdicts from the DryRunBackend; these numbers mean nothing.\n\n`;
+  md += `## Headline\n\n`;
+  md += `- **(a) verdict match**: ${pct(H.verdict_lenient)} lenient, ${pct(H.verdict_strict)} strict\n`;
+  md += `- **(b) decisive objection found** (listed or other surviving): ${pct(H.decisive_found)}; found or partially: ${pct(H.decisive_found_or_partial)}; found but refuted by the critic: ${H.decisive_found_but_refuted}\n`;
+  md += `- **(c) earlier attempt found** (in prior attempts): ${pct(H.earlier_found)}; retrieved but not listed: ${H.earlier_retrieved_only}; not in corpus (corpus gap, excluded): ${H.earlier_not_in_corpus}\n`;
+  md += `- **(d) false-alarm rate on accepted cases**: ${pct(H.false_alarm_rate)}\n\n`;
+  md += `### Split by post_training_cutoff\n\nThe models may remember how famous old threads ended; post-cutoff cases cannot be recalled.\n\n`;
+  md += headlineTable([["all", H], ["post-cutoff (true)", rep.by_post_training_cutoff.true], ["pre-cutoff (false)", rep.by_post_training_cutoff.false], ...(rep.by_post_training_cutoff.unknown.cases ? [["unknown", rep.by_post_training_cutoff.unknown] as [string, Headline]] : [])]);
+  md += `\n### Split by outcome\n\n` + headlineTable(Object.entries(rep.by_outcome).filter(([, h]) => h.cases));
+  md += `\n### Split by level of the decisive objection\n\nLevel from a separate classify call that saw only the case, never the roast (idea = rubric 1a/1b, design = 2, implementation = 3-5). An implementation-level reason (a bug, a buildfarm failure) is usually invisible to an idea-level review of the first few emails; those cases are shown apart, not averaged in.\n\n`;
+  md += headlineTable(Object.entries(rep.by_decisive_level).filter(([, h]) => h.cases));
+  const inWin = rep.scores.filter((s) => s.decisive_in_window);
+  if (inWin.length) {
+    md += `\n> **Decisive objection inside the shown window** (position <= blind_n; the roast could read it): ${[...new Set(inWin.map((s) => s.case_id))].join(", ")}. A "found" here may be reading, not foresight.\n`;
+  }
+  if (rep.by_repeat) {
+    md += `\n### Per repeat\n\n` + headlineTable(rep.by_repeat.map((h, i) => [`r${i + 1}`, h] as [string, Headline]));
+    md += `\nVerdict agreement across repeats:\n\n| case | verdicts | modal share |\n|---|---|---:|\n`;
+    md += rep.agreement!.map((a) => `| ${a.case_id} | ${a.verdicts.map((v) => v ?? "error").join(", ")} | ${Math.round(a.modal_share * 100)}% |`).join("\n") + "\n";
+  }
+  md += `\n## Cases\n\n| case | outcome | post-cutoff | precedent cutoff | decisive level | verdict | (a) lenient | strict | (b) decisive | bucket | (c) earlier | fatal surviving | calls (roast+judge) | cost | wall |\n|---|---|---|---|---|---|---|---|---|---|---|---:|---|---:|---:|\n`;
+  for (const s of rep.scores) {
+    const tag = (rep.repeats > 1 ? `${s.case_id}/r${s.repeat}` : s.case_id) + (s.decisive_in_window ? " ⚑" : "");
+    const lvl = s.decisive_level ?? "–";
+    const co = s.cutoffs;
+    const cut = co.precedent_cutoff
+      ? `${co.precedent_cutoff}${co.precedent_cutoff < co.case_cutoff_date ? ` (case ${co.case_cutoff_date}; last shown ${co.last_shown_date ?? "?"})` : ""}`
+      : `? (case ${co.case_cutoff_date})`;
+    if (s.error) {
+      md += `| ${tag} | ${s.outcome} | ${yn(s.post_training_cutoff)} | ${cut} | ${lvl} | ERROR: ${s.error.replace(/\|/g, "/").slice(0, 80)} | | | | | | | ${s.cost.roast_calls}+${s.cost.judge_calls} | | |\n`;
+      continue;
+    }
+    const d = s.decisive!;
+    md += `| ${tag} | ${s.outcome} | ${yn(s.post_training_cutoff)} | ${cut} | ${lvl} | ${s.verdict} (${s.confidence}) | ${yn(s.verdict_lenient)} | ${yn(s.verdict_strict)} | ${d.status === "scored" ? d.result : d.status} | ${d.bucket ?? "–"} | ${s.earlier!.status} | ${s.surviving_fatal} | ${s.cost.roast_calls}+${s.cost.judge_calls} | ${s.cost.cost_usd === null ? "–" : `$${s.cost.cost_usd.toFixed(2)}`} | ${s.cost.wall_ms === null ? "–" : `${Math.round(s.cost.wall_ms / 1000)} s`} |\n`;
+  }
+  const t = rep.totals;
+  md += `\n⚑ = decisive objection inside the shown window. Precedent cutoff = min(case cutoff_date, last shown message + 1 day); both dates are shown when the window made it earlier.\n`;
+  md += `\nTotals: ${t.roast_calls} roast calls + ${t.judge_calls} judge calls; cost ${t.cost_usd === null ? "not reported" : `$${t.cost_usd}`}; wall ${Math.round(t.wall_ms / 60000)} min (sum over cases).\n`;
+
+  md += `\n## Misses\n\n`;
+  const misses = rep.scores.filter((s) => !s.error && (s.verdict_lenient === false || (s.decisive?.status === "scored" && !s.decisive.found_surviving) || s.earlier?.status === "not_found" || s.earlier?.status === "retrieved_only"));
+  if (!misses.length) md += `None.\n`;
+  for (const s of misses) {
+    const c = byId.get(s.case_id)!;
+    md += `### ${s.case_id}${rep.repeats > 1 ? ` r${s.repeat}` : ""}: ${c.title}\n\n`;
+    if (s.verdict_lenient === false) md += `- **Verdict**: ${s.verdict}, but the outcome was ${s.outcome}.${s.false_alarm ? ` False alarm (${s.surviving_fatal} surviving fatal).` : ""}\n`;
+    if (s.decisive?.status === "scored" && !s.decisive.found_surviving) {
+      md += `- **Decisive objection** (${s.decisive_level ?? "unclassified"}-level${s.decisive_in_window ? ", inside the shown window" : ""}): ${s.decisive.result}${s.decisive.bucket ? ` (best match ${s.decisive.objection_id}, ${s.decisive.bucket})` : ""}. ${s.decisive.reason}\n`;
+      for (const [i, d] of c.decisive_objections.entries()) {
+        const lv = s.decisive_levels.find((x) => x.decisive_index === i + 1)?.level;
+        md += `  - D${i + 1}${lv ? ` [${lv}]` : ""} ${d.who ?? ""}${d.date ? `, ${d.date}` : ""}: "${String(d.quote ?? "").replace(/\s+/g, " ").slice(0, 300)}"${d.message_url ? ` ([message](${d.message_url}))` : d.source ? ` (source: ${String(d.source).slice(0, 120)})` : ""}\n`;
+      }
+    }
+    if (s.earlier?.status === "not_found" || s.earlier?.status === "retrieved_only") {
+      md += `- **Earlier attempt** ${s.earlier.status === "retrieved_only" ? "retrieved but not listed as a prior attempt" : "not found"}: ${c.earlier_attempts.map((e) => e.thread_url).join(", ")}`;
+      md += ` (in corpus as ${s.earlier.in_corpus.map((x) => `${x.rows.join("/")}${x.retrievable_before_cutoff ? "" : ", dated on/after cutoff"}`).join("; ")})\n`;
+    }
+    md += `\n`;
+  }
+  return md;
+}
+
+export function writeReport(rep: Report, cases: EvalCase[]): { md: string; json: string } {
+  const dir = join(RESULTS_DIR, rep.run_id);
+  const md = join(dir, "report.md"), json = join(dir, "report.json");
+  writeFileSync(json, JSON.stringify(rep, null, 2) + "\n");
+  writeFileSync(md, renderReport(rep, cases));
+  return { md, json };
+}

package/src/eval/run.ts

@@ -0,0 +1,153 @@
+// Eval runner: roast each case in blind mode, as
+//   bun run src/cli.ts <thread_url> --blind <blind_n> [--rounds N]
+// would. Messages: blind_n only. Precedent cutoff (roast `before`, set after
+// loading so it filters no message) = min(cutoff_date, last shown message
+// date + 1 day). Writes to
+// eval/results/<run-id>/<case-id>[/r<k>]/. Resumable.
+
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { CliBackend, DryRunBackend, modelsFromEnv, pickBackend, type BackendName } from "../roast/llm.ts";
+import { closePrecedents } from "../roast/precedents.ts";
+import { runRoast } from "../roast/pipeline.ts";
+import { renderCompactMarkdown, renderMarkdown } from "../roast/render.ts";
+import { loadProposal } from "../thread/load.ts";
+import { REPO_ROOT, type EvalCase } from "./cases.ts";
+
+export const RESULTS_DIR = join(REPO_ROOT, "eval", "results");
+
+export interface RunEvalOptions {
+  runId: string;
+  cases: EvalCase[];
+  /** pipeline reflection rounds, as `src/cli.ts --rounds` */
+  rounds?: number;
+  /** independent repetitions of each case (variance); default 1 */
+  repeat?: number;
+  concurrency?: number;
+  dryRun?: boolean;
+  backend?: BackendName;
+  offline?: boolean;
+  log?: (m: string) => void;
+  /** injected for tests */
+  roastFn?: typeof runRoast;
+  loadFn?: typeof loadProposal;
+}
+
+export interface PrecedentCutoff {
+  /** the case file's cutoff_date */
+  case_cutoff_date: string;
+  /** UTC date of the last message shown to the roast */
+  last_shown_date: string;
+  /** what the roast used: the earlier of case_cutoff_date and last_shown_date + 1 day */
+  precedent_cutoff: string;
+}
+
+/**
+ * Precedent cutoff = min(cutoff_date, last shown message date + 1 day), so a
+ * case whose shown window is from 2018 gets no precedent dated 2019 or later.
+ * Precedents are kept only if every date they have is strictly before it.
+ */
+export function precedentCutoff(caseCutoff: string, lastShownIso: string): PrecedentCutoff {
+  const last = lastShownIso.slice(0, 10);
+  const d = new Date(last + "T00:00:00Z");
+  d.setUTCDate(d.getUTCDate() + 1);
+  const dayAfter = d.toISOString().slice(0, 10);
+  const cc = caseCutoff.slice(0, 10);
+  return { case_cutoff_date: cc, last_shown_date: last, precedent_cutoff: cc < dayAfter ? cc : dayAfter };
+}
+
+export function caseDir(runId: string, caseId: string, repeat: number, repeats: number): string {
+  return repeats > 1 ? join(RESULTS_DIR, runId, caseId, `r${repeat}`) : join(RESULTS_DIR, runId, caseId);
+}
+
+export interface CaseMeta {
+  case_id: string;
+  repeat: number;
+  started_at: string;
+  wall_ms: number;
+  llm_calls: number;
+  cost_usd: number | null;
+  options: {
+    thread_url: string; blind: number; message_filter: string; rounds: number; hint: string | null; backend: string; models: unknown;
+  } & Partial<PrecedentCutoff>;
+  error: string | null;
+}
+
+export async function runEval(o: RunEvalOptions): Promise<{ done: number; skipped: number; failed: number }> {
+  const log = o.log ?? ((m: string) => console.error(m));
+  const repeats = Math.max(1, o.repeat ?? 1);
+  const backendName: BackendName = o.dryRun ? (o.backend ?? "claude") : pickBackend(o.backend);
+  const models = modelsFromEnv(process.env, backendName);
+  const roast = o.roastFn ?? runRoast;
+  const load = o.loadFn ?? loadProposal;
+  const jobs: { c: EvalCase; k: number }[] = [];
+  for (const c of o.cases) for (let k = 1; k <= repeats; k++) jobs.push({ c, k });
+  const res = { done: 0, skipped: 0, failed: 0 };
+  mkdirSync(join(RESULTS_DIR, o.runId), { recursive: true });
+
+  const one = async ({ c, k }: { c: EvalCase; k: number }) => {
+    const dir = caseDir(o.runId, c.id, k, repeats);
+    const tag = repeats > 1 ? `${c.id}/r${k}` : c.id;
+    if (existsSync(join(dir, "roast.json"))) {
+      res.skipped++;
+      log(`[${tag}] roast.json exists, skipped`);
+      return;
+    }
+    mkdirSync(dir, { recursive: true });
+    rmSync(join(dir, "error.json"), { force: true });
+    const t0 = Date.now();
+    const transcriptDir = join(dir, "llm");
+    const backend = o.dryRun
+      ? new DryRunBackend({ transcriptDir, print: () => undefined })
+      : new CliBackend({ backend: backendName, transcriptDir, log: (m) => log(`[${tag}] ${m}`) });
+    const meta: CaseMeta = {
+      case_id: c.id, repeat: k, started_at: new Date(t0).toISOString(), wall_ms: 0, llm_calls: 0, cost_usd: null,
+      options: {
+        thread_url: c.thread_url, blind: c.blind_n,
+        message_filter: "blind only (first N messages); cutoff_date not applied to messages", rounds: o.rounds ?? 0,
+        hint: null, backend: o.dryRun ? "dry-run" : backendName, models,
+      },
+      error: null,
+    };
+    try {
+      log(`[${tag}] loading ${c.thread_url} (first ${c.blind_n} message(s))`);
+      // Messages: blind_n only (first N of the flat thread). cutoff_date is the
+      // precedent cutoff only: set after loading so it filters no message.
+      const proposal = await load(c.thread_url, { blind: c.blind_n, offline: o.offline });
+      const lastShown = proposal.thread.messages.at(-1)?.date ?? proposal.threadStart;
+      const pc = precedentCutoff(c.cutoff_date, lastShown);
+      proposal.filters.before = pc.precedent_cutoff;
+      Object.assign(meta.options, pc);
+      log(`[${tag}] precedent cutoff ${pc.precedent_cutoff} (case cutoff ${pc.case_cutoff_date}, last shown message ${pc.last_shown_date})`);
+      const r = await roast(proposal, {
+        backend, models, backendName: o.dryRun ? "dry-run" : backendName, hint: null, enrich: true,
+        mode: "review", rounds: o.rounds ?? 0, log: (m) => log(`[${tag}] ${m}`),
+      });
+      r.stats.wall_ms = Date.now() - t0;
+      writeFileSync(join(dir, "roast.md"), renderCompactMarkdown(r));
+      writeFileSync(join(dir, "roast-full.md"), renderMarkdown(r));
+      // roast.json last: its presence marks the case done
+      writeFileSync(join(dir, "roast.json"), JSON.stringify(r, null, 2) + "\n");
+      Object.assign(meta, { wall_ms: r.stats.wall_ms, llm_calls: r.stats.llm_calls, cost_usd: r.stats.cost_usd });
+      res.done++;
+      log(`[${tag}] verdict ${r.verdict} (${r.confidence}); ${r.stats.llm_calls} calls; ${Math.round(r.stats.wall_ms / 1000)} s`);
+    } catch (e) {
+      meta.error = String((e as Error)?.stack ?? e);
+      meta.wall_ms = Date.now() - t0;
+      meta.llm_calls = backend.stats().calls;
+      meta.cost_usd = backend.stats().costUsd;
+      writeFileSync(join(dir, "error.json"), JSON.stringify({ error: meta.error }, null, 2) + "\n");
+      res.failed++;
+      log(`[${tag}] FAILED: ${(e as Error)?.message ?? e}`);
+    }
+    writeFileSync(join(dir, "case-meta.json"), JSON.stringify(meta, null, 2) + "\n");
+  };
+
+  const queue = [...jobs];
+  const workers = Array.from({ length: Math.max(1, Math.min(o.concurrency ?? 2, jobs.length || 1)) }, async () => {
+    for (let j = queue.shift(); j; j = queue.shift()) await one(j);
+  });
+  await Promise.all(workers);
+  await closePrecedents();
+  return res;
+}