pgroast 0.0.0-stage → 0.1.0

package/src/corpus/search.ts

@@ -0,0 +1,201 @@
+// Precedent search. Imported directly by the roaster.
+//
+// 1. AND path: websearch_to_tsquery('english', q), ranked by ts_rank_cd over
+//    the weighted tsv (title A, summary+tags B, objection C).
+// 2. If that yields fewer than `limit` rows: OR-of-terms fallback — the
+//    query's english lexemes, each prefix-matched, OR'ed (to_tsquery('simple')
+//    so the already-stemmed lexemes are not stemmed again), expanded with the
+//    small SYNONYMS map, ranked by the number of distinct query terms covered
+//    (title matches count twice), then ts_rank_cd.
+// 3. If pg_trgm is installed: similarity(title, q) is added to the score and
+//    titles with similarity > 0.3 are also candidates.
+// Only stored rows are returned, so every URL is one the build read from a source.
+
+import type { SQL } from "bun";
+import { connect, hasTrgm } from "./db.ts";
+import type { Kind } from "./util.ts";
+
+export interface Precedent {
+  id: string;
+  kind: Kind;
+  title: string;
+  status: string;
+  date: string | null;       // decision date, YYYY-MM-DD
+  first_date: string | null; // thread start / reverted commit, YYYY-MM-DD
+  url: string;
+  thread_urls: string[];
+  authors: string[];
+  reviewers: string[];
+  objectors: string[];
+  tags: string;
+  summary: string;
+  objection: string;
+  touch_points: string[];
+  score: number;
+  match: "and" | "or" | "trgm";
+}
+
+export interface SearchOptions {
+  query: string;
+  /** YYYY-MM-DD: keep only rows whose every known date is strictly before this. */
+  before?: string;
+  kinds?: Kind[];
+  limit?: number;
+  /** With `before`: also keep rows that have no date at all (wiki items). */
+  includeUndated?: boolean;
+  sql?: SQL;
+}
+
+// words that carry no signal in proposal titles; dropped in the OR fallback only
+const WEAK = new Set(["add", "adding", "support", "allow", "new", "option", "use", "make", "patch", "feature", "pg"]);
+
+// Small curated synonym map over english-stemmed lexemes, used in the OR
+// fallback only. A query term counts as covered if it or any synonym matches.
+export const SYNONYMS: Record<string, string[]> = {
+  planner: ["optim"],
+  optim: ["planner"],
+  pooler: ["pool", "pgbouncer", "pgpool"],
+  pool: ["pooler", "pgbouncer", "pgpool"],
+  posit: ["order", "reorder", "attnum"],
+  timestamp: ["time", "date"],
+};
+
+// SQL keywords that the english config drops as stop words; when present in
+// the query they are matched against the title in the 'simple' config (tsv_kw).
+export const KEYWORD_STOPWORDS = new Set(["where", "having", "between", "into", "only", "any", "all", "off", "over"]);
+
+export interface OrTerm {
+  /** 'tsv' = weighted english vector; 'kw' = title in the simple config */
+  c: "tsv" | "kw";
+  /** tsquery text (simple config) matching anywhere in the vector */
+  q: string;
+  /** same, title (weight A) only */
+  tq: string;
+  label: string;
+}
+
+/**
+ * Terms of the OR fallback. Built only from english lexemes, SQL keywords and
+ * identifiers that match [a-z0-9]+, so no user text reaches tsquery syntax.
+ */
+export function orTerms(query: string, lexemes: string[]): OrTerm[] {
+  const terms: OrTerm[] = [];
+  for (const l of lexemes) {
+    const group = [l, ...(SYNONYMS[l] ?? [])];
+    terms.push({
+      c: "tsv", label: l,
+      q: group.map((x) => `'${x}':*`).join(" | "),
+      tq: group.map((x) => `'${x}':*A`).join(" | "),
+    });
+  }
+  const words = query.toLowerCase().match(/[a-z0-9_]+/g) ?? [];
+  for (const w of new Set(words)) {
+    if (KEYWORD_STOPWORDS.has(w)) terms.push({ c: "kw", label: w, q: `'${w}'`, tq: `'${w}'` });
+    const parts = w.split("_").filter(Boolean);
+    if (parts.length > 1 && parts.every((x) => /^[a-z0-9]+$/.test(x))) {
+      const ph = parts.map((x) => `'${x}'`).join(" <-> ");
+      terms.push({ c: "kw", label: w, q: ph, tq: ph });
+    }
+  }
+  return terms;
+}
+
+let shared: SQL | null = null;
+let trgmCache: boolean | null = null;
+
+export async function closeSearch() {
+  if (shared) await shared.close();
+  shared = null;
+}
+
+export async function searchPrecedents(o: SearchOptions): Promise<Precedent[]> {
+  const sql = o.sql ?? (shared ??= connect());
+  const limit = Math.max(1, Math.min(o.limit ?? 10, 200));
+  const q = o.query.trim();
+  if (!q) return [];
+  if (o.before && !/^\d{4}-\d\d-\d\d$/.test(o.before)) throw new Error("before must be YYYY-MM-DD");
+  if (trgmCache === null || o.sql) trgmCache = await hasTrgm(sql);
+  const trgm = trgmCache;
+  const kinds = o.kinds?.length ? o.kinds : null;
+  const before = o.before ?? null;
+  const undated = !!o.includeUndated;
+
+  const filters = sql`
+    (${kinds === null} or p.kind in (select jsonb_array_elements_text((${JSON.stringify(kinds ?? [])}::text)::jsonb)))
+    and (${before}::date is null
+         or (coalesce(p.first_date, p.date) < ${before}::date and coalesce(p.date, p.first_date) < ${before}::date)
+         or (${undated} and p.first_date is null and p.date is null))`;
+  const trgmExpr = trgm ? sql`similarity(p.title, ${q})` : sql`0::real`;
+  const cols = sql`
+    p.id, p.kind, p.title, p.status, to_char(p.date, 'YYYY-MM-DD') as date,
+    to_char(p.first_date, 'YYYY-MM-DD') as first_date, p.url,
+    array_to_json(p.thread_urls)::text as thread_urls, array_to_json(p.authors)::text as authors,
+    array_to_json(p.reviewers)::text as reviewers, array_to_json(p.objectors)::text as objectors,
+    p.tags, p.summary, p.objection, array_to_json(p.touch_points)::text as touch_points`;
+
+  const andRows = await sql`
+    select ${cols}, ts_rank_cd(p.tsv, w.q, 1) + 0.5 * ${trgmExpr} as score, 'and' as match
+    from precedent p, websearch_to_tsquery('english', ${q}) w(q)
+    where numnode(w.q) > 0 and p.tsv @@ w.q and ${filters}
+    order by score desc, p.id limit ${limit}`;
+  let rows = [...andRows];
+
+  if (rows.length < limit) {
+    const lex = await sql`
+      select coalesce(array_agg(l order by l), '{}') as lexemes
+      from unnest(tsvector_to_array(to_tsvector('english', ${q}))) l
+      where l ~ '^[[:alnum:]_.-]+$'`;
+    const lexemes = ((lex[0]?.lexemes ?? []) as string[]).filter((l) => !WEAK.has(l));
+    const seen = rows.map((r) => r.id as string);
+    const terms = orTerms(q, lexemes);
+    if (terms.length) {
+      // document frequency per term -> idf weight
+      const tj = JSON.stringify(terms);
+      const df = await sql`
+        select (select count(*) from precedent)::float8 as n,
+          coalesce(json_agg((select count(*) from precedent p where
+            case when t->>'c' = 'kw' then p.tsv_kw @@ to_tsquery('simple', t->>'q')
+                 else p.tsv @@ to_tsquery('simple', t->>'q') end) order by o), '[]')::text as df
+        from jsonb_array_elements((${tj}::text)::jsonb) with ordinality as x(t, o)`;
+      const n = Number(df[0]!.n);
+      const dfs = JSON.parse(df[0]!.df) as number[];
+      const weighted = JSON.stringify(terms.map((t, i) => ({ ...t, w: Math.log((n + 1) / ((dfs[i] ?? 0) + 1)) })));
+      // score = sum over terms of idf * (matches anywhere + matches in title) + ts_rank_cd
+      const cover = sql`(select coalesce(sum((t->>'w')::float8 * (
+            (case when t->>'c' = 'kw' then p.tsv_kw @@ to_tsquery('simple', t->>'q')
+                  else p.tsv @@ to_tsquery('simple', t->>'q') end)::int
+          + (case when t->>'c' = 'kw' then p.tsv_kw @@ to_tsquery('simple', t->>'q')
+                  else p.tsv @@ to_tsquery('simple', t->>'tq') end)::int)), 0)
+        from jsonb_array_elements((${weighted}::text)::jsonb) t)`;
+      const anyq = terms.filter((t) => t.c === "tsv").map((t) => t.q).join(" | ");
+      const orRows = await sql`
+        select * from (
+          select ${cols}, ${cover}
+            + coalesce(ts_rank_cd(p.tsv, to_tsquery('simple', nullif(${anyq}, '')), 1), 0)
+            + 0.5 * ${trgmExpr} as score, 'or' as match
+          from precedent p
+          where ${filters}
+            and p.id not in (select jsonb_array_elements_text((${JSON.stringify(seen)}::text)::jsonb))
+        ) s
+        where s.score > 0.0001
+        order by score desc, id limit ${limit - rows.length}`;
+      rows = rows.concat([...orRows]);
+    }
+    if (trgm && rows.length < limit) {
+      const seen2 = rows.map((r) => r.id as string);
+      const tRows = await sql`
+        select ${cols}, 0.5 * similarity(p.title, ${q}) as score, 'trgm' as match
+        from precedent p
+        where similarity(p.title, ${q}) > 0.3 and ${filters} and p.id not in (select jsonb_array_elements_text((${JSON.stringify(seen2)}::text)::jsonb))
+        order by score desc, p.id limit ${limit - rows.length}`;
+      rows = rows.concat([...tRows]);
+    }
+  }
+  return rows.map((r) => ({
+    id: r.id, kind: r.kind, title: r.title, status: r.status, date: r.date, first_date: r.first_date,
+    url: r.url, thread_urls: JSON.parse(r.thread_urls), authors: JSON.parse(r.authors),
+    reviewers: JSON.parse(r.reviewers), objectors: JSON.parse(r.objectors), tags: r.tags,
+    summary: r.summary, objection: r.objection, touch_points: JSON.parse(r.touch_points),
+    score: Math.round(Number(r.score) * 10000) / 10000, match: r.match,
+  }));
+}

package/src/corpus/util.ts

@@ -0,0 +1,133 @@
+// Shared helpers: HTML text extraction, message-id links, touch-point rules,
+// and the record type every source produces.
+
+export type Kind = "cf" | "revert" | "do_not_want";
+
+/** One row as produced by a source extractor (before it is stored). */
+export interface PrecedentRecord {
+  id: string;
+  kind: Kind;
+  rawRef: string; // stable source key; unique per kind
+  title: string;
+  status: string;
+  date: string | null; // decision date, ISO "YYYY-MM-DD[THH:MM:SS]"
+  firstDate: string | null;
+  url: string;
+  threadUrls: string[];
+  authors: string[];
+  reviewers: string[];
+  objectors: string[];
+  tags: string;
+  summary: string;
+  objection: string;
+  touchPoints: string[];
+  otherLinks: string[];
+  provenance: Record<string, unknown>;
+}
+
+const NAMED: Record<string, string> = {
+  amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", ndash: "–", mdash: "—",
+  hellip: "…", laquo: "«", raquo: "»", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”",
+  copy: "©", eacute: "é", oslash: "ø", ouml: "ö", uuml: "ü", auml: "ä", aacute: "á",
+};
+
+export function unescapeHtml(s: string): string {
+  return s.replace(/&(#x[0-9a-fA-F]+|#\d+|[A-Za-z]+);/g, (m, e: string) => {
+    if (e[0] === "#") {
+      const cp = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+      return Number.isFinite(cp) ? String.fromCodePoint(cp) : m;
+    }
+    return NAMED[e] ?? m;
+  });
+}
+
+export function stripTags(s: string): string {
+  return unescapeHtml(s.replace(/<br\s*\/?>/gi, "\n").replace(/<[^>]+>/g, " "))
+    .replace(/[ \t\r\f\v\u00a0]+/g, " ")
+    .replace(/\s*\n\s*/g, "\n")
+    .trim();
+}
+
+/** "a (x), b (y)" -> ["a (x)", "b (y)"] */
+export function splitPeople(s: string): string[] {
+  return s.split(/,\s*(?![^()]*\))/).map((x) => x.trim()).filter(Boolean);
+}
+
+// ---------------------------------------------------------------- message-ids
+
+export const FLAT = "https://www.postgresql.org/message-id/flat/";
+const SAFE = /[A-Za-z0-9@+=._\-~!$&'*,;:]/;
+
+function safeDecode(s: string): string {
+  try {
+    return decodeURIComponent(s);
+  } catch {
+    return s;
+  }
+}
+
+/** Canonical form of a message-id inside a flat URL (decode, then re-quote). */
+export function canonMsgid(raw: string): string {
+  const mid = safeDecode(raw.trim()).replace(/^<|>$/g, "").trim();
+  let out = "";
+  for (const ch of mid) {
+    if (SAFE.test(ch)) out += ch;
+    else for (const b of new TextEncoder().encode(ch)) out += "%" + b.toString(16).toUpperCase().padStart(2, "0");
+  }
+  return out;
+}
+
+export const flatUrl = (rawMsgid: string) => FLAT + canonMsgid(rawMsgid);
+
+const MSGID_URL_RE =
+  /https?:\/\/(?:www\.)?postgresql\.org\/message-id\/(?:flat\/|raw\/|mbox\/)?([^\s"'<>()\[\]]+)|https?:\/\/postgr\.es\/m\/([^\s"'<>()\[\]]+)/g;
+
+export function isMsgidUrl(u: string): boolean {
+  return /^https?:\/\/(?:www\.)?postgresql\.org\/message-id\/|^https?:\/\/postgr\.es\/m\//.test(u);
+}
+
+/** Raw message-ids found as URLs in text, in order, deduped. */
+export function extractMsgids(text: string): string[] {
+  const out: string[] = [];
+  for (const m of text.matchAll(MSGID_URL_RE)) {
+    const mid = (m[1] ?? m[2] ?? "").replace(/[.,;:]+$/, "");
+    if (!mid || mid.startsWith("attachment/")) continue;
+    if (!safeDecode(mid).includes("@")) continue;
+    if (!out.includes(mid)) out.push(mid);
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------- touch points
+// Cheap deterministic keyword rules (no LLM). Hints only.
+
+const TOUCH_RULES: [string, RegExp][] = [
+  ["pg_dump / restore", /\bpg_dump(all)?\b|\bpg_restore\b|\bdump\b/i],
+  ["psql", /\bpsql\b/i],
+  ["new GUC", /\bgucs?\b|\bpostgresql\.conf\b|\bconfiguration (parameter|variable|option)/i],
+  ["WAL / replication / standby", /\bwal\b|replication|\bstandby\b|\breplica\b|\bwalsender\b|\bwalreceiver\b|\bfailover\b|logical decoding|\bslot\b|\brecovery\b|\barchiv/i],
+  ["planner", /\bplanner\b|\boptimi[sz]er\b|\bplan\b|\bexplain\b|\bjoin\b|\bcost(ing)?\b|\bhint/i],
+  ["wire protocol / libpq", /\bprotocol\b|\blibpq\b|\bwire\b/i],
+  ["catalog changes", /\bcatalog\b|\bpg_class\b|\bpg_attribute\b|\bsystem view\b|\bpg_proc\b/i],
+  ["on-disk format / pg_upgrade", /\bpg_upgrade\b|on-disk|\bpage format\b|\btuple format\b|\bheap format\b/i],
+  ["security / privileges", /\bsecurity\b|\bprivilege|\bgrant\b|\brevoke\b|\brole\b|\bpassword\b|\bauth|\bssl\b|\btls\b|\bscram\b|\brls\b|row.level security/i],
+  ["SQL syntax / standard", /\bsyntax\b|sql standard|\bsql:20\d\d\b|\bkeyword\b|\bgrammar\b/i],
+  ["ABI / extension API", /\bhook\b|\bextension\b|\babi\b|\btable am\b|\bindex am\b|\bcustom scan\b/i],
+  ["performance on the hot path", /\bperformance\b|\bspeed ?up\b|\bfaster\b|\bhot path\b|\blatency\b/i],
+];
+
+export function touchPoints(text: string): string[] {
+  return TOUCH_RULES.filter(([, rx]) => rx.test(text)).map(([n]) => n);
+}
+
+/** "2024-09-12 11:08:15" -> "2024-09-12T11:08:15" */
+export function normTs(s: string | null | undefined): string | null {
+  if (!s) return null;
+  const m = /(\d{4}-\d\d-\d\d)(?:[ T](\d\d:\d\d(?::\d\d)?))?/.exec(s);
+  if (!m) return null;
+  return m[1] + (m[2] ? "T" + m[2] : "");
+}
+
+export function log(msg: string) {
+  console.error(msg);
+}

package/src/corpus/wiki.ts

@@ -0,0 +1,80 @@
+// Wiki source: https://wiki.postgresql.org/wiki/Not_Worth_Doing
+// "Features We Do Not Want" is an <h2> on that page (not on wiki/Todo; the
+// Todo page says retired items move to Not_Worth_Doing). Items under that
+// heading get status "Not wanted"; the other retired items "Not worth doing".
+
+import { fetchPage } from "./fetch.ts";
+import {
+  extractMsgids, flatUrl, isMsgidUrl, stripTags, touchPoints, unescapeHtml,
+  type PrecedentRecord,
+} from "./util.ts";
+
+export const WIKI_PAGE = "https://wiki.postgresql.org/wiki/Not_Worth_Doing";
+export const DNW_HEADING = "Features We Do Not Want";
+
+export interface WikiItem {
+  anchor: string; section: string; index: number; title: string;
+  description: string; links: string[]; linkTexts: string[]; wikiLinks: string[]; doNotWant: boolean;
+}
+
+export function parseWiki(h: string): WikiItem[] {
+  let body = h.slice(Math.max(0, h.indexOf('id="mw-content-text"')));
+  const cat = body.indexOf('id="catlinks"');
+  if (cat >= 0) body = body.slice(0, cat);
+  // headings with "/" carry a URL-safe legacy anchor span first; prefer it
+  const re = /<h([23])>(?:<span id="([^"]+)"><\/span>)?<span class="mw-headline" id="([^"]+)">([\s\S]*?)<\/span><\/h[23]>/g;
+  const heads = [...body.matchAll(re)];
+  const items: WikiItem[] = [];
+  let h2: string | null = null;
+  heads.forEach((m, i) => {
+    const level = m[1], anchor = m[2] ?? m[3]!, label = stripTags(m[4]!);
+    if (level === "2") h2 = label;
+    const start = m.index! + m[0].length;
+    const end = heads[i + 1]?.index ?? body.length;
+    const content = body.slice(start, end);
+    const section = label === h2 ? label : `${h2} / ${label}`;
+    let n = 0;
+    for (const dm of content.matchAll(/<dl>\s*<dt[^>]*>([\s\S]*?)<\/dt>\s*<dd>([\s\S]*?)<\/dd>\s*<\/dl>/g)) {
+      n++;
+      const dt = dm[1]!, dd = dm[2]!;
+      const both = dt + dd;
+      items.push({
+        anchor, section, index: n,
+        title: stripTags(dt.split("<ul>")[0]!),
+        description: stripTags(dd.replace(/<ul>[\s\S]*?<\/ul>/g, "")),
+        links: [...both.matchAll(/href="(https?:\/\/[^"]+)"/g)].map((x) => unescapeHtml(x[1]!)),
+        linkTexts: [...both.matchAll(/<a [^>]*href="https?:\/\/[^"]+"[^>]*>([\s\S]*?)<\/a>/g)].map((x) => stripTags(x[1]!)),
+        wikiLinks: [...dd.matchAll(/href="(\/wiki\/(?!File:)[^"#]+)"/g)].map((x) => "https://wiki.postgresql.org" + unescapeHtml(x[1]!)),
+        doNotWant: h2 === DNW_HEADING,
+      });
+    }
+  });
+  return items;
+}
+
+export async function collectWiki(opts: { offline?: boolean }) {
+  const h = await fetchPage(WIKI_PAGE, opts);
+  if (!h) throw new Error("cannot fetch " + WIKI_PAGE);
+  const items = parseWiki(h);
+  const recs: PrecedentRecord[] = items.map((it) => {
+    const note = it.linkTexts.filter((t) => t && !t.startsWith("http")).join("; ");
+    return {
+      id: `wiki:${it.anchor}:${it.index}`, kind: "do_not_want", rawRef: `Not_Worth_Doing#${it.anchor}:${it.index}`,
+      title: it.title, status: it.doNotWant ? "Not wanted" : "Not worth doing",
+      date: null, firstDate: null, url: `${WIKI_PAGE}#${it.anchor}`,
+      threadUrls: [...new Set(extractMsgids(it.links.join(" ")).map(flatUrl))],
+      authors: [], reviewers: [], objectors: [],
+      tags: it.section,
+      summary: `${it.title}\nSection: ${it.section}` + (note ? `\nLinked: ${note}` : ""),
+      objection: it.description,
+      touchPoints: touchPoints([it.title, it.section, it.description].join(" ")),
+      otherLinks: [...it.links.filter((u) => !isMsgidUrl(u)), ...it.wikiLinks],
+      provenance: { source: "wiki page " + WIKI_PAGE, section: it.section, item_index: it.index },
+    };
+  });
+  return {
+    recs,
+    cov: { items: items.length, do_not_want_items: items.filter((i) => i.doNotWant).length },
+    complete: true,
+  };
+}

package/src/eval/cases.ts

@@ -0,0 +1,118 @@
+// Eval cases: eval/cases.json (owned by the evalset agent). This loader is
+// tolerant: it accepts the documented fields, reports problems per case, and
+// keeps unknown fields so the report can show them.
+
+import { existsSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { canonMsgid } from "../corpus/util.ts";
+
+export const REPO_ROOT = resolve(import.meta.dir, "..", "..");
+export const CASES_PATH = join(REPO_ROOT, "eval", "cases.json");
+
+export type Outcome = "rejected" | "accepted" | "reverted";
+
+export interface DecisiveObjection {
+  who?: string;
+  date?: string;
+  message_url?: string;
+  quote?: string;
+  category?: string;
+  /** where the quote comes from when there is no message_url (e.g. a revert commit message) */
+  source?: string;
+  [k: string]: unknown;
+}
+
+export interface EarlierAttempt {
+  thread_url: string;
+  [k: string]: unknown;
+}
+
+export interface EvalCase {
+  id: string;
+  title: string;
+  outcome: Outcome;
+  thread_url: string;
+  first_message_id?: string;
+  cutoff_date: string;
+  blind_n: number;
+  decisive_objections: DecisiveObjection[];
+  earlier_attempts: EarlierAttempt[];
+  post_training_cutoff: boolean | null;
+  commit?: string | null;
+  revert_commit?: string | null;
+  /** 1-based position of the decisive message in the flat thread, if known. */
+  decisive_objection_position?: number | null;
+  /** Hint for directed precedent lookup, only if the case file provides one. */
+  hint?: string | null;
+  [k: string]: unknown;
+}
+
+const OUTCOMES = new Set(["rejected", "accepted", "reverted"]);
+
+/** Parse cases; throws on a file-level problem, returns per-case problems. */
+export function parseCases(json: unknown): { cases: EvalCase[]; problems: string[] } {
+  const arr = Array.isArray(json) ? json : (json as { cases?: unknown })?.cases;
+  if (!Array.isArray(arr)) throw new Error("cases.json: expected an array or {cases: [...]}");
+  const cases: EvalCase[] = [];
+  const problems: string[] = [];
+  const ids = new Set<string>();
+  arr.forEach((raw, i) => {
+    const c = raw as Record<string, unknown>;
+    const where = `case[${i}]${typeof c?.id === "string" ? ` (${c.id})` : ""}`;
+    const bad: string[] = [];
+    if (typeof c?.id !== "string" || !c.id) bad.push("id missing");
+    else if (ids.has(c.id)) bad.push("duplicate id");
+    const outcome = typeof c?.outcome === "string" ? c.outcome.toLowerCase() : "";
+    if (!OUTCOMES.has(outcome)) bad.push(`outcome must be rejected|accepted|reverted, got ${JSON.stringify(c?.outcome)}`);
+    if (typeof c?.thread_url !== "string" || !c.thread_url) bad.push("thread_url missing");
+    if (typeof c?.cutoff_date !== "string" || !/^\d{4}-\d\d-\d\d/.test(c.cutoff_date)) bad.push("cutoff_date missing or not YYYY-MM-DD");
+    const blind = Number(c?.blind_n);
+    if (!Number.isInteger(blind) || blind < 1) bad.push("blind_n must be a positive integer");
+    if (bad.length) {
+      problems.push(`${where}: ${bad.join("; ")}`);
+      return;
+    }
+    ids.add(c.id as string);
+    const ptc = c.post_training_cutoff;
+    cases.push({
+      ...c,
+      id: c.id as string,
+      title: typeof c.title === "string" ? c.title : (c.id as string),
+      outcome: outcome as Outcome,
+      thread_url: c.thread_url as string,
+      cutoff_date: (c.cutoff_date as string).slice(0, 10),
+      blind_n: blind,
+      decisive_objections: Array.isArray(c.decisive_objections) ? (c.decisive_objections as DecisiveObjection[]) : [],
+      earlier_attempts: (Array.isArray(c.earlier_attempts) ? (c.earlier_attempts as EarlierAttempt[]) : [])
+        .filter((e) => e && typeof e.thread_url === "string" && e.thread_url),
+      post_training_cutoff: typeof ptc === "boolean" ? ptc : null,
+      decisive_objection_position: Number.isInteger(c.decisive_objection_position) ? (c.decisive_objection_position as number) : null,
+    });
+  });
+  return { cases, problems };
+}
+
+export function loadCases(path = CASES_PATH): { cases: EvalCase[]; problems: string[] } {
+  if (!existsSync(path)) throw new Error(`${path} does not exist yet`);
+  return parseCases(JSON.parse(readFileSync(path, "utf8")));
+}
+
+/**
+ * Message-id of a thread/message URL (or a bare id), in the corpus's
+ * canonical form: the part after /message-id/[flat|raw|mbox/] or postgr.es/m/,
+ * without query/fragment, percent-decoded, <> stripped, re-quoted.
+ */
+export function msgidOf(urlOrId: string): string | null {
+  let s = urlOrId.trim();
+  const m = /(?:postgresql\.org\/message-id\/(?:flat\/|raw\/|mbox\/)?|postgr\.es\/m\/)([^?#\s]+)/.exec(s);
+  if (m) s = m[1]!;
+  else if (/^https?:/.test(s)) return null;
+  s = s.replace(/\/+$/, "");
+  const c = canonMsgid(s);
+  return c.includes("@") ? c : null;
+}
+
+/** True when the decisive objection's message is inside the shown window (position <= blind_n). */
+export function decisiveInWindow(c: EvalCase): boolean {
+  return c.decisive_objection_position != null && c.decisive_objection_position <= c.blind_n;
+}

package/src/eval/cli.ts

@@ -0,0 +1,105 @@
+// PG Roast evaluation CLI (rules: docs/eval.md).
+//
+//   bun run src/eval/cli.ts run   [--run-id ID] [--case ID ...] [--rounds N] [--repeat K] [--concurrency 2] [--dry-run] [--backend claude|codex] [--offline]
+//   bun run src/eval/cli.ts score <run-id> [--no-judge] [--case ID ...]
+//   bun run src/eval/cli.ts list
+
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import type { BackendName } from "../roast/llm.ts";
+import { CASES_PATH, loadCases, type EvalCase } from "./cases.ts";
+import { renderReport, scoreRun, writeReport } from "./report.ts";
+import { RESULTS_DIR, runEval } from "./run.ts";
+
+const USAGE = `usage:
+  bun run src/eval/cli.ts run   [--run-id ID] [--case ID ...] [--rounds N] [--repeat K] [--concurrency 2] [--dry-run] [--backend claude|codex] [--offline] [--cases FILE]
+  bun run src/eval/cli.ts score <run-id> [--no-judge] [--case ID ...] [--backend claude|codex] [--cases FILE]
+  bun run src/eval/cli.ts list [--cases FILE]
+--rounds N passes through to the pipeline (extra reflection rounds, as src/cli.ts --rounds).
+--repeat K runs every case K times (r1..rK) to measure variance.`;
+
+interface Args { cmd: string; pos: string[]; cases: string[]; flags: Record<string, string | true> }
+
+export function parseArgs(argv: string[]): Args {
+  const a: Args = { cmd: argv[0] ?? "", pos: [], cases: [], flags: {} };
+  const boolFlags = new Set(["dry-run", "no-judge", "offline", "help"]);
+  for (let i = 1; i < argv.length; i++) {
+    const t = argv[i]!;
+    if (t.startsWith("--")) {
+      const k = t.slice(2);
+      if (boolFlags.has(k)) { a.flags[k] = true; continue; }
+      const v = argv[++i];
+      if (v === undefined) throw new Error(`--${k} needs a value`);
+      if (k === "case") a.cases.push(...v.split(","));
+      else a.flags[k] = v;
+    } else a.pos.push(t);
+  }
+  return a;
+}
+
+function selectCases(all: EvalCase[], ids: string[]): EvalCase[] {
+  if (!ids.length) return all;
+  const unknown = ids.filter((id) => !all.some((c) => c.id === id));
+  if (unknown.length) throw new Error(`unknown case id(s): ${unknown.join(", ")}`);
+  return all.filter((c) => ids.includes(c.id));
+}
+
+const int = (v: string | true | undefined, def: number, name: string, min = 0, max = 100) => {
+  if (v === undefined) return def;
+  const n = Number(v);
+  if (!Number.isInteger(n) || n < min || n > max) throw new Error(`--${name} must be an integer ${min}..${max}`);
+  return n;
+};
+
+async function main(): Promise<number> {
+  const a = parseArgs(process.argv.slice(2));
+  if (!a.cmd || a.flags.help) { console.log(USAGE); return a.cmd ? 0 : 2; }
+  const casesPath = typeof a.flags.cases === "string" ? a.flags.cases : CASES_PATH;
+  const { cases, problems } = loadCases(casesPath);
+  for (const p of problems) console.error(`cases.json: skipped ${p}`);
+
+  if (a.cmd === "list") {
+    for (const c of cases) {
+      console.log(`${c.id.padEnd(24)} ${c.outcome.padEnd(9)} blind ${String(c.blind_n).padStart(3)}  before ${c.cutoff_date}  post-cutoff ${c.post_training_cutoff ?? "?"}  decisive ${c.decisive_objections.length}  earlier ${c.earlier_attempts.length}  ${c.title}`);
+    }
+    const runs = existsSync(RESULTS_DIR) ? readdirSync(RESULTS_DIR).filter((d) => !d.startsWith(".")) : [];
+    console.log(`\nruns: ${runs.join(", ") || "(none)"}`);
+    return 0;
+  }
+  if (a.cmd === "run") {
+    const runId = typeof a.flags["run-id"] === "string" ? a.flags["run-id"] : new Date().toISOString().slice(0, 16).replace(/[-:]/g, "").replace("T", "-");
+    if (!/^[A-Za-z0-9._-]+$/.test(runId)) throw new Error("--run-id: letters, digits, . _ - only");
+    const sel = selectCases(cases, a.cases);
+    console.error(`run ${runId}: ${sel.length} case(s) -> ${join(RESULTS_DIR, runId)}${a.flags["dry-run"] ? " (DRY RUN, no model calls)" : ""}`);
+    const r = await runEval({
+      runId, cases: sel,
+      rounds: int(a.flags.rounds, 0, "rounds", 0, 6),
+      repeat: int(a.flags.repeat, 1, "repeat", 1, 10),
+      concurrency: int(a.flags.concurrency, 2, "concurrency", 1, 8),
+      dryRun: !!a.flags["dry-run"],
+      backend: a.flags.backend as BackendName | undefined,
+      offline: !!a.flags.offline,
+    });
+    console.error(`run ${runId}: ${r.done} done, ${r.skipped} skipped (already done), ${r.failed} failed`);
+    console.error(`next: bun run src/eval/cli.ts score ${runId}`);
+    return r.failed ? 1 : 0;
+  }
+  if (a.cmd === "score") {
+    const runId = a.pos[0];
+    if (!runId) throw new Error("score needs a run id");
+    const rep = await scoreRun({
+      runId, cases: selectCases(cases, a.cases), judge: !a.flags["no-judge"],
+      backend: a.flags.backend as BackendName | undefined,
+    });
+    const out = writeReport(rep, cases);
+    console.log(renderReport(rep, cases).split("\n## Cases")[0]);
+    console.error(`wrote ${out.md}\nwrote ${out.json}`);
+    return 0;
+  }
+  console.error(USAGE);
+  return 2;
+}
+
+if (import.meta.main) {
+  main().then((c) => process.exit(c), (e) => { console.error(`eval: ${(e as Error).message}`); process.exit(1); });
+}