pgroast 0.0.0-stage → 0.1.0

package/src/cli.ts

@@ -0,0 +1,391 @@
+#!/usr/bin/env bun
+// pgroast <cf-id|cf-url|message-id|thread-url> [--blind N] [--before DATE] [--hint TEXT] [--out DIR]
+//         [--backend claude|codex] [--rounds N] [--self] [--dry-run]
+// pgroast --self --proposal FILE [--patch FILE ...]
+// pgroast page <roast.json> [--out FILE] [--site DIR [--base-url URL]]
+// pgroast pushback <cf-id|cf-url|thread-url> [--out DIR] [--backend claude|codex] [--dry-run] [--offline]
+// pgroast doctor
+//
+// Writes roast.md, roast.json and roast-summary.txt to a local directory. It never posts
+// anything anywhere.
+
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { gunzipSync } from "node:zlib";
+import { dirname, join, resolve } from "node:path";
+import { DEFAULT_DSN, parseDsn } from "./corpus/db.ts";
+import { renderPage, safeUrl, siteFiles } from "./roast/page.ts";
+import type { Roast } from "./roast/schema.ts";
+import { loadPushbackInput, pushbackSlug, renderPushbackMarkdown, runPushback } from "./roast/pushback.ts";
+import { DEFAULT_OUT_ROOT, REPO_ROOT, stats as fetchStats } from "./thread/fetch.ts";
+import { loadProposal, validateDate } from "./thread/load.ts";
+import { loadLocalProposal } from "./thread/local.ts";
+import { runDoctor } from "./roast/doctor.ts";
+import { CliBackend, DryRunBackend, modelsFromEnv, pickBackend, type BackendName } from "./roast/llm.ts";
+import { closePrecedents } from "./roast/precedents.ts";
+import { runRoast, slugFor } from "./roast/pipeline.ts";
+import { renderCompactMarkdown, renderMarkdown, renderSummary } from "./roast/render.ts";
+
+const USAGE = `Usage: bun run src/cli.ts <cf-id|cf-url|message-id|thread-url> [options]
+       bun run src/cli.ts --self --proposal FILE [--patch FILE ...]
+       bun run src/cli.ts page <roast.json> [--out FILE] [--site DIR [--base-url URL]]
+       bun run src/cli.ts pushback <cf-id|cf-url|thread-url> [--out DIR] [--backend NAME] [--dry-run] [--offline]
+       bun run src/cli.ts doctor      check the installed CLIs and their login
+       bun run src/cli.ts init-db     create the precedent database and load the shipped index
+
+  --self           your own proposal, before you send it: same roast, written for the author
+                   ("objections to address before you send": fix it, or rebut it in the cover
+                   letter); the verdict stays in roast.json and moves down in roast.md
+  --proposal FILE  with --self: a draft email on disk instead of a thread
+  --patch FILE     with --proposal: a patch file (repeatable); only its commit message, file
+                   list and doc hunks are read
+  --rounds N       extra rounds after the first pass (default 0): 1 asks every roaster "What
+                   did you forget?"; 2 also asks "Are you happy with the results?"; 3 and more
+                   add supervisor follow-ups on areas nobody examined
+
+  --blind N        read only the first N messages of the thread (evaluation: judge the idea itself)
+  --before DATE    read only messages strictly before DATE (YYYY-MM-DD); also the precedent cutoff
+  --hint TEXT      directed precedent lookup, e.g. "the 2018 suggestion of doing X"
+  --out DIR        output directory (default: out/<slug>)
+  --backend NAME   claude or codex (default: PGROAST_BACKEND, else claude if installed, else codex)
+  --no-enrich      do not read precedent threads and do not filter precedents for relevance
+  --dry-run        print the prompts, call no model
+  --offline        use only cached pages
+  -h, --help
+
+pushback: for a long-stuck entry, "what is the real pushback?". Reads the whole thread (every
+thread of the CommitFest entry) in parts, one model call per part plus one to merge, and writes
+pushback.md and pushback.json: the blocking objections with verified quotes and message links,
+who raised them, whether they were resolved, when the thread last moved, and a bucket.
+--dry-run prints the number of parts and calls and calls no model.
+
+page: renders a finished roast.json, with no model call, to one self-contained HTML file
+(default: roast.html next to the JSON; --out FILE), or with --site DIR to a static layout:
+DIR/cf/<cf-id>.html (embeddable card), DIR/cf/<cf-id>.json, DIR/cf/<cf-id>/full.html
+(DIR/msg/<message-id>.* for a thread without a CommitFest entry). The card links to
+{{PGROAST_BASE_URL}}/cf/<cf-id>/full.html unless --base-url is given. Local files only:
+nothing is uploaded or published.
+
+The model runs through an installed Claude Code or Codex CLI on your subscription login.
+No API key is used: API key variables are removed from the CLI's environment.
+
+Environment: PGROAST_BACKEND; for claude: PGROAST_ROASTER_MODEL (default opus), PGROAST_CRITIC_MODEL
+(default claude-mythos-5-2; also used for synthesis), Opus, Mythos and Fable only; for codex:
+PGROAST_CODEX_MODEL (default: the CLI's configured model); PGROAST_CONCURRENCY (3),
+PGROAST_LLM_TIMEOUT_MS (900000).
+
+Output: roast.md, roast.json, roast-summary.txt (a draft of at most 10 lines, to rewrite in your
+own words) and llm/ (every prompt and reply). Local files only.`;
+
+export interface CliArgs {
+  input: string;
+  blind?: number;
+  before?: string;
+  hint?: string;
+  out?: string;
+  backend?: BackendName;
+  doctor?: boolean;
+  initDb?: boolean;
+  noEnrich?: boolean;
+  self?: boolean;
+  proposal?: string;
+  patches?: string[];
+  rounds?: number;
+  page?: { json: string; out?: string; site?: string; baseUrl?: string };
+  pushback?: boolean;
+  dryRun: boolean;
+  offline: boolean;
+}
+
+export function parseArgs(argv: string[]): CliArgs | "help" {
+  const a: CliArgs = { input: "", dryRun: false, offline: false };
+  if (argv[0] === "page") return parsePageArgs(argv.slice(1));
+  const rest = [...argv];
+  if (rest[0] === "pushback") {
+    rest.shift();
+    a.pushback = true;
+  }
+  const value = (flag: string, inline: string | undefined): string => {
+    const v = inline ?? rest.shift();
+    if (v == null || v === "") throw new Error(`${flag} needs a value`);
+    return v;
+  };
+  while (rest.length) {
+    const tok = rest.shift()!;
+    const eq = tok.startsWith("--") && tok.includes("=") ? tok.indexOf("=") : -1;
+    const flag = eq > 0 ? tok.slice(0, eq) : tok;
+    const inline = eq > 0 ? tok.slice(eq + 1) : undefined;
+    switch (flag) {
+      case "-h":
+      case "--help":
+        return "help";
+      case "--blind": {
+        const n = Number(value(flag, inline));
+        if (!Number.isInteger(n) || n < 1) throw new Error("--blind needs a positive integer");
+        a.blind = n;
+        break;
+      }
+      case "--before":
+        a.before = validateDate(value(flag, inline));
+        break;
+      case "--hint":
+        a.hint = value(flag, inline);
+        break;
+      case "--out":
+        a.out = value(flag, inline);
+        break;
+      case "--backend": {
+        const b = value(flag, inline).toLowerCase();
+        if (b !== "claude" && b !== "codex") throw new Error("--backend must be claude or codex");
+        a.backend = b;
+        break;
+      }
+      case "--self":
+        a.self = true;
+        break;
+      case "--proposal":
+        a.proposal = value(flag, inline);
+        break;
+      case "--patch":
+        (a.patches ??= []).push(value(flag, inline));
+        break;
+      case "--rounds": {
+        const n = Number(value(flag, inline));
+        if (!Number.isInteger(n) || n < 0 || n > 6) throw new Error("--rounds needs an integer from 0 to 6");
+        a.rounds = n;
+        break;
+      }
+      case "--no-enrich":
+        a.noEnrich = true;
+        break;
+      case "--dry-run":
+        a.dryRun = true;
+        break;
+      case "--offline":
+        a.offline = true;
+        break;
+      default:
+        if (flag.startsWith("--")) throw new Error(`unknown option ${flag}`);
+        if (a.input) throw new Error(`unexpected argument "${tok}"`);
+        a.input = tok;
+    }
+  }
+  if (a.pushback) {
+    if (!a.input) throw new Error("pushback needs a CommitFest patch id or URL, or a thread URL");
+    for (const [k, name] of [["blind", "--blind"], ["before", "--before"], ["hint", "--hint"], ["self", "--self"], ["proposal", "--proposal"], ["rounds", "--rounds"], ["noEnrich", "--no-enrich"]] as const) {
+      if (a[k] != null) throw new Error(`${name} does not apply to pushback (it reads the whole thread)`);
+    }
+    if (a.patches?.length) throw new Error("--patch does not apply to pushback");
+    return a;
+  }
+  if (a.input === "doctor") a.doctor = true;
+  if (a.input === "init-db") a.initDb = true;
+  if (a.proposal) {
+    if (!a.self) throw new Error("--proposal is for your own draft: use it with --self");
+    if (a.input) throw new Error("give either a thread or --proposal FILE, not both");
+    if (a.blind != null || a.before) throw new Error("--blind and --before do not apply to a draft on disk");
+    return a;
+  }
+  if (a.patches?.length) throw new Error("--patch needs --self --proposal FILE");
+  if (!a.input) throw new Error("missing input: a CommitFest patch id or URL, a message-id, or a thread URL (or --self --proposal FILE)");
+  return a;
+}
+
+function parsePageArgs(argv: string[]): CliArgs | "help" {
+  const page: NonNullable<CliArgs["page"]> = { json: "" };
+  const rest = [...argv];
+  while (rest.length) {
+    const tok = rest.shift()!;
+    const eq = tok.startsWith("--") && tok.includes("=") ? tok.indexOf("=") : -1;
+    const flag = eq > 0 ? tok.slice(0, eq) : tok;
+    const inline = eq > 0 ? tok.slice(eq + 1) : undefined;
+    const value = (): string => {
+      const v = inline ?? rest.shift();
+      if (v == null || v === "") throw new Error(`${flag} needs a value`);
+      return v;
+    };
+    if (flag === "-h" || flag === "--help") return "help";
+    else if (flag === "--out") page.out = value();
+    else if (flag === "--site") page.site = value();
+    else if (flag === "--base-url") {
+      const u = value();
+      if (!safeUrl(u)) throw new Error("--base-url must be an http(s) URL");
+      page.baseUrl = u;
+    } else if (flag.startsWith("--")) throw new Error(`unknown option ${flag} for page`);
+    else if (page.json) throw new Error(`unexpected argument "${tok}"`);
+    else page.json = tok;
+  }
+  if (!page.json) throw new Error("page needs the path of a roast.json");
+  if (page.out && page.site) throw new Error("give --out FILE or --site DIR, not both");
+  if (page.baseUrl && !page.site) throw new Error("--base-url is for --site");
+  return { input: "page", dryRun: false, offline: false, page };
+}
+
+/** `pgroast page`: roast.json -> static HTML. No model, no network, local files only. */
+export function runPage(page: NonNullable<CliArgs["page"]>, log: (m: string) => void = () => undefined): string[] {
+  let roast: Roast;
+  try {
+    roast = JSON.parse(readFileSync(page.json, "utf8")) as Roast;
+  } catch (e) {
+    throw new Error(`cannot read ${page.json}: ${(e as Error).message}`);
+  }
+  const missing = ["verdict", "objections", "other_surviving", "refuted", "withdrawn", "touch_points", "questions", "committability", "priority", "rounds", "invariants_loaded", "disclosure", "brief", "input"].filter(
+    (k) => (roast as unknown as Record<string, unknown>)?.[k] == null,
+  );
+  if (missing.length) throw new Error(`${page.json} is not a roast.json of this version (missing: ${missing.join(", ")}); run the roast again`);
+  const written: string[] = [];
+  const write = (file: string, content: string) => {
+    mkdirSync(dirname(file), { recursive: true });
+    writeFileSync(file, content);
+    written.push(file);
+    log(`wrote ${file}`);
+  };
+  if (page.site) {
+    const dir = resolve(page.site);
+    for (const f of siteFiles(roast, page.baseUrl)) write(join(dir, f.path), f.content);
+  } else {
+    write(resolve(page.out ?? join(dirname(page.json), "roast.html")), renderPage(roast));
+  }
+  return written;
+}
+
+/**
+ * Create the precedent database and load the shipped index. Uses createdb, gunzip and psql with
+ * the usual libpq environment (PGHOST, PGPORT, PGUSER); the database name is `pgroast`, or
+ * the dbname of PGROAST_DSN.
+ */
+function initDb(): number {
+  const dump = join(REPO_ROOT, "data", "precedents.sql.gz");
+  if (!existsSync(dump)) {
+    console.error(`pgroast: ${dump} is missing`);
+    return 1;
+  }
+  const kv = parseDsn(process.env.PGROAST_DSN ?? DEFAULT_DSN);
+  const db = kv.dbname ?? "pgroast";
+  const env = { ...process.env, ...(kv.host ? { PGHOST: kv.host } : {}), ...(kv.port ? { PGPORT: kv.port } : {}), ...(kv.user ? { PGUSER: kv.user } : {}) };
+  const run = (cmd: string, argv: string[], input?: Buffer) => spawnSync(cmd, argv, { env, input, encoding: "utf8", maxBuffer: 1 << 28 });
+  const has = run("psql", ["-XAtq", "-d", db, "-c", "select to_regclass('public.precedent') is not null"]);
+  if (has.error) {
+    console.error("pgroast: psql was not found on PATH; install the Postgres client tools");
+    return 1;
+  }
+  if (has.status === 0 && has.stdout.trim() === "t") {
+    const n = run("psql", ["-XAtq", "-d", db, "-c", "select count(*) from precedent"]);
+    console.log(`database "${db}" already holds the precedent index (${n.stdout.trim()} rows); nothing to do`);
+    return 0;
+  }
+  if (has.status !== 0) {
+    const c = run("createdb", [db]);
+    if (c.status !== 0) {
+      console.error(`pgroast: createdb ${db} failed: ${(c.stderr || c.error?.message || "").trim()}\nIs Postgres running? Set PGROAST_DSN (for example "dbname=pgroast host=localhost") if it is not on the local socket.`);
+      return 1;
+    }
+    console.log(`created database "${db}"`);
+  }
+  const load = run("psql", ["-Xq", "-v", "ON_ERROR_STOP=1", "-d", db], gunzipSync(readFileSync(dump)));
+  if (load.status !== 0) {
+    console.error(`pgroast: loading the index failed: ${(load.stderr || "").trim().split("\n").slice(0, 5).join("\n")}`);
+    return 1;
+  }
+  const n = run("psql", ["-XAtq", "-d", db, "-c", "select count(*) from precedent"]);
+  console.log(`loaded the precedent index into "${db}": ${n.stdout.trim()} rows`);
+  return 0;
+}
+
+async function main(): Promise<number> {
+  let args: CliArgs | "help";
+  try {
+    args = parseArgs(process.argv.slice(2));
+  } catch (e) {
+    console.error(`pgroast: ${(e as Error).message}\n\n${USAGE}`);
+    return 2;
+  }
+  if (args === "help") {
+    console.log(USAGE);
+    return 0;
+  }
+  if (args.page) {
+    runPage(args.page, (m) => console.error(m));
+    return 0;
+  }
+  if (args.initDb) return initDb();
+  if (args.doctor) {
+    const d = await runDoctor();
+    console.log(d.text);
+    return d.ok ? 0 : 1;
+  }
+  const t0 = Date.now();
+  const log = (m: string) => console.error(m);
+  const backendName: BackendName = args.dryRun ? (args.backend ?? "claude") : pickBackend(args.backend);
+  const models = modelsFromEnv(process.env, backendName);
+
+  if (args.pushback) {
+    log(`reading ${args.input} (whole thread) ...`);
+    const input = await loadPushbackInput(args.input, { offline: args.offline, noAttachments: true });
+    const outDir = resolve(args.out ?? join(DEFAULT_OUT_ROOT, pushbackSlug(input)));
+    mkdirSync(outDir, { recursive: true });
+    const transcriptDir = join(outDir, "llm");
+    const backend = args.dryRun ? new DryRunBackend({ transcriptDir, print: () => undefined }) : new CliBackend({ backend: backendName, transcriptDir, log });
+    const result = await runPushback(input, { backend, models: { map: models.roaster, reduce: models.critic }, backendName: args.dryRun ? "dry-run" : backendName, log });
+    result.stats.wall_ms = Date.now() - t0;
+    writeFileSync(join(outDir, "pushback.json"), JSON.stringify(result, null, 2) + "\n");
+    writeFileSync(join(outDir, "pushback.md"), renderPushbackMarkdown(result));
+    log("");
+    if (args.dryRun) log(`dry run: no model was called; prompts are in ${transcriptDir}`);
+    else {
+      log(`bucket: ${result.bucket}`);
+      log(`objections: ${result.objections.length} (${result.objections.filter((o) => o.blocking).length} blocking), from ${result.candidates.length} candidates`);
+      for (const w of result.warnings) log(`warning: ${w}`);
+    }
+    log(`model calls: ${result.stats.llm_calls}; parts: ${result.stats.chunks}; wall time: ${Math.round(result.stats.wall_ms / 1000)} s`);
+    log(`wrote ${join(outDir, "pushback.md")}`);
+    log(`wrote ${join(outDir, "pushback.json")}`);
+    return 0;
+  }
+
+  log(`reading ${args.proposal ?? args.input} ...`);
+  const proposal = args.proposal
+    ? loadLocalProposal(args.proposal, args.patches ?? [])
+    : await loadProposal(args.input, { blind: args.blind, before: args.before, offline: args.offline });
+  log(
+    args.proposal
+      ? `draft: "${proposal.thread.subject}", ${proposal.patchProse.length} patch file(s) read`
+      : `thread: "${proposal.thread.subject}", ${proposal.thread.messages.length} of ${proposal.totalMessages} messages, ${proposal.patchProse.length} patch file(s) read; fetched ${fetchStats.fetched}, cached ${fetchStats.cached}`,
+  );
+
+  const outDir = resolve(args.out ?? join(DEFAULT_OUT_ROOT, slugFor(proposal)));
+  mkdirSync(outDir, { recursive: true });
+  const transcriptDir = join(outDir, "llm");
+  const backend = args.dryRun ? new DryRunBackend({ transcriptDir }) : new CliBackend({ backend: backendName, transcriptDir, log });
+  if (!args.dryRun) log(`backend: ${backendName} CLI (subscription login, no tools)`);
+
+  const roast = await runRoast(proposal, { backend, models, backendName: args.dryRun ? "dry-run" : backendName, hint: args.hint ?? null, enrich: !args.noEnrich, mode: args.self ? "self" : "review", rounds: args.rounds ?? 0, log });
+  await closePrecedents();
+  roast.stats.wall_ms = Date.now() - t0;
+
+  writeFileSync(join(outDir, "roast.json"), JSON.stringify(roast, null, 2) + "\n");
+  writeFileSync(join(outDir, "roast.md"), renderCompactMarkdown(roast));
+  writeFileSync(join(outDir, "roast-full.md"), renderMarkdown(roast));
+  writeFileSync(join(outDir, "roast-summary.txt"), renderSummary(roast));
+  log("");
+  log(`${args.dryRun ? "DRY RUN, placeholder verdict" : "verdict"}: ${roast.verdict} (confidence: ${roast.confidence})`);
+  log(`objections: ${roast.objections.length} listed, ${roast.other_surviving.length} more survived, ${roast.refuted.length} refuted; prior attempts: ${roast.prior_attempts.length}`);
+  for (const x of roast.rounds.slice(1)) log(`round ${x.round} (${x.kind}): ${x.llm_calls} calls, ${x.new_objections} new objection(s), ${x.new_survived} survived, ${x.revised} revised, ${x.withdrawn} withdrawn`);
+  log(`LLM calls: ${roast.stats.llm_calls}; wall time: ${Math.round(roast.stats.wall_ms / 1000)} s`);
+  if (!args.dryRun) for (const w of roast.warnings) log(`warning: ${w}`);
+  else log(`dry run: prompts are in ${transcriptDir}; roast.md and roast.json hold placeholders`);
+  log(`wrote ${join(outDir, "roast.md")} (short report; roast-full.md has everything)`);
+  log(`wrote ${join(outDir, "roast.json")}`);
+  log(`wrote ${join(outDir, "roast-summary.txt")}`);
+  return 0;
+}
+
+if (import.meta.main) {
+  main().then(
+    (code) => process.exit(code),
+    (e) => {
+      console.error(`pgroast: ${(e as Error).message}`);
+      process.exit(1);
+    },
+  );
+}

package/src/corpus/cf.ts

@@ -0,0 +1,220 @@
+// CommitFest source: Rejected / Returned with feedback / Withdrawn patches.
+
+import { fetchPage, fetchStats, isCached } from "./fetch.ts";
+import {
+  flatUrl, log, normTs, splitPeople, stripTags, touchPoints, unescapeHtml,
+  type PrecedentRecord,
+} from "./util.ts";
+
+export const CF_BASE = "https://commitfest.postgresql.org";
+export const CF_STATUSES: Record<number, string> = {
+  6: "Rejected",
+  7: "Returned with feedback",
+  8: "Withdrawn",
+};
+const NEGATIVE = new Set(Object.values(CF_STATUSES).map((s) => s.toLowerCase()));
+
+export interface CfInfo { id: number; name: string | null; state: string | null; start: string | null; end: string | null }
+
+export function parseArchive(h: string): CfInfo[] {
+  const out = new Map<number, CfInfo>();
+  const re = /href="\/(\d+)\/">([^<]+)<\/a>\s*\(([^-)]+?)\s*-\s*(\d{4}-\d\d-\d\d)\s*\S\s*(\d{4}-\d\d-\d\d)\)/g;
+  for (const m of h.matchAll(re)) {
+    out.set(Number(m[1]), { id: Number(m[1]), name: m[2]!.trim(), state: m[3]!.trim(), start: m[4]!, end: m[5]! });
+  }
+  for (const m of h.matchAll(/href="\/(\d+)\/"/g)) {
+    const id = Number(m[1]);
+    if (!out.has(id)) out.set(id, { id, name: null, state: null, start: null, end: null });
+  }
+  return [...out.values()].sort((a, b) => a.id - b.id);
+}
+
+export interface CfListRow {
+  id: number; title: string; status: string | null; tags: string[];
+  authors: string; reviewers: string; committer: string; latestMail: string | null;
+}
+
+export function parseList(h: string): CfListRow[] {
+  const rows: CfListRow[] = [];
+  for (const m of h.matchAll(/<tr>\s*<td class="wrap-text patch-column">([\s\S]*?)<\/tr>/g)) {
+    const body = m[1]!;
+    const pm = /<a href="\/patch\/(\d+)\/">([\s\S]*?)<\/a>/.exec(body);
+    if (!pm) continue;
+    const st = /<span class="badge[^"]*">([^<]+)<\/span>/.exec(body);
+    const tags = [...body.matchAll(/<span class="badge" style="[^"]*"(?: title="[^"]*")?>([^<]+)<\/span>/g)]
+      .map((t) => unescapeHtml(t[1]!));
+    const lm = /title="(\d{4}-\d\d-\d\d[ \d:]*)"/.exec(body);
+    const wraps = [...body.matchAll(/<td class="wrap-text">([\s\S]*?)<\/td>/g)].map((w) => stripTags(w[1]!));
+    rows.push({
+      id: Number(pm[1]), title: stripTags(pm[2]!), status: st ? st[1]!.trim() : null, tags,
+      authors: wraps[0] ?? "", reviewers: wraps[1] ?? "", committer: wraps[2] ?? "",
+      latestMail: lm ? lm[1]!.trim() : null,
+    });
+  }
+  return rows;
+}
+
+export interface CfThread { msgid: string; subject: string; firstAt: string | null; firstBy: string; latestAt: string | null }
+export interface CfPatch {
+  title: string; created: string | null;
+  statuses: { cf: number; cfName: string; status: string }[]; // as listed (newest first)
+  tags: string[]; targetVersion: string; authors: string; reviewers: string; committer: string;
+  threads: CfThread[]; history: { when: string | null; who: string; what: string }[]; otherLinks: string[];
+}
+
+function fields(h: string): Map<string, string> {
+  const out = new Map<string, string>();
+  for (const m of h.matchAll(/<th[^>]*>\s*([^<]+?)\s*<\/th>\s*<td[^>]*>([\s\S]*?)<\/td>\s*<\/tr>/g)) {
+    if (!out.has(m[1]!.trim())) out.set(m[1]!.trim(), m[2]!);
+  }
+  return out;
+}
+
+export function parsePatch(h: string): CfPatch | null {
+  if (!h.includes("<th>Title</th>")) return null;
+  const f = fields(h);
+  const statuses = [...(f.get("Status") ?? "").matchAll(
+    /<a href="\/(\d+)\/">([^<]+)<\/a>[^:<]*:\s*<span class="badge[^"]*">([^<]+)<\/span>/g,
+  )].map((m) => ({ cf: Number(m[1]), cfName: m[2]!.trim(), status: m[3]!.trim() }));
+  const tags = [...(f.get("Tags") ?? "").matchAll(/<span class="badge"[^>]*>([^<]+)<\/span>/g)]
+    .map((m) => unescapeHtml(m[1]!));
+  const reviewers = (f.get("Reviewers") ?? "").replace(/<a class="btn[\s\S]*?<\/a>/g, "");
+  const threads: CfThread[] = [];
+  for (const dm of (f.get("Emails") ?? "").matchAll(/<dt>([\s\S]*?)<\/dt>\s*<dd>([\s\S]*?)<\/dd>/g)) {
+    const a = /<a href="(https:\/\/www\.postgresql\.org\/message-id\/flat\/([^"]+))">([\s\S]*?)<\/a>/.exec(dm[1]!);
+    if (!a) continue;
+    const first = /First at <a[^>]*>([^<]+)<\/a> by ([^<]*?)(?:<br|$)/.exec(dm[2]!);
+    const latest = /Latest at <a[^>]*>([^<]+)<\/a>/.exec(dm[2]!);
+    threads.push({
+      msgid: unescapeHtml(a[2]!),
+      subject: stripTags(a[3]!),
+      firstAt: first ? normTs(first[1]) : null,
+      firstBy: first ? unescapeHtml(first[2]!).replace(/\s*<.*$/, "").trim() : "",
+      latestAt: latest ? normTs(latest[1]) : null,
+    });
+  }
+  const history: CfPatch["history"] = [];
+  const hm = /<th>History<\/th>([\s\S]*?)<\/table>/.exec(h);
+  if (hm) {
+    for (const r of hm[1]!.matchAll(/<tr>\s*<td[^>]*>([^<]+)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/g)) {
+      history.push({ when: normTs(r[1]), who: stripTags(r[2]!), what: stripTags(r[3]!) });
+    }
+  }
+  return {
+    title: stripTags(f.get("Title") ?? ""),
+    created: normTs(stripTags(f.get("Created") ?? "")),
+    statuses, tags,
+    targetVersion: stripTags(f.get("Target version") ?? ""),
+    authors: stripTags(f.get("Authors") ?? f.get("Author") ?? ""),
+    reviewers: stripTags(reviewers),
+    committer: stripTags(f.get("Committer") ?? ""),
+    threads, history,
+    otherLinks: [...(f.get("Links") ?? "").matchAll(/href="(https?:\/\/[^"]+)"/g)].map((m) => unescapeHtml(m[1]!)),
+  };
+}
+
+/** Final status = status in the highest CF id (the site lists newest first). */
+export function finalStatus(p: CfPatch): string | null {
+  if (!p.statuses.length) return null;
+  return p.statuses.reduce((a, b) => (b.cf > a.cf ? b : a)).status;
+}
+
+export function patchRecord(pid: number, p: CfPatch, final: string): PrecedentRecord {
+  let close: string | null = null;
+  for (const h of p.history) {
+    const m = /^Closed in commitfest (\S+) with status: (.+)$/.exec(h.what);
+    if (m && m[2]!.trim().toLowerCase() === final.toLowerCase() && h.when && (!close || h.when > close)) close = h.when;
+  }
+  const firsts = p.threads.map((t) => t.firstAt).filter((x): x is string => !!x).sort();
+  const firstDate = firsts[0] ?? p.created;
+  const subjects = p.threads.map((t) => t.subject).filter(Boolean);
+  let summary = p.title;
+  if (p.tags.length) summary += "\nTags: " + p.tags.join(", ");
+  if (subjects.length) summary += "\nThreads: " + subjects.join("; ");
+  const url = `${CF_BASE}/patch/${pid}/`;
+  return {
+    id: `cf:${pid}`, kind: "cf", rawRef: `patch/${pid}`,
+    title: p.title, status: final, date: close, firstDate, url,
+    threadUrls: [...new Set(p.threads.map((t) => flatUrl(t.msgid)))],
+    authors: splitPeople(p.authors), reviewers: splitPeople(p.reviewers), objectors: [],
+    tags: p.tags.join(", "), summary, objection: "",
+    touchPoints: touchPoints([p.title, ...p.tags, ...subjects].join(" ")),
+    otherLinks: p.otherLinks,
+    provenance: {
+      source: "commitfest patch page", cache_url: url, committer: p.committer,
+      target_version: p.targetVersion, cf_history: p.statuses,
+      decided_at: close, first_at: firstDate,
+      date_source: close ? "history 'Closed in commitfest' line" : null,
+    },
+  };
+}
+
+function listOnlyRecord(pid: number, cf: number, row: CfListRow): PrecedentRecord {
+  return {
+    id: `cf:${pid}`, kind: "cf", rawRef: `patch/${pid}`,
+    title: row.title, status: row.status ?? "", date: null, firstDate: null,
+    url: `${CF_BASE}/patch/${pid}/`, threadUrls: [],
+    authors: splitPeople(row.authors), reviewers: splitPeople(row.reviewers), objectors: [],
+    tags: row.tags.join(", "),
+    summary: row.title + (row.tags.length ? "\nTags: " + row.tags.join(", ") : ""),
+    objection: "", touchPoints: touchPoints(row.title + " " + row.tags.join(" ")), otherLinks: [],
+    provenance: { source: "commitfest list page only (patch page not fetched)", cache_url: `${CF_BASE}/${cf}/`, cf, latest_mail: row.latestMail },
+  };
+}
+
+export async function crawlCommitfest(opts: { limit?: number; offline?: boolean }) {
+  const arch = await fetchPage(CF_BASE + "/archive/", opts);
+  if (!arch) throw new Error("cannot read CommitFest archive page");
+  const cfs = parseArchive(arch);
+  const order: number[] = [];
+  const latestRow = new Map<number, { cf: number; row: CfListRow }>();
+  const cov: Record<string, unknown> = { commitfests: `${cfs[0]?.id}..${cfs.at(-1)?.id} (${cfs.length})` };
+  let listOk = 0;
+  const listMissing: string[] = [];
+  for (const st of [6, 7, 8]) { // Rejected first, then RwF, then Withdrawn
+    for (const cf of cfs) {
+      const url = `${CF_BASE}/${cf.id}/?status=${st}`;
+      const h = await fetchPage(url, opts);
+      if (h === null) { listMissing.push(url); continue; }
+      listOk++;
+      for (const row of parseList(h)) {
+        if (row.status !== CF_STATUSES[st]) continue; // defensive
+        if (!latestRow.has(row.id)) order.push(row.id);
+        const prev = latestRow.get(row.id);
+        if (!prev || cf.id > prev.cf) latestRow.set(row.id, { cf: cf.id, row });
+      }
+    }
+  }
+  Object.assign(cov, { list_pages_ok: listOk, list_pages_missing: listMissing, unique_patches_in_lists: order.length });
+  log(`cf: ${listOk} list pages, ${order.length} unique patches`);
+  const todo = opts.limit ? order.slice(0, opts.limit) : order;
+  const recs: PrecedentRecord[] = [];
+  const c = { patch_pages_ok: 0, patch_pages_missing: 0, patch_pages_unparsed: 0, dropped_final_not_negative: 0, list_only_rows: 0 };
+  const finalElsewhere: Record<string, number> = {};
+  let n = 0;
+  for (const pid of todo) {
+    n++;
+    const url = `${CF_BASE}/patch/${pid}/`;
+    const wasCached = isCached(url);
+    const h = await fetchPage(url, opts);
+    if (!wasCached && n % 100 === 0) log(`  patch pages: ${n}/${todo.length} ${JSON.stringify(fetchStats)}`);
+    const lr = latestRow.get(pid)!;
+    const p = h ? parsePatch(h) : null;
+    if (!p) {
+      if (h) { c.patch_pages_unparsed++; log(`  patch ${pid}: page did not parse`); } else c.patch_pages_missing++;
+      c.list_only_rows++;
+      recs.push(listOnlyRecord(pid, lr.cf, lr.row));
+      continue;
+    }
+    c.patch_pages_ok++;
+    const final = finalStatus(p) ?? lr.row.status ?? "";
+    if (!NEGATIVE.has(final.toLowerCase())) {
+      c.dropped_final_not_negative++;
+      finalElsewhere[final] = (finalElsewhere[final] ?? 0) + 1;
+      continue;
+    }
+    recs.push(patchRecord(pid, p, CF_STATUSES[[6, 7, 8].find((k) => CF_STATUSES[k]!.toLowerCase() === final.toLowerCase())!]!));
+  }
+  Object.assign(cov, c, { final_status_of_dropped: finalElsewhere, skipped_by_limit: order.length - todo.length });
+  return { recs, cov, complete: !opts.limit && c.patch_pages_missing === 0 };
+}