pgroast 0.0.0-stage → 0.1.0

package/src/corpus/cli.ts

@@ -0,0 +1,189 @@
+// PG Roast precedent corpus CLI.
+//
+//   bun run src/corpus/cli.ts build [--source cf|reverts|wiki|all] [--limit N] [--offline]
+//   bun run src/corpus/cli.ts search "<query>" [--before YYYY-MM-DD] [--kind cf,revert,do_not_want] [-n 10] [--json] [--include-undated]
+//   bun run src/corpus/cli.ts stats
+//   bun run src/corpus/cli.ts selftest
+
+import { crawlCommitfest } from "./cf.ts";
+import { applySchema, connect, hasTrgm, store } from "./db.ts";
+import { fetchStats, readCached } from "./fetch.ts";
+import { collectReverts, git, PG_REPO } from "./reverts.ts";
+import { closeSearch, searchPrecedents } from "./search.ts";
+import { canonMsgid, extractMsgids, log, unescapeHtml, type Kind } from "./util.ts";
+import { collectWiki, WIKI_PAGE } from "./wiki.ts";
+
+type Args = { _: string[]; [k: string]: string | boolean | string[] };
+
+function parseArgs(argv: string[]): Args {
+  const a: Args = { _: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const t = argv[i]!;
+    if (t.startsWith("-")) {
+      const k = t.replace(/^-+/, "");
+      const next = argv[i + 1];
+      if (["json", "offline", "include-undated"].includes(k) || next === undefined) a[k] = true;
+      else { a[k] = next; i++; }
+    } else a._.push(t);
+  }
+  return a;
+}
+
+const KIND_OF: Record<string, Kind> = { cf: "cf", reverts: "revert", wiki: "do_not_want" };
+
+async function build(a: Args) {
+  const source = (a.source as string) ?? "all";
+  if (!["cf", "reverts", "wiki", "all"].includes(source)) throw new Error("bad --source");
+  const limit = a.limit ? Number(a.limit) : undefined;
+  const offline = !!a.offline;
+  const sql = connect();
+  const sch = await applySchema(sql);
+  log(sch.trgm ? "pg_trgm: available" : `pg_trgm: NOT available (${sch.trgmError}); title similarity disabled`);
+  for (const s of source === "all" ? ["wiki", "reverts", "cf"] : [source]) {
+    log(`== building ${s}`);
+    const r = s === "cf" ? await crawlCommitfest({ limit, offline })
+      : s === "reverts" ? await collectReverts({ limit })
+      : await collectWiki({ offline });
+    const cov = { ...r.cov, records: r.recs.length, fetch_stats: { ...fetchStats } };
+    const st = await store(sql, KIND_OF[s]!, r.recs, r.complete, cov);
+    console.log(JSON.stringify({ [s]: { ...cov, ...st, complete: r.complete } }, null, 1));
+  }
+  await stats(sql);
+  await sql.close();
+}
+
+async function stats(sql = connect()) {
+  const rows = await sql`
+    select kind, status, count(*)::int n, min(first_date)::text f1, max(first_date)::text f2,
+           min(date)::text d1, max(date)::text d2,
+           count(*) filter (where cardinality(thread_urls) > 0)::int with_threads
+    from precedent group by 1, 2 order by 1, 2`;
+  console.log("kind        status                   rows  w/thread  first_date range        date range");
+  for (const r of rows) {
+    console.log(`${r.kind.padEnd(11)} ${r.status.padEnd(24)} ${String(r.n).padStart(5)} ${String(r.with_threads).padStart(8)}  ` +
+      `${r.f1 ?? "-"}..${r.f2 ?? "-"}  ${r.d1 ?? "-"}..${r.d2 ?? "-"}`);
+  }
+  const [t] = await sql`select count(*)::int n from precedent`;
+  console.log(`total ${t.n} rows; pg_trgm ${(await hasTrgm(sql)) ? "on" : "off"}`);
+}
+
+async function search(a: Args) {
+  const query = a._[1];
+  if (!query) throw new Error('usage: search "<query>" [--before YYYY-MM-DD] [--kind ...] [-n 10] [--json]');
+  const res = await searchPrecedents({
+    query,
+    before: a.before as string | undefined,
+    kinds: a.kind ? (String(a.kind).split(",") as Kind[]) : undefined,
+    limit: a.n ? Number(a.n) : 10,
+    includeUndated: !!a["include-undated"],
+  });
+  await closeSearch();
+  if (a.json) {
+    console.log(JSON.stringify({ query, before: a.before ?? null, results: res }, null, 1));
+    return;
+  }
+  if (!res.length) { console.log("no precedent found"); return; }
+  res.forEach((r, i) => {
+    console.log(`${String(i + 1).padStart(2)}. [${r.kind}/${r.status}] ${r.date ?? r.first_date ?? "undated"}  ${r.title}  (${r.match} ${r.score})`);
+    console.log(`    ${r.url}`);
+    for (const t of r.thread_urls.slice(0, 2)) console.log(`    thread: ${t}`);
+    if (r.objection) {
+      const o = r.objection.replace(/\s+/g, " ");
+      console.log(`    objection: ${o.slice(0, 200)}${o.length > 200 ? "..." : ""}`);
+    }
+  });
+}
+
+// ------------------------------------------------------------------ selftest
+
+async function selftest() {
+  const fails: string[] = [];
+  const check = (ok: unknown, msg: string) => { if (!ok) fails.push(msg); };
+  const sql = connect();
+  try {
+    await sql`select 1`;
+  } catch (e) {
+    console.log("FAIL database not reachable:", String(e));
+    process.exit(1);
+  }
+  const [schema] = await sql`
+    select to_regclass('precedent') is not null as t,
+           exists (select 1 from pg_attribute where attrelid = to_regclass('precedent') and attname = 'tsv' and attgenerated = 's') as gen,
+           exists (select 1 from pg_indexes where tablename = 'precedent' and indexdef ilike '%using gin (tsv)%') as gin,
+           exists (select 1 from pg_constraint where conname = 'precedent_kind_raw_ref_key') as uniq`;
+  check(schema.t && schema.gen && schema.gin && schema.uniq, `schema incomplete: ${JSON.stringify(schema)}`);
+  const [cnt] = await sql`select count(*)::int n, count(*) filter (where tsv = ''::tsvector)::int empty from precedent`;
+  check(cnt.n > 0, "precedent table is empty");
+  check(cnt.empty === 0, `${cnt.empty} rows with empty tsvector`);
+  const kinds = await sql`select kind, count(*)::int n from precedent group by 1`;
+  for (const k of ["cf", "revert", "do_not_want"]) check(kinds.some((r: any) => r.kind === k && r.n > 0), `no rows of kind ${k}`);
+  const r1 = await searchPrecedents({ query: "optimizer hints", sql });
+  check(r1.length > 0, "search 'optimizer hints' returned nothing");
+  const r2 = await searchPrecedents({ query: `'); drop table precedent; -- " OR *:`, sql });
+  check(Array.isArray(r2), "hostile query raised");
+  const r3 = await searchPrecedents({ query: "revert", before: "2010-01-01", sql });
+  check(r3.every((r) => (r.date ?? "") < "2010-01-01" && (r.first_date ?? "") < "2010-01-01" && (r.date || r.first_date)),
+    "--before leaked a row dated on/after the cutoff or an undated row");
+
+  // URL patterns, in SQL
+  const bad = await sql`
+    select id, url from precedent where not (
+      (kind = 'cf' and url ~ '^https://commitfest\\.postgresql\\.org/patch/[0-9]+/$' and url = 'https://commitfest.postgresql.org/' || raw_ref || '/')
+      or (kind = 'revert' and url ~ '^https://git\\.postgresql\\.org/gitweb/\\?p=postgresql\\.git;a=commit;h=[0-9a-f]{40}$' and url like '%=' || raw_ref)
+      or (kind = 'do_not_want' and url ~ '^https://wiki\\.postgresql\\.org/wiki/Not_Worth_Doing#[A-Za-z0-9_.-]+$'))
+    union all
+    select id, t from precedent, unnest(thread_urls) t
+    where t !~ '^https://www\\.postgresql\\.org/message-id/flat/[^[:space:]/?#]+@[^[:space:]/?#]+$'
+    union all
+    select id, t from precedent, unnest(other_links) t where t !~ '^https?://[^[:space:]]+$'`;
+  for (const b of bad) check(false, `${b.id}: bad url ${b.url}`);
+
+  // provenance: every thread message-id appears in the cached source page / git output
+  const wikiHtml = readCached(WIKI_PAGE) ?? "";
+  const rows = await sql`select id, kind, raw_ref, url, thread_urls, provenance from precedent`;
+  let prov = 0;
+  for (const r of rows) {
+    let ids: Set<string>;
+    if (r.kind === "cf") {
+      const pid = /^patch\/(\d+)$/.exec(r.raw_ref)?.[1];
+      const page = readCached(r.provenance.cache_url ?? r.url) ?? "";
+      check(pid && page.includes(`/patch/${pid}`), `${r.id}: patch id not found in cached source page`);
+      ids = new Set([...page.matchAll(/message-id\/flat\/([^"]+)"/g)].map((m) => canonMsgid(unescapeHtml(m[1]!))));
+    } else if (r.kind === "revert") {
+      const reverted = ((r.provenance.reverted_commits ?? []) as { hash: string }[]).map((c) => c.hash);
+      const g = await git("show", "-s", "--format=%B", r.raw_ref, ...reverted);
+      check(g.code === 0, `${r.id}: commit not in ${PG_REPO}`);
+      ids = new Set(extractMsgids(g.out).map(canonMsgid));
+    } else {
+      const anchor = r.url.split("#")[1];
+      check(wikiHtml.includes(`id="${anchor}"`), `${r.id}: anchor not on cached wiki page`);
+      ids = new Set(extractMsgids(wikiHtml).map(canonMsgid));
+    }
+    for (const t of r.thread_urls as string[]) {
+      prov++;
+      check(ids.has(t.replace("https://www.postgresql.org/message-id/flat/", "")), `${r.id}: thread ${t} not found in its source`);
+    }
+  }
+  console.log(JSON.stringify({ rows: cnt.n, kinds, url_violations: bad.length, threads_checked: prov, failures: fails.length, trgm: await hasTrgm(sql) }));
+  for (const f of fails.slice(0, 30)) console.log("FAIL", f);
+  await sql.close();
+  await closeSearch();
+  if (fails.length) process.exit(1);
+  console.log("selftest OK");
+}
+
+const a = parseArgs(process.argv.slice(2));
+const cmd = a._[0];
+try {
+  if (cmd === "build") await build(a);
+  else if (cmd === "search") await search(a);
+  else if (cmd === "stats") { const s = connect(); await stats(s); await s.close(); }
+  else if (cmd === "selftest") await selftest();
+  else {
+    console.error("usage: cli.ts build|search|stats|selftest   (see header of src/corpus/cli.ts)");
+    process.exit(2);
+  }
+} catch (e) {
+  console.error(String((e as Error).stack ?? e));
+  process.exit(1);
+}

package/src/corpus/db.ts

@@ -0,0 +1,119 @@
+// Postgres access for the precedent corpus (Bun.sql, no npm driver).
+//
+// PGROAST_DSN accepts either a libpq keyword string ("dbname=pgroast host=/tmp")
+// or a URL ("postgres://localhost/pgroast?host=/tmp"). Bun 1.3.5 forwards an
+// unknown URL query parameter such as ?host= to the server as a startup GUC,
+// which fails, so the DSN is parsed here and a socket directory is turned
+// into Bun's `path` option (<dir>/.s.PGSQL.<port>).
+
+import { SQL } from "bun";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { ROOT } from "./fetch.ts";
+import type { Kind, PrecedentRecord } from "./util.ts";
+
+export const DEFAULT_DSN = "dbname=pgroast host=/tmp";
+
+export function parseDsn(dsn: string): Record<string, string> {
+  const kv: Record<string, string> = {};
+  if (/^postgres(ql)?:\/\//.test(dsn)) {
+    const u = new URL(dsn);
+    if (u.hostname) kv.host = decodeURIComponent(u.hostname);
+    if (u.port) kv.port = u.port;
+    if (u.username) kv.user = decodeURIComponent(u.username);
+    if (u.password) kv.password = decodeURIComponent(u.password);
+    const db = decodeURIComponent(u.pathname.replace(/^\//, ""));
+    if (db) kv.dbname = db;
+    for (const [k, v] of u.searchParams) kv[k === "database" ? "dbname" : k] = v;
+  } else {
+    for (const m of dsn.matchAll(/(\w+)\s*=\s*('(?:[^'\\]|\\.)*'|\S+)/g)) {
+      kv[m[1]!] = m[2]!.startsWith("'") ? m[2]!.slice(1, -1).replace(/\\(.)/g, "$1") : m[2]!;
+    }
+  }
+  return kv;
+}
+
+export function connect(dsn = process.env.PGROAST_DSN ?? DEFAULT_DSN): SQL {
+  const kv = parseDsn(dsn);
+  const port = Number(kv.port ?? 5432);
+  const opts: Record<string, unknown> = {
+    database: kv.dbname ?? "pgroast",
+    username: kv.user ?? process.env.PGUSER ?? process.env.USER,
+    max: 4,
+    idleTimeout: 5,
+  };
+  if (kv.password) opts.password = kv.password;
+  const host = kv.host ?? "/tmp";
+  if (host.startsWith("/")) opts.path = join(host, `.s.PGSQL.${port}`);
+  else Object.assign(opts, { hostname: host, port });
+  return new SQL(opts as ConstructorParameters<typeof SQL>[0]);
+}
+
+/** Apply sql/schema.sql; try pg_trgm. Returns whether pg_trgm is usable. */
+export async function applySchema(sql: SQL): Promise<{ trgm: boolean; trgmError?: string }> {
+  await sql.unsafe(readFileSync(join(ROOT, "sql", "schema.sql"), "utf8"));
+  try {
+    await sql.unsafe("create extension if not exists pg_trgm");
+    await sql.unsafe("create index if not exists precedent_title_trgm_idx on precedent using gin (title gin_trgm_ops)");
+    return { trgm: true };
+  } catch (e) {
+    return { trgm: false, trgmError: String((e as Error).message ?? e).split("\n")[0] };
+  }
+}
+
+export async function hasTrgm(sql: SQL): Promise<boolean> {
+  const r = await sql`select 1 from pg_extension where extname = 'pg_trgm'`;
+  return r.length > 0;
+}
+
+/**
+ * Upsert records on (kind, raw_ref). When `complete`, rows of this kind that
+ * the source no longer produced are deleted, so a full rebuild is exact.
+ */
+export async function store(sql: SQL, kind: Kind, recs: PrecedentRecord[], complete: boolean, coverage: unknown) {
+  const payload = JSON.stringify(recs.map((r) => ({
+    id: r.id, kind: r.kind, raw_ref: r.rawRef, title: r.title, status: r.status,
+    date: r.date, first_date: r.firstDate, url: r.url, thread_urls: r.threadUrls,
+    authors: r.authors, reviewers: r.reviewers, objectors: r.objectors, tags: r.tags,
+    summary: r.summary, objection: r.objection, touch_points: r.touchPoints,
+    other_links: r.otherLinks, provenance: r.provenance,
+  })));
+  let deleted = 0;
+  await sql.begin(async (tx) => {
+    await tx`
+      insert into precedent as p (id, kind, raw_ref, title, status, date, first_date, url,
+        thread_urls, authors, reviewers, objectors, tags, summary, objection,
+        touch_points, other_links, provenance, built_at)
+      select r.id, r.kind, r.raw_ref, r.title, r.status, left(r.date, 10)::date, left(r.first_date, 10)::date, r.url,
+        array(select jsonb_array_elements_text(r.thread_urls)),
+        array(select jsonb_array_elements_text(r.authors)),
+        array(select jsonb_array_elements_text(r.reviewers)),
+        array(select jsonb_array_elements_text(r.objectors)),
+        r.tags, r.summary, r.objection,
+        array(select jsonb_array_elements_text(r.touch_points)),
+        array(select jsonb_array_elements_text(r.other_links)),
+        r.provenance, now()
+      from jsonb_to_recordset((${payload}::text)::jsonb) as r(
+        id text, kind text, raw_ref text, title text, status text, date text, first_date text, url text,
+        thread_urls jsonb, authors jsonb, reviewers jsonb, objectors jsonb, tags text, summary text,
+        objection text, touch_points jsonb, other_links jsonb, provenance jsonb)
+      on conflict (kind, raw_ref) do update set
+        id = excluded.id, title = excluded.title, status = excluded.status, date = excluded.date,
+        first_date = excluded.first_date, url = excluded.url, thread_urls = excluded.thread_urls,
+        authors = excluded.authors, reviewers = excluded.reviewers, objectors = excluded.objectors,
+        tags = excluded.tags, summary = excluded.summary, objection = excluded.objection,
+        touch_points = excluded.touch_points, other_links = excluded.other_links,
+        provenance = excluded.provenance, built_at = excluded.built_at`;
+    if (complete) {
+      const keys = JSON.stringify(recs.map((r) => r.rawRef));
+      const d = await tx`
+        delete from precedent where kind = ${kind}
+          and raw_ref not in (select jsonb_array_elements_text((${keys}::text)::jsonb))`;
+      deleted = d.count ?? 0;
+    }
+    await tx`
+      insert into build_log (kind, built_at, coverage) values (${kind}, now(), (${JSON.stringify(coverage)}::text)::jsonb)
+      on conflict (kind) do update set built_at = excluded.built_at, coverage = excluded.coverage`;
+  });
+  return { upserted: recs.length, deleted };
+}

package/src/corpus/fetch.ts

@@ -0,0 +1,104 @@
+// Polite, cached HTTP fetcher for the precedent corpus.
+//
+//  * User-Agent "pgroast-research/0.1", GET only, no cookies, no sign-in
+//  * >= 1.05 s between requests to the same host
+//  * every 200 response is cached under data/cache/<host>/ and never refetched;
+//    a 404 leaves a "<file>.404" marker
+//  * 429 / 5xx: honour Retry-After, else exponential back-off 5 s .. 300 s, 6 tries
+//
+// The cache file naming is byte-compatible with the earlier Python builder so
+// pages crawled by it are reused.
+
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+export const USER_AGENT = "pgroast-research/0.1";
+const MIN_INTERVAL_MS = 1050;
+export const ROOT = resolve(import.meta.dir, "..", "..");
+export const CACHE_DIR = process.env.PGROAST_CACHE_DIR ?? join(ROOT, "data", "cache");
+
+export const fetchStats = { fetched: 0, cached: 0, failed: 0, notFound: 0 };
+const lastRequest = new Map<string, number>();
+
+export function cachePath(url: string): string {
+  const u = new URL(url);
+  let name = u.pathname.replace(/^\/+|\/+$/g, "") || "_root";
+  name = name.replace(/[^A-Za-z0-9._-]+/g, "_");
+  const query = u.search.replace(/^\?/, "");
+  if (query) name += "__" + query.replace(/[^A-Za-z0-9._=-]+/g, "_");
+  if (name.length > 150) {
+    name = name.slice(0, 100) + "_" + createHash("sha1").update(name).digest("hex").slice(0, 16);
+  }
+  return join(CACHE_DIR, u.host, name + ".html");
+}
+
+export function isCached(url: string): boolean {
+  const p = cachePath(url);
+  return existsSync(p) || existsSync(p + ".404");
+}
+
+export function readCached(url: string): string | null {
+  const p = cachePath(url);
+  return existsSync(p) ? readFileSync(p, "utf8") : null;
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/** Page text, or null for 404 / failure / offline cache miss. */
+export async function fetchPage(url: string, opts: { offline?: boolean } = {}): Promise<string | null> {
+  const p = cachePath(url);
+  if (existsSync(p)) {
+    fetchStats.cached++;
+    return readFileSync(p, "utf8");
+  }
+  if (existsSync(p + ".404")) {
+    fetchStats.notFound++;
+    return null;
+  }
+  if (opts.offline) return null;
+  const host = new URL(url).host;
+  let delay = 5000;
+  for (let attempt = 0; attempt < 6; attempt++) {
+    const wait = (lastRequest.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
+    if (wait > 0) await sleep(wait);
+    lastRequest.set(host, Date.now());
+    try {
+      const r = await fetch(url, {
+        headers: { "User-Agent": USER_AGENT },
+        redirect: "follow",
+        signal: AbortSignal.timeout(60_000),
+      });
+      if (r.ok) {
+        const body = await r.text();
+        mkdirSync(dirname(p), { recursive: true });
+        writeFileSync(p + ".tmp", body);
+        renameSync(p + ".tmp", p);
+        fetchStats.fetched++;
+        return body;
+      }
+      if (r.status === 404) {
+        mkdirSync(dirname(p), { recursive: true });
+        writeFileSync(p + ".404", "");
+        fetchStats.notFound++;
+        return null;
+      }
+      if (r.status === 429 || r.status >= 500) {
+        const ra = r.headers.get("retry-after");
+        const ms = ra && /^\d+$/.test(ra) ? Number(ra) * 1000 : delay;
+        console.error(`  HTTP ${r.status} on ${url}; backing off ${ms / 1000}s`);
+        await sleep(ms);
+        delay = Math.min(delay * 2, 300_000);
+        continue;
+      }
+      console.error(`  HTTP ${r.status} on ${url}; giving up`);
+      break;
+    } catch (e) {
+      console.error(`  network error on ${url}: ${e}; backing off ${delay / 1000}s`);
+      await sleep(delay);
+      delay = Math.min(delay * 2, 300_000);
+    }
+  }
+  fetchStats.failed++;
+  return null;
+}

package/src/corpus/reverts.ts

@@ -0,0 +1,133 @@
+// Git source: commits whose message has a line starting with "Revert"
+// (git log --grep='^Revert' -i) in the local Postgres clone. Read-only.
+
+import { resolve as pathResolve } from "node:path";
+import { extractMsgids, flatUrl, touchPoints, type PrecedentRecord } from "./util.ts";
+
+export const PG_REPO = process.env.PGROAST_PG_REPO ?? pathResolve(import.meta.dir, "..", "..", "..", "postgres");
+export const GITWEB = "https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=";
+
+export async function git(...args: string[]): Promise<{ out: string; code: number }> {
+  const p = Bun.spawn(["git", "-C", PG_REPO, ...args], { stdout: "pipe", stderr: "pipe" });
+  const out = await new Response(p.stdout).text();
+  const code = await p.exited;
+  return { out, code };
+}
+
+async function gitOk(...args: string[]): Promise<string> {
+  const r = await git(...args);
+  if (r.code !== 0) throw new Error(`git ${args.join(" ")} failed (${r.code})`);
+  return r.out;
+}
+
+const utc = (iso: string) => new Date(iso).toISOString().slice(0, 19);
+
+interface Commit { hash: string; author: string; date: string; subject: string; body: string }
+
+const commitCache = new Map<string, Commit>();
+async function commit(h: string): Promise<Commit> {
+  const hit = commitCache.get(h);
+  if (hit) return hit;
+  const out = await gitOk("log", "-1", "--format=%H%x00%an%x00%cI%x00%s%x00%B", h);
+  const [hash, author, cd, subject, ...rest] = out.split("\x00");
+  const c = { hash: hash!, author: author!, date: utc(cd!), subject: subject!, body: rest.join("\x00").trim() };
+  commitCache.set(h, c);
+  return c;
+}
+
+async function resolve(short: string): Promise<string | null> {
+  const r = await git("rev-parse", "-q", "--verify", short + "^{commit}");
+  return r.code === 0 ? r.out.trim() : null;
+}
+
+let subjects: Map<string, [string, string][]> | null = null;
+async function subjectIndex() {
+  if (!subjects) {
+    subjects = new Map();
+    for (const ln of (await gitOk("log", "--format=%H%x00%cI%x00%s")).split("\n")) {
+      if (!ln) continue;
+      const [H, d, s] = ln.split("\x00");
+      const k = s!.trim();
+      if (!subjects.has(k)) subjects.set(k, []);
+      subjects.get(k)!.push([H!, utc(d!)]);
+    }
+  }
+  return subjects;
+}
+
+/**
+ * CVS-era commits often have the whole message on the subject line. Use its
+ * first sentence (max 160 chars) as the title; the full subject stays in summary.
+ */
+export function shortTitle(subject: string): string {
+  if (subject.length <= 160) return subject;
+  const m = /^(.{20,160}?[.;:])\s/.exec(subject);
+  return m ? m[1]! : subject.slice(0, 157).replace(/\s+\S*$/, "") + "...";
+}
+
+const TRAILER = /^(Discussion|Backpatch-through|Back-patch-through|Reported-by|Author|Reviewed-by|Tested-by|Security|Co-authored-by|Bug|Suggested-by|Diagnosed-by|Backpatch|Analyzed-by)\s*:/i;
+
+/** Revert message minus subject line and trailers (verbatim otherwise). */
+export function revertReason(body: string, subject: string): string {
+  let lines = body.split("\n");
+  if (lines[0]?.trim() === subject.trim()) lines = lines.slice(1);
+  return lines.filter((l) => !TRAILER.test(l.trim())).join("\n").trim();
+}
+
+export async function collectReverts(opts: { limit?: number }) {
+  const hashes = (await gitOk("log", "--grep=^Revert", "-i", "--format=%H")).split(/\s+/).filter(Boolean);
+  const cov = {
+    git_grep_matches: hashes.length, kept_subject_revert: 0, kept_partial_revert: 0,
+    skipped_no_revert_target: 0, reverted_commit_resolved: 0, reverted_by_subject_match: 0,
+  };
+  const recs: PrecedentRecord[] = [];
+  for (const h of opts.limit ? hashes.slice(0, opts.limit) : hashes) {
+    const c = await commit(h);
+    const subjRev = c.subject.toLowerCase().startsWith("revert");
+    const targets: string[] = [];
+    for (const m of c.body.matchAll(/\b([0-9a-f]{7,40})\b/g)) {
+      const ctx = c.body.slice(Math.max(0, m.index! - 120), m.index!).toLowerCase();
+      if (!subjRev && !ctx.includes("revert")) continue;
+      const full = await resolve(m[1]!);
+      if (full && full !== c.hash && !targets.includes(full)) targets.push(full);
+    }
+    let how: string | null = targets.length ? "hash named in message" : null;
+    if (subjRev && !targets.length) {
+      const sm = /^Revert\s+"(.+)"\s*\.?$/i.exec(c.subject);
+      if (sm) {
+        const cands = ((await subjectIndex()).get(sm[1]!.trim()) ?? []).filter(([H, d]) => d < c.date && H !== c.hash);
+        if (cands.length) {
+          targets.push(cands.reduce((a, b) => (b[1] > a[1] ? b : a))[0]);
+          how = "exact subject match, latest earlier commit";
+          cov.reverted_by_subject_match++;
+        }
+      }
+    }
+    if (!subjRev && !targets.length) { cov.skipped_no_revert_target++; continue; }
+    if (subjRev) cov.kept_subject_revert++; else cov.kept_partial_revert++;
+    const tinfo = await Promise.all(targets.slice(0, 5).map(commit));
+    if (tinfo.length) cov.reverted_commit_resolved++;
+    const msgids = extractMsgids(c.body);
+    for (const t of tinfo) for (const mid of extractMsgids(t.body)) if (!msgids.includes(mid)) msgids.push(mid);
+    const firstDate = [c.date, ...tinfo.map((t) => t.date)].sort()[0]!;
+    const revSubjects = tinfo.map((t) => t.subject);
+    recs.push({
+      id: `revert:${c.hash}`, kind: "revert", rawRef: c.hash,
+      title: shortTitle(c.subject), status: subjRev ? "Reverted" : "Partially reverted",
+      date: c.date, firstDate, url: GITWEB + c.hash,
+      threadUrls: [...new Set(msgids.map(flatUrl))],
+      authors: [...new Set(tinfo.map((t) => t.author))], reviewers: [], objectors: [],
+      tags: "",
+      summary: c.subject + (revSubjects.length ? "\nReverted commit(s): " + revSubjects.join("; ") : ""),
+      objection: revertReason(c.body, c.subject),
+      touchPoints: touchPoints([c.subject, ...revSubjects, c.body].join(" ")),
+      otherLinks: tinfo.map((t) => GITWEB + t.hash),
+      provenance: {
+        source: "git log of postgresql.git (local clone)", commit: c.hash, reverted_by: c.author, reverted_commit_match: how,
+        decided_at: c.date, first_at: firstDate,
+        reverted_commits: tinfo.map((t) => ({ hash: t.hash, subject: t.subject, date: t.date, author: t.author })),
+      },
+    });
+  }
+  return { recs, cov, complete: !opts.limit };
+}