pgroast 0.0.0-stage → 0.1.0

package/src/roast/tone.ts

@@ -0,0 +1,216 @@
+// Tone lint: a deterministic check over every prose field of a roast, before files are
+// written. What a person may forward must read as a calm, respectful, evidence-first review.
+// On a hit, one repair call asks the synthesis model to rephrase only the flagged sentences;
+// what is still flagged afterwards is listed in roast.json (`tone_lint.remaining`).
+//
+// Verbatim quotes from the archives are not prose of ours and are not checked.
+
+import type { LlmBackend } from "./llm.ts";
+import { scrubUrls, type AllowedUrls } from "./sanitize.ts";
+import { TONE_REPAIR_SCHEMA, type SanitizerReport, type FinalObjection, type Roast, type ToneHit, type ToneLint } from "./schema.ts";
+
+export interface ToneRule {
+  id: string;
+  re: RegExp;
+}
+
+const words = (id: string, list: string[]): ToneRule => ({
+  id,
+  re: new RegExp(`\\b(?:${list.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/ /g, "\\s+")).join("|")})\\b`, "i"),
+});
+
+export const TONE_RULES: ToneRule[] = [
+  words("intensifier", [
+    "obviously", "clearly", "simply", "of course", "trivially", "needless to say", "plainly wrong",
+    "any competent", "everyone knows", "everybody knows", "it goes without saying", "no-brainer", "by definition wrong",
+  ]),
+  words("hyperbole", [
+    "nonsense", "ridiculous", "absurd", "laughable", "insane", "crazy", "stupid", "dumb", "idiotic", "garbage", "terrible",
+    "horrible", "awful", "atrocious", "disaster", "disastrous", "nightmare", "horror story", "abomination",
+    "hopeless", "worthless", "useless", "a joke", "dead on arrival", "doomed", "utterly", "completely broken", "totally",
+  ]),
+  words("sarcasm", [
+    "good luck", "nice try", "yeah right", "surprise surprise", "surprise, surprise", "so-called", "lol", "sigh", "oh great",
+    "what could possibly go wrong", "who would have thought", "brilliant idea", "bless", "pray tell", "hand-waving", "handwaving",
+    "hand-wavy", "handwavy",
+  ]),
+  words("roast vocabulary", ["roast", "roasts", "roasted", "roasting", "roaster", "roasters", "beat down", "beatdown", "beat-down", "takedown", "smackdown", "rejector", "kill the patch", "shoot down", "shot down", "tear apart", "torn apart"]),
+  words("about the author", [
+    "you failed to", "you forgot", "you neglected", "you ignored", "you did not bother", "you didn't bother", "you clearly", "you should have known",
+    "the author failed", "the author fails", "the author forgot", "the author neglected", "the author ignored", "the author ignores",
+    "the author did not bother", "the author didn't bother", "the author does not understand", "the author doesn't understand",
+    "the author is unaware", "the author has no idea", "the author never bothered", "the author is wrong", "authors failed",
+    "lazy", "sloppy", "amateur", "amateurish", "incompetent", "clueless", "ignorant",
+  ]),
+  // Writing rules (prompts/writing.md): filler, stock review phrases, words with a simpler form.
+  words("filler", [
+    "importantly", "notably", "crucially", "essentially", "basically", "it is worth noting", "it's worth noting", "worth noting that",
+    "it should be noted", "needless to say", "as such", "at the end of the day", "here is the catch", "here's the catch",
+  ]),
+  words("stock phrase", ["the material does not describe", "nothing in the material", "remains unclear", "raises the question", "it is not specified"]),
+  words("simpler word exists", ["utilize", "utilizes", "utilized", "utilizing", "utilization", "facilitate", "facilitates", "facilitated", "in order to", "approximately", "leverage", "leverages", "leveraging"]),
+  // "!" is fine in "!=" and inside `code`; code spans are removed before the check.
+  { id: "exclamation mark", re: /!(?!=)/ },
+];
+
+/** Rule id for an objection's question to the author that is not a question. */
+export const NO_QUESTION = "does not end in a question";
+
+const stripCode = (s: string) => s.replace(/`[^`]*`/g, (m) => " ".repeat(m.length));
+
+/** Sentences of a text, as substrings of it (so a rewrite can replace exactly one). */
+export function sentences(text: string): string[] {
+  return text
+    .split(/(?<=[.?!]["')\]]*)\s+|\n+/)
+    .map((x) => x.trim())
+    .filter((x) => x.length > 0);
+}
+
+export function lintText(text: string, path = ""): ToneHit[] {
+  const hits: ToneHit[] = [];
+  for (const sentence of sentences(text)) {
+    const bare = stripCode(sentence);
+    for (const rule of TONE_RULES) {
+      const m = bare.match(rule.re);
+      if (m) hits.push({ path, rule: rule.id, match: m[0], sentence });
+    }
+  }
+  return hits;
+}
+
+export interface ProseField {
+  path: string;
+  get(): string;
+  set(v: string): void;
+  /** The objection's question to the author: must end with "?". */
+  question?: boolean;
+}
+
+/** Every prose field a model wrote and a person may read. Quotes from the archives are left out. */
+export function proseFields(r: Roast): ProseField[] {
+  const out: ProseField[] = [];
+  const add = <T extends object>(obj: T, key: keyof T, path: string, question = false) => {
+    if (typeof obj[key] === "string") out.push({ path, get: () => obj[key] as string, set: (v) => void ((obj[key] as string) = v), question });
+  };
+  const list = (arr: string[], path: string) => arr.forEach((_, i) => out.push({ path: `${path}[${i}]`, get: () => arr[i]!, set: (v) => void (arr[i] = v) }));
+  const objection = (o: FinalObjection, path: string, ranked: boolean) => {
+    add(o, "title", `${path}.title`);
+    add(o, "failure_scenario", `${path}.failure_scenario`);
+    add(o, "argument", `${path}.argument`);
+    add(o, "author_must_show", `${path}.author_must_show`, ranked);
+    add(o.critic, "reasoning", `${path}.critic.reasoning`);
+    if (o.not_ranked_reason != null) add(o, "not_ranked_reason", `${path}.not_ranked_reason`);
+    if (o.withdrawal) add(o.withdrawal, "reason", `${path}.withdrawal.reason`);
+  };
+
+  add(r, "verdict_reasoning", "verdict_reasoning");
+  add(r, "provisional_note", "provisional_note");
+  add(r.brief, "problem", "brief.problem");
+  add(r.brief, "proposed_design", "brief.proposed_design");
+  add(r.brief, "thread_state", "brief.thread_state");
+  list(r.brief.user_visible_surface, "brief.user_visible_surface");
+  list(r.brief.author_claims, "brief.author_claims");
+  list(r.questions, "questions");
+  r.objections.forEach((o, i) => objection(o, `objections[${i}]`, true));
+  r.other_surviving.forEach((o, i) => objection(o, `other_surviving[${i}]`, true));
+  r.refuted.forEach((o, i) => objection(o, `refuted[${i}]`, false));
+  r.withdrawn.forEach((o, i) => objection(o, `withdrawn[${i}]`, false));
+  r.prior_attempts.forEach((p, i) => {
+    add(p, "what_is_different", `prior_attempts[${i}].what_is_different`);
+    if (p.enriched) add(p, "reason", `prior_attempts[${i}].reason`); // an AI summary, not a quote
+  });
+  r.touch_points.forEach((t, i) => add(t, "reason", `touch_points[${i}].reason`));
+  add(r.committability, "owner_note", "committability.owner_note");
+  add(r.committability, "liability", "committability.liability");
+  add(r.committability, "author_follow_up", "committability.author_follow_up");
+  add(r.not_now, "evidence_that_would_change_it", "not_now.evidence_that_would_change_it");
+  add(r.priority, "reason", "priority.reason");
+  r.self_advice.forEach((a, i) => add(a, "how", `self_advice[${i}].how`));
+  r.roaster_notes.forEach((n, i) => add(n, "note", `roaster_notes[${i}].note`));
+  r.rounds.forEach((x, i) => add(x, "note", `rounds[${i}].note`));
+  return out;
+}
+
+function lintField(f: ProseField): ToneHit[] {
+  const text = f.get();
+  const hits = lintText(text, f.path);
+  if (f.question && text.trim() && !/\?["')\]]*\s*$/.test(text)) {
+    const all = sentences(text);
+    hits.push({ path: f.path, rule: NO_QUESTION, match: "", sentence: all[all.length - 1] ?? text });
+  }
+  return hits;
+}
+
+/** All hits in a roast. Pure: no model, no change. */
+export function lintRoast(r: Roast): { fields: number; hits: ToneHit[] } {
+  const fields = proseFields(r);
+  return { fields: fields.length, hits: fields.flatMap(lintField) };
+}
+
+export interface ToneRepairOptions {
+  backend: LlmBackend;
+  model: string;
+  /** System prompt (prompts/tone.md plus the output contract). */
+  system: string;
+  allowed: AllowedUrls;
+  report: SanitizerReport;
+  log?: (m: string) => void;
+}
+
+/**
+ * Lint; on hits, one repair call for the flagged sentences only; lint again. Changes the roast
+ * in place and returns the record for roast.json. Without `repair` (or in a dry run) it only lints.
+ */
+export function toneCheck(r: Roast, repair?: ToneRepairOptions): Promise<ToneLint> {
+  return toneCheckFields(() => proseFields(r), repair);
+}
+
+/** The same check over any set of prose fields (the pushback report uses it too). */
+export async function toneCheckFields(getFields: () => ProseField[], repair?: ToneRepairOptions): Promise<ToneLint> {
+  const lint = () => {
+    const fields = getFields();
+    return { fields: fields.length, hits: fields.flatMap(lintField) };
+  };
+  const first = lint();
+  const result: ToneLint = { fields_checked: first.fields, hits_found: first.hits.length, sentences_rewritten: 0, repair_call: "not needed", remaining: first.hits };
+  if (first.hits.length === 0) return result;
+  if (!repair || repair.backend.dryRun) return { ...result, repair_call: "skipped" };
+
+  // One item per flagged sentence of a field.
+  const fields = new Map(getFields().map((f) => [f.path, f]));
+  const items: { n: number; path: string; sentence: string; flagged: string[] }[] = [];
+  for (const h of first.hits) {
+    const item = items.find((x) => x.path === h.path && x.sentence === h.sentence);
+    const why = h.rule === NO_QUESTION ? "must end with one question the author can answer" : `${h.rule}: "${h.match}"`;
+    if (item) item.flagged.push(why);
+    else items.push({ n: items.length + 1, path: h.path, sentence: h.sentence, flagged: [why] });
+  }
+  try {
+    const reply = scrubUrls(
+      await repair.backend.complete<{ rewrites: { n: number; text: string }[] }>({
+        label: "tone-repair",
+        model: repair.model,
+        system: repair.system,
+        prompt: `# Flagged sentences\n\n${JSON.stringify(items.map(({ n, sentence, flagged }) => ({ n, sentence, flagged })), null, 1)}\n\n# Task\n\nRewrite each flagged sentence. One rewrite per number. JSON only.`,
+        schema: TONE_REPAIR_SCHEMA,
+      }),
+      repair.allowed,
+      "tone-repair",
+      repair.report,
+    );
+    for (const rw of reply.rewrites) {
+      const item = items.find((x) => x.n === rw.n);
+      const f = item && fields.get(item.path);
+      const text = rw.text.replace(/\s+/g, " ").trim();
+      if (!item || !f || !text || !f.get().includes(item.sentence)) continue;
+      f.set(f.get().replace(item.sentence, () => text));
+      result.sentences_rewritten++;
+    }
+    result.repair_call = "done";
+  } catch (e) {
+    repair.log?.(`tone repair failed: ${(e as Error).message.split("\n")[0]}`);
+    result.repair_call = "failed";
+  }
+  result.remaining = lint().hits;
+  return result;
+}

package/src/thread/fetch.ts

@@ -0,0 +1,172 @@
+// Polite, read-only HTTP GET with an on-disk cache.
+//
+// Rules (see docs/design.md):
+//  - GET only. No cookies, no credentials, no POST: this module cannot post anything anywhere.
+//  - User-Agent "pgroast/0.1".
+//  - At least MIN_INTERVAL_MS between requests to one host.
+//  - A cached URL is never fetched again (404s are cached as well).
+//  - 429 / 5xx / network errors: retry with exponential backoff, honouring Retry-After.
+
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+
+export const USER_AGENT = "pgroast/0.1";
+export const MIN_INTERVAL_MS = 1100;
+export const REPO_ROOT = resolve(import.meta.dir, "..", "..");
+/** True when running from an installed package (npx, bunx, global install), not from a clone. */
+export const INSTALLED = /[\\/]node_modules[\\/]/.test(REPO_ROOT);
+/**
+ * Where fetched pages are cached: PGROAST_CACHE_DIR; else data/cache in a clone; else the user's
+ * cache directory, because a package directory is not a place to keep data.
+ */
+export const CACHE_ROOT = process.env.PGROAST_CACHE_DIR
+  ? resolve(process.env.PGROAST_CACHE_DIR)
+  : INSTALLED
+    ? join(process.env.XDG_CACHE_HOME || join(homedir(), ".cache"), "pgroast")
+    : join(REPO_ROOT, "data", "cache");
+export const DEFAULT_CACHE_DIR = join(CACHE_ROOT, "threads");
+/** Where results go by default: out/ in a clone, ./pgroast-out in the current directory when installed. */
+export const DEFAULT_OUT_ROOT = INSTALLED ? resolve("pgroast-out") : join(REPO_ROOT, "out");
+
+/** Hosts this tool is allowed to read. Anything else is refused. */
+export const ALLOWED_HOSTS = new Set(["www.postgresql.org", "postgr.es", "commitfest.postgresql.org"]);
+
+export interface FetchOptions {
+  cacheDir?: string;
+  /** Never touch the network; a cache miss returns null. */
+  offline?: boolean;
+  maxTries?: number;
+  /** Refuse bodies larger than this (bytes). The body is not cached when refused. */
+  maxBytes?: number;
+  /** Injected for tests. */
+  fetchImpl?: typeof fetch;
+  sleepImpl?: (ms: number) => Promise<void>;
+  log?: (msg: string) => void;
+}
+
+export interface FetchStats {
+  fetched: number;
+  cached: number;
+  notFound: number;
+  failed: number;
+}
+
+export const stats: FetchStats = { fetched: 0, cached: 0, notFound: 0, failed: 0 };
+
+const lastRequestAt = new Map<string, number>();
+const hostQueue = new Map<string, Promise<unknown>>();
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+/** Map a URL to a stable file path under <cacheDir>/<host>/. */
+export function cachePath(url: string, cacheDir: string = DEFAULT_CACHE_DIR): string {
+  const u = new URL(url);
+  let name = (u.pathname.replace(/^\/+|\/+$/g, "") || "_root").replace(/[^A-Za-z0-9._-]+/g, "_");
+  if (u.search) name += "__" + u.search.slice(1).replace(/[^A-Za-z0-9._=-]+/g, "_");
+  if (name.length > 150) {
+    name = name.slice(0, 100) + "_" + createHash("sha1").update(name).digest("hex").slice(0, 16);
+  }
+  return join(cacheDir, u.host, name);
+}
+
+function parseRetryAfter(v: string | null): number | null {
+  if (!v) return null;
+  if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000;
+  const t = Date.parse(v);
+  return Number.isNaN(t) ? null : Math.max(0, t - Date.now());
+}
+
+/**
+ * GET a URL as text. Returns null for 404, for an offline cache miss, for an oversized body,
+ * and after all retries failed.
+ */
+export async function fetchText(url: string, opts: FetchOptions = {}): Promise<string | null> {
+  const u = new URL(url);
+  if (u.protocol !== "https:" || !ALLOWED_HOSTS.has(u.host)) {
+    throw new Error(`refusing to fetch ${url}: host not in the read-only allow list`);
+  }
+  const p = cachePath(url, opts.cacheDir);
+  if (existsSync(p)) {
+    stats.cached++;
+    return readFileSync(p, "utf8");
+  }
+  if (existsSync(p + ".404")) {
+    stats.notFound++;
+    return null;
+  }
+  if (opts.offline) return null;
+
+  // Serialise requests per host so the minimum interval holds under concurrency.
+  const prev = hostQueue.get(u.host) ?? Promise.resolve();
+  const run = prev.then(() => fetchUncached(url, p, opts));
+  hostQueue.set(
+    u.host,
+    run.catch(() => undefined),
+  );
+  return run;
+}
+
+async function fetchUncached(url: string, p: string, opts: FetchOptions): Promise<string | null> {
+  const host = new URL(url).host;
+  const doFetch = opts.fetchImpl ?? fetch;
+  const doSleep = opts.sleepImpl ?? sleep;
+  const log = opts.log ?? ((m: string) => console.error(m));
+  const maxTries = opts.maxTries ?? 5;
+  let delay = 5000;
+
+  for (let attempt = 0; attempt < maxTries; attempt++) {
+    const wait = (lastRequestAt.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
+    if (wait > 0) await doSleep(wait);
+    lastRequestAt.set(host, Date.now());
+    try {
+      const res = await doFetch(url, {
+        method: "GET",
+        headers: { "User-Agent": USER_AGENT, Accept: "text/html,text/plain,*/*" },
+        redirect: "follow",
+        credentials: "omit",
+        signal: AbortSignal.timeout(60_000),
+      });
+      if (res.status === 404 || res.status === 410) {
+        mkdirSync(dirname(p), { recursive: true });
+        writeFileSync(p + ".404", "");
+        stats.notFound++;
+        return null;
+      }
+      if (res.status === 429 || res.status >= 500) {
+        const ra = parseRetryAfter(res.headers.get("retry-after"));
+        const pause = Math.min(ra ?? delay, 300_000);
+        log(`  HTTP ${res.status} on ${url}; backing off ${Math.round(pause / 1000)}s`);
+        await doSleep(pause);
+        delay = Math.min(delay * 2, 300_000);
+        continue;
+      }
+      if (!res.ok) {
+        log(`  HTTP ${res.status} on ${url}; giving up`);
+        break;
+      }
+      const len = Number(res.headers.get("content-length") ?? "0");
+      if (opts.maxBytes && len > opts.maxBytes) {
+        log(`  ${url}: ${len} bytes, over the ${opts.maxBytes} limit; skipped`);
+        return null;
+      }
+      const body = await res.text();
+      if (opts.maxBytes && body.length > opts.maxBytes) {
+        log(`  ${url}: ${body.length} bytes, over the ${opts.maxBytes} limit; skipped`);
+        return null;
+      }
+      mkdirSync(dirname(p), { recursive: true });
+      writeFileSync(p + ".tmp", body);
+      renameSync(p + ".tmp", p);
+      stats.fetched++;
+      return body;
+    } catch (e) {
+      log(`  network error on ${url}: ${(e as Error).message}; backing off ${Math.round(delay / 1000)}s`);
+      await doSleep(delay);
+      delay = Math.min(delay * 2, 300_000);
+    }
+  }
+  stats.failed++;
+  return null;
+}

package/src/thread/load.ts

@@ -0,0 +1,227 @@
+// Resolve a CLI argument to a CommitFest entry and/or a mailing-list thread, fetch it
+// (cached, polite), apply --blind / --before, and collect the prose of the proposal's patches.
+
+import { fetchText } from "./fetch.ts";
+import { flatThreadUrl, parseCommitfestPatch, parseFlatThread } from "./parse.ts";
+import { extractPatchProse, isReadableAttachment, MAX_DOC_HUNK_CHARS } from "./patchprose.ts";
+import type {
+  CfEntry,
+  InputRef,
+  LoadOptions,
+  PatchProse,
+  ProposalInput,
+  Thread,
+  ThreadMessage,
+} from "./types.ts";
+
+/** Attachments larger than this are not downloaded. */
+export const MAX_ATTACHMENT_BYTES = 1536 * 1024;
+/** At most this many attachments are read per run. */
+export const MAX_ATTACHMENTS = 8;
+
+export function parseInputRef(arg: string): InputRef {
+  const s = arg.trim();
+  if (/^\d+$/.test(s)) return { kind: "cf", id: Number(s) };
+
+  const cf = s.match(/^https?:\/\/commitfest\.postgresql\.org\/(?:patch\/(\d+)|\d+\/(\d+))\/?/i);
+  if (cf) return { kind: "cf", id: Number(cf[1] ?? cf[2]) };
+
+  const arch = s.match(
+    /^https?:\/\/(?:www\.)?postgresql\.org\/message-id\/(?:flat\/|raw\/|mbox\/)?([^?#]+?)\/?(?:[?#].*)?$/i,
+  );
+  const short = s.match(/^https?:\/\/postgr\.es\/m\/([^?#]+?)\/?(?:[?#].*)?$/i);
+  const raw = arch?.[1] ?? short?.[1];
+  if (raw) {
+    let id = raw;
+    try {
+      id = decodeURIComponent(raw);
+    } catch {
+      // keep as is
+    }
+    return { kind: "msgid", messageId: id };
+  }
+  if (/^<?[^\s<>@]+@[^\s<>@]+>?$/.test(s)) return { kind: "msgid", messageId: s.replace(/^<|>$/g, "") };
+  throw new Error(
+    `cannot understand input "${arg}": expected a CommitFest patch id or URL, a message-id, or a postgresql.org thread URL`,
+  );
+}
+
+export function validateDate(d: string): string {
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(d) || Number.isNaN(Date.parse(d + "T00:00:00Z"))) {
+    throw new Error(`bad date "${d}": expected YYYY-MM-DD`);
+  }
+  return d;
+}
+
+/** Apply --before (strictly before the date, UTC) and --blind (first N). Both may apply. */
+export function filterMessages(
+  messages: ThreadMessage[],
+  opts: { blind?: number | null; before?: string | null },
+): ThreadMessage[] {
+  let out = messages;
+  if (opts.before) {
+    const cutoff = validateDate(opts.before) + "T00:00:00Z";
+    out = out.filter((m) => m.date !== "" && m.date < cutoff);
+  }
+  if (opts.blind != null) {
+    if (!Number.isInteger(opts.blind) || opts.blind < 1) throw new Error("--blind needs a positive integer");
+    out = out.slice(0, opts.blind);
+  }
+  return out;
+}
+
+/**
+ * Remove what would reveal the outcome. With --blind the whole status/history/reviewer
+ * part goes; with only --before, history entries before the cutoff are kept.
+ */
+export function redactCf(cf: CfEntry, opts: { blind?: number | null; before?: string | null }): CfEntry {
+  if (opts.blind == null && !opts.before) return cf;
+  const cutoff = opts.before ? opts.before + "T00:00:00Z" : null;
+  return {
+    ...cf,
+    status: null,
+    statuses: [],
+    reviewers: "",
+    committer: "",
+    targetVersion: "",
+    links: [],
+    history:
+      opts.blind == null && cutoff
+        ? cf.history.filter((h) => h.when < cutoff && !/^Closed in commitfest/i.test(h.what))
+        : [],
+    threads: cf.threads.map((t) => ({ ...t, latestAt: null })),
+    redacted: true,
+  };
+}
+
+/** Messages whose attachments describe the proposal: the first with a patch, and the author's latest. */
+export function pickProposalMessages(messages: ThreadMessage[]): ThreadMessage[] {
+  const withPatch = messages.filter((m) => m.attachments.some((a) => isReadableAttachment(a.name, a.contentType)));
+  if (withPatch.length === 0) return [];
+  const first = withPatch[0]!;
+  const author = messages[0]?.from ?? first.from;
+  const latestByAuthor = [...withPatch].reverse().find((m) => m.from === author) ?? first;
+  return latestByAuthor === first ? [first] : [latestByAuthor, first];
+}
+
+async function loadPatchProse(
+  messages: ThreadMessage[],
+  opts: LoadOptions,
+  warnings: string[],
+): Promise<PatchProse[]> {
+  const out: PatchProse[] = [];
+  let budget = MAX_ATTACHMENTS;
+  for (const msg of pickProposalMessages(messages)) {
+    for (const att of msg.attachments) {
+      if (!isReadableAttachment(att.name, att.contentType)) continue;
+      if (budget <= 0) {
+        warnings.push(`attachment limit (${MAX_ATTACHMENTS}) reached; ${att.name} not read`);
+        continue;
+      }
+      if (att.sizeBytes != null && att.sizeBytes > MAX_ATTACHMENT_BYTES) {
+        warnings.push(`attachment ${att.name} (${att.sizeText}) is too large; not read`);
+        continue;
+      }
+      budget--;
+      const text = await fetchText(att.url, {
+        offline: opts.offline,
+        cacheDir: opts.cacheDir,
+        maxBytes: MAX_ATTACHMENT_BYTES,
+      });
+      if (text == null) {
+        warnings.push(`attachment ${att.name} could not be read`);
+        continue;
+      }
+      if (text.includes("\u0000")) {
+        warnings.push(`attachment ${att.name} looks binary; skipped`);
+        continue;
+      }
+      const prose = extractPatchProse(text);
+      if (prose.files.length === 0 && prose.commitMessages.length === 0) {
+        // Not a diff. Keep it only when it is plainly a prose file (README, notes).
+        if (!/README|\.(md|txt)$/i.test(att.name)) continue;
+        const cut = text.length > MAX_DOC_HUNK_CHARS;
+        out.push({
+          attachment: att,
+          fromMessageIndex: msg.index,
+          commitMessages: [],
+          files: [],
+          docHunks: [{ path: att.name, text: cut ? text.slice(0, MAX_DOC_HUNK_CHARS) + "\n[... truncated ...]" : text }],
+          truncated: cut,
+        });
+        continue;
+      }
+      out.push({ attachment: att, fromMessageIndex: msg.index, ...prose });
+    }
+  }
+  return out;
+}
+
+export async function loadCfEntry(id: number, opts: LoadOptions = {}): Promise<CfEntry> {
+  const url = `https://commitfest.postgresql.org/patch/${id}/`;
+  const html = await fetchText(url, { offline: opts.offline, cacheDir: opts.cacheDir });
+  if (html == null) throw new Error(`CommitFest patch ${id} not found (${url})`);
+  return parseCommitfestPatch(html, id);
+}
+
+export async function loadThread(messageId: string, opts: LoadOptions = {}): Promise<Thread> {
+  const url = flatThreadUrl(messageId);
+  const html = await fetchText(url, { offline: opts.offline, cacheDir: opts.cacheDir });
+  if (html == null) throw new Error(`thread not found: ${url}`);
+  return parseFlatThread(html);
+}
+
+export async function loadProposal(arg: string, opts: LoadOptions = {}): Promise<ProposalInput> {
+  const ref = parseInputRef(arg);
+  const warnings: string[] = [];
+  const filters = { blind: opts.blind ?? null, before: opts.before ? validateDate(opts.before) : null };
+
+  let cf: CfEntry | null = null;
+  let messageId: string;
+  if (ref.kind === "cf") {
+    const full = await loadCfEntry(ref.id, opts);
+    if (full.threads.length === 0) throw new Error(`CommitFest patch ${ref.id} has no mailing-list thread attached`);
+    // The proposal thread is the one that started first.
+    const sorted = [...full.threads].sort((a, b) => (a.firstAt ?? "9999").localeCompare(b.firstAt ?? "9999"));
+    messageId = sorted[0]!.messageId;
+    if (sorted.length > 1) {
+      warnings.push(
+        `CommitFest entry has ${sorted.length} threads; only the earliest is read (${sorted[0]!.url}). Others: ` +
+          sorted
+            .slice(1)
+            .map((t) => t.url)
+            .join(", "),
+      );
+    }
+    cf = redactCf(full, filters);
+  } else if (ref.kind === "msgid") {
+    messageId = ref.messageId;
+  } else {
+    throw new Error("a local draft is loaded with loadLocalProposal(), not from the archives");
+  }
+
+  const full = await loadThread(messageId, opts);
+  const messages = filterMessages(full.messages, filters);
+  if (messages.length === 0) {
+    throw new Error(
+      `no messages left after filtering (thread has ${full.messages.length}, first on ${full.messages[0]!.date}; --before ${filters.before ?? "-"}, --blind ${filters.blind ?? "-"})`,
+    );
+  }
+  if (ref.kind === "msgid" && full.rootMessageId !== ref.messageId) {
+    warnings.push(`the given message is not the thread root; reading the whole thread from ${full.threadUrl}`);
+  }
+  const thread: Thread = { ...full, messages };
+  const patchProse = opts.noAttachments ? [] : await loadPatchProse(messages, opts, warnings);
+
+  return {
+    input: arg,
+    ref,
+    cf,
+    thread,
+    totalMessages: full.messages.length,
+    filters,
+    threadStart: full.messages[0]!.date,
+    patchProse,
+    warnings,
+  };
+}