pgroast 0.0.0-stage → 0.1.0

package/src/thread/local.ts

@@ -0,0 +1,73 @@
+// Self mode without a thread: the author's draft email and optional patch files on disk.
+// The same rule as for attachments applies: only the prose of a patch reaches the model.
+
+import { readFileSync, statSync } from "node:fs";
+import { basename } from "node:path";
+import { extractPatchProse } from "./patchprose.ts";
+import type { PatchProse, ProposalInput } from "./types.ts";
+
+export const MAX_LOCAL_PATCH_BYTES = 8 * 1024 * 1024;
+
+export function loadLocalProposal(proposalFile: string, patchFiles: string[] = [], now: Date = new Date()): ProposalInput {
+  const text = readFileSync(proposalFile, "utf8").replace(/\r\n/g, "\n");
+  if (!text.trim()) throw new Error(`${proposalFile} is empty`);
+  const warnings: string[] = [];
+
+  // A draft may start with mail headers; "Subject:" gives the title, else the first line does.
+  const lines = text.split("\n");
+  const subj = lines.slice(0, 15).find((l) => /^Subject:/i.test(l));
+  const subject = (subj ? subj.replace(/^Subject:\s*/i, "") : (lines.find((l) => l.trim()) ?? "")).trim().slice(0, 200);
+  const name = basename(proposalFile);
+  const date = now.toISOString().replace(/\.\d+Z$/, "Z");
+
+  const patchProse: PatchProse[] = [];
+  for (const f of patchFiles) {
+    const size = statSync(f).size;
+    const att = { name: basename(f), url: "", contentType: "text/x-patch", sizeText: `${(size / 1024).toFixed(1)} KB`, sizeBytes: size };
+    if (size > MAX_LOCAL_PATCH_BYTES) {
+      warnings.push(`patch ${att.name} (${att.sizeText}) is too large; not read`);
+      continue;
+    }
+    const raw = readFileSync(f, "utf8");
+    if (raw.includes("\u0000")) {
+      warnings.push(`patch ${att.name} looks binary; skipped`);
+      continue;
+    }
+    const prose = extractPatchProse(raw);
+    if (prose.files.length === 0 && prose.commitMessages.length === 0) {
+      // Not a diff: it could be source code, and code never goes to the model.
+      warnings.push(`${att.name} is not a patch (no diff found); not read`);
+      continue;
+    }
+    patchProse.push({ attachment: att, fromMessageIndex: 1, ...prose });
+  }
+
+  return {
+    input: name,
+    ref: { kind: "local", name },
+    cf: null,
+    thread: {
+      threadUrl: "",
+      rootMessageId: "",
+      subject,
+      lists: ["(draft, not sent)"],
+      messages: [
+        {
+          index: 1,
+          messageId: "",
+          url: "",
+          from: "The author (draft, not sent yet)",
+          date,
+          subject,
+          body: text.trim(),
+          attachments: patchProse.map((p) => p.attachment),
+        },
+      ],
+    },
+    totalMessages: 1,
+    filters: { blind: null, before: null },
+    threadStart: date,
+    patchProse,
+    warnings,
+  };
+}

package/src/thread/parse.ts

@@ -0,0 +1,243 @@
+// HTML parsers for postgresql.org flat thread pages and CommitFest patch pages.
+// Regex-based on purpose: no dependencies, and the two page layouts are stable and simple.
+
+import type { Attachment, CfEntry, CfHistoryItem, CfThreadRef, Thread, ThreadMessage } from "./types.ts";
+
+const ARCHIVE = "https://www.postgresql.org";
+
+const NAMED_ENTITIES: Record<string, string> = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+  ndash: "–",
+  mdash: "—",
+  hellip: "…",
+  copy: "©",
+};
+
+export function decodeEntities(s: string): string {
+  return s.replace(/&(#x[0-9a-fA-F]+|#\d+|[a-zA-Z]+);/g, (m, e: string) => {
+    if (e[0] === "#") {
+      const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : m;
+    }
+    return NAMED_ENTITIES[e] ?? m;
+  });
+}
+
+/** Strip tags and collapse whitespace: for single-line fields. */
+export function stripTags(s: string): string {
+  return decodeEntities(s.replace(/<[^>]*>/g, " "))
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/** Convert a message body to plain text, keeping line structure. */
+export function htmlToText(html: string): string {
+  const s = html
+    .replace(/\r/g, "")
+    .replace(/\n/g, " ")
+    .replace(/<br\s*\/?>/gi, "\n")
+    .replace(/<\/p>\s*<p[^>]*>/gi, "\n\n")
+    .replace(/<\/?(p|div|pre|blockquote)[^>]*>/gi, "\n")
+    .replace(/<[^>]*>/g, "");
+  return decodeEntities(s)
+    .split("\n")
+    .map((l) => l.replace(/[ \t]+$/g, "").replace(/^ (?=\S)/, ""))
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/** "2024-09-12 11:08:15" -> "2024-09-12T11:08:15Z"; null when not a timestamp. */
+export function normTimestamp(s: string): string | null {
+  const m = s.match(/(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}(?::\d{2})?)/);
+  if (!m) return null;
+  const time = m[2]!.length === 5 ? m[2] + ":00" : m[2];
+  return `${m[1]}T${time}Z`;
+}
+
+export function parseSize(s: string): number | null {
+  const m = s.trim().match(/^([\d.]+)\s*(bytes|B|KB|kB|MB|GB)$/i);
+  if (!m) return null;
+  const unit = m[2]!.toLowerCase();
+  const mult = unit === "kb" ? 1024 : unit === "mb" ? 1024 ** 2 : unit === "gb" ? 1024 ** 3 : 1;
+  return Math.round(Number(m[1]) * mult);
+}
+
+export function messageUrl(messageId: string): string {
+  return `${ARCHIVE}/message-id/${encodeURIComponent(messageId)}`;
+}
+
+export function flatThreadUrl(messageId: string): string {
+  return `${ARCHIVE}/message-id/flat/${encodeURIComponent(messageId)}`;
+}
+
+function headerField(block: string, name: string): string {
+  const re = new RegExp(`<th[^>]*>\\s*${name}:\\s*</th>\\s*<td[^>]*>([\\s\\S]*?)</td>`, "i");
+  const m = block.match(re);
+  return m ? stripTags(m[1]!) : "";
+}
+
+function parseAttachments(block: string): Attachment[] {
+  const out: Attachment[] = [];
+  const table = block.match(/<table[^>]*message-attachments[^>]*>([\s\S]*?)<\/table>/i);
+  if (!table) return out;
+  const rowRe =
+    /<tr>\s*<th[^>]*>\s*<a href="([^"]+)"[^>]*>([\s\S]*?)<\/a>\s*<\/th>\s*<td[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/gi;
+  for (const m of table[1]!.matchAll(rowRe)) {
+    const href = decodeEntities(m[1]!);
+    const sizeText = stripTags(m[4]!);
+    out.push({
+      name: stripTags(m[2]!),
+      url: href.startsWith("http") ? href : ARCHIVE + href,
+      contentType: stripTags(m[3]!),
+      sizeText,
+      sizeBytes: parseSize(sizeText),
+    });
+  }
+  return out;
+}
+
+/**
+ * Parse https://www.postgresql.org/message-id/flat/<msgid>.
+ * Throws when the page holds no message (e.g. an error page).
+ */
+export function parseFlatThread(html: string): Thread {
+  const subjM = html.match(/<h1[^>]*id="flatMsgSubject"[^>]*>([\s\S]*?)<\/h1>/i);
+  const lists = [
+    ...new Set([...html.matchAll(/<span class="listname"><a[^>]*>([^<]+)<\/a>/g)].map((m) => m[1]!.trim())),
+  ];
+
+  // Each message starts with a header table that holds a "From:" row. The page-level
+  // header table (only "Lists:") is skipped by that test.
+  const parts = html.split(/<table[^>]*class="[^"]*message-header[^"]*"[^>]*>/i).slice(1);
+  const messages: ThreadMessage[] = [];
+  for (const part of parts) {
+    const headerEnd = part.indexOf("</table>");
+    if (headerEnd < 0) continue;
+    const header = part.slice(0, headerEnd);
+    if (!/<th[^>]*>\s*From:/i.test(header)) continue;
+    const rest = part.slice(headerEnd);
+
+    const idM = header.match(/<th[^>]*>\s*Message-ID:\s*<\/th>\s*<td[^>]*>\s*<a[^>]*>([\s\S]*?)<\/a>/i);
+    const messageId = idM ? stripTags(idM[1]!) : "";
+    if (!messageId) continue;
+    const bodyM = rest.match(/<div class="message-content">([\s\S]*?)<\/div>/i);
+    const date = normTimestamp(headerField(header, "Date"));
+    messages.push({
+      index: messages.length + 1,
+      messageId,
+      url: messageUrl(messageId),
+      from: headerField(header, "From"),
+      date: date ?? "",
+      subject: headerField(header, "Subject"),
+      body: bodyM ? htmlToText(bodyM[1]!) : "",
+      attachments: parseAttachments(rest),
+    });
+  }
+  if (messages.length === 0) throw new Error("no messages found: not a flat thread page");
+  const root = messages[0]!;
+  return {
+    threadUrl: flatThreadUrl(root.messageId),
+    rootMessageId: root.messageId,
+    subject: subjM ? stripTags(subjM[1]!) : root.subject,
+    lists,
+    messages,
+  };
+}
+
+function cfFields(html: string): Map<string, string> {
+  const out = new Map<string, string>();
+  const tableM = html.match(/<table class="table table-bordered">([\s\S]*)/i);
+  if (!tableM) return out;
+  // Top-level rows only: "History" holds a nested table, so cut it off first.
+  const histAt = tableM[1]!.search(/<th>\s*History\s*<\/th>/i);
+  const main = histAt >= 0 ? tableM[1]!.slice(0, histAt) : tableM[1]!;
+  for (const m of main.matchAll(/<th[^>]*>([\s\S]*?)<\/th>\s*<td[^>]*>([\s\S]*?)(?=<th[^>]*>|$)/gi)) {
+    out.set(stripTags(m[1]!), m[2]!);
+  }
+  // "Emails" is the last row before "History" and can hold a nested table (annotations) with
+  // its own <th> cells: take the whole rest of the main table for it.
+  const emailsAt = main.search(/<th>\s*Emails\s*<\/th>/i);
+  if (emailsAt >= 0) out.set("Emails", main.slice(emailsAt).replace(/^<th>\s*Emails\s*<\/th>\s*<td[^>]*>/i, ""));
+  return out;
+}
+
+/** Parse https://commitfest.postgresql.org/patch/<id>/. */
+export function parseCommitfestPatch(html: string, id: number): CfEntry {
+  const f = cfFields(html);
+  const get = (k: string) => f.get(k) ?? "";
+  const title = stripTags(get("Title"));
+  if (!title) throw new Error(`CommitFest patch ${id}: no title found (not a patch page?)`);
+
+  const statuses = [
+    ...get("Status").matchAll(
+      /<a href="\/(\d+)\/">([^<]+)<\/a>[^<]*<span class="badge[^"]*">([^<]+)<\/span>/g,
+    ),
+  ].map((m) => ({ cfId: Number(m[1]), cfName: m[2]!.trim(), status: m[3]!.trim() }));
+  const latest = [...statuses].sort((a, b) => b.cfId - a.cfId)[0];
+
+  const tags = [...get("Tags").matchAll(/<span class="badge"[^>]*>([^<]+)<\/span>/g)].map((m) =>
+    decodeEntities(m[1]!).trim(),
+  );
+
+  const threads: CfThreadRef[] = [];
+  for (const m of get("Emails").matchAll(/<dt>([\s\S]*?)<\/dt>\s*<dd>([\s\S]*?)<\/dd>/g)) {
+    const a = m[1]!.match(/<a href="https:\/\/www\.postgresql\.org\/message-id\/flat\/([^"]+)">([\s\S]*?)<\/a>/);
+    if (!a) continue;
+    let messageId = decodeEntities(a[1]!);
+    try {
+      messageId = decodeURIComponent(messageId);
+    } catch {
+      // keep as is
+    }
+    const first = m[2]!.match(/First at <a[^>]*>([^<]+)<\/a> by ([\s\S]*?)<br/);
+    const last = m[2]!.match(/Latest at <a[^>]*>([^<]+)<\/a>/);
+    threads.push({
+      messageId,
+      url: flatThreadUrl(messageId),
+      subject: stripTags(a[2]!),
+      firstAt: first ? normTimestamp(first[1]!) : null,
+      firstBy: first ? stripTags(first[2]!).replace(/\s*<.*$/, "") : "",
+      latestAt: last ? normTimestamp(last[1]!) : null,
+    });
+  }
+
+  const history: CfHistoryItem[] = [];
+  const histM = html.match(/<th>\s*History\s*<\/th>([\s\S]*?)<\/table>/i);
+  if (histM) {
+    for (const r of histM[1]!.matchAll(
+      /<tr>\s*<td[^>]*>([^<]+)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/g,
+    )) {
+      history.push({
+        when: normTimestamp(r[1]!) ?? r[1]!.trim(),
+        who: stripTags(r[2]!),
+        what: stripTags(r[3]!),
+      });
+    }
+  }
+
+  const reviewers = stripTags(get("Reviewers").replace(/<a class="btn[\s\S]*?<\/a>/g, ""));
+  return {
+    id,
+    url: `https://commitfest.postgresql.org/patch/${id}/`,
+    title,
+    status: latest?.status ?? null,
+    statuses,
+    authors: stripTags(get("Authors") || get("Author")),
+    reviewers,
+    committer: stripTags(get("Committer")),
+    tags,
+    topic: stripTags(get("Topic")),
+    targetVersion: stripTags(get("Target version")),
+    created: normTimestamp(stripTags(get("Created"))),
+    threads,
+    history,
+    links: [...get("Links").matchAll(/href="(https?:\/\/[^"]+)"/g)].map((m) => decodeEntities(m[1]!)),
+    redacted: false,
+  };
+}

package/src/thread/patchprose.ts

@@ -0,0 +1,143 @@
+// Extract the prose of a patch file: commit message(s), the list of touched paths, and
+// diff hunks for documentation files only. Code hunks are dropped here, in code, so no
+// later stage can feed them to the model: PG Roast judges the idea, not the lines.
+
+import type { CommitMessage } from "./types.ts";
+
+export interface ProseExtract {
+  commitMessages: CommitMessage[];
+  files: string[];
+  docHunks: { path: string; text: string }[];
+  truncated: boolean;
+}
+
+export const MAX_COMMIT_MESSAGE_CHARS = 6000;
+export const MAX_DOC_HUNK_CHARS = 6000;
+export const MAX_DOC_TOTAL_CHARS = 16000;
+export const MAX_FILES = 80;
+
+/** Paths whose diff hunks are prose, not code. */
+export function isDocPath(path: string): boolean {
+  return (
+    /^doc\//.test(path) ||
+    /(^|\/)README[^/]*$/i.test(path) ||
+    /\.(md|sgml|rst|txt)$/i.test(path) && !/(^|\/)(expected|output|sql|data)\//.test(path)
+  );
+}
+
+function clip(s: string, max: number): { text: string; cut: boolean } {
+  if (s.length <= max) return { text: s, cut: false };
+  return { text: s.slice(0, max) + "\n[... truncated ...]", cut: true };
+}
+
+/** RFC 2047 "=?UTF-8?q?...?=" subjects, as git format-patch emits for non-ASCII. */
+function decodeMimeWords(s: string): string {
+  return s.replace(/=\?([^?]+)\?([bBqQ])\?([^?]*)\?=\s*/g, (m, _cs: string, enc: string, data: string) => {
+    try {
+      if (enc.toLowerCase() === "b") return Buffer.from(data, "base64").toString("utf8");
+      const bytes = data
+        .replace(/_/g, " ")
+        .replace(/=([0-9A-Fa-f]{2})/g, (_x, h: string) => String.fromCharCode(parseInt(h, 16)));
+      return Buffer.from(bytes, "latin1").toString("utf8");
+    } catch {
+      return m;
+    }
+  });
+}
+
+export function extractPatchProse(raw: string): ProseExtract {
+  const text = raw.replace(/\r\n/g, "\n");
+  let truncated = false;
+
+  // Split into per-file diffs. Everything before the first "diff --git" of a commit is header.
+  const lines = text.split("\n");
+  const commitMessages: CommitMessage[] = [];
+  const files: string[] = [];
+  const docHunks: { path: string; text: string }[] = [];
+  let docTotal = 0;
+
+  let header: string[] = [];
+  let cur: { path: string; lines: string[] } | null = null;
+
+  const flushFile = () => {
+    if (!cur) return;
+    if (!files.includes(cur.path)) files.push(cur.path);
+    if (isDocPath(cur.path) && docTotal < MAX_DOC_TOTAL_CHARS) {
+      const c = clip(cur.lines.join("\n"), Math.min(MAX_DOC_HUNK_CHARS, MAX_DOC_TOTAL_CHARS - docTotal));
+      truncated ||= c.cut;
+      docTotal += c.text.length;
+      docHunks.push({ path: cur.path, text: c.text });
+    } else if (isDocPath(cur.path)) {
+      truncated = true;
+    }
+    cur = null;
+  };
+
+  const flushHeader = () => {
+    const cm = parseCommitHeader(header);
+    if (cm) {
+      const c = clip(cm.body, MAX_COMMIT_MESSAGE_CHARS);
+      truncated ||= c.cut;
+      commitMessages.push({ subject: cm.subject, body: c.text });
+    }
+    header = [];
+  };
+
+  for (const line of lines) {
+    const d = line.match(/^diff --git a\/(\S+) b\/(\S+)/);
+    if (d) {
+      if (cur) flushFile();
+      else flushHeader();
+      cur = { path: d[2]!, lines: [line] };
+      continue;
+    }
+    // A new mail header inside a multi-commit file ends the previous commit's last file.
+    if (/^From [0-9a-f]{40} /.test(line)) {
+      if (cur) flushFile();
+      else flushHeader();
+      header = [line];
+      continue;
+    }
+    if (cur) cur.lines.push(line);
+    else header.push(line);
+  }
+  if (cur) flushFile();
+  else flushHeader();
+
+  if (files.length > MAX_FILES) {
+    truncated = true;
+    files.length = MAX_FILES;
+  }
+  return { commitMessages, files, docHunks, truncated };
+}
+
+/** Parse a git format-patch mail header block: Subject (with continuation lines) + body. */
+function parseCommitHeader(header: string[]): CommitMessage | null {
+  const si = header.findIndex((l) => /^Subject:/i.test(l));
+  if (si < 0) return null;
+  let subject = header[si]!.replace(/^Subject:\s*/i, "");
+  let i = si + 1;
+  while (i < header.length && /^[ \t]+\S/.test(header[i]!)) {
+    subject += " " + header[i]!.trim();
+    i++;
+  }
+  subject = decodeMimeWords(subject)
+    .replace(/^\[PATCH[^\]]*\]\s*/i, "")
+    .trim();
+  // Skip remaining mail headers up to the blank line.
+  while (i < header.length && header[i]!.trim() !== "") i++;
+  const bodyLines: string[] = [];
+  for (i = i + 1; i < header.length; i++) {
+    const l = header[i]!;
+    if (l === "---") break; // start of the diffstat
+    bodyLines.push(l);
+  }
+  return { subject, body: bodyLines.join("\n").trim() };
+}
+
+/** True when an attachment name looks like a text patch or a prose file worth reading. */
+export function isReadableAttachment(name: string, contentType: string): boolean {
+  if (/\.(gz|bz2|xz|zst|zip|tgz|tar|png|jpe?g|gif|pdf|svg|odt|docx?|xlsx?|bin)$/i.test(name)) return false;
+  if (/\.(patch|diff|txt|md|nocfbot)$/i.test(name) || /README/i.test(name)) return true;
+  return /^text\//i.test(contentType);
+}

package/src/thread/types.ts

@@ -0,0 +1,120 @@
+// Types for the thread / CommitFest input layer.
+
+export interface Attachment {
+  name: string;
+  url: string;
+  contentType: string;
+  /** Size as shown by the archive, e.g. "33.5 KB". */
+  sizeText: string;
+  /** Approximate size in bytes parsed from sizeText, or null. */
+  sizeBytes: number | null;
+}
+
+export interface ThreadMessage {
+  /** 1-based position in the flat thread (archive order = by date). */
+  index: number;
+  messageId: string;
+  /** Permalink of this message in the archive. */
+  url: string;
+  from: string;
+  /** ISO 8601, UTC (the archive shows UTC). */
+  date: string;
+  subject: string;
+  body: string;
+  attachments: Attachment[];
+}
+
+export interface Thread {
+  /** https://www.postgresql.org/message-id/flat/<root message-id> */
+  threadUrl: string;
+  rootMessageId: string;
+  subject: string;
+  lists: string[];
+  messages: ThreadMessage[];
+}
+
+export interface CfThreadRef {
+  messageId: string;
+  url: string;
+  subject: string;
+  firstAt: string | null;
+  firstBy: string;
+  latestAt: string | null;
+}
+
+export interface CfHistoryItem {
+  when: string;
+  who: string;
+  what: string;
+}
+
+export interface CfEntry {
+  id: number;
+  url: string;
+  title: string;
+  /** Status in the most recent CommitFest, e.g. "Needs review". Null when redacted. */
+  status: string | null;
+  statuses: { cfId: number; cfName: string; status: string }[];
+  authors: string;
+  reviewers: string;
+  committer: string;
+  tags: string[];
+  topic: string;
+  targetVersion: string;
+  created: string | null;
+  threads: CfThreadRef[];
+  history: CfHistoryItem[];
+  links: string[];
+  /** True when outcome-bearing fields were removed for --blind / --before. */
+  redacted: boolean;
+}
+
+export interface CommitMessage {
+  subject: string;
+  body: string;
+}
+
+/** The prose parts of one patch attachment. Never contains code hunks. */
+export interface PatchProse {
+  attachment: Attachment;
+  fromMessageIndex: number;
+  commitMessages: CommitMessage[];
+  /** Paths touched (from the diff headers). Names only. */
+  files: string[];
+  /** Diff hunks for doc/, README* and *.md / *.sgml / *.rst files only. */
+  docHunks: { path: string; text: string }[];
+  truncated: boolean;
+}
+
+export interface LoadOptions {
+  /** Keep only the first N messages. */
+  blind?: number;
+  /** Keep only messages strictly before this date (YYYY-MM-DD, UTC). */
+  before?: string;
+  offline?: boolean;
+  cacheDir?: string;
+  /** Skip attachment download. */
+  noAttachments?: boolean;
+}
+
+export type InputRef =
+  | { kind: "cf"; id: number }
+  | { kind: "msgid"; messageId: string }
+  /** A draft on disk (self mode): no thread exists yet. */
+  | { kind: "local"; name: string };
+
+export interface ProposalInput {
+  /** The CLI argument as given. */
+  input: string;
+  ref: InputRef;
+  cf: CfEntry | null;
+  /** The thread after --blind / --before filtering. */
+  thread: Thread;
+  /** Number of messages in the thread before filtering. */
+  totalMessages: number;
+  filters: { blind: number | null; before: string | null };
+  /** ISO date-time of the first message of the thread. */
+  threadStart: string;
+  patchProse: PatchProse[];
+  warnings: string[];
+}