tumwater 0.2.0 → 0.3.0

package/dist/src/backlog/backlog-structure.js

@@ -1,17 +1,31 @@
-// vocabulary: plain
 import path from "node:path";
 import { increment } from "../collections.js";
 import { readTextOrNull } from "../files/files.js";
 import { cachedTextByStat } from "../files/stat-cache.js";
 import { BACKLOG_FILES, FINISHED_ARCHIVES, fenceTracker, headingMetadata, parseEntryDetails, sectionBodyLines, fenceAwareHeadingLines, headingTitle, } from "./backlog-md.js";
 import { changeBaseRev, fileContentAt } from "../git/git.js";
-import { allLiveClaims } from "../loop/loop-state.js";
+import { plural } from "../text/phrases.js";
 import { collapseWhitespace, truncate } from "../text/text.js";
-import { entryHoldAgainst, holdMark, plannedPartKeys, seriesPart, } from "./backlog-eligibility.js";
-import { plannedPlanEntries } from "./backlog.js";
+import { seriesPart } from "./backlog-parts.js";
 /** The archive files of PLANS.md and BUGS.md (FINISHED_ARCHIVES). */
 const PLANS_ARCHIVE = FINISHED_ARCHIVES[0].archive;
 const BUGS_ARCHIVE = FINISHED_ARCHIVES[1].archive;
+/** The `## ` section titles each backlog file is allowed to carry, keyed by file name. A `## `
+ * title outside its file's list is an unknown heading — quoted in no reader's section, so every
+ * entry under it would be invisible. backlogStructureReason treats known sections as load-bearing
+ * (dropping one is rejected) but unknown ones as mistakes: ADDING one stays a legitimate project
+ * extension (this repo grew `## Verified`; a doc-only stack can grow `## Notes`), while DROPPING
+ * one is allowed — demoting a misplaced `## ` heading to `### ` under a known section is how such
+ * a heading is repaired (BUGS.md 2026-10-23). BUGS.md's Verified section is a real section this
+ * repo grew beyond its template; the archive and questions files keep the one section their
+ * template seeds. */
+const KNOWN_SECTIONS = {
+    "PLANS.md": ["Planned", "Done"],
+    "PLANS-DONE.md": ["Done"],
+    "BUGS.md": ["Open", "Fixed", "Verified"],
+    "BUGS-FIXED.md": ["Fixed"],
+    "QUESTIONS.md": ["Open", "Answered"],
+};
 /** The stamp verbs a backlog heading's trailing `(planned 2026-09-29)`-style parenthetical
  * opens with — the same list the backlog writers stamp and the dashboards parse. One home
  * for the verb list: the TUI's team alerts strip the suffix (stripEntryStamp) and the
@@ -24,7 +38,7 @@ export const ENTRY_STAMP_META_SOURCE = String.raw `\s*\(((?:planned|reported|fou
  * (case-insensitive, note captured), read through stripEntryStamp below. */
 const ENTRY_STAMP_META_RE = new RegExp(ENTRY_STAMP_META_SOURCE, "i");
 /** A heading or title with its trailing `(planned 2026-09-29)`-style stamp suffix removed,
- * whitespace otherwise untouched — the shared span of this module's indexTitle, eligibility's
+ * whitespace otherwise untouched — the shared span of backlog-index's indexTitle, eligibility's
  * entryKey and seriesPart, and the TUI team alerts, which all read a heading stamp-free. */
 export function stripEntryStamp(title) {
     return title.replace(ENTRY_STAMP_META_RE, "");
@@ -68,7 +82,7 @@ const DONE_STAMP = /[—,(;]\s*done \d{4}-\d{2}-\d{2}/;
  * but no `done YYYY-MM-DD` — written as plans, never stamped done, and invisible to every
  * Planned reader (`plannedPlanEntries` reads only `## Planned`); and (b) entries under
  * `## Planned` whose heading already carries `done YYYY-MM-DD` — finished and never moved,
- * so the feature loop may implement them again. Fence-aware through backlog-md.ts's shared
+ * so the feature role may implement them again. Fence-aware through backlog-md.ts's shared
  * machinery: a `### `/`## ` line inside a fenced code block is quoted content, not structure,
  * and heading dates are matched against the same joined metadata entryDates matches against.
  * BUGS.md and QUESTIONS.md are deliberately out of scope — their Fixed-section headings do
@@ -144,6 +158,36 @@ function sectionTitleCounts(md) {
 export function duplicateHeadings(md) {
     return [...sectionTitleCounts(md)].filter(([, n]) => n > 1).map(([title]) => title);
 }
+/** The `### ` headings `head` adds that `base` already carries, as extra copies: for each
+ * heading (whitespace-collapsed, every `## ` section, fence-aware) present in `base` and more
+ * often in `head`, one entry per added copy, in the head's first-appearance order. The archive
+ * files' re-pasted-history signal: on 2026-10-19 a conflict hand-back re-pasted 383 + 310 archived
+ * entries, and the commit-time sweep prepended them to PLANS-DONE.md and BUGS-FIXED.md again. A
+ * heading the base already duplicates never counts unless the head adds yet another copy. */
+export function readdedHeadings(base, head) {
+    const count = (md) => {
+        const out = new Map();
+        for (const line of fenceAwareHeadingLines(md, "### ")) {
+            const title = headingTitle(line, "### ");
+            const key = collapseWhitespace(title);
+            const seen = out.get(key);
+            if (seen)
+                seen.n++;
+            else
+                out.set(key, { n: 1, title });
+        }
+        return out;
+    };
+    const baseCounts = count(base);
+    const readded = [];
+    for (const [key, { n, title }] of count(head)) {
+        const had = baseCounts.get(key)?.n ?? 0;
+        if (had > 0 && n > had)
+            for (let i = had; i < n; i++)
+                readded.push(title);
+    }
+    return readded;
+}
 /** The `## Fixed` entry headings whose body holds more than one `**Symptom:**` or
  * `**Fix:**` block — the shape a record with no `### ` heading leaves when parseEntryDetails
  * absorbs it into the entry above (an entry opens only at a `### ` line, so the headingless
@@ -177,9 +221,13 @@ export function doubleBlockFixedEntries(md) {
  * against, so a stacked batch is judged change by change — and rejects when (a) the head carries
  * MORE of a title than the base carried (a duplicate `## Done` added, or a second one where the
  * base had none), or (b) a title present on the base is gone from the head (a whole section
- * dropped). The rule is deliberately about heading sets, not section names: a project whose BUGS.md
- * adds `## Verified` (this repo's does) or a fresh repo seeded from src/init/init.ts's templates
- * passes unchanged. A base that already carries a duplicate never blocks unrelated edits — rule (a)
+ * dropped). Dropping is judged against the file's KNOWN_SECTIONS — PLANS.md Planned and Done,
+ * its archive Done, BUGS.md Open, Fixed and Verified, its archive Fixed, QUESTIONS.md Open and
+ * Answered: an unknown `## ` heading (a misplaced one) is free to be dropped — demoting it to
+ * `### ` under a known section is the repair (BUGS.md 2026-10-23) — while dropping a known
+ * section stays rejected. ADDING a `## ` heading is still a legitimate project extension (this
+ * repo's `## Verified`, a doc-only stack's `## Notes`), so the rule stays about heading sets, not
+ * section names. A base that already carries a duplicate never blocks unrelated edits — rule (a)
  * fires only when the head's count EXCEEDS the base's, so a change that removes a duplicate always
  * passes. For PLANS.md there is a third rule (part 4/4): a head that ADDS a `### ` entry directly
  * under `## Done` whose joined heading metadata carries `(planned YYYY-MM-DD` but no `done
@@ -187,11 +235,14 @@ export function doubleBlockFixedEntries(md) {
  * the wrong section — rejected with a reason naming the entry and the one-line fix. Judged against
  * the merge-base (the same base the heading rules use), so a Planned → Done move of an entry the
  * base already had passes, a stacked batch is measured change by change, and a pre-existing
- * stranded entry (whose key the base has) never blocks unrelated merges — the clean loop's repair
+ * stranded entry (whose key the base has) never blocks unrelated merges — the clean role's repair
  * (part 3/4) owns those. BUGS.md has one more: a head `## Fixed` entry that holds more than one
  * `**Symptom:**` or `**Fix:**` block (doubleBlockFixedEntries) absorbed a headingless record into
  * its body; a base body that already had the shape is never re-flagged, so only a newly merged
- * record is rejected (BUGS.md 2026-10-08). Detection is deterministic markdown reading — no pi — so
+ * record is rejected (BUGS.md 2026-10-08). The archive files (PLANS-DONE.md, BUGS-FIXED.md) have
+ * one more: a head that adds a copy of a `### ` heading the base archive already carries
+ * (readdedHeadings) is pasting finished history back, rejected with the count and the first
+ * heading. Detection is deterministic markdown reading — no pi — so
  * both the review gate (exempt and code diffs alike) and the in-lock merge re-check can afford it
  * on every merge (plans: "Backlog structure check", part 2/4). */
 export async function backlogStructureReason(wt, mainBranch, files) {
@@ -209,10 +260,25 @@ export async function backlogStructureReason(wt, mainBranch, files) {
             if (count > 1 && count > (baseCounts.get(title) ?? 0))
                 return (`${file} adds another "## ${title}" heading — readers take the first section of a ` +
                     `name, so anything under the duplicate is invisible; keep one "## ${title}" per file`);
+        // A `## ` title that is not one of this file's known sections (KNOWN_SECTIONS) is not
+        // load-bearing: ADDING one is a legitimate project extension (this repo grew `## Verified`,
+        // a doc-only stack can grow `## Notes`), so a new one passes — but DROPPING one is the repair
+        // for a misplaced heading, so rule (b) guards only known sections and leaves an unknown one
+        // free to be demoted to `### ` under a known section (BUGS.md 2026-10-23).
+        const known = KNOWN_SECTIONS[file] ?? [];
         for (const title of baseCounts.keys())
-            if (!headCounts.has(title))
+            if (!headCounts.has(title) && known.includes(title))
                 return (`${file} drops the "## ${title}" section the base had — entries filed under it would ` +
                     `be invisible to every section reader; restore "## ${title}"`);
+        // An archive only grows by new headings: a copy of one the base archive already carries is
+        // finished history pasted back (readdedHeadings), never a new finished entry.
+        if (file === PLANS_ARCHIVE || file === BUGS_ARCHIVE) {
+            const readded = readdedHeadings(base, head);
+            if (readded.length > 0)
+                return (`${file} adds ${plural(readded.length, '"### " heading', '"### " headings')} it already carries ` +
+                    `(first: "${truncate(readded[0], 160)}") — the archive holds each finished entry once, so the ` +
+                    `copies duplicate its history; remove them, and edit an archived entry in place instead`);
+        }
         // Part 4/4's rule, PLANS.md and its archive (see this function's doc comment): a NEW plan
         // filed directly under ## Done with no done date. strandedPlanEntries already does the
         // fence-aware, joined-metadata reading the first two conditions need; the base's key set —
@@ -225,9 +291,18 @@ export async function backlogStructureReason(wt, mainBranch, files) {
             ]);
             const newcomer = strandedPlanEntries(head).find((e) => e.section === "Done" && !baseKeys.has(planHeadingKey(e.title)));
             if (newcomer)
-                return (`${file} files "${newcomer.title}" directly under "## Done" with no done date — a plan ` +
-                    `written into the done section is invisible to every Planned reader; file it under ` +
-                    `"## Planned"${file === "PLANS.md" ? "" : " in PLANS.md"} instead`);
+                // The fault and its fix differ by file: in PLANS.md a NEW plan (the base never carried
+                // it in any section) was pasted under ## Done unfinished and belongs under ## Planned;
+                // in the archive the same heading is a finished plan the commit-time sweep already moved
+                // here, whose only fault is a missing `done YYYY-MM-DD` token — telling its author to
+                // "file it under ## Planned" is the opposite of the fix and wastes a revision.
+                return file === PLANS_ARCHIVE
+                    ? `${file} files "${newcomer.title}" directly under "## Done" with no done date — the ` +
+                        `entry is finished and archived, so the only fix is to add \`done YYYY-MM-DD\` to its ` +
+                        `"### " heading; do not move it back under "## Planned"`
+                    : `${file} files "${newcomer.title}" directly under "## Done" with no done date — a plan ` +
+                        `written into the done section is invisible to every Planned reader; file it under ` +
+                        `"## Planned" instead`;
         }
         // A new headingless record cannot hide in a Fixed entry's body: reject a head whose
         // multi-block entries the base did not carry (an existing one never blocks unrelated
@@ -249,7 +324,7 @@ export async function backlogStructureReason(wt, mainBranch, files) {
 const strandedPlanCache = new Map();
 /** `strandedPlanEntries` for the primary checkout's PLANS.md and PLANS-DONE.md (where finished
  * plans are archived, plans/backlog-archive.md), each entry tagged with its file, stat-cached per
- * file path (stat-cache.cachedTextByStat): the clean loop asks for it every tick, `tumwater role`
+ * file path (stat-cache.cachedTextByStat): the clean role asks for it every run, `tumwater role`
  * previews the same block and doctor reports it, and the archive grows without bound, so an
  * unchanged file costs one stat instead of a full read and section walk. A missing or
  * unreadable file reads as no entries. The result is copied out, so each caller owns its
@@ -260,8 +335,8 @@ export function strandedPlanEntriesFor(root) {
         return (cachedTextByStat(strandedPlanCache, full, full, (md) => strandedPlanEntries(md).map((e) => ({ ...e, file })), (entries) => entries.slice()) ?? []);
     });
 }
-/** The clean loop's `<backlog-structure>` prompt block for the primary checkout at `root`
- * (the same root the telemetry digest and qa coverage blocks read): a rendered block listing
+/** The clean role's `<backlog-structure>` prompt block for the primary checkout at `root`
+ * (the same root the selfcheck digest and qa coverage blocks read): a rendered block listing
  * each stranded heading with its current section, or undefined when PLANS.md is missing,
  * unreadable, or clean — an unreadable or clean file gives no block, so the prompt is
  * unchanged in the common case. The stranded entries are stat-cached (strandedPlanEntriesFor),
@@ -285,7 +360,7 @@ ${listed}
  * order: the fence-aware walk sectionLines and parseEntryDetails share, carrying the line
  * numbers neither returns. An entry starts at a `### ` line and ends at the line before the
  * next `### ` or `## ` line (or EOF) — the same boundary parseEntryDetails closes bodies on —
- * so a loop reading lines `start`..`end` sees exactly the entry's heading and body. A `## ` or
+ * so a role reading lines `start`..`end` sees exactly the entry's heading and body. A `## ` or
  * `### ` line inside a fenced code block is body content, never a boundary, through
  * backlog-md.ts's shared fenceTracker. */
 export function actionableEntryRanges(md, sectionTitle) {
@@ -322,7 +397,7 @@ export function actionableEntryRanges(md, sectionTitle) {
 /** Per-file cache of a section's actionable ranges (actionableEntryRangesFor). */
 const actionableRangesCache = new Map();
 /** `actionableEntryRanges` for one backlog file, stat-cached per (file path, section title):
- * the per-poll and per-tick readers (eligibility's eligibleEntries, claims' listedKeys) ask for
+ * the per-poll and per-run readers (eligibility's eligibleEntries, claims' listedKeys) ask for
  * the same section of the same file every couple of seconds, and BUGS.md's closed history —
  * which the whole-file walk reads — grows without bound, so an unchanged file costs one stat
  * instead of a full read, split, and section walk. Keyed by file path so distinct roots never
@@ -332,43 +407,6 @@ export function actionableEntryRangesFor(root, file, sectionTitle) {
     const full = path.join(root, file);
     return (cachedTextByStat(actionableRangesCache, `${full}\u0000${sectionTitle}`, full, (md) => actionableEntryRanges(md, sectionTitle), (ranges) => ranges.slice()) ?? []);
 }
-/** Max characters an index title keeps before a trailing ellipsis, so one very long BUGS.md
- * heading cannot dominate the block. */
-const INDEX_TITLE_MAX = 160;
-/** The index title for a `### ` heading: its stamp suffix (`(reported … 2026-…)`) removed via
- * the shared stripEntryStamp and whitespace collapsed, then clipped to INDEX_TITLE_MAX
- * through text.ts's truncate — the one ellipsis rule, so the cut cannot drift from the
- * other trimmed labels and never splits a surrogate pair. */
-function indexTitle(heading) {
-    return truncate(collapseWhitespace(stripEntryStamp(heading)), INDEX_TITLE_MAX);
-}
-/** The files and sections the actionable index covers, in the order loops read them: PLANS.md's
- * planned features, BUGS.md's open bugs, QUESTIONS.md's open questions. */
-const INDEX_SECTIONS = [
-    { file: "PLANS.md", section: "Planned" },
-    { file: "BUGS.md", section: "Open" },
-    { file: "QUESTIONS.md", section: "Open" },
-];
-/** How many closed entries per closed section the index lists. */
-const RECENT_CLOSED_MAX = 5;
-/** The closed sections whose newest entries the index also lists, each read from the live file
- * and then its archive (FINISHED_ARCHIVES, plans/backlog-archive.md); every one keeps its entries
- * newest first, so the first entries in file order are the recent ones. */
-const CLOSED_SECTIONS = FINISHED_ARCHIVES.map((a) => ({
-    files: [a.file, a.archive],
-    section: a.section,
-}));
-/** Per-file cache of the parsed actionable entries (indexSectionCache). */
-const indexSectionCache = new Map();
-/** Per-file cache of the newest closed entries' ranges; separate from indexSectionCache, which is
- * keyed by the file path alone and PLANS.md would collide. */
-const closedSectionCache = new Map();
-/** One `IndexRow` as an ordinary index line: range, truncated stamp-free title, hold mark, and
- * the in-flight suffix when a live claim names it. The collapsed-series line is built by
- * collapseBlockedSeries, returned alongside these. */
-function renderIndexRow(row) {
-    return `- ${row.start}-${row.end}: ${indexTitle(row.title)}${row.mark}${row.claimer ? ` [in flight by ${row.claimer}]` : ""}`;
-}
 /** True when `row` is a candidate for series-collapsing: it is a series part held only by an
  * unmerged prerequisite (`{ blockedBy }`, not a Refused/needs-review/needs-replan note) and no live
  * claim names it. Returns the row's series part when it collapses, else null. */
@@ -415,7 +453,7 @@ export function partsLabel(rows) {
 }
 /** Renders `rows` to index lines, collapsing a multi-part series' blocked parts to one line per
  * series (this plan, "Compact backlog index, part 1/1"): a long series' chain of held parts would
- * otherwise fill every loop prompt with lines no role can act on. An entry collapses when it is a
+ * otherwise fill every role prompt with lines no role can act on. An entry collapses when it is a
  * series part held only by an unmerged prerequisite and no live claim names it; a series with two
  * or more such parts renders as one line at the position of its first collapsed part, spanning from
  * that part's start to the last's end, with the part list (`2/4–4/4`) and a `[blocked: each waits
@@ -423,9 +461,9 @@ export function partsLabel(rows) {
  * (Refused/needs-review/needs-replan) part and a claimed part keep their ordinary lines, so the
  * eligibility rules the feature role's charter teaches are unchanged; the collapsed mark keeps the
  * `[blocked: …]` prefix it already skips. */
-/** The grouping half of collapseBlockedSeries, on any rows carrying a title, hold and claimer:
- * each series with two or more collapsible blocked parts becomes one `series` group at its first
- * part's place; every other row stays a `row` group in order. */
+/** The grouping half of backlog-index's collapseBlockedSeries, on any rows carrying a title,
+ * hold and claimer: each series with two or more collapsible blocked parts becomes one `series`
+ * group at its first part's place; every other row stays a `row` group in order. */
 export function groupBlockedSeries(rows) {
     // Group the collapsible rows by series, keyed on the lowercased series name.
     const bySeries = new Map();
@@ -461,98 +499,3 @@ export function groupBlockedSeries(rows) {
     }
     return groups;
 }
-function collapseBlockedSeries(rows) {
-    return groupBlockedSeries(rows).map((g) => {
-        if (g.kind === "row")
-            return renderIndexRow(g.row);
-        const first = g.rows[0];
-        const last = g.rows[g.rows.length - 1];
-        return (`- ${first.start}-${last.end}: ${indexTitle(g.series)}, ${g.rows.length} blocked parts ` +
-            `(${partsLabel(g.rows)}) [blocked: each waits on an earlier part]`);
-    });
-}
-/** The `<backlog-index>` prompt block: each actionable entry (PLANS.md ## Planned, BUGS.md
- * ## Open, QUESTIONS.md ## Open) with its 1-based line range, rendered from the primary
- * checkout `root`. Replaces the per-tick `grep -n '^##'` heading map, whose pattern also matched
- * every `###` title in BUGS.md's closed history and so re-sent ~108 KB on every turn (BUGS.md
- * 2026-10-06). A missing or unreadable file contributes no sections; undefined when no section
- * has entries, so an empty backlog leaves the prompt unchanged. The per-file actionable entries
- * are stat-cached (indexSectionCache), so an unchanged backlog file is not re-read or re-walked. */
-export function renderBacklogIndexBlock(root) {
-    // The still-planned set every section's holds are judged against. A prerequisite clause rides
-    // in the entry heading, and the eligibility gate (backlog-eligibility.ts's eligibleEntries)
-    // and `tumwater backlog`'s holds read it for BUGS.md's Open entries too, so the same set must
-    // reach this block's Open section as well — otherwise a held bug reads as eligible here while
-    // the gate skips it. plannedPlanEntries is stat-cached by sectionEntries, so an unchanged
-    // PLANS.md costs one stat per call.
-    const planned = plannedPlanEntries(root);
-    const plannedParts = plannedPartKeys(planned);
-    // The cross-role in-flight hold (BUGS.md 2026-10-09): every role's prompts share this one
-    // section-agnostic index, and the role it is assembled for is not available here, so an entry
-    // claimed by ANY role is marked in flight — the owner's own index marks it too, which is harmless
-    // (its assignment note is the instruction), and every other role reads it as taken instead of
-    // re-implementing a change someone's merge is already carrying to main.
-    const claimed = allLiveClaims(root);
-    const listed = [];
-    for (const { file, section } of INDEX_SECTIONS) {
-        const full = path.join(root, file);
-        const entries = cachedTextByStat(indexSectionCache, full, full, (md) => {
-            const ranges = actionableEntryRanges(md, section);
-            if (ranges.length === 0)
-                return [];
-            const lines = md.split(/\r?\n/);
-            // The body slice is the lines after the heading through the last body line, so a note
-            // is found whether it sits first or last (the heading is at 1-based line e.start).
-            return ranges.map((e) => ({
-                start: e.start,
-                end: e.end,
-                title: e.title,
-                body: lines.slice(e.start, e.end).join("\n"),
-            }));
-        }, (v) => v.slice());
-        if (entries === null || entries.length === 0)
-            continue;
-        listed.push(`${file} ## ${section}`);
-        // Build the rows once per section, then collapse a PLANS.md series' blocked parts (this
-        // plan) — BUGS.md and QUESTIONS.md sections keep every entry on its own line.
-        const rows = entries.map((e) => {
-            const hold = entryHoldAgainst({ title: e.title, body: e.body }, plannedParts);
-            const key = collapseWhitespace(stripEntryStamp(e.title)).toLowerCase();
-            return {
-                start: e.start,
-                end: e.end,
-                title: e.title,
-                hold,
-                mark: holdMark(hold),
-                claimer: claimed.get(key),
-            };
-        });
-        const isPlans = file === "PLANS.md" && section === "Planned";
-        listed.push(...(isPlans ? collapseBlockedSeries(rows) : rows.map(renderIndexRow)));
-    }
-    for (const { files, section } of CLOSED_SECTIONS) {
-        // The newest RECENT_CLOSED_MAX across the live section (normally empty) and the archive.
-        let room = RECENT_CLOSED_MAX;
-        for (const file of files) {
-            if (room <= 0)
-                break;
-            const full = path.join(root, file);
-            const entries = cachedTextByStat(closedSectionCache, full, full, (md) => actionableEntryRanges(md, section).slice(0, RECENT_CLOSED_MAX), (v) => v.slice())?.slice(0, room);
-            if (entries === undefined || entries.length === 0)
-                continue;
-            room -= entries.length;
-            listed.push(`${file} ## ${section} (newest ${entries.length})`);
-            for (const e of entries)
-                listed.push(`- ${e.start}-${e.end}: ${indexTitle(e.title)}`);
-        }
-    }
-    if (listed.length === 0)
-        return undefined;
-    return `Backlog index — each actionable entry, then the newest closed ones, with the 1-based line range it occupies in its file.
-Read the entries you need by those ranges; the archive files (PLANS-DONE.md, BUGS-FIXED.md) grow
-with every merge, so search them rather than reading them. Blocked parts of a multi-part series are
-shown as one line per series.
-<backlog-index>
-${listed.join("\n")}
-</backlog-index>`;
-}