tumwater 0.2.0 → 0.3.0

package/dist/src/doctor/doctor-contention-checks.js

@@ -0,0 +1,380 @@
+/** The worktree/process contention checks, split out of doctor-checks.ts (the project's
+ * largest module, ~6x the file-size median, holding four unrelated check domains): the merge
+ * lock, the worktree pool, the three `_wait` contention rows (slot, provider, check), and the
+ * parallel-work-instance, tumwater-pin, leftover-role-branch and provider-concurrency audits.
+ * Read-only like every doctor check; they share the wait fold (sumWaits) and the pin/worktree
+ * leftovers helpers, all of which live here beside the checks that use them. The environment
+ * and repo-setup checks stay in doctor-checks.ts; the project's own state, disk and build
+ * checks live in doctor-project-checks.ts; model readiness lives in doctor-model-checks.ts;
+ * the per-document backlog checks live in doctor-backlog-plans.ts and doctor-backlog-bugs.ts
+ * beside doctor-backlog.ts's shared helpers and queue checks;
+ * doctor.ts remains the composition layer
+ * that calls each check in fixed order. */
+import path from "node:path";
+import { defaultConfig, knownRoleIds, slotCount } from "../config/config.js";
+import { CUSTOM_NAME_RE } from "../config/config-schema.js";
+import { subdirectoryNames } from "../files/files.js";
+import { finiteNumber, stringList } from "../files/json-object.js";
+import { commitExists, listBranches, refsUnder } from "../git/git.js";
+import { readSlotsState } from "../git/slots-state.js";
+import { eventsSince } from "../events/event-window.js";
+import { DAY_MS } from "../text/datetime.js";
+import { shortSha } from "../text/format.js";
+import { errorMessage, joinParts, textOr } from "../text/text.js";
+import { rankCountEntries } from "../collections.js";
+import { DIRECTOR_ROLE } from "../roles/roles.js";
+import { INSTANCE_ROLES, runnerEnabled, runnerIds } from "../roles/runner-ids.js";
+import { CLAIM_IDLE_MAX_MS, listedKeys } from "../scheduling/claims.js";
+import { piModelsPath } from "../pi/pi-models.js";
+import { piSettingsPath, readPiDefaultModel } from "../pi/pi-settings.js";
+import { seamConcurrencyKey } from "../config/config-views.js";
+import { seamsInUse } from "../config/config-fallbacks.js";
+import { enforcedKeyCaps } from "../concurrency/semaphore.js";
+import { classifyLock, readLockPid } from "../concurrency/lock.js";
+import { branchName, mergeLockDir, worktreesDir } from "../paths.js";
+import { loadRunnerState } from "../runner/runner-state-load.js";
+import { agree, plural, shortSpanPhrase } from "../text/phrases.js";
+import { failOutcome, okOutcome, warnOutcome } from "./outcome.js";
+/** Merge lock — read-only classification via classifyLock (the same three cases the breaker
+ * uses). A stale lock is a warning, not a failure: it self-heals on the next merge. */
+export function checkMergeLock(root) {
+    const dir = mergeLockDir(root);
+    switch (classifyLock(dir)) {
+        case "absent":
+            return okOutcome("not held");
+        case "live": {
+            const pid = readLockPid(dir);
+            return okOutcome(pid !== null ? `held by running role (pid ${pid})` : "held (pid not yet written)");
+        }
+        case "stale":
+            return warnOutcome("stale — will be broken on next merge");
+    }
+}
+/** A pin older than a day is the doctor's stuck-pin signal: a paused or retired role is holding
+ * a pooled checkout nobody will reuse (plans/worktree-pool.md, "Observability"). */
+const PIN_STALE_MS = DAY_MS;
+/** Directory names under the worktrees root that the pool no longer owns: a legacy per-role
+ * `<role>` checkout (the pre-pool layout) or a `_land-<role>` checkout from the merge queue. The
+ * dedicated `director`, `_slot-<n>`, `_merge`, `_main`, `_gate-main` and `_build` checkouts are
+ * expected and never reported. An unreadable worktrees dir means nothing to report. */
+function leftoverWorktreeDirs(root) {
+    return subdirectoryNames(worktreesDir(root), (name) => name !== DIRECTOR_ROLE && !(name.startsWith("_") && !name.startsWith("_land-")));
+}
+/** The window of `slot_wait` events the pool row sums, and the total wait inside it that warns. */
+const SLOT_WAIT_CHECK_WINDOW_MS = DAY_MS;
+const SLOT_WAIT_WARN_TOTAL_MS = 10 * 60_000;
+const PROVIDER_WAIT_CHECK_WINDOW_MS = DAY_MS;
+const PROVIDER_WAIT_WARN_TOTAL_MS = 10 * 60_000;
+const CHECK_WAIT_CHECK_WINDOW_MS = DAY_MS;
+const CHECK_WAIT_WARN_TOTAL_MS = 10 * 60_000;
+/** Fold a window's `*_wait` events into the three numbers every contention check reads: how
+ * many waits there were (each event counts once), the total ms they waited, and the longest
+ * single wait — plus the longest event itself, for the check that names the role/purpose off
+ * it. The single home of that fold for the three contention checks (checkWorktreePool's
+ * `slot_wait` half, checkProviderContention, checkCheckContention), so the sum and the
+ * longest-wait selection cannot drift between them. A wait that omits or corrupts `waitedMs`
+ * reads as 0 for the sum, exactly as each inline `finiteNumber(ev.waitedMs, 0)` fold did;
+ * the longest-wait tie keeps the first event that reached it. */
+function sumWaits(waits) {
+    let count = 0;
+    let totalMs = 0;
+    let longestMs = 0;
+    let longest = waits[0];
+    for (const ev of waits) {
+        count += 1;
+        const waited = finiteNumber(ev.waitedMs, 0);
+        totalMs += waited;
+        if (waited > longestMs) {
+            longestMs = waited;
+            longest = ev;
+        }
+    }
+    return { count, totalMs, longestMs, longest };
+}
+/** Worktree pool (plans/worktree-pool.md, part 5/5): report the pooled slots and warn on the
+ * states an operator should act on — a legacy `<role>` or `_land-*` checkout the pool migration
+ * left behind, a pin older than PIN_STALE_MS (a paused role holding a checkout), and the last
+ * 24 h of `slot_wait` events totalling SLOT_WAIT_WARN_TOTAL_MS (the pool is too small). Read-only,
+ * like every doctor check; `now` is a test seam for the pin-age arithmetic. */
+export function checkWorktreePool(root, now = Date.now()) {
+    const slots = readSlotsState(root).slots;
+    const leased = slots.filter((slot) => slot.lease !== null);
+    const pins = slots.filter((slot) => slot.pinnedFor !== null);
+    const stale = pins.filter((slot) => slot.pinnedAt != null && now - slot.pinnedAt > PIN_STALE_MS);
+    const leftover = leftoverWorktreeDirs(root);
+    const parts = [`${plural(slots.length, "workspace")}, ${leased.length} leased, ${pins.length} pinned`];
+    if (leftover.length > 0)
+        parts.push(`legacy worktrees still present: ${leftover.join(", ")}`);
+    if (stale.length > 0) {
+        parts.push(stale
+            .map((slot) => `${textOr(slot.pinnedFor)} pinned on ${path.basename(slot.dir)} for ${Math.round((now - (slot.pinnedAt ?? now)) / 3_600_000)}h`)
+            .join("; "));
+    }
+    let contended = false;
+    try {
+        const waits = eventsSince(root, SLOT_WAIT_CHECK_WINDOW_MS, now, (ev) => ev.type === "slot_wait");
+        if (waits.length > 0) {
+            const { count, totalMs, longestMs, longest } = sumWaits(waits);
+            contended = totalMs >= SLOT_WAIT_WARN_TOTAL_MS;
+            const purpose = longest.purpose === "verify" ? "check" : "run";
+            parts.push(`${plural(count, "wait")} to start in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)} (${textOr(longest.role)} ${purpose})${contended ? ` — raise worktreeSlots above ${slots.length}` : ""}`);
+        }
+    }
+    catch (err) {
+        parts.push(`could not read the event log for waits to start: ${errorMessage(err)}`);
+    }
+    const warn = leftover.length > 0 || stale.length > 0 || contended;
+    return (warn ? warnOutcome : okOutcome)(parts.join(" — "));
+}
+/** Provider-slot contention: sums the last 24 h of `provider_wait` events (each a role run
+ * whose semaphore admission waited PROVIDER_WAIT_EVENT_MS or more, recorded in
+ * orchestrator-launch.ts) and warns when the total reaches PROVIDER_WAIT_WARN_TOTAL_MS — the
+ * provider cap is too small. Names every key that waited, busiest first (the key-less
+ * bucket, which also takes every team-cap wait whatever its key, reads "the run cap") and, on
+ * a warn, each key's maxConcurrentPerProvider cap to raise, plus maxConcurrent for the run cap.
+ * A counterpoint to checkWorktreePool's slot-wait half; read-only, like every doctor check.
+ * `now` is a test seam for the window. */
+export function checkProviderContention(root, config, now = Date.now()) {
+    const perKey = new Map();
+    let count = 0;
+    let totalMs = 0;
+    let longestMs = 0;
+    try {
+        const waits = eventsSince(root, PROVIDER_WAIT_CHECK_WINDOW_MS, now, (ev) => ev.type === "provider_wait");
+        const sum = sumWaits(waits);
+        count = sum.count;
+        totalMs = sum.totalMs;
+        longestMs = sum.longestMs;
+        for (const ev of waits) {
+            const waited = finiteNumber(ev.waitedMs, 0);
+            const key = ev.cap !== "team" && typeof ev.key === "string" && ev.key.length > 0 ? ev.key : "";
+            perKey.set(key, (perKey.get(key) ?? 0) + waited);
+        }
+    }
+    catch (err) {
+        // A read error leaves the level ok, exactly like the slot-wait branch's own read: the
+        // check cannot measure, so it says so rather than crying wolf.
+        return okOutcome(`could not read the event log for provider waits: ${errorMessage(err)}`);
+    }
+    if (count === 0)
+        return okOutcome("no provider waits");
+    // Rank the buckets by wait total, ties breaking on key name so the detail is deterministic.
+    // The key-less bucket reads "the run cap" — maxConcurrent, since no seam keys to a provider.
+    const ranked = rankCountEntries(perKey).map(([key, total]) => ({ key, total }));
+    const bucketLabel = (key) => (key === "" ? "the run cap" : `the ${key} slot`);
+    const cells = ranked.map((b) => `${bucketLabel(b.key)} ${shortSpanPhrase(b.total)}`);
+    // A single bucket keeps the pre-existing wording, a bare `(the X slot)` without the total
+    // (the total is already named by the `shortSpanPhrase(total)` in the clause before it);
+    // two or more name each bucket's total so the ranking reads at a glance.
+    const parenthetical = ranked.length === 1
+        ? `(${bucketLabel(ranked[0].key)})`
+        : `(${cells[0]}, also ${cells.slice(1).join(", ")})`;
+    let detail = `${plural(count, "wait")} for a provider slot in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)} ${parenthetical}`;
+    const contended = totalMs >= PROVIDER_WAIT_WARN_TOTAL_MS;
+    if (contended) {
+        const advice = ranked
+            .filter((b) => b.key !== "" && config?.maxConcurrentPerProvider?.[b.key] !== undefined)
+            .map((b) => `raise maxConcurrentPerProvider.${b.key} above ${config?.maxConcurrentPerProvider?.[b.key]}`)
+            .join(" and ");
+        const teamAdvice = perKey.has("") && config !== undefined ? [`raise maxConcurrent above ${config.maxConcurrent}`] : [];
+        const all = [...(advice ? [advice] : []), ...teamAdvice].join(" and ");
+        if (all)
+            detail += ` — ${all}`;
+    }
+    return (contended ? warnOutcome : okOutcome)(detail);
+}
+/** Names the kind of check a `heldBy` label stands for, in checkCheckContention's tally. */
+function checkHolderKind(label) {
+    if (label === "main baseline")
+        return "the main baseline";
+    const scope = label.slice(label.lastIndexOf(" ") + 1);
+    return scope === "gate" || scope === "merge" || scope === "batch" ? `${scope} checks` : "other checks";
+}
+/** Check-slot contention: sums the last 24 h of `check_wait` events (each a scoped check whose
+ * permit wait reached CHECK_WAIT_EVENT_MS, recorded in build-check-events.ts) and warns when
+ * the total reaches CHECK_WAIT_WARN_TOTAL_MS — maxConcurrentChecks is too small. The third
+ * sibling of checkWorktreePool's slot-wait half and checkProviderContention; read-only.
+ * When events carry `heldBy` (the labels of the permit holders the check parked behind), the
+ * detail gains `(behind <kind> <span>, …)` after the longest clause, kinds busiest first: a
+ * wait counts once toward each distinct kind that held a slot. `now` is a test seam for the
+ * window. */
+export function checkCheckContention(root, config, now = Date.now()) {
+    const perKind = new Map();
+    let count = 0;
+    let totalMs = 0;
+    let longestMs = 0;
+    try {
+        const waits = eventsSince(root, CHECK_WAIT_CHECK_WINDOW_MS, now, (e) => e.type === "check_wait");
+        const sum = sumWaits(waits);
+        count = sum.count;
+        totalMs = sum.totalMs;
+        longestMs = sum.longestMs;
+        for (const ev of waits) {
+            const waited = finiteNumber(ev.waitedMs, 0);
+            for (const kind of new Set(stringList(ev.heldBy).map(checkHolderKind))) {
+                perKind.set(kind, (perKind.get(kind) ?? 0) + waited);
+            }
+        }
+    }
+    catch (err) {
+        return okOutcome(`could not read the event log for check waits: ${errorMessage(err)}`);
+    }
+    if (count === 0)
+        return okOutcome("no check waits");
+    let detail = `${plural(count, "wait")} for a check slot in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)}`;
+    if (perKind.size > 0) {
+        const ranked = rankCountEntries(perKind);
+        detail += ` (behind ${ranked.map(([kind, ms]) => `${kind} ${shortSpanPhrase(ms)}`).join(", ")})`;
+    }
+    if (totalMs < CHECK_WAIT_WARN_TOTAL_MS)
+        return okOutcome(detail);
+    detail += ` — raise maxConcurrentChecks above ${config?.maxConcurrentChecks ?? defaultConfig().maxConcurrentChecks}`;
+    return warnOutcome(detail);
+}
+/** The "leftover role branches" row: `tumwater/<id>` branches whose role is no longer enabled
+ * (disabled in tumwater.json, or an instance above the configured count). Retirable ids are
+ * the ones `tumwater retire --role <id>` accepts (knownRoleIds, or any well-formed custom-role name
+ * with a leftover branch); any other id is listed by
+ * full branch name with a manual-delete hint. The director and enabled roles are skipped.
+ * Read-only, like every check: it only lists refs; an unreadable tumwater.json skips it. */
+export async function checkLeftoverRoleBranches(root, config) {
+    if (!config)
+        return okOutcome("no readable tumwater.json — nothing to verify");
+    const prefix = branchName("");
+    const known = new Set(knownRoleIds(config));
+    const retirable = [];
+    const unknown = [];
+    for (const branch of await listBranches(root)) {
+        if (!branch.startsWith(prefix))
+            continue;
+        const id = branch.slice(prefix.length);
+        if (!id || id.includes("/") || id === DIRECTOR_ROLE || runnerEnabled(config, id))
+            continue;
+        if (known.has(id) || CUSTOM_NAME_RE.test(id))
+            retirable.push(id);
+        else
+            unknown.push(branch);
+    }
+    const parts = [];
+    if (retirable.length > 0) {
+        const n = retirable.length;
+        parts.push(`${retirable.join(", ")} ${agree(n, "is", "are")} disabled but still ${agree(n, "has", "have")} a branch — \`tumwater retire --role <id>\` removes each (\`--dry-run\` previews)`);
+    }
+    if (unknown.length > 0) {
+        parts.push(`${unknown.join(", ")} ${agree(unknown.length, "names", "name")} no role retire accepts — delete with \`git branch -D\` once its commits are not needed`);
+    }
+    return parts.length === 0 ? okOutcome("no branches left by disabled roles") : warnOutcome(parts.join("; "));
+}
+/** The "tumwater pins" row: fails when a `refs/tumwater/` pin (merge, rejected or legacy landing)
+ * names a commit that is gone. It fails rather than warns because such a pin stops `git gc`
+ * (and the `gc --auto` commits trigger), and a slot reset or leftover recovery only drops one
+ * when it reaches it. Read-only: it lists refs and names the `git update-ref -d` fix. */
+export async function checkTumwaterPins(root) {
+    const pins = await refsUnder(root, "refs/tumwater/");
+    const broken = [];
+    for (const pin of pins)
+        if (!(await commitExists(root, pin.sha)))
+            broken.push(pin);
+    if (broken.length === 0) {
+        return okOutcome(pins.length === 0 ? "no tumwater pins" : `${plural(pins.length, "tumwater pin")}, each naming a commit that exists`);
+    }
+    const n = broken.length;
+    return failOutcome(`${broken.map((p) => `${p.ref} names ${shortSha(p.sha)}, which is gone`).join("; ")} — git gc and a detached checkout abort on ${agree(n, "it", "them")}; \`git update-ref -d <ref>\` drops each (the work ${agree(n, "it", "they")} pinned is already lost)`);
+}
+/** The ok detail of the instance check: non-zero counts, plans first, each pluralized. */
+function claimCounts(claims) {
+    const bugs = claims.filter((c) => c.file === "BUGS.md").length;
+    const plans = claims.length - bugs;
+    const joined = joinParts([
+        plans > 0 ? plural(plans, "claimed plan") : "",
+        bugs > 0 ? plural(bugs, "claimed bug") : "",
+    ], ", ");
+    return joined || "no claimed plans or bugs";
+}
+/** Parallel work instances (plans/parallel-work-instances.md "Observability", part 6/7):
+ * each instance's claimed plan or bug and the two states an operator should act on — a claim
+ * whose key is no longer listed under its role's section (its instance is stuck on an entry
+ * that moved), and a claim idle for more than CLAIM_IDLE_MAX_MS (the scheduler releases
+ * these, so seeing one means the reader's view and the scheduler's disagree). An enabled
+ * multi-instance role configured with fewer pooled slots than maxConcurrent also warns:
+ * instances then contend for the same checkouts. A disabled role runs none of its
+ * configured instances, so it never contends and
+ * stays out of that warning.
+ * Read-only, like every check; `now` is the claim-age test seam, and an unreadable
+ * tumwater.json (config undefined) skips the walk — checkInit reports the config itself. */
+export function checkWorkInstances(root, config, now = Date.now()) {
+    if (!config)
+        return okOutcome("no readable tumwater.json — nothing to verify");
+    const claims = [];
+    for (const id of runnerIds(config)) {
+        const claim = loadRunnerState(root, id).claim;
+        if (claim)
+            claims.push({ id, title: claim.title, key: claim.key, at: claim.at, file: claim.file });
+    }
+    const gone = claims.filter((c) => !listedKeys(root, c.id).has(c.key));
+    const stale = claims.filter((c) => !gone.includes(c) && now - c.at > CLAIM_IDLE_MAX_MS);
+    const multi = Object.entries(config.roles)
+        .filter(([id, role]) => INSTANCE_ROLES.has(id) && role.enabled && (role.instances ?? 1) > 1)
+        .map(([id]) => id);
+    const slots = slotCount(config);
+    const smallPool = multi.length > 0 && slots < config.maxConcurrent;
+    if (gone.length === 0 && stale.length === 0 && !smallPool)
+        return okOutcome(claimCounts(claims));
+    const noun = (c) => (c.file === "BUGS.md" ? "bug" : "plan");
+    const parts = [
+        ...gone.map((c) => `claimed ${noun(c)} "${c.title}" (${c.id}) names an entry no longer listed`),
+        ...stale.map((c) => `claimed ${noun(c)} "${c.title}" (${c.id}) has been idle ${Math.round((now - c.at) / 3_600_000)}h`),
+    ];
+    if (smallPool)
+        parts.push(`${multi.join(", ")} run instances but worktreeSlots ${slots} is below maxConcurrent ${config.maxConcurrent}`);
+    return warnOutcome(parts.join("; "));
+}
+/** Provider concurrency caps (plans/provider-concurrency.md §8): audit the
+ * `maxConcurrentPerProvider` map against the seams the config can actually spawn — each role,
+ * the review gate, the resolver, the director, and each tier's fallback — so the operator
+ * sees whether every cap names a provider in use and whether any cap actually binds. Warns on
+ * a cap key no seam resolves to (a typo, or a provider not in use), a cap at or above
+ * `maxConcurrent` (which never binds — the team total is the real ceiling), and a capped
+ * team whose seams include one with no key (pi's default provider is unreadable, so that
+ * seam cannot be capped). A config with no per-provider map or no caps is fine, and so is a
+ * well-formed map; a broken config is reported by the init check. Takes `root` to read pi's
+ * own default model (piSettingsPath), the same seam the orchestrator's admission uses, so a
+ * bare no-model pi seam keys exactly as the team would key it. */
+export function checkProviderConcurrency(config, root) {
+    if (!config)
+        return okOutcome("no config to audit");
+    const caps = config.maxConcurrentPerProvider;
+    const capEntries = enforcedKeyCaps(caps);
+    if (capEntries.length === 0)
+        return okOutcome("no per-provider caps configured");
+    // The pi default model the orchestrator's admission seam carries: a bare no-model pi seam
+    // resolves its key only through it, so a team whose default is readable has keyed seams and
+    // only an unreadable default leaves a key-less seam to warn about.
+    const piDefault = readPiDefaultModel(piSettingsPath(piModelsPath()), root);
+    const seams = seamsInUse(config);
+    const keyed = [];
+    for (const s of seams) {
+        const k = seamConcurrencyKey(s, piDefault);
+        if (k !== undefined && !keyed.includes(k))
+            keyed.push(k);
+    }
+    const keySet = new Set(keyed);
+    const capKeys = capEntries.map(([k]) => k);
+    const warnings = [];
+    for (const key of capKeys) {
+        if (!keySet.has(key))
+            warnings.push(`cap key "${key}" resolves to no model job (a typo, or a provider not in use)`);
+    }
+    for (const [key, cap] of capEntries) {
+        if (cap >= config.maxConcurrent)
+            warnings.push(`cap "${key}" ${cap} is at or above maxConcurrent ${config.maxConcurrent} — it never binds`);
+    }
+    const capSet = new Set(capKeys);
+    const keyless = seams.filter((s) => seamConcurrencyKey(s, piDefault) === undefined);
+    if (capSet.size > 0 && keyless.length > 0) {
+        warnings.push(`${keyless.length} model job(s) resolve to no provider key (pi's default provider is unreadable, so they cannot be capped): ${keyless.map((s) => s.label).join(", ")}`);
+    }
+    if (warnings.length === 0)
+        return okOutcome(`${plural(capEntries.length, "provider cap")} — all model jobs resolve`);
+    return warnOutcome(warnings.join("; "));
+}

package/dist/src/doctor/doctor-holds.js

@@ -1,4 +1,3 @@
-// vocabulary: plain
 import { roleLabel } from "../roles/role-labels.js";
 import { pausedRoles, pausedRolesUntil, standingTeamPause } from "../team/team-state.js";
 import { MODEL_TIERS } from "../config/config-schema.js";
@@ -9,34 +8,37 @@ import { roleCapVerdicts } from "../gates/role-cap-gates.js";
 import { readMergeQueue } from "../merge/merge-queue.js";
 import { MAINTENANCE_DAILY_FLOOR } from "../gates/maintenance-quota.js";
 import { heldByMainRed, mainCheckForPoll, maintenanceQuotaForPoll } from "../status/status-polls.js";
-import { loadLoopState } from "../loop/loop-state.js";
-import { baseRoleOf, loopIds, loopPaused } from "../roles/loop-ids.js";
-import { ERROR_STREAK_BREAKER, ERROR_STREAK_WARN } from "../tick/tick-apply.js";
+import { loadRunnerState } from "../runner/runner-state-load.js";
+import { baseRoleOf, runnerIds, runnerPaused } from "../roles/runner-ids.js";
+import { ERROR_STREAK_BREAKER, ERROR_STREAK_WARN } from "../run/run-apply.js";
 import { squash } from "../text/text.js";
 import { usd } from "../text/format.js";
 import { parseQuietHours, quietHoursStatus, quietWindowEnd, roleQuietHold } from "../scheduling/quiet-hours.js";
-import { humanUntil } from "../text/datetime.js";
+import { DAY_MS, HOUR_MS, humanUntil } from "../text/datetime.js";
 import { budgetPhrase, pauseReasonSuffix, plural } from "../text/phrases.js";
-import { claudeUtilizationPct, claudeWindowEntries, claudeWindowLabel, readClaudeAccount } from "../agent/claude/claude-account.js";
-import { claudeWindowObservation } from "../team/team-polls.js";
+import { readAccount, utilizationPct, windowEntries, windowLabel } from "../agent/subscription-account.js";
+import { backendDisplayName } from "../agent/backend-traits.js";
+import { windowObservation } from "../team/team-polls.js";
 import { readEventsSince } from "../events/event-window.js";
 import { errorMessage } from "../text/text.js";
 import { okOutcome, warnOutcome } from "./outcome.js";
+import { redMainWarning, revertWarnings } from "../baseline/main-guardrails.js";
+import { loadQuarantine } from "../review/known-flakes.js";
 /** The standing-hold checks: `pause` (checkPause), `quiet hours` (checkQuietHours), and `budget`
- * (checkBudget), `claude window` (checkClaudeWindow), `maintenance quota`
- * (checkMaintenanceQuota), and `error streaks` (checkErrorStreaks), all read-only reports of
- * holds on ticking. */
-export function loadLoopStates(root, config) {
-    return loopIds(config)
-        .map((id) => loadLoopState(root, id))
+ * (checkBudget), the `claude window`/`codex window` rows (checkSubscriptionWindow),
+ * `maintenance quota` (checkMaintenanceQuota), and `error streaks` (checkErrorStreaks), all
+ * read-only reports of holds on running. */
+export function loadRunnerStates(root, config) {
+    return runnerIds(config)
+        .map((id) => loadRunnerState(root, id))
         .sort((a, b) => a.role.localeCompare(b.role));
 }
 /** The `pause` check: names a standing operator pause so doctor does not print "ready to run"
- * over a team that starts no tick. Read-only: it reads the same team and per-role pause
+ * over a team that starts no run. Read-only: it reads the same team and per-role pause
  * markers the scheduler reads (standingTeamPause, pausedRoles, pausedRolesUntil — all treat
  * an expired marker as absent), so its verdict cannot disagree with what the team does. A
- * team pause is a warn because it stops every loop; paused roles are ok because the operator
- * chose them by name and every other loop keeps ticking. `now` only drives the countdowns. */
+ * team pause is a warn because it stops every role; paused roles are ok because the operator
+ * chose them by name and every other role keeps running. `now` only drives the countdowns. */
 export function checkPause(root, now = Date.now()) {
     const team = standingTeamPause(root);
     const roles = pausedRoles(root);
@@ -53,10 +55,10 @@ export function checkPause(root, now = Date.now()) {
     return warnOutcome(`${head}${pauseReasonSuffix(team.reason)}${rolesPhrase ? `; ${rolesPhrase}` : ""}`);
 }
 /** The `quiet hours` check: names the team quiet window and any per-role windows holding now,
- * so doctor does not print "ready to run" over a team that starts no tick until the window
+ * so doctor does not print "ready to run" over a team that starts no run until the window
  * ends. Read-only: membership comes from the scheduler's own parser and predicate
  * (quietHoursStatus, roleQuietHold), so the verdict cannot disagree with the gate. A team
- * window holding is a warn because it stops every loop (the director is exempt); held per-role
+ * window holding is a warn because it stops every role (the director is exempt); held per-role
  * windows are ok because the operator chose those roles by name. A malformed value reads as
  * unset here — checkInit owns that error. `now` is injected so tests never read the clock. */
 export function checkQuietHours(config, now = new Date()) {
@@ -89,13 +91,13 @@ export function checkQuietHours(config, now = new Date()) {
  * own `maxDailyCostUsdPerRole` cap. Read-only and built on the scheduler's own definitions
  * (teamDailyCost, budgetReached, roleCapVerdicts), so the verdict cannot disagree with the gate.
  * `published` is the running scheduler's figure (it includes runs charged since the last save)
- * and wins over the persisted sum, as in status. A reached team cap is a warn because role loops
+ * and wins over the persisted sum, as in status. A reached team cap is a warn because roles
  * run on their tier fallback or hold until local midnight (the `fallback` row says which);
  * per-role caps reached alone are ok because the operator chose them by name. */
 export function checkBudget(root, config, published, now = Date.now()) {
     if (config === undefined)
         return okOutcome("not checked — tumwater.json unreadable");
-    const states = loadLoopStates(root, config);
+    const states = loadRunnerStates(root, config);
     const spent = published?.spentUsd ?? teamDailyCost(states, now);
     const caps = config.maxDailyCostUsdPerRole;
     const capped = [...roleCapVerdicts(states.map((s) => ({ role: s.role, state: s })), caps, now)]
@@ -116,63 +118,65 @@ export function checkBudget(root, config, published, now = Date.now()) {
     return okOutcome(`${phrase} today${cappedPhrase}`);
 }
 /** The `maintenance quota` check: rolling-24 h maintenance merges against the allowance
- * (`maintenancePerWorkMerge` per work merge + the daily floor), naming the maintenance loops held
+ * (`maintenancePerWorkMerge` per work merge + the daily floor), naming the maintenance roles held
  * when it is full. Read-only and built on the same verdict the scheduler and status use
- * (maintenanceQuotaForPoll). Only a live team's running loops and merge queue count as in flight,
+ * (maintenanceQuotaForPoll). Only a live team's running roles and merge queue count as in flight,
  * since a dead team's leftover `running` flag and queue drain nothing. A held allowance is a warn
  * because it stops a whole tier. */
 export function checkMaintenanceQuota(root, config, aliveTeam, now = Date.now()) {
     if (config === undefined)
         return okOutcome("not checked — tumwater.json unreadable");
-    const loops = loadLoopStates(root, config).map((s) => ({ role: s.role, running: aliveTeam && s.running === true }));
+    const runners = loadRunnerStates(root, config).map((s) => ({ role: s.role, running: aliveTeam && s.running === true }));
     const queued = aliveTeam ? new Set(readMergeQueue(root).entries.map((e) => e.role)) : new Set();
-    const { allowance, used, held } = maintenanceQuotaForPoll(root, config.maintenancePerWorkMerge, loops, queued, now);
+    const { allowance, used, held } = maintenanceQuotaForPoll(root, config.maintenancePerWorkMerge, runners, queued, now);
     if (held.length > 0) {
         return warnOutcome(`allowance reached — ${used}/${allowance} maintenance merges in the last 24h; held until the window rolls under: ${[...held].sort().join(", ")}`);
     }
     const base = `${used}/${allowance} maintenance merges in the last 24h (${config.maintenancePerWorkMerge} per work merge + ${MAINTENANCE_DAILY_FLOOR} floor)`;
-    const anyMaintenance = loops.some((l) => commitTier(l.role) === "maintenance");
+    const anyMaintenance = runners.some((l) => commitTier(l.role) === "maintenance");
     return okOutcome(anyMaintenance ? base : `${base}; no maintenance role enabled`);
 }
-/** The `claude window` check: whether a rejected subscription window holds the subscription claude
- * seams. Read-only and built on claudeWindowObservation, the verdict the scheduler's team hold
- * uses, so it cannot disagree with the hold. A hold is a warn because it stops every subscription
- * claude seam until the window resets; otherwise the most constrained window's state is reported
- * as ok (a rejected window spending overage reads `rejected` here: overage spends, it does not
- * hold). API-billed claude never holds. */
-export function checkClaudeWindow(root, now = Date.now(), chain, config) {
-    const account = readClaudeAccount(root, now);
+/** The `<backend> window` check, one row per subscription backend in use (claude and codex):
+ * whether a rejected subscription window holds the subscription `<backend>` seams. Read-only and
+ * built on windowObservation (the backend's own), the verdict the scheduler's team hold uses, so
+ * it cannot disagree with the hold. A hold is a warn because it stops every subscription seam on
+ * that backend until the window resets; otherwise the most constrained window's state is
+ * reported as ok (a rejected window spending overage reads `rejected` here: overage spends, it
+ * does not hold). Returns null (omitting the row from doctor) when the backend has no
+ * subscription account — the same omit-a-row pattern as `pi extensions` — because an absent or
+ * API-billed account never holds. */
+export function checkSubscriptionWindow(backend, root, now = Date.now(), chain, config) {
+    const account = readAccount(root, backend, now);
     if (account?.billing !== "subscription")
-        return okOutcome("no subscription window reported");
-    const held = claudeWindowObservation(account, now);
+        return null;
+    const held = windowObservation(backend, account, now);
     if (held?.kind === "subscription-window" && held.windowType !== undefined && held.resetsAt !== undefined) {
-        const label = claudeWindowLabel(held.windowType);
+        const label = windowLabel(held.windowType);
         const resets = humanUntil(held.resetsAt, now);
-        const moved = MODEL_TIERS.filter((t) => {
-            const reason = chain?.[t]?.reason;
-            return reason === "window" || reason === "reserve";
-        });
+        const moved = MODEL_TIERS.filter((t) => (chain?.[t]?.reason === "window" || chain?.[t]?.reason === "reserve") &&
+            config !== undefined &&
+            withModelOverrides(config, t, {}).agent === backend);
         if (moved.length > 0) {
-            const waiting = MODEL_TIERS.filter((t) => !chain?.[t] && config !== undefined && withModelOverrides(config, t, {}).agent === "claude");
+            const waiting = MODEL_TIERS.filter((t) => !chain?.[t] && config !== undefined && withModelOverrides(config, t, {}).agent === backend);
             return warnOutcome(`${label} window rejected — ${moved.join(", ")} moved to fallbacks until it resets in ${resets}${waiting.length > 0 ? `; ${waiting.join(", ")} wait` : ""}`);
         }
-        return warnOutcome(`${label} window rejected — subscription claude model jobs wait until it resets in ${resets}`);
+        return warnOutcome(`${label} window rejected — subscription ${backendDisplayName(backend)} model jobs wait until it resets in ${resets}`);
     }
-    const top = claudeWindowEntries(account)[0];
+    const top = windowEntries(account)[0];
     if (top === undefined)
         return okOutcome("subscription; no window reported yet");
-    const pct = typeof top.utilization === "number" ? ` (${claudeUtilizationPct(top.utilization)}% used)` : "";
-    const label = claudeWindowLabel(top.type);
+    const pct = typeof top.utilization === "number" ? ` (${utilizationPct(top.utilization)}% used)` : "";
+    const label = windowLabel(top.type);
     return okOutcome(top.status === "allowed_warning" ? `${label} window nearing its limit${pct}` : `${label} window ${top.status}${pct}`);
 }
 /** How far back the `provider hold` check reads for hold/resume crossings. The longest a
  * hold can stay is HOLD_CAP_MS (team-hold's 15-minute cap that escalation doubles against),
  * so a window a comfortable multiple of that covers any standing hold's crossing event plus
  * its resume — an older resumed event must not be missed or a re-opened provider reads held. */
-const HOLD_CHECK_WINDOW_MS = 2 * 3600_000;
+const HOLD_CHECK_WINDOW_MS = 2 * HOUR_MS;
 /** The `provider hold` check: names each provider whose team-wide failure hold
  * (src/team/team-hold.ts, polled by pollTeamHold) stands now — a 429/backend-failure storm
- * that stops the roles on that provider from starting new ticks until it re-opens. Read-only:
+ * that stops the roles on that provider from starting new runs until it re-opens. Read-only:
  * it infers the standing set from the event log's own `rate_limit_hold`/`rate_limit_resumed`
  * crossings, the events pollTeamHold writes, so it cannot disagree with the hold. A standing
  * hold is a warn (as with the `claude window` row) because it stops every role on that
@@ -210,16 +214,16 @@ export function checkProviderHolds(root, now = Date.now()) {
     });
     return warnOutcome(lines.join("; "));
 }
-/** The `error streaks` check: names each enabled loop whose `consecutiveErrors` is at or past
+/** The `error streaks` check: names each enabled role whose `consecutiveErrors` is at or past
  * ERROR_STREAK_WARN, with its streak and last error, and whether it is paused. Read-only; it reads
  * the same `consecutiveErrors` the error-streak breaker and `teamAlerts` read, and the same
  * `pausedRoles` marker the scheduler reads. The marker does not record who wrote it, so a paused
- * loop at ERROR_STREAK_BREAKER's bar is described, not attributed to the breaker. A warn, not a
- * fail: a failing loop does not stop the team from running. */
+ * role at ERROR_STREAK_BREAKER's bar is described, not attributed to the breaker. A warn, not a
+ * fail: a failing role does not stop the team from running. */
 export function checkErrorStreaks(root, config) {
     if (config === undefined)
         return okOutcome("not checked — tumwater.json unreadable");
-    const failing = loadLoopStates(root, config)
+    const failing = loadRunnerStates(root, config)
         .filter((s) => (s.consecutiveErrors ?? 0) >= ERROR_STREAK_WARN);
     if (failing.length === 0)
         return okOutcome("no role failing repeatedly");
@@ -227,7 +231,7 @@ export function checkErrorStreaks(root, config) {
     const phrases = failing.map((s) => {
         const n = s.consecutiveErrors ?? 0;
         let phrase = `${roleLabel(s.role)}: ${n} failed runs`;
-        if (loopPaused(paused, s.role)) {
+        if (runnerPaused(paused, s.role)) {
             const key = paused.has(s.role) ? s.role : paused.has(baseRoleOf(s.role)) ? baseRoleOf(s.role) : `${s.role}-1`;
             phrase +=
                 n >= ERROR_STREAK_BREAKER
@@ -240,23 +244,35 @@ export function checkErrorStreaks(root, config) {
     });
     return warnOutcome(`failing run after run: ${phrases.join("; ")}`);
 }
-/** The `main red` check: names the code loops the red-main gate (mainRedGate) holds, ending
- * each blocked tick `main_red` while main's suite is red. Read-only: it asks heldByMainRed,
- * the predicate the status phase's `main red` label reads, so the two cannot disagree. A warn,
- * not a fail: a red main does not stop the team from running. */
-export function checkMainRed(root, config) {
+/** The `main red` check: names the code roles the red-main gate (mainRedGate) holds, ending
+ * each blocked run `main_red` while main's suite is red, and warns when main has been red over
+ * RED_MAIN_WARN_MS even with no role held (the red-main gate's guardrail, G1). Read-only: it
+ * asks heldByMainRed, the predicate the status phase's `main red` label reads, so the two
+ * cannot disagree on the held roles, and redMainWarning over the event trail, so both phrases
+ * can appear in one row. A warn, not a fail: a red main does not stop the team from running.
+ * `now` is injected so a test can pin a red's age. */
+export function checkMainRed(root, config, now = Date.now()) {
     if (config === undefined)
         return okOutcome("not checked — tumwater.json unreadable");
     const mainCheck = mainCheckForPoll(root, config);
-    const ids = loopIds(config);
-    const held = loadLoopStates(root, config)
+    const ids = runnerIds(config);
+    const held = loadRunnerStates(root, config)
         .filter((s) => heldByMainRed(s, mainCheck));
-    if (held.length === 0)
+    const phrases = [];
+    if (held.length > 0) {
+        const err = held.find((s) => s.lastError)?.lastError;
+        const cause = err ? ` — ${squash(err, 160)}` : "";
+        const healer = ids.some((id) => baseRoleOf(id) === "bugfix")
+            ? "; the bugfix role is exempt and repairs it"
+            : "; no bugfix role is enabled to repair it — fix main by hand or enable bugfix";
+        phrases.push(`main's check is failing; ${plural(held.length, "code role")} waiting: ${held.map((s) => roleLabel(s.role)).join(", ")}${cause}${healer}`);
+    }
+    const events = readEventsSince(root, DAY_MS).events;
+    const red = redMainWarning(events, now);
+    if (red !== null)
+        phrases.push(red);
+    phrases.push(...revertWarnings(events, loadQuarantine(root), now));
+    if (phrases.length === 0)
         return okOutcome("no role waiting on main's check");
-    const err = held.find((s) => s.lastError)?.lastError;
-    const cause = err ? ` — ${squash(err, 160)}` : "";
-    const healer = ids.some((id) => baseRoleOf(id) === "bugfix")
-        ? "; the bugfix role is exempt and repairs it"
-        : "; no bugfix role is enabled to repair it — fix main by hand or enable bugfix";
-    return warnOutcome(`main's check is failing; ${plural(held.length, "code role")} waiting: ${held.map((s) => roleLabel(s.role)).join(", ")}${cause}${healer}`);
+    return warnOutcome(phrases.join("; "));
 }