tumwater 0.1.2 → 0.2.0

package/dist/src/doctor/doctor-holds.js

@@ -1,50 +1,66 @@
-import { pausedRoles, pausedRolesUntil, standingFleetPause } from "../fleet/fleet-state.js";
+// vocabulary: plain
+import { roleLabel } from "../roles/role-labels.js";
+import { pausedRoles, pausedRolesUntil, standingTeamPause } from "../team/team-state.js";
+import { MODEL_TIERS } from "../config/config-schema.js";
+import { withModelOverrides } from "../config/config-views.js";
 import { commitTier, DIRECTOR_ROLE } from "../roles/roles.js";
-import { budgetReached, budgetWarning, fleetDailyCost } from "../budget/budget.js";
+import { budgetReached, budgetWarning, teamDailyCost } from "../budget/budget.js";
 import { roleCapVerdicts } from "../gates/role-cap-gates.js";
-import { readLandingQueue } from "../landing/landing-queue.js";
+import { readMergeQueue } from "../merge/merge-queue.js";
 import { MAINTENANCE_DAILY_FLOOR } from "../gates/maintenance-quota.js";
-import { maintenanceQuotaForPoll } from "../status/status-polls.js";
+import { heldByMainRed, mainCheckForPoll, maintenanceQuotaForPoll } from "../status/status-polls.js";
 import { loadLoopState } from "../loop/loop-state.js";
-import { loopIds } from "../roles/loop-ids.js";
+import { baseRoleOf, loopIds, loopPaused } from "../roles/loop-ids.js";
+import { ERROR_STREAK_BREAKER, ERROR_STREAK_WARN } from "../tick/tick-apply.js";
+import { squash } from "../text/text.js";
 import { usd } from "../text/format.js";
 import { parseQuietHours, quietHoursStatus, quietWindowEnd, roleQuietHold } from "../scheduling/quiet-hours.js";
-import { humanSeconds, secondsUntil } from "../text/datetime.js";
-import { budgetPhrase, pauseReasonSuffix } from "../text/phrases.js";
+import { humanUntil } from "../text/datetime.js";
+import { budgetPhrase, pauseReasonSuffix, plural } from "../text/phrases.js";
+import { claudeUtilizationPct, claudeWindowEntries, claudeWindowLabel, readClaudeAccount } from "../agent/claude/claude-account.js";
+import { claudeWindowObservation } from "../team/team-polls.js";
+import { readEventsSince } from "../events/event-window.js";
+import { errorMessage } from "../text/text.js";
 import { okOutcome, warnOutcome } from "./outcome.js";
 /** The standing-hold checks: `pause` (checkPause), `quiet hours` (checkQuietHours), and `budget`
- * (checkBudget), and `maintenance quota` (checkMaintenanceQuota), all read-only reports of holds on ticking.
- *
- * The `pause` check: names a standing operator pause so doctor does not print "ready to run"
- * over a fleet that starts no tick. Read-only: it reads the same fleet and per-role pause
- * markers the scheduler reads (standingFleetPause, pausedRoles, pausedRolesUntil — all treat
- * an expired marker as absent), so its verdict cannot disagree with what the fleet does. A
- * fleet pause is a warn because it stops every loop; paused roles are ok because the operator
+ * (checkBudget), `claude window` (checkClaudeWindow), `maintenance quota`
+ * (checkMaintenanceQuota), and `error streaks` (checkErrorStreaks), all read-only reports of
+ * holds on ticking. */
+export function loadLoopStates(root, config) {
+    return loopIds(config)
+        .map((id) => loadLoopState(root, id))
+        .sort((a, b) => a.role.localeCompare(b.role));
+}
+/** The `pause` check: names a standing operator pause so doctor does not print "ready to run"
+ * over a team that starts no tick. Read-only: it reads the same team and per-role pause
+ * markers the scheduler reads (standingTeamPause, pausedRoles, pausedRolesUntil — all treat
+ * an expired marker as absent), so its verdict cannot disagree with what the team does. A
+ * team pause is a warn because it stops every loop; paused roles are ok because the operator
  * chose them by name and every other loop keeps ticking. `now` only drives the countdowns. */
 export function checkPause(root, now = Date.now()) {
-    const fleet = standingFleetPause(root);
+    const team = standingTeamPause(root);
     const roles = pausedRoles(root);
     let rolesPhrase = "";
     if (roles.length > 0) {
         const until = pausedRolesUntil(root);
-        rolesPhrase = `paused loops: ${roles.join(", ")}${until !== undefined ? ` (auto-resume in ${humanSeconds(secondsUntil(until, now))})` : ""}`;
+        rolesPhrase = `paused roles: ${roles.join(", ")}${until !== undefined ? ` (auto-resume in ${humanUntil(until, now)})` : ""}`;
     }
-    if (!fleet)
+    if (!team)
         return rolesPhrase ? okOutcome(rolesPhrase) : okOutcome("not paused");
-    const head = fleet.until !== undefined
-        ? `fleet paused — auto-resumes in ${humanSeconds(secondsUntil(fleet.until, now))}`
-        : "fleet paused — no loop starts a tick until `tumwater resume`";
-    return warnOutcome(`${head}${pauseReasonSuffix(fleet.reason)}${rolesPhrase ? `; ${rolesPhrase}` : ""}`);
+    const head = team.until !== undefined
+        ? `team paused — auto-resumes in ${humanUntil(team.until, now)}`
+        : "team paused — no role starts a run until `tumwater resume`";
+    return warnOutcome(`${head}${pauseReasonSuffix(team.reason)}${rolesPhrase ? `; ${rolesPhrase}` : ""}`);
 }
-/** The `quiet hours` check: names the fleet quiet window and any per-role windows holding now,
- * so doctor does not print "ready to run" over a fleet that starts no tick until the window
+/** The `quiet hours` check: names the team quiet window and any per-role windows holding now,
+ * so doctor does not print "ready to run" over a team that starts no tick until the window
  * ends. Read-only: membership comes from the scheduler's own parser and predicate
- * (quietHoursStatus, roleQuietHold), so the verdict cannot disagree with the gate. A fleet
+ * (quietHoursStatus, roleQuietHold), so the verdict cannot disagree with the gate. A team
  * window holding is a warn because it stops every loop (the director is exempt); held per-role
  * windows are ok because the operator chose those roles by name. A malformed value reads as
  * unset here — checkInit owns that error. `now` is injected so tests never read the clock. */
 export function checkQuietHours(config, now = new Date()) {
-    const fleet = quietHoursStatus(config?.quietHours, now);
+    const team = quietHoursStatus(config?.quietHours, now);
     const perRole = config?.quietHoursPerRole;
     const roleKeys = Object.keys(perRole ?? {}).filter((k) => k !== DIRECTOR_ROLE);
     const held = roleKeys.filter((k) => roleQuietHold(perRole, k, now)).sort();
@@ -53,13 +69,13 @@ export function checkQuietHours(config, now = new Date()) {
         return parsed.ok && parsed.window !== null;
     });
     const phrases = [];
-    if (fleet.window !== null) {
-        phrases.push(fleet.inWindow
-            ? `quiet now (${fleet.window}) — no role loop starts a tick until ${quietWindowEnd(fleet.window)}; the director is exempt`
-            : `${fleet.window} (not in the window now)`);
+    if (team.window !== null) {
+        phrases.push(team.inWindow
+            ? `quiet now (${team.window}) — no role starts a run until ${quietWindowEnd(team.window)}; the director is exempt`
+            : `${team.window} (not in the window now)`);
     }
     if (held.length > 0) {
-        phrases.push(`quiet loops: ${held.map((k) => `${k} until ${quietWindowEnd(String(perRole?.[k]))}`).join(", ")}`);
+        phrases.push(`quiet roles: ${held.map((k) => `${k} until ${quietWindowEnd(String(perRole?.[k]))}`).join(", ")}`);
     }
     else if (anyRoleWindow) {
         phrases.push("per-role windows set, none holding now");
@@ -67,56 +83,180 @@ export function checkQuietHours(config, now = new Date()) {
     if (phrases.length === 0)
         return okOutcome("none configured");
     const detail = phrases.join("; ");
-    return fleet.inWindow ? warnOutcome(detail) : okOutcome(detail);
+    return team.inWindow ? warnOutcome(detail) : okOutcome(detail);
 }
-/** The `budget` check: today's fleet spend against `maxDailyCostUsd`, plus base roles over their
+/** The `budget` check: today's team spend against `maxDailyCostUsd`, plus base roles over their
  * own `maxDailyCostUsdPerRole` cap. Read-only and built on the scheduler's own definitions
- * (fleetDailyCost, budgetReached, roleCapVerdicts), so the verdict cannot disagree with the gate.
+ * (teamDailyCost, budgetReached, roleCapVerdicts), so the verdict cannot disagree with the gate.
  * `published` is the running scheduler's figure (it includes runs charged since the last save)
- * and wins over the persisted sum, as in status. A reached fleet cap is a warn because role loops
+ * and wins over the persisted sum, as in status. A reached team cap is a warn because role loops
  * run on their tier fallback or hold until local midnight (the `fallback` row says which);
  * per-role caps reached alone are ok because the operator chose them by name. */
 export function checkBudget(root, config, published, now = Date.now()) {
     if (config === undefined)
         return okOutcome("not checked — tumwater.json unreadable");
-    const states = loopIds(config).map((id) => loadLoopState(root, id));
-    const spent = published?.spentUsd ?? fleetDailyCost(states, now);
+    const states = loadLoopStates(root, config);
+    const spent = published?.spentUsd ?? teamDailyCost(states, now);
     const caps = config.maxDailyCostUsdPerRole;
     const capped = [...roleCapVerdicts(states.map((s) => ({ role: s.role, state: s })), caps, now)]
         .filter(([, v]) => v.over)
         .sort(([a], [b]) => a.localeCompare(b))
         .map(([base, v]) => `${base} (${budgetPhrase(v.spentUsd, caps?.[base])})`);
-    const cappedPhrase = capped.length > 0 ? `; capped loops: ${capped.join(", ")}` : "";
+    const cappedPhrase = capped.length > 0 ? `; capped roles: ${capped.join(", ")}` : "";
     const cap = config.maxDailyCostUsd;
     if (!(cap > 0))
         return okOutcome(`no daily cap; ${usd(spent)} spent today${cappedPhrase}`);
     const view = { spentUsd: spent, capUsd: cap };
     const phrase = budgetPhrase(spent, cap);
     if (budgetReached(view)) {
-        return warnOutcome(`daily cap reached — ${phrase} today; role loops run on their tier fallback or hold until local midnight (see the fallback row)${cappedPhrase}`);
+        return warnOutcome(`daily cap reached — ${phrase} today; roles run on their over-cap model or hold until local midnight (see the over-cap model row)${cappedPhrase}`);
     }
     if (budgetWarning(view))
         return okOutcome(`${phrase} today — past 80% of the cap${cappedPhrase}`);
     return okOutcome(`${phrase} today${cappedPhrase}`);
 }
-/** The `maintenance quota` check: rolling-24 h maintenance landings against the allowance
- * (`maintenancePerWorkLanding` per work landing + the daily floor), naming the maintenance loops
- * held when it is full. Read-only and built on the same verdict the scheduler and status use
- * (maintenanceQuotaForPoll). Only a live fleet's running loops and landing queue count as in
- * flight, since a dead fleet's leftover `running` flag and queue drain nothing. A held allowance
- * is a warn because it stops a whole tier. */
-export function checkMaintenanceQuota(root, config, aliveFleet, now = Date.now()) {
+/** The `maintenance quota` check: rolling-24 h maintenance merges against the allowance
+ * (`maintenancePerWorkMerge` per work merge + the daily floor), naming the maintenance loops held
+ * when it is full. Read-only and built on the same verdict the scheduler and status use
+ * (maintenanceQuotaForPoll). Only a live team's running loops and merge queue count as in flight,
+ * since a dead team's leftover `running` flag and queue drain nothing. A held allowance is a warn
+ * because it stops a whole tier. */
+export function checkMaintenanceQuota(root, config, aliveTeam, now = Date.now()) {
     if (config === undefined)
         return okOutcome("not checked — tumwater.json unreadable");
-    const loops = loopIds(config)
-        .map((id) => loadLoopState(root, id))
-        .map((s) => ({ role: s.role, running: aliveFleet && s.running === true }));
-    const queued = aliveFleet ? new Set(readLandingQueue(root).entries.map((e) => e.role)) : new Set();
-    const { allowance, used, held } = maintenanceQuotaForPoll(root, config.maintenancePerWorkLanding, loops, queued, now);
+    const loops = loadLoopStates(root, config).map((s) => ({ role: s.role, running: aliveTeam && s.running === true }));
+    const queued = aliveTeam ? new Set(readMergeQueue(root).entries.map((e) => e.role)) : new Set();
+    const { allowance, used, held } = maintenanceQuotaForPoll(root, config.maintenancePerWorkMerge, loops, queued, now);
     if (held.length > 0) {
-        return warnOutcome(`allowance reached — ${used}/${allowance} maintenance landings in the last 24h; held until the window rolls under: ${[...held].sort().join(", ")}`);
+        return warnOutcome(`allowance reached — ${used}/${allowance} maintenance merges in the last 24h; held until the window rolls under: ${[...held].sort().join(", ")}`);
     }
-    const base = `${used}/${allowance} maintenance landings in the last 24h (${config.maintenancePerWorkLanding} per work landing + ${MAINTENANCE_DAILY_FLOOR} floor)`;
+    const base = `${used}/${allowance} maintenance merges in the last 24h (${config.maintenancePerWorkMerge} per work merge + ${MAINTENANCE_DAILY_FLOOR} floor)`;
     const anyMaintenance = loops.some((l) => commitTier(l.role) === "maintenance");
-    return okOutcome(anyMaintenance ? base : `${base}; no maintenance loop enabled`);
+    return okOutcome(anyMaintenance ? base : `${base}; no maintenance role enabled`);
+}
+/** The `claude window` check: whether a rejected subscription window holds the subscription claude
+ * seams. Read-only and built on claudeWindowObservation, the verdict the scheduler's team hold
+ * uses, so it cannot disagree with the hold. A hold is a warn because it stops every subscription
+ * claude seam until the window resets; otherwise the most constrained window's state is reported
+ * as ok (a rejected window spending overage reads `rejected` here: overage spends, it does not
+ * hold). API-billed claude never holds. */
+export function checkClaudeWindow(root, now = Date.now(), chain, config) {
+    const account = readClaudeAccount(root, now);
+    if (account?.billing !== "subscription")
+        return okOutcome("no subscription window reported");
+    const held = claudeWindowObservation(account, now);
+    if (held?.kind === "subscription-window" && held.windowType !== undefined && held.resetsAt !== undefined) {
+        const label = claudeWindowLabel(held.windowType);
+        const resets = humanUntil(held.resetsAt, now);
+        const moved = MODEL_TIERS.filter((t) => {
+            const reason = chain?.[t]?.reason;
+            return reason === "window" || reason === "reserve";
+        });
+        if (moved.length > 0) {
+            const waiting = MODEL_TIERS.filter((t) => !chain?.[t] && config !== undefined && withModelOverrides(config, t, {}).agent === "claude");
+            return warnOutcome(`${label} window rejected — ${moved.join(", ")} moved to fallbacks until it resets in ${resets}${waiting.length > 0 ? `; ${waiting.join(", ")} wait` : ""}`);
+        }
+        return warnOutcome(`${label} window rejected — subscription claude model jobs wait until it resets in ${resets}`);
+    }
+    const top = claudeWindowEntries(account)[0];
+    if (top === undefined)
+        return okOutcome("subscription; no window reported yet");
+    const pct = typeof top.utilization === "number" ? ` (${claudeUtilizationPct(top.utilization)}% used)` : "";
+    const label = claudeWindowLabel(top.type);
+    return okOutcome(top.status === "allowed_warning" ? `${label} window nearing its limit${pct}` : `${label} window ${top.status}${pct}`);
+}
+/** How far back the `provider hold` check reads for hold/resume crossings. The longest a
+ * hold can stay is HOLD_CAP_MS (team-hold's 15-minute cap that escalation doubles against),
+ * so a window a comfortable multiple of that covers any standing hold's crossing event plus
+ * its resume — an older resumed event must not be missed or a re-opened provider reads held. */
+const HOLD_CHECK_WINDOW_MS = 2 * 3600_000;
+/** The `provider hold` check: names each provider whose team-wide failure hold
+ * (src/team/team-hold.ts, polled by pollTeamHold) stands now — a 429/backend-failure storm
+ * that stops the roles on that provider from starting new ticks until it re-opens. Read-only:
+ * it infers the standing set from the event log's own `rate_limit_hold`/`rate_limit_resumed`
+ * crossings, the events pollTeamHold writes, so it cannot disagree with the hold. A standing
+ * hold is a warn (as with the `claude window` row) because it stops every role on that
+ * provider; `now` is injected so a test can pin a hold standing before its deadline and
+ * re-opened after. The ended hold's kind rides the resumed event (see pollTeamHold), and any
+ * resumed event ends that provider's current hold whatever its kind. */
+export function checkProviderHolds(root, now = Date.now()) {
+    let events;
+    try {
+        events = readEventsSince(root, HOLD_CHECK_WINDOW_MS).events;
+    }
+    catch (err) {
+        return warnOutcome(`could not read the event log for provider holds: ${errorMessage(err)}`);
+    }
+    const standing = new Map();
+    for (const ev of events) {
+        if (ev.type === "rate_limit_hold") {
+            const provider = typeof ev.provider === "string" ? ev.provider : undefined;
+            const end = (typeof ev.ts === "number" ? ev.ts : 0) + (typeof ev.holdMs === "number" ? ev.holdMs : 0);
+            standing.set(provider, { end, kind: ev.kind, roles: ev.roles });
+        }
+        else if (ev.type === "rate_limit_resumed") {
+            standing.delete(typeof ev.provider === "string" ? ev.provider : undefined);
+        }
+    }
+    const held = [...standing.entries()]
+        .filter(([, h]) => h.end > now)
+        .sort(([a], [b]) => (a ?? "").localeCompare(b ?? ""));
+    if (held.length === 0)
+        return okOutcome("no provider holds");
+    const lines = held.map(([provider, h]) => {
+        const name = provider ?? "pi's default backend";
+        const roles = Array.isArray(h.roles) && h.roles.length > 0 ? `; roles: ${h.roles.join(", ")}` : "";
+        return `${name} (${h.kind}) holds until it re-opens in ${humanUntil(h.end, now)}${roles}`;
+    });
+    return warnOutcome(lines.join("; "));
+}
+/** The `error streaks` check: names each enabled loop whose `consecutiveErrors` is at or past
+ * ERROR_STREAK_WARN, with its streak and last error, and whether it is paused. Read-only; it reads
+ * the same `consecutiveErrors` the error-streak breaker and `teamAlerts` read, and the same
+ * `pausedRoles` marker the scheduler reads. The marker does not record who wrote it, so a paused
+ * loop at ERROR_STREAK_BREAKER's bar is described, not attributed to the breaker. A warn, not a
+ * fail: a failing loop does not stop the team from running. */
+export function checkErrorStreaks(root, config) {
+    if (config === undefined)
+        return okOutcome("not checked — tumwater.json unreadable");
+    const failing = loadLoopStates(root, config)
+        .filter((s) => (s.consecutiveErrors ?? 0) >= ERROR_STREAK_WARN);
+    if (failing.length === 0)
+        return okOutcome("no role failing repeatedly");
+    const paused = new Set(pausedRoles(root));
+    const phrases = failing.map((s) => {
+        const n = s.consecutiveErrors ?? 0;
+        let phrase = `${roleLabel(s.role)}: ${n} failed runs`;
+        if (loopPaused(paused, s.role)) {
+            const key = paused.has(s.role) ? s.role : paused.has(baseRoleOf(s.role)) ? baseRoleOf(s.role) : `${s.role}-1`;
+            phrase +=
+                n >= ERROR_STREAK_BREAKER
+                    ? `, paused at the repeated-failure pause's bar — \`tumwater resume --role ${key}\` once the cause is fixed`
+                    : ", paused";
+        }
+        if (s.lastError)
+            phrase += ` — last error: ${squash(s.lastError, 120)}`;
+        return phrase;
+    });
+    return warnOutcome(`failing run after run: ${phrases.join("; ")}`);
+}
+/** The `main red` check: names the code loops the red-main gate (mainRedGate) holds, ending
+ * each blocked tick `main_red` while main's suite is red. Read-only: it asks heldByMainRed,
+ * the predicate the status phase's `main red` label reads, so the two cannot disagree. A warn,
+ * not a fail: a red main does not stop the team from running. */
+export function checkMainRed(root, config) {
+    if (config === undefined)
+        return okOutcome("not checked — tumwater.json unreadable");
+    const mainCheck = mainCheckForPoll(root, config);
+    const ids = loopIds(config);
+    const held = loadLoopStates(root, config)
+        .filter((s) => heldByMainRed(s, mainCheck));
+    if (held.length === 0)
+        return okOutcome("no role waiting on main's check");
+    const err = held.find((s) => s.lastError)?.lastError;
+    const cause = err ? ` — ${squash(err, 160)}` : "";
+    const healer = ids.some((id) => baseRoleOf(id) === "bugfix")
+        ? "; the bugfix role is exempt and repairs it"
+        : "; no bugfix role is enabled to repair it — fix main by hand or enable bugfix";
+    return warnOutcome(`main's check is failing; ${plural(held.length, "code role")} waiting: ${held.map((s) => roleLabel(s.role)).join(", ")}${cause}${healer}`);
 }

package/dist/src/doctor/doctor-install.js

@@ -0,0 +1,30 @@
+/** The doctor's `root install` row: whether the root checkout's node_modules still matches
+ * package-lock.json's direct pins. A failed post-merge re-sync (syncRootInstall) leaves the root
+ * behind, and every check then installs into its own tree until a later merge retries. */
+import fs from "node:fs";
+import path from "node:path";
+import { installDrift } from "../build/dep-install.js";
+import { readEventsSince } from "../events/event-window.js";
+import { plural, shortSpanPhrase } from "../text/phrases.js";
+import { errorMessage, textOr } from "../text/text.js";
+import { okOutcome, warnOutcome } from "./outcome.js";
+/** How far back a failed `dep_install` event may sit and still explain the drift. */
+const DEP_INSTALL_CHECK_WINDOW_MS = 24 * 3600_000;
+export function checkRootInstall(root, now = Date.now()) {
+    if (!fs.existsSync(path.join(root, "package-lock.json")))
+        return okOutcome("no package-lock.json — nothing pinned");
+    const drift = installDrift(root);
+    if (drift.length === 0)
+        return okOutcome("node_modules matches package-lock.json");
+    let detail = `${plural(drift.length, "direct dependency", "direct dependencies")} not installed as package-lock.json pins (${drift.join(", ")}) — run npm install in ${root}; until then each check installs into its own tree`;
+    try {
+        const installs = readEventsSince(root, DEP_INSTALL_CHECK_WINDOW_MS).events.filter((ev) => ev.type === "dep_install");
+        const ev = installs[installs.length - 1];
+        if (ev?.status === "failed")
+            detail += `; the last re-sync failed ${shortSpanPhrase(now - ev.ts)} ago: ${textOr(ev.error)}`;
+    }
+    catch (err) {
+        detail += `; could not read the event log: ${errorMessage(err)}`;
+    }
+    return warnOutcome(detail);
+}

package/dist/src/doctor/doctor-launch-services.js

@@ -1,10 +1,10 @@
 /** Doctor's `mach ports` line: a warning at or past launchservicesd's port threshold, a warning
- * when the count cannot be read on macOS, ok otherwise. Never a failure — the fleet runs fine on
+ * when the count cannot be read on macOS, ok otherwise. Never a failure — the team runs fine on
  * a leaking Mac; it is the Mac that does not. Split from process/launch-services.ts (2026-10-07,
  * organize) so the doctor's environment checks live together (doctor-orphans.ts reads the process
  * table the same way) and the process layer no longer imports the doctor's report contract. The
  * launchservicesd facts this reads — the thresholds and the warning text — stay in
- * src/process/launch-services.ts, shared with the running fleet's watch. `platform` is a
+ * src/process/launch-services.ts, shared with the running team's watch. `platform` is a
  * parameter so the macOS branches are testable on any host. */
 import { okOutcome, warnOutcome } from "./outcome.js";
 import { systemProcessProbe } from "../process/process-table.js";