tumwater 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/README.md +1 -1
  2. package/dist/build-info.json +2 -2
  3. package/dist/src/agent/agent-run.js +2 -1
  4. package/dist/src/agent/claude/claude-account.js +6 -20
  5. package/dist/src/agent/codex/codex-account.js +5 -6
  6. package/dist/src/agent/codex/codex-args.js +5 -5
  7. package/dist/src/agent/codex/codex-rollout.js +1 -1
  8. package/dist/src/agent/opencode/opencode-backend.js +24 -22
  9. package/dist/src/agent/opencode/opencode-catalog.js +35 -9
  10. package/dist/src/agent/session-ledger.js +22 -26
  11. package/dist/src/agent/subscription-account.js +2 -83
  12. package/dist/src/backlog/backlog-archive.js +3 -4
  13. package/dist/src/backlog/backlog-eligibility.js +14 -36
  14. package/dist/src/backlog/backlog-question-answer.js +1 -1
  15. package/dist/src/backlog/backlog.js +2 -1
  16. package/dist/src/baseline/main-guardrails.js +2 -2
  17. package/dist/src/baseline/main-watch.js +84 -50
  18. package/dist/src/brief/brief.js +26 -13
  19. package/dist/src/budget/budget.js +4 -5
  20. package/dist/src/build/build-check-report.js +10 -0
  21. package/dist/src/build/build-check-scoped.js +0 -2
  22. package/dist/src/build/build-stage.js +6 -4
  23. package/dist/src/change/change-data.js +1 -4
  24. package/dist/src/cli/cli-completion-command.js +75 -0
  25. package/dist/src/cli/cli-doctor-command.js +103 -0
  26. package/dist/src/cli/cli-init.js +2 -2
  27. package/dist/src/cli/cli-query-commands.js +1 -1
  28. package/dist/src/cli/cli-role-command.js +2 -41
  29. package/dist/src/cli/cli-start-summary.js +4 -3
  30. package/dist/src/cli/cli-start-supervisor.js +190 -0
  31. package/dist/src/cli/cli-start.js +4 -187
  32. package/dist/src/cli/completion.js +3 -3
  33. package/dist/src/cli/follow-view.js +20 -2
  34. package/dist/src/cli/help.js +2 -7
  35. package/dist/src/cli.js +19 -171
  36. package/dist/src/collections.js +11 -14
  37. package/dist/src/concurrency/lock.js +1 -1
  38. package/dist/src/config/config-keys.js +61 -20
  39. package/dist/src/config/config-request.js +11 -1
  40. package/dist/src/config/config-schema.js +1 -1
  41. package/dist/src/config/config-validation-roles.js +6 -22
  42. package/dist/src/config/config-validation.js +0 -1
  43. package/dist/src/config/config-views.js +17 -16
  44. package/dist/src/config/config-write.js +122 -79
  45. package/dist/src/config/config.js +1 -9
  46. package/dist/src/doctor/doctor-backlog-plans.js +29 -96
  47. package/dist/src/doctor/doctor-backlog.js +10 -10
  48. package/dist/src/doctor/doctor-checks.js +4 -4
  49. package/dist/src/doctor/doctor-claude-checks.js +27 -9
  50. package/dist/src/doctor/doctor-contention-checks.js +13 -5
  51. package/dist/src/doctor/doctor-fallback-checks.js +46 -24
  52. package/dist/src/doctor/doctor-holds.js +2 -2
  53. package/dist/src/doctor/doctor-orphans.js +3 -2
  54. package/dist/src/doctor/doctor-project-checks.js +21 -8
  55. package/dist/src/events/event-format.js +15 -11
  56. package/dist/src/events/event-legacy.js +2 -2
  57. package/dist/src/events/event-notify.js +4 -4
  58. package/dist/src/events/event-window.js +1 -1
  59. package/dist/src/events/events.js +8 -0
  60. package/dist/src/failure/failure-cluster.js +4 -13
  61. package/dist/src/failure/failure-data.js +4 -4
  62. package/dist/src/failure/failure-render.js +2 -2
  63. package/dist/src/failure/failure-state-change.js +50 -86
  64. package/dist/src/gates/gate-polls.js +5 -6
  65. package/dist/src/gates/startup-gates.js +2 -1
  66. package/dist/src/git/git-diff.js +6 -2
  67. package/dist/src/gui/gui-args.js +12 -0
  68. package/dist/src/gui/gui-config-commands.js +2 -64
  69. package/dist/src/gui/gui-endpoint-commands.js +1 -1
  70. package/dist/src/gui/gui-endpoint-common.js +19 -1
  71. package/dist/src/gui/gui-operator-commands.js +21 -25
  72. package/dist/src/gui/gui-prompt-commands.js +2 -2
  73. package/dist/src/gui/gui-server.js +1 -2
  74. package/dist/src/gui/gui-team-commands.js +2 -1
  75. package/dist/src/history/history.js +4 -4
  76. package/dist/src/merge/merge-check-failures.js +12 -9
  77. package/dist/src/merge/merge-core.js +6 -31
  78. package/dist/src/merge/merge-drain.js +3 -2
  79. package/dist/src/merge/merge-post-review-drain.js +1 -1
  80. package/dist/src/merge/merge-queue-render.js +3 -3
  81. package/dist/src/merge/merge-stack.js +17 -18
  82. package/dist/src/merge/merge-to-main.js +13 -58
  83. package/dist/src/merge/merge-verify.js +4 -3
  84. package/dist/src/operator/operator-commands.js +4 -3
  85. package/dist/src/orchestrator/orchestrator-poll.js +18 -38
  86. package/dist/src/orchestrator/orchestrator-scheduling.js +5 -9
  87. package/dist/src/orchestrator/scheduling-scope.js +17 -26
  88. package/dist/src/orchestrator/scheduling-skip.js +9 -13
  89. package/dist/src/paths.js +2 -2
  90. package/dist/src/pi/pi-event-line.js +27 -0
  91. package/dist/src/pi/pi-listed-models.js +13 -0
  92. package/dist/src/pi/pi-stream.js +9 -2
  93. package/dist/src/pi/pi-watchdogs.js +31 -10
  94. package/dist/src/process/process-group.js +2 -21
  95. package/dist/src/process/process.js +12 -4
  96. package/dist/src/process/run-marker.js +2 -2
  97. package/dist/src/prompt/gate-prompts.js +1 -44
  98. package/dist/src/redeploy/redeployer.js +21 -1
  99. package/dist/src/review/diff-rules.js +35 -21
  100. package/dist/src/review/post-merge-review.js +47 -40
  101. package/dist/src/review/review-precheck.js +3 -3
  102. package/dist/src/review/review.js +13 -22
  103. package/dist/src/roles/legacy-role-migration.js +6 -3
  104. package/dist/src/roles/role-render.js +6 -32
  105. package/dist/src/roles/role-view.js +2 -13
  106. package/dist/src/run/ci-parity.js +19 -8
  107. package/dist/src/run/run-apply.js +0 -9
  108. package/dist/src/run/run-stage.js +6 -6
  109. package/dist/src/run/run-verdict.js +13 -4
  110. package/dist/src/run/stage-check.js +17 -24
  111. package/dist/src/runner/runner-leftover.js +9 -26
  112. package/dist/src/runner/runner-outcome.js +1 -1
  113. package/dist/src/runner/runner-pi.js +6 -12
  114. package/dist/src/runner/runner-recovery.js +4 -12
  115. package/dist/src/runner/runner-revision.js +1 -2
  116. package/dist/src/runner/runner-run.js +7 -16
  117. package/dist/src/runner/runner-state-load.js +6 -19
  118. package/dist/src/runner/runner-state.js +0 -14
  119. package/dist/src/scheduling/backoff.js +8 -3
  120. package/dist/src/scheduling/claim-entries.js +2 -1
  121. package/dist/src/status/status-budget.js +4 -43
  122. package/dist/src/status/status-data.js +34 -38
  123. package/dist/src/status/status-roles.js +1 -2
  124. package/dist/src/team/team-hold.js +7 -5
  125. package/dist/src/team/team-orchestrator-info.js +53 -21
  126. package/dist/src/team/team-polls.js +36 -86
  127. package/dist/src/text/datetime.js +16 -7
  128. package/dist/src/text/phrases.js +9 -0
  129. package/dist/src/text/suggest.js +1 -1
  130. package/dist/src/text/text.js +3 -2
  131. package/dist/src/ui/alert-feeds.js +0 -41
  132. package/dist/src/ui/badges.js +14 -75
  133. package/dist/src/ui/gui/gui-client-boot.js +0 -6
  134. package/dist/src/ui/gui/gui-client-drawer-config.js +1 -16
  135. package/dist/src/ui/gui/gui-client-feed.js +1 -1
  136. package/dist/src/ui/gui/gui-client-operator.js +7 -5
  137. package/dist/src/ui/gui/gui-client-team.js +8 -2
  138. package/dist/src/ui/progress-data.js +54 -4
  139. package/dist/src/ui/run-progress-model.js +12 -3
  140. package/dist/src/ui/status-payload.js +10 -25
  141. package/dist/src/ui/status-render.js +10 -17
  142. package/dist/src/ui/team-alert-backlog.js +15 -205
  143. package/dist/src/ui/team-alert-shared.js +6 -29
  144. package/dist/src/ui/team-alerts.js +4 -10
  145. package/dist/src/ui/tui/tui-backlog.js +2 -2
  146. package/dist/src/ui/tui/tui-editors.js +5 -4
  147. package/dist/src/ui/tui/tui-input.js +8 -6
  148. package/dist/src/ui/tui/tui-pane-activity.js +2 -1
  149. package/dist/src/ui/tui/tui-pane-health.js +21 -23
  150. package/dist/src/ui/tui/tui-pane-history.js +7 -5
  151. package/dist/src/ui/tui/tui-pane-pending.js +19 -15
  152. package/dist/src/ui/tui/tui-pane-session.js +4 -3
  153. package/dist/src/ui/tui/tui-panes.js +9 -1
  154. package/dist/src/ui/tui/tui.js +3 -5
  155. package/dist/src/verdict/fix-claim.js +36 -7
  156. package/dist/src/verdict/reply-contract.js +3 -4
  157. package/package.json +2 -2
  158. package/dist/src/review/review-followup.js +0 -87
  159. package/dist/src/review/review-run.js +0 -395
  160. /package/dist/src/cli/{brief-command.js → cli-brief-command.js} +0 -0
  161. /package/dist/src/cli/{config-commands.js → cli-config-commands.js} +0 -0
  162. /package/dist/src/cli/{log-commands.js → cli-log-commands.js} +0 -0
  163. /package/dist/src/cli/{notify-command.js → cli-notify-command.js} +0 -0
  164. /package/dist/src/cli/{question-commands.js → cli-question-commands.js} +0 -0
@@ -18,7 +18,7 @@ import { finiteNumber, stringList } from "../files/json-object.js";
18
18
  import { commitExists, deleteRefIfAt, listBranches, refSha, refsUnder } from "../git/git.js";
19
19
  import { readSlotsState } from "../git/slots-state.js";
20
20
  import { eventsSince } from "../events/event-window.js";
21
- import { DAY_MS } from "../text/datetime.js";
21
+ import { DAY_MS, HOUR_MS } from "../text/datetime.js";
22
22
  import { shortSha } from "../text/format.js";
23
23
  import { errorMessage, joinParts, textOr } from "../text/text.js";
24
24
  import { rankCountEntries } from "../collections.js";
@@ -72,7 +72,7 @@ export function staleWorkspacePins(root, now = Date.now()) {
72
72
  * clause renders it through textOr like the row always did, so a malformed pin can never
73
73
  * print nothing where the phrase expects a role. */
74
74
  function stalePinPhrase(pin) {
75
- return `${textOr(pin.role)} pinned on ${pin.dir} for ${Math.round(pin.ageMs / 3_600_000)}h`;
75
+ return `${textOr(pin.role)} pinned on ${pin.dir} for ${Math.round(pin.ageMs / HOUR_MS)}h`;
76
76
  }
77
77
  /** Directory names under the worktrees root that the pool no longer owns: a legacy per-role
78
78
  * `<role>` checkout (the pre-pool layout) or a `_land-<role>` checkout from the merge queue. The
@@ -112,6 +112,14 @@ function sumWaits(waits) {
112
112
  }
113
113
  return { count, totalMs, longestMs, longest };
114
114
  }
115
+ /** The leading wait-summary clause every `*_wait` contention row shares — the wait count, the
116
+ * total and the longest span over the trailing day. Only the verb naming the waited-for thing
117
+ * differs between the three rows ("to start", "for a provider slot", "for a check slot"), so
118
+ * this single home keeps the shared wording — the same `plural("wait")`/`shortSpanPhrase`
119
+ * bucketing — from drifting between them. Each row appends its own bucket/knob tail after it. */
120
+ function waitDetail(count, verb, totalMs, longestMs) {
121
+ return `${plural(count, "wait")} ${verb} in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)}`;
122
+ }
115
123
  /** The `slot_wait` half of checkWorktreePool's rule: whether the window's waits cross
116
124
  * SLOT_WAIT_WARN_TOTAL_MS and the pool row's exact detail clause — the wait count, total and
117
125
  * longest spans (naming the role and whether it was a check or a run), and, on a warn, the
@@ -121,7 +129,7 @@ function slotWaitContention(waits, slotCount) {
121
129
  const { count, totalMs, longestMs, longest } = sumWaits(waits);
122
130
  const contended = totalMs >= SLOT_WAIT_WARN_TOTAL_MS;
123
131
  const purpose = longest.purpose === "verify" ? "check" : "run";
124
- const detail = `${plural(count, "wait")} to start in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)} (${textOr(longest.role)} ${purpose})${contended ? ` — raise worktreeSlots above ${slotCount}` : ""}`;
132
+ const detail = `${waitDetail(count, "to start", totalMs, longestMs)} (${textOr(longest.role)} ${purpose})${contended ? ` — raise worktreeSlots above ${slotCount}` : ""}`;
125
133
  return { warn: contended, detail };
126
134
  }
127
135
  /** Provider-slot contention's rule (the body of checkProviderContention from `sumWaits` on):
@@ -150,7 +158,7 @@ function providerWaitContention(waits, config) {
150
158
  const parenthetical = ranked.length === 1
151
159
  ? `(${bucketLabel(ranked[0].key)})`
152
160
  : `(${cells[0]}, also ${cells.slice(1).join(", ")})`;
153
- let detail = `${plural(count, "wait")} for a provider slot in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)} ${parenthetical}`;
161
+ let detail = `${waitDetail(count, "for a provider slot", totalMs, longestMs)} ${parenthetical}`;
154
162
  const contended = totalMs >= PROVIDER_WAIT_WARN_TOTAL_MS;
155
163
  // The one configurable bound is the team total, so a warn names `maxConcurrent` to raise —
156
164
  // but only for a team-cap wait, the only kind that bound can affect (a keyed-only wait from
@@ -175,7 +183,7 @@ function checkWaitContention(waits, config, load) {
175
183
  perKind.set(kind, (perKind.get(kind) ?? 0) + waited);
176
184
  }
177
185
  }
178
- let detail = `${plural(count, "wait")} for a check slot in the last 24h, ${shortSpanPhrase(totalMs)} total, longest ${shortSpanPhrase(longestMs)}`;
186
+ let detail = waitDetail(count, "for a check slot", totalMs, longestMs);
179
187
  if (perKind.size > 0) {
180
188
  const ranked = rankCountEntries(perKind);
181
189
  detail += ` (behind ${ranked.map(([kind, ms]) => `${kind} ${shortSpanPhrase(ms)}`).join(", ")})`;
@@ -77,6 +77,48 @@ const isLocalUrl = (url) => {
77
77
  return false;
78
78
  }
79
79
  };
80
+ /** Which URL this step probes, and did it answer. A provider that names a backend other than
81
+ * pi (claude today; opencode from part 2) never asks pi for a base URL or runs pi's price
82
+ * probe — only a pi step may. Returns the step's backend-ness, the resolved base URL and
83
+ * whether a local pi step answered; a non-local or non-pi step never probes (up stays null). */
84
+ async function resolveStepProbe(modelsPath, pair, probe) {
85
+ const nonPiBackend = pair.backend !== undefined && pair.backend !== "pi";
86
+ const base = !nonPiBackend && pair.provider ? piProviderBaseUrl(modelsPath, pair.provider) : null;
87
+ const local = base !== null && isLocalUrl(base);
88
+ const up = local ? await probe(`${base}/models`) : null;
89
+ return { nonPiBackend, base, up };
90
+ }
91
+ /** The claude step's billing display: extra usage when the step follows an earlier claude
92
+ * model, else the persisted account's billing answer (or unknowns when the account is absent). */
93
+ function claudeVerdict(extra, account) {
94
+ return extra ? "extra usage" : (account?.billing ?? "billing unknown");
95
+ }
96
+ /** Why this step is not runnable now (or is, when no arm fires): the five-way verdict, and
97
+ * whether the chain is down. The `up === false` arm is the only one that marks the chain down.
98
+ * `accounts` prices a free/claude pair for the one claude account held by the caller. */
99
+ function stepVerdict(pair, ctx) {
100
+ const { up, base, cap, spent, providers, accounts } = ctx;
101
+ if (up === false) {
102
+ return { verdict: `not running (GET ${base}/models failed)`, down: true };
103
+ }
104
+ else if (cap !== undefined && spent >= cap) {
105
+ return { verdict: `cap reached (${usd(spent)}/${usdCap(cap)})`, down: false };
106
+ }
107
+ else if (pair.backend !== undefined && pair.backend !== "pi") {
108
+ return { verdict: `claude ${claudeVerdict(pair.extraUsage, accounts("claude"))}`, down: false };
109
+ }
110
+ else if (piPairFree(providers, pair, accounts)) {
111
+ return { verdict: up ? "free · server up" : "free", down: false };
112
+ }
113
+ return {
114
+ verdict: cap !== undefined ? `priced, ${usd(spent)}/${usdCap(cap)} today` : "priced, team cap only",
115
+ down: false,
116
+ };
117
+ }
118
+ /** One chain fragment's line: the tier, the 1-based step number and the step's pair and verdict. */
119
+ function chainFragment(tier, step, pair, verdict) {
120
+ return `${tier} ${step + 1}: ${pair.provider ?? "?"}/${pair.model ?? "?"} — ${verdict}`;
121
+ }
80
122
  /** The `fallback chain` check: one fragment per step of every tier's chain saying whether the
81
123
  * step can run now — a local pi server that does not answer (warn), a step whose own daily cap is
82
124
  * spent, a free pair, a claude step with its billing, or a priced pair. Reads config, pi's
@@ -98,32 +140,12 @@ export async function checkFallbackChain(root, modelsPath = piModelsPath(), prob
98
140
  let down = false;
99
141
  const fragments = [];
100
142
  for (const { tier, step, pair } of steps) {
101
- // A provider that names a backend other than pi (claude today; opencode from part 2) never
102
- // asks pi for a base URL or runs pi's price probe — only a pi step may.
103
- const nonPiBackend = pair.backend !== undefined && pair.backend !== "pi";
104
- const base = !nonPiBackend && pair.provider ? piProviderBaseUrl(modelsPath, pair.provider) : null;
105
- const local = base !== null && isLocalUrl(base);
106
- const up = local ? await probe(`${base}/models`) : null;
143
+ const { base, up } = await resolveStepProbe(modelsPath, pair, probe);
107
144
  const cap = pair.maxDailyCostUsd;
108
145
  const spent = cap !== undefined ? pairDailyCost(states, modelPairName(pair), now) : 0;
109
- let verdict;
110
- if (up === false) {
111
- down = true;
112
- verdict = `not running (GET ${base}/models failed)`;
113
- }
114
- else if (cap !== undefined && spent >= cap) {
115
- verdict = `cap reached (${usd(spent)}/${usdCap(cap)})`;
116
- }
117
- else if (nonPiBackend) {
118
- verdict = `claude ${pair.extraUsage ? "extra usage" : (account?.billing ?? "billing unknown")}`;
119
- }
120
- else if (piPairFree(providers, pair, accounts)) {
121
- verdict = up ? "free · server up" : "free";
122
- }
123
- else {
124
- verdict = cap !== undefined ? `priced, ${usd(spent)}/${usdCap(cap)} today` : "priced, team cap only";
125
- }
126
- fragments.push(`${tier} ${step + 1}: ${pair.provider ?? "?"}/${pair.model ?? "?"} — ${verdict}`);
146
+ const { verdict, down: isDown } = stepVerdict(pair, { up, base, cap, spent, providers, accounts });
147
+ down ||= isDown;
148
+ fragments.push(chainFragment(tier, step, pair, verdict));
127
149
  }
128
150
  const detail = fragments.join("; ");
129
151
  return down ? warnOutcome(detail) : okOutcome(detail);
@@ -207,8 +207,8 @@ export function checkErrorStreaks(root, config) {
207
207
  return warnOutcome(`failing run after run: ${phrases.join("; ")}`);
208
208
  }
209
209
  /** The `main red` check: the row lists the runs the merge-side `main_red` pin ended — a red
210
- * main is still recorded as the `main_red` outcome that ended those runs until parts 3d/3–3f/3
211
- * land — and warns when main has been red over RED_MAIN_WARN_MS even with no role held (the
210
+ * main is still recorded as the `main_red` outcome that ended those runs until part 3f/3
211
+ * lands — and warns when main has been red over RED_MAIN_WARN_MS even with no role held (the
212
212
  * guardrail, G1). Read-only: it asks heldByMainRed, the predicate the status phase's `main red`
213
213
  * label reads, so the two cannot disagree on the held runs, and redMainWarning over the event
214
214
  * trail, so both phrases can appear in one row. A warn, not a fail: a red main does not stop
@@ -9,6 +9,7 @@ import { warnEventBestEffort, TUMWATER_SOURCE } from "../events/events.js";
9
9
  import { errorMessage, truncate } from "../text/text.js";
10
10
  import { plural } from "../text/phrases.js";
11
11
  import { failOutcome, okOutcome, warnOutcome } from "./doctor-outcome.js";
12
+ import { distinctOrdered } from "../collections.js";
12
13
  /** The orphan-process check of `tumwater doctor`, split out of doctor.ts: it is the one
13
14
  * doctor check that inspects the operating system's process table (argv, cwd, parent-child
14
15
  * tree) rather than the repo's own files, so it carries a different shape of logic — path
@@ -330,7 +331,7 @@ async function killTrees(root, rows, chosen, signals) {
330
331
  }
331
332
  };
332
333
  const trees = chosen.map((r) => ({ r, pids: [r.pid, ...descendantPids(children, r.pid)] }));
333
- const all = [...new Set(trees.flatMap((t) => t.pids))];
334
+ const all = distinctOrdered(trees.flatMap((t) => t.pids));
334
335
  for (const pid of all)
335
336
  send(pid, "SIGTERM");
336
337
  const stillAlive = () => all.filter((pid) => !refused.has(pid) && signals.alive(pid));
@@ -400,7 +401,7 @@ export async function reapPreviousTeamOrphans(root, previousPid, probe = systemP
400
401
  * orphan may still be a live run's process that findOrphans read as parentless mid-hand-off,
401
402
  * and ten minutes is far longer than any such attribution would hold. The operator's
402
403
  * `--kill-orphans`/dashboard/TUI click bypasses this — doctors judge with their eyes. */
403
- const ORPHAN_REAP_MIN_AGE_MS = 10 * 60 * 1000;
404
+ const ORPHAN_REAP_MIN_AGE_MS = 10 * 60_000;
404
405
  /** The orchestrator's once-per-interval sweep of the orphans `checkOrphans` already reports:
405
406
  * kill the same set the operator's `--kill-orphans` kills, once each is old enough that no
406
407
  * live run can still own it. Reuses findOrphans' attribution and killTrees' signal-and-wait,
@@ -25,7 +25,7 @@ import { formatGB, shortSha } from "../text/format.js";
25
25
  import { eventLogState, readEventsSince } from "../events/event-window.js";
26
26
  import { EVENTS_KEEP } from "../events/events.js";
27
27
  import { finiteNumber } from "../files/json-object.js";
28
- import { agree, plural, shortSpanPhrase } from "../text/phrases.js";
28
+ import { agree, plural, shortSpanPhrase, trailingDayMedianPhrase } from "../text/phrases.js";
29
29
  import { andMore } from "./doctor-backlog.js";
30
30
  import { KNOWN_FLAKE_WINDOW_MS, loadQuarantine, repeatedFlakeCounts } from "../review/known-flakes.js";
31
31
  import { POST_MERGE_REJECT_WINDOW_MS, postMergeChurn } from "../review/post-merge-review.js";
@@ -204,7 +204,7 @@ export function checkDurationPhrase(events) {
204
204
  return undefined;
205
205
  const median = lowerMedian(durations);
206
206
  const slowest = Math.max(...durations);
207
- return `passing merge checks in the last 24h took ${shortSpanPhrase(median)} median, ${shortSpanPhrase(slowest)} slowest (${plural(durations.length, "run")})`;
207
+ return `passing merge checks in the last 24h took ${trailingDayMedianPhrase(median, slowest, "slowest", durations.length, "run")}`;
208
208
  }
209
209
  /** Declared project check — names what the review gate's deterministic pre-check, the
210
210
  * red-main baseline, and redeploy's green check will run, without running it: a configured
@@ -278,7 +278,7 @@ export function checkPostMergeChurn(root) {
278
278
  return okOutcome("no post-merge rejection burst in the last day");
279
279
  return warnOutcome(`${plural(churn.count, "post-merge review rejection")} in the last day (${churn.roles
280
280
  .map((r) => `${roleLabel(r.role)} ×${r.count}`)
281
- .join(", ")}) — each commit had already reached main and stays there with the reviewer's reasons; narrow \`review.afterMerge\` for the noisy role or raise the authoring bar`);
281
+ .join(", ")}) — each commit had already reached main and stays there with the reviewer's reasons; raise the authoring bar`);
282
282
  }
283
283
  /** The one home of the doctor's `flaky checks` sentence: names each check that flapped more than
284
284
  * once, most frequent first, with how many times it flapped — the one sentence the doctor's
@@ -321,7 +321,10 @@ running = null) {
321
321
  ? `; auto-restart is BLOCKED (${running.restartBlocked}) and will not retry until main moves or the block clears`
322
322
  : running?.restartPending
323
323
  ? "; auto-restart is under way"
324
- : "; run `npm run build` and restart `tumwater start` (auto-restart does this for a running team)";
324
+ : running?.waitingForCheck
325
+ ? `; auto-restart waits for a commit that passes main's check — none newer than ` +
326
+ `${shortSha(running.checkedHead ?? info.sha)} has yet (the main's check row says why)`
327
+ : "; run `npm run build` and restart `tumwater start` (auto-restart does this for a running team)";
325
328
  return warnOutcome(`dist/ from ${sha} is stale — main has ${plural(stale.aheadCommits, "later commit")} touching ${STALE_INPUTS_LABEL}${next}`);
326
329
  }
327
330
  return okOutcome(`dist/ from ${sha}, matches main`);
@@ -396,7 +399,7 @@ export function checkQaCoverage(root, now = Date.now()) {
396
399
  /** How far back the `main diff rules` row looks — one day, the same 24 h trailing-day convention
397
400
  * every event-reading doctor row uses, so a diff-rule failure the main watch logged stays a
398
401
  * standing warn until a day passes. Owned here (the check that reads it). */
399
- const MAIN_RULE_FAILURE_WINDOW_MS = 24 * 60 * 60_000;
402
+ const MAIN_RULE_FAILURE_WINDOW_MS = DAY_MS;
400
403
  /** The exact substring src/baseline/main-watch.ts's `watchRules` logs in its warning, so this
401
404
  * check matches a real rule failure and cannot drift from the wording the watch writes. */
402
405
  const MAIN_RULE_FAILURE_MARKER = "fails the diff rules";
@@ -407,9 +410,19 @@ const MAIN_RULE_FAILURE_MARKER = "fails the diff rules";
407
410
  * blind spot sits on main with only this event log to say so. Reads only the `warning` event
408
411
  * the watch already writes, so the row cannot drift from the watch's wording; the most recent
409
412
  * hit names the tip and the rule. ok/warn only — it never fails a scripted pre-flight. */
413
+ /** The `warning` events whose message contains `marker` — the read both marker-driven
414
+ * project rows (checkMainDiffRules and checkDroppedPostMergeReviews) ask for, so their
415
+ * filter predicate cannot drift: a warning whose message is absent (an older event) is
416
+ * never a hit, and the marker match is a plain substring like the watch's wording.
417
+ * `e.type === "warning"` gates on the literal type, mirroring the warn shape warnEvent
418
+ * writes. Since every hit is a `warning`, `String` on its message downstream reads safely
419
+ * for the events this returns. */
420
+ function warningEventsWithMarker(events, marker) {
421
+ return events.filter((e) => e.type === "warning" && typeof e.message === "string" && e.message.includes(marker));
422
+ }
410
423
  export function checkMainDiffRules(root) {
411
424
  const { events } = readEventsSince(root, MAIN_RULE_FAILURE_WINDOW_MS);
412
- const hits = events.filter((e) => e.type === "warning" && typeof e.message === "string" && e.message.includes(MAIN_RULE_FAILURE_MARKER));
425
+ const hits = warningEventsWithMarker(events, MAIN_RULE_FAILURE_MARKER);
413
426
  if (hits.length === 0)
414
427
  return okOutcome("no diff-rule failure on main in the last day");
415
428
  const latest = hits[hits.length - 1];
@@ -426,7 +439,7 @@ export function checkMainDiffRules(root) {
426
439
  const DROPPED_REVIEW_MARKER = "dropping the post-merge review of";
427
440
  /** How far back the `dropped post-merge reviews` row looks — one day, the same 24 h trailing-day
428
441
  * convention every event-reading doctor row uses. Owned here (the check that reads it). */
429
- const DROPPED_REVIEW_WINDOW_MS = 24 * 60 * 60_000;
442
+ const DROPPED_REVIEW_WINDOW_MS = DAY_MS;
430
443
  /** The doctor's `dropped post-merge reviews` row: a `warning` the post-merge-review drain
431
444
  * (src/merge/merge-post-review-drain.ts) wrote when it dropped a review at
432
445
  * POST_REVIEW_FAILURE_LIMIT — after failed runs its commit stays on main unreviewed, and
@@ -439,7 +452,7 @@ const DROPPED_REVIEW_WINDOW_MS = 24 * 60 * 60_000;
439
452
  * unreviewed commit is operator signal, not a broken environment. */
440
453
  export function checkDroppedPostMergeReviews(root) {
441
454
  const { events } = readEventsSince(root, DROPPED_REVIEW_WINDOW_MS);
442
- const hits = events.filter((e) => e.type === "warning" && typeof e.message === "string" && e.message.includes(DROPPED_REVIEW_MARKER));
455
+ const hits = warningEventsWithMarker(events, DROPPED_REVIEW_MARKER);
443
456
  if (hits.length === 0)
444
457
  return okOutcome("no post-merge review was dropped in the last day");
445
458
  const latest = hits[hits.length - 1];
@@ -49,6 +49,14 @@ function elapsed(ms) {
49
49
  return "";
50
50
  return ` (in ${shortSpanPhrase(n)})`;
51
51
  }
52
+ /** The `waited <duration>` head of the three wait-family events (slot_wait, provider_wait,
53
+ * check_wait) — each rendered its own `waited ${shortSpanPhrase(finiteNumber(e.waitedMs, 0))}`
54
+ * inline, so the corrupt-field rule and the s/m bucketing cannot drift between them. Each
55
+ * caller keeps its own `to start`/`for a run slot`/`for a check slot` tail and its own
56
+ * parenthetical after it. */
57
+ function waitedSpan(waitedMs) {
58
+ return `waited ${shortSpanPhrase(finiteNumber(waitedMs, 0))}`;
59
+ }
52
60
  /** A disk event's GB field as "12.3 GB" text. A torn or hand-edited event whose value is
53
61
  * missing or non-numeric reads as "0.0 GB", never "NaN" — the same corrupt-field rule
54
62
  * budgetPhrase applies to spentUsd/capUsd, since HarnessEvent's index signature hands every
@@ -183,7 +191,7 @@ export function eventFeedItem(e) {
183
191
  }
184
192
  /** The feed's `--grep`/Search match rule: case-insensitive substring over the raw event type
185
193
  * prefixed to the rendered line — exactly the haystack the operator would otherwise read. One
186
- * home shared by the CLI's `tumwater logs --grep` (cli/log-commands.ts) and the dashboard's
194
+ * home shared by the CLI's `tumwater logs --grep` (cli/cli-log-commands.ts) and the dashboard's
187
195
  * /api/logs search, so the two surfaces cannot disagree about what a pattern matches. */
188
196
  export function eventMatchesGrep(e, patternLower) {
189
197
  return `${e.type} ${formatEvent(e)}`.toLowerCase().includes(patternLower);
@@ -312,22 +320,19 @@ function mergeEventMessage(e) {
312
320
  default: return textOr(e.type);
313
321
  }
314
322
  }
315
- /** " after merge" when the review is a post-merge review (review.afterMerge), else "". */
323
+ /** " after merge" when the review is a post-merge review, else "". */
316
324
  function afterMergeSuffix(e) {
317
325
  return e.afterMerge === true ? " after merge" : "";
318
326
  }
319
327
  function reviewEventMessage(e) {
320
328
  switch (e.type) {
321
329
  case "review_start": {
322
- // A resumed review continues a timed-out reviewer's session (PLANS.md, review-retry part
323
- // 3/3). The verb names that, because a bare "reviewing" line reads as a from-scratch run;
324
- // the revision and model suffixes are unchanged and independent of the flag.
325
- // afterMerge is the tier flag reviewMergedCommit writes for a review.afterMerge role; the
330
+ // afterMerge is the tier flag reviewMergedCommit writes for a post-merge review; the
326
331
  // lineage pins review_start/review_verdict/review_rejected to the same discriminator.
327
332
  const tier = e.afterMerge === true ? " after merge" : " before merge";
328
333
  const revision = e.revision !== undefined ? ` (revision ${textOr(e.revision)})` : "";
329
334
  const model = e.model !== undefined ? ` on ${textOr(e.model)}` : "";
330
- return `${e.resumed ? "review resumed" : "reviewing"} ${shortSha(e.head)}${revision}${tier}${model}`;
335
+ return `reviewing ${shortSha(e.head)}${revision}${tier}${model}`;
331
336
  }
332
337
  case "review_verdict":
333
338
  return `review approved ${shortSha(e.head)}${afterMergeSuffix(e)}${e.reason ? ` — ${textOr(e.reason)}` : ""}${elapsed(e.durationMs)}`;
@@ -657,14 +662,13 @@ function checkWaitEventMessage(e) {
657
662
  // working as designed. Names the wait, the pool it waited on, and how many slots are
658
663
  // pinned, so the operator can raise worktreeSlots with the numbers in hand.
659
664
  const purpose = e.purpose === "verify" ? "check" : "run";
660
- return `waited ${shortSpanPhrase(finiteNumber(e.waitedMs, 0))} to start (${textOr(e.role)} ${purpose}; ${textOr(e.slots)} workspaces, ${textOr(e.pinned)} pinned)`;
665
+ return `${waitedSpan(e.waitedMs)} to start (${textOr(e.role)} ${purpose}; ${textOr(e.slots)} workspaces, ${textOr(e.pinned)} pinned)`;
661
666
  }
662
667
  case "provider_wait": {
663
668
  // Routine-with-explanation, like slot_wait: waiting for the team slot is the cap
664
669
  // working as designed, so no warning prefix. Names the wait, so the operator can size
665
670
  // maxConcurrent with the number in hand.
666
- const span = shortSpanPhrase(finiteNumber(e.waitedMs, 0));
667
- return `waited ${span} for a run slot (${textOr(e.role)})`;
671
+ return `${waitedSpan(e.waitedMs)} for a run slot (${textOr(e.role)})`;
668
672
  }
669
673
  case "check_wait":
670
674
  // Routine-with-explanation, like slot_wait: waiting for a check permit is the cap working
@@ -674,7 +678,7 @@ function checkWaitEventMessage(e) {
674
678
  const heldBy = stringList(e.heldBy);
675
679
  const limit = finiteNumber(e.limit, null);
676
680
  const behind = heldBy.length > 0 ? `; held by ${heldBy.join(", ")}${limit === null ? "" : ` of maxConcurrentChecks ${limit}`}` : "";
677
- return `waited ${shortSpanPhrase(finiteNumber(e.waitedMs, 0))} for a check slot (${textOr(e.role)} ${textOr(e.scope)} check${behind})`;
681
+ return `${waitedSpan(e.waitedMs)} for a check slot (${textOr(e.role)} ${textOr(e.scope)} check${behind})`;
678
682
  }
679
683
  default: return textOr(e.type);
680
684
  }
@@ -108,8 +108,8 @@ const LEGACY_EVENT_REWRITES = [
108
108
  },
109
109
  },
110
110
  {
111
- // A retired role id on an old line reads as the role's current id. The table is empty
112
- // until a switch part adds a row, so this changes nothing yet.
111
+ // A retired role id on an old line reads as the role's current id. The table grows a row
112
+ // per retired role as future switch parts rename; a row already present applies here.
113
113
  name: "legacy-role-ids",
114
114
  apply(ev) {
115
115
  const e = ev;
@@ -14,8 +14,8 @@ import { childEnv } from "../process/process.js";
14
14
  * role-cap-gates.ts and disk-gates.ts, src/baseline/main-red.ts, src/team/team-polls.ts,
15
15
  * merge-slot.ts, merge-questions.ts,
16
16
  * src/redeploy/redeployer.ts). `supervisor_exit` is logged by the supervisor after the
17
- * orchestrator is gone, so src/cli/cli-start.ts pages it directly with notifyTeamDown; events
18
- * appended by
17
+ * orchestrator is gone, so src/cli/cli-start-supervisor.ts pages it directly with notifyTeamDown;
18
+ * events appended by
19
19
  * operator CLI commands go straight to the file and are out of scope by design.
20
20
  *
21
21
  * One knob, an opinionated fixed allowlist, no per-event selection — a command that can post
@@ -45,7 +45,7 @@ export const NOTIFY_EVENT_TYPES = [
45
45
  export const NOTIFY_MIN_GAP_MS = 60_000;
46
46
  /** The child environment the notify hook runs with: the three TUMWATER_EVENT_* variables over
47
47
  * the harness's own environment. One home for the variable names, shared by the notifier's
48
- * spawn and the `tumwater notify --test` diagnostic (src/cli/notify-command.ts), so the two
48
+ * spawn and the `tumwater notify --test` diagnostic (src/cli/cli-notify-command.ts), so the two
49
49
  * cannot drift on what a hook sees. */
50
50
  export function notifyCommandEnv(type, source, message) {
51
51
  return childEnv({
@@ -76,7 +76,7 @@ export function notifyExitWarning(type, code, signal) {
76
76
  * not found", 126 its "found but not executable" — most often the configured `notify` command
77
77
  * is a typo or an uninstalled/non-executable binary, so a bare "code 127" hides the fix the
78
78
  * gloss names. Shared by the notifier's warning (notifyExitWarning above) and `tumwater
79
- * notify --test`'s diagnostic (src/cli/notify-command.ts) so both surfaces name the same fix
79
+ * notify --test`'s diagnostic (src/cli/cli-notify-command.ts) so both surfaces name the same fix
80
80
  * for a hook that cannot launch. Any other code gets no gloss. */
81
81
  export function notifyExitGloss(code) {
82
82
  return code === 127 ? " (command not found)" : code === 126 ? " (found but not executable)" : "";
@@ -321,7 +321,7 @@ export function readEventsSinceJoined(root, sinceMs, joinBackslackMs) {
321
321
  (window.events[firstNarrowIdx]?.ts ?? Number.POSITIVE_INFINITY) <= cutoff;
322
322
  return { cutoff, events, join: { cutoff: joinCutoff, events: joinEvents }, covered };
323
323
  }
324
- /** The duration-shaped windowed read behind `logs --since` (log-commands.ts): the cutoff is
324
+ /** The duration-shaped windowed read behind `logs --since` (cli-log-commands.ts): the cutoff is
325
325
  * now − sinceMs, the rotation-spanning read is keyed on the cutoff's local calendar day
326
326
  * (dayKey — the shared dayKey helper eventDayKey buckets events with, so the read's day keys
327
327
  * cannot disagree with the ts filter), the day-keyed read may include earlier hours of that
@@ -97,6 +97,14 @@ export function logEventBestEffort(root, event) {
97
97
  process.stderr.write(`tumwater: ${stripTerminalControls(detail)}\n`);
98
98
  }
99
99
  }
100
+ /** Log one `image_blind` event (plans/image-input.md "Detecting a blind read"): an agent read an
101
+ * image its model cannot see — the authoring run was killed (action "fail") or a gate run merely
102
+ * recorded the read (action "logged"). The runner and both review paths used to build this exact
103
+ * event object themselves (source + type + run + path + model + action); it lives here so the
104
+ * payload has one home and cannot drift between the three callers. */
105
+ export function logBlindRead(root, source, run, path, model, action) {
106
+ logEventBestEffort(root, { source, type: "image_blind", run, path, model, action });
107
+ }
100
108
  /** warnEvent's twin of logEventBestEffort: the "something is off but the loop continues"
101
109
  * warning must not be the thing that ends the role it warns about. The fallback keeps the
102
110
  * warning's own text on stderr — when the feed is the failure, this line is the only place the
@@ -86,24 +86,15 @@ const RUN_TIMEOUT_PROGRESSING_KEY = "timed out after <dur> while still making pr
86
86
  // resumes, so its timeout names what actually happens — the commit is kept, the next
87
87
  // attempt re-reviews from scratch. Same cause, same knob (review.timeoutSeconds), one row.
88
88
  const RUN_TIMEOUT_PROGRESSING_REVIEW_KEY = "timed out after <dur> while still making progress — the commit is kept; the next attempt reviews it from scratch";
89
- // The review run's wording when a resumable session remains:
90
- // the next attempt continues this review instead of re-reviewing. Continuing the timed-out
91
- // session is a different cause from a from-scratch re-review, so this wording is its own
92
- // cluster rather than joining the pooled plain run-timeout row. Exported for its unit tests
93
- // (like RUN_TIMEOUT_KEY), so a display or digest test can pin the mapping by name.
94
- export const RUN_TIMEOUT_PROGRESSING_REVIEW_RESUME_KEY = "timed out after <dur> while still making progress — the commit is kept; the next attempt continues this review";
95
- /** The known progressing-timeout wordings and the key each pools under. The two review
96
- * wordings name one cause on one knob (review.timeoutSeconds): a still-making-progress kill
89
+ /** The known progressing-timeout wordings and the key each pools under. The review
90
+ * wording names one cause on one knob (review.timeoutSeconds): a still-making-progress kill
97
91
  * pools into the plain run-timeout row (a mixed team of plain and progressing kills is one
98
92
  * cause's agent-hours, not two half-size rows the top-N cut can drop — the same pooling the
99
- * error-storm reducer applies, src/team/team-error-storm.ts), and a resumed review's wording maps
100
- * to itself, because continuing the timed-out session is a different cause from a from-scratch
101
- * re-review. Enumerated so a future wording edit lands in
102
- * this table rather than silently changing which row its timeouts count under. */
93
+ * error-storm reducer applies, src/team/team-error-storm.ts). Enumerated so a future wording
94
+ * edit lands in this table rather than silently changing which row its timeouts count under. */
103
95
  const TIMEOUT_POOL = new Map([
104
96
  [RUN_TIMEOUT_PROGRESSING_KEY, RUN_TIMEOUT_KEY],
105
97
  [RUN_TIMEOUT_PROGRESSING_REVIEW_KEY, RUN_TIMEOUT_KEY],
106
- [RUN_TIMEOUT_PROGRESSING_REVIEW_RESUME_KEY, RUN_TIMEOUT_PROGRESSING_REVIEW_RESUME_KEY],
107
98
  ]);
108
99
  /** The cluster key a normalized error clusters under: the pooled wordings above map to their
109
100
  * row, and every other cause stands as normalizeClusterKey rendered it. */
@@ -3,10 +3,10 @@ import { eventDayKey, eventRole } from "../events/event-read.js";
3
3
  import { baseRoleOf } from "../roles/runner-ids.js";
4
4
  import { scopeBases } from "../report/report-fold.js";
5
5
  import { timeAndSpend } from "./failure-time-spend.js";
6
- import { dayAt, dayKey, dayWindow, formatDate } from "../text/datetime.js";
6
+ import { dayAt, dayKey, dayWindow, DAY_MS, formatDate } from "../text/datetime.js";
7
7
  import { describeStateChange, STATE_CHANGE_TOP, STATE_CHANGE_TYPES } from "./failure-state-change.js";
8
8
  import { clusterMessages, errorTextOrPlaceholder, truncateExample } from "./failure-cluster.js";
9
- import { addTo, getOrCreate, memoSetLru, rankByCount } from "../collections.js";
9
+ import { addTo, distinctOrdered, getOrCreate, memoSetLru, rankByCount } from "../collections.js";
10
10
  import { firstReason } from "../text/phrases.js";
11
11
  import { sameStatIdentity, statIdentityOrZero, statOrNull } from "../files/files.js";
12
12
  import { nonNegativeNumber } from "../files/json-object.js";
@@ -200,7 +200,7 @@ export function collectFailureReportSince(root, sinceMs, role, untilMs) {
200
200
  prior.push(ev);
201
201
  }
202
202
  return assembleFailureReport(current, prior, raw.events, eventWindowCovers(raw, priorFrom), raw.logState, {
203
- days: Math.ceil(length / 86_400_000),
203
+ days: Math.ceil(length / DAY_MS),
204
204
  from: dayKey(cutoff),
205
205
  to: dayKey(end),
206
206
  ...slice,
@@ -298,7 +298,7 @@ function outcomeRows(runEvents) {
298
298
  function deltaRows(current, prior) {
299
299
  const curStats = roleStats(current);
300
300
  const prevStats = roleStats(prior);
301
- const roles = [...new Set([...curStats.keys(), ...prevStats.keys()])];
301
+ const roles = distinctOrdered([...curStats.keys(), ...prevStats.keys()]);
302
302
  return roles
303
303
  .map((role) => {
304
304
  const c = curStats.get(role) ?? ZERO_ROLE_STATS;
@@ -1,6 +1,6 @@
1
1
  import { agree, plural } from "../text/phrases.js";
2
2
  import { shortSha, usd } from "../text/format.js";
3
- import { boundedWindowLine, clockHm, dayKey, dayLabel, monthDay, reportWindow, sinceWindowLine } from "../text/datetime.js";
3
+ import { HOUR_MS, boundedWindowLine, clockHm, dayKey, dayLabel, monthDay, reportWindow, sinceWindowLine } from "../text/datetime.js";
4
4
  import { durationLabel } from "../cli/cli-args.js";
5
5
  import { eventsRotationLabel } from "../events/events.js";
6
6
  import { eventTypeLabel } from "../events/event-labels.js";
@@ -99,7 +99,7 @@ function spendCell(cell) {
99
99
  * from shortSpanPhrase's minutes-or-seconds (an event-feed phrase), because a summed 8-hour
100
100
  * timeout episode must not read as "480m". */
101
101
  function hoursPhrase(ms) {
102
- return `${(ms / 3_600_000).toFixed(1)} h`;
102
+ return `${(ms / HOUR_MS).toFixed(1)} h`;
103
103
  }
104
104
  /** Render the digest as bounded Markdown. Byte bound: for a given team the tables grow only with
105
105
  * the number of configured roles (fixed by config), the RESULT_ORDER vocabulary (fixed by