tumwater 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/build-info.json +2 -2
- package/dist/src/agent/agent-run.js +1 -0
- package/dist/src/agent/opencode/write-guard-plugin/index.js +11 -8
- package/dist/src/agent/subscription-account.js +68 -3
- package/dist/src/agent/write-guard.js +50 -13
- package/dist/src/backlog/backlog-eligibility.js +8 -4
- package/dist/src/backlog/backlog-index.js +2 -2
- package/dist/src/backlog/backlog-md.js +13 -2
- package/dist/src/backlog/backlog-release.js +3 -11
- package/dist/src/backlog/backlog-write.js +19 -11
- package/dist/src/baseline/main-baseline.js +47 -72
- package/dist/src/baseline/main-guardrails.js +11 -7
- package/dist/src/baseline/main-red.js +100 -12
- package/dist/src/baseline/main-watch.js +66 -20
- package/dist/src/budget/budget-fallback-view.js +96 -28
- package/dist/src/build/build-check-detect.js +22 -3
- package/dist/src/build/build-check-events.js +25 -5
- package/dist/src/build/build-check-report.js +40 -4
- package/dist/src/build/build-check-rerun.js +62 -0
- package/dist/src/build/build-check-scoped.js +13 -2
- package/dist/src/build/build-check.js +2 -1
- package/dist/src/build/build-info.js +4 -0
- package/dist/src/build/dep-install.js +1 -2
- package/dist/src/build/host-sleep.js +9 -0
- package/dist/src/change/change-render.js +2 -1
- package/dist/src/cli/cli-args.js +34 -2
- package/dist/src/cli/cli-command-args.js +12 -4
- package/dist/src/cli/cli-init.js +5 -1
- package/dist/src/cli/cli-query-commands.js +10 -6
- package/dist/src/cli/cli-review-commands.js +3 -3
- package/dist/src/cli/cli-role-command.js +338 -244
- package/dist/src/cli/cli-start.js +10 -2
- package/dist/src/cli/completion.js +6 -2
- package/dist/src/cli/help.js +1 -1
- package/dist/src/cli.js +111 -139
- package/dist/src/collections.js +21 -0
- package/dist/src/concurrency/check-permit.js +30 -3
- package/dist/src/concurrency/lock.js +73 -14
- package/dist/src/config/config-editable-keys.js +9 -0
- package/dist/src/config/config-fallbacks.js +16 -5
- package/dist/src/config/config-schema.js +3 -3
- package/dist/src/config/config-validation.js +2 -0
- package/dist/src/config/config-views.js +55 -33
- package/dist/src/config/config.js +12 -5
- package/dist/src/doctor/doctor-backlog-bugs.js +33 -1
- package/dist/src/doctor/doctor-backlog.js +6 -8
- package/dist/src/doctor/doctor-check-names.js +2 -0
- package/dist/src/doctor/doctor-checks.js +13 -6
- package/dist/src/doctor/doctor-claude-checks.js +3 -3
- package/dist/src/doctor/doctor-contention-checks.js +7 -6
- package/dist/src/doctor/doctor-fallback-checks.js +37 -2
- package/dist/src/doctor/doctor-holds.js +7 -10
- package/dist/src/doctor/doctor-model-checks.js +7 -0
- package/dist/src/doctor/doctor-opencode-checks.js +2 -2
- package/dist/src/doctor/doctor-orphans.js +103 -27
- package/dist/src/doctor/doctor-project-checks.js +37 -20
- package/dist/src/doctor/doctor.js +5 -3
- package/dist/src/events/event-format.js +63 -98
- package/dist/src/events/event-labels.js +2 -1
- package/dist/src/events/events.js +3 -3
- package/dist/src/failure/failure-data.js +6 -4
- package/dist/src/failure/failure-state-change.js +14 -4
- package/dist/src/files/file-queue.js +32 -19
- package/dist/src/files/files.js +2 -2
- package/dist/src/files/json-object.js +3 -2
- package/dist/src/gates/budget-gates.js +37 -2
- package/dist/src/gates/gate-polls.js +8 -7
- package/dist/src/gates/{maintenance-quota.js → maintenance-quota-gates.js} +1 -1
- package/dist/src/gates/startup-gates.js +18 -1
- package/dist/src/git/git.js +8 -0
- package/dist/src/git/worktree-pool.js +18 -5
- package/dist/src/gui/gui-admin-endpoints.js +11 -1
- package/dist/src/gui/gui-args.js +2 -2
- package/dist/src/gui/gui-backlog-endpoints.js +2 -1
- package/dist/src/gui/gui-history-endpoints.js +3 -3
- package/dist/src/gui/gui-operator-commands.js +2 -3
- package/dist/src/history/history.js +10 -6
- package/dist/src/inbox/director-request.js +92 -0
- package/dist/src/merge/merge-backlog-conflicts.js +33 -11
- package/dist/src/merge/merge-batch.js +13 -235
- package/dist/src/merge/merge-cancel.js +2 -1
- package/dist/src/merge/merge-core.js +13 -22
- package/dist/src/merge/merge-drain.js +103 -99
- package/dist/src/merge/merge-pipeline.js +6 -4
- package/dist/src/merge/merge-post-review-drain.js +8 -2
- package/dist/src/merge/merge-questions.js +3 -3
- package/dist/src/merge/merge-queue-render.js +2 -2
- package/dist/src/merge/merge-queue.js +2 -1
- package/dist/src/merge/merge-requeue.js +62 -19
- package/dist/src/merge/merge-slot.js +41 -10
- package/dist/src/merge/merge-to-main.js +16 -165
- package/dist/src/merge/merge-top.js +2 -1
- package/dist/src/merge/merge-verify.js +5 -3
- package/dist/src/model-check/model-check-gate.js +206 -0
- package/dist/src/model-check/model-check-probe.js +109 -0
- package/dist/src/model-check/model-check.js +101 -0
- package/dist/src/operator/operator-commands.js +6 -23
- package/dist/src/orchestrator/orchestrator-launch.js +6 -1
- package/dist/src/orchestrator/orchestrator-poll.js +32 -5
- package/dist/src/orchestrator/orchestrator-scheduling.js +7 -7
- package/dist/src/orchestrator/orchestrator.js +14 -1
- package/dist/src/paths.js +22 -0
- package/dist/src/pi/pi-run-result.js +16 -0
- package/dist/src/pi/pi-watchdogs.js +15 -4
- package/dist/src/process/process.js +24 -0
- package/dist/src/prompt/gate-notes.js +10 -4
- package/dist/src/prompt/gate-prompts.js +13 -14
- package/dist/src/prompt/principles.js +13 -3
- package/dist/src/prompt/prompt-followup.js +14 -5
- package/dist/src/redeploy/redeploy-probes.js +53 -18
- package/dist/src/redeploy/redeploy-target.js +100 -0
- package/dist/src/redeploy/redeployer.js +99 -45
- package/dist/src/report/report-main-red.js +2 -2
- package/dist/src/report/report-render.js +2 -2
- package/dist/src/review/known-flakes.js +4 -2
- package/dist/src/review/post-merge-review-accept.js +2 -1
- package/dist/src/review/post-merge-review.js +6 -1
- package/dist/src/review/review-precheck.js +72 -28
- package/dist/src/roles/role-catalog.js +13 -3
- package/dist/src/roles/role-render.js +158 -83
- package/dist/src/roles/role-view.js +272 -71
- package/dist/src/roles/roles.js +1 -1
- package/dist/src/run/ci-parity.js +15 -2
- package/dist/src/run/run-apply.js +0 -6
- package/dist/src/run/run-detail-render.js +2 -2
- package/dist/src/run/run-finalize.js +4 -0
- package/dist/src/run/run-timing.js +2 -2
- package/dist/src/run/run-usage.js +9 -1
- package/dist/src/run/run-verdict.js +79 -51
- package/dist/src/runner/runner-outcome.js +0 -1
- package/dist/src/runner/runner-pi.js +6 -0
- package/dist/src/runner/runner-recovery.js +5 -4
- package/dist/src/runner/runner-revision.js +2 -1
- package/dist/src/runner/runner-run.js +154 -108
- package/dist/src/runner/runner-state-load.js +88 -42
- package/dist/src/runner/runner-state.js +3 -2
- package/dist/src/runner/runner.js +39 -20
- package/dist/src/{orchestrator/scheduling-admit.js → scheduling/admit.js} +2 -2
- package/dist/src/scheduling/backoff.js +6 -6
- package/dist/src/scheduling/claims.js +10 -12
- package/dist/src/{orchestrator/scheduling-scope.js → scheduling/scope.js} +9 -59
- package/dist/src/{orchestrator/scheduling-skip.js → scheduling/skip.js} +112 -78
- package/dist/src/status/status-budget.js +38 -4
- package/dist/src/status/status-data.js +23 -14
- package/dist/src/status/status-polls.js +82 -25
- package/dist/src/status/status-role-notes.js +25 -7
- package/dist/src/status/status-roles.js +56 -25
- package/dist/src/team/team-orchestrator-info.js +51 -0
- package/dist/src/team/team-state.js +25 -9
- package/dist/src/text/format.js +20 -2
- package/dist/src/text/phrases.js +57 -10
- package/dist/src/text/suggest.js +14 -1
- package/dist/src/text/text.js +30 -1
- package/dist/src/ui/badges.js +102 -17
- package/dist/src/ui/gui/gui-client-drawer-config.js +12 -6
- package/dist/src/ui/gui/gui-client-model.js +4 -0
- package/dist/src/ui/gui/gui-client-operator.js +11 -14
- package/dist/src/ui/gui/gui-client-pending.js +10 -3
- package/dist/src/ui/gui/gui-client-report-chart.js +22 -15
- package/dist/src/ui/gui/gui-client-settings.js +5 -1
- package/dist/src/ui/gui/gui-client-team.js +7 -0
- package/dist/src/ui/status-model.js +21 -6
- package/dist/src/ui/status-payload.js +19 -5
- package/dist/src/ui/status-render.js +14 -8
- package/dist/src/ui/team-alert-shared.js +2 -2
- package/dist/src/ui/team-alert-standings.js +96 -0
- package/dist/src/ui/team-alert-status.js +39 -12
- package/dist/src/ui/team-alerts.js +8 -4
- package/dist/src/ui/tui/tui-backlog.js +12 -0
- package/dist/src/ui/tui/tui-editor-interval.js +2 -2
- package/dist/src/ui/tui/tui-editor-role-model.js +5 -1
- package/dist/src/ui/tui/tui-input.js +2 -2
- package/dist/src/ui/tui/tui-pane-backlog.js +2 -2
- package/dist/src/ui/tui/tui-pane-health.js +25 -21
- package/dist/src/ui/tui/tui-pane-history.js +156 -471
- package/dist/src/ui/tui/tui-pane-pending-selection.js +346 -0
- package/dist/src/ui/tui/tui-pane-pending.js +46 -322
- package/dist/src/ui/tui/tui-pane-queued.js +2 -2
- package/dist/src/ui/tui/tui-pane-trail.js +420 -0
- package/dist/src/ui/tui/tui-pane.js +3 -3
- package/dist/src/ui/tui/tui.js +2 -1
- package/dist/src/verdict/fix-claim.js +10 -4
- package/dist/src/verdict/reply-contract.js +2 -4
- package/package.json +9 -7
- package/dist/src/merge/merge-check-failures.js +0 -212
- package/dist/src/merge/merge-stack.js +0 -235
|
@@ -115,13 +115,17 @@ function redFailingTest(newest) {
|
|
|
115
115
|
return Array.isArray(newest.failingTests) && typeof newest.failingTests[0] === "string" ? newest.failingTests[0] : undefined;
|
|
116
116
|
}
|
|
117
117
|
/** The blame clause naming the merge the main watch blamed, when the newest `culprit` in the
|
|
118
|
-
* window
|
|
119
|
-
function redBlameClause(culprit, newest) {
|
|
118
|
+
* window names the spell's opener or its newest red. Owns the blame clause. */
|
|
119
|
+
function redBlameClause(culprit, startEv, newest) {
|
|
120
120
|
// The blame names the merge the main watch blamed for the red — the same `main_culprit`
|
|
121
121
|
// commit-equals-tip rule the dashboard alert uses (status-polls only publishes a blame whose
|
|
122
122
|
// commit is the derived tip sha), so a red the watch did not blame never borrows another
|
|
123
123
|
// head's culprit. A culprit naming another commit, or none, leaves the phrase as today.
|
|
124
|
-
|
|
124
|
+
// The watch names the commit that OPENED the red (its first red tip since the last green)
|
|
125
|
+
// even when a later tip re-checks red, so the match accepts the spell's opener as well as the
|
|
126
|
+
// newest red tip: the opener is what the watch blames for the ongoing spell, and a
|
|
127
|
+
// pre-fix log that named the newest tip instead still matches its own newest red.
|
|
128
|
+
const shouldBlame = culprit !== undefined && (culprit.commit === newest.sha || culprit.commit === startEv.sha);
|
|
125
129
|
return shouldBlame
|
|
126
130
|
? typeof culprit.runner === "string" && culprit.runner.length > 0
|
|
127
131
|
? `; the main watch blames ${roleLabel(culprit.runner)}'s merge ${shortSha(culprit.commit)}`
|
|
@@ -134,9 +138,9 @@ function redBlameClause(culprit, newest) {
|
|
|
134
138
|
* ignored; a pass (including one naming only quarantined tests, recorded passed) ends the red.
|
|
135
139
|
* The phrase names how long main has had no passing full check, the sha it has been red since,
|
|
136
140
|
* its clock time, the newest red's first failing test when one is named, and — when the newest
|
|
137
|
-
* `main_culprit` in the window
|
|
138
|
-
* blamed (`; the main watch blames <role>'s merge <sha>`, or `; the main watch blames
|
|
139
|
-
* <sha>` when no runner is named), the one rule the dashboard alert and this phrase share.
|
|
141
|
+
* `main_culprit` in the window names the spell's opener or its newest red tip — the merge the
|
|
142
|
+
* main watch blamed (`; the main watch blames <role>'s merge <sha>`, or `; the main watch blames
|
|
143
|
+
* merge <sha>` when no runner is named), the one rule the dashboard alert and this phrase share.
|
|
140
144
|
* Returns null when main is green, or red for RED_MAIN_WARN_MS or less. The per-job work is
|
|
141
145
|
* split into the module-private helpers `scanRedWindow`, `redSpell`, `redSpellDuration`,
|
|
142
146
|
* `redFailingTest` and `redBlameClause`, one per job, so each is found by name. */
|
|
@@ -148,7 +152,7 @@ export function redMainWarning(events, now) {
|
|
|
148
152
|
const duration = redSpellDuration(spell);
|
|
149
153
|
const head = `main has had no passing full check for ${duration}: red since ${shortSha(spell.startEv.sha)} at ${clockHm(spell.startEv.ts)}`;
|
|
150
154
|
const failing = redFailingTest(spell.newest);
|
|
151
|
-
const blame = redBlameClause(window.culprit, spell.newest);
|
|
155
|
+
const blame = redBlameClause(window.culprit, spell.startEv, spell.newest);
|
|
152
156
|
return failing ? `${head} — ${squash(failing, 120)}${blame}` : `${head}${blame}`;
|
|
153
157
|
}
|
|
154
158
|
/** G3 over `entries` (the loadQuarantine list): warns when the list holds more than
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import { baseRoleOf } from "../roles/runner-ids.js";
|
|
2
2
|
import { enqueueRolePrompt } from "../inbox/inbox.js";
|
|
3
3
|
import { liveConfig } from "../config/config.js";
|
|
4
|
-
import { mergeCommandOf } from "../build/build-check-detect.js";
|
|
4
|
+
import { mergeCommandOf, rerunCommandOf } from "../build/build-check-detect.js";
|
|
5
|
+
import { canRerun, rerunCommandLine, rerunTests } from "../build/build-check-rerun.js";
|
|
5
6
|
import { failureHeadline, FAILING_TESTS_MAX } from "../build/build-check-report.js";
|
|
6
7
|
import { baselineCheckEventLogger } from "../build/build-check-events.js";
|
|
7
|
-
import { checkMainBaseline, watchedBaseline } from "./main-baseline.js";
|
|
8
|
+
import { checkMainBaseline, newestObservedBaseline, watchedBaseline } from "./main-baseline.js";
|
|
8
9
|
import { buildMainRedNote } from "../prompt/gate-notes.js";
|
|
9
10
|
import { logEventBestEffort, warnEventBestEffort, TUMWATER_SOURCE } from "../events/events.js";
|
|
10
11
|
import { errorMessage } from "../text/text.js";
|
|
11
12
|
import { mainRedPhrase, plural } from "../text/phrases.js";
|
|
12
13
|
import { clipToWidth } from "../text/text-width.js";
|
|
13
14
|
import { shortSha } from "../text/format.js";
|
|
14
|
-
import { branchHead } from "../git/git.js";
|
|
15
|
+
import { branchHead, headOf, isMergedInto } from "../git/git.js";
|
|
15
16
|
import { gateMainWorktreePath } from "../paths.js";
|
|
16
17
|
import { useWorktree } from "../git/worktree-use.js";
|
|
17
18
|
import { ensureDetachedWorktree } from "../git/worktree.js";
|
|
@@ -41,8 +42,8 @@ let lastHealKey = null;
|
|
|
41
42
|
/** The failing tests a red verdict names, when the list is complete — non-empty and short of
|
|
42
43
|
* FAILING_TESTS_MAX, which may have been cut short (the same rule as onlyKnownMainFailures).
|
|
43
44
|
* Empty when it names none: main's red is then not a set of known failures, and the heal is
|
|
44
|
-
* keyed to the red SHA rather than a failing set (the event reads "
|
|
45
|
-
*
|
|
45
|
+
* keyed to the red SHA rather than a failing set (the event reads "a change whose own check
|
|
46
|
+
* fails is kept until main moves"). */
|
|
46
47
|
export function namedFailures(red) {
|
|
47
48
|
const tests = red.failingTests ?? [];
|
|
48
49
|
return tests.length > 0 && tests.length < FAILING_TESTS_MAX ? tests : [];
|
|
@@ -81,7 +82,7 @@ function warnMainRedOnce(root, red, foundBy) {
|
|
|
81
82
|
...(named.length > 0 ? { failingTests: named } : {}),
|
|
82
83
|
message: named.length > 0
|
|
83
84
|
? `${mainRedPhrase(red.sha)} (${plural(named.length, "failing test")}: ${clipToWidth(named[0] ?? "", 80)}${named.length > 1 ? `, +${named.length - 1} more` : ""}); changes that add no new failures still merge`
|
|
84
|
-
: redMainMessage(red, "
|
|
85
|
+
: redMainMessage(red, "a change whose own check fails is kept until main moves"),
|
|
85
86
|
});
|
|
86
87
|
// The ci role is the healer, and it schedules as an observer, so a red main discovered by any
|
|
87
88
|
// other path could sit unrepaired until ci's own interval or a coincidental wake.
|
|
@@ -116,8 +117,22 @@ export function mainRedHealPrompt(sha, detail) {
|
|
|
116
117
|
if (!detail)
|
|
117
118
|
return base;
|
|
118
119
|
const tests = detail.failingTests.length > 0 ? ` Failing: ${detail.failingTests.join("; ")}.` : "";
|
|
120
|
+
const rerun = detail.rerun ? ` ${rerunAdvice(detail.rerun)}` : "";
|
|
119
121
|
const tail = detail.outputTail && detail.outputTail.length > 0 ? ` Output tail: ${detail.outputTail.join(" | ")}` : "";
|
|
120
|
-
return `${base} The watch named ${shortSha(detail.culprit)} as the commit that turned it red.${tests}${tail}`;
|
|
122
|
+
return `${base} The watch named ${shortSha(detail.culprit)} as the commit that turned it red.${tests}${rerun}${tail}`;
|
|
123
|
+
}
|
|
124
|
+
/** The heal's one instruction about re-running: the command that runs only main's failing tests,
|
|
125
|
+
* for reproducing them and for proving the fix, instead of the whole suite. */
|
|
126
|
+
function rerunAdvice(command) {
|
|
127
|
+
return `Run only the failing tests, to reproduce them and again to prove the fix, with: ${command}`;
|
|
128
|
+
}
|
|
129
|
+
/** The command line that re-runs `red`'s named failures on their own (rerunCommandLine), or
|
|
130
|
+
* undefined when they cannot be re-run alone (canRerun) — the healer is then left to the full
|
|
131
|
+
* check. */
|
|
132
|
+
function healRerunLine(root, config, red) {
|
|
133
|
+
const tests = namedFailures(red);
|
|
134
|
+
const command = canRerun(root, config, tests) ? rerunCommandOf(root, config) : undefined;
|
|
135
|
+
return command === undefined ? undefined : rerunCommandLine(command, tests, red.failingTestFiles);
|
|
121
136
|
}
|
|
122
137
|
/** Queue the heal prompt for ci with the culprit's detail: the one way a named red-main culprit
|
|
123
138
|
* is answered (Forward, never back, part 4/6): the watch's onCulprit hand-off routes through
|
|
@@ -128,7 +143,8 @@ export function mainRedHealPrompt(sha, detail) {
|
|
|
128
143
|
* the orchestrator. */
|
|
129
144
|
export function queueCulpritHeal(root, tip, detail) {
|
|
130
145
|
try {
|
|
131
|
-
|
|
146
|
+
const rerun = healRerunLine(root, liveConfig(root), { failingTests: [...detail.failingTests], failingTestFiles: detail.failingTestFiles });
|
|
147
|
+
enqueueRolePrompt(root, "ci", mainRedHealPrompt(tip, { ...detail, ...(rerun ? { rerun } : {}) }));
|
|
132
148
|
}
|
|
133
149
|
catch (err) {
|
|
134
150
|
warnEventBestEffort(root, TUMWATER_SOURCE, `could not queue the heal prompt for ci: ${errorMessage(err)}`);
|
|
@@ -138,8 +154,8 @@ export function queueCulpritHeal(root, tip, detail) {
|
|
|
138
154
|
* that paid for it, with its duration — the gate's build_check sibling, so both halves of the
|
|
139
155
|
* team's deterministic verification are priced in the feed. The one logger for every baseline
|
|
140
156
|
* check this module runs — the ci handoff, the main watch, and mainTipVerdict's tip
|
|
141
|
-
* check (which review
|
|
142
|
-
*
|
|
157
|
+
* check (which the review pre-check reaches through it) — so the event's shape cannot drift
|
|
158
|
+
* between them. */
|
|
143
159
|
export function baselineCheckLogger(root, role) {
|
|
144
160
|
return baselineCheckEventLogger(role, (event) => logEventBestEffort(root, event));
|
|
145
161
|
}
|
|
@@ -169,7 +185,7 @@ export async function mainRedHealNote(root, role, wt) {
|
|
|
169
185
|
if (red?.status !== "red")
|
|
170
186
|
return undefined;
|
|
171
187
|
warnMainRedOnce(root, red, role);
|
|
172
|
-
return buildMainRedNote(red.sha, red.script, failureHeadline(red.outputTail), red.outputTail);
|
|
188
|
+
return buildMainRedNote(red.sha, red.script, failureHeadline(red.outputTail), red.outputTail, healRerunLine(wt, config, red));
|
|
173
189
|
}
|
|
174
190
|
/** Serializes mainTipVerdict's use of its one worktree: the merge pipeline's verifies run their
|
|
175
191
|
* gates concurrently, and a second gate re-pointing the checkout at a newer main while the first's
|
|
@@ -233,7 +249,7 @@ async function verdictAtMainTipIn(root, role, mainBranch, config, forceFresh) {
|
|
|
233
249
|
return { status: "green", sha: baseline.sha };
|
|
234
250
|
if (baseline?.status === "red") {
|
|
235
251
|
warnMainRedOnce(root, baseline, role);
|
|
236
|
-
return { status: "red", sha: baseline.sha };
|
|
252
|
+
return { status: "red", sha: baseline.sha, failingTests: baseline.failingTests ?? [] };
|
|
237
253
|
}
|
|
238
254
|
return {
|
|
239
255
|
status: "unavailable",
|
|
@@ -248,3 +264,75 @@ async function verdictAtMainTipIn(root, role, mainBranch, config, forceFresh) {
|
|
|
248
264
|
return { status: "unavailable", why: errorMessage(err) };
|
|
249
265
|
}
|
|
250
266
|
}
|
|
267
|
+
/** Main's verdict at its tip on a gate's twice-failed check, for an attribution that has to run
|
|
268
|
+
* something: when the failures name their tests and those can be re-run alone (canRerun), only
|
|
269
|
+
* they run, in the `_gate-main` mirror at main's tip — a red names the ones that fail there too,
|
|
270
|
+
* a green says none does. Otherwise, or when the re-run reaches no verdict, main's full check runs
|
|
271
|
+
* fresh (mainTipVerdict with forceFresh), as before. A re-run is no full check: it never touches
|
|
272
|
+
* the verdict cache and warns nothing. Never throws. */
|
|
273
|
+
export async function mainTipFailures(root, role, wt, mainBranch, config, failed) {
|
|
274
|
+
const rerun = failed ? await rerunAtMainTip(root, role, wt, mainBranch, config, failed) : null;
|
|
275
|
+
return rerun ?? mainTipVerdict(root, role, mainBranch, config, true);
|
|
276
|
+
}
|
|
277
|
+
/** mainTipFailures's re-run alone: `failed`'s tests run on their own in the `_gate-main` mirror at
|
|
278
|
+
* main's tip, queued behind the gates and the main watch that share it. Null when they cannot be
|
|
279
|
+
* re-run alone (canRerun) or the re-run reaches no verdict. Never throws. */
|
|
280
|
+
export async function rerunAtMainTip(root, role, wt, mainBranch, config, failed) {
|
|
281
|
+
if (!canRerun(wt, config, failed.tests))
|
|
282
|
+
return null;
|
|
283
|
+
const rerun = await withMainTipWorktree(root, mainBranch, async (mirror) => {
|
|
284
|
+
const result = await rerunTests(root, role, mirror, config, failed.tests, failed.files);
|
|
285
|
+
return result ? { result, sha: await headOf(mirror, "HEAD") } : null;
|
|
286
|
+
}).catch(() => null);
|
|
287
|
+
if (!rerun)
|
|
288
|
+
return null;
|
|
289
|
+
const { result, sha } = rerun;
|
|
290
|
+
return result.failing.length === 0 ? { status: "green", sha } : { status: "red", sha, failingTests: result.failing };
|
|
291
|
+
}
|
|
292
|
+
/** The main watch's newest observed verdict at or below `tip` (newestObservedBaseline), and
|
|
293
|
+
* whether it covers `base`: it was observed on `base` itself or on a commit that contains it, so
|
|
294
|
+
* the tree the change was rebased onto is part of what the watch checked. Shared by the gate's
|
|
295
|
+
* attribution and the kept change's re-queue, so the two agree on when the wait is over. Never
|
|
296
|
+
* throws: a git failure reads as not covered. */
|
|
297
|
+
export async function watchVerdictCovering(wt, tip, base) {
|
|
298
|
+
const verdict = await newestObservedBaseline(wt, tip);
|
|
299
|
+
const covers = verdict !== null && (verdict.sha === base || (await isMergedInto(wt, base, verdict.sha)));
|
|
300
|
+
return { verdict, covers };
|
|
301
|
+
}
|
|
302
|
+
/** Main's verdict for attributing a gate check that failed twice, read from the main watch's
|
|
303
|
+
* verdicts instead of a full check of the gate's own (plans/fast-merge.md part 6/14). The gate
|
|
304
|
+
* used to run main's whole suite here, queued behind the watch's own runs in `_gate-main`: on
|
|
305
|
+
* 2026-10-30 that took 166–2,282 s for a 30 s gate, and two of five runs repeated a commit the
|
|
306
|
+
* watch or an earlier attribution had just checked.
|
|
307
|
+
*
|
|
308
|
+
* `base` is the commit the change was rebased onto (its merge-base with main). When the watch's
|
|
309
|
+
* newest observed verdict covers it (watchVerdictCovering), that verdict answers at once. When the
|
|
310
|
+
* watch is still behind it, the answer is `awaiting`: the gate keeps the change with no strike
|
|
311
|
+
* and the merge pipeline re-queues it once the watch catches up. Waiting instead of judging
|
|
312
|
+
* against an older verdict matters: a commit the watch has not reached yet may be what broke the
|
|
313
|
+
* test, and the change would be rejected for main's failure.
|
|
314
|
+
*
|
|
315
|
+
* `waited` is set on the retry after such a wait, and the gate does not wait twice: after
|
|
316
|
+
* `covered` the newest observed verdict answers even if main moved past it again; after
|
|
317
|
+
* `timed_out` (the watch never got there — held, failing, or main's tip only carried doc commits
|
|
318
|
+
* past its last observed check) main is asked here (mainTipFailures: the gate's `failed` tests
|
|
319
|
+
* re-run alone on main's tip when they can be, else main's full check). The same fallback answers a
|
|
320
|
+
* `covered` retry that finds no observed verdict at all. A red is not warned again here: the watch
|
|
321
|
+
* that observed it reported it. Never throws. */
|
|
322
|
+
export async function watchedMainVerdict(root, role, wt, mainBranch, config, base, waited, failed) {
|
|
323
|
+
const fallback = () => mainTipFailures(root, role, wt, mainBranch, config, failed);
|
|
324
|
+
if (waited === "timed_out")
|
|
325
|
+
return fallback();
|
|
326
|
+
const tip = await branchHead(root, mainBranch);
|
|
327
|
+
if (!tip)
|
|
328
|
+
return { status: "unavailable", why: `${mainBranch} is unreadable` };
|
|
329
|
+
const { verdict, covers } = await watchVerdictCovering(wt, tip, base);
|
|
330
|
+
if (verdict && (covers || waited === "covered")) {
|
|
331
|
+
return verdict.status === "red"
|
|
332
|
+
? { status: "red", sha: verdict.sha, failingTests: verdict.failingTests ?? [] }
|
|
333
|
+
: { status: "green", sha: verdict.sha };
|
|
334
|
+
}
|
|
335
|
+
if (waited === "covered")
|
|
336
|
+
return fallback();
|
|
337
|
+
return { status: "awaiting", ...(verdict ? { sha: verdict.sha } : {}) };
|
|
338
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { distinctOrdered } from "../collections.js";
|
|
2
|
-
import { checkMainBaseline } from "./main-baseline.js";
|
|
2
|
+
import { checkMainBaseline, settleRedByRerun } from "./main-baseline.js";
|
|
3
|
+
import { canRerun, rerunTests } from "../build/build-check-rerun.js";
|
|
3
4
|
import { baselineCheckLogger, clearHealKeyOnGreen, withMainTipWorktree } from "./main-red.js";
|
|
4
5
|
import { CHECK_TIER } from "../concurrency/check-permit.js";
|
|
5
6
|
import { FAILING_TESTS_MAX } from "../build/build-check-report.js";
|
|
@@ -32,6 +33,14 @@ function mainCheckEvent(baseline, red, quarantined, covered, durationMs) {
|
|
|
32
33
|
}
|
|
33
34
|
/** The failing tests of each root's newest watch verdict, keyed by root. */
|
|
34
35
|
const knownFailures = new Map();
|
|
36
|
+
/** The commit each root's red spell was opened at, keyed by root — the tip of the first red
|
|
37
|
+
* verdict since the last green. It stays the named culprit while the spell is open: a later tip
|
|
38
|
+
* re-checks red (the red never inherits across a doc commit) but did not open it, so it is not
|
|
39
|
+
* re-attributed. A green verdict clears it, so a red that opens after a green names its own
|
|
40
|
+
* opener. In-memory only: a restart mid-red loses the spell's opener, and the next settled red
|
|
41
|
+
* tip then names itself as culprit — the pre-fix behaviour, and a bounded one (one tip per
|
|
42
|
+
* restart, since a spell's later tips inherit the red and settle without a re-run). */
|
|
43
|
+
const redSpellOpeners = new Map();
|
|
35
44
|
/** Main's known failures: the failing tests of the watch's newest verdict — main's tip, or the
|
|
36
45
|
* nearest tip it checked while the newest head waits — and every quarantined flaky test
|
|
37
46
|
* (review/known-flakes.ts). Empty when the verdict is green, red with no parsed test, or none has
|
|
@@ -138,10 +147,14 @@ export function createMainWatch(root, mainBranch, onCulprit) {
|
|
|
138
147
|
}
|
|
139
148
|
}
|
|
140
149
|
};
|
|
141
|
-
/** Run the declared build check on main's tip,
|
|
142
|
-
* whose
|
|
143
|
-
*
|
|
144
|
-
*
|
|
150
|
+
/** Run the declared build check on main's tip, confirming a red once and quarantining a flake
|
|
151
|
+
* whose confirmation passes. A red whose failing tests can be re-run on their own (canRerun) is
|
|
152
|
+
* confirmed by re-running just those tests, so the full check runs once per tip: a test that
|
|
153
|
+
* passes there is quarantined, and the red narrows to the tests that failed again (or turns
|
|
154
|
+
* green when none did). A re-run that reaches no verdict leaves the first red standing. Without
|
|
155
|
+
* a re-run command the whole check runs again, as before. Returns the check's result and
|
|
156
|
+
* whether any full run actually executed (false for a cache hit, a skip, or a sleep-spanned
|
|
157
|
+
* run). `config` and `root` are captured from `watchOnce`. */
|
|
145
158
|
async function runBaselineCheck(wt) {
|
|
146
159
|
let ran = false;
|
|
147
160
|
const onRun = baselineCheckLogger(root, TUMWATER_SOURCE);
|
|
@@ -150,13 +163,27 @@ export function createMainWatch(root, mainBranch, onCulprit) {
|
|
|
150
163
|
onRun(run);
|
|
151
164
|
};
|
|
152
165
|
let check = await checkMainBaseline(wt, config, hook, false, false, undefined, false, CHECK_TIER.merge);
|
|
153
|
-
if (check.baseline?.status
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
166
|
+
if (check.baseline?.status !== "red")
|
|
167
|
+
return { check, ran };
|
|
168
|
+
const first = check.baseline;
|
|
169
|
+
const named = first.failingTests ?? [];
|
|
170
|
+
if (canRerun(wt, config, named)) {
|
|
171
|
+
const rerun = await rerunTests(root, TUMWATER_SOURCE, wt, config, named, first.failingTestFiles, CHECK_TIER.merge);
|
|
172
|
+
if (rerun) {
|
|
173
|
+
// Each test that failed and then passed on this same tree is a flake.
|
|
174
|
+
const passed = named.filter((t) => !rerun.failing.includes(t));
|
|
175
|
+
if (passed.length > 0)
|
|
176
|
+
quarantineTests(root, passed, passed.join("\n"));
|
|
177
|
+
const settled = settleRedByRerun(first.sha, rerun.failing);
|
|
178
|
+
if (settled)
|
|
179
|
+
check = { baseline: settled };
|
|
180
|
+
}
|
|
181
|
+
return { check, ran };
|
|
159
182
|
}
|
|
183
|
+
check = await checkMainBaseline(wt, config, hook, true, false, undefined, false, CHECK_TIER.merge);
|
|
184
|
+
// A red the same tree's confirming re-run turns green is a flake: quarantine its tests.
|
|
185
|
+
if (check.baseline?.status === "green")
|
|
186
|
+
quarantineTests(root, named, named.join("\n"));
|
|
160
187
|
return { check, ran };
|
|
161
188
|
}
|
|
162
189
|
/** The covered-commit count for a main_check event: the first-parent commits between the
|
|
@@ -179,24 +206,37 @@ export function createMainWatch(root, mainBranch, onCulprit) {
|
|
|
179
206
|
logEventBestEffort(root, mainCheckEvent(baseline, red, quarantined, covered, ran ? Date.now() - started : 0));
|
|
180
207
|
return red;
|
|
181
208
|
}
|
|
182
|
-
/** Name a settled red tip as the culprit:
|
|
183
|
-
* the onCulprit callback with the tip and its failing tests.
|
|
209
|
+
/** Name a settled red tip's spell as the culprit: the merging runner, the main_culprit
|
|
210
|
+
* event, and the onCulprit callback with the tip and its failing tests. The commit named is
|
|
211
|
+
* the one that OPENED the red — the first red tip since the last green — not the newest red
|
|
212
|
+
* tip: a doc-exempt tip that lands after the red opened cannot have changed the suite, so
|
|
213
|
+
* blaming it would name a merge in the G1 phrase (src/baseline/main-guardrails.ts) and the
|
|
214
|
+
* ci heal prompt (src/baseline/main-red.ts) that did not break anything. The opener is named
|
|
215
|
+
* once per spell with the tip's failing tests and, when the newest merged event names the
|
|
216
|
+
* opener, its runner; while the spell is open a later tip re-settling red re-attributes
|
|
217
|
+
* nothing. */
|
|
184
218
|
function reportCulprit(baseline) {
|
|
185
|
-
//
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
//
|
|
189
|
-
|
|
219
|
+
// A red spell that is already open keeps its opener named: the first red tip since the
|
|
220
|
+
// last green caused the red, so a later tip re-settling red (a doc-exempt tip cannot
|
|
221
|
+
// change the suite, and a code tip does not open a SECOND spell until a green closed the
|
|
222
|
+
// first) is not re-attributed — no new event, no new heal prompt (the heal dedup already
|
|
223
|
+
// re-queues when the failing set returns after a green).
|
|
224
|
+
if (redSpellOpeners.has(root))
|
|
225
|
+
return;
|
|
226
|
+
const culprit = baseline.sha;
|
|
227
|
+
redSpellOpeners.set(root, culprit);
|
|
228
|
+
const runner = mergingRoleOf(root, culprit);
|
|
190
229
|
logEventBestEffort(root, {
|
|
191
230
|
source: TUMWATER_SOURCE,
|
|
192
231
|
type: "main_culprit",
|
|
193
|
-
commit:
|
|
232
|
+
commit: culprit,
|
|
194
233
|
...(runner !== undefined ? { runner } : {}),
|
|
195
234
|
failingTests: baseline.failingTests ?? [],
|
|
196
235
|
});
|
|
197
236
|
onCulprit?.({
|
|
198
|
-
commit:
|
|
237
|
+
commit: culprit,
|
|
199
238
|
failingTests: baseline.failingTests ?? [],
|
|
239
|
+
...(baseline.failingTestFiles ? { failingTestFiles: baseline.failingTestFiles } : {}),
|
|
200
240
|
tip: baseline.sha,
|
|
201
241
|
});
|
|
202
242
|
}
|
|
@@ -232,8 +272,14 @@ export function createMainWatch(root, mainBranch, onCulprit) {
|
|
|
232
272
|
// is red.
|
|
233
273
|
if (!red) {
|
|
234
274
|
clearHealKeyOnGreen();
|
|
275
|
+
redSpellOpeners.delete(root); // a green closes the red spell; the next red names its own opener
|
|
235
276
|
return;
|
|
236
277
|
}
|
|
278
|
+
// The spell's opener is blamed once: the first settled red tip since the last green names
|
|
279
|
+
// itself, and a later tip re-settling the same open red re-checks the suite without re-
|
|
280
|
+
// naming the culprit (BUGS.md 2026-10-30 mis-attribution). The map holds the open spell's
|
|
281
|
+
// opener, so a second red while the spell is still open re-checks without re-naming; a
|
|
282
|
+
// green clears the entry (above) so a later red names its own opener.
|
|
237
283
|
reportCulprit(baseline);
|
|
238
284
|
});
|
|
239
285
|
}
|
|
@@ -4,18 +4,19 @@
|
|
|
4
4
|
* stage-check.ts). This module owns everything pollBudgetGate does from its `onFallback` seam
|
|
5
5
|
* through the end of the function: the seam/backends walk, the motion key (`resolvedKey`,
|
|
6
6
|
* `fromKey`), the fallback view derivation (`fallbackConfig`), `chainInfo`, and the `pairs`
|
|
7
|
-
* handback list. It no longer moves any tier for a used-up subscription window (part 5d/6):
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
7
|
+
* handback list. It no longer moves any tier for a used-up subscription window (part 5d/6): a
|
|
8
|
+
* spent `maxDailyCostUsd` moves every priced tier to its free seam, and, at any spend, a tier
|
|
9
|
+
* whose own primary failed its model check moves to a verified step of its chain
|
|
10
|
+
* (plans/model-check.md "Fallback"). Its one entry reads the inputs pollBudgetGate's first half
|
|
11
|
+
* already resolved and returns the view fields `BudgetGatePoll` carries, so pollBudgetGate keeps
|
|
12
|
+
* its call signature and src/gates/gate-polls.ts's destructure of it is unchanged. The plan
|
|
13
|
+
* references the moved comments carry (part 5b/8, part 5c/8, 6a/7) hold here. */
|
|
13
14
|
import { MODEL_TIERS, } from "../config/config-schema.js";
|
|
14
15
|
import { modelPairName, pairDailyCost } from "../budget/budget.js";
|
|
15
16
|
import { fallbackServingPair, } from "../budget/fallback-breaker.js";
|
|
16
17
|
import { applyFallbackModel, tierChainSteps, } from "../config/config-fallbacks.js";
|
|
17
18
|
import { seamAgent } from "../config/config-model-selector.js";
|
|
18
|
-
import { withModelOverrides } from "../config/config-views.js";
|
|
19
|
+
import { seamPairName, withModelOverrides } from "../config/config-views.js";
|
|
19
20
|
import { piPairFree, seamFree } from "../pi/pi-models.js";
|
|
20
21
|
import { dayAt } from "../text/datetime.js";
|
|
21
22
|
/** Can the account not take extra usage: some window is rejected, or the newest non-reset
|
|
@@ -44,21 +45,41 @@ export function subscriptionBackendOf(x, accounts) {
|
|
|
44
45
|
const account = accounts(id);
|
|
45
46
|
return account?.billing === "subscription" ? id : null;
|
|
46
47
|
}
|
|
48
|
+
/** The pair a tier's own primary resolves to under the live config — the name the model check
|
|
49
|
+
* keys it by. */
|
|
50
|
+
function tierPrimaryPair(config, tier) {
|
|
51
|
+
return seamPairName(withModelOverrides(config, tier, {}));
|
|
52
|
+
}
|
|
53
|
+
/** `base` with each check-moved tier's entry replaced by its verified step. */
|
|
54
|
+
function withCheckMoves(base, checkResolved, checkMoved) {
|
|
55
|
+
const out = { ...base };
|
|
56
|
+
for (const tier of checkMoved)
|
|
57
|
+
out[tier] = checkResolved[tier];
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
47
60
|
/** Install `state.fallbackConfig` when the fallback changed the view key: at the cap, the free
|
|
48
61
|
* pair installed top-level and every per-role/reviewer model override dropped, so no seam can
|
|
49
62
|
* reach a priced model. A breaker-demoted fallback keeps the view too: its gate is `paused`,
|
|
50
63
|
* but a run parked in the semaphore when it tripped, the half-open probe, and the merges must
|
|
51
64
|
* still run on the free pair — a demotion must never promote them to the priced model the cap
|
|
52
|
-
* already spent.
|
|
53
|
-
|
|
65
|
+
* already spent. A tier the model check moved (deriveCheckMovedTiers) installs its verified
|
|
66
|
+
* step at any spend, and only its own primary seams move: every other tier, and a role or
|
|
67
|
+
* reviewer pinned to some other pair, keeps its model. */
|
|
68
|
+
function installFallbackConfig(a, fromKey, checkMoved) {
|
|
54
69
|
const { state, config: liveConfig, providers, catalog, accounts, priceResolved, reached } = a;
|
|
55
|
-
if (reached && (state.from !== liveConfig || state.fromKey !== fromKey)) {
|
|
70
|
+
if ((reached || checkMoved.size > 0) && (state.from !== liveConfig || state.fromKey !== fromKey)) {
|
|
56
71
|
state.from = liveConfig;
|
|
57
72
|
state.fromKey = fromKey;
|
|
73
|
+
const primaries = new Map([...checkMoved].map((t) => [t, tierPrimaryPair(liveConfig, t)]));
|
|
74
|
+
const checkKeeps = (seam, tier) => primaries.get(tier) !== seamPairName(seam);
|
|
58
75
|
// At the cap only free seams are available (a subscription claude seam is free money), so
|
|
59
76
|
// every role/reviewer override that resolves to a priced seam drops to the fallback pair.
|
|
60
|
-
const
|
|
61
|
-
|
|
77
|
+
const keeps = reached
|
|
78
|
+
? (seam, tier) => seamFree(providers, seam, accounts, catalog) && checkKeeps(seam, tier)
|
|
79
|
+
: checkKeeps;
|
|
80
|
+
// The cap's base view is the check-aware resolution while the check is on (a cap-moved
|
|
81
|
+
// tier's failed step stays out of the config its roles run under), else the price view.
|
|
82
|
+
state.fallbackConfig = applyFallbackModel(liveConfig, withCheckMoves(a.checkFreeResolved ?? priceResolved, a.checkResolved, checkMoved), keeps);
|
|
62
83
|
}
|
|
63
84
|
}
|
|
64
85
|
/** The tiers `applyFallbackModel` moved at the cap: those whose own seam fails the `available`
|
|
@@ -73,28 +94,64 @@ function deriveMovedTiers(a) {
|
|
|
73
94
|
}
|
|
74
95
|
return movedTiers;
|
|
75
96
|
}
|
|
97
|
+
/** The tiers the model check moves (plans/model-check.md "Fallback"): a tier still on its own
|
|
98
|
+
* primary — every tier below the cap, at it only one the cap leaves in place (`capMoved` holds
|
|
99
|
+
* the rest) — whose primary failed its check while the tier's verified-only resolution has a
|
|
100
|
+
* step to run on. A tier whose primary failed with no verified step stays held on it. */
|
|
101
|
+
function deriveCheckMovedTiers(a, capMoved) {
|
|
102
|
+
const moved = new Set();
|
|
103
|
+
if (a.checkResolved === undefined || a.checkFailed === undefined)
|
|
104
|
+
return moved;
|
|
105
|
+
for (const tier of MODEL_TIERS) {
|
|
106
|
+
if (capMoved.has(tier) || a.checkResolved[tier].pair === null)
|
|
107
|
+
continue;
|
|
108
|
+
if (a.checkFailed(tierPrimaryPair(a.config, tier)))
|
|
109
|
+
moved.add(tier);
|
|
110
|
+
}
|
|
111
|
+
return moved;
|
|
112
|
+
}
|
|
76
113
|
/** Derive the fallback view for one poll from the verdict half's already-resolved inputs
|
|
77
|
-
* (`priceResolved`, `servingResolved`, `
|
|
78
|
-
* the shared reads (`state`, `config`, `accounts`,
|
|
79
|
-
* cross-poll view state (`from`/`fromKey
|
|
80
|
-
*
|
|
81
|
-
*
|
|
114
|
+
* (`priceResolved`, `servingResolved`, `checkResolved`, `checkFreeResolved`, `chainPairs`,
|
|
115
|
+
* `pairNames`) and the `reached` flag and the shared reads (`state`, `config`, `accounts`,
|
|
116
|
+
* `providers`, `catalog`), writing the cross-poll view state (`from`/`fromKey`/
|
|
117
|
+
* `fallbackConfig`) and returning the view
|
|
118
|
+
* fields pollBudgetGate hands back. The view is installed at the cap, and below it while a tier
|
|
119
|
+
* the model check moved runs on its verified step; `onFallback` (the image-reroute regime) stays
|
|
120
|
+
* the cap's alone. The director always keeps the live config (part 5d/6): with the window move
|
|
121
|
+
* gone no input moves it, so `directorConfig` is the live config by construction. */
|
|
82
122
|
export function budgetFallbackView(a) {
|
|
83
123
|
const { state, config: liveConfig, reached, priceResolved } = a;
|
|
84
124
|
const onFallback = reached && a.pairNames.length > 0;
|
|
125
|
+
const movedTiers = deriveMovedTiers(a);
|
|
126
|
+
const checkMoved = deriveCheckMovedTiers(a, movedTiers);
|
|
85
127
|
// The resolved pairs are part of the key: a priced step crossing its own daily cap changes
|
|
86
|
-
// the resolution without a config or window change, and the view must follow it.
|
|
128
|
+
// the resolution without a config or window change, and the view must follow it. So are the
|
|
129
|
+
// model check's moves, appended only while there are some.
|
|
87
130
|
const resolvedKey = MODEL_TIERS.map((t) => {
|
|
88
131
|
const rp = priceResolved[t].pair;
|
|
89
132
|
return rp ? modelPairName(rp) : "-";
|
|
90
133
|
}).join(",");
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
134
|
+
const checkKey = MODEL_TIERS.map((t) => (checkMoved.has(t) ? modelPairName(a.checkResolved[t].pair) : "-")).join(",");
|
|
135
|
+
// The cap view's check-aware pairs follow the price pairs when present: a check verdict
|
|
136
|
+
// failing or clearing a step changes the installed config without a price change, so the
|
|
137
|
+
// view must re-install on the next poll. Absent while the check is off, so the key is
|
|
138
|
+
// unchanged then.
|
|
139
|
+
const checkFreeKey = a.checkFreeResolved
|
|
140
|
+
? "|" +
|
|
141
|
+
MODEL_TIERS.map((t) => {
|
|
142
|
+
const rp = a.checkFreeResolved[t].pair;
|
|
143
|
+
return rp ? modelPairName(rp) : "-";
|
|
144
|
+
}).join(",")
|
|
145
|
+
: "";
|
|
146
|
+
const fromKey = `${reached ? "c" : ""}|${resolvedKey}${checkMoved.size > 0 ? `|${checkKey}` : ""}${checkFreeKey}`;
|
|
147
|
+
installFallbackConfig(a, fromKey, checkMoved);
|
|
94
148
|
const chain = chainInfo({
|
|
95
149
|
config: liveConfig,
|
|
96
|
-
resolved: a.servingResolved,
|
|
97
|
-
moved: movedTiers,
|
|
150
|
+
resolved: withCheckMoves(a.servingResolved, a.checkResolved, checkMoved),
|
|
151
|
+
moved: new Set([...movedTiers, ...checkMoved]),
|
|
152
|
+
checkMoved,
|
|
153
|
+
...(a.checkFailed !== undefined ? { modelCheckFailed: a.checkFailed } : {}),
|
|
154
|
+
...(a.checkVerified !== undefined ? { modelCheckVerified: a.checkVerified } : {}),
|
|
98
155
|
breakers: state.breakers,
|
|
99
156
|
states: a.states,
|
|
100
157
|
now: a.now,
|
|
@@ -104,16 +161,20 @@ export function budgetFallbackView(a) {
|
|
|
104
161
|
return {
|
|
105
162
|
onFallback,
|
|
106
163
|
chain,
|
|
107
|
-
roleConfig: onFallback ? state.fallbackConfig : liveConfig,
|
|
164
|
+
roleConfig: onFallback || checkMoved.size > 0 ? state.fallbackConfig : liveConfig,
|
|
108
165
|
pairs: distinctPairs(priceResolved, a.chainPairs),
|
|
109
166
|
directorConfig: liveConfig,
|
|
167
|
+
checkMoved,
|
|
110
168
|
};
|
|
111
169
|
}
|
|
112
170
|
/** The `chain` block: for each tier this poll moved off its primary, the step it runs on and
|
|
113
171
|
* why the position before it cannot run (plans/fallback-chain.md "Events and surfaces").
|
|
114
172
|
* `moved` names the tiers the poll moved; a tier the gate pauses (`pair: null`) gets no entry.
|
|
115
173
|
* The reason is read from the position just before the active step — the previous chain step,
|
|
116
|
-
* or the tier's primary at step 0 — in the order demoted, step-cap, overage, cap.
|
|
174
|
+
* or the tier's primary at step 0 — in the order demoted, step-cap, overage, model-check, cap.
|
|
175
|
+
* `checkMoved` names the moved tiers whose primary failed its model check rather than the cap
|
|
176
|
+
* moving them; the model check's verdicts say whether a previous step failed (or, for such a
|
|
177
|
+
* tier, which moves only to verified steps, has not answered yet). */
|
|
117
178
|
export function chainInfo(a) {
|
|
118
179
|
const out = {};
|
|
119
180
|
const steps = tierChainSteps(a.config);
|
|
@@ -127,7 +188,7 @@ export function chainInfo(a) {
|
|
|
127
188
|
pair: modelPairName(r.pair),
|
|
128
189
|
step: position.step,
|
|
129
190
|
of: position.of,
|
|
130
|
-
...movedTierReason(a, position.prior, position.priorName, midnight),
|
|
191
|
+
...movedTierReason(a, tier, position.prior, position.priorName, midnight),
|
|
131
192
|
};
|
|
132
193
|
Object.assign(info, movedTierSpend(a, r.pair));
|
|
133
194
|
out[tier] = info;
|
|
@@ -146,10 +207,11 @@ function movedTierPosition(steps, tier, r) {
|
|
|
146
207
|
return { step: idx + 1, of: own.length, prior, priorName: prior ? modelPairName(prior) : "" };
|
|
147
208
|
}
|
|
148
209
|
/** Why the position before the active step cannot run, in INV-BN1's fixed order (demoted,
|
|
149
|
-
* step-cap, overage, cap). A demoted or
|
|
210
|
+
* step-cap, overage, model-check, cap). A demoted, overage or model-check result carries no
|
|
211
|
+
* `returnsAt` (a model check lifts when a re-check answers, not at a clock time); a step-cap or
|
|
150
212
|
* cap result pins it to `midnight` (INV-BN2: local midnight only for step-cap and cap).
|
|
151
213
|
* Answers "why can't the position before the active step run, and when does it come back?". */
|
|
152
|
-
function movedTierReason(a, prior, priorName, midnight) {
|
|
214
|
+
function movedTierReason(a, tier, prior, priorName, midnight) {
|
|
153
215
|
if (prior && a.breakers[priorName] !== undefined && !fallbackServingPair(a.breakers, priorName)) {
|
|
154
216
|
return { reason: "demoted" };
|
|
155
217
|
}
|
|
@@ -163,6 +225,12 @@ function movedTierReason(a, prior, priorName, midnight) {
|
|
|
163
225
|
claudeOverageRefused(a.accounts("claude"))) {
|
|
164
226
|
return { reason: "overage" };
|
|
165
227
|
}
|
|
228
|
+
const checkMoved = a.checkMoved?.has(tier) === true;
|
|
229
|
+
const priorChecked = prior === null
|
|
230
|
+
? checkMoved
|
|
231
|
+
: a.modelCheckFailed?.(priorName) === true || (checkMoved && a.modelCheckVerified?.(priorName) !== true);
|
|
232
|
+
if (priorChecked)
|
|
233
|
+
return { reason: "model-check" };
|
|
166
234
|
return { reason: "cap", returnsAt: midnight };
|
|
167
235
|
}
|
|
168
236
|
/** What the active step has spent against its own cap: both keys absent unless the step is a
|
|
@@ -119,9 +119,8 @@ export function detectBuildCheck(startDir, config, maxLevels = WALK_UP_LEVELS) {
|
|
|
119
119
|
/** The configured per-change gate command (`check.gateCommand`, PLANS.md Land-queue speed 3e) when
|
|
120
120
|
* it is a non-blank string, else undefined — off. Opt-in: when no `mergeCommand` is declared, a
|
|
121
121
|
* project that sets one merges on it, not the full check (mergeCommandOf's fallback, PLANS.md
|
|
122
|
-
* "Fast merge" "Defaults"). One reading
|
|
123
|
-
* "gate")
|
|
124
|
-
* full check's), so the two cannot disagree about whether it is on. */
|
|
122
|
+
* "Fast merge" "Defaults"). One reading every caller shares (runScopedBuildCheck runs it at
|
|
123
|
+
* scope "gate"), so none can disagree about whether it is on. */
|
|
125
124
|
export function gateCommandOf(config) {
|
|
126
125
|
const g = config?.check?.gateCommand;
|
|
127
126
|
return isNonBlankString(g) ? g : undefined;
|
|
@@ -141,6 +140,26 @@ export function mergeCommandOf(config) {
|
|
|
141
140
|
return gateCommandOf(config);
|
|
142
141
|
return isNonBlankString(m) ? m : undefined;
|
|
143
142
|
}
|
|
143
|
+
/** The npm script the re-run of named tests auto-detects (rerunCommandOf). */
|
|
144
|
+
const RERUN_SCRIPT = "test:rerun";
|
|
145
|
+
/** The command that re-runs only named tests (`check.rerunCommand`, build-check-rerun.ts), or
|
|
146
|
+
* undefined — off, and its callers run the full check instead. A configured non-blank string
|
|
147
|
+
* wins; a blank one (`""`) is the explicit opt-out. Absent, a project whose check is the npm
|
|
148
|
+
* walk-up (no `check.command`) re-runs with `npm run test:rerun --` when the package.json the
|
|
149
|
+
* walk-up finds declares that script, so an npm project opts in from its own manifest. Never
|
|
150
|
+
* throws. */
|
|
151
|
+
export function rerunCommandOf(startDir, config) {
|
|
152
|
+
const r = config?.check?.rerunCommand;
|
|
153
|
+
if (r !== undefined)
|
|
154
|
+
return isNonBlankString(r) ? r : undefined;
|
|
155
|
+
if (isNonBlankString(config?.check?.command))
|
|
156
|
+
return undefined;
|
|
157
|
+
const root = walkUp(startDir, WALK_UP_LEVELS, (dir) => (hasInstall(dir) ? dir : null));
|
|
158
|
+
if (root === null)
|
|
159
|
+
return undefined;
|
|
160
|
+
const scripts = readJsonFile(path.join(root, "package.json"))?.scripts;
|
|
161
|
+
return isJsonObject(scripts) && isCheckScript(scripts, RERUN_SCRIPT) ? `npm run ${RERUN_SCRIPT} --` : undefined;
|
|
162
|
+
}
|
|
144
163
|
/** Resolve `node_modules/<rel>` by walking UP from `startDir` — the same climb detectBuildCheck
|
|
145
164
|
* makes, and the same one npm's run-script PATH walk makes: a tumwater worktree has no install
|
|
146
165
|
* of its own (node_modules is gitignored, so it exists only where someone ran npm install), and
|