tumwater 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/build-info.json +2 -2
- package/dist/src/agent/agent-backend.js +6 -6
- package/dist/src/agent/{run-agent.js → agent-run.js} +25 -12
- package/dist/src/agent/{run-settle.js → agent-settle.js} +48 -24
- package/dist/src/agent/{run-spawn.js → agent-spawn.js} +21 -12
- package/dist/src/agent/{run-stream.js → agent-stream.js} +4 -4
- package/dist/src/agent/backend-traits.js +2 -2
- package/dist/src/agent/claude/claude-account.js +1 -8
- package/dist/src/agent/claude/claude-backend.js +3 -3
- package/dist/src/agent/claude/claude-env.js +27 -31
- package/dist/src/agent/claude/claude-stream.js +111 -54
- package/dist/src/agent/codex/codex-account.js +9 -0
- package/dist/src/agent/codex/codex-backend.js +2 -2
- package/dist/src/agent/codex/codex-env.js +28 -15
- package/dist/src/agent/codex/codex-hook-trust.js +39 -20
- package/dist/src/agent/codex/codex-rollout.js +93 -56
- package/dist/src/agent/codex/codex-stream.js +1 -1
- package/dist/src/agent/env-scrub.js +18 -0
- package/dist/src/agent/image-input.js +68 -11
- package/dist/src/agent/notes-shim.js +2 -2
- package/dist/src/agent/opencode/opencode-backend.js +2 -2
- package/dist/src/agent/opencode/opencode-catalog.js +50 -27
- package/dist/src/agent/opencode/opencode-config.js +32 -14
- package/dist/src/agent/opencode/opencode-env.js +12 -13
- package/dist/src/agent/opencode/opencode-stream.js +47 -25
- package/dist/src/agent/session-ledger.js +1 -1
- package/dist/src/agent/subscription-account.js +3 -3
- package/dist/src/backlog/backlog-cli-view.js +81 -41
- package/dist/src/backlog/backlog-cli.js +57 -26
- package/dist/src/backlog/backlog-eligibility.js +58 -102
- package/dist/src/backlog/backlog-md.js +5 -5
- package/dist/src/backlog/backlog-parts.js +65 -27
- package/dist/src/backlog/backlog-release.js +12 -12
- package/dist/src/backlog/backlog-structure.js +114 -61
- package/dist/src/backlog/backlog.js +1 -1
- package/dist/src/baseline/main-baseline.js +148 -84
- package/dist/src/budget/budget-fallback-view.js +100 -243
- package/dist/src/budget/budget-wiring.js +29 -65
- package/dist/src/budget/fallback-breaker.js +2 -2
- package/dist/src/build/build-check-detect.js +43 -14
- package/dist/src/build/build-check-events.js +2 -2
- package/dist/src/build/build-check-scoped.js +40 -16
- package/dist/src/build/build-check.js +13 -5
- package/dist/src/cli/cli-command-args.js +89 -52
- package/dist/src/cli/cli-flag-specs.js +42 -22
- package/dist/src/cli/cli-init.js +34 -12
- package/dist/src/cli/cli-query-commands.js +1 -1
- package/dist/src/cli/cli-repo-gate.js +2 -2
- package/dist/src/cli/cli-start.js +91 -58
- package/dist/src/cli/completion.js +82 -46
- package/dist/src/cli/config-commands.js +82 -71
- package/dist/src/cli/help.js +4 -6
- package/dist/src/cli/log-commands.js +43 -23
- package/dist/src/cli/notify-command.js +2 -2
- package/dist/src/cli/question-commands.js +86 -60
- package/dist/src/cli.js +181 -135
- package/dist/src/collections.js +31 -16
- package/dist/src/concurrency/check-slots.js +70 -0
- package/dist/src/concurrency/lock.js +22 -0
- package/dist/src/concurrency/run-permit.js +49 -0
- package/dist/src/concurrency/semaphore.js +46 -250
- package/dist/src/config/config-editable-keys.js +4 -7
- package/dist/src/config/config-fallbacks.js +81 -101
- package/dist/src/config/config-field-checks.js +66 -37
- package/dist/src/config/config-keys.js +5 -20
- package/dist/src/config/config-live.js +4 -28
- package/dist/src/config/{model-selector.js → config-model-selector.js} +4 -4
- package/dist/src/config/config-request.js +153 -102
- package/dist/src/config/config-schema.js +9 -11
- package/dist/src/config/config-selector-checks.js +1 -1
- package/dist/src/config/config-validation-agents.js +2 -2
- package/dist/src/config/config-validation-roles.js +4 -52
- package/dist/src/config/config-validation.js +4 -38
- package/dist/src/config/config-views.js +9 -34
- package/dist/src/config/config-write.js +6 -23
- package/dist/src/config/config.js +4 -5
- package/dist/src/doctor/doctor-backlog-bugs.js +7 -6
- package/dist/src/doctor/doctor-backlog-plans.js +24 -24
- package/dist/src/doctor/doctor-backlog.js +7 -5
- package/dist/src/doctor/doctor-check-names.js +3 -6
- package/dist/src/doctor/doctor-checks.js +5 -5
- package/dist/src/doctor/doctor-claude-checks.js +44 -28
- package/dist/src/doctor/doctor-codex-checks.js +6 -6
- package/dist/src/doctor/doctor-contention-checks.js +44 -112
- package/dist/src/doctor/doctor-fallback-checks.js +2 -2
- package/dist/src/doctor/doctor-holds.js +6 -45
- package/dist/src/doctor/doctor-install.js +1 -1
- package/dist/src/doctor/doctor-launch-services.js +1 -1
- package/dist/src/doctor/doctor-model-checks.js +178 -94
- package/dist/src/doctor/doctor-notify.js +2 -2
- package/dist/src/doctor/doctor-opencode-checks.js +5 -4
- package/dist/src/doctor/doctor-orphans.js +120 -20
- package/dist/src/doctor/{outcome.js → doctor-outcome.js} +7 -0
- package/dist/src/doctor/doctor-pi-checks.js +2 -2
- package/dist/src/doctor/doctor-principles.js +3 -3
- package/dist/src/doctor/doctor-project-checks.js +3 -3
- package/dist/src/doctor/doctor-role-notes.js +1 -1
- package/dist/src/doctor/doctor.js +8 -11
- package/dist/src/events/event-format.js +30 -54
- package/dist/src/events/event-labels.js +1 -5
- package/dist/src/events/event-legacy.js +1 -3
- package/dist/src/events/{notify.js → event-notify.js} +2 -2
- package/dist/src/events/event-read.js +1 -1
- package/dist/src/failure/failure-cluster.js +6 -6
- package/dist/src/failure/failure-data.js +8 -4
- package/dist/src/failure/failure-render.js +44 -34
- package/dist/src/failure/failure-state-change.js +4 -34
- package/dist/src/failure/{time-spend.js → failure-time-spend.js} +117 -84
- package/dist/src/files/files.js +30 -0
- package/dist/src/files/json-object.js +8 -8
- package/dist/src/files/stat-cache.js +12 -8
- package/dist/src/gates/budget-gates.js +85 -84
- package/dist/src/gates/gate-polls.js +27 -68
- package/dist/src/gates/pause-gates.js +16 -2
- package/dist/src/gates/readiness.js +1 -1
- package/dist/src/gates/role-cap-gates.js +44 -35
- package/dist/src/gates/{startup-gate.js → startup-gates.js} +7 -7
- package/dist/src/gates/{streak-gate.js → streak-gates.js} +75 -52
- package/dist/src/git/git.js +75 -45
- package/dist/src/git/worktree-pool.js +3 -6
- package/dist/src/gui/gui-admin-endpoints.js +3 -3
- package/dist/src/gui/gui-backlog-commands.js +4 -4
- package/dist/src/gui/gui-backlog-endpoints.js +61 -41
- package/dist/src/gui/gui-command.js +43 -24
- package/dist/src/gui/gui-history-endpoints.js +100 -52
- package/dist/src/gui/gui-operator-commands.js +22 -23
- package/dist/src/gui/gui-prompt-commands.js +23 -23
- package/dist/src/gui/gui-report-endpoints.js +1 -1
- package/dist/src/history/history-data.js +71 -44
- package/dist/src/history/history-queued.js +49 -27
- package/dist/src/history/history.js +51 -33
- package/dist/src/inbox/prompt-commands.js +59 -42
- package/dist/src/init/init.js +88 -40
- package/dist/src/merge/merge-backlog-conflicts.js +1 -1
- package/dist/src/merge/merge-core.js +1 -1
- package/dist/src/merge/merge-drain.js +8 -11
- package/dist/src/merge/merge-pipeline.js +6 -7
- package/dist/src/merge/merge-post-review-drain.js +10 -13
- package/dist/src/merge/merge-requeue.js +1 -1
- package/dist/src/merge/merge-slot.js +0 -8
- package/dist/src/merge/merge-verify.js +8 -18
- package/dist/src/notes-cli.js +52 -42
- package/dist/src/operator/operator-commands.js +76 -44
- package/dist/src/operator/operator-intent.js +3 -3
- package/dist/src/orchestrator/orchestrator-launch.js +31 -63
- package/dist/src/orchestrator/orchestrator-poll.js +27 -32
- package/dist/src/orchestrator/orchestrator.js +8 -21
- package/dist/src/orchestrator/scheduling-admit.js +101 -68
- package/dist/src/orchestrator/scheduling-scope.js +1 -1
- package/dist/src/orchestrator/scheduling-skip.js +1 -1
- package/dist/src/orchestrator/startup-migrations.js +1 -1
- package/dist/src/paths.js +8 -7
- package/dist/src/pi/pi-args.js +5 -2
- package/dist/src/pi/pi-backend.js +2 -2
- package/dist/src/pi/pi-bin.js +3 -3
- package/dist/src/pi/pi-event-line.js +1 -1
- package/dist/src/pi/pi-models.js +43 -29
- package/dist/src/pi/pi-settings.js +43 -20
- package/dist/src/pi/pi-stream.js +32 -6
- package/dist/src/pi/pi-watchdogs.js +12 -5
- package/dist/src/pi/pi.js +2 -2
- package/dist/src/pi-extension/context-shake.js +32 -20
- package/dist/src/pi-extension/kill-guard.js +69 -0
- package/dist/src/process/process-group.js +49 -5
- package/dist/src/process/process.js +8 -8
- package/dist/src/process/run-marker.js +31 -13
- package/dist/src/process/supervisor.js +40 -28
- package/dist/src/prompt/dialect.js +1 -1
- package/dist/src/prompt/gate-notes.js +2 -2
- package/dist/src/prompt/notes-backlog.js +66 -37
- package/dist/src/prompt/principles.js +6 -4
- package/dist/src/prompt/prompt-followup.js +12 -4
- package/dist/src/prompt/prompt.js +88 -38
- package/dist/src/report/report-data.js +53 -37
- package/dist/src/report/report-fold.js +16 -9
- package/dist/src/report/report-render.js +21 -194
- package/dist/src/report/report-since-data.js +4 -3
- package/dist/src/report/report-since-render.js +173 -0
- package/dist/src/report/report-usage-fold.js +8 -8
- package/dist/src/report/report.js +46 -22
- package/dist/src/review/post-merge-review-render.js +9 -3
- package/dist/src/review/post-merge-review.js +1 -1
- package/dist/src/review/review-precheck.js +11 -1
- package/dist/src/review/review.js +40 -153
- package/dist/src/review/secret-leaks.js +26 -16
- package/dist/src/roles/role-catalog.js +11 -3
- package/dist/src/roles/role-guidance.js +16 -12
- package/dist/src/roles/role-render.js +1 -14
- package/dist/src/roles/role-view.js +9 -21
- package/dist/src/run/acceptance-greps.js +88 -45
- package/dist/src/run/run-apply.js +12 -6
- package/dist/src/run/run-detail-data.js +121 -68
- package/dist/src/run/run-detail-render.js +92 -35
- package/dist/src/run/run-detail.js +72 -41
- package/dist/src/run/run-prompt.js +63 -50
- package/dist/src/run/run-resume.js +76 -29
- package/dist/src/run/run-stage.js +135 -84
- package/dist/src/run/run-timing.js +8 -2
- package/dist/src/run/run-usage.js +31 -1
- package/dist/src/run/run-verdict.js +119 -50
- package/dist/src/run/stage-check.js +49 -32
- package/dist/src/run/stage-checks-shared.js +28 -0
- package/dist/src/run/stage-name-checks.js +111 -92
- package/dist/src/run/stage-path-checks.js +69 -42
- package/dist/src/run/stage-prose-checks.js +55 -40
- package/dist/src/runner/runner-outcome.js +6 -1
- package/dist/src/runner/runner-pi.js +3 -10
- package/dist/src/runner/runner-recovery.js +1 -1
- package/dist/src/runner/{revision.js → runner-revision.js} +1 -1
- package/dist/src/runner/runner-run.js +97 -35
- package/dist/src/runner/runner-state-load.js +12 -9
- package/dist/src/runner/runner.js +41 -25
- package/dist/src/scheduling/claims.js +0 -4
- package/dist/src/scheduling/scheduling.js +53 -66
- package/dist/src/status/status-data.js +2 -2
- package/dist/src/status/status-role-notes.js +2 -2
- package/dist/src/status/status-roles.js +9 -17
- package/dist/src/team/{error-storm.js → team-error-storm.js} +1 -1
- package/dist/src/team/{failure-spread.js → team-failure-spread.js} +5 -5
- package/dist/src/team/team-hold.js +98 -65
- package/dist/src/team/{orchestrator-info.js → team-orchestrator-info.js} +2 -15
- package/dist/src/team/team-polls.js +28 -126
- package/dist/src/team/{reclaim.js → team-reclaim.js} +61 -31
- package/dist/src/team/team-state.js +4 -13
- package/dist/src/text/markdown.js +1 -1
- package/dist/src/text/phrases.js +20 -15
- package/dist/src/text/text.js +3 -3
- package/dist/src/ui/alert-feeds.js +49 -73
- package/dist/src/ui/badges.js +3 -13
- package/dist/src/ui/gui/gui-client-boot.js +2 -2
- package/dist/src/ui/gui/gui-client-composer.js +1 -1
- package/dist/src/ui/gui/gui-client-drawer-config.js +7 -17
- package/dist/src/ui/gui/gui-client-drawer.js +1 -1
- package/dist/src/ui/gui/gui-client-history.js +28 -14
- package/dist/src/ui/gui/gui-client-model.js +0 -12
- package/dist/src/ui/gui/gui-client-pending.js +7 -6
- package/dist/src/ui/gui/gui-client-report-chart.js +5 -5
- package/dist/src/ui/gui/gui-client-report.js +1 -1
- package/dist/src/ui/gui/gui-client-roles.js +0 -1
- package/dist/src/ui/gui/gui-client-settings.js +43 -84
- package/dist/src/ui/gui/gui-client-team.js +4 -4
- package/dist/src/ui/gui/gui-client.js +7 -1
- package/dist/src/ui/progress-data.js +53 -40
- package/dist/src/ui/run-progress-model.js +9 -2
- package/dist/src/ui/status-model.js +2 -24
- package/dist/src/ui/status-payload.js +3 -7
- package/dist/src/ui/status-render.js +5 -9
- package/dist/src/ui/team-alert-backlog.js +6 -6
- package/dist/src/ui/team-alert-status.js +9 -170
- package/dist/src/ui/team-alert-tail.js +19 -215
- package/dist/src/ui/team-alerts.js +7 -8
- package/dist/src/ui/transcript-run.js +27 -16
- package/dist/src/ui/tui/tui-editor-answer.js +12 -10
- package/dist/src/ui/tui/tui-editor-attach.js +4 -3
- package/dist/src/ui/tui/tui-editor-backends.js +4 -3
- package/dist/src/ui/tui/tui-editor-brief.js +6 -9
- package/dist/src/ui/tui/tui-editor-budget.js +4 -3
- package/dist/src/ui/tui/tui-editor-config.js +80 -68
- package/dist/src/ui/tui/tui-editor-entry.js +12 -20
- package/dist/src/ui/tui/tui-editor-pause.js +51 -57
- package/dist/src/ui/tui/tui-editor-prompt.js +9 -13
- package/dist/src/ui/tui/tui-editor-queued.js +11 -14
- package/dist/src/ui/tui/tui-editor-quiet.js +4 -3
- package/dist/src/ui/tui/tui-editor-release.js +4 -3
- package/dist/src/ui/tui/tui-editor-retire.js +7 -7
- package/dist/src/ui/tui/tui-editor-role-enabled.js +4 -3
- package/dist/src/ui/tui/tui-editor-role-file.js +4 -4
- package/dist/src/ui/tui/tui-editor-role-key.js +4 -3
- package/dist/src/ui/tui/tui-editor-role-model.js +97 -15
- package/dist/src/ui/tui/tui-editor-search.js +4 -3
- package/dist/src/ui/tui/tui-editor-session.js +17 -4
- package/dist/src/ui/tui/tui-editors.js +1 -7
- package/dist/src/ui/tui/tui-frame.js +182 -134
- package/dist/src/ui/tui/tui-guard.js +20 -0
- package/dist/src/ui/tui/tui-input.js +159 -92
- package/dist/src/ui/tui/tui-keymap.js +51 -29
- package/dist/src/ui/tui/tui-keys.js +1 -1
- package/dist/src/ui/tui/tui-pane-activity.js +3 -3
- package/dist/src/ui/tui/tui-pane-backlog.js +74 -36
- package/dist/src/ui/tui/tui-pane-ctx.js +1 -1
- package/dist/src/ui/tui/tui-pane-health.js +252 -228
- package/dist/src/ui/tui/tui-pane-history.js +61 -70
- package/dist/src/ui/tui/tui-pane-pending.js +284 -247
- package/dist/src/ui/tui/tui-pane-queued.js +25 -27
- package/dist/src/ui/tui/tui-pane-report.js +2 -1
- package/dist/src/ui/tui/tui-pane-session.js +11 -7
- package/dist/src/ui/tui/tui-pane-transcript.js +19 -31
- package/dist/src/ui/tui/tui-pane.js +14 -11
- package/dist/src/ui/tui/tui-panes.js +29 -32
- package/dist/src/ui/tui/tui.js +3 -4
- package/dist/src/verdict/fix-claim.js +76 -54
- package/dist/src/verdict/refusal.js +1 -1
- package/package.json +5 -5
- package/dist/src/concurrency/provider-lease.js +0 -105
- package/dist/src/runner/model-fallback.js +0 -76
- package/dist/src/runner/runner-fallback.js +0 -66
- package/dist/src/ui/tui/tui-editor-claude-reserve.js +0 -24
- package/dist/src/ui/tui/tui-editor-provider-caps.js +0 -27
- /package/dist/src/backlog/{question-answer.js → backlog-question-answer.js} +0 -0
- /package/dist/src/doctor/{probe.js → doctor-probe.js} +0 -0
- /package/dist/src/gates/{disk-gate.js → disk-gates.js} +0 -0
- /package/dist/src/operator/{retire.js → operator-retire.js} +0 -0
- /package/dist/src/operator/{revert.js → operator-revert.js} +0 -0
- /package/dist/src/runner/{leftover.js → runner-leftover.js} +0 -0
|
@@ -10,11 +10,9 @@ import { fallbackServingPair, rekeyFallbackBreakers, } from "../budget/fallback-
|
|
|
10
10
|
import { fallbackPair, resolveTierFallbacks, tierChainSteps, tierFallbackLabels, tiersResolveDistinctPairs, } from "../config/config-fallbacks.js";
|
|
11
11
|
import { DIRECTOR_ROLE } from "../roles/roles.js";
|
|
12
12
|
import { logEventBestEffort, TUMWATER_SOURCE } from "../events/events.js";
|
|
13
|
-
import { windowReserveFor } from "../config/config-views.js";
|
|
14
13
|
import { piPairFree, readPiProviders } from "../pi/pi-models.js";
|
|
15
14
|
import { readOpencodeCache } from "../agent/opencode/opencode-catalog.js";
|
|
16
15
|
import { readAccount } from "../agent/subscription-account.js";
|
|
17
|
-
import { claudeWindowUsedUp } from "../agent/claude/claude-account.js";
|
|
18
16
|
import { budgetFallbackView, chainInfo, claudeOverageRefused, subscriptionBackendOf, } from "../budget/budget-fallback-view.js";
|
|
19
17
|
// Re-export the names moved into budget-fallback-view.ts so no importer's import changes
|
|
20
18
|
// (gate-polls.ts reads subscriptionBackendOf; budget-chain-info.test.ts reads chainInfo).
|
|
@@ -28,31 +26,25 @@ export function newBudgetGateState(config) {
|
|
|
28
26
|
engaged: null,
|
|
29
27
|
from: null,
|
|
30
28
|
fromKey: "",
|
|
31
|
-
prevWindowMoved: new Set(),
|
|
32
|
-
directorFrom: null,
|
|
33
|
-
directorView: config,
|
|
34
|
-
directorKey: "",
|
|
35
29
|
fallbackConfig: config,
|
|
36
30
|
warned: false,
|
|
37
31
|
prevChain: {},
|
|
38
32
|
};
|
|
39
33
|
}
|
|
40
|
-
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
const
|
|
54
|
-
// Who spent the most today, read once for whichever of the warning or a transition fires.
|
|
55
|
-
const topRoles = (warn && !state.warned) || gate !== state.prevGate ? topDailySpenders(states, now) : [];
|
|
34
|
+
/** Which `budget_*` type a gate crossing logs: the resumed/fallback/paused pick on the
|
|
35
|
+
* direction of the transition (resumed only back to open; paused only when neither the
|
|
36
|
+
* default nor, with review on, the strong tier serves). */
|
|
37
|
+
function budgetTransitionType(gate) {
|
|
38
|
+
return gate === "open" ? "budget_resumed" : gate === "fallback" ? "budget_fallback" : "budget_paused";
|
|
39
|
+
}
|
|
40
|
+
/** The 80%-of-cap warning edge (PLANS.md 2026-09-30), edge-triggered with reset-on-below — the
|
|
41
|
+
* whole state machine: crossing the threshold while the gate is still open logs one
|
|
42
|
+
* budget_warning, midnight or a cap raise drops spend back under and re-arms it, and a
|
|
43
|
+
* second crossing the same day warns again. Warn only while the gate is open: a poll that
|
|
44
|
+
* engages fallback/paused already logs its own transition event, so the warning never
|
|
45
|
+
* doubles the page on the poll the cap itself is reached. */
|
|
46
|
+
function budgetWarningEdge(input, warn, topRoles) {
|
|
47
|
+
const { root, state, spentUsd, capUsd } = input;
|
|
56
48
|
if (warn && !state.warned) {
|
|
57
49
|
logEventBestEffort(root, {
|
|
58
50
|
source: TUMWATER_SOURCE,
|
|
@@ -66,49 +58,71 @@ function logBudgetGateEvents(input) {
|
|
|
66
58
|
else if (!warn) {
|
|
67
59
|
state.warned = false;
|
|
68
60
|
}
|
|
61
|
+
}
|
|
62
|
+
/** The identity fields a `budget_fallback` event carries so a reader can resolve the operator's
|
|
63
|
+
* "why this and not the other one" — which free pair took over, which configured pair was
|
|
64
|
+
* refused because pi's definitions do not price it at zero, or which free pair the breaker
|
|
65
|
+
* demoted after its runs kept failing (and after how many). */
|
|
66
|
+
function fallbackTransitionFields(input) {
|
|
67
|
+
const { gate, pair, servingResolved } = input;
|
|
68
|
+
return gate === "fallback"
|
|
69
|
+
? {
|
|
70
|
+
provider: pair?.provider,
|
|
71
|
+
model: pair?.model,
|
|
72
|
+
// The per-tier story (part 7b/8): which pair each tier resolved to, borrowed
|
|
73
|
+
// ones marked. Carried only when the tiers resolve to two or more distinct
|
|
74
|
+
// pairs — a single fallback (its pair borrowed by the other tiers) keeps the
|
|
75
|
+
// event tier-free, so today's single-pair readers keep today's text.
|
|
76
|
+
...(tiersResolveDistinctPairs(servingResolved)
|
|
77
|
+
? { tiers: tierFallbackLabels(servingResolved) }
|
|
78
|
+
: {}),
|
|
79
|
+
}
|
|
80
|
+
: {};
|
|
81
|
+
}
|
|
82
|
+
/** The paused gate's cause, named by the pair story that produced it. The default tier is the
|
|
83
|
+
* cause only when it failed to serve: a price-free default the breaker dropped is its own
|
|
84
|
+
* demotion (with the failure count), and a default with no price-free pair at all names the
|
|
85
|
+
* declared pair the price check refused. When the default serves, the review-on pause is the
|
|
86
|
+
* strong tier's: name its price-free pair when the breaker demoted it, and leave the cause
|
|
87
|
+
* unset when strong simply had no pair to borrow. */
|
|
88
|
+
function pausedTransitionFields(input) {
|
|
89
|
+
const { gate, state, pair, pairName, rejectedPair, priceResolved, servingResolved } = input;
|
|
90
|
+
return gate === "paused" && servingResolved.default.pair === null && pair
|
|
91
|
+
? { fallbackDemoted: pairName, failures: state.breakers[pairName]?.failures ?? 0 }
|
|
92
|
+
: gate === "paused" && servingResolved.default.pair === null && rejectedPair
|
|
93
|
+
? { fallbackRejected: modelPairName(rejectedPair) }
|
|
94
|
+
: gate === "paused" && priceResolved.strong.pair !== null
|
|
95
|
+
? {
|
|
96
|
+
fallbackDemoted: modelPairName(priceResolved.strong.pair),
|
|
97
|
+
failures: state.breakers[modelPairName(priceResolved.strong.pair)]?.failures ?? 0,
|
|
98
|
+
}
|
|
99
|
+
: {};
|
|
100
|
+
}
|
|
101
|
+
/** The budget gate's warning-and-transition half (split out of pollBudgetGate so a
|
|
102
|
+
* budget_warning event field or an edge-trigger rule is found by name): edge-triggers the
|
|
103
|
+
* 80%-of-cap warning into one budget_warning, computes the top spenders for whichever event
|
|
104
|
+
* fires, logs exactly one budget_resumed/budget_fallback/budget_paused on a gate crossing
|
|
105
|
+
* with the per-tier story and the pause cause, and advances state.prevGate. */
|
|
106
|
+
function logBudgetGateEvents(input) {
|
|
107
|
+
const { root, state, states, now, gate, spentUsd, capUsd, } = input;
|
|
108
|
+
// The 80% early warning, computed once below budgetWarningEdge (which holds its state
|
|
109
|
+
// machine): a poll at or past the 80% threshold while still open warrants the edge.
|
|
110
|
+
const warn = gate === "open" && budgetWarning({ spentUsd, capUsd });
|
|
111
|
+
// Who spent the most today, read once for whichever of the warning or a transition fires.
|
|
112
|
+
const topRoles = (warn && !state.warned) || gate !== state.prevGate ? topDailySpenders(states, now) : [];
|
|
113
|
+
budgetWarningEdge(input, warn, topRoles);
|
|
69
114
|
if (gate !== state.prevGate) {
|
|
70
115
|
logEventBestEffort(root, {
|
|
71
116
|
source: TUMWATER_SOURCE,
|
|
72
|
-
type: gate
|
|
117
|
+
type: budgetTransitionType(gate),
|
|
73
118
|
spentUsd,
|
|
74
119
|
capUsd,
|
|
75
120
|
// The cap pages say where the money went, since a poll can jump past the cap with no
|
|
76
121
|
// warning first; a resume does not (midnight reads zero spend, a cap raise is the
|
|
77
122
|
// operator's own act).
|
|
78
123
|
...(gate !== "open" && topRoles.length > 0 ? { topRoles } : {}),
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
// refused because pi's definitions do not price it at zero, or which free pair the
|
|
82
|
-
// breaker demoted after its runs kept failing (and after how many).
|
|
83
|
-
...(gate === "fallback"
|
|
84
|
-
? {
|
|
85
|
-
provider: pair?.provider,
|
|
86
|
-
model: pair?.model,
|
|
87
|
-
// The per-tier story (part 7b/8): which pair each tier resolved to, borrowed
|
|
88
|
-
// ones marked. Carried only when the tiers resolve to two or more distinct
|
|
89
|
-
// pairs — a single fallback (its pair borrowed by the other tiers) keeps the
|
|
90
|
-
// event tier-free, so today's single-pair readers keep today's text.
|
|
91
|
-
...(tiersResolveDistinctPairs(servingResolved)
|
|
92
|
-
? { tiers: tierFallbackLabels(servingResolved) }
|
|
93
|
-
: {}),
|
|
94
|
-
}
|
|
95
|
-
: {}),
|
|
96
|
-
// The pause's cause, named by the pair story that produced it. The default tier is the
|
|
97
|
-
// cause only when it failed to serve: a price-free default the breaker dropped is its
|
|
98
|
-
// own demotion (with the failure count), and a default with no price-free pair at all
|
|
99
|
-
// names the declared pair the price check refused. When the default serves, the
|
|
100
|
-
// review-on pause is the strong tier's: name its price-free pair when the breaker
|
|
101
|
-
// demoted it, and leave the cause unset when strong simply had no pair to borrow.
|
|
102
|
-
...(gate === "paused" && servingResolved.default.pair === null && pair
|
|
103
|
-
? { fallbackDemoted: pairName, failures: state.breakers[pairName]?.failures ?? 0 }
|
|
104
|
-
: gate === "paused" && servingResolved.default.pair === null && rejectedPair
|
|
105
|
-
? { fallbackRejected: modelPairName(rejectedPair) }
|
|
106
|
-
: gate === "paused" && priceResolved.strong.pair !== null
|
|
107
|
-
? {
|
|
108
|
-
fallbackDemoted: modelPairName(priceResolved.strong.pair),
|
|
109
|
-
failures: state.breakers[modelPairName(priceResolved.strong.pair)]?.failures ?? 0,
|
|
110
|
-
}
|
|
111
|
-
: {}),
|
|
124
|
+
...fallbackTransitionFields(input),
|
|
125
|
+
...pausedTransitionFields(input),
|
|
112
126
|
});
|
|
113
127
|
state.prevGate = gate;
|
|
114
128
|
}
|
|
@@ -126,13 +140,12 @@ export function runOnPair(runModelPair, pair) {
|
|
|
126
140
|
runModelPair.model === pair.model);
|
|
127
141
|
}
|
|
128
142
|
/** The config a runner runs on while the budget gate stands: the director keeps the live config —
|
|
129
|
-
* an explicit human prompt outranks the autonomous-spend cap
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
* granted to one more role in one copy and not the other. */
|
|
143
|
+
* an explicit human prompt outranks the autonomous-spend cap (and, part 5d/6, no usage window
|
|
144
|
+
* moves it either) — so it takes `directorConfig` (always the live config; omitted means the live
|
|
145
|
+
* config). Every other role takes the gate's derived view (pollBudgetGate's returned `roleConfig`).
|
|
146
|
+
* Single home for the rule, so the scheduler's per-poll assignment (src/budget/budget-wiring.ts)
|
|
147
|
+
* and a throwaway merge author's construction (src/merge/merge-verify.ts's resolveAuthor) cannot
|
|
148
|
+
* drift apart — say, an exemption granted to one more role in one copy and not the other. */
|
|
136
149
|
export function gateRoleConfig(role, liveConfig, roleConfig, directorConfig = liveConfig) {
|
|
137
150
|
return role === DIRECTOR_ROLE ? directorConfig : roleConfig;
|
|
138
151
|
}
|
|
@@ -179,13 +192,8 @@ export function pollBudgetGate(state, ctx) {
|
|
|
179
192
|
// verdict. The resolution below deliberately prices alone (the map's keys are the pairs
|
|
180
193
|
// that would run absent a demotion, so a demotion cannot churn them); part 5c/8 wires the
|
|
181
194
|
// full predicate into the resolution so a demoted pair re-resolves only the tiers using it.
|
|
182
|
-
// A
|
|
183
|
-
//
|
|
184
|
-
// same window. `windowReserveFor(config, backend)` makes a window past its reserve fraction
|
|
185
|
-
// count as used up for the team; the director's own reading (directorWindowOutFor) ignores
|
|
186
|
-
// it, as it runs the operator's prompts.
|
|
187
|
-
const windowOutFor = (backend) => claudeWindowUsedUp(accounts(backend), now, true, windowReserveFor(liveConfig, backend));
|
|
188
|
-
const directorWindowOutFor = (backend) => claudeWindowUsedUp(accounts(backend), now);
|
|
195
|
+
// A used-up subscription window no longer moves a tier to its fallback (part 5d/6): only a
|
|
196
|
+
// spent maxDailyCostUsd does, so the free-ness predicate below is purely the cap rule.
|
|
189
197
|
// A priced step (one pi does not price at zero, or claude on API billing) is usable only
|
|
190
198
|
// while the team is under its cap and the step is under its own `maxDailyCostUsd`, so no cap
|
|
191
199
|
// can be passed; at the cap only free steps remain, as before.
|
|
@@ -196,18 +204,13 @@ export function pollBudgetGate(state, ctx) {
|
|
|
196
204
|
// (extraUsage, claude-only) may spend extra usage within its cap while no window is
|
|
197
205
|
// rejected or refuses overage.
|
|
198
206
|
if (!p.extraUsage)
|
|
199
|
-
return
|
|
200
|
-
// Past the reserve this step would spend the reserved quota before reaching extra usage:
|
|
201
|
-
// it waits until a reading shows the account already spending overage.
|
|
207
|
+
return piPairFree(providers, p, accounts, catalog);
|
|
202
208
|
const subAccount = accounts(subBackend);
|
|
203
|
-
if (windowOutFor(subBackend)?.reserve === true &&
|
|
204
|
-
!Object.values(subAccount.windows).some((w) => w.isUsingOverage === true))
|
|
205
|
-
return false;
|
|
206
209
|
return (!reached &&
|
|
207
210
|
!claudeOverageRefused(subAccount) &&
|
|
208
211
|
(p.maxDailyCostUsd === undefined || pairDailyCost(states, modelPairName(p), now) < p.maxDailyCostUsd));
|
|
209
212
|
}
|
|
210
|
-
// The subscription
|
|
213
|
+
// The subscription rule lives entirely in the branch above (it returns early for
|
|
211
214
|
// any subscription pair); below that branch a pair is never a subscription, so only the
|
|
212
215
|
// free-ness and cap rules apply.
|
|
213
216
|
if (piPairFree(providers, p, accounts, catalog))
|
|
@@ -285,11 +288,11 @@ export function pollBudgetGate(state, ctx) {
|
|
|
285
288
|
priceResolved,
|
|
286
289
|
servingResolved,
|
|
287
290
|
});
|
|
288
|
-
// The fallback config view — the derived config roles run under, the
|
|
289
|
-
//
|
|
290
|
-
//
|
|
291
|
-
//
|
|
292
|
-
//
|
|
291
|
+
// The fallback config view — the derived config roles run under, the chain block, and the
|
|
292
|
+
// handback pairs — lives in src/budget/budget-fallback-view.ts (the view half, plan's parts
|
|
293
|
+
// 5b/8, 5c/8, 6a/7): it reads these verdict half's resolved inputs and writes the cross-poll
|
|
294
|
+
// view state, and pollBudgetGate keeps this call signature so gate-polls.ts's destructure is
|
|
295
|
+
// unchanged.
|
|
293
296
|
const view = budgetFallbackView({
|
|
294
297
|
state,
|
|
295
298
|
config: liveConfig,
|
|
@@ -298,8 +301,6 @@ export function pollBudgetGate(state, ctx) {
|
|
|
298
301
|
providers,
|
|
299
302
|
catalog,
|
|
300
303
|
accounts,
|
|
301
|
-
windowOutFor,
|
|
302
|
-
directorWindowOutFor,
|
|
303
304
|
priceResolved,
|
|
304
305
|
servingResolved,
|
|
305
306
|
chainPairs,
|
|
@@ -1,23 +1,20 @@
|
|
|
1
1
|
import { applyBudgetGateWiring, budgetPausedSubjects } from "../budget/budget-wiring.js";
|
|
2
2
|
import { newBudgetGateState, pollBudgetGate, } from "./budget-gates.js";
|
|
3
|
-
import { backendsInUse } from "./startup-gate.js";
|
|
4
3
|
import { newPauseGateState, pollPauseGates } from "./pause-gates.js";
|
|
5
|
-
import { newStreakGateState, pollStreakGate } from "./streak-
|
|
4
|
+
import { newStreakGateState, pollStreakGate } from "./streak-gates.js";
|
|
6
5
|
import { newRoleCapGateState, pollRoleCapGate, } from "./role-cap-gates.js";
|
|
7
6
|
import { newMaintenanceQuotaGateState, pollMaintenanceQuotaGate, } from "./maintenance-quota.js";
|
|
8
7
|
import { newBootstrapGateState, pollBootstrapGate, } from "./bootstrap-gates.js";
|
|
9
8
|
import { newQuietHoursGateState, pollQuietHoursGate, roleQuietHold, } from "../scheduling/quiet-hours.js";
|
|
10
|
-
import { BYTES_PER_GB, newDiskGateState, pollDiskGate, sampleFreeBytes, } from "./disk-
|
|
11
|
-
import { pollErrorStorm, pollFailureSpread, pollTeamHold
|
|
12
|
-
import { readAccount } from "../agent/subscription-account.js";
|
|
9
|
+
import { BYTES_PER_GB, newDiskGateState, pollDiskGate, sampleFreeBytes, } from "./disk-gates.js";
|
|
10
|
+
import { pollErrorStorm, pollFailureSpread, pollTeamHold } from "../team/team-polls.js";
|
|
13
11
|
import { standingHolds } from "../team/team-hold.js";
|
|
14
|
-
import { ERROR_STORM_QUIET } from "../team/error-storm.js";
|
|
15
|
-
import { FAILURE_SPREAD_QUIET } from "../team/failure-spread.js";
|
|
12
|
+
import { ERROR_STORM_QUIET } from "../team/team-error-storm.js";
|
|
13
|
+
import { FAILURE_SPREAD_QUIET } from "../team/team-failure-spread.js";
|
|
16
14
|
import { DIRECTOR_ROLE } from "../roles/roles.js";
|
|
17
15
|
import { baseRoleOf, instancePauseKey, runnerEnabled } from "../roles/runner-ids.js";
|
|
18
16
|
import { queuedMergeRoles } from "../merge/merge-queue.js";
|
|
19
|
-
import { publishInfoFields } from "../team/orchestrator-info.js";
|
|
20
|
-
import { windowReserveFor } from "../config/config-views.js";
|
|
17
|
+
import { publishInfoFields } from "../team/team-orchestrator-info.js";
|
|
21
18
|
/** A fresh gate family: nothing seen yet, so the first poll of an unpaused, unheld team
|
|
22
19
|
* logs nothing. The budget gate seeds its breaker and derived fallback view from the
|
|
23
20
|
* startup config; a live reload replaces the config on the next poll. */
|
|
@@ -32,28 +29,19 @@ export function newTeamGateStates(config) {
|
|
|
32
29
|
quiet: newQuietHoursGateState(),
|
|
33
30
|
disk: newDiskGateState(),
|
|
34
31
|
teamHold: new Map(),
|
|
35
|
-
claudeWindowWarned: new Set(),
|
|
36
|
-
claudeWindowReserved: new Set(),
|
|
37
32
|
errorStorm: ERROR_STORM_QUIET,
|
|
38
33
|
failureSpread: FAILURE_SPREAD_QUIET,
|
|
39
34
|
};
|
|
40
35
|
}
|
|
41
36
|
/** Poll every team-wide gate and alarm, advancing `states` in place. */
|
|
42
37
|
export function pollTeamGates(states, ctx) {
|
|
43
|
-
const
|
|
44
|
-
// The per-backend subscription account lookup (part 2e/9): each backend's account is read on
|
|
45
|
-
// demand — the window hold/reserve/warning loop, the window-move and the handback reads each
|
|
46
|
-
// call accounts fresh — so a second subscription backend is judged from its own account, never
|
|
47
|
-
// from claude's.
|
|
48
|
-
const accounts = (backend) => readAccount(root, backend, now);
|
|
49
|
-
const budget = pollBudgetPhase(states, ctx, accounts);
|
|
38
|
+
const budget = pollBudgetPhase(states, ctx);
|
|
50
39
|
const holds = pollHoldGates(states, ctx);
|
|
51
40
|
const quietDisk = pollQuietAndDiskGates(states, ctx);
|
|
52
|
-
pollTeamFailureAlarms(states, ctx
|
|
41
|
+
pollTeamFailureAlarms(states, ctx);
|
|
53
42
|
return {
|
|
54
43
|
gate: budget.gate,
|
|
55
44
|
roleConfig: budget.roleConfig,
|
|
56
|
-
directorConfig: budget.directorConfig,
|
|
57
45
|
userPaused: holds.userPaused,
|
|
58
46
|
pausedRoles: holds.pausedRoles,
|
|
59
47
|
capPaused: holds.capPaused,
|
|
@@ -63,7 +51,6 @@ export function pollTeamGates(states, ctx) {
|
|
|
63
51
|
servingResolved: budget.servingResolved,
|
|
64
52
|
priceResolved: budget.priceResolved,
|
|
65
53
|
budgetActive: budget.budgetActive,
|
|
66
|
-
windowMovedRoles: budget.windowMovedRoles,
|
|
67
54
|
roleQuietHeld: quietDisk.roleQuietHeld,
|
|
68
55
|
quietNow: quietDisk.quietNow,
|
|
69
56
|
diskHeld: quietDisk.diskHeld,
|
|
@@ -75,66 +62,54 @@ export function pollTeamGates(states, ctx) {
|
|
|
75
62
|
// owns only the wiring — the demotion publish below and the per-runner assignment at
|
|
76
63
|
// the bottom of this block — and the per-tier budget pause (src/budget/budget-wiring.ts,
|
|
77
64
|
// part 5c/8): a role whose model tier resolved to no usable free pair while the cap is
|
|
78
|
-
// reached starts no new runs
|
|
79
|
-
|
|
80
|
-
function pollBudgetPhase(states, ctx, accounts) {
|
|
65
|
+
// reached starts no new runs.
|
|
66
|
+
function pollBudgetPhase(states, ctx) {
|
|
81
67
|
const { root, runners, liveConfig, modelsPath, info } = ctx;
|
|
82
|
-
const { gate, roleConfig, spentUsd, capUsd, onFallback, resumed: gateResumed, pairs: fallbackPairs, priceResolved, servingResolved, budgetActive,
|
|
68
|
+
const { gate, roleConfig, spentUsd, capUsd, onFallback, resumed: gateResumed, pairs: fallbackPairs, priceResolved, servingResolved, budgetActive, chain, } = pollBudgetGate(states.budget, {
|
|
83
69
|
root,
|
|
84
70
|
states: runners.map((r) => r.state),
|
|
85
71
|
liveConfig,
|
|
86
72
|
modelsPath,
|
|
87
73
|
});
|
|
88
74
|
// The budget wiring (demotions publish, fallback_switched event, per-runner config/cap
|
|
89
|
-
// assignment, cap-reopen
|
|
90
|
-
// (part 5c/8
|
|
75
|
+
// assignment, cap-reopen hand-back) lives in src/budget/budget-wiring.ts
|
|
76
|
+
// (part 5c/8) along with the whole budget concern.
|
|
91
77
|
applyBudgetGateWiring({
|
|
92
78
|
root,
|
|
93
79
|
runners,
|
|
94
80
|
liveConfig,
|
|
95
81
|
info,
|
|
96
|
-
accounts,
|
|
97
82
|
budget: states.budget,
|
|
98
83
|
view: {
|
|
99
84
|
roleConfig,
|
|
100
|
-
directorConfig,
|
|
101
85
|
onFallback,
|
|
102
86
|
resumed: gateResumed,
|
|
103
87
|
pairs: fallbackPairs,
|
|
104
|
-
priceResolved,
|
|
105
88
|
budgetActive,
|
|
106
|
-
windowTiers,
|
|
107
|
-
windowResumedBackends,
|
|
108
89
|
chain,
|
|
109
90
|
spentUsd,
|
|
110
91
|
capUsd,
|
|
111
92
|
},
|
|
112
93
|
});
|
|
113
94
|
// The per-tier budget pause (src/budget/budget-wiring.ts, part 5c/8): a role whose model
|
|
114
|
-
// tier resolved to no usable free pair while the cap is reached starts no new runs
|
|
115
|
-
// role on a window-moved tier whose own backend owns the out window is moved off it — the
|
|
95
|
+
// tier resolved to no usable free pair while the cap is reached starts no new runs — the
|
|
116
96
|
// stateless verdict recomputed every poll from the SAME resolution the gate's value came
|
|
117
97
|
// from, beside `capPaused` (the scheduling pass folds the sets; the director is exempt, as
|
|
118
98
|
// under every autonomous gate, and the demoted pair's probe pierces the set for its own
|
|
119
99
|
// tier's roles only — probeRoles in the orchestrator's scheduling pass).
|
|
120
|
-
const { budgetPausedRoles
|
|
100
|
+
const { budgetPausedRoles } = budgetPausedSubjects({
|
|
121
101
|
runners,
|
|
122
102
|
liveConfig,
|
|
123
|
-
accounts,
|
|
124
103
|
budgetActive,
|
|
125
104
|
servingResolved,
|
|
126
|
-
windowTiers,
|
|
127
105
|
});
|
|
128
106
|
return {
|
|
129
107
|
gate,
|
|
130
108
|
roleConfig,
|
|
131
|
-
directorConfig,
|
|
132
109
|
budgetPausedRoles,
|
|
133
|
-
windowMovedRoles,
|
|
134
110
|
servingResolved,
|
|
135
111
|
priceResolved,
|
|
136
112
|
budgetActive,
|
|
137
|
-
windowTiers,
|
|
138
113
|
};
|
|
139
114
|
}
|
|
140
115
|
// The pause gates (src/gates/pause-gates.ts owns the reads, the edge-triggered pause/resume
|
|
@@ -145,7 +120,7 @@ function pollBudgetPhase(states, ctx, accounts) {
|
|
|
145
120
|
function pollHoldGates(states, ctx) {
|
|
146
121
|
const { root, runners, liveConfig, now } = ctx;
|
|
147
122
|
const { userPaused, pausedRoles } = pollPauseGates(root, states.pause);
|
|
148
|
-
// The error-streak circuit breaker (src/gates/streak-
|
|
123
|
+
// The error-streak circuit breaker (src/gates/streak-gates.ts): a role past ERROR_STREAK_BREAKER
|
|
149
124
|
// consecutive failed runs is paused through the same per-role marker the operator's
|
|
150
125
|
// `pause --role` writes — act-on-it where the warn bar and the storm alarms only talk. The
|
|
151
126
|
// director is not exempt; a failing director cannot process prompts anyway. Roles it just
|
|
@@ -240,19 +215,21 @@ function pollQuietAndDiskGates(states, ctx) {
|
|
|
240
215
|
}
|
|
241
216
|
// Each runner as the two provider-failure polls read it (HoldInputs) is built ONCE here so
|
|
242
217
|
// the hold and the spread read the same list, like holdObservations on the reducer side.
|
|
243
|
-
// Keying on the effective provider is what lets a storm on the fallback pair form its
|
|
244
|
-
// hold while the abandoned primary stays clear
|
|
218
|
+
// Keying on the effective provider is what lets a storm on the budget fallback pair form its
|
|
219
|
+
// own hold while the abandoned primary stays clear.
|
|
245
220
|
//
|
|
246
221
|
// Each stamp (lastRateLimit/lastBackendFailure) carries the hold key of the backend that
|
|
247
222
|
// ACTUALLY failed — a reviewer run folded through the author's role stamps `claude`, not
|
|
248
223
|
// the pi author's provider — and the polls' observation mapper files each stamp under that
|
|
249
224
|
// key, falling back to runProvider here for a stamp without one (PLANS.md "Mixed claude +
|
|
250
|
-
// pi fleets, part 2/4"). The spreads below carry the stamps wholesale, provider included
|
|
251
|
-
|
|
252
|
-
|
|
225
|
+
// pi fleets, part 2/4"). The spreads below carry the stamps wholesale, provider included;
|
|
226
|
+
// the provider is each runner's resolved seam key (runProvider), naming the budget fallback
|
|
227
|
+
// pair when a cap episode is active — the pair the run actually uses.
|
|
228
|
+
function pollTeamFailureAlarms(states, ctx) {
|
|
229
|
+
const { root, runners, now, info } = ctx;
|
|
253
230
|
const holdInputs = (rs) => rs.map((r) => ({
|
|
254
231
|
role: r.role,
|
|
255
|
-
provider: r.runProvider(
|
|
232
|
+
provider: r.runProvider(),
|
|
256
233
|
...(r.lastRateLimit ? { lastRateLimit: r.lastRateLimit } : {}),
|
|
257
234
|
...(r.lastBackendFailure ? { lastBackendFailure: r.lastBackendFailure } : {}),
|
|
258
235
|
}));
|
|
@@ -275,16 +252,7 @@ function pollTeamFailureAlarms(states, ctx, accounts, windowTiers) {
|
|
|
275
252
|
// back to states.teamHold — one hold per provider, lifted holds kept keyed for their relapse
|
|
276
253
|
// memory but never reading as held — and the permit-time closures read the LATEST poll's map
|
|
277
254
|
// there, so a waiter granted its permit after a later poll sees that poll's world.
|
|
278
|
-
|
|
279
|
-
// The subscription-window hold, reserve and warning, read per backend in use (part 2e/9): a
|
|
280
|
-
// second subscription backend gets its own hold, reserve and warning memory keyed by its own
|
|
281
|
-
// `backend`, so claude's roles, hold and events are untouched by another backend's window.
|
|
282
|
-
// One account read per backend feeds both the hold observations and the reserve/warning polls;
|
|
283
|
-
// a backend whose account implies no hold contributes no observation.
|
|
284
|
-
const windowObservations = windowAccountReads
|
|
285
|
-
.map(([backend, account]) => windowObservation(backend, account, now, windowReserveFor(liveConfig, backend)))
|
|
286
|
-
.filter((o) => o !== null);
|
|
287
|
-
states.teamHold = pollTeamHold(root, states.teamHold, holdInputs(runners), now, windowObservations, windowTiers);
|
|
255
|
+
states.teamHold = pollTeamHold(root, states.teamHold, holdInputs(runners), now);
|
|
288
256
|
// The standings hold, published for observers (PLANS.md 2026-10-17 "the dashboards and
|
|
289
257
|
// `tumwater status` name a standing provider failure hold"): `standingHolds` turns the holds
|
|
290
258
|
// map into the per-provider list the status snapshot and the alert banner name, the same
|
|
@@ -294,23 +262,14 @@ function pollTeamFailureAlarms(states, ctx, accounts, windowTiers) {
|
|
|
294
262
|
publishInfoFields(root, info, [
|
|
295
263
|
["providerHolds", providerHolds.length > 0 ? providerHolds : undefined],
|
|
296
264
|
]);
|
|
297
|
-
//
|
|
298
|
-
// events keyed by its own backend (the `backend:type:resetsAt` keys), so claude's own reserve
|
|
299
|
-
// and warning memory is unaffected by another backend's crossings (part 2e/9).
|
|
300
|
-
for (const [backend, account] of windowAccountReads) {
|
|
301
|
-
states.claudeWindowReserved = pollClaudeWindowReserve(root, states.claudeWindowReserved, account, now, backend, windowReserveFor(liveConfig, backend));
|
|
302
|
-
// The subscription usage window's warning is observational (the hold above is the gate): the
|
|
303
|
-
// first allowed_warning logs once per window, and the memory lives beside the holds map.
|
|
304
|
-
states.claudeWindowWarned = pollClaudeWindowWarning(root, states.claudeWindowWarned, account, now, backend);
|
|
305
|
-
}
|
|
306
|
-
// Team-wide error-storm warning (src/team/error-storm.ts): when several roles' run streaks
|
|
265
|
+
// Team-wide error-storm warning (src/team/team-error-storm.ts): when several roles' run streaks
|
|
307
266
|
// fail consecutively on one shared cause, each role's own "consecutive run failures"
|
|
308
267
|
// warning still fires alone — this adds the one team-level warning that names the
|
|
309
268
|
// cause (and the config knob, when the cause has one) instead of leaving the operator
|
|
310
269
|
// to diff 14 streak lines. Observational only: it gates nothing, so unlike the failure
|
|
311
270
|
// hold above there is no re-open event — the members' own recoveries tell that story.
|
|
312
271
|
states.errorStorm = pollErrorStorm(root, states.errorStorm, runners);
|
|
313
|
-
// Team-wide wide-shallow storm alarm (src/team/failure-spread.ts): when many roles each
|
|
272
|
+
// Team-wide wide-shallow storm alarm (src/team/team-failure-spread.ts): when many roles each
|
|
314
273
|
// fail a few times on one provider failure kind, the streak bar needs one role deep,
|
|
315
274
|
// the error storm needs several roles deep, and the hold needs the failures close
|
|
316
275
|
// together — this counts raw failures of one kind across roles in a rolling window,
|
|
@@ -27,6 +27,15 @@ export function newPauseGateState() {
|
|
|
27
27
|
* empty, so a logged `until` is the live deadline. */
|
|
28
28
|
export function pollPauseGates(root, state) {
|
|
29
29
|
const marker = standingTeamPause(root);
|
|
30
|
+
const userPaused = pollTeamPauseCrossing(root, state, marker);
|
|
31
|
+
const { roles: pausedRolesNow, until: rolesUntil } = standingRolePause(root);
|
|
32
|
+
const pausedRolesSet = pollRolePauseCrossing(root, state, pausedRolesNow, rolesUntil);
|
|
33
|
+
return { userPaused, pausedRoles: pausedRolesSet };
|
|
34
|
+
}
|
|
35
|
+
/** The team-pause crossing: one `team_paused`/`team_resumed` event on the poll where the
|
|
36
|
+
* marker's presence flips, carrying a timed pause's `until`/`reason` or the `expired` mark
|
|
37
|
+
* when the previous poll's deadline ran out. Returns whether the team is paused now. */
|
|
38
|
+
function pollTeamPauseCrossing(root, state, marker) {
|
|
30
39
|
const userPaused = marker !== null;
|
|
31
40
|
if (userPaused !== state.prevUserPaused) {
|
|
32
41
|
logEventBestEffort(root, marker
|
|
@@ -44,7 +53,12 @@ export function pollPauseGates(root, state) {
|
|
|
44
53
|
state.prevUserPaused = userPaused;
|
|
45
54
|
}
|
|
46
55
|
state.prevTeamUntil = marker?.until;
|
|
47
|
-
|
|
56
|
+
return userPaused;
|
|
57
|
+
}
|
|
58
|
+
/** The per-role crossings: one `role_paused` per role that just entered the set, and one
|
|
59
|
+
* `role_resumed` per role that left it, carrying the shared deadline or `expired` mark when
|
|
60
|
+
* the previous poll's deadline ran out. Returns the now-paused role set. */
|
|
61
|
+
function pollRolePauseCrossing(root, state, pausedRolesNow, rolesUntil) {
|
|
48
62
|
const rolesExpired = state.prevRolesUntil !== undefined && state.prevRolesUntil <= Date.now();
|
|
49
63
|
const pausedRolesSet = new Set(pausedRolesNow);
|
|
50
64
|
for (const r of pausedRolesNow)
|
|
@@ -55,5 +69,5 @@ export function pollPauseGates(root, state) {
|
|
|
55
69
|
logEventBestEffort(root, { source: TUMWATER_SOURCE, type: "role_resumed", role: r, ...(rolesExpired ? { expired: true } : {}) });
|
|
56
70
|
state.prevPausedRoles = pausedRolesSet;
|
|
57
71
|
state.prevRolesUntil = rolesUntil;
|
|
58
|
-
return
|
|
72
|
+
return pausedRolesSet;
|
|
59
73
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/** The repo preconditions' shared wording behind the readiness gate. startup-
|
|
1
|
+
/** The repo preconditions' shared wording behind the readiness gate. startup-gates.ts answers
|
|
2
2
|
* with the first unmet precondition (cmdStart and requireReadyRepo fail on it, the self-redeploy
|
|
3
3
|
* refuses a swap on it) and doctor.ts reports each one individually, but every surface
|
|
4
4
|
* describes the same problems — so the wording lives here once and cannot drift.
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* and nothing can get stuck"): the verdict is recomputed from the runner state every poll, no
|
|
8
8
|
* marker is written, and local midnight lifts it by itself.
|
|
9
9
|
*
|
|
10
|
-
* Deliberate differences from the streak gate's shape (src/gates/streak-
|
|
10
|
+
* Deliberate differences from the streak gate's shape (src/gates/streak-gates.ts), stated in
|
|
11
11
|
* the plan so no implementer re-litigates them:
|
|
12
12
|
* - This gate writes NOTHING to the shared per-role pause marker (src/team/team-state.ts's
|
|
13
13
|
* pauseRole). The marker is anonymous — after a restart across midnight the harness could
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* module owns the only event emission — the orchestrator's poll owns only the wiring. The
|
|
25
25
|
* in-memory `prev` and `warned` sets are the whole cross-poll memory: a restart with a
|
|
26
26
|
* still-over-cap role re-logs one `role_cap_paused` on the first poll, and a role already past
|
|
27
|
-
* 80% re-logs one `role_cap_warning` (the durable-cause honest report the streak-
|
|
27
|
+
* 80% re-logs one `role_cap_warning` (the durable-cause honest report the streak-gates doc
|
|
28
28
|
* accepts). */
|
|
29
29
|
import { groupBy } from "../collections.js";
|
|
30
30
|
import { budgetWarning, dailyCost, todayStamp } from "../budget/budget.js";
|
|
@@ -50,7 +50,7 @@ export function roleCapPaused(state, cap, now = Date.now()) {
|
|
|
50
50
|
* ids so a caller can log the base (the gate) or flatten the ids (the observer). */
|
|
51
51
|
export function roleCapVerdicts(runners, caps, now) {
|
|
52
52
|
// Group by base role via the shared Map-of-arrays step (collections.groupBy, which
|
|
53
|
-
// time-spend, history-data, and team-polls also fold), then drop the exempt director's
|
|
53
|
+
// failure-time-spend, history-data, and team-polls also fold), then drop the exempt director's
|
|
54
54
|
// group so it never holds a verdict; first-appearance order is preserved.
|
|
55
55
|
const groups = groupBy(runners, (r) => baseRoleOf(r.role));
|
|
56
56
|
groups.delete(DIRECTOR_ROLE);
|
|
@@ -71,6 +71,45 @@ export function roleCapVerdicts(runners, caps, now) {
|
|
|
71
71
|
export function newRoleCapGateState() {
|
|
72
72
|
return { prev: new Set(), warned: new Set() };
|
|
73
73
|
}
|
|
74
|
+
/** One base role's pause/resume edge, logged by the poll's loop: reads whether the role was
|
|
75
|
+
* already judged over its cap at the last poll (`state.prev`) and on ENTER logs one
|
|
76
|
+
* `role_cap_paused` (role, spentUsd, capUsd); on EXIT logs one `role_cap_resumed` (role). */
|
|
77
|
+
function logCapCrossing(root, base, verdict, cap, state) {
|
|
78
|
+
const was = state.prev.has(base);
|
|
79
|
+
if (verdict.over && !was) {
|
|
80
|
+
logEventBestEffort(root, {
|
|
81
|
+
source: TUMWATER_SOURCE,
|
|
82
|
+
type: "role_cap_paused",
|
|
83
|
+
role: base,
|
|
84
|
+
spentUsd: verdict.spentUsd,
|
|
85
|
+
capUsd: cap,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
else if (!verdict.over && was) {
|
|
89
|
+
logEventBestEffort(root, { source: TUMWATER_SOURCE, type: "role_cap_resumed", role: base });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/** The 80% early warning (PLANS.md 2026-10-09), the per-role twin of the team's
|
|
93
|
+
* budget_warning and edge-triggered with the same reset-on-below: crossing the threshold
|
|
94
|
+
* while still under the cap logs one `role_cap_warning`, a dip back under re-arms it. Warn
|
|
95
|
+
* only while the cap is open — a poll that jumps straight to the cap already logs its own
|
|
96
|
+
* `role_cap_paused`, so the warning never doubles the page on the poll the cap is reached. */
|
|
97
|
+
function logCapWarning(root, base, verdict, cap, state) {
|
|
98
|
+
const warn = !verdict.over && cap !== undefined && budgetWarning({ spentUsd: verdict.spentUsd, capUsd: cap });
|
|
99
|
+
if (warn && !state.warned.has(base)) {
|
|
100
|
+
logEventBestEffort(root, {
|
|
101
|
+
source: TUMWATER_SOURCE,
|
|
102
|
+
type: "role_cap_warning",
|
|
103
|
+
role: base,
|
|
104
|
+
spentUsd: verdict.spentUsd,
|
|
105
|
+
capUsd: cap,
|
|
106
|
+
});
|
|
107
|
+
state.warned.add(base);
|
|
108
|
+
}
|
|
109
|
+
else if (!warn) {
|
|
110
|
+
state.warned.delete(base);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
74
113
|
/** Step the gate by one orchestrator poll. Per runner (the director exempt, like every
|
|
75
114
|
* autonomous gate): judge `roleCapPaused` against its configured cap; on ENTER log one
|
|
76
115
|
* `role_cap_paused` carrying the role, its spend, and its cap; on EXIT log one
|
|
@@ -86,39 +125,9 @@ export function pollRoleCapGate(root, state, runners, caps, now) {
|
|
|
86
125
|
const paused = new Set();
|
|
87
126
|
const overBases = new Set();
|
|
88
127
|
for (const [base, verdict] of verdicts) {
|
|
89
|
-
const was = state.prev.has(base);
|
|
90
|
-
if (verdict.over && !was) {
|
|
91
|
-
logEventBestEffort(root, {
|
|
92
|
-
source: TUMWATER_SOURCE,
|
|
93
|
-
type: "role_cap_paused",
|
|
94
|
-
role: base,
|
|
95
|
-
spentUsd: verdict.spentUsd,
|
|
96
|
-
capUsd: caps?.[base],
|
|
97
|
-
});
|
|
98
|
-
}
|
|
99
|
-
else if (!verdict.over && was) {
|
|
100
|
-
logEventBestEffort(root, { source: TUMWATER_SOURCE, type: "role_cap_resumed", role: base });
|
|
101
|
-
}
|
|
102
|
-
// The 80% early warning (PLANS.md 2026-10-09), the per-role twin of the team's
|
|
103
|
-
// budget_warning and edge-triggered with the same reset-on-below: crossing the threshold
|
|
104
|
-
// while still under the cap logs one role_cap_warning, a dip back under re-arms it. Warn
|
|
105
|
-
// only while the cap is open — a poll that jumps straight to the cap already logs its own
|
|
106
|
-
// role_cap_paused, so the warning never doubles the page on the poll the cap is reached.
|
|
107
128
|
const cap = caps?.[base];
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
logEventBestEffort(root, {
|
|
111
|
-
source: TUMWATER_SOURCE,
|
|
112
|
-
type: "role_cap_warning",
|
|
113
|
-
role: base,
|
|
114
|
-
spentUsd: verdict.spentUsd,
|
|
115
|
-
capUsd: cap,
|
|
116
|
-
});
|
|
117
|
-
state.warned.add(base);
|
|
118
|
-
}
|
|
119
|
-
else if (!warn) {
|
|
120
|
-
state.warned.delete(base);
|
|
121
|
-
}
|
|
129
|
+
logCapCrossing(root, base, verdict, cap, state);
|
|
130
|
+
logCapWarning(root, base, verdict, cap, state);
|
|
122
131
|
if (verdict.over) {
|
|
123
132
|
overBases.add(base);
|
|
124
133
|
for (const id of verdict.roleIds)
|