tumwater 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -108
- package/dist/build-info.json +2 -2
- package/dist/src/agent/agent-args.js +24 -0
- package/dist/src/agent/agent-backend.js +2 -2
- package/dist/src/agent/backend-traits.js +60 -0
- package/dist/src/agent/claude/claude-account.js +61 -8
- package/dist/src/agent/claude/claude-args.js +12 -15
- package/dist/src/agent/claude/claude-backend.js +51 -41
- package/dist/src/agent/claude/claude-env.js +7 -3
- package/dist/src/agent/claude/claude-stream.js +7 -16
- package/dist/src/agent/image-input.js +107 -0
- package/dist/src/agent/notes-shim.js +41 -0
- package/dist/src/agent/opencode/opencode-args.js +61 -0
- package/dist/src/agent/opencode/opencode-backend.js +255 -0
- package/dist/src/agent/opencode/opencode-catalog.js +184 -0
- package/dist/src/agent/opencode/opencode-config.js +188 -0
- package/dist/src/agent/opencode/opencode-env.js +36 -0
- package/dist/src/agent/opencode/opencode-session-id.js +37 -0
- package/dist/src/agent/opencode/opencode-stream.js +278 -0
- package/dist/src/agent/role-notes-write.js +16 -32
- package/dist/src/agent/run-agent.js +95 -18
- package/dist/src/agent/session-ledger.js +57 -1
- package/dist/src/backlog/backlog-archive.js +296 -0
- package/dist/src/backlog/backlog-cli.js +573 -0
- package/dist/src/backlog/backlog-commit.js +31 -0
- package/dist/src/backlog/backlog-eligibility.js +357 -33
- package/dist/src/backlog/backlog-md.js +98 -18
- package/dist/src/backlog/backlog-render.js +62 -11
- package/dist/src/backlog/backlog-structure.js +261 -58
- package/dist/src/backlog/backlog-write.js +197 -319
- package/dist/src/backlog/backlog.js +113 -16
- package/dist/src/backlog/question-answer.js +339 -0
- package/dist/src/baseline/main-baseline.js +75 -88
- package/dist/src/baseline/main-red.js +38 -30
- package/dist/src/brief/brief-edit.js +43 -0
- package/dist/src/{brief.js → brief/brief.js} +66 -10
- package/dist/src/budget/budget.js +32 -15
- package/dist/src/budget/fallback-breaker.js +8 -7
- package/dist/src/build/build-check-detect.js +8 -7
- package/dist/src/build/build-check-events.js +68 -22
- package/dist/src/build/build-check-report.js +22 -23
- package/dist/src/build/build-check-scoped.js +51 -62
- package/dist/src/build/build-check.js +15 -14
- package/dist/src/build/build-info.js +2 -1
- package/dist/src/build/build-stage.js +23 -22
- package/dist/src/build/dep-install.js +22 -21
- package/dist/src/build/host-sleep.js +1 -1
- package/dist/src/change/change-data.js +25 -14
- package/dist/src/change/change-render.js +11 -10
- package/dist/src/cli/brief-command.js +58 -0
- package/dist/src/cli/cli-args.js +25 -5
- package/dist/src/cli/cli-command-args.js +46 -14
- package/dist/src/cli/cli-editor.js +64 -0
- package/dist/src/cli/cli-flag-specs.js +54 -16
- package/dist/src/cli/cli-marker-commands.js +13 -10
- package/dist/src/cli/cli-output.js +9 -0
- package/dist/src/cli/cli-query-commands.js +458 -49
- package/dist/src/cli/cli-repo-gate.js +1 -1
- package/dist/src/cli/cli-run.js +208 -59
- package/dist/src/cli/completion.js +207 -0
- package/dist/src/cli/config-commands.js +14 -11
- package/dist/src/cli/help.js +305 -110
- package/dist/src/cli/log-commands.js +127 -21
- package/dist/src/cli/notify-command.js +33 -18
- package/dist/src/cli/question-commands.js +29 -224
- package/dist/src/cli.js +121 -40
- package/dist/src/collections.js +66 -16
- package/dist/src/concurrency/check-permit.js +16 -17
- package/dist/src/concurrency/lock.js +6 -5
- package/dist/src/concurrency/provider-lease.js +69 -0
- package/dist/src/concurrency/semaphore.js +258 -36
- package/dist/src/config/config-aliases.js +106 -0
- package/dist/src/config/config-editable-keys.js +3 -3
- package/dist/src/config/config-fallbacks.js +379 -0
- package/dist/src/config/config-field-checks.js +19 -10
- package/dist/src/config/config-keys.js +160 -0
- package/dist/src/config/config-live.js +64 -12
- package/dist/src/config/config-schema.js +34 -22
- package/dist/src/config/config-validation.js +132 -48
- package/dist/src/config/config-views.js +119 -337
- package/dist/src/config/config-write.js +138 -177
- package/dist/src/config/config.js +60 -49
- package/dist/src/config/model-selector.js +28 -2
- package/dist/src/doctor/doctor-backlog.js +546 -49
- package/dist/src/doctor/doctor-checks.js +534 -56
- package/dist/src/doctor/doctor-holds.js +193 -53
- package/dist/src/doctor/doctor-install.js +30 -0
- package/dist/src/doctor/doctor-launch-services.js +2 -2
- package/dist/src/doctor/doctor-model-checks.js +549 -157
- package/dist/src/doctor/doctor-notify.js +27 -0
- package/dist/src/doctor/doctor-orphans.js +10 -9
- package/dist/src/doctor/doctor-render.js +36 -9
- package/dist/src/doctor/doctor.js +80 -22
- package/dist/src/doctor/image-probe.js +244 -0
- package/dist/src/doctor/probe.js +64 -0
- package/dist/src/events/event-format.js +226 -102
- package/dist/src/events/event-labels.js +116 -0
- package/dist/src/events/event-legacy.js +140 -0
- package/dist/src/events/event-read.js +58 -22
- package/dist/src/events/event-window.js +27 -7
- package/dist/src/events/events.js +18 -14
- package/dist/src/events/notify.js +137 -38
- package/dist/src/failure/failure-cluster.js +12 -9
- package/dist/src/failure/failure-data.js +96 -48
- package/dist/src/failure/failure-render.js +69 -58
- package/dist/src/failure/failure-state-change.js +33 -26
- package/dist/src/failure/time-spend.js +79 -59
- package/dist/src/files/file-queue.js +16 -4
- package/dist/src/files/files.js +18 -6
- package/dist/src/files/json-object.js +18 -3
- package/dist/src/files/tail.js +1 -1
- package/dist/src/gates/bootstrap-gates.js +5 -4
- package/dist/src/gates/budget-gates.js +250 -34
- package/dist/src/gates/disk-gate.js +18 -18
- package/dist/src/gates/gate-polls.js +117 -69
- package/dist/src/gates/maintenance-quota.js +29 -29
- package/dist/src/gates/pause-gates.js +11 -11
- package/dist/src/gates/readiness.js +2 -1
- package/dist/src/gates/role-cap-gates.js +16 -21
- package/dist/src/gates/startup-gate.js +75 -31
- package/dist/src/gates/streak-gate.js +18 -16
- package/dist/src/git/commit-message.js +12 -5
- package/dist/src/git/git-diff.js +16 -16
- package/dist/src/git/git-run.js +8 -8
- package/dist/src/git/git.js +86 -55
- package/dist/src/git/slots-state.js +14 -5
- package/dist/src/git/worktree-pool.js +25 -18
- package/dist/src/git/worktree-use.js +15 -14
- package/dist/src/git/worktree.js +18 -18
- package/dist/src/gui/gui-args.js +10 -14
- package/dist/src/gui/gui-backlog-commands.js +273 -0
- package/dist/src/gui/gui-command.js +1 -1
- package/dist/src/gui/gui-config-commands.js +440 -0
- package/dist/src/gui/gui-endpoint-commands.js +16 -669
- package/dist/src/gui/gui-endpoint-common.js +49 -0
- package/dist/src/gui/gui-endpoints.js +116 -56
- package/dist/src/gui/gui-operator-commands.js +145 -0
- package/dist/src/gui/gui-prompt-commands.js +234 -0
- package/dist/src/gui/gui-server.js +28 -8
- package/dist/src/gui/gui-team-commands.js +164 -0
- package/dist/src/gui/http-body.js +17 -1
- package/dist/src/history/history-data.js +194 -98
- package/dist/src/history/history-render.js +76 -0
- package/dist/src/history/history.js +37 -58
- package/dist/src/inbox/inbox-attachments.js +75 -14
- package/dist/src/inbox/inbox-cancel.js +7 -5
- package/dist/src/inbox/inbox-edit.js +1 -1
- package/dist/src/inbox/inbox-submit.js +44 -4
- package/dist/src/inbox/inbox-top.js +83 -0
- package/dist/src/inbox/inbox.js +8 -2
- package/dist/src/inbox/pending-prompt.js +1 -1
- package/dist/src/inbox/prompt-commands.js +170 -80
- package/dist/src/init/init-templates.js +16 -14
- package/dist/src/init/init.js +12 -11
- package/dist/src/loop/leftover.js +28 -27
- package/dist/src/loop/loop-fallback.js +2 -2
- package/dist/src/loop/loop-pi.js +106 -52
- package/dist/src/loop/loop-recovery.js +15 -15
- package/dist/src/loop/loop-state.js +139 -13
- package/dist/src/loop/loop-tick.js +194 -86
- package/dist/src/loop/loop.js +72 -36
- package/dist/src/loop/model-fallback.js +3 -2
- package/dist/src/loop/revision.js +16 -7
- package/dist/src/merge/backlog-conflicts.js +155 -0
- package/dist/src/{landing/landing-batch.js → merge/merge-batch.js} +90 -89
- package/dist/src/{landing/landing-check-failures.js → merge/merge-check-failures.js} +87 -41
- package/dist/src/{landing/landing-core.js → merge/merge-core.js} +119 -95
- package/dist/src/{landing/landing-diff.js → merge/merge-diff.js} +1 -1
- package/dist/src/merge/merge-drain.js +212 -0
- package/dist/src/{landing/landing-git.js → merge/merge-git.js} +94 -21
- package/dist/src/merge/merge-pipeline.js +99 -0
- package/dist/src/{landing/landing-questions.js → merge/merge-questions.js} +9 -6
- package/dist/src/merge/merge-queue-render.js +75 -0
- package/dist/src/{landing/landing-queue.js → merge/merge-queue.js} +95 -36
- package/dist/src/merge/merge-slot.js +340 -0
- package/dist/src/{landing/landing-stack.js → merge/merge-stack.js} +66 -62
- package/dist/src/{landing/landing-merge.js → merge/merge-to-main.js} +72 -62
- package/dist/src/merge/merge-verify.js +212 -0
- package/dist/src/notes-cli.js +97 -0
- package/dist/src/operator/operator-commands.js +154 -47
- package/dist/src/operator/operator-intent.js +87 -49
- package/dist/src/operator/operator-requests.js +52 -52
- package/dist/src/operator/retire.js +62 -27
- package/dist/src/operator/revert.js +71 -0
- package/dist/src/orchestrator/orchestrator-launch.js +74 -14
- package/dist/src/orchestrator/orchestrator-scheduling.js +129 -44
- package/dist/src/orchestrator/orchestrator.js +158 -100
- package/dist/src/orchestrator/retention.js +29 -12
- package/dist/src/orchestrator/startup-migration-steps.js +51 -0
- package/dist/src/orchestrator/startup-migrations.js +40 -0
- package/dist/src/paths.js +128 -37
- package/dist/src/pi/pi-args.js +26 -3
- package/dist/src/pi/pi-backend.js +15 -23
- package/dist/src/pi/pi-bin.js +18 -12
- package/dist/src/pi/pi-event-line.js +17 -1
- package/dist/src/pi/pi-failure.js +87 -0
- package/dist/src/pi/pi-models.js +126 -38
- package/dist/src/pi/pi-run-result.js +1 -1
- package/dist/src/pi/pi-settings.js +86 -0
- package/dist/src/pi/pi-stream.js +131 -107
- package/dist/src/pi/pi-watchdogs.js +47 -43
- package/dist/src/pi-extension/bounded-output.js +20 -5
- package/dist/src/pi-extension/context-budget.js +3 -1
- package/dist/src/pi-extension/context-shake.js +54 -21
- package/dist/src/pi-extension/context-window.js +37 -0
- package/dist/src/pi-extension/full-output.js +2 -2
- package/dist/src/pi-extension/strip-thinking.js +80 -0
- package/dist/src/pi-extension/trim-system-prompt.js +38 -0
- package/dist/src/process/launch-services.js +11 -10
- package/dist/src/process/process-table.js +1 -1
- package/dist/src/process/process.js +7 -7
- package/dist/src/process/run-marker.js +29 -6
- package/dist/src/process/supervisor.js +47 -25
- package/dist/src/prompt/dialect.js +30 -10
- package/dist/src/prompt/gate-prompts.js +170 -67
- package/dist/src/prompt/principles.js +1 -0
- package/dist/src/prompt/prompt-followup.js +35 -20
- package/dist/src/prompt/prompt.js +249 -91
- package/dist/src/redeploy/redeploy-policy.js +10 -10
- package/dist/src/redeploy/redeploy.js +6 -4
- package/dist/src/redeploy/redeployer.js +63 -55
- package/dist/src/redeploy/self-reload.js +1 -1
- package/dist/src/report/report-data.js +264 -255
- package/dist/src/report/report-fold.js +353 -0
- package/dist/src/report/report-render.js +270 -33
- package/dist/src/request-timeouts.js +1 -1
- package/dist/src/review/known-flakes.js +34 -14
- package/dist/src/review/protected-edits.js +33 -0
- package/dist/src/review/review-followup.js +16 -4
- package/dist/src/review/review-precheck.js +14 -13
- package/dist/src/review/review-verdict.js +14 -0
- package/dist/src/review/review.js +81 -39
- package/dist/src/review/secret-leaks.js +127 -0
- package/dist/src/review/suite-rerun.js +4 -3
- package/dist/src/roles/legacy-role-ids.js +21 -0
- package/dist/src/roles/legacy-role-migration.js +62 -0
- package/dist/src/roles/loop-ids.js +19 -2
- package/dist/src/roles/role-catalog.js +125 -135
- package/dist/src/roles/role-guidance.js +64 -31
- package/dist/src/roles/role-labels.js +10 -0
- package/dist/src/roles/role-render.js +194 -18
- package/dist/src/roles/role-view.js +216 -17
- package/dist/src/roles/roles.js +18 -15
- package/dist/src/scheduling/backoff.js +29 -19
- package/dist/src/scheduling/claims.js +25 -9
- package/dist/src/scheduling/once-round.js +11 -11
- package/dist/src/scheduling/quiet-hours.js +25 -32
- package/dist/src/scheduling/scheduling.js +43 -24
- package/dist/src/scheduling/{work-landed-cache.js → work-merged-cache.js} +14 -14
- package/dist/src/status/status-budget.js +115 -0
- package/dist/src/status/status-data.js +77 -184
- package/dist/src/status/status-inbox.js +47 -0
- package/dist/src/status/status-install.js +27 -0
- package/dist/src/status/status-polls.js +68 -33
- package/dist/src/status/status-roles.js +105 -0
- package/dist/src/{fleet → team}/error-storm.js +10 -10
- package/dist/src/{fleet → team}/failure-spread.js +8 -8
- package/dist/src/{fleet → team}/orchestrator-info.js +86 -18
- package/dist/src/{fleet → team}/reclaim.js +5 -4
- package/dist/src/{fleet/fleet-hold.js → team/team-hold.js} +55 -21
- package/dist/src/{fleet/fleet-polls.js → team/team-polls.js} +116 -65
- package/dist/src/{fleet/fleet-state.js → team/team-state.js} +62 -37
- package/dist/src/text/datetime.js +42 -2
- package/dist/src/text/format.js +10 -1
- package/dist/src/text/markdown.js +1 -1
- package/dist/src/text/phrases.js +35 -34
- package/dist/src/text/suggest.js +3 -2
- package/dist/src/text/text.js +2 -2
- package/dist/src/tick/qa-coverage.js +2 -1
- package/dist/src/tick/stage-check.js +147 -26
- package/dist/src/tick/telemetry-digest.js +10 -9
- package/dist/src/tick/tick-apply.js +101 -72
- package/dist/src/tick/tick-detail-data.js +32 -18
- package/dist/src/tick/tick-detail.js +62 -21
- package/dist/src/tick/tick-finalize.js +29 -22
- package/dist/src/tick/tick-outcome.js +2 -1
- package/dist/src/tick/tick-prompt.js +94 -26
- package/dist/src/tick/tick-resume.js +29 -13
- package/dist/src/tick/tick-stage.js +91 -47
- package/dist/src/tick/tick-timing.js +60 -59
- package/dist/src/tick/tick-usage.js +53 -23
- package/dist/src/tick/tick-verdict.js +76 -10
- package/dist/src/ui/badges.js +118 -67
- package/dist/src/ui/gui/gui-client-boot.js +209 -30
- package/dist/src/ui/gui/gui-client-composer.js +102 -23
- package/dist/src/ui/gui/gui-client-drawer.js +330 -75
- package/dist/src/ui/gui/gui-client-history.js +96 -38
- package/dist/src/ui/gui/gui-client-labels.js +24 -0
- package/dist/src/ui/gui/gui-client-loops.js +94 -52
- package/dist/src/ui/gui/gui-client-markdown.js +1 -0
- package/dist/src/ui/gui/gui-client-model.js +63 -43
- package/dist/src/ui/gui/gui-client-operator.js +129 -39
- package/dist/src/ui/gui/gui-client-pending.js +36 -32
- package/dist/src/ui/gui/gui-client-report.js +339 -46
- package/dist/src/ui/gui/gui-client-settings.js +148 -14
- package/dist/src/ui/gui/gui-client-sound.js +2 -1
- package/dist/src/ui/gui/{gui-client-fleet.js → gui-client-team.js} +196 -72
- package/dist/src/ui/gui/gui-client.js +26 -20
- package/dist/src/ui/gui/gui-page.js +19 -19
- package/dist/src/ui/gui/gui-styles.js +12 -4
- package/dist/src/ui/progress-data.js +5 -3
- package/dist/src/ui/status-model.js +184 -122
- package/dist/src/ui/status-payload.js +119 -32
- package/dist/src/ui/status-render.js +122 -52
- package/dist/src/ui/team-alerts.js +341 -0
- package/dist/src/ui/tick-progress-model.js +14 -14
- package/dist/src/ui/tone.js +11 -8
- package/dist/src/ui/transcript-tail.js +1 -0
- package/dist/src/ui/transcript.js +23 -16
- package/dist/src/ui/tui/tui-app.js +1 -0
- package/dist/src/ui/tui/tui-backlog.js +85 -11
- package/dist/src/ui/tui/tui-editor-answer.js +97 -0
- package/dist/src/ui/tui/tui-editor-attach.js +107 -0
- package/dist/src/ui/tui/tui-editor-brief.js +107 -0
- package/dist/src/ui/tui/tui-editor-budget.js +155 -0
- package/dist/src/ui/tui/tui-editor-ctx.js +1 -0
- package/dist/src/ui/tui/tui-editor-entry.js +230 -0
- package/dist/src/ui/tui/tui-editor-interval.js +122 -0
- package/dist/src/ui/tui/tui-editor-notes.js +114 -0
- package/dist/src/ui/tui/tui-editor-pause.js +266 -0
- package/dist/src/ui/tui/tui-editor-prompt.js +95 -0
- package/dist/src/ui/tui/tui-editor-queued.js +98 -0
- package/dist/src/ui/tui/tui-editor-quiet.js +106 -0
- package/dist/src/ui/tui/tui-editor-retire.js +83 -0
- package/dist/src/ui/tui/tui-editor-search.js +73 -0
- package/dist/src/ui/tui/tui-editor-session.js +97 -0
- package/dist/src/ui/tui/tui-editors.js +152 -0
- package/dist/src/ui/tui/tui-frame.js +67 -55
- package/dist/src/ui/tui/tui-history.js +12 -7
- package/dist/src/ui/tui/tui-input.js +49 -44
- package/dist/src/ui/tui/tui-keymap.js +1 -0
- package/dist/src/ui/tui/tui-keys.js +91 -1704
- package/dist/src/ui/tui/tui-pane-activity.js +99 -0
- package/dist/src/ui/tui/tui-pane-backlog.js +224 -0
- package/dist/src/ui/tui/tui-pane-brief.js +30 -0
- package/dist/src/ui/tui/tui-pane-ctx.js +11 -0
- package/dist/src/ui/tui/tui-pane-health.js +124 -0
- package/dist/src/ui/tui/tui-pane-history.js +316 -0
- package/dist/src/ui/tui/tui-pane-pending.js +74 -0
- package/dist/src/ui/tui/tui-pane-queued.js +143 -0
- package/dist/src/ui/tui/tui-pane-report.js +122 -0
- package/dist/src/ui/tui/tui-pane-session.js +101 -0
- package/dist/src/ui/tui/tui-pane-transcript.js +150 -0
- package/dist/src/ui/tui/tui-pane.js +66 -28
- package/dist/src/ui/tui/tui-panes.js +294 -0
- package/dist/src/ui/tui/tui.js +45 -25
- package/dist/src/verdict/fix-claim.js +141 -38
- package/dist/src/verdict/no-change.js +19 -0
- package/dist/src/verdict/refusal.js +92 -8
- package/dist/src/verdict/reply-contract.js +17 -5
- package/dist/src/version.js +3 -2
- package/package.json +4 -2
- package/dist/src/agent/mcp/role-notes-server.js +0 -160
- package/dist/src/landing/backlog-conflicts.js +0 -116
- package/dist/src/landing/landing-drain.js +0 -204
- package/dist/src/landing/landing-pipeline.js +0 -96
- package/dist/src/landing/landing-queue-render.js +0 -31
- package/dist/src/landing/landing-slot.js +0 -246
- package/dist/src/landing/landing-vetting.js +0 -196
- package/dist/src/pi-extension/role-notes.js +0 -40
- package/dist/src/ui/fleet-alerts.js +0 -174
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
// vocabulary: plain
|
|
1
2
|
import { asTestCounts } from "../build/build-check-counts.js";
|
|
2
|
-
import {
|
|
3
|
+
import { isFullCheckScope } from "../build/build-check-events.js";
|
|
4
|
+
import { isMergeChange, readMergeMarker } from "../merge/merge-slot.js";
|
|
3
5
|
import { defaultConfig, loadConfigCached } from "../config/config.js";
|
|
4
6
|
import { DEFAULT_EVENT_TAIL, readEvents } from "../events/event-read.js";
|
|
5
7
|
import { currentBranchFromHeadFile, readBranchHead, targetBranch } from "../git/git.js";
|
|
@@ -8,9 +10,18 @@ import { eventsLogPath, statePath } from "../paths.js";
|
|
|
8
10
|
import { cachedByStat } from "../files/stat-cache.js";
|
|
9
11
|
import { countMaintenanceWindow, maintenanceQuota, newMaintenanceWindowCounter, } from "../gates/maintenance-quota.js";
|
|
10
12
|
import { commitTier } from "../roles/roles.js";
|
|
13
|
+
/** Is a value one of MainCheckStatus's three statuses? The one home of the membership test:
|
|
14
|
+
* the event log is loose-typed by design, and a check whose status is not one of the three
|
|
15
|
+
* (a hand-edited or future-format block) is malformed — dropped like any other malformed
|
|
16
|
+
* check (see MainCheckScan.check's note) rather than narrowed with a cast into a status it
|
|
17
|
+
* cannot prove, which would otherwise leak a mystery word to the header badge and the GUI
|
|
18
|
+
* payload (mainCheckVerdict renders any unmatched status verbatim). */
|
|
19
|
+
function isMainCheckStatus(status) {
|
|
20
|
+
return status === "passed" || status === "failed" || status === "skipped";
|
|
21
|
+
}
|
|
11
22
|
/** How far mainCheckForPoll may grow its event tail. The per-poll default
|
|
12
|
-
* (event-read.ts's DEFAULT_EVENT_TAIL) covers a healthy
|
|
13
|
-
* every
|
|
23
|
+
* (event-read.ts's DEFAULT_EVENT_TAIL) covers a healthy team — a merge-scope check rides
|
|
24
|
+
* every merge and every redeploy — but a burst of quiet or failing ticks logs hundreds
|
|
14
25
|
* of events without moving main, and a tail
|
|
15
26
|
* that ends before the last check makes the header badge vanish and reappear as ticks tick
|
|
16
27
|
* by. Growing until the check is found keeps "absent" meaning what the field's contract says
|
|
@@ -21,9 +32,9 @@ import { commitTier } from "../roles/roles.js";
|
|
|
21
32
|
* Past the cap the badge drops, the same graceful loss log rotation already imposes. */
|
|
22
33
|
const MAIN_CHECK_SCAN_MAX_EVENTS = 5_000;
|
|
23
34
|
/** Per-root note that the previous scan had to grow past the default event tail
|
|
24
|
-
* (DEFAULT_EVENT_TAIL) without finding a merge-scope check. A check rides every
|
|
35
|
+
* (DEFAULT_EVENT_TAIL) without finding a merge-scope check. A check rides every merge, so a poll
|
|
25
36
|
* that grew all the way to the cap is a burst of quiet or failing ticks — hundreds of events
|
|
26
|
-
* between
|
|
37
|
+
* between merges — and the next poll's growth starts straight at the cap instead of re-scanning
|
|
27
38
|
* the ×4 ladder's intermediate windows (200→800→3200→5000 ≈ 9.2k re-parsed events per
|
|
28
39
|
* fresh tail, vs 5.2k for 200→cap; measured on a 6k-event check-free log, ~2.3 ms → ~1.2 ms per
|
|
29
40
|
* append+poll). A poll that finds a check clears the note, so the ordinary find-in-200 /
|
|
@@ -32,21 +43,21 @@ const MAIN_CHECK_SCAN_MAX_EVENTS = 5_000;
|
|
|
32
43
|
const mainCheckGrewFull = new Set();
|
|
33
44
|
const MAIN_CHECK_FULL_SET_MAX = 64;
|
|
34
45
|
/** Per-root scan cache (stat-cache.cachedByStat): both dashboards poll snapshot every second,
|
|
35
|
-
* but events.jsonl is written only at tick boundaries and
|
|
46
|
+
* but events.jsonl is written only at tick boundaries and merges, so an unchanged log serves
|
|
36
47
|
* the whole scan from one stat instead of re-listing the ladder, re-parsing its windows, and
|
|
37
|
-
* re-collecting the
|
|
48
|
+
* re-collecting the merged events. Rotation swaps the inode and every append changes the size,
|
|
38
49
|
* so both invalidate through the same freshness check; keyed by root so distinct repos never
|
|
39
50
|
* collide. Bounded inside cachedByStat. */
|
|
40
51
|
const mainCheckScanCache = new Map();
|
|
41
52
|
/** Scan the log for MainCheckScan, growing the tail until a merge-scope check is inside it (see
|
|
42
53
|
* MAIN_CHECK_SCAN_MAX_EVENTS): a check older than the default tail must still badge the header,
|
|
43
|
-
* and every event after the check — the
|
|
54
|
+
* and every event after the check — the merged pairing the sha needs — is newer, so one window
|
|
44
55
|
* holds the whole derivation. */
|
|
45
56
|
function scanMainCheck(root) {
|
|
46
57
|
let check;
|
|
47
|
-
// The final window's
|
|
58
|
+
// The final window's merged events, in scan order — collected in the same pass that
|
|
48
59
|
// finds the check instead of a second full scan of the window below.
|
|
49
|
-
let
|
|
60
|
+
let mergeDone = [];
|
|
50
61
|
const grewFull = mainCheckGrewFull.has(root);
|
|
51
62
|
// A set note means the previous scan reached the cap without finding a check, so start
|
|
52
63
|
// there: starting at DEFAULT_EVENT_TAIL would only re-read (and re-parse on a cache miss)
|
|
@@ -55,16 +66,16 @@ function scanMainCheck(root) {
|
|
|
55
66
|
for (let window = grewFull ? MAIN_CHECK_SCAN_MAX_EVENTS : DEFAULT_EVENT_TAIL;; window = grewFull ? MAIN_CHECK_SCAN_MAX_EVENTS : Math.min(window * 4, MAIN_CHECK_SCAN_MAX_EVENTS)) {
|
|
56
67
|
const events = readEvents(root, window);
|
|
57
68
|
check = undefined;
|
|
58
|
-
|
|
69
|
+
mergeDone = [];
|
|
59
70
|
// One pass collects both facts the derivation needs: the newest merge-scope check and
|
|
60
|
-
// every
|
|
61
|
-
// over the
|
|
71
|
+
// every merged event. The merges are re-read against the check's ts below — a pass
|
|
72
|
+
// over the merges alone, not a second pass over every event in the window.
|
|
62
73
|
for (const e of events) {
|
|
63
|
-
if (e.type === "build_check" && (e.scope
|
|
74
|
+
if (e.type === "build_check" && isFullCheckScope(e.scope)) {
|
|
64
75
|
check = e;
|
|
65
76
|
}
|
|
66
|
-
else if (e.type === "
|
|
67
|
-
|
|
77
|
+
else if (e.type === "merge_done" && typeof e.ts === "number") {
|
|
78
|
+
mergeDone.push(e);
|
|
68
79
|
}
|
|
69
80
|
}
|
|
70
81
|
// Found, the log is shorter than the window (nothing older exists to find), or the cap
|
|
@@ -80,21 +91,21 @@ function scanMainCheck(root) {
|
|
|
80
91
|
mainCheckGrewFull.delete(root);
|
|
81
92
|
else
|
|
82
93
|
mainCheckGrewFull.add(root);
|
|
83
|
-
if (!check || typeof check.ts !== "number" ||
|
|
94
|
+
if (!check || typeof check.ts !== "number" || !isMainCheckStatus(check.status)) {
|
|
84
95
|
return { check: undefined };
|
|
85
96
|
}
|
|
86
|
-
let
|
|
87
|
-
let
|
|
88
|
-
for (const e of
|
|
97
|
+
let mergeDoneAfter;
|
|
98
|
+
let mergeDoneBefore;
|
|
99
|
+
for (const e of mergeDone) {
|
|
89
100
|
if (e.ts > check.ts)
|
|
90
|
-
|
|
101
|
+
mergeDoneAfter = e;
|
|
91
102
|
else
|
|
92
|
-
|
|
103
|
+
mergeDoneBefore = e;
|
|
93
104
|
}
|
|
94
105
|
return {
|
|
95
106
|
check,
|
|
96
|
-
...(typeof
|
|
97
|
-
...(typeof
|
|
107
|
+
...(typeof mergeDoneAfter?.commit === "string" ? { mergeDoneAfterCommit: mergeDoneAfter.commit } : {}),
|
|
108
|
+
...(typeof mergeDoneBefore?.commit === "string" ? { mergeDoneBeforeCommit: mergeDoneBefore.commit } : {}),
|
|
98
109
|
};
|
|
99
110
|
}
|
|
100
111
|
/** The cached scan for `root`, computing it through scanMainCheck on a cache miss. The clone
|
|
@@ -103,6 +114,21 @@ function scanMainCheck(root) {
|
|
|
103
114
|
function mainCheckScanFor(root) {
|
|
104
115
|
return (cachedByStat(mainCheckScanCache, root, eventsLogPath(root), () => scanMainCheck(root), (v) => ({ ...v, ...(v.check ? { check: { ...v.check } } : {}) })) ?? { check: undefined });
|
|
105
116
|
}
|
|
117
|
+
/** Has main's newest merge-scope check retired a loop's stale `main_red` verdict? Only a
|
|
118
|
+
* PASSED check stamped strictly after the loop's red tick ended counts — the team-level
|
|
119
|
+
* green is newer evidence than the loop's own red (BUGS.md 2026-09-30). A failed or skipped
|
|
120
|
+
* newest check, a check from before the red tick, or a loop with no known tick end all leave
|
|
121
|
+
* the verdict standing. Single-homed here beside heldByMainRed. */
|
|
122
|
+
function mainCheckRecovered(mainCheck, s) {
|
|
123
|
+
return mainCheck?.status === "passed"
|
|
124
|
+
&& s.lastRunEndedAt !== undefined
|
|
125
|
+
&& mainCheck.at > s.lastRunEndedAt;
|
|
126
|
+
}
|
|
127
|
+
/** Is a loop's tick verdict still the red-main hold? The one predicate behind the status
|
|
128
|
+
* phase's `main red` label and doctor's `main red` check, so the two cannot disagree. */
|
|
129
|
+
export function heldByMainRed(s, mainCheck) {
|
|
130
|
+
return s.lastResult === "main_red" && s.queuedSummary === undefined && !mainCheckRecovered(mainCheck, s);
|
|
131
|
+
}
|
|
106
132
|
/** The newest merge-scope build_check in the event tail, with the main sha it verified
|
|
107
133
|
* (mainCheck's derivation — see the field's comment). A gate-scope check is excluded: it
|
|
108
134
|
* verified a role worktree, not main. The log-derived scan is stat-cached per events.jsonl
|
|
@@ -111,10 +137,10 @@ function mainCheckScanFor(root) {
|
|
|
111
137
|
export function mainCheckForPoll(root, cfg) {
|
|
112
138
|
const scan = mainCheckScanFor(root);
|
|
113
139
|
const check = scan.check;
|
|
114
|
-
if (!check || typeof check.ts !== "number" ||
|
|
140
|
+
if (!check || typeof check.ts !== "number" || !isMainCheckStatus(check.status))
|
|
115
141
|
return undefined;
|
|
116
142
|
const branch = targetBranch(cfg.baseBranch, currentBranchFromHeadFile(root));
|
|
117
|
-
const sha = scan.
|
|
143
|
+
const sha = scan.mergeDoneAfterCommit ?? readBranchHead(root, branch) ?? scan.mergeDoneBeforeCommit;
|
|
118
144
|
const counts = asTestCounts(check.counts);
|
|
119
145
|
return {
|
|
120
146
|
...(sha ? { sha } : {}),
|
|
@@ -128,18 +154,27 @@ export function mainCheckForPoll(root, cfg) {
|
|
|
128
154
|
* marker displays only while its sha is still queued; a marker with per-change records keeps
|
|
129
155
|
* just its records whose sha is still queued — a change whose entry was dropped is finished,
|
|
130
156
|
* whatever its record last said — and displays only while one of those is not `done`. */
|
|
131
|
-
export function
|
|
157
|
+
export function liveMergeMarker(marker, queued) {
|
|
132
158
|
if (!Array.isArray(marker.changes))
|
|
133
159
|
return queued.has(marker.sha) ? marker : undefined;
|
|
134
160
|
// Drop foreign entries before reading their sha/status: a `null`/scalar record in a
|
|
135
161
|
// hand-edited marker would otherwise throw out of the every-second status poll.
|
|
136
|
-
const changes = marker.changes.filter(
|
|
162
|
+
const changes = marker.changes.filter(isMergeChange).filter((c) => queued.has(c.sha));
|
|
137
163
|
return changes.some((c) => c.status !== "done") ? { ...marker, changes } : undefined;
|
|
138
164
|
}
|
|
165
|
+
/** The marker observers display for a queue read: undefined unless the orchestrator is running
|
|
166
|
+
* (`running`) and the marker has a live record for a queued entry (liveMergeMarker). The one
|
|
167
|
+
* home of this cross-check for the status snapshot, `tumwater queue`, and the TUI Pending pane. */
|
|
168
|
+
export function queueInFlight(root, entries, running) {
|
|
169
|
+
if (!running)
|
|
170
|
+
return undefined;
|
|
171
|
+
const marker = readMergeMarker(root);
|
|
172
|
+
return marker ? liveMergeMarker(marker, new Set(entries.map((e) => e.sha))) : undefined;
|
|
173
|
+
}
|
|
139
174
|
// The last config each root loaded successfully. snapshot is polled every second by the
|
|
140
175
|
// TUI and GUI, where a throw would kill (or blind) the observer — but tumwater.json can be
|
|
141
176
|
// transiently broken while a user edits it live-reload style. On failure we keep showing
|
|
142
|
-
// the
|
|
177
|
+
// the team with the last known-good role set; the orchestrator does the same in its own
|
|
143
178
|
// reload poll and surfaces the error as a warning event visible in `tumwater logs`.
|
|
144
179
|
const lastGoodConfig = new Map();
|
|
145
180
|
/** The config to display loop state against: fresh when valid, otherwise the last
|
|
@@ -178,10 +213,10 @@ const maintenanceCounters = new Map();
|
|
|
178
213
|
/** The maintenance allowance verdict as an observer sees it for this poll (status-data's
|
|
179
214
|
* `maintenanceQuota` field): the rolling-window counts over the same event log the gate reads,
|
|
180
215
|
* plus the enabled maintenance loops currently in flight (running a tick or holding a queued
|
|
181
|
-
*
|
|
216
|
+
* merge, so the `used` figure matches the gate's). `held` names the enabled maintenance loop
|
|
182
217
|
* ids the scheduler holds while `used >= allowance`. Pure read — unlike the gate it logs nothing
|
|
183
218
|
* and keeps no hold edge, so it can run in the TUI/GUI process every second. */
|
|
184
|
-
export function maintenanceQuotaForPoll(root,
|
|
219
|
+
export function maintenanceQuotaForPoll(root, perWorkMerge, loops, queuedMergeRoles, now) {
|
|
185
220
|
let counter = maintenanceCounters.get(root);
|
|
186
221
|
if (!counter) {
|
|
187
222
|
counter = newMaintenanceWindowCounter();
|
|
@@ -194,9 +229,9 @@ export function maintenanceQuotaForPoll(root, perWorkLanding, loops, queuedLandi
|
|
|
194
229
|
if (commitTier(loop.role) !== "maintenance")
|
|
195
230
|
continue;
|
|
196
231
|
maintenanceRoles.push(loop.role);
|
|
197
|
-
if (loop.running ||
|
|
232
|
+
if (loop.running || queuedMergeRoles.has(loop.role))
|
|
198
233
|
inFlight++;
|
|
199
234
|
}
|
|
200
|
-
const verdict = maintenanceQuota({ work, maint, inFlight,
|
|
235
|
+
const verdict = maintenanceQuota({ work, maint, inFlight, perWorkMerge });
|
|
201
236
|
return { allowance: verdict.allowance, used: verdict.used, held: verdict.held ? maintenanceRoles : [] };
|
|
202
237
|
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// vocabulary: plain
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { configForRole, modelSelectorField, roleSeamTier, } from "../config/config-views.js";
|
|
4
|
+
import { modelFallbackView } from "../config/config-fallbacks.js";
|
|
5
|
+
import { isCustomRole } from "../config/config.js";
|
|
6
|
+
import { baseRoleOf, loopIds } from "../roles/loop-ids.js";
|
|
7
|
+
import { formatModelSelector } from "../config/model-selector.js";
|
|
8
|
+
import { piSettingsPath, readPiDefaultModel } from "../pi/pi-settings.js";
|
|
9
|
+
import { isJsonObject } from "../files/json-object.js";
|
|
10
|
+
import { loopStateForPoll } from "./status-polls.js";
|
|
11
|
+
import { roleImageInfo, seamImageLabel } from "../agent/image-input.js";
|
|
12
|
+
import { imageProbeEntryFresh, readImageProbeCache } from "../doctor/image-probe.js";
|
|
13
|
+
/** One row per enabled loop id (part 5b/7). `configForRole`, `modelFallbackView`, the leased /
|
|
14
|
+
* pinned slot and the pi-default seam all resolve against the values `snapshot()` already
|
|
15
|
+
* read once for the poll — the info and slots pins (threaded as `slots`) keep rows, `running`
|
|
16
|
+
* and `disk` on the same shared reads, and `now` is the snapshot's pinned clock so every row
|
|
17
|
+
* agrees on what "active" means. One read of pi's own saved default per poll (docs/backends.md)
|
|
18
|
+
* serves the rows whose seam runs pi and resolves no model — its settings.json sits beside the
|
|
19
|
+
* models catalog this snapshot reads (piSettingsPath), so one seam serves both. */
|
|
20
|
+
export function collectRoleRows(root, cfg, now, slots, modelsPath) {
|
|
21
|
+
const roles = loopIds(cfg);
|
|
22
|
+
const piDefault = readPiDefaultModel(piSettingsPath(modelsPath), root);
|
|
23
|
+
// One probe-cache read per poll serves every row's image-input surface (same no-data policy
|
|
24
|
+
// as every other degraded read: a missing cache is an empty one).
|
|
25
|
+
const probes = readImageProbeCache(root);
|
|
26
|
+
return roles.map((r) => {
|
|
27
|
+
const base = { ...loopStateForPoll(root, r), custom: isCustomRole(cfg, r) };
|
|
28
|
+
const leased = slots.find((slot) => slot.lease?.role === r);
|
|
29
|
+
const pinned = slots.find((slot) => slot.pinnedFor === r);
|
|
30
|
+
const slot = leased ?? pinned;
|
|
31
|
+
// Every row carries the selector its config resolves to, so the dashboards show which
|
|
32
|
+
// model a role rides even with a single string `model`. A tier-name `roles.<id>.model`
|
|
33
|
+
// and a role selector override both resolve through configForRole — the same view the
|
|
34
|
+
// seam consumes. Part 7a (model-tiers.md "Observability"): the seam tier tag is added
|
|
35
|
+
// only with a tier map declared, where it names the tier a role rides.
|
|
36
|
+
const eff = configForRole(cfg, r);
|
|
37
|
+
// The image-input surface (plans/image-input.md "Surfaces", part 5/5): the loop's own
|
|
38
|
+
// capability and its reroute target, one shared resolver so the status rows, the role
|
|
39
|
+
// page, and the submit gate cannot disagree about who reads images or where a blind
|
|
40
|
+
// prompt would run.
|
|
41
|
+
const image = roleImageInfo(cfg, r, modelsPath, probes);
|
|
42
|
+
// Model failure fallback, part 2/2: while the role's episode is active its ticks run on
|
|
43
|
+
// the tier-resolved fallback pair, so the row names that pair, the episode's start, and
|
|
44
|
+
// the failure that tripped it. `now` is the poll's own clock (the snapshot seam), so
|
|
45
|
+
// every row agrees on what "active" means.
|
|
46
|
+
const episode = modelFallbackView(base.modelFallback, cfg, r, now);
|
|
47
|
+
// A seam that runs pi and resolves neither a provider nor a model starts on pi's own
|
|
48
|
+
// saved default — name it in the row (modelFromPi) so the operator sees which model the
|
|
49
|
+
// loop runs. The config's `thinking` wins over the default's because tumwater passes it
|
|
50
|
+
// to pi as `--thinking` (src/pi/pi-args.ts). Any other row is unchanged and carries no
|
|
51
|
+
// `modelFromPi`.
|
|
52
|
+
const runsPiWithoutAModel = (eff.agent === undefined || eff.agent === "pi") &&
|
|
53
|
+
eff.provider === undefined &&
|
|
54
|
+
eff.model === undefined;
|
|
55
|
+
const modelFromPi = runsPiWithoutAModel && piDefault !== null;
|
|
56
|
+
return {
|
|
57
|
+
...base,
|
|
58
|
+
...modelSelectorField(eff),
|
|
59
|
+
imageInput: image.imageInput,
|
|
60
|
+
imageRerouteLabel: image.imageReroute,
|
|
61
|
+
...(modelFromPi
|
|
62
|
+
? {
|
|
63
|
+
model: formatModelSelector({
|
|
64
|
+
...piDefault,
|
|
65
|
+
thinking: eff.thinking ?? piDefault.thinking,
|
|
66
|
+
}),
|
|
67
|
+
modelFromPi: true,
|
|
68
|
+
}
|
|
69
|
+
: {}),
|
|
70
|
+
...(isJsonObject(cfg.model) ? { modelTier: roleSeamTier(cfg, r) } : {}),
|
|
71
|
+
...(episode !== null ? { fallback: episode } : {}),
|
|
72
|
+
...(slot ? { slot: path.basename(slot.dir), slotPinned: pinned !== undefined } : {}),
|
|
73
|
+
...(baseRoleOf(r) === r ? {} : { instanceOf: baseRoleOf(r) }),
|
|
74
|
+
};
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
/** The doctor's live-probe failures (plans/image-input.md "Surfaces", part 5/5): every cached
|
|
78
|
+
* probe verdict of `fail` that is still fresh (the claim is unchanged and within the 7-day
|
|
79
|
+
* TTL), labelled as `<provider>/<model>`. A claimed model that failed the probe reads `false`
|
|
80
|
+
* everywhere (seamImageInput honors a fresh failure), but the observability value is naming
|
|
81
|
+
* it — a claim the doctor contradicted is exactly what the amber probe alert and the role/status
|
|
82
|
+
* rows should surface, not just "this loop is text-only". A torn or missing cache answers
|
|
83
|
+
* empty (the same no-data policy as the per-loop read). */
|
|
84
|
+
export function collectImageProbeFailures(root, now = Date.now()) {
|
|
85
|
+
const probes = readImageProbeCache(root);
|
|
86
|
+
const out = [];
|
|
87
|
+
for (const [key, entry] of Object.entries(probes)) {
|
|
88
|
+
if (entry.verdict !== "fail" || !imageProbeEntryFresh(entry, now))
|
|
89
|
+
continue;
|
|
90
|
+
// The cache key is agent/provider/model, and a pi model id is itself fully qualified (the
|
|
91
|
+
// key's third segment can carry embedded '/' — parseModelSelector keeps the id whole), so
|
|
92
|
+
// split only the first two separators and keep the rest as the model. A naive split("/")
|
|
93
|
+
// destructured to three parts would truncate a qualified id to its org, mislabelling the
|
|
94
|
+
// model the doctor flagged.
|
|
95
|
+
const first = key.indexOf("/");
|
|
96
|
+
const second = key.indexOf("/", first + 1);
|
|
97
|
+
if (first === -1 || second === -1)
|
|
98
|
+
continue;
|
|
99
|
+
const agent = key.slice(0, first) || undefined;
|
|
100
|
+
const provider = key.slice(first + 1, second) || undefined;
|
|
101
|
+
const model = key.slice(second + 1) || undefined;
|
|
102
|
+
out.push(seamImageLabel(agent, provider, model));
|
|
103
|
+
}
|
|
104
|
+
return out.sort();
|
|
105
|
+
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
/** The
|
|
1
|
+
/** The team-wide error-storm warning (BUGS.md 2026-09-29): when three or more roles' ticks
|
|
2
2
|
* are failing consecutively on one shared cause, only each role's own "consecutive tick
|
|
3
|
-
* failures" warning fires — nothing
|
|
3
|
+
* failures" warning fires — nothing team-wide names the shared cause. The 2026-09-22
|
|
4
4
|
* meltdown (all 14 roles timing out at the 1800 s default for 8 hours) surfaced as 14
|
|
5
|
-
* independent streak warnings; an operator scanning the feed saw a quiet
|
|
6
|
-
* the shape of the
|
|
5
|
+
* independent streak warnings; an operator scanning the feed saw a quiet team. POLICY, on
|
|
6
|
+
* the shape of the team-wide hold (src/team/team-hold.ts): a pure reducer over the
|
|
7
7
|
* observations the orchestrator collects each poll and the previous storm, so the rule is
|
|
8
|
-
* unit-testable without a
|
|
8
|
+
* unit-testable without a team and the orchestrator owns only the wiring (the warning
|
|
9
9
|
* event). Like the hold it has memory — which storm is active is a fact about the past
|
|
10
10
|
* no single poll's inputs carry — so it is a reducer rather than a stateless predicate. */
|
|
11
11
|
import { getOrCreate, rankByCount } from "../collections.js";
|
|
@@ -18,19 +18,19 @@ import { ERROR_STREAK_WARN } from "../tick/tick-apply.js";
|
|
|
18
18
|
* standard the 2026-09-22 storm's diagnosis needed. Two would cry wolf on a pair of roles that
|
|
19
19
|
* happen to hit the same broken merge target; the per-role warnings already cover that pair. */
|
|
20
20
|
export const ERROR_STORM_ROLES = 3;
|
|
21
|
-
/** The
|
|
21
|
+
/** The team's starting state: no storm. */
|
|
22
22
|
export const ERROR_STORM_QUIET = { key: null, roles: [] };
|
|
23
23
|
/** The config knob a shared cause points at, when one is known. Only the timeout cause is
|
|
24
24
|
* mapped: `timed out after <dur>` is the reducer's pooled rendering of the tick-timeout errors
|
|
25
|
-
* (src/agent/run-agent.ts's two `timed out after ${
|
|
25
|
+
* (src/agent/run-agent.ts's two `timed out after ${runTimeoutSeconds}s` shapes, pooled into one
|
|
26
26
|
* key by poolTimeoutKey — src/failure/failure-cluster.ts owns the two shapes and their
|
|
27
|
-
* pooling), and a
|
|
27
|
+
* pooling), and a team-wide
|
|
28
28
|
* run of it means the tick budget does not fit the serving model — the one cause with a knob
|
|
29
29
|
* to name. Every other cause is left unmapped: the storm warning still names it, but inventing
|
|
30
30
|
* a knob for a cause no setting controls would send an operator turning the wrong dial.
|
|
31
31
|
* Exported for its unit tests. */
|
|
32
32
|
export function errorStormKnob(key) {
|
|
33
|
-
return key === TICK_TIMEOUT_KEY ? "
|
|
33
|
+
return key === TICK_TIMEOUT_KEY ? "runTimeoutSeconds" : undefined;
|
|
34
34
|
}
|
|
35
35
|
/** Step the storm by one orchestrator poll: from `prev` and every role's current streak,
|
|
36
36
|
* decide what is storming now. A role counts toward a cause once its streak has reached the
|
|
@@ -41,7 +41,7 @@ export function errorStormKnob(key) {
|
|
|
41
41
|
* (the wiring logs exactly one event), a poll that still sees the same storm returns `prev`
|
|
42
42
|
* itself so the caller can tell nothing changed, and a storm whose members drop below the
|
|
43
43
|
* bar goes quiet and re-arms — a recurrence warns again. A storm on a different cause is a
|
|
44
|
-
* new episode: the old cause no longer explains the
|
|
44
|
+
* new episode: the old cause no longer explains the team, so the new one names itself. */
|
|
45
45
|
export function errorStorm(prev, observations) {
|
|
46
46
|
const rolesByKey = new Map();
|
|
47
47
|
for (const { role, consecutiveErrors, lastError } of observations) {
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
/** The
|
|
1
|
+
/** The team-wide wide-shallow storm alarm (BUGS.md 2026-09-30, part (2) of the
|
|
2
2
|
* 2026-09-29 connection-storm entry): when many roles each fail a few times on one provider
|
|
3
3
|
* failure kind, no existing bar trips — the per-role streak alarm needs one role DEEP
|
|
4
4
|
* (ERROR_STREAK_WARN consecutive, src/tick/tick-apply.ts), the error-storm warning needs several
|
|
5
|
-
* roles deep at once (src/
|
|
6
|
-
* a two-minute window (src/
|
|
5
|
+
* roles deep at once (src/team/error-storm.ts), and the team hold needs failures COALESCED within
|
|
6
|
+
* a two-minute window (src/team/team-hold.ts). A degraded backend that fails the team widely and
|
|
7
7
|
* shallowly — six ticks across as many roles, each failure minutes apart — rides all three
|
|
8
8
|
* gaps in silence. This alarm counts raw failures of one kind across roles in a rolling
|
|
9
9
|
* window, independent of any role's streak depth: the fourth bar, shaped for the spread the
|
|
10
|
-
* other three cannot see. POLICY, on the shape of its siblings (src/
|
|
11
|
-
* src/
|
|
12
|
-
* poll and the previous state, so the rule is unit-testable without a
|
|
10
|
+
* other three cannot see. POLICY, on the shape of its siblings (src/team/error-storm.ts,
|
|
11
|
+
* src/team/team-hold.ts): a pure reducer over the observations the orchestrator collects each
|
|
12
|
+
* poll and the previous state, so the rule is unit-testable without a team and the
|
|
13
13
|
* orchestrator owns only the wiring (the warning event). Like both siblings it has memory —
|
|
14
14
|
* which failures still sit inside the window is a fact about the past no single poll's inputs
|
|
15
15
|
* carry — so it is a reducer rather than a stateless predicate. */
|
|
@@ -24,10 +24,10 @@ import { increment, rankCountEntries } from "../collections.js";
|
|
|
24
24
|
export const FAILURE_SPREAD_COUNT = 6;
|
|
25
25
|
/** How recent a failure must be to count toward the alarm. Thirty minutes: wide enough that
|
|
26
26
|
* failures spaced minutes apart — the shape a degraded backend produces against a staggered
|
|
27
|
-
*
|
|
27
|
+
* team — read as one episode; narrow enough that a bad half-hour this morning does not alarm
|
|
28
28
|
* all afternoon. */
|
|
29
29
|
export const FAILURE_SPREAD_WINDOW_MS = 30 * 60_000;
|
|
30
|
-
/** The
|
|
30
|
+
/** The team's starting state: quiet, with no history. */
|
|
31
31
|
export const FAILURE_SPREAD_QUIET = { recent: [], active: false, kind: null };
|
|
32
32
|
/** Step the alarm by one orchestrator poll: fold each observation into the window (skipping
|
|
33
33
|
* ones older than the window and ones already folded — the same runner observation re-read
|
|
@@ -2,7 +2,7 @@ import { readJsonFile, writeJsonAtomic } from "../files/json-files.js";
|
|
|
2
2
|
import { finiteNumber, isJsonObject } from "../files/json-object.js";
|
|
3
3
|
import { pidAlive } from "../process/process.js";
|
|
4
4
|
import { orchestratorStatePath } from "../paths.js";
|
|
5
|
-
import { warnEventBestEffort } from "../events/events.js";
|
|
5
|
+
import { warnEventBestEffort, TUMWATER_SOURCE } from "../events/events.js";
|
|
6
6
|
import { errorMessage } from "../text/text.js";
|
|
7
7
|
/** Assign one field of the orchestrator info file in place, but only when the value differs
|
|
8
8
|
* from what it already carries, and report whether it changed. This is the shared rule for the
|
|
@@ -11,12 +11,30 @@ import { errorMessage } from "../text/text.js";
|
|
|
11
11
|
* once after several fields. The comparison uses JSON serialization, matching
|
|
12
12
|
* writeOrchestratorInfo's
|
|
13
13
|
* own output, so a value equal to the stored one is never treated as a change. */
|
|
14
|
-
|
|
14
|
+
function assignInfoFieldIfChanged(info, field, value) {
|
|
15
15
|
if (JSON.stringify(value) === JSON.stringify(info[field]))
|
|
16
16
|
return false;
|
|
17
17
|
info[field] = value;
|
|
18
18
|
return true;
|
|
19
19
|
}
|
|
20
|
+
/** Assign several non-recomputable info fields in place (each via assignInfoFieldIfChanged) and
|
|
21
|
+
* publish the file best-effort once when any of them changed — the publish idiom the
|
|
22
|
+
* orchestrator's poll (permits, build) and gate-polls (the budget trio, disk) apply to every
|
|
23
|
+
* field observers cannot recompute. Every field is staged regardless of the others (this is
|
|
24
|
+
* not a short-circuiting find), so a batch like the budget trio lands as exactly one write no
|
|
25
|
+
* matter how many of its fields changed, while an unchanged batch leaves the file untouched.
|
|
26
|
+
* Returns whether the file was written. The staging primitive and the best-effort writer are
|
|
27
|
+
* private to this module now that publishInfoFields is the only publish path. */
|
|
28
|
+
export function publishInfoFields(root, info, fields) {
|
|
29
|
+
let changed = false;
|
|
30
|
+
for (const [field, value] of fields) {
|
|
31
|
+
if (assignInfoFieldIfChanged(info, field, value))
|
|
32
|
+
changed = true;
|
|
33
|
+
}
|
|
34
|
+
if (changed)
|
|
35
|
+
writeOrchestratorInfoBestEffort(root, info);
|
|
36
|
+
return changed;
|
|
37
|
+
}
|
|
20
38
|
/** Read the running orchestrator's info file; null when it is missing or unreadable.
|
|
21
39
|
* Never throws — a torn write (e.g. a crash mid-write) must not take down observers
|
|
22
40
|
* that poll this every second (TUI, GUI, status). The nested fields cross the same
|
|
@@ -67,6 +85,36 @@ function parseBuild(v) {
|
|
|
67
85
|
build.restartBlocked = v.restartBlocked;
|
|
68
86
|
return build;
|
|
69
87
|
}
|
|
88
|
+
/** The published permit snapshot, or undefined for any foreign shape. Each `byKey` entry must
|
|
89
|
+
* carry its two numbers; `limit` is kept only when it is a number, so a malformed provider
|
|
90
|
+
* map is dropped to `undefined` rather than reaching a render that reads it as a valid cap. */
|
|
91
|
+
/** One entry of the published provider-holds list, or undefined for any foreign shape: a
|
|
92
|
+
* well-formed entry is an object with a finite `until` and a string `kind`; a `provider` that
|
|
93
|
+
* is present but not a string drops the entry, so a guarded list keeps its well-formed
|
|
94
|
+
* siblings rather than dropping wholesale. */
|
|
95
|
+
function parseProviderHold(v) {
|
|
96
|
+
if (!isJsonObject(v) || !isFiniteNumber(v.until) || typeof v.kind !== "string")
|
|
97
|
+
return undefined;
|
|
98
|
+
if (v.provider !== undefined && typeof v.provider !== "string")
|
|
99
|
+
return undefined;
|
|
100
|
+
return { ...(v.provider !== undefined ? { provider: v.provider } : {}), until: v.until, kind: v.kind };
|
|
101
|
+
}
|
|
102
|
+
function parsePermits(v) {
|
|
103
|
+
if (!isJsonObject(v) || !isFiniteNumber(v.inUse) || !isFiniteNumber(v.limit) || !isFiniteNumber(v.reserve))
|
|
104
|
+
return undefined;
|
|
105
|
+
if (!isJsonObject(v.byKey))
|
|
106
|
+
return undefined;
|
|
107
|
+
const byKey = {};
|
|
108
|
+
for (const [key, value] of Object.entries(v.byKey)) {
|
|
109
|
+
if (!isJsonObject(value) || !isFiniteNumber(value.inUse) || !isFiniteNumber(value.waiting))
|
|
110
|
+
continue;
|
|
111
|
+
const entry = { inUse: value.inUse, waiting: value.waiting };
|
|
112
|
+
if (isFiniteNumber(value.limit))
|
|
113
|
+
entry.limit = value.limit;
|
|
114
|
+
byKey[key] = entry;
|
|
115
|
+
}
|
|
116
|
+
return { inUse: v.inUse, limit: v.limit, reserve: v.reserve, byKey };
|
|
117
|
+
}
|
|
70
118
|
function isLastReclaim(v) {
|
|
71
119
|
if (!isJsonObject(v))
|
|
72
120
|
return false;
|
|
@@ -75,19 +123,23 @@ function isLastReclaim(v) {
|
|
|
75
123
|
isFiniteNumber(v.freedGB) &&
|
|
76
124
|
(mode === "pressure" || mode === "idle" || mode === "manual"));
|
|
77
125
|
}
|
|
126
|
+
/** Parse one nested field through a field parser and either assign its parsed shape or, when
|
|
127
|
+
* the parser rejects the value as foreign, delete the field so a degraded value never reaches
|
|
128
|
+
* a render that reads it as valid. The only fields with a dedicated parser (build,
|
|
129
|
+
* fallbackDemoted, permits) keep their values unchanged when well-formed. */
|
|
130
|
+
function parseOrDelete(rec, key, parse) {
|
|
131
|
+
const parsed = parse(rec[key]);
|
|
132
|
+
if (parsed === undefined)
|
|
133
|
+
delete rec[key];
|
|
134
|
+
else
|
|
135
|
+
rec[key] = parsed;
|
|
136
|
+
}
|
|
78
137
|
/** Degrade each known nested field to a usable shape or drop it, mutating the freshly parsed
|
|
79
138
|
* object in place so a well-formed file keeps its values unchanged. */
|
|
80
139
|
function normalizeInfoFields(rec) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
else
|
|
85
|
-
rec.build = build;
|
|
86
|
-
const demoted = parseDemotion(rec.fallbackDemoted);
|
|
87
|
-
if (demoted === undefined)
|
|
88
|
-
delete rec.fallbackDemoted;
|
|
89
|
-
else
|
|
90
|
-
rec.fallbackDemoted = demoted;
|
|
140
|
+
parseOrDelete(rec, "build", parseBuild);
|
|
141
|
+
parseOrDelete(rec, "fallbackDemoted", parseDemotion);
|
|
142
|
+
parseOrDelete(rec, "permits", parsePermits);
|
|
91
143
|
const demotions = rec.fallbackDemotions;
|
|
92
144
|
if (!isJsonObject(demotions))
|
|
93
145
|
delete rec.fallbackDemotions;
|
|
@@ -103,6 +155,22 @@ function normalizeInfoFields(rec) {
|
|
|
103
155
|
const budget = rec.budget;
|
|
104
156
|
if (!isJsonObject(budget) || !isFiniteNumber(budget.spentUsd))
|
|
105
157
|
delete rec.budget;
|
|
158
|
+
if (rec.chain !== undefined && !isJsonObject(rec.chain))
|
|
159
|
+
delete rec.chain;
|
|
160
|
+
if (rec.stopsAt !== undefined && !isFiniteNumber(rec.stopsAt))
|
|
161
|
+
delete rec.stopsAt;
|
|
162
|
+
const providerHolds = rec.providerHolds;
|
|
163
|
+
if (!Array.isArray(providerHolds))
|
|
164
|
+
delete rec.providerHolds;
|
|
165
|
+
else {
|
|
166
|
+
const kept = [];
|
|
167
|
+
for (const value of providerHolds) {
|
|
168
|
+
const parsed = parseProviderHold(value);
|
|
169
|
+
if (parsed !== undefined)
|
|
170
|
+
kept.push(parsed);
|
|
171
|
+
}
|
|
172
|
+
rec.providerHolds = kept;
|
|
173
|
+
}
|
|
106
174
|
const disk = rec.disk;
|
|
107
175
|
if (!isJsonObject(disk) ||
|
|
108
176
|
!isFiniteNumber(disk.freeGB) ||
|
|
@@ -118,25 +186,25 @@ function normalizeInfoFields(rec) {
|
|
|
118
186
|
/** Publish the running orchestrator's info file atomically (writeJsonAtomic's tmp + rename).
|
|
119
187
|
* The only writer of that path, so a process killed mid-write — the orchestrator's own poll
|
|
120
188
|
* publishes budget/disk/build changes while `tumwater run` in another terminal reads it to
|
|
121
|
-
* refuse a second
|
|
189
|
+
* refuse a second team — leaves either the previous complete file or the new one, never a
|
|
122
190
|
* torn body that readOrchestratorInfo reports as no running orchestrator. */
|
|
123
191
|
export function writeOrchestratorInfo(root, info) {
|
|
124
192
|
writeJsonAtomic(orchestratorStatePath(root), info);
|
|
125
193
|
}
|
|
126
194
|
/** Publish the info file for a caller on a path that must not throw — the orchestrator's
|
|
127
195
|
* catch-less poll loop, whose build/budget/disk updates run every couple of seconds. The info
|
|
128
|
-
* file is an observation surface, not
|
|
129
|
-
* path replaced by a directory) must degrade the published snapshot, not end the
|
|
196
|
+
* file is an observation surface, not team state: an unwritable state dir (ENOSPC, EACCES, the
|
|
197
|
+
* path replaced by a directory) must degrade the published snapshot, not end the team. The
|
|
130
198
|
* failure is reported as a best-effort warning; the caller keeps polling and the field is
|
|
131
199
|
* retried the next time its value changes. Startup's first write stays strict
|
|
132
|
-
* (writeOrchestratorInfo): it is what refuses a second
|
|
200
|
+
* (writeOrchestratorInfo): it is what refuses a second team, so a repo that cannot publish it
|
|
133
201
|
* must fail fast rather than boot. */
|
|
134
|
-
|
|
202
|
+
function writeOrchestratorInfoBestEffort(root, info) {
|
|
135
203
|
try {
|
|
136
204
|
writeOrchestratorInfo(root, info);
|
|
137
205
|
}
|
|
138
206
|
catch (err) {
|
|
139
|
-
warnEventBestEffort(root,
|
|
207
|
+
warnEventBestEffort(root, TUMWATER_SOURCE, `could not publish orchestrator info: ${errorMessage(err)}`);
|
|
140
208
|
}
|
|
141
209
|
}
|
|
142
210
|
/** True when the recorded orchestrator's pid is still alive (see pidAlive). Callers that have
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// vocabulary: plain
|
|
1
2
|
/** Pressure reclaim of build outputs (plans/disk-floor.md, "Reclaiming build outputs", part
|
|
2
3
|
* 2/4): when the worktrees volume drops below `diskReclaimGB`, delete the files git ignores
|
|
3
4
|
* (`git clean -fdX`) in idle harness worktrees, least recently used first, before part 1/4's
|
|
@@ -10,7 +11,7 @@
|
|
|
10
11
|
import path from "node:path";
|
|
11
12
|
import { worktreesDir } from "../paths.js";
|
|
12
13
|
import { git, gitTry } from "../git/git-run.js";
|
|
13
|
-
import { logEventBestEffort } from "../events/events.js";
|
|
14
|
+
import { logEventBestEffort, TUMWATER_SOURCE } from "../events/events.js";
|
|
14
15
|
import { loadLoopState } from "../loop/loop-state.js";
|
|
15
16
|
import { BYTES_PER_GB, sampleFreeBytes } from "../gates/disk-gate.js";
|
|
16
17
|
import { subdirectoryNames } from "../files/files.js";
|
|
@@ -29,7 +30,7 @@ const HOUR_MS = 60 * 60 * 1000;
|
|
|
29
30
|
* the listing instead of being stamped used and delaying a later reclaim. */
|
|
30
31
|
export function reclaimCandidates(root, now = Date.now(), opts = {}) {
|
|
31
32
|
const dir = worktreesDir(root);
|
|
32
|
-
// A missing worktrees dir (a
|
|
33
|
+
// A missing worktrees dir (a team that has not started) yields no candidates.
|
|
33
34
|
const names = subdirectoryNames(dir, (name) => !RESERVED_WORKTREES.has(name));
|
|
34
35
|
// Seed never-seen worktrees as used-now under the cross-process lock, and remember which of
|
|
35
36
|
// this pass's names that covered so they stay out of it. A later pass (or a restart) then sees
|
|
@@ -89,7 +90,7 @@ function isIdleExpired(candidate, idleHours, now) {
|
|
|
89
90
|
/** True when `dir` is a linked worktree and not the primary checkout: `git rev-parse --git-dir`
|
|
90
91
|
* and `--git-common-dir` differ for a linked worktree and are equal for the primary checkout.
|
|
91
92
|
* Reading the primary checkout's `.gitignore` with `clean -X` would delete `.tumwater/`, the
|
|
92
|
-
*
|
|
93
|
+
* team's entire state, so both this and the path check below are load-bearing. */
|
|
93
94
|
export async function isLinkedWorktree(dir) {
|
|
94
95
|
const gitDir = await gitTry(dir, "rev-parse", "--git-dir");
|
|
95
96
|
const commonDir = await gitTry(dir, "rev-parse", "--git-common-dir");
|
|
@@ -167,7 +168,7 @@ export async function reclaimPass(root, mode, opts) {
|
|
|
167
168
|
durationMs: Date.now() - startedAt,
|
|
168
169
|
};
|
|
169
170
|
logEventBestEffort(root, {
|
|
170
|
-
|
|
171
|
+
source: TUMWATER_SOURCE,
|
|
171
172
|
type: "disk_reclaim",
|
|
172
173
|
mode,
|
|
173
174
|
worktrees: cleaned,
|