code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/drive_queue.py
ADDED
|
@@ -0,0 +1,2298 @@
|
|
|
1
|
+
"""Pure decision half of the ``coord drive`` queue (#1754, DQ-2).
|
|
2
|
+
|
|
3
|
+
Phase B of #1750. DQ-1 gave the queue a home (``drive_queue``, one row per
|
|
4
|
+
``(repo, issue)``, dense 0-based ``position``, an ``after_json`` pre-req list
|
|
5
|
+
nothing interpreted yet). This module is what interprets it: given the queue
|
|
6
|
+
rows, a typed projection of the board, and a concurrency ceiling, it returns a
|
|
7
|
+
:class:`TickPlan` — what to reconcile, the ONE entry to launch, what to block,
|
|
8
|
+
what merely deferred and why, and the single queue-level alert.
|
|
9
|
+
|
|
10
|
+
STRUCTURE — the split is copied verbatim from ``coord/drive.py``, for the
|
|
11
|
+
reason that module's docstring gives: *"every bug the bash version shipped was
|
|
12
|
+
in the decision half, which is why that half is where the tests are."* Nothing
|
|
13
|
+
in this file runs a subprocess, opens a socket, touches the DB, or reads the
|
|
14
|
+
clock. ``coord/commands/drive_queue.py`` is the thin I/O shell that fetches,
|
|
15
|
+
calls :func:`plan_tick`, and executes what comes back. #1794 needs wall-clock
|
|
16
|
+
age, so the clock is *passed in* (``plan_tick(..., now=time.time())``) rather
|
|
17
|
+
than read here — the rule is "no ambient state", not "no time".
|
|
18
|
+
|
|
19
|
+
TWO RULES THIS FILE EXISTS TO ENFORCE, both learned the hard way:
|
|
20
|
+
|
|
21
|
+
1. **Capacity comes from BOARD STATE, not from a session count.** ``coord
|
|
22
|
+
drive`` returns ``EXIT_DEADLINE`` (3) when the *observer* gives up; the
|
|
23
|
+
worker, test and review keep running on the fleet (#1660). Such a drive is
|
|
24
|
+
invisible to ``coord drive-sessions`` but is still occupying a machine. So
|
|
25
|
+
an entry occupies capacity when its tmux session is alive **or** it still
|
|
26
|
+
has a live work-like assignment on the board — see
|
|
27
|
+
:func:`_reconcile_running`. Getting this wrong reproduces the 2026-08-01
|
|
28
|
+
incident where a sequential batch became concurrent on the fleet.
|
|
29
|
+
|
|
30
|
+
2. **Typed state, never CLI prose** (#1523 §2). Everything here reads dicts
|
|
31
|
+
that came off ``GET /board`` and ``coord drive-sessions --json``. Both bugs
|
|
32
|
+
in the ad-hoc overnight sequencer were prose-parsing and both failed
|
|
33
|
+
*silently*.
|
|
34
|
+
|
|
35
|
+
DELIBERATELY NOT HERE: auto-demotion. A deferral increments a counter and
|
|
36
|
+
records a reason; it never reorders the queue (see #1750's design note). The
|
|
37
|
+
head of the queue stays the head until an operator moves it.
|
|
38
|
+
|
|
39
|
+
#1757 (DEPLOY GATES) adds a third rule: **merged is not live.** An entry may
|
|
40
|
+
be marked ``--hold-after``, and when the tick transitions THAT entry to
|
|
41
|
+
``done`` the queue stops launching — even with free capacity and a fully
|
|
42
|
+
eligible successor — until a human deploys and releases it. That is not a
|
|
43
|
+
niche case; it is the shape of every change here that crosses a deploy lane
|
|
44
|
+
(``docs/OPERATING_GOTCHAS.md`` opens with the matrix). A queue that models
|
|
45
|
+
merge but not deploy would confidently sequence work into that trap overnight.
|
|
46
|
+
The gate's decision half is :func:`plan_tick`'s hold resolution below; running
|
|
47
|
+
the optional ``resume_when`` probe is the shell's job, and its result comes
|
|
48
|
+
back in as data (:class:`ProbeResult`) so this file stays pure.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
from __future__ import annotations
|
|
52
|
+
|
|
53
|
+
import json
|
|
54
|
+
from dataclasses import dataclass, field
|
|
55
|
+
from typing import Any, Iterable, Mapping, Sequence
|
|
56
|
+
|
|
57
|
+
from coord.drive_state import TERMINAL_STATUSES, WORK_LIKE
|
|
58
|
+
from coord.gate_a import is_gate_a_refusal_reason
|
|
59
|
+
from coord.merge_queue import is_ci_infra_reason, is_ci_pending_reason
|
|
60
|
+
|
|
61
|
+
# ── queue states ─────────────────────────────────────────────────────────────
|
|
62
|
+
#
|
|
63
|
+
# `waiting` and `running` are the live states; `done`/`blocked`/`failed` are
|
|
64
|
+
# terminal and stay in the table until an operator removes them, so
|
|
65
|
+
# `coord drive-queue list` doubles as a short run history (coord/db.py's
|
|
66
|
+
# drive_queue comment states that contract).
|
|
67
|
+
|
|
68
|
+
STATE_WAITING = "waiting"
|
|
69
|
+
STATE_RUNNING = "running"
|
|
70
|
+
STATE_DONE = "done"
|
|
71
|
+
STATE_BLOCKED = "blocked"
|
|
72
|
+
STATE_FAILED = "failed"
|
|
73
|
+
# #1891: a drive that died while ITS OWN issue's merge was refused for
|
|
74
|
+
# nothing stronger than "CI checks have not reported yet" (see
|
|
75
|
+
# `coord.merge_queue.is_ci_pending_reason`) — as opposed to genuinely dead,
|
|
76
|
+
# genuinely refused, or genuinely out of attempts. Deliberately NOT in
|
|
77
|
+
# `TERMINAL_QUEUE_STATES`: unlike `blocked`, this is not a state an operator
|
|
78
|
+
# must release — `plan_tick` re-checks it every tick (see the pre-pass in
|
|
79
|
+
# `plan_tick`) and flips it straight back to `waiting` — without spending an
|
|
80
|
+
# attempt — the moment the board shows the gate has cleared. The whole
|
|
81
|
+
# feature this state exists for is "one GitHub Actions outage costs zero
|
|
82
|
+
# interventions", so a queue read (`coord drive-queue list`/`status`) must
|
|
83
|
+
# render it distinctly from both `waiting` (nothing wrong) and `blocked`
|
|
84
|
+
# (needs a human) — see `_STATE_ORDER` in `coord/commands/drive_queue.py`.
|
|
85
|
+
#
|
|
86
|
+
# #1892 extends the SAME state to a second trigger:
|
|
87
|
+
# `coord.merge_queue.is_ci_infra_reason` — a CI verdict that DID arrive but
|
|
88
|
+
# said nothing about the code (never assigned a runner, or died before
|
|
89
|
+
# checkout). There the "more real time" that un-parks the entry is the
|
|
90
|
+
# in-flight auto-rerun (`MAX_CI_INFRA_RERUNS`) landing, not a verdict that
|
|
91
|
+
# simply hasn't shown up yet — but the queue-level treatment is identical:
|
|
92
|
+
# relaunching a fresh `coord drive` right now would just observe the same
|
|
93
|
+
# rerun-in-progress and wait again, so this parks instead of spending an
|
|
94
|
+
# attempt. See `build_board_view`'s population of `merge_ci_pending` below.
|
|
95
|
+
STATE_PARKED = "parked"
|
|
96
|
+
|
|
97
|
+
TERMINAL_QUEUE_STATES: frozenset[str] = frozenset(
|
|
98
|
+
{STATE_DONE, STATE_BLOCKED, STATE_FAILED}
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
# ── deploy-gate states (#1757) ───────────────────────────────────────────────
|
|
102
|
+
#
|
|
103
|
+
# `hold_state` is the gate's LIFECYCLE, orthogonal to the entry's queue
|
|
104
|
+
# `state`. A gate is `armed` from the moment the operator declares it
|
|
105
|
+
# (`coord drive-queue add --hold-after`, written by `enqueue_drive_queue`),
|
|
106
|
+
# `fired` the tick the entry reaches `done`, and `released` once a human ran
|
|
107
|
+
# `coord drive-queue resume` or the entry's `resume_when` probe exited 0.
|
|
108
|
+
# `''` means the entry carries no gate at all.
|
|
109
|
+
#
|
|
110
|
+
# The queue is held for exactly as long as SOME entry sits at `fired` — the
|
|
111
|
+
# release, not the entry leaving the queue, is what unblocks the successors.
|
|
112
|
+
HOLD_NONE = ""
|
|
113
|
+
HOLD_ARMED = "armed"
|
|
114
|
+
HOLD_FIRED = "fired"
|
|
115
|
+
HOLD_RELEASED = "released"
|
|
116
|
+
|
|
117
|
+
# Wall-clock ceiling for one `resume_when` run. The shell enforces it; it
|
|
118
|
+
# lives here so the CLI's help text, the alert prose and the test all quote one
|
|
119
|
+
# number. A wedged probe must never wedge the tick (a tick that stops running
|
|
120
|
+
# is indistinguishable from a queue with nothing to do — #1616's lesson).
|
|
121
|
+
RESUME_PROBE_TIMEOUT_SECONDS = 5.0
|
|
122
|
+
|
|
123
|
+
# Launch attempts a single entry gets before it is blocked and escalated. An
|
|
124
|
+
# attempt is only consumed when a launched drive DIED without landing the work
|
|
125
|
+
# — a deferral (pre-req not satisfied yet) never touches it, and neither does
|
|
126
|
+
# an unsatisfiable pre-req.
|
|
127
|
+
DEFAULT_MAX_ATTEMPTS = 2
|
|
128
|
+
|
|
129
|
+
# ── the per-repo ceiling (#1972) ─────────────────────────────────────────────
|
|
130
|
+
#
|
|
131
|
+
# `--max-parallel` is one GLOBAL counter, which makes the queue answer the
|
|
132
|
+
# wrong question. The hazard that forced serialisation in the first place is
|
|
133
|
+
# strictly INTRA-repo: a merge stales the Test verdict of every other queued
|
|
134
|
+
# branch in that repo, because #1479's freshness keys on the base of the
|
|
135
|
+
# branch's own repo. A vimcode merge cannot stale a quadraui branch. So repo
|
|
136
|
+
# is precisely the boundary along which parallelism is safe — within a repo is
|
|
137
|
+
# the risky case, across repos is nearly free.
|
|
138
|
+
#
|
|
139
|
+
# Counting one global slot conflates the two. With `--max-parallel 3` and a
|
|
140
|
+
# queue of 39 claude-coordinator entries followed by one quadraui entry, the
|
|
141
|
+
# tick launches claude-coordinator #2 and #3 — the two launches most likely to
|
|
142
|
+
# stale each other — and never reaches the quadraui entry that could have run
|
|
143
|
+
# alongside them for free. Getting the wanted behaviour meant hand-chaining
|
|
144
|
+
# `--after` across 38 entries: tedious, fragile, and wrong the moment the queue
|
|
145
|
+
# is reordered.
|
|
146
|
+
#
|
|
147
|
+
# So occupancy is counted per repo as well as globally, and an entry whose repo
|
|
148
|
+
# is already at this ceiling DEFERS (position unchanged, no attempt consumed,
|
|
149
|
+
# no escalation — a "not yet", exactly like an unsatisfied `after`). The walk
|
|
150
|
+
# then lands naturally on the first entry from a repo that still has headroom.
|
|
151
|
+
#
|
|
152
|
+
# The default is 1 — today's effective behaviour for the single-repo queues
|
|
153
|
+
# that are the common case, since `--max-parallel` itself defaults to 1. It is
|
|
154
|
+
# configurable rather than hardcoded because #1715 (batch revalidation) closed,
|
|
155
|
+
# which makes intra-repo parallelism materially less punishing than it was; 0
|
|
156
|
+
# disables the per-repo ceiling entirely and restores the pre-#1972 behaviour.
|
|
157
|
+
#
|
|
158
|
+
# CAVEAT worth stating where the constant lives: per-repo occupancy inherits
|
|
159
|
+
# rule 1 above — it is counted from BOARD state, not live sessions (#1660). A
|
|
160
|
+
# drive whose observer died still holds its repo's slot until something
|
|
161
|
+
# reconciles it. That is strictly better than before (a wedged drive now
|
|
162
|
+
# blocks one repo instead of the whole queue) but it is also quieter, which is
|
|
163
|
+
# why `render_plan` prints the per-repo breakdown and says where it came from.
|
|
164
|
+
DEFAULT_MAX_PARALLEL_PER_REPO = 1
|
|
165
|
+
|
|
166
|
+
# ── the startup grace window (#1794) ─────────────────────────────────────────
|
|
167
|
+
#
|
|
168
|
+
# A drive is NOT established the instant `coord drive --tmux` exits 0. #1606's
|
|
169
|
+
# verification proves a tmux session exists and its run log has been written
|
|
170
|
+
# to; it does NOT prove the drive has registered anywhere the tick can see it.
|
|
171
|
+
# Between the launch and the first dispatch there is a window in which the
|
|
172
|
+
# entry has:
|
|
173
|
+
#
|
|
174
|
+
# * no live session in `board.live_sessions` — that snapshot is a
|
|
175
|
+
# `tmux list-sessions` reading, and `list_drive_sessions()` returns `[]`
|
|
176
|
+
# for "tmux unavailable" / "no server running" / "the call timed out"
|
|
177
|
+
# exactly as it does for "no sessions", so one bad reading makes EVERY
|
|
178
|
+
# running entry look dead at once;
|
|
179
|
+
# * no `active_work` on the board — the drive has not dispatched yet.
|
|
180
|
+
#
|
|
181
|
+
# Before #1794 that fell straight through all three non-death branches of
|
|
182
|
+
# `_reconcile_running` into `retry`. On 2026-08-03 a tick 40s after a launch
|
|
183
|
+
# declared a healthy drive dead, spent an attempt, and launched a SECOND
|
|
184
|
+
# `coord drive` for the same issue. Left alone that walks the entry to
|
|
185
|
+
# `attempts=2/2` and `blocked`, i.e. an unattended queue parks healthy work and
|
|
186
|
+
# reports it as failed. The two ticks were 40s apart because DRIVE_QUEUE.md §2's
|
|
187
|
+
# install sequence is `systemctl --user enable --now …timer` immediately
|
|
188
|
+
# followed by a verification `systemctl --user start …service` — i.e. the
|
|
189
|
+
# documented install reliably produces the back-to-back ticks that trigger it.
|
|
190
|
+
#
|
|
191
|
+
# So an entry launched within this window is `starting`, not dead: it OCCUPIES
|
|
192
|
+
# capacity and is never a retry candidate. The measured startup on a loaded
|
|
193
|
+
# dellserver was ~2 minutes (19:13:09 launch → 19:15:22 `drive loop started`),
|
|
194
|
+
# and this is 5 — deliberately >2x that, and still well under the timer's
|
|
195
|
+
# 15-minute cadence so a genuinely dead drive is only ever delayed by ONE
|
|
196
|
+
# interval before the retry path sees it.
|
|
197
|
+
#
|
|
198
|
+
# The window is also applied to the LAUNCH decision (see `_startup_cooldown`),
|
|
199
|
+
# so no code path in the tick — not a retry, not a hand-edited row — can start
|
|
200
|
+
# a second `coord drive` for an issue whose last launch is this recent.
|
|
201
|
+
# `coord drive`'s per-issue flock stays the last line of defence; the queue no
|
|
202
|
+
# longer relies on it.
|
|
203
|
+
DRIVE_STARTUP_GRACE_SECONDS = 300.0
|
|
204
|
+
|
|
205
|
+
# ── the queue-level alert's synthetic escalation key ─────────────────────────
|
|
206
|
+
#
|
|
207
|
+
# #1754 asks for "one queue-level record per tick, written through the DQ-1
|
|
208
|
+
# seam OR `record_drive_escalation` with a synthetic issue key — pick one and
|
|
209
|
+
# state it in the code comment, don't leave both live".
|
|
210
|
+
#
|
|
211
|
+
# CHOSEN: `record_drive_escalation` under the synthetic key below. Reasons:
|
|
212
|
+
# the alert is exactly the shape `drive_escalations` already stores (stage +
|
|
213
|
+
# reason + gate readings + a proposed command), that table's UNIQUE(repo_name,
|
|
214
|
+
# issue_number) + ON CONFLICT DO UPDATE gives "exactly one record, replaced
|
|
215
|
+
# each tick" for free, and `coord escalate list` / the TUI's escalation
|
|
216
|
+
# plumbing pick it up with no new wire type. The alternative — a synthetic
|
|
217
|
+
# `drive_queue` row — would have to be filtered out of `list`, `move`,
|
|
218
|
+
# `plan_tick`, and the dense-position renumbering, i.e. a special case in
|
|
219
|
+
# every function in this file. The DQ-1 seam stays strictly "real entries".
|
|
220
|
+
#
|
|
221
|
+
# The repo name is deliberately not a valid coordinator.yml repo, so this row
|
|
222
|
+
# can never collide with a real issue's escalation or match a Pipeline row.
|
|
223
|
+
QUEUE_ALERT_REPO = "(drive-queue)"
|
|
224
|
+
QUEUE_ALERT_ISSUE = 0
|
|
225
|
+
QUEUE_ALERT_STAGE = "drive-queue"
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class QueueError(ValueError):
|
|
229
|
+
"""A queue mutation was refused before it was written.
|
|
230
|
+
|
|
231
|
+
Carries a message naming the offending issue and the violated constraint,
|
|
232
|
+
the same posture ``coord milestone write-order`` takes for ``## Work
|
|
233
|
+
order`` (``coord.milestone_order.WorkOrderError``): validate, then write —
|
|
234
|
+
never write and then discover.
|
|
235
|
+
"""
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
# ── keys ─────────────────────────────────────────────────────────────────────
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def entry_key(repo: str, issue: int) -> str:
|
|
242
|
+
"""The fully-qualified queue/pre-req key for an issue: ``"repo#N"``.
|
|
243
|
+
|
|
244
|
+
This is the on-disk form DQ-1 stores in ``after_json`` — one column
|
|
245
|
+
carries a cross-repo queue with no second column.
|
|
246
|
+
"""
|
|
247
|
+
return f"{repo}#{int(issue)}"
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def parse_key(key: str) -> tuple[str, int] | None:
|
|
251
|
+
"""Inverse of :func:`entry_key`; ``None`` when *key* isn't ``repo#N``.
|
|
252
|
+
|
|
253
|
+
Splits on the LAST ``#`` so a repo name containing one still parses, and
|
|
254
|
+
requires the tail to be a bare number.
|
|
255
|
+
"""
|
|
256
|
+
repo, sep, num = str(key).rpartition("#")
|
|
257
|
+
if not sep or not repo or not num.isdigit():
|
|
258
|
+
return None
|
|
259
|
+
return repo, int(num)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def parse_after_spec(raw: str | Iterable[str], default_repo: str) -> list[str]:
|
|
263
|
+
"""Normalise a ``--after`` spec into fully-qualified ``repo#N`` keys.
|
|
264
|
+
|
|
265
|
+
Accepts ``N`` or ``REPO#N``, comma-separated, and (for repeatable Click
|
|
266
|
+
options) an iterable of either. Bare numbers resolve against
|
|
267
|
+
*default_repo* — the queue is usually single-repo, and typing the repo
|
|
268
|
+
name twice is the kind of friction that gets a flag skipped.
|
|
269
|
+
|
|
270
|
+
Raises :class:`QueueError` on anything that isn't one of those two forms,
|
|
271
|
+
rather than silently dropping it (a dropped pre-req launches work early,
|
|
272
|
+
which is the whole failure this feature exists to prevent).
|
|
273
|
+
"""
|
|
274
|
+
chunks: list[str] = []
|
|
275
|
+
items: Iterable[str] = [raw] if isinstance(raw, str) else raw
|
|
276
|
+
for item in items:
|
|
277
|
+
chunks.extend(str(item).split(","))
|
|
278
|
+
|
|
279
|
+
keys: list[str] = []
|
|
280
|
+
for chunk in chunks:
|
|
281
|
+
text = chunk.strip()
|
|
282
|
+
if not text:
|
|
283
|
+
continue
|
|
284
|
+
if text.isdigit():
|
|
285
|
+
keys.append(entry_key(default_repo, int(text)))
|
|
286
|
+
continue
|
|
287
|
+
parsed = parse_key(text.lstrip("#"))
|
|
288
|
+
if parsed is None:
|
|
289
|
+
raise QueueError(
|
|
290
|
+
f"malformed --after entry {text!r} (expected 'N' or 'REPO#N')"
|
|
291
|
+
)
|
|
292
|
+
keys.append(entry_key(*parsed))
|
|
293
|
+
# De-duplicate, preserving declaration order.
|
|
294
|
+
seen: set[str] = set()
|
|
295
|
+
out: list[str] = []
|
|
296
|
+
for key in keys:
|
|
297
|
+
if key not in seen:
|
|
298
|
+
seen.add(key)
|
|
299
|
+
out.append(key)
|
|
300
|
+
return out
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# ── the queue row ────────────────────────────────────────────────────────────
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@dataclass(frozen=True)
|
|
307
|
+
class QueueEntry:
|
|
308
|
+
"""One ``drive_queue`` row, typed.
|
|
309
|
+
|
|
310
|
+
Built from the dicts DQ-1's ``list_drive_queue()`` returns — identical
|
|
311
|
+
whether they came off the local DB or the daemon's ``/drive-queue``, which
|
|
312
|
+
is what lets the whole tick run unchanged on a thin client.
|
|
313
|
+
"""
|
|
314
|
+
|
|
315
|
+
repo: str
|
|
316
|
+
issue: int
|
|
317
|
+
position: int = 0
|
|
318
|
+
machine: str = ""
|
|
319
|
+
after: tuple[str, ...] = ()
|
|
320
|
+
state: str = STATE_WAITING
|
|
321
|
+
attempts: int = 0
|
|
322
|
+
deferrals: int = 0
|
|
323
|
+
last_reason: str = ""
|
|
324
|
+
session_name: str = ""
|
|
325
|
+
launched_at: float | None = None
|
|
326
|
+
# #1870: the short hostname of the machine whose tick launched THIS
|
|
327
|
+
# session — stamped alongside `session_name`/`launched_at` when the
|
|
328
|
+
# launch succeeds. '' for a row predating this column or hand-flipped to
|
|
329
|
+
# `running`, which degrades to the pre-#1870 behaviour exactly (see
|
|
330
|
+
# `_reconcile_running`). Liveness (`list_drive_sessions`) is always a
|
|
331
|
+
# LOCAL tmux read; this is what lets a tick tell "no session because it's
|
|
332
|
+
# dead" apart from "no session because it's not MY session to see".
|
|
333
|
+
launch_host: str = ""
|
|
334
|
+
# #1757 deploy gate. `hold_after`/`hold_reason`/`resume_when` are
|
|
335
|
+
# operator-declared (written by `enqueue`); `hold_state`/`hold_probes` are
|
|
336
|
+
# the tick's run state.
|
|
337
|
+
hold_after: bool = False
|
|
338
|
+
hold_reason: str = ""
|
|
339
|
+
resume_when: str = ""
|
|
340
|
+
hold_state: str = HOLD_NONE
|
|
341
|
+
hold_probes: int = 0
|
|
342
|
+
|
|
343
|
+
@property
|
|
344
|
+
def key(self) -> str:
|
|
345
|
+
return entry_key(self.repo, self.issue)
|
|
346
|
+
|
|
347
|
+
@property
|
|
348
|
+
def gate_reason(self) -> str:
|
|
349
|
+
"""What to tell the operator when this entry's gate fires.
|
|
350
|
+
|
|
351
|
+
Never empty: an operator who used ``--hold-after`` without a reason
|
|
352
|
+
still gets a sentence naming the entry, because an alert that says
|
|
353
|
+
only "HELD" is one the operator has to go and reconstruct.
|
|
354
|
+
"""
|
|
355
|
+
return self.hold_reason or f"deploy gate declared on {self.key}"
|
|
356
|
+
|
|
357
|
+
@classmethod
|
|
358
|
+
def from_row(cls, row: Mapping[str, Any]) -> "QueueEntry":
|
|
359
|
+
"""Type one raw queue row.
|
|
360
|
+
|
|
361
|
+
``after_json`` arrives as a real list over the wire (DQ-1 put it in
|
|
362
|
+
``coord.dao._JSON_COLUMNS``) but as a JSON *string* when a caller
|
|
363
|
+
reads the table directly, so both are accepted; anything unparseable
|
|
364
|
+
degrades to ``()`` rather than blowing up the whole tick.
|
|
365
|
+
"""
|
|
366
|
+
raw_after: Any = row.get("after_json")
|
|
367
|
+
if isinstance(raw_after, str):
|
|
368
|
+
try:
|
|
369
|
+
raw_after = json.loads(raw_after)
|
|
370
|
+
except (TypeError, ValueError):
|
|
371
|
+
raw_after = []
|
|
372
|
+
if not isinstance(raw_after, list):
|
|
373
|
+
raw_after = []
|
|
374
|
+
launched_at = row.get("launched_at")
|
|
375
|
+
return cls(
|
|
376
|
+
repo=str(row.get("repo_name") or ""),
|
|
377
|
+
issue=int(row.get("issue_number") or 0),
|
|
378
|
+
position=int(row.get("position") or 0),
|
|
379
|
+
machine=str(row.get("machine") or ""),
|
|
380
|
+
after=tuple(str(a) for a in raw_after),
|
|
381
|
+
state=str(row.get("state") or STATE_WAITING),
|
|
382
|
+
attempts=int(row.get("attempts") or 0),
|
|
383
|
+
deferrals=int(row.get("deferrals") or 0),
|
|
384
|
+
last_reason=str(row.get("last_reason") or ""),
|
|
385
|
+
session_name=str(row.get("session_name") or ""),
|
|
386
|
+
launched_at=None if launched_at is None else float(launched_at),
|
|
387
|
+
launch_host=str(row.get("launch_host") or ""),
|
|
388
|
+
# SQLite hands `hold_after` back as 0/1; a JSON client may send a
|
|
389
|
+
# real bool. `bool(...)` accepts both and, for a row written
|
|
390
|
+
# before #1757's migration ran, an absent key reads as False —
|
|
391
|
+
# i.e. no gate, which is the pre-#1757 behaviour exactly.
|
|
392
|
+
hold_after=bool(row.get("hold_after") or 0),
|
|
393
|
+
hold_reason=str(row.get("hold_reason") or ""),
|
|
394
|
+
resume_when=str(row.get("resume_when") or ""),
|
|
395
|
+
hold_state=str(row.get("hold_state") or HOLD_NONE),
|
|
396
|
+
hold_probes=int(row.get("hold_probes") or 0),
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def entries_from_rows(rows: Iterable[Mapping[str, Any]]) -> list[QueueEntry]:
|
|
401
|
+
"""Type a whole queue read, in ``position`` order."""
|
|
402
|
+
return sorted(
|
|
403
|
+
(QueueEntry.from_row(r) for r in rows), key=lambda e: (e.position, e.key)
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
# ── the board projection ─────────────────────────────────────────────────────
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
@dataclass(frozen=True)
|
|
411
|
+
class IssueFacts:
|
|
412
|
+
"""Everything the tick needs to know about one issue, and nothing else.
|
|
413
|
+
|
|
414
|
+
All four fields come off ``GET /board`` — no ``gh`` call, no CLI prose.
|
|
415
|
+
``known=False`` means the board has never heard of this issue at all
|
|
416
|
+
(unsynced, or a typo'd number), which is deliberately NOT the same as
|
|
417
|
+
"open": an unknown pre-req is unsatisfiable, an open one merely defers.
|
|
418
|
+
"""
|
|
419
|
+
|
|
420
|
+
known: bool = False
|
|
421
|
+
issue_state: str = "" # "open" / "closed" / "" when the board has no row
|
|
422
|
+
merged: bool = False # a work-like assignment with status == 'merged'
|
|
423
|
+
active_work: bool = False # a NON-terminal work-like assignment
|
|
424
|
+
# #1891: this issue's CURRENT merge-queue entry is refused for nothing
|
|
425
|
+
# stronger than "CI checks have not reported yet" — see
|
|
426
|
+
# `build_board_view`'s population of this field for exactly which board
|
|
427
|
+
# sections it reads (and why it reads BOTH of them) and
|
|
428
|
+
# `_reconcile_running`'s `parked` outcome for the one place it changes a
|
|
429
|
+
# decision.
|
|
430
|
+
merge_ci_pending: bool = False
|
|
431
|
+
# The actual board/queue reason text `merge_ci_pending` was derived from
|
|
432
|
+
# (e.g. ``"CI running: build, lint"``) — carried alongside the bool
|
|
433
|
+
# purely for diagnostics, so a `parked` reconcile's `reason` can quote
|
|
434
|
+
# the SAME text an operator would see on `IssueState.merge_reason`
|
|
435
|
+
# instead of a generic synthesised sentence.
|
|
436
|
+
merge_ci_pending_reason: str = ""
|
|
437
|
+
|
|
438
|
+
@property
|
|
439
|
+
def open(self) -> bool:
|
|
440
|
+
return self.issue_state == "open"
|
|
441
|
+
|
|
442
|
+
@property
|
|
443
|
+
def closed(self) -> bool:
|
|
444
|
+
return self.issue_state == "closed"
|
|
445
|
+
|
|
446
|
+
@property
|
|
447
|
+
def landed(self) -> bool:
|
|
448
|
+
"""The work is done, by either witness.
|
|
449
|
+
|
|
450
|
+
Both are checked because #611 leaves merged work with ``branch=None``
|
|
451
|
+
rows the merge projection can miss, and quadraui-style repos can merge
|
|
452
|
+
a PR into ``develop`` while the linked issue stays open — so neither
|
|
453
|
+
signal alone is reliable.
|
|
454
|
+
"""
|
|
455
|
+
return self.merged or self.closed
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
@dataclass(frozen=True)
|
|
459
|
+
class BoardView:
|
|
460
|
+
"""The whole board reduced to per-issue facts plus live drive sessions."""
|
|
461
|
+
|
|
462
|
+
issues: Mapping[str, IssueFacts] = field(default_factory=dict)
|
|
463
|
+
live_sessions: frozenset[str] = frozenset()
|
|
464
|
+
|
|
465
|
+
def facts(self, key: str) -> IssueFacts:
|
|
466
|
+
return self.issues.get(key, IssueFacts())
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def build_board_view(
|
|
470
|
+
payload: Mapping[str, Any],
|
|
471
|
+
live_sessions: Iterable[Mapping[str, Any] | str] = (),
|
|
472
|
+
) -> BoardView:
|
|
473
|
+
"""Reduce a ``/board`` payload + ``drive-sessions --json`` to a :class:`BoardView`.
|
|
474
|
+
|
|
475
|
+
Pure: *payload* is whatever ``coord.drive_state.BoardFetcher.fetch()``
|
|
476
|
+
returned and *live_sessions* is whatever ``coord.drive.list_drive_sessions()``
|
|
477
|
+
returned (dicts with ``repo``/``issue``), or a plain iterable of
|
|
478
|
+
``"repo#N"`` keys for tests.
|
|
479
|
+
"""
|
|
480
|
+
facts: dict[str, dict[str, Any]] = {}
|
|
481
|
+
|
|
482
|
+
def slot(key: str) -> dict[str, Any]:
|
|
483
|
+
return facts.setdefault(
|
|
484
|
+
key,
|
|
485
|
+
{"known": True, "issue_state": "", "merged": False, "active_work": False},
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
for row in payload.get("assignments") or []:
|
|
489
|
+
if (row.get("type") or "") not in WORK_LIKE:
|
|
490
|
+
continue
|
|
491
|
+
repo = row.get("repo_name") or ""
|
|
492
|
+
number = row.get("issue_number")
|
|
493
|
+
if not repo or number is None:
|
|
494
|
+
continue
|
|
495
|
+
entry = slot(entry_key(repo, int(number)))
|
|
496
|
+
status = row.get("status") or ""
|
|
497
|
+
if status == "merged":
|
|
498
|
+
entry["merged"] = True
|
|
499
|
+
if status not in TERMINAL_STATUSES:
|
|
500
|
+
entry["active_work"] = True
|
|
501
|
+
|
|
502
|
+
for row in payload.get("issues") or []:
|
|
503
|
+
repo = row.get("repo_name") or ""
|
|
504
|
+
number = row.get("number")
|
|
505
|
+
if not repo or number is None:
|
|
506
|
+
continue
|
|
507
|
+
entry = slot(entry_key(repo, int(number)))
|
|
508
|
+
entry["issue_state"] = str(row.get("state") or "").lower()
|
|
509
|
+
|
|
510
|
+
# #1891: `merge_ci_pending` — mirrors `drive_state._merge_entry`'s OWN
|
|
511
|
+
# reason resolution exactly (live `merge_plan` reason, falling back to
|
|
512
|
+
# the raw `merge_queue` row's persisted `error` when the plan's
|
|
513
|
+
# re-evaluation comes back empty) rather than importing that per-issue
|
|
514
|
+
# function and calling it once per queue entry: this is a single O(N)
|
|
515
|
+
# pass over the SAME two board sections `_merge_entry` scans, building a
|
|
516
|
+
# dict up front the way every other fact in this function already does.
|
|
517
|
+
# See `coord.merge_queue.CI_PENDING_PREFIX`'s docstring for why the raw
|
|
518
|
+
# row is a required second read, not a belt-and-braces extra one.
|
|
519
|
+
plan_reasons: dict[str, str] = {}
|
|
520
|
+
for row in payload.get("merge_plan") or []:
|
|
521
|
+
repo = row.get("repo_name") or ""
|
|
522
|
+
number = row.get("issue_number")
|
|
523
|
+
if not repo or number is None:
|
|
524
|
+
continue
|
|
525
|
+
plan_reasons[entry_key(repo, int(number))] = str(row.get("reason") or "")
|
|
526
|
+
|
|
527
|
+
for row in payload.get("merge_queue") or []:
|
|
528
|
+
repo = row.get("repo_name") or ""
|
|
529
|
+
number = row.get("issue_number")
|
|
530
|
+
if not repo or number is None:
|
|
531
|
+
continue
|
|
532
|
+
key = entry_key(repo, int(number))
|
|
533
|
+
plan_reason = plan_reasons.get(key) or ""
|
|
534
|
+
raw_reason = str(row.get("error") or "")
|
|
535
|
+
reason = plan_reason or raw_reason
|
|
536
|
+
# #1892: same recovery `drive_state._merge_entry` applies — the
|
|
537
|
+
# plan's own reason is `_entry_gate_status`'s fresh re-derivation at
|
|
538
|
+
# board-build time, which never computes the CI_INFRA_PREFIX
|
|
539
|
+
# classification (it needs an extra `gh api .../jobs` call the
|
|
540
|
+
# board *read* path must never make — see `coord.gate_snapshot`'s
|
|
541
|
+
# Invariant 1). Only a LIVE `coord merge` attempt computes it and
|
|
542
|
+
# persists it onto the raw row. Prefer the raw reading whenever it
|
|
543
|
+
# carries the classification and the plan's fresher one doesn't —
|
|
544
|
+
# otherwise a verdictless failure would never park here at all.
|
|
545
|
+
if is_ci_infra_reason(raw_reason) and not is_ci_infra_reason(plan_reason):
|
|
546
|
+
reason = raw_reason
|
|
547
|
+
if is_ci_pending_reason(reason) or is_ci_infra_reason(reason):
|
|
548
|
+
got = slot(key)
|
|
549
|
+
got["merge_ci_pending"] = True
|
|
550
|
+
got["merge_ci_pending_reason"] = reason
|
|
551
|
+
|
|
552
|
+
sessions: set[str] = set()
|
|
553
|
+
for item in live_sessions:
|
|
554
|
+
if isinstance(item, str):
|
|
555
|
+
sessions.add(item)
|
|
556
|
+
continue
|
|
557
|
+
repo = item.get("repo") or ""
|
|
558
|
+
number = item.get("issue")
|
|
559
|
+
if repo and number is not None:
|
|
560
|
+
sessions.add(entry_key(repo, int(number)))
|
|
561
|
+
|
|
562
|
+
return BoardView(
|
|
563
|
+
issues={key: IssueFacts(**value) for key, value in facts.items()},
|
|
564
|
+
live_sessions=frozenset(sessions),
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
# ── the plan ─────────────────────────────────────────────────────────────────
|
|
569
|
+
#
|
|
570
|
+
# Every item carries an explicit `updates` mapping of DQ-1-whitelisted columns
|
|
571
|
+
# (see `_DRIVE_QUEUE_UPDATABLE` in coord/state.py). The shell's apply loop is
|
|
572
|
+
# therefore a single uniform `update_drive_queue_entry(repo, issue, **updates)`
|
|
573
|
+
# per item — it never re-derives a decision, and a plan with no updates is
|
|
574
|
+
# provably a no-op, which is what makes `--dry-run` trustworthy.
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
@dataclass(frozen=True)
|
|
578
|
+
class Reconcile:
|
|
579
|
+
"""The resolved outcome for one ``running`` entry.
|
|
580
|
+
|
|
581
|
+
``outcome`` is one of:
|
|
582
|
+
|
|
583
|
+
* ``alive`` — a live ``coord-drive-*`` tmux session. Occupies.
|
|
584
|
+
* ``starting`` — launched inside :data:`DRIVE_STARTUP_GRACE_SECONDS` and
|
|
585
|
+
not yet visible anywhere else (#1794). Occupies; never a death.
|
|
586
|
+
* ``held`` — session gone but work still ACTIVE on the board (the
|
|
587
|
+
#1660 observer-deadline case). Occupies; never a death.
|
|
588
|
+
* ``unknown`` — this entry's ``launch_host`` names a DIFFERENT machine
|
|
589
|
+
than the one running this tick (#1870). Liveness is always a LOCAL
|
|
590
|
+
tmux read, so a foreign host's session is invisible here — that is not
|
|
591
|
+
evidence of anything. Occupies; never a death, never a retry.
|
|
592
|
+
* ``done`` — merged, or the issue closed.
|
|
593
|
+
* ``refused`` — #1844: the drive's own exit was a PERMANENT pre-dispatch
|
|
594
|
+
guard refusal (``coord.drive.EXIT_DISPATCH_REFUSED``). Goes straight to
|
|
595
|
+
``blocked``; costs NO attempt — pairs with a :class:`Blocked`.
|
|
596
|
+
* ``dead_end`` — #2019: the drive's own exit was ``coord.drive.
|
|
597
|
+
EXIT_DEAD_END`` — its dead-end predicate found the row terminal and
|
|
598
|
+
unactionable (nothing active, every stage terminal, no gate transition
|
|
599
|
+
available). Same disposition as ``refused`` (straight to ``blocked``,
|
|
600
|
+
NO attempt spent, pairs with a :class:`Blocked`); a distinct outcome
|
|
601
|
+
only so the journal line names the right cause.
|
|
602
|
+
* ``parked`` — #1891: no session, no active work, nothing landed — same
|
|
603
|
+
evidence as ``retry`` — but the board's OWN current read of this
|
|
604
|
+
entry's merge gate names nothing stronger than "CI checks have not
|
|
605
|
+
reported yet" (``IssueFacts.merge_ci_pending``, sourced independently
|
|
606
|
+
of whatever killed the drive). Goes straight to :data:`STATE_PARKED`;
|
|
607
|
+
costs NO attempt — a missing verdict is not a failed one, and no
|
|
608
|
+
number of relaunches changes it, only more real time. Re-checked
|
|
609
|
+
every tick by the pre-pass in :func:`plan_tick`, which flips it back
|
|
610
|
+
to ``waiting`` — no human, no escalation — the moment the board shows
|
|
611
|
+
the gate has cleared.
|
|
612
|
+
* ``retry`` — genuinely dead: no session, no active work, and past the
|
|
613
|
+
startup grace window. Costs one attempt.
|
|
614
|
+
* ``exhausted`` — as ``retry``, but out of attempts; pairs with a
|
|
615
|
+
:class:`Blocked`.
|
|
616
|
+
"""
|
|
617
|
+
|
|
618
|
+
key: str
|
|
619
|
+
outcome: str # alive | starting | held | unknown | done | refused | parked | retry | exhausted
|
|
620
|
+
reason: str
|
|
621
|
+
occupies: bool = False
|
|
622
|
+
updates: Mapping[str, Any] = field(default_factory=dict)
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
@dataclass(frozen=True)
|
|
626
|
+
class Blocked:
|
|
627
|
+
"""An entry to mark ``blocked`` and escalate."""
|
|
628
|
+
|
|
629
|
+
key: str
|
|
630
|
+
reason: str
|
|
631
|
+
updates: Mapping[str, Any] = field(default_factory=dict)
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
@dataclass(frozen=True)
|
|
635
|
+
class Deferral:
|
|
636
|
+
"""An entry whose pre-reqs aren't satisfied YET. Position unchanged.
|
|
637
|
+
|
|
638
|
+
``counted=False`` marks a REPORT-ONLY deferral: an entry the walk reached
|
|
639
|
+
only after a launch had already been chosen, so it never actually
|
|
640
|
+
competed for a free slot. Its ``updates`` are empty, so it mutates
|
|
641
|
+
nothing — it exists purely so ``--dry-run`` can answer "and why isn't the
|
|
642
|
+
rest of the queue going?" in the same breath. Counting it would ruin the
|
|
643
|
+
signal ``deferrals`` carries: *how many times this entry was passed over
|
|
644
|
+
while a slot was actually available*.
|
|
645
|
+
|
|
646
|
+
``repo_limited=True`` marks the #1972 deferral — the entry was otherwise
|
|
647
|
+
fully eligible and lost its turn only because its REPO was already at
|
|
648
|
+
``max_parallel_per_repo``. It is flagged rather than string-matched
|
|
649
|
+
because the tick has to tell that case apart from a genuine stall: a queue
|
|
650
|
+
whose remaining entries are all waiting on their own repo's in-flight work
|
|
651
|
+
is the queue working exactly as designed, so it raises no queue-level
|
|
652
|
+
alert — the same posture the global at-capacity return takes.
|
|
653
|
+
|
|
654
|
+
``cordoned=True`` is #2101's twin of that flag: the entry is pinned to a
|
|
655
|
+
machine that is draining for a release right now. Same posture and same
|
|
656
|
+
reason — a drain is the fleet working, lasts minutes, and ends by itself,
|
|
657
|
+
so escalating it every tick is how an alert channel gets muted. A
|
|
658
|
+
separate flag rather than reusing ``repo_limited`` because the two produce
|
|
659
|
+
different prose and different remedies, and a render that blames the
|
|
660
|
+
repo limit for a cordon is a render that sends the operator to the wrong
|
|
661
|
+
knob.
|
|
662
|
+
"""
|
|
663
|
+
|
|
664
|
+
key: str
|
|
665
|
+
reason: str
|
|
666
|
+
updates: Mapping[str, Any] = field(default_factory=dict)
|
|
667
|
+
counted: bool = True
|
|
668
|
+
repo_limited: bool = False
|
|
669
|
+
cordoned: bool = False
|
|
670
|
+
|
|
671
|
+
@property
|
|
672
|
+
def benign(self) -> bool:
|
|
673
|
+
"""Is this a "the fleet is working" deferral rather than a stall?
|
|
674
|
+
|
|
675
|
+
The single predicate both the queue-level alert and `render_plan`
|
|
676
|
+
consult, so a future third benign cause cannot be added to one and
|
|
677
|
+
forgotten in the other.
|
|
678
|
+
"""
|
|
679
|
+
return self.repo_limited or self.cordoned
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
@dataclass(frozen=True)
|
|
683
|
+
class ProbeResult:
|
|
684
|
+
"""The outcome of ONE ``resume_when`` run, handed back in by the shell.
|
|
685
|
+
|
|
686
|
+
Exit 0 (``ok=True``) releases the gate; anything else — non-zero, a
|
|
687
|
+
timeout, or a command that could not be spawned at all — keeps it held.
|
|
688
|
+
Fail-CLOSED is the only safe default here: a gate that releases because
|
|
689
|
+
its probe blew up is a gate that did not exist.
|
|
690
|
+
"""
|
|
691
|
+
|
|
692
|
+
key: str
|
|
693
|
+
ok: bool
|
|
694
|
+
detail: str = ""
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
@dataclass(frozen=True)
|
|
698
|
+
class Hold:
|
|
699
|
+
"""One entry's deploy gate, as resolved by this tick (#1757).
|
|
700
|
+
|
|
701
|
+
``outcome``:
|
|
702
|
+
|
|
703
|
+
* ``fired`` — the entry reached ``done`` on THIS tick and the gate has
|
|
704
|
+
just closed the queue. ``updates`` arms the run state.
|
|
705
|
+
* ``held`` — the gate was already ``fired`` and is still closed
|
|
706
|
+
(either no probe is declared, or the probe ran and failed).
|
|
707
|
+
* ``released`` — the probe exited 0; the walk continues in this same tick.
|
|
708
|
+
|
|
709
|
+
``blocking`` is the single thing the tick acts on, so a future outcome
|
|
710
|
+
can be added without every caller re-deriving the rule.
|
|
711
|
+
"""
|
|
712
|
+
|
|
713
|
+
key: str
|
|
714
|
+
outcome: str
|
|
715
|
+
reason: str
|
|
716
|
+
resume_when: str = ""
|
|
717
|
+
probes: int = 0
|
|
718
|
+
probe_detail: str = ""
|
|
719
|
+
updates: Mapping[str, Any] = field(default_factory=dict)
|
|
720
|
+
|
|
721
|
+
@property
|
|
722
|
+
def blocking(self) -> bool:
|
|
723
|
+
return self.outcome in ("fired", "held")
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
@dataclass(frozen=True)
|
|
727
|
+
class QueueAlert:
|
|
728
|
+
"""The one queue-level record a tick may raise (see QUEUE_ALERT_REPO).
|
|
729
|
+
|
|
730
|
+
``command`` is the proposed fix written into the escalation record. It is
|
|
731
|
+
carried here rather than derived in the shell so the alert's prose and its
|
|
732
|
+
one-key remedy are decided together — a "HELD" alert whose command says
|
|
733
|
+
``coord drive-queue list`` teaches the operator to ignore the field.
|
|
734
|
+
"""
|
|
735
|
+
|
|
736
|
+
reason: str
|
|
737
|
+
details: tuple[str, ...] = ()
|
|
738
|
+
command: str = "coord drive-queue list"
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
@dataclass(frozen=True)
|
|
742
|
+
class TickPlan:
|
|
743
|
+
"""Everything one tick decided, and nothing it has done yet."""
|
|
744
|
+
|
|
745
|
+
reconciles: tuple[Reconcile, ...] = ()
|
|
746
|
+
launch: QueueEntry | None = None
|
|
747
|
+
blocked: tuple[Blocked, ...] = ()
|
|
748
|
+
deferrals: tuple[Deferral, ...] = ()
|
|
749
|
+
holds: tuple[Hold, ...] = ()
|
|
750
|
+
alert: QueueAlert | None = None
|
|
751
|
+
occupied: int = 0
|
|
752
|
+
capacity: int = 0
|
|
753
|
+
# #1972: the same occupancy, broken down by repo, plus the per-repo ceiling
|
|
754
|
+
# it is measured against. `repo_occupied` holds only repos that actually
|
|
755
|
+
# occupy something (a repo with no live drive is simply absent, not 0), and
|
|
756
|
+
# it is the PRE-launch reading — the same instant `occupied` is taken — so
|
|
757
|
+
# the two never disagree. `repo_capacity == 0` means no per-repo ceiling
|
|
758
|
+
# was applied at all, which is what a `TickPlan` built by hand (or by a
|
|
759
|
+
# pre-#1972 caller) gets, and what makes `render_plan` fall back to the
|
|
760
|
+
# original single-line capacity render.
|
|
761
|
+
repo_occupied: Mapping[str, int] = field(default_factory=dict)
|
|
762
|
+
repo_capacity: int = 0
|
|
763
|
+
# #2101: non-empty when THIS host is under a release cordon, in which case
|
|
764
|
+
# nothing launched this tick and `launch` is guaranteed None. Carried as
|
|
765
|
+
# the cordon's own sentence ("cordoned: draining for v0.5.31") rather than
|
|
766
|
+
# a bool, because a queue that stops with no stated reason is the failure
|
|
767
|
+
# the cordon mechanism exists to stop repeating.
|
|
768
|
+
cordon_reason: str = ""
|
|
769
|
+
|
|
770
|
+
@property
|
|
771
|
+
def free_slots(self) -> int:
|
|
772
|
+
return max(0, self.capacity - self.occupied)
|
|
773
|
+
|
|
774
|
+
@property
|
|
775
|
+
def held(self) -> Hold | None:
|
|
776
|
+
"""The gate holding the queue shut, if any (lowest position wins)."""
|
|
777
|
+
for item in self.holds:
|
|
778
|
+
if item.blocking:
|
|
779
|
+
return item
|
|
780
|
+
return None
|
|
781
|
+
|
|
782
|
+
def writes(self) -> list[tuple[str, Mapping[str, Any]]]:
|
|
783
|
+
"""``(key, updates)`` for every row this plan mutates, in apply order.
|
|
784
|
+
|
|
785
|
+
The launch is NOT here: its row is written by the shell only after
|
|
786
|
+
``coord drive --tmux`` has confirmed a live session, so a launch that
|
|
787
|
+
dies immediately is recorded as a failed attempt rather than as a
|
|
788
|
+
running entry (#1606 makes that exit code trustworthy).
|
|
789
|
+
|
|
790
|
+
Holds come straight after reconciles: the reconcile that moved an
|
|
791
|
+
entry to ``done`` and the hold that fires off it touch the same row,
|
|
792
|
+
and the gate's run state must land after the state that triggered it.
|
|
793
|
+
"""
|
|
794
|
+
out: list[tuple[str, Mapping[str, Any]]] = []
|
|
795
|
+
for item in (*self.reconciles, *self.holds, *self.blocked, *self.deferrals):
|
|
796
|
+
if item.updates:
|
|
797
|
+
out.append((item.key, dict(item.updates)))
|
|
798
|
+
return out
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
# ── cycle detection ──────────────────────────────────────────────────────────
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
def find_cycle(edges: Mapping[str, Sequence[str]]) -> list[str] | None:
|
|
805
|
+
"""Return one cycle in *edges* (``key -> pre-req keys``), or ``None``.
|
|
806
|
+
|
|
807
|
+
Same three-colour DFS as ``coord.milestone_order._check_cycles`` — the
|
|
808
|
+
validation posture ``coord milestone write-order`` applies to ``## Work
|
|
809
|
+
order``, applied to the same shape of graph. Edges pointing outside
|
|
810
|
+
*edges* (a pre-req that isn't itself queued) are ignored: they cannot
|
|
811
|
+
close a loop.
|
|
812
|
+
"""
|
|
813
|
+
white, gray, black = 0, 1, 2
|
|
814
|
+
color = {key: white for key in edges}
|
|
815
|
+
|
|
816
|
+
def visit(node: str, path: list[str]) -> list[str] | None:
|
|
817
|
+
color[node] = gray
|
|
818
|
+
path.append(node)
|
|
819
|
+
for dep in edges.get(node, ()): # noqa: SIM118 — Mapping, not dict
|
|
820
|
+
if dep not in color:
|
|
821
|
+
continue
|
|
822
|
+
if color[dep] == gray:
|
|
823
|
+
return path[path.index(dep):] + [dep]
|
|
824
|
+
if color[dep] == white:
|
|
825
|
+
found = visit(dep, path)
|
|
826
|
+
if found is not None:
|
|
827
|
+
return found
|
|
828
|
+
path.pop()
|
|
829
|
+
color[node] = black
|
|
830
|
+
return None
|
|
831
|
+
|
|
832
|
+
for key in edges:
|
|
833
|
+
if color[key] == white:
|
|
834
|
+
found = visit(key, [])
|
|
835
|
+
if found is not None:
|
|
836
|
+
return found
|
|
837
|
+
return None
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def validate_enqueue(
|
|
841
|
+
entries: Sequence[QueueEntry],
|
|
842
|
+
repo: str,
|
|
843
|
+
issue: int,
|
|
844
|
+
after: Sequence[str],
|
|
845
|
+
) -> None:
|
|
846
|
+
"""Refuse an ``add`` that would be malformed, BEFORE anything is written.
|
|
847
|
+
|
|
848
|
+
Checks, in the order an operator is most likely to hit them: a self-edge,
|
|
849
|
+
then a cycle across the queue as it would look *after* this write (the
|
|
850
|
+
entry being added replaces its own current edges, because ``enqueue``
|
|
851
|
+
upserts). Raises :class:`QueueError`; the caller writes nothing.
|
|
852
|
+
|
|
853
|
+
A pre-req that isn't queued is NOT an error here — the point of `--after`
|
|
854
|
+
is often "run this after that other thing merges", and that thing may
|
|
855
|
+
never be queued at all. Whether such an edge is satisfiable is a *tick*
|
|
856
|
+
question (:func:`plan_tick`), answered against the board.
|
|
857
|
+
"""
|
|
858
|
+
key = entry_key(repo, issue)
|
|
859
|
+
normalised = [str(a) for a in after]
|
|
860
|
+
if key in normalised:
|
|
861
|
+
raise QueueError(f"{key} cannot depend on itself")
|
|
862
|
+
|
|
863
|
+
edges: dict[str, list[str]] = {
|
|
864
|
+
e.key: list(e.after) for e in entries if e.key != key
|
|
865
|
+
}
|
|
866
|
+
edges[key] = normalised
|
|
867
|
+
cycle = find_cycle(edges)
|
|
868
|
+
if cycle is not None:
|
|
869
|
+
raise QueueError("dependency cycle: " + " -> ".join(cycle))
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
# ── pre-req resolution ───────────────────────────────────────────────────────
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
@dataclass(frozen=True)
|
|
876
|
+
class _Verdict:
|
|
877
|
+
satisfied: bool
|
|
878
|
+
unsatisfiable: bool = False
|
|
879
|
+
reason: str = ""
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
def _resolve_prereqs(
|
|
883
|
+
entry: QueueEntry,
|
|
884
|
+
board: BoardView,
|
|
885
|
+
states: Mapping[str, str],
|
|
886
|
+
cycle_keys: Mapping[str, str],
|
|
887
|
+
) -> _Verdict:
|
|
888
|
+
"""Decide whether *entry* may launch now.
|
|
889
|
+
|
|
890
|
+
Three outcomes, and the difference between the last two is the whole
|
|
891
|
+
point: an *unsatisfied* pre-req will plausibly clear on a later tick, so
|
|
892
|
+
the entry defers and keeps its position; an *unsatisfiable* one never
|
|
893
|
+
will, so waiting forever is the silent-stall failure mode this feature
|
|
894
|
+
exists to remove — it blocks and escalates instead.
|
|
895
|
+
"""
|
|
896
|
+
if entry.key in cycle_keys:
|
|
897
|
+
return _Verdict(False, True, cycle_keys[entry.key])
|
|
898
|
+
|
|
899
|
+
for dep in entry.after:
|
|
900
|
+
facts = board.facts(dep)
|
|
901
|
+
if facts.landed:
|
|
902
|
+
continue
|
|
903
|
+
dep_state = states.get(dep)
|
|
904
|
+
if dep_state is not None:
|
|
905
|
+
if dep_state in (STATE_BLOCKED, STATE_FAILED):
|
|
906
|
+
return _Verdict(
|
|
907
|
+
False,
|
|
908
|
+
True,
|
|
909
|
+
f"pre-req {dep} is queued but {dep_state} — it will never satisfy",
|
|
910
|
+
)
|
|
911
|
+
return _Verdict(
|
|
912
|
+
False, False, f"waiting on {dep} (queued, {dep_state})"
|
|
913
|
+
)
|
|
914
|
+
if facts.open:
|
|
915
|
+
return _Verdict(
|
|
916
|
+
False, False, f"waiting on {dep} (open, not queued)"
|
|
917
|
+
)
|
|
918
|
+
if facts.active_work:
|
|
919
|
+
# No `issues` row (the standalone `serialize_board` payload ships
|
|
920
|
+
# assignments only) but live work-like assignment rows — the issue
|
|
921
|
+
# is demonstrably in flight, so this defers rather than blocking.
|
|
922
|
+
return _Verdict(
|
|
923
|
+
False, False, f"waiting on {dep} (work in flight, not queued)"
|
|
924
|
+
)
|
|
925
|
+
return _Verdict(
|
|
926
|
+
False,
|
|
927
|
+
True,
|
|
928
|
+
f"pre-req {dep} is not queued, not merged and not open on the board "
|
|
929
|
+
f"(unknown issue, or the board has not synced it — try `coord sync`)",
|
|
930
|
+
)
|
|
931
|
+
return _Verdict(True)
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
# ── reconciliation ───────────────────────────────────────────────────────────
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def _startup_age(entry: QueueEntry, now: float | None) -> float | None:
|
|
938
|
+
"""Seconds since *entry*'s drive was launched, or ``None`` when unknowable.
|
|
939
|
+
|
|
940
|
+
``None`` — meaning "no startup grace applies" — for three distinct cases,
|
|
941
|
+
all of which must degrade to the pre-#1794 behaviour rather than to an
|
|
942
|
+
entry that can never be retried:
|
|
943
|
+
|
|
944
|
+
* the caller passed no clock (``now is None``): a pure-logic caller that
|
|
945
|
+
does not care about the window, e.g. a test pinning pre-req resolution;
|
|
946
|
+
* the row has no ``launched_at``: a row written before DQ-1 shipped the
|
|
947
|
+
column, or one a human flipped to ``running`` by hand;
|
|
948
|
+
* the stamp is in the FUTURE (negative age): a clock that jumped backwards
|
|
949
|
+
must not be able to pin an entry inside the grace window indefinitely.
|
|
950
|
+
"""
|
|
951
|
+
if now is None or entry.launched_at is None:
|
|
952
|
+
return None
|
|
953
|
+
age = now - entry.launched_at
|
|
954
|
+
return age if age >= 0.0 else None
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _startup_cooldown(
|
|
958
|
+
entry: QueueEntry, now: float | None, grace_seconds: float
|
|
959
|
+
) -> float | None:
|
|
960
|
+
"""The entry's age when it is still inside the startup window, else ``None``.
|
|
961
|
+
|
|
962
|
+
The age is returned (rather than a bare bool) so every caller can put the
|
|
963
|
+
real number in its reason string — a journal line that says "launched 41s
|
|
964
|
+
ago" is diagnosable; one that says "still starting" is not.
|
|
965
|
+
"""
|
|
966
|
+
age = _startup_age(entry, now)
|
|
967
|
+
if age is None or age >= grace_seconds:
|
|
968
|
+
return None
|
|
969
|
+
return age
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _reconcile_running(
|
|
973
|
+
entry: QueueEntry,
|
|
974
|
+
board: BoardView,
|
|
975
|
+
max_attempts: int,
|
|
976
|
+
*,
|
|
977
|
+
now: float | None = None,
|
|
978
|
+
grace_seconds: float = DRIVE_STARTUP_GRACE_SECONDS,
|
|
979
|
+
local_host: str | None = None,
|
|
980
|
+
exit_reasons: Mapping[str, str] | None = None,
|
|
981
|
+
exit_refused: Mapping[str, bool] | None = None,
|
|
982
|
+
exit_dead_end: Mapping[str, bool] | None = None,
|
|
983
|
+
) -> tuple[Reconcile, Blocked | None]:
|
|
984
|
+
"""Resolve one ``running`` entry against the board.
|
|
985
|
+
|
|
986
|
+
The branch ORDER is the contract; each non-death branch exists because a
|
|
987
|
+
real incident proved the fall-through to ``retry`` was wrong:
|
|
988
|
+
|
|
989
|
+
* ``held`` is rule 1 from this module's docstring and the reason capacity
|
|
990
|
+
is not a session count: ``coord drive`` exits ``EXIT_DEADLINE`` (3) when
|
|
991
|
+
the observer's budget runs out, but the worker/test/review it was
|
|
992
|
+
watching keep running on the fleet (#1660). Such an entry has no tmux
|
|
993
|
+
session and no merge yet — counting it as free is exactly the 2026-08-01
|
|
994
|
+
incident, where five expired drives were each stacked on top of.
|
|
995
|
+
* ``refused``/``dead_end`` are #1844/#2019: a drive that exited on a
|
|
996
|
+
PERMANENT pre-dispatch guard refusal — or, since #2019, on a
|
|
997
|
+
terminal-and-unactionable board row — is definitively finished for this
|
|
998
|
+
launch. Both share one branch below; only the wording differs.
|
|
999
|
+
Checked right after ``held``,
|
|
1000
|
+
BEFORE the #1870 cross-host guard and the #1794 startup grace window,
|
|
1001
|
+
because this evidence (the drive's own audit trail, scoped to this
|
|
1002
|
+
exact launch) is stronger than anything a local tmux read or the
|
|
1003
|
+
startup clock can offer; neither of those exists to protect a
|
|
1004
|
+
conclusion this certain. See the extended note below.
|
|
1005
|
+
* ``unknown`` is #1870: ``board.live_sessions`` is always a LOCAL tmux
|
|
1006
|
+
read, but the queue is fleet-global. When *entry* was launched on a
|
|
1007
|
+
DIFFERENT host than *local_host*, an absent local session proves
|
|
1008
|
+
nothing — the drive may be 47 minutes into Test on the machine that
|
|
1009
|
+
actually launched it. Checked AFTER ``held``/``refused`` (real
|
|
1010
|
+
evidence always wins) and BEFORE the grace window / death (neither of
|
|
1011
|
+
which may run on evidence this tick cannot see).
|
|
1012
|
+
* ``starting`` is #1794: a drive that has been launched but has not yet
|
|
1013
|
+
registered a session reading OR put work on the board is not dead, it is
|
|
1014
|
+
young. See :data:`DRIVE_STARTUP_GRACE_SECONDS`.
|
|
1015
|
+
|
|
1016
|
+
``retry`` is therefore reachable only when the session is absent, no work
|
|
1017
|
+
is active, nothing landed, the drive's own exit was not a permanent
|
|
1018
|
+
refusal, the launch host is this host (or unrecorded), AND the launch is
|
|
1019
|
+
older than the grace window — i.e. when death is the only remaining
|
|
1020
|
+
explanation.
|
|
1021
|
+
|
|
1022
|
+
*local_host* is the shell's identity for the machine THIS tick is running
|
|
1023
|
+
on (``None`` disables the check entirely — the pre-#1870 behaviour, same
|
|
1024
|
+
posture as ``now=None`` disabling the grace window). An entry with no
|
|
1025
|
+
recorded ``launch_host`` (predates #1870, or a hand-edited row) is always
|
|
1026
|
+
treated as launched here, so it degrades to today's behaviour exactly.
|
|
1027
|
+
|
|
1028
|
+
#1845/#1844: "no session, no active work, nothing landed" is also exactly
|
|
1029
|
+
what a drive that exited *deliberately* — a clean, non-crash exit after it
|
|
1030
|
+
diagnosed its own blocker and gave up — looks like from here. The drive
|
|
1031
|
+
already wrote the true reason to the audit trail (``drive_exited``,
|
|
1032
|
+
``coord.drive.Driver._drive_exit_summary``); nothing downstream of that
|
|
1033
|
+
write used to read it, so every one of those orderly exits was reported
|
|
1034
|
+
as "drive session died" — a crash where there was none. *exit_reasons*
|
|
1035
|
+
(keyed by :attr:`QueueEntry.key`, fetched by the shell from
|
|
1036
|
+
:func:`coord.audit.query_audit_log` for the current run only — never a
|
|
1037
|
+
stale reason from a prior attempt on the same entry) is that write,
|
|
1038
|
+
threaded through as data so this function stays pure.
|
|
1039
|
+
|
|
1040
|
+
*exit_refused* (same keying, same "this run only" scoping) is #1844's
|
|
1041
|
+
addition: ``True`` when that exit carried ``coord.drive.
|
|
1042
|
+
EXIT_DISPATCH_REFUSED`` rather than a generic non-zero code — i.e. a
|
|
1043
|
+
PERMANENT pre-dispatch guard refusal, not a transient death. That one
|
|
1044
|
+
boolean is the only thing that changes the state transition: an entry
|
|
1045
|
+
with ``exit_refused=True`` goes straight to ``blocked`` (the ``refused``
|
|
1046
|
+
branch above), attempts untouched, on the FIRST tick that observes it —
|
|
1047
|
+
never ``retry``, because nothing about waiting and relaunching can change
|
|
1048
|
+
a condition a retry cannot affect.
|
|
1049
|
+
|
|
1050
|
+
*exit_dead_end* (#2019) is the SAME contract for a second permanent cause:
|
|
1051
|
+
``True`` when the exit carried ``coord.drive.EXIT_DEAD_END`` — the drive's
|
|
1052
|
+
own dead-end predicate (``coord.dead_end.detect_dead_end``) found the row
|
|
1053
|
+
terminal and unactionable, with nothing active on the fleet and no gate
|
|
1054
|
+
transition available. Relaunching a drive against an unchanged dead-end
|
|
1055
|
+
row reproduces the dead end exactly, so it too blocks without spending an
|
|
1056
|
+
attempt; only the reason wording differs from ``exit_refused``'s. Before
|
|
1057
|
+
#2019 this shape did not even reach here — the drive never exited, it
|
|
1058
|
+
counted ``no state change`` against a held tmux session, a held queue slot
|
|
1059
|
+
and (since #1972) a whole repo's capacity lane for 140 minutes.
|
|
1060
|
+
|
|
1061
|
+
Every other exit reason — present or
|
|
1062
|
+
absent, refused or not — only ever changes the WORDING below; whether the
|
|
1063
|
+
entry gets another attempt is otherwise unaffected by #1845/#1844 (still
|
|
1064
|
+
``retry`` until ``max_attempts``, still ``exhausted`` → ``blocked`` after).
|
|
1065
|
+
"""
|
|
1066
|
+
facts = board.facts(entry.key)
|
|
1067
|
+
|
|
1068
|
+
if entry.key in board.live_sessions:
|
|
1069
|
+
return (
|
|
1070
|
+
Reconcile(entry.key, "alive", "drive session is live", occupies=True),
|
|
1071
|
+
None,
|
|
1072
|
+
)
|
|
1073
|
+
|
|
1074
|
+
if facts.landed:
|
|
1075
|
+
witness = "merged" if facts.merged else "issue closed"
|
|
1076
|
+
return (
|
|
1077
|
+
Reconcile(
|
|
1078
|
+
entry.key,
|
|
1079
|
+
"done",
|
|
1080
|
+
f"drive finished ({witness})",
|
|
1081
|
+
occupies=False,
|
|
1082
|
+
updates={
|
|
1083
|
+
"state": STATE_DONE,
|
|
1084
|
+
"last_reason": f"done ({witness})",
|
|
1085
|
+
"session_name": None,
|
|
1086
|
+
},
|
|
1087
|
+
),
|
|
1088
|
+
None,
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
if facts.active_work:
|
|
1092
|
+
return (
|
|
1093
|
+
Reconcile(
|
|
1094
|
+
entry.key,
|
|
1095
|
+
"held",
|
|
1096
|
+
"drive session is gone but work is still ACTIVE on the board "
|
|
1097
|
+
"(observer deadline, #1660) — still occupying a machine",
|
|
1098
|
+
occupies=True,
|
|
1099
|
+
updates={
|
|
1100
|
+
"last_reason": "session gone, work still active on the board",
|
|
1101
|
+
},
|
|
1102
|
+
),
|
|
1103
|
+
None,
|
|
1104
|
+
)
|
|
1105
|
+
|
|
1106
|
+
# #1844: a drive that exited on a PERMANENT pre-dispatch guard refusal
|
|
1107
|
+
# (`coord.dispatch.enforce_oracle_readiness`, `enforce_epic_dispatch_
|
|
1108
|
+
# guard`, or any other check `coord assign`/`coord approve-plan`/`coord
|
|
1109
|
+
# fix` raises a plain `ValueError` for — see `coord.drive.
|
|
1110
|
+
# EXIT_DISPATCH_REFUSED`'s docstring) is definitively FINISHED for this
|
|
1111
|
+
# launch, not merely absent from this tick's evidence. Checked before the
|
|
1112
|
+
# #1870 cross-host guard and the #1794 startup grace window below — both
|
|
1113
|
+
# of which exist only to withhold judgement on WEAK evidence (an absent
|
|
1114
|
+
# local tmux session proves nothing about a foreign host, or about a
|
|
1115
|
+
# drive that has not had time to start yet). This is the strongest
|
|
1116
|
+
# evidence available: the drive's own audit trail, scoped to THIS launch
|
|
1117
|
+
# by the shell (`since=entry.launched_at`), naming its own exit code.
|
|
1118
|
+
# Retrying a deterministic refusal costs a full tick cycle and changes
|
|
1119
|
+
# nothing — the #1817 overnight incident this issue is named for spent
|
|
1120
|
+
# both of its attempts on an identical, guaranteed-to-fail dispatch
|
|
1121
|
+
# before exhausting to `blocked` anyway. So this goes straight to
|
|
1122
|
+
# `blocked`, WITHOUT incrementing `attempts` — there was never anything
|
|
1123
|
+
# to retry.
|
|
1124
|
+
#
|
|
1125
|
+
# #2019 rides the SAME branch with a second cause: `exit_dead_end`. The
|
|
1126
|
+
# evidence is identically strong (the drive's own audit trail, this launch,
|
|
1127
|
+
# naming its own exit code) and the conclusion is identical (relaunching
|
|
1128
|
+
# against an unchanged row reproduces the outcome exactly), so only the
|
|
1129
|
+
# wording and the reported outcome differ. `exit_refused` is checked FIRST
|
|
1130
|
+
# purely for stability — the two codes are mutually exclusive by
|
|
1131
|
+
# construction (`_drive_exit_summary` records exactly one), so the order
|
|
1132
|
+
# is never actually load-bearing.
|
|
1133
|
+
own_reason = (exit_reasons or {}).get(entry.key)
|
|
1134
|
+
|
|
1135
|
+
# #2063 rides the SAME evidence as `refused` below but reaches the
|
|
1136
|
+
# OPPOSITE conclusion, so it is checked first. A Gate-A "no recorded
|
|
1137
|
+
# human sign-off" refusal is not permanent: it is an explicitly
|
|
1138
|
+
# operator-fixable condition with a one-command remedy (`coord gate-a
|
|
1139
|
+
# --approved`), and it self-clears the moment that verdict is recorded.
|
|
1140
|
+
# Landing it in terminal `blocked` — which nothing re-evaluates and
|
|
1141
|
+
# `coord drive-queue add` will not clear (#2040) — would leave the entry
|
|
1142
|
+
# dead AFTER the human approved, requiring an undocumented remove+add.
|
|
1143
|
+
# So it parks (#1891 semantics: re-checked every tick, no attempt spent),
|
|
1144
|
+
# and `plan_tick`'s pre-pass below un-parks it once the verdict exists.
|
|
1145
|
+
if own_reason and is_gate_a_refusal_reason(own_reason):
|
|
1146
|
+
reason = (
|
|
1147
|
+
f"{own_reason} — parking without spending an attempt; the queue "
|
|
1148
|
+
"resumes it automatically once a human records the verdict, no "
|
|
1149
|
+
"queue surgery needed (#2063)"
|
|
1150
|
+
)
|
|
1151
|
+
return (
|
|
1152
|
+
Reconcile(
|
|
1153
|
+
entry.key,
|
|
1154
|
+
"parked",
|
|
1155
|
+
reason,
|
|
1156
|
+
occupies=False,
|
|
1157
|
+
updates={
|
|
1158
|
+
"state": STATE_PARKED,
|
|
1159
|
+
"last_reason": reason,
|
|
1160
|
+
"session_name": None,
|
|
1161
|
+
},
|
|
1162
|
+
),
|
|
1163
|
+
None,
|
|
1164
|
+
)
|
|
1165
|
+
|
|
1166
|
+
permanent: tuple[str, str] | None = None
|
|
1167
|
+
if own_reason and (exit_refused or {}).get(entry.key):
|
|
1168
|
+
permanent = (
|
|
1169
|
+
"refused",
|
|
1170
|
+
"refused by a pre-dispatch guard, which cannot change on retry "
|
|
1171
|
+
"(#1844); blocking without spending an attempt",
|
|
1172
|
+
)
|
|
1173
|
+
elif own_reason and (exit_dead_end or {}).get(entry.key):
|
|
1174
|
+
permanent = (
|
|
1175
|
+
"dead_end",
|
|
1176
|
+
"the board row is terminal and unactionable (nothing active, no "
|
|
1177
|
+
"gate transition available), which cannot change on retry "
|
|
1178
|
+
"(#2019); blocking without spending an attempt",
|
|
1179
|
+
)
|
|
1180
|
+
if permanent is not None:
|
|
1181
|
+
outcome, explanation = permanent
|
|
1182
|
+
reason = f"{own_reason} — {explanation}"
|
|
1183
|
+
# `Reconcile.updates` is deliberately EMPTY, same as `exhausted`
|
|
1184
|
+
# below — the paired `Blocked` carries every write, applied once by
|
|
1185
|
+
# `TickPlan.writes()`. `attempts` is absent from BOTH: there is
|
|
1186
|
+
# nothing to spend, unlike `exhausted`'s Blocked which stamps the
|
|
1187
|
+
# final attempt count.
|
|
1188
|
+
return (
|
|
1189
|
+
Reconcile(entry.key, outcome, reason, occupies=False),
|
|
1190
|
+
Blocked(
|
|
1191
|
+
entry.key,
|
|
1192
|
+
reason,
|
|
1193
|
+
updates={
|
|
1194
|
+
"state": STATE_BLOCKED,
|
|
1195
|
+
"last_reason": reason,
|
|
1196
|
+
"session_name": None,
|
|
1197
|
+
},
|
|
1198
|
+
),
|
|
1199
|
+
)
|
|
1200
|
+
|
|
1201
|
+
if (
|
|
1202
|
+
local_host is not None
|
|
1203
|
+
and entry.launch_host
|
|
1204
|
+
and entry.launch_host.lower() != local_host.lower()
|
|
1205
|
+
):
|
|
1206
|
+
# #1870. This tick's tmux read is LOCAL; it cannot see a session on
|
|
1207
|
+
# the host that actually launched this entry, so its absence here is
|
|
1208
|
+
# not evidence of anything. Fail-soft exactly like an unreachable
|
|
1209
|
+
# probe would: occupy the slot, touch neither `state` nor `attempts`,
|
|
1210
|
+
# and never relaunch — the same posture #1794 established for "tmux
|
|
1211
|
+
# unavailable" / "no server running" / "timed out".
|
|
1212
|
+
reason = (
|
|
1213
|
+
f"drive was launched on {entry.launch_host!r}, not this host "
|
|
1214
|
+
f"({local_host!r}) — liveness cannot be verified from here, so "
|
|
1215
|
+
f"this is UNKNOWN, not dead (#1870); still occupying a slot, no "
|
|
1216
|
+
f"attempt spent"
|
|
1217
|
+
)
|
|
1218
|
+
return (
|
|
1219
|
+
Reconcile(
|
|
1220
|
+
entry.key,
|
|
1221
|
+
"unknown",
|
|
1222
|
+
reason,
|
|
1223
|
+
occupies=True,
|
|
1224
|
+
updates={"last_reason": reason},
|
|
1225
|
+
),
|
|
1226
|
+
None,
|
|
1227
|
+
)
|
|
1228
|
+
|
|
1229
|
+
age = _startup_cooldown(entry, now, grace_seconds)
|
|
1230
|
+
if age is not None:
|
|
1231
|
+
# #1794. Launched, but not yet visible as a session and not yet
|
|
1232
|
+
# visible as work. A tick that fires inside this window sees exactly
|
|
1233
|
+
# what a dead drive looks like, so it must not be allowed to conclude
|
|
1234
|
+
# anything: the entry keeps its state, keeps its attempts, and keeps
|
|
1235
|
+
# its slot.
|
|
1236
|
+
reason = (
|
|
1237
|
+
f"drive is still starting — launched {age:.0f}s ago, inside the "
|
|
1238
|
+
f"{grace_seconds:.0f}s startup grace window (#1794); "
|
|
1239
|
+
f"not a death, still occupying a machine"
|
|
1240
|
+
)
|
|
1241
|
+
return (
|
|
1242
|
+
Reconcile(
|
|
1243
|
+
entry.key,
|
|
1244
|
+
"starting",
|
|
1245
|
+
reason,
|
|
1246
|
+
occupies=True,
|
|
1247
|
+
updates={"last_reason": reason},
|
|
1248
|
+
),
|
|
1249
|
+
None,
|
|
1250
|
+
)
|
|
1251
|
+
|
|
1252
|
+
# Past the grace window (or with no launch stamp to measure), with no
|
|
1253
|
+
# session, no active work and nothing landed: this entry did not land the
|
|
1254
|
+
# work by any board-visible path. #1845/#1844: that no longer means
|
|
1255
|
+
# "died" — the drive may have exited deliberately, with its own reason
|
|
1256
|
+
# already on the audit trail. Prefer that reason when one was recorded
|
|
1257
|
+
# for this run; fall back to the synthesised wording (with the launch age
|
|
1258
|
+
# quoted, so a journal reader can tell a genuine death from a grace
|
|
1259
|
+
# window that was set too short) when it wasn't — e.g. no audit row at
|
|
1260
|
+
# all, a crash that never reached the `drive_exited` write, or a shell
|
|
1261
|
+
# that failed to fetch it.
|
|
1262
|
+
since = _startup_age(entry, now)
|
|
1263
|
+
launched = f", launched {since:.0f}s ago" if since is not None else ""
|
|
1264
|
+
# `own_reason` was already resolved above (before the cross-host/startup
|
|
1265
|
+
# checks) so the `refused` branch could use it; reused here unchanged —
|
|
1266
|
+
# a non-refusal exit reason (a genuine death that still narrated why)
|
|
1267
|
+
# still wins over the synthesised wording, same as #1845.
|
|
1268
|
+
|
|
1269
|
+
# #1891: checked BEFORE the retry/exhausted computation below, and
|
|
1270
|
+
# deliberately independent of `own_reason`/`exit_refused` — it does not
|
|
1271
|
+
# matter WHY this drive is no longer visible (a deadline, a crash, a
|
|
1272
|
+
# machine reboot mid-wait); what matters is whether the board's OWN
|
|
1273
|
+
# current read of this entry's issue still shows nothing stronger than
|
|
1274
|
+
# "CI checks have not reported yet". Relaunching a fresh `coord drive`
|
|
1275
|
+
# right now would just observe the identical silence and wait again — so
|
|
1276
|
+
# this parks instead, without spending an attempt (mirrors `refused`
|
|
1277
|
+
# just above: `Reconcile.updates` carries the whole transition, no paired
|
|
1278
|
+
# `Blocked`, because unlike `refused` this is not a terminal condition —
|
|
1279
|
+
# see `plan_tick`'s pre-pass, which is what un-parks it).
|
|
1280
|
+
if facts.merge_ci_pending:
|
|
1281
|
+
reason = (
|
|
1282
|
+
f"{facts.merge_ci_pending_reason or 'CI checks have not reported yet'}"
|
|
1283
|
+
f"{launched} — parking without spending an attempt; the queue "
|
|
1284
|
+
"resumes it automatically once they do, no operator needed (#1891)"
|
|
1285
|
+
)
|
|
1286
|
+
return (
|
|
1287
|
+
Reconcile(
|
|
1288
|
+
entry.key,
|
|
1289
|
+
"parked",
|
|
1290
|
+
reason,
|
|
1291
|
+
occupies=False,
|
|
1292
|
+
updates={
|
|
1293
|
+
"state": STATE_PARKED,
|
|
1294
|
+
"last_reason": reason,
|
|
1295
|
+
"session_name": None,
|
|
1296
|
+
},
|
|
1297
|
+
),
|
|
1298
|
+
None,
|
|
1299
|
+
)
|
|
1300
|
+
|
|
1301
|
+
attempts = entry.attempts + 1
|
|
1302
|
+
if attempts < max_attempts:
|
|
1303
|
+
if own_reason:
|
|
1304
|
+
reason = (
|
|
1305
|
+
f"{own_reason} (attempt {attempts}/{max_attempts}) — "
|
|
1306
|
+
f"requeued at position {entry.position}"
|
|
1307
|
+
)
|
|
1308
|
+
else:
|
|
1309
|
+
reason = (
|
|
1310
|
+
f"drive session died without landing the work"
|
|
1311
|
+
f"{launched} (attempt {attempts}/{max_attempts}) — requeued at "
|
|
1312
|
+
f"position {entry.position}"
|
|
1313
|
+
)
|
|
1314
|
+
return (
|
|
1315
|
+
Reconcile(
|
|
1316
|
+
entry.key,
|
|
1317
|
+
"retry",
|
|
1318
|
+
reason,
|
|
1319
|
+
occupies=False,
|
|
1320
|
+
updates={
|
|
1321
|
+
"state": STATE_WAITING,
|
|
1322
|
+
"attempts": attempts,
|
|
1323
|
+
"last_reason": reason,
|
|
1324
|
+
"session_name": None,
|
|
1325
|
+
},
|
|
1326
|
+
),
|
|
1327
|
+
None,
|
|
1328
|
+
)
|
|
1329
|
+
|
|
1330
|
+
if own_reason:
|
|
1331
|
+
reason = (
|
|
1332
|
+
f"{own_reason} ({attempts}/{max_attempts} attempts) — giving up"
|
|
1333
|
+
)
|
|
1334
|
+
else:
|
|
1335
|
+
reason = (
|
|
1336
|
+
f"drive session died without landing the work"
|
|
1337
|
+
f"{launched} {attempts}/{max_attempts} times — giving up"
|
|
1338
|
+
)
|
|
1339
|
+
return (
|
|
1340
|
+
Reconcile(entry.key, "exhausted", reason, occupies=False),
|
|
1341
|
+
Blocked(
|
|
1342
|
+
entry.key,
|
|
1343
|
+
reason,
|
|
1344
|
+
updates={
|
|
1345
|
+
"state": STATE_BLOCKED,
|
|
1346
|
+
"attempts": attempts,
|
|
1347
|
+
"last_reason": reason,
|
|
1348
|
+
"session_name": None,
|
|
1349
|
+
},
|
|
1350
|
+
),
|
|
1351
|
+
)
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
# ── deploy gates (#1757) ─────────────────────────────────────────────────────
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def pending_probe_targets(entries: Sequence[QueueEntry]) -> list[QueueEntry]:
|
|
1358
|
+
"""Entries whose ``resume_when`` the shell should run BEFORE this tick.
|
|
1359
|
+
|
|
1360
|
+
Only an ALREADY-``fired`` gate is probed: a gate that fires during this
|
|
1361
|
+
tick's own reconcile holds unconditionally for one interval, which is the
|
|
1362
|
+
issue's rule ("a ``fired`` hold makes each SUBSEQUENT tick run the
|
|
1363
|
+
command") and also the honest one — the deploy cannot have happened in the
|
|
1364
|
+
microseconds since the merge was observed.
|
|
1365
|
+
|
|
1366
|
+
Pure and position-ordered, so the shell has no decision left to make: it
|
|
1367
|
+
runs exactly this list, in this order, and hands the results back to
|
|
1368
|
+
:func:`plan_tick`.
|
|
1369
|
+
"""
|
|
1370
|
+
return [
|
|
1371
|
+
e
|
|
1372
|
+
for e in sorted(entries, key=lambda e: (e.position, e.key))
|
|
1373
|
+
if e.hold_state == HOLD_FIRED and e.resume_when
|
|
1374
|
+
]
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def fired_holds(entries: Sequence[QueueEntry]) -> list[QueueEntry]:
|
|
1378
|
+
"""Entries whose gate has fired and is still holding the queue shut.
|
|
1379
|
+
|
|
1380
|
+
What ``coord drive-queue resume`` releases and what ``status`` reports.
|
|
1381
|
+
Position-ordered so "the hold" is always the same entry in both.
|
|
1382
|
+
"""
|
|
1383
|
+
return [
|
|
1384
|
+
e
|
|
1385
|
+
for e in sorted(entries, key=lambda e: (e.position, e.key))
|
|
1386
|
+
if e.hold_state == HOLD_FIRED
|
|
1387
|
+
]
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
def _resolve_holds(
|
|
1391
|
+
ordered: Sequence[QueueEntry],
|
|
1392
|
+
reconciled_states: Mapping[str, str],
|
|
1393
|
+
probes: Mapping[str, ProbeResult],
|
|
1394
|
+
) -> list[Hold]:
|
|
1395
|
+
"""Fire / probe / release every gate, in position order.
|
|
1396
|
+
|
|
1397
|
+
*reconciled_states* is each entry's queue state AFTER step 1 of the tick,
|
|
1398
|
+
which is what makes "fires on ``done`` only" checkable here: a
|
|
1399
|
+
``--hold-after`` entry that reconciled to ``blocked`` never reaches this
|
|
1400
|
+
branch, so it produces the existing escalation and NOT a second alert (the
|
|
1401
|
+
issue's explicit rule — two alerts for one condition is how an alert
|
|
1402
|
+
channel gets muted).
|
|
1403
|
+
"""
|
|
1404
|
+
holds: list[Hold] = []
|
|
1405
|
+
for entry in ordered:
|
|
1406
|
+
if not entry.hold_after:
|
|
1407
|
+
continue
|
|
1408
|
+
|
|
1409
|
+
# ARMED → FIRED, the tick the entry lands. Nothing else fires a gate:
|
|
1410
|
+
# `blocked`/`failed` already stop the queue through the escalation
|
|
1411
|
+
# path, and `waiting`/`running` have not finished anything yet.
|
|
1412
|
+
if (
|
|
1413
|
+
entry.hold_state == HOLD_ARMED
|
|
1414
|
+
and reconciled_states.get(entry.key) == STATE_DONE
|
|
1415
|
+
):
|
|
1416
|
+
holds.append(
|
|
1417
|
+
Hold(
|
|
1418
|
+
key=entry.key,
|
|
1419
|
+
outcome="fired",
|
|
1420
|
+
reason=entry.gate_reason,
|
|
1421
|
+
resume_when=entry.resume_when,
|
|
1422
|
+
probes=0,
|
|
1423
|
+
updates={"hold_state": HOLD_FIRED, "hold_probes": 0},
|
|
1424
|
+
)
|
|
1425
|
+
)
|
|
1426
|
+
continue
|
|
1427
|
+
|
|
1428
|
+
if entry.hold_state != HOLD_FIRED:
|
|
1429
|
+
# `''` (no gate yet armed), `armed` on an entry that has not
|
|
1430
|
+
# landed, or `released` — none of which hold anything.
|
|
1431
|
+
continue
|
|
1432
|
+
|
|
1433
|
+
probe = probes.get(entry.key)
|
|
1434
|
+
if probe is None:
|
|
1435
|
+
# No probe declared, or the shell did not run one. Manual resume
|
|
1436
|
+
# only; the count does not move, so a hold that nobody probes
|
|
1437
|
+
# never grows a fake attempt number.
|
|
1438
|
+
holds.append(
|
|
1439
|
+
Hold(
|
|
1440
|
+
key=entry.key,
|
|
1441
|
+
outcome="held",
|
|
1442
|
+
reason=entry.gate_reason,
|
|
1443
|
+
resume_when=entry.resume_when,
|
|
1444
|
+
probes=entry.hold_probes,
|
|
1445
|
+
)
|
|
1446
|
+
)
|
|
1447
|
+
continue
|
|
1448
|
+
|
|
1449
|
+
if probe.ok:
|
|
1450
|
+
holds.append(
|
|
1451
|
+
Hold(
|
|
1452
|
+
key=entry.key,
|
|
1453
|
+
outcome="released",
|
|
1454
|
+
reason=entry.gate_reason,
|
|
1455
|
+
resume_when=entry.resume_when,
|
|
1456
|
+
probes=entry.hold_probes,
|
|
1457
|
+
probe_detail=probe.detail,
|
|
1458
|
+
updates={"hold_state": HOLD_RELEASED, "hold_probes": 0},
|
|
1459
|
+
)
|
|
1460
|
+
)
|
|
1461
|
+
continue
|
|
1462
|
+
|
|
1463
|
+
attempts = entry.hold_probes + 1
|
|
1464
|
+
holds.append(
|
|
1465
|
+
Hold(
|
|
1466
|
+
key=entry.key,
|
|
1467
|
+
outcome="held",
|
|
1468
|
+
reason=entry.gate_reason,
|
|
1469
|
+
resume_when=entry.resume_when,
|
|
1470
|
+
probes=attempts,
|
|
1471
|
+
probe_detail=probe.detail,
|
|
1472
|
+
updates={"hold_probes": attempts},
|
|
1473
|
+
)
|
|
1474
|
+
)
|
|
1475
|
+
return holds
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
def _norm_host(name: str | None) -> str:
|
|
1479
|
+
"""#2101: the one host-identity normalisation this module compares by.
|
|
1480
|
+
|
|
1481
|
+
Machine names come from ``coordinator.yml`` (``dellserver``) and host
|
|
1482
|
+
identities from ``socket.gethostname()`` (``dellserver.local``), and a
|
|
1483
|
+
cordon that fails to match because of a domain suffix is a cordon that
|
|
1484
|
+
silently does nothing — the exact class of failure #1563 closed for
|
|
1485
|
+
pause. Same normalisation `coord/commands/drive_queue.py`'s
|
|
1486
|
+
``_local_host_id`` already applies: short hostname, lowercased.
|
|
1487
|
+
"""
|
|
1488
|
+
return str(name or "").split(".")[0].strip().lower()
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
def _normalized_cordons(cordons: Mapping[str, str] | None) -> dict[str, str]:
|
|
1492
|
+
return {
|
|
1493
|
+
_norm_host(name): str(reason)
|
|
1494
|
+
for name, reason in (cordons or {}).items()
|
|
1495
|
+
if _norm_host(name) and reason
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def _cordon_alert(host: str, reason: str) -> QueueAlert:
|
|
1500
|
+
"""The queue-level record for "this host is cordoned" (#2101 trap E).
|
|
1501
|
+
|
|
1502
|
+
Mirrors :func:`_hold_alert`: a queue that stops must say why, in the same
|
|
1503
|
+
channel and with the same one-key remedy field, or "stopped" and "wedged"
|
|
1504
|
+
look identical from the outside — which is how a fleet sits eleven
|
|
1505
|
+
releases behind for a day with every readout silent.
|
|
1506
|
+
"""
|
|
1507
|
+
return QueueAlert(
|
|
1508
|
+
reason=(
|
|
1509
|
+
f"no launch — {host} is {reason}. In-flight drives are draining; "
|
|
1510
|
+
"the queue resumes automatically the moment this host is rolled "
|
|
1511
|
+
"and uncordoned (#2101)."
|
|
1512
|
+
),
|
|
1513
|
+
details=(
|
|
1514
|
+
"release cordons expire on their own if the propagate run that "
|
|
1515
|
+
"set one dies, so this can never wedge the queue permanently",
|
|
1516
|
+
),
|
|
1517
|
+
command=f"coord release cordon --clear {host}",
|
|
1518
|
+
)
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
def _hold_alert(hold: Hold) -> QueueAlert:
|
|
1522
|
+
"""The one queue-level record a closed gate raises.
|
|
1523
|
+
|
|
1524
|
+
Carries the operator's own ``hold_reason`` verbatim in ``reason`` — that
|
|
1525
|
+
string is the entire point of the feature (it is the runbook line for the
|
|
1526
|
+
deploy the queue is waiting on), so it must survive into the alert without
|
|
1527
|
+
being summarised.
|
|
1528
|
+
"""
|
|
1529
|
+
details = [f"held after {hold.key} — nothing will launch until this is released"]
|
|
1530
|
+
if hold.resume_when:
|
|
1531
|
+
outcome = (
|
|
1532
|
+
f"attempt {hold.probes} failed"
|
|
1533
|
+
if hold.probes
|
|
1534
|
+
else "not probed yet (fires on the next tick)"
|
|
1535
|
+
)
|
|
1536
|
+
if hold.probe_detail:
|
|
1537
|
+
outcome += f": {hold.probe_detail}"
|
|
1538
|
+
details.append(f"resume-when: {hold.resume_when} ({outcome})")
|
|
1539
|
+
else:
|
|
1540
|
+
details.append("no --resume-when probe: release manually")
|
|
1541
|
+
return QueueAlert(
|
|
1542
|
+
reason=f"QUEUE HELD — {hold.reason}",
|
|
1543
|
+
details=tuple(details),
|
|
1544
|
+
command="coord drive-queue resume",
|
|
1545
|
+
)
|
|
1546
|
+
|
|
1547
|
+
|
|
1548
|
+
# ── the tick ─────────────────────────────────────────────────────────────────
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
def plan_tick(
|
|
1552
|
+
entries: Sequence[QueueEntry],
|
|
1553
|
+
board: BoardView,
|
|
1554
|
+
capacity: int,
|
|
1555
|
+
*,
|
|
1556
|
+
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
|
1557
|
+
max_parallel_per_repo: int = DEFAULT_MAX_PARALLEL_PER_REPO,
|
|
1558
|
+
probes: Mapping[str, ProbeResult] | None = None,
|
|
1559
|
+
now: float | None = None,
|
|
1560
|
+
grace_seconds: float = DRIVE_STARTUP_GRACE_SECONDS,
|
|
1561
|
+
local_host: str | None = None,
|
|
1562
|
+
exit_reasons: Mapping[str, str] | None = None,
|
|
1563
|
+
exit_refused: Mapping[str, bool] | None = None,
|
|
1564
|
+
exit_dead_end: Mapping[str, bool] | None = None,
|
|
1565
|
+
gate_a_pending: Mapping[str, bool] | None = None,
|
|
1566
|
+
cordons: Mapping[str, str] | None = None,
|
|
1567
|
+
) -> TickPlan:
|
|
1568
|
+
"""Decide one tick. Pure; the caller executes the returned plan.
|
|
1569
|
+
|
|
1570
|
+
*capacity* is the CEILING (``--max-parallel``), not the number of free
|
|
1571
|
+
slots — how many slots are already occupied is a decision (rule 1 above),
|
|
1572
|
+
and decisions live in here, not in the shell.
|
|
1573
|
+
|
|
1574
|
+
*max_parallel_per_repo* is the SECOND ceiling (#1972), applied per repo
|
|
1575
|
+
after the global one: an entry whose repo already occupies this many slots
|
|
1576
|
+
defers, so the walk lands on the first entry from a repo with headroom.
|
|
1577
|
+
Both ceilings apply, global first. ``0`` disables it (pre-#1972
|
|
1578
|
+
behaviour); the default of 1 is per-repo serialisation, which for a
|
|
1579
|
+
single-repo queue at ``--max-parallel 1`` is exactly what the queue did
|
|
1580
|
+
before. See :data:`DEFAULT_MAX_PARALLEL_PER_REPO` for why repo is the
|
|
1581
|
+
right axis and what the board-derived counting means for a wedged drive.
|
|
1582
|
+
|
|
1583
|
+
*probes* maps an entry key to the :class:`ProbeResult` the shell got from
|
|
1584
|
+
running that entry's ``resume_when`` (see :func:`pending_probe_targets`);
|
|
1585
|
+
an absent key simply means no probe ran.
|
|
1586
|
+
|
|
1587
|
+
*exit_reasons* maps a ``running`` entry's key to the drive's own
|
|
1588
|
+
``drive_exited`` audit summary for THIS launch (#1845/#1844) — see
|
|
1589
|
+
:func:`_reconcile_running` for what it changes (wording, and — when
|
|
1590
|
+
*exit_refused* also marks the entry — the ``retry``/``exhausted``
|
|
1591
|
+
decision itself) and why "for this launch" matters (a stale reason from a
|
|
1592
|
+
prior attempt on the same entry must never be replayed as if it explained
|
|
1593
|
+
the current one).
|
|
1594
|
+
|
|
1595
|
+
*exit_refused* maps the same keys to ``True`` when that exit was a
|
|
1596
|
+
PERMANENT pre-dispatch guard refusal (``coord.drive.
|
|
1597
|
+
EXIT_DISPATCH_REFUSED``) rather than a transient death (#1844). Unlike
|
|
1598
|
+
*exit_reasons*, this DOES change the decision: such an entry reconciles
|
|
1599
|
+
straight to ``blocked`` with ``attempts`` unchanged, never ``retry`` —
|
|
1600
|
+
see :func:`_reconcile_running`'s ``refused`` branch.
|
|
1601
|
+
|
|
1602
|
+
*exit_dead_end* is the #2019 twin: ``True`` when the exit was
|
|
1603
|
+
``coord.drive.EXIT_DEAD_END`` (the row was terminal and unactionable).
|
|
1604
|
+
Same disposition, same branch, different wording — a relaunch against an
|
|
1605
|
+
unchanged dead-end row reproduces the dead end exactly, so it too costs no
|
|
1606
|
+
attempt.
|
|
1607
|
+
|
|
1608
|
+
*now* is the shell's ``time.time()``, passed in rather than read here (see
|
|
1609
|
+
the module docstring). It powers #1794's startup grace window on both
|
|
1610
|
+
sides of the tick: a recently-launched entry never reconciles to ``retry``
|
|
1611
|
+
(:func:`_reconcile_running`), and no entry is relaunched while its last
|
|
1612
|
+
launch is still that recent (step 4 below). ``None`` disables the window
|
|
1613
|
+
entirely, which is the pre-#1794 behaviour — the production shell always
|
|
1614
|
+
passes a real clock.
|
|
1615
|
+
|
|
1616
|
+
*local_host* is the shell's identity for the machine THIS tick is running
|
|
1617
|
+
on (#1870). It powers the cross-host guard in :func:`_reconcile_running`:
|
|
1618
|
+
an entry whose ``launch_host`` names a DIFFERENT machine reconciles to
|
|
1619
|
+
``unknown`` rather than ``retry``, because this tick's tmux read cannot
|
|
1620
|
+
see that host at all. ``None`` disables the check entirely, the
|
|
1621
|
+
pre-#1870 behaviour — the production shell always passes its own
|
|
1622
|
+
hostname.
|
|
1623
|
+
|
|
1624
|
+
*cordons* maps a machine name to the reason it is under a #2101 release
|
|
1625
|
+
cordon ("cordoned: draining for v0.5.31"). Two distinct effects, because
|
|
1626
|
+
a drive occupies two machines:
|
|
1627
|
+
|
|
1628
|
+
* this tick's own host (*local_host*) cordoned ⇒ launch NOTHING at all,
|
|
1629
|
+
because ``coord drive --tmux`` starts its session HERE. This is the
|
|
1630
|
+
hole #2101 names outright: `coord/drive.py` checks pause only when
|
|
1631
|
+
routing a *worker*, so before this the main launcher walked straight
|
|
1632
|
+
through the cordon and the fleet could never drain.
|
|
1633
|
+
* an entry pinned (``--machine``) to a cordoned host ⇒ that ENTRY defers,
|
|
1634
|
+
position unchanged, no attempt spent; the walk moves on to the next
|
|
1635
|
+
entry, which may well be launchable on an uncordoned machine.
|
|
1636
|
+
|
|
1637
|
+
Reconciliation (steps 1/1b) still runs under a cordon, and that is the
|
|
1638
|
+
whole point: a cordon that also froze the queue's view of reality would
|
|
1639
|
+
leave a finished drive's `running` row pinning propagation forever — the
|
|
1640
|
+
#2110 deadlock, re-created by the very mechanism meant to end it. A
|
|
1641
|
+
cordoned tick is exactly `--reconcile-only`.
|
|
1642
|
+
|
|
1643
|
+
The algorithm, from #1754, plus #1757's step 2, #1891's step 1b, and
|
|
1644
|
+
#2055's extension of it:
|
|
1645
|
+
|
|
1646
|
+
1. Reconcile every ``running`` entry (:func:`_reconcile_running`).
|
|
1647
|
+
1b. Re-check every ``parked``/``blocked``/``failed`` entry against the
|
|
1648
|
+
CURRENT board: landed ⇒ ``done`` (#1891 for ``parked``, #2055 for
|
|
1649
|
+
``blocked``/``failed``). For ``parked`` only, not-yet-landed then
|
|
1650
|
+
also checks the gate: cleared ⇒ ``waiting`` (falls into step 4 on
|
|
1651
|
+
this SAME tick); still shut ⇒ untouched, no write, nothing to
|
|
1652
|
+
report. ``blocked``/``failed`` that haven't landed are left alone
|
|
1653
|
+
entirely — this never resurrects them for dispatch, only lets a
|
|
1654
|
+
finished one stop claiming to be unfinished. Never spends an
|
|
1655
|
+
attempt either way — a missing CI verdict is not a failed one.
|
|
1656
|
+
2. Resolve deploy gates (:func:`_resolve_holds`). ANY gate left closed
|
|
1657
|
+
returns immediately with no launch and a HELD alert — before the
|
|
1658
|
+
capacity check, and regardless of how eligible the rest of the queue
|
|
1659
|
+
is. That "even with free capacity and an eligible successor" clause is
|
|
1660
|
+
the entire feature: the successor is exactly the thing that must not
|
|
1661
|
+
run until the deploy lands.
|
|
1662
|
+
3. ``free = capacity - occupied``; ``<= 0`` returns with no launch and no
|
|
1663
|
+
alert — being at capacity is the queue working, not a problem to
|
|
1664
|
+
report.
|
|
1665
|
+
4. Walk ``waiting`` by ``position``, FIRST ELIGIBLE WINS: an entry still
|
|
1666
|
+
inside its startup grace window defers (#1794); an entry whose own
|
|
1667
|
+
issue is already landed (merged or closed) reconciles straight to
|
|
1668
|
+
``done`` without ever launching (#1873) — checked before its `after=`
|
|
1669
|
+
graph, so a landed entry is never blocked or deferred on account of its
|
|
1670
|
+
own now-irrelevant pre-reqs; unsatisfiable blocks and escalates,
|
|
1671
|
+
unsatisfied defers (position unchanged); an entry whose REPO is already
|
|
1672
|
+
at *max_parallel_per_repo* defers too (#1972, checked LAST — a broken
|
|
1673
|
+
pre-req is a permanent fact and must still escalate, whatever the
|
|
1674
|
+
repo's occupancy is doing this tick); the first eligible entry is the
|
|
1675
|
+
launch. Everything after the launch is walked in REPORT-ONLY mode
|
|
1676
|
+
(``Deferral.counted=False``, no updates) so ``--dry-run`` can explain
|
|
1677
|
+
the rest of the queue — including against the launch's own repo, which
|
|
1678
|
+
the report-only pass counts as occupied.
|
|
1679
|
+
5. No launch with at least one entry STILL genuinely waiting (deferred or
|
|
1680
|
+
blocked — #1873 reconciliations do not count, see below) ⇒ exactly ONE
|
|
1681
|
+
queue-level alert. #1972's repo-limit deferrals do not count either:
|
|
1682
|
+
a queue whose every remaining entry is waiting on its own repo's
|
|
1683
|
+
in-flight work is saturated, not stalled, and alerting on it every tick
|
|
1684
|
+
is how an alert channel gets muted (same reasoning as step 3).
|
|
1685
|
+
|
|
1686
|
+
An entry reconciled from ``running`` back to ``waiting`` in step 1 IS
|
|
1687
|
+
walked in step 4 — its attempt was already consumed, so a drive that died
|
|
1688
|
+
early relaunches on the same tick instead of idling a whole interval. A
|
|
1689
|
+
gate RELEASED in step 2 likewise falls straight through into step 4, so a
|
|
1690
|
+
probe that starts passing launches in the same tick rather than costing
|
|
1691
|
+
the queue a whole interval.
|
|
1692
|
+
|
|
1693
|
+
#1794 puts one bound on that same-tick relaunch, and it is the reason the
|
|
1694
|
+
grace window is checked TWICE. Step 1 can only produce a ``retry`` for an
|
|
1695
|
+
entry whose launch is older than *grace_seconds*, so the relaunch is only
|
|
1696
|
+
ever of a drive the tick is confident is gone; and step 4 refuses the
|
|
1697
|
+
launch outright for anything launched more recently, whatever put it back
|
|
1698
|
+
in ``waiting``. Between them, no single tick can start a second ``coord
|
|
1699
|
+
drive`` for an issue whose first one may still be coming up.
|
|
1700
|
+
"""
|
|
1701
|
+
ordered = sorted(entries, key=lambda e: (e.position, e.key))
|
|
1702
|
+
states: dict[str, str] = {e.key: e.state for e in ordered}
|
|
1703
|
+
by_key = {e.key: e for e in ordered}
|
|
1704
|
+
|
|
1705
|
+
reconciles: list[Reconcile] = []
|
|
1706
|
+
blocked: list[Blocked] = []
|
|
1707
|
+
deferrals: list[Deferral] = []
|
|
1708
|
+
occupied = 0
|
|
1709
|
+
# #1972: the same count, keyed by repo. Populated from the SAME
|
|
1710
|
+
# `reconcile.occupies` verdict as `occupied` above — one source of truth, so
|
|
1711
|
+
# the per-repo view can never claim a slot the global view does not.
|
|
1712
|
+
repo_occupied: dict[str, int] = {}
|
|
1713
|
+
repo_capacity = max(0, int(max_parallel_per_repo))
|
|
1714
|
+
|
|
1715
|
+
for entry in ordered:
|
|
1716
|
+
if entry.state != STATE_RUNNING:
|
|
1717
|
+
continue
|
|
1718
|
+
reconcile, block = _reconcile_running(
|
|
1719
|
+
entry,
|
|
1720
|
+
board,
|
|
1721
|
+
max_attempts,
|
|
1722
|
+
now=now,
|
|
1723
|
+
grace_seconds=grace_seconds,
|
|
1724
|
+
local_host=local_host,
|
|
1725
|
+
exit_reasons=exit_reasons,
|
|
1726
|
+
exit_refused=exit_refused,
|
|
1727
|
+
exit_dead_end=exit_dead_end,
|
|
1728
|
+
)
|
|
1729
|
+
reconciles.append(reconcile)
|
|
1730
|
+
if reconcile.occupies:
|
|
1731
|
+
occupied += 1
|
|
1732
|
+
repo_occupied[entry.repo] = repo_occupied.get(entry.repo, 0) + 1
|
|
1733
|
+
new_state = reconcile.updates.get("state")
|
|
1734
|
+
if new_state:
|
|
1735
|
+
states[entry.key] = str(new_state)
|
|
1736
|
+
if block is not None:
|
|
1737
|
+
blocked.append(block)
|
|
1738
|
+
states[entry.key] = STATE_BLOCKED
|
|
1739
|
+
|
|
1740
|
+
# #1891 step 1b: re-check every `parked` entry against the CURRENT board,
|
|
1741
|
+
# independent of capacity/holds below — mirrors step 1's own `done` check
|
|
1742
|
+
# (an entry can land while parked exactly as it can while running) and,
|
|
1743
|
+
# like step 1, never spends an attempt either way. `entry.landed` wins
|
|
1744
|
+
# unconditionally over "still gated", same ordering `_reconcile_running`
|
|
1745
|
+
# uses for a `running` entry. A gate that CLEARED flips `states` straight
|
|
1746
|
+
# to `waiting` here — not `by_key`, which stays whatever DQ-1 loaded — so
|
|
1747
|
+
# it falls into the SAME step-4 walk below, on the SAME tick, exactly
|
|
1748
|
+
# like a deploy gate released in step 2 (see this function's docstring
|
|
1749
|
+
# for why that same-tick fall-through matters). A gate that is STILL
|
|
1750
|
+
# shut is left alone entirely: no reconcile, no write, nothing to
|
|
1751
|
+
# report — the parked row itself, rendered by `coord drive-queue list`/
|
|
1752
|
+
# `status`, already answers "why isn't this launching".
|
|
1753
|
+
#
|
|
1754
|
+
# #2055 extends the SAME `landed` check to `blocked` and `failed`
|
|
1755
|
+
# entries. `blocked`/`failed` are terminal for dispatch — the queue gave
|
|
1756
|
+
# up on them, and this loop must NOT resurrect them for a relaunch, the
|
|
1757
|
+
# way the `parked` branch below resumes to `waiting` on cleared CI. But
|
|
1758
|
+
# "the queue gave up" and "the work is done" are independent facts: a
|
|
1759
|
+
# human fixes a blocked/failed issue by hand and merges it out of band
|
|
1760
|
+
# exactly as often as a parked one lands while its gate is still shut.
|
|
1761
|
+
# Without this, that merge is invisible forever — `blocked`/`failed`
|
|
1762
|
+
# have no other re-check, so the board keeps reporting finished work as
|
|
1763
|
+
# outstanding until someone notices and runs
|
|
1764
|
+
# `coord drive-queue remove`. See #1956 for a live instance.
|
|
1765
|
+
for entry in ordered:
|
|
1766
|
+
if entry.state not in (STATE_PARKED, STATE_BLOCKED, STATE_FAILED):
|
|
1767
|
+
continue
|
|
1768
|
+
facts = board.facts(entry.key)
|
|
1769
|
+
if facts.landed:
|
|
1770
|
+
witness = "merged" if facts.merged else "closed"
|
|
1771
|
+
reason = f"done — issue already {witness} while {entry.state} (#2055)"
|
|
1772
|
+
reconciles.append(
|
|
1773
|
+
Reconcile(
|
|
1774
|
+
entry.key,
|
|
1775
|
+
"done",
|
|
1776
|
+
reason,
|
|
1777
|
+
occupies=False,
|
|
1778
|
+
updates={
|
|
1779
|
+
"state": STATE_DONE,
|
|
1780
|
+
"last_reason": reason,
|
|
1781
|
+
"session_name": None,
|
|
1782
|
+
},
|
|
1783
|
+
)
|
|
1784
|
+
)
|
|
1785
|
+
states[entry.key] = STATE_DONE
|
|
1786
|
+
continue
|
|
1787
|
+
if entry.state != STATE_PARKED:
|
|
1788
|
+
# `blocked`/`failed` entries are terminal for dispatch: the
|
|
1789
|
+
# landed check above is the only re-check they get. Never fall
|
|
1790
|
+
# through to the parked-only CI resume below — that would
|
|
1791
|
+
# relaunch a gave-up entry outside the `blocked`/`failed`
|
|
1792
|
+
# attempt-tracking this function's docstring describes (#2055).
|
|
1793
|
+
continue
|
|
1794
|
+
if facts.merge_ci_pending:
|
|
1795
|
+
continue
|
|
1796
|
+
# #2063: a Gate-A park is gated on a HUMAN, not on the board, so the
|
|
1797
|
+
# `merge_ci_pending` predicate above says nothing about it. Without
|
|
1798
|
+
# this branch such an entry would resume on the very next tick and
|
|
1799
|
+
# relaunch straight back into the identical refusal, forever — the
|
|
1800
|
+
# hot loop that "park, don't block" is supposed to avoid. The
|
|
1801
|
+
# shell resolves `gate_a_pending` by re-reading the recorded verdict
|
|
1802
|
+
# for the (repo, milestone) embedded in the park reason's marker (a
|
|
1803
|
+
# local board read, no `gh` call per entry); an entry it can't
|
|
1804
|
+
# resolve stays parked, which fails closed exactly like the guard.
|
|
1805
|
+
if is_gate_a_refusal_reason(entry.last_reason):
|
|
1806
|
+
if (gate_a_pending or {}).get(entry.key, True):
|
|
1807
|
+
continue
|
|
1808
|
+
reason = (
|
|
1809
|
+
f"Gate A sign-off recorded for {entry.key} — resuming from "
|
|
1810
|
+
"parked without spending an attempt (#2063)"
|
|
1811
|
+
)
|
|
1812
|
+
reconciles.append(
|
|
1813
|
+
Reconcile(
|
|
1814
|
+
entry.key,
|
|
1815
|
+
"resumed",
|
|
1816
|
+
reason,
|
|
1817
|
+
occupies=False,
|
|
1818
|
+
updates={"state": STATE_WAITING, "last_reason": reason},
|
|
1819
|
+
)
|
|
1820
|
+
)
|
|
1821
|
+
states[entry.key] = STATE_WAITING
|
|
1822
|
+
continue
|
|
1823
|
+
reason = (
|
|
1824
|
+
f"CI checks for {entry.key} have reported — resuming from "
|
|
1825
|
+
"parked without spending an attempt (#1891)"
|
|
1826
|
+
)
|
|
1827
|
+
reconciles.append(
|
|
1828
|
+
Reconcile(
|
|
1829
|
+
entry.key,
|
|
1830
|
+
"resumed",
|
|
1831
|
+
reason,
|
|
1832
|
+
occupies=False,
|
|
1833
|
+
updates={"state": STATE_WAITING, "last_reason": reason},
|
|
1834
|
+
)
|
|
1835
|
+
)
|
|
1836
|
+
states[entry.key] = STATE_WAITING
|
|
1837
|
+
|
|
1838
|
+
# #1757 step 2: deploy gates. Resolved from the POST-reconcile states, so
|
|
1839
|
+
# a `--hold-after` entry that reconciled to `blocked` cannot also fire a
|
|
1840
|
+
# gate, and `released` falls through to the walk below in this same tick.
|
|
1841
|
+
holds = _resolve_holds(ordered, states, probes or {})
|
|
1842
|
+
|
|
1843
|
+
# NOTE: "reconciles" is deliberately NOT in plan_base. The waiting-entry
|
|
1844
|
+
# walk below (#1873) can append to `reconciles` too — a `waiting` entry
|
|
1845
|
+
# whose own issue already landed reconciles to `done` there — so every
|
|
1846
|
+
# return site passes `reconciles=tuple(reconciles)` explicitly, taken at
|
|
1847
|
+
# the point of that return rather than frozen here before the walk runs.
|
|
1848
|
+
plan_base = {
|
|
1849
|
+
"holds": tuple(holds),
|
|
1850
|
+
"occupied": occupied,
|
|
1851
|
+
"capacity": capacity,
|
|
1852
|
+
# A copy, not the live dict: the walk below mutates its own projection
|
|
1853
|
+
# of these counts (it charges the launch to its repo) and the plan must
|
|
1854
|
+
# report the reading that `occupied` was taken from.
|
|
1855
|
+
"repo_occupied": dict(repo_occupied),
|
|
1856
|
+
"repo_capacity": repo_capacity,
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
gate = next((h for h in holds if h.blocking), None)
|
|
1860
|
+
if gate is not None:
|
|
1861
|
+
# Launch NOTHING. Not "launch if there is spare capacity", not
|
|
1862
|
+
# "launch anything whose pre-reqs don't mention the held entry" — the
|
|
1863
|
+
# deploy this gate is waiting on is invisible to the dependency graph,
|
|
1864
|
+
# which is exactly why an explicit operator-declared gate exists.
|
|
1865
|
+
return TickPlan(
|
|
1866
|
+
**plan_base,
|
|
1867
|
+
reconciles=tuple(reconciles),
|
|
1868
|
+
blocked=tuple(blocked),
|
|
1869
|
+
deferrals=(),
|
|
1870
|
+
alert=_hold_alert(gate),
|
|
1871
|
+
launch=None,
|
|
1872
|
+
)
|
|
1873
|
+
|
|
1874
|
+
# #2101 step 2b: is THIS host cordoned? Checked after reconciliation (so
|
|
1875
|
+
# a cordoned host still drains its view of reality — see the docstring)
|
|
1876
|
+
# and after the deploy gate (which is the older, narrower stop and keeps
|
|
1877
|
+
# its own alert), but BEFORE capacity: a cordoned host launches nothing
|
|
1878
|
+
# however many slots are free, which is the entire mechanism.
|
|
1879
|
+
cordon_map = _normalized_cordons(cordons)
|
|
1880
|
+
local_cordon = cordon_map.get(_norm_host(local_host)) if local_host else None
|
|
1881
|
+
if local_cordon:
|
|
1882
|
+
return TickPlan(
|
|
1883
|
+
**plan_base,
|
|
1884
|
+
reconciles=tuple(reconciles),
|
|
1885
|
+
blocked=tuple(blocked),
|
|
1886
|
+
deferrals=(),
|
|
1887
|
+
alert=_cordon_alert(local_host or "this host", local_cordon),
|
|
1888
|
+
launch=None,
|
|
1889
|
+
cordon_reason=local_cordon,
|
|
1890
|
+
)
|
|
1891
|
+
|
|
1892
|
+
if capacity - occupied <= 0:
|
|
1893
|
+
return TickPlan(
|
|
1894
|
+
**plan_base,
|
|
1895
|
+
reconciles=tuple(reconciles),
|
|
1896
|
+
blocked=tuple(blocked),
|
|
1897
|
+
deferrals=(),
|
|
1898
|
+
alert=None,
|
|
1899
|
+
launch=None,
|
|
1900
|
+
)
|
|
1901
|
+
|
|
1902
|
+
# Cycles are re-checked here, not just at `add` time: `remove` can leave
|
|
1903
|
+
# the surviving edges in a shape `add` never validated, and a hand-edited
|
|
1904
|
+
# DB row is always possible. A cycle makes every member unsatisfiable.
|
|
1905
|
+
cycle_keys: dict[str, str] = {}
|
|
1906
|
+
cycle = find_cycle({e.key: list(e.after) for e in ordered})
|
|
1907
|
+
if cycle is not None:
|
|
1908
|
+
message = "dependency cycle: " + " -> ".join(cycle)
|
|
1909
|
+
for key in cycle:
|
|
1910
|
+
cycle_keys[key] = message
|
|
1911
|
+
|
|
1912
|
+
def _cooldown_reason(candidate: QueueEntry) -> str:
|
|
1913
|
+
"""#1794's launch-side guard: '' unless this entry was just launched.
|
|
1914
|
+
|
|
1915
|
+
A `waiting` row carrying a recent `launched_at` means SOMETHING put a
|
|
1916
|
+
drive up for this issue moments ago — a retry decided on stale
|
|
1917
|
+
evidence, a launch subprocess whose exit code lied, an operator's hand
|
|
1918
|
+
edit. Whatever it was, starting a second `coord drive` now is the
|
|
1919
|
+
failure #1794 exists to prevent, so the entry defers and tries again
|
|
1920
|
+
on the next tick, by which point the reconcile branches above have
|
|
1921
|
+
real evidence to work with.
|
|
1922
|
+
"""
|
|
1923
|
+
age = _startup_cooldown(candidate, now, grace_seconds)
|
|
1924
|
+
if age is None:
|
|
1925
|
+
return ""
|
|
1926
|
+
return (
|
|
1927
|
+
f"launched {age:.0f}s ago — inside the {grace_seconds:.0f}s startup "
|
|
1928
|
+
f"grace window, so a second `coord drive` is refused (#1794)"
|
|
1929
|
+
)
|
|
1930
|
+
|
|
1931
|
+
# #1972's projection of per-repo occupancy AS THE WALK SEES IT: the board
|
|
1932
|
+
# reading above, plus this tick's own launch once one is chosen. Kept
|
|
1933
|
+
# separate from `repo_occupied` (reported in the plan) so the launch's own
|
|
1934
|
+
# slot is charged to the report-only pass — otherwise `--dry-run` would
|
|
1935
|
+
# cheerfully explain that the next same-repo entry is eligible, one line
|
|
1936
|
+
# under the launch that just took its repo's last slot.
|
|
1937
|
+
repo_slots: dict[str, int] = dict(repo_occupied)
|
|
1938
|
+
|
|
1939
|
+
def _repo_limit_reason(candidate: QueueEntry) -> str:
|
|
1940
|
+
"""#1972's per-repo ceiling: '' unless this entry's repo is full.
|
|
1941
|
+
|
|
1942
|
+
A DEFER, never a block: nothing is wrong with the entry, its position
|
|
1943
|
+
does not move, no attempt is spent and nothing escalates. It is the
|
|
1944
|
+
same "not yet" an unsatisfied `after` produces — the difference is only
|
|
1945
|
+
that what it is waiting on is its own repo's in-flight drive rather
|
|
1946
|
+
than a named pre-req.
|
|
1947
|
+
"""
|
|
1948
|
+
if not repo_capacity:
|
|
1949
|
+
return ""
|
|
1950
|
+
used = repo_slots.get(candidate.repo, 0)
|
|
1951
|
+
if used < repo_capacity:
|
|
1952
|
+
return ""
|
|
1953
|
+
return (
|
|
1954
|
+
f"repo {candidate.repo} at its limit ({used}/{repo_capacity}) — "
|
|
1955
|
+
"deferring so a different repo can launch"
|
|
1956
|
+
)
|
|
1957
|
+
|
|
1958
|
+
def _cordon_reason(candidate: QueueEntry) -> str:
|
|
1959
|
+
"""#2101: '' unless this entry is PINNED to a cordoned machine.
|
|
1960
|
+
|
|
1961
|
+
A DEFER, never a block: nothing is wrong with the entry, its position
|
|
1962
|
+
does not move and no attempt is spent — its destination is simply
|
|
1963
|
+
draining for a release right now and will take work again in minutes.
|
|
1964
|
+
Only an explicit ``--machine`` pin is checked here; an unpinned entry
|
|
1965
|
+
auto-picks its host at dispatch time, where `coord.drive_state`'s
|
|
1966
|
+
machine picker already skips paused machines (a cordon IS a routing
|
|
1967
|
+
pause — see `coord.machine_pause`), so guessing a destination here
|
|
1968
|
+
would be a second, weaker copy of that decision.
|
|
1969
|
+
"""
|
|
1970
|
+
if not candidate.machine:
|
|
1971
|
+
return ""
|
|
1972
|
+
reason = cordon_map.get(_norm_host(candidate.machine))
|
|
1973
|
+
if not reason:
|
|
1974
|
+
return ""
|
|
1975
|
+
return (
|
|
1976
|
+
f"{candidate.machine} is {reason} — deferring rather than "
|
|
1977
|
+
"dispatching into a host that is draining for a release (#2101)"
|
|
1978
|
+
)
|
|
1979
|
+
|
|
1980
|
+
launch: QueueEntry | None = None
|
|
1981
|
+
# #1873: keys that reconciled straight to `done` in the walk below —
|
|
1982
|
+
# landed under someone else's branch/PR, closed by hand as obsolete, or
|
|
1983
|
+
# picked up by `coord reconcile-merges` — WITHOUT this queue ever
|
|
1984
|
+
# launching them. These must not count toward the queue-level alert
|
|
1985
|
+
# below: they were neither deferred nor blocked, so they have nothing to
|
|
1986
|
+
# show up in `details`, and counting them in "considered N" without a
|
|
1987
|
+
# matching detail line is exactly the "considered N, N-1 explained"
|
|
1988
|
+
# contradiction the "considered N" comment below warns about — see the
|
|
1989
|
+
# #1864 incident this branch exists to fix, where the ENTIRE queue was
|
|
1990
|
+
# this case and the tick has nothing to be stalled about.
|
|
1991
|
+
landed_keys: set[str] = set()
|
|
1992
|
+
waiting = [e for e in ordered if states.get(e.key) == STATE_WAITING]
|
|
1993
|
+
for entry in waiting:
|
|
1994
|
+
if launch is not None:
|
|
1995
|
+
# Report-only pass over the tail of the queue. The launch above
|
|
1996
|
+
# already won this tick, so nothing here is mutated (see
|
|
1997
|
+
# Deferral.counted) — this exists so `--dry-run` explains the rest
|
|
1998
|
+
# of the queue instead of going silent after the first line.
|
|
1999
|
+
cooldown = _cooldown_reason(entry)
|
|
2000
|
+
if cooldown:
|
|
2001
|
+
deferrals.append(Deferral(entry.key, cooldown, counted=False))
|
|
2002
|
+
continue
|
|
2003
|
+
verdict = _resolve_prereqs(entry, board, states, cycle_keys)
|
|
2004
|
+
if not verdict.satisfied:
|
|
2005
|
+
deferrals.append(
|
|
2006
|
+
Deferral(entry.key, verdict.reason, counted=False)
|
|
2007
|
+
)
|
|
2008
|
+
continue
|
|
2009
|
+
cordoned = _cordon_reason(entry)
|
|
2010
|
+
if cordoned:
|
|
2011
|
+
deferrals.append(
|
|
2012
|
+
Deferral(entry.key, cordoned, counted=False, cordoned=True)
|
|
2013
|
+
)
|
|
2014
|
+
continue
|
|
2015
|
+
repo_limit = _repo_limit_reason(entry)
|
|
2016
|
+
if repo_limit:
|
|
2017
|
+
deferrals.append(
|
|
2018
|
+
Deferral(
|
|
2019
|
+
entry.key, repo_limit, counted=False, repo_limited=True
|
|
2020
|
+
)
|
|
2021
|
+
)
|
|
2022
|
+
continue
|
|
2023
|
+
cooldown = _cooldown_reason(entry)
|
|
2024
|
+
if cooldown:
|
|
2025
|
+
deferrals.append(
|
|
2026
|
+
Deferral(
|
|
2027
|
+
entry.key,
|
|
2028
|
+
cooldown,
|
|
2029
|
+
updates={
|
|
2030
|
+
"deferrals": entry.deferrals + 1,
|
|
2031
|
+
"last_reason": cooldown,
|
|
2032
|
+
},
|
|
2033
|
+
)
|
|
2034
|
+
)
|
|
2035
|
+
continue
|
|
2036
|
+
# #1873: checked BEFORE `_resolve_prereqs`, not after. The entry's
|
|
2037
|
+
# own board state is unconditional — if this issue is already landed,
|
|
2038
|
+
# its `after=` graph is irrelevant, including when that graph is
|
|
2039
|
+
# itself unsatisfiable (unknown pre-req, cycle, a pre-req that is
|
|
2040
|
+
# `blocked`/`failed`). Checking prereqs first would route a landed
|
|
2041
|
+
# entry with a broken pre-req into the BLOCKED branch below, which
|
|
2042
|
+
# escalates and demands a manual `remove && add` for an entry that
|
|
2043
|
+
# needs neither — it is already done. `_reconcile_running` catches
|
|
2044
|
+
# this same fact for entries that WERE launched (:813); a `waiting`
|
|
2045
|
+
# entry never enters that function at all, so nothing had checked the
|
|
2046
|
+
# board against the entry's own issue until now.
|
|
2047
|
+
facts = board.facts(entry.key)
|
|
2048
|
+
if facts.landed:
|
|
2049
|
+
witness = "merged" if facts.merged else "closed"
|
|
2050
|
+
reason = (
|
|
2051
|
+
f"done — issue already {witness}, never launched by this queue"
|
|
2052
|
+
)
|
|
2053
|
+
reconciles.append(
|
|
2054
|
+
Reconcile(
|
|
2055
|
+
entry.key,
|
|
2056
|
+
"done",
|
|
2057
|
+
reason,
|
|
2058
|
+
occupies=False,
|
|
2059
|
+
# attempts is deliberately NOT incremented: nothing was
|
|
2060
|
+
# ever launched for this entry, so charging it a retry
|
|
2061
|
+
# would be charging it for work that landed elsewhere
|
|
2062
|
+
# (same reasoning as the BLOCKED branch's "operator's
|
|
2063
|
+
# typo" comment just below).
|
|
2064
|
+
updates={
|
|
2065
|
+
"state": STATE_DONE,
|
|
2066
|
+
"last_reason": reason,
|
|
2067
|
+
},
|
|
2068
|
+
)
|
|
2069
|
+
)
|
|
2070
|
+
states[entry.key] = STATE_DONE
|
|
2071
|
+
landed_keys.add(entry.key)
|
|
2072
|
+
continue
|
|
2073
|
+
verdict = _resolve_prereqs(entry, board, states, cycle_keys)
|
|
2074
|
+
if verdict.unsatisfiable:
|
|
2075
|
+
blocked.append(
|
|
2076
|
+
Blocked(
|
|
2077
|
+
entry.key,
|
|
2078
|
+
verdict.reason,
|
|
2079
|
+
# attempts is deliberately NOT incremented: nothing was
|
|
2080
|
+
# ever launched for this entry, so charging it a retry
|
|
2081
|
+
# would be charging it for the operator's typo.
|
|
2082
|
+
updates={
|
|
2083
|
+
"state": STATE_BLOCKED,
|
|
2084
|
+
"last_reason": verdict.reason,
|
|
2085
|
+
},
|
|
2086
|
+
)
|
|
2087
|
+
)
|
|
2088
|
+
states[entry.key] = STATE_BLOCKED
|
|
2089
|
+
continue
|
|
2090
|
+
if not verdict.satisfied:
|
|
2091
|
+
deferrals.append(
|
|
2092
|
+
Deferral(
|
|
2093
|
+
entry.key,
|
|
2094
|
+
verdict.reason,
|
|
2095
|
+
updates={
|
|
2096
|
+
"deferrals": entry.deferrals + 1,
|
|
2097
|
+
"last_reason": verdict.reason,
|
|
2098
|
+
},
|
|
2099
|
+
)
|
|
2100
|
+
)
|
|
2101
|
+
continue
|
|
2102
|
+
# #2101, checked with the same "facts about the ENTRY come first"
|
|
2103
|
+
# rule #1972 states below: a landed entry still reconciles to `done`
|
|
2104
|
+
# and a broken pre-req still blocks and escalates, whatever its
|
|
2105
|
+
# pinned machine's cordon is doing this tick. Only the LAUNCH is
|
|
2106
|
+
# withheld.
|
|
2107
|
+
cordoned = _cordon_reason(entry)
|
|
2108
|
+
if cordoned:
|
|
2109
|
+
deferrals.append(
|
|
2110
|
+
Deferral(
|
|
2111
|
+
entry.key,
|
|
2112
|
+
cordoned,
|
|
2113
|
+
updates={
|
|
2114
|
+
"deferrals": entry.deferrals + 1,
|
|
2115
|
+
"last_reason": cordoned,
|
|
2116
|
+
},
|
|
2117
|
+
# Same posture as #1972's repo limit: this is the fleet
|
|
2118
|
+
# working as designed (a host draining so it can be
|
|
2119
|
+
# rolled), not a stalled queue, so it must not raise the
|
|
2120
|
+
# queue-level alert every tick for the duration of a
|
|
2121
|
+
# drain. The cordon has its OWN alert when it is THIS
|
|
2122
|
+
# host that is stopped — see `_cordon_alert`.
|
|
2123
|
+
cordoned=True,
|
|
2124
|
+
)
|
|
2125
|
+
)
|
|
2126
|
+
continue
|
|
2127
|
+
# #1972, checked LAST: everything above is a fact about the ENTRY (is
|
|
2128
|
+
# it still starting, has it already landed, are its pre-reqs sound),
|
|
2129
|
+
# and those verdicts must not change because some unrelated drive in
|
|
2130
|
+
# the same repo happens to be up. In particular an unsatisfiable
|
|
2131
|
+
# pre-req still blocks and escalates here rather than hiding behind a
|
|
2132
|
+
# repo-limit deferral that would silently postpone it forever.
|
|
2133
|
+
repo_limit = _repo_limit_reason(entry)
|
|
2134
|
+
if repo_limit:
|
|
2135
|
+
deferrals.append(
|
|
2136
|
+
Deferral(
|
|
2137
|
+
entry.key,
|
|
2138
|
+
repo_limit,
|
|
2139
|
+
updates={
|
|
2140
|
+
"deferrals": entry.deferrals + 1,
|
|
2141
|
+
"last_reason": repo_limit,
|
|
2142
|
+
},
|
|
2143
|
+
repo_limited=True,
|
|
2144
|
+
)
|
|
2145
|
+
)
|
|
2146
|
+
continue
|
|
2147
|
+
launch = by_key[entry.key]
|
|
2148
|
+
# The launch takes its repo's slot for the rest of THIS walk, so the
|
|
2149
|
+
# report-only tail explains the remaining same-repo entries correctly.
|
|
2150
|
+
repo_slots[launch.repo] = repo_slots.get(launch.repo, 0) + 1
|
|
2151
|
+
|
|
2152
|
+
alert: QueueAlert | None = None
|
|
2153
|
+
# `waiting`, minus anything the walk above reconciled straight to `done`
|
|
2154
|
+
# (#1873) — those were never deferred or blocked, so they have no line in
|
|
2155
|
+
# `details` and must not be counted as "considered" either. What is left
|
|
2156
|
+
# is exactly the set of entries that are genuinely still waiting: deferred
|
|
2157
|
+
# or blocked, each with a matching `details` entry.
|
|
2158
|
+
still_waiting = [e for e in waiting if e.key not in landed_keys]
|
|
2159
|
+
# #1972: minus anything whose ONLY reason for standing still is that its
|
|
2160
|
+
# own repo is busy. That is the queue doing its job — the same condition
|
|
2161
|
+
# the global at-capacity return above answers with `alert=None` — and a
|
|
2162
|
+
# 39-entry single-repo queue would otherwise escalate on every tick for the
|
|
2163
|
+
# duration of the batch. A MIXED tick still alerts: if even one entry is
|
|
2164
|
+
# deferred on a pre-req or blocked outright, something really is stuck and
|
|
2165
|
+
# the alert names all of it, repo-limit lines included.
|
|
2166
|
+
# #2101 adds the release cordon to that same set: an entry pinned to a
|
|
2167
|
+
# host that is draining for a release is waiting on the fleet working, not
|
|
2168
|
+
# on something wedged. See `Deferral.benign`.
|
|
2169
|
+
benign_keys = {item.key for item in deferrals if item.benign}
|
|
2170
|
+
stalled = [e for e in still_waiting if e.key not in benign_keys]
|
|
2171
|
+
if launch is None and stalled:
|
|
2172
|
+
details = [f"{item.key}: {item.reason}" for item in deferrals]
|
|
2173
|
+
details += [f"{item.key}: BLOCKED — {item.reason}" for item in blocked]
|
|
2174
|
+
alert = QueueAlert(
|
|
2175
|
+
# "considered N" rather than "N waiting": some of those entries are
|
|
2176
|
+
# blocked by the time this line is written, and an alert that
|
|
2177
|
+
# contradicts `coord drive-queue status` two lines below it is an
|
|
2178
|
+
# alert operators learn to distrust.
|
|
2179
|
+
reason=(
|
|
2180
|
+
f"nothing eligible to launch: considered {len(still_waiting)} "
|
|
2181
|
+
f"waiting entr{'y' if len(still_waiting) == 1 else 'ies'}, "
|
|
2182
|
+
f"{capacity - occupied} free slot(s)"
|
|
2183
|
+
),
|
|
2184
|
+
details=tuple(details),
|
|
2185
|
+
)
|
|
2186
|
+
|
|
2187
|
+
return TickPlan(
|
|
2188
|
+
**plan_base,
|
|
2189
|
+
reconciles=tuple(reconciles),
|
|
2190
|
+
blocked=tuple(blocked),
|
|
2191
|
+
deferrals=tuple(deferrals),
|
|
2192
|
+
alert=alert,
|
|
2193
|
+
launch=launch,
|
|
2194
|
+
)
|
|
2195
|
+
|
|
2196
|
+
|
|
2197
|
+
# ── rendering (pure, so `--dry-run` is testable without a CLI) ───────────────
|
|
2198
|
+
|
|
2199
|
+
|
|
2200
|
+
def render_plan(plan: TickPlan, *, dry_run: bool = False) -> list[str]:
|
|
2201
|
+
"""The human-readable form of a :class:`TickPlan`, one line per element."""
|
|
2202
|
+
prefix = "would " if dry_run else ""
|
|
2203
|
+
lines = [
|
|
2204
|
+
f"capacity: {plan.occupied}/{plan.capacity} occupied, "
|
|
2205
|
+
f"{plan.free_slots} free"
|
|
2206
|
+
]
|
|
2207
|
+
if plan.repo_capacity:
|
|
2208
|
+
# #1972: "1/3 occupied" alone cannot answer "so why didn't item 2 go?"
|
|
2209
|
+
# — the answer is per-repo, so print the breakdown rather than making
|
|
2210
|
+
# the operator read the code. The provenance is spelled out because
|
|
2211
|
+
# this counter inherits rule 1 (board state, not live sessions): a
|
|
2212
|
+
# drive whose observer died still holds its repo's slot, and after
|
|
2213
|
+
# #1972 that wedges ONE repo instead of the whole queue, which is
|
|
2214
|
+
# better but also much quieter.
|
|
2215
|
+
detail = ", ".join(
|
|
2216
|
+
f"{repo} {count}/{plan.repo_capacity}"
|
|
2217
|
+
for repo, count in sorted(plan.repo_occupied.items())
|
|
2218
|
+
)
|
|
2219
|
+
lines.append(
|
|
2220
|
+
f" per-repo: {detail or 'no repo occupied'} (limit "
|
|
2221
|
+
f"{plan.repo_capacity}/repo, counted from board state — a drive "
|
|
2222
|
+
"whose observer died still holds its repo's slot)"
|
|
2223
|
+
)
|
|
2224
|
+
for item in plan.reconciles:
|
|
2225
|
+
lines.append(f" reconcile {item.key}: {item.outcome} — {item.reason}")
|
|
2226
|
+
# #1757: the gate line goes directly under its reconcile, because "1753
|
|
2227
|
+
# done" immediately followed by "and therefore nothing launches" is the
|
|
2228
|
+
# sentence an operator reading a timer log needs to read as one thought.
|
|
2229
|
+
for item in plan.holds:
|
|
2230
|
+
probe = ""
|
|
2231
|
+
if item.resume_when:
|
|
2232
|
+
probe = f" [resume-when: {item.resume_when}"
|
|
2233
|
+
if item.probes:
|
|
2234
|
+
probe += f", {item.probes} failed attempt(s)"
|
|
2235
|
+
if item.probe_detail:
|
|
2236
|
+
probe += f" — {item.probe_detail}"
|
|
2237
|
+
probe += "]"
|
|
2238
|
+
lines.append(f" hold {item.key}: {item.outcome} — {item.reason}{probe}")
|
|
2239
|
+
for item in plan.blocked:
|
|
2240
|
+
lines.append(f" {prefix}block {item.key}: {item.reason}")
|
|
2241
|
+
# Counted deferrals come BEFORE the launch line and report-only ones after,
|
|
2242
|
+
# so the output reads in the order the walk actually happened: these lost
|
|
2243
|
+
# their turn while a slot was free; that one took it; the rest were never
|
|
2244
|
+
# reached.
|
|
2245
|
+
for item in plan.deferrals:
|
|
2246
|
+
if item.counted:
|
|
2247
|
+
lines.append(f" defer {item.key}: {item.reason}")
|
|
2248
|
+
if plan.launch is not None:
|
|
2249
|
+
target = plan.launch
|
|
2250
|
+
pinned = f" on {target.machine}" if target.machine else ""
|
|
2251
|
+
lines.append(f" {prefix}launch {target.key}{pinned}")
|
|
2252
|
+
elif plan.held is not None:
|
|
2253
|
+
lines.append(
|
|
2254
|
+
f" no launch — HELD by the deploy gate on {plan.held.key} "
|
|
2255
|
+
f"(release with `coord drive-queue resume`)"
|
|
2256
|
+
)
|
|
2257
|
+
elif plan.cordon_reason:
|
|
2258
|
+
# #2101 trap E: naming the cordon here is the difference between a
|
|
2259
|
+
# journal that reads "the fleet is upgrading itself" and one that
|
|
2260
|
+
# reads "the queue mysteriously stopped".
|
|
2261
|
+
lines.append(
|
|
2262
|
+
f" no launch — this host is {plan.cordon_reason}; in-flight "
|
|
2263
|
+
"drives are draining and the queue resumes once it is rolled"
|
|
2264
|
+
)
|
|
2265
|
+
elif plan.capacity and plan.free_slots == 0:
|
|
2266
|
+
# Naming the reason matters more here than anywhere else in this
|
|
2267
|
+
# render: #1794 was diagnosed entirely from a journal, and "no launch"
|
|
2268
|
+
# on its own is indistinguishable from a stalled queue.
|
|
2269
|
+
lines.append(
|
|
2270
|
+
f" no launch — at capacity ({plan.occupied}/{plan.capacity} occupied)"
|
|
2271
|
+
)
|
|
2272
|
+
elif plan.deferrals and all(item.benign for item in plan.deferrals):
|
|
2273
|
+
# Same reasoning as the at-capacity line above: with free GLOBAL slots
|
|
2274
|
+
# and no launch, a bare "no launch" reads as a stalled queue in a
|
|
2275
|
+
# journal. This one is saturated per repo (or draining for a release
|
|
2276
|
+
# — #2101), not stalled, and unlike the global case it raises no
|
|
2277
|
+
# alert, so this line is the only place it is ever said.
|
|
2278
|
+
if any(item.cordoned for item in plan.deferrals):
|
|
2279
|
+
lines.append(
|
|
2280
|
+
" no launch — every waiting entry is pinned to a machine "
|
|
2281
|
+
"under a release cordon (draining to be rolled)"
|
|
2282
|
+
)
|
|
2283
|
+
else:
|
|
2284
|
+
lines.append(
|
|
2285
|
+
f" no launch — every waiting entry's repo is at its per-repo "
|
|
2286
|
+
f"limit ({plan.repo_capacity}/repo)"
|
|
2287
|
+
)
|
|
2288
|
+
else:
|
|
2289
|
+
lines.append(" no launch")
|
|
2290
|
+
for item in plan.deferrals:
|
|
2291
|
+
if not item.counted:
|
|
2292
|
+
lines.append(
|
|
2293
|
+
f" defer {item.key}: {item.reason} (not reached this tick)"
|
|
2294
|
+
)
|
|
2295
|
+
if plan.alert is not None:
|
|
2296
|
+
lines.append(f" {prefix}alert: {plan.alert.reason}")
|
|
2297
|
+
lines.extend(f" {detail}" for detail in plan.alert.details)
|
|
2298
|
+
return lines
|