code-coordinator 0.5.332__py3-none-any.whl → 0.5.334__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {code_coordinator-0.5.332.dist-info → code_coordinator-0.5.334.dist-info}/METADATA +1 -1
- {code_coordinator-0.5.332.dist-info → code_coordinator-0.5.334.dist-info}/RECORD +11 -11
- coord/dashboard/index.html +6 -6
- coord/dashboard/server.py +137 -16
- coord/health/fleet_snapshot.py +48 -0
- coord/machine_metrics.py +173 -0
- coord/serve_app.py +138 -4
- {code_coordinator-0.5.332.dist-info → code_coordinator-0.5.334.dist-info}/WHEEL +0 -0
- {code_coordinator-0.5.332.dist-info → code_coordinator-0.5.334.dist-info}/entry_points.txt +0 -0
- {code_coordinator-0.5.332.dist-info → code_coordinator-0.5.334.dist-info}/licenses/LICENSE +0 -0
- {code_coordinator-0.5.332.dist-info → code_coordinator-0.5.334.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
code_coordinator-0.5.
|
|
1
|
+
code_coordinator-0.5.334.dist-info/licenses/LICENSE,sha256=851BVZZwf_pEv2ydSlMmcZrwnwSS5YJ98wdLze7lyL8,4212
|
|
2
2
|
coord/__init__.py,sha256=3FN2qrrxtW87xSsSD557YcMwHLDahnFK8LvqQI7Wlf4,9783
|
|
3
3
|
coord/_board_mapping.py,sha256=SGq8hC-gsyXHEZe2L58qy6pyOVQvswTpQ5FzPa_vZ8w,11012
|
|
4
4
|
coord/acceptance.py,sha256=v7k1e_shUIjWKolKDQ9oJv67o0O8L659El3grvcF8ns,75452
|
|
@@ -63,7 +63,7 @@ coord/interactive.py,sha256=AEK-5S2aMknpP6k3Xx-aBuTEulw0_c-gbYPEKTwUjRE,219083
|
|
|
63
63
|
coord/issue_store.py,sha256=abywea0vpXE_BZ5F3QHLDxLKYKDYF7iQYabZhppVYto,69451
|
|
64
64
|
coord/issues_sync_status.py,sha256=_6ucx7oW3TUGZZvs93wRbzR_SUQcNlz-7RptOzr4alo,9262
|
|
65
65
|
coord/liveness_auditor.py,sha256=sf_xBklt37DUnrIEeD4fVp4_gWPDaXbydZgfuWAPFRE,12478
|
|
66
|
-
coord/machine_metrics.py,sha256=
|
|
66
|
+
coord/machine_metrics.py,sha256=paWqBTZgeIi7w1pgH5QSJL59UxwetxE8bXblzzcspSI,16300
|
|
67
67
|
coord/machine_onboard.py,sha256=BTUAC7bdERcDyaIvO2yKkQ8xPwsdR063yHUrOznmHWk,58555
|
|
68
68
|
coord/machine_pause.py,sha256=6s7Dtr5zLs1qB2NWI2-cfbMdPyigK62FTtLjzmY6slE,47521
|
|
69
69
|
coord/merge_queue.py,sha256=UdwyoyfE9DplS6Mia4_aY6TK5a-f933M04Ns5mVHnsw,392677
|
|
@@ -108,7 +108,7 @@ coord/revalidate.py,sha256=fQaYCvObqUawSZWMEOkVWFtYswz9PQQwSMApAyP5zeA,66266
|
|
|
108
108
|
coord/review.py,sha256=hsY6HUif44HL3pY52H7SoqNuMMZzA_9gfHfJKnC-fWE,183282
|
|
109
109
|
coord/scorecard.py,sha256=axe_doBJBBFWtN21JZDRSijwDI0xaZrt2oVYbwJPW7U,20211
|
|
110
110
|
coord/self_health.py,sha256=9PLVdaEReFLUY6y2QXMGVd6U1CdHAh9_8MzduwBOzAY,10651
|
|
111
|
-
coord/serve_app.py,sha256=
|
|
111
|
+
coord/serve_app.py,sha256=8xveD7Y9zmP-J-dIE6GcJJJl0CVMIubIZqJFojrbIw0,516163
|
|
112
112
|
coord/smoke.py,sha256=INLh9Wu57WOcCoOY4CMOcbzArIhMxMolsV2D8hAW7zw,70481
|
|
113
113
|
coord/spend_ceiling.py,sha256=hfuiXlnD4or7Wqoqf6BHcOMT2CXd92FBZuVdnQVlI2M,13573
|
|
114
114
|
coord/split_work.py,sha256=Gp_cNtYBe7nDxBN7cDRs8CJG3V_Y0C1QUi35fHGcvnY,8502
|
|
@@ -163,8 +163,8 @@ coord/commands/test_gate.py,sha256=8y_u-otEXUIBoY0m6gzcqdOq78cbG4lSW0o1N0u1z9Y,3
|
|
|
163
163
|
coord/commands/tui.py,sha256=GX4LsgMk08itlURCgkOKqWgjnv38uIfFSWM3xrz9T_A,18193
|
|
164
164
|
coord/dashboard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
165
165
|
coord/dashboard/fixture.py,sha256=YQrm_EvpsddqAa8d5fT7iEbUXLH2KQNxEST5NA1NgpA,18941
|
|
166
|
-
coord/dashboard/index.html,sha256=
|
|
167
|
-
coord/dashboard/server.py,sha256=
|
|
166
|
+
coord/dashboard/index.html,sha256=XzipoURZAP46Tggas-pSMq8Fb1ETX9TyAJy2ItNVigc,31463
|
|
167
|
+
coord/dashboard/server.py,sha256=rzygqs9P7D8Mu9f0LtqS8fcAeDNutYjmbvmLG6CVIhc,142424
|
|
168
168
|
coord/dashboard/terminal.py,sha256=wR5rfWQCZ0yO5bXCkqa0t-a1g5YAQ-20LC6icZdGWHM,14647
|
|
169
169
|
coord/deploy/README.md,sha256=DG_q36KeW0LQ32ElU11wjfjULO7xGUoiPIFEGPW41k0,1391
|
|
170
170
|
coord/deploy/coord-agent.service,sha256=GSo_2iqCx4zv4-85LGnaINsu-BdPHpG64I8xfULKcUk,6392
|
|
@@ -190,7 +190,7 @@ coord/health/__init__.py,sha256=fvda5KAAaycBk7kXcNm_7wkLm28BTU3_TjyF6Nr3DnA,1990
|
|
|
190
190
|
coord/health/aggregate.py,sha256=MFPEe5JiS5gRsD0uc4zXAg86YhqnKFvEeSa1e9RAUnY,5694
|
|
191
191
|
coord/health/cli.py,sha256=CYMWO_UsjpmjvVOSk3AuRa26zmfIEiz8_pnx_QvFxYQ,5931
|
|
192
192
|
coord/health/context.py,sha256=pbc5qq-1DMbgjsFKTR52hg-igHtK7HazGLq-LTgzwuU,4060
|
|
193
|
-
coord/health/fleet_snapshot.py,sha256=
|
|
193
|
+
coord/health/fleet_snapshot.py,sha256=H0oqWFsdVw6vWW35P7w84pG_4n9aA1zzVkjJsA4Qh1Y,25595
|
|
194
194
|
coord/health/models.py,sha256=h-rT3SqYjaM5obrldbhMN5BC97WKBVvN-4wDxFCM890,11516
|
|
195
195
|
coord/health/pypi.py,sha256=L1CDtuWYrmaF_XGKiHh2R9PkDQNMjBdTxAWIpWZpS4c,10409
|
|
196
196
|
coord/health/registry.py,sha256=TLzPSY-HCkKGWRamJdOkPc9A-2EYmQxTqS4n-kr3cqg,15713
|
|
@@ -249,8 +249,8 @@ coord/skills/release-now/SKILL.md,sha256=u5JY0Jm0IiZYs2JGjRSqNj265CUWcIA6pMaeulb
|
|
|
249
249
|
coord/skills/review-verdict-recovery/SKILL.md,sha256=Grg8GWkJLK20ct2YiigbTFfNOkit7tNrdRIhdocKGJk,3941
|
|
250
250
|
coord/skills/tui-quadraui-workflow/SKILL.md,sha256=4hUZza9wSSnPhf0eRAOh2r_MQZxK2pXYO5g-kxmJq-U,3832
|
|
251
251
|
coord/skills/update-issue/SKILL.md,sha256=VrPinEmLZrCSNDrnz96hQomyO403o5c4i0RdsnUnWA0,3642
|
|
252
|
-
code_coordinator-0.5.
|
|
253
|
-
code_coordinator-0.5.
|
|
254
|
-
code_coordinator-0.5.
|
|
255
|
-
code_coordinator-0.5.
|
|
256
|
-
code_coordinator-0.5.
|
|
252
|
+
code_coordinator-0.5.334.dist-info/METADATA,sha256=a8GsHRhSqTf55gDNiGuVjRuRQyQrOhRyOStbDDLkjMs,27178
|
|
253
|
+
code_coordinator-0.5.334.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
254
|
+
code_coordinator-0.5.334.dist-info/entry_points.txt,sha256=o-Hq8pvG7ReGePvN_4iOKjLNqiq-MJ43HbPurqjNPZo,41
|
|
255
|
+
code_coordinator-0.5.334.dist-info/top_level.txt,sha256=w5sTrXesTCebhXyYv2qufOZGo0PoD1Lxgn0Q3MAXjh4,6
|
|
256
|
+
code_coordinator-0.5.334.dist-info/RECORD,,
|
coord/dashboard/index.html
CHANGED
|
@@ -521,12 +521,12 @@ async function loadMachines() {
|
|
|
521
521
|
const a = m.assignments.active[0];
|
|
522
522
|
const spec = a.spec || {};
|
|
523
523
|
detail = `<div class="meta">${badge('busy', 'busy')} #${spec.issue_number}: ${E(spec.issue_title || '')}</div>`;
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
524
|
+
// #3023: GET /api/machines no longer carries `a.progress` (live
|
|
525
|
+
// STATUS:/STUCK: tail of the owning machine's own log file) — that
|
|
526
|
+
// data only ever existed via a fresh per-agent probe, and serving it
|
|
527
|
+
// here would reintroduce the exact per-request fleet fan-out this
|
|
528
|
+
// endpoint was reworked to remove. Use `coord status` / `coord log
|
|
529
|
+
// <id> -f` for live per-worker progress instead.
|
|
530
530
|
}
|
|
531
531
|
return `
|
|
532
532
|
<div class="card">
|
coord/dashboard/server.py
CHANGED
|
@@ -51,7 +51,6 @@ from coord.drive_queue import (
|
|
|
51
51
|
summarize_drive_queue,
|
|
52
52
|
)
|
|
53
53
|
from coord.models import Assignment
|
|
54
|
-
from coord.network import check_all, fetch_status
|
|
55
54
|
from coord.openapi import build_spec, dataclass_schema, openapi_and_docs_routes
|
|
56
55
|
from coord.pipeline import PipelineView
|
|
57
56
|
from coord.state import list_drive_queue, load_proposals
|
|
@@ -1142,6 +1141,94 @@ def build_app(
|
|
|
1142
1141
|
return
|
|
1143
1142
|
write_board(board)
|
|
1144
1143
|
|
|
1144
|
+
def _read_board_and_machine_health() -> tuple: # -> tuple[Board, dict[str, dict]]
|
|
1145
|
+
"""The board plus every configured machine's latest daemon-tick-
|
|
1146
|
+
refreshed health row, keyed by machine name (#3023).
|
|
1147
|
+
|
|
1148
|
+
Backs ``GET /api/machines``. Fetches both in ONE daemon round trip
|
|
1149
|
+
when this dashboard is a thin client (``board_service`` configured)
|
|
1150
|
+
instead of two independent ``GET /board`` calls — the review found
|
|
1151
|
+
that ``_read_machine_health()`` fetching its own payload and then
|
|
1152
|
+
``_active_assignments_by_machine(_read_board())`` fetching a SECOND
|
|
1153
|
+
one doubled this endpoint's daemon I/O per client poll, which is
|
|
1154
|
+
exactly the per-request amplification #3023 exists to eliminate
|
|
1155
|
+
(just at a smaller scale than the fan-out it replaced). A thin
|
|
1156
|
+
client reads the daemon-published payload's
|
|
1157
|
+
``fleet_health.machine_health`` sibling key — the tick-refreshed
|
|
1158
|
+
output of ``coord.health.fleet_snapshot.FleetHealthRefresher``
|
|
1159
|
+
(#1630) a board poll already carries — never a fresh per-request
|
|
1160
|
+
probe of the fleet. A dashboard co-located with the daemon (no
|
|
1161
|
+
``board_service``) has no network cost to share in the first place:
|
|
1162
|
+
it reads the board from the local DB and the health rows straight
|
|
1163
|
+
out of the same local DB via ``coord.state.load_machine_health`` +
|
|
1164
|
+
``coord.health.fleet_snapshot.machine_health_rows`` (the public
|
|
1165
|
+
wrapper ``FleetHealthRefresher.refresh()`` itself calls right after
|
|
1166
|
+
persisting) — so the two modes can't drift.
|
|
1167
|
+
|
|
1168
|
+
A machine absent from the health dict was never polled (fresh
|
|
1169
|
+
install, or nothing has ticked the health refresher yet) — callers
|
|
1170
|
+
must treat that the same as an ``unknown`` state, never as healthy
|
|
1171
|
+
(#1485's failure mode).
|
|
1172
|
+
"""
|
|
1173
|
+
from coord import board_service # noqa: PLC0415
|
|
1174
|
+
|
|
1175
|
+
svc = board_service.resolve()
|
|
1176
|
+
if svc is not None:
|
|
1177
|
+
from coord.client import board_from_payload, fetch_board_payload # noqa: PLC0415
|
|
1178
|
+
|
|
1179
|
+
payload = fetch_board_payload(svc)
|
|
1180
|
+
board = board_from_payload(payload)
|
|
1181
|
+
rows = (payload.get("fleet_health") or {}).get("machine_health") or []
|
|
1182
|
+
return board, {row["machine"]: row for row in rows}
|
|
1183
|
+
|
|
1184
|
+
from coord.health.fleet_snapshot import machine_health_rows # noqa: PLC0415
|
|
1185
|
+
from coord.state import load_machine_health # noqa: PLC0415
|
|
1186
|
+
|
|
1187
|
+
board = _read_board()
|
|
1188
|
+
raw = load_machine_health()
|
|
1189
|
+
machine_names = [m.name for m in config.machines]
|
|
1190
|
+
rows = machine_health_rows(machine_names, raw, now=time.time())
|
|
1191
|
+
return board, {row["machine"]: row for row in rows}
|
|
1192
|
+
|
|
1193
|
+
def _active_assignments_by_machine(board) -> dict[str, list[dict]]: # noqa: ANN001
|
|
1194
|
+
"""Group *board*'s RUNNING active assignments by machine (#3023).
|
|
1195
|
+
|
|
1196
|
+
Backs the legacy dashboard's per-machine "busy" card
|
|
1197
|
+
(``coord/dashboard/index.html``'s ``loadMachines()``), which used
|
|
1198
|
+
to come from a live per-agent ``GET /status`` probe. The board
|
|
1199
|
+
already carries the same fact — which issue a machine is currently
|
|
1200
|
+
running — from the normal board read path, so grouping it here
|
|
1201
|
+
costs no extra daemon I/O beyond what ``_read_board_and_machine_health()``
|
|
1202
|
+
already fetches.
|
|
1203
|
+
|
|
1204
|
+
Filters to ``status == "running"`` exactly like
|
|
1205
|
+
``Board.idle_machines()`` and ``Board.active_files_by_repo()`` in
|
|
1206
|
+
``coord/models.py`` do before treating an active-list entry as real,
|
|
1207
|
+
in-progress work — an unfiltered read of ``board.active`` would be a
|
|
1208
|
+
second, independent answer to "is this machine busy" that could
|
|
1209
|
+
diverge from those two (epic #2096 "one question, one answer").
|
|
1210
|
+
Every ``board.active.append()`` call site happens to set
|
|
1211
|
+
``status="running"`` at append time today, and reconcile removes
|
|
1212
|
+
non-running entries the same tick it retags them, so this filter is
|
|
1213
|
+
currently a no-op in practice — which is exactly why the existing
|
|
1214
|
+
code defends against it twice already rather than trusting that
|
|
1215
|
+
invariant to hold forever.
|
|
1216
|
+
"""
|
|
1217
|
+
out: dict[str, list[dict]] = {}
|
|
1218
|
+
for a in board.active:
|
|
1219
|
+
if a.status != "running":
|
|
1220
|
+
continue
|
|
1221
|
+
out.setdefault(a.machine_name, []).append({
|
|
1222
|
+
"assignment_id": a.assignment_id,
|
|
1223
|
+
"status": a.status,
|
|
1224
|
+
"spec": {
|
|
1225
|
+
"issue_number": a.issue_number,
|
|
1226
|
+
"issue_title": a.issue_title,
|
|
1227
|
+
"repo_name": a.repo_name,
|
|
1228
|
+
},
|
|
1229
|
+
})
|
|
1230
|
+
return out
|
|
1231
|
+
|
|
1145
1232
|
def _read_drive_queue() -> list[dict]:
|
|
1146
1233
|
"""Every drive-queue row — seeded fixture or the daemon/local DB (#2428).
|
|
1147
1234
|
|
|
@@ -1350,27 +1437,61 @@ def build_app(
|
|
|
1350
1437
|
})
|
|
1351
1438
|
|
|
1352
1439
|
async def api_machines(request: Request) -> JSONResponse:
|
|
1440
|
+
"""GET /api/machines — the fleet panel, from daemon-refreshed state (#3023).
|
|
1441
|
+
|
|
1442
|
+
Used to do a synchronous fan-out probe on every request: ``GET
|
|
1443
|
+
/health`` against every configured machine, then ``GET /status``
|
|
1444
|
+
against every one that answered — seconds of worst-case latency,
|
|
1445
|
+
multiplied by every connected client polling this panel live. Now
|
|
1446
|
+
serves the SAME state the board daemon's tick loop already
|
|
1447
|
+
refreshes on its own cadence, plus the board this dashboard reads
|
|
1448
|
+
for every other panel anyway, both from ONE combined read
|
|
1449
|
+
(``_read_board_and_machine_health()``) — zero per-request network
|
|
1450
|
+
fan-out to the fleet, whether this dashboard is co-located with the
|
|
1451
|
+
daemon or a thin client of it.
|
|
1452
|
+
|
|
1453
|
+
Design decision (#3023 review): each active assignment's ``spec``
|
|
1454
|
+
(issue/repo) is included, but the live ``STATUS:``/``STUCK:``
|
|
1455
|
+
per-worker progress tail (``a.progress.updates`` / ``a.progress.stuck``
|
|
1456
|
+
in the pre-#3023 shape) is NOT. That data only ever existed as a
|
|
1457
|
+
fresh tail-read of the OWNING machine's own log file — see
|
|
1458
|
+
``AgentServer.progress()``/``list_assignments()`` in ``coord/agent.py``
|
|
1459
|
+
("only this machine can see its own log file — the coordinator
|
|
1460
|
+
cannot stat a remote path") — so serving it here would mean putting
|
|
1461
|
+
the exact per-request fan-out probe this endpoint exists to remove
|
|
1462
|
+
right back in, just hidden one level down. ``coord/dashboard/index.html``
|
|
1463
|
+
no longer reads those two keys; live per-worker progress is still
|
|
1464
|
+
available via ``coord status`` / ``coord log <id> -f``, which DO pay
|
|
1465
|
+
for a targeted probe of the one machine in question rather than
|
|
1466
|
+
every machine on every dashboard poll.
|
|
1467
|
+
"""
|
|
1353
1468
|
if _fixture is not None:
|
|
1354
1469
|
# Seeded reachability — never probe the fleet in fixture mode.
|
|
1355
1470
|
return JSONResponse(_fixture.machines())
|
|
1356
|
-
|
|
1471
|
+
board, health_by_name = _read_board_and_machine_health()
|
|
1472
|
+
assignments_by_machine = _active_assignments_by_machine(board)
|
|
1357
1473
|
result = []
|
|
1358
|
-
for
|
|
1474
|
+
for m in config.machines:
|
|
1475
|
+
row = health_by_name.get(m.name) or {}
|
|
1359
1476
|
machine_data = {
|
|
1360
|
-
"name":
|
|
1361
|
-
"host":
|
|
1362
|
-
"repos":
|
|
1363
|
-
"state":
|
|
1364
|
-
"reason":
|
|
1365
|
-
"latency_ms":
|
|
1477
|
+
"name": m.name,
|
|
1478
|
+
"host": m.host,
|
|
1479
|
+
"repos": m.repos,
|
|
1480
|
+
"state": row.get("state", "unknown"),
|
|
1481
|
+
"reason": row.get("reason", ""),
|
|
1482
|
+
"latency_ms": row.get("latency_ms"),
|
|
1483
|
+
# #3023: agent's own running version — compared against
|
|
1484
|
+
# this coordinator's local version by consumers (coord-tui's
|
|
1485
|
+
# `machine_detail_list` shows it red on a mismatch) — and
|
|
1486
|
+
# total on-disk size of this agent's git worktrees. Both
|
|
1487
|
+
# ride the same tick-refreshed health blob as everything
|
|
1488
|
+
# else here; see `coord.health.fleet_snapshot.refresh`.
|
|
1489
|
+
"agent_version": row.get("agent_runtime_version"),
|
|
1490
|
+
"worktree_bytes": row.get("worktree_bytes"),
|
|
1366
1491
|
}
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
machine_data["assignments"] = status_result.data
|
|
1371
|
-
else:
|
|
1372
|
-
machine_data["assignments"] = None
|
|
1373
|
-
machine_data["status_error"] = status_result.error
|
|
1492
|
+
active = assignments_by_machine.get(m.name)
|
|
1493
|
+
if active:
|
|
1494
|
+
machine_data["assignments"] = {"active": active}
|
|
1374
1495
|
result.append(machine_data)
|
|
1375
1496
|
return JSONResponse(result)
|
|
1376
1497
|
|
coord/health/fleet_snapshot.py
CHANGED
|
@@ -209,6 +209,19 @@ def _effective_severity(entry: dict, *, now: float) -> tuple[str, bool]:
|
|
|
209
209
|
return (severity or "unknown"), stale
|
|
210
210
|
|
|
211
211
|
|
|
212
|
+
def machine_health_rows(machine_names: list[str], raw: dict, *, now: float) -> list[dict]:
|
|
213
|
+
"""Public wrapper around :func:`_machine_health_rows` (#3023).
|
|
214
|
+
|
|
215
|
+
``coord.dashboard.server`` reads this from outside the module (the
|
|
216
|
+
local-mode half of ``GET /api/machines``, mirroring the thin-client half
|
|
217
|
+
that reads the daemon-published ``fleet_health.machine_health`` block
|
|
218
|
+
``FleetHealthRefresher.refresh()`` builds from this same function) —
|
|
219
|
+
exposed as a real entry point instead of reaching across the module
|
|
220
|
+
boundary for the underscore-prefixed helper below.
|
|
221
|
+
"""
|
|
222
|
+
return _machine_health_rows(machine_names, raw, now=now)
|
|
223
|
+
|
|
224
|
+
|
|
212
225
|
def _machine_health_rows(machine_names: list[str], raw: dict, *, now: float) -> list[dict]:
|
|
213
226
|
rows: list[dict] = []
|
|
214
227
|
for name in machine_names:
|
|
@@ -234,6 +247,11 @@ def _machine_health_rows(machine_names: list[str], raw: dict, *, now: float) ->
|
|
|
234
247
|
"severity": severity,
|
|
235
248
|
"checked_at": checks.get("checked_at"),
|
|
236
249
|
"results": checks.get("results", []),
|
|
250
|
+
# #3023: sibling facts `refresh()` stashes on the same
|
|
251
|
+
# persisted `health` blob — see its docstring comment for
|
|
252
|
+
# why they live here instead of inside `results`.
|
|
253
|
+
"worktree_bytes": checks.get("worktree_bytes"),
|
|
254
|
+
"agent_runtime_version": checks.get("agent_runtime_version"),
|
|
237
255
|
}
|
|
238
256
|
)
|
|
239
257
|
return rows
|
|
@@ -310,6 +328,36 @@ class FleetHealthRefresher:
|
|
|
310
328
|
health_block = (
|
|
311
329
|
status.health.get("health") if status.health else None
|
|
312
330
|
) or last_known_health
|
|
331
|
+
# #3023: the coord-web Machines panel (GET /api/machines) needs
|
|
332
|
+
# two more whole-payload facts that live OUTSIDE the H-1
|
|
333
|
+
# check-registry block above — the agent's own running
|
|
334
|
+
# `version` (compared against the coordinator's local version to
|
|
335
|
+
# flag skew) and `worktree_bytes` (total on-disk size of this
|
|
336
|
+
# agent's git worktrees). Both are real facts about the agent
|
|
337
|
+
# PROCESS, not a health check result, so they ride as sibling
|
|
338
|
+
# keys on the same persisted `health` blob rather than a schema
|
|
339
|
+
# change — advisory only, same as everything else in this
|
|
340
|
+
# module (see the module docstring). Freshly measured whenever
|
|
341
|
+
# this poll actually reached the agent; carried forward from the
|
|
342
|
+
# last successful poll otherwise, same "last-known, not
|
|
343
|
+
# blanked" posture as the H-1 `results` above, so a machine
|
|
344
|
+
# that's merely offline for one tick doesn't flash to "no data"
|
|
345
|
+
# in the panel. Named `agent_runtime_version` (not
|
|
346
|
+
# `agent_version`) to avoid colliding with the unrelated
|
|
347
|
+
# `agent_version` H-1 *check* already inside `results`, which
|
|
348
|
+
# measures installed-vs-latest-on-PyPI skew, not this.
|
|
349
|
+
if status.health is not None:
|
|
350
|
+
worktree_bytes = status.health.get("worktree_bytes")
|
|
351
|
+
agent_runtime_version = status.health.get("version")
|
|
352
|
+
else:
|
|
353
|
+
worktree_bytes = (last_known_health or {}).get("worktree_bytes")
|
|
354
|
+
agent_runtime_version = (last_known_health or {}).get("agent_runtime_version")
|
|
355
|
+
if health_block is not None:
|
|
356
|
+
health_block = {
|
|
357
|
+
**health_block,
|
|
358
|
+
"worktree_bytes": worktree_bytes,
|
|
359
|
+
"agent_runtime_version": agent_runtime_version,
|
|
360
|
+
}
|
|
313
361
|
state.save_machine_health(
|
|
314
362
|
machine.name,
|
|
315
363
|
state=status.state,
|
coord/machine_metrics.py
CHANGED
|
@@ -48,10 +48,12 @@ for exactly this reason) are not this module's job.
|
|
|
48
48
|
from __future__ import annotations
|
|
49
49
|
|
|
50
50
|
import logging
|
|
51
|
+
import re
|
|
51
52
|
import time
|
|
52
53
|
from collections import deque
|
|
53
54
|
from concurrent.futures import ThreadPoolExecutor
|
|
54
55
|
from dataclasses import dataclass
|
|
56
|
+
from datetime import datetime
|
|
55
57
|
|
|
56
58
|
import httpx
|
|
57
59
|
|
|
@@ -215,3 +217,174 @@ class MachineMetricsSampler:
|
|
|
215
217
|
except Exception as exc: # noqa: BLE001 — one bad machine must never abort the refresh
|
|
216
218
|
log.warning("machine-metrics poll: %s raised %s", machine.name, exc)
|
|
217
219
|
return _unknown(f"poll raised: {exc}")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ── read path: GET /machines/metrics (#3021) ────────────────────────────────
|
|
223
|
+
#
|
|
224
|
+
# Everything below is pure — no I/O, no dependence on the sampler's internal
|
|
225
|
+
# deques beyond the plain ``list[dict]`` returned by ``series()``/
|
|
226
|
+
# ``all_series()``. That keeps it trivially unit-testable and reusable from
|
|
227
|
+
# `coord.serve_app`'s request handler without either side needing to know
|
|
228
|
+
# about the other's internals.
|
|
229
|
+
|
|
230
|
+
_DURATION_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([smhdw])\s*$", re.IGNORECASE)
|
|
231
|
+
_UNIT_SECONDS = {"s": 1.0, "m": 60.0, "h": 3600.0, "d": 86400.0, "w": 604800.0}
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def resolve_since(raw: str | None, *, now: float | None = None) -> float | None:
|
|
235
|
+
"""Resolve a ``?since=`` query value to an absolute epoch timestamp.
|
|
236
|
+
|
|
237
|
+
Accepts three forms (mirrors the vocabulary the issue asks for,
|
|
238
|
+
``<epoch|duration>``, plus ISO-8601 for parity with ``coord audit``'s
|
|
239
|
+
``since``/``until``):
|
|
240
|
+
|
|
241
|
+
- ``None`` or ``""`` → ``None``, meaning "no lower bound" (the full
|
|
242
|
+
retained window — the endpoint's documented default).
|
|
243
|
+
- A duration like ``"6h"``, ``"90m"``, ``"3d"`` (units: s, m, h, d, w) →
|
|
244
|
+
``now - duration``.
|
|
245
|
+
- An epoch number (``"1699999999"``) or an ISO-8601 timestamp
|
|
246
|
+
(``"2026-01-01T00:00:00Z"``) → that instant, verbatim.
|
|
247
|
+
|
|
248
|
+
Raises :class:`ValueError` on anything else, so the HTTP handler can
|
|
249
|
+
turn it into a 400 with the bad value quoted back.
|
|
250
|
+
"""
|
|
251
|
+
if raw is None or raw == "":
|
|
252
|
+
return None
|
|
253
|
+
now = now if now is not None else time.time()
|
|
254
|
+
match = _DURATION_RE.match(raw)
|
|
255
|
+
if match:
|
|
256
|
+
return now - float(match.group(1)) * _UNIT_SECONDS[match.group(2).lower()]
|
|
257
|
+
try:
|
|
258
|
+
return float(raw)
|
|
259
|
+
except (TypeError, ValueError):
|
|
260
|
+
pass
|
|
261
|
+
try:
|
|
262
|
+
return datetime.fromisoformat(raw.replace("Z", "+00:00")).timestamp()
|
|
263
|
+
except ValueError as exc:
|
|
264
|
+
raise ValueError(
|
|
265
|
+
f"not an epoch number, ISO-8601 timestamp, or duration: {raw!r} "
|
|
266
|
+
"(duration units: s, m, h, d, w)"
|
|
267
|
+
) from exc
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _filter_since(samples: list[dict], since: float | None) -> list[dict]:
|
|
271
|
+
if since is None:
|
|
272
|
+
return list(samples)
|
|
273
|
+
return [s for s in samples if s["timestamp"] >= since]
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _sample_intensity(sample: dict) -> float:
|
|
277
|
+
"""How "interesting" a sample is for peak-preserving downsampling.
|
|
278
|
+
|
|
279
|
+
An ``unknown`` sample never outranks a real one — it always scores
|
|
280
|
+
below every possible real reading (percentages are 0-100) — so a bucket
|
|
281
|
+
with even one real sample surfaces that reading rather than the gap.
|
|
282
|
+
"""
|
|
283
|
+
if sample.get("status") != STATUS_OK:
|
|
284
|
+
return -1.0
|
|
285
|
+
cpu = sample.get("cpu_percent") or 0.0
|
|
286
|
+
mem = sample.get("mem_percent") or 0.0
|
|
287
|
+
return max(cpu, mem)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _pick_representative(bucket: list[dict]) -> dict:
|
|
291
|
+
"""The single sample that best represents ``bucket``.
|
|
292
|
+
|
|
293
|
+
Picks the highest-intensity real sample in the bucket (see
|
|
294
|
+
:func:`_sample_intensity`) so a brief spike survives downsampling
|
|
295
|
+
instead of being averaged away or dropped by naive striding. A bucket
|
|
296
|
+
with no real samples at all — every poll in that span came back
|
|
297
|
+
``unknown`` — reports its first ``unknown`` sample, so a renderer still
|
|
298
|
+
sees the gap rather than the bucket silently vanishing.
|
|
299
|
+
"""
|
|
300
|
+
best = bucket[0]
|
|
301
|
+
best_score = _sample_intensity(best)
|
|
302
|
+
for sample in bucket[1:]:
|
|
303
|
+
score = _sample_intensity(sample)
|
|
304
|
+
if score > best_score:
|
|
305
|
+
best, best_score = sample, score
|
|
306
|
+
return best
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def downsample(samples: list[dict], resolution: int) -> list[dict]:
|
|
310
|
+
"""Reduce ``samples`` (oldest-first) to at most ``resolution`` points.
|
|
311
|
+
|
|
312
|
+
A 6h window at the default ~15s cadence is ~1440 points per machine; a
|
|
313
|
+
phone asking for ``resolution=100`` must get **at most** 100 back, and
|
|
314
|
+
the server must do the reduction — never ship the raw buffer and let
|
|
315
|
+
the client thin it (the whole point of #3021).
|
|
316
|
+
|
|
317
|
+
Buckets ``samples`` into ``resolution`` roughly-equal, chronologically
|
|
318
|
+
contiguous spans and keeps one representative per bucket (see
|
|
319
|
+
:func:`_pick_representative`) — naive fixed-stride sampling (every Nth
|
|
320
|
+
point) would silently skip over a spike that lands on a dropped index;
|
|
321
|
+
this always surfaces the bucket's peak instead.
|
|
322
|
+
|
|
323
|
+
Raises :class:`ValueError` if ``resolution`` is not positive.
|
|
324
|
+
"""
|
|
325
|
+
if resolution <= 0:
|
|
326
|
+
raise ValueError("resolution must be positive")
|
|
327
|
+
n = len(samples)
|
|
328
|
+
if n <= resolution:
|
|
329
|
+
return list(samples)
|
|
330
|
+
out: list[dict] = []
|
|
331
|
+
for i in range(resolution):
|
|
332
|
+
lo = (i * n) // resolution
|
|
333
|
+
hi = ((i + 1) * n) // resolution
|
|
334
|
+
bucket = samples[lo:hi]
|
|
335
|
+
if bucket:
|
|
336
|
+
out.append(_pick_representative(bucket))
|
|
337
|
+
return out
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def build_metrics_response(
|
|
341
|
+
series_by_machine: dict[str, list[dict]],
|
|
342
|
+
*,
|
|
343
|
+
machine: str | None = None,
|
|
344
|
+
since: float | None = None,
|
|
345
|
+
resolution: int | None = None,
|
|
346
|
+
now: float | None = None,
|
|
347
|
+
) -> dict:
|
|
348
|
+
"""Assemble the versioned ``GET /machines/metrics`` response body.
|
|
349
|
+
|
|
350
|
+
``series_by_machine`` is exactly :meth:`MachineMetricsSampler.all_series`'s
|
|
351
|
+
shape (name → oldest-first list of :meth:`MetricsSample.to_dict`), kept
|
|
352
|
+
as a plain argument rather than the sampler itself so this stays a pure
|
|
353
|
+
function a test can call with a synthetic dict.
|
|
354
|
+
|
|
355
|
+
- ``machine`` narrows to a single machine's series; a name with no
|
|
356
|
+
buffer yet (never polled, or simply unknown) comes back as ``[]``,
|
|
357
|
+
same as :meth:`MachineMetricsSampler.series` — never a 404, since "no
|
|
358
|
+
data yet" and "no such machine" are indistinguishable from here and
|
|
359
|
+
both mean the same thing to a renderer: nothing to draw.
|
|
360
|
+
- ``since`` filters to samples at or after that epoch timestamp
|
|
361
|
+
(:func:`resolve_since` turns the query string into this).
|
|
362
|
+
- ``resolution`` server-side downsamples each machine's filtered series
|
|
363
|
+
independently (:func:`downsample`); omitted means the full filtered
|
|
364
|
+
series.
|
|
365
|
+
|
|
366
|
+
``schema: 1`` — versioned explicitly, matching
|
|
367
|
+
``FleetHealthSnapshot.to_dict()``'s posture, so a client can detect a
|
|
368
|
+
future incompatible reshape rather than guess from field presence.
|
|
369
|
+
"""
|
|
370
|
+
now = now if now is not None else time.time()
|
|
371
|
+
selected = (
|
|
372
|
+
{machine: series_by_machine.get(machine, [])}
|
|
373
|
+
if machine is not None
|
|
374
|
+
else series_by_machine
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
machines_out: dict[str, list[dict]] = {}
|
|
378
|
+
for name, series in selected.items():
|
|
379
|
+
windowed = _filter_since(series, since)
|
|
380
|
+
if resolution is not None:
|
|
381
|
+
windowed = downsample(windowed, resolution)
|
|
382
|
+
machines_out[name] = windowed
|
|
383
|
+
|
|
384
|
+
return {
|
|
385
|
+
"schema": 1,
|
|
386
|
+
"generated_at": now,
|
|
387
|
+
"since": since,
|
|
388
|
+
"resolution": resolution,
|
|
389
|
+
"machines": machines_out,
|
|
390
|
+
}
|
coord/serve_app.py
CHANGED
|
@@ -4957,6 +4957,91 @@ def openapi_spec() -> dict:
|
|
|
4957
4957
|
},
|
|
4958
4958
|
}
|
|
4959
4959
|
},
|
|
4960
|
+
"/machines/metrics": {
|
|
4961
|
+
"get": {
|
|
4962
|
+
"summary": (
|
|
4963
|
+
"#3021: server-downsampled cpu/mem history from #3020's "
|
|
4964
|
+
"bounded per-machine ring buffers (~6h at ~15s cadence). "
|
|
4965
|
+
"Never ships the raw buffer for the client to thin — pass "
|
|
4966
|
+
"`resolution` and the server reduces it."
|
|
4967
|
+
),
|
|
4968
|
+
"parameters": [
|
|
4969
|
+
{
|
|
4970
|
+
"name": "since",
|
|
4971
|
+
"in": "query",
|
|
4972
|
+
"schema": {"type": "string"},
|
|
4973
|
+
"description": (
|
|
4974
|
+
"epoch seconds, ISO-8601, or a duration like '6h' "
|
|
4975
|
+
"('now minus 6h'). Omitted = the full retained window."
|
|
4976
|
+
),
|
|
4977
|
+
},
|
|
4978
|
+
{
|
|
4979
|
+
"name": "resolution",
|
|
4980
|
+
"in": "query",
|
|
4981
|
+
"schema": {"type": "integer"},
|
|
4982
|
+
"description": (
|
|
4983
|
+
"max points per machine per series in the response, "
|
|
4984
|
+
"e.g. 100. Downsampling is peak-preserving (keeps the "
|
|
4985
|
+
"highest cpu/mem reading per bucket, never naive "
|
|
4986
|
+
"striding). Omitted = full filtered series."
|
|
4987
|
+
),
|
|
4988
|
+
},
|
|
4989
|
+
{
|
|
4990
|
+
"name": "machine",
|
|
4991
|
+
"in": "query",
|
|
4992
|
+
"schema": {"type": "string"},
|
|
4993
|
+
"description": "narrow to a single machine's series",
|
|
4994
|
+
},
|
|
4995
|
+
],
|
|
4996
|
+
"responses": {
|
|
4997
|
+
"200": {
|
|
4998
|
+
"description": "OK",
|
|
4999
|
+
"content": {
|
|
5000
|
+
"application/json": {
|
|
5001
|
+
"schema": {
|
|
5002
|
+
"type": "object",
|
|
5003
|
+
"properties": {
|
|
5004
|
+
"schema": {"type": "integer", "enum": [1]},
|
|
5005
|
+
"generated_at": {"type": "number"},
|
|
5006
|
+
"since": {"type": ["number", "null"]},
|
|
5007
|
+
"resolution": {"type": ["integer", "null"]},
|
|
5008
|
+
"machines": {
|
|
5009
|
+
"type": "object",
|
|
5010
|
+
"description": (
|
|
5011
|
+
"machine name -> oldest-first list "
|
|
5012
|
+
"of samples; status=unknown carries "
|
|
5013
|
+
"no cpu/mem values so a renderer "
|
|
5014
|
+
"draws a gap instead of interpolating"
|
|
5015
|
+
),
|
|
5016
|
+
"additionalProperties": {
|
|
5017
|
+
"type": "array",
|
|
5018
|
+
"items": {
|
|
5019
|
+
"type": "object",
|
|
5020
|
+
"properties": {
|
|
5021
|
+
"timestamp": {"type": "number"},
|
|
5022
|
+
"status": {
|
|
5023
|
+
"type": "string",
|
|
5024
|
+
"enum": ["ok", "unknown"],
|
|
5025
|
+
},
|
|
5026
|
+
"cpu_percent": {"type": ["number", "null"]},
|
|
5027
|
+
"mem_percent": {"type": ["number", "null"]},
|
|
5028
|
+
"mem_used_mb": {"type": ["number", "null"]},
|
|
5029
|
+
"mem_total_mb": {"type": ["number", "null"]},
|
|
5030
|
+
"reason": {"type": "string"},
|
|
5031
|
+
},
|
|
5032
|
+
},
|
|
5033
|
+
},
|
|
5034
|
+
},
|
|
5035
|
+
},
|
|
5036
|
+
"required": ["schema", "generated_at", "machines"],
|
|
5037
|
+
}
|
|
5038
|
+
}
|
|
5039
|
+
},
|
|
5040
|
+
},
|
|
5041
|
+
"400": {"description": "Bad `since` or `resolution`"},
|
|
5042
|
+
},
|
|
5043
|
+
}
|
|
5044
|
+
},
|
|
4960
5045
|
"/report": {
|
|
4961
5046
|
"get": {
|
|
4962
5047
|
"summary": (
|
|
@@ -5304,11 +5389,23 @@ def openapi_spec() -> dict:
|
|
|
5304
5389
|
)
|
|
5305
5390
|
|
|
5306
5391
|
|
|
5307
|
-
def build_app(
|
|
5392
|
+
def build_app(
|
|
5393
|
+
store: CoordStore,
|
|
5394
|
+
config: Config,
|
|
5395
|
+
*,
|
|
5396
|
+
token: str | None = None,
|
|
5397
|
+
machine_metrics_sampler: MachineMetricsSampler | None = None,
|
|
5398
|
+
) -> Starlette:
|
|
5308
5399
|
"""Build the read-only control-center Starlette app bound to *store* + *config*.
|
|
5309
5400
|
|
|
5310
5401
|
*token* — when set, every endpoint except ``/healthz`` requires
|
|
5311
5402
|
``Authorization: Bearer <token>``.
|
|
5403
|
+
|
|
5404
|
+
*machine_metrics_sampler* — injectable for tests (#3021): a fresh
|
|
5405
|
+
:class:`~coord.machine_metrics.MachineMetricsSampler` is created when
|
|
5406
|
+
omitted, exactly as before. Passing a pre-seeded instance lets a test
|
|
5407
|
+
drive ``GET /machines/metrics`` against known ring-buffer contents
|
|
5408
|
+
without waiting on the tick loop or a live agent poll.
|
|
5312
5409
|
"""
|
|
5313
5410
|
# #1081: track the backing coordinator.yml's mtime so the handlers below
|
|
5314
5411
|
# can swap in a freshly-reloaded Config when it changes on disk, instead
|
|
@@ -5343,11 +5440,12 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
|
|
|
5343
5440
|
# #3020: same shape again — polling every agent's own /metrics is real
|
|
5344
5441
|
# per-machine I/O, so it runs on the tick loop's cadence
|
|
5345
5442
|
# (`_machine_metrics_loop` below), never inline off a request handler.
|
|
5346
|
-
# Series live only in this sampler's bounded in-memory ring buffers;
|
|
5347
|
-
#
|
|
5443
|
+
# Series live only in this sampler's bounded in-memory ring buffers;
|
|
5444
|
+
# `GET /machines/metrics` (#3021, below) is the read endpoint and does
|
|
5445
|
+
# bare in-memory reads only. Nothing here touches /board.
|
|
5348
5446
|
from coord.machine_metrics import MachineMetricsSampler # noqa: PLC0415
|
|
5349
5447
|
|
|
5350
|
-
_machine_metrics_sampler = MachineMetricsSampler()
|
|
5448
|
+
_machine_metrics_sampler = machine_metrics_sampler or MachineMetricsSampler()
|
|
5351
5449
|
|
|
5352
5450
|
# Short-TTL cache for the computed /board projection so burst polls from the
|
|
5353
5451
|
# TUI don't each pay the full board_projection + merge-plan + stage-projection
|
|
@@ -8882,6 +8980,39 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
|
|
|
8882
8980
|
)
|
|
8883
8981
|
return JSONResponse(result.to_dict())
|
|
8884
8982
|
|
|
8983
|
+
async def get_machine_metrics(request: Request) -> Response:
|
|
8984
|
+
# #3021: the read side of #3020's sampler. Bare in-memory reads +
|
|
8985
|
+
# pure filtering/downsampling — no I/O, so (unlike /board) this
|
|
8986
|
+
# never needs a threadpool hop and can't be slowed by a hanging
|
|
8987
|
+
# agent (that's `_machine_metrics_loop`'s problem, not this
|
|
8988
|
+
# handler's).
|
|
8989
|
+
from coord.machine_metrics import build_metrics_response, resolve_since # noqa: PLC0415
|
|
8990
|
+
|
|
8991
|
+
qp = request.query_params
|
|
8992
|
+
|
|
8993
|
+
resolution_raw = qp.get("resolution")
|
|
8994
|
+
try:
|
|
8995
|
+
resolution = int(resolution_raw) if resolution_raw else None
|
|
8996
|
+
if resolution is not None and resolution <= 0:
|
|
8997
|
+
raise ValueError("resolution must be a positive integer")
|
|
8998
|
+
except ValueError as e:
|
|
8999
|
+
return JSONResponse(
|
|
9000
|
+
{"error": f"bad resolution={resolution_raw!r}: {e}"}, status_code=400
|
|
9001
|
+
)
|
|
9002
|
+
|
|
9003
|
+
try:
|
|
9004
|
+
since = resolve_since(qp.get("since"))
|
|
9005
|
+
except ValueError as e:
|
|
9006
|
+
return JSONResponse({"error": str(e)}, status_code=400)
|
|
9007
|
+
|
|
9008
|
+
result = build_metrics_response(
|
|
9009
|
+
_machine_metrics_sampler.all_series(),
|
|
9010
|
+
machine=qp.get("machine") or None,
|
|
9011
|
+
since=since,
|
|
9012
|
+
resolution=resolution,
|
|
9013
|
+
)
|
|
9014
|
+
return JSONResponse(result)
|
|
9015
|
+
|
|
8885
9016
|
async def post_merge(request: Request) -> Response:
|
|
8886
9017
|
# #584: the merge queue + board live in THIS (canonical) DB, and gh is
|
|
8887
9018
|
# authenticated here — so a thin client's `coord merge` / TUI 'Go' routes
|
|
@@ -10292,6 +10423,9 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
|
|
|
10292
10423
|
# read-only, both alongside /audit because they read the same data.
|
|
10293
10424
|
Route("/report", get_report_catalogue, methods=["GET"]),
|
|
10294
10425
|
Route("/report/{report_id}", get_report, methods=["GET"]),
|
|
10426
|
+
# #3021: read side of #3020's sampler — server-downsampled machine
|
|
10427
|
+
# cpu/mem history for the coord-web Machines panel.
|
|
10428
|
+
Route("/machines/metrics", get_machine_metrics, methods=["GET"]),
|
|
10295
10429
|
Route("/config", serve_config, methods=["GET"]),
|
|
10296
10430
|
Route("/result", post_result, methods=["POST"]),
|
|
10297
10431
|
Route("/completion", post_completion, methods=["POST"]),
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|