code-coordinator 0.5.46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_coordinator-0.5.46.dist-info/METADATA +625 -0
- code_coordinator-0.5.46.dist-info/RECORD +295 -0
- code_coordinator-0.5.46.dist-info/WHEEL +5 -0
- code_coordinator-0.5.46.dist-info/entry_points.txt +2 -0
- code_coordinator-0.5.46.dist-info/licenses/LICENSE +110 -0
- code_coordinator-0.5.46.dist-info/top_level.txt +1 -0
- coord/__init__.py +176 -0
- coord/_board_mapping.py +229 -0
- coord/acceptance.py +468 -0
- coord/acceptance_drivers.py +632 -0
- coord/agent.py +7517 -0
- coord/agent_app.py +1555 -0
- coord/agent_update.py +417 -0
- coord/agents/opencode/.gitignore +13 -0
- coord/agents/opencode/agents/work.md +129 -0
- coord/agents/opencode/routing.jsonc +49 -0
- coord/audit.py +301 -0
- coord/auto_loop.py +1440 -0
- coord/board_bool_guard.py +72 -0
- coord/board_service.py +141 -0
- coord/board_wire.py +309 -0
- coord/brain.py +581 -0
- coord/branch_model.py +214 -0
- coord/cargo_cache.py +258 -0
- coord/ci_github.py +386 -0
- coord/ci_store.py +560 -0
- coord/claim.py +353 -0
- coord/cli.py +454 -0
- coord/client.py +610 -0
- coord/commands/__init__.py +1 -0
- coord/commands/_common.py +329 -0
- coord/commands/acceptance.py +916 -0
- coord/commands/agent_ops.py +1339 -0
- coord/commands/audit.py +131 -0
- coord/commands/chat.py +320 -0
- coord/commands/dispatch.py +1780 -0
- coord/commands/dispatch_workers.py +4894 -0
- coord/commands/drive.py +616 -0
- coord/commands/drive_queue.py +1203 -0
- coord/commands/gate_a.py +217 -0
- coord/commands/gates.py +89 -0
- coord/commands/issues.py +681 -0
- coord/commands/lifecycle.py +513 -0
- coord/commands/merge.py +1900 -0
- coord/commands/milestone.py +2081 -0
- coord/commands/plan_followup.py +1243 -0
- coord/commands/plans.py +156 -0
- coord/commands/release.py +2232 -0
- coord/commands/report.py +341 -0
- coord/commands/review.py +1523 -0
- coord/commands/scorecard.py +252 -0
- coord/commands/sessions.py +1930 -0
- coord/commands/setup.py +576 -0
- coord/commands/status.py +2089 -0
- coord/commands/terminal.py +385 -0
- coord/commands/test_gate.py +775 -0
- coord/commands/tui.py +288 -0
- coord/comments.py +718 -0
- coord/config.py +3032 -0
- coord/conflict_fix.py +633 -0
- coord/dao.py +483 -0
- coord/dashboard/__init__.py +0 -0
- coord/dashboard/fixture.py +376 -0
- coord/dashboard/index.html +658 -0
- coord/dashboard/server.py +1894 -0
- coord/dashboard/terminal.py +382 -0
- coord/dashboard/webapp/.gitignore +9 -0
- coord/dashboard/webapp/components.json +17 -0
- coord/dashboard/webapp/dist/assets/Gallery-da3qNiIw.js +71 -0
- coord/dashboard/webapp/dist/assets/Terminal-9CEnUXvW.css +32 -0
- coord/dashboard/webapp/dist/assets/Terminal-skVFCxPU.js +63 -0
- coord/dashboard/webapp/dist/assets/index-DltfZR5f.js +184 -0
- coord/dashboard/webapp/dist/assets/index-Dq4kwTdw.css +1 -0
- coord/dashboard/webapp/dist/assets/workbox-window.prod.es5-BqEJf4Xk.js +2 -0
- coord/dashboard/webapp/dist/icons/icon-192.png +0 -0
- coord/dashboard/webapp/dist/icons/icon-512.png +0 -0
- coord/dashboard/webapp/dist/icons/icon.svg +5 -0
- coord/dashboard/webapp/dist/index.html +38 -0
- coord/dashboard/webapp/dist/manifest.webmanifest +1 -0
- coord/dashboard/webapp/dist/sw.js +1 -0
- coord/dashboard/webapp/dist/workbox-e4022e15.js +1 -0
- coord/dashboard/webapp/e2e/available-gates-terminal.spec.ts +75 -0
- coord/dashboard/webapp/e2e/deep-link.spec.ts +172 -0
- coord/dashboard/webapp/e2e/fixtureServer.ts +155 -0
- coord/dashboard/webapp/e2e/live-update-fixture.spec.ts +113 -0
- coord/dashboard/webapp/e2e/realtime.spec.ts +238 -0
- coord/dashboard/webapp/e2e/shell.spec.ts +309 -0
- coord/dashboard/webapp/e2e/smoke.spec.ts +191 -0
- coord/dashboard/webapp/e2e/terminal.spec.ts +420 -0
- coord/dashboard/webapp/e2e/theme.spec.ts +138 -0
- coord/dashboard/webapp/eslint.config.js +20 -0
- coord/dashboard/webapp/index.html +37 -0
- coord/dashboard/webapp/node_modules/flatted/python/flatted.py +144 -0
- coord/dashboard/webapp/package-lock.json +10584 -0
- coord/dashboard/webapp/package.json +63 -0
- coord/dashboard/webapp/playwright.acceptance.config.ts +166 -0
- coord/dashboard/webapp/playwright.config.ts +93 -0
- coord/dashboard/webapp/postcss.config.js +6 -0
- coord/dashboard/webapp/public/icons/icon-192.png +0 -0
- coord/dashboard/webapp/public/icons/icon-512.png +0 -0
- coord/dashboard/webapp/public/icons/icon.svg +5 -0
- coord/dashboard/webapp/src/App.tsx +140 -0
- coord/dashboard/webapp/src/api/client.ts +199 -0
- coord/dashboard/webapp/src/api/generated.ts +176 -0
- coord/dashboard/webapp/src/components/ConnectionBadge.tsx +52 -0
- coord/dashboard/webapp/src/components/Detail.tsx +800 -0
- coord/dashboard/webapp/src/components/Gallery.tsx +341 -0
- coord/dashboard/webapp/src/components/Home.tsx +435 -0
- coord/dashboard/webapp/src/components/MobileKeyBar.tsx +280 -0
- coord/dashboard/webapp/src/components/PanelHeader.tsx +59 -0
- coord/dashboard/webapp/src/components/PipelineCard.tsx +168 -0
- coord/dashboard/webapp/src/components/SessionCard.tsx +99 -0
- coord/dashboard/webapp/src/components/SessionDetail.tsx +140 -0
- coord/dashboard/webapp/src/components/SessionsList.tsx +81 -0
- coord/dashboard/webapp/src/components/Terminal.tsx +376 -0
- coord/dashboard/webapp/src/components/__tests__/ConnectionBadge.test.tsx +81 -0
- coord/dashboard/webapp/src/components/__tests__/Detail.test.tsx +680 -0
- coord/dashboard/webapp/src/components/__tests__/Gallery.test.tsx +83 -0
- coord/dashboard/webapp/src/components/__tests__/Home.test.tsx +271 -0
- coord/dashboard/webapp/src/components/__tests__/MobileKeyBar.test.tsx +197 -0
- coord/dashboard/webapp/src/components/__tests__/PipelineCard.test.tsx +143 -0
- coord/dashboard/webapp/src/components/__tests__/SessionCard.test.tsx +106 -0
- coord/dashboard/webapp/src/components/__tests__/Terminal.test.tsx +504 -0
- coord/dashboard/webapp/src/components/ui/badge.tsx +41 -0
- coord/dashboard/webapp/src/components/ui/button.tsx +54 -0
- coord/dashboard/webapp/src/components/ui/card.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/dialog.tsx +99 -0
- coord/dashboard/webapp/src/components/ui/dropdown-menu.tsx +189 -0
- coord/dashboard/webapp/src/components/ui/empty-state.tsx +35 -0
- coord/dashboard/webapp/src/components/ui/sheet.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/skeleton.tsx +9 -0
- coord/dashboard/webapp/src/components/ui/tabs.tsx +55 -0
- coord/dashboard/webapp/src/components/ui/theme-provider.tsx +78 -0
- coord/dashboard/webapp/src/components/ui/theme-toggle.tsx +20 -0
- coord/dashboard/webapp/src/components/ui/toast.tsx +123 -0
- coord/dashboard/webapp/src/components/ui/toaster.tsx +30 -0
- coord/dashboard/webapp/src/components/ui/tooltip.tsx +26 -0
- coord/dashboard/webapp/src/components/ui/use-toast.ts +134 -0
- coord/dashboard/webapp/src/index.css +210 -0
- coord/dashboard/webapp/src/lib/pipeline.ts +29 -0
- coord/dashboard/webapp/src/lib/utils.ts +6 -0
- coord/dashboard/webapp/src/main.tsx +46 -0
- coord/dashboard/webapp/src/realtime/RealtimeProvider.tsx +112 -0
- coord/dashboard/webapp/src/realtime/__tests__/RealtimeProvider.test.tsx +189 -0
- coord/dashboard/webapp/src/realtime/__tests__/connection.test.ts +255 -0
- coord/dashboard/webapp/src/realtime/connection.ts +227 -0
- coord/dashboard/webapp/src/realtime/events.ts +100 -0
- coord/dashboard/webapp/src/routes/__tests__/paths.test.ts +92 -0
- coord/dashboard/webapp/src/routes/paths.ts +92 -0
- coord/dashboard/webapp/src/shell/ActivityRail.tsx +335 -0
- coord/dashboard/webapp/src/shell/AppShell.tsx +276 -0
- coord/dashboard/webapp/src/shell/ComingSoon.tsx +33 -0
- coord/dashboard/webapp/src/shell/EmptyDetail.tsx +26 -0
- coord/dashboard/webapp/src/shell/RouteNotFound.tsx +33 -0
- coord/dashboard/webapp/src/shell/ShellLayout.tsx +147 -0
- coord/dashboard/webapp/src/shell/StatusBar.tsx +46 -0
- coord/dashboard/webapp/src/shell/__tests__/ShellLayout.test.tsx +520 -0
- coord/dashboard/webapp/src/shell/__tests__/shellState.test.ts +95 -0
- coord/dashboard/webapp/src/shell/__tests__/stubViewport.ts +40 -0
- coord/dashboard/webapp/src/shell/breakpoints.ts +87 -0
- coord/dashboard/webapp/src/shell/railItems.ts +105 -0
- coord/dashboard/webapp/src/shell/shellState.ts +174 -0
- coord/dashboard/webapp/src/shell/useRegionFocus.ts +95 -0
- coord/dashboard/webapp/src/test-setup.ts +41 -0
- coord/dashboard/webapp/src/vite-env.d.ts +2 -0
- coord/dashboard/webapp/tailwind.config.js +140 -0
- coord/dashboard/webapp/tsconfig.json +25 -0
- coord/dashboard/webapp/tsconfig.node.json +11 -0
- coord/dashboard/webapp/vite.config.ts +71 -0
- coord/db.py +1076 -0
- coord/dead_end.py +332 -0
- coord/deploy/README.md +33 -0
- coord/deploy/coord-agent.service +89 -0
- coord/deploy/coord-db-backup.service +60 -0
- coord/deploy/coord-db-backup.sh +74 -0
- coord/deploy/coord-db-backup.timer +18 -0
- coord/deploy/coord-drive-queue.service +117 -0
- coord/deploy/coord-drive-queue.timer +39 -0
- coord/deploy/coord-notify.service +48 -0
- coord/deploy/coord-notify.timer +24 -0
- coord/deploy/coord-release-propagate.service +83 -0
- coord/deploy/coord-release-propagate.timer +38 -0
- coord/deploy/coord-release-window.service +119 -0
- coord/deploy/coord-release-window.timer +36 -0
- coord/deploy/coord-serve.service +82 -0
- coord/deploy/coord-web-dist-build.service +43 -0
- coord/deploy/coord-web-dist-build.timer +36 -0
- coord/deploy/coord-web.service +125 -0
- coord/deploy_manifest.py +80 -0
- coord/deploy_units.py +384 -0
- coord/deps.py +115 -0
- coord/diagnose.py +1623 -0
- coord/dispatch.py +1009 -0
- coord/dist_name.py +123 -0
- coord/drive.py +3101 -0
- coord/drive_queue.py +2298 -0
- coord/drive_state.py +870 -0
- coord/events.py +381 -0
- coord/failure_class.py +914 -0
- coord/filelock.py +168 -0
- coord/fleet_config_health.py +300 -0
- coord/freshness.py +206 -0
- coord/gate_a.py +469 -0
- coord/gate_b.py +411 -0
- coord/gate_snapshot.py +385 -0
- coord/gates.py +582 -0
- coord/github_ops.py +1954 -0
- coord/goal.py +125 -0
- coord/graph_health.py +348 -0
- coord/health/__init__.py +69 -0
- coord/health/aggregate.py +129 -0
- coord/health/checks/__init__.py +13 -0
- coord/health/checks/agent_install.py +280 -0
- coord/health/checks/cargo_targets.py +171 -0
- coord/health/checks/claude_binary.py +65 -0
- coord/health/checks/deploy_lane_facts.py +458 -0
- coord/health/checks/disk.py +99 -0
- coord/health/checks/fleet_board.py +89 -0
- coord/health/checks/fleet_deploy_lanes.py +469 -0
- coord/health/checks/fleet_phantom.py +69 -0
- coord/health/checks/fleet_unit_drift.py +151 -0
- coord/health/checks/graph.py +192 -0
- coord/health/checks/plan_usage.py +88 -0
- coord/health/checks/repo_state.py +161 -0
- coord/health/checks/spawned_coord.py +465 -0
- coord/health/checks/timer_active.py +254 -0
- coord/health/checks/toolchain.py +547 -0
- coord/health/checks/unit_drift.py +648 -0
- coord/health/checks/unit_enablement.py +171 -0
- coord/health/checks/worktrees.py +96 -0
- coord/health/cli.py +121 -0
- coord/health/context.py +106 -0
- coord/health/fleet_snapshot.py +477 -0
- coord/health/models.py +250 -0
- coord/health/pypi.py +231 -0
- coord/health/registry.py +240 -0
- coord/health/render.py +82 -0
- coord/health/units.py +60 -0
- coord/hooks.py +106 -0
- coord/housekeeping.py +204 -0
- coord/interactive.py +4286 -0
- coord/issue_store.py +1496 -0
- coord/liveness_auditor.py +293 -0
- coord/machine_pause.py +755 -0
- coord/merge_queue.py +4681 -0
- coord/milestone_chat.py +600 -0
- coord/milestone_dispatch.py +943 -0
- coord/milestone_gate.py +709 -0
- coord/milestone_order.py +840 -0
- coord/mock_author.py +334 -0
- coord/models.py +891 -0
- coord/network.py +269 -0
- coord/new_issue_chat.py +229 -0
- coord/notify.py +3226 -0
- coord/openapi.py +404 -0
- coord/overlap_fence.py +133 -0
- coord/parentage.py +200 -0
- coord/parentage_github.py +58 -0
- coord/pipeline.py +481 -0
- coord/plan_parser.py +266 -0
- coord/plans.py +543 -0
- coord/platform_paths.py +43 -0
- coord/pr_body_lint.py +67 -0
- coord/prereqs.py +533 -0
- coord/progress.py +425 -0
- coord/providers/__init__.py +683 -0
- coord/providers/base.py +218 -0
- coord/providers/claude.py +284 -0
- coord/providers/claude_pty.py +610 -0
- coord/providers/opencode.py +896 -0
- coord/reconcile.py +2233 -0
- coord/refine_chat.py +485 -0
- coord/release_cordon.py +525 -0
- coord/release_propagate.py +1176 -0
- coord/release_verify.py +777 -0
- coord/release_window.py +322 -0
- coord/reports.py +1643 -0
- coord/revalidate.py +1101 -0
- coord/review.py +3317 -0
- coord/scorecard.py +484 -0
- coord/serve_app.py +7192 -0
- coord/skills/update-issue/SKILL.md +93 -0
- coord/smoke.py +1030 -0
- coord/split_work.py +210 -0
- coord/stage_projection.py +650 -0
- coord/state.py +5720 -0
- coord/test_author.py +1064 -0
- coord/test_chat.py +352 -0
- coord/test_orchestrator.py +494 -0
- coord/test_report.py +178 -0
- coord/tui_release.py +271 -0
- coord/usage.py +753 -0
- coord/usage_limits.py +358 -0
- coord/usage_rollup.py +709 -0
- coord/worker_events.py +954 -0
coord/worker_events.py
ADDED
|
@@ -0,0 +1,954 @@
|
|
|
1
|
+
"""Parse stream-json worker logs into typed events and summaries.
|
|
2
|
+
|
|
3
|
+
The worker (claude -p) is invoked with `--output-format stream-json --verbose`,
|
|
4
|
+
which emits one JSON object per line to stdout. The agent writes that stream
|
|
5
|
+
verbatim to ``~/.coord/logs/<assignment_id>.log``.
|
|
6
|
+
|
|
7
|
+
This module knows how to:
|
|
8
|
+
|
|
9
|
+
* Detect whether a log is stream-json (vs. plain text from older workers).
|
|
10
|
+
* Parse a single line into a :class:`WorkerEvent`.
|
|
11
|
+
* Walk the log and build a rolling :class:`WorkerSummary` (turns, cost,
|
|
12
|
+
tools used, files edited, bash commands, rate-limit state, etc.).
|
|
13
|
+
* Spot anomaly patterns (repeated bash, rate-limit hits, permission denials).
|
|
14
|
+
* Render events as a concise one-line-per-event human-readable form.
|
|
15
|
+
|
|
16
|
+
The implementation is intentionally permissive — the stream-json shape has
|
|
17
|
+
changed over time and varies between claude versions. We accept a handful of
|
|
18
|
+
plausible field paths for each thing we care about, and ignore anything we
|
|
19
|
+
don't recognise.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import re
|
|
26
|
+
from collections import Counter
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Iterable
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# ── Data classes ────────────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class WorkerEvent:
|
|
37
|
+
"""One JSON object from the stream-json log."""
|
|
38
|
+
|
|
39
|
+
type: str
|
|
40
|
+
subtype: str | None = None
|
|
41
|
+
raw: dict = field(default_factory=dict)
|
|
42
|
+
|
|
43
|
+
def to_dict(self) -> dict:
|
|
44
|
+
return {"type": self.type, "subtype": self.subtype, "raw": self.raw}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class WorkerSummary:
|
|
49
|
+
"""Rolling summary built from a stream of WorkerEvents."""
|
|
50
|
+
|
|
51
|
+
session_id: str | None = None
|
|
52
|
+
model_used: str | None = None
|
|
53
|
+
num_turns: int = 0
|
|
54
|
+
total_cost_usd: float = 0.0
|
|
55
|
+
stop_reason: str | None = None
|
|
56
|
+
permission_denials: list[str] = field(default_factory=list)
|
|
57
|
+
rate_limited: bool = False
|
|
58
|
+
rate_limit_resets_at: float | None = None
|
|
59
|
+
tools_used: list[str] = field(default_factory=list)
|
|
60
|
+
last_tool: str | None = None
|
|
61
|
+
files_edited: list[str] = field(default_factory=list)
|
|
62
|
+
bash_commands: list[str] = field(default_factory=list)
|
|
63
|
+
duration_ms: int | None = None
|
|
64
|
+
# Token counts from the result event (may be zero if the log predates
|
|
65
|
+
# token reporting or the worker didn't emit usage data).
|
|
66
|
+
input_tokens: int = 0
|
|
67
|
+
output_tokens: int = 0
|
|
68
|
+
cache_creation_tokens: int = 0
|
|
69
|
+
cache_read_tokens: int = 0
|
|
70
|
+
# #1584: `is_error` off the LAST `result` event seen (update_summary
|
|
71
|
+
# overwrites these on every `result` line it processes, in log order —
|
|
72
|
+
# never OR'd together), so a worker that hit a transient API error,
|
|
73
|
+
# retried internally, and finished cleanly ends with `is_error=False`
|
|
74
|
+
# here, exactly like any other successful run. `terminal_reason` and
|
|
75
|
+
# `api_error_status` are the same event's diagnostic fields (e.g.
|
|
76
|
+
# `"api_error"` / `529`); `result_text` is its raw `result` string, kept
|
|
77
|
+
# so :func:`format_api_error_reason` can pull a human phrase (e.g.
|
|
78
|
+
# "Overloaded") out of it. All four are blank/False for a log with no
|
|
79
|
+
# `result` event at all, or whose last one wasn't an error.
|
|
80
|
+
is_error: bool = False
|
|
81
|
+
terminal_reason: str | None = None
|
|
82
|
+
api_error_status: int | None = None
|
|
83
|
+
result_text: str | None = None
|
|
84
|
+
|
|
85
|
+
def to_dict(self) -> dict:
|
|
86
|
+
return {
|
|
87
|
+
"session_id": self.session_id,
|
|
88
|
+
"model_used": self.model_used,
|
|
89
|
+
"num_turns": self.num_turns,
|
|
90
|
+
"total_cost_usd": self.total_cost_usd,
|
|
91
|
+
"stop_reason": self.stop_reason,
|
|
92
|
+
"permission_denials": list(self.permission_denials),
|
|
93
|
+
"rate_limited": self.rate_limited,
|
|
94
|
+
"rate_limit_resets_at": self.rate_limit_resets_at,
|
|
95
|
+
"tools_used": list(self.tools_used),
|
|
96
|
+
"last_tool": self.last_tool,
|
|
97
|
+
"files_edited": list(self.files_edited),
|
|
98
|
+
"bash_commands": list(self.bash_commands),
|
|
99
|
+
"duration_ms": self.duration_ms,
|
|
100
|
+
"input_tokens": self.input_tokens,
|
|
101
|
+
"output_tokens": self.output_tokens,
|
|
102
|
+
"cache_creation_tokens": self.cache_creation_tokens,
|
|
103
|
+
"cache_read_tokens": self.cache_read_tokens,
|
|
104
|
+
"is_error": self.is_error,
|
|
105
|
+
"terminal_reason": self.terminal_reason,
|
|
106
|
+
"api_error_status": self.api_error_status,
|
|
107
|
+
"result_text": self.result_text,
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ── Line-level parsing ──────────────────────────────────────────────────────
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def parse_event(line: str) -> WorkerEvent | None:
|
|
115
|
+
"""Parse a single NDJSON line into a :class:`WorkerEvent`.
|
|
116
|
+
|
|
117
|
+
Returns ``None`` for blank lines, lines that aren't valid JSON, or lines
|
|
118
|
+
that don't decode to a JSON object (e.g. arrays, scalars). The log file
|
|
119
|
+
legitimately contains a leading ``# argv=…`` comment line written by the
|
|
120
|
+
agent itself; we just skip past those.
|
|
121
|
+
"""
|
|
122
|
+
if not line or not line.strip():
|
|
123
|
+
return None
|
|
124
|
+
try:
|
|
125
|
+
data = json.loads(line)
|
|
126
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
127
|
+
return None
|
|
128
|
+
if not isinstance(data, dict):
|
|
129
|
+
return None
|
|
130
|
+
return WorkerEvent(
|
|
131
|
+
type=data.get("type", "unknown"),
|
|
132
|
+
subtype=data.get("subtype"),
|
|
133
|
+
raw=data,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def is_stream_json(log_path: str | Path) -> bool:
|
|
138
|
+
"""Heuristic: is *log_path* a stream-json log?
|
|
139
|
+
|
|
140
|
+
The agent prepends a ``# agent=… argv=…`` comment line before spawning the
|
|
141
|
+
worker, so we skip past comment lines and check whether the first
|
|
142
|
+
non-comment line starts with ``{``. Returns ``False`` for missing or
|
|
143
|
+
empty files.
|
|
144
|
+
"""
|
|
145
|
+
p = Path(log_path)
|
|
146
|
+
if not p.exists():
|
|
147
|
+
return False
|
|
148
|
+
try:
|
|
149
|
+
with open(p, "r", encoding="utf-8", errors="replace") as f:
|
|
150
|
+
for _ in range(50): # Bound the scan.
|
|
151
|
+
line = f.readline()
|
|
152
|
+
if not line:
|
|
153
|
+
return False
|
|
154
|
+
stripped = line.lstrip()
|
|
155
|
+
if not stripped:
|
|
156
|
+
continue
|
|
157
|
+
if stripped.startswith("#"):
|
|
158
|
+
continue
|
|
159
|
+
return stripped.startswith("{")
|
|
160
|
+
except OSError:
|
|
161
|
+
return False
|
|
162
|
+
return False
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# ── Usage-limit kill detection (#1461) ──────────────────────────────────────
|
|
166
|
+
#
|
|
167
|
+
# A worker (claude -p) that hits the account's Max/Pro *session* usage limit
|
|
168
|
+
# mid-flight prints a terminal line like:
|
|
169
|
+
#
|
|
170
|
+
# "You've hit your session limit · resets 8:30pm (America/Chicago)"
|
|
171
|
+
#
|
|
172
|
+
# and exits — with no structured event marking what happened, so the reap
|
|
173
|
+
# path records a bare FAILED (or, if the CLI ends the turn gracefully before
|
|
174
|
+
# any commit, ADVISORY) indistinguishable from a real defect. This is a
|
|
175
|
+
# different signal from the *API* `rate_limit_event` handled above: that one
|
|
176
|
+
# is a structured stream-json event describing 429 throttling; this is a
|
|
177
|
+
# plain-text kill message from the CLI's own subscription-limit handling, and
|
|
178
|
+
# it never arrives as a `rate_limit_event`.
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@dataclass
|
|
182
|
+
class UsageLimitKill:
|
|
183
|
+
"""Diagnostic: the transcript shows the worker was killed by hitting the
|
|
184
|
+
account's session usage limit — not an API rate limit, and not a genuine
|
|
185
|
+
defect. A worker in this state is safe to re-dispatch unchanged once the
|
|
186
|
+
limit resets; it must never be diagnosed as a bug or escalated (e.g. via
|
|
187
|
+
`coord fix`'s model bump).
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
reset_at_raw: str
|
|
191
|
+
excerpt: str
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# A stable, greppable prefix stamped onto `Assignment.failure_reason` (and
|
|
195
|
+
# recognised by `coord/drive.py`'s state machine) whenever a kill is detected
|
|
196
|
+
# — see `format_usage_limit_reason`/`is_usage_limit_reason` below.
|
|
197
|
+
USAGE_LIMIT_REASON_PREFIX = "usage limit — resets "
|
|
198
|
+
|
|
199
|
+
# Matches the CLI's own message regardless of whether the apostrophe/middle
|
|
200
|
+
# dot appear as literal unicode glyphs (a plain trailing line) or as \uXXXX
|
|
201
|
+
# escapes (embedded in a JSON string field) — only the stable ASCII words
|
|
202
|
+
# around them are required, so this matches either encoding without first
|
|
203
|
+
# decoding the line as JSON. Tolerant of "usage limit" phrasing too, in case
|
|
204
|
+
# the CLI's wording changes.
|
|
205
|
+
_USAGE_LIMIT_RE = re.compile(
|
|
206
|
+
r"(?:session|usage) limit[^\r\n]{0,40}?resets?\s+([^\r\n\"\\]+)",
|
|
207
|
+
re.IGNORECASE,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# How many of the transcript's final non-blank, non-comment lines to search.
|
|
211
|
+
# Bounded to exactly the literal last one — the issue's own evidence was
|
|
212
|
+
# "the literal last line of the raw transcript", and a bare substring/regex
|
|
213
|
+
# search over the whole log would false-positive on a worker that merely
|
|
214
|
+
# *discusses* usage limits mid-conversation (this very issue's own worker
|
|
215
|
+
# transcript, for instance). Blank lines and coordinator-appended
|
|
216
|
+
# `# reap: ...` comments (written to the SAME log file after the worker
|
|
217
|
+
# exits, e.g. by the push-attempt bookkeeping in `_reap`) are skipped so this
|
|
218
|
+
# still reaches the worker's own real last line despite them.
|
|
219
|
+
_USAGE_LIMIT_TAIL_LINES = 1
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def format_usage_limit_reason(kill: UsageLimitKill) -> str:
|
|
223
|
+
"""Render *kill* as the one-liner stamped onto ``failure_reason``."""
|
|
224
|
+
return f"{USAGE_LIMIT_REASON_PREFIX}{kill.reset_at_raw}"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def is_usage_limit_reason(reason: str | None) -> bool:
|
|
228
|
+
"""True iff *reason* is a `failure_reason` stamped by this detector."""
|
|
229
|
+
return bool(reason) and reason.startswith(USAGE_LIMIT_REASON_PREFIX)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def detect_usage_limit_kill(text: str) -> UsageLimitKill | None:
|
|
233
|
+
"""Scan *text* (a transcript, or any tail slice of one) for the "hit your
|
|
234
|
+
session limit" kill message.
|
|
235
|
+
|
|
236
|
+
Only the last few meaningful (non-blank, non ``#``-comment) lines are
|
|
237
|
+
considered — see ``_USAGE_LIMIT_TAIL_LINES`` — so an incidental mention
|
|
238
|
+
of "session limit" earlier in a normal, successfully-completed
|
|
239
|
+
conversation is never mistaken for a kill. Returns ``None`` when no match
|
|
240
|
+
is found.
|
|
241
|
+
"""
|
|
242
|
+
lines = [
|
|
243
|
+
ln for ln in text.splitlines()
|
|
244
|
+
if ln.strip() and not ln.lstrip().startswith("#")
|
|
245
|
+
]
|
|
246
|
+
for line in reversed(lines[-_USAGE_LIMIT_TAIL_LINES:]):
|
|
247
|
+
m = _USAGE_LIMIT_RE.search(line)
|
|
248
|
+
if not m:
|
|
249
|
+
continue
|
|
250
|
+
reset = m.group(1).strip().strip("\"'.,;: \t")
|
|
251
|
+
if reset:
|
|
252
|
+
return UsageLimitKill(reset_at_raw=reset, excerpt=line[:500])
|
|
253
|
+
return None
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def detect_usage_limit_kill_in_log(
|
|
257
|
+
log_path: str | Path, tail_bytes: int = 65536
|
|
258
|
+
) -> UsageLimitKill | None:
|
|
259
|
+
""":func:`detect_usage_limit_kill` over the tail of *log_path*.
|
|
260
|
+
|
|
261
|
+
Reads at most *tail_bytes* from the end of the file — a kill message is
|
|
262
|
+
always the transcript's last line, so there is never a need to read the
|
|
263
|
+
whole (potentially multi-MB) log. Returns ``None`` for a missing file or
|
|
264
|
+
any read error (best-effort, mirrors ``is_stream_json``).
|
|
265
|
+
"""
|
|
266
|
+
p = Path(log_path)
|
|
267
|
+
try:
|
|
268
|
+
size = p.stat().st_size
|
|
269
|
+
with open(p, "rb") as f:
|
|
270
|
+
if size > tail_bytes:
|
|
271
|
+
f.seek(-tail_bytes, 2)
|
|
272
|
+
data = f.read()
|
|
273
|
+
except OSError:
|
|
274
|
+
return None
|
|
275
|
+
return detect_usage_limit_kill(data.decode("utf-8", errors="replace"))
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
# ── Terminal API-error classification (#1584) ───────────────────────────────
|
|
279
|
+
#
|
|
280
|
+
# `is_error: true` on a worker's TERMINAL `result` event (`WorkerSummary
|
|
281
|
+
# .is_error`, populated above) means the session ended on a real failure —
|
|
282
|
+
# most often a transient upstream problem (529 Overloaded, 500, a network
|
|
283
|
+
# drop) that killed the worker before it did anything:
|
|
284
|
+
#
|
|
285
|
+
# {"is_error": true, "num_turns": 1, "stop_reason": "stop_sequence",
|
|
286
|
+
# "terminal_reason": "api_error", "api_error_status": 529,
|
|
287
|
+
# "result": "API Error: 529 Overloaded. This is a server-side issue,
|
|
288
|
+
# usually temporary...", "total_cost_usd": 0.026247}
|
|
289
|
+
#
|
|
290
|
+
# Before this, nothing mapped `is_error` to assignment status (it was read
|
|
291
|
+
# only for `coord watch`'s display string — see `format_important_event`
|
|
292
|
+
# below) so this recorded a clean `done`, indistinguishable from a real
|
|
293
|
+
# success. `format_api_error_reason` renders the three diagnostic fields
|
|
294
|
+
# into the one-line reason `AgentServer._reap` stamps onto
|
|
295
|
+
# `AgentAssignment.api_error_reason` when it flips the assignment to FAILED.
|
|
296
|
+
|
|
297
|
+
# Pulls the short phrase (e.g. "Overloaded") out of the raw `result` text
|
|
298
|
+
# that follows an "API Error: <status>" prefix, when present.
|
|
299
|
+
_API_ERROR_PHRASE_RE = re.compile(r"API Error:\s*\d+\s+([^.\r\n]+)")
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def format_api_error_reason(
|
|
303
|
+
*,
|
|
304
|
+
terminal_reason: str | None,
|
|
305
|
+
api_error_status: int | None,
|
|
306
|
+
result_text: str | None = None,
|
|
307
|
+
) -> str:
|
|
308
|
+
"""Render a terminal API-error `result` event as a one-line failure reason.
|
|
309
|
+
|
|
310
|
+
Prefers ``"<status> <phrase>"`` (e.g. ``"529 Overloaded"``) when both the
|
|
311
|
+
structured *api_error_status* and a matching phrase in *result_text* are
|
|
312
|
+
available — the shape from #1584's own worked example. Falls back to
|
|
313
|
+
whatever subset of the three fields is present, so a future
|
|
314
|
+
``terminal_reason``/status combination this doesn't specifically
|
|
315
|
+
recognise still renders something greppable rather than raising or
|
|
316
|
+
going silent.
|
|
317
|
+
"""
|
|
318
|
+
phrase: str | None = None
|
|
319
|
+
if result_text:
|
|
320
|
+
m = _API_ERROR_PHRASE_RE.search(result_text)
|
|
321
|
+
if m:
|
|
322
|
+
phrase = m.group(1).strip().rstrip(".") or None
|
|
323
|
+
if api_error_status is not None and phrase:
|
|
324
|
+
return f"{api_error_status} {phrase}"
|
|
325
|
+
if api_error_status is not None:
|
|
326
|
+
# Only append the parenthetical when `terminal_reason` says something
|
|
327
|
+
# `"api_error"` alone doesn't already — the common case (this exact
|
|
328
|
+
# field is almost always the literal string `"api_error"`) would
|
|
329
|
+
# otherwise render the redundant `"api_error 500 (api_error)"`.
|
|
330
|
+
if terminal_reason and terminal_reason != "api_error":
|
|
331
|
+
return f"api_error {api_error_status} ({terminal_reason})"
|
|
332
|
+
return f"api_error {api_error_status}"
|
|
333
|
+
if terminal_reason:
|
|
334
|
+
return f"api_error: {terminal_reason}"
|
|
335
|
+
return "api_error"
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# ── Field extraction helpers ────────────────────────────────────────────────
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _is_bash_tool_use(event: WorkerEvent) -> bool:
|
|
342
|
+
"""True iff this event represents a Bash tool invocation."""
|
|
343
|
+
if event.type not in ("tool_use", "assistant"):
|
|
344
|
+
return False
|
|
345
|
+
tool_name = _tool_name_from_event(event)
|
|
346
|
+
return tool_name == "Bash"
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _tool_name_from_event(event: WorkerEvent) -> str | None:
|
|
350
|
+
"""Try a few plausible field paths for the tool name."""
|
|
351
|
+
raw = event.raw
|
|
352
|
+
if event.type == "tool_use":
|
|
353
|
+
return raw.get("name") or raw.get("tool") or raw.get("tool_name")
|
|
354
|
+
# Assistant events may embed a tool_use block in `message.content[*]`.
|
|
355
|
+
if event.type == "assistant":
|
|
356
|
+
message = raw.get("message") or {}
|
|
357
|
+
for block in _iter_content_blocks(message):
|
|
358
|
+
if block.get("type") == "tool_use":
|
|
359
|
+
return block.get("name")
|
|
360
|
+
return None
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _iter_content_blocks(message: dict) -> Iterable[dict]:
|
|
364
|
+
"""Yield content blocks from an Anthropic-style message payload."""
|
|
365
|
+
content = message.get("content")
|
|
366
|
+
if isinstance(content, list):
|
|
367
|
+
for block in content:
|
|
368
|
+
if isinstance(block, dict):
|
|
369
|
+
yield block
|
|
370
|
+
elif isinstance(content, dict):
|
|
371
|
+
yield content
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _bash_command_from_event(event: WorkerEvent) -> str | None:
|
|
375
|
+
raw = event.raw
|
|
376
|
+
# Direct tool_use form: {"type":"tool_use","name":"Bash","input":{"command":"..."}}
|
|
377
|
+
if event.type == "tool_use" and raw.get("name") == "Bash":
|
|
378
|
+
return _command_from_input(raw.get("input"))
|
|
379
|
+
if event.type == "assistant":
|
|
380
|
+
message = raw.get("message") or {}
|
|
381
|
+
for block in _iter_content_blocks(message):
|
|
382
|
+
if block.get("type") == "tool_use" and block.get("name") == "Bash":
|
|
383
|
+
return _command_from_input(block.get("input"))
|
|
384
|
+
return None
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _command_from_input(input_obj: object) -> str | None:
|
|
388
|
+
if not isinstance(input_obj, dict):
|
|
389
|
+
return None
|
|
390
|
+
cmd = input_obj.get("command")
|
|
391
|
+
if isinstance(cmd, str):
|
|
392
|
+
return cmd
|
|
393
|
+
return None
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _file_path_from_event(event: WorkerEvent) -> str | None:
|
|
397
|
+
"""Pull file_path out of an Edit/Write tool_use, if present."""
|
|
398
|
+
raw = event.raw
|
|
399
|
+
name = _tool_name_from_event(event)
|
|
400
|
+
if name not in ("Edit", "Write", "NotebookEdit"):
|
|
401
|
+
return None
|
|
402
|
+
if event.type == "tool_use":
|
|
403
|
+
return _file_from_input(raw.get("input"))
|
|
404
|
+
if event.type == "assistant":
|
|
405
|
+
message = raw.get("message") or {}
|
|
406
|
+
for block in _iter_content_blocks(message):
|
|
407
|
+
if block.get("type") == "tool_use" and block.get("name") in (
|
|
408
|
+
"Edit",
|
|
409
|
+
"Write",
|
|
410
|
+
"NotebookEdit",
|
|
411
|
+
):
|
|
412
|
+
return _file_from_input(block.get("input"))
|
|
413
|
+
return None
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _file_from_input(input_obj: object) -> str | None:
|
|
417
|
+
if not isinstance(input_obj, dict):
|
|
418
|
+
return None
|
|
419
|
+
for key in ("file_path", "path", "notebook_path"):
|
|
420
|
+
v = input_obj.get(key)
|
|
421
|
+
if isinstance(v, str):
|
|
422
|
+
return v
|
|
423
|
+
return None
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
# ── rate_limit_event wire shape (#1466) ─────────────────────────────────────
|
|
427
|
+
#
|
|
428
|
+
# Claude Code v2.1.220 emits a `rate_limit_event` on essentially every run —
|
|
429
|
+
# it is the *healthy* case, not a throttle signal:
|
|
430
|
+
#
|
|
431
|
+
# {"type": "rate_limit_event",
|
|
432
|
+
# "rate_limit_info": {"status": "allowed", "resetsAt": 1785133800,
|
|
433
|
+
# "rateLimitType": "five_hour", "overageStatus": "rejected",
|
|
434
|
+
# "overageDisabledReason": "org_level_disabled", "isUsingOverage": false},
|
|
435
|
+
# "uuid": "...", "session_id": "..."}
|
|
436
|
+
#
|
|
437
|
+
# Everything lives nested under `rate_limit_info`, camelCase. There is no
|
|
438
|
+
# top-level `resets_at`/`reset_at` — that shape was invented (pre-#1466) and
|
|
439
|
+
# the real CLI never emits it, which is why `rate_limit_resets_at` was always
|
|
440
|
+
# ``None`` and `render_event` always printed `resets_at=?`. Only
|
|
441
|
+
# `allowed_warning` and `rejected` mean the account is actually throttled;
|
|
442
|
+
# `allowed` is the normal, common case and must never set `rate_limited`.
|
|
443
|
+
#
|
|
444
|
+
# `format_important_event` already read this shape correctly (nested,
|
|
445
|
+
# camelCase, status-gated) — this helper is the single place both it and
|
|
446
|
+
# `update_summary`/`render_event` now go through, so the two paths can't
|
|
447
|
+
# disagree about the wire format again.
|
|
448
|
+
_RATE_LIMIT_THROTTLED_STATUSES = frozenset({"allowed_warning", "rejected"})
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _rate_limit_info(raw: dict) -> tuple[str | None, float | None]:
|
|
452
|
+
"""Pull ``(status, resets_at)`` out of a ``rate_limit_event``'s payload.
|
|
453
|
+
|
|
454
|
+
Returns ``(None, None)`` when ``rate_limit_info`` is missing or not a
|
|
455
|
+
dict — a shape we don't recognise, not something to guess at.
|
|
456
|
+
"""
|
|
457
|
+
info = raw.get("rate_limit_info")
|
|
458
|
+
if not isinstance(info, dict):
|
|
459
|
+
return None, None
|
|
460
|
+
status = info.get("status")
|
|
461
|
+
if not isinstance(status, str):
|
|
462
|
+
status = None
|
|
463
|
+
resets = info.get("resetsAt")
|
|
464
|
+
if not isinstance(resets, (int, float)):
|
|
465
|
+
resets = None
|
|
466
|
+
else:
|
|
467
|
+
resets = float(resets)
|
|
468
|
+
return status, resets
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _is_rate_limit_throttled(status: str | None) -> bool:
|
|
472
|
+
"""True iff *status* means the account is actually being throttled.
|
|
473
|
+
|
|
474
|
+
``allowed`` (no status, or any other value) is the healthy/normal case.
|
|
475
|
+
"""
|
|
476
|
+
return status in _RATE_LIMIT_THROTTLED_STATUSES
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _assistant_text(event: WorkerEvent) -> str:
|
|
480
|
+
"""First text block from an assistant message, truncated for display."""
|
|
481
|
+
raw = event.raw
|
|
482
|
+
message = raw.get("message") or {}
|
|
483
|
+
for block in _iter_content_blocks(message):
|
|
484
|
+
if block.get("type") == "text":
|
|
485
|
+
txt = block.get("text") or ""
|
|
486
|
+
if isinstance(txt, str):
|
|
487
|
+
return txt.strip()
|
|
488
|
+
# Some shapes carry top-level text on the event itself.
|
|
489
|
+
direct = raw.get("text")
|
|
490
|
+
if isinstance(direct, str):
|
|
491
|
+
return direct.strip()
|
|
492
|
+
return ""
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
# ── Streaming summary update ───────────────────────────────────────────────
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def update_summary(summary: WorkerSummary, event: WorkerEvent) -> None:
|
|
499
|
+
"""Fold *event* into *summary* in-place."""
|
|
500
|
+
raw = event.raw
|
|
501
|
+
|
|
502
|
+
if event.type == "system" and event.subtype == "init":
|
|
503
|
+
sid = raw.get("session_id") or raw.get("id")
|
|
504
|
+
if isinstance(sid, str):
|
|
505
|
+
summary.session_id = sid
|
|
506
|
+
model = raw.get("model") or (raw.get("config") or {}).get("model")
|
|
507
|
+
if isinstance(model, str) and not summary.model_used:
|
|
508
|
+
summary.model_used = model
|
|
509
|
+
return
|
|
510
|
+
|
|
511
|
+
if event.type == "assistant":
|
|
512
|
+
summary.num_turns += 1
|
|
513
|
+
message = raw.get("message") or {}
|
|
514
|
+
model = message.get("model") or raw.get("model")
|
|
515
|
+
if isinstance(model, str):
|
|
516
|
+
summary.model_used = model
|
|
517
|
+
# Tool uses can be nested in the assistant message content.
|
|
518
|
+
for block in _iter_content_blocks(message):
|
|
519
|
+
if block.get("type") == "tool_use":
|
|
520
|
+
name = block.get("name")
|
|
521
|
+
if isinstance(name, str):
|
|
522
|
+
summary.tools_used.append(name)
|
|
523
|
+
summary.last_tool = name
|
|
524
|
+
if name == "Bash":
|
|
525
|
+
cmd = _command_from_input(block.get("input"))
|
|
526
|
+
if cmd:
|
|
527
|
+
summary.bash_commands.append(cmd)
|
|
528
|
+
elif name in ("Edit", "Write", "NotebookEdit"):
|
|
529
|
+
fp = _file_from_input(block.get("input"))
|
|
530
|
+
if fp:
|
|
531
|
+
summary.files_edited.append(fp)
|
|
532
|
+
return
|
|
533
|
+
|
|
534
|
+
if event.type == "tool_use":
|
|
535
|
+
name = _tool_name_from_event(event)
|
|
536
|
+
if name:
|
|
537
|
+
summary.tools_used.append(name)
|
|
538
|
+
summary.last_tool = name
|
|
539
|
+
if name == "Bash":
|
|
540
|
+
cmd = _bash_command_from_event(event)
|
|
541
|
+
if cmd:
|
|
542
|
+
summary.bash_commands.append(cmd)
|
|
543
|
+
elif name in ("Edit", "Write", "NotebookEdit"):
|
|
544
|
+
fp = _file_path_from_event(event)
|
|
545
|
+
if fp:
|
|
546
|
+
summary.files_edited.append(fp)
|
|
547
|
+
return
|
|
548
|
+
|
|
549
|
+
if event.type == "rate_limit_event":
|
|
550
|
+
status, resets = _rate_limit_info(raw)
|
|
551
|
+
if _is_rate_limit_throttled(status):
|
|
552
|
+
summary.rate_limited = True
|
|
553
|
+
if resets is not None:
|
|
554
|
+
summary.rate_limit_resets_at = resets
|
|
555
|
+
return
|
|
556
|
+
|
|
557
|
+
if event.type == "result":
|
|
558
|
+
# #1584: overwrite (never OR/append) on every `result` event so the
|
|
559
|
+
# final state after a full parse reflects only the LAST one — a
|
|
560
|
+
# worker that hit a transient API error, retried internally, and
|
|
561
|
+
# finished cleanly has an earlier `result` line with `is_error: true`
|
|
562
|
+
# followed by a final one without it, and only the latter must
|
|
563
|
+
# survive.
|
|
564
|
+
summary.is_error = bool(raw.get("is_error"))
|
|
565
|
+
tr = raw.get("terminal_reason")
|
|
566
|
+
summary.terminal_reason = tr if isinstance(tr, str) else None
|
|
567
|
+
aes = raw.get("api_error_status")
|
|
568
|
+
summary.api_error_status = aes if isinstance(aes, int) else None
|
|
569
|
+
rtext = raw.get("result")
|
|
570
|
+
summary.result_text = rtext if isinstance(rtext, str) else None
|
|
571
|
+
cost = raw.get("total_cost_usd") or raw.get("cost_usd")
|
|
572
|
+
if isinstance(cost, (int, float)):
|
|
573
|
+
summary.total_cost_usd = float(cost)
|
|
574
|
+
stop = raw.get("stop_reason") or raw.get("subtype")
|
|
575
|
+
if isinstance(stop, str):
|
|
576
|
+
summary.stop_reason = stop
|
|
577
|
+
turns = raw.get("num_turns")
|
|
578
|
+
if isinstance(turns, int) and turns >= summary.num_turns:
|
|
579
|
+
# Prefer the explicit count from claude when available.
|
|
580
|
+
summary.num_turns = turns
|
|
581
|
+
dur = raw.get("duration_ms") or raw.get("duration")
|
|
582
|
+
if isinstance(dur, (int, float)):
|
|
583
|
+
summary.duration_ms = int(dur)
|
|
584
|
+
denials = raw.get("permission_denials") or []
|
|
585
|
+
if isinstance(denials, list):
|
|
586
|
+
for d in denials:
|
|
587
|
+
if isinstance(d, str):
|
|
588
|
+
summary.permission_denials.append(d)
|
|
589
|
+
elif isinstance(d, dict):
|
|
590
|
+
label = (
|
|
591
|
+
d.get("tool_name")
|
|
592
|
+
or d.get("tool")
|
|
593
|
+
or d.get("name")
|
|
594
|
+
or json.dumps(d, sort_keys=True)
|
|
595
|
+
)
|
|
596
|
+
summary.permission_denials.append(str(label))
|
|
597
|
+
# Extract token counts. Claude may report them under a nested
|
|
598
|
+
# ``usage`` object or at the top level — try both forms.
|
|
599
|
+
usage_obj = raw.get("usage") or {}
|
|
600
|
+
if not isinstance(usage_obj, dict):
|
|
601
|
+
usage_obj = {}
|
|
602
|
+
|
|
603
|
+
def _tok(key: str, *alt_keys: str) -> int:
|
|
604
|
+
"""Return first non-zero int found across key variants."""
|
|
605
|
+
for k in (key, *alt_keys):
|
|
606
|
+
v = usage_obj.get(k) or raw.get(k)
|
|
607
|
+
if isinstance(v, int) and v > 0:
|
|
608
|
+
return v
|
|
609
|
+
return 0
|
|
610
|
+
|
|
611
|
+
summary.input_tokens = _tok("input_tokens")
|
|
612
|
+
summary.output_tokens = _tok("output_tokens")
|
|
613
|
+
summary.cache_creation_tokens = _tok(
|
|
614
|
+
"cache_creation_input_tokens", "cache_creation_tokens"
|
|
615
|
+
)
|
|
616
|
+
summary.cache_read_tokens = _tok(
|
|
617
|
+
"cache_read_input_tokens", "cache_read_tokens"
|
|
618
|
+
)
|
|
619
|
+
return
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
# ── File-level helpers ──────────────────────────────────────────────────────
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _read_tail(path: Path, tail_bytes: int) -> str:
|
|
626
|
+
size = path.stat().st_size
|
|
627
|
+
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
|
628
|
+
if tail_bytes and size > tail_bytes:
|
|
629
|
+
f.seek(size - tail_bytes)
|
|
630
|
+
f.readline() # discard partial line
|
|
631
|
+
return f.read()
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def iter_events_from_text(text: str) -> Iterable[WorkerEvent]:
|
|
635
|
+
"""Yield :class:`WorkerEvent` for each parseable line in an in-memory
|
|
636
|
+
string. Shared by :func:`iter_events` (file-backed) and callers that
|
|
637
|
+
already have the log text in hand (e.g. fetched over HTTP from a
|
|
638
|
+
remote agent) and want to avoid a redundant write-then-read."""
|
|
639
|
+
for line in text.splitlines():
|
|
640
|
+
ev = parse_event(line)
|
|
641
|
+
if ev is not None:
|
|
642
|
+
yield ev
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def iter_events(log_path: str | Path, *, tail_bytes: int = 0) -> Iterable[WorkerEvent]:
|
|
646
|
+
"""Yield :class:`WorkerEvent` for each parseable line in *log_path*.
|
|
647
|
+
|
|
648
|
+
With ``tail_bytes`` > 0, only the last *tail_bytes* of the file is read
|
|
649
|
+
(after skipping a partial leading line). Use this for cheap polling of
|
|
650
|
+
live, long-running assignments.
|
|
651
|
+
"""
|
|
652
|
+
p = Path(log_path)
|
|
653
|
+
if not p.exists():
|
|
654
|
+
return
|
|
655
|
+
try:
|
|
656
|
+
text = _read_tail(p, tail_bytes)
|
|
657
|
+
except OSError:
|
|
658
|
+
return
|
|
659
|
+
yield from iter_events_from_text(text)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
# ── Single-latest-turn extraction (#2048) ───────────────────────────────────
|
|
663
|
+
#
|
|
664
|
+
# The liveness auditor (coord/liveness_auditor.py) must see ONLY the single
|
|
665
|
+
# most recent assistant turn — never the transcript. These helpers pick that
|
|
666
|
+
# one turn's text (or, for a tool-only turn with no text block, a compact
|
|
667
|
+
# summary of which tools it called) out of a stream-json log.
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _turn_text_or_tool_summary(event: WorkerEvent) -> str:
|
|
671
|
+
text = _assistant_text(event)
|
|
672
|
+
if text:
|
|
673
|
+
return text
|
|
674
|
+
message = event.raw.get("message") or {}
|
|
675
|
+
tool_names = [
|
|
676
|
+
block.get("name")
|
|
677
|
+
for block in _iter_content_blocks(message)
|
|
678
|
+
if block.get("type") == "tool_use"
|
|
679
|
+
]
|
|
680
|
+
tool_names = [t for t in tool_names if t]
|
|
681
|
+
return f"[tool_use: {', '.join(tool_names)}]" if tool_names else ""
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def latest_assistant_turn_text_from_text(text: str) -> str | None:
|
|
685
|
+
"""Return the most recent assistant turn's text (or tool-use summary)
|
|
686
|
+
found in *text*, or ``None`` if the text contains no assistant turn at
|
|
687
|
+
all. An empty string is a real, meaningful result (the last turn
|
|
688
|
+
produced neither text nor a recognised tool call) and is distinct from
|
|
689
|
+
``None`` (no turn found to look at)."""
|
|
690
|
+
found = False
|
|
691
|
+
last_text = ""
|
|
692
|
+
for event in iter_events_from_text(text):
|
|
693
|
+
if event.type != "assistant":
|
|
694
|
+
continue
|
|
695
|
+
found = True
|
|
696
|
+
last_text = _turn_text_or_tool_summary(event)
|
|
697
|
+
return last_text if found else None
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def latest_assistant_turn_text(
|
|
701
|
+
log_path: str | Path, *, tail_bytes: int = 65536
|
|
702
|
+
) -> str | None:
|
|
703
|
+
"""File-backed counterpart to
|
|
704
|
+
:func:`latest_assistant_turn_text_from_text` — reads only the tail of
|
|
705
|
+
*log_path* (a full stream-json transcript can be multi-MB; the auditor
|
|
706
|
+
only ever needs the last turn) and returns ``None`` for a missing file,
|
|
707
|
+
read error, or a tail slice with no assistant turn in it."""
|
|
708
|
+
p = Path(log_path)
|
|
709
|
+
if not p.exists():
|
|
710
|
+
return None
|
|
711
|
+
try:
|
|
712
|
+
text = _read_tail(p, tail_bytes)
|
|
713
|
+
except OSError:
|
|
714
|
+
return None
|
|
715
|
+
return latest_assistant_turn_text_from_text(text)
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def parse_log(log_path: str | Path, tail_bytes: int = 65536) -> WorkerSummary:
|
|
719
|
+
"""Parse a stream-json log file into a :class:`WorkerSummary`.
|
|
720
|
+
|
|
721
|
+
For active assignments we only read the tail to stay cheap. The fields
|
|
722
|
+
that come from the ``init`` event (session_id, model) and per-turn
|
|
723
|
+
accumulations (cost, turns) are still useful even from a tail read,
|
|
724
|
+
though session_id may be missing if the head of the log has rolled off.
|
|
725
|
+
Callers that need a fully reliable summary should pass ``tail_bytes=0``.
|
|
726
|
+
"""
|
|
727
|
+
summary = WorkerSummary()
|
|
728
|
+
for event in iter_events(log_path, tail_bytes=tail_bytes):
|
|
729
|
+
update_summary(summary, event)
|
|
730
|
+
return summary
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
# ── Human-readable rendering ────────────────────────────────────────────────
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _truncate(text: str, n: int = 80) -> str:
|
|
737
|
+
text = text.replace("\n", " ").strip()
|
|
738
|
+
if len(text) <= n:
|
|
739
|
+
return text
|
|
740
|
+
return text[: n - 1] + "…"
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _format_duration(ms: int | None) -> str:
|
|
744
|
+
if ms is None:
|
|
745
|
+
return "?"
|
|
746
|
+
seconds = ms / 1000.0
|
|
747
|
+
if seconds < 60:
|
|
748
|
+
return f"{seconds:.1f}s"
|
|
749
|
+
minutes, secs = divmod(int(seconds), 60)
|
|
750
|
+
if minutes < 60:
|
|
751
|
+
return f"{minutes}m {secs}s"
|
|
752
|
+
hours, minutes = divmod(minutes, 60)
|
|
753
|
+
return f"{hours}h {minutes}m"
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def render_event(event: WorkerEvent, *, turn_counter: list[int] | None = None) -> str | None:
|
|
757
|
+
"""Render an event as a single human-readable line. Returns None to skip."""
|
|
758
|
+
raw = event.raw
|
|
759
|
+
|
|
760
|
+
if event.type == "system" and event.subtype == "init":
|
|
761
|
+
model = raw.get("model") or (raw.get("config") or {}).get("model") or "?"
|
|
762
|
+
sid = raw.get("session_id") or raw.get("id") or "?"
|
|
763
|
+
return f"[init] model={model} session={sid}"
|
|
764
|
+
|
|
765
|
+
if event.type == "assistant":
|
|
766
|
+
if turn_counter is not None:
|
|
767
|
+
turn_counter[0] += 1
|
|
768
|
+
n = turn_counter[0]
|
|
769
|
+
else:
|
|
770
|
+
n = 0
|
|
771
|
+
text = _assistant_text(event)
|
|
772
|
+
# If this assistant turn is purely a tool call, the text block may
|
|
773
|
+
# be empty — render a placeholder so the timeline still ticks.
|
|
774
|
+
if text:
|
|
775
|
+
return f"[assistant] Turn {n}: {_truncate(text, 100)!r}"
|
|
776
|
+
# Try to summarise the tool calls.
|
|
777
|
+
message = raw.get("message") or {}
|
|
778
|
+
tool_names = [
|
|
779
|
+
block.get("name")
|
|
780
|
+
for block in _iter_content_blocks(message)
|
|
781
|
+
if block.get("type") == "tool_use"
|
|
782
|
+
]
|
|
783
|
+
tool_names = [t for t in tool_names if t]
|
|
784
|
+
if tool_names:
|
|
785
|
+
return f"[assistant] Turn {n}: tool_use={','.join(tool_names)}"
|
|
786
|
+
return f"[assistant] Turn {n}"
|
|
787
|
+
|
|
788
|
+
if event.type == "tool_use":
|
|
789
|
+
name = _tool_name_from_event(event) or "?"
|
|
790
|
+
if name == "Bash":
|
|
791
|
+
cmd = _bash_command_from_event(event) or ""
|
|
792
|
+
return f"[tool] Bash: {_truncate(cmd, 100)}"
|
|
793
|
+
if name in ("Edit", "Write", "NotebookEdit"):
|
|
794
|
+
fp = _file_path_from_event(event)
|
|
795
|
+
return f"[tool] {name}: {fp or '?'}"
|
|
796
|
+
return f"[tool] {name}"
|
|
797
|
+
|
|
798
|
+
if event.type == "tool_result":
|
|
799
|
+
# Tool results are usually noisy — keep a compact form.
|
|
800
|
+
tool_use_id = raw.get("tool_use_id") or "?"
|
|
801
|
+
is_error = raw.get("is_error")
|
|
802
|
+
tag = " error" if is_error else ""
|
|
803
|
+
return f"[tool_result{tag}] {tool_use_id}"
|
|
804
|
+
|
|
805
|
+
if event.type == "rate_limit_event":
|
|
806
|
+
status, resets = _rate_limit_info(raw)
|
|
807
|
+
return f"[rate_limit] status={status or '?'} resets_at={resets if resets is not None else '?'}"
|
|
808
|
+
|
|
809
|
+
if event.type == "result":
|
|
810
|
+
cost = raw.get("total_cost_usd") or raw.get("cost_usd") or 0.0
|
|
811
|
+
stop = raw.get("stop_reason") or raw.get("subtype") or "?"
|
|
812
|
+
turns = raw.get("num_turns") or "?"
|
|
813
|
+
dur = _format_duration(raw.get("duration_ms"))
|
|
814
|
+
return (
|
|
815
|
+
f"[result] completed in {dur}, {turns} turns, "
|
|
816
|
+
f"${float(cost):.2f}, stop={stop}"
|
|
817
|
+
)
|
|
818
|
+
|
|
819
|
+
# Anything else: render type/subtype only — keep one line.
|
|
820
|
+
if event.subtype:
|
|
821
|
+
return f"[{event.type}] {event.subtype}"
|
|
822
|
+
return f"[{event.type}]"
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def render_log(log_path: str | Path) -> Iterable[str]:
|
|
826
|
+
"""Yield rendered lines for every event in *log_path*."""
|
|
827
|
+
turn_counter = [0]
|
|
828
|
+
for event in iter_events(log_path):
|
|
829
|
+
line = render_event(event, turn_counter=turn_counter)
|
|
830
|
+
if line is not None:
|
|
831
|
+
yield line
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def format_important_event(event: WorkerEvent) -> str | None:
|
|
835
|
+
"""Format an event for ``coord watch`` output.
|
|
836
|
+
|
|
837
|
+
Returns a human-readable string if the event is *important* (i.e. worth
|
|
838
|
+
showing in filtered live output), or ``None`` to skip it.
|
|
839
|
+
"""
|
|
840
|
+
raw = event.raw
|
|
841
|
+
|
|
842
|
+
if event.type == "system" and event.subtype == "init":
|
|
843
|
+
model = raw.get("model") or (raw.get("config") or {}).get("model") or "unknown"
|
|
844
|
+
session = str(raw.get("session_id") or raw.get("id") or "?")[:8]
|
|
845
|
+
return f"[init] {model} session {session}"
|
|
846
|
+
|
|
847
|
+
if event.type == "rate_limit_event":
|
|
848
|
+
status, resets = _rate_limit_info(raw)
|
|
849
|
+
# Only surface throttled events — `allowed` is the healthy, common
|
|
850
|
+
# case (fires on essentially every run) and must stay silent.
|
|
851
|
+
if _is_rate_limit_throttled(status):
|
|
852
|
+
return f"[rate_limit] {status}, resets at {resets if resets is not None else '?'}"
|
|
853
|
+
return None
|
|
854
|
+
|
|
855
|
+
if event.type == "result":
|
|
856
|
+
dur = (raw.get("duration_ms") or 0) / 1000
|
|
857
|
+
turns = raw.get("num_turns") or 0
|
|
858
|
+
cost = raw.get("total_cost_usd") or raw.get("cost_usd") or 0
|
|
859
|
+
stop = raw.get("stop_reason") or raw.get("subtype") or "?"
|
|
860
|
+
is_err = raw.get("is_error", False)
|
|
861
|
+
mins, secs = divmod(int(dur), 60)
|
|
862
|
+
result_status = "failed" if is_err else "completed"
|
|
863
|
+
base = f"[result] {result_status} in {mins}m {secs}s, {turns} turns, ${float(cost):.2f}, stop={stop}"
|
|
864
|
+
# Surface permission denials attached to the result event
|
|
865
|
+
denials = raw.get("permission_denials") or []
|
|
866
|
+
denial_lines: list[str] = []
|
|
867
|
+
if isinstance(denials, list):
|
|
868
|
+
for d in denials:
|
|
869
|
+
if isinstance(d, str):
|
|
870
|
+
denial_lines.append(f"[denied] {d}")
|
|
871
|
+
elif isinstance(d, dict):
|
|
872
|
+
label = (
|
|
873
|
+
d.get("tool_name")
|
|
874
|
+
or d.get("tool")
|
|
875
|
+
or d.get("name")
|
|
876
|
+
or d.get("reason")
|
|
877
|
+
or str(d)
|
|
878
|
+
)
|
|
879
|
+
reason = d.get("reason") or d.get("message") or ""
|
|
880
|
+
if reason:
|
|
881
|
+
denial_lines.append(f"[denied] {label}: {reason}")
|
|
882
|
+
else:
|
|
883
|
+
denial_lines.append(f"[denied] {label}")
|
|
884
|
+
if denial_lines:
|
|
885
|
+
return base + "\n" + "\n".join(denial_lines)
|
|
886
|
+
return base
|
|
887
|
+
|
|
888
|
+
if event.type == "assistant":
|
|
889
|
+
# Scan text blocks for STUCK: signal
|
|
890
|
+
message = raw.get("message") or {}
|
|
891
|
+
content = message.get("content") or []
|
|
892
|
+
if isinstance(content, list):
|
|
893
|
+
for block in content:
|
|
894
|
+
if isinstance(block, dict) and block.get("type") == "text":
|
|
895
|
+
text = block.get("text") or ""
|
|
896
|
+
if "STUCK:" in text:
|
|
897
|
+
stuck_line = next(
|
|
898
|
+
(ln for ln in text.split("\n") if "STUCK:" in ln),
|
|
899
|
+
text[:200],
|
|
900
|
+
)
|
|
901
|
+
return f"[stuck] {stuck_line.strip()}"
|
|
902
|
+
return None
|
|
903
|
+
|
|
904
|
+
return None
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
# ── Anomaly detection ──────────────────────────────────────────────────────
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def detect_anomalies(log_path: str | Path, *, tail_bytes: int = 65536) -> list[str]:
|
|
911
|
+
"""Scan a stream-json log for anomaly patterns. Returns warning strings."""
|
|
912
|
+
warnings: list[str] = []
|
|
913
|
+
summary = WorkerSummary()
|
|
914
|
+
bash_cmds: list[str] = []
|
|
915
|
+
saw_commit = False
|
|
916
|
+
|
|
917
|
+
for event in iter_events(log_path, tail_bytes=tail_bytes):
|
|
918
|
+
update_summary(summary, event)
|
|
919
|
+
cmd = _bash_command_from_event(event)
|
|
920
|
+
if cmd:
|
|
921
|
+
bash_cmds.append(cmd)
|
|
922
|
+
# A `git commit` command (with or without flags) breaks the
|
|
923
|
+
# "many turns, no commit" pattern.
|
|
924
|
+
if cmd.lstrip().startswith("git commit"):
|
|
925
|
+
saw_commit = True
|
|
926
|
+
|
|
927
|
+
# Repeated identical bash invocations.
|
|
928
|
+
if bash_cmds:
|
|
929
|
+
counts = Counter(bash_cmds)
|
|
930
|
+
for cmd, n in counts.items():
|
|
931
|
+
if n >= 3:
|
|
932
|
+
warnings.append(
|
|
933
|
+
f"bash command repeated {n}x: {_truncate(cmd, 60)}"
|
|
934
|
+
)
|
|
935
|
+
|
|
936
|
+
# Rate-limit hit anywhere in the log.
|
|
937
|
+
if summary.rate_limited:
|
|
938
|
+
resets = summary.rate_limit_resets_at
|
|
939
|
+
warnings.append(
|
|
940
|
+
f"rate limited (resets at {resets})" if resets else "rate limited"
|
|
941
|
+
)
|
|
942
|
+
|
|
943
|
+
# Permission denials in the final result.
|
|
944
|
+
if summary.permission_denials:
|
|
945
|
+
joined = ", ".join(summary.permission_denials[:5])
|
|
946
|
+
warnings.append(f"permission denials: {joined}")
|
|
947
|
+
|
|
948
|
+
# Many turns without a commit — possible runaway / lost worker.
|
|
949
|
+
if summary.num_turns >= 15 and not saw_commit:
|
|
950
|
+
warnings.append(
|
|
951
|
+
f"{summary.num_turns} turns without a git commit"
|
|
952
|
+
)
|
|
953
|
+
|
|
954
|
+
return warnings
|