agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
"""The ingest runs: `aut scan`, `aut export` and `aut backfill`, each with its print_* report.
|
|
2
|
+
|
|
3
|
+
A run returns its data and leaves the final summary to its print_* helper; cli.py only parses the
|
|
4
|
+
command line and calls the pair.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import csv
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from dataclasses import replace
|
|
14
|
+
from typing import Any, Callable, Iterator, Sequence, TextIO, TypedDict
|
|
15
|
+
|
|
16
|
+
from . import diagnostics
|
|
17
|
+
from .collectors import REGISTRY, Collect, drain
|
|
18
|
+
from .config import Config
|
|
19
|
+
from .event import (ActivityEvent, Batch, CollectorUnavailable, Row, TimeInterval, UsageEvent,
|
|
20
|
+
parse_ts, valid_ts)
|
|
21
|
+
from .push import (COLUMNS, ScanStatus, deliver_scan_rows, error_type_name, make_sender,
|
|
22
|
+
redacted_error, replay_outbox, scan_row, send_activity_rows, send_session_rows)
|
|
23
|
+
from .repo_keys import refuse_new_salt, require_readable_salt
|
|
24
|
+
from .state import collector_wm, load_state, save_state
|
|
25
|
+
|
|
26
|
+
BATCH = 500
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ModelTally(TypedDict):
|
|
30
|
+
events: int
|
|
31
|
+
total: int
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AgentTally(TypedDict):
|
|
35
|
+
"""One collector's line of the scan report."""
|
|
36
|
+
events: int
|
|
37
|
+
human: int
|
|
38
|
+
input: int
|
|
39
|
+
output: int
|
|
40
|
+
cache_read: int
|
|
41
|
+
cache_write: int
|
|
42
|
+
reasoning: int
|
|
43
|
+
total: int
|
|
44
|
+
skipped_no_ts: int
|
|
45
|
+
duplicates: int
|
|
46
|
+
by_model: dict[str, ModelTally]
|
|
47
|
+
activity: dict[str, int]
|
|
48
|
+
intervals: int
|
|
49
|
+
time: dict[str | None, float] # category -> seconds
|
|
50
|
+
relevant_s: float
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class HistoryCounts(TypedDict):
|
|
54
|
+
"""The report of a history backfill."""
|
|
55
|
+
rows: int
|
|
56
|
+
by_origin: dict[str, int]
|
|
57
|
+
turn_null: int
|
|
58
|
+
subagent_no_parent: int
|
|
59
|
+
activity: int
|
|
60
|
+
by_kind: dict[str, int]
|
|
61
|
+
via: dict[str, int]
|
|
62
|
+
link: dict[str, dict[str, int]]
|
|
63
|
+
failed: set[str]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _blank_agent_tally() -> AgentTally:
|
|
67
|
+
return {"events": 0, "human": 0, "input": 0, "output": 0, "cache_read": 0,
|
|
68
|
+
"cache_write": 0, "reasoning": 0, "total": 0, "skipped_no_ts": 0,
|
|
69
|
+
"duplicates": 0, "by_model": {}, "activity": {},
|
|
70
|
+
"intervals": 0, "time": {}, "relevant_s": 0.0}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _tally_event(agg: AgentTally, e: Row) -> None:
|
|
74
|
+
if isinstance(e, TimeInterval): # active time; no tokens, kept out of events/human
|
|
75
|
+
secs = (datetime.fromisoformat(e.end_ts) - datetime.fromisoformat(e.ts)).total_seconds()
|
|
76
|
+
agg["intervals"] += 1
|
|
77
|
+
agg["time"][e.category] = agg["time"].get(e.category, 0.0) + secs
|
|
78
|
+
agg["relevant_s"] += secs if e.work_relevant else 0.0
|
|
79
|
+
return
|
|
80
|
+
if isinstance(e, ActivityEvent): # no tokens; tallied by kind, kept out of events/human
|
|
81
|
+
agg["activity"][e.kind] = agg["activity"].get(e.kind, 0) + 1
|
|
82
|
+
return
|
|
83
|
+
if e.kind == "human": # a user prompt: no tokens, kept out of the token/model rollup
|
|
84
|
+
agg["human"] += 1
|
|
85
|
+
return
|
|
86
|
+
agg["events"] += 1
|
|
87
|
+
agg["input"] += e.input_tokens
|
|
88
|
+
agg["output"] += e.output_tokens
|
|
89
|
+
agg["cache_read"] += e.cache_read_tokens
|
|
90
|
+
agg["cache_write"] += e.cache_write_tokens
|
|
91
|
+
agg["reasoning"] += e.reasoning_tokens or 0
|
|
92
|
+
agg["total"] += e.total_tokens
|
|
93
|
+
m = agg["by_model"].setdefault(e.model or "unknown", {"events": 0, "total": 0})
|
|
94
|
+
m["events"] += 1
|
|
95
|
+
m["total"] += e.total_tokens
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class _CollectorPass:
|
|
99
|
+
"""One collector's rows on their way to the sink: tallied, buffered, sent in batches."""
|
|
100
|
+
|
|
101
|
+
def __init__(self, agg: AgentTally, sender: Callable[[list[Row]], None] | None) -> None:
|
|
102
|
+
self.agg, self.sender = agg, sender # sender None: a dry run, nothing is sent
|
|
103
|
+
# The same session can sit at two paths -- Claude Code re-encodes a project directory
|
|
104
|
+
# (e.g. '-Users-me-proj' and 'prefix---Users-me-proj'), so one session file is globbed
|
|
105
|
+
# twice and every turn in it is emitted twice. Both copies produce the same event_id, so
|
|
106
|
+
# the sink stores one and the numbers there are right; without this set the PRINTED tally
|
|
107
|
+
# counted both and overstated the scan. Per-agent, discarded when the agent finishes.
|
|
108
|
+
self.seen: set[str] = set()
|
|
109
|
+
self.buf: list[Row] = []
|
|
110
|
+
self.advances: list[Callable[[], None]] = []
|
|
111
|
+
self.newest_ts: datetime | None = None
|
|
112
|
+
self.diagnostics = diagnostics.Diagnostics()
|
|
113
|
+
self.status: ScanStatus = "ok"
|
|
114
|
+
self.error: str | None = None
|
|
115
|
+
|
|
116
|
+
def drain_all(self, collect: Collect, cfg: Config, wm: dict[str, Any], name: str) -> None:
|
|
117
|
+
for batches in drain(collect, cfg, wm, name):
|
|
118
|
+
try:
|
|
119
|
+
for batch in batches:
|
|
120
|
+
self.take(batch)
|
|
121
|
+
except CollectorUnavailable as exc: # what it could read still goes out below
|
|
122
|
+
print(f"aut: {exc}", file=sys.stderr)
|
|
123
|
+
self.status, self.error = "degraded", error_type_name(exc.__cause__ or exc)
|
|
124
|
+
self.flush()
|
|
125
|
+
|
|
126
|
+
def take(self, batch: Batch) -> None:
|
|
127
|
+
events, advance_watermark = batch
|
|
128
|
+
for e in events:
|
|
129
|
+
ts = parse_ts(e.ts)
|
|
130
|
+
if ts is None:
|
|
131
|
+
self.agg["skipped_no_ts"] += 1
|
|
132
|
+
continue
|
|
133
|
+
ts = ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)
|
|
134
|
+
self.newest_ts = ts if self.newest_ts is None or ts > self.newest_ts else self.newest_ts
|
|
135
|
+
if e.event_id in self.seen:
|
|
136
|
+
self.agg["duplicates"] += 1 # counted once, above
|
|
137
|
+
else:
|
|
138
|
+
self.seen.add(e.event_id)
|
|
139
|
+
_tally_event(self.agg, e)
|
|
140
|
+
# Still queued for the sink either way: deduping the PUSH would mean a copy whose batch
|
|
141
|
+
# failed could not be re-sent from the other path, and ON CONFLICT already makes the
|
|
142
|
+
# repeat free.
|
|
143
|
+
if self.sender is not None:
|
|
144
|
+
self.buf.append(e)
|
|
145
|
+
self.advances.append(advance_watermark)
|
|
146
|
+
if len(self.buf) >= BATCH:
|
|
147
|
+
self.flush()
|
|
148
|
+
|
|
149
|
+
def flush(self) -> None:
|
|
150
|
+
if self.buf and self.sender is not None: # a dry run fills no buf; this narrows sender
|
|
151
|
+
self.sender(self.buf) # raises on delivery failure -> advances skipped
|
|
152
|
+
for advance in self.advances: # always advance the (in-memory) watermark so the
|
|
153
|
+
advance() # drain terminates; only scan persists it
|
|
154
|
+
self.buf.clear()
|
|
155
|
+
self.advances.clear()
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def run_scan(cfg: Config, agents: Sequence[str], *, dry_run: bool) -> dict[str, AgentTally]:
|
|
159
|
+
cfg = replace(cfg, agents=tuple(dict.fromkeys((*cfg.agents, *agents))))
|
|
160
|
+
state = load_state()
|
|
161
|
+
sender = None if dry_run else make_sender(cfg)
|
|
162
|
+
if dry_run: # a dry run still makes the salt (the collectors' RepoKeys)
|
|
163
|
+
refuse_new_salt(cfg.repo_key_salt, cfg.pg_dsn, cfg.machine)
|
|
164
|
+
else:
|
|
165
|
+
require_readable_salt(cfg.repo_key_salt, cfg.pg_dsn, cfg.machine)
|
|
166
|
+
if cfg.pg_dsn:
|
|
167
|
+
replay_outbox(cfg)
|
|
168
|
+
report: dict[str, AgentTally] = {}
|
|
169
|
+
scan_rows: list[dict[str, Any]] = [] # one heartbeat row per collector that ran (agents.collector_scan)
|
|
170
|
+
observations: list[dict[str, Any]] = []
|
|
171
|
+
|
|
172
|
+
for name in agents:
|
|
173
|
+
collector = REGISTRY[name]
|
|
174
|
+
wm = collector_wm(state, name)
|
|
175
|
+
agg = report.setdefault(name, _blank_agent_tally())
|
|
176
|
+
run = _CollectorPass(agg, sender)
|
|
177
|
+
started_at = datetime.now(timezone.utc)
|
|
178
|
+
failure: BaseException | None = None
|
|
179
|
+
|
|
180
|
+
try:
|
|
181
|
+
with diagnostics.capture(run.diagnostics):
|
|
182
|
+
run.drain_all(collector.collect, cfg, wm, name)
|
|
183
|
+
if collector.after_scan and not dry_run: # a dry run changes nothing
|
|
184
|
+
collector.after_scan(cfg)
|
|
185
|
+
except BaseException as exc:
|
|
186
|
+
run.status, run.error = "error", error_type_name(exc)
|
|
187
|
+
failure = exc
|
|
188
|
+
if not dry_run:
|
|
189
|
+
try:
|
|
190
|
+
save_state(state, [name])
|
|
191
|
+
except OSError as exc: # never ahead of the heartbeat, or of the pass's own failure
|
|
192
|
+
if failure:
|
|
193
|
+
print(f"aut: warning: watermark not saved: {redacted_error(exc)}", file=sys.stderr)
|
|
194
|
+
else:
|
|
195
|
+
run.status, run.error = "error", error_type_name(exc)
|
|
196
|
+
failure = exc
|
|
197
|
+
scan_rows.append(scan_row(cfg, name, started_at, status=run.status, error=run.error,
|
|
198
|
+
events=agg["events"] + agg["intervals"],
|
|
199
|
+
human=agg["human"], newest_ts=run.newest_ts))
|
|
200
|
+
observations.append({
|
|
201
|
+
"collector": name, "status": run.status, "error": run.error,
|
|
202
|
+
"started_at": started_at.isoformat(),
|
|
203
|
+
"finished_at": datetime.now(timezone.utc).isoformat(),
|
|
204
|
+
"usage_events": agg["events"], "human_classifications": agg["human"],
|
|
205
|
+
"activity": dict(agg["activity"]), "diagnostics": run.diagnostics.snapshot(),
|
|
206
|
+
})
|
|
207
|
+
# Ctrl-C leaves at once. Diagnostic persistence cannot replace the original failure.
|
|
208
|
+
if failure and not isinstance(failure, KeyboardInterrupt):
|
|
209
|
+
row = None
|
|
210
|
+
try:
|
|
211
|
+
row = _record_coverage(cfg, observations)
|
|
212
|
+
except Exception as exc:
|
|
213
|
+
print(f"aut: coverage observation failed ({type(exc).__name__})", file=sys.stderr)
|
|
214
|
+
# A failing collector must not hide a new coverage issue.
|
|
215
|
+
deliver_scan_rows(cfg, scan_rows + ([row] if row else []))
|
|
216
|
+
if failure:
|
|
217
|
+
raise failure # the scan still fails
|
|
218
|
+
|
|
219
|
+
if not dry_run:
|
|
220
|
+
row = None
|
|
221
|
+
try:
|
|
222
|
+
row = _record_coverage(cfg, observations)
|
|
223
|
+
finally: # the collectors' heartbeat goes out whatever happens
|
|
224
|
+
deliver_scan_rows(cfg, scan_rows + ([row] if row else []))
|
|
225
|
+
return report
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _record_coverage(cfg: Config, observations: list[dict[str, Any]]) -> dict[str, Any] | None:
|
|
229
|
+
"""Save the coverage receipt and return its `coverage` scan row (None if unavailable)."""
|
|
230
|
+
from .coverage import ALERT_STATUS, build_report, save_report, save_scan_observations
|
|
231
|
+
|
|
232
|
+
started_at = datetime.now(timezone.utc)
|
|
233
|
+
try:
|
|
234
|
+
save_scan_observations(observations)
|
|
235
|
+
assessment = build_report(cfg)
|
|
236
|
+
saved = save_report(assessment)
|
|
237
|
+
except (OSError, ValueError) as exc:
|
|
238
|
+
print(f"aut: coverage receipt unavailable ({type(exc).__name__}); "
|
|
239
|
+
"inspect with aut doctor --json", file=sys.stderr)
|
|
240
|
+
return None
|
|
241
|
+
if saved is None: # an overlapping scan saved a newer assessment; only its row counts
|
|
242
|
+
return None
|
|
243
|
+
changed, alert = saved
|
|
244
|
+
if changed and assessment["issues"]:
|
|
245
|
+
print(f"aut: coverage changed: {len(assessment['issues'])} reported issue(s) "
|
|
246
|
+
"within inspected sources; inspect with aut doctor --json", file=sys.stderr)
|
|
247
|
+
# The row returns to ok once an issue is known: the effort view marks a pair stale without ok.
|
|
248
|
+
row = scan_row(cfg, "coverage", started_at, status=ALERT_STATUS if alert else "ok", error=alert)
|
|
249
|
+
# Ordered by the assessment it reports, not by when this scan reached delivery.
|
|
250
|
+
row["finished_at"] = datetime.fromisoformat(assessment["generated_at"])
|
|
251
|
+
return row
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _print_tokens(name: str, agg: AgentTally) -> None:
|
|
255
|
+
print(f"{name:<12} {agg['events']:>6} turns {agg['total']:>12,} tokens "
|
|
256
|
+
f"(in {agg['input']:,} / out {agg['output']:,} / "
|
|
257
|
+
f"cacheR {agg['cache_read']:,} / cacheW {agg['cache_write']:,}"
|
|
258
|
+
+ (f" / reason {agg['reasoning']:,}" if agg['reasoning'] else "") + ")")
|
|
259
|
+
for model, mv in sorted(agg["by_model"].items(), key=lambda kv: -kv[1]["total"]):
|
|
260
|
+
print(f" {model:<28} {mv['events']:>5} turns {mv['total']:>12,} tokens")
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _print_time(name: str, agg: AgentTally) -> None:
|
|
264
|
+
total_s = sum(agg["time"].values())
|
|
265
|
+
print(f"{name:<12} {agg['intervals']:>6} intervals {total_s / 60:,.1f} min active "
|
|
266
|
+
f"({agg['relevant_s'] / 60:,.1f} min work-relevant)")
|
|
267
|
+
for cat, secs in sorted(agg["time"].items(), key=lambda kv: -kv[1]):
|
|
268
|
+
print(f" {cat:<28} {secs / 60:>9,.1f} min")
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _print_agent(name: str, agg: AgentTally) -> None:
|
|
272
|
+
if agg["events"] or agg["total"]:
|
|
273
|
+
_print_tokens(name, agg)
|
|
274
|
+
if agg["human"]:
|
|
275
|
+
print(f"{name:<12} {agg['human']:>6} human turns (prompts you sent)")
|
|
276
|
+
if agg["intervals"]:
|
|
277
|
+
_print_time(name, agg)
|
|
278
|
+
if agg["activity"]:
|
|
279
|
+
print(f"{name:<12} activity rows {json.dumps(agg['activity'], sort_keys=True)}")
|
|
280
|
+
if agg["skipped_no_ts"]:
|
|
281
|
+
print(f" (skipped {agg['skipped_no_ts']} records with missing/invalid timestamp)")
|
|
282
|
+
if agg["duplicates"]:
|
|
283
|
+
print(f" (counted once: {agg['duplicates']:,} repeat records from sessions stored "
|
|
284
|
+
"under more than one project path)")
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def print_scan(cfg: Config, report: dict[str, AgentTally], *, dry_run: bool) -> None:
|
|
288
|
+
mode = "DRY-RUN (nothing sent)" if dry_run else "scan"
|
|
289
|
+
print(f"\naut {mode} machine={cfg.machine}\n" + "=" * 60)
|
|
290
|
+
grand = 0
|
|
291
|
+
for name, agg in report.items():
|
|
292
|
+
if not (agg["events"] or agg["human"] or agg["activity"] or agg["skipped_no_ts"]
|
|
293
|
+
or agg["duplicates"] or agg["intervals"]):
|
|
294
|
+
print(f"{name:<12} - no new usage")
|
|
295
|
+
continue
|
|
296
|
+
grand += agg["total"]
|
|
297
|
+
_print_agent(name, agg)
|
|
298
|
+
print("-" * 60)
|
|
299
|
+
print(f"{'TOTAL':<12} {grand:>32,} tokens")
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def reset_watermarks(agents: Sequence[str]) -> None:
|
|
303
|
+
"""Forget the selected collectors' watermarks, so the next scan reads all their history."""
|
|
304
|
+
state = load_state()
|
|
305
|
+
for name in agents:
|
|
306
|
+
state.get("collectors", {}).pop(name, None)
|
|
307
|
+
save_state(state, agents)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _one_shot_batches(name: str, cfg: Config, wm: dict[str, Any], failed: set[str],
|
|
311
|
+
keeps: set[str]) -> Iterator[Batch]:
|
|
312
|
+
"""All of a collector's batches for a one-shot command, which calls each batch's advance
|
|
313
|
+
itself: an unavailable source ends that pass, and its name goes into `failed` when the collector
|
|
314
|
+
can emit a kind of row in `keeps`. Otherwise the command is missing nothing."""
|
|
315
|
+
for batches in drain(REGISTRY[name].collect, cfg, wm, name):
|
|
316
|
+
try:
|
|
317
|
+
yield from batches
|
|
318
|
+
except CollectorUnavailable as exc:
|
|
319
|
+
print(f"aut: {exc}", file=sys.stderr)
|
|
320
|
+
if REGISTRY[name].rows & keeps:
|
|
321
|
+
failed.add(name)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def run_export(cfg: Config, agents: Sequence[str], out: TextIO) -> tuple[int, int, set[str]]:
|
|
325
|
+
"""Write all normalized events (from scratch, ignoring the watermark) as CSV to `out`.
|
|
326
|
+
Column order matches push.COLUMNS / the sink table, so it loads via COPY. Useful for
|
|
327
|
+
an offline bulk backfill or a bridged load where the live transport is not wired yet.
|
|
328
|
+
Returns (written, skipped, failed): skipped counts records with a missing/invalid ts, failed
|
|
329
|
+
names the usage collectors with a source they could not read, so the CSV is partial."""
|
|
330
|
+
cfg = replace(cfg, agents=tuple(dict.fromkeys((*cfg.agents, *agents))))
|
|
331
|
+
writer = csv.writer(out)
|
|
332
|
+
writer.writerow(COLUMNS)
|
|
333
|
+
n = skipped = 0
|
|
334
|
+
failed: set[str] = set()
|
|
335
|
+
for name in agents:
|
|
336
|
+
# Must dedup here, unlike the push path: this CSV is meant to be loaded with COPY, which
|
|
337
|
+
# has no ON CONFLICT to fall back on, so a session stored under two project paths would
|
|
338
|
+
# land twice in the target table. See the note in run_scan.
|
|
339
|
+
seen: set[str] = set()
|
|
340
|
+
for events, advance_watermark in _one_shot_batches(name, cfg, {}, failed, {"usage"}):
|
|
341
|
+
for e in events:
|
|
342
|
+
if not isinstance(e, UsageEvent): # the CSV is agents.usage_event only
|
|
343
|
+
continue
|
|
344
|
+
if not valid_ts(e.ts):
|
|
345
|
+
skipped += 1
|
|
346
|
+
continue
|
|
347
|
+
if e.event_id in seen:
|
|
348
|
+
continue
|
|
349
|
+
seen.add(e.event_id)
|
|
350
|
+
d = e.to_dict()
|
|
351
|
+
writer.writerow(["" if d[c] is None else d[c] for c in COLUMNS])
|
|
352
|
+
n += 1
|
|
353
|
+
advance_watermark() # so the next pass continues past the cap
|
|
354
|
+
return n, skipped, failed
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def print_export(written: int, skipped: int, out_path: str | None) -> None:
|
|
358
|
+
dest = f" -> {out_path}" if out_path else ""
|
|
359
|
+
skip_note = f" ({skipped} skipped: missing/invalid ts)" if skipped else ""
|
|
360
|
+
print(f"exported {written} events{skip_note}{dest}", file=sys.stderr)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _tally_activity(counts: HistoryCounts, e: ActivityEvent) -> None:
|
|
364
|
+
counts["activity"] += 1
|
|
365
|
+
counts["by_kind"][e.kind] = counts["by_kind"].get(e.kind, 0) + 1
|
|
366
|
+
if e.kind == "steer":
|
|
367
|
+
v = e.attrs["via"]
|
|
368
|
+
counts["via"][v] = counts["via"].get(v, 0) + 1
|
|
369
|
+
elif e.kind == "link": # rows per role, and how many carry each key
|
|
370
|
+
r = counts["link"].setdefault(e.attrs["role"], {"rows": 0})
|
|
371
|
+
r["rows"] += 1
|
|
372
|
+
for k, v in e.attrs.items():
|
|
373
|
+
if k != "role" and v:
|
|
374
|
+
r[k] = r.get(k, 0) + 1
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _tally_session(counts: HistoryCounts, row: dict[str, Any]) -> None:
|
|
378
|
+
counts["rows"] += 1
|
|
379
|
+
origin = row["run_origin"]
|
|
380
|
+
counts["by_origin"][origin] = counts["by_origin"].get(origin, 0) + 1
|
|
381
|
+
counts["turn_null"] += row["turn_id"] is None
|
|
382
|
+
counts["subagent_no_parent"] += origin == "subagent" and row["parent_session_id"] is None
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
class _History:
|
|
386
|
+
"""The rows of a history backfill: tallied, buffered and sent in batches."""
|
|
387
|
+
|
|
388
|
+
def __init__(self, dsn: str, dry_run: bool, *, sessions: bool, activity: bool) -> None:
|
|
389
|
+
self.dsn, self.dry_run, self.sessions, self.activity = dsn, dry_run, sessions, activity
|
|
390
|
+
self.counts: HistoryCounts = {"rows": 0, "by_origin": {}, "turn_null": 0,
|
|
391
|
+
"subagent_no_parent": 0, "activity": 0, "by_kind": {},
|
|
392
|
+
"via": {}, "link": {}, "failed": set()}
|
|
393
|
+
self.session_buf: list[UsageEvent] = []
|
|
394
|
+
self.activity_buf: list[ActivityEvent] = []
|
|
395
|
+
|
|
396
|
+
def flush(self) -> None:
|
|
397
|
+
if self.session_buf and not self.dry_run:
|
|
398
|
+
send_session_rows(self.dsn, self.session_buf) # raises -> the command fails, nothing to undo
|
|
399
|
+
if self.activity_buf and not self.dry_run:
|
|
400
|
+
send_activity_rows(self.dsn, self.activity_buf)
|
|
401
|
+
self.session_buf.clear()
|
|
402
|
+
self.activity_buf.clear()
|
|
403
|
+
|
|
404
|
+
def take(self, e: Row, seen: set[str], name: str) -> None:
|
|
405
|
+
if isinstance(e, ActivityEvent):
|
|
406
|
+
if self.activity:
|
|
407
|
+
self._take_activity(e, seen)
|
|
408
|
+
elif isinstance(e, UsageEvent) and self.sessions: # a time row is neither
|
|
409
|
+
self._take_session(e, seen, name)
|
|
410
|
+
|
|
411
|
+
def _take_activity(self, e: ActivityEvent, seen: set[str]) -> None:
|
|
412
|
+
if valid_ts(e.ts) and e.event_id not in seen:
|
|
413
|
+
seen.add(e.event_id)
|
|
414
|
+
_tally_activity(self.counts, e)
|
|
415
|
+
self.activity_buf.append(e)
|
|
416
|
+
if len(self.activity_buf) >= BATCH:
|
|
417
|
+
self.flush()
|
|
418
|
+
|
|
419
|
+
def _take_session(self, e: UsageEvent, seen: set[str], name: str) -> None:
|
|
420
|
+
row = e.session_row()
|
|
421
|
+
# no row for an event the usage path drops (bad ts), or one already sent
|
|
422
|
+
if row is None or not valid_ts(e.ts) or e.event_id in seen:
|
|
423
|
+
return
|
|
424
|
+
seen.add(e.event_id)
|
|
425
|
+
_tally_session(self.counts, row)
|
|
426
|
+
self.session_buf.append(e)
|
|
427
|
+
if len(self.session_buf) >= BATCH:
|
|
428
|
+
self.flush()
|
|
429
|
+
if self.counts["rows"] % (BATCH * 20) == 0:
|
|
430
|
+
print(f"aut: {name}: {self.counts['rows']:,} session rows so far", file=sys.stderr)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def run_history_backfill(cfg: Config, agents: Sequence[str], *, dry_run: bool,
|
|
434
|
+
sessions: bool = True, activity: bool = False) -> HistoryCounts:
|
|
435
|
+
"""Send agents.event_session rows (sessions) and/or agents.activity_event rows (activity) for
|
|
436
|
+
ALL history, never usage rows: re-reading files from byte 0 re-emits (event_id, ts) pairs that
|
|
437
|
+
can differ from the stored ones (see the ts note in collectors/claude_code.py), and those would
|
|
438
|
+
land as new duplicates. Works on a throwaway watermark, so the live one is neither read nor
|
|
439
|
+
saved. ON CONFLICT makes a re-run free."""
|
|
440
|
+
cfg = replace(cfg, agents=tuple(dict.fromkeys((*cfg.agents, *agents))))
|
|
441
|
+
dsn = "" if dry_run else cfg.require_pg("backfill --sessions-only/--activity-only without "
|
|
442
|
+
"--dry-run")
|
|
443
|
+
if dry_run: # a dry run still makes the salt (the collectors' RepoKeys)
|
|
444
|
+
refuse_new_salt(cfg.repo_key_salt, cfg.pg_dsn, cfg.machine)
|
|
445
|
+
else:
|
|
446
|
+
require_readable_salt(cfg.repo_key_salt, cfg.pg_dsn, cfg.machine)
|
|
447
|
+
history = _History(dsn, dry_run, sessions=sessions, activity=activity)
|
|
448
|
+
keeps = {kind for kind, on in (("session", sessions), ("activity", activity)) if on}
|
|
449
|
+
for name in agents:
|
|
450
|
+
seen: set[str] = set() # same session under two paths: one row
|
|
451
|
+
for events, advance_watermark in _one_shot_batches(name, cfg, {},
|
|
452
|
+
history.counts["failed"], keeps):
|
|
453
|
+
for e in events:
|
|
454
|
+
history.take(e, seen, name)
|
|
455
|
+
advance_watermark() # advances only the throwaway wm
|
|
456
|
+
history.flush()
|
|
457
|
+
print(f"aut: {name}: {len(seen):,} {'session/activity' if activity else 'session'} rows "
|
|
458
|
+
f"{'counted' if dry_run else 'sent'}", file=sys.stderr)
|
|
459
|
+
return history.counts
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def print_history_backfill(c: HistoryCounts, *, dry_run: bool, sessions: bool, activity: bool) -> None:
|
|
463
|
+
mode = "DRY-RUN (nothing sent)" if dry_run else "sent"
|
|
464
|
+
if sessions:
|
|
465
|
+
print(f"sessions-only backfill {mode}: {c['rows']} rows, by run_origin "
|
|
466
|
+
f"{json.dumps(c['by_origin'], sort_keys=True)}, turn_id NULL {c['turn_null']}, "
|
|
467
|
+
f"subagent rows without a parent {c['subagent_no_parent']}")
|
|
468
|
+
if activity:
|
|
469
|
+
print(f"activity-only backfill {mode}: {c['activity']} rows, by kind "
|
|
470
|
+
f"{json.dumps(c['by_kind'], sort_keys=True)}, steer via "
|
|
471
|
+
f"{json.dumps(c['via'], sort_keys=True)}, link rows with each key by role "
|
|
472
|
+
f"{json.dumps(c['link'], sort_keys=True)}")
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Retry scheduled scans and daily jobs without overlapping work or leaking SSH tunnels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import signal
|
|
8
|
+
import socket
|
|
9
|
+
import subprocess
|
|
10
|
+
import time
|
|
11
|
+
from datetime import date, datetime
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from types import FrameType
|
|
14
|
+
from typing import TextIO
|
|
15
|
+
|
|
16
|
+
from .state import LockBusy, _locked, atomic_write_text
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _Stopped(BaseException):
|
|
20
|
+
def __init__(self, signum: int) -> None:
|
|
21
|
+
self.signum = signum
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _stop(signum: int, frame: FrameType | None) -> None:
|
|
25
|
+
raise _Stopped(signum)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _terminate(process: subprocess.Popen[bytes] | None) -> None:
|
|
29
|
+
if process is not None and process.poll() is None:
|
|
30
|
+
process.terminate()
|
|
31
|
+
try:
|
|
32
|
+
process.wait(timeout=5)
|
|
33
|
+
except subprocess.TimeoutExpired:
|
|
34
|
+
process.kill()
|
|
35
|
+
process.wait()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _listening(port: int) -> bool:
|
|
39
|
+
try:
|
|
40
|
+
with socket.create_connection(("localhost", port), timeout=0.2):
|
|
41
|
+
return True
|
|
42
|
+
except OSError:
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _run(conf: Path, log: TextIO) -> int:
|
|
47
|
+
aut = os.environ.get("AUT_BIN", str(Path.home() / ".local/bin/aut"))
|
|
48
|
+
host, spec = os.environ.get("AUT_TUNNEL_HOST", ""), os.environ.get("AUT_TUNNEL_SPEC", "")
|
|
49
|
+
# Fallback routes share the spec, so each must reach the same sink (e.g. a LAN address).
|
|
50
|
+
fallbacks = os.environ.get("AUT_TUNNEL_FALLBACK_HOSTS", "").replace(",", " ").split()
|
|
51
|
+
hosts = [host, *fallbacks]
|
|
52
|
+
retry = conf / "retry-pending"
|
|
53
|
+
tunnel: subprocess.Popen[bytes] | None = None
|
|
54
|
+
child: subprocess.Popen[bytes] | None = None
|
|
55
|
+
|
|
56
|
+
def write(message: str) -> None:
|
|
57
|
+
print(f"{datetime.now():%Y-%m-%dT%H:%M:%S} {message}", file=log, flush=True)
|
|
58
|
+
|
|
59
|
+
def command(*args: str) -> int:
|
|
60
|
+
nonlocal child
|
|
61
|
+
write(f"{args[0]} start")
|
|
62
|
+
child = subprocess.Popen([aut, *args], stdout=log, stderr=log)
|
|
63
|
+
rc = child.wait()
|
|
64
|
+
write(f"{args[0]} done (exit {rc})")
|
|
65
|
+
return rc
|
|
66
|
+
|
|
67
|
+
if not shutil.which(aut):
|
|
68
|
+
write("AUT_BIN is not executable; refusing to scan")
|
|
69
|
+
return 2
|
|
70
|
+
port = 0
|
|
71
|
+
if host:
|
|
72
|
+
try:
|
|
73
|
+
forward, remote_port = spec.rsplit(":", 1)
|
|
74
|
+
local_port, remote_host = forward.split(":", 1)
|
|
75
|
+
port = int(local_port)
|
|
76
|
+
if not remote_host or not 1 <= port <= 65535 or not 1 <= int(remote_port) <= 65535:
|
|
77
|
+
raise ValueError
|
|
78
|
+
except ValueError:
|
|
79
|
+
write("AUT_TUNNEL_HOST requires AUT_TUNNEL_SPEC=port:host:port; refusing to scan")
|
|
80
|
+
return 2
|
|
81
|
+
if any(target.startswith("-") for target in hosts):
|
|
82
|
+
write("tunnel host must be user@host; refusing to scan") # else ssh reads an option
|
|
83
|
+
return 2
|
|
84
|
+
if not shutil.which("ssh"):
|
|
85
|
+
write("ssh is not installed; refusing to scan")
|
|
86
|
+
return 2
|
|
87
|
+
|
|
88
|
+
handlers = {sig: signal.signal(sig, _stop) for sig in (signal.SIGINT, signal.SIGTERM)}
|
|
89
|
+
scanning = False
|
|
90
|
+
try:
|
|
91
|
+
if host and not _listening(port):
|
|
92
|
+
for index, target in enumerate(hosts):
|
|
93
|
+
wait = 15 if index == 0 else 5 # LAN fallbacks answer fast or not at all
|
|
94
|
+
tunnel = subprocess.Popen(
|
|
95
|
+
["ssh", "-N", "-o", "ControlMaster=no", "-o", "ControlPath=none",
|
|
96
|
+
"-o", "ForkAfterAuthentication=no", "-o", "ExitOnForwardFailure=yes",
|
|
97
|
+
"-o", "BatchMode=yes", "-o", f"ConnectTimeout={wait}",
|
|
98
|
+
"-L", spec, target], stdout=log, stderr=log)
|
|
99
|
+
deadline = time.monotonic() + wait + 2
|
|
100
|
+
while tunnel.poll() is None and time.monotonic() < deadline:
|
|
101
|
+
if _listening(port):
|
|
102
|
+
break
|
|
103
|
+
time.sleep(0.1)
|
|
104
|
+
if _listening(port):
|
|
105
|
+
write("tunnel up via " + ("primary" if index == 0 else f"fallback {index}"))
|
|
106
|
+
break
|
|
107
|
+
# A half-dead ssh must not hold the port; `finally` only reaps the last one.
|
|
108
|
+
_terminate(tunnel)
|
|
109
|
+
tunnel = None
|
|
110
|
+
else:
|
|
111
|
+
write(f"tunnel FAILED on all {len(hosts)} route(s); armed retry-pending "
|
|
112
|
+
"(retries until a host is reachable)")
|
|
113
|
+
retry.touch()
|
|
114
|
+
return 1
|
|
115
|
+
elif host:
|
|
116
|
+
write("tunnel reused existing listener")
|
|
117
|
+
|
|
118
|
+
scanning = True
|
|
119
|
+
rc = command("scan")
|
|
120
|
+
scanning = False
|
|
121
|
+
if rc:
|
|
122
|
+
retry.touch()
|
|
123
|
+
return rc
|
|
124
|
+
retry.unlink(missing_ok=True)
|
|
125
|
+
today = date.today().isoformat()
|
|
126
|
+
jobs = [("outcomes", ["outcomes"])]
|
|
127
|
+
if os.environ.get("AUT_LABEL") == "1":
|
|
128
|
+
args = ["label", "run"]
|
|
129
|
+
if os.environ.get("AUT_LABEL_MODE") == "local":
|
|
130
|
+
args.append("--local-estimates")
|
|
131
|
+
jobs.append(("label", args))
|
|
132
|
+
for name, args in jobs:
|
|
133
|
+
marker = conf / f"{name}-day"
|
|
134
|
+
if marker.exists() and marker.read_text().strip() == today:
|
|
135
|
+
continue
|
|
136
|
+
if command(*args) == 0:
|
|
137
|
+
atomic_write_text(marker, today + "\n")
|
|
138
|
+
return rc
|
|
139
|
+
except _Stopped as exc:
|
|
140
|
+
if scanning:
|
|
141
|
+
retry.touch()
|
|
142
|
+
return 128 + exc.signum
|
|
143
|
+
finally:
|
|
144
|
+
# Disable repeat interrupts while reaping only the children this job created.
|
|
145
|
+
for sig in handlers:
|
|
146
|
+
signal.signal(sig, signal.SIG_IGN)
|
|
147
|
+
try:
|
|
148
|
+
_terminate(child)
|
|
149
|
+
_terminate(tunnel)
|
|
150
|
+
finally:
|
|
151
|
+
for sig, handler in handlers.items():
|
|
152
|
+
signal.signal(sig, handler)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def main() -> int:
|
|
156
|
+
conf = Path.home() / ".config/agent-usage-tracker"
|
|
157
|
+
conf.mkdir(parents=True, exist_ok=True)
|
|
158
|
+
try:
|
|
159
|
+
# Keep the inode: removing a lock file lets another process lock a replacement.
|
|
160
|
+
with _locked(conf / "scan-job", blocking=False):
|
|
161
|
+
with (conf / "scan.log").open("a") as log:
|
|
162
|
+
return _run(conf, log)
|
|
163
|
+
except LockBusy:
|
|
164
|
+
return 0
|