agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,530 @@
|
|
|
1
|
+
"""Transport: push normalized rows to the sink.
|
|
2
|
+
|
|
3
|
+
A scan's usage rows go over the n8n webhook (behind Cloudflare Access) when one is configured, and
|
|
4
|
+
fall back to direct Postgres (transport.pg_dsn) when it fails, or go to Postgres alone without a
|
|
5
|
+
webhook. Everything else (session, activity, time, outcome, label and heartbeat rows) goes over
|
|
6
|
+
pg_dsn only. All of it is idempotent at the sink via ON CONFLICT, so re-delivery is harmless.
|
|
7
|
+
|
|
8
|
+
A sender that cannot deliver or durably queue rows raises, so the watermark stays unchanged.
|
|
9
|
+
Webhook-only machines keep session, activity and time rows in a private local SQLite outbox.
|
|
10
|
+
Postgres delivery replays that outbox before sending new rows.
|
|
11
|
+
|
|
12
|
+
Delivery is strict on purpose: redirects are NOT followed (a 3xx that turns the POST into a
|
|
13
|
+
GET could otherwise look like success and advance the watermark over data the sink never
|
|
14
|
+
stored), and any non-2xx is a hard failure.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import sqlite3
|
|
22
|
+
import sys
|
|
23
|
+
import urllib.error
|
|
24
|
+
import urllib.request
|
|
25
|
+
from contextlib import contextmanager
|
|
26
|
+
from dataclasses import asdict, dataclass
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any, Callable, Iterator, Literal
|
|
30
|
+
from urllib.parse import urlparse
|
|
31
|
+
|
|
32
|
+
from . import __version__
|
|
33
|
+
from .config import AutError, Config
|
|
34
|
+
from .event import ActivityEvent, Row, TimeInterval, UsageEvent
|
|
35
|
+
from .net import HTTP_ERRORS, LOCAL_HOSTS, NoRedirect
|
|
36
|
+
from .redact import redact
|
|
37
|
+
from .sink import connect, write_many
|
|
38
|
+
from .state import state_path
|
|
39
|
+
|
|
40
|
+
# Column order shared with the homelab schema (agents.usage_event). Keep in sync.
|
|
41
|
+
COLUMNS = (
|
|
42
|
+
"event_id", "ts", "machine", "agent", "kind", "model", "provider",
|
|
43
|
+
"input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens",
|
|
44
|
+
"reasoning_tokens", "total_tokens", "est_cost_usd", "project",
|
|
45
|
+
)
|
|
46
|
+
TABLE = "agents.usage_event"
|
|
47
|
+
|
|
48
|
+
# One row per collector per scan (agents.collector_scan): proof a machine ran and how it ended.
|
|
49
|
+
SCAN_COLUMNS = (
|
|
50
|
+
"machine", "collector", "started_at", "finished_at", "status", "events", "human",
|
|
51
|
+
"newest_ts", "error", "aut_version",
|
|
52
|
+
)
|
|
53
|
+
SCAN_TABLE = "agents.collector_scan"
|
|
54
|
+
|
|
55
|
+
# One row per usage event_id (agents.event_session): which session, parent and turn it belongs to.
|
|
56
|
+
SESSION_COLUMNS = (
|
|
57
|
+
"event_id", "machine", "agent", "session_id", "parent_session_id", "run_origin", "turn_id",
|
|
58
|
+
)
|
|
59
|
+
SESSION_TABLE = "agents.event_session"
|
|
60
|
+
|
|
61
|
+
# One row per non-token activity (agents.activity_event): steers, commits, turns. attrs is jsonb.
|
|
62
|
+
ACTIVITY_COLUMNS = ("event_id", "ts", "machine", "agent", "kind", "session_id", "turn_id", "attrs")
|
|
63
|
+
ACTIVITY_TABLE = "agents.activity_event"
|
|
64
|
+
|
|
65
|
+
# One row per active-time interval (agents.time_interval): ActivityWatch window time and manual
|
|
66
|
+
# entries. title is private (redacted on this machine; retention is off by default, see CONTRACT.md).
|
|
67
|
+
TIME_COLUMNS = ("event_id", "ts", "end_ts", "machine", "source", "app", "title", "category",
|
|
68
|
+
"work_relevant", "attrs")
|
|
69
|
+
TIME_TABLE = "agents.time_interval"
|
|
70
|
+
_ERROR_MAX = 300
|
|
71
|
+
_JSONB_COLUMN = "attrs" # the one jsonb column of these tables
|
|
72
|
+
ScanStatus = Literal["ok", "degraded", "error"]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class DeliveryError(RuntimeError):
|
|
76
|
+
"""Rows could not be delivered and no route is left: an outage main reports on one line."""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def post_webhook(url: str, secret: str, batch: list[UsageEvent],
|
|
80
|
+
extra_headers: dict[str, str] | None = None) -> None:
|
|
81
|
+
body = json.dumps({"events": [e.to_dict() for e in batch]}).encode("utf-8")
|
|
82
|
+
headers = {"Content-Type": "application/json", "X-AUT-Secret": secret}
|
|
83
|
+
if extra_headers: # e.g. CF-Access-Client-Id/Secret to clear Cloudflare Access
|
|
84
|
+
headers.update(extra_headers)
|
|
85
|
+
req = urllib.request.Request(url, data=body, method="POST", headers=headers)
|
|
86
|
+
try:
|
|
87
|
+
with urllib.request.build_opener(NoRedirect).open(req, timeout=30) as resp:
|
|
88
|
+
status = resp.status
|
|
89
|
+
except urllib.error.HTTPError as exc: # 3xx (no-follow) / 4xx / 5xx
|
|
90
|
+
raise RuntimeError(f"webhook HTTP {exc.code}") from exc
|
|
91
|
+
if not (200 <= status < 300):
|
|
92
|
+
raise RuntimeError(f"webhook HTTP {status}")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _insert_sql(table: str, columns: tuple[str, ...], conflict: str) -> str:
|
|
96
|
+
marks = ", ".join("%s::jsonb" if c == _JSONB_COLUMN else "%s" for c in columns)
|
|
97
|
+
return (f"INSERT INTO {table} ({', '.join(columns)}) VALUES " # nosec B608 # constants only
|
|
98
|
+
f"({marks}) ON CONFLICT ({conflict}) DO NOTHING")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _params(row: dict[str, Any], columns: tuple[str, ...]) -> tuple[Any, ...]:
|
|
102
|
+
return tuple(json.dumps(row[c], sort_keys=True) if c == _JSONB_COLUMN else row[c]
|
|
103
|
+
for c in columns)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _session_rows(batch: list[UsageEvent]) -> list[tuple[Any, ...]]:
|
|
107
|
+
return [_params(r, SESSION_COLUMNS) for r in map(UsageEvent.session_row, batch) if r]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
_USAGE_SQL = _insert_sql(TABLE, COLUMNS, "event_id, ts")
|
|
111
|
+
_SESSION_SQL = _insert_sql(SESSION_TABLE, SESSION_COLUMNS, "event_id")
|
|
112
|
+
_ACTIVITY_SQL = _insert_sql(ACTIVITY_TABLE, ACTIVITY_COLUMNS, "event_id, ts")
|
|
113
|
+
_TIME_SQL = _insert_sql(TIME_TABLE, TIME_COLUMNS, "event_id, ts")
|
|
114
|
+
_SCAN_SQL = _insert_sql(SCAN_TABLE, SCAN_COLUMNS, "machine, collector, finished_at")
|
|
115
|
+
_OUTBOX_TYPES = {"session": (_SESSION_SQL, SESSION_COLUMNS),
|
|
116
|
+
"activity": (_ACTIVITY_SQL, ACTIVITY_COLUMNS), "time": (_TIME_SQL, TIME_COLUMNS)}
|
|
117
|
+
_OUTBOX_BATCH = 500
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def outbox_path() -> Path:
|
|
121
|
+
"""Private deferred sink rows, kept independently of collector watermarks."""
|
|
122
|
+
return state_path().parent / "delivery-outbox" / "pending.sqlite3"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@contextmanager
|
|
126
|
+
def _outbox() -> Iterator[sqlite3.Connection]:
|
|
127
|
+
path = outbox_path()
|
|
128
|
+
try:
|
|
129
|
+
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
130
|
+
os.chmod(path.parent, 0o700)
|
|
131
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600)
|
|
132
|
+
os.close(fd)
|
|
133
|
+
os.chmod(path, 0o600)
|
|
134
|
+
conn = sqlite3.connect(path, timeout=30)
|
|
135
|
+
try:
|
|
136
|
+
conn.execute("PRAGMA secure_delete=ON")
|
|
137
|
+
conn.execute("CREATE TABLE IF NOT EXISTS pending ("
|
|
138
|
+
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
|
139
|
+
"kind TEXT NOT NULL, event_id TEXT NOT NULL, ts TEXT NOT NULL, "
|
|
140
|
+
"payload TEXT NOT NULL, UNIQUE(kind, event_id, ts))")
|
|
141
|
+
yield conn
|
|
142
|
+
finally:
|
|
143
|
+
conn.close()
|
|
144
|
+
except (sqlite3.Error, OSError, ValueError, KeyError, TypeError) as exc:
|
|
145
|
+
raise DeliveryError("local delivery outbox failed; watermark unchanged. "
|
|
146
|
+
f"Keep {path} and fix its storage or permissions: {exc}") from exc
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def outbox_status() -> dict[str, int]:
|
|
150
|
+
"""Pending rows by sink type. Corrupt or unreadable storage raises, never appears empty."""
|
|
151
|
+
if not outbox_path().exists():
|
|
152
|
+
return {}
|
|
153
|
+
with _outbox() as conn:
|
|
154
|
+
return dict(conn.execute("SELECT kind, COUNT(*) FROM pending GROUP BY kind"))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _queue_rest(batch: list[Row]) -> None:
|
|
158
|
+
with _outbox() as conn:
|
|
159
|
+
with conn:
|
|
160
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
161
|
+
for event in batch:
|
|
162
|
+
if isinstance(event, UsageEvent):
|
|
163
|
+
row = event.session_row()
|
|
164
|
+
if row is None:
|
|
165
|
+
continue
|
|
166
|
+
kind = "session"
|
|
167
|
+
else:
|
|
168
|
+
row = asdict(event)
|
|
169
|
+
kind = "activity" if isinstance(event, ActivityEvent) else "time"
|
|
170
|
+
snapshot = (kind == "activity" and row["kind"] in ("turn", "link")
|
|
171
|
+
and "revision_sources" in row["attrs"])
|
|
172
|
+
if snapshot:
|
|
173
|
+
_revision_sources(row["attrs"])
|
|
174
|
+
old = conn.execute("SELECT payload FROM pending "
|
|
175
|
+
"WHERE kind=? AND event_id=? AND ts=?",
|
|
176
|
+
(kind, event.event_id, event.ts or "")).fetchone()
|
|
177
|
+
if old and not _snapshot_covers(json.loads(old[0])["attrs"], row["attrs"]):
|
|
178
|
+
continue
|
|
179
|
+
conn.execute("INSERT INTO pending(kind, event_id, ts, payload) VALUES (?, ?, ?, ?) "
|
|
180
|
+
"ON CONFLICT(kind, event_id, ts) DO UPDATE SET payload=excluded.payload "
|
|
181
|
+
"WHERE ?", (kind, event.event_id, event.ts or "",
|
|
182
|
+
json.dumps(row, sort_keys=True), snapshot))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def replay_outbox(cfg: Config) -> int:
|
|
186
|
+
"""Replay a finite FIFO snapshot, at most 500 rows in memory, deleting after PG commit only."""
|
|
187
|
+
dsn = cfg.require_pg("outbox replay")
|
|
188
|
+
if not outbox_path().exists():
|
|
189
|
+
return 0
|
|
190
|
+
delivered = 0
|
|
191
|
+
deferred = False
|
|
192
|
+
after = 0
|
|
193
|
+
with _outbox() as conn:
|
|
194
|
+
ceiling = conn.execute("SELECT COALESCE(MAX(rowid), 0) FROM pending").fetchone()[0]
|
|
195
|
+
while True:
|
|
196
|
+
with conn:
|
|
197
|
+
# A writer cannot replace a row between its delivery and local acknowledgement.
|
|
198
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
199
|
+
pending = conn.execute("SELECT rowid, kind, payload FROM pending "
|
|
200
|
+
"WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ?",
|
|
201
|
+
(after, ceiling, _OUTBOX_BATCH)).fetchall()
|
|
202
|
+
if not pending:
|
|
203
|
+
break
|
|
204
|
+
records = [(rowid, kind, json.loads(payload)) for rowid, kind, payload in pending]
|
|
205
|
+
blocked: set[int] = set()
|
|
206
|
+
with connect(dsn) as pg:
|
|
207
|
+
with pg.cursor() as cur:
|
|
208
|
+
for kind, (sql, columns) in _OUTBOX_TYPES.items():
|
|
209
|
+
rows = [row for _, k, row in records if k == kind]
|
|
210
|
+
if kind == "activity":
|
|
211
|
+
waiting = _write_activities(cur, rows)
|
|
212
|
+
keys = {(r["event_id"], r["ts"]) for r in waiting}
|
|
213
|
+
blocked.update(rowid for rowid, k, row in records if k == kind
|
|
214
|
+
and (row["event_id"], row["ts"]) in keys)
|
|
215
|
+
elif rows:
|
|
216
|
+
cur.executemany(sql, [_params(row, columns) for row in rows])
|
|
217
|
+
if any(kind not in _OUTBOX_TYPES for _, kind, _ in pending):
|
|
218
|
+
raise DeliveryError("unknown row type in local delivery outbox")
|
|
219
|
+
pg.commit()
|
|
220
|
+
conn.executemany("DELETE FROM pending WHERE rowid=?",
|
|
221
|
+
[(r[0],) for r in pending if r[0] not in blocked])
|
|
222
|
+
delivered += len(pending) - len(blocked)
|
|
223
|
+
deferred |= bool(blocked)
|
|
224
|
+
after = pending[-1][0]
|
|
225
|
+
if deferred:
|
|
226
|
+
_snapshot_permission_note()
|
|
227
|
+
return delivered
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _snapshot_permission_note() -> None:
|
|
231
|
+
print(f"aut: note: turn/link snapshot updates queued privately at {outbox_path()}; "
|
|
232
|
+
"the sink role needs UPDATE(attrs) on agents.activity_event. "
|
|
233
|
+
"Grant that permission, then run aut scan to replay them.", file=sys.stderr)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _write_activities(cur: Any, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
237
|
+
"""Ordinary events are immutable; covered turn snapshots may refresh their aggregate counts."""
|
|
238
|
+
snapshots, ordinary = [], []
|
|
239
|
+
deferred = []
|
|
240
|
+
can_update = None
|
|
241
|
+
for row in rows:
|
|
242
|
+
if row["kind"] in ("turn", "link") and "revision_sources" in row["attrs"]:
|
|
243
|
+
snapshots.append(row)
|
|
244
|
+
else:
|
|
245
|
+
ordinary.append(row)
|
|
246
|
+
if ordinary:
|
|
247
|
+
cur.executemany(_ACTIVITY_SQL, [_params(row, ACTIVITY_COLUMNS) for row in ordinary])
|
|
248
|
+
for row in sorted(snapshots, key=lambda r: r["event_id"]):
|
|
249
|
+
_revision_sources(row["attrs"])
|
|
250
|
+
cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (row["event_id"],))
|
|
251
|
+
# Match event_id alone because older Pi snapshots used the turn's end timestamp.
|
|
252
|
+
cur.execute("SELECT ts, agent, kind, machine, attrs FROM agents.activity_event "
|
|
253
|
+
"WHERE event_id=%s", (row["event_id"],))
|
|
254
|
+
existing = cur.fetchall()
|
|
255
|
+
if not existing:
|
|
256
|
+
cur.executemany(_ACTIVITY_SQL, [_params(row, ACTIVITY_COLUMNS)])
|
|
257
|
+
continue
|
|
258
|
+
for ts, agent, kind, machine, attrs in existing:
|
|
259
|
+
if (agent, kind) != (row["agent"], row["kind"]):
|
|
260
|
+
raise DeliveryError("turn snapshot event identity conflicts with an existing row")
|
|
261
|
+
for ts, agent, kind, machine, attrs in existing:
|
|
262
|
+
if attrs != row["attrs"] and _snapshot_covers(attrs, row["attrs"]):
|
|
263
|
+
if can_update is None:
|
|
264
|
+
cur.execute("SELECT has_column_privilege(current_user, "
|
|
265
|
+
"'agents.activity_event', 'attrs', 'UPDATE')", ())
|
|
266
|
+
can_update = bool(cur.fetchone()[0])
|
|
267
|
+
if not can_update:
|
|
268
|
+
deferred.append(row)
|
|
269
|
+
break
|
|
270
|
+
cur.execute("UPDATE agents.activity_event SET attrs=%s::jsonb "
|
|
271
|
+
"WHERE event_id=%s AND ts=%s",
|
|
272
|
+
(json.dumps(row["attrs"], sort_keys=True), row["event_id"], ts))
|
|
273
|
+
return deferred
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _revision_sources(attrs: dict[str, Any]) -> set[str]:
|
|
277
|
+
sources = attrs.get("revision_sources")
|
|
278
|
+
if not isinstance(sources, list) or not sources or any(
|
|
279
|
+
not isinstance(source, str) or not source for source in sources):
|
|
280
|
+
raise DeliveryError("turn snapshot has invalid revision source coverage")
|
|
281
|
+
return set(sources)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _snapshot_covers(old: dict[str, Any], new: dict[str, Any]) -> bool:
|
|
285
|
+
"""A shorter or divergent scan cannot replace a more complete turn snapshot."""
|
|
286
|
+
sources = _revision_sources(new)
|
|
287
|
+
if "revision_sources" in old and not _revision_sources(old) <= sources:
|
|
288
|
+
return False
|
|
289
|
+
for key in ("tool_calls", "tool_errors"):
|
|
290
|
+
before, after = old.get(key, 0), new.get(key, 0)
|
|
291
|
+
if type(before) is not int or type(after) is not int or after < before:
|
|
292
|
+
return False
|
|
293
|
+
return True
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _split(batch: list[Row]) -> tuple[list[UsageEvent], list[ActivityEvent], list[TimeInterval]]:
|
|
297
|
+
usage = [e for e in batch if isinstance(e, UsageEvent)]
|
|
298
|
+
acts = [e for e in batch if isinstance(e, ActivityEvent)]
|
|
299
|
+
return usage, acts, [e for e in batch if isinstance(e, TimeInterval)]
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def insert_pg(dsn: str, batch: list[Row], with_usage: bool = True) -> None:
|
|
303
|
+
"""Deliver rows together; durably queue covered snapshot changes lacking UPDATE permission.
|
|
304
|
+
A raise leaves the watermark unchanged, so committed rows retry with stable IDs.
|
|
305
|
+
with_usage=False: the webhook already carried the usage rows."""
|
|
306
|
+
usage, acts, times = _split(batch)
|
|
307
|
+
rows = [_params(e.to_dict(), COLUMNS) for e in usage] if with_usage else []
|
|
308
|
+
sessions = _session_rows(usage)
|
|
309
|
+
deferred = []
|
|
310
|
+
with connect(dsn) as conn:
|
|
311
|
+
with conn.cursor() as cur:
|
|
312
|
+
if rows:
|
|
313
|
+
cur.executemany(_USAGE_SQL, rows)
|
|
314
|
+
if sessions:
|
|
315
|
+
cur.executemany(_SESSION_SQL, sessions)
|
|
316
|
+
if acts:
|
|
317
|
+
deferred = _write_activities(cur, [asdict(a) for a in acts])
|
|
318
|
+
if times:
|
|
319
|
+
cur.executemany(_TIME_SQL, [_params(asdict(t), TIME_COLUMNS) for t in times])
|
|
320
|
+
conn.commit()
|
|
321
|
+
if deferred:
|
|
322
|
+
_queue_rest([ActivityEvent(**row) for row in deferred])
|
|
323
|
+
_snapshot_permission_note()
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def send_activity_rows(dsn: str, acts: list[ActivityEvent]) -> None:
|
|
327
|
+
"""Activity rows only (the activity backfill). Idempotent on (event_id, ts); raises on failure."""
|
|
328
|
+
if acts:
|
|
329
|
+
insert_pg(dsn, list(acts))
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def send_session_rows(dsn: str, events: list[UsageEvent]) -> None:
|
|
333
|
+
"""The events' session rows only (the sessions-only backfill). Idempotent on event_id; raises
|
|
334
|
+
on failure."""
|
|
335
|
+
write_many(dsn, _SESSION_SQL, _session_rows(events))
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def webhook_url_allowed(url: str) -> bool:
|
|
339
|
+
"""Allowlist, not blocklist: only https, or http to localhost for testing. Anything else
|
|
340
|
+
(http to a remote host, or ftp:/file:/no-scheme) could leak the secret or misfire."""
|
|
341
|
+
try:
|
|
342
|
+
parsed = urlparse(url)
|
|
343
|
+
return parsed.scheme.lower() == "https" or (
|
|
344
|
+
parsed.scheme.lower() == "http" and (parsed.hostname or "").lower() in LOCAL_HOSTS)
|
|
345
|
+
except ValueError:
|
|
346
|
+
return False
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _check_webhook(cfg: Config) -> None:
|
|
350
|
+
"""Raises if a configured webhook is missing its secret or would leak it."""
|
|
351
|
+
url = cfg.webhook_url or ""
|
|
352
|
+
if not cfg.webhook_secret:
|
|
353
|
+
raise AutError(
|
|
354
|
+
"transport.webhook_url is set but webhook_secret is missing or unresolved; "
|
|
355
|
+
"refusing to send unauthenticated. Set webhook_secret (or its env: value)."
|
|
356
|
+
)
|
|
357
|
+
parsed = urlparse(url)
|
|
358
|
+
scheme = parsed.scheme.lower()
|
|
359
|
+
host = (parsed.hostname or "").lower()
|
|
360
|
+
if not webhook_url_allowed(url):
|
|
361
|
+
raise AutError(
|
|
362
|
+
f"refusing to send the webhook secret to host '{host}': only https:// "
|
|
363
|
+
f"(or http:// to localhost for testing) is allowed, not '{scheme or 'no'}-scheme'."
|
|
364
|
+
)
|
|
365
|
+
# A warning, not a refusal: a webhook outside Access works today without these headers.
|
|
366
|
+
if bool(cfg.cf_access_client_id) != bool(cfg.cf_access_client_secret):
|
|
367
|
+
print("aut: warning: only one of cf_access_client_id and cf_access_client_secret is "
|
|
368
|
+
"set, so no Cloudflare Access headers are sent.", file=sys.stderr)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
class _Sender:
|
|
372
|
+
"""send(batch): usage over the webhook, or over pg_dsn without one or when it fails, and every
|
|
373
|
+
other row over pg_dsn. Each stderr note shows once per scan."""
|
|
374
|
+
|
|
375
|
+
def __init__(self, cfg: Config) -> None:
|
|
376
|
+
self.cfg = cfg
|
|
377
|
+
self.note_shown = self.fallback_shown = False
|
|
378
|
+
self.replayed = False
|
|
379
|
+
|
|
380
|
+
def __call__(self, batch: list[Row]) -> None:
|
|
381
|
+
if self.cfg.pg_dsn and not self.replayed:
|
|
382
|
+
replay_outbox(self.cfg)
|
|
383
|
+
self.replayed = True
|
|
384
|
+
if not batch:
|
|
385
|
+
return
|
|
386
|
+
cfg = self.cfg
|
|
387
|
+
webhook_err = None
|
|
388
|
+
if cfg.webhook_url and cfg.webhook_secret:
|
|
389
|
+
webhook_err = self._post_usage(batch, cfg.webhook_url, cfg.webhook_secret)
|
|
390
|
+
if webhook_err is None:
|
|
391
|
+
self._send_the_rest(batch)
|
|
392
|
+
return
|
|
393
|
+
if cfg.pg_dsn:
|
|
394
|
+
insert_pg(cfg.pg_dsn, batch)
|
|
395
|
+
return
|
|
396
|
+
raise DeliveryError("webhook delivery failed and no PG fallback: "
|
|
397
|
+
f"{redacted_error(webhook_err) if webhook_err else 'not sent'}") \
|
|
398
|
+
from webhook_err
|
|
399
|
+
|
|
400
|
+
def _post_usage(self, batch: list[Row], url: str, secret: str) -> Exception | None:
|
|
401
|
+
"""The webhook's failure, if any: the caller falls back to pg_dsn on it."""
|
|
402
|
+
usage = _split(batch)[0]
|
|
403
|
+
try:
|
|
404
|
+
if usage:
|
|
405
|
+
post_webhook(url, secret, usage, self.cfg.cf_access_headers)
|
|
406
|
+
except (*HTTP_ERRORS, RuntimeError) as exc:
|
|
407
|
+
if self.cfg.pg_dsn and not self.fallback_shown: # printed first: a PG failure shows both
|
|
408
|
+
self.fallback_shown = True
|
|
409
|
+
print(f"aut: warning: webhook delivery failed, using transport.pg_dsn: "
|
|
410
|
+
f"{redacted_error(exc)}", file=sys.stderr)
|
|
411
|
+
return exc
|
|
412
|
+
return None
|
|
413
|
+
|
|
414
|
+
def _send_the_rest(self, batch: list[Row]) -> None:
|
|
415
|
+
"""What the webhook does not carry goes over pg_dsn; its failure raises, watermark stays."""
|
|
416
|
+
usage, acts, times = _split(batch)
|
|
417
|
+
if not (acts or times or any(e.session for e in usage)):
|
|
418
|
+
return
|
|
419
|
+
if self.cfg.pg_dsn:
|
|
420
|
+
insert_pg(self.cfg.pg_dsn, batch, with_usage=False)
|
|
421
|
+
else:
|
|
422
|
+
_queue_rest(batch)
|
|
423
|
+
if not self.note_shown:
|
|
424
|
+
self.note_shown = True
|
|
425
|
+
print(f"aut: note: session/activity/time rows queued privately at {outbox_path()}; "
|
|
426
|
+
"set transport.pg_dsn and run aut scan to replay them.", file=sys.stderr)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def make_sender(cfg: Config) -> Callable[[list[Row]], None]:
|
|
430
|
+
"""Return a send(batch), see _Sender. Raises (before any send) if nothing is configured or a
|
|
431
|
+
configured webhook is missing its secret."""
|
|
432
|
+
if not cfg.webhook_url and not cfg.pg_dsn:
|
|
433
|
+
raise AutError(
|
|
434
|
+
"no transport configured: set [transport].webhook_url (and/or pg_dsn) in your "
|
|
435
|
+
"config, or run with --dry-run to preview without sending."
|
|
436
|
+
)
|
|
437
|
+
if cfg.webhook_url:
|
|
438
|
+
_check_webhook(cfg)
|
|
439
|
+
return _Sender(cfg)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def error_type_name(exc: BaseException) -> str:
|
|
443
|
+
"""The error stored in a scan row: the exception TYPE only. Messages can carry a file path
|
|
444
|
+
(which names a project) or part of a credential, and no redaction pattern catches every
|
|
445
|
+
form, so the full message stays on this machine's stderr and never reaches the sink."""
|
|
446
|
+
return type(exc).__name__[:_ERROR_MAX]
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def redacted_error(exc: BaseException) -> str:
|
|
450
|
+
"""The error for this machine's stderr: type and message on one line, credentials redacted."""
|
|
451
|
+
return f"{type(exc).__name__}: {redact(' '.join(str(exc).split()))}"
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def is_operational_failure(exc: Exception) -> bool:
|
|
455
|
+
"""An outage, not a bug: a sink or webhook that failed, or an OSError, the OS's own report on a
|
|
456
|
+
file or socket (the line keeps its errno and path). Any other RuntimeError is a bug.
|
|
457
|
+
By module name because the driver is an optional extra; its errors are not imported."""
|
|
458
|
+
return isinstance(exc, (*HTTP_ERRORS, DeliveryError)) \
|
|
459
|
+
or type(exc).__module__.partition(".")[0] == "psycopg"
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def scan_row(cfg: Config, collector: str, started_at: datetime, *, status: ScanStatus = "ok",
|
|
463
|
+
error: str | None = None, events: int = 0, human: int = 0,
|
|
464
|
+
newest_ts: datetime | None = None) -> dict[str, Any]:
|
|
465
|
+
"""One agents.collector_scan row, finished now."""
|
|
466
|
+
return {"machine": cfg.machine, "collector": collector, "started_at": started_at,
|
|
467
|
+
"finished_at": datetime.now(timezone.utc), "status": status, "events": events,
|
|
468
|
+
"human": human, "newest_ts": newest_ts, "error": error, "aut_version": __version__}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def send_scan_rows(dsn: str, rows: list[dict[str, Any]]) -> None:
|
|
472
|
+
"""Scan heartbeat rows. Idempotent on (machine, collector, finished_at); raises on failure."""
|
|
473
|
+
def write(batch: list[dict[str, Any]]) -> None:
|
|
474
|
+
# 5 s, not 15: usage is already delivered, so an unreachable sink should not hold the exit.
|
|
475
|
+
write_many(dsn, _SCAN_SQL, [_params(r, SCAN_COLUMNS) for r in batch], timeout=5)
|
|
476
|
+
|
|
477
|
+
try:
|
|
478
|
+
write(rows)
|
|
479
|
+
except Exception as exc:
|
|
480
|
+
# A CHECK with only ok and error refuses a degraded row, and with it the whole delivery:
|
|
481
|
+
# every collector of the scan would look dead. By name: the driver is an optional extra.
|
|
482
|
+
if type(exc).__name__ != "CheckViolation" or all(r["status"] != "degraded" for r in rows):
|
|
483
|
+
raise
|
|
484
|
+
write([{**r, "status": "error"} if r["status"] == "degraded" else r for r in rows])
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def deliver_scan_rows(cfg: Config, rows: list[dict[str, Any]], what: str = "scan") -> None:
|
|
488
|
+
"""The heartbeat is best effort: whatever goes wrong here must not touch the job it reports
|
|
489
|
+
on, and that includes a missing driver's AutError. No pg_dsn: skipped with a note."""
|
|
490
|
+
if not rows:
|
|
491
|
+
return
|
|
492
|
+
if not cfg.pg_dsn:
|
|
493
|
+
# shortcut: webhook path for scan rows not built; upgrade when a machine runs webhook-only.
|
|
494
|
+
print(f"aut: note: {what} heartbeat not sent (needs transport.pg_dsn)", file=sys.stderr)
|
|
495
|
+
return
|
|
496
|
+
try:
|
|
497
|
+
send_scan_rows(cfg.pg_dsn, rows)
|
|
498
|
+
except Exception as exc:
|
|
499
|
+
print(f"aut: warning: {what} heartbeat not delivered: {redacted_error(exc)}", file=sys.stderr)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
@dataclass(slots=True)
|
|
503
|
+
class JobBeat:
|
|
504
|
+
"""What a one-row job's heartbeat will say; the job's body fills it in."""
|
|
505
|
+
status: ScanStatus = "ok"
|
|
506
|
+
error: str | None = None
|
|
507
|
+
events: int = 0
|
|
508
|
+
|
|
509
|
+
def fail(self, exc: BaseException) -> None:
|
|
510
|
+
self.status, self.error = "error", error_type_name(exc)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
@contextmanager
|
|
514
|
+
def job_heartbeat(cfg: Config, name: str, started: datetime, *,
|
|
515
|
+
dry_run: bool) -> Iterator[JobBeat]:
|
|
516
|
+
"""One heartbeat row when the body ends: a failure still reports and re-raises, Ctrl-C leaves
|
|
517
|
+
at once and sends nothing, a dry run stays silent. run_scan keeps its own: it sends one row per
|
|
518
|
+
collector in a single delivery."""
|
|
519
|
+
beat = JobBeat()
|
|
520
|
+
interrupted = False
|
|
521
|
+
try:
|
|
522
|
+
yield beat
|
|
523
|
+
except BaseException as exc:
|
|
524
|
+
beat.fail(exc)
|
|
525
|
+
interrupted = isinstance(exc, KeyboardInterrupt)
|
|
526
|
+
raise
|
|
527
|
+
finally:
|
|
528
|
+
if not dry_run and not interrupted:
|
|
529
|
+
deliver_scan_rows(cfg, [scan_row(cfg, name, started, status=beat.status,
|
|
530
|
+
error=beat.error, events=beat.events)], name)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""The privacy rule for local text that may leave the machine or reach a page: titles, steer text,
|
|
2
|
+
review snippets. Owned here because every reader needs it, whichever collectors are enabled.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
_TEXT_MAX = 300
|
|
10
|
+
|
|
11
|
+
_REDACTED = "[redacted]"
|
|
12
|
+
# (name, pattern, replacement). Order matters: URL credentials before emails, specific before
|
|
13
|
+
# the generic long run.
|
|
14
|
+
PATTERNS = (
|
|
15
|
+
# These read every word of a pasted log, so no pattern may rescan a long run from each of its
|
|
16
|
+
# characters. Anchored on "://": matching the scheme too was quadratic on dotted text.
|
|
17
|
+
("url_credentials", re.compile(r"://[^\s/@:]+:[^\s/@]+@"), "://" + _REDACTED + "@"),
|
|
18
|
+
# no leading \b: env names (PGPASSWORD=, DB_PASSWORD=, GITHUB_TOKEN=) carry a prefix, and it
|
|
19
|
+
# stays in the text untouched, so matching it too (cubic on a word run) would add nothing
|
|
20
|
+
("assignment", re.compile(r"(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key)"
|
|
21
|
+
r"(\s*[=:]\s*)\S+", re.I), r"\1\2" + _REDACTED),
|
|
22
|
+
("bearer", re.compile(r"\b(bearer\s+)\S+", re.I), r"\1" + _REDACTED),
|
|
23
|
+
("jwt", re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+"), _REDACTED),
|
|
24
|
+
("sk_key", re.compile(r"\bsk-[\w-]{16,}"), _REDACTED),
|
|
25
|
+
("github_pat", re.compile(r"\bgithub_pat_\w{20,}"), _REDACTED),
|
|
26
|
+
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), _REDACTED),
|
|
27
|
+
("slack_token", re.compile(r"\bxox[abprs]-[\w-]{10,}"), _REDACTED),
|
|
28
|
+
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), _REDACTED),
|
|
29
|
+
# 64: the longest local part an address can have, so a run with no @ is never rescanned in full
|
|
30
|
+
("email", re.compile(r"[\w.+-]{1,64}@[\w-]+\.[\w.-]*[\w-]"), _REDACTED),
|
|
31
|
+
# long hex and base64 runs; also takes long slugs and paths, which is the safe side
|
|
32
|
+
("long_run", re.compile(r"[A-Za-z0-9+/_=-]{32,}"), _REDACTED),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def redact(text: str) -> str:
|
|
37
|
+
"""The redaction alone."""
|
|
38
|
+
for _, pattern, repl in PATTERNS:
|
|
39
|
+
text = pattern.sub(repl, text)
|
|
40
|
+
return text
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def clean_text(text: str) -> str:
|
|
44
|
+
"""Redacted, then cut to 300 characters."""
|
|
45
|
+
return redact(text)[:_TEXT_MAX]
|