agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,530 @@
1
+ """Transport: push normalized rows to the sink.
2
+
3
+ A scan's usage rows go over the n8n webhook (behind Cloudflare Access) when one is configured, and
4
+ fall back to direct Postgres (transport.pg_dsn) when it fails, or go to Postgres alone without a
5
+ webhook. Everything else (session, activity, time, outcome, label and heartbeat rows) goes over
6
+ pg_dsn only. All of it is idempotent at the sink via ON CONFLICT, so re-delivery is harmless.
7
+
8
+ A sender that cannot deliver or durably queue rows raises, so the watermark stays unchanged.
9
+ Webhook-only machines keep session, activity and time rows in a private local SQLite outbox.
10
+ Postgres delivery replays that outbox before sending new rows.
11
+
12
+ Delivery is strict on purpose: redirects are NOT followed (a 3xx that turns the POST into a
13
+ GET could otherwise look like success and advance the watermark over data the sink never
14
+ stored), and any non-2xx is a hard failure.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import os
21
+ import sqlite3
22
+ import sys
23
+ import urllib.error
24
+ import urllib.request
25
+ from contextlib import contextmanager
26
+ from dataclasses import asdict, dataclass
27
+ from datetime import datetime, timezone
28
+ from pathlib import Path
29
+ from typing import Any, Callable, Iterator, Literal
30
+ from urllib.parse import urlparse
31
+
32
+ from . import __version__
33
+ from .config import AutError, Config
34
+ from .event import ActivityEvent, Row, TimeInterval, UsageEvent
35
+ from .net import HTTP_ERRORS, LOCAL_HOSTS, NoRedirect
36
+ from .redact import redact
37
+ from .sink import connect, write_many
38
+ from .state import state_path
39
+
40
+ # Column order shared with the homelab schema (agents.usage_event). Keep in sync.
41
+ COLUMNS = (
42
+ "event_id", "ts", "machine", "agent", "kind", "model", "provider",
43
+ "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens",
44
+ "reasoning_tokens", "total_tokens", "est_cost_usd", "project",
45
+ )
46
+ TABLE = "agents.usage_event"
47
+
48
+ # One row per collector per scan (agents.collector_scan): proof a machine ran and how it ended.
49
+ SCAN_COLUMNS = (
50
+ "machine", "collector", "started_at", "finished_at", "status", "events", "human",
51
+ "newest_ts", "error", "aut_version",
52
+ )
53
+ SCAN_TABLE = "agents.collector_scan"
54
+
55
+ # One row per usage event_id (agents.event_session): which session, parent and turn it belongs to.
56
+ SESSION_COLUMNS = (
57
+ "event_id", "machine", "agent", "session_id", "parent_session_id", "run_origin", "turn_id",
58
+ )
59
+ SESSION_TABLE = "agents.event_session"
60
+
61
+ # One row per non-token activity (agents.activity_event): steers, commits, turns. attrs is jsonb.
62
+ ACTIVITY_COLUMNS = ("event_id", "ts", "machine", "agent", "kind", "session_id", "turn_id", "attrs")
63
+ ACTIVITY_TABLE = "agents.activity_event"
64
+
65
+ # One row per active-time interval (agents.time_interval): ActivityWatch window time and manual
66
+ # entries. title is private (redacted on this machine; retention is off by default, see CONTRACT.md).
67
+ TIME_COLUMNS = ("event_id", "ts", "end_ts", "machine", "source", "app", "title", "category",
68
+ "work_relevant", "attrs")
69
+ TIME_TABLE = "agents.time_interval"
70
+ _ERROR_MAX = 300
71
+ _JSONB_COLUMN = "attrs" # the one jsonb column of these tables
72
+ ScanStatus = Literal["ok", "degraded", "error"]
73
+
74
+
75
+ class DeliveryError(RuntimeError):
76
+ """Rows could not be delivered and no route is left: an outage main reports on one line."""
77
+
78
+
79
+ def post_webhook(url: str, secret: str, batch: list[UsageEvent],
80
+ extra_headers: dict[str, str] | None = None) -> None:
81
+ body = json.dumps({"events": [e.to_dict() for e in batch]}).encode("utf-8")
82
+ headers = {"Content-Type": "application/json", "X-AUT-Secret": secret}
83
+ if extra_headers: # e.g. CF-Access-Client-Id/Secret to clear Cloudflare Access
84
+ headers.update(extra_headers)
85
+ req = urllib.request.Request(url, data=body, method="POST", headers=headers)
86
+ try:
87
+ with urllib.request.build_opener(NoRedirect).open(req, timeout=30) as resp:
88
+ status = resp.status
89
+ except urllib.error.HTTPError as exc: # 3xx (no-follow) / 4xx / 5xx
90
+ raise RuntimeError(f"webhook HTTP {exc.code}") from exc
91
+ if not (200 <= status < 300):
92
+ raise RuntimeError(f"webhook HTTP {status}")
93
+
94
+
95
+ def _insert_sql(table: str, columns: tuple[str, ...], conflict: str) -> str:
96
+ marks = ", ".join("%s::jsonb" if c == _JSONB_COLUMN else "%s" for c in columns)
97
+ return (f"INSERT INTO {table} ({', '.join(columns)}) VALUES " # nosec B608 # constants only
98
+ f"({marks}) ON CONFLICT ({conflict}) DO NOTHING")
99
+
100
+
101
+ def _params(row: dict[str, Any], columns: tuple[str, ...]) -> tuple[Any, ...]:
102
+ return tuple(json.dumps(row[c], sort_keys=True) if c == _JSONB_COLUMN else row[c]
103
+ for c in columns)
104
+
105
+
106
+ def _session_rows(batch: list[UsageEvent]) -> list[tuple[Any, ...]]:
107
+ return [_params(r, SESSION_COLUMNS) for r in map(UsageEvent.session_row, batch) if r]
108
+
109
+
110
+ _USAGE_SQL = _insert_sql(TABLE, COLUMNS, "event_id, ts")
111
+ _SESSION_SQL = _insert_sql(SESSION_TABLE, SESSION_COLUMNS, "event_id")
112
+ _ACTIVITY_SQL = _insert_sql(ACTIVITY_TABLE, ACTIVITY_COLUMNS, "event_id, ts")
113
+ _TIME_SQL = _insert_sql(TIME_TABLE, TIME_COLUMNS, "event_id, ts")
114
+ _SCAN_SQL = _insert_sql(SCAN_TABLE, SCAN_COLUMNS, "machine, collector, finished_at")
115
+ _OUTBOX_TYPES = {"session": (_SESSION_SQL, SESSION_COLUMNS),
116
+ "activity": (_ACTIVITY_SQL, ACTIVITY_COLUMNS), "time": (_TIME_SQL, TIME_COLUMNS)}
117
+ _OUTBOX_BATCH = 500
118
+
119
+
120
+ def outbox_path() -> Path:
121
+ """Private deferred sink rows, kept independently of collector watermarks."""
122
+ return state_path().parent / "delivery-outbox" / "pending.sqlite3"
123
+
124
+
125
+ @contextmanager
126
+ def _outbox() -> Iterator[sqlite3.Connection]:
127
+ path = outbox_path()
128
+ try:
129
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
130
+ os.chmod(path.parent, 0o700)
131
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600)
132
+ os.close(fd)
133
+ os.chmod(path, 0o600)
134
+ conn = sqlite3.connect(path, timeout=30)
135
+ try:
136
+ conn.execute("PRAGMA secure_delete=ON")
137
+ conn.execute("CREATE TABLE IF NOT EXISTS pending ("
138
+ "id INTEGER PRIMARY KEY AUTOINCREMENT, "
139
+ "kind TEXT NOT NULL, event_id TEXT NOT NULL, ts TEXT NOT NULL, "
140
+ "payload TEXT NOT NULL, UNIQUE(kind, event_id, ts))")
141
+ yield conn
142
+ finally:
143
+ conn.close()
144
+ except (sqlite3.Error, OSError, ValueError, KeyError, TypeError) as exc:
145
+ raise DeliveryError("local delivery outbox failed; watermark unchanged. "
146
+ f"Keep {path} and fix its storage or permissions: {exc}") from exc
147
+
148
+
149
+ def outbox_status() -> dict[str, int]:
150
+ """Pending rows by sink type. Corrupt or unreadable storage raises, never appears empty."""
151
+ if not outbox_path().exists():
152
+ return {}
153
+ with _outbox() as conn:
154
+ return dict(conn.execute("SELECT kind, COUNT(*) FROM pending GROUP BY kind"))
155
+
156
+
157
+ def _queue_rest(batch: list[Row]) -> None:
158
+ with _outbox() as conn:
159
+ with conn:
160
+ conn.execute("BEGIN IMMEDIATE")
161
+ for event in batch:
162
+ if isinstance(event, UsageEvent):
163
+ row = event.session_row()
164
+ if row is None:
165
+ continue
166
+ kind = "session"
167
+ else:
168
+ row = asdict(event)
169
+ kind = "activity" if isinstance(event, ActivityEvent) else "time"
170
+ snapshot = (kind == "activity" and row["kind"] in ("turn", "link")
171
+ and "revision_sources" in row["attrs"])
172
+ if snapshot:
173
+ _revision_sources(row["attrs"])
174
+ old = conn.execute("SELECT payload FROM pending "
175
+ "WHERE kind=? AND event_id=? AND ts=?",
176
+ (kind, event.event_id, event.ts or "")).fetchone()
177
+ if old and not _snapshot_covers(json.loads(old[0])["attrs"], row["attrs"]):
178
+ continue
179
+ conn.execute("INSERT INTO pending(kind, event_id, ts, payload) VALUES (?, ?, ?, ?) "
180
+ "ON CONFLICT(kind, event_id, ts) DO UPDATE SET payload=excluded.payload "
181
+ "WHERE ?", (kind, event.event_id, event.ts or "",
182
+ json.dumps(row, sort_keys=True), snapshot))
183
+
184
+
185
+ def replay_outbox(cfg: Config) -> int:
186
+ """Replay a finite FIFO snapshot, at most 500 rows in memory, deleting after PG commit only."""
187
+ dsn = cfg.require_pg("outbox replay")
188
+ if not outbox_path().exists():
189
+ return 0
190
+ delivered = 0
191
+ deferred = False
192
+ after = 0
193
+ with _outbox() as conn:
194
+ ceiling = conn.execute("SELECT COALESCE(MAX(rowid), 0) FROM pending").fetchone()[0]
195
+ while True:
196
+ with conn:
197
+ # A writer cannot replace a row between its delivery and local acknowledgement.
198
+ conn.execute("BEGIN IMMEDIATE")
199
+ pending = conn.execute("SELECT rowid, kind, payload FROM pending "
200
+ "WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ?",
201
+ (after, ceiling, _OUTBOX_BATCH)).fetchall()
202
+ if not pending:
203
+ break
204
+ records = [(rowid, kind, json.loads(payload)) for rowid, kind, payload in pending]
205
+ blocked: set[int] = set()
206
+ with connect(dsn) as pg:
207
+ with pg.cursor() as cur:
208
+ for kind, (sql, columns) in _OUTBOX_TYPES.items():
209
+ rows = [row for _, k, row in records if k == kind]
210
+ if kind == "activity":
211
+ waiting = _write_activities(cur, rows)
212
+ keys = {(r["event_id"], r["ts"]) for r in waiting}
213
+ blocked.update(rowid for rowid, k, row in records if k == kind
214
+ and (row["event_id"], row["ts"]) in keys)
215
+ elif rows:
216
+ cur.executemany(sql, [_params(row, columns) for row in rows])
217
+ if any(kind not in _OUTBOX_TYPES for _, kind, _ in pending):
218
+ raise DeliveryError("unknown row type in local delivery outbox")
219
+ pg.commit()
220
+ conn.executemany("DELETE FROM pending WHERE rowid=?",
221
+ [(r[0],) for r in pending if r[0] not in blocked])
222
+ delivered += len(pending) - len(blocked)
223
+ deferred |= bool(blocked)
224
+ after = pending[-1][0]
225
+ if deferred:
226
+ _snapshot_permission_note()
227
+ return delivered
228
+
229
+
230
+ def _snapshot_permission_note() -> None:
231
+ print(f"aut: note: turn/link snapshot updates queued privately at {outbox_path()}; "
232
+ "the sink role needs UPDATE(attrs) on agents.activity_event. "
233
+ "Grant that permission, then run aut scan to replay them.", file=sys.stderr)
234
+
235
+
236
+ def _write_activities(cur: Any, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
237
+ """Ordinary events are immutable; covered turn snapshots may refresh their aggregate counts."""
238
+ snapshots, ordinary = [], []
239
+ deferred = []
240
+ can_update = None
241
+ for row in rows:
242
+ if row["kind"] in ("turn", "link") and "revision_sources" in row["attrs"]:
243
+ snapshots.append(row)
244
+ else:
245
+ ordinary.append(row)
246
+ if ordinary:
247
+ cur.executemany(_ACTIVITY_SQL, [_params(row, ACTIVITY_COLUMNS) for row in ordinary])
248
+ for row in sorted(snapshots, key=lambda r: r["event_id"]):
249
+ _revision_sources(row["attrs"])
250
+ cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (row["event_id"],))
251
+ # Match event_id alone because older Pi snapshots used the turn's end timestamp.
252
+ cur.execute("SELECT ts, agent, kind, machine, attrs FROM agents.activity_event "
253
+ "WHERE event_id=%s", (row["event_id"],))
254
+ existing = cur.fetchall()
255
+ if not existing:
256
+ cur.executemany(_ACTIVITY_SQL, [_params(row, ACTIVITY_COLUMNS)])
257
+ continue
258
+ for ts, agent, kind, machine, attrs in existing:
259
+ if (agent, kind) != (row["agent"], row["kind"]):
260
+ raise DeliveryError("turn snapshot event identity conflicts with an existing row")
261
+ for ts, agent, kind, machine, attrs in existing:
262
+ if attrs != row["attrs"] and _snapshot_covers(attrs, row["attrs"]):
263
+ if can_update is None:
264
+ cur.execute("SELECT has_column_privilege(current_user, "
265
+ "'agents.activity_event', 'attrs', 'UPDATE')", ())
266
+ can_update = bool(cur.fetchone()[0])
267
+ if not can_update:
268
+ deferred.append(row)
269
+ break
270
+ cur.execute("UPDATE agents.activity_event SET attrs=%s::jsonb "
271
+ "WHERE event_id=%s AND ts=%s",
272
+ (json.dumps(row["attrs"], sort_keys=True), row["event_id"], ts))
273
+ return deferred
274
+
275
+
276
+ def _revision_sources(attrs: dict[str, Any]) -> set[str]:
277
+ sources = attrs.get("revision_sources")
278
+ if not isinstance(sources, list) or not sources or any(
279
+ not isinstance(source, str) or not source for source in sources):
280
+ raise DeliveryError("turn snapshot has invalid revision source coverage")
281
+ return set(sources)
282
+
283
+
284
+ def _snapshot_covers(old: dict[str, Any], new: dict[str, Any]) -> bool:
285
+ """A shorter or divergent scan cannot replace a more complete turn snapshot."""
286
+ sources = _revision_sources(new)
287
+ if "revision_sources" in old and not _revision_sources(old) <= sources:
288
+ return False
289
+ for key in ("tool_calls", "tool_errors"):
290
+ before, after = old.get(key, 0), new.get(key, 0)
291
+ if type(before) is not int or type(after) is not int or after < before:
292
+ return False
293
+ return True
294
+
295
+
296
+ def _split(batch: list[Row]) -> tuple[list[UsageEvent], list[ActivityEvent], list[TimeInterval]]:
297
+ usage = [e for e in batch if isinstance(e, UsageEvent)]
298
+ acts = [e for e in batch if isinstance(e, ActivityEvent)]
299
+ return usage, acts, [e for e in batch if isinstance(e, TimeInterval)]
300
+
301
+
302
+ def insert_pg(dsn: str, batch: list[Row], with_usage: bool = True) -> None:
303
+ """Deliver rows together; durably queue covered snapshot changes lacking UPDATE permission.
304
+ A raise leaves the watermark unchanged, so committed rows retry with stable IDs.
305
+ with_usage=False: the webhook already carried the usage rows."""
306
+ usage, acts, times = _split(batch)
307
+ rows = [_params(e.to_dict(), COLUMNS) for e in usage] if with_usage else []
308
+ sessions = _session_rows(usage)
309
+ deferred = []
310
+ with connect(dsn) as conn:
311
+ with conn.cursor() as cur:
312
+ if rows:
313
+ cur.executemany(_USAGE_SQL, rows)
314
+ if sessions:
315
+ cur.executemany(_SESSION_SQL, sessions)
316
+ if acts:
317
+ deferred = _write_activities(cur, [asdict(a) for a in acts])
318
+ if times:
319
+ cur.executemany(_TIME_SQL, [_params(asdict(t), TIME_COLUMNS) for t in times])
320
+ conn.commit()
321
+ if deferred:
322
+ _queue_rest([ActivityEvent(**row) for row in deferred])
323
+ _snapshot_permission_note()
324
+
325
+
326
+ def send_activity_rows(dsn: str, acts: list[ActivityEvent]) -> None:
327
+ """Activity rows only (the activity backfill). Idempotent on (event_id, ts); raises on failure."""
328
+ if acts:
329
+ insert_pg(dsn, list(acts))
330
+
331
+
332
+ def send_session_rows(dsn: str, events: list[UsageEvent]) -> None:
333
+ """The events' session rows only (the sessions-only backfill). Idempotent on event_id; raises
334
+ on failure."""
335
+ write_many(dsn, _SESSION_SQL, _session_rows(events))
336
+
337
+
338
+ def webhook_url_allowed(url: str) -> bool:
339
+ """Allowlist, not blocklist: only https, or http to localhost for testing. Anything else
340
+ (http to a remote host, or ftp:/file:/no-scheme) could leak the secret or misfire."""
341
+ try:
342
+ parsed = urlparse(url)
343
+ return parsed.scheme.lower() == "https" or (
344
+ parsed.scheme.lower() == "http" and (parsed.hostname or "").lower() in LOCAL_HOSTS)
345
+ except ValueError:
346
+ return False
347
+
348
+
349
+ def _check_webhook(cfg: Config) -> None:
350
+ """Raises if a configured webhook is missing its secret or would leak it."""
351
+ url = cfg.webhook_url or ""
352
+ if not cfg.webhook_secret:
353
+ raise AutError(
354
+ "transport.webhook_url is set but webhook_secret is missing or unresolved; "
355
+ "refusing to send unauthenticated. Set webhook_secret (or its env: value)."
356
+ )
357
+ parsed = urlparse(url)
358
+ scheme = parsed.scheme.lower()
359
+ host = (parsed.hostname or "").lower()
360
+ if not webhook_url_allowed(url):
361
+ raise AutError(
362
+ f"refusing to send the webhook secret to host '{host}': only https:// "
363
+ f"(or http:// to localhost for testing) is allowed, not '{scheme or 'no'}-scheme'."
364
+ )
365
+ # A warning, not a refusal: a webhook outside Access works today without these headers.
366
+ if bool(cfg.cf_access_client_id) != bool(cfg.cf_access_client_secret):
367
+ print("aut: warning: only one of cf_access_client_id and cf_access_client_secret is "
368
+ "set, so no Cloudflare Access headers are sent.", file=sys.stderr)
369
+
370
+
371
+ class _Sender:
372
+ """send(batch): usage over the webhook, or over pg_dsn without one or when it fails, and every
373
+ other row over pg_dsn. Each stderr note shows once per scan."""
374
+
375
+ def __init__(self, cfg: Config) -> None:
376
+ self.cfg = cfg
377
+ self.note_shown = self.fallback_shown = False
378
+ self.replayed = False
379
+
380
+ def __call__(self, batch: list[Row]) -> None:
381
+ if self.cfg.pg_dsn and not self.replayed:
382
+ replay_outbox(self.cfg)
383
+ self.replayed = True
384
+ if not batch:
385
+ return
386
+ cfg = self.cfg
387
+ webhook_err = None
388
+ if cfg.webhook_url and cfg.webhook_secret:
389
+ webhook_err = self._post_usage(batch, cfg.webhook_url, cfg.webhook_secret)
390
+ if webhook_err is None:
391
+ self._send_the_rest(batch)
392
+ return
393
+ if cfg.pg_dsn:
394
+ insert_pg(cfg.pg_dsn, batch)
395
+ return
396
+ raise DeliveryError("webhook delivery failed and no PG fallback: "
397
+ f"{redacted_error(webhook_err) if webhook_err else 'not sent'}") \
398
+ from webhook_err
399
+
400
+ def _post_usage(self, batch: list[Row], url: str, secret: str) -> Exception | None:
401
+ """The webhook's failure, if any: the caller falls back to pg_dsn on it."""
402
+ usage = _split(batch)[0]
403
+ try:
404
+ if usage:
405
+ post_webhook(url, secret, usage, self.cfg.cf_access_headers)
406
+ except (*HTTP_ERRORS, RuntimeError) as exc:
407
+ if self.cfg.pg_dsn and not self.fallback_shown: # printed first: a PG failure shows both
408
+ self.fallback_shown = True
409
+ print(f"aut: warning: webhook delivery failed, using transport.pg_dsn: "
410
+ f"{redacted_error(exc)}", file=sys.stderr)
411
+ return exc
412
+ return None
413
+
414
+ def _send_the_rest(self, batch: list[Row]) -> None:
415
+ """What the webhook does not carry goes over pg_dsn; its failure raises, watermark stays."""
416
+ usage, acts, times = _split(batch)
417
+ if not (acts or times or any(e.session for e in usage)):
418
+ return
419
+ if self.cfg.pg_dsn:
420
+ insert_pg(self.cfg.pg_dsn, batch, with_usage=False)
421
+ else:
422
+ _queue_rest(batch)
423
+ if not self.note_shown:
424
+ self.note_shown = True
425
+ print(f"aut: note: session/activity/time rows queued privately at {outbox_path()}; "
426
+ "set transport.pg_dsn and run aut scan to replay them.", file=sys.stderr)
427
+
428
+
429
+ def make_sender(cfg: Config) -> Callable[[list[Row]], None]:
430
+ """Return a send(batch), see _Sender. Raises (before any send) if nothing is configured or a
431
+ configured webhook is missing its secret."""
432
+ if not cfg.webhook_url and not cfg.pg_dsn:
433
+ raise AutError(
434
+ "no transport configured: set [transport].webhook_url (and/or pg_dsn) in your "
435
+ "config, or run with --dry-run to preview without sending."
436
+ )
437
+ if cfg.webhook_url:
438
+ _check_webhook(cfg)
439
+ return _Sender(cfg)
440
+
441
+
442
+ def error_type_name(exc: BaseException) -> str:
443
+ """The error stored in a scan row: the exception TYPE only. Messages can carry a file path
444
+ (which names a project) or part of a credential, and no redaction pattern catches every
445
+ form, so the full message stays on this machine's stderr and never reaches the sink."""
446
+ return type(exc).__name__[:_ERROR_MAX]
447
+
448
+
449
+ def redacted_error(exc: BaseException) -> str:
450
+ """The error for this machine's stderr: type and message on one line, credentials redacted."""
451
+ return f"{type(exc).__name__}: {redact(' '.join(str(exc).split()))}"
452
+
453
+
454
+ def is_operational_failure(exc: Exception) -> bool:
455
+ """An outage, not a bug: a sink or webhook that failed, or an OSError, the OS's own report on a
456
+ file or socket (the line keeps its errno and path). Any other RuntimeError is a bug.
457
+ By module name because the driver is an optional extra; its errors are not imported."""
458
+ return isinstance(exc, (*HTTP_ERRORS, DeliveryError)) \
459
+ or type(exc).__module__.partition(".")[0] == "psycopg"
460
+
461
+
462
+ def scan_row(cfg: Config, collector: str, started_at: datetime, *, status: ScanStatus = "ok",
463
+ error: str | None = None, events: int = 0, human: int = 0,
464
+ newest_ts: datetime | None = None) -> dict[str, Any]:
465
+ """One agents.collector_scan row, finished now."""
466
+ return {"machine": cfg.machine, "collector": collector, "started_at": started_at,
467
+ "finished_at": datetime.now(timezone.utc), "status": status, "events": events,
468
+ "human": human, "newest_ts": newest_ts, "error": error, "aut_version": __version__}
469
+
470
+
471
+ def send_scan_rows(dsn: str, rows: list[dict[str, Any]]) -> None:
472
+ """Scan heartbeat rows. Idempotent on (machine, collector, finished_at); raises on failure."""
473
+ def write(batch: list[dict[str, Any]]) -> None:
474
+ # 5 s, not 15: usage is already delivered, so an unreachable sink should not hold the exit.
475
+ write_many(dsn, _SCAN_SQL, [_params(r, SCAN_COLUMNS) for r in batch], timeout=5)
476
+
477
+ try:
478
+ write(rows)
479
+ except Exception as exc:
480
+ # A CHECK with only ok and error refuses a degraded row, and with it the whole delivery:
481
+ # every collector of the scan would look dead. By name: the driver is an optional extra.
482
+ if type(exc).__name__ != "CheckViolation" or all(r["status"] != "degraded" for r in rows):
483
+ raise
484
+ write([{**r, "status": "error"} if r["status"] == "degraded" else r for r in rows])
485
+
486
+
487
+ def deliver_scan_rows(cfg: Config, rows: list[dict[str, Any]], what: str = "scan") -> None:
488
+ """The heartbeat is best effort: whatever goes wrong here must not touch the job it reports
489
+ on, and that includes a missing driver's AutError. No pg_dsn: skipped with a note."""
490
+ if not rows:
491
+ return
492
+ if not cfg.pg_dsn:
493
+ # shortcut: webhook path for scan rows not built; upgrade when a machine runs webhook-only.
494
+ print(f"aut: note: {what} heartbeat not sent (needs transport.pg_dsn)", file=sys.stderr)
495
+ return
496
+ try:
497
+ send_scan_rows(cfg.pg_dsn, rows)
498
+ except Exception as exc:
499
+ print(f"aut: warning: {what} heartbeat not delivered: {redacted_error(exc)}", file=sys.stderr)
500
+
501
+
502
+ @dataclass(slots=True)
503
+ class JobBeat:
504
+ """What a one-row job's heartbeat will say; the job's body fills it in."""
505
+ status: ScanStatus = "ok"
506
+ error: str | None = None
507
+ events: int = 0
508
+
509
+ def fail(self, exc: BaseException) -> None:
510
+ self.status, self.error = "error", error_type_name(exc)
511
+
512
+
513
+ @contextmanager
514
+ def job_heartbeat(cfg: Config, name: str, started: datetime, *,
515
+ dry_run: bool) -> Iterator[JobBeat]:
516
+ """One heartbeat row when the body ends: a failure still reports and re-raises, Ctrl-C leaves
517
+ at once and sends nothing, a dry run stays silent. run_scan keeps its own: it sends one row per
518
+ collector in a single delivery."""
519
+ beat = JobBeat()
520
+ interrupted = False
521
+ try:
522
+ yield beat
523
+ except BaseException as exc:
524
+ beat.fail(exc)
525
+ interrupted = isinstance(exc, KeyboardInterrupt)
526
+ raise
527
+ finally:
528
+ if not dry_run and not interrupted:
529
+ deliver_scan_rows(cfg, [scan_row(cfg, name, started, status=beat.status,
530
+ error=beat.error, events=beat.events)], name)
@@ -0,0 +1,45 @@
1
+ """The privacy rule for local text that may leave the machine or reach a page: titles, steer text,
2
+ review snippets. Owned here because every reader needs it, whichever collectors are enabled.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ import re
8
+
9
+ _TEXT_MAX = 300
10
+
11
+ _REDACTED = "[redacted]"
12
+ # (name, pattern, replacement). Order matters: URL credentials before emails, specific before
13
+ # the generic long run.
14
+ PATTERNS = (
15
+ # These read every word of a pasted log, so no pattern may rescan a long run from each of its
16
+ # characters. Anchored on "://": matching the scheme too was quadratic on dotted text.
17
+ ("url_credentials", re.compile(r"://[^\s/@:]+:[^\s/@]+@"), "://" + _REDACTED + "@"),
18
+ # no leading \b: env names (PGPASSWORD=, DB_PASSWORD=, GITHUB_TOKEN=) carry a prefix, and it
19
+ # stays in the text untouched, so matching it too (cubic on a word run) would add nothing
20
+ ("assignment", re.compile(r"(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key)"
21
+ r"(\s*[=:]\s*)\S+", re.I), r"\1\2" + _REDACTED),
22
+ ("bearer", re.compile(r"\b(bearer\s+)\S+", re.I), r"\1" + _REDACTED),
23
+ ("jwt", re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+"), _REDACTED),
24
+ ("sk_key", re.compile(r"\bsk-[\w-]{16,}"), _REDACTED),
25
+ ("github_pat", re.compile(r"\bgithub_pat_\w{20,}"), _REDACTED),
26
+ ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), _REDACTED),
27
+ ("slack_token", re.compile(r"\bxox[abprs]-[\w-]{10,}"), _REDACTED),
28
+ ("aws_key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), _REDACTED),
29
+ # 64: the longest local part an address can have, so a run with no @ is never rescanned in full
30
+ ("email", re.compile(r"[\w.+-]{1,64}@[\w-]+\.[\w.-]*[\w-]"), _REDACTED),
31
+ # long hex and base64 runs; also takes long slugs and paths, which is the safe side
32
+ ("long_run", re.compile(r"[A-Za-z0-9+/_=-]{32,}"), _REDACTED),
33
+ )
34
+
35
+
36
+ def redact(text: str) -> str:
37
+ """The redaction alone."""
38
+ for _, pattern, repl in PATTERNS:
39
+ text = pattern.sub(repl, text)
40
+ return text
41
+
42
+
43
+ def clean_text(text: str) -> str:
44
+ """Redacted, then cut to 300 characters."""
45
+ return redact(text)[:_TEXT_MAX]