agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,365 @@
1
+ """Private, content-free external observations. Unknown counts remain None."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ import re
10
+ import sqlite3
11
+ from datetime import datetime
12
+ from typing import Any, Iterable
13
+
14
+ MAX_FILE_BYTES = 8 * 1024 * 1024
15
+ MAX_RECORD_BYTES = 4096
16
+ MAX_BATCH_RECORDS = 10000
17
+ MAX_STORE_RECORDS = 1000000
18
+ COUNTS = (
19
+ "input_tokens",
20
+ "input_tokens_total",
21
+ "output_tokens",
22
+ "reasoning_tokens",
23
+ "cache_read_tokens",
24
+ "cache_write_tokens",
25
+ )
26
+ COMMON = {"version", "id", "kind", "timestamp"}
27
+ FIELDS = {
28
+ "usage": set(COUNTS) | {"start", "temporality", "counter"},
29
+ "quota": {"remaining", "limit", "unit", "period", "reset", "fresh_until", "scope"},
30
+ "tool": {"status", "duration_ms"},
31
+ }
32
+ IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/-]{0,127}\Z")
33
+
34
+
35
+ def timestamp(value: Any) -> datetime:
36
+ if not isinstance(value, str) or "T" not in value:
37
+ raise ValueError("timestamp must be an ISO timezone timestamp")
38
+ try:
39
+ result = datetime.fromisoformat(value.replace("Z", "+00:00"))
40
+ except ValueError:
41
+ raise ValueError("invalid timestamp") from None
42
+ if result.tzinfo is None:
43
+ raise ValueError("timestamp requires timezone")
44
+ return result
45
+
46
+
47
+ def identifier(value: Any) -> str:
48
+ if not isinstance(value, str) or not IDENTIFIER.fullmatch(value):
49
+ raise ValueError("invalid identifier")
50
+ return value
51
+
52
+
53
+ def _components_valid(r: dict[str, Any]) -> bool:
54
+ total = r["input_tokens_total"]
55
+ known_input = sum(r[key] for key in ("input_tokens", "cache_read_tokens") if r[key] is not None)
56
+ if r["input_tokens"] is not None and r["cache_write_tokens"] is not None:
57
+ known_input += r["cache_write_tokens"]
58
+ if total is not None and known_input > total:
59
+ return False
60
+ if total is not None and all(
61
+ r[key] is not None for key in ("input_tokens", "cache_read_tokens", "cache_write_tokens")
62
+ ) and known_input != total:
63
+ return False
64
+ return not (
65
+ r["reasoning_tokens"] is not None
66
+ and r["output_tokens"] is not None
67
+ and r["reasoning_tokens"] > r["output_tokens"]
68
+ )
69
+
70
+
71
+ def validate_record(record: Any) -> dict[str, Any]:
72
+ """Validate schema v1. Reject text and unknown fields at this trust boundary."""
73
+ if not isinstance(record, dict):
74
+ raise ValueError("record must be object")
75
+ r = dict(record)
76
+ kind = r.get("kind")
77
+ if (
78
+ not isinstance(kind, str)
79
+ or kind not in FIELDS
80
+ or type(r.get("version")) is not int
81
+ or r["version"] != 1
82
+ ):
83
+ raise ValueError("unknown schema version or kind")
84
+ if set(r) - COMMON - FIELDS[kind] or not COMMON <= set(r):
85
+ raise ValueError("missing or unknown fields")
86
+ identifier(r["id"])
87
+ timestamp(r["timestamp"])
88
+ for key in ("start", "reset", "fresh_until"):
89
+ if r.get(key) is not None:
90
+ timestamp(r[key])
91
+ for key in (*COUNTS, "remaining", "limit", "duration_ms"):
92
+ if key in r and r[key] is not None:
93
+ if type(r[key]) is not int or not 0 <= r[key] <= 2**63 - 1:
94
+ raise ValueError("counts must be nonnegative int64 or null")
95
+ if kind == "usage":
96
+ for key in COUNTS:
97
+ r.setdefault(key, None)
98
+ if not any(r[k] is not None for k in COUNTS):
99
+ raise ValueError("usage needs a known count")
100
+ if r.get("temporality", "delta") not in ("delta", "cumulative"):
101
+ raise ValueError("invalid temporality")
102
+ if ("start" in r or "temporality" in r) and "counter" not in r:
103
+ raise ValueError("counter windows require a series identifier")
104
+ if "counter" in r:
105
+ identifier(r["counter"])
106
+ if not _components_valid(r):
107
+ raise ValueError("input components disagree with total or reasoning subset exceeds total")
108
+ if r.get("start") is not None and timestamp(r["start"]) >= timestamp(r["timestamp"]):
109
+ raise ValueError("invalid usage window")
110
+ if kind == "quota":
111
+ required = {"remaining", "limit", "unit", "period", "reset", "fresh_until", "scope"}
112
+ if not required <= set(r):
113
+ raise ValueError("quota needs explicit values, including null unknowns")
114
+ if r["unit"] not in ("tokens", "requests", "percent", "credits", "unknown"):
115
+ raise ValueError("invalid quota unit")
116
+ if r["period"] not in ("minute", "hour", "day", "week", "month", "session", "unknown"):
117
+ raise ValueError("invalid quota period")
118
+ identifier(r["scope"])
119
+ if kind == "tool" and r.get("status") not in ("ok", "error", "unknown"):
120
+ raise ValueError("invalid tool status")
121
+ if len(json.dumps(r).encode()) > MAX_RECORD_BYTES:
122
+ raise ValueError("record too large")
123
+ return r
124
+
125
+
126
+ def _connect(store: str | Path) -> sqlite3.Connection:
127
+ path = Path(store)
128
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
129
+ fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
130
+ os.close(fd)
131
+ os.chmod(path, 0o600)
132
+ db = sqlite3.connect(path)
133
+ db.execute(
134
+ "CREATE TABLE IF NOT EXISTS observations "
135
+ "(source TEXT, stream TEXT, role TEXT, id TEXT, body TEXT, "
136
+ "PRIMARY KEY(source, stream, id))"
137
+ )
138
+ db.execute(
139
+ "CREATE TABLE IF NOT EXISTS counters "
140
+ "(source TEXT, stream TEXT, counter TEXT, timestamp TEXT, temporality TEXT, "
141
+ "PRIMARY KEY(source, stream, counter))"
142
+ )
143
+ db.execute(
144
+ "CREATE TABLE IF NOT EXISTS streams "
145
+ "(source TEXT, stream TEXT, role TEXT, representation TEXT, "
146
+ "PRIMARY KEY(source, stream))"
147
+ )
148
+ db.execute(
149
+ "CREATE TABLE IF NOT EXISTS tool_disputes "
150
+ "(source TEXT, stream TEXT, id TEXT, body TEXT, "
151
+ "PRIMARY KEY(source, stream, id))"
152
+ )
153
+ return db
154
+
155
+
156
+ def ingest(
157
+ store: str | Path, records: Iterable[dict[str, Any]], *, source: str, stream: str, role: str
158
+ ) -> dict[str, Any]:
159
+ identifier(source)
160
+ identifier(stream)
161
+ if role not in ("primary", "crosscheck"):
162
+ raise ValueError("invalid role")
163
+ batch: list[dict[str, Any]] = []
164
+ for raw in records:
165
+ if len(batch) >= MAX_BATCH_RECORDS:
166
+ raise ValueError("batch too large")
167
+ batch.append(validate_record(raw))
168
+ result = dict(accepted=0, duplicates=0, conflicts=0, excluded=0)
169
+ db = _connect(store)
170
+ try:
171
+ with db:
172
+ db.execute("BEGIN IMMEDIATE")
173
+ size = db.execute("SELECT count(*) FROM observations").fetchone()[0]
174
+ for r in batch:
175
+ mode = ("counter" if "counter" in r else "event") if r["kind"] == "usage" else None
176
+ ownership = db.execute(
177
+ "SELECT role, representation FROM streams WHERE source=? AND stream=?",
178
+ (source, stream),
179
+ ).fetchone()
180
+ if ownership:
181
+ if ownership[0] != role:
182
+ raise ValueError("stream role changed; use a separate crosscheck stream")
183
+ body = json.dumps(r, sort_keys=True, separators=(",", ":"))
184
+ old = db.execute(
185
+ "SELECT body, role FROM observations WHERE source=? AND stream=? AND id=?",
186
+ (source, stream, r["id"]),
187
+ ).fetchone()
188
+ if old:
189
+ disputed = db.execute(
190
+ "SELECT 1 FROM tool_disputes WHERE source=? AND stream=? AND id=?",
191
+ (source, stream, r["id"]),
192
+ ).fetchone()
193
+ conflict = old != (body, role) or bool(disputed)
194
+ if conflict and (r["kind"] == "tool" or json.loads(old[0])["kind"] == "tool"):
195
+ # Retain conflicting evidence without rewriting the original observation.
196
+ db.execute(
197
+ "INSERT OR IGNORE INTO tool_disputes VALUES (?,?,?,?)",
198
+ (source, stream, r["id"], body),
199
+ )
200
+ result["conflicts" if conflict else "duplicates"] += 1
201
+ continue
202
+ if ownership:
203
+ if mode is not None and ownership[1] is not None and mode != ownership[1]:
204
+ raise ValueError("incompatible usage representation")
205
+ mode = mode or ownership[1]
206
+ db.execute(
207
+ "INSERT OR REPLACE INTO streams VALUES (?,?,?,?)", (source, stream, role, mode)
208
+ )
209
+ if size >= MAX_STORE_RECORDS:
210
+ raise ValueError("store limit reached")
211
+ # Primary ownership is source-wide: alternate streams must be crosschecks.
212
+ owners = db.execute(
213
+ "SELECT DISTINCT stream FROM observations WHERE source=? AND role='primary'",
214
+ (source,),
215
+ ).fetchall()
216
+ if role == "primary" and owners and (stream,) not in owners:
217
+ raise ValueError("overlapping primary streams; use crosscheck")
218
+ if r["kind"] == "usage" and "counter" in r:
219
+ identity = (source, stream, r["counter"])
220
+ last = db.execute(
221
+ "SELECT timestamp, temporality FROM counters WHERE source=? AND stream=? AND counter=?",
222
+ identity,
223
+ ).fetchone()
224
+ if last and last[1] != r.get("temporality", "delta"):
225
+ raise ValueError("counter temporality changed")
226
+ if last and timestamp(r["timestamp"]) <= timestamp(last[0]):
227
+ raise ValueError("out-of-order counter observation")
228
+ db.execute(
229
+ "INSERT OR REPLACE INTO counters VALUES (?,?,?,?,?)",
230
+ (*identity, r["timestamp"], r.get("temporality", "delta")),
231
+ )
232
+ db.execute(
233
+ "INSERT INTO observations VALUES (?,?,?,?,?)",
234
+ (source, stream, role, r["id"], body),
235
+ )
236
+ size += 1
237
+ result["accepted"] += 1
238
+ result["excluded"] += role == "crosscheck"
239
+ finally:
240
+ db.close()
241
+ return result
242
+
243
+
244
+ def disputed_tool_ids(store: str | Path, *, source: str, stream: str) -> set[str]:
245
+ """Return quarantined identities without loading unrelated observations."""
246
+ db = _connect(store)
247
+ try:
248
+ return {row[0] for row in db.execute(
249
+ "SELECT DISTINCT id FROM tool_disputes WHERE source=? AND stream=?", (source, stream)
250
+ )}
251
+ finally:
252
+ db.close()
253
+
254
+
255
+ def records(store: str | Path) -> list[dict[str, Any]]:
256
+ """Return normalized evidence; only additive=True observations can feed totals."""
257
+ db = _connect(store)
258
+ try:
259
+ rows = db.execute("SELECT source, stream, role, body FROM observations").fetchall()
260
+ disputed = set(db.execute("SELECT source, stream, id FROM tool_disputes").fetchall())
261
+ finally:
262
+ db.close()
263
+ result = []
264
+ for source, stream, role, body in rows:
265
+ r = json.loads(body)
266
+ r.update(source=source, stream=stream, role=role)
267
+ if r["kind"] == "tool" and (source, stream, r["id"]) in disputed:
268
+ r["status"] = "unknown"
269
+ r["disputed"] = True
270
+ r["additive"] = (
271
+ role == "primary"
272
+ and r["kind"] == "usage"
273
+ and (r.get("temporality", "delta") == "delta")
274
+ )
275
+ result.append(r)
276
+ # Windows are checked across all stored evidence, including late arrivals.
277
+ windows: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
278
+ for r in result:
279
+ if r["kind"] == "usage" and "counter" in r:
280
+ windows.setdefault((r["source"], r["stream"], r["counter"]), []).append(r)
281
+ for group in windows.values():
282
+ group.sort(key=lambda r: timestamp(r["timestamp"]))
283
+ previous = None
284
+ for r in group:
285
+ start = r.get("start")
286
+ if r.get("temporality") == "cumulative":
287
+ r["additive"] = False
288
+ r["coverage"] = "unknown_first_point"
289
+ r["delta"] = None
290
+ if start is not None:
291
+ same_start = previous is not None and previous.get("start") == start
292
+ baseline: dict[str, Any] = (
293
+ previous
294
+ if same_start and previous is not None
295
+ else dict.fromkeys(COUNTS, 0)
296
+ )
297
+ delta = {
298
+ key: (
299
+ r[key] - baseline[key]
300
+ if r[key] is not None and baseline[key] is not None
301
+ else None
302
+ )
303
+ for key in COUNTS
304
+ }
305
+ if any(value is not None and value < 0 for value in delta.values()):
306
+ r["coverage"] = "counter_decreased"
307
+ elif (
308
+ previous is not None
309
+ and not same_start
310
+ and (timestamp(start) < timestamp(previous["timestamp"]))
311
+ ):
312
+ r["coverage"] = "overlapping_reset"
313
+ elif not _components_valid(delta):
314
+ r["coverage"] = "invalid_components"
315
+ else:
316
+ r["delta"] = delta
317
+ r["additive"] = r["role"] == "primary"
318
+ r["coverage"] = "delta" if same_start else "known_start_reset"
319
+ elif previous is not None:
320
+ r["coverage"] = "unknown_start"
321
+ elif start is None or (
322
+ previous is not None and timestamp(start) < timestamp(previous["timestamp"])
323
+ ):
324
+ r["additive"] = False
325
+ r["coverage"] = "unknown_or_overlapping_window"
326
+ if previous is not None:
327
+ previous["additive"] = False
328
+ if r.get("coverage") not in ("counter_decreased", "invalid_components"):
329
+ previous = r
330
+ latest: dict[tuple[str, str, str], dict[str, Any]] = {}
331
+ for r in result:
332
+ if r["kind"] == "quota":
333
+ key = (r["source"], r["stream"], r["scope"])
334
+ if key not in latest or timestamp(r["timestamp"]) > timestamp(latest[key]["timestamp"]):
335
+ latest[key] = r
336
+ for r in result:
337
+ if r["kind"] == "quota":
338
+ r["latest"] = latest[(r["source"], r["stream"], r["scope"])] is r
339
+ return sorted(result, key=lambda r: (r["timestamp"], r["id"]))
340
+
341
+
342
+ def stable_id(value: Any) -> str:
343
+ return hashlib.sha256(
344
+ json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
345
+ ).hexdigest()
346
+
347
+
348
+ def import_file(
349
+ store: str | Path, path: str | Path, *, format: str, source: str, stream: str, role: str
350
+ ) -> dict[str, Any]:
351
+ with Path(path).open("rb") as handle:
352
+ data = handle.read(MAX_FILE_BYTES + 1)
353
+ if len(data) > MAX_FILE_BYTES:
354
+ raise ValueError("file too large")
355
+ if format == "jsonl":
356
+ parsed = [json.loads(line) for line in data.splitlines() if line.strip()]
357
+ return ingest(store, parsed, source=source, stream=stream, role=role)
358
+ if format == "otlp":
359
+ from .telemetry import normalize_otlp
360
+
361
+ normalized, unsupported = normalize_otlp(json.loads(data))
362
+ result = ingest(store, normalized, source=source, stream=stream, role=role)
363
+ result["unsupported"] = unsupported
364
+ return result
365
+ raise ValueError("unsupported format")