agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,291 @@
1
+ """A locked local cycle with a hard subprocess deadline and durable progress."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import html
6
+ import argparse
7
+ import json
8
+ import math
9
+ import os
10
+ import signal
11
+ import threading
12
+ from contextlib import ExitStack, contextmanager
13
+ from collections.abc import Iterator
14
+ import sys
15
+ import time
16
+ import uuid
17
+ from datetime import datetime, timezone
18
+ from pathlib import Path
19
+ from typing import Any
20
+ from types import FrameType
21
+
22
+ from ..config import AutError
23
+ from ..state import LockBusy, _locked, atomic_write_text
24
+ from .judge import diagnose
25
+ from .store import Store, audit
26
+ from .trial import command
27
+
28
+
29
+ class _BudgetExpired(BaseException):
30
+ # Model loading wraps ordinary exceptions as failures; cancellation must pass through.
31
+ pass
32
+
33
+
34
+ @contextmanager
35
+ def _cooperative_deadline(deadline: float) -> Iterator[None]:
36
+ remaining = deadline - time.monotonic()
37
+ if remaining <= 0:
38
+ raise _BudgetExpired
39
+ if not hasattr(signal, "setitimer") or threading.current_thread() is not threading.main_thread():
40
+ yield
41
+ return
42
+
43
+ def expired(signum: int, frame: FrameType | None) -> None:
44
+ raise _BudgetExpired
45
+
46
+ previous_handler = signal.getsignal(signal.SIGALRM)
47
+ previous_timer = signal.getitimer(signal.ITIMER_REAL)
48
+ started = time.monotonic()
49
+ signal.signal(signal.SIGALRM, expired)
50
+ signal.setitimer(signal.ITIMER_REAL, remaining)
51
+ try:
52
+ yield
53
+ finally:
54
+ signal.setitimer(signal.ITIMER_REAL, 0)
55
+ signal.signal(signal.SIGALRM, previous_handler)
56
+ if previous_timer[0]:
57
+ signal.setitimer(signal.ITIMER_REAL,
58
+ max(.001, previous_timer[0] - (time.monotonic() - started)),
59
+ previous_timer[1])
60
+
61
+
62
+ def inbox(store: Store, judgment_ids: list[str]) -> None:
63
+ rows = [store.get("judgments", key) for key in judgment_ids]
64
+ entries = []
65
+ for row in rows:
66
+ review = row.get("review") or {}
67
+ entries.append("<tr><td>" + html.escape(row["rubric"]) + "</td><td>"
68
+ + html.escape(row["verdict"]) + "</td><td>"
69
+ + html.escape(review.get("rationale", "Local decision-model estimate"))
70
+ + "</td><td><code>" + html.escape(row["case_id"][:12]) + "</code></td></tr>")
71
+ page = ("<!doctype html><html lang=en><meta charset=utf-8>"
72
+ "<meta name=viewport content='width=device-width, initial-scale=1'>"
73
+ "<title>Private improvement inbox</title><style>"
74
+ "body{font:16px system-ui;margin:2rem;max-width:75rem;line-height:1.5}"
75
+ "td,th{text-align:left;padding:.7rem;border-bottom:1px solid #ccc}"
76
+ "table{border-collapse:collapse;width:100%} .scroll{overflow:auto}"
77
+ "</style><h1>Private improvement inbox</h1>"
78
+ "<p>Local model estimates. Unknowns remain unresolved. These are not human labels "
79
+ "or proof of task quality. Routine review does not require a human.</p>"
80
+ "<div class=scroll role=region aria-label='Model judgments' tabindex=0><table>"
81
+ "<tr><th>Criterion</th><th>Estimate</th><th>Evidence review</th><th>Case</th></tr>"
82
+ + "".join(entries) + "</table></div></html>")
83
+ atomic_write_text(store.root / "inbox.html", page)
84
+
85
+
86
+ def _pending(store: Store) -> dict[str, Any]:
87
+ path = store.root / "pending_cycle.json"
88
+ return json.loads(path.read_text()) if path.exists() else {}
89
+
90
+
91
+ def _pending_write(store: Store) -> tuple[int, int] | None:
92
+ """Identity of the last pending_cycle.json write; every atomic write makes a new file."""
93
+ try:
94
+ info = os.stat(store.root / "pending_cycle.json")
95
+ except FileNotFoundError:
96
+ return None
97
+ return info.st_ino, info.st_mtime_ns
98
+
99
+
100
+ def _reusable_audit(store: Store, pending: dict[str, Any], days: int,
101
+ limit: int) -> dict[str, Any] | None:
102
+ """The pending frozen audit when it was built for these days and limit.
103
+ Budget and call limits do not change the evidence, so they no longer discard it."""
104
+ frozen = pending.get("audit")
105
+ if not frozen:
106
+ return None
107
+ try:
108
+ source = store.get("audits", frozen["audit_id"]).get("source_policy")
109
+ except (AutError, KeyError, TypeError): # missing or changed record: rebuild, never fail nightly
110
+ return None
111
+ if source is None: # pre-v4 manifests: the days and limit of the policy that froze it
112
+ try:
113
+ source = store.get("cycle_policies", pending["policy_id"])
114
+ except (AutError, KeyError, TypeError):
115
+ return None
116
+ return frozen if (source.get("days"), source.get("limit")) == (days, limit) else None
117
+
118
+
119
+ def freeze_cycle_audit(store: Store, *, days: int = 7, limit: int = 30,
120
+ budget_s: int = 1500) -> dict[str, Any]:
121
+ """Freeze the next scheduled cycle's audit, so the reviewer spends its budget on judging."""
122
+ if not 1 <= budget_s <= 3600:
123
+ raise AutError("improve: audit budget must be 1..3600 seconds")
124
+ try:
125
+ with _locked(store.root / "cycle_audit", blocking=False):
126
+ result = _freeze(store, days, limit, budget_s)
127
+ except LockBusy:
128
+ result = {"status": "busy"}
129
+ store.append("cycle_audit", status=result["status"], audit_id=result.get("audit_id"))
130
+ return result
131
+
132
+
133
+ def _freeze(store: Store, days: int, limit: int, budget_s: int) -> dict[str, Any]:
134
+ # Content alone cannot tell a cycle that finished during the build from last night's.
135
+ before = _pending_write(store)
136
+ if (kept := _reusable_audit(store, _pending(store), days, limit)):
137
+ return {"status": "kept", **kept}
138
+ # Build without the cycle lock: after sleep launchd fires this and the reviewer together,
139
+ # and the reviewer must not lose its night to a busy lock. The worst case is a duplicate build.
140
+ try:
141
+ with _cooperative_deadline(time.monotonic() + budget_s):
142
+ fresh = audit(store, Path.home(), days=days, limit=limit)
143
+ except _BudgetExpired:
144
+ return {"status": "budget"}
145
+ try:
146
+ with _locked(store.root / "cycle", blocking=False):
147
+ if (kept := _reusable_audit(store, _pending(store), days, limit)):
148
+ return {"status": "kept", **kept}
149
+ if _pending_write(store) != before: # a cycle finished and judged this evidence
150
+ return {"status": "superseded", "audit_id": fresh["audit_id"]}
151
+ atomic_write_text(store.root / "pending_cycle.json", json.dumps({"audit": fresh}) + "\n")
152
+ return {"status": "frozen", **fresh}
153
+ except LockBusy: # a running cycle owns pending; the built audit record is harmless
154
+ return {"status": "busy", "audit_id": fresh["audit_id"]}
155
+
156
+
157
+ def run_once(store: Store, *, days: int = 7, limit: int = 30, case_limit: int = 12,
158
+ budget_s: int = 300, max_calls: int = 40,
159
+ models: tuple[str, ...] = ("pplx-decider-flash", "clef-flash"),
160
+ explain_model: str | None = None, audit_id: str | None = None,
161
+ audit_budget_s: int = 0) -> dict[str, Any]:
162
+ if not 1 <= budget_s <= 3600:
163
+ raise AutError("improve: cycle budget must be 1..3600 seconds")
164
+ if not 0 <= audit_budget_s <= 3600:
165
+ raise AutError("improve: audit budget must be 0..3600 seconds")
166
+ reserve = min(5., budget_s / 10)
167
+ policy = {"version": 1, "days": days, "limit": limit, "case_limit": case_limit,
168
+ "budget_s": budget_s, "max_calls": max_calls, "models": list(models),
169
+ "explain_model": explain_model, "audit_budget_s": audit_budget_s}
170
+ if audit_id is not None:
171
+ store.get("audits", audit_id)
172
+ policy["audit_id"] = audit_id
173
+ key = store.put("cycle_policies", policy)
174
+ run_id = uuid.uuid4().hex
175
+ try:
176
+ with _locked(store.root / "cycle", blocking=False):
177
+ # A run that must build its own audit gets that time on top of the judge's budget.
178
+ needs_audit = audit_id is None and audit_budget_s and not _reusable_audit(
179
+ store, _pending(store), days, limit)
180
+ deadline = time.monotonic() + budget_s + (audit_budget_s if needs_audit else 0)
181
+ argv = [sys.executable, "-I", "-m", "agent_usage_tracker.improve.schedule", "--root",
182
+ str(store.root), "--policy", key, "--run-id", run_id,
183
+ "--deadline", str(deadline)]
184
+ result = command(argv, store.root, max(.001, deadline - time.monotonic() - reserve / 2))
185
+ report: dict[str, Any] = {"status": "timeout" if result["timeout"] else "error",
186
+ "policy_id": key, "exit_code": result["exit_code"]}
187
+ if not result["accepted"]:
188
+ report["error_id"] = store.put("cycle_errors", {
189
+ "run_id": run_id, "policy_id": key, **result})
190
+ try:
191
+ saved = json.loads((store.root / "worker_result.json").read_text())
192
+ if saved["run_id"] != run_id or saved["report"]["policy_id"] != key:
193
+ raise ValueError("stale worker result")
194
+ if result["accepted"]:
195
+ if saved["report"]["status"] not in {"ok", "deferred"}:
196
+ raise ValueError("unfinished worker result")
197
+ report = saved["report"]
198
+ else:
199
+ report["progress"] = saved["report"]
200
+ except (OSError, ValueError, KeyError, TypeError) as exc:
201
+ if result["accepted"]:
202
+ raise AutError("improve: worker did not preserve its matching result") from exc
203
+ store.append("cycle_finished", **report)
204
+ atomic_write_text(store.root / "last_cycle.json", json.dumps(report, indent=2) + "\n")
205
+ inbox(store, report.get("diagnosis", report.get("progress", {}).get("diagnosis", {}))
206
+ .get("judgment_ids", []))
207
+ return report
208
+ except LockBusy:
209
+ return {"status": "busy"}
210
+
211
+
212
+ def _worker(store: Store, policy_id: str, run_id: str, *,
213
+ deadline: float | None = None) -> dict[str, Any]:
214
+ started = time.monotonic()
215
+ started_at = datetime.now(timezone.utc).isoformat()
216
+ policy = store.get("cycle_policies", policy_id)
217
+ if deadline is not None and not math.isfinite(deadline):
218
+ raise AutError("improve: invalid worker deadline")
219
+ budget_s = policy["budget_s"]
220
+ pending_path = store.root / "pending_cycle.json"
221
+ provided = policy.get("audit_id")
222
+ reuse = None if provided else _reusable_audit(store, _pending(store), policy["days"],
223
+ policy["limit"])
224
+ allowance = 0 if provided or reuse else policy.get("audit_budget_s", 0)
225
+ outer = min(deadline if deadline is not None else started + budget_s + allowance,
226
+ started + budget_s + allowance)
227
+ judged: dict[str, Any] = {"complete": False, "reason": "budget", "calls": 0,
228
+ "finished": 0, "judgment_ids": []}
229
+ report: dict[str, Any] = {"status": "running", "policy_id": policy_id, "audit": None,
230
+ "diagnosis": judged, "worker_started_at": started_at, "audit_s": 0}
231
+ if provided:
232
+ store.get("audits", provided)
233
+ frozen = {"audit_id": provided}
234
+ elif reuse:
235
+ frozen = reuse
236
+ else:
237
+ try:
238
+ # With an audit allowance the audit has its own cap; without one it keeps today's
239
+ # shared deadline, which only the outer subprocess timeout enforces.
240
+ with _cooperative_deadline(min(outer, started + allowance)) if allowance else ExitStack():
241
+ frozen = audit(store, Path.home(), days=policy["days"], limit=policy["limit"])
242
+ except _BudgetExpired:
243
+ report.update(status="deferred", audit_s=round(time.monotonic() - started, 3))
244
+ atomic_write_text(store.root / "worker_result.json",
245
+ json.dumps({"run_id": run_id, "report": report}) + "\n")
246
+ return report
247
+ atomic_write_text(pending_path, json.dumps({"policy_id": policy_id, "audit": frozen}) + "\n")
248
+ report["audit_s"] = round(time.monotonic() - started, 3)
249
+ report["audit"] = frozen
250
+ # The judge's own budget starts after the audit; the outer deadline still bounds both.
251
+ deadline = min(outer, time.monotonic() + budget_s) - min(5., budget_s / 10)
252
+
253
+ cleared = False
254
+
255
+ def checkpoint(progress: dict[str, Any]) -> None:
256
+ nonlocal judged, cleared
257
+ judged = progress
258
+ report["diagnosis"] = judged
259
+ # Blocking work done: release the audit now, so an alarm or a kill during a non-blocking
260
+ # rubric's call cannot hold it open for the next night.
261
+ if judged.get("complete") and not provided and not cleared:
262
+ atomic_write_text(pending_path, json.dumps({"policy_id": policy_id, "audit": None}) + "\n")
263
+ cleared = True
264
+ atomic_write_text(store.root / "worker_result.json",
265
+ json.dumps({"run_id": run_id, "report": report}) + "\n")
266
+
267
+ checkpoint(judged)
268
+ try:
269
+ with _cooperative_deadline(deadline):
270
+ judged = diagnose(store, frozen["audit_id"], models=tuple(policy["models"]),
271
+ limit=policy["case_limit"], budget_s=policy["budget_s"],
272
+ max_calls=policy["max_calls"], explain_model=policy["explain_model"],
273
+ deadline=deadline, progress=checkpoint)
274
+ except _BudgetExpired: # keeps the last checkpoint's complete: blocking work may be done
275
+ judged = {**judged, "reason": "budget"}
276
+ report.update(status="ok" if judged["complete"] else "deferred", diagnosis=judged)
277
+ if judged["complete"] and not provided and not cleared:
278
+ atomic_write_text(pending_path, json.dumps({"policy_id": policy_id, "audit": None}) + "\n")
279
+ atomic_write_text(store.root / "worker_result.json",
280
+ json.dumps({"run_id": run_id, "report": report}) + "\n")
281
+ return report
282
+
283
+
284
+ if __name__ == "__main__":
285
+ parser = argparse.ArgumentParser(description="Internal isolated local improvement worker")
286
+ parser.add_argument("--root", type=Path, required=True)
287
+ parser.add_argument("--policy", required=True)
288
+ parser.add_argument("--run-id", required=True)
289
+ parser.add_argument("--deadline", type=float)
290
+ args = parser.parse_args()
291
+ _worker(Store(args.root), args.policy, args.run_id, deadline=args.deadline)