agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""agent-usage-tracker (aut): passive, subscription-safe tracker of coding-agent work.
|
|
2
|
+
|
|
3
|
+
Reads what each agent already writes to disk (no proxy, no API key), normalizes it into usage,
|
|
4
|
+
session, activity and active-time rows, and pushes them to a self-hosted sink. Commit outcomes and
|
|
5
|
+
local-model labels are separate pipelines over the same sink.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""`aut backup [--check]`: an encrypted, deduplicated restic snapshot of what aut reads and keeps.
|
|
2
|
+
|
|
3
|
+
The sink stores counts and keys, never text, and a re-read of logs that are gone is impossible. So
|
|
4
|
+
a dead or replaced laptop loses its transcripts, its ActivityWatch history, the repo-key salt and
|
|
5
|
+
the label gold files, unless they live somewhere else too. This copies them to a restic repository
|
|
6
|
+
once a day (packaging/launchd/dev.aut.backup.plist). docs/new-laptop.md is the restore.
|
|
7
|
+
|
|
8
|
+
Settings come from the [backup] table of config.toml: repository and password_command (restic runs
|
|
9
|
+
the command itself, so aut never holds the password), and optional paths (replace the defaults)
|
|
10
|
+
and excludes (added to the defaults). A run that cannot or should not happen now is skipped with
|
|
11
|
+
exit 0 and a log line: another backup still running, the laptop on battery below 20%, the
|
|
12
|
+
repository unreachable (off the VPN). A wrong password or a missing repository is an error. Each
|
|
13
|
+
good run records its time in backup.json next to the state file, which `aut status` shows, then
|
|
14
|
+
runs the retention: restic forget of this machine's aut snapshots, with --prune once a week.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import subprocess # nosec B404 # restic and pmset, argv lists, no shell
|
|
23
|
+
import sys
|
|
24
|
+
from glob import glob
|
|
25
|
+
from datetime import datetime, timedelta, timezone
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from .config import AutError, Config, config_path
|
|
30
|
+
from .state import atomic_write_text, load_json_dict, state_path
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
import fcntl
|
|
34
|
+
except ImportError: # Windows: no flock; restic's own repository lock still applies
|
|
35
|
+
fcntl = None # type: ignore[assignment]
|
|
36
|
+
|
|
37
|
+
MIN_BATTERY = 20 # percent: below this, on battery, the run waits for the next one
|
|
38
|
+
_REACH_TIMEOUT_S = 120
|
|
39
|
+
# restic exit codes (0.17+): 3 = snapshot saved but some files could not be read
|
|
40
|
+
_PARTIAL, _NO_REPO, _BAD_PASSWORD = 3, 10, 12
|
|
41
|
+
# the one retention for this repository (other docs point here): keep 14 daily, 8 weekly
|
|
42
|
+
# and 12 monthly snapshots
|
|
43
|
+
RETENTION = ("--keep-daily", "14", "--keep-weekly", "8", "--keep-monthly", "12")
|
|
44
|
+
PRUNE_EVERY = timedelta(days=7)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def default_paths(cfg: Config) -> list[Path]:
|
|
48
|
+
from .label.core import data_dir # here, not at the top: `aut` itself loads no label module
|
|
49
|
+
from .collectors.cursor import cursor_db
|
|
50
|
+
h = cfg.home
|
|
51
|
+
cursor = cursor_db(cfg)
|
|
52
|
+
cursor_paths = [cursor, Path(str(cursor) + "-wal"), Path(str(cursor) + "-shm")] \
|
|
53
|
+
if cfg.cursor_db_path else [cursor.parent.parent]
|
|
54
|
+
manifests = [Path(p) for p in glob(os.path.expanduser(cfg.cursor_sdk_manifests), recursive=True)] \
|
|
55
|
+
if cfg.cursor_sdk_manifests else []
|
|
56
|
+
return [h / ".claude" / "projects", h / ".claude" / "history.jsonl",
|
|
57
|
+
h / ".codex" / "sessions", h / ".codex" / "history.jsonl",
|
|
58
|
+
h / ".codex" / "archived_sessions", h / ".codex" / "session_index.jsonl",
|
|
59
|
+
h / ".cursor" / "chats", h / ".cursor" / "projects", *cursor_paths,
|
|
60
|
+
h / ".pi" / "agent" / "sessions", *manifests,
|
|
61
|
+
h / "Library" / "Application Support" / "activitywatch",
|
|
62
|
+
config_path().parent, # config.toml, label.json, scan.env
|
|
63
|
+
data_dir(), # label state and eval/ gold files; models excluded
|
|
64
|
+
state_path().parent] # repo_key.salt, the ActivityWatch watermark
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def default_excludes() -> list[str]:
|
|
68
|
+
from .label.core import data_dir
|
|
69
|
+
return [str(data_dir() / "label" / "models")] # model caches: GBs, downloaded again
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _status_path() -> Path:
|
|
73
|
+
return state_path().parent / "backup.json"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _log(msg: str) -> None:
|
|
77
|
+
print(f"{datetime.now().astimezone().isoformat(timespec='seconds')} aut backup: {msg}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _record(result: str, ok: bool = False) -> None:
|
|
81
|
+
st = load_json_dict(_status_path())
|
|
82
|
+
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
83
|
+
st.update(last_run=now, last_result=result, **({"last_ok": now} if ok else {}))
|
|
84
|
+
atomic_write_text(_status_path(), json.dumps(st))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def describe() -> str:
|
|
88
|
+
"""One line for `aut status`: the age of the last good backup."""
|
|
89
|
+
st = load_json_dict(_status_path())
|
|
90
|
+
if not st:
|
|
91
|
+
return "never ran"
|
|
92
|
+
ok = st.get("last_ok")
|
|
93
|
+
if not ok:
|
|
94
|
+
return f"never succeeded (last run: {st.get('last_result')})"
|
|
95
|
+
hours = int((datetime.now(timezone.utc) - datetime.fromisoformat(ok)).total_seconds() // 3600)
|
|
96
|
+
age = f"{hours} h" if hours < 48 else f"{hours // 24} d"
|
|
97
|
+
return f"last good {age} ago (last run: {st.get('last_result')})"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _battery() -> tuple[bool, int] | None:
|
|
101
|
+
"""(on battery, percent) from pmset, or None where there is no pmset or no battery."""
|
|
102
|
+
try:
|
|
103
|
+
out = subprocess.run(["pmset", "-g", "batt"], capture_output=True, text=True, # nosec B603 B607
|
|
104
|
+
timeout=10).stdout
|
|
105
|
+
except (OSError, subprocess.SubprocessError):
|
|
106
|
+
return None
|
|
107
|
+
m = re.search(r"(\d+)%", out)
|
|
108
|
+
return ("Battery Power" in out, int(m.group(1))) if m else None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _restic(args: list[str], env: dict[str, str],
|
|
112
|
+
timeout: float | None = None) -> subprocess.CompletedProcess[str]:
|
|
113
|
+
try:
|
|
114
|
+
return subprocess.run(["restic", *args], capture_output=True, text=True, # nosec B603 B607
|
|
115
|
+
env=env, timeout=timeout)
|
|
116
|
+
except FileNotFoundError:
|
|
117
|
+
raise AutError("aut backup: restic is not installed or not on PATH (brew install restic; "
|
|
118
|
+
"a launchd job needs PATH set, see dev.aut.backup.plist)") from None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _env(cfg: Config) -> dict[str, str]:
|
|
122
|
+
if not cfg.backup_repository or not cfg.backup_password_command:
|
|
123
|
+
raise AutError("aut backup: set repository and password_command in the [backup] table of "
|
|
124
|
+
"config.toml (see config.example.toml)")
|
|
125
|
+
env = {k: v for k, v in os.environ.items()
|
|
126
|
+
if k not in ("RESTIC_PASSWORD", "RESTIC_PASSWORD_FILE")}
|
|
127
|
+
env.update(RESTIC_REPOSITORY=cfg.backup_repository,
|
|
128
|
+
RESTIC_PASSWORD_COMMAND=cfg.backup_password_command)
|
|
129
|
+
return env
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _tail(r: subprocess.CompletedProcess[str]) -> str:
|
|
133
|
+
return " | ".join((r.stderr or r.stdout or "").strip().splitlines()[-3:])
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _reachable(env: dict[str, str]) -> bool:
|
|
137
|
+
try:
|
|
138
|
+
r = _restic(["cat", "config"], env, timeout=_REACH_TIMEOUT_S)
|
|
139
|
+
except subprocess.TimeoutExpired:
|
|
140
|
+
return False
|
|
141
|
+
if r.returncode == _BAD_PASSWORD:
|
|
142
|
+
raise AutError("aut backup: restic rejected the password (check password_command)")
|
|
143
|
+
if r.returncode == _NO_REPO:
|
|
144
|
+
raise AutError("aut backup: the repository does not exist; create it once with "
|
|
145
|
+
"`restic init` (docs/new-laptop.md)")
|
|
146
|
+
return r.returncode == 0
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def run_backup(cfg: Config, check: bool = False) -> int:
|
|
150
|
+
"""One backup (or with `check`, a `restic check` of the repository, sending nothing)."""
|
|
151
|
+
env = _env(cfg)
|
|
152
|
+
lock_path = state_path().parent / "backup.lock"
|
|
153
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
with open(lock_path, "a") as lock:
|
|
155
|
+
if fcntl is not None:
|
|
156
|
+
try:
|
|
157
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
158
|
+
except BlockingIOError:
|
|
159
|
+
_log("another backup is still running; skipped")
|
|
160
|
+
return 0
|
|
161
|
+
return _locked(cfg, env, check)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _skip(reason: str, check: bool) -> int:
|
|
165
|
+
_log(f"skipped: {reason}")
|
|
166
|
+
if not check:
|
|
167
|
+
_record(f"skipped: {reason}")
|
|
168
|
+
return 0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _locked(cfg: Config, env: dict[str, str], check: bool) -> int:
|
|
172
|
+
batt = _battery()
|
|
173
|
+
if batt and batt[0] and batt[1] < MIN_BATTERY:
|
|
174
|
+
return _skip(f"on battery at {batt[1]}%", check)
|
|
175
|
+
if not _reachable(env):
|
|
176
|
+
return _skip("repository unreachable", check)
|
|
177
|
+
if check:
|
|
178
|
+
r = _restic(["check"], env)
|
|
179
|
+
if r.returncode != 0:
|
|
180
|
+
raise AutError(f"aut backup: restic check failed (exit {r.returncode}): {_tail(r)}")
|
|
181
|
+
_log("restic check: no errors")
|
|
182
|
+
_log(f"last good backup: {describe()}")
|
|
183
|
+
return 0
|
|
184
|
+
paths = [str(p) for p in (Path(x).expanduser() for x in cfg.backup_paths)] \
|
|
185
|
+
if cfg.backup_paths else [str(p) for p in default_paths(cfg)]
|
|
186
|
+
paths = [p for p in paths if Path(p).exists()]
|
|
187
|
+
if not paths:
|
|
188
|
+
raise AutError("aut backup: none of the paths to back up exist")
|
|
189
|
+
excludes: list[Any] = []
|
|
190
|
+
for x in [*default_excludes(), *cfg.backup_excludes]:
|
|
191
|
+
excludes += ["--exclude", x]
|
|
192
|
+
# --host: the machine name, so a new laptop's first snapshot finds the old one as its parent
|
|
193
|
+
r = _restic(["backup", "--tag", "aut", "--host", cfg.machine, "--exclude-caches",
|
|
194
|
+
*excludes, *paths], env)
|
|
195
|
+
if r.returncode not in (0, _PARTIAL):
|
|
196
|
+
_record(f"failed (exit {r.returncode})")
|
|
197
|
+
raise AutError(f"aut backup: restic backup failed (exit {r.returncode}): {_tail(r)}")
|
|
198
|
+
for line in r.stdout.splitlines():
|
|
199
|
+
if line.startswith(("Files:", "Added to the repository", "snapshot ")):
|
|
200
|
+
_log(line.strip())
|
|
201
|
+
if r.returncode == _PARTIAL:
|
|
202
|
+
print(f"aut backup: warning: some files could not be read: {_tail(r)}", file=sys.stderr)
|
|
203
|
+
_record("ok" if r.returncode == 0 else "ok, some files unreadable", ok=True)
|
|
204
|
+
if cfg.backup_retention:
|
|
205
|
+
_forget(cfg, env)
|
|
206
|
+
return 0
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _forget(cfg: Config, env: dict[str, str]) -> None:
|
|
210
|
+
"""The retention, after a good backup: this machine's aut snapshots only. The prune (it reads
|
|
211
|
+
and rewrites pack files) runs at most once a week; the forget alone is cheap."""
|
|
212
|
+
st = load_json_dict(_status_path())
|
|
213
|
+
last = st.get("last_prune")
|
|
214
|
+
prune = not last or datetime.now(timezone.utc) - datetime.fromisoformat(last) >= PRUNE_EVERY
|
|
215
|
+
r = _restic(["forget", "--tag", "aut", "--host", cfg.machine, *RETENTION,
|
|
216
|
+
*(["--prune"] if prune else [])], env)
|
|
217
|
+
if r.returncode != 0:
|
|
218
|
+
_record(f"ok, forget failed (exit {r.returncode})")
|
|
219
|
+
raise AutError(f"aut backup: the backup is saved, but restic forget failed "
|
|
220
|
+
f"(exit {r.returncode}): {_tail(r)}")
|
|
221
|
+
if prune:
|
|
222
|
+
st = load_json_dict(_status_path())
|
|
223
|
+
st["last_prune"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
224
|
+
atomic_write_text(_status_path(), json.dumps(st))
|
|
225
|
+
_log("forget + prune done")
|