agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,248 @@
1
+ """API-equivalent model price map, synced from LiteLLM (primary), then models.dev, then
2
+ pydantic/genai-prices, each filling only the models the ones before it lack.
3
+
4
+ Follows the usual price-registry sync pattern, EXTENDED to also extract the two cache-cost
5
+ fields such syncs normally skip (cache_read_input_token_cost /
6
+ cache_creation_input_token_cost). Nothing in aut reads the file; it is kept for readers outside
7
+ aut (see the README).
8
+ Zero runtime deps: the fetch uses stdlib urllib. Prices are NOTIONAL / API-equivalent.
9
+
10
+ Map shape: {normalized_key: {output_cost, input_cost?, cache_read_cost?, cache_write_cost?}}
11
+ in USD per token.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import urllib.request
18
+ from datetime import date
19
+ from pathlib import Path
20
+ from typing import Any, Callable
21
+
22
+ from .config import AutError
23
+ from .net import HTTP_ERRORS
24
+ from .state import atomic_write_text
25
+
26
+ _MODELS_DEV_PRICE_SCALE = 1_000_000 # models.dev prices are per-1M tokens
27
+
28
+ # LiteLLM field -> our field
29
+ _FIELD_MAP = (
30
+ ("input_cost_per_token", "input_cost"),
31
+ ("cache_read_input_token_cost", "cache_read_cost"),
32
+ ("cache_creation_input_token_cost", "cache_write_cost"),
33
+ )
34
+
35
+
36
+ def normalize_model_key(model_id: str) -> str:
37
+ """Lowercase; strip a trailing version suffix (model@20240229), an alias prefix
38
+ (alias@model), then a provider path (provider/model), so a tracker model string and a
39
+ LiteLLM key reduce to the same token."""
40
+ key = model_id.strip().lower()
41
+ if "@" in key:
42
+ head, _, tail = key.rpartition("@")
43
+ if head and tail.isdigit():
44
+ key = head
45
+ if "@" in key:
46
+ head, _, tail = key.rpartition("@")
47
+ key = tail or head
48
+ if "/" in key:
49
+ key = key.rpartition("/")[2] or key
50
+ return key
51
+
52
+
53
+ def _coerce_float(value: Any) -> float | None:
54
+ if isinstance(value, bool): # bool is an int; a stray `true` must not become 1.0
55
+ return None
56
+ try:
57
+ return float(value)
58
+ except (TypeError, ValueError):
59
+ return None
60
+
61
+
62
+ def build_rate_map(
63
+ litellm: dict[str, Any],
64
+ models_dev: dict[str, Any] | None = None,
65
+ genai_prices: Any = None,
66
+ ) -> dict[str, dict[str, float]]:
67
+ """Prune + normalize the sources. An entry survives only with an output price (always
68
+ present and the discriminating cost). LiteLLM is primary; the others fill gaps."""
69
+ rates: dict[str, dict[str, float]] = {}
70
+ for model_id, spec in litellm.items():
71
+ if not isinstance(spec, dict):
72
+ continue
73
+ out = _coerce_float(spec.get("output_cost_per_token"))
74
+ if out is None:
75
+ continue
76
+ entry: dict[str, float] = {"output_cost": out}
77
+ for src, dst in _FIELD_MAP:
78
+ val = _coerce_float(spec.get(src))
79
+ if val is not None:
80
+ entry[dst] = val
81
+ rates[normalize_model_key(model_id)] = entry
82
+ _merge_models_dev(rates, models_dev)
83
+ _merge_genai_prices(rates, genai_prices)
84
+ return rates
85
+
86
+
87
+ def _merge_models_dev(
88
+ rates: dict[str, dict[str, float]],
89
+ models_dev: dict[str, Any] | None,
90
+ ) -> None:
91
+ """Fill keys LiteLLM lacked from models.dev (nested provider -> models -> cost;
92
+ per-1M -> per-token). Output price required. Mutates `rates` in place."""
93
+ if not isinstance(models_dev, dict):
94
+ return
95
+ for _provider, block in models_dev.items():
96
+ models = block.get("models") if isinstance(block, dict) else None
97
+ if not isinstance(models, dict):
98
+ continue
99
+ for model_id, spec in models.items():
100
+ key = normalize_model_key(model_id)
101
+ if key in rates or not isinstance(spec, dict):
102
+ continue
103
+ cost = spec.get("cost")
104
+ if not isinstance(cost, dict):
105
+ continue
106
+ out = _coerce_float(cost.get("output"))
107
+ if out is None:
108
+ continue
109
+ entry: dict[str, float] = {"output_cost": out / _MODELS_DEV_PRICE_SCALE}
110
+ inp = _coerce_float(cost.get("input"))
111
+ if inp is not None:
112
+ entry["input_cost"] = inp / _MODELS_DEV_PRICE_SCALE
113
+ cache_read = _coerce_float(cost.get("cache_read"))
114
+ if cache_read is not None:
115
+ entry["cache_read_cost"] = cache_read / _MODELS_DEV_PRICE_SCALE
116
+ rates[key] = entry
117
+
118
+
119
+ # genai-prices field -> our field
120
+ _GENAI_FIELD_MAP = (
121
+ ("output_mtok", "output_cost"),
122
+ ("input_mtok", "input_cost"),
123
+ ("cache_read_mtok", "cache_read_cost"),
124
+ ("cache_write_mtok", "cache_write_cost"),
125
+ )
126
+
127
+
128
+ def _current_genai_prices(prices: Any, today: str) -> dict[str, Any] | None:
129
+ """A model's price table today. A list holds the base entry, then entries from a start_date in
130
+ date order. Entries for an hour of the day are skipped: this map holds one price per model."""
131
+ if isinstance(prices, dict):
132
+ return prices
133
+ current = None
134
+ for entry in prices if isinstance(prices, list) else ():
135
+ if not isinstance(entry, dict):
136
+ continue
137
+ constraint = entry.get("constraint")
138
+ if constraint is None or (isinstance(constraint, dict)
139
+ and str(constraint.get("start_date", "9")) <= today):
140
+ current = entry.get("prices")
141
+ return current if isinstance(current, dict) else None
142
+
143
+
144
+ def _merge_genai_prices(
145
+ rates: dict[str, dict[str, float]],
146
+ genai_prices: Any,
147
+ today: str | None = None,
148
+ ) -> None:
149
+ """Fill keys still missing from genai-prices (a list of providers, each with models; per-1M ->
150
+ per-token). A tiered price {"base": x, "tiers": [...]} counts as its base: this map holds one
151
+ price per token type, not per prompt size. Output price required. Mutates `rates` in place."""
152
+ if not isinstance(genai_prices, list):
153
+ return
154
+ today = today or date.today().isoformat()
155
+ for provider in genai_prices:
156
+ models = provider.get("models") if isinstance(provider, dict) else None
157
+ for spec in models if isinstance(models, list) else ():
158
+ model_id = spec.get("id") if isinstance(spec, dict) else None
159
+ if not isinstance(model_id, str):
160
+ continue
161
+ key = normalize_model_key(model_id)
162
+ prices = _current_genai_prices(spec.get("prices"), today)
163
+ if key in rates or prices is None:
164
+ continue
165
+ entry: dict[str, float] = {}
166
+ for src, dst in _GENAI_FIELD_MAP:
167
+ raw = prices.get(src)
168
+ val = _coerce_float(raw.get("base") if isinstance(raw, dict) else raw)
169
+ if val is not None:
170
+ entry[dst] = val / _MODELS_DEV_PRICE_SCALE
171
+ if "output_cost" in entry:
172
+ rates[key] = entry
173
+
174
+
175
+ LITELLM_URL = (
176
+ "https://raw.githubusercontent.com/BerriAI/litellm/main/"
177
+ "model_prices_and_context_window.json"
178
+ )
179
+ MODELS_DEV_URL = "https://models.dev/api.json"
180
+ GENAI_PRICES_URL = "https://raw.githubusercontent.com/pydantic/genai-prices/main/prices/new_data/v2/data.json"
181
+
182
+
183
+ def _fetch_json(
184
+ url: str, timeout: int = 30
185
+ ) -> Any:
186
+ """GET a JSON object or list via stdlib urllib (no third-party HTTP client).
187
+ None on any failure. Raises AutError for a URL that is not http(s): urllib would read file:."""
188
+ if not url.lower().startswith(("http://", "https://")):
189
+ raise AutError("sync-prices: a price source URL must be http(s)")
190
+ req = urllib.request.Request(
191
+ url,
192
+ headers={
193
+ "Accept": "application/json",
194
+ "User-Agent": "agent-usage-tracker/price-sync",
195
+ },
196
+ )
197
+ try:
198
+ with urllib.request.urlopen(req, timeout=timeout) as resp: # nosec B310 # http(s) only
199
+ data = json.loads(resp.read().decode("utf-8"))
200
+ except (*HTTP_ERRORS, ValueError):
201
+ return None
202
+ return data if isinstance(data, (dict, list)) else None
203
+
204
+
205
+ def sync_prices(
206
+ out: str | Path,
207
+ *,
208
+ litellm_url: str = LITELLM_URL,
209
+ models_dev_url: str = MODELS_DEV_URL,
210
+ genai_prices_url: str = GENAI_PRICES_URL,
211
+ fetch: Callable[[str], Any] | None = None,
212
+ ) -> int:
213
+ """Fetch -> build -> atomic write. Returns the number of priced models. Raises
214
+ AutError if the primary source is unreachable or the map is empty, so a
215
+ good file is never overwritten with junk. `fetch` defaults to the network
216
+ fetcher; inject a fake in tests."""
217
+ fetch = fetch or _fetch_json
218
+ litellm = fetch(litellm_url)
219
+ if not litellm or not isinstance(litellm, dict):
220
+ raise AutError(
221
+ "sync-prices: LiteLLM source unreachable; keeping the existing "
222
+ "price file."
223
+ )
224
+ rates = build_rate_map(litellm, fetch(models_dev_url), fetch(genai_prices_url))
225
+ if not rates:
226
+ raise AutError(
227
+ "sync-prices: built an empty rate map; keeping the existing "
228
+ "price file."
229
+ )
230
+ # Atomic, so a concurrent reader never sees a half-written file. Shared: no secret in it, and
231
+ # --out can point at a path another user or service reads.
232
+ atomic_write_text(Path(out).expanduser(), json.dumps(rates, indent=2, sort_keys=True),
233
+ shared=True)
234
+ return len(rates)
235
+
236
+
237
+ def load_prices(path: str | Path) -> dict[str, dict[str, float]]:
238
+ """Load a price file; {} if absent."""
239
+ p = Path(path).expanduser()
240
+ if not p.exists():
241
+ return {}
242
+ try:
243
+ obj = json.loads(p.read_text(encoding="utf-8"))
244
+ except (ValueError, OSError) as exc:
245
+ raise AutError(f"price file {p} is unreadable/corrupt; re-run `aut sync-prices`: {exc}")
246
+ if not isinstance(obj, dict) or not all(isinstance(v, dict) for v in obj.values()):
247
+ raise AutError(f"price file {p} is not a table of models; re-run `aut sync-prices`")
248
+ return obj