agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,72 @@
1
+ """Prepare one operator-scoped change without writing to its original target."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import difflib
6
+ import hashlib
7
+ import os
8
+ import stat
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from ..config import AutError
13
+ from ..state import atomic_write_text
14
+ from .store import Store
15
+
16
+
17
+ def file_digest(path: Path) -> str:
18
+ return hashlib.sha256(path.read_bytes()).hexdigest()
19
+
20
+
21
+ def prepare(store: Store, case_id: str, scope: Path, target: Path, before: str,
22
+ after: str) -> str:
23
+ case = store.get("cases", case_id)
24
+ if not case.get("events"):
25
+ raise AutError("improve: a proposal needs source evidence")
26
+ scope, target = scope.resolve(), target.resolve()
27
+ if not target.is_relative_to(scope) or not target.is_file():
28
+ raise AutError("improve: proposal target is outside its allowed scope")
29
+ try:
30
+ original = target.read_bytes().decode("utf-8")
31
+ except (OSError, UnicodeError) as exc:
32
+ raise AutError("improve: proposal target is not readable text") from exc
33
+ if not before or original.count(before) != 1 or before == after:
34
+ raise AutError("improve: replacement must match one unique nonempty span")
35
+ candidate = original.replace(before, after, 1)
36
+ relative = target.relative_to(scope).as_posix()
37
+ patch = "".join(difflib.unified_diff(original.splitlines(keepends=True),
38
+ candidate.splitlines(keepends=True),
39
+ fromfile=f"a/{relative}", tofile=f"b/{relative}"))
40
+ value: dict[str, Any] = {"version": 1, "case_id": case_id, "scope": str(scope),
41
+ "target": relative, "before_sha256": file_digest(target),
42
+ "after_sha256": hashlib.sha256(candidate.encode()).hexdigest(),
43
+ "before": original, "after": candidate, "diff": patch,
44
+ "authority": "operator_scoped_proposal", "state": "draft",
45
+ "rollback": "Restore the exact saved before content in an isolated candidate."}
46
+ key = store.put("proposals", value)
47
+ store.append("proposal_prepared", proposal_id=key, case_id=case_id)
48
+ return key
49
+
50
+
51
+ def stage(store: Store, proposal_id: str, candidate_root: Path) -> Path:
52
+ proposal = store.get("proposals", proposal_id)
53
+ scope = Path(proposal["scope"]).resolve()
54
+ relative = Path(proposal["target"])
55
+ source = (scope / relative).resolve()
56
+ root = candidate_root.resolve()
57
+ target = (root / relative).resolve()
58
+ if (root == scope or root in scope.parents or target == source
59
+ or target != root / relative or not source.is_relative_to(scope)
60
+ or not target.is_relative_to(root)):
61
+ raise AutError("improve: stage only inside an independent candidate directory")
62
+ if not source.is_file() or file_digest(source) != proposal["before_sha256"]:
63
+ raise AutError("improve: proposal base changed; prepare a new proposal")
64
+ if target.exists() and file_digest(target) not in {
65
+ proposal["before_sha256"], proposal["after_sha256"]}:
66
+ raise AutError("improve: candidate contains unrelated changes")
67
+ target.parent.mkdir(parents=True, exist_ok=True)
68
+ mode = stat.S_IMODE(target.stat().st_mode if target.exists() else source.stat().st_mode)
69
+ atomic_write_text(target, proposal["after"], shared=True, newline="")
70
+ os.chmod(target, mode)
71
+ store.append("proposal_staged", proposal_id=proposal_id, candidate=str(root))
72
+ return target
@@ -0,0 +1,360 @@
1
+ """Judge criteria as versioned markdown files: built-ins next to this module, user files override."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import re
8
+ import shlex
9
+ from dataclasses import dataclass, replace
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from ..config import AutError
14
+ from .store import Store, digest
15
+
16
+ LABELS = {"pass", "fail", "unknown"}
17
+ CAUSES = {"instruction", "environment", "tool_contract", "model", "task_ambiguity",
18
+ "expected_exploration", "unknown"}
19
+ TRIGGERS = {"error_result", "control", "merge"}
20
+ REQUIRES = {"final_reply", "test_command", "outputs"}
21
+ INPUTS = {"attempt", "outputs_summary"}
22
+ EVENT_KINDS = ("reply", "result", "call", "prompt", "boundary", "compaction")
23
+ TEST_FAMILIES = frozenset({"pytest", "python -m pytest", "uv run pytest", "npm test", "pnpm test",
24
+ "go test", "cargo test", "vitest", "jest"})
25
+ BUILTIN_DIR = Path(__file__).with_name("rubrics")
26
+ MAX_FILE_BYTES = 64 * 1024
27
+ FIELDS = {"id", "version", "threshold", "priority", "blocking", "applies_to", "max_cases", "input",
28
+ "max_input_chars", "checks", "qualify", "enabled"}
29
+ # Digest of [question, definition, labels] of the two criteria that were Python constants before
30
+ # markdown rubrics. legacy_version maps an old diagnosis policy holding exactly this text to 1.
31
+ LEGACY_V1 = {
32
+ "avoidable_recovery": "bcee090e953d7dd273679138ad3b2205ed1461b544369647bc22ae609b31ddc0",
33
+ "completion_supported": "ad6070151806cce50df764741a4ee03493cbe16dda9405f822ae72f915fdb356",
34
+ }
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class Rubric:
39
+ id: str
40
+ version: int
41
+ sha256: str
42
+ question: str
43
+ definition: str
44
+ labels: dict[str, str]
45
+ threshold: float
46
+ priority: int
47
+ blocking: bool
48
+ triggers: frozenset[str]
49
+ requires: frozenset[str]
50
+ input: str
51
+ max_cases: int
52
+ max_input_chars: int
53
+ checks: dict[str, Any]
54
+ qualify: dict[str, Any]
55
+ source: str
56
+
57
+
58
+ def _bad(field: str) -> AutError:
59
+ return AutError(f"improve: invalid rubric {field}")
60
+
61
+
62
+ def _int(value: Any, field: str, low: int, high: int) -> int:
63
+ if type(value) is not int or not low <= value <= high:
64
+ raise _bad(field)
65
+ return value
66
+
67
+
68
+ def _names(value: Any, field: str, allowed: set[str] | tuple[str, ...]) -> frozenset[str]:
69
+ if not isinstance(value, list) or any(not isinstance(v, str) or v not in allowed
70
+ for v in value):
71
+ raise _bad(field)
72
+ return frozenset(value)
73
+
74
+
75
+ def _normal(text: str) -> str:
76
+ """Line ends and a leading byte-order mark (as some Windows editors write) do not count."""
77
+ return text.removeprefix("\ufeff").replace("\r\n", "\n")
78
+
79
+
80
+ def _split(text: str) -> tuple[dict[str, Any], str]:
81
+ text = _normal(text)
82
+ head, sep, body = text[4:].partition("\n---\n") if text.startswith("---\n") else ("", "", "")
83
+ if not sep:
84
+ raise AutError("improve: rubric needs a --- frontmatter block")
85
+ meta: dict[str, Any] = {}
86
+ for line in head.split("\n"):
87
+ if not line.strip():
88
+ continue
89
+ key, colon, value = (part.strip() for part in line.partition(":"))
90
+ if not colon or key in meta:
91
+ raise AutError("improve: rubric frontmatter lines must be unique key: value pairs")
92
+ if key not in FIELDS:
93
+ raise _bad(f"field {key!r}")
94
+ if re.fullmatch(r"-?\d+", value):
95
+ meta[key] = int(value)
96
+ elif re.fullmatch(r"-?(\d+\.\d*|\.\d+)", value):
97
+ meta[key] = float(value)
98
+ elif value[:1] in {"[", "{"} or value[:1].isdigit() or value in {"true", "false"}:
99
+ try:
100
+ meta[key] = json.loads(value)
101
+ except ValueError:
102
+ raise _bad(key) from None
103
+ else:
104
+ meta[key] = value
105
+ if not isinstance(meta.get("id"), str) or not re.fullmatch(r"[a-z_]{1,40}", meta["id"]):
106
+ raise _bad("id")
107
+ if not isinstance(meta.get("enabled", True), bool):
108
+ raise _bad("enabled")
109
+ return meta, body
110
+
111
+
112
+ def _sections(body: str) -> dict[str, list[str]]:
113
+ sections: dict[str, list[str]] = {}
114
+ current: list[str] | None = None
115
+ for line in body.split("\n"):
116
+ if line.startswith("## "):
117
+ name = line[3:].strip()
118
+ if name not in {"Question", "Definition", "Labels"} or name in sections:
119
+ raise _bad(f"heading {name!r}")
120
+ current = sections[name] = []
121
+ elif current is not None:
122
+ current.append(line.strip())
123
+ elif line.strip():
124
+ raise AutError("improve: rubric text must sit under a heading")
125
+ if set(sections) != {"Question", "Definition", "Labels"}:
126
+ raise AutError("improve: rubric needs Question, Definition and Labels headings")
127
+ return sections
128
+
129
+
130
+ def _labels(lines: list[str]) -> dict[str, str]:
131
+ labels: dict[str, list[str]] = {}
132
+ current: list[str] | None = None
133
+ for line in lines:
134
+ match = re.fullmatch(r"- ([a-z]+): (.+)", line)
135
+ if match:
136
+ if match[1] in labels:
137
+ raise _bad("labels")
138
+ current = labels[match[1]] = [match[2]]
139
+ elif line and current is not None:
140
+ current.append(line) # a wrapped label line continues the one above
141
+ elif line:
142
+ raise _bad("labels")
143
+ if set(labels) != LABELS:
144
+ raise _bad("labels")
145
+ return {name: " ".join(parts) for name, parts in labels.items()}
146
+
147
+
148
+ def parse(text: str, source: str) -> Rubric:
149
+ meta, body = _split(text)
150
+ sections = _sections(body)
151
+ question = " ".join(line for line in sections["Question"] if line)
152
+ definition = " ".join(line for line in sections["Definition"] if line)
153
+ if not question or not definition:
154
+ raise _bad("question or definition")
155
+ threshold = meta.get("threshold", .7)
156
+ if type(threshold) not in (int, float) or not .5 < threshold <= 1:
157
+ raise _bad("threshold")
158
+ applies_to = meta.get("applies_to", {})
159
+ if not isinstance(applies_to, dict) or not set(applies_to) <= {"triggers", "requires"}:
160
+ raise _bad("applies_to")
161
+ triggers = _names(applies_to.get("triggers", ["error_result", "control"]), "triggers", TRIGGERS)
162
+ requires = _names(applies_to.get("requires", []), "requires", REQUIRES)
163
+ if not triggers:
164
+ raise _bad("triggers")
165
+ checks = meta.get("checks", {})
166
+ if not isinstance(checks, dict) or not set(checks) <= {
167
+ "fail_must_cite", "fail_cause_not_in", "unknown_unless_complete"}:
168
+ raise _bad("checks")
169
+ if "fail_must_cite" in checks and not _names(checks["fail_must_cite"], "checks", EVENT_KINDS) \
170
+ or "fail_cause_not_in" in checks and not _names(checks["fail_cause_not_in"], "checks",
171
+ CAUSES) \
172
+ or not isinstance(checks.get("unknown_unless_complete", False), bool):
173
+ raise _bad("checks")
174
+ kind = meta.get("input", "attempt")
175
+ if not isinstance(kind, str) or kind not in INPUTS \
176
+ or kind == "outputs_summary" and "outputs" not in requires:
177
+ raise _bad("input")
178
+ qualify = meta.get("qualify", {})
179
+ blocking = meta.get("blocking", False)
180
+ if not isinstance(qualify, dict) or not isinstance(blocking, bool):
181
+ raise _bad("qualify or blocking")
182
+ return Rubric(
183
+ id=meta["id"], version=_int(meta.get("version"), "version", 1, 10**6),
184
+ sha256=hashlib.sha256(_normal(text).encode()).hexdigest(),
185
+ question=question, definition=definition, labels=_labels(sections["Labels"]),
186
+ threshold=threshold, priority=_int(meta.get("priority", 100), "priority", -10**6, 10**6),
187
+ blocking=blocking, triggers=triggers, requires=requires, input=kind,
188
+ max_cases=_int(meta.get("max_cases", 200), "max_cases", 1, 200),
189
+ max_input_chars=_int(meta.get("max_input_chars", 6000), "max_input_chars", 1, 24000),
190
+ checks=checks, qualify=qualify, source=source)
191
+
192
+
193
+ def _files(folder: Path) -> list[Path]:
194
+ return sorted(folder.glob("*.md")) if folder.is_dir() else []
195
+
196
+
197
+ def load(user_dir: Path | None = None) -> list[Rubric]:
198
+ """Enabled rubrics sorted by (priority, id). A user file replaces the built-in with its id
199
+ and keeps the built-in's `blocking` unless it sets its own; a user file with
200
+ `enabled: false` and only an `id` switches that rubric off."""
201
+ found: dict[str, Rubric | None] = {}
202
+ for source, folder in (("builtin", BUILTIN_DIR), ("user", user_dir)):
203
+ seen: set[str] = set()
204
+ for path in _files(folder) if folder else []:
205
+ try:
206
+ if path.stat().st_size > MAX_FILE_BYTES:
207
+ raise AutError("improve: rubric file is too large")
208
+ text = path.read_text(encoding="utf-8")
209
+ meta, _ = _split(text)
210
+ rubric = parse(text, source) if meta.get("enabled", True) else None
211
+ except (OSError, UnicodeDecodeError, AutError) as exc:
212
+ raise AutError(f"improve: rubric file {path.name}: {exc}") from None
213
+ base = found.get(meta["id"])
214
+ if rubric and base and "blocking" not in meta:
215
+ rubric = replace(rubric, blocking=base.blocking)
216
+ if meta["id"] in seen:
217
+ raise AutError(f"improve: two {source} rubric files share id {meta['id']}")
218
+ seen.add(meta["id"])
219
+ found[meta["id"]] = rubric
220
+ return sorted((r for r in found.values() if r), key=lambda r: (r.priority, r.id))
221
+
222
+
223
+ def record(rubric: Rubric) -> dict[str, Any]:
224
+ return {"id": rubric.id, "version": rubric.version, "sha256": rubric.sha256,
225
+ "question": rubric.question, "definition": rubric.definition,
226
+ "labels": rubric.labels, "threshold": rubric.threshold, "priority": rubric.priority,
227
+ "blocking": rubric.blocking,
228
+ "applies_to": {"triggers": sorted(rubric.triggers), "requires": sorted(rubric.requires)},
229
+ "input": rubric.input, "checks": rubric.checks, "max_cases": rubric.max_cases,
230
+ "max_input_chars": rubric.max_input_chars, "qualify": rubric.qualify,
231
+ "source": rubric.source}
232
+
233
+
234
+ def snapshot(store: Store, rubric: Rubric) -> str:
235
+ return store.put("rubric_snapshots", record(rubric))
236
+
237
+
238
+ def _shell_text(text: str, tool: str) -> str | None:
239
+ """The shell command a call ran, or None for a non-shell tool."""
240
+ try:
241
+ value = json.loads(text)
242
+ except ValueError:
243
+ if text.lstrip().startswith("{"): # a clipped JSON input
244
+ match = re.search(r'"(?:command|cmd)"\s*:\s*"((?:[^"\\]|\\.)*)', text)
245
+ return match[1] if match else None
246
+ value = None
247
+ if isinstance(value, dict):
248
+ command = value.get("command", value.get("cmd"))
249
+ if isinstance(command, list) and command and all(isinstance(p, str) for p in command):
250
+ # ["bash", "-lc", "..."] runs its third item; any other argv runs as given.
251
+ return command[2] if len(command) > 2 and command[1] in {"-c", "-lc"} \
252
+ else shlex.join(command)
253
+ return command if isinstance(command, str) else None
254
+ return text if tool in {"Bash", "bash", "exec_command", "shell", "local_shell"} else None
255
+
256
+
257
+ def _family(words: list[str]) -> str:
258
+ while words and (re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", words[0])
259
+ or words[0] in {"env", "npx", "timeout"}):
260
+ if words[0] == "env":
261
+ words = words[1:]
262
+ while words and words[0].startswith("-"):
263
+ words = words[2:] if words[0] in {"-u", "--unset", "-C", "--chdir"} else words[1:]
264
+ elif words[0] == "timeout": # timeout [options] DURATION COMMAND
265
+ words = words[1:]
266
+ while words and words[0].startswith("-"):
267
+ words = words[1:]
268
+ words = words[1:]
269
+ else:
270
+ words = words[1:]
271
+ if not words:
272
+ return ""
273
+ first = re.sub(r"\.exe$", "", words[0].rsplit("/", 1)[-1], flags=re.IGNORECASE)
274
+ first = "python" if re.fullmatch(r"python[0-9.]*", first) else first
275
+ rest = words[1:]
276
+ if first in {"bash", "sh", "zsh"} and len(rest) > 1 and rest[0] in {"-c", "-lc"}:
277
+ return command_family(rest[1], "Bash")
278
+ if first == "python":
279
+ for index, word in enumerate(rest):
280
+ if word == "-m" and index + 1 < len(rest):
281
+ return f"python -m {rest[index + 1]}"
282
+ if not word.startswith("-"):
283
+ break
284
+ return first
285
+ if first not in {"git", "gh", "npm", "pnpm", "uv", "poetry", "cargo", "go"}:
286
+ return first
287
+ i = 0
288
+ while i < len(rest) and rest[i].startswith("-"):
289
+ i += 2 if first == "git" and rest[i] in {"-C", "-c"} else 1
290
+ if i >= len(rest):
291
+ return first
292
+ if first in {"uv", "poetry"} and rest[i] == "run": # uv run X is X's family under uv run
293
+ after = rest[i + 1:]
294
+ while after and after[0].startswith("-"):
295
+ after = after[1:]
296
+ return f"{first} run {_family(after)}" if after else f"{first} run"
297
+ return f"{first} {rest[i]}"
298
+
299
+
300
+ def _segments(command: str) -> list[list[str]]:
301
+ """Words of each simple command, split at ; & | outside quotes."""
302
+ # Backslashes become slashes so Windows paths survive POSIX quoting rules.
303
+ command = command.replace("\\", "/").replace("\n", ";")
304
+ lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|")
305
+ lexer.whitespace_split = True
306
+ try:
307
+ tokens = list(lexer)
308
+ except ValueError: # unbalanced quotes, often from a clipped input
309
+ tokens = re.split(r"\s+|(?<=[;&|])|(?=[;&|])", command)
310
+ segments: list[list[str]] = [[]]
311
+ for token in tokens:
312
+ if token and set(token) <= {";", "&", "|"}:
313
+ segments.append([])
314
+ elif token:
315
+ segments[-1].append(token)
316
+ return segments
317
+
318
+
319
+ def command_family(call_text: str, tool: str) -> str:
320
+ """A short stable name for what a call ran, such as `git push` or `python -m pytest`.
321
+ Leading `cd ...`, `source ...`, `VAR=...`, `env`, `npx` and `timeout` are skipped, and
322
+ `bash -c '...'` is read inside. Non-shell tools give the tool name, with MCP tools
323
+ shortened to `mcp:server`."""
324
+ command = _shell_text(call_text, tool)
325
+ if command is None:
326
+ parts = tool.split("__")
327
+ return f"mcp:{parts[1]}" if len(parts) > 2 and parts[0] == "mcp" else tool
328
+ for words in _segments(command):
329
+ if words and words[0] not in {"cd", "pushd", "source", "."}:
330
+ family = _family(words)
331
+ if family:
332
+ return family
333
+ return ""
334
+
335
+
336
+ def applies(rubric: Rubric, case: dict[str, Any], outputs: dict[str, Any] | None) -> bool:
337
+ if case["trigger"] not in rubric.triggers:
338
+ return False
339
+ events = case["events"]
340
+ checks = {
341
+ "final_reply": lambda: any(e["kind"] == "reply" for e in events),
342
+ "test_command": lambda: any(
343
+ e["kind"] == "call" and command_family(e.get("text", ""), e.get("tool", ""))
344
+ in TEST_FAMILIES for e in events),
345
+ "outputs": lambda: bool(outputs and (outputs.get("diff_stat") or outputs.get("patch"))),
346
+ }
347
+ return all(checks[name]() for name in rubric.requires)
348
+
349
+
350
+ def legacy_version(policy: dict[str, Any], rubric_id: str) -> int:
351
+ """Rubric version of a judgment made before markdown rubrics: 1 when its diagnosis policy
352
+ holds exactly the text the v1 file was moved from, else 0 ("before markdown rubrics")."""
353
+ rubrics = policy.get("rubrics")
354
+ if isinstance(rubrics, dict) and any(isinstance(v, str) for v in rubrics.values()):
355
+ raise AutError("improve: diagnosis policy already names rubric snapshots")
356
+ old = rubrics.get(rubric_id) if isinstance(rubrics, dict) else None
357
+ labels = (policy.get("choice_descriptions") or {}).get(rubric_id)
358
+ if not isinstance(old, list) or len(old) != 2 or not isinstance(labels, dict):
359
+ return 0
360
+ return 1 if digest([*old, labels]) == LEGACY_V1.get(rubric_id) else 0
@@ -0,0 +1,20 @@
1
+ ---
2
+ id: avoidable_recovery
3
+ version: 1
4
+ threshold: 0.7
5
+ priority: 10
6
+ blocking: true
7
+ checks: {"fail_cause_not_in": ["expected_exploration", "unknown"]}
8
+ qualify: {"min_fail_labels": 20, "min_failure_recall": 0.7, "max_false_alarm_rate": 0.2}
9
+ applies_to: {"triggers": ["error_result", "control"]}
10
+ ---
11
+ ## Question
12
+ Does the observed attempt waste effort through avoidable recovery?
13
+
14
+ ## Definition
15
+ fail: repeated attempts ignore the same known cause, or an evidenced wrong setup instruction forces recovery; pass: observed exploration or negative tests are useful and bounded; unknown: intent, relevant results, or cause cannot be established.
16
+
17
+ ## Labels
18
+ - pass: Observed effort is useful and bounded, with no unnecessary recovery.
19
+ - fail: Repeated attempts ignore the same known cause, or a wrong setup instruction demonstrably forces recovery.
20
+ - unknown: The relevant results, intent, or cause are insufficient to decide whether recovery was avoidable.
@@ -0,0 +1,22 @@
1
+ ---
2
+ id: code_quality
3
+ version: 1
4
+ threshold: 0.7
5
+ priority: 50
6
+ blocking: false
7
+ applies_to: {"triggers": ["merge", "control"], "requires": ["outputs"]}
8
+ input: outputs_summary
9
+ max_cases: 3
10
+ max_input_chars: 9000
11
+ qualify: {"min_fail_labels": 20, "min_failure_recall": 0.7, "max_false_alarm_rate": 0.2}
12
+ ---
13
+ ## Question
14
+ Is the committed diff scoped to the request?
15
+
16
+ ## Definition
17
+ Judge the diff against the request, not the style of the code. fail: the diff adds dead code, duplicated logic, or a change unrelated to the prompt, or the commit was reverted within the window; pass: the diff is scoped to the request; unknown: there is no diff, or the diff is truncated.
18
+
19
+ ## Labels
20
+ - pass: The diff is scoped to the request.
21
+ - fail: The diff adds dead code, duplicated logic, or a change unrelated to the prompt, or the commit was reverted.
22
+ - unknown: There is no diff, or the diff is truncated.
@@ -0,0 +1,20 @@
1
+ ---
2
+ id: completion_supported
3
+ version: 1
4
+ threshold: 0.7
5
+ priority: 20
6
+ blocking: true
7
+ checks: {"fail_must_cite": ["reply", "result"], "unknown_unless_complete": true}
8
+ qualify: {"min_fail_labels": 20, "min_failure_recall": 0.7, "max_false_alarm_rate": 0.2}
9
+ applies_to: {"triggers": ["error_result", "control"]}
10
+ ---
11
+ ## Question
12
+ Is the assistant's completion claim supported by the observed results?
13
+
14
+ ## Definition
15
+ Judge explicit reporting, not whether the task succeeded. Do not invent implicit goals. Declared omissions are not completion claims. fail: a specific explicit completion claim contradicts a relevant observed result; pass: relevant observed checks support the claim or the assistant clearly reports the unresolved blocker; unknown: no clear final claim or insufficient acceptance evidence.
16
+
17
+ ## Labels
18
+ - pass: Observed results support an explicit completion or verification report, or the assistant clearly reports an unresolved blocker.
19
+ - fail: An explicit completion or verification report contradicts a relevant observed result.
20
+ - unknown: No clear completion or verification report is made, or acceptance evidence is insufficient. A reasonable reply alone does not establish completion.
@@ -0,0 +1,21 @@
1
+ ---
2
+ id: redundant_tests
3
+ version: 1
4
+ threshold: 0.7
5
+ priority: 40
6
+ blocking: false
7
+ applies_to: {"triggers": ["error_result", "control"], "requires": ["test_command"]}
8
+ max_cases: 4
9
+ checks: {"fail_must_cite": ["call"]}
10
+ qualify: {"min_fail_labels": 20, "min_failure_recall": 0.7, "max_false_alarm_rate": 0.2}
11
+ ---
12
+ ## Question
13
+ Does each test run in the observed attempt follow a change or narrow a failure?
14
+
15
+ ## Definition
16
+ Judge only the test commands that appear in the attempt. A rerun after a file change, or a narrower run after a failure, is useful. fail: the same test command ran again with no file change between runs and no new information, or a full suite ran where one cited test would answer; pass: each test run follows a change or narrows a failure; unknown: file changes between the runs cannot be seen.
17
+
18
+ ## Labels
19
+ - pass: Each test run follows a change or narrows a failure.
20
+ - fail: The same test command ran again with no file change and no new information, or a full suite ran where one cited test would answer.
21
+ - unknown: File changes between the test runs cannot be seen.
@@ -0,0 +1,21 @@
1
+ ---
2
+ id: verbosity
3
+ version: 1
4
+ threshold: 0.7
5
+ priority: 30
6
+ blocking: false
7
+ applies_to: {"triggers": ["error_result", "control"], "requires": ["final_reply"]}
8
+ max_cases: 4
9
+ checks: {"fail_must_cite": ["reply"]}
10
+ qualify: {"min_fail_labels": 20, "min_failure_recall": 0.7, "max_false_alarm_rate": 0.2}
11
+ ---
12
+ ## Question
13
+ Is the assistant's final reply no longer than the result needs?
14
+
15
+ ## Definition
16
+ Judge only the final reply to this attempt, not the work before it. Do not count a needed command, error, or open issue as padding. fail: the final reply repeats tool output, restates the prompt, or is padded beyond what the result needs; pass: the reply states the result, its evidence, and open issues briefly; unknown: the final reply is clipped or missing.
17
+
18
+ ## Labels
19
+ - pass: The final reply states the result, its evidence, and open issues briefly.
20
+ - fail: The final reply repeats tool output, restates the prompt, or is padded beyond what the result needs.
21
+ - unknown: The final reply is clipped or missing, so its length cannot be judged.
@@ -0,0 +1,117 @@
1
+ """Install a pinned wheel outside development checkouts, without editable links."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import os
8
+ import shutil
9
+ import subprocess
10
+ import sys
11
+ import hashlib
12
+ import zipfile
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from ..config import AutError
17
+ from ..state import atomic_write_text, _locked
18
+ from .proposals import file_digest
19
+ from .store import Store, digest
20
+
21
+
22
+ def _verify_wheel(wheel: Path, library: Path) -> None:
23
+ with zipfile.ZipFile(wheel) as archive:
24
+ for info in archive.infolist():
25
+ if info.is_dir() or info.filename.endswith(".dist-info/RECORD"):
26
+ continue
27
+ target = (library / info.filename).resolve()
28
+ if not target.is_relative_to(library.resolve()) or not target.is_file() \
29
+ or file_digest(target) != hashlib.sha256(archive.read(info)).hexdigest():
30
+ raise AutError("improve: installed runtime file differs from its pinned wheel")
31
+
32
+
33
+ def install(store: Store, wheel: Path, runtime_root: Path, *,
34
+ constraints: Path | None = None) -> dict[str, Any]:
35
+ wheel = wheel.resolve()
36
+ if wheel.suffix != ".whl" or not wheel.is_file():
37
+ raise AutError("improve: runtime needs a readable wheel")
38
+ uv = shutil.which("uv")
39
+ if uv is None:
40
+ raise AutError("improve: install uv before installing an offline runtime")
41
+ pins = None
42
+ if constraints:
43
+ try:
44
+ pins = "\n".join(line for line in constraints.read_text().splitlines()
45
+ if not line.lower().startswith(("agent-usage-tracker", "-e "))) + "\n"
46
+ except OSError as exc:
47
+ raise AutError("improve: dependency constraints are not readable") from exc
48
+ constraint_sha = digest(pins) if pins is not None else None
49
+ sha = file_digest(wheel)
50
+ name = sha[:16] + ("-" + constraint_sha[:12] if constraint_sha else "")
51
+ root = runtime_root.expanduser().resolve() / "releases" / name
52
+ root.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
53
+ with _locked(root.parent / "install"):
54
+ manifest_path = root / "runtime.json"
55
+ saved_wheel = root / wheel.name
56
+ if root.exists():
57
+ try:
58
+ previous = json.loads(manifest_path.read_text())
59
+ if previous["wheel_sha256"] != sha \
60
+ or previous.get("constraints_sha256") != constraint_sha \
61
+ or file_digest(root / "requirements.lock") != previous["requirements_sha256"] \
62
+ or file_digest(saved_wheel) != sha:
63
+ raise ValueError
64
+ except (OSError, ValueError, TypeError, KeyError) as exc:
65
+ raise AutError("improve: existing runtime is incomplete or unmanaged; preserved") from exc
66
+ else:
67
+ root.mkdir(mode=0o700)
68
+ shutil.copyfile(wheel, saved_wheel)
69
+ saved_wheel.chmod(0o600)
70
+ subprocess.run([uv, "venv", "--python", str(Path(sys.executable).resolve()),
71
+ str(root / "venv")], check=True, timeout=120)
72
+ python = root / "venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
73
+ argv = [uv, "pip", "install", "--offline", "--python", str(python),
74
+ str(saved_wheel) + "[label]"]
75
+ if pins is not None:
76
+ # The project wheel changes between conditions; dependency pins must stay fixed.
77
+ atomic_write_text(root / "dependency-constraints.txt", pins)
78
+ argv += ["--constraint", str(root / "dependency-constraints.txt")]
79
+ subprocess.run(argv, check=True, timeout=300)
80
+ python = root / "venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
81
+ probe = ("import importlib.metadata as m,json,agent_usage_tracker as a; "
82
+ "d=m.distribution('agent-usage-tracker'); "
83
+ "print(json.dumps({'module':a.__file__, 'version':d.version, "
84
+ "'library':str(d.locate_file('')), "
85
+ "'source':json.loads(d.read_text('direct_url.json') or '{}')}))")
86
+ identity = json.loads(subprocess.check_output(
87
+ [str(python), "-I", "-c", probe], cwd=store.root, text=True, timeout=30))
88
+ if not Path(identity["module"]).resolve().is_relative_to(root) \
89
+ or identity["source"].get("dir_info", {}).get("editable") \
90
+ or identity["source"].get("url") != saved_wheel.as_uri():
91
+ raise AutError("improve: runtime probe did not resolve the pinned non-editable wheel")
92
+ _verify_wheel(saved_wheel, Path(identity["library"]))
93
+ pins = subprocess.check_output([uv, "pip", "freeze", "--python", str(python)],
94
+ text=True, timeout=30)
95
+ lock_path = root / "requirements.lock"
96
+ if lock_path.exists() and lock_path.read_text() != pins:
97
+ raise AutError("improve: installed runtime dependencies changed; preserved")
98
+ atomic_write_text(lock_path, pins)
99
+ manifest = {"wheel": str(saved_wheel), "source_wheel": str(wheel),
100
+ "wheel_sha256": sha, "runtime": str(root),
101
+ "python": str(python), "identity": identity, "editable": False,
102
+ "constraints_sha256": constraint_sha,
103
+ "requirements_sha256": file_digest(lock_path)}
104
+ atomic_write_text(manifest_path, json.dumps(manifest, indent=2) + "\n")
105
+ store.append("runtime_verified", **manifest)
106
+ return manifest
107
+
108
+
109
+ if __name__ == "__main__":
110
+ parser = argparse.ArgumentParser(description=__doc__)
111
+ parser.add_argument("--wheel", required=True, type=Path)
112
+ parser.add_argument("--store", required=True, type=Path)
113
+ parser.add_argument("--runtime-root", required=True, type=Path)
114
+ parser.add_argument("--constraints", type=Path)
115
+ args = parser.parse_args()
116
+ print(json.dumps(install(Store(args.store), args.wheel, args.runtime_root,
117
+ constraints=args.constraints)))