agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,318 @@
1
+ """Execute explicit trusted trial cases and retain every baseline/candidate result."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import random
7
+ import json
8
+ import shutil
9
+ import signal
10
+ import statistics
11
+ import sys
12
+ import subprocess
13
+ import threading
14
+ import time
15
+ import uuid
16
+ from pathlib import Path
17
+ from types import FrameType
18
+ from typing import Any
19
+
20
+ from ..config import AutError
21
+ from ..redact import redact
22
+ from ..state import _locked
23
+ from .proposals import file_digest
24
+ from .store import Store, digest
25
+
26
+
27
+ def command(argv: list[str], cwd: Path, timeout_s: float, *,
28
+ source_pythonpath: bool = True) -> dict[str, Any]:
29
+ env = {k: v for k, v in os.environ.items()
30
+ if k in {"PATH", "HOME", "LANG", "LC_ALL", "TZ", "SYSTEMROOT", "TEMP", "TMP"}}
31
+ env.update(PYTHONDONTWRITEBYTECODE="1", HF_HUB_OFFLINE="1")
32
+ if source_pythonpath:
33
+ env["PYTHONPATH"] = str(cwd / "src")
34
+ started = time.monotonic()
35
+ process: subprocess.Popen[bytes] | None = None
36
+ timed_out = False
37
+ output = bytearray()
38
+ output_limit = threading.Event()
39
+ cleanup_deadline: float | None = None
40
+ cleanup_timeout = False
41
+
42
+ def cleanup_remaining() -> float:
43
+ assert cleanup_deadline is not None
44
+ return max(0., cleanup_deadline - time.monotonic())
45
+
46
+ def reap() -> None:
47
+ nonlocal cleanup_timeout
48
+ assert process is not None
49
+ try:
50
+ process.wait(timeout=cleanup_remaining())
51
+ except subprocess.TimeoutExpired:
52
+ cleanup_timeout = True
53
+
54
+ def stop() -> None:
55
+ nonlocal cleanup_deadline
56
+ if process is None:
57
+ return
58
+ # A child can remain unreaped after SIGKILL; do not block its supervisor indefinitely.
59
+ if cleanup_deadline is None:
60
+ cleanup_deadline = time.monotonic() + 1
61
+ try:
62
+ if os.name == "posix":
63
+ os.killpg(process.pid, signal.SIGKILL)
64
+ else:
65
+ process.kill()
66
+ except ProcessLookupError:
67
+ pass
68
+
69
+ def read_output() -> None:
70
+ assert process is not None and process.stdout is not None
71
+ with process.stdout:
72
+ while chunk := process.stdout.read(65536):
73
+ remaining = 256 * 1024 - len(output)
74
+ output.extend(chunk[:remaining])
75
+ if len(chunk) > remaining:
76
+ output_limit.set()
77
+ stop()
78
+ break
79
+
80
+ reader = threading.Thread(target=read_output, daemon=True)
81
+ previous = None
82
+
83
+ def interrupt(signum: int, frame: FrameType | None) -> None:
84
+ raise KeyboardInterrupt
85
+
86
+ if threading.current_thread() is threading.main_thread():
87
+ previous = signal.signal(signal.SIGTERM, interrupt)
88
+ try:
89
+ process = subprocess.Popen(argv, cwd=cwd, env=env, stdout=subprocess.PIPE,
90
+ stderr=subprocess.STDOUT, start_new_session=os.name == "posix")
91
+ reader.start()
92
+ try:
93
+ process.wait(timeout=max(0., started + timeout_s - time.monotonic()))
94
+ timed_out = time.monotonic() >= started + timeout_s
95
+ except subprocess.TimeoutExpired:
96
+ timed_out = True
97
+ stop()
98
+ reap()
99
+ finally:
100
+ if process is not None and process.poll() is None:
101
+ stop()
102
+ reap()
103
+ if reader.ident is not None:
104
+ reader.join(timeout=1 if cleanup_deadline is None else cleanup_remaining())
105
+ if reader.is_alive():
106
+ stop()
107
+ reader.join(timeout=cleanup_remaining())
108
+ cleanup_timeout |= reader.is_alive()
109
+ if previous is not None:
110
+ signal.signal(signal.SIGTERM, previous)
111
+ assert process is not None
112
+ return {"exit_code": process.returncode, "timeout": timed_out,
113
+ "output_limit": output_limit.is_set(),
114
+ "accepted": process.returncode == 0 and not timed_out and not output_limit.is_set()
115
+ and not cleanup_timeout,
116
+ "cleanup_timeout": cleanup_timeout,
117
+ **({"pid": process.pid} if cleanup_timeout else {}),
118
+ "elapsed_s": round(time.monotonic() - started, 4),
119
+ "output": redact(output.decode("utf-8", errors="replace"))[:12000],
120
+ "output_truncated": output_limit.is_set() or len(output) > 12000}
121
+
122
+
123
+ def _spec(store: Store, spec_id: str) -> dict[str, Any]:
124
+ spec = store.get("trial_specs", spec_id)
125
+ try:
126
+ directories = spec["directories"]
127
+ if spec["version"] != 1 or spec["kind"] not in {"command_check", "agent_task"} \
128
+ or set(directories) != {"baseline", "candidate"}:
129
+ raise ValueError
130
+ roots = [Path(value).resolve() for value in directories.values()]
131
+ if roots[0] == roots[1]:
132
+ raise AutError("improve: trial directories must be independent")
133
+ if any(not root.is_dir() for root in roots):
134
+ raise ValueError
135
+ if type(spec["repeats"]) is not int or not 1 <= spec["repeats"] <= 10:
136
+ raise ValueError
137
+ if not 1 <= len(spec["cases"]) <= 50:
138
+ raise ValueError
139
+ ids = set()
140
+ for case in spec["cases"]:
141
+ if not isinstance(case["id"], str) or not case["id"] or case["id"] in ids:
142
+ raise ValueError
143
+ ids.add(case["id"])
144
+ argv = case["argv"]
145
+ if not isinstance(argv, list) or not argv \
146
+ or any(not isinstance(arg, str) or "\0" in arg for arg in argv):
147
+ raise ValueError
148
+ if type(case["timeout_s"]) not in (int, float) or not 0 < case["timeout_s"] <= 300:
149
+ raise ValueError
150
+ if spec["kind"] == "agent_task":
151
+ if not all(spec.get("files", {}).get(name) for name in directories):
152
+ raise ValueError
153
+ accept = case["acceptance_argv"]
154
+ if not isinstance(accept, list) or not accept or any(
155
+ not isinstance(arg, str) or "\0" in arg for arg in accept):
156
+ raise ValueError
157
+ if "{metrics_file}" not in argv or not case["acceptance_files"]:
158
+ raise ValueError
159
+ _acceptance_inputs(case, roots)
160
+ elif "acceptance_files" in case:
161
+ for root in roots:
162
+ _command_acceptance(case, roots, root)
163
+ except (KeyError, TypeError, ValueError) as exc:
164
+ raise AutError("improve: invalid trusted trial specification") from exc
165
+ return spec
166
+
167
+
168
+ def _acceptance_inputs(case: dict[str, Any], roots: list[Path], cwd: Path | None = None) -> None:
169
+ for index, arg in enumerate(case["acceptance_argv"]):
170
+ path = Path(arg)
171
+ if not path.is_absolute() and cwd is None:
172
+ continue
173
+ path = ((cwd or Path.cwd()) / path).resolve()
174
+ if path.is_file() and not (
175
+ index == 0 and path.resolve() == Path(sys.executable).resolve()
176
+ ) and path not in {Path(p).resolve() for p in case["acceptance_files"]}:
177
+ raise AutError("improve: acceptance command input is not frozen")
178
+ for name, expected in case["acceptance_files"].items():
179
+ path = Path(name)
180
+ if not path.is_absolute() or any(path.resolve().is_relative_to(root) for root in roots):
181
+ raise AutError("improve: acceptance inputs must stay outside agent templates")
182
+ if not path.is_file() or file_digest(path) != expected:
183
+ raise AutError("improve: acceptance input changed")
184
+
185
+
186
+ def _command_acceptance(case: dict[str, Any], roots: list[Path], cwd: Path) -> None:
187
+ if "acceptance_files" not in case:
188
+ return
189
+ if not isinstance(case["acceptance_files"], dict) or not case["acceptance_files"]:
190
+ raise AutError("improve: command acceptance files must be a nonempty hash map")
191
+ _acceptance_inputs({"acceptance_argv": case["argv"],
192
+ "acceptance_files": case["acceptance_files"]}, roots, cwd)
193
+
194
+
195
+ def _agent_run(store: Store, spec: dict[str, Any], case: dict[str, Any], condition: str,
196
+ key: str, deadline: float) -> dict[str, Any]:
197
+ source = Path(spec["directories"][condition]).resolve()
198
+ directory = store.root / "agent_runs" / key / uuid.uuid4().hex
199
+ workspace = directory / "workspace"
200
+ workspace.mkdir(mode=0o700, parents=True)
201
+ for name, expected in spec["files"][condition].items():
202
+ src, target = (source / name).resolve(), workspace / name
203
+ if not src.is_relative_to(source) or not target.resolve().is_relative_to(workspace):
204
+ raise AutError("improve: agent fixture path escapes its workspace")
205
+ if file_digest(src) != expected:
206
+ raise AutError("improve: agent fixture changed")
207
+ target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
208
+ shutil.copyfile(src, target)
209
+ target.chmod(src.stat().st_mode & 0o700)
210
+ metrics_path = directory / "metrics.json"
211
+ argv = [str(metrics_path) if arg == "{metrics_file}" else arg for arg in case["argv"]]
212
+ result = command(argv, workspace, min(case["timeout_s"], max(.01, deadline-time.monotonic())))
213
+ execution_id = store.put("agent_executions", {"workspace": str(workspace), **result})
214
+ acceptance = None
215
+ metrics = None
216
+ error = None
217
+ try:
218
+ roots = [Path(p).resolve() for p in spec["directories"].values()]
219
+ _acceptance_inputs(case, roots, directory)
220
+ accept_argv = [str(workspace) if arg == "{workspace}" else arg
221
+ for arg in case["acceptance_argv"]]
222
+ acceptance = command(accept_argv, directory,
223
+ min(30, max(.01, deadline-time.monotonic())), source_pythonpath=False)
224
+ _acceptance_inputs(case, roots, directory)
225
+ if metrics_path.is_file():
226
+ if metrics_path.stat().st_size > 16384 or metrics_path.is_symlink():
227
+ raise ValueError
228
+ metrics = json.loads(metrics_path.read_text())
229
+ if set(metrics) != {"tool_calls", "interventions"} or any(
230
+ type(value) is not int or value < 0 for value in metrics.values()):
231
+ raise ValueError
232
+ else:
233
+ raise ValueError("missing agent metrics")
234
+ except (AutError, OSError, ValueError, TypeError, AttributeError) as exc:
235
+ error = redact(str(exc)) or "invalid captured agent metrics"
236
+ metrics = None
237
+ return {**result, "execution_id": execution_id, "workspace": str(workspace),
238
+ "acceptance": acceptance, "metrics": metrics, "validation_error": error,
239
+ "accepted": bool(result["accepted"] and acceptance and acceptance["accepted"]
240
+ and metrics is not None and error is None)}
241
+
242
+
243
+ def run_trial(store: Store, spec_id: str, *, execute: bool = False,
244
+ budget_s: int = 600) -> dict[str, Any]:
245
+ spec = _spec(store, spec_id)
246
+ if not 1 <= budget_s <= 3600:
247
+ raise AutError("improve: trial budget must be 1..3600 seconds")
248
+ directories = spec["directories"]
249
+ results = {row["evaluation_key"]: row for key in store.keys("trial_results")
250
+ if (row := store.get("trial_results", key))["spec_id"] == spec_id}
251
+ total = 2 * spec["repeats"] * len(spec["cases"])
252
+ executed = 0
253
+ deadline = time.monotonic() + budget_s
254
+ if execute:
255
+ with _locked(store.root / "trial", blocking=False):
256
+ for condition, root_name in directories.items():
257
+ root = Path(root_name).resolve()
258
+ for relative, expected in spec.get("files", {}).get(condition, {}).items():
259
+ target = (root / relative).resolve()
260
+ if not target.is_relative_to(root) or file_digest(target) != expected:
261
+ raise AutError("improve: trial input changed; previous results are preserved")
262
+ for repeat in range(spec["repeats"]):
263
+ for case in spec["cases"]:
264
+ order = ["baseline", "candidate"]
265
+ random.Random(f"{spec.get('seed', 0)}:{repeat}:{case['id']}").shuffle(order)
266
+ for condition in order:
267
+ identity = {"spec_id": spec_id, "repeat": repeat,
268
+ "case_id": case["id"], "condition": condition}
269
+ key = digest(identity)
270
+ if key in results:
271
+ continue
272
+ remaining = deadline - time.monotonic()
273
+ if remaining <= 0:
274
+ break
275
+ root = Path(directories[condition]).resolve()
276
+ for relative, expected in spec.get("files", {}).get(condition, {}).items():
277
+ target = (root / relative).resolve()
278
+ if not target.is_relative_to(root) or file_digest(target) != expected:
279
+ raise AutError("improve: trial input changed; prepare a new specification")
280
+ roots = [Path(name).resolve() for name in directories.values()]
281
+ if spec["kind"] == "command_check":
282
+ _command_acceptance(case, roots, root)
283
+ outcome = (_agent_run(store, spec, case, condition, key, deadline)
284
+ if spec["kind"] == "agent_task" else
285
+ command(case["argv"], root, min(case["timeout_s"], remaining)))
286
+ validation_error = None
287
+ if spec["kind"] == "command_check" and "acceptance_files" in case:
288
+ try:
289
+ _command_acceptance(case, roots, root)
290
+ except (AutError, OSError) as exc:
291
+ validation_error = redact(str(exc))
292
+ outcome.update(accepted=False, validation_error=validation_error)
293
+ result = {**identity, "evaluation_key": key, **outcome}
294
+ result_id = store.put("trial_results", result)
295
+ results[key] = result
296
+ executed += 1
297
+ if validation_error is not None:
298
+ store.append("trial_acceptance_rejected", spec_id=spec_id,
299
+ result_id=result_id, validation_error=validation_error)
300
+ raise AutError("improve: protected command acceptance changed; "
301
+ f"rejected result {result_id}")
302
+ report = {"spec_id": spec_id, "complete": len(results) == total,
303
+ "finished": len(results), "total": total, "executed": executed,
304
+ "accepted": {condition: sum(row["condition"] == condition and row["accepted"]
305
+ for row in results.values()) for condition in directories},
306
+ "timeouts": sum(row["timeout"] for row in results.values()),
307
+ "trial_kind": spec["kind"], "agent_behaviour_verified": False}
308
+ if spec["kind"] == "agent_task":
309
+ report["conditions"] = {name: {
310
+ "runs": len(rows := [r for r in results.values() if r["condition"] == name]),
311
+ "median_elapsed_s": statistics.median([r["elapsed_s"] for r in rows]) if rows else None,
312
+ "tool_calls": sum(r["metrics"]["tool_calls"] for r in rows if r.get("metrics")),
313
+ "interventions": sum(r["metrics"]["interventions"] for r in rows if r.get("metrics")),
314
+ "missing_metrics": sum(r.get("metrics") is None for r in rows),
315
+ } for name in directories}
316
+ report["improvement_established"] = False
317
+ store.append("trial", **report)
318
+ return report
@@ -0,0 +1,148 @@
1
+ """Compare frozen judge decisions with explicit, independently executable references."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ import re
8
+ import sys
9
+ from collections import Counter
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from ..config import AutError
14
+ from . import rubric
15
+ from .judge import LABELS, evaluation_key, evaluator
16
+ from .proposals import file_digest
17
+ from .store import Store, digest
18
+ from .trial import command
19
+
20
+
21
+ def evidence_key(case: dict[str, Any]) -> str:
22
+ return digest({name: case[name] for name in (
23
+ "events", "coverage", "omitted_task_events")})
24
+
25
+
26
+ def _oracle_inputs(spec: dict[str, Any]) -> None:
27
+ for index, arg in enumerate(spec["oracle_argv"]):
28
+ path = (Path(spec["oracle_cwd"]) / arg).resolve()
29
+ if path.is_file() and not (
30
+ index == 0 and path.resolve() == Path(sys.executable).resolve()
31
+ ) and path not in {Path(p).resolve() for p in spec["oracle_files"]}:
32
+ raise AutError("improve: reference oracle command input is not frozen")
33
+ for name, expected in spec["oracle_files"].items():
34
+ path = Path(name)
35
+ if not path.is_absolute() or not path.is_file() or file_digest(path) != expected:
36
+ raise AutError("improve: reference oracle input changed")
37
+
38
+
39
+ def validate(store: Store, spec_id: str, diagnosis_id: str, *,
40
+ execute: bool = False) -> dict[str, Any]:
41
+ spec = store.get("reference_specs", spec_id)
42
+ policy = store.get("diagnoses", diagnosis_id)
43
+ try:
44
+ references = spec["references"]
45
+ argv = spec["oracle_argv"]
46
+ timeout = spec["timeout_s"]
47
+ if spec["version"] != 1 or spec["split"] not in {"development", "holdout"} \
48
+ or not isinstance(references, list) or not 1 <= len(references) <= 200 \
49
+ or not isinstance(argv, list) or not argv \
50
+ or any(not isinstance(a, str) or "\0" in a for a in argv) \
51
+ or type(timeout) not in (int, float) or not math.isfinite(timeout) \
52
+ or not 0 < timeout <= 300 or not spec["oracle_files"] \
53
+ or "{reference_input}" not in argv:
54
+ raise ValueError
55
+ identities = set()
56
+ # A policy names its rubrics; one without any predates them and had only the two core ones.
57
+ known = policy.get("rubrics")
58
+ if not isinstance(known, dict):
59
+ known = set(rubric.LEGACY_V1)
60
+ for ref in references:
61
+ if ref["rubric"] not in known or ref["expected"] not in LABELS \
62
+ or not re.fullmatch(r"[a-f0-9]{64}", ref["evidence_sha256"]):
63
+ raise ValueError
64
+ identity = ref["evidence_sha256"], ref["rubric"]
65
+ if identity in identities:
66
+ raise ValueError
67
+ identities.add(identity)
68
+ cwd = Path(spec["oracle_cwd"])
69
+ if not cwd.is_absolute() or not cwd.is_dir():
70
+ raise ValueError
71
+ _oracle_inputs(spec)
72
+ except (KeyError, TypeError, ValueError, AttributeError) as exc:
73
+ raise AutError("improve: invalid executable reference specification") from exc
74
+ cases: dict[str, str] = {}
75
+ for case_id in policy["cases"]:
76
+ key = evidence_key(store.get("cases", case_id))
77
+ if key in cases:
78
+ raise AutError("improve: ambiguous reference evidence in diagnosis")
79
+ cases[key] = case_id
80
+ if any(ref["evidence_sha256"] not in cases for ref in references):
81
+ raise AutError("improve: reference evidence is absent from this diagnosis")
82
+ if not execute:
83
+ return {"spec_id": spec_id, "diagnosis_id": diagnosis_id,
84
+ "execution_required": True, "references": len(references), "split": spec["split"]}
85
+ payload = {"references": [{"evidence_sha256": ref["evidence_sha256"],
86
+ "rubric": ref["rubric"], "case": store.get("cases", cases[ref["evidence_sha256"]])}
87
+ for ref in references]}
88
+ input_id = store.put("reference_inputs", payload)
89
+ input_path = store.path("reference_inputs", input_id)
90
+ argv = [str(input_path) if arg == "{reference_input}" else arg for arg in argv]
91
+ oracle = command(argv, cwd, timeout, source_pythonpath=False)
92
+ oracle_id = store.put("reference_oracles", {"spec_id": spec_id, **oracle})
93
+ _oracle_inputs(spec)
94
+ store.get("reference_inputs", input_id)
95
+ try:
96
+ if not oracle["accepted"] or oracle["output_truncated"]:
97
+ raise ValueError
98
+ observed = json.loads(oracle["output"])
99
+ # Short digest parts survive text redaction without exempting arbitrary long tokens.
100
+ bound = [{"evidence_sha256": "".join(row["evidence_sha256_parts"]),
101
+ "rubric": row["rubric"], "expected": row["expected"]}
102
+ for row in observed["references"]]
103
+ if bound != references:
104
+ raise ValueError
105
+ except (ValueError, TypeError, KeyError) as exc:
106
+ raise AutError(f"improve: reference oracle failed or disagreed; record {oracle_id}") from exc
107
+ judgments = {value["evaluation_key"]: (key, value) for key in store.keys("judgments")
108
+ if (value := store.get("judgments", key))}
109
+ counts: Counter[str] = Counter()
110
+ rows = []
111
+ for ref in references:
112
+ key = evaluation_key(evaluator(policy, ref["rubric"]), cases[ref["evidence_sha256"]],
113
+ ref["rubric"])
114
+ if key not in judgments:
115
+ counts["missing"] += 1
116
+ rows.append({**ref, "observed": None, "judgment_id": None})
117
+ continue
118
+ judgment_id, judgment = judgments[key]
119
+ verdict = judgment["verdict"]
120
+ if verdict not in LABELS:
121
+ raise AutError("improve: reference judgment has an invalid verdict")
122
+ expected = ref["expected"]
123
+ counts[f"reference_{expected}"] += 1
124
+ counts["abstentions"] += verdict == "unknown"
125
+ counts["detected_failures"] += expected == "fail" and verdict == "fail"
126
+ counts["missed_failures"] += expected == "fail" and verdict != "fail"
127
+ counts["false_alarms"] += expected == "pass" and verdict == "fail"
128
+ counts["supported_passes"] += expected == "pass" and verdict == "pass"
129
+ counts["decisions_on_unknown"] += expected == "unknown" and verdict != "unknown"
130
+ rows.append({**ref, "observed": verdict, "judgment_id": judgment_id})
131
+
132
+ def rate(numerator: str, denominator: str) -> float | None:
133
+ return counts[numerator] / counts[denominator] if counts[denominator] \
134
+ and not counts["missing"] else None
135
+
136
+ report = {"spec_id": spec_id, "diagnosis_id": diagnosis_id, "oracle_id": oracle_id,
137
+ "input_id": input_id,
138
+ "complete": not counts["missing"], "split": spec["split"], "counts": dict(counts),
139
+ "failure_recall": rate("detected_failures", "reference_fail"),
140
+ "false_alarm_rate": rate("false_alarms", "reference_pass"),
141
+ "pass_recall": rate("supported_passes", "reference_pass"), "cases": rows,
142
+ "scope": "executable references only; not real-work accuracy",
143
+ "human_verified": False, "automation_qualified": False,
144
+ "holdout_independence_verified": False}
145
+ key = store.put("reference_results", report)
146
+ store.append("reference_validation", report_id=key, diagnosis_id=diagnosis_id,
147
+ spec_id=spec_id, complete=report["complete"])
148
+ return {"report_id": key, **report}