agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
"""Execute explicit trusted trial cases and retain every baseline/candidate result."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import json
|
|
8
|
+
import shutil
|
|
9
|
+
import signal
|
|
10
|
+
import statistics
|
|
11
|
+
import sys
|
|
12
|
+
import subprocess
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
import uuid
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from types import FrameType
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from ..config import AutError
|
|
21
|
+
from ..redact import redact
|
|
22
|
+
from ..state import _locked
|
|
23
|
+
from .proposals import file_digest
|
|
24
|
+
from .store import Store, digest
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def command(argv: list[str], cwd: Path, timeout_s: float, *,
|
|
28
|
+
source_pythonpath: bool = True) -> dict[str, Any]:
|
|
29
|
+
env = {k: v for k, v in os.environ.items()
|
|
30
|
+
if k in {"PATH", "HOME", "LANG", "LC_ALL", "TZ", "SYSTEMROOT", "TEMP", "TMP"}}
|
|
31
|
+
env.update(PYTHONDONTWRITEBYTECODE="1", HF_HUB_OFFLINE="1")
|
|
32
|
+
if source_pythonpath:
|
|
33
|
+
env["PYTHONPATH"] = str(cwd / "src")
|
|
34
|
+
started = time.monotonic()
|
|
35
|
+
process: subprocess.Popen[bytes] | None = None
|
|
36
|
+
timed_out = False
|
|
37
|
+
output = bytearray()
|
|
38
|
+
output_limit = threading.Event()
|
|
39
|
+
cleanup_deadline: float | None = None
|
|
40
|
+
cleanup_timeout = False
|
|
41
|
+
|
|
42
|
+
def cleanup_remaining() -> float:
|
|
43
|
+
assert cleanup_deadline is not None
|
|
44
|
+
return max(0., cleanup_deadline - time.monotonic())
|
|
45
|
+
|
|
46
|
+
def reap() -> None:
|
|
47
|
+
nonlocal cleanup_timeout
|
|
48
|
+
assert process is not None
|
|
49
|
+
try:
|
|
50
|
+
process.wait(timeout=cleanup_remaining())
|
|
51
|
+
except subprocess.TimeoutExpired:
|
|
52
|
+
cleanup_timeout = True
|
|
53
|
+
|
|
54
|
+
def stop() -> None:
|
|
55
|
+
nonlocal cleanup_deadline
|
|
56
|
+
if process is None:
|
|
57
|
+
return
|
|
58
|
+
# A child can remain unreaped after SIGKILL; do not block its supervisor indefinitely.
|
|
59
|
+
if cleanup_deadline is None:
|
|
60
|
+
cleanup_deadline = time.monotonic() + 1
|
|
61
|
+
try:
|
|
62
|
+
if os.name == "posix":
|
|
63
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
64
|
+
else:
|
|
65
|
+
process.kill()
|
|
66
|
+
except ProcessLookupError:
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
def read_output() -> None:
|
|
70
|
+
assert process is not None and process.stdout is not None
|
|
71
|
+
with process.stdout:
|
|
72
|
+
while chunk := process.stdout.read(65536):
|
|
73
|
+
remaining = 256 * 1024 - len(output)
|
|
74
|
+
output.extend(chunk[:remaining])
|
|
75
|
+
if len(chunk) > remaining:
|
|
76
|
+
output_limit.set()
|
|
77
|
+
stop()
|
|
78
|
+
break
|
|
79
|
+
|
|
80
|
+
reader = threading.Thread(target=read_output, daemon=True)
|
|
81
|
+
previous = None
|
|
82
|
+
|
|
83
|
+
def interrupt(signum: int, frame: FrameType | None) -> None:
|
|
84
|
+
raise KeyboardInterrupt
|
|
85
|
+
|
|
86
|
+
if threading.current_thread() is threading.main_thread():
|
|
87
|
+
previous = signal.signal(signal.SIGTERM, interrupt)
|
|
88
|
+
try:
|
|
89
|
+
process = subprocess.Popen(argv, cwd=cwd, env=env, stdout=subprocess.PIPE,
|
|
90
|
+
stderr=subprocess.STDOUT, start_new_session=os.name == "posix")
|
|
91
|
+
reader.start()
|
|
92
|
+
try:
|
|
93
|
+
process.wait(timeout=max(0., started + timeout_s - time.monotonic()))
|
|
94
|
+
timed_out = time.monotonic() >= started + timeout_s
|
|
95
|
+
except subprocess.TimeoutExpired:
|
|
96
|
+
timed_out = True
|
|
97
|
+
stop()
|
|
98
|
+
reap()
|
|
99
|
+
finally:
|
|
100
|
+
if process is not None and process.poll() is None:
|
|
101
|
+
stop()
|
|
102
|
+
reap()
|
|
103
|
+
if reader.ident is not None:
|
|
104
|
+
reader.join(timeout=1 if cleanup_deadline is None else cleanup_remaining())
|
|
105
|
+
if reader.is_alive():
|
|
106
|
+
stop()
|
|
107
|
+
reader.join(timeout=cleanup_remaining())
|
|
108
|
+
cleanup_timeout |= reader.is_alive()
|
|
109
|
+
if previous is not None:
|
|
110
|
+
signal.signal(signal.SIGTERM, previous)
|
|
111
|
+
assert process is not None
|
|
112
|
+
return {"exit_code": process.returncode, "timeout": timed_out,
|
|
113
|
+
"output_limit": output_limit.is_set(),
|
|
114
|
+
"accepted": process.returncode == 0 and not timed_out and not output_limit.is_set()
|
|
115
|
+
and not cleanup_timeout,
|
|
116
|
+
"cleanup_timeout": cleanup_timeout,
|
|
117
|
+
**({"pid": process.pid} if cleanup_timeout else {}),
|
|
118
|
+
"elapsed_s": round(time.monotonic() - started, 4),
|
|
119
|
+
"output": redact(output.decode("utf-8", errors="replace"))[:12000],
|
|
120
|
+
"output_truncated": output_limit.is_set() or len(output) > 12000}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _spec(store: Store, spec_id: str) -> dict[str, Any]:
|
|
124
|
+
spec = store.get("trial_specs", spec_id)
|
|
125
|
+
try:
|
|
126
|
+
directories = spec["directories"]
|
|
127
|
+
if spec["version"] != 1 or spec["kind"] not in {"command_check", "agent_task"} \
|
|
128
|
+
or set(directories) != {"baseline", "candidate"}:
|
|
129
|
+
raise ValueError
|
|
130
|
+
roots = [Path(value).resolve() for value in directories.values()]
|
|
131
|
+
if roots[0] == roots[1]:
|
|
132
|
+
raise AutError("improve: trial directories must be independent")
|
|
133
|
+
if any(not root.is_dir() for root in roots):
|
|
134
|
+
raise ValueError
|
|
135
|
+
if type(spec["repeats"]) is not int or not 1 <= spec["repeats"] <= 10:
|
|
136
|
+
raise ValueError
|
|
137
|
+
if not 1 <= len(spec["cases"]) <= 50:
|
|
138
|
+
raise ValueError
|
|
139
|
+
ids = set()
|
|
140
|
+
for case in spec["cases"]:
|
|
141
|
+
if not isinstance(case["id"], str) or not case["id"] or case["id"] in ids:
|
|
142
|
+
raise ValueError
|
|
143
|
+
ids.add(case["id"])
|
|
144
|
+
argv = case["argv"]
|
|
145
|
+
if not isinstance(argv, list) or not argv \
|
|
146
|
+
or any(not isinstance(arg, str) or "\0" in arg for arg in argv):
|
|
147
|
+
raise ValueError
|
|
148
|
+
if type(case["timeout_s"]) not in (int, float) or not 0 < case["timeout_s"] <= 300:
|
|
149
|
+
raise ValueError
|
|
150
|
+
if spec["kind"] == "agent_task":
|
|
151
|
+
if not all(spec.get("files", {}).get(name) for name in directories):
|
|
152
|
+
raise ValueError
|
|
153
|
+
accept = case["acceptance_argv"]
|
|
154
|
+
if not isinstance(accept, list) or not accept or any(
|
|
155
|
+
not isinstance(arg, str) or "\0" in arg for arg in accept):
|
|
156
|
+
raise ValueError
|
|
157
|
+
if "{metrics_file}" not in argv or not case["acceptance_files"]:
|
|
158
|
+
raise ValueError
|
|
159
|
+
_acceptance_inputs(case, roots)
|
|
160
|
+
elif "acceptance_files" in case:
|
|
161
|
+
for root in roots:
|
|
162
|
+
_command_acceptance(case, roots, root)
|
|
163
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
164
|
+
raise AutError("improve: invalid trusted trial specification") from exc
|
|
165
|
+
return spec
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _acceptance_inputs(case: dict[str, Any], roots: list[Path], cwd: Path | None = None) -> None:
|
|
169
|
+
for index, arg in enumerate(case["acceptance_argv"]):
|
|
170
|
+
path = Path(arg)
|
|
171
|
+
if not path.is_absolute() and cwd is None:
|
|
172
|
+
continue
|
|
173
|
+
path = ((cwd or Path.cwd()) / path).resolve()
|
|
174
|
+
if path.is_file() and not (
|
|
175
|
+
index == 0 and path.resolve() == Path(sys.executable).resolve()
|
|
176
|
+
) and path not in {Path(p).resolve() for p in case["acceptance_files"]}:
|
|
177
|
+
raise AutError("improve: acceptance command input is not frozen")
|
|
178
|
+
for name, expected in case["acceptance_files"].items():
|
|
179
|
+
path = Path(name)
|
|
180
|
+
if not path.is_absolute() or any(path.resolve().is_relative_to(root) for root in roots):
|
|
181
|
+
raise AutError("improve: acceptance inputs must stay outside agent templates")
|
|
182
|
+
if not path.is_file() or file_digest(path) != expected:
|
|
183
|
+
raise AutError("improve: acceptance input changed")
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _command_acceptance(case: dict[str, Any], roots: list[Path], cwd: Path) -> None:
|
|
187
|
+
if "acceptance_files" not in case:
|
|
188
|
+
return
|
|
189
|
+
if not isinstance(case["acceptance_files"], dict) or not case["acceptance_files"]:
|
|
190
|
+
raise AutError("improve: command acceptance files must be a nonempty hash map")
|
|
191
|
+
_acceptance_inputs({"acceptance_argv": case["argv"],
|
|
192
|
+
"acceptance_files": case["acceptance_files"]}, roots, cwd)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _agent_run(store: Store, spec: dict[str, Any], case: dict[str, Any], condition: str,
|
|
196
|
+
key: str, deadline: float) -> dict[str, Any]:
|
|
197
|
+
source = Path(spec["directories"][condition]).resolve()
|
|
198
|
+
directory = store.root / "agent_runs" / key / uuid.uuid4().hex
|
|
199
|
+
workspace = directory / "workspace"
|
|
200
|
+
workspace.mkdir(mode=0o700, parents=True)
|
|
201
|
+
for name, expected in spec["files"][condition].items():
|
|
202
|
+
src, target = (source / name).resolve(), workspace / name
|
|
203
|
+
if not src.is_relative_to(source) or not target.resolve().is_relative_to(workspace):
|
|
204
|
+
raise AutError("improve: agent fixture path escapes its workspace")
|
|
205
|
+
if file_digest(src) != expected:
|
|
206
|
+
raise AutError("improve: agent fixture changed")
|
|
207
|
+
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
208
|
+
shutil.copyfile(src, target)
|
|
209
|
+
target.chmod(src.stat().st_mode & 0o700)
|
|
210
|
+
metrics_path = directory / "metrics.json"
|
|
211
|
+
argv = [str(metrics_path) if arg == "{metrics_file}" else arg for arg in case["argv"]]
|
|
212
|
+
result = command(argv, workspace, min(case["timeout_s"], max(.01, deadline-time.monotonic())))
|
|
213
|
+
execution_id = store.put("agent_executions", {"workspace": str(workspace), **result})
|
|
214
|
+
acceptance = None
|
|
215
|
+
metrics = None
|
|
216
|
+
error = None
|
|
217
|
+
try:
|
|
218
|
+
roots = [Path(p).resolve() for p in spec["directories"].values()]
|
|
219
|
+
_acceptance_inputs(case, roots, directory)
|
|
220
|
+
accept_argv = [str(workspace) if arg == "{workspace}" else arg
|
|
221
|
+
for arg in case["acceptance_argv"]]
|
|
222
|
+
acceptance = command(accept_argv, directory,
|
|
223
|
+
min(30, max(.01, deadline-time.monotonic())), source_pythonpath=False)
|
|
224
|
+
_acceptance_inputs(case, roots, directory)
|
|
225
|
+
if metrics_path.is_file():
|
|
226
|
+
if metrics_path.stat().st_size > 16384 or metrics_path.is_symlink():
|
|
227
|
+
raise ValueError
|
|
228
|
+
metrics = json.loads(metrics_path.read_text())
|
|
229
|
+
if set(metrics) != {"tool_calls", "interventions"} or any(
|
|
230
|
+
type(value) is not int or value < 0 for value in metrics.values()):
|
|
231
|
+
raise ValueError
|
|
232
|
+
else:
|
|
233
|
+
raise ValueError("missing agent metrics")
|
|
234
|
+
except (AutError, OSError, ValueError, TypeError, AttributeError) as exc:
|
|
235
|
+
error = redact(str(exc)) or "invalid captured agent metrics"
|
|
236
|
+
metrics = None
|
|
237
|
+
return {**result, "execution_id": execution_id, "workspace": str(workspace),
|
|
238
|
+
"acceptance": acceptance, "metrics": metrics, "validation_error": error,
|
|
239
|
+
"accepted": bool(result["accepted"] and acceptance and acceptance["accepted"]
|
|
240
|
+
and metrics is not None and error is None)}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def run_trial(store: Store, spec_id: str, *, execute: bool = False,
|
|
244
|
+
budget_s: int = 600) -> dict[str, Any]:
|
|
245
|
+
spec = _spec(store, spec_id)
|
|
246
|
+
if not 1 <= budget_s <= 3600:
|
|
247
|
+
raise AutError("improve: trial budget must be 1..3600 seconds")
|
|
248
|
+
directories = spec["directories"]
|
|
249
|
+
results = {row["evaluation_key"]: row for key in store.keys("trial_results")
|
|
250
|
+
if (row := store.get("trial_results", key))["spec_id"] == spec_id}
|
|
251
|
+
total = 2 * spec["repeats"] * len(spec["cases"])
|
|
252
|
+
executed = 0
|
|
253
|
+
deadline = time.monotonic() + budget_s
|
|
254
|
+
if execute:
|
|
255
|
+
with _locked(store.root / "trial", blocking=False):
|
|
256
|
+
for condition, root_name in directories.items():
|
|
257
|
+
root = Path(root_name).resolve()
|
|
258
|
+
for relative, expected in spec.get("files", {}).get(condition, {}).items():
|
|
259
|
+
target = (root / relative).resolve()
|
|
260
|
+
if not target.is_relative_to(root) or file_digest(target) != expected:
|
|
261
|
+
raise AutError("improve: trial input changed; previous results are preserved")
|
|
262
|
+
for repeat in range(spec["repeats"]):
|
|
263
|
+
for case in spec["cases"]:
|
|
264
|
+
order = ["baseline", "candidate"]
|
|
265
|
+
random.Random(f"{spec.get('seed', 0)}:{repeat}:{case['id']}").shuffle(order)
|
|
266
|
+
for condition in order:
|
|
267
|
+
identity = {"spec_id": spec_id, "repeat": repeat,
|
|
268
|
+
"case_id": case["id"], "condition": condition}
|
|
269
|
+
key = digest(identity)
|
|
270
|
+
if key in results:
|
|
271
|
+
continue
|
|
272
|
+
remaining = deadline - time.monotonic()
|
|
273
|
+
if remaining <= 0:
|
|
274
|
+
break
|
|
275
|
+
root = Path(directories[condition]).resolve()
|
|
276
|
+
for relative, expected in spec.get("files", {}).get(condition, {}).items():
|
|
277
|
+
target = (root / relative).resolve()
|
|
278
|
+
if not target.is_relative_to(root) or file_digest(target) != expected:
|
|
279
|
+
raise AutError("improve: trial input changed; prepare a new specification")
|
|
280
|
+
roots = [Path(name).resolve() for name in directories.values()]
|
|
281
|
+
if spec["kind"] == "command_check":
|
|
282
|
+
_command_acceptance(case, roots, root)
|
|
283
|
+
outcome = (_agent_run(store, spec, case, condition, key, deadline)
|
|
284
|
+
if spec["kind"] == "agent_task" else
|
|
285
|
+
command(case["argv"], root, min(case["timeout_s"], remaining)))
|
|
286
|
+
validation_error = None
|
|
287
|
+
if spec["kind"] == "command_check" and "acceptance_files" in case:
|
|
288
|
+
try:
|
|
289
|
+
_command_acceptance(case, roots, root)
|
|
290
|
+
except (AutError, OSError) as exc:
|
|
291
|
+
validation_error = redact(str(exc))
|
|
292
|
+
outcome.update(accepted=False, validation_error=validation_error)
|
|
293
|
+
result = {**identity, "evaluation_key": key, **outcome}
|
|
294
|
+
result_id = store.put("trial_results", result)
|
|
295
|
+
results[key] = result
|
|
296
|
+
executed += 1
|
|
297
|
+
if validation_error is not None:
|
|
298
|
+
store.append("trial_acceptance_rejected", spec_id=spec_id,
|
|
299
|
+
result_id=result_id, validation_error=validation_error)
|
|
300
|
+
raise AutError("improve: protected command acceptance changed; "
|
|
301
|
+
f"rejected result {result_id}")
|
|
302
|
+
report = {"spec_id": spec_id, "complete": len(results) == total,
|
|
303
|
+
"finished": len(results), "total": total, "executed": executed,
|
|
304
|
+
"accepted": {condition: sum(row["condition"] == condition and row["accepted"]
|
|
305
|
+
for row in results.values()) for condition in directories},
|
|
306
|
+
"timeouts": sum(row["timeout"] for row in results.values()),
|
|
307
|
+
"trial_kind": spec["kind"], "agent_behaviour_verified": False}
|
|
308
|
+
if spec["kind"] == "agent_task":
|
|
309
|
+
report["conditions"] = {name: {
|
|
310
|
+
"runs": len(rows := [r for r in results.values() if r["condition"] == name]),
|
|
311
|
+
"median_elapsed_s": statistics.median([r["elapsed_s"] for r in rows]) if rows else None,
|
|
312
|
+
"tool_calls": sum(r["metrics"]["tool_calls"] for r in rows if r.get("metrics")),
|
|
313
|
+
"interventions": sum(r["metrics"]["interventions"] for r in rows if r.get("metrics")),
|
|
314
|
+
"missing_metrics": sum(r.get("metrics") is None for r in rows),
|
|
315
|
+
} for name in directories}
|
|
316
|
+
report["improvement_established"] = False
|
|
317
|
+
store.append("trial", **report)
|
|
318
|
+
return report
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Compare frozen judge decisions with explicit, independently executable references."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ..config import AutError
|
|
14
|
+
from . import rubric
|
|
15
|
+
from .judge import LABELS, evaluation_key, evaluator
|
|
16
|
+
from .proposals import file_digest
|
|
17
|
+
from .store import Store, digest
|
|
18
|
+
from .trial import command
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def evidence_key(case: dict[str, Any]) -> str:
|
|
22
|
+
return digest({name: case[name] for name in (
|
|
23
|
+
"events", "coverage", "omitted_task_events")})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _oracle_inputs(spec: dict[str, Any]) -> None:
|
|
27
|
+
for index, arg in enumerate(spec["oracle_argv"]):
|
|
28
|
+
path = (Path(spec["oracle_cwd"]) / arg).resolve()
|
|
29
|
+
if path.is_file() and not (
|
|
30
|
+
index == 0 and path.resolve() == Path(sys.executable).resolve()
|
|
31
|
+
) and path not in {Path(p).resolve() for p in spec["oracle_files"]}:
|
|
32
|
+
raise AutError("improve: reference oracle command input is not frozen")
|
|
33
|
+
for name, expected in spec["oracle_files"].items():
|
|
34
|
+
path = Path(name)
|
|
35
|
+
if not path.is_absolute() or not path.is_file() or file_digest(path) != expected:
|
|
36
|
+
raise AutError("improve: reference oracle input changed")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def validate(store: Store, spec_id: str, diagnosis_id: str, *,
|
|
40
|
+
execute: bool = False) -> dict[str, Any]:
|
|
41
|
+
spec = store.get("reference_specs", spec_id)
|
|
42
|
+
policy = store.get("diagnoses", diagnosis_id)
|
|
43
|
+
try:
|
|
44
|
+
references = spec["references"]
|
|
45
|
+
argv = spec["oracle_argv"]
|
|
46
|
+
timeout = spec["timeout_s"]
|
|
47
|
+
if spec["version"] != 1 or spec["split"] not in {"development", "holdout"} \
|
|
48
|
+
or not isinstance(references, list) or not 1 <= len(references) <= 200 \
|
|
49
|
+
or not isinstance(argv, list) or not argv \
|
|
50
|
+
or any(not isinstance(a, str) or "\0" in a for a in argv) \
|
|
51
|
+
or type(timeout) not in (int, float) or not math.isfinite(timeout) \
|
|
52
|
+
or not 0 < timeout <= 300 or not spec["oracle_files"] \
|
|
53
|
+
or "{reference_input}" not in argv:
|
|
54
|
+
raise ValueError
|
|
55
|
+
identities = set()
|
|
56
|
+
# A policy names its rubrics; one without any predates them and had only the two core ones.
|
|
57
|
+
known = policy.get("rubrics")
|
|
58
|
+
if not isinstance(known, dict):
|
|
59
|
+
known = set(rubric.LEGACY_V1)
|
|
60
|
+
for ref in references:
|
|
61
|
+
if ref["rubric"] not in known or ref["expected"] not in LABELS \
|
|
62
|
+
or not re.fullmatch(r"[a-f0-9]{64}", ref["evidence_sha256"]):
|
|
63
|
+
raise ValueError
|
|
64
|
+
identity = ref["evidence_sha256"], ref["rubric"]
|
|
65
|
+
if identity in identities:
|
|
66
|
+
raise ValueError
|
|
67
|
+
identities.add(identity)
|
|
68
|
+
cwd = Path(spec["oracle_cwd"])
|
|
69
|
+
if not cwd.is_absolute() or not cwd.is_dir():
|
|
70
|
+
raise ValueError
|
|
71
|
+
_oracle_inputs(spec)
|
|
72
|
+
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
|
73
|
+
raise AutError("improve: invalid executable reference specification") from exc
|
|
74
|
+
cases: dict[str, str] = {}
|
|
75
|
+
for case_id in policy["cases"]:
|
|
76
|
+
key = evidence_key(store.get("cases", case_id))
|
|
77
|
+
if key in cases:
|
|
78
|
+
raise AutError("improve: ambiguous reference evidence in diagnosis")
|
|
79
|
+
cases[key] = case_id
|
|
80
|
+
if any(ref["evidence_sha256"] not in cases for ref in references):
|
|
81
|
+
raise AutError("improve: reference evidence is absent from this diagnosis")
|
|
82
|
+
if not execute:
|
|
83
|
+
return {"spec_id": spec_id, "diagnosis_id": diagnosis_id,
|
|
84
|
+
"execution_required": True, "references": len(references), "split": spec["split"]}
|
|
85
|
+
payload = {"references": [{"evidence_sha256": ref["evidence_sha256"],
|
|
86
|
+
"rubric": ref["rubric"], "case": store.get("cases", cases[ref["evidence_sha256"]])}
|
|
87
|
+
for ref in references]}
|
|
88
|
+
input_id = store.put("reference_inputs", payload)
|
|
89
|
+
input_path = store.path("reference_inputs", input_id)
|
|
90
|
+
argv = [str(input_path) if arg == "{reference_input}" else arg for arg in argv]
|
|
91
|
+
oracle = command(argv, cwd, timeout, source_pythonpath=False)
|
|
92
|
+
oracle_id = store.put("reference_oracles", {"spec_id": spec_id, **oracle})
|
|
93
|
+
_oracle_inputs(spec)
|
|
94
|
+
store.get("reference_inputs", input_id)
|
|
95
|
+
try:
|
|
96
|
+
if not oracle["accepted"] or oracle["output_truncated"]:
|
|
97
|
+
raise ValueError
|
|
98
|
+
observed = json.loads(oracle["output"])
|
|
99
|
+
# Short digest parts survive text redaction without exempting arbitrary long tokens.
|
|
100
|
+
bound = [{"evidence_sha256": "".join(row["evidence_sha256_parts"]),
|
|
101
|
+
"rubric": row["rubric"], "expected": row["expected"]}
|
|
102
|
+
for row in observed["references"]]
|
|
103
|
+
if bound != references:
|
|
104
|
+
raise ValueError
|
|
105
|
+
except (ValueError, TypeError, KeyError) as exc:
|
|
106
|
+
raise AutError(f"improve: reference oracle failed or disagreed; record {oracle_id}") from exc
|
|
107
|
+
judgments = {value["evaluation_key"]: (key, value) for key in store.keys("judgments")
|
|
108
|
+
if (value := store.get("judgments", key))}
|
|
109
|
+
counts: Counter[str] = Counter()
|
|
110
|
+
rows = []
|
|
111
|
+
for ref in references:
|
|
112
|
+
key = evaluation_key(evaluator(policy, ref["rubric"]), cases[ref["evidence_sha256"]],
|
|
113
|
+
ref["rubric"])
|
|
114
|
+
if key not in judgments:
|
|
115
|
+
counts["missing"] += 1
|
|
116
|
+
rows.append({**ref, "observed": None, "judgment_id": None})
|
|
117
|
+
continue
|
|
118
|
+
judgment_id, judgment = judgments[key]
|
|
119
|
+
verdict = judgment["verdict"]
|
|
120
|
+
if verdict not in LABELS:
|
|
121
|
+
raise AutError("improve: reference judgment has an invalid verdict")
|
|
122
|
+
expected = ref["expected"]
|
|
123
|
+
counts[f"reference_{expected}"] += 1
|
|
124
|
+
counts["abstentions"] += verdict == "unknown"
|
|
125
|
+
counts["detected_failures"] += expected == "fail" and verdict == "fail"
|
|
126
|
+
counts["missed_failures"] += expected == "fail" and verdict != "fail"
|
|
127
|
+
counts["false_alarms"] += expected == "pass" and verdict == "fail"
|
|
128
|
+
counts["supported_passes"] += expected == "pass" and verdict == "pass"
|
|
129
|
+
counts["decisions_on_unknown"] += expected == "unknown" and verdict != "unknown"
|
|
130
|
+
rows.append({**ref, "observed": verdict, "judgment_id": judgment_id})
|
|
131
|
+
|
|
132
|
+
def rate(numerator: str, denominator: str) -> float | None:
|
|
133
|
+
return counts[numerator] / counts[denominator] if counts[denominator] \
|
|
134
|
+
and not counts["missing"] else None
|
|
135
|
+
|
|
136
|
+
report = {"spec_id": spec_id, "diagnosis_id": diagnosis_id, "oracle_id": oracle_id,
|
|
137
|
+
"input_id": input_id,
|
|
138
|
+
"complete": not counts["missing"], "split": spec["split"], "counts": dict(counts),
|
|
139
|
+
"failure_recall": rate("detected_failures", "reference_fail"),
|
|
140
|
+
"false_alarm_rate": rate("false_alarms", "reference_pass"),
|
|
141
|
+
"pass_recall": rate("supported_passes", "reference_pass"), "cases": rows,
|
|
142
|
+
"scope": "executable references only; not real-work accuracy",
|
|
143
|
+
"human_verified": False, "automation_qualified": False,
|
|
144
|
+
"holdout_independence_verified": False}
|
|
145
|
+
key = store.put("reference_results", report)
|
|
146
|
+
store.append("reference_validation", report_id=key, diagnosis_id=diagnosis_id,
|
|
147
|
+
spec_id=spec_id, complete=report["complete"])
|
|
148
|
+
return {"report_id": key, **report}
|