agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""Private content-addressed evidence and an append-only progress journal."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import subprocess
|
|
10
|
+
import time
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ..collectors import claude_code, codex
|
|
17
|
+
from ..config import AutError
|
|
18
|
+
from ..event import CollectorUnavailable
|
|
19
|
+
from ..git_safe import git_argv, git_env
|
|
20
|
+
from ..state import _locked, atomic_write_text
|
|
21
|
+
from .evidence import read_session
|
|
22
|
+
from . import t3
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def canonical(value: Any) -> str:
|
|
26
|
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def digest(value: Any) -> str:
|
|
30
|
+
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Store:
|
|
34
|
+
def __init__(self, root: Path) -> None:
|
|
35
|
+
self.root = root.expanduser().resolve()
|
|
36
|
+
marker = self.root / ".aut-improve"
|
|
37
|
+
if self.root.exists() and any(self.root.iterdir()) and not marker.is_file():
|
|
38
|
+
raise AutError("improve: refusing a nonempty directory without its private marker")
|
|
39
|
+
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
40
|
+
os.chmod(self.root, 0o700)
|
|
41
|
+
if marker.exists() and marker.read_text() != "1\n":
|
|
42
|
+
raise AutError("improve: unsupported private store version")
|
|
43
|
+
if not marker.exists():
|
|
44
|
+
atomic_write_text(marker, "1\n")
|
|
45
|
+
atomic_write_text(self.root / ".gitignore", "*\n")
|
|
46
|
+
|
|
47
|
+
def path(self, kind: str, key: str) -> Path:
|
|
48
|
+
if not re.fullmatch(r"[a-z_]{1,40}", kind) or not re.fullmatch(r"[a-f0-9]{64}", key):
|
|
49
|
+
raise AutError("improve: invalid record identity")
|
|
50
|
+
path = self.root / kind / f"{key}.json"
|
|
51
|
+
if not path.resolve().is_relative_to(self.root):
|
|
52
|
+
raise AutError("improve: record path escapes its private store")
|
|
53
|
+
return path
|
|
54
|
+
|
|
55
|
+
def put(self, kind: str, value: dict[str, Any]) -> str:
|
|
56
|
+
try:
|
|
57
|
+
if not isinstance(value, dict):
|
|
58
|
+
raise ValueError("record is not an object")
|
|
59
|
+
key = digest(value)
|
|
60
|
+
except (TypeError, ValueError) as exc:
|
|
61
|
+
raise AutError("improve: record must be a finite JSON object") from exc
|
|
62
|
+
path = self.path(kind, key)
|
|
63
|
+
with _locked(self.root / "write"):
|
|
64
|
+
if path.exists():
|
|
65
|
+
self.get(kind, key)
|
|
66
|
+
else:
|
|
67
|
+
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
68
|
+
atomic_write_text(path, canonical(value) + "\n")
|
|
69
|
+
return key
|
|
70
|
+
|
|
71
|
+
def get(self, kind: str, key: str) -> dict[str, Any]:
|
|
72
|
+
path = self.path(kind, key)
|
|
73
|
+
try:
|
|
74
|
+
value = json.loads(path.read_text())
|
|
75
|
+
valid = isinstance(value, dict) and digest(value) == key
|
|
76
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
77
|
+
raise AutError("improve: private record is missing or invalid") from exc
|
|
78
|
+
if not valid:
|
|
79
|
+
raise AutError("improve: private record changed; refusing to continue")
|
|
80
|
+
return value
|
|
81
|
+
|
|
82
|
+
def keys(self, kind: str) -> list[str]:
|
|
83
|
+
self.path(kind, "0" * 64)
|
|
84
|
+
return sorted(p.stem for p in (self.root / kind).glob("*.json"))
|
|
85
|
+
|
|
86
|
+
def append(self, kind: str, **attrs: Any) -> None:
|
|
87
|
+
row = {"at": datetime.now(timezone.utc).isoformat(), "kind": kind, **attrs}
|
|
88
|
+
with _locked(self.root / "journal"):
|
|
89
|
+
fd = os.open(self.root / "events.jsonl", os.O_CREAT | os.O_APPEND | os.O_WRONLY, 0o600)
|
|
90
|
+
with os.fdopen(fd, "a", encoding="utf-8") as stream:
|
|
91
|
+
stream.write(canonical(row) + "\n")
|
|
92
|
+
stream.flush()
|
|
93
|
+
os.fsync(stream.fileno())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def repository(path: Path) -> str | None:
|
|
97
|
+
try:
|
|
98
|
+
result = subprocess.run(git_argv(str(path), "rev-parse", "--path-format=absolute",
|
|
99
|
+
"--git-common-dir"),
|
|
100
|
+
capture_output=True, text=True, timeout=5, env=git_env())
|
|
101
|
+
return str(Path(result.stdout.strip()).resolve()) if result.returncode == 0 else None
|
|
102
|
+
except (OSError, subprocess.SubprocessError):
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _source_session(path: Path, agent: str, budget: int) -> str | None:
|
|
107
|
+
try:
|
|
108
|
+
with path.open("rb") as stream:
|
|
109
|
+
record = json.loads(stream.readline(min(budget, 65536)))
|
|
110
|
+
meta = record.get("payload", {}) if agent == "codex" else record
|
|
111
|
+
session = (meta.get("id") or meta.get("session_id")) if agent == "codex" \
|
|
112
|
+
else meta.get("sessionId", path.stem)
|
|
113
|
+
return session if isinstance(session, str) else None
|
|
114
|
+
except (OSError, ValueError, AttributeError):
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def audit(store: Store, home: Path, *, days: int = 7, limit: int = 40,
|
|
119
|
+
project: Path | None = None, idle_s: int = 120,
|
|
120
|
+
since_epoch: float | None = None, max_bytes: int = 16 * 1024 * 1024) -> dict[str, Any]:
|
|
121
|
+
if not 1 <= days <= 90 or not 1 <= limit <= 200 or idle_s < 0:
|
|
122
|
+
raise AutError("improve: days must be 1..90 and session limit 1..200")
|
|
123
|
+
now = time.time()
|
|
124
|
+
since = since_epoch if since_epoch is not None else now - days * 86400
|
|
125
|
+
files: list[tuple[float, str, Path]] = []
|
|
126
|
+
counts: Counter[str] = Counter()
|
|
127
|
+
for agent, discover in (("claude_code", claude_code.main_session_files),
|
|
128
|
+
("codex", codex.session_files)):
|
|
129
|
+
try:
|
|
130
|
+
discovered = discover(home)
|
|
131
|
+
except (OSError, CollectorUnavailable):
|
|
132
|
+
counts["unreadable_sources"] += 1
|
|
133
|
+
continue
|
|
134
|
+
for name in discovered:
|
|
135
|
+
path = Path(name)
|
|
136
|
+
try:
|
|
137
|
+
stamp = path.stat().st_mtime
|
|
138
|
+
if stamp >= since:
|
|
139
|
+
if stamp > now - idle_s:
|
|
140
|
+
counts["active_sources"] += 1
|
|
141
|
+
else:
|
|
142
|
+
files.append((stamp, agent, path))
|
|
143
|
+
except OSError:
|
|
144
|
+
counts["unreadable_sources"] += 1
|
|
145
|
+
sessions: list[str] = []
|
|
146
|
+
imports = codex.imported_until(home)
|
|
147
|
+
seen: set[tuple[str, str]] = set()
|
|
148
|
+
identities: dict[tuple[str, str], str] = {}
|
|
149
|
+
copies: dict[tuple[str, str], list[str]] = {}
|
|
150
|
+
project_repo = repository(project) if project else None
|
|
151
|
+
if project and project_repo is None:
|
|
152
|
+
raise AutError("improve: project must be a readable local Git repository")
|
|
153
|
+
t3_rows, t3_counts = t3.snapshots(home, since=since, limit=limit,
|
|
154
|
+
idle_s=idle_s, max_bytes=max_bytes)
|
|
155
|
+
counts.update(t3_counts)
|
|
156
|
+
mirrors: dict[tuple[str, str], str] = {}
|
|
157
|
+
supplement_reads = 0
|
|
158
|
+
# App threads join provider handoffs; native mirrors remain referenced, not counted twice.
|
|
159
|
+
candidates: list[tuple[float, str, Path | dict[str, Any]]] = [
|
|
160
|
+
(0, "t3", row) for row in t3_rows]
|
|
161
|
+
candidates.extend(sorted(files, reverse=True))
|
|
162
|
+
for _stamp, agent, candidate in candidates:
|
|
163
|
+
if len(sessions) >= limit:
|
|
164
|
+
linked_identity = (agent, _source_session(candidate, agent, max_bytes)) \
|
|
165
|
+
if isinstance(candidate, Path) else None
|
|
166
|
+
if linked_identity not in mirrors or supplement_reads >= 2 * limit:
|
|
167
|
+
counts["unread_sources"] += 1
|
|
168
|
+
continue
|
|
169
|
+
supplement_reads += 1
|
|
170
|
+
try:
|
|
171
|
+
evidence = candidate if isinstance(candidate, dict) else read_session(
|
|
172
|
+
candidate, agent, max_bytes=max_bytes, since_epoch=since,
|
|
173
|
+
imports=imports, exclude_automated=True, recent=True)
|
|
174
|
+
except (OSError, ValueError, TypeError, OverflowError):
|
|
175
|
+
counts["unreadable_sources"] += 1
|
|
176
|
+
continue
|
|
177
|
+
if evidence is None:
|
|
178
|
+
counts["automated_sources"] += 1
|
|
179
|
+
continue
|
|
180
|
+
if (agent, evidence["session_id"]) in mirrors:
|
|
181
|
+
source = (agent, evidence["source"]["sha256"])
|
|
182
|
+
if source in seen:
|
|
183
|
+
counts["duplicates"] += 1
|
|
184
|
+
continue
|
|
185
|
+
seen.add(source)
|
|
186
|
+
primary_key = mirrors[(agent, evidence["session_id"])]
|
|
187
|
+
supplement = store.put("sessions", evidence)
|
|
188
|
+
primary = store.get("sessions", primary_key)
|
|
189
|
+
linked = set(primary.get("supplemental_sources", []))
|
|
190
|
+
linked.add(supplement)
|
|
191
|
+
primary["supplemental_sources"] = sorted(linked)
|
|
192
|
+
primary["source_family_link"] = "Native linkage is not proof of equivalent coverage."
|
|
193
|
+
updated_key = store.put("sessions", primary)
|
|
194
|
+
sessions[sessions.index(primary_key)] = updated_key
|
|
195
|
+
mirrors = {pair: updated_key if key == primary_key else key
|
|
196
|
+
for pair, key in mirrors.items()}
|
|
197
|
+
copies[("t3", primary["session_id"])] = [updated_key]
|
|
198
|
+
counts["t3_native_mirrors"] += 1
|
|
199
|
+
continue
|
|
200
|
+
if evidence["origin"] in {"subagent", "automated"}:
|
|
201
|
+
counts["automated_sources"] += 1
|
|
202
|
+
continue
|
|
203
|
+
if not evidence["events"]:
|
|
204
|
+
counts["no_recent_evidence"] += 1
|
|
205
|
+
continue
|
|
206
|
+
if project and (not evidence["cwd"] or repository(Path(evidence["cwd"])) != project_repo):
|
|
207
|
+
counts["other_projects"] += 1
|
|
208
|
+
continue
|
|
209
|
+
source = (agent, evidence["source"]["sha256"])
|
|
210
|
+
if source in seen:
|
|
211
|
+
counts["duplicates"] += 1
|
|
212
|
+
continue
|
|
213
|
+
seen.add(source)
|
|
214
|
+
identity = (agent, evidence["session_id"])
|
|
215
|
+
if identity in identities:
|
|
216
|
+
evidence["coverage"]["divergent_copy"] = True
|
|
217
|
+
evidence["coverage"]["complete"] = False
|
|
218
|
+
counts["divergent_copies"] += 1
|
|
219
|
+
identities[identity] = source[1]
|
|
220
|
+
key = store.put("sessions", evidence)
|
|
221
|
+
sessions.append(key)
|
|
222
|
+
if agent == "t3":
|
|
223
|
+
mirrors.update((tuple(pair), key) for pair in evidence.get("native_sessions", []))
|
|
224
|
+
copies.setdefault(identity, []).append(key)
|
|
225
|
+
coverage = evidence["coverage"]
|
|
226
|
+
counts["complete_sessions"] += bool(coverage["complete"])
|
|
227
|
+
for name in ("calls", "matched_results", "unmatched_calls", "orphan_results",
|
|
228
|
+
"truncated_events", "malformed_records", "byte_cap_reached"):
|
|
229
|
+
counts[name] += coverage.get(name, 0)
|
|
230
|
+
counts["error_results"] += sum(e["kind"] == "result" and e["status"] == "error"
|
|
231
|
+
for e in evidence["events"])
|
|
232
|
+
ambiguous = sorted(key for group in copies.values() if len(group) > 1 for key in group)
|
|
233
|
+
counts["complete_sessions"] = sum(store.get("sessions", key)["coverage"]["complete"]
|
|
234
|
+
and key not in ambiguous for key in sessions)
|
|
235
|
+
manifest = {"version": 4, "session_ids": sorted(sessions),
|
|
236
|
+
"ambiguous_session_ids": ambiguous, "coverage": dict(counts),
|
|
237
|
+
"project": str(project.resolve()) if project else None,
|
|
238
|
+
"source_policy": {"days": days, "limit": limit, "max_bytes": max_bytes,
|
|
239
|
+
"idle_s": idle_s, "agents": ["t3", "claude_code", "codex"],
|
|
240
|
+
"additional_native_reads": 2 * limit,
|
|
241
|
+
"preference": "T3 app thread before native provider mirrors"}}
|
|
242
|
+
key = store.put("audits", manifest)
|
|
243
|
+
store.append("audit", audit_id=key, sessions=len(sessions), coverage=dict(counts))
|
|
244
|
+
return {"audit_id": key, "sessions": len(sessions), **counts}
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Freeze prospective policies and report bounded evidence without inferring causality."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..config import AutError
|
|
10
|
+
from .store import Store, digest
|
|
11
|
+
from .study_decision import measure, observed_experiment
|
|
12
|
+
from .trial import _spec
|
|
13
|
+
from .validation import evidence_key
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def fields(value: Any, names: str) -> None:
|
|
17
|
+
if not isinstance(value, dict) or set(value) != set(names.split()):
|
|
18
|
+
raise AutError("improve: invalid study fields")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def strings(value: Any, *, empty: bool = False) -> None:
|
|
22
|
+
if not isinstance(value, list) or (not value and not empty) or any(
|
|
23
|
+
not isinstance(item, str) or not item.strip() for item in value) \
|
|
24
|
+
or len(value) != len(set(value)):
|
|
25
|
+
raise AutError("improve: expected distinct nonempty strings")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def identity(store: Store, case_id: str) -> dict[str, str]:
|
|
29
|
+
case = store.get("cases", case_id)
|
|
30
|
+
session = store.get("sessions", case["session_snapshot"])
|
|
31
|
+
# Copies can change event ids, line numbers and snapshot coverage without changing the task.
|
|
32
|
+
content = [{k: v for k, v in event.items() if k not in {"id", "line", "byte_offset", "timestamp"}}
|
|
33
|
+
for event in case["events"]]
|
|
34
|
+
return {"session": digest([session["agent"], session["session_id"]]),
|
|
35
|
+
"content": digest(content), "evidence": evidence_key(case)}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def shortlist(store: Store, audit_id: str) -> dict[str, Any]:
|
|
39
|
+
audit = store.get("audits", audit_id)
|
|
40
|
+
incidents: dict[str, dict[str, Any]] = {}
|
|
41
|
+
sessions = set()
|
|
42
|
+
for snapshot in audit["session_ids"]:
|
|
43
|
+
session = store.get("sessions", snapshot)
|
|
44
|
+
source = digest([session["agent"], session["session_id"]])
|
|
45
|
+
sessions.add(source)
|
|
46
|
+
for event in session["events"]:
|
|
47
|
+
if event["kind"] != "result" or event.get("status") != "error":
|
|
48
|
+
continue
|
|
49
|
+
key = digest([source, event["id"]])
|
|
50
|
+
if key in incidents:
|
|
51
|
+
incidents[key]["snapshots"].append(snapshot)
|
|
52
|
+
continue
|
|
53
|
+
incidents[key] = {"incident_id": key, "source_session": source,
|
|
54
|
+
"event_id": event["id"], "snapshots": [snapshot],
|
|
55
|
+
"signal": digest([event.get("tool"), event.get("text")])}
|
|
56
|
+
counts = Counter(row["signal"] for row in incidents.values())
|
|
57
|
+
return {"audit_id": audit_id, "distinct_source_sessions": len(sessions),
|
|
58
|
+
"incidents": list(incidents.values()), "signal_counts": dict(counts),
|
|
59
|
+
"scope": "Error groups are investigation signals, not causal findings."}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _manifest(store: Store, value: dict[str, Any]) -> dict[str, Any]:
|
|
63
|
+
fields(value, "version discovery_cases control_cases task_families proposal_id trial_spec_id "
|
|
64
|
+
"primary_metric quality_constraints budget_s provenance reference_result_ids cost_inputs "
|
|
65
|
+
"decision_policy")
|
|
66
|
+
if value["version"] != 1:
|
|
67
|
+
raise AutError("improve: unsupported study version")
|
|
68
|
+
for name in ("discovery_cases", "control_cases", "quality_constraints"):
|
|
69
|
+
strings(value[name])
|
|
70
|
+
strings(value["reference_result_ids"], empty=True)
|
|
71
|
+
ids = value["discovery_cases"] + value["control_cases"]
|
|
72
|
+
if len(ids) != len(set(ids)):
|
|
73
|
+
raise AutError("improve: duplicate study case")
|
|
74
|
+
families = value["task_families"]
|
|
75
|
+
if not isinstance(families, dict) or not set(ids) <= set(families) or any(
|
|
76
|
+
not isinstance(v, str) or not v.strip() for v in families.values()):
|
|
77
|
+
raise AutError("improve: every case needs a task family")
|
|
78
|
+
identities = [identity(store, key) for key in ids]
|
|
79
|
+
for key in families:
|
|
80
|
+
identity(store, key)
|
|
81
|
+
if len({row["content"] for row in identities}) != len(ids):
|
|
82
|
+
raise AutError("improve: copied case evidence")
|
|
83
|
+
discovery = [identity(store, key) for key in value["discovery_cases"]]
|
|
84
|
+
controls = [identity(store, key) for key in value["control_cases"]]
|
|
85
|
+
if {r["session"] for r in discovery} & {r["session"] for r in controls}:
|
|
86
|
+
raise AutError("improve: control source sessions overlap discovery")
|
|
87
|
+
proposal = store.get("proposals", value["proposal_id"])
|
|
88
|
+
if proposal["case_id"] not in value["discovery_cases"]:
|
|
89
|
+
raise AutError("improve: proposal must link discovery evidence")
|
|
90
|
+
trial = _spec(store, value["trial_spec_id"])
|
|
91
|
+
if trial["kind"] == "command_check":
|
|
92
|
+
from pathlib import Path
|
|
93
|
+
from .trial import _acceptance_inputs
|
|
94
|
+
for case in trial["cases"]:
|
|
95
|
+
if case.get("acceptance_files"):
|
|
96
|
+
protected = {"acceptance_argv": case["argv"],
|
|
97
|
+
"acceptance_files": case["acceptance_files"]}
|
|
98
|
+
roots = [Path(root).resolve() for root in trial["directories"].values()]
|
|
99
|
+
for root in roots:
|
|
100
|
+
_acceptance_inputs(protected, roots, root)
|
|
101
|
+
policy = value["decision_policy"]
|
|
102
|
+
fields(policy, "direction minimum_improvement require_no_acceptance_regressions "
|
|
103
|
+
"require_all_candidate_accepted")
|
|
104
|
+
if policy["direction"] not in {"increase", "decrease"} or type(
|
|
105
|
+
policy["minimum_improvement"]) not in (int, float) or not (
|
|
106
|
+
0 <= policy["minimum_improvement"] < float("inf")) or any(
|
|
107
|
+
type(policy[name]) is not bool for name in (
|
|
108
|
+
"require_no_acceptance_regressions", "require_all_candidate_accepted")):
|
|
109
|
+
raise AutError("improve: invalid prospective decision policy")
|
|
110
|
+
if value["primary_metric"] not in {"accepted", "elapsed_s", "tool_calls", "interventions"}:
|
|
111
|
+
raise AutError("improve: unsupported primary metric")
|
|
112
|
+
if type(value["budget_s"]) is not int or not 1 <= value["budget_s"] <= 3600:
|
|
113
|
+
raise AutError("improve: study budget must be 1..3600 seconds")
|
|
114
|
+
fields(value["provenance"], "setup model tools instructions")
|
|
115
|
+
if any(v is not None and (not isinstance(v, str) or not v.strip())
|
|
116
|
+
for v in value["provenance"].values()):
|
|
117
|
+
raise AutError("improve: provenance must be an explicit source or null for unknown")
|
|
118
|
+
fields(value["cost_inputs"], "operator_minutes money")
|
|
119
|
+
for v in value["cost_inputs"].values():
|
|
120
|
+
if v is not None and (type(v) not in (int, float) or not 0 <= v < float("inf")):
|
|
121
|
+
raise AutError("improve: cost inputs must be finite nonnegative numbers or null")
|
|
122
|
+
for key in value["reference_result_ids"]:
|
|
123
|
+
store.get("reference_results", key)
|
|
124
|
+
return value
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def create(store: Store, value: dict[str, Any]) -> dict[str, Any]:
|
|
128
|
+
try:
|
|
129
|
+
_manifest(store, value)
|
|
130
|
+
if observed_experiment(store, value["trial_spec_id"]):
|
|
131
|
+
raise AutError("improve: prospective policy cannot include pre-existing trial results")
|
|
132
|
+
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
|
133
|
+
raise AutError("improve: invalid study specification") from exc
|
|
134
|
+
key = store.put("studies", {**value, "known_case_ids": store.keys("cases"),
|
|
135
|
+
"created_at": datetime.now(timezone.utc).isoformat()})
|
|
136
|
+
store.append("study_created", study_id=key)
|
|
137
|
+
return {"study_id": key, "prospective": True}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def references(store: Store, study: dict[str, Any]) -> dict[str, Any]:
|
|
141
|
+
frozen = [identity(store, key) for key in study["discovery_cases"] + study["control_cases"]]
|
|
142
|
+
reports = [store.get("reference_results", key) for key in study["reference_result_ids"]]
|
|
143
|
+
relevant = {row["evidence_sha256"] for report in reports for row in report["cases"]}
|
|
144
|
+
lookup: dict[str, list[dict[str, str]]] = {}
|
|
145
|
+
unattributable: set[str] = set()
|
|
146
|
+
unattributable_cases = 0
|
|
147
|
+
unindexable_cases = 0
|
|
148
|
+
for key in store.keys("cases") if relevant else []:
|
|
149
|
+
case = store.get("cases", key)
|
|
150
|
+
if not {"events", "coverage", "omitted_task_events"} <= case.keys():
|
|
151
|
+
unindexable_cases += 1
|
|
152
|
+
continue
|
|
153
|
+
evidence = evidence_key(case)
|
|
154
|
+
if evidence not in relevant:
|
|
155
|
+
continue
|
|
156
|
+
snapshot = case.get("session_snapshot")
|
|
157
|
+
session = store.get("sessions", snapshot) if snapshot is not None else {}
|
|
158
|
+
if not all(isinstance(session.get(name), str) and session[name].strip()
|
|
159
|
+
for name in ("agent", "session_id")) or not isinstance(case["events"], list) \
|
|
160
|
+
or any(not isinstance(event, dict) for event in case["events"]):
|
|
161
|
+
unattributable.add(evidence)
|
|
162
|
+
unattributable_cases += 1
|
|
163
|
+
continue
|
|
164
|
+
row = identity(store, key)
|
|
165
|
+
lookup.setdefault(evidence, []).append({**row, "family":
|
|
166
|
+
study["task_families"].get(key, "")})
|
|
167
|
+
development = [r for report in reports if report["split"] == "development"
|
|
168
|
+
for r in report["cases"]]
|
|
169
|
+
unknown_development = sum(not lookup.get(row["evidence_sha256"]) or
|
|
170
|
+
row["evidence_sha256"] in unattributable for row in development)
|
|
171
|
+
blocked = list(frozen)
|
|
172
|
+
for row in development:
|
|
173
|
+
blocked.extend(lookup.get(row["evidence_sha256"], []))
|
|
174
|
+
blocked_families = {study["task_families"][key] for key in
|
|
175
|
+
study["discovery_cases"] + study["control_cases"]}
|
|
176
|
+
blocked_families.update(group["family"] for row in development
|
|
177
|
+
for group in lookup.get(row["evidence_sha256"], [])
|
|
178
|
+
if group["family"])
|
|
179
|
+
counts: dict[str, Counter[str]] = {}
|
|
180
|
+
seen = set()
|
|
181
|
+
conflicts = 0
|
|
182
|
+
holdout = 0
|
|
183
|
+
overlap = 0
|
|
184
|
+
unknown = 0
|
|
185
|
+
for report in reports:
|
|
186
|
+
for row in report["cases"]:
|
|
187
|
+
marker = (row["evidence_sha256"], row["rubric"])
|
|
188
|
+
if marker in seen:
|
|
189
|
+
conflicts += 1
|
|
190
|
+
continue
|
|
191
|
+
seen.add(marker)
|
|
192
|
+
count = counts.setdefault(row["rubric"], Counter({name: 0 for name in (
|
|
193
|
+
"total", "missing", "abstentions", "misses", "false_alarms",
|
|
194
|
+
"reference_pass", "reference_fail", "reference_unknown")}))
|
|
195
|
+
expected, observed = row["expected"], row.get("observed")
|
|
196
|
+
count["total"] += 1
|
|
197
|
+
count[f"reference_{expected}"] += 1
|
|
198
|
+
count["missing"] += observed is None
|
|
199
|
+
count["abstentions"] += observed == "unknown"
|
|
200
|
+
count["misses"] += expected == "fail" and observed in {"pass", "unknown"}
|
|
201
|
+
count["false_alarms"] += expected == "pass" and observed == "fail"
|
|
202
|
+
if report["split"] == "holdout":
|
|
203
|
+
holdout += 1
|
|
204
|
+
groups = lookup.get(row["evidence_sha256"], [])
|
|
205
|
+
if not groups or row["evidence_sha256"] in unattributable or any(
|
|
206
|
+
not g["family"] for g in groups):
|
|
207
|
+
unknown += 1
|
|
208
|
+
if any(g["session"] == b["session"] or g["content"] == b["content"]
|
|
209
|
+
for g in groups for b in blocked) or any(
|
|
210
|
+
g["family"] in blocked_families for g in groups if g["family"]):
|
|
211
|
+
overlap += 1
|
|
212
|
+
rubrics = {}
|
|
213
|
+
for name, count in counts.items():
|
|
214
|
+
rubrics[name] = {**dict(count),
|
|
215
|
+
"abstention_rate": count["abstentions"] / count["total"],
|
|
216
|
+
"missing_rate": count["missing"] / count["total"], "miss_rate": count["misses"] / count["reference_fail"]
|
|
217
|
+
if count["reference_fail"] and not count["missing"] else None,
|
|
218
|
+
"false_alarm_rate": count["false_alarms"] / count["reference_pass"]
|
|
219
|
+
if count["reference_pass"] and not count["missing"] else None}
|
|
220
|
+
return {"rubrics": rubrics, "reports": study["reference_result_ids"],
|
|
221
|
+
"duplicate_references": conflicts, "holdout_overlap": overlap,
|
|
222
|
+
"unknown_holdout_groups": unknown,
|
|
223
|
+
"unknown_development_groups": unknown_development,
|
|
224
|
+
"unattributable_reference_cases": unattributable_cases,
|
|
225
|
+
"unindexable_cases": unindexable_cases,
|
|
226
|
+
"holdout_independence_verified": bool(holdout and not (overlap or unknown or unknown_development or conflicts)),
|
|
227
|
+
"human_calibrated": False, "scope": "Executable references only; human labels unknown."}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def assess(store: Store, study_id: str) -> dict[str, Any]:
|
|
231
|
+
study = store.get("studies", study_id)
|
|
232
|
+
proposal = store.get("proposals", study["proposal_id"])
|
|
233
|
+
spec = store.get("trial_specs", study["trial_spec_id"])
|
|
234
|
+
rows = [store.get("trial_results", key) for key in store.keys("trial_results")]
|
|
235
|
+
rows = [row for row in rows if row["spec_id"] == study["trial_spec_id"]]
|
|
236
|
+
total = 2 * spec["repeats"] * len(spec["cases"])
|
|
237
|
+
trial = {"spec_id": study["trial_spec_id"], "finished": len(rows), "total": total,
|
|
238
|
+
"complete": len(rows) == total, "trial_kind": spec["kind"],
|
|
239
|
+
"accepted": {name: sum(row["condition"] == name and row["accepted"] for row in rows)
|
|
240
|
+
for name in ("baseline", "candidate")}, "agent_behaviour_verified": False,
|
|
241
|
+
"elapsed_s": sum(row["elapsed_s"] for row in rows),
|
|
242
|
+
"missing_metrics": sum(row.get("metrics") is None for row in rows),
|
|
243
|
+
"conditions": {name: {"runs": len(selected := [row for row in rows
|
|
244
|
+
if row["condition"] == name]),
|
|
245
|
+
"elapsed_s": sum(row["elapsed_s"] for row in selected),
|
|
246
|
+
"tool_calls": sum(row["metrics"]["tool_calls"] for row in selected
|
|
247
|
+
if row.get("metrics")) if selected and all(
|
|
248
|
+
row.get("metrics") for row in selected) else None,
|
|
249
|
+
"interventions": sum(row["metrics"]["interventions"] for row in selected
|
|
250
|
+
if row.get("metrics")) if selected and all(
|
|
251
|
+
row.get("metrics") for row in selected) else None}
|
|
252
|
+
for name in ("baseline", "candidate")}}
|
|
253
|
+
followups = [{"followup_id": key, **row} for key in store.keys("study_followups")
|
|
254
|
+
if (row := store.get("study_followups", key))["study_id"] == study_id]
|
|
255
|
+
transfers = [{"transfer_id": key, **row} for key in store.keys("study_transfers")
|
|
256
|
+
if (row := store.get("study_transfers", key))["study_id"] == study_id]
|
|
257
|
+
decision = measure(store, study["trial_spec_id"], study["primary_metric"],
|
|
258
|
+
study["decision_policy"], proposal)
|
|
259
|
+
fresh_followup = any(row["freshness"]["work_after_policy_verified"] and
|
|
260
|
+
row["independence"]["verified"] and
|
|
261
|
+
measure(store, row["spec"]["trial_spec_id"], study["primary_metric"],
|
|
262
|
+
study["decision_policy"], proposal)["decision"] == "supported" for row in followups)
|
|
263
|
+
return {"study_id": study_id, "scoped_trial_decision": decision,
|
|
264
|
+
"scoped_decision": decision["decision"] if decision["decision"] != "supported"
|
|
265
|
+
or fresh_followup else "inconclusive",
|
|
266
|
+
"fresh_followup_supported": fresh_followup, "decision_policy": study["decision_policy"],
|
|
267
|
+
"stages": {"discovery": {
|
|
268
|
+
"cases": len(study["discovery_cases"]), "controls": len(study["control_cases"])},
|
|
269
|
+
"references": references(store, study), "trial": trial,
|
|
270
|
+
"followup": followups, "transfer": transfers},
|
|
271
|
+
"provenance": study["provenance"], "cost_inputs": study["cost_inputs"],
|
|
272
|
+
"observed_cost": {"trial_elapsed_s": trial["elapsed_s"]},
|
|
273
|
+
"primary_metric": study["primary_metric"],
|
|
274
|
+
"observed_effects": {"accepted_count_delta":
|
|
275
|
+
trial["accepted"]["candidate"] - trial["accepted"]["baseline"]
|
|
276
|
+
if trial["complete"] and not decision["invalid_observations"] else None,
|
|
277
|
+
"scope": "command/code correctness" if trial["trial_kind"] == "command_check"
|
|
278
|
+
else "bounded agent-task observations"},
|
|
279
|
+
"quality_constraints": study["quality_constraints"],
|
|
280
|
+
"rollback": {"proposal_id": study["proposal_id"], "target": proposal["target"],
|
|
281
|
+
"before_sha256": proposal["before_sha256"],
|
|
282
|
+
"after_sha256": proposal["after_sha256"]},
|
|
283
|
+
"improvement_established": False,
|
|
284
|
+
"unknowns": ["causal attribution", "independent human calibration",
|
|
285
|
+
"general productivity", "loaded instruction exposure"]}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Expose study records through the existing private improvement command."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from ..config import AutError
|
|
11
|
+
from .store import Store
|
|
12
|
+
from .studies import assess, create, shortlist
|
|
13
|
+
from .study_followup import followup, transfer
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def register(subparsers: Any) -> None:
|
|
17
|
+
parser = subparsers.add_parser("study", help="freeze and assess a prospective proof policy")
|
|
18
|
+
actions = parser.add_subparsers(dest="study_cmd", required=True)
|
|
19
|
+
action = actions.add_parser("shortlist", help="deduplicate frozen incident evidence")
|
|
20
|
+
action.add_argument("--audit", required=True)
|
|
21
|
+
action = actions.add_parser("create", help="freeze an operator-authored prospective policy")
|
|
22
|
+
action.add_argument("--spec-file", type=Path, required=True)
|
|
23
|
+
action = actions.add_parser("assess", help="report each stored proof stage")
|
|
24
|
+
action.add_argument("--study", required=True)
|
|
25
|
+
for name in ("followup", "transfer"):
|
|
26
|
+
action = actions.add_parser(name)
|
|
27
|
+
action.add_argument("--study", required=True)
|
|
28
|
+
action.add_argument("--spec-file", type=Path, required=True)
|
|
29
|
+
if name == "followup":
|
|
30
|
+
action.add_argument("--execute", action="store_true")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def run(args: argparse.Namespace, store: Store) -> dict[str, Any]:
|
|
34
|
+
if args.study_cmd == "shortlist":
|
|
35
|
+
return shortlist(store, args.audit)
|
|
36
|
+
if args.study_cmd == "assess":
|
|
37
|
+
return assess(store, args.study)
|
|
38
|
+
try:
|
|
39
|
+
spec = json.loads(args.spec_file.read_text())
|
|
40
|
+
except (OSError, ValueError) as exc:
|
|
41
|
+
raise AutError("improve: study specification is not readable JSON") from exc
|
|
42
|
+
if args.study_cmd == "create":
|
|
43
|
+
return create(store, spec)
|
|
44
|
+
if args.study_cmd == "followup":
|
|
45
|
+
return followup(store, args.study, spec, execute=args.execute)
|
|
46
|
+
return transfer(store, args.study, spec)
|