agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,244 @@
1
+ """Private content-addressed evidence and an append-only progress journal."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import re
9
+ import subprocess
10
+ import time
11
+ from collections import Counter
12
+ from datetime import datetime, timezone
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from ..collectors import claude_code, codex
17
+ from ..config import AutError
18
+ from ..event import CollectorUnavailable
19
+ from ..git_safe import git_argv, git_env
20
+ from ..state import _locked, atomic_write_text
21
+ from .evidence import read_session
22
+ from . import t3
23
+
24
+
25
+ def canonical(value: Any) -> str:
26
+ return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False)
27
+
28
+
29
+ def digest(value: Any) -> str:
30
+ return hashlib.sha256(canonical(value).encode()).hexdigest()
31
+
32
+
33
+ class Store:
34
+ def __init__(self, root: Path) -> None:
35
+ self.root = root.expanduser().resolve()
36
+ marker = self.root / ".aut-improve"
37
+ if self.root.exists() and any(self.root.iterdir()) and not marker.is_file():
38
+ raise AutError("improve: refusing a nonempty directory without its private marker")
39
+ self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
40
+ os.chmod(self.root, 0o700)
41
+ if marker.exists() and marker.read_text() != "1\n":
42
+ raise AutError("improve: unsupported private store version")
43
+ if not marker.exists():
44
+ atomic_write_text(marker, "1\n")
45
+ atomic_write_text(self.root / ".gitignore", "*\n")
46
+
47
+ def path(self, kind: str, key: str) -> Path:
48
+ if not re.fullmatch(r"[a-z_]{1,40}", kind) or not re.fullmatch(r"[a-f0-9]{64}", key):
49
+ raise AutError("improve: invalid record identity")
50
+ path = self.root / kind / f"{key}.json"
51
+ if not path.resolve().is_relative_to(self.root):
52
+ raise AutError("improve: record path escapes its private store")
53
+ return path
54
+
55
+ def put(self, kind: str, value: dict[str, Any]) -> str:
56
+ try:
57
+ if not isinstance(value, dict):
58
+ raise ValueError("record is not an object")
59
+ key = digest(value)
60
+ except (TypeError, ValueError) as exc:
61
+ raise AutError("improve: record must be a finite JSON object") from exc
62
+ path = self.path(kind, key)
63
+ with _locked(self.root / "write"):
64
+ if path.exists():
65
+ self.get(kind, key)
66
+ else:
67
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
68
+ atomic_write_text(path, canonical(value) + "\n")
69
+ return key
70
+
71
+ def get(self, kind: str, key: str) -> dict[str, Any]:
72
+ path = self.path(kind, key)
73
+ try:
74
+ value = json.loads(path.read_text())
75
+ valid = isinstance(value, dict) and digest(value) == key
76
+ except (OSError, TypeError, ValueError) as exc:
77
+ raise AutError("improve: private record is missing or invalid") from exc
78
+ if not valid:
79
+ raise AutError("improve: private record changed; refusing to continue")
80
+ return value
81
+
82
+ def keys(self, kind: str) -> list[str]:
83
+ self.path(kind, "0" * 64)
84
+ return sorted(p.stem for p in (self.root / kind).glob("*.json"))
85
+
86
+ def append(self, kind: str, **attrs: Any) -> None:
87
+ row = {"at": datetime.now(timezone.utc).isoformat(), "kind": kind, **attrs}
88
+ with _locked(self.root / "journal"):
89
+ fd = os.open(self.root / "events.jsonl", os.O_CREAT | os.O_APPEND | os.O_WRONLY, 0o600)
90
+ with os.fdopen(fd, "a", encoding="utf-8") as stream:
91
+ stream.write(canonical(row) + "\n")
92
+ stream.flush()
93
+ os.fsync(stream.fileno())
94
+
95
+
96
+ def repository(path: Path) -> str | None:
97
+ try:
98
+ result = subprocess.run(git_argv(str(path), "rev-parse", "--path-format=absolute",
99
+ "--git-common-dir"),
100
+ capture_output=True, text=True, timeout=5, env=git_env())
101
+ return str(Path(result.stdout.strip()).resolve()) if result.returncode == 0 else None
102
+ except (OSError, subprocess.SubprocessError):
103
+ return None
104
+
105
+
106
+ def _source_session(path: Path, agent: str, budget: int) -> str | None:
107
+ try:
108
+ with path.open("rb") as stream:
109
+ record = json.loads(stream.readline(min(budget, 65536)))
110
+ meta = record.get("payload", {}) if agent == "codex" else record
111
+ session = (meta.get("id") or meta.get("session_id")) if agent == "codex" \
112
+ else meta.get("sessionId", path.stem)
113
+ return session if isinstance(session, str) else None
114
+ except (OSError, ValueError, AttributeError):
115
+ return None
116
+
117
+
118
+ def audit(store: Store, home: Path, *, days: int = 7, limit: int = 40,
119
+ project: Path | None = None, idle_s: int = 120,
120
+ since_epoch: float | None = None, max_bytes: int = 16 * 1024 * 1024) -> dict[str, Any]:
121
+ if not 1 <= days <= 90 or not 1 <= limit <= 200 or idle_s < 0:
122
+ raise AutError("improve: days must be 1..90 and session limit 1..200")
123
+ now = time.time()
124
+ since = since_epoch if since_epoch is not None else now - days * 86400
125
+ files: list[tuple[float, str, Path]] = []
126
+ counts: Counter[str] = Counter()
127
+ for agent, discover in (("claude_code", claude_code.main_session_files),
128
+ ("codex", codex.session_files)):
129
+ try:
130
+ discovered = discover(home)
131
+ except (OSError, CollectorUnavailable):
132
+ counts["unreadable_sources"] += 1
133
+ continue
134
+ for name in discovered:
135
+ path = Path(name)
136
+ try:
137
+ stamp = path.stat().st_mtime
138
+ if stamp >= since:
139
+ if stamp > now - idle_s:
140
+ counts["active_sources"] += 1
141
+ else:
142
+ files.append((stamp, agent, path))
143
+ except OSError:
144
+ counts["unreadable_sources"] += 1
145
+ sessions: list[str] = []
146
+ imports = codex.imported_until(home)
147
+ seen: set[tuple[str, str]] = set()
148
+ identities: dict[tuple[str, str], str] = {}
149
+ copies: dict[tuple[str, str], list[str]] = {}
150
+ project_repo = repository(project) if project else None
151
+ if project and project_repo is None:
152
+ raise AutError("improve: project must be a readable local Git repository")
153
+ t3_rows, t3_counts = t3.snapshots(home, since=since, limit=limit,
154
+ idle_s=idle_s, max_bytes=max_bytes)
155
+ counts.update(t3_counts)
156
+ mirrors: dict[tuple[str, str], str] = {}
157
+ supplement_reads = 0
158
+ # App threads join provider handoffs; native mirrors remain referenced, not counted twice.
159
+ candidates: list[tuple[float, str, Path | dict[str, Any]]] = [
160
+ (0, "t3", row) for row in t3_rows]
161
+ candidates.extend(sorted(files, reverse=True))
162
+ for _stamp, agent, candidate in candidates:
163
+ if len(sessions) >= limit:
164
+ linked_identity = (agent, _source_session(candidate, agent, max_bytes)) \
165
+ if isinstance(candidate, Path) else None
166
+ if linked_identity not in mirrors or supplement_reads >= 2 * limit:
167
+ counts["unread_sources"] += 1
168
+ continue
169
+ supplement_reads += 1
170
+ try:
171
+ evidence = candidate if isinstance(candidate, dict) else read_session(
172
+ candidate, agent, max_bytes=max_bytes, since_epoch=since,
173
+ imports=imports, exclude_automated=True, recent=True)
174
+ except (OSError, ValueError, TypeError, OverflowError):
175
+ counts["unreadable_sources"] += 1
176
+ continue
177
+ if evidence is None:
178
+ counts["automated_sources"] += 1
179
+ continue
180
+ if (agent, evidence["session_id"]) in mirrors:
181
+ source = (agent, evidence["source"]["sha256"])
182
+ if source in seen:
183
+ counts["duplicates"] += 1
184
+ continue
185
+ seen.add(source)
186
+ primary_key = mirrors[(agent, evidence["session_id"])]
187
+ supplement = store.put("sessions", evidence)
188
+ primary = store.get("sessions", primary_key)
189
+ linked = set(primary.get("supplemental_sources", []))
190
+ linked.add(supplement)
191
+ primary["supplemental_sources"] = sorted(linked)
192
+ primary["source_family_link"] = "Native linkage is not proof of equivalent coverage."
193
+ updated_key = store.put("sessions", primary)
194
+ sessions[sessions.index(primary_key)] = updated_key
195
+ mirrors = {pair: updated_key if key == primary_key else key
196
+ for pair, key in mirrors.items()}
197
+ copies[("t3", primary["session_id"])] = [updated_key]
198
+ counts["t3_native_mirrors"] += 1
199
+ continue
200
+ if evidence["origin"] in {"subagent", "automated"}:
201
+ counts["automated_sources"] += 1
202
+ continue
203
+ if not evidence["events"]:
204
+ counts["no_recent_evidence"] += 1
205
+ continue
206
+ if project and (not evidence["cwd"] or repository(Path(evidence["cwd"])) != project_repo):
207
+ counts["other_projects"] += 1
208
+ continue
209
+ source = (agent, evidence["source"]["sha256"])
210
+ if source in seen:
211
+ counts["duplicates"] += 1
212
+ continue
213
+ seen.add(source)
214
+ identity = (agent, evidence["session_id"])
215
+ if identity in identities:
216
+ evidence["coverage"]["divergent_copy"] = True
217
+ evidence["coverage"]["complete"] = False
218
+ counts["divergent_copies"] += 1
219
+ identities[identity] = source[1]
220
+ key = store.put("sessions", evidence)
221
+ sessions.append(key)
222
+ if agent == "t3":
223
+ mirrors.update((tuple(pair), key) for pair in evidence.get("native_sessions", []))
224
+ copies.setdefault(identity, []).append(key)
225
+ coverage = evidence["coverage"]
226
+ counts["complete_sessions"] += bool(coverage["complete"])
227
+ for name in ("calls", "matched_results", "unmatched_calls", "orphan_results",
228
+ "truncated_events", "malformed_records", "byte_cap_reached"):
229
+ counts[name] += coverage.get(name, 0)
230
+ counts["error_results"] += sum(e["kind"] == "result" and e["status"] == "error"
231
+ for e in evidence["events"])
232
+ ambiguous = sorted(key for group in copies.values() if len(group) > 1 for key in group)
233
+ counts["complete_sessions"] = sum(store.get("sessions", key)["coverage"]["complete"]
234
+ and key not in ambiguous for key in sessions)
235
+ manifest = {"version": 4, "session_ids": sorted(sessions),
236
+ "ambiguous_session_ids": ambiguous, "coverage": dict(counts),
237
+ "project": str(project.resolve()) if project else None,
238
+ "source_policy": {"days": days, "limit": limit, "max_bytes": max_bytes,
239
+ "idle_s": idle_s, "agents": ["t3", "claude_code", "codex"],
240
+ "additional_native_reads": 2 * limit,
241
+ "preference": "T3 app thread before native provider mirrors"}}
242
+ key = store.put("audits", manifest)
243
+ store.append("audit", audit_id=key, sessions=len(sessions), coverage=dict(counts))
244
+ return {"audit_id": key, "sessions": len(sessions), **counts}
@@ -0,0 +1,285 @@
1
+ """Freeze prospective policies and report bounded evidence without inferring causality."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import Counter
6
+ from datetime import datetime, timezone
7
+ from typing import Any
8
+
9
+ from ..config import AutError
10
+ from .store import Store, digest
11
+ from .study_decision import measure, observed_experiment
12
+ from .trial import _spec
13
+ from .validation import evidence_key
14
+
15
+
16
+ def fields(value: Any, names: str) -> None:
17
+ if not isinstance(value, dict) or set(value) != set(names.split()):
18
+ raise AutError("improve: invalid study fields")
19
+
20
+
21
+ def strings(value: Any, *, empty: bool = False) -> None:
22
+ if not isinstance(value, list) or (not value and not empty) or any(
23
+ not isinstance(item, str) or not item.strip() for item in value) \
24
+ or len(value) != len(set(value)):
25
+ raise AutError("improve: expected distinct nonempty strings")
26
+
27
+
28
+ def identity(store: Store, case_id: str) -> dict[str, str]:
29
+ case = store.get("cases", case_id)
30
+ session = store.get("sessions", case["session_snapshot"])
31
+ # Copies can change event ids, line numbers and snapshot coverage without changing the task.
32
+ content = [{k: v for k, v in event.items() if k not in {"id", "line", "byte_offset", "timestamp"}}
33
+ for event in case["events"]]
34
+ return {"session": digest([session["agent"], session["session_id"]]),
35
+ "content": digest(content), "evidence": evidence_key(case)}
36
+
37
+
38
+ def shortlist(store: Store, audit_id: str) -> dict[str, Any]:
39
+ audit = store.get("audits", audit_id)
40
+ incidents: dict[str, dict[str, Any]] = {}
41
+ sessions = set()
42
+ for snapshot in audit["session_ids"]:
43
+ session = store.get("sessions", snapshot)
44
+ source = digest([session["agent"], session["session_id"]])
45
+ sessions.add(source)
46
+ for event in session["events"]:
47
+ if event["kind"] != "result" or event.get("status") != "error":
48
+ continue
49
+ key = digest([source, event["id"]])
50
+ if key in incidents:
51
+ incidents[key]["snapshots"].append(snapshot)
52
+ continue
53
+ incidents[key] = {"incident_id": key, "source_session": source,
54
+ "event_id": event["id"], "snapshots": [snapshot],
55
+ "signal": digest([event.get("tool"), event.get("text")])}
56
+ counts = Counter(row["signal"] for row in incidents.values())
57
+ return {"audit_id": audit_id, "distinct_source_sessions": len(sessions),
58
+ "incidents": list(incidents.values()), "signal_counts": dict(counts),
59
+ "scope": "Error groups are investigation signals, not causal findings."}
60
+
61
+
62
+ def _manifest(store: Store, value: dict[str, Any]) -> dict[str, Any]:
63
+ fields(value, "version discovery_cases control_cases task_families proposal_id trial_spec_id "
64
+ "primary_metric quality_constraints budget_s provenance reference_result_ids cost_inputs "
65
+ "decision_policy")
66
+ if value["version"] != 1:
67
+ raise AutError("improve: unsupported study version")
68
+ for name in ("discovery_cases", "control_cases", "quality_constraints"):
69
+ strings(value[name])
70
+ strings(value["reference_result_ids"], empty=True)
71
+ ids = value["discovery_cases"] + value["control_cases"]
72
+ if len(ids) != len(set(ids)):
73
+ raise AutError("improve: duplicate study case")
74
+ families = value["task_families"]
75
+ if not isinstance(families, dict) or not set(ids) <= set(families) or any(
76
+ not isinstance(v, str) or not v.strip() for v in families.values()):
77
+ raise AutError("improve: every case needs a task family")
78
+ identities = [identity(store, key) for key in ids]
79
+ for key in families:
80
+ identity(store, key)
81
+ if len({row["content"] for row in identities}) != len(ids):
82
+ raise AutError("improve: copied case evidence")
83
+ discovery = [identity(store, key) for key in value["discovery_cases"]]
84
+ controls = [identity(store, key) for key in value["control_cases"]]
85
+ if {r["session"] for r in discovery} & {r["session"] for r in controls}:
86
+ raise AutError("improve: control source sessions overlap discovery")
87
+ proposal = store.get("proposals", value["proposal_id"])
88
+ if proposal["case_id"] not in value["discovery_cases"]:
89
+ raise AutError("improve: proposal must link discovery evidence")
90
+ trial = _spec(store, value["trial_spec_id"])
91
+ if trial["kind"] == "command_check":
92
+ from pathlib import Path
93
+ from .trial import _acceptance_inputs
94
+ for case in trial["cases"]:
95
+ if case.get("acceptance_files"):
96
+ protected = {"acceptance_argv": case["argv"],
97
+ "acceptance_files": case["acceptance_files"]}
98
+ roots = [Path(root).resolve() for root in trial["directories"].values()]
99
+ for root in roots:
100
+ _acceptance_inputs(protected, roots, root)
101
+ policy = value["decision_policy"]
102
+ fields(policy, "direction minimum_improvement require_no_acceptance_regressions "
103
+ "require_all_candidate_accepted")
104
+ if policy["direction"] not in {"increase", "decrease"} or type(
105
+ policy["minimum_improvement"]) not in (int, float) or not (
106
+ 0 <= policy["minimum_improvement"] < float("inf")) or any(
107
+ type(policy[name]) is not bool for name in (
108
+ "require_no_acceptance_regressions", "require_all_candidate_accepted")):
109
+ raise AutError("improve: invalid prospective decision policy")
110
+ if value["primary_metric"] not in {"accepted", "elapsed_s", "tool_calls", "interventions"}:
111
+ raise AutError("improve: unsupported primary metric")
112
+ if type(value["budget_s"]) is not int or not 1 <= value["budget_s"] <= 3600:
113
+ raise AutError("improve: study budget must be 1..3600 seconds")
114
+ fields(value["provenance"], "setup model tools instructions")
115
+ if any(v is not None and (not isinstance(v, str) or not v.strip())
116
+ for v in value["provenance"].values()):
117
+ raise AutError("improve: provenance must be an explicit source or null for unknown")
118
+ fields(value["cost_inputs"], "operator_minutes money")
119
+ for v in value["cost_inputs"].values():
120
+ if v is not None and (type(v) not in (int, float) or not 0 <= v < float("inf")):
121
+ raise AutError("improve: cost inputs must be finite nonnegative numbers or null")
122
+ for key in value["reference_result_ids"]:
123
+ store.get("reference_results", key)
124
+ return value
125
+
126
+
127
+ def create(store: Store, value: dict[str, Any]) -> dict[str, Any]:
128
+ try:
129
+ _manifest(store, value)
130
+ if observed_experiment(store, value["trial_spec_id"]):
131
+ raise AutError("improve: prospective policy cannot include pre-existing trial results")
132
+ except (KeyError, TypeError, ValueError, AttributeError) as exc:
133
+ raise AutError("improve: invalid study specification") from exc
134
+ key = store.put("studies", {**value, "known_case_ids": store.keys("cases"),
135
+ "created_at": datetime.now(timezone.utc).isoformat()})
136
+ store.append("study_created", study_id=key)
137
+ return {"study_id": key, "prospective": True}
138
+
139
+
140
+ def references(store: Store, study: dict[str, Any]) -> dict[str, Any]:
141
+ frozen = [identity(store, key) for key in study["discovery_cases"] + study["control_cases"]]
142
+ reports = [store.get("reference_results", key) for key in study["reference_result_ids"]]
143
+ relevant = {row["evidence_sha256"] for report in reports for row in report["cases"]}
144
+ lookup: dict[str, list[dict[str, str]]] = {}
145
+ unattributable: set[str] = set()
146
+ unattributable_cases = 0
147
+ unindexable_cases = 0
148
+ for key in store.keys("cases") if relevant else []:
149
+ case = store.get("cases", key)
150
+ if not {"events", "coverage", "omitted_task_events"} <= case.keys():
151
+ unindexable_cases += 1
152
+ continue
153
+ evidence = evidence_key(case)
154
+ if evidence not in relevant:
155
+ continue
156
+ snapshot = case.get("session_snapshot")
157
+ session = store.get("sessions", snapshot) if snapshot is not None else {}
158
+ if not all(isinstance(session.get(name), str) and session[name].strip()
159
+ for name in ("agent", "session_id")) or not isinstance(case["events"], list) \
160
+ or any(not isinstance(event, dict) for event in case["events"]):
161
+ unattributable.add(evidence)
162
+ unattributable_cases += 1
163
+ continue
164
+ row = identity(store, key)
165
+ lookup.setdefault(evidence, []).append({**row, "family":
166
+ study["task_families"].get(key, "")})
167
+ development = [r for report in reports if report["split"] == "development"
168
+ for r in report["cases"]]
169
+ unknown_development = sum(not lookup.get(row["evidence_sha256"]) or
170
+ row["evidence_sha256"] in unattributable for row in development)
171
+ blocked = list(frozen)
172
+ for row in development:
173
+ blocked.extend(lookup.get(row["evidence_sha256"], []))
174
+ blocked_families = {study["task_families"][key] for key in
175
+ study["discovery_cases"] + study["control_cases"]}
176
+ blocked_families.update(group["family"] for row in development
177
+ for group in lookup.get(row["evidence_sha256"], [])
178
+ if group["family"])
179
+ counts: dict[str, Counter[str]] = {}
180
+ seen = set()
181
+ conflicts = 0
182
+ holdout = 0
183
+ overlap = 0
184
+ unknown = 0
185
+ for report in reports:
186
+ for row in report["cases"]:
187
+ marker = (row["evidence_sha256"], row["rubric"])
188
+ if marker in seen:
189
+ conflicts += 1
190
+ continue
191
+ seen.add(marker)
192
+ count = counts.setdefault(row["rubric"], Counter({name: 0 for name in (
193
+ "total", "missing", "abstentions", "misses", "false_alarms",
194
+ "reference_pass", "reference_fail", "reference_unknown")}))
195
+ expected, observed = row["expected"], row.get("observed")
196
+ count["total"] += 1
197
+ count[f"reference_{expected}"] += 1
198
+ count["missing"] += observed is None
199
+ count["abstentions"] += observed == "unknown"
200
+ count["misses"] += expected == "fail" and observed in {"pass", "unknown"}
201
+ count["false_alarms"] += expected == "pass" and observed == "fail"
202
+ if report["split"] == "holdout":
203
+ holdout += 1
204
+ groups = lookup.get(row["evidence_sha256"], [])
205
+ if not groups or row["evidence_sha256"] in unattributable or any(
206
+ not g["family"] for g in groups):
207
+ unknown += 1
208
+ if any(g["session"] == b["session"] or g["content"] == b["content"]
209
+ for g in groups for b in blocked) or any(
210
+ g["family"] in blocked_families for g in groups if g["family"]):
211
+ overlap += 1
212
+ rubrics = {}
213
+ for name, count in counts.items():
214
+ rubrics[name] = {**dict(count),
215
+ "abstention_rate": count["abstentions"] / count["total"],
216
+ "missing_rate": count["missing"] / count["total"], "miss_rate": count["misses"] / count["reference_fail"]
217
+ if count["reference_fail"] and not count["missing"] else None,
218
+ "false_alarm_rate": count["false_alarms"] / count["reference_pass"]
219
+ if count["reference_pass"] and not count["missing"] else None}
220
+ return {"rubrics": rubrics, "reports": study["reference_result_ids"],
221
+ "duplicate_references": conflicts, "holdout_overlap": overlap,
222
+ "unknown_holdout_groups": unknown,
223
+ "unknown_development_groups": unknown_development,
224
+ "unattributable_reference_cases": unattributable_cases,
225
+ "unindexable_cases": unindexable_cases,
226
+ "holdout_independence_verified": bool(holdout and not (overlap or unknown or unknown_development or conflicts)),
227
+ "human_calibrated": False, "scope": "Executable references only; human labels unknown."}
228
+
229
+
230
+ def assess(store: Store, study_id: str) -> dict[str, Any]:
231
+ study = store.get("studies", study_id)
232
+ proposal = store.get("proposals", study["proposal_id"])
233
+ spec = store.get("trial_specs", study["trial_spec_id"])
234
+ rows = [store.get("trial_results", key) for key in store.keys("trial_results")]
235
+ rows = [row for row in rows if row["spec_id"] == study["trial_spec_id"]]
236
+ total = 2 * spec["repeats"] * len(spec["cases"])
237
+ trial = {"spec_id": study["trial_spec_id"], "finished": len(rows), "total": total,
238
+ "complete": len(rows) == total, "trial_kind": spec["kind"],
239
+ "accepted": {name: sum(row["condition"] == name and row["accepted"] for row in rows)
240
+ for name in ("baseline", "candidate")}, "agent_behaviour_verified": False,
241
+ "elapsed_s": sum(row["elapsed_s"] for row in rows),
242
+ "missing_metrics": sum(row.get("metrics") is None for row in rows),
243
+ "conditions": {name: {"runs": len(selected := [row for row in rows
244
+ if row["condition"] == name]),
245
+ "elapsed_s": sum(row["elapsed_s"] for row in selected),
246
+ "tool_calls": sum(row["metrics"]["tool_calls"] for row in selected
247
+ if row.get("metrics")) if selected and all(
248
+ row.get("metrics") for row in selected) else None,
249
+ "interventions": sum(row["metrics"]["interventions"] for row in selected
250
+ if row.get("metrics")) if selected and all(
251
+ row.get("metrics") for row in selected) else None}
252
+ for name in ("baseline", "candidate")}}
253
+ followups = [{"followup_id": key, **row} for key in store.keys("study_followups")
254
+ if (row := store.get("study_followups", key))["study_id"] == study_id]
255
+ transfers = [{"transfer_id": key, **row} for key in store.keys("study_transfers")
256
+ if (row := store.get("study_transfers", key))["study_id"] == study_id]
257
+ decision = measure(store, study["trial_spec_id"], study["primary_metric"],
258
+ study["decision_policy"], proposal)
259
+ fresh_followup = any(row["freshness"]["work_after_policy_verified"] and
260
+ row["independence"]["verified"] and
261
+ measure(store, row["spec"]["trial_spec_id"], study["primary_metric"],
262
+ study["decision_policy"], proposal)["decision"] == "supported" for row in followups)
263
+ return {"study_id": study_id, "scoped_trial_decision": decision,
264
+ "scoped_decision": decision["decision"] if decision["decision"] != "supported"
265
+ or fresh_followup else "inconclusive",
266
+ "fresh_followup_supported": fresh_followup, "decision_policy": study["decision_policy"],
267
+ "stages": {"discovery": {
268
+ "cases": len(study["discovery_cases"]), "controls": len(study["control_cases"])},
269
+ "references": references(store, study), "trial": trial,
270
+ "followup": followups, "transfer": transfers},
271
+ "provenance": study["provenance"], "cost_inputs": study["cost_inputs"],
272
+ "observed_cost": {"trial_elapsed_s": trial["elapsed_s"]},
273
+ "primary_metric": study["primary_metric"],
274
+ "observed_effects": {"accepted_count_delta":
275
+ trial["accepted"]["candidate"] - trial["accepted"]["baseline"]
276
+ if trial["complete"] and not decision["invalid_observations"] else None,
277
+ "scope": "command/code correctness" if trial["trial_kind"] == "command_check"
278
+ else "bounded agent-task observations"},
279
+ "quality_constraints": study["quality_constraints"],
280
+ "rollback": {"proposal_id": study["proposal_id"], "target": proposal["target"],
281
+ "before_sha256": proposal["before_sha256"],
282
+ "after_sha256": proposal["after_sha256"]},
283
+ "improvement_established": False,
284
+ "unknowns": ["causal attribution", "independent human calibration",
285
+ "general productivity", "loaded instruction exposure"]}
@@ -0,0 +1,46 @@
1
+ """Expose study records through the existing private improvement command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ from ..config import AutError
11
+ from .store import Store
12
+ from .studies import assess, create, shortlist
13
+ from .study_followup import followup, transfer
14
+
15
+
16
+ def register(subparsers: Any) -> None:
17
+ parser = subparsers.add_parser("study", help="freeze and assess a prospective proof policy")
18
+ actions = parser.add_subparsers(dest="study_cmd", required=True)
19
+ action = actions.add_parser("shortlist", help="deduplicate frozen incident evidence")
20
+ action.add_argument("--audit", required=True)
21
+ action = actions.add_parser("create", help="freeze an operator-authored prospective policy")
22
+ action.add_argument("--spec-file", type=Path, required=True)
23
+ action = actions.add_parser("assess", help="report each stored proof stage")
24
+ action.add_argument("--study", required=True)
25
+ for name in ("followup", "transfer"):
26
+ action = actions.add_parser(name)
27
+ action.add_argument("--study", required=True)
28
+ action.add_argument("--spec-file", type=Path, required=True)
29
+ if name == "followup":
30
+ action.add_argument("--execute", action="store_true")
31
+
32
+
33
+ def run(args: argparse.Namespace, store: Store) -> dict[str, Any]:
34
+ if args.study_cmd == "shortlist":
35
+ return shortlist(store, args.audit)
36
+ if args.study_cmd == "assess":
37
+ return assess(store, args.study)
38
+ try:
39
+ spec = json.loads(args.spec_file.read_text())
40
+ except (OSError, ValueError) as exc:
41
+ raise AutError("improve: study specification is not readable JSON") from exc
42
+ if args.study_cmd == "create":
43
+ return create(store, spec)
44
+ if args.study_cmd == "followup":
45
+ return followup(store, args.study, spec, execute=args.execute)
46
+ return transfer(store, args.study, spec)