agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
"""`aut outcomes`: what became of each commit an agent made, read from the local clones.
|
|
2
|
+
|
|
3
|
+
Reads the commit rows (agents.activity_event kind commit) and the git clones under
|
|
4
|
+
outcome_repo_roots, and writes agents.commit_outcome rows: one per change of state, never a URL,
|
|
5
|
+
path, subject or file name. repo_key -> clone is held in memory for the run only.
|
|
6
|
+
|
|
7
|
+
States (a commit moves through them by new rows; readers take the latest):
|
|
8
|
+
merged ancestor of origin/HEAD, a single-commit patch-id twin on it, or (network on) a
|
|
9
|
+
merged GitHub PR found by gh; survival = share of its added lines still there
|
|
10
|
+
14 days after the merge
|
|
11
|
+
reverted "This reverts commit X" for it or its twin within 14 days after the merge
|
|
12
|
+
reworked merged, but under half of its added lines survive 14 days
|
|
13
|
+
open reachable from a branch only
|
|
14
|
+
abandoned unreachable, no twin, no merged PR
|
|
15
|
+
superseded a pre-amend SHA (unreachable, and the session's next commit was an amend)
|
|
16
|
+
undecided younger than 14 days (from the merge, or from the commit row when not merged), or
|
|
17
|
+
not decidable this run (default ref older than merge + 14 days, gh failed, a git
|
|
18
|
+
read of the clone failed)
|
|
19
|
+
A run that cannot see or decide a commit never replaces its stored state, except a stored
|
|
20
|
+
"reverted" that a stale default ref's reads show no revert for: that row goes to undecided.
|
|
21
|
+
|
|
22
|
+
`aut outcome <shipped|ruled-out|open>`: the manual outcome of the newest interactive session of
|
|
23
|
+
the current project, written to agents.manual_outcome.
|
|
24
|
+
|
|
25
|
+
The git and gh reads live in outcome_git; this module decides the states and writes them.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import re
|
|
33
|
+
import sys
|
|
34
|
+
from collections import Counter, defaultdict
|
|
35
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
36
|
+
from dataclasses import dataclass
|
|
37
|
+
from datetime import datetime, timezone
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Any, Literal, NamedTuple, NotRequired, TypedDict
|
|
40
|
+
|
|
41
|
+
from .collectors import claude_code
|
|
42
|
+
from .config import AutError, Config
|
|
43
|
+
from .outcome_git import (DAY, DECIDE_S, Clone, CloneRead, GitFailed, ResolvedCommit, fetch_clones,
|
|
44
|
+
find_clones, gh_merged, read_clone, resolve_shas, run_git, survival)
|
|
45
|
+
from .push import job_heartbeat
|
|
46
|
+
from .repo_keys import RepoKeys, refuse_new_salt
|
|
47
|
+
from .sink import connect, write_many
|
|
48
|
+
from .state import state_path, atomic_write_text, load_json_dict
|
|
49
|
+
|
|
50
|
+
CACHE_NAME = "outcomes_cache.json"
|
|
51
|
+
_CACHE_V = 2 # 2: a failed git read no longer counts as a survival score
|
|
52
|
+
_SHA = re.compile(r"^[0-9a-f]{7,40}$")
|
|
53
|
+
_WORKERS = 8
|
|
54
|
+
MANUAL = {"shipped": "shipped", "ruled-out": "ruled_out", "open": "open"}
|
|
55
|
+
|
|
56
|
+
OutcomeState = Literal["merged", "reverted", "reworked", "open", "abandoned", "superseded",
|
|
57
|
+
"undecided"]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Outcome(TypedDict):
|
|
61
|
+
sha: str
|
|
62
|
+
repo_key: str
|
|
63
|
+
state: OutcomeState
|
|
64
|
+
survival: float | None
|
|
65
|
+
sticky: NotRequired[bool] # this run could not see or decide it: keep any stored state
|
|
66
|
+
no_revert: NotRequired[bool] # sticky, but its reads found no revert after the merge
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class _Candidate(NamedTuple):
|
|
70
|
+
clone: int # index into the run's clones
|
|
71
|
+
full: str
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class _Pick(NamedTuple):
|
|
75
|
+
clone: Clone
|
|
76
|
+
read: CloneRead
|
|
77
|
+
full: str
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(slots=True)
|
|
81
|
+
class _Anchor:
|
|
82
|
+
"""One (sha, repo_key) of the commit rows."""
|
|
83
|
+
sha: str
|
|
84
|
+
key: str | None
|
|
85
|
+
ts: float
|
|
86
|
+
amended: bool = False
|
|
87
|
+
pick: _Pick | None = None # the clone that knows most of it
|
|
88
|
+
unread: bool = False # a clone that might know more could not be read
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class _SurvivalJob(NamedTuple):
|
|
92
|
+
clone: Clone
|
|
93
|
+
target: str
|
|
94
|
+
merge_t: float
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class _GhJob(NamedTuple):
|
|
98
|
+
github_slug: str
|
|
99
|
+
full: str
|
|
100
|
+
ts: float
|
|
101
|
+
on_branch: bool
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class _JobResult(NamedTuple):
|
|
105
|
+
"""What a finished job says about its outcome; evaluate applies it on its own thread."""
|
|
106
|
+
state: OutcomeState
|
|
107
|
+
survival: float | None
|
|
108
|
+
sticky: bool
|
|
109
|
+
counts: dict[str, int] # the stats to add
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _anchors(rows: list[dict[str, Any]], stats: Counter[str]) -> list[_Anchor]:
|
|
113
|
+
"""The distinct commits of the rows, a pre-amend one marked as amended."""
|
|
114
|
+
anchors: dict[tuple[str, str | None], _Anchor] = {}
|
|
115
|
+
by_session = defaultdict(list)
|
|
116
|
+
for r in rows:
|
|
117
|
+
sha = (r["sha"] or "").lower()
|
|
118
|
+
if not _SHA.match(sha):
|
|
119
|
+
stats["junk_sha"] += 1
|
|
120
|
+
continue
|
|
121
|
+
a = anchors.setdefault((sha, r["repo_key"]), _Anchor(sha, r["repo_key"], r["ts"]))
|
|
122
|
+
a.ts = min(a.ts, r["ts"])
|
|
123
|
+
by_session[r["session_id"]].append((r["ts"], r["amend"], a))
|
|
124
|
+
for seq in by_session.values():
|
|
125
|
+
seq.sort(key=lambda x: (x[0], bool(x[1]), x[2].sha)) # an amend sorts after a tie
|
|
126
|
+
for (_, _, a), (_, amend, _) in zip(seq, seq[1:]):
|
|
127
|
+
a.amended |= bool(amend)
|
|
128
|
+
stats["anchors"] = len(anchors)
|
|
129
|
+
return list(anchors.values())
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _candidates(a: _Anchor, readable: dict[int, dict[str, ResolvedCommit]],
|
|
133
|
+
clones: list[Clone]) -> list[_Candidate]:
|
|
134
|
+
"""The clones that may hold the anchor, its own repo's first."""
|
|
135
|
+
cands = [_Candidate(i, h[a.sha].full) for i, h in readable.items()
|
|
136
|
+
if a.sha in h and abs(h[a.sha].committed - a.ts) <= DAY]
|
|
137
|
+
own = [c for c in cands if a.key and clones[c.clone].key == a.key]
|
|
138
|
+
return own or sorted(cands, key=lambda c: clones[c.clone].key is None)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _distance_from_merged(read: CloneRead, full: str) -> int:
|
|
142
|
+
"""How far a commit is from merged in one clone: merged 0, open 1, unreachable 2."""
|
|
143
|
+
return 0 if full in read.on_default or full in read.twin else 1 if full in read.on_branch else 2
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _resolve(anchors: list[_Anchor], clones: list[Clone], stats: Counter[str]) -> None:
|
|
147
|
+
"""Point each anchor at the clone where it is closest to merged (among equals the keyed one,
|
|
148
|
+
then the freshest default ref, which sees the most reverts), and mark the ones a failed git read
|
|
149
|
+
could change."""
|
|
150
|
+
shas = sorted({a.sha for a in anchors})
|
|
151
|
+
held = {i: resolve_shas(c, shas) for i, c in enumerate(clones)} if shas else {}
|
|
152
|
+
failed = [i for i, h in held.items() if h is None]
|
|
153
|
+
readable = {i: h for i, h in held.items() if h is not None}
|
|
154
|
+
found: list[tuple[_Anchor, list[_Candidate]]] = []
|
|
155
|
+
per_clone: dict[int, list[tuple[str, int]]] = defaultdict(list)
|
|
156
|
+
for a in anchors:
|
|
157
|
+
cands = _candidates(a, readable, clones)
|
|
158
|
+
has_own = any(a.key and clones[c.clone].key == a.key for c in cands)
|
|
159
|
+
a.unread = any(not has_own or clones[i].key == a.key for i in failed)
|
|
160
|
+
if cands:
|
|
161
|
+
found.append((a, cands))
|
|
162
|
+
for i, full in cands:
|
|
163
|
+
per_clone[i].append((full, readable[i][a.sha].committed))
|
|
164
|
+
else:
|
|
165
|
+
stats["not_found"] += 1
|
|
166
|
+
reads = {i: read_clone(clones[i], int(min(t for _, t in got)) - 2 * DAY, [f for f, _ in got])
|
|
167
|
+
for i, got in per_clone.items()}
|
|
168
|
+
for a, cands in found:
|
|
169
|
+
best = min(cands, key=lambda c: (_distance_from_merged(reads[c.clone], c.full),
|
|
170
|
+
clones[c.clone].key is None, -reads[c.clone].tip_t))
|
|
171
|
+
a.pick = _Pick(clones[best.clone], reads[best.clone], best.full)
|
|
172
|
+
a.unread |= not all(reads[j].ok for j, _ in cands)
|
|
173
|
+
stats["found"] += 1
|
|
174
|
+
stats["null_key_resolved"] += a.key is None and clones[best.clone].key is not None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _decide_merged(o: Outcome, pick: _Pick, target: str, now_ts: float, cache: dict[str, Any],
|
|
178
|
+
stats: Counter[str]) -> _SurvivalJob | None:
|
|
179
|
+
"""Settle a commit (or its twin) on the default branch once 14 days have passed; the job that
|
|
180
|
+
measures its survival, if it needs one."""
|
|
181
|
+
c, read, full = pick
|
|
182
|
+
merge_t = read.on_default[target] # when it landed on the default branch
|
|
183
|
+
if now_ts - merge_t < DECIDE_S:
|
|
184
|
+
return None
|
|
185
|
+
window = (merge_t, merge_t + DECIDE_S)
|
|
186
|
+
if read.reverted_between(full, *window) or read.reverted_between(target, *window):
|
|
187
|
+
o["state"] = "reverted"
|
|
188
|
+
return None
|
|
189
|
+
if target in cache or max(read.tip_t, c.fetched_at) >= merge_t + DECIDE_S:
|
|
190
|
+
return _SurvivalJob(c, target, merge_t)
|
|
191
|
+
# shortcut: the default ref was last updated before merge + 14 d, so survival
|
|
192
|
+
# cannot be measured yet; a repo nobody fetches stays undecided. Upgrade with a
|
|
193
|
+
# per-repo "inactive" signal if that holds months back.
|
|
194
|
+
o["sticky"] = o["no_revert"] = True
|
|
195
|
+
stats["stale_default"] += 1
|
|
196
|
+
return None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _decide_unmerged(o: Outcome, a: _Anchor, pick: _Pick, young: bool, cache: dict[str, Any],
|
|
200
|
+
use_gh: bool) -> _GhJob | None:
|
|
201
|
+
"""Settle a commit off the default branch; the job that asks gh, if it needs one."""
|
|
202
|
+
c, read, full = pick
|
|
203
|
+
on_branch = full in read.on_branch
|
|
204
|
+
if not on_branch and a.amended:
|
|
205
|
+
o["state"] = "superseded"
|
|
206
|
+
elif not young:
|
|
207
|
+
# a squash-merged PR leaves its commits off the default branch; a cached answer needs no gh
|
|
208
|
+
if f"gh:{full}" in cache or (use_gh and c.github_slug):
|
|
209
|
+
return _GhJob(c.github_slug or "", full, a.ts, on_branch)
|
|
210
|
+
o["state"] = "open" if on_branch else "abandoned"
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _decide(a: _Anchor, now_ts: float, cache: dict[str, Any], use_gh: bool,
|
|
215
|
+
stats: Counter[str]) -> tuple[Outcome, _SurvivalJob | _GhJob | None]:
|
|
216
|
+
"""The anchor's outcome, and the job that finishes it if one is needed. Returning with the
|
|
217
|
+
state untouched and no job means undecided."""
|
|
218
|
+
o: Outcome = {"sha": a.sha, "repo_key": a.key or "", "state": "undecided", "survival": None}
|
|
219
|
+
if a.pick:
|
|
220
|
+
o["repo_key"] = a.key or a.pick.clone.key or ""
|
|
221
|
+
if a.unread: # git could not answer: decide nothing, keep what is stored
|
|
222
|
+
o["sticky"] = True
|
|
223
|
+
stats["repo_unreadable"] += 1
|
|
224
|
+
return o, None
|
|
225
|
+
young = now_ts - a.ts < DECIDE_S
|
|
226
|
+
if a.pick is None:
|
|
227
|
+
# A missing clone cannot prove abandonment; retain any prior decision.
|
|
228
|
+
o["sticky"] = True
|
|
229
|
+
return o, None
|
|
230
|
+
_c, read, full = a.pick
|
|
231
|
+
target = full if full in read.on_default else read.twin.get(full)
|
|
232
|
+
if target:
|
|
233
|
+
return o, _decide_merged(o, a.pick, target, now_ts, cache, stats)
|
|
234
|
+
return o, _decide_unmerged(o, a, a.pick, young, cache, use_gh)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _ask_gh(job: _GhJob, now_ts: float, cache: dict[str, Any]) -> _JobResult:
|
|
238
|
+
"""Settle a commit off the default branch by its PR."""
|
|
239
|
+
github_slug, full, ts, on_branch = job
|
|
240
|
+
k = f"gh:{full}"
|
|
241
|
+
looked = 0
|
|
242
|
+
merged: bool | None
|
|
243
|
+
if cache.get(k) is True:
|
|
244
|
+
merged = True
|
|
245
|
+
else:
|
|
246
|
+
cache.pop(k, None)
|
|
247
|
+
merged = gh_merged(github_slug, full)
|
|
248
|
+
looked = 1
|
|
249
|
+
if merged is None: # gh could not answer: decide nothing, keep what is stored
|
|
250
|
+
return _JobResult("undecided", None, True, {"gh_lookups": 1, "gh_failed": 1})
|
|
251
|
+
if merged:
|
|
252
|
+
cache[k] = True
|
|
253
|
+
state: OutcomeState = "merged" if merged else "open" if on_branch else "abandoned"
|
|
254
|
+
return _JobResult(state, None, False, {"gh_lookups": looked, "gh_failed": 0})
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _measure(job: _SurvivalJob, cache: dict[str, Any]) -> _JobResult:
|
|
258
|
+
"""Measure a merged commit's survival."""
|
|
259
|
+
c, target, merge_t = job
|
|
260
|
+
fresh = target not in cache
|
|
261
|
+
if fresh:
|
|
262
|
+
try:
|
|
263
|
+
cache[target] = survival(c, target, merge_t)
|
|
264
|
+
except GitFailed: # not cached: the next run measures it again
|
|
265
|
+
return _JobResult("undecided", None, True, {"survival_failed": 1})
|
|
266
|
+
alive = cache[target]
|
|
267
|
+
return _JobResult("reworked" if alive is not None and alive < 0.5 else "merged", alive, False,
|
|
268
|
+
{"survival_computed" if fresh else "survival_cached": 1})
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _apply(o: Outcome, result: _JobResult, stats: Counter[str]) -> None:
|
|
272
|
+
o["state"], o["survival"] = result.state, result.survival
|
|
273
|
+
if result.sticky:
|
|
274
|
+
o["sticky"] = True
|
|
275
|
+
stats.update(result.counts)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def evaluate(rows: list[dict[str, Any]], clones: list[Clone], now_ts: float, cache: dict[str, Any],
|
|
279
|
+
use_gh: bool = False) -> tuple[list[Outcome], Counter[str]]:
|
|
280
|
+
"""One outcome per (sha, repo_key) of the commit rows, as of the epoch now_ts. rows: sha, repo_key, ts (epoch),
|
|
281
|
+
session_id, amend. cache: full sha -> survival (filled in place); gh answers sit under 'gh:', its version under 'v'."""
|
|
282
|
+
stats: Counter[str] = Counter(rows=len(rows))
|
|
283
|
+
anchors = _anchors(rows, stats)
|
|
284
|
+
_resolve(anchors, clones, stats)
|
|
285
|
+
outs: list[Outcome] = []
|
|
286
|
+
gh_outs: list[Outcome] = []
|
|
287
|
+
gh_jobs: list[_GhJob] = []
|
|
288
|
+
survival_outs: list[Outcome] = []
|
|
289
|
+
survival_jobs: list[_SurvivalJob] = []
|
|
290
|
+
for a in anchors:
|
|
291
|
+
o, job = _decide(a, now_ts, cache, use_gh, stats)
|
|
292
|
+
outs.append(o)
|
|
293
|
+
if isinstance(job, _GhJob):
|
|
294
|
+
gh_outs.append(o)
|
|
295
|
+
gh_jobs.append(job)
|
|
296
|
+
elif job:
|
|
297
|
+
survival_outs.append(o)
|
|
298
|
+
survival_jobs.append(job)
|
|
299
|
+
with ThreadPoolExecutor(_WORKERS) as pool: # outcomes and tallies change on this thread only
|
|
300
|
+
for o, result in zip(gh_outs, pool.map(lambda j: _ask_gh(j, now_ts, cache), gh_jobs)):
|
|
301
|
+
_apply(o, result, stats)
|
|
302
|
+
for o, result in zip(survival_outs, pool.map(lambda j: _measure(j, cache), survival_jobs)):
|
|
303
|
+
_apply(o, result, stats)
|
|
304
|
+
# a NULL-key row resolved to a clone can repeat a keyed row of the same commit: keep the keyed
|
|
305
|
+
# anchor's, whatever order the rows came in, because it alone says which repo it means
|
|
306
|
+
kept: dict[tuple[str, str], Outcome] = {}
|
|
307
|
+
for a, o in zip(anchors, outs):
|
|
308
|
+
if a.key is not None or (o["sha"], o["repo_key"]) not in kept:
|
|
309
|
+
kept[(o["sha"], o["repo_key"])] = o
|
|
310
|
+
return list(kept.values()), stats
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def pending(outs: list[Outcome], stored: dict[tuple[str, str], tuple[str, float | None]]) -> list[Outcome]:
|
|
314
|
+
"""The outcomes whose (state, survival) differs from the latest stored row. A sticky outcome
|
|
315
|
+
(this run could not see the commit or decide it) never replaces a stored row of its sha,
|
|
316
|
+
under any repo_key: readers match on the sha alone. The exception is a stored 'reverted' the
|
|
317
|
+
run's reads contradict: an earlier bug counted a revert from before the merge."""
|
|
318
|
+
have = {sha for sha, _ in stored}
|
|
319
|
+
|
|
320
|
+
def holds(o: Outcome) -> bool:
|
|
321
|
+
latest = stored.get((o["sha"], o["repo_key"]))
|
|
322
|
+
wrong = o.get("no_revert") and latest is not None and latest[0] == "reverted"
|
|
323
|
+
return bool(o.get("sticky")) and o["sha"] in have and not wrong
|
|
324
|
+
|
|
325
|
+
return [o for o in outs if not holds(o)
|
|
326
|
+
and stored.get((o["sha"], o["repo_key"])) != (o["state"], o["survival"])]
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _cache_path() -> Path:
|
|
330
|
+
return state_path().parent / CACHE_NAME
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _load_cache() -> dict[str, Any]:
|
|
334
|
+
"""The cache; survivals of an older version are dropped, so they are measured again."""
|
|
335
|
+
cache = load_json_dict(_cache_path())
|
|
336
|
+
cache = {k: v for k, v in cache.items() if not k.startswith("gh:") or v is True}
|
|
337
|
+
if cache.get("v") != _CACHE_V:
|
|
338
|
+
cache = {k: v for k, v in cache.items() if k.startswith("gh:")}
|
|
339
|
+
cache["v"] = _CACHE_V
|
|
340
|
+
return cache
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def load_rows(dsn: str) -> list[dict[str, Any]]:
|
|
344
|
+
with connect(dsn) as conn:
|
|
345
|
+
rows = conn.execute(
|
|
346
|
+
"SELECT session_id, ts, attrs->>'sha', attrs->>'repo_key', "
|
|
347
|
+
"coalesce((attrs->>'amend')::boolean, false) "
|
|
348
|
+
"FROM agents.activity_event WHERE kind = 'commit'").fetchall()
|
|
349
|
+
return [{"session_id": s, "ts": ts.timestamp(), "sha": sha, "repo_key": k, "amend": am}
|
|
350
|
+
for s, ts, sha, k, am in rows]
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def load_stored(dsn: str) -> dict[tuple[str, str], tuple[str, float | None]]:
|
|
354
|
+
with connect(dsn) as conn:
|
|
355
|
+
rows = conn.execute(
|
|
356
|
+
"SELECT DISTINCT ON (sha, repo_key) sha, repo_key, state, survival "
|
|
357
|
+
"FROM agents.commit_outcome ORDER BY sha, repo_key, decided_at DESC").fetchall()
|
|
358
|
+
return {(sha, k): (st, None if sv is None else round(float(sv), 4))
|
|
359
|
+
for sha, k, st, sv in rows}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def insert_outcomes(dsn: str, rows: list[dict[str, Any]]) -> None:
|
|
363
|
+
write_many(dsn, "INSERT INTO agents.commit_outcome (sha, repo_key, state, survival, decided_at) "
|
|
364
|
+
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT DO NOTHING",
|
|
365
|
+
[(r["sha"], r["repo_key"], r["state"], r["survival"], r["decided_at"])
|
|
366
|
+
for r in rows])
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
_READ_FAILURES = ("repo_unreadable", "fetch_failed", "gh_failed", "survival_failed")
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def run_outcomes(cfg: Config, *, refresh: bool, dry_run: bool,
|
|
373
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
374
|
+
"""Score every commit row and write the changed states. refresh (--fetch, outcome_fetch) git
|
|
375
|
+
fetches the clones and turns on gh. A dry run reads the sink and the clones as they are and
|
|
376
|
+
changes nothing: no rows, no cache, no heartbeat, no fetch, no gh, even with refresh."""
|
|
377
|
+
dsn = cfg.require_pg("outcomes (it reads the commit rows)")
|
|
378
|
+
refuse_new_salt(cfg.repo_key_salt, dsn, cfg.machine)
|
|
379
|
+
started = datetime.now(timezone.utc)
|
|
380
|
+
decided = (now or started).astimezone(timezone.utc)
|
|
381
|
+
with job_heartbeat(cfg, "outcomes", started, dry_run=dry_run) as beat:
|
|
382
|
+
rows = load_rows(dsn)
|
|
383
|
+
clones = find_clones(list(cfg.outcome_repo_roots), RepoKeys(cfg.repo_key_salt, create=not dry_run))
|
|
384
|
+
stats: dict[str, Any] = {"clones": len(clones)}
|
|
385
|
+
if refresh and dry_run:
|
|
386
|
+
stats["fetch_skipped_dry_run"] = 1
|
|
387
|
+
refresh = refresh and not dry_run
|
|
388
|
+
if refresh:
|
|
389
|
+
# only repos the agents committed to: fetch talks to the remote, so a clone that
|
|
390
|
+
# merely sits under a root (and holds a matching short SHA) is never fetched
|
|
391
|
+
keys = {r["repo_key"] for r in rows if r["repo_key"]}
|
|
392
|
+
want = [c for c in clones if c.key in keys]
|
|
393
|
+
stats["fetch_ok"], stats["fetch_failed"] = fetch_clones(want)
|
|
394
|
+
cache = _load_cache()
|
|
395
|
+
outs, ev = evaluate(rows, clones, decided.timestamp(), cache, use_gh=refresh)
|
|
396
|
+
stats.update(ev)
|
|
397
|
+
stats["states"] = Counter(o["state"] for o in outs)
|
|
398
|
+
failed = {k: stats[k] for k in _READ_FAILURES if stats.get(k)}
|
|
399
|
+
if failed: # the run still writes what it could decide
|
|
400
|
+
beat.status, beat.error = "degraded", "GitReadFailed"
|
|
401
|
+
print(f"aut: outcomes: some reads failed, those commits stay undecided: "
|
|
402
|
+
f"{json.dumps(failed, sort_keys=True)}", file=sys.stderr)
|
|
403
|
+
if not dry_run:
|
|
404
|
+
atomic_write_text(_cache_path(), json.dumps(cache)) # before the sink: a failing write never redoes the blame
|
|
405
|
+
todo = pending(outs, load_stored(dsn))
|
|
406
|
+
stats["to_write"] = len(todo)
|
|
407
|
+
if not dry_run:
|
|
408
|
+
insert_outcomes(dsn, [{"sha": o["sha"], "repo_key": o["repo_key"], "state": o["state"],
|
|
409
|
+
"survival": o["survival"], "decided_at": decided} for o in todo])
|
|
410
|
+
beat.events = len(todo)
|
|
411
|
+
return stats
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def print_outcomes(report: dict[str, Any], *, dry_run: bool) -> None:
|
|
415
|
+
states = dict(sorted(report["states"].items()))
|
|
416
|
+
rest = {k: v for k, v in sorted(report.items()) if k != "states"}
|
|
417
|
+
print(f"outcomes {'DRY-RUN (nothing written)' if dry_run else 'written'}: "
|
|
418
|
+
f"states {json.dumps(states)}, {json.dumps(rest)}")
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _project_paths(cwd: str) -> list[str]:
|
|
422
|
+
"""cwd and its parents up to the repo's top level (only cwd outside a repo)."""
|
|
423
|
+
top = (run_git(cwd, "rev-parse", "--show-toplevel", timeout=10) or "").strip()
|
|
424
|
+
root = os.path.realpath(top) if top else None
|
|
425
|
+
out, p = [], Path(cwd)
|
|
426
|
+
while True:
|
|
427
|
+
out.append(str(p))
|
|
428
|
+
if not root or os.path.realpath(p) == root or p.parent == p:
|
|
429
|
+
break
|
|
430
|
+
p = p.parent
|
|
431
|
+
return out + ([root] if root and root not in out else [])
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def latest_session(home: Path, cwd: str) -> str | None:
|
|
435
|
+
# shortcut: Claude Code sessions only; upgrade when /outcome runs inside another agent.
|
|
436
|
+
files = [Path(f) for p in _project_paths(cwd)
|
|
437
|
+
for f in claude_code.project_session_files(home, p)]
|
|
438
|
+
for f in sorted(set(files), key=lambda f: f.stat().st_mtime, reverse=True):
|
|
439
|
+
if claude_code.is_interactive(f):
|
|
440
|
+
return f.stem
|
|
441
|
+
return None
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def build_manual_outcome(cfg: Config, outcome: str, cwd: str, session: str | None = None, *,
|
|
445
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
446
|
+
"""One agents.manual_outcome row, or AutError with the reason it was refused."""
|
|
447
|
+
if outcome not in MANUAL:
|
|
448
|
+
raise AutError(f"outcome: one of {', '.join(MANUAL)}")
|
|
449
|
+
sid = session or latest_session(cfg.home, cwd)
|
|
450
|
+
if not sid:
|
|
451
|
+
raise AutError("outcome: no interactive session found for this directory; "
|
|
452
|
+
"pass --session ID")
|
|
453
|
+
refuse_new_salt(cfg.repo_key_salt, cfg.pg_dsn, cfg.machine)
|
|
454
|
+
ts = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
|
455
|
+
return {"session_id": sid, "outcome": MANUAL[outcome], "ts": ts,
|
|
456
|
+
"repo_key": RepoKeys(cfg.repo_key_salt).of_cwd(cwd)}
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def insert_manual_outcome(dsn: str, row: dict[str, Any]) -> None:
|
|
460
|
+
with connect(dsn) as conn:
|
|
461
|
+
conn.execute("INSERT INTO agents.manual_outcome (session_id, outcome, ts, repo_key) "
|
|
462
|
+
"VALUES (%s, %s, %s, %s)",
|
|
463
|
+
(row["session_id"], row["outcome"], row["ts"], row["repo_key"]))
|
|
464
|
+
conn.commit()
|