agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Find known harnesses from bundle names, commands and data-path metadata, never file contents."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from itertools import islice
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
import stat
|
|
11
|
+
import sys
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
MAX_PER_ROOT = 512
|
|
15
|
+
MAX_TOTAL = 1024
|
|
16
|
+
MAX_WALK_PER_ENTRY = 256 # per catalog entry, so a large tree cannot starve later entries
|
|
17
|
+
MAX_WALK_DEPTH = 4
|
|
18
|
+
MAX_PATHS = 8
|
|
19
|
+
MAX_COMMANDS = 4
|
|
20
|
+
RECENT_DAYS = 7
|
|
21
|
+
_COMMAND = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def application_dirs(home: Path) -> tuple[Path, Path]:
|
|
25
|
+
return Path("/Applications"), home / "Applications"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def command_dirs(home: Path) -> list[Path]:
|
|
29
|
+
# launchd's PATH lacks ~/.local/bin, where several harness CLIs live.
|
|
30
|
+
extra = [Path(part) for part in os.environ.get("PATH", "").split(os.pathsep) if part]
|
|
31
|
+
return [home / ".local" / "bin", Path("/opt/homebrew/bin"), Path("/usr/local/bin"), *extra]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _strings(value: object, limit: int) -> list[str]:
|
|
35
|
+
return [item for item in value[:limit] if isinstance(item, str)] if isinstance(value, list) else []
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _detect(entry: dict[str, Any], home: Path) -> tuple[list[Path], list[str]]:
|
|
39
|
+
detect = entry.get("detect")
|
|
40
|
+
detect = detect if isinstance(detect, dict) else {}
|
|
41
|
+
paths = [home / raw[2:] for raw in _strings(detect.get("paths"), MAX_PATHS)
|
|
42
|
+
if raw.startswith("~/") and ".." not in Path(raw).parts]
|
|
43
|
+
commands = [name for name in _strings(detect.get("commands"), MAX_COMMANDS)
|
|
44
|
+
if _COMMAND.fullmatch(name)]
|
|
45
|
+
return paths, commands
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _has_command(dirs: list[Path], name: str) -> bool:
|
|
49
|
+
for directory in dirs:
|
|
50
|
+
candidate = directory / name
|
|
51
|
+
try:
|
|
52
|
+
if stat.S_ISREG(os.stat(candidate).st_mode) and os.access(candidate, os.X_OK):
|
|
53
|
+
return True
|
|
54
|
+
except OSError:
|
|
55
|
+
continue
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _newest(paths: list[Path]) -> tuple[bool, float | None, bool]:
|
|
60
|
+
"""(any path exists, newest file mtime, walk finished within bounds). Reads metadata only.
|
|
61
|
+
Only files date activity: a directory's mtime moves when a skill-sync tool adds `skills/`.
|
|
62
|
+
Skips symlinks and `skills` trees, which those tools touch without any agent use."""
|
|
63
|
+
exists, newest, complete, budget = False, None, True, MAX_WALK_PER_ENTRY
|
|
64
|
+
stack: list[tuple[str, int]] = []
|
|
65
|
+
for path in paths:
|
|
66
|
+
try:
|
|
67
|
+
info = os.lstat(path)
|
|
68
|
+
except OSError:
|
|
69
|
+
continue
|
|
70
|
+
exists = True
|
|
71
|
+
if stat.S_ISLNK(info.st_mode):
|
|
72
|
+
continue
|
|
73
|
+
if stat.S_ISDIR(info.st_mode):
|
|
74
|
+
stack.append((str(path), 1))
|
|
75
|
+
else:
|
|
76
|
+
newest = max(newest or 0., info.st_mtime)
|
|
77
|
+
while stack:
|
|
78
|
+
directory, depth = stack.pop()
|
|
79
|
+
try:
|
|
80
|
+
with os.scandir(directory) as entries:
|
|
81
|
+
for entry in entries:
|
|
82
|
+
if budget <= 0:
|
|
83
|
+
return exists, newest, False
|
|
84
|
+
budget -= 1
|
|
85
|
+
try:
|
|
86
|
+
if entry.is_symlink() or entry.name == "skills":
|
|
87
|
+
continue
|
|
88
|
+
if entry.is_dir(follow_symlinks=False):
|
|
89
|
+
if depth < MAX_WALK_DEPTH:
|
|
90
|
+
stack.append((entry.path, depth + 1))
|
|
91
|
+
else:
|
|
92
|
+
complete = False
|
|
93
|
+
continue
|
|
94
|
+
newest = max(newest or 0., entry.stat(follow_symlinks=False).st_mtime)
|
|
95
|
+
except OSError: # gone or unreadable mid-walk: keep walking the directory
|
|
96
|
+
complete = False
|
|
97
|
+
except OSError:
|
|
98
|
+
complete = False
|
|
99
|
+
return exists, newest, complete
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _name(value: str) -> str:
|
|
103
|
+
value = value.casefold()
|
|
104
|
+
if value.endswith(".app"):
|
|
105
|
+
value = value[:-4]
|
|
106
|
+
value = re.sub(r"[^a-z0-9]", "", value)
|
|
107
|
+
return value[:-4] if value.endswith("beta") else value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _known_names(catalog: list[dict[str, Any]]) -> dict[str, set[str]]:
|
|
111
|
+
known: dict[str, set[str]] = {}
|
|
112
|
+
for entry in catalog[:128]:
|
|
113
|
+
if not isinstance(entry, dict):
|
|
114
|
+
continue
|
|
115
|
+
identifier = entry.get("id")
|
|
116
|
+
if not isinstance(identifier, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,127}", identifier):
|
|
117
|
+
continue
|
|
118
|
+
aliases = entry.get("aliases", [])
|
|
119
|
+
aliases = aliases[:32] if isinstance(aliases, list) else []
|
|
120
|
+
for value in [identifier, entry.get("name"), *aliases]:
|
|
121
|
+
if isinstance(value, str) and len(value) <= 256 and (key := _name(value)):
|
|
122
|
+
known.setdefault(key, set()).add(identifier)
|
|
123
|
+
return known
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def discover(home: Path, catalog: list[dict[str, Any]], *, readers: frozenset[str] = frozenset(),
|
|
127
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
128
|
+
"""Match bundle names, probe declared commands and walk declared data paths, metadata only."""
|
|
129
|
+
report: dict[str, Any] = {
|
|
130
|
+
"status": "unknown", "observations": [], "completeness": "unknown", "roots": [],
|
|
131
|
+
"entries_inspected": 0,
|
|
132
|
+
"limits": {"entries_per_root": MAX_PER_ROOT, "entries_total": MAX_TOTAL},
|
|
133
|
+
"claim_scope": "bundle_name_candidates_do_not_prove_installation_or_use",
|
|
134
|
+
}
|
|
135
|
+
if sys.platform != "darwin":
|
|
136
|
+
report["reason"] = "platform_inventory_unavailable"
|
|
137
|
+
return report
|
|
138
|
+
known = _known_names(catalog)
|
|
139
|
+
found: set[str] = set()
|
|
140
|
+
limited = False
|
|
141
|
+
for index, root in enumerate(application_dirs(home)[:2]):
|
|
142
|
+
probe: dict[str, Any] = {"scope": "system" if index == 0 else "user",
|
|
143
|
+
"status": "available", "entries_inspected": 0}
|
|
144
|
+
remaining = min(MAX_PER_ROOT, MAX_TOTAL - report["entries_inspected"])
|
|
145
|
+
try:
|
|
146
|
+
if root.is_symlink():
|
|
147
|
+
probe["status"] = "symlink_omitted"
|
|
148
|
+
limited = True
|
|
149
|
+
elif remaining <= 0:
|
|
150
|
+
probe["status"] = "limited"
|
|
151
|
+
limited = True
|
|
152
|
+
else:
|
|
153
|
+
with os.scandir(root) as entries:
|
|
154
|
+
for entry in islice(entries, remaining):
|
|
155
|
+
probe["entries_inspected"] += 1
|
|
156
|
+
report["entries_inspected"] += 1
|
|
157
|
+
if not entry.name.casefold().endswith(".app"):
|
|
158
|
+
continue
|
|
159
|
+
if entry.is_symlink():
|
|
160
|
+
probe["status"] = "symlinks_omitted"
|
|
161
|
+
limited = True
|
|
162
|
+
continue
|
|
163
|
+
if not entry.is_dir(follow_symlinks=False):
|
|
164
|
+
continue
|
|
165
|
+
matches = known.get(_name(entry.name), set())
|
|
166
|
+
if len(matches) == 1:
|
|
167
|
+
found.update(matches)
|
|
168
|
+
if probe["entries_inspected"] == remaining:
|
|
169
|
+
# Reaching the bound leaves completeness unknown without reading another name.
|
|
170
|
+
probe["status"] = "limited"
|
|
171
|
+
limited = True
|
|
172
|
+
except FileNotFoundError:
|
|
173
|
+
probe["status"] = "missing"
|
|
174
|
+
except (OSError, ValueError):
|
|
175
|
+
probe["status"] = "unreadable"
|
|
176
|
+
limited = True
|
|
177
|
+
report["roots"].append(probe)
|
|
178
|
+
report["status"] = "limited" if limited else "available"
|
|
179
|
+
evidence: dict[str, set[str]] = {identifier: {"bundle_directory_name"} for identifier in found}
|
|
180
|
+
activity: dict[str, tuple[str, str | None]] = {}
|
|
181
|
+
qualified: set[str] = set()
|
|
182
|
+
dirs = command_dirs(home)
|
|
183
|
+
cutoff = (now or datetime.now(timezone.utc)).timestamp() - RECENT_DAYS * 86400
|
|
184
|
+
for harness in catalog[:128]:
|
|
185
|
+
identifier = harness.get("id") if isinstance(harness, dict) else None
|
|
186
|
+
if not isinstance(identifier, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,127}", identifier):
|
|
187
|
+
continue
|
|
188
|
+
paths, commands = _detect(harness, home)
|
|
189
|
+
if any(_has_command(dirs, name) for name in commands):
|
|
190
|
+
evidence.setdefault(identifier, set()).add("command")
|
|
191
|
+
if set(_strings(harness.get("read_by"), 8)) & readers:
|
|
192
|
+
# Already read by a configured collector: its activity changes nothing, so skip the walk.
|
|
193
|
+
qualified.add(identifier)
|
|
194
|
+
if any(os.path.lexists(path) for path in paths):
|
|
195
|
+
evidence.setdefault(identifier, set()).add("path_metadata")
|
|
196
|
+
continue
|
|
197
|
+
exists, newest, complete = _newest(paths)
|
|
198
|
+
if exists:
|
|
199
|
+
evidence.setdefault(identifier, set()).add("path_metadata")
|
|
200
|
+
if newest is not None and (newest >= cutoff or complete):
|
|
201
|
+
day = datetime.fromtimestamp(newest, timezone.utc).date().isoformat()
|
|
202
|
+
activity[identifier] = ("recent" if newest >= cutoff else "stale", day)
|
|
203
|
+
report["observations"] = [
|
|
204
|
+
{"harness_id": identifier, "evidence": sorted(kinds), "installation": "candidate",
|
|
205
|
+
"activity": activity.get(identifier, ("unknown", None))[0],
|
|
206
|
+
"last_active": activity.get(identifier, ("unknown", None))[1],
|
|
207
|
+
"reader_qualified": identifier in qualified} for identifier, kinds in sorted(evidence.items())]
|
|
208
|
+
return report
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""`aut history-steers`: daily steer estimates for days whose session logs are gone.
|
|
2
|
+
|
|
3
|
+
Claude Code: the 2 March to 19 April 2026 gap. Codex: its whole history file, 11 February to
|
|
4
|
+
2 October 2026, except days that already have counted Codex steers, so no day counts twice.
|
|
5
|
+
The CLI history files still hold what was typed.
|
|
6
|
+
Each day and agent becomes one activity_event row of kind `steer_estimate`, with a count and no
|
|
7
|
+
text. It is a separate kind, so no view counts it as a steer and it can be deleted on its own:
|
|
8
|
+
DELETE FROM agents.activity_event WHERE kind = 'steer_estimate'. The files are only read.
|
|
9
|
+
|
|
10
|
+
Slash and shell commands are left out (the owner's rule), although a logged steer includes them.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from collections import Counter
|
|
17
|
+
from datetime import date, datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from .collectors.claude_code import _is_injected
|
|
21
|
+
from .collectors.codex import is_scheduled_automation
|
|
22
|
+
from .config import Config
|
|
23
|
+
from .event import ActivityEvent, make_event_id
|
|
24
|
+
from .push import send_activity_rows
|
|
25
|
+
from .sink import check_ledger_zone, connect
|
|
26
|
+
|
|
27
|
+
# agent -> (file under home, text key, time key, time units per second, first day, last day)
|
|
28
|
+
SOURCES = {"claude_code": (".claude/history.jsonl", "display", "timestamp", 1000,
|
|
29
|
+
date(2026, 3, 2), date(2026, 4, 19)),
|
|
30
|
+
"codex": (".codex/history.jsonl", "text", "ts", 1,
|
|
31
|
+
date(2026, 2, 11), date(2026, 10, 2))}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def is_steer(text: object) -> bool:
|
|
35
|
+
if not isinstance(text, str) or not text.strip():
|
|
36
|
+
return False
|
|
37
|
+
return text.lstrip()[0] not in "/!" and not _is_injected(text) \
|
|
38
|
+
and not is_scheduled_automation(text)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def daily_counts(path: Path, text_key: str, time_key: str, per_second: int, first: date,
|
|
42
|
+
last: date) -> Counter[date]:
|
|
43
|
+
"""Steers per local day in the window. A missing file is no rows; a bad line is skipped."""
|
|
44
|
+
days: Counter[date] = Counter()
|
|
45
|
+
if not path.is_file():
|
|
46
|
+
return days
|
|
47
|
+
with path.open(encoding="utf-8", errors="replace") as f:
|
|
48
|
+
for line in f:
|
|
49
|
+
try:
|
|
50
|
+
o = json.loads(line)
|
|
51
|
+
day = datetime.fromtimestamp(o[time_key] / per_second).date()
|
|
52
|
+
except (ValueError, KeyError, TypeError, OSError):
|
|
53
|
+
continue
|
|
54
|
+
if first <= day <= last and is_steer(o.get(text_key)):
|
|
55
|
+
days[day] += 1
|
|
56
|
+
return days
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def counted_days(dsn: str, zone: str) -> set[tuple[str, date]]:
|
|
60
|
+
"""(agent, ledger day) pairs that already have counted steers."""
|
|
61
|
+
with connect(dsn) as conn:
|
|
62
|
+
rows = conn.execute("SELECT DISTINCT agent, (ts AT TIME ZONE %s)::date "
|
|
63
|
+
"FROM agents.activity_event WHERE kind = 'steer'", (zone,)).fetchall()
|
|
64
|
+
return {(a, d) for a, d in rows}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def estimates(cfg: Config, counted: set[tuple[str, date]]) -> list[ActivityEvent]:
|
|
68
|
+
rows = []
|
|
69
|
+
for agent, (rel, text_key, time_key, per_second, first, last) in SOURCES.items():
|
|
70
|
+
counts = daily_counts(cfg.home / rel, text_key, time_key, per_second, first, last)
|
|
71
|
+
for day, n in sorted(counts.items()):
|
|
72
|
+
if (agent, day) in counted:
|
|
73
|
+
continue
|
|
74
|
+
rows.append(ActivityEvent(
|
|
75
|
+
event_id=make_event_id(agent, "steer_estimate", day), agent=agent,
|
|
76
|
+
ts=datetime(day.year, day.month, day.day, 12).astimezone().isoformat(),
|
|
77
|
+
kind="steer_estimate", session_id=None, turn_id=None,
|
|
78
|
+
attrs={"count": n, "source": "history"}, machine=cfg.machine))
|
|
79
|
+
return rows
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def run(cfg: Config, dry_run: bool) -> Counter[str]:
|
|
83
|
+
dsn = cfg.require_pg("history-steers")
|
|
84
|
+
check_ledger_zone(dsn, cfg.ledger_timezone)
|
|
85
|
+
rows = estimates(cfg, counted_days(dsn, cfg.ledger_timezone))
|
|
86
|
+
if not dry_run:
|
|
87
|
+
send_activity_rows(dsn, rows)
|
|
88
|
+
return Counter({a: sum(r.attrs["count"] for r in rows if r.agent == a) for a in SOURCES})
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional private evidence, diagnosis, and controlled setup improvements."""
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Inspect frozen local evidence without changing model eligibility or opening raw sources."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from html import escape
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from ..config import AutError
|
|
11
|
+
from ..state import atomic_write_text
|
|
12
|
+
from .store import Store
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _attempt(session: dict[str, Any], case: dict[str, Any]) -> list[dict[str, Any]]:
|
|
16
|
+
try:
|
|
17
|
+
events, selected = session["events"], case["events"]
|
|
18
|
+
if not isinstance(events, list) or not isinstance(selected, list) or not selected \
|
|
19
|
+
or len({event["id"] for event in events}) != len(events) \
|
|
20
|
+
or len({event["id"] for event in selected}) != len(selected):
|
|
21
|
+
raise ValueError
|
|
22
|
+
indices = [events.index(event) for event in selected]
|
|
23
|
+
if indices != sorted(indices):
|
|
24
|
+
raise ValueError
|
|
25
|
+
first = indices[0]
|
|
26
|
+
start = max((i + 1 for i in range(first) if events[i]["kind"] == "boundary"), default=0)
|
|
27
|
+
start = max((i for i in range(start, first + 1) if events[i]["kind"] == "prompt"),
|
|
28
|
+
default=start)
|
|
29
|
+
end = next((i if events[i]["kind"] == "prompt" else i + 1
|
|
30
|
+
for i in range(start + 1, len(events))
|
|
31
|
+
if events[i]["kind"] in {"prompt", "boundary"}), len(events))
|
|
32
|
+
if indices[-1] >= end:
|
|
33
|
+
raise ValueError
|
|
34
|
+
return list(events[start:end])
|
|
35
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
36
|
+
raise AutError("improve: case does not match one captured source attempt") from exc
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def write_case_view(store: Store, case_id: str) -> dict[str, Any]:
|
|
40
|
+
case = store.get("cases", case_id)
|
|
41
|
+
try:
|
|
42
|
+
session = store.get("sessions", case["session_snapshot"])
|
|
43
|
+
except KeyError as exc:
|
|
44
|
+
raise AutError("improve: case has no source snapshot") from exc
|
|
45
|
+
events = _attempt(session, case)
|
|
46
|
+
selected = {event["id"] for event in case["events"]}
|
|
47
|
+
omitted = len(events) - len(selected)
|
|
48
|
+
clipped = sum(bool(event.get("truncated")) for event in events)
|
|
49
|
+
eligible = bool(case.get("coverage", {}).get("complete") and not omitted
|
|
50
|
+
and not case.get("omitted_task_events") and not clipped)
|
|
51
|
+
source = session.get("source", {})
|
|
52
|
+
if not isinstance(source, dict):
|
|
53
|
+
raise AutError("improve: source metadata is not a source record")
|
|
54
|
+
path = Path(str(source.get("path", "")))
|
|
55
|
+
source_link = (f'<a href="{escape(path.as_uri(), quote=True)}">Original local trace</a>'
|
|
56
|
+
if path.is_absolute() else "Original trace link unavailable")
|
|
57
|
+
supplements = []
|
|
58
|
+
for key in session.get("supplemental_sources", []):
|
|
59
|
+
store.get("sessions", key)
|
|
60
|
+
link = store.path("sessions", key).as_uri()
|
|
61
|
+
supplements.append(f'<p><a href="{escape(link, quote=True)}">Linked native snapshot</a>'
|
|
62
|
+
' (supplemental evidence, not a separate judge observation).</p>')
|
|
63
|
+
blocks = []
|
|
64
|
+
labels = {"prompt": "User request", "call": "Tool call", "result": "Tool result",
|
|
65
|
+
"reply": "Assistant reply", "boundary": "Turn boundary", "compaction": "Compaction"}
|
|
66
|
+
for index, event in enumerate(events):
|
|
67
|
+
seen = event["id"] in selected
|
|
68
|
+
location = (f"line {event['line']}" if event.get("line") is not None else
|
|
69
|
+
f"byte {event['byte_offset']}" if event.get("byte_offset") is not None else
|
|
70
|
+
f"{event['table']} record {event['record_id']}" if event.get("record_id") else
|
|
71
|
+
"source position unavailable")
|
|
72
|
+
title = labels.get(event["kind"], str(event["kind"]))
|
|
73
|
+
details = "Used in model input" if seen else "Captured context omitted from model input"
|
|
74
|
+
if event.get("truncated"):
|
|
75
|
+
details += "; body clipped in the frozen snapshot"
|
|
76
|
+
blocks.append(f'<section id="event-{index}" class="{"selected" if seen else "context"}">'
|
|
77
|
+
f'<h2>{escape(title)}</h2><p>{escape(details)}</p>'
|
|
78
|
+
f'<p class="reference">{escape(location)} · event '
|
|
79
|
+
f'<code>{escape(str(event["id"]))}</code> · '
|
|
80
|
+
f'{escape(str(event.get("status", "unknown")))}</p>'
|
|
81
|
+
f'<pre>{escape(str(event.get("text", "")))}</pre></section>')
|
|
82
|
+
status = ("Model input passes basic completeness checks; this does not establish task success."
|
|
83
|
+
if eligible else "Model input remains incomplete; this view does not upgrade a judgment.")
|
|
84
|
+
snapshot = store.path("sessions", case["session_snapshot"])
|
|
85
|
+
page = ('<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
|
86
|
+
'<meta name="viewport" content="width=device-width,initial-scale=1">'
|
|
87
|
+
'<meta name="referrer" content="no-referrer">'
|
|
88
|
+
'<title>AUT private case evidence</title><style>'
|
|
89
|
+
'body{font:16px/1.6 system-ui;margin:0;background:#f6f8fb;color:#172033}'
|
|
90
|
+
'main{max-width:64rem;margin:auto;padding:1.5rem}h1{line-height:1.2}'
|
|
91
|
+
'section{background:white;padding:1rem;margin:1rem 0;border-left:4px solid #aab4c2}'
|
|
92
|
+
'.selected{border-color:#226fb2}.status{padding:1rem;background:#fff1cf}'
|
|
93
|
+
'pre{white-space:pre-wrap;overflow-wrap:anywhere;font-size:.9rem}'
|
|
94
|
+
'code,.reference{overflow-wrap:anywhere}.reference{font-size:.85rem;color:#43516a}'
|
|
95
|
+
'a{color:#175b94}a:focus-visible{outline:3px solid #175b94}'
|
|
96
|
+
'</style></head><body><main><h1>Private case evidence</h1>'
|
|
97
|
+
'<p>Review the captured request, commands, recovery and final reply together.</p>'
|
|
98
|
+
f'<p class="status">{escape(status)}</p>'
|
|
99
|
+
f'<p>Captured events: {len(events)}. Model input events: {len(selected)}. '
|
|
100
|
+
f'Omitted from model input: {omitted}. Clipped bodies: {clipped}.</p>'
|
|
101
|
+
'<details><summary>Source references</summary>'
|
|
102
|
+
'<p>This view uses the frozen snapshot. The original link opens the current local file.</p>'
|
|
103
|
+
f'<p>{source_link}</p><p><code>{escape(str(source.get("path", "unavailable")))}</code></p>'
|
|
104
|
+
f'<p>Recorded source hash: <code>{escape(str(source.get("sha256", "unavailable")))}</code></p>'
|
|
105
|
+
f'<p><a href="{escape(snapshot.as_uri(), quote=True)}">Frozen source snapshot</a></p>'
|
|
106
|
+
+ ''.join(supplements) + '</details>' + ''.join(blocks) + '</main></body></html>')
|
|
107
|
+
target = store.path("case_views", case_id).with_suffix(".html")
|
|
108
|
+
atomic_write_text(target, page)
|
|
109
|
+
return {"case_id": case_id, "path": str(target), "captured_events": len(events),
|
|
110
|
+
"model_events": len(selected), "omitted_model_events": omitted,
|
|
111
|
+
"clipped_events": clipped, "eligible": eligible, "private": True,
|
|
112
|
+
"html_sha256": hashlib.sha256(page.encode()).hexdigest()}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Private improvement commands, independent from the configured sink."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..config import AutError
|
|
12
|
+
from ..label.core import data_dir
|
|
13
|
+
from .proposals import prepare, stage
|
|
14
|
+
from .schedule import freeze_cycle_audit, run_once
|
|
15
|
+
from .store import Store, audit
|
|
16
|
+
from .trial import run_trial
|
|
17
|
+
from .validation import validate
|
|
18
|
+
from . import study_cli
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def register(sub: Any) -> None:
|
|
22
|
+
parser = sub.add_parser("improve", help="private local diagnosis and controlled setup trials")
|
|
23
|
+
parser.add_argument("--root", type=Path, help="private local store (never sent to the sink)")
|
|
24
|
+
actions = parser.add_subparsers(dest="improve_cmd", required=True)
|
|
25
|
+
for name in ("audit", "run"):
|
|
26
|
+
action = actions.add_parser(name, help="freeze local evidence" if name == "audit"
|
|
27
|
+
else "run one locked, bounded local cycle")
|
|
28
|
+
action.add_argument("--days", type=int, default=7)
|
|
29
|
+
action.add_argument("--limit", type=int, default=30)
|
|
30
|
+
if name == "audit":
|
|
31
|
+
action.add_argument("--project", type=Path)
|
|
32
|
+
action.add_argument("--cycle", action="store_true",
|
|
33
|
+
help="freeze the next scheduled cycle's audit unless one is pending")
|
|
34
|
+
action.add_argument("--budget-s", type=int, default=1500,
|
|
35
|
+
help="deadline for the --cycle audit step")
|
|
36
|
+
else:
|
|
37
|
+
action.add_argument("--case-limit", type=int, default=12)
|
|
38
|
+
action.add_argument("--audit-budget-s", type=int, default=0,
|
|
39
|
+
help="extra seconds to build an audit when none is frozen")
|
|
40
|
+
decision = actions.add_parser("diagnose", help="review frozen evidence with cached local models")
|
|
41
|
+
decision.add_argument("--audit", required=True)
|
|
42
|
+
decision.add_argument("--limit", type=int, default=12)
|
|
43
|
+
for action in (decision, actions.choices["run"]):
|
|
44
|
+
action.add_argument("--models", default="pplx-decider-flash,clef-flash")
|
|
45
|
+
action.add_argument("--budget-s", type=int, default=300)
|
|
46
|
+
action.add_argument("--max-calls", type=int, default=40)
|
|
47
|
+
action.add_argument("--explain-model", help="cached local instruct model for evidence review")
|
|
48
|
+
inspection = actions.add_parser("inspect", help="write a private view of captured case evidence")
|
|
49
|
+
inspection.add_argument("--case", required=True)
|
|
50
|
+
proposal = actions.add_parser("propose", help="draft one explicit scoped text replacement")
|
|
51
|
+
proposal.add_argument("--case", required=True)
|
|
52
|
+
proposal.add_argument("--scope", type=Path, required=True)
|
|
53
|
+
proposal.add_argument("--target", type=Path, required=True)
|
|
54
|
+
proposal.add_argument("--before-file", type=Path, required=True)
|
|
55
|
+
proposal.add_argument("--after-file", type=Path, required=True)
|
|
56
|
+
proposal.add_argument("--candidate", type=Path, help="stage only in an independent directory")
|
|
57
|
+
trial = actions.add_parser("trial", help="preview or execute an operator-supplied trial")
|
|
58
|
+
source = trial.add_mutually_exclusive_group(required=True)
|
|
59
|
+
source.add_argument("--spec")
|
|
60
|
+
source.add_argument("--spec-file", type=Path)
|
|
61
|
+
trial.add_argument("--execute", action="store_true", help="execute the trusted argv cases")
|
|
62
|
+
trial.add_argument("--budget-s", type=int, default=600)
|
|
63
|
+
reference = actions.add_parser("validate", help="check judgments against executable references")
|
|
64
|
+
reference.add_argument("--spec-file", type=Path, required=True)
|
|
65
|
+
reference.add_argument("--diagnosis", required=True)
|
|
66
|
+
reference.add_argument("--execute", action="store_true", help="execute the trusted reference oracle")
|
|
67
|
+
desktop = actions.add_parser("desktop", help="import private native-app command evidence")
|
|
68
|
+
desktop.add_argument("--trace", type=Path, required=True, help="raw trace file or segment directory")
|
|
69
|
+
desktop.add_argument("--responses", type=Path, help="local response companion JSONL")
|
|
70
|
+
desktop.add_argument("--mapping", type=Path, required=True, help="explicit task/session/parent JSON")
|
|
71
|
+
study_cli.register(actions)
|
|
72
|
+
actions.add_parser("status", help="show local progress counts without trace text")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def run(args: argparse.Namespace) -> None:
|
|
76
|
+
root = args.root or Path(os.environ.get("AUT_IMPROVE_ROOT", str(data_dir() / "improve")))
|
|
77
|
+
store = Store(root)
|
|
78
|
+
if args.improve_cmd == "study":
|
|
79
|
+
result = study_cli.run(args, store)
|
|
80
|
+
elif args.improve_cmd == "desktop":
|
|
81
|
+
from .desktop import import_desktop
|
|
82
|
+
try:
|
|
83
|
+
mapping = json.loads(args.mapping.read_text())
|
|
84
|
+
except (OSError, ValueError) as exc:
|
|
85
|
+
raise AutError("improve: desktop mapping is not readable JSON") from exc
|
|
86
|
+
if not isinstance(mapping, dict):
|
|
87
|
+
raise AutError("improve: desktop mapping must be a JSON object")
|
|
88
|
+
result = import_desktop(store, args.trace, args.responses, mapping)
|
|
89
|
+
elif args.improve_cmd == "audit" and args.cycle:
|
|
90
|
+
if args.project:
|
|
91
|
+
raise AutError("improve: --cycle freezes the scheduled audit and takes no --project")
|
|
92
|
+
result = freeze_cycle_audit(store, days=args.days, limit=args.limit, budget_s=args.budget_s)
|
|
93
|
+
elif args.improve_cmd == "audit":
|
|
94
|
+
result = audit(store, Path.home(), days=args.days, limit=args.limit, project=args.project)
|
|
95
|
+
elif args.improve_cmd == "diagnose":
|
|
96
|
+
report = run_once(store, audit_id=args.audit, models=tuple(args.models.split(",")),
|
|
97
|
+
case_limit=args.limit, budget_s=args.budget_s, max_calls=args.max_calls,
|
|
98
|
+
explain_model=args.explain_model)
|
|
99
|
+
result = report.get("diagnosis", report)
|
|
100
|
+
elif args.improve_cmd == "inspect":
|
|
101
|
+
from .case_view import write_case_view
|
|
102
|
+
result = write_case_view(store, args.case)
|
|
103
|
+
elif args.improve_cmd == "propose":
|
|
104
|
+
key = prepare(store, args.case, args.scope, args.target,
|
|
105
|
+
args.before_file.read_bytes().decode("utf-8"),
|
|
106
|
+
args.after_file.read_bytes().decode("utf-8"))
|
|
107
|
+
if args.candidate:
|
|
108
|
+
stage(store, key, args.candidate)
|
|
109
|
+
result = {"proposal_id": key, "staged": bool(args.candidate)}
|
|
110
|
+
elif args.improve_cmd == "trial":
|
|
111
|
+
key = args.spec
|
|
112
|
+
if args.spec_file:
|
|
113
|
+
try:
|
|
114
|
+
value = json.loads(args.spec_file.read_text())
|
|
115
|
+
except (OSError, ValueError) as exc:
|
|
116
|
+
raise AutError("improve: trial specification is not readable JSON") from exc
|
|
117
|
+
key = store.put("trial_specs", value)
|
|
118
|
+
result = run_trial(store, key, execute=args.execute, budget_s=args.budget_s)
|
|
119
|
+
elif args.improve_cmd == "run":
|
|
120
|
+
result = run_once(store, days=args.days, limit=args.limit, case_limit=args.case_limit,
|
|
121
|
+
budget_s=args.budget_s, max_calls=args.max_calls,
|
|
122
|
+
models=tuple(args.models.split(",")), explain_model=args.explain_model,
|
|
123
|
+
audit_budget_s=args.audit_budget_s)
|
|
124
|
+
elif args.improve_cmd == "validate":
|
|
125
|
+
try:
|
|
126
|
+
value = json.loads(args.spec_file.read_text())
|
|
127
|
+
except (OSError, ValueError) as exc:
|
|
128
|
+
raise AutError("improve: reference specification is not readable JSON") from exc
|
|
129
|
+
key = store.put("reference_specs", value)
|
|
130
|
+
result = validate(store, key, args.diagnosis, execute=args.execute)
|
|
131
|
+
else:
|
|
132
|
+
kinds = ("audits", "sessions", "cases", "judgments", "proposals", "trial_results")
|
|
133
|
+
result = {"records": {kind: len(store.keys(kind)) for kind in kinds},
|
|
134
|
+
"private": True, "human_verified": False}
|
|
135
|
+
print(json.dumps(result, sort_keys=True))
|
|
136
|
+
if args.improve_cmd in {"run", "diagnose", "audit"} \
|
|
137
|
+
and result.get("status") in {"timeout", "error", "budget"}:
|
|
138
|
+
raise AutError(f"improve: {args.improve_cmd} {result['status']}")
|