agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""Optional measured execution settings. They never change the decision recipe."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import multiprocessing
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import re
|
|
12
|
+
import signal
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from typing import Any, Callable
|
|
18
|
+
|
|
19
|
+
from ..config import config_path
|
|
20
|
+
from .backends import BackendUnavailable, make_backend
|
|
21
|
+
from .core import Choice, Request
|
|
22
|
+
from .local import _number, combine, score_member
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Profile:
|
|
27
|
+
group_size: int
|
|
28
|
+
workers: int
|
|
29
|
+
peaks_mb: tuple[float, ...]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class MemoryAdmissionError(MemoryError):
|
|
33
|
+
"""No workers started; the original serial path remains available."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_profile(task: str, row: dict[str, Any], path: Path | None = None) -> Profile | None:
|
|
37
|
+
path = path or config_path().with_name("label-local-runtime.json")
|
|
38
|
+
try:
|
|
39
|
+
value = json.loads(path.read_text())
|
|
40
|
+
if type(value["version"]) is not int or value["version"] != 1:
|
|
41
|
+
raise ValueError("Unsupported runtime profile")
|
|
42
|
+
entry = value["tasks"].get(task)
|
|
43
|
+
if entry is None:
|
|
44
|
+
return None
|
|
45
|
+
expected = hashlib.sha256(
|
|
46
|
+
json.dumps(row, sort_keys=True, allow_nan=False).encode()
|
|
47
|
+
).hexdigest()
|
|
48
|
+
if entry["recipe_sha256"] != expected:
|
|
49
|
+
raise ValueError("Runtime profile does not match the decision recipe")
|
|
50
|
+
if (
|
|
51
|
+
type(entry["workers"]) is not int
|
|
52
|
+
or entry["workers"] != 2
|
|
53
|
+
or type(entry["group_size"]) is not int
|
|
54
|
+
or not 1 <= entry["group_size"] <= 200
|
|
55
|
+
or not isinstance(entry["benchmark_sha256"], str)
|
|
56
|
+
or not re.fullmatch("[a-f0-9]{64}", entry["benchmark_sha256"])
|
|
57
|
+
):
|
|
58
|
+
raise ValueError("Invalid execution settings")
|
|
59
|
+
members, measured = row["recipe"]["members"], entry["members"]
|
|
60
|
+
if len(measured) != len(members) or len(members) < entry["workers"]:
|
|
61
|
+
raise ValueError("Every member needs a measured peak")
|
|
62
|
+
peaks = []
|
|
63
|
+
for member, observation in zip(members, measured):
|
|
64
|
+
if (
|
|
65
|
+
any(observation[k] != member[k] for k in ("backend", "model_id", "model_rev"))
|
|
66
|
+
or not _number(observation["peak_memory_mb"])
|
|
67
|
+
or observation["peak_memory_mb"] <= 0
|
|
68
|
+
):
|
|
69
|
+
raise ValueError("Runtime member changed")
|
|
70
|
+
peaks.append(float(observation["peak_memory_mb"]))
|
|
71
|
+
return Profile(entry["group_size"], entry["workers"], tuple(peaks))
|
|
72
|
+
except FileNotFoundError:
|
|
73
|
+
return None
|
|
74
|
+
except (OSError, ValueError, KeyError, TypeError, AttributeError):
|
|
75
|
+
print("aut: warning: invalid local runtime profile; using serial scoring", file=sys.stderr)
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def admits(profile: Profile, total: float, available: float) -> bool:
|
|
80
|
+
if (
|
|
81
|
+
not _number(total)
|
|
82
|
+
or not _number(available)
|
|
83
|
+
or not 0 < available <= total
|
|
84
|
+
or not profile.peaks_mb
|
|
85
|
+
or profile.workers > len(profile.peaks_mb)
|
|
86
|
+
or any(not _number(n) or n <= 0 for n in profile.peaks_mb)
|
|
87
|
+
):
|
|
88
|
+
return False
|
|
89
|
+
demand = sum(sorted(profile.peaks_mb, reverse=True)[: profile.workers]) * 1e6 * 1.5
|
|
90
|
+
demand += profile.workers * 2**30
|
|
91
|
+
return available - demand >= total * 0.15
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def memory_headroom() -> tuple[int, float]:
|
|
95
|
+
if sys.platform != "darwin":
|
|
96
|
+
raise MemoryAdmissionError("Parallel memory admission is available on macOS only")
|
|
97
|
+
try:
|
|
98
|
+
total = int(
|
|
99
|
+
subprocess.check_output(["/usr/sbin/sysctl", "-n", "hw.memsize"], text=True, timeout=10)
|
|
100
|
+
)
|
|
101
|
+
output = subprocess.check_output(["/usr/bin/memory_pressure", "-Q"], text=True, timeout=10)
|
|
102
|
+
match = re.search(r"memory free percentage:\s*(\d+)%", output)
|
|
103
|
+
if not match or not 0 <= int(match[1]) <= 100:
|
|
104
|
+
raise ValueError("Invalid free memory measurement")
|
|
105
|
+
return total, total * int(match[1]) / 100
|
|
106
|
+
except (OSError, ValueError, subprocess.SubprocessError) as exc:
|
|
107
|
+
raise MemoryAdmissionError("Parallel memory headroom is unavailable") from exc
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
_STOP: Any = None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _initialize(stop: Any) -> None:
|
|
114
|
+
global _STOP
|
|
115
|
+
_STOP = stop
|
|
116
|
+
for signum in (signal.SIGTERM, signal.SIGINT):
|
|
117
|
+
signal.signal(signum, lambda *_args: stop.set())
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _member(
|
|
121
|
+
index: int,
|
|
122
|
+
member: dict[str, Any],
|
|
123
|
+
requests: list[Request],
|
|
124
|
+
label_model: str,
|
|
125
|
+
llama_server: str,
|
|
126
|
+
deadline: float,
|
|
127
|
+
) -> tuple[int, list[dict[str, float]]]:
|
|
128
|
+
if _STOP is None:
|
|
129
|
+
raise BackendUnavailable("label: parallel worker has no supervisor")
|
|
130
|
+
|
|
131
|
+
def factory(row: dict[str, Any]) -> Any:
|
|
132
|
+
return make_backend(
|
|
133
|
+
requests[0].task,
|
|
134
|
+
Choice(backend=row["backend"], model_id=row["model_id"], model_rev=row["model_rev"]),
|
|
135
|
+
label_model,
|
|
136
|
+
llama_server,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
try:
|
|
140
|
+
return index, score_member(member, requests, deadline, factory, _STOP.is_set)
|
|
141
|
+
except BaseException:
|
|
142
|
+
_STOP.set()
|
|
143
|
+
raise
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
Worker = Callable[
|
|
147
|
+
[int, dict[str, Any], list[Request], str, str, float], tuple[int, list[dict[str, float]]]
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def score_parallel(
|
|
152
|
+
requests: list[Request],
|
|
153
|
+
row: dict[str, Any],
|
|
154
|
+
label_model: str,
|
|
155
|
+
llama_server: str,
|
|
156
|
+
profile: Profile,
|
|
157
|
+
deadline: float,
|
|
158
|
+
*,
|
|
159
|
+
worker: Worker = _member,
|
|
160
|
+
) -> dict[str, dict[str, float]]:
|
|
161
|
+
if (
|
|
162
|
+
not requests
|
|
163
|
+
or len(requests) > profile.group_size
|
|
164
|
+
or len(requests) > 200
|
|
165
|
+
or len({r.unit_id for r in requests}) != len(requests)
|
|
166
|
+
or len({r.task for r in requests}) != 1
|
|
167
|
+
):
|
|
168
|
+
raise BackendUnavailable("label: invalid parallel request group")
|
|
169
|
+
if time.monotonic() >= deadline:
|
|
170
|
+
raise BackendUnavailable("label: local model budget reached; unfinished batch retries")
|
|
171
|
+
if not admits(profile, *memory_headroom()):
|
|
172
|
+
raise MemoryAdmissionError("Insufficient parallel memory headroom")
|
|
173
|
+
context = multiprocessing.get_context("spawn")
|
|
174
|
+
stop = context.Event()
|
|
175
|
+
previous = (
|
|
176
|
+
{s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)}
|
|
177
|
+
if threading.current_thread() is threading.main_thread()
|
|
178
|
+
else {}
|
|
179
|
+
)
|
|
180
|
+
pool = None
|
|
181
|
+
futures = {}
|
|
182
|
+
closing = False
|
|
183
|
+
|
|
184
|
+
def terminate(_signum: int, _frame: Any) -> None:
|
|
185
|
+
stop.set()
|
|
186
|
+
if not closing:
|
|
187
|
+
raise KeyboardInterrupt("Local model scoring interrupted")
|
|
188
|
+
|
|
189
|
+
try:
|
|
190
|
+
for signum in previous:
|
|
191
|
+
signal.signal(signum, terminate)
|
|
192
|
+
pool = ProcessPoolExecutor(
|
|
193
|
+
max_workers=profile.workers,
|
|
194
|
+
mp_context=context,
|
|
195
|
+
initializer=_initialize,
|
|
196
|
+
initargs=(stop,),
|
|
197
|
+
)
|
|
198
|
+
futures = {
|
|
199
|
+
pool.submit(worker, i, member, requests, label_model, llama_server, deadline): i
|
|
200
|
+
for i, member in enumerate(row["recipe"]["members"])
|
|
201
|
+
}
|
|
202
|
+
results = {}
|
|
203
|
+
for future in as_completed(futures, timeout=max(0, deadline - time.monotonic())):
|
|
204
|
+
index, scores = future.result()
|
|
205
|
+
if type(index) is not int or index != futures[future] or len(scores) != len(requests):
|
|
206
|
+
raise BackendUnavailable("label: incomplete parallel member result")
|
|
207
|
+
results[index] = scores
|
|
208
|
+
return {
|
|
209
|
+
request.unit_id: combine(
|
|
210
|
+
row["recipe"]["method"],
|
|
211
|
+
[results[k][i] for k in range(len(futures))],
|
|
212
|
+
row["recipe"].get("stack"),
|
|
213
|
+
)
|
|
214
|
+
for i, request in enumerate(requests)
|
|
215
|
+
}
|
|
216
|
+
except Exception as exc:
|
|
217
|
+
if isinstance(exc, BackendUnavailable):
|
|
218
|
+
raise
|
|
219
|
+
raise BackendUnavailable(
|
|
220
|
+
"label: parallel scoring stopped; unfinished batch retries"
|
|
221
|
+
) from exc
|
|
222
|
+
finally:
|
|
223
|
+
closing = True
|
|
224
|
+
stop.set()
|
|
225
|
+
for future in futures:
|
|
226
|
+
future.cancel()
|
|
227
|
+
try:
|
|
228
|
+
if pool is not None:
|
|
229
|
+
pool.shutdown(wait=True, cancel_futures=True)
|
|
230
|
+
finally:
|
|
231
|
+
for signum, handler in previous.items():
|
|
232
|
+
signal.signal(signum, handler)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Fold a LoRA into its base once, so mlx-lm can load it: run
|
|
2
|
+
|
|
3
|
+
python -m agent_usage_tracker.label.merge evalengine-4b
|
|
4
|
+
|
|
5
|
+
with the extra aut[merge] (torch and peft, needed for this command only) and both repos in the
|
|
6
|
+
Hugging Face cache. It writes core.data_dir()/label/models/<name>; aut then needs only mlx-lm.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import shutil
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .backends import MODELS, cached_snapshot, merged_dir
|
|
17
|
+
|
|
18
|
+
BASES = {"evalengine-4b": ("Qwen/Qwen3.5-4B", "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a")}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def merge(name: str) -> Path:
|
|
22
|
+
import torch
|
|
23
|
+
from huggingface_hub import try_to_load_from_cache
|
|
24
|
+
from peft import PeftModel
|
|
25
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
26
|
+
|
|
27
|
+
m = MODELS[name]
|
|
28
|
+
base_repo, base_rev = BASES[name]
|
|
29
|
+
base = cached_snapshot(try_to_load_from_cache, base_repo, base_rev)
|
|
30
|
+
adapter = try_to_load_from_cache(m.repo, "adapter_config.json", revision=m.revision)
|
|
31
|
+
if base is None or not isinstance(adapter, str):
|
|
32
|
+
raise SystemExit(f"label merge: fetch both first: hf download {base_repo} --revision "
|
|
33
|
+
f"{base_rev}, and hf download {m.repo} --revision {m.revision}")
|
|
34
|
+
# AutoModelForCausalLM: another class silently dropped the adapter, hence the check.
|
|
35
|
+
model = PeftModel.from_pretrained(
|
|
36
|
+
AutoModelForCausalLM.from_pretrained(str(base), dtype=torch.bfloat16),
|
|
37
|
+
str(Path(adapter).parent))
|
|
38
|
+
if not any("lora_" in k for k, _ in model.named_parameters()):
|
|
39
|
+
raise SystemExit(f"label merge: {m.repo} did not attach to {base_repo}; nothing written")
|
|
40
|
+
out = merged_dir(name)
|
|
41
|
+
tmp = out.with_name(out.name + ".tmp")
|
|
42
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
43
|
+
model.merge_and_unload().save_pretrained(tmp)
|
|
44
|
+
AutoTokenizer.from_pretrained(str(base)).save_pretrained(tmp)
|
|
45
|
+
cfg = json.loads((tmp / "config.json").read_text())
|
|
46
|
+
(tmp / "config.json").write_text(json.dumps({**cfg, "model_type": "qwen3_5"}, indent=2))
|
|
47
|
+
(tmp / "aut_merge.json").write_text(json.dumps(
|
|
48
|
+
{"adapter": m.repo, "adapter_rev": m.revision, "base": base_repo, "base_rev": base_rev}))
|
|
49
|
+
shutil.rmtree(out, ignore_errors=True)
|
|
50
|
+
tmp.rename(out) # a stopped merge leaves nothing that looks finished
|
|
51
|
+
return out
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
if __name__ == "__main__":
|
|
55
|
+
if len(sys.argv) != 2 or sys.argv[1] not in BASES:
|
|
56
|
+
raise SystemExit(f"usage: python -m agent_usage_tracker.label.merge {{{','.join(BASES)}}}")
|
|
57
|
+
print(merge(sys.argv[1]))
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Pure recent/mature planner used by opt-in local coverage; activation defaults off."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from collections import deque
|
|
8
|
+
from dataclasses import asdict
|
|
9
|
+
|
|
10
|
+
from .core import Request
|
|
11
|
+
from .tasks import OUTCOME_DAYS
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def coverage_key(request: Request, policy_id: str) -> tuple[str, bool]:
|
|
15
|
+
"""A delivered mature request stays done as idle days increase, until its evidence changes."""
|
|
16
|
+
age = request.facts.get("days_idle")
|
|
17
|
+
if request.task != "soft_outcome" or type(age) is not int or age < 0:
|
|
18
|
+
raise ValueError("outcome coverage requires a nonnegative integer idle age")
|
|
19
|
+
mature = age >= OUTCOME_DAYS
|
|
20
|
+
value = asdict(request)
|
|
21
|
+
if mature:
|
|
22
|
+
del value["facts"]["days_idle"]
|
|
23
|
+
encoded = json.dumps({"request": value, "policy": policy_id,
|
|
24
|
+
"phase": "mature" if mature else "recent"},
|
|
25
|
+
sort_keys=True, allow_nan=False).encode()
|
|
26
|
+
return hashlib.sha256(encoded).hexdigest(), mature
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def pending_outcomes(requests: list[Request], acknowledged: set[str],
|
|
30
|
+
policy_id: str, *, start_mature: bool = False) -> list[tuple[str, Request]]:
|
|
31
|
+
"""Interleave both queues, starting with the caller's persisted next phase.
|
|
32
|
+
|
|
33
|
+
Callers must freeze one source snapshot and record keys only after delivery succeeds.
|
|
34
|
+
Persist the opposite of the last delivered phase for the next invocation. Restarting
|
|
35
|
+
recent-first each night cannot guarantee fairness with a one-request budget.
|
|
36
|
+
Unknown predictions are delivered attempts too; this planner never interprets a label.
|
|
37
|
+
"""
|
|
38
|
+
recent: deque[tuple[str, Request]] = deque()
|
|
39
|
+
mature: deque[tuple[str, Request]] = deque()
|
|
40
|
+
seen: dict[tuple[str, str], str] = {}
|
|
41
|
+
for request in sorted(requests, key=lambda r: (r.ts, r.unit_id)):
|
|
42
|
+
key, is_mature = coverage_key(request, policy_id)
|
|
43
|
+
identity = request.unit, request.unit_id
|
|
44
|
+
if identity in seen:
|
|
45
|
+
if seen[identity] != key:
|
|
46
|
+
raise ValueError("outcome coverage requires one current snapshot per unit")
|
|
47
|
+
continue
|
|
48
|
+
seen[identity] = key
|
|
49
|
+
if key not in acknowledged:
|
|
50
|
+
(mature if is_mature else recent).append((key, request))
|
|
51
|
+
out = []
|
|
52
|
+
queues = (mature, recent) if start_mature else (recent, mature)
|
|
53
|
+
while recent or mature:
|
|
54
|
+
for queue in queues:
|
|
55
|
+
if queue:
|
|
56
|
+
out.append(queue.popleft())
|
|
57
|
+
return out
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Private, bounded historical jobs. No teacher calls or nightly watermark changes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
import time
|
|
9
|
+
import uuid
|
|
10
|
+
from dataclasses import asdict
|
|
11
|
+
from datetime import date, datetime, timezone
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Callable
|
|
14
|
+
|
|
15
|
+
from ..config import AutError, Config
|
|
16
|
+
from ..push import send_activity_rows
|
|
17
|
+
from ..repo_keys import RepoKeys
|
|
18
|
+
from ..state import LockBusy, _locked, atomic_write_text
|
|
19
|
+
from .backends import BackendUnavailable
|
|
20
|
+
from .core import Choice, Request, data_dir, decide, label_event, local_midnight
|
|
21
|
+
from .local import (TASK_NAMES, LocalBackend, decorate, load_policy, reserve_label_time,
|
|
22
|
+
writing_policy_path)
|
|
23
|
+
from .tasks import Source, units
|
|
24
|
+
|
|
25
|
+
BATCH = 10
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _digest(value: Any) -> str:
|
|
29
|
+
return hashlib.sha256(json.dumps(value, sort_keys=True, allow_nan=False).encode()).hexdigest()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _write(path: Path, value: Any) -> None:
|
|
33
|
+
atomic_write_text(path, json.dumps(value, ensure_ascii=False, allow_nan=False))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _identity(cfg: Config) -> str:
|
|
37
|
+
value = RepoKeys(cfg.repo_key_salt, create=False).of("aut-label-replay")
|
|
38
|
+
if not value:
|
|
39
|
+
raise AutError("label replay: restore the existing repository salt before preparing a job")
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _directory(job_id: str) -> Path:
|
|
44
|
+
if not re.fullmatch(r"[0-9a-f]{32}", job_id):
|
|
45
|
+
raise AutError("label replay: invalid job ID")
|
|
46
|
+
return data_dir() / "label" / "replay" / job_id
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read(path: Path) -> dict[str, Any]:
|
|
50
|
+
try:
|
|
51
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
52
|
+
if not isinstance(value, dict):
|
|
53
|
+
raise ValueError("not an object")
|
|
54
|
+
return value
|
|
55
|
+
except (OSError, ValueError) as exc:
|
|
56
|
+
raise AutError("label replay: job state is missing or invalid; it was not reset") from exc
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _report(manifest: dict[str, Any], progress: dict[str, Any], delivered: int = 0,
|
|
60
|
+
reason: str | None = None) -> dict[str, Any]:
|
|
61
|
+
total = len(manifest["requests"])
|
|
62
|
+
return {"job_id": manifest["job_id"], "task": manifest["task"],
|
|
63
|
+
"since": manifest["since"], "until": manifest["until"], "total": total,
|
|
64
|
+
"missing_local_text": manifest["missing_local_text"],
|
|
65
|
+
"partial_local_text": manifest["partial_local_text"],
|
|
66
|
+
"incomplete_outcome_context": manifest.get("incomplete_outcome_context", 0),
|
|
67
|
+
"delivered": delivered, "remaining": total - progress["next"],
|
|
68
|
+
"complete": progress["next"] == total, "reason": reason}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def prepare(cfg: Config, task: str, since: date, until: date, *, src: Source | None = None,
|
|
72
|
+
now: datetime | None = None) -> dict[str, Any]:
|
|
73
|
+
"""Freeze redacted local inputs and preview counts, without scoring or sink writes."""
|
|
74
|
+
now = now or datetime.now(timezone.utc)
|
|
75
|
+
if task not in TASK_NAMES or since >= until or until > now.astimezone().date():
|
|
76
|
+
raise AutError("label replay: choose a local task and a nonempty range of completed dates")
|
|
77
|
+
cfg.require_pg("label replay preview")
|
|
78
|
+
identity = _identity(cfg)
|
|
79
|
+
policy, policy_id = load_policy(writing_policy_path()) if task == "writing_activity" else load_policy()
|
|
80
|
+
if task not in policy["tasks"]:
|
|
81
|
+
raise AutError("label replay: this task has no saved local recipe")
|
|
82
|
+
start, end = local_midnight(since), local_midnight(until)
|
|
83
|
+
src = src or Source(cfg, local_context=True)
|
|
84
|
+
missing = len(getattr(src, "missing_text_units", []))
|
|
85
|
+
partial = len(getattr(src, "partial_text_units", []))
|
|
86
|
+
incomplete = len(getattr(src, "incomplete_outcome_units", []))
|
|
87
|
+
# Outcome facts describe the current snapshot; daily work stops at the selected date.
|
|
88
|
+
reqs = [r for r in units(task, src, start, now if task == "soft_outcome" else end)
|
|
89
|
+
if start <= datetime.fromisoformat(r.ts) < end]
|
|
90
|
+
reqs.sort(key=lambda r: (r.ts, r.unit, r.unit_id))
|
|
91
|
+
if len({(r.task, r.unit, r.unit_id) for r in reqs}) != len(reqs):
|
|
92
|
+
raise AutError("label replay: duplicate unit identities in the preview")
|
|
93
|
+
job_id = uuid.uuid4().hex
|
|
94
|
+
directory = _directory(job_id)
|
|
95
|
+
labelled_at = reserve_label_time(directory.parent, now)
|
|
96
|
+
manifest = {"version": 1, "job_id": job_id, "task": task, "since": since.isoformat(),
|
|
97
|
+
"until": until.isoformat(), "asof": now.astimezone(timezone.utc).isoformat(),
|
|
98
|
+
"labelled_at": labelled_at, "machine": cfg.machine, "identity": identity,
|
|
99
|
+
"policy_id": policy_id, "task_policy": policy["tasks"][task],
|
|
100
|
+
"missing_local_text": sum(start <= ts < end for ts in
|
|
101
|
+
getattr(src, "missing_text_units", [])[missing:]),
|
|
102
|
+
"partial_local_text": sum(start <= ts < end for ts in
|
|
103
|
+
getattr(src, "partial_text_units", [])[partial:]),
|
|
104
|
+
"incomplete_outcome_context": sum(start <= ts < end for ts in
|
|
105
|
+
getattr(src, "incomplete_outcome_units", [])[incomplete:]),
|
|
106
|
+
"requests": [asdict(r) for r in reqs]}
|
|
107
|
+
progress = {"manifest_sha256": _digest(manifest), "next": 0}
|
|
108
|
+
directory.mkdir(mode=0o700)
|
|
109
|
+
_write(directory / "manifest.json", manifest)
|
|
110
|
+
_write(directory / "progress.json", progress)
|
|
111
|
+
return _report(manifest, progress)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def resume(cfg: Config, job_id: str, *, apply: bool = False, budget_s: int = 600,
|
|
115
|
+
backend_for: Callable[[str, dict[str, Any], str, Config], LocalBackend] | None = None,
|
|
116
|
+
) -> dict[str, Any]:
|
|
117
|
+
"""Resume one private generation. Only acknowledged delivery advances its cursor."""
|
|
118
|
+
if type(budget_s) is not int or not 1 <= budget_s <= 3600:
|
|
119
|
+
raise AutError("label replay: budget must be between 1 and 3600 seconds")
|
|
120
|
+
directory = _directory(job_id)
|
|
121
|
+
if not directory.is_dir():
|
|
122
|
+
raise AutError("label replay: job does not exist")
|
|
123
|
+
progress_path = directory / "progress.json"
|
|
124
|
+
try:
|
|
125
|
+
with _locked(progress_path, blocking=False):
|
|
126
|
+
return _resume(cfg, job_id, directory, apply, budget_s, backend_for)
|
|
127
|
+
except LockBusy as exc:
|
|
128
|
+
raise AutError("label replay: this job is already running") from exc
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _resume(cfg: Config, job_id: str, directory: Path, apply: bool, budget_s: int,
|
|
132
|
+
backend_for: Callable[[str, dict[str, Any], str, Config], LocalBackend] | None,
|
|
133
|
+
) -> dict[str, Any]:
|
|
134
|
+
manifest = _read(directory / "manifest.json")
|
|
135
|
+
progress = _read(directory / "progress.json")
|
|
136
|
+
if (manifest.get("version") != 1 or manifest.get("job_id") != job_id
|
|
137
|
+
or progress.get("manifest_sha256") != _digest(manifest)
|
|
138
|
+
or type(progress.get("next")) is not int
|
|
139
|
+
or not 0 <= progress["next"] <= len(manifest.get("requests", []))):
|
|
140
|
+
raise AutError("label replay: job snapshot or cursor changed; refusing to continue")
|
|
141
|
+
if manifest["machine"] != cfg.machine or manifest["identity"] != _identity(cfg):
|
|
142
|
+
raise AutError("label replay: machine identity or repository salt changed")
|
|
143
|
+
task = manifest["task"]
|
|
144
|
+
policy, _current_id = load_policy(writing_policy_path()) if task == "writing_activity" else load_policy()
|
|
145
|
+
if _digest(policy["tasks"].get(task)) != _digest(manifest["task_policy"]):
|
|
146
|
+
raise AutError("label replay: the task recipe changed; prepare a new job")
|
|
147
|
+
if not apply or progress["next"] == len(manifest["requests"]):
|
|
148
|
+
return _report(manifest, progress)
|
|
149
|
+
dsn = cfg.require_pg("label replay")
|
|
150
|
+
backend = (backend_for or LocalBackend)(task, manifest["task_policy"],
|
|
151
|
+
manifest["policy_id"], cfg)
|
|
152
|
+
deadline = time.monotonic() + budget_s
|
|
153
|
+
delivered = 0
|
|
154
|
+
prepared: list[Request] = []
|
|
155
|
+
prepared_start = progress["next"]
|
|
156
|
+
try:
|
|
157
|
+
while progress["next"] < len(manifest["requests"]):
|
|
158
|
+
if time.monotonic() >= deadline:
|
|
159
|
+
return _report(manifest, progress, delivered, "budget")
|
|
160
|
+
if not prepared or progress["next"] >= prepared_start + len(prepared):
|
|
161
|
+
prepared_start = progress["next"]
|
|
162
|
+
group_size = getattr(getattr(backend, "runtime", None), "group_size", BATCH)
|
|
163
|
+
prefetch = max(BATCH, group_size // BATCH * BATCH)
|
|
164
|
+
prepared = [
|
|
165
|
+
Request(**r)
|
|
166
|
+
for r in manifest["requests"][prepared_start : prepared_start + prefetch]
|
|
167
|
+
]
|
|
168
|
+
for req in prepared:
|
|
169
|
+
req.rev = f"replay:{job_id}:{req.rev or ''}"
|
|
170
|
+
backend.prepare_requests(prepared, deadline)
|
|
171
|
+
offset = progress["next"] - prepared_start
|
|
172
|
+
chunk = prepared[offset : offset + BATCH]
|
|
173
|
+
events = []
|
|
174
|
+
for req in chunk:
|
|
175
|
+
if time.monotonic() >= deadline:
|
|
176
|
+
return _report(manifest, progress, delivered, "budget")
|
|
177
|
+
raw = backend.distribution(req)
|
|
178
|
+
response = decide(req, raw, Choice(), backend.model_id, backend.model_rev)
|
|
179
|
+
event = decorate(label_event(req, response, manifest["labelled_at"]), req,
|
|
180
|
+
backend, response.prob)
|
|
181
|
+
event.machine = cfg.machine
|
|
182
|
+
event.attrs.update(replay_job_id=job_id, input_revision=req.rev)
|
|
183
|
+
events.append(event)
|
|
184
|
+
send_activity_rows(dsn, events)
|
|
185
|
+
progress["next"] += len(chunk)
|
|
186
|
+
_write(directory / "progress.json", progress)
|
|
187
|
+
delivered += len(chunk)
|
|
188
|
+
except BackendUnavailable:
|
|
189
|
+
if time.monotonic() >= deadline:
|
|
190
|
+
return _report(manifest, progress, delivered, "budget")
|
|
191
|
+
raise
|
|
192
|
+
finally:
|
|
193
|
+
backend.close()
|
|
194
|
+
return _report(manifest, progress, delivered)
|