agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,232 @@
1
+ """Optional measured execution settings. They never change the decision recipe."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from concurrent.futures import ProcessPoolExecutor, as_completed
6
+ from dataclasses import dataclass
7
+ import hashlib
8
+ import json
9
+ import multiprocessing
10
+ from pathlib import Path
11
+ import re
12
+ import signal
13
+ import subprocess
14
+ import sys
15
+ import threading
16
+ import time
17
+ from typing import Any, Callable
18
+
19
+ from ..config import config_path
20
+ from .backends import BackendUnavailable, make_backend
21
+ from .core import Choice, Request
22
+ from .local import _number, combine, score_member
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class Profile:
27
+ group_size: int
28
+ workers: int
29
+ peaks_mb: tuple[float, ...]
30
+
31
+
32
+ class MemoryAdmissionError(MemoryError):
33
+ """No workers started; the original serial path remains available."""
34
+
35
+
36
+ def load_profile(task: str, row: dict[str, Any], path: Path | None = None) -> Profile | None:
37
+ path = path or config_path().with_name("label-local-runtime.json")
38
+ try:
39
+ value = json.loads(path.read_text())
40
+ if type(value["version"]) is not int or value["version"] != 1:
41
+ raise ValueError("Unsupported runtime profile")
42
+ entry = value["tasks"].get(task)
43
+ if entry is None:
44
+ return None
45
+ expected = hashlib.sha256(
46
+ json.dumps(row, sort_keys=True, allow_nan=False).encode()
47
+ ).hexdigest()
48
+ if entry["recipe_sha256"] != expected:
49
+ raise ValueError("Runtime profile does not match the decision recipe")
50
+ if (
51
+ type(entry["workers"]) is not int
52
+ or entry["workers"] != 2
53
+ or type(entry["group_size"]) is not int
54
+ or not 1 <= entry["group_size"] <= 200
55
+ or not isinstance(entry["benchmark_sha256"], str)
56
+ or not re.fullmatch("[a-f0-9]{64}", entry["benchmark_sha256"])
57
+ ):
58
+ raise ValueError("Invalid execution settings")
59
+ members, measured = row["recipe"]["members"], entry["members"]
60
+ if len(measured) != len(members) or len(members) < entry["workers"]:
61
+ raise ValueError("Every member needs a measured peak")
62
+ peaks = []
63
+ for member, observation in zip(members, measured):
64
+ if (
65
+ any(observation[k] != member[k] for k in ("backend", "model_id", "model_rev"))
66
+ or not _number(observation["peak_memory_mb"])
67
+ or observation["peak_memory_mb"] <= 0
68
+ ):
69
+ raise ValueError("Runtime member changed")
70
+ peaks.append(float(observation["peak_memory_mb"]))
71
+ return Profile(entry["group_size"], entry["workers"], tuple(peaks))
72
+ except FileNotFoundError:
73
+ return None
74
+ except (OSError, ValueError, KeyError, TypeError, AttributeError):
75
+ print("aut: warning: invalid local runtime profile; using serial scoring", file=sys.stderr)
76
+ return None
77
+
78
+
79
+ def admits(profile: Profile, total: float, available: float) -> bool:
80
+ if (
81
+ not _number(total)
82
+ or not _number(available)
83
+ or not 0 < available <= total
84
+ or not profile.peaks_mb
85
+ or profile.workers > len(profile.peaks_mb)
86
+ or any(not _number(n) or n <= 0 for n in profile.peaks_mb)
87
+ ):
88
+ return False
89
+ demand = sum(sorted(profile.peaks_mb, reverse=True)[: profile.workers]) * 1e6 * 1.5
90
+ demand += profile.workers * 2**30
91
+ return available - demand >= total * 0.15
92
+
93
+
94
+ def memory_headroom() -> tuple[int, float]:
95
+ if sys.platform != "darwin":
96
+ raise MemoryAdmissionError("Parallel memory admission is available on macOS only")
97
+ try:
98
+ total = int(
99
+ subprocess.check_output(["/usr/sbin/sysctl", "-n", "hw.memsize"], text=True, timeout=10)
100
+ )
101
+ output = subprocess.check_output(["/usr/bin/memory_pressure", "-Q"], text=True, timeout=10)
102
+ match = re.search(r"memory free percentage:\s*(\d+)%", output)
103
+ if not match or not 0 <= int(match[1]) <= 100:
104
+ raise ValueError("Invalid free memory measurement")
105
+ return total, total * int(match[1]) / 100
106
+ except (OSError, ValueError, subprocess.SubprocessError) as exc:
107
+ raise MemoryAdmissionError("Parallel memory headroom is unavailable") from exc
108
+
109
+
110
+ _STOP: Any = None
111
+
112
+
113
+ def _initialize(stop: Any) -> None:
114
+ global _STOP
115
+ _STOP = stop
116
+ for signum in (signal.SIGTERM, signal.SIGINT):
117
+ signal.signal(signum, lambda *_args: stop.set())
118
+
119
+
120
+ def _member(
121
+ index: int,
122
+ member: dict[str, Any],
123
+ requests: list[Request],
124
+ label_model: str,
125
+ llama_server: str,
126
+ deadline: float,
127
+ ) -> tuple[int, list[dict[str, float]]]:
128
+ if _STOP is None:
129
+ raise BackendUnavailable("label: parallel worker has no supervisor")
130
+
131
+ def factory(row: dict[str, Any]) -> Any:
132
+ return make_backend(
133
+ requests[0].task,
134
+ Choice(backend=row["backend"], model_id=row["model_id"], model_rev=row["model_rev"]),
135
+ label_model,
136
+ llama_server,
137
+ )
138
+
139
+ try:
140
+ return index, score_member(member, requests, deadline, factory, _STOP.is_set)
141
+ except BaseException:
142
+ _STOP.set()
143
+ raise
144
+
145
+
146
+ Worker = Callable[
147
+ [int, dict[str, Any], list[Request], str, str, float], tuple[int, list[dict[str, float]]]
148
+ ]
149
+
150
+
151
+ def score_parallel(
152
+ requests: list[Request],
153
+ row: dict[str, Any],
154
+ label_model: str,
155
+ llama_server: str,
156
+ profile: Profile,
157
+ deadline: float,
158
+ *,
159
+ worker: Worker = _member,
160
+ ) -> dict[str, dict[str, float]]:
161
+ if (
162
+ not requests
163
+ or len(requests) > profile.group_size
164
+ or len(requests) > 200
165
+ or len({r.unit_id for r in requests}) != len(requests)
166
+ or len({r.task for r in requests}) != 1
167
+ ):
168
+ raise BackendUnavailable("label: invalid parallel request group")
169
+ if time.monotonic() >= deadline:
170
+ raise BackendUnavailable("label: local model budget reached; unfinished batch retries")
171
+ if not admits(profile, *memory_headroom()):
172
+ raise MemoryAdmissionError("Insufficient parallel memory headroom")
173
+ context = multiprocessing.get_context("spawn")
174
+ stop = context.Event()
175
+ previous = (
176
+ {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)}
177
+ if threading.current_thread() is threading.main_thread()
178
+ else {}
179
+ )
180
+ pool = None
181
+ futures = {}
182
+ closing = False
183
+
184
+ def terminate(_signum: int, _frame: Any) -> None:
185
+ stop.set()
186
+ if not closing:
187
+ raise KeyboardInterrupt("Local model scoring interrupted")
188
+
189
+ try:
190
+ for signum in previous:
191
+ signal.signal(signum, terminate)
192
+ pool = ProcessPoolExecutor(
193
+ max_workers=profile.workers,
194
+ mp_context=context,
195
+ initializer=_initialize,
196
+ initargs=(stop,),
197
+ )
198
+ futures = {
199
+ pool.submit(worker, i, member, requests, label_model, llama_server, deadline): i
200
+ for i, member in enumerate(row["recipe"]["members"])
201
+ }
202
+ results = {}
203
+ for future in as_completed(futures, timeout=max(0, deadline - time.monotonic())):
204
+ index, scores = future.result()
205
+ if type(index) is not int or index != futures[future] or len(scores) != len(requests):
206
+ raise BackendUnavailable("label: incomplete parallel member result")
207
+ results[index] = scores
208
+ return {
209
+ request.unit_id: combine(
210
+ row["recipe"]["method"],
211
+ [results[k][i] for k in range(len(futures))],
212
+ row["recipe"].get("stack"),
213
+ )
214
+ for i, request in enumerate(requests)
215
+ }
216
+ except Exception as exc:
217
+ if isinstance(exc, BackendUnavailable):
218
+ raise
219
+ raise BackendUnavailable(
220
+ "label: parallel scoring stopped; unfinished batch retries"
221
+ ) from exc
222
+ finally:
223
+ closing = True
224
+ stop.set()
225
+ for future in futures:
226
+ future.cancel()
227
+ try:
228
+ if pool is not None:
229
+ pool.shutdown(wait=True, cancel_futures=True)
230
+ finally:
231
+ for signum, handler in previous.items():
232
+ signal.signal(signum, handler)
@@ -0,0 +1,57 @@
1
+ """Fold a LoRA into its base once, so mlx-lm can load it: run
2
+
3
+ python -m agent_usage_tracker.label.merge evalengine-4b
4
+
5
+ with the extra aut[merge] (torch and peft, needed for this command only) and both repos in the
6
+ Hugging Face cache. It writes core.data_dir()/label/models/<name>; aut then needs only mlx-lm.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import shutil
13
+ import sys
14
+ from pathlib import Path
15
+
16
+ from .backends import MODELS, cached_snapshot, merged_dir
17
+
18
+ BASES = {"evalengine-4b": ("Qwen/Qwen3.5-4B", "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a")}
19
+
20
+
21
+ def merge(name: str) -> Path:
22
+ import torch
23
+ from huggingface_hub import try_to_load_from_cache
24
+ from peft import PeftModel
25
+ from transformers import AutoModelForCausalLM, AutoTokenizer
26
+
27
+ m = MODELS[name]
28
+ base_repo, base_rev = BASES[name]
29
+ base = cached_snapshot(try_to_load_from_cache, base_repo, base_rev)
30
+ adapter = try_to_load_from_cache(m.repo, "adapter_config.json", revision=m.revision)
31
+ if base is None or not isinstance(adapter, str):
32
+ raise SystemExit(f"label merge: fetch both first: hf download {base_repo} --revision "
33
+ f"{base_rev}, and hf download {m.repo} --revision {m.revision}")
34
+ # AutoModelForCausalLM: another class silently dropped the adapter, hence the check.
35
+ model = PeftModel.from_pretrained(
36
+ AutoModelForCausalLM.from_pretrained(str(base), dtype=torch.bfloat16),
37
+ str(Path(adapter).parent))
38
+ if not any("lora_" in k for k, _ in model.named_parameters()):
39
+ raise SystemExit(f"label merge: {m.repo} did not attach to {base_repo}; nothing written")
40
+ out = merged_dir(name)
41
+ tmp = out.with_name(out.name + ".tmp")
42
+ shutil.rmtree(tmp, ignore_errors=True)
43
+ model.merge_and_unload().save_pretrained(tmp)
44
+ AutoTokenizer.from_pretrained(str(base)).save_pretrained(tmp)
45
+ cfg = json.loads((tmp / "config.json").read_text())
46
+ (tmp / "config.json").write_text(json.dumps({**cfg, "model_type": "qwen3_5"}, indent=2))
47
+ (tmp / "aut_merge.json").write_text(json.dumps(
48
+ {"adapter": m.repo, "adapter_rev": m.revision, "base": base_repo, "base_rev": base_rev}))
49
+ shutil.rmtree(out, ignore_errors=True)
50
+ tmp.rename(out) # a stopped merge leaves nothing that looks finished
51
+ return out
52
+
53
+
54
+ if __name__ == "__main__":
55
+ if len(sys.argv) != 2 or sys.argv[1] not in BASES:
56
+ raise SystemExit(f"usage: python -m agent_usage_tracker.label.merge {{{','.join(BASES)}}}")
57
+ print(merge(sys.argv[1]))
@@ -0,0 +1,57 @@
1
+ """Pure recent/mature planner used by opt-in local coverage; activation defaults off."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from collections import deque
8
+ from dataclasses import asdict
9
+
10
+ from .core import Request
11
+ from .tasks import OUTCOME_DAYS
12
+
13
+
14
+ def coverage_key(request: Request, policy_id: str) -> tuple[str, bool]:
15
+ """A delivered mature request stays done as idle days increase, until its evidence changes."""
16
+ age = request.facts.get("days_idle")
17
+ if request.task != "soft_outcome" or type(age) is not int or age < 0:
18
+ raise ValueError("outcome coverage requires a nonnegative integer idle age")
19
+ mature = age >= OUTCOME_DAYS
20
+ value = asdict(request)
21
+ if mature:
22
+ del value["facts"]["days_idle"]
23
+ encoded = json.dumps({"request": value, "policy": policy_id,
24
+ "phase": "mature" if mature else "recent"},
25
+ sort_keys=True, allow_nan=False).encode()
26
+ return hashlib.sha256(encoded).hexdigest(), mature
27
+
28
+
29
+ def pending_outcomes(requests: list[Request], acknowledged: set[str],
30
+ policy_id: str, *, start_mature: bool = False) -> list[tuple[str, Request]]:
31
+ """Interleave both queues, starting with the caller's persisted next phase.
32
+
33
+ Callers must freeze one source snapshot and record keys only after delivery succeeds.
34
+ Persist the opposite of the last delivered phase for the next invocation. Restarting
35
+ recent-first each night cannot guarantee fairness with a one-request budget.
36
+ Unknown predictions are delivered attempts too; this planner never interprets a label.
37
+ """
38
+ recent: deque[tuple[str, Request]] = deque()
39
+ mature: deque[tuple[str, Request]] = deque()
40
+ seen: dict[tuple[str, str], str] = {}
41
+ for request in sorted(requests, key=lambda r: (r.ts, r.unit_id)):
42
+ key, is_mature = coverage_key(request, policy_id)
43
+ identity = request.unit, request.unit_id
44
+ if identity in seen:
45
+ if seen[identity] != key:
46
+ raise ValueError("outcome coverage requires one current snapshot per unit")
47
+ continue
48
+ seen[identity] = key
49
+ if key not in acknowledged:
50
+ (mature if is_mature else recent).append((key, request))
51
+ out = []
52
+ queues = (mature, recent) if start_mature else (recent, mature)
53
+ while recent or mature:
54
+ for queue in queues:
55
+ if queue:
56
+ out.append(queue.popleft())
57
+ return out
@@ -0,0 +1,194 @@
1
+ """Private, bounded historical jobs. No teacher calls or nightly watermark changes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import re
8
+ import time
9
+ import uuid
10
+ from dataclasses import asdict
11
+ from datetime import date, datetime, timezone
12
+ from pathlib import Path
13
+ from typing import Any, Callable
14
+
15
+ from ..config import AutError, Config
16
+ from ..push import send_activity_rows
17
+ from ..repo_keys import RepoKeys
18
+ from ..state import LockBusy, _locked, atomic_write_text
19
+ from .backends import BackendUnavailable
20
+ from .core import Choice, Request, data_dir, decide, label_event, local_midnight
21
+ from .local import (TASK_NAMES, LocalBackend, decorate, load_policy, reserve_label_time,
22
+ writing_policy_path)
23
+ from .tasks import Source, units
24
+
25
+ BATCH = 10
26
+
27
+
28
+ def _digest(value: Any) -> str:
29
+ return hashlib.sha256(json.dumps(value, sort_keys=True, allow_nan=False).encode()).hexdigest()
30
+
31
+
32
+ def _write(path: Path, value: Any) -> None:
33
+ atomic_write_text(path, json.dumps(value, ensure_ascii=False, allow_nan=False))
34
+
35
+
36
+ def _identity(cfg: Config) -> str:
37
+ value = RepoKeys(cfg.repo_key_salt, create=False).of("aut-label-replay")
38
+ if not value:
39
+ raise AutError("label replay: restore the existing repository salt before preparing a job")
40
+ return value
41
+
42
+
43
+ def _directory(job_id: str) -> Path:
44
+ if not re.fullmatch(r"[0-9a-f]{32}", job_id):
45
+ raise AutError("label replay: invalid job ID")
46
+ return data_dir() / "label" / "replay" / job_id
47
+
48
+
49
+ def _read(path: Path) -> dict[str, Any]:
50
+ try:
51
+ value = json.loads(path.read_text(encoding="utf-8"))
52
+ if not isinstance(value, dict):
53
+ raise ValueError("not an object")
54
+ return value
55
+ except (OSError, ValueError) as exc:
56
+ raise AutError("label replay: job state is missing or invalid; it was not reset") from exc
57
+
58
+
59
+ def _report(manifest: dict[str, Any], progress: dict[str, Any], delivered: int = 0,
60
+ reason: str | None = None) -> dict[str, Any]:
61
+ total = len(manifest["requests"])
62
+ return {"job_id": manifest["job_id"], "task": manifest["task"],
63
+ "since": manifest["since"], "until": manifest["until"], "total": total,
64
+ "missing_local_text": manifest["missing_local_text"],
65
+ "partial_local_text": manifest["partial_local_text"],
66
+ "incomplete_outcome_context": manifest.get("incomplete_outcome_context", 0),
67
+ "delivered": delivered, "remaining": total - progress["next"],
68
+ "complete": progress["next"] == total, "reason": reason}
69
+
70
+
71
+ def prepare(cfg: Config, task: str, since: date, until: date, *, src: Source | None = None,
72
+ now: datetime | None = None) -> dict[str, Any]:
73
+ """Freeze redacted local inputs and preview counts, without scoring or sink writes."""
74
+ now = now or datetime.now(timezone.utc)
75
+ if task not in TASK_NAMES or since >= until or until > now.astimezone().date():
76
+ raise AutError("label replay: choose a local task and a nonempty range of completed dates")
77
+ cfg.require_pg("label replay preview")
78
+ identity = _identity(cfg)
79
+ policy, policy_id = load_policy(writing_policy_path()) if task == "writing_activity" else load_policy()
80
+ if task not in policy["tasks"]:
81
+ raise AutError("label replay: this task has no saved local recipe")
82
+ start, end = local_midnight(since), local_midnight(until)
83
+ src = src or Source(cfg, local_context=True)
84
+ missing = len(getattr(src, "missing_text_units", []))
85
+ partial = len(getattr(src, "partial_text_units", []))
86
+ incomplete = len(getattr(src, "incomplete_outcome_units", []))
87
+ # Outcome facts describe the current snapshot; daily work stops at the selected date.
88
+ reqs = [r for r in units(task, src, start, now if task == "soft_outcome" else end)
89
+ if start <= datetime.fromisoformat(r.ts) < end]
90
+ reqs.sort(key=lambda r: (r.ts, r.unit, r.unit_id))
91
+ if len({(r.task, r.unit, r.unit_id) for r in reqs}) != len(reqs):
92
+ raise AutError("label replay: duplicate unit identities in the preview")
93
+ job_id = uuid.uuid4().hex
94
+ directory = _directory(job_id)
95
+ labelled_at = reserve_label_time(directory.parent, now)
96
+ manifest = {"version": 1, "job_id": job_id, "task": task, "since": since.isoformat(),
97
+ "until": until.isoformat(), "asof": now.astimezone(timezone.utc).isoformat(),
98
+ "labelled_at": labelled_at, "machine": cfg.machine, "identity": identity,
99
+ "policy_id": policy_id, "task_policy": policy["tasks"][task],
100
+ "missing_local_text": sum(start <= ts < end for ts in
101
+ getattr(src, "missing_text_units", [])[missing:]),
102
+ "partial_local_text": sum(start <= ts < end for ts in
103
+ getattr(src, "partial_text_units", [])[partial:]),
104
+ "incomplete_outcome_context": sum(start <= ts < end for ts in
105
+ getattr(src, "incomplete_outcome_units", [])[incomplete:]),
106
+ "requests": [asdict(r) for r in reqs]}
107
+ progress = {"manifest_sha256": _digest(manifest), "next": 0}
108
+ directory.mkdir(mode=0o700)
109
+ _write(directory / "manifest.json", manifest)
110
+ _write(directory / "progress.json", progress)
111
+ return _report(manifest, progress)
112
+
113
+
114
+ def resume(cfg: Config, job_id: str, *, apply: bool = False, budget_s: int = 600,
115
+ backend_for: Callable[[str, dict[str, Any], str, Config], LocalBackend] | None = None,
116
+ ) -> dict[str, Any]:
117
+ """Resume one private generation. Only acknowledged delivery advances its cursor."""
118
+ if type(budget_s) is not int or not 1 <= budget_s <= 3600:
119
+ raise AutError("label replay: budget must be between 1 and 3600 seconds")
120
+ directory = _directory(job_id)
121
+ if not directory.is_dir():
122
+ raise AutError("label replay: job does not exist")
123
+ progress_path = directory / "progress.json"
124
+ try:
125
+ with _locked(progress_path, blocking=False):
126
+ return _resume(cfg, job_id, directory, apply, budget_s, backend_for)
127
+ except LockBusy as exc:
128
+ raise AutError("label replay: this job is already running") from exc
129
+
130
+
131
+ def _resume(cfg: Config, job_id: str, directory: Path, apply: bool, budget_s: int,
132
+ backend_for: Callable[[str, dict[str, Any], str, Config], LocalBackend] | None,
133
+ ) -> dict[str, Any]:
134
+ manifest = _read(directory / "manifest.json")
135
+ progress = _read(directory / "progress.json")
136
+ if (manifest.get("version") != 1 or manifest.get("job_id") != job_id
137
+ or progress.get("manifest_sha256") != _digest(manifest)
138
+ or type(progress.get("next")) is not int
139
+ or not 0 <= progress["next"] <= len(manifest.get("requests", []))):
140
+ raise AutError("label replay: job snapshot or cursor changed; refusing to continue")
141
+ if manifest["machine"] != cfg.machine or manifest["identity"] != _identity(cfg):
142
+ raise AutError("label replay: machine identity or repository salt changed")
143
+ task = manifest["task"]
144
+ policy, _current_id = load_policy(writing_policy_path()) if task == "writing_activity" else load_policy()
145
+ if _digest(policy["tasks"].get(task)) != _digest(manifest["task_policy"]):
146
+ raise AutError("label replay: the task recipe changed; prepare a new job")
147
+ if not apply or progress["next"] == len(manifest["requests"]):
148
+ return _report(manifest, progress)
149
+ dsn = cfg.require_pg("label replay")
150
+ backend = (backend_for or LocalBackend)(task, manifest["task_policy"],
151
+ manifest["policy_id"], cfg)
152
+ deadline = time.monotonic() + budget_s
153
+ delivered = 0
154
+ prepared: list[Request] = []
155
+ prepared_start = progress["next"]
156
+ try:
157
+ while progress["next"] < len(manifest["requests"]):
158
+ if time.monotonic() >= deadline:
159
+ return _report(manifest, progress, delivered, "budget")
160
+ if not prepared or progress["next"] >= prepared_start + len(prepared):
161
+ prepared_start = progress["next"]
162
+ group_size = getattr(getattr(backend, "runtime", None), "group_size", BATCH)
163
+ prefetch = max(BATCH, group_size // BATCH * BATCH)
164
+ prepared = [
165
+ Request(**r)
166
+ for r in manifest["requests"][prepared_start : prepared_start + prefetch]
167
+ ]
168
+ for req in prepared:
169
+ req.rev = f"replay:{job_id}:{req.rev or ''}"
170
+ backend.prepare_requests(prepared, deadline)
171
+ offset = progress["next"] - prepared_start
172
+ chunk = prepared[offset : offset + BATCH]
173
+ events = []
174
+ for req in chunk:
175
+ if time.monotonic() >= deadline:
176
+ return _report(manifest, progress, delivered, "budget")
177
+ raw = backend.distribution(req)
178
+ response = decide(req, raw, Choice(), backend.model_id, backend.model_rev)
179
+ event = decorate(label_event(req, response, manifest["labelled_at"]), req,
180
+ backend, response.prob)
181
+ event.machine = cfg.machine
182
+ event.attrs.update(replay_job_id=job_id, input_revision=req.rev)
183
+ events.append(event)
184
+ send_activity_rows(dsn, events)
185
+ progress["next"] += len(chunk)
186
+ _write(directory / "progress.json", progress)
187
+ delivered += len(chunk)
188
+ except BackendUnavailable:
189
+ if time.monotonic() >= deadline:
190
+ return _report(manifest, progress, delivered, "budget")
191
+ raise
192
+ finally:
193
+ backend.close()
194
+ return _report(manifest, progress, delivered)