rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/watch.py
ADDED
|
@@ -0,0 +1,742 @@
|
|
|
1
|
+
"""Prefix-available analysis over durably sealed local rollout steps."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import importlib.metadata
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import platform
|
|
10
|
+
import time
|
|
11
|
+
from dataclasses import replace
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import pandas as pd
|
|
15
|
+
|
|
16
|
+
from ._sampling import resolve_sampling_units
|
|
17
|
+
from ._watch_report import (
|
|
18
|
+
load_watch_report,
|
|
19
|
+
render_watch_report,
|
|
20
|
+
resolve_watch_exemplars,
|
|
21
|
+
validate_watch_report,
|
|
22
|
+
)
|
|
23
|
+
from ._watch_storage import (
|
|
24
|
+
WatchStore,
|
|
25
|
+
_capture,
|
|
26
|
+
_decode,
|
|
27
|
+
_json,
|
|
28
|
+
_open,
|
|
29
|
+
_read,
|
|
30
|
+
seal_watch_step,
|
|
31
|
+
)
|
|
32
|
+
from .adapters import load_trl, load_verl
|
|
33
|
+
from .comparison import canonical_fingerprint
|
|
34
|
+
from .features import _extract_features, _retain_features
|
|
35
|
+
from .pipeline import _analyze_prepared, analyze_rollouts
|
|
36
|
+
from .report import _identifier, _value, build_report
|
|
37
|
+
from .schema import validate_rollouts
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"WatchSession",
|
|
41
|
+
"analyze_watch_prefix",
|
|
42
|
+
"load_watch_report",
|
|
43
|
+
"load_watch_source",
|
|
44
|
+
"render_watch_report",
|
|
45
|
+
"resolve_watch_exemplars",
|
|
46
|
+
"seal_watch_step",
|
|
47
|
+
"validate_watch_report",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
POLICY_ID = "prefix-peak-v2"
|
|
51
|
+
_LEGACY_POLICY = "prefix-peak-v1"
|
|
52
|
+
_BASELINE = (
|
|
53
|
+
"baseline_n",
|
|
54
|
+
"baseline_start",
|
|
55
|
+
"baseline_end",
|
|
56
|
+
"baseline_mean",
|
|
57
|
+
"baseline_sd",
|
|
58
|
+
"baseline_se_floor",
|
|
59
|
+
"numerical_floor",
|
|
60
|
+
"scale",
|
|
61
|
+
"scale_source",
|
|
62
|
+
"numerical_floor_dominant",
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _code_state():
|
|
67
|
+
return {
|
|
68
|
+
path.name: hashlib.sha256(path.read_bytes()).hexdigest()
|
|
69
|
+
for path in sorted(Path(__file__).parent.glob("*.py"))
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _runtime():
|
|
74
|
+
return {
|
|
75
|
+
"python": platform.python_version(),
|
|
76
|
+
"implementation": platform.python_implementation(),
|
|
77
|
+
"system": platform.system(),
|
|
78
|
+
"machine": platform.machine(),
|
|
79
|
+
"libraries": {
|
|
80
|
+
name: importlib.metadata.version(name)
|
|
81
|
+
for name in ("numpy", "pandas", "pyarrow", "pydantic")
|
|
82
|
+
},
|
|
83
|
+
"threads": {
|
|
84
|
+
name: os.environ.get(name)
|
|
85
|
+
for name in (
|
|
86
|
+
"OMP_NUM_THREADS",
|
|
87
|
+
"OPENBLAS_NUM_THREADS",
|
|
88
|
+
"MKL_NUM_THREADS",
|
|
89
|
+
"VECLIB_MAXIMUM_THREADS",
|
|
90
|
+
)
|
|
91
|
+
},
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _unit_records(mapping, run):
|
|
96
|
+
if mapping is None:
|
|
97
|
+
return None
|
|
98
|
+
names = ["run", "prompt_id", "sampling_unit_id"]
|
|
99
|
+
if not isinstance(mapping, pd.DataFrame) or set(mapping.columns) != set(names):
|
|
100
|
+
raise ValueError(
|
|
101
|
+
"watch sampling units require run, prompt_id and sampling_unit_id"
|
|
102
|
+
)
|
|
103
|
+
if mapping.duplicated(["run", "prompt_id"]).any():
|
|
104
|
+
raise ValueError("duplicate watch sampling-unit mapping")
|
|
105
|
+
if (
|
|
106
|
+
any(
|
|
107
|
+
not isinstance(x, str) or not x.strip()
|
|
108
|
+
for x in mapping[names].to_numpy().flat
|
|
109
|
+
)
|
|
110
|
+
or not mapping.run.eq(run).all()
|
|
111
|
+
):
|
|
112
|
+
raise ValueError(
|
|
113
|
+
"watch sampling-unit identifiers must belong to the selected run"
|
|
114
|
+
)
|
|
115
|
+
return mapping[names].sort_values(["run", "prompt_id"]).to_dict("records")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _units(rows, records):
|
|
119
|
+
if records is None:
|
|
120
|
+
return None
|
|
121
|
+
mapping = pd.DataFrame(records, columns=["run", "prompt_id", "sampling_unit_id"])
|
|
122
|
+
selected = mapping.loc[mapping.prompt_id.isin(rows.prompt_id)]
|
|
123
|
+
return resolve_sampling_units(rows, selected)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def analyze_watch_prefix(
|
|
127
|
+
rollouts, *, max_sparse_per_tier=2000, sampling_units=None, retrospective=False
|
|
128
|
+
):
|
|
129
|
+
"""Batch reference for admitted rows under the fixed watch analysis policy."""
|
|
130
|
+
rows = validate_rollouts(rollouts)
|
|
131
|
+
runs = rows.run.unique()
|
|
132
|
+
if len(runs) > 1:
|
|
133
|
+
raise ValueError("watch prefixes contain one run")
|
|
134
|
+
records = _unit_records(sampling_units, runs[0]) if len(runs) else None
|
|
135
|
+
return analyze_rollouts(
|
|
136
|
+
rows,
|
|
137
|
+
max_sparse_per_tier=max_sparse_per_tier,
|
|
138
|
+
sampling_units=_units(rows, records),
|
|
139
|
+
retrospective=retrospective,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _table(frame, columns=None):
|
|
144
|
+
names = list(frame.columns) if columns is None else columns
|
|
145
|
+
return [
|
|
146
|
+
{name: _value(name, value) for name, value in row.items()}
|
|
147
|
+
for row in frame[names].to_dict("records")
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _digest(value):
|
|
152
|
+
return hashlib.sha256(_json(value).encode()).hexdigest()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class _WatchConfiguration(dict):
|
|
156
|
+
"""Use the same type-sensitive equality as the storage configuration digest.
|
|
157
|
+
|
|
158
|
+
WatchStore compares ordinary JSON dictionaries to this pin under lock, both
|
|
159
|
+
when opening the state and when validating each historical manifest.
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
def __eq__(self, other):
|
|
163
|
+
if not isinstance(other, dict):
|
|
164
|
+
return NotImplemented
|
|
165
|
+
return _json(self) == _json(other)
|
|
166
|
+
|
|
167
|
+
def __ne__(self, other):
|
|
168
|
+
if not isinstance(other, dict):
|
|
169
|
+
return NotImplemented
|
|
170
|
+
return _json(self) != _json(other)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _select_policy(state, configuration, requested):
|
|
174
|
+
"""Read the initialized policy. WatchStore rechecks the complete config under lock."""
|
|
175
|
+
path = Path(state) / "configuration.json"
|
|
176
|
+
recorded = _decode(_read(path)) if path.exists() or path.is_symlink() else None
|
|
177
|
+
legacy = recorded is not None and recorded.get("policy") == _LEGACY_POLICY
|
|
178
|
+
if recorded is None:
|
|
179
|
+
enabled = requested is True
|
|
180
|
+
elif legacy:
|
|
181
|
+
enabled = True
|
|
182
|
+
pins = _decode(
|
|
183
|
+
Path(__file__).with_name("watch-compatibility.json").read_bytes()
|
|
184
|
+
)
|
|
185
|
+
if (
|
|
186
|
+
_digest(recorded.get("code")) not in pins["legacy_code_sha256s"].values()
|
|
187
|
+
or _digest(configuration["code"]) != pins["current_code_sha256"]
|
|
188
|
+
):
|
|
189
|
+
raise ValueError(
|
|
190
|
+
"unsupported legacy watch code configuration; use the recorded build"
|
|
191
|
+
)
|
|
192
|
+
configuration["policy"] = _LEGACY_POLICY
|
|
193
|
+
configuration["code"] = recorded["code"]
|
|
194
|
+
else:
|
|
195
|
+
analysis = recorded.get("analysis")
|
|
196
|
+
enabled = analysis.get("retrospective") if isinstance(analysis, dict) else None
|
|
197
|
+
if recorded.get("policy") != POLICY_ID or type(enabled) is not bool:
|
|
198
|
+
raise ValueError("unknown recorded watch policy configuration")
|
|
199
|
+
if requested is not None and requested != enabled:
|
|
200
|
+
raise ValueError("retrospective setting differs from the recorded watch policy")
|
|
201
|
+
if not legacy:
|
|
202
|
+
analysis = configuration["analysis"]
|
|
203
|
+
analysis["retrospective"] = enabled
|
|
204
|
+
if not enabled:
|
|
205
|
+
for key in (
|
|
206
|
+
"retrospective_dependence",
|
|
207
|
+
"retrospective_penalty_multiplier",
|
|
208
|
+
"min_segment_steps",
|
|
209
|
+
):
|
|
210
|
+
del analysis[key]
|
|
211
|
+
if recorded is not None:
|
|
212
|
+
if configuration["runtime"] != recorded.get("runtime"):
|
|
213
|
+
raise ValueError(
|
|
214
|
+
"watch runtime configuration differs; use the recorded runtime"
|
|
215
|
+
)
|
|
216
|
+
# JSON comparison also distinguishes booleans from integer settings.
|
|
217
|
+
if _json(configuration) != _json(
|
|
218
|
+
{k: v for k, v in recorded.items() if k != "_source_identity"}
|
|
219
|
+
):
|
|
220
|
+
raise ValueError("watch configuration does not match initialized state")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _write(path, value):
|
|
224
|
+
path.write_text(_json(value), encoding="utf-8")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _load_rows(directory, configuration):
|
|
228
|
+
return (
|
|
229
|
+
load_trl(
|
|
230
|
+
directory,
|
|
231
|
+
run=configuration["run"],
|
|
232
|
+
reward_column=configuration["reward_column"],
|
|
233
|
+
)
|
|
234
|
+
if configuration["format"] == "trl"
|
|
235
|
+
else load_verl(directory, run=configuration["run"])
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class WatchSession:
|
|
240
|
+
"""One locked watcher. Each poll yields receipts only after durable commits.
|
|
241
|
+
|
|
242
|
+
New steps reuse extraction and recompute the complete admitted prefix. Idle
|
|
243
|
+
polls inspect source metadata only. Omit retrospective to inherit a saved
|
|
244
|
+
policy or start a sequential-only session. True enables experimental localization.
|
|
245
|
+
"""
|
|
246
|
+
|
|
247
|
+
def __init__(
|
|
248
|
+
self,
|
|
249
|
+
source,
|
|
250
|
+
state,
|
|
251
|
+
*,
|
|
252
|
+
format,
|
|
253
|
+
run,
|
|
254
|
+
start_step,
|
|
255
|
+
step_stride=1,
|
|
256
|
+
reward_column=None,
|
|
257
|
+
max_sparse_per_tier=2000,
|
|
258
|
+
sampling_units=None,
|
|
259
|
+
retrospective: bool | None = None,
|
|
260
|
+
):
|
|
261
|
+
_identifier(run)
|
|
262
|
+
if retrospective is not None and type(retrospective) is not bool:
|
|
263
|
+
raise ValueError("retrospective must be Boolean or None")
|
|
264
|
+
if type(max_sparse_per_tier) is not int or max_sparse_per_tier < 0:
|
|
265
|
+
raise ValueError("max_sparse_per_tier must be a nonnegative integer")
|
|
266
|
+
self._execution_code = _code_state()
|
|
267
|
+
self.configuration = {
|
|
268
|
+
"policy": POLICY_ID,
|
|
269
|
+
"run": run,
|
|
270
|
+
"format": format,
|
|
271
|
+
"reward_column": (reward_column or "reward")
|
|
272
|
+
if format == "trl"
|
|
273
|
+
else reward_column,
|
|
274
|
+
"start_step": start_step,
|
|
275
|
+
"step_stride": step_stride,
|
|
276
|
+
"max_sparse_per_tier": max_sparse_per_tier,
|
|
277
|
+
"min_prevalence": 0.05,
|
|
278
|
+
"analysis": {
|
|
279
|
+
"baseline_steps": 15,
|
|
280
|
+
"z_threshold": 4.0,
|
|
281
|
+
"sustain": 3,
|
|
282
|
+
"min_prevalence_change": 0.1,
|
|
283
|
+
"numerical_floor": 1e-9,
|
|
284
|
+
"retrospective_dependence": "ar1",
|
|
285
|
+
"retrospective_penalty_multiplier": 3.0,
|
|
286
|
+
"min_segment_steps": 5,
|
|
287
|
+
"replication_tolerance": 10,
|
|
288
|
+
"replication_coverage_policy": "contiguous_prefix",
|
|
289
|
+
"corroboration_radius": 10,
|
|
290
|
+
"persistence_effect_fraction": 0.5,
|
|
291
|
+
},
|
|
292
|
+
"sampling_units": _unit_records(sampling_units, run),
|
|
293
|
+
"code": self._execution_code.copy(),
|
|
294
|
+
"runtime": _runtime(),
|
|
295
|
+
}
|
|
296
|
+
_select_policy(state, self.configuration, retrospective)
|
|
297
|
+
selected = Path(source)
|
|
298
|
+
if format == "trl" and (selected / "completions").is_dir():
|
|
299
|
+
selected = selected / "completions"
|
|
300
|
+
self.store = WatchStore(selected, state, self.configuration)
|
|
301
|
+
self.store.configuration = _WatchConfiguration(self.store.configuration)
|
|
302
|
+
self.rows = self.dense = self.sparse = None
|
|
303
|
+
self.ledger = {
|
|
304
|
+
"schema_version": 1,
|
|
305
|
+
"catalog": [],
|
|
306
|
+
"admissions": {},
|
|
307
|
+
"baselines": {},
|
|
308
|
+
"emissions": {},
|
|
309
|
+
"observations": {},
|
|
310
|
+
}
|
|
311
|
+
self._fixed_evidence_cache = {}
|
|
312
|
+
self.extracted_rows = 0
|
|
313
|
+
self.analysis_count = 0
|
|
314
|
+
self._entered = False
|
|
315
|
+
self._failed = False
|
|
316
|
+
|
|
317
|
+
@property
|
|
318
|
+
def records(self):
|
|
319
|
+
return self.store.records
|
|
320
|
+
|
|
321
|
+
@property
|
|
322
|
+
def next_step(self):
|
|
323
|
+
return self.store.next_step
|
|
324
|
+
|
|
325
|
+
def __enter__(self):
|
|
326
|
+
self._pins()
|
|
327
|
+
self.store.__enter__()
|
|
328
|
+
try:
|
|
329
|
+
if self.records:
|
|
330
|
+
last = self.records[-1]
|
|
331
|
+
metadata = last.manifest["metadata"]
|
|
332
|
+
expected = {"canonical_sha256", "policy"}
|
|
333
|
+
if (
|
|
334
|
+
self.configuration["policy"] == _LEGACY_POLICY
|
|
335
|
+
and "execution_code_sha256" in metadata
|
|
336
|
+
):
|
|
337
|
+
expected.add("execution_code_sha256")
|
|
338
|
+
if metadata["execution_code_sha256"] != _digest(
|
|
339
|
+
self._execution_code
|
|
340
|
+
):
|
|
341
|
+
raise ValueError("unsupported legacy watch execution code")
|
|
342
|
+
if (
|
|
343
|
+
set(metadata) != expected
|
|
344
|
+
or metadata.get("policy") != self.configuration["policy"]
|
|
345
|
+
or not isinstance(metadata.get("canonical_sha256"), str)
|
|
346
|
+
or len(metadata["canonical_sha256"]) != 64
|
|
347
|
+
or any(
|
|
348
|
+
char not in "0123456789abcdef"
|
|
349
|
+
for char in metadata["canonical_sha256"]
|
|
350
|
+
)
|
|
351
|
+
):
|
|
352
|
+
raise ValueError("invalid watch analysis metadata")
|
|
353
|
+
self.rows = _load_rows(last.path / "source", self.store.configuration)
|
|
354
|
+
if (
|
|
355
|
+
canonical_fingerprint(self.rows)
|
|
356
|
+
!= last.manifest["metadata"]["canonical_sha256"]
|
|
357
|
+
):
|
|
358
|
+
raise ValueError("watch restored prefix fingerprint differs")
|
|
359
|
+
self.dense = pd.concat(
|
|
360
|
+
[pd.read_parquet(r.path / "dense.parquet") for r in self.records],
|
|
361
|
+
ignore_index=True,
|
|
362
|
+
)
|
|
363
|
+
self.sparse = pd.concat(
|
|
364
|
+
[pd.read_parquet(r.path / "sparse.parquet") for r in self.records],
|
|
365
|
+
ignore_index=True,
|
|
366
|
+
)
|
|
367
|
+
self.ledger = _decode((last.path / "ledger.json").read_bytes())
|
|
368
|
+
expected = {
|
|
369
|
+
"schema_version",
|
|
370
|
+
"catalog",
|
|
371
|
+
"admissions",
|
|
372
|
+
"baselines",
|
|
373
|
+
"emissions",
|
|
374
|
+
"observations",
|
|
375
|
+
}
|
|
376
|
+
if (
|
|
377
|
+
set(self.ledger) != expected
|
|
378
|
+
or type(self.ledger["schema_version"]) is not int
|
|
379
|
+
or self.ledger["schema_version"] != 1
|
|
380
|
+
or type(self.ledger["catalog"]) is not list
|
|
381
|
+
or any(not isinstance(name, str) for name in self.ledger["catalog"])
|
|
382
|
+
or any(
|
|
383
|
+
type(self.ledger[name]) is not dict
|
|
384
|
+
for name in (
|
|
385
|
+
"admissions",
|
|
386
|
+
"baselines",
|
|
387
|
+
"emissions",
|
|
388
|
+
"observations",
|
|
389
|
+
)
|
|
390
|
+
)
|
|
391
|
+
):
|
|
392
|
+
raise ValueError("invalid watch derived ledger")
|
|
393
|
+
saved = load_watch_report(last.path / "report.json")
|
|
394
|
+
if (
|
|
395
|
+
saved["sequence"] != len(self.records)
|
|
396
|
+
or saved["policy"]["id"] != self.configuration["policy"]
|
|
397
|
+
or (saved["retrospective"] is not None) != self._retrospective
|
|
398
|
+
or saved["as_of_step"] != last.manifest["step"]
|
|
399
|
+
or saved["source"]["canonical_sha256"]
|
|
400
|
+
!= metadata["canonical_sha256"]
|
|
401
|
+
or saved["policy"]["configuration_sha256"]
|
|
402
|
+
!= self.store.configuration_sha256
|
|
403
|
+
or saved["catalog"]["active"] != len(set(self.ledger["catalog"]))
|
|
404
|
+
or self.ledger["emissions"]
|
|
405
|
+
!= {item["feature"]: item for item in saved["emissions"]}
|
|
406
|
+
):
|
|
407
|
+
raise ValueError("watch derived ledger disagrees with its snapshot")
|
|
408
|
+
self._entered = True
|
|
409
|
+
return self
|
|
410
|
+
except BaseException:
|
|
411
|
+
self.store.__exit__(None, None, None)
|
|
412
|
+
raise
|
|
413
|
+
|
|
414
|
+
def __exit__(self, *args):
|
|
415
|
+
self._entered = False
|
|
416
|
+
return self.store.__exit__(*args)
|
|
417
|
+
|
|
418
|
+
def _configuration_unchanged(self):
|
|
419
|
+
pinned = {
|
|
420
|
+
key: value
|
|
421
|
+
for key, value in self.store.configuration.items()
|
|
422
|
+
if key != "_source_identity"
|
|
423
|
+
}
|
|
424
|
+
if _json(self.configuration) != _json(pinned):
|
|
425
|
+
raise ValueError("watch effective configuration changed")
|
|
426
|
+
|
|
427
|
+
def _pins(self):
|
|
428
|
+
self._configuration_unchanged()
|
|
429
|
+
if self._entered:
|
|
430
|
+
recorded = _decode(_read(self.store.state / "configuration.json"))
|
|
431
|
+
if self.store.configuration != recorded:
|
|
432
|
+
raise ValueError("watch disk configuration changed")
|
|
433
|
+
if (
|
|
434
|
+
self._execution_code != _code_state()
|
|
435
|
+
or self.configuration["runtime"] != _runtime()
|
|
436
|
+
):
|
|
437
|
+
raise ValueError("watch code or runtime configuration changed")
|
|
438
|
+
|
|
439
|
+
@property
|
|
440
|
+
def _retrospective(self):
|
|
441
|
+
return (
|
|
442
|
+
self.configuration["policy"] == _LEGACY_POLICY
|
|
443
|
+
or self.configuration["analysis"]["retrospective"]
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
def _snapshot(self, analysis, rows, stage, sequence):
|
|
447
|
+
step = stage.input_record["step"]
|
|
448
|
+
legacy = self.configuration["policy"] == _LEGACY_POLICY
|
|
449
|
+
if legacy:
|
|
450
|
+
analysis = replace(analysis, retrospective_metadata=None)
|
|
451
|
+
current = build_report(analysis, run=self.configuration["run"], benchmark=None)
|
|
452
|
+
ledger = json.loads(json.dumps(self.ledger))
|
|
453
|
+
active = set(analysis.features.catalog.feature)
|
|
454
|
+
old = set(ledger["catalog"])
|
|
455
|
+
added, removed = sorted(active - old), sorted(old - active)
|
|
456
|
+
ledger["catalog"] = sorted(active)
|
|
457
|
+
for name in sorted(active | {"scalar:reward"}):
|
|
458
|
+
ledger["admissions"].setdefault(name, step)
|
|
459
|
+
for diagnostic in analysis.sequential.diagnostics.to_dict("records"):
|
|
460
|
+
if diagnostic["baseline_n"] < 15:
|
|
461
|
+
continue
|
|
462
|
+
baseline = {name: _value(name, diagnostic[name]) for name in _BASELINE}
|
|
463
|
+
previous = ledger["baselines"].setdefault(diagnostic["feature"], baseline)
|
|
464
|
+
if previous != baseline:
|
|
465
|
+
raise ValueError("watch frozen baseline changed")
|
|
466
|
+
members = {}
|
|
467
|
+
for event in current["runs"][0]["events"]:
|
|
468
|
+
for member in event["members"]:
|
|
469
|
+
members[member["feature"]] = {
|
|
470
|
+
"member": member,
|
|
471
|
+
"event": {
|
|
472
|
+
k: event[k]
|
|
473
|
+
for k in (
|
|
474
|
+
"event_id",
|
|
475
|
+
"rank",
|
|
476
|
+
"representative_feature",
|
|
477
|
+
"n_members",
|
|
478
|
+
)
|
|
479
|
+
},
|
|
480
|
+
}
|
|
481
|
+
records = []
|
|
482
|
+
if added or removed:
|
|
483
|
+
records.append(
|
|
484
|
+
{
|
|
485
|
+
"id": f"{sequence}:catalog",
|
|
486
|
+
"type": "catalog",
|
|
487
|
+
"step": step,
|
|
488
|
+
"added": added,
|
|
489
|
+
"removed": removed,
|
|
490
|
+
}
|
|
491
|
+
)
|
|
492
|
+
previous_observations = ledger["observations"]
|
|
493
|
+
for feature in sorted(set(ledger["emissions"]) | set(members)):
|
|
494
|
+
member = members.get(feature)
|
|
495
|
+
signature = _digest(member) if member is not None else None
|
|
496
|
+
previous = previous_observations.get(feature)
|
|
497
|
+
identity = hashlib.sha256(feature.encode()).hexdigest()
|
|
498
|
+
if feature not in ledger["emissions"]:
|
|
499
|
+
detail = member["member"]
|
|
500
|
+
emission = {
|
|
501
|
+
"feature": feature,
|
|
502
|
+
"first_admitted_step": ledger["admissions"][feature],
|
|
503
|
+
"first_emitted_step": step,
|
|
504
|
+
"first_emission_sequence": sequence,
|
|
505
|
+
"onset_step": detail["onset_step"],
|
|
506
|
+
"alert_step": detail["alert_step"],
|
|
507
|
+
"direction": detail["direction"],
|
|
508
|
+
"active": True,
|
|
509
|
+
}
|
|
510
|
+
ledger["emissions"][feature] = emission
|
|
511
|
+
records.append({"id": f"alert:{identity}", "type": "alert", **emission})
|
|
512
|
+
elif previous is None or previous["signature"] != signature:
|
|
513
|
+
records.append(
|
|
514
|
+
{
|
|
515
|
+
"id": f"{sequence}:update:{identity}",
|
|
516
|
+
"type": "update",
|
|
517
|
+
"feature": feature,
|
|
518
|
+
"step": step,
|
|
519
|
+
"previous_sequence": previous["sequence"] if previous else None,
|
|
520
|
+
"reason": "evidence_or_event_changed"
|
|
521
|
+
if member
|
|
522
|
+
else "not_in_current_snapshot",
|
|
523
|
+
"previous_signature": previous["signature"]
|
|
524
|
+
if previous
|
|
525
|
+
else None,
|
|
526
|
+
"signature": signature,
|
|
527
|
+
}
|
|
528
|
+
)
|
|
529
|
+
ledger["emissions"][feature]["active"] = member is not None
|
|
530
|
+
previous_observations[feature] = {
|
|
531
|
+
"signature": signature,
|
|
532
|
+
"sequence": sequence,
|
|
533
|
+
}
|
|
534
|
+
inputs = [r.manifest["input"] for r in self.records] + [stage.input_record]
|
|
535
|
+
fingerprint = canonical_fingerprint(rows)
|
|
536
|
+
snapshot = {
|
|
537
|
+
"report_type": "watch",
|
|
538
|
+
"schema_version": 1 if legacy else 2,
|
|
539
|
+
"sequence": sequence,
|
|
540
|
+
"as_of_step": step,
|
|
541
|
+
"policy": {
|
|
542
|
+
"id": self.configuration["policy"],
|
|
543
|
+
"configuration_sha256": self.store.configuration_sha256,
|
|
544
|
+
},
|
|
545
|
+
"source": {
|
|
546
|
+
"run": self.configuration["run"],
|
|
547
|
+
"format": self.configuration["format"],
|
|
548
|
+
"reward_column": self.store.configuration["reward_column"],
|
|
549
|
+
"canonical_sha256": fingerprint,
|
|
550
|
+
"files": [
|
|
551
|
+
{
|
|
552
|
+
"path": item["file"],
|
|
553
|
+
"sha256": item["sha256"],
|
|
554
|
+
"size_bytes": item["size_bytes"],
|
|
555
|
+
"n_rollouts": item["n_rollouts"],
|
|
556
|
+
}
|
|
557
|
+
for item in inputs
|
|
558
|
+
],
|
|
559
|
+
},
|
|
560
|
+
"catalog": {
|
|
561
|
+
"active": len(active),
|
|
562
|
+
"added": len(added),
|
|
563
|
+
"removed": len(removed),
|
|
564
|
+
},
|
|
565
|
+
"emissions": [
|
|
566
|
+
ledger["emissions"][name] for name in sorted(ledger["emissions"])
|
|
567
|
+
],
|
|
568
|
+
"analysis": current,
|
|
569
|
+
"retrospective": None,
|
|
570
|
+
}
|
|
571
|
+
if self._retrospective:
|
|
572
|
+
snapshot["retrospective"] = {
|
|
573
|
+
"as_of_step": step,
|
|
574
|
+
"screening": _table(analysis.retrospective.screening),
|
|
575
|
+
"changes": _table(
|
|
576
|
+
analysis.retrospective.changepoints,
|
|
577
|
+
[
|
|
578
|
+
"feature",
|
|
579
|
+
"kind",
|
|
580
|
+
"tier",
|
|
581
|
+
"changepoint_step",
|
|
582
|
+
"previous_step",
|
|
583
|
+
"delta",
|
|
584
|
+
],
|
|
585
|
+
),
|
|
586
|
+
}
|
|
587
|
+
snapshot = validate_watch_report(snapshot)
|
|
588
|
+
records.append(
|
|
589
|
+
{
|
|
590
|
+
"id": f"{sequence}:snapshot",
|
|
591
|
+
"type": "snapshot",
|
|
592
|
+
"step": step,
|
|
593
|
+
"sequence": sequence,
|
|
594
|
+
"supersedes": sequence - 1 if sequence > 1 else None,
|
|
595
|
+
"report_sha256": _digest(snapshot),
|
|
596
|
+
"retrospective": "as_of_prefix" if self._retrospective else "disabled",
|
|
597
|
+
}
|
|
598
|
+
)
|
|
599
|
+
journal = {
|
|
600
|
+
"schema_version": 1,
|
|
601
|
+
"configuration_sha256": self.store.configuration_sha256,
|
|
602
|
+
"sequence": sequence,
|
|
603
|
+
"step": step,
|
|
604
|
+
"records": records,
|
|
605
|
+
}
|
|
606
|
+
return snapshot, journal, ledger, fingerprint
|
|
607
|
+
|
|
608
|
+
def poll_once(self, *, max_steps=None):
|
|
609
|
+
"""Yield receipts for a finite completed inventory; idle polls yield none."""
|
|
610
|
+
if not self._entered or self._failed:
|
|
611
|
+
raise ValueError("watch session is closed or failed")
|
|
612
|
+
if max_steps is not None and (type(max_steps) is not int or max_steps < 1):
|
|
613
|
+
raise ValueError("max_steps must be positive")
|
|
614
|
+
try:
|
|
615
|
+
self._configuration_unchanged()
|
|
616
|
+
candidates = self.store.discover()
|
|
617
|
+
if max_steps is not None:
|
|
618
|
+
candidates = candidates[:max_steps]
|
|
619
|
+
for candidate in candidates:
|
|
620
|
+
start = time.perf_counter()
|
|
621
|
+
self._pins()
|
|
622
|
+
stage = self.store.stage(candidate)
|
|
623
|
+
rows = validate_rollouts(
|
|
624
|
+
stage.rows
|
|
625
|
+
if self.rows is None
|
|
626
|
+
else pd.concat([self.rows, stage.rows], ignore_index=True)
|
|
627
|
+
)
|
|
628
|
+
units = _units(rows, self.configuration["sampling_units"])
|
|
629
|
+
new_dense, new_sparse = _extract_features(stage.rows)
|
|
630
|
+
self.extracted_rows += len(stage.rows)
|
|
631
|
+
dense = (
|
|
632
|
+
new_dense
|
|
633
|
+
if self.dense is None
|
|
634
|
+
else pd.concat([self.dense, new_dense], ignore_index=True)
|
|
635
|
+
)
|
|
636
|
+
sparse = (
|
|
637
|
+
new_sparse
|
|
638
|
+
if self.sparse is None
|
|
639
|
+
else pd.concat([self.sparse, new_sparse], ignore_index=True)
|
|
640
|
+
)
|
|
641
|
+
features = _retain_features(
|
|
642
|
+
rows,
|
|
643
|
+
dense,
|
|
644
|
+
sparse,
|
|
645
|
+
max_sparse_per_tier=self.configuration["max_sparse_per_tier"],
|
|
646
|
+
)
|
|
647
|
+
analyzed = time.perf_counter()
|
|
648
|
+
self.analysis_count += 1
|
|
649
|
+
analysis = _analyze_prepared(
|
|
650
|
+
rows,
|
|
651
|
+
features,
|
|
652
|
+
sampling_units=units,
|
|
653
|
+
retrospective=self._retrospective,
|
|
654
|
+
_fixed_evidence_cache=self._fixed_evidence_cache,
|
|
655
|
+
_fixed_evidence_context=self.store.configuration_sha256,
|
|
656
|
+
)
|
|
657
|
+
analysis_seconds = time.perf_counter() - analyzed
|
|
658
|
+
sequence = len(self.records) + 1
|
|
659
|
+
snapshot, journal, ledger, fingerprint = self._snapshot(
|
|
660
|
+
analysis, rows, stage, sequence
|
|
661
|
+
)
|
|
662
|
+
new_dense.to_parquet(stage.path / "dense.parquet", index=False)
|
|
663
|
+
new_sparse.to_parquet(stage.path / "sparse.parquet", index=False)
|
|
664
|
+
_write(
|
|
665
|
+
stage.path / "catalog.json",
|
|
666
|
+
{"schema_version": 1, "features": ledger["catalog"]},
|
|
667
|
+
)
|
|
668
|
+
_write(stage.path / "ledger.json", ledger)
|
|
669
|
+
_write(stage.path / "report.json", snapshot)
|
|
670
|
+
(stage.path / "report.html").write_text(
|
|
671
|
+
render_watch_report(snapshot), encoding="utf-8"
|
|
672
|
+
)
|
|
673
|
+
_write(stage.path / "journal.json", journal)
|
|
674
|
+
self._pins()
|
|
675
|
+
metadata = {
|
|
676
|
+
"canonical_sha256": fingerprint,
|
|
677
|
+
"policy": self.configuration["policy"],
|
|
678
|
+
}
|
|
679
|
+
if self.configuration["policy"] == _LEGACY_POLICY:
|
|
680
|
+
metadata["execution_code_sha256"] = _digest(self._execution_code)
|
|
681
|
+
record = self.store.commit(stage, metadata)
|
|
682
|
+
self.rows, self.dense, self.sparse, self.ledger = (
|
|
683
|
+
rows,
|
|
684
|
+
dense,
|
|
685
|
+
sparse,
|
|
686
|
+
ledger,
|
|
687
|
+
)
|
|
688
|
+
yield {
|
|
689
|
+
"type": "committed",
|
|
690
|
+
"sequence": sequence,
|
|
691
|
+
"step": candidate.step,
|
|
692
|
+
"new_alerts": sum(x["type"] == "alert" for x in journal["records"]),
|
|
693
|
+
"updates": sum(x["type"] == "update" for x in journal["records"]),
|
|
694
|
+
"report": str(
|
|
695
|
+
record.path.relative_to(self.store.state) / "report.json"
|
|
696
|
+
),
|
|
697
|
+
"new_rows": len(stage.rows),
|
|
698
|
+
"prefix_rows": len(rows),
|
|
699
|
+
"analysis_seconds": analysis_seconds,
|
|
700
|
+
"total_seconds": time.perf_counter() - start,
|
|
701
|
+
}
|
|
702
|
+
except BaseException:
|
|
703
|
+
self._failed = True
|
|
704
|
+
raise
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def load_watch_source(report, path, *, format, run, reward_column=None):
|
|
708
|
+
"""Verify and load exactly a saved report's private frozen source inventory."""
|
|
709
|
+
from tempfile import TemporaryDirectory
|
|
710
|
+
|
|
711
|
+
snapshot = validate_watch_report(report)
|
|
712
|
+
source = snapshot["source"]
|
|
713
|
+
reward = (reward_column or "reward") if format == "trl" else reward_column
|
|
714
|
+
if (format, run, reward) != (
|
|
715
|
+
source["format"],
|
|
716
|
+
source["run"],
|
|
717
|
+
source["reward_column"],
|
|
718
|
+
):
|
|
719
|
+
raise ValueError("watch source loader settings differ")
|
|
720
|
+
root = Path(os.path.abspath(path))
|
|
721
|
+
with _open(root, directory=True) as descriptor:
|
|
722
|
+
names = os.listdir(descriptor)
|
|
723
|
+
if set(names) != {entry["path"] for entry in source["files"]}:
|
|
724
|
+
raise ValueError("watch source inventory differs from the frozen prefix")
|
|
725
|
+
with TemporaryDirectory(prefix="rollout-watch-inspect-") as temporary:
|
|
726
|
+
directory = Path(temporary)
|
|
727
|
+
for entry in source["files"]:
|
|
728
|
+
destination = directory / entry["path"]
|
|
729
|
+
with destination.open("wb") as stream:
|
|
730
|
+
metadata = _capture(root / entry["path"], stream)
|
|
731
|
+
if (
|
|
732
|
+
metadata["sha256"] != entry["sha256"]
|
|
733
|
+
or metadata["size_bytes"] != entry["size_bytes"]
|
|
734
|
+
):
|
|
735
|
+
raise ValueError("watch source file fingerprint differs")
|
|
736
|
+
with _open(root, directory=True) as descriptor:
|
|
737
|
+
if set(os.listdir(descriptor)) != set(names):
|
|
738
|
+
raise ValueError("watch source inventory changed during inspection")
|
|
739
|
+
rows = _load_rows(directory, source)
|
|
740
|
+
if canonical_fingerprint(rows) != source["canonical_sha256"]:
|
|
741
|
+
raise ValueError("watch prefix fingerprint differs")
|
|
742
|
+
return rows
|