rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/watch.py ADDED
@@ -0,0 +1,742 @@
1
+ """Prefix-available analysis over durably sealed local rollout steps."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import importlib.metadata
7
+ import json
8
+ import os
9
+ import platform
10
+ import time
11
+ from dataclasses import replace
12
+ from pathlib import Path
13
+
14
+ import pandas as pd
15
+
16
+ from ._sampling import resolve_sampling_units
17
+ from ._watch_report import (
18
+ load_watch_report,
19
+ render_watch_report,
20
+ resolve_watch_exemplars,
21
+ validate_watch_report,
22
+ )
23
+ from ._watch_storage import (
24
+ WatchStore,
25
+ _capture,
26
+ _decode,
27
+ _json,
28
+ _open,
29
+ _read,
30
+ seal_watch_step,
31
+ )
32
+ from .adapters import load_trl, load_verl
33
+ from .comparison import canonical_fingerprint
34
+ from .features import _extract_features, _retain_features
35
+ from .pipeline import _analyze_prepared, analyze_rollouts
36
+ from .report import _identifier, _value, build_report
37
+ from .schema import validate_rollouts
38
+
39
+ __all__ = [
40
+ "WatchSession",
41
+ "analyze_watch_prefix",
42
+ "load_watch_report",
43
+ "load_watch_source",
44
+ "render_watch_report",
45
+ "resolve_watch_exemplars",
46
+ "seal_watch_step",
47
+ "validate_watch_report",
48
+ ]
49
+
50
+ POLICY_ID = "prefix-peak-v2"
51
+ _LEGACY_POLICY = "prefix-peak-v1"
52
+ _BASELINE = (
53
+ "baseline_n",
54
+ "baseline_start",
55
+ "baseline_end",
56
+ "baseline_mean",
57
+ "baseline_sd",
58
+ "baseline_se_floor",
59
+ "numerical_floor",
60
+ "scale",
61
+ "scale_source",
62
+ "numerical_floor_dominant",
63
+ )
64
+
65
+
66
+ def _code_state():
67
+ return {
68
+ path.name: hashlib.sha256(path.read_bytes()).hexdigest()
69
+ for path in sorted(Path(__file__).parent.glob("*.py"))
70
+ }
71
+
72
+
73
+ def _runtime():
74
+ return {
75
+ "python": platform.python_version(),
76
+ "implementation": platform.python_implementation(),
77
+ "system": platform.system(),
78
+ "machine": platform.machine(),
79
+ "libraries": {
80
+ name: importlib.metadata.version(name)
81
+ for name in ("numpy", "pandas", "pyarrow", "pydantic")
82
+ },
83
+ "threads": {
84
+ name: os.environ.get(name)
85
+ for name in (
86
+ "OMP_NUM_THREADS",
87
+ "OPENBLAS_NUM_THREADS",
88
+ "MKL_NUM_THREADS",
89
+ "VECLIB_MAXIMUM_THREADS",
90
+ )
91
+ },
92
+ }
93
+
94
+
95
+ def _unit_records(mapping, run):
96
+ if mapping is None:
97
+ return None
98
+ names = ["run", "prompt_id", "sampling_unit_id"]
99
+ if not isinstance(mapping, pd.DataFrame) or set(mapping.columns) != set(names):
100
+ raise ValueError(
101
+ "watch sampling units require run, prompt_id and sampling_unit_id"
102
+ )
103
+ if mapping.duplicated(["run", "prompt_id"]).any():
104
+ raise ValueError("duplicate watch sampling-unit mapping")
105
+ if (
106
+ any(
107
+ not isinstance(x, str) or not x.strip()
108
+ for x in mapping[names].to_numpy().flat
109
+ )
110
+ or not mapping.run.eq(run).all()
111
+ ):
112
+ raise ValueError(
113
+ "watch sampling-unit identifiers must belong to the selected run"
114
+ )
115
+ return mapping[names].sort_values(["run", "prompt_id"]).to_dict("records")
116
+
117
+
118
+ def _units(rows, records):
119
+ if records is None:
120
+ return None
121
+ mapping = pd.DataFrame(records, columns=["run", "prompt_id", "sampling_unit_id"])
122
+ selected = mapping.loc[mapping.prompt_id.isin(rows.prompt_id)]
123
+ return resolve_sampling_units(rows, selected)
124
+
125
+
126
+ def analyze_watch_prefix(
127
+ rollouts, *, max_sparse_per_tier=2000, sampling_units=None, retrospective=False
128
+ ):
129
+ """Batch reference for admitted rows under the fixed watch analysis policy."""
130
+ rows = validate_rollouts(rollouts)
131
+ runs = rows.run.unique()
132
+ if len(runs) > 1:
133
+ raise ValueError("watch prefixes contain one run")
134
+ records = _unit_records(sampling_units, runs[0]) if len(runs) else None
135
+ return analyze_rollouts(
136
+ rows,
137
+ max_sparse_per_tier=max_sparse_per_tier,
138
+ sampling_units=_units(rows, records),
139
+ retrospective=retrospective,
140
+ )
141
+
142
+
143
+ def _table(frame, columns=None):
144
+ names = list(frame.columns) if columns is None else columns
145
+ return [
146
+ {name: _value(name, value) for name, value in row.items()}
147
+ for row in frame[names].to_dict("records")
148
+ ]
149
+
150
+
151
+ def _digest(value):
152
+ return hashlib.sha256(_json(value).encode()).hexdigest()
153
+
154
+
155
+ class _WatchConfiguration(dict):
156
+ """Use the same type-sensitive equality as the storage configuration digest.
157
+
158
+ WatchStore compares ordinary JSON dictionaries to this pin under lock, both
159
+ when opening the state and when validating each historical manifest.
160
+ """
161
+
162
+ def __eq__(self, other):
163
+ if not isinstance(other, dict):
164
+ return NotImplemented
165
+ return _json(self) == _json(other)
166
+
167
+ def __ne__(self, other):
168
+ if not isinstance(other, dict):
169
+ return NotImplemented
170
+ return _json(self) != _json(other)
171
+
172
+
173
+ def _select_policy(state, configuration, requested):
174
+ """Read the initialized policy. WatchStore rechecks the complete config under lock."""
175
+ path = Path(state) / "configuration.json"
176
+ recorded = _decode(_read(path)) if path.exists() or path.is_symlink() else None
177
+ legacy = recorded is not None and recorded.get("policy") == _LEGACY_POLICY
178
+ if recorded is None:
179
+ enabled = requested is True
180
+ elif legacy:
181
+ enabled = True
182
+ pins = _decode(
183
+ Path(__file__).with_name("watch-compatibility.json").read_bytes()
184
+ )
185
+ if (
186
+ _digest(recorded.get("code")) not in pins["legacy_code_sha256s"].values()
187
+ or _digest(configuration["code"]) != pins["current_code_sha256"]
188
+ ):
189
+ raise ValueError(
190
+ "unsupported legacy watch code configuration; use the recorded build"
191
+ )
192
+ configuration["policy"] = _LEGACY_POLICY
193
+ configuration["code"] = recorded["code"]
194
+ else:
195
+ analysis = recorded.get("analysis")
196
+ enabled = analysis.get("retrospective") if isinstance(analysis, dict) else None
197
+ if recorded.get("policy") != POLICY_ID or type(enabled) is not bool:
198
+ raise ValueError("unknown recorded watch policy configuration")
199
+ if requested is not None and requested != enabled:
200
+ raise ValueError("retrospective setting differs from the recorded watch policy")
201
+ if not legacy:
202
+ analysis = configuration["analysis"]
203
+ analysis["retrospective"] = enabled
204
+ if not enabled:
205
+ for key in (
206
+ "retrospective_dependence",
207
+ "retrospective_penalty_multiplier",
208
+ "min_segment_steps",
209
+ ):
210
+ del analysis[key]
211
+ if recorded is not None:
212
+ if configuration["runtime"] != recorded.get("runtime"):
213
+ raise ValueError(
214
+ "watch runtime configuration differs; use the recorded runtime"
215
+ )
216
+ # JSON comparison also distinguishes booleans from integer settings.
217
+ if _json(configuration) != _json(
218
+ {k: v for k, v in recorded.items() if k != "_source_identity"}
219
+ ):
220
+ raise ValueError("watch configuration does not match initialized state")
221
+
222
+
223
+ def _write(path, value):
224
+ path.write_text(_json(value), encoding="utf-8")
225
+
226
+
227
+ def _load_rows(directory, configuration):
228
+ return (
229
+ load_trl(
230
+ directory,
231
+ run=configuration["run"],
232
+ reward_column=configuration["reward_column"],
233
+ )
234
+ if configuration["format"] == "trl"
235
+ else load_verl(directory, run=configuration["run"])
236
+ )
237
+
238
+
239
+ class WatchSession:
240
+ """One locked watcher. Each poll yields receipts only after durable commits.
241
+
242
+ New steps reuse extraction and recompute the complete admitted prefix. Idle
243
+ polls inspect source metadata only. Omit retrospective to inherit a saved
244
+ policy or start a sequential-only session. True enables experimental localization.
245
+ """
246
+
247
+ def __init__(
248
+ self,
249
+ source,
250
+ state,
251
+ *,
252
+ format,
253
+ run,
254
+ start_step,
255
+ step_stride=1,
256
+ reward_column=None,
257
+ max_sparse_per_tier=2000,
258
+ sampling_units=None,
259
+ retrospective: bool | None = None,
260
+ ):
261
+ _identifier(run)
262
+ if retrospective is not None and type(retrospective) is not bool:
263
+ raise ValueError("retrospective must be Boolean or None")
264
+ if type(max_sparse_per_tier) is not int or max_sparse_per_tier < 0:
265
+ raise ValueError("max_sparse_per_tier must be a nonnegative integer")
266
+ self._execution_code = _code_state()
267
+ self.configuration = {
268
+ "policy": POLICY_ID,
269
+ "run": run,
270
+ "format": format,
271
+ "reward_column": (reward_column or "reward")
272
+ if format == "trl"
273
+ else reward_column,
274
+ "start_step": start_step,
275
+ "step_stride": step_stride,
276
+ "max_sparse_per_tier": max_sparse_per_tier,
277
+ "min_prevalence": 0.05,
278
+ "analysis": {
279
+ "baseline_steps": 15,
280
+ "z_threshold": 4.0,
281
+ "sustain": 3,
282
+ "min_prevalence_change": 0.1,
283
+ "numerical_floor": 1e-9,
284
+ "retrospective_dependence": "ar1",
285
+ "retrospective_penalty_multiplier": 3.0,
286
+ "min_segment_steps": 5,
287
+ "replication_tolerance": 10,
288
+ "replication_coverage_policy": "contiguous_prefix",
289
+ "corroboration_radius": 10,
290
+ "persistence_effect_fraction": 0.5,
291
+ },
292
+ "sampling_units": _unit_records(sampling_units, run),
293
+ "code": self._execution_code.copy(),
294
+ "runtime": _runtime(),
295
+ }
296
+ _select_policy(state, self.configuration, retrospective)
297
+ selected = Path(source)
298
+ if format == "trl" and (selected / "completions").is_dir():
299
+ selected = selected / "completions"
300
+ self.store = WatchStore(selected, state, self.configuration)
301
+ self.store.configuration = _WatchConfiguration(self.store.configuration)
302
+ self.rows = self.dense = self.sparse = None
303
+ self.ledger = {
304
+ "schema_version": 1,
305
+ "catalog": [],
306
+ "admissions": {},
307
+ "baselines": {},
308
+ "emissions": {},
309
+ "observations": {},
310
+ }
311
+ self._fixed_evidence_cache = {}
312
+ self.extracted_rows = 0
313
+ self.analysis_count = 0
314
+ self._entered = False
315
+ self._failed = False
316
+
317
+ @property
318
+ def records(self):
319
+ return self.store.records
320
+
321
+ @property
322
+ def next_step(self):
323
+ return self.store.next_step
324
+
325
+ def __enter__(self):
326
+ self._pins()
327
+ self.store.__enter__()
328
+ try:
329
+ if self.records:
330
+ last = self.records[-1]
331
+ metadata = last.manifest["metadata"]
332
+ expected = {"canonical_sha256", "policy"}
333
+ if (
334
+ self.configuration["policy"] == _LEGACY_POLICY
335
+ and "execution_code_sha256" in metadata
336
+ ):
337
+ expected.add("execution_code_sha256")
338
+ if metadata["execution_code_sha256"] != _digest(
339
+ self._execution_code
340
+ ):
341
+ raise ValueError("unsupported legacy watch execution code")
342
+ if (
343
+ set(metadata) != expected
344
+ or metadata.get("policy") != self.configuration["policy"]
345
+ or not isinstance(metadata.get("canonical_sha256"), str)
346
+ or len(metadata["canonical_sha256"]) != 64
347
+ or any(
348
+ char not in "0123456789abcdef"
349
+ for char in metadata["canonical_sha256"]
350
+ )
351
+ ):
352
+ raise ValueError("invalid watch analysis metadata")
353
+ self.rows = _load_rows(last.path / "source", self.store.configuration)
354
+ if (
355
+ canonical_fingerprint(self.rows)
356
+ != last.manifest["metadata"]["canonical_sha256"]
357
+ ):
358
+ raise ValueError("watch restored prefix fingerprint differs")
359
+ self.dense = pd.concat(
360
+ [pd.read_parquet(r.path / "dense.parquet") for r in self.records],
361
+ ignore_index=True,
362
+ )
363
+ self.sparse = pd.concat(
364
+ [pd.read_parquet(r.path / "sparse.parquet") for r in self.records],
365
+ ignore_index=True,
366
+ )
367
+ self.ledger = _decode((last.path / "ledger.json").read_bytes())
368
+ expected = {
369
+ "schema_version",
370
+ "catalog",
371
+ "admissions",
372
+ "baselines",
373
+ "emissions",
374
+ "observations",
375
+ }
376
+ if (
377
+ set(self.ledger) != expected
378
+ or type(self.ledger["schema_version"]) is not int
379
+ or self.ledger["schema_version"] != 1
380
+ or type(self.ledger["catalog"]) is not list
381
+ or any(not isinstance(name, str) for name in self.ledger["catalog"])
382
+ or any(
383
+ type(self.ledger[name]) is not dict
384
+ for name in (
385
+ "admissions",
386
+ "baselines",
387
+ "emissions",
388
+ "observations",
389
+ )
390
+ )
391
+ ):
392
+ raise ValueError("invalid watch derived ledger")
393
+ saved = load_watch_report(last.path / "report.json")
394
+ if (
395
+ saved["sequence"] != len(self.records)
396
+ or saved["policy"]["id"] != self.configuration["policy"]
397
+ or (saved["retrospective"] is not None) != self._retrospective
398
+ or saved["as_of_step"] != last.manifest["step"]
399
+ or saved["source"]["canonical_sha256"]
400
+ != metadata["canonical_sha256"]
401
+ or saved["policy"]["configuration_sha256"]
402
+ != self.store.configuration_sha256
403
+ or saved["catalog"]["active"] != len(set(self.ledger["catalog"]))
404
+ or self.ledger["emissions"]
405
+ != {item["feature"]: item for item in saved["emissions"]}
406
+ ):
407
+ raise ValueError("watch derived ledger disagrees with its snapshot")
408
+ self._entered = True
409
+ return self
410
+ except BaseException:
411
+ self.store.__exit__(None, None, None)
412
+ raise
413
+
414
+ def __exit__(self, *args):
415
+ self._entered = False
416
+ return self.store.__exit__(*args)
417
+
418
+ def _configuration_unchanged(self):
419
+ pinned = {
420
+ key: value
421
+ for key, value in self.store.configuration.items()
422
+ if key != "_source_identity"
423
+ }
424
+ if _json(self.configuration) != _json(pinned):
425
+ raise ValueError("watch effective configuration changed")
426
+
427
+ def _pins(self):
428
+ self._configuration_unchanged()
429
+ if self._entered:
430
+ recorded = _decode(_read(self.store.state / "configuration.json"))
431
+ if self.store.configuration != recorded:
432
+ raise ValueError("watch disk configuration changed")
433
+ if (
434
+ self._execution_code != _code_state()
435
+ or self.configuration["runtime"] != _runtime()
436
+ ):
437
+ raise ValueError("watch code or runtime configuration changed")
438
+
439
+ @property
440
+ def _retrospective(self):
441
+ return (
442
+ self.configuration["policy"] == _LEGACY_POLICY
443
+ or self.configuration["analysis"]["retrospective"]
444
+ )
445
+
446
+ def _snapshot(self, analysis, rows, stage, sequence):
447
+ step = stage.input_record["step"]
448
+ legacy = self.configuration["policy"] == _LEGACY_POLICY
449
+ if legacy:
450
+ analysis = replace(analysis, retrospective_metadata=None)
451
+ current = build_report(analysis, run=self.configuration["run"], benchmark=None)
452
+ ledger = json.loads(json.dumps(self.ledger))
453
+ active = set(analysis.features.catalog.feature)
454
+ old = set(ledger["catalog"])
455
+ added, removed = sorted(active - old), sorted(old - active)
456
+ ledger["catalog"] = sorted(active)
457
+ for name in sorted(active | {"scalar:reward"}):
458
+ ledger["admissions"].setdefault(name, step)
459
+ for diagnostic in analysis.sequential.diagnostics.to_dict("records"):
460
+ if diagnostic["baseline_n"] < 15:
461
+ continue
462
+ baseline = {name: _value(name, diagnostic[name]) for name in _BASELINE}
463
+ previous = ledger["baselines"].setdefault(diagnostic["feature"], baseline)
464
+ if previous != baseline:
465
+ raise ValueError("watch frozen baseline changed")
466
+ members = {}
467
+ for event in current["runs"][0]["events"]:
468
+ for member in event["members"]:
469
+ members[member["feature"]] = {
470
+ "member": member,
471
+ "event": {
472
+ k: event[k]
473
+ for k in (
474
+ "event_id",
475
+ "rank",
476
+ "representative_feature",
477
+ "n_members",
478
+ )
479
+ },
480
+ }
481
+ records = []
482
+ if added or removed:
483
+ records.append(
484
+ {
485
+ "id": f"{sequence}:catalog",
486
+ "type": "catalog",
487
+ "step": step,
488
+ "added": added,
489
+ "removed": removed,
490
+ }
491
+ )
492
+ previous_observations = ledger["observations"]
493
+ for feature in sorted(set(ledger["emissions"]) | set(members)):
494
+ member = members.get(feature)
495
+ signature = _digest(member) if member is not None else None
496
+ previous = previous_observations.get(feature)
497
+ identity = hashlib.sha256(feature.encode()).hexdigest()
498
+ if feature not in ledger["emissions"]:
499
+ detail = member["member"]
500
+ emission = {
501
+ "feature": feature,
502
+ "first_admitted_step": ledger["admissions"][feature],
503
+ "first_emitted_step": step,
504
+ "first_emission_sequence": sequence,
505
+ "onset_step": detail["onset_step"],
506
+ "alert_step": detail["alert_step"],
507
+ "direction": detail["direction"],
508
+ "active": True,
509
+ }
510
+ ledger["emissions"][feature] = emission
511
+ records.append({"id": f"alert:{identity}", "type": "alert", **emission})
512
+ elif previous is None or previous["signature"] != signature:
513
+ records.append(
514
+ {
515
+ "id": f"{sequence}:update:{identity}",
516
+ "type": "update",
517
+ "feature": feature,
518
+ "step": step,
519
+ "previous_sequence": previous["sequence"] if previous else None,
520
+ "reason": "evidence_or_event_changed"
521
+ if member
522
+ else "not_in_current_snapshot",
523
+ "previous_signature": previous["signature"]
524
+ if previous
525
+ else None,
526
+ "signature": signature,
527
+ }
528
+ )
529
+ ledger["emissions"][feature]["active"] = member is not None
530
+ previous_observations[feature] = {
531
+ "signature": signature,
532
+ "sequence": sequence,
533
+ }
534
+ inputs = [r.manifest["input"] for r in self.records] + [stage.input_record]
535
+ fingerprint = canonical_fingerprint(rows)
536
+ snapshot = {
537
+ "report_type": "watch",
538
+ "schema_version": 1 if legacy else 2,
539
+ "sequence": sequence,
540
+ "as_of_step": step,
541
+ "policy": {
542
+ "id": self.configuration["policy"],
543
+ "configuration_sha256": self.store.configuration_sha256,
544
+ },
545
+ "source": {
546
+ "run": self.configuration["run"],
547
+ "format": self.configuration["format"],
548
+ "reward_column": self.store.configuration["reward_column"],
549
+ "canonical_sha256": fingerprint,
550
+ "files": [
551
+ {
552
+ "path": item["file"],
553
+ "sha256": item["sha256"],
554
+ "size_bytes": item["size_bytes"],
555
+ "n_rollouts": item["n_rollouts"],
556
+ }
557
+ for item in inputs
558
+ ],
559
+ },
560
+ "catalog": {
561
+ "active": len(active),
562
+ "added": len(added),
563
+ "removed": len(removed),
564
+ },
565
+ "emissions": [
566
+ ledger["emissions"][name] for name in sorted(ledger["emissions"])
567
+ ],
568
+ "analysis": current,
569
+ "retrospective": None,
570
+ }
571
+ if self._retrospective:
572
+ snapshot["retrospective"] = {
573
+ "as_of_step": step,
574
+ "screening": _table(analysis.retrospective.screening),
575
+ "changes": _table(
576
+ analysis.retrospective.changepoints,
577
+ [
578
+ "feature",
579
+ "kind",
580
+ "tier",
581
+ "changepoint_step",
582
+ "previous_step",
583
+ "delta",
584
+ ],
585
+ ),
586
+ }
587
+ snapshot = validate_watch_report(snapshot)
588
+ records.append(
589
+ {
590
+ "id": f"{sequence}:snapshot",
591
+ "type": "snapshot",
592
+ "step": step,
593
+ "sequence": sequence,
594
+ "supersedes": sequence - 1 if sequence > 1 else None,
595
+ "report_sha256": _digest(snapshot),
596
+ "retrospective": "as_of_prefix" if self._retrospective else "disabled",
597
+ }
598
+ )
599
+ journal = {
600
+ "schema_version": 1,
601
+ "configuration_sha256": self.store.configuration_sha256,
602
+ "sequence": sequence,
603
+ "step": step,
604
+ "records": records,
605
+ }
606
+ return snapshot, journal, ledger, fingerprint
607
+
608
+ def poll_once(self, *, max_steps=None):
609
+ """Yield receipts for a finite completed inventory; idle polls yield none."""
610
+ if not self._entered or self._failed:
611
+ raise ValueError("watch session is closed or failed")
612
+ if max_steps is not None and (type(max_steps) is not int or max_steps < 1):
613
+ raise ValueError("max_steps must be positive")
614
+ try:
615
+ self._configuration_unchanged()
616
+ candidates = self.store.discover()
617
+ if max_steps is not None:
618
+ candidates = candidates[:max_steps]
619
+ for candidate in candidates:
620
+ start = time.perf_counter()
621
+ self._pins()
622
+ stage = self.store.stage(candidate)
623
+ rows = validate_rollouts(
624
+ stage.rows
625
+ if self.rows is None
626
+ else pd.concat([self.rows, stage.rows], ignore_index=True)
627
+ )
628
+ units = _units(rows, self.configuration["sampling_units"])
629
+ new_dense, new_sparse = _extract_features(stage.rows)
630
+ self.extracted_rows += len(stage.rows)
631
+ dense = (
632
+ new_dense
633
+ if self.dense is None
634
+ else pd.concat([self.dense, new_dense], ignore_index=True)
635
+ )
636
+ sparse = (
637
+ new_sparse
638
+ if self.sparse is None
639
+ else pd.concat([self.sparse, new_sparse], ignore_index=True)
640
+ )
641
+ features = _retain_features(
642
+ rows,
643
+ dense,
644
+ sparse,
645
+ max_sparse_per_tier=self.configuration["max_sparse_per_tier"],
646
+ )
647
+ analyzed = time.perf_counter()
648
+ self.analysis_count += 1
649
+ analysis = _analyze_prepared(
650
+ rows,
651
+ features,
652
+ sampling_units=units,
653
+ retrospective=self._retrospective,
654
+ _fixed_evidence_cache=self._fixed_evidence_cache,
655
+ _fixed_evidence_context=self.store.configuration_sha256,
656
+ )
657
+ analysis_seconds = time.perf_counter() - analyzed
658
+ sequence = len(self.records) + 1
659
+ snapshot, journal, ledger, fingerprint = self._snapshot(
660
+ analysis, rows, stage, sequence
661
+ )
662
+ new_dense.to_parquet(stage.path / "dense.parquet", index=False)
663
+ new_sparse.to_parquet(stage.path / "sparse.parquet", index=False)
664
+ _write(
665
+ stage.path / "catalog.json",
666
+ {"schema_version": 1, "features": ledger["catalog"]},
667
+ )
668
+ _write(stage.path / "ledger.json", ledger)
669
+ _write(stage.path / "report.json", snapshot)
670
+ (stage.path / "report.html").write_text(
671
+ render_watch_report(snapshot), encoding="utf-8"
672
+ )
673
+ _write(stage.path / "journal.json", journal)
674
+ self._pins()
675
+ metadata = {
676
+ "canonical_sha256": fingerprint,
677
+ "policy": self.configuration["policy"],
678
+ }
679
+ if self.configuration["policy"] == _LEGACY_POLICY:
680
+ metadata["execution_code_sha256"] = _digest(self._execution_code)
681
+ record = self.store.commit(stage, metadata)
682
+ self.rows, self.dense, self.sparse, self.ledger = (
683
+ rows,
684
+ dense,
685
+ sparse,
686
+ ledger,
687
+ )
688
+ yield {
689
+ "type": "committed",
690
+ "sequence": sequence,
691
+ "step": candidate.step,
692
+ "new_alerts": sum(x["type"] == "alert" for x in journal["records"]),
693
+ "updates": sum(x["type"] == "update" for x in journal["records"]),
694
+ "report": str(
695
+ record.path.relative_to(self.store.state) / "report.json"
696
+ ),
697
+ "new_rows": len(stage.rows),
698
+ "prefix_rows": len(rows),
699
+ "analysis_seconds": analysis_seconds,
700
+ "total_seconds": time.perf_counter() - start,
701
+ }
702
+ except BaseException:
703
+ self._failed = True
704
+ raise
705
+
706
+
707
+ def load_watch_source(report, path, *, format, run, reward_column=None):
708
+ """Verify and load exactly a saved report's private frozen source inventory."""
709
+ from tempfile import TemporaryDirectory
710
+
711
+ snapshot = validate_watch_report(report)
712
+ source = snapshot["source"]
713
+ reward = (reward_column or "reward") if format == "trl" else reward_column
714
+ if (format, run, reward) != (
715
+ source["format"],
716
+ source["run"],
717
+ source["reward_column"],
718
+ ):
719
+ raise ValueError("watch source loader settings differ")
720
+ root = Path(os.path.abspath(path))
721
+ with _open(root, directory=True) as descriptor:
722
+ names = os.listdir(descriptor)
723
+ if set(names) != {entry["path"] for entry in source["files"]}:
724
+ raise ValueError("watch source inventory differs from the frozen prefix")
725
+ with TemporaryDirectory(prefix="rollout-watch-inspect-") as temporary:
726
+ directory = Path(temporary)
727
+ for entry in source["files"]:
728
+ destination = directory / entry["path"]
729
+ with destination.open("wb") as stream:
730
+ metadata = _capture(root / entry["path"], stream)
731
+ if (
732
+ metadata["sha256"] != entry["sha256"]
733
+ or metadata["size_bytes"] != entry["size_bytes"]
734
+ ):
735
+ raise ValueError("watch source file fingerprint differs")
736
+ with _open(root, directory=True) as descriptor:
737
+ if set(os.listdir(descriptor)) != set(names):
738
+ raise ValueError("watch source inventory changed during inspection")
739
+ rows = _load_rows(directory, source)
740
+ if canonical_fingerprint(rows) != source["canonical_sha256"]:
741
+ raise ValueError("watch prefix fingerprint differs")
742
+ return rows