rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,800 @@
1
+ """Private snapshots and atomic local transactions for passive rollout watching."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import re
9
+ import stat
10
+ from contextlib import contextmanager
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+ from tempfile import TemporaryDirectory
14
+ from uuid import uuid4
15
+
16
+ import pandas as pd
17
+
18
+ from . import adapters
19
+ from .schema import validate_rollouts
20
+
21
+ _MARKER_SUFFIX = ".complete.json"
22
+ _MARKER_FIELDS = {
23
+ "schema_version",
24
+ "run",
25
+ "format",
26
+ "reward_column",
27
+ "step",
28
+ "file",
29
+ "size_bytes",
30
+ "sha256",
31
+ "groups",
32
+ }
33
+ _MANIFEST_FIELDS = {
34
+ "schema_version",
35
+ "sequence",
36
+ "step",
37
+ "previous_sha256",
38
+ "configuration",
39
+ "configuration_sha256",
40
+ "input",
41
+ "artifacts",
42
+ "metadata",
43
+ }
44
+
45
+
46
+ def _json(value):
47
+ return (
48
+ json.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2)
49
+ + "\n"
50
+ )
51
+
52
+
53
+ def _sha(content):
54
+ return hashlib.sha256(content).hexdigest()
55
+
56
+
57
+ def _object(pairs):
58
+ value = {}
59
+ for key, item in pairs:
60
+ if key in value:
61
+ raise ValueError("duplicate JSON field")
62
+ value[key] = item
63
+ return value
64
+
65
+
66
+ def _decode(content):
67
+ def invalid(value):
68
+ raise ValueError("nonfinite JSON value")
69
+
70
+ try:
71
+ value = json.loads(content, object_pairs_hook=_object, parse_constant=invalid)
72
+ except (UnicodeDecodeError, json.JSONDecodeError):
73
+ raise ValueError("invalid watch JSON") from None
74
+ if type(value) is not dict:
75
+ raise ValueError("watch JSON must be an object")
76
+ return value
77
+
78
+
79
+ def _platform():
80
+ if os.name != "posix" or not all(
81
+ hasattr(os, name) for name in ("O_NOFOLLOW", "O_DIRECTORY", "O_NONBLOCK")
82
+ ):
83
+ raise ValueError("watch requires macOS or Linux with no-follow file access")
84
+ import fcntl
85
+
86
+ return fcntl
87
+
88
+
89
+ @contextmanager
90
+ def _open(path, *, directory=False):
91
+ """Open every path component relative to a no-follow directory descriptor."""
92
+ path = Path(os.path.abspath(path))
93
+ descriptors = []
94
+ try:
95
+ try:
96
+ descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY)
97
+ descriptors.append(descriptor)
98
+ for index, part in enumerate(path.parts[1:]):
99
+ flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
100
+ if directory or index < len(path.parts) - 2:
101
+ flags |= os.O_DIRECTORY
102
+ descriptor = os.open(part, flags, dir_fd=descriptor)
103
+ descriptors.append(descriptor)
104
+ mode = os.fstat(descriptor).st_mode
105
+ if not (stat.S_ISDIR(mode) if directory else stat.S_ISREG(mode)):
106
+ raise ValueError(
107
+ "watch paths must be regular files or directories without symbolic links"
108
+ )
109
+ except OSError as exc:
110
+ raise ValueError(
111
+ "watch path is missing, changed, or contains a symbolic link"
112
+ ) from exc
113
+ yield descriptor
114
+ finally:
115
+ for descriptor in reversed(descriptors):
116
+ os.close(descriptor)
117
+
118
+
119
+ def _stat(value):
120
+ return {
121
+ "device": value.st_dev,
122
+ "inode": value.st_ino,
123
+ "size_bytes": value.st_size,
124
+ "mtime_ns": value.st_mtime_ns,
125
+ "ctime_ns": value.st_ctime_ns,
126
+ }
127
+
128
+
129
+ def _capture(path, destination=None, *, content=False):
130
+ digest, size, chunks = hashlib.sha256(), 0, []
131
+ with _open(path) as descriptor:
132
+ before = _stat(os.fstat(descriptor))
133
+ while chunk := os.read(descriptor, 1024 * 1024):
134
+ digest.update(chunk)
135
+ size += len(chunk)
136
+ if destination is not None:
137
+ destination.write(chunk)
138
+ if content:
139
+ chunks.append(chunk)
140
+ after = _stat(os.fstat(descriptor))
141
+ if before != after or size != after["size_bytes"]:
142
+ raise ValueError("watch input changed while reading")
143
+ result = {"sha256": digest.hexdigest(), "size_bytes": size, "stat": after}
144
+ return (result, b"".join(chunks)) if content else result
145
+
146
+
147
+ def _read(path):
148
+ return _capture(path, content=True)[1]
149
+
150
+
151
+ def _sync_directory(path):
152
+ with _open(path, directory=True) as descriptor:
153
+ os.fsync(descriptor)
154
+
155
+
156
+ def _publish(path, content):
157
+ """Publish a single immutable bootstrap or completion file without clobbering."""
158
+ path = Path(path)
159
+ with _open(path.parent, directory=True) as parent:
160
+ temporary = ".watch-" + uuid4().hex
161
+ descriptor = os.open(
162
+ temporary,
163
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
164
+ 0o600,
165
+ dir_fd=parent,
166
+ )
167
+ try:
168
+ with os.fdopen(descriptor, "wb") as stream:
169
+ stream.write(content)
170
+ stream.flush()
171
+ os.fsync(stream.fileno())
172
+ os.link(
173
+ temporary,
174
+ path.name,
175
+ src_dir_fd=parent,
176
+ dst_dir_fd=parent,
177
+ follow_symlinks=False,
178
+ )
179
+ os.fsync(parent)
180
+ finally:
181
+ os.unlink(temporary, dir_fd=parent)
182
+ os.fsync(parent)
183
+
184
+
185
+ def _groups(value):
186
+ if type(value) is not dict or not value:
187
+ raise ValueError("expected groups must be a nonempty mapping")
188
+ if any(
189
+ not isinstance(key, str)
190
+ or not key.strip()
191
+ or type(count) is not int
192
+ or count < 1
193
+ for key, count in value.items()
194
+ ):
195
+ raise ValueError(
196
+ "group counts must be positive integers with nonblank prompt IDs"
197
+ )
198
+ return dict(value)
199
+
200
+
201
+ def _settings(format, run, reward_column):
202
+ if format not in {"trl", "verl"} or not isinstance(run, str) or not run.strip():
203
+ raise ValueError("watch requires a supported format and nonblank run")
204
+ if reward_column is not None and (
205
+ not isinstance(reward_column, str) or not reward_column.strip()
206
+ ):
207
+ raise ValueError("reward column must be nonblank")
208
+ if format == "verl" and reward_column is not None:
209
+ raise ValueError("reward column is supported only for TRL")
210
+ return (reward_column or "reward") if format == "trl" else None
211
+
212
+
213
+ def _pattern(format):
214
+ return adapters._TRL_FILE if format == "trl" else adapters._VERL_FILE
215
+
216
+
217
+ def _load(path, configuration):
218
+ return (
219
+ adapters.load_trl(
220
+ path, run=configuration["run"], reward_column=configuration["reward_column"]
221
+ )
222
+ if configuration["format"] == "trl"
223
+ else adapters.load_verl(path, run=configuration["run"])
224
+ )
225
+
226
+
227
+ def _population(rows, groups):
228
+ expected = _groups(groups)
229
+ if rows.empty or rows.groupby("prompt_id").size().to_dict() != expected:
230
+ raise ValueError("completed input does not match expected group counts")
231
+
232
+
233
+ def _marker(value, configuration, name):
234
+ match = _pattern(configuration["format"]).fullmatch(name)
235
+ if (
236
+ set(value) != _MARKER_FIELDS
237
+ or type(value["schema_version"]) is not int
238
+ or value["schema_version"] != 1
239
+ or value["run"] != configuration["run"]
240
+ or value["format"] != configuration["format"]
241
+ or value["reward_column"] != configuration["reward_column"]
242
+ or type(value["step"]) is not int
243
+ or match is None
244
+ or value["step"] != int(match.group(1))
245
+ or value["file"] != name
246
+ or type(value["size_bytes"]) is not int
247
+ or value["size_bytes"] < 1
248
+ or not isinstance(value["sha256"], str)
249
+ or re.fullmatch(r"[a-f0-9]{64}", value["sha256"]) is None
250
+ ):
251
+ raise ValueError("invalid completion marker or loader settings")
252
+ _groups(value["groups"])
253
+ return value
254
+
255
+
256
+ def _input_record(value, configuration, step):
257
+ fields = {
258
+ "file",
259
+ "step",
260
+ "sha256",
261
+ "size_bytes",
262
+ "live_stat",
263
+ "marker",
264
+ "marker_sha256",
265
+ "marker_size_bytes",
266
+ "marker_stat",
267
+ "groups",
268
+ "n_rollouts",
269
+ "n_prompts",
270
+ }
271
+ if type(value) is not dict or set(value) != fields:
272
+ raise ValueError("invalid watch input manifest")
273
+ if not isinstance(value["file"], str) or value["step"] != step:
274
+ raise ValueError("invalid watch input step or filename")
275
+ _marker(
276
+ {
277
+ "schema_version": 1,
278
+ **{key: configuration[key] for key in ("run", "format", "reward_column")},
279
+ **{
280
+ key: value[key]
281
+ for key in ("step", "file", "sha256", "size_bytes", "groups")
282
+ },
283
+ },
284
+ configuration,
285
+ value["file"],
286
+ )
287
+ if (
288
+ value["marker"] != value["file"] + _MARKER_SUFFIX
289
+ or not isinstance(value["marker_sha256"], str)
290
+ or re.fullmatch(r"[a-f0-9]{64}", value["marker_sha256"]) is None
291
+ or type(value["marker_size_bytes"]) is not int
292
+ or value["marker_size_bytes"] < 1
293
+ or type(value["n_rollouts"]) is not int
294
+ or value["n_rollouts"] != sum(value["groups"].values())
295
+ or type(value["n_prompts"]) is not int
296
+ or value["n_prompts"] != len(value["groups"])
297
+ ):
298
+ raise ValueError("invalid watch input counts or completion marker")
299
+ for name, size in (
300
+ ("live_stat", value["size_bytes"]),
301
+ ("marker_stat", value["marker_size_bytes"]),
302
+ ):
303
+ metadata = value[name]
304
+ if (
305
+ type(metadata) is not dict
306
+ or set(metadata)
307
+ != {"device", "inode", "size_bytes", "mtime_ns", "ctime_ns"}
308
+ or any(type(item) is not int for item in metadata.values())
309
+ or metadata["size_bytes"] != size
310
+ ):
311
+ raise ValueError("invalid watch input metadata")
312
+
313
+
314
+ def seal_watch_step(file, *, format, run, expected_groups, reward_column=None):
315
+ """Seal a closed file against independently supplied prompt completion counts."""
316
+ _platform()
317
+ reward_column = _settings(format, run, reward_column)
318
+ groups = _groups(expected_groups)
319
+ path = Path(os.path.abspath(file))
320
+ match = _pattern(format).fullmatch(path.name)
321
+ if match is None:
322
+ raise ValueError("unsupported rollout filename")
323
+ marker = path.with_name(path.name + _MARKER_SUFFIX)
324
+ with _open(path.parent, directory=True) as parent:
325
+ try:
326
+ os.stat(marker.name, dir_fd=parent, follow_symlinks=False)
327
+ except FileNotFoundError:
328
+ pass
329
+ else:
330
+ raise FileExistsError("completion marker already exists")
331
+ configuration = {"format": format, "run": run, "reward_column": reward_column}
332
+ with TemporaryDirectory(prefix="rollout-watch-seal-") as temporary:
333
+ copied = Path(temporary) / path.name
334
+ with copied.open("xb") as destination:
335
+ captured = _capture(path, destination)
336
+ rows = _load(copied, configuration)
337
+ _population(rows, groups)
338
+ with _open(path) as descriptor:
339
+ if _stat(os.fstat(descriptor)) != captured["stat"]:
340
+ raise ValueError("watch input changed before sealing")
341
+ value = {
342
+ "schema_version": 1,
343
+ **configuration,
344
+ "step": int(match.group(1)),
345
+ "file": path.name,
346
+ "size_bytes": captured["size_bytes"],
347
+ "sha256": captured["sha256"],
348
+ "groups": groups,
349
+ }
350
+ _publish(marker, _json(value).encode("utf-8"))
351
+ return marker
352
+
353
+
354
+ @dataclass(frozen=True)
355
+ class Candidate:
356
+ file: Path
357
+ marker: Path
358
+ step: int
359
+
360
+
361
+ @dataclass
362
+ class Stage:
363
+ path: Path
364
+ new_file: Path
365
+ rows: pd.DataFrame
366
+ input_record: dict
367
+
368
+
369
+ @dataclass(frozen=True)
370
+ class Record:
371
+ path: Path
372
+ manifest: dict
373
+ sha256: str
374
+
375
+
376
+ def _tree(path):
377
+ result = []
378
+ with _open(path, directory=True) as descriptor:
379
+ for name in sorted(os.listdir(descriptor)):
380
+ mode = os.stat(name, dir_fd=descriptor, follow_symlinks=False).st_mode
381
+ child = path / name
382
+ if stat.S_ISDIR(mode):
383
+ result.extend(_tree(child))
384
+ elif stat.S_ISREG(mode):
385
+ result.append(child)
386
+ else:
387
+ raise ValueError(
388
+ "state artifacts must be regular files without symbolic links"
389
+ )
390
+ return result
391
+
392
+
393
+ def _artifacts(path):
394
+ result = {}
395
+ for file in _tree(path):
396
+ if file == path / "manifest.json":
397
+ continue
398
+ captured = _capture(file)
399
+ result[file.relative_to(path).as_posix()] = {
400
+ key: captured[key] for key in ("sha256", "size_bytes")
401
+ }
402
+ return result
403
+
404
+
405
+ class WatchStore:
406
+ """Own one locked state and commit complete prefixes by directory rename."""
407
+
408
+ def __init__(self, source, state, configuration):
409
+ self.source = Path(os.path.abspath(source))
410
+ self.state = Path(os.path.abspath(state))
411
+ self.configuration = _decode(_json(configuration))
412
+ self.configuration["reward_column"] = _settings(
413
+ self.configuration.get("format"),
414
+ self.configuration.get("run"),
415
+ self.configuration.get("reward_column"),
416
+ )
417
+ for name, minimum in (("start_step", 0), ("step_stride", 1)):
418
+ value = self.configuration.get(name)
419
+ if type(value) is not int or value < minimum:
420
+ raise ValueError(f"invalid watch {name}")
421
+ self.records = []
422
+ self.commits = self.state / "commits"
423
+ self._lock = None
424
+ self._failed = False
425
+ self._rows = None
426
+
427
+ @property
428
+ def next_step(self):
429
+ return (
430
+ self.configuration["start_step"]
431
+ + len(self.records) * self.configuration["step_stride"]
432
+ )
433
+
434
+ def __enter__(self):
435
+ fcntl = _platform()
436
+ if (
437
+ self.source == self.state
438
+ or self.source in self.state.parents
439
+ or self.state in self.source.parents
440
+ ):
441
+ raise ValueError("source and state directories must be disjoint")
442
+ with _open(self.source, directory=True) as descriptor:
443
+ source_stat = os.fstat(descriptor)
444
+ self.configuration["_source_identity"] = {
445
+ "path": str(self.source),
446
+ "device": source_stat.st_dev,
447
+ "inode": source_stat.st_ino,
448
+ }
449
+ self.configuration_sha256 = _sha(_json(self.configuration).encode("utf-8"))
450
+ try:
451
+ with _open(self.state.parent, directory=True) as parent:
452
+ try:
453
+ os.mkdir(self.state.name, mode=0o700, dir_fd=parent)
454
+ os.fsync(parent)
455
+ except FileExistsError:
456
+ pass
457
+ with _open(self.state, directory=True) as directory:
458
+ current = os.fstat(directory)
459
+ if (
460
+ current.st_uid != os.geteuid()
461
+ or stat.S_IMODE(current.st_mode) & 0o077
462
+ ):
463
+ raise ValueError(
464
+ "watch state must be private and owned by the current user"
465
+ )
466
+ self._lock = os.open(
467
+ "lock",
468
+ os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
469
+ 0o600,
470
+ dir_fd=directory,
471
+ )
472
+ lock_stat = os.fstat(self._lock)
473
+ if (
474
+ not stat.S_ISREG(lock_stat.st_mode)
475
+ or lock_stat.st_uid != os.geteuid()
476
+ ):
477
+ raise ValueError("watch lock must be a regular owned file")
478
+ try:
479
+ fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
480
+ except BlockingIOError:
481
+ raise ValueError(
482
+ "watch state is locked by another writer"
483
+ ) from None
484
+ if _stat(
485
+ os.stat("lock", dir_fd=directory, follow_symlinks=False)
486
+ ) != _stat(lock_stat):
487
+ raise ValueError("watch lock changed")
488
+ names = set(os.listdir(directory))
489
+ if names - {"lock", "commits", "configuration.json"}:
490
+ raise ValueError("unknown watch state files")
491
+ if "commits" not in names:
492
+ os.mkdir("commits", mode=0o700, dir_fd=directory)
493
+ os.fsync(directory)
494
+ configuration_file = self.state / "configuration.json"
495
+ if configuration_file.exists() or configuration_file.is_symlink():
496
+ if _decode(_read(configuration_file)) != self.configuration:
497
+ raise ValueError(
498
+ "watch configuration does not match initialized state"
499
+ )
500
+ else:
501
+ with _open(self.commits, directory=True) as descriptor:
502
+ if os.listdir(descriptor):
503
+ raise ValueError("watch state configuration is missing")
504
+ _publish(configuration_file, _json(self.configuration).encode("utf-8"))
505
+ self._root_ids = {self.source: (source_stat.st_dev, source_stat.st_ino)}
506
+ for path in (self.state, self.commits):
507
+ with _open(path, directory=True) as descriptor:
508
+ observed = os.fstat(descriptor)
509
+ self._root_ids[path] = (observed.st_dev, observed.st_ino)
510
+ self._recover()
511
+ return self
512
+ except BaseException:
513
+ self.__exit__(None, None, None)
514
+ raise
515
+
516
+ def __exit__(self, *args):
517
+ if self._lock is not None:
518
+ os.close(self._lock)
519
+ self._lock = None
520
+
521
+ def _ready(self):
522
+ if self._lock is None:
523
+ raise ValueError("watch store is not open")
524
+ if self._failed:
525
+ raise ValueError("watch publication failed; reopen and verify state")
526
+ if _sha(_json(self.configuration).encode("utf-8")) != self.configuration_sha256:
527
+ raise ValueError("watch configuration changed")
528
+ for path, expected in self._root_ids.items():
529
+ with _open(path, directory=True) as descriptor:
530
+ current = os.fstat(descriptor)
531
+ if (current.st_dev, current.st_ino) != expected:
532
+ raise ValueError("watch directory identity changed")
533
+ with _open(self.state / "lock") as descriptor:
534
+ current, held = os.fstat(descriptor), os.fstat(self._lock)
535
+ if (current.st_dev, current.st_ino) != (held.st_dev, held.st_ino):
536
+ raise ValueError("watch lock changed")
537
+
538
+ def _live(self, record, *, hashes=False):
539
+ for name, expected, digest in (
540
+ (record["file"], record["live_stat"], record["sha256"]),
541
+ (record["marker"], record["marker_stat"], record["marker_sha256"]),
542
+ ):
543
+ path = self.source / name
544
+ with _open(path) as descriptor:
545
+ if _stat(os.fstat(descriptor)) != expected:
546
+ raise ValueError("committed watch input changed")
547
+ if hashes and _capture(path)["sha256"] != digest:
548
+ raise ValueError("committed watch input hash changed")
549
+
550
+ def _recover(self):
551
+ self.records = []
552
+ with _open(self.commits, directory=True) as descriptor:
553
+ names = sorted(os.listdir(descriptor))
554
+ for name in names:
555
+ mode = os.stat(name, dir_fd=descriptor, follow_symlinks=False).st_mode
556
+ if not stat.S_ISDIR(mode):
557
+ raise ValueError("invalid committed watch directory")
558
+ if name.startswith(".pending-"):
559
+ continue
560
+ if name != f"{len(self.records) + 1:012d}":
561
+ raise ValueError("noncontiguous watch commit chain")
562
+ path = self.commits / name
563
+ encoded = _read(path / "manifest.json")
564
+ manifest = _decode(encoded)
565
+ previous = self.records[-1].sha256 if self.records else None
566
+ if (
567
+ set(manifest) != _MANIFEST_FIELDS
568
+ or type(manifest["schema_version"]) is not int
569
+ or manifest["schema_version"] != 1
570
+ or type(manifest["sequence"]) is not int
571
+ or manifest["sequence"] != len(self.records) + 1
572
+ or type(manifest["step"]) is not int
573
+ or manifest["step"] != self.next_step
574
+ or manifest["previous_sha256"] != previous
575
+ or manifest["configuration"] != self.configuration
576
+ or manifest["configuration_sha256"] != self.configuration_sha256
577
+ or type(manifest["input"]) is not dict
578
+ or type(manifest["metadata"]) is not dict
579
+ ):
580
+ raise ValueError("invalid watch manifest chain or configuration")
581
+ self._verify_artifacts(path, manifest["artifacts"], manifest["input"])
582
+ self._live(manifest["input"], hashes=True)
583
+ self.records.append(Record(path, manifest, _sha(encoded)))
584
+ if self.records:
585
+ # A surviving rename may precede an interrupted parent-directory fsync.
586
+ _sync_directory(self.commits)
587
+ self._rows = _load(self.records[-1].path / "source", self.configuration)
588
+ self.discover()
589
+
590
+ def _verify_artifacts(self, path, expected, incoming):
591
+ _input_record(incoming, self.configuration, self.next_step)
592
+ if type(expected) is not dict or _artifacts(path) != expected:
593
+ raise ValueError("watch artifact inventory or hash changed")
594
+ inputs = [record.manifest["input"] for record in self.records] + [incoming]
595
+ sources = {
596
+ "source/" + record["file"]: {
597
+ key: record[key] for key in ("sha256", "size_bytes")
598
+ }
599
+ for record in inputs
600
+ }
601
+ if {
602
+ name: digest
603
+ for name, digest in expected.items()
604
+ if name.startswith("source/")
605
+ } != sources:
606
+ raise ValueError(
607
+ "watch source artifact prefix does not match committed inputs"
608
+ )
609
+ marker = expected.get(incoming["marker"])
610
+ if marker != {
611
+ "sha256": incoming["marker_sha256"],
612
+ "size_bytes": incoming["marker_size_bytes"],
613
+ }:
614
+ raise ValueError("watch completion marker artifact changed")
615
+ completion = _marker(
616
+ _decode(_read(path / incoming["marker"])),
617
+ self.configuration,
618
+ incoming["file"],
619
+ )
620
+ if any(
621
+ completion[key] != incoming[key]
622
+ for key in ("file", "step", "sha256", "size_bytes", "groups")
623
+ ):
624
+ raise ValueError(
625
+ "watch input manifest does not match its completion marker"
626
+ )
627
+
628
+ def discover(self):
629
+ """Capture eligible names without allowing future failures to gate a prefix."""
630
+ self._ready()
631
+ for record in self.records:
632
+ self._live(record.manifest["input"])
633
+ pattern = _pattern(self.configuration["format"])
634
+ inventory = {}
635
+ with _open(self.source, directory=True) as descriptor:
636
+ for name in os.listdir(descriptor):
637
+ basename = name.removesuffix(_MARKER_SUFFIX)
638
+ match = pattern.fullmatch(basename)
639
+ if match is not None:
640
+ step = int(match.group(1))
641
+ if step >= self.configuration["start_step"]:
642
+ inventory.setdefault(step, set()).add(name)
643
+ accepted = {
644
+ record.manifest["step"]: record.manifest["input"]["file"]
645
+ for record in self.records
646
+ }
647
+ for step, names in sorted(inventory.items()):
648
+ if step >= self.next_step:
649
+ break
650
+ if (step - self.configuration["start_step"]) % self.configuration[
651
+ "step_stride"
652
+ ]:
653
+ raise ValueError(
654
+ "input step does not match configured watch stride"
655
+ )
656
+ aliases = {name.removesuffix(_MARKER_SUFFIX) for name in names}
657
+ if aliases != {accepted.get(step)}:
658
+ raise ValueError("late input alias at finalized watch step")
659
+ candidates = []
660
+ expected = self.next_step
661
+ for step, names in sorted(inventory.items()):
662
+ if step < self.next_step:
663
+ continue
664
+ if step != expected:
665
+ break
666
+ try:
667
+ candidate = self._candidate(step, names, descriptor)
668
+ except ValueError:
669
+ if not candidates:
670
+ raise
671
+ break
672
+ if candidate is None:
673
+ break
674
+ candidates.append(candidate)
675
+ expected += self.configuration["step_stride"]
676
+ return candidates
677
+
678
+ def _candidate(self, step, names, descriptor):
679
+ aliases = {name.removesuffix(_MARKER_SUFFIX) for name in names}
680
+ if len(aliases) != 1:
681
+ raise ValueError("multiple input aliases for one watch step")
682
+ name = next(iter(aliases))
683
+ marker = name + _MARKER_SUFFIX
684
+ if marker not in names:
685
+ return None
686
+ if name not in names:
687
+ raise ValueError("published completion marker has missing input")
688
+ for entry in (name, marker):
689
+ try:
690
+ mode = os.stat(entry, dir_fd=descriptor, follow_symlinks=False).st_mode
691
+ except OSError as exc:
692
+ raise ValueError("watch input changed during discovery") from exc
693
+ if not stat.S_ISREG(mode):
694
+ raise ValueError(
695
+ "watch inputs must be regular files without symbolic links"
696
+ )
697
+ return Candidate(self.source / name, self.source / marker, step)
698
+
699
+ def stage(self, candidate):
700
+ candidates = self.discover()
701
+ if not candidates or candidate != candidates[0]:
702
+ raise ValueError("only the next expected watch step may be staged")
703
+ if self.records:
704
+ last = self.records[-1]
705
+ # Validate previous private bytes before reusing their inodes.
706
+ if _artifacts(last.path) != last.manifest["artifacts"]:
707
+ raise ValueError("watch artifact changed before staging")
708
+ directory = self.commits / (".pending-" + uuid4().hex)
709
+ directory.mkdir(mode=0o700)
710
+ snapshot = directory / "source"
711
+ snapshot.mkdir(mode=0o700)
712
+ if self.records:
713
+ for file in (self.records[-1].path / "source").iterdir():
714
+ os.link(file, snapshot / file.name, follow_symlinks=False)
715
+ marker_copy = directory / candidate.marker.name
716
+ with marker_copy.open("xb") as stream:
717
+ marker_info, content = _capture(candidate.marker, stream, content=True)
718
+ marker = _marker(_decode(content), self.configuration, candidate.file.name)
719
+ copied = snapshot / candidate.file.name
720
+ with copied.open("xb") as stream:
721
+ captured = _capture(candidate.file, stream)
722
+ if any(captured[key] != marker[key] for key in ("sha256", "size_bytes")):
723
+ raise ValueError("completed input hash or size does not match its marker")
724
+ rows = _load(copied, self.configuration)
725
+ _population(rows, marker["groups"])
726
+ if self._rows is not None:
727
+ validate_rollouts(pd.concat([self._rows, rows], ignore_index=True))
728
+ record = {
729
+ "file": candidate.file.name,
730
+ "step": candidate.step,
731
+ "sha256": captured["sha256"],
732
+ "size_bytes": captured["size_bytes"],
733
+ "live_stat": captured["stat"],
734
+ "marker": candidate.marker.name,
735
+ "marker_sha256": marker_info["sha256"],
736
+ "marker_size_bytes": marker_info["size_bytes"],
737
+ "marker_stat": marker_info["stat"],
738
+ "groups": marker["groups"],
739
+ "n_rollouts": len(rows),
740
+ "n_prompts": rows.prompt_id.nunique(),
741
+ }
742
+ self._live(record)
743
+ copied.chmod(0o400)
744
+ return Stage(directory, copied, rows, record)
745
+
746
+ def commit(self, stage, metadata):
747
+ self._ready()
748
+ if type(metadata) is not dict:
749
+ raise ValueError("watch metadata must be an object")
750
+ if (
751
+ stage.path.parent != self.commits
752
+ or not stage.path.name.startswith(".pending-")
753
+ or stage.input_record["step"] != self.next_step
754
+ ):
755
+ raise ValueError("invalid watch stage or sequence")
756
+ self.discover()
757
+ self._live(stage.input_record)
758
+ sequence = len(self.records) + 1
759
+ artifacts = _artifacts(stage.path)
760
+ self._verify_artifacts(stage.path, artifacts, stage.input_record)
761
+ manifest = {
762
+ "schema_version": 1,
763
+ "sequence": sequence,
764
+ "step": self.next_step,
765
+ "previous_sha256": self.records[-1].sha256 if self.records else None,
766
+ "configuration": self.configuration,
767
+ "configuration_sha256": self.configuration_sha256,
768
+ "input": stage.input_record,
769
+ "artifacts": artifacts,
770
+ "metadata": metadata,
771
+ }
772
+ encoded = _json(manifest).encode("utf-8")
773
+ destination = self.commits / f"{sequence:012d}"
774
+ try:
775
+ _publish(stage.path / "manifest.json", encoded)
776
+ for file in _tree(stage.path):
777
+ with _open(file) as descriptor:
778
+ os.fsync(descriptor)
779
+ directories = {file.parent for file in _tree(stage.path)} | {stage.path}
780
+ for directory in sorted(
781
+ directories, key=lambda path: len(path.parts), reverse=True
782
+ ):
783
+ _sync_directory(directory)
784
+ if destination.exists() or destination.is_symlink():
785
+ raise ValueError("watch commit destination already exists")
786
+ self.discover()
787
+ self._live(stage.input_record)
788
+ os.rename(stage.path, destination)
789
+ _sync_directory(self.commits)
790
+ except BaseException:
791
+ self._failed = True
792
+ raise
793
+ record = Record(destination, _decode(encoded), _sha(encoded))
794
+ self.records.append(record)
795
+ self._rows = (
796
+ stage.rows.copy()
797
+ if self._rows is None
798
+ else pd.concat([self._rows, stage.rows], ignore_index=True)
799
+ )
800
+ return record