rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,800 @@
|
|
|
1
|
+
"""Private snapshots and atomic local transactions for passive rollout watching."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import stat
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from tempfile import TemporaryDirectory
|
|
14
|
+
from uuid import uuid4
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from . import adapters
|
|
19
|
+
from .schema import validate_rollouts
|
|
20
|
+
|
|
21
|
+
_MARKER_SUFFIX = ".complete.json"
|
|
22
|
+
_MARKER_FIELDS = {
|
|
23
|
+
"schema_version",
|
|
24
|
+
"run",
|
|
25
|
+
"format",
|
|
26
|
+
"reward_column",
|
|
27
|
+
"step",
|
|
28
|
+
"file",
|
|
29
|
+
"size_bytes",
|
|
30
|
+
"sha256",
|
|
31
|
+
"groups",
|
|
32
|
+
}
|
|
33
|
+
_MANIFEST_FIELDS = {
|
|
34
|
+
"schema_version",
|
|
35
|
+
"sequence",
|
|
36
|
+
"step",
|
|
37
|
+
"previous_sha256",
|
|
38
|
+
"configuration",
|
|
39
|
+
"configuration_sha256",
|
|
40
|
+
"input",
|
|
41
|
+
"artifacts",
|
|
42
|
+
"metadata",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _json(value):
|
|
47
|
+
return (
|
|
48
|
+
json.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2)
|
|
49
|
+
+ "\n"
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _sha(content):
|
|
54
|
+
return hashlib.sha256(content).hexdigest()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _object(pairs):
|
|
58
|
+
value = {}
|
|
59
|
+
for key, item in pairs:
|
|
60
|
+
if key in value:
|
|
61
|
+
raise ValueError("duplicate JSON field")
|
|
62
|
+
value[key] = item
|
|
63
|
+
return value
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _decode(content):
|
|
67
|
+
def invalid(value):
|
|
68
|
+
raise ValueError("nonfinite JSON value")
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
value = json.loads(content, object_pairs_hook=_object, parse_constant=invalid)
|
|
72
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
73
|
+
raise ValueError("invalid watch JSON") from None
|
|
74
|
+
if type(value) is not dict:
|
|
75
|
+
raise ValueError("watch JSON must be an object")
|
|
76
|
+
return value
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _platform():
|
|
80
|
+
if os.name != "posix" or not all(
|
|
81
|
+
hasattr(os, name) for name in ("O_NOFOLLOW", "O_DIRECTORY", "O_NONBLOCK")
|
|
82
|
+
):
|
|
83
|
+
raise ValueError("watch requires macOS or Linux with no-follow file access")
|
|
84
|
+
import fcntl
|
|
85
|
+
|
|
86
|
+
return fcntl
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@contextmanager
|
|
90
|
+
def _open(path, *, directory=False):
|
|
91
|
+
"""Open every path component relative to a no-follow directory descriptor."""
|
|
92
|
+
path = Path(os.path.abspath(path))
|
|
93
|
+
descriptors = []
|
|
94
|
+
try:
|
|
95
|
+
try:
|
|
96
|
+
descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY)
|
|
97
|
+
descriptors.append(descriptor)
|
|
98
|
+
for index, part in enumerate(path.parts[1:]):
|
|
99
|
+
flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
|
|
100
|
+
if directory or index < len(path.parts) - 2:
|
|
101
|
+
flags |= os.O_DIRECTORY
|
|
102
|
+
descriptor = os.open(part, flags, dir_fd=descriptor)
|
|
103
|
+
descriptors.append(descriptor)
|
|
104
|
+
mode = os.fstat(descriptor).st_mode
|
|
105
|
+
if not (stat.S_ISDIR(mode) if directory else stat.S_ISREG(mode)):
|
|
106
|
+
raise ValueError(
|
|
107
|
+
"watch paths must be regular files or directories without symbolic links"
|
|
108
|
+
)
|
|
109
|
+
except OSError as exc:
|
|
110
|
+
raise ValueError(
|
|
111
|
+
"watch path is missing, changed, or contains a symbolic link"
|
|
112
|
+
) from exc
|
|
113
|
+
yield descriptor
|
|
114
|
+
finally:
|
|
115
|
+
for descriptor in reversed(descriptors):
|
|
116
|
+
os.close(descriptor)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _stat(value):
|
|
120
|
+
return {
|
|
121
|
+
"device": value.st_dev,
|
|
122
|
+
"inode": value.st_ino,
|
|
123
|
+
"size_bytes": value.st_size,
|
|
124
|
+
"mtime_ns": value.st_mtime_ns,
|
|
125
|
+
"ctime_ns": value.st_ctime_ns,
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _capture(path, destination=None, *, content=False):
|
|
130
|
+
digest, size, chunks = hashlib.sha256(), 0, []
|
|
131
|
+
with _open(path) as descriptor:
|
|
132
|
+
before = _stat(os.fstat(descriptor))
|
|
133
|
+
while chunk := os.read(descriptor, 1024 * 1024):
|
|
134
|
+
digest.update(chunk)
|
|
135
|
+
size += len(chunk)
|
|
136
|
+
if destination is not None:
|
|
137
|
+
destination.write(chunk)
|
|
138
|
+
if content:
|
|
139
|
+
chunks.append(chunk)
|
|
140
|
+
after = _stat(os.fstat(descriptor))
|
|
141
|
+
if before != after or size != after["size_bytes"]:
|
|
142
|
+
raise ValueError("watch input changed while reading")
|
|
143
|
+
result = {"sha256": digest.hexdigest(), "size_bytes": size, "stat": after}
|
|
144
|
+
return (result, b"".join(chunks)) if content else result
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _read(path):
|
|
148
|
+
return _capture(path, content=True)[1]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _sync_directory(path):
|
|
152
|
+
with _open(path, directory=True) as descriptor:
|
|
153
|
+
os.fsync(descriptor)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _publish(path, content):
|
|
157
|
+
"""Publish a single immutable bootstrap or completion file without clobbering."""
|
|
158
|
+
path = Path(path)
|
|
159
|
+
with _open(path.parent, directory=True) as parent:
|
|
160
|
+
temporary = ".watch-" + uuid4().hex
|
|
161
|
+
descriptor = os.open(
|
|
162
|
+
temporary,
|
|
163
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
164
|
+
0o600,
|
|
165
|
+
dir_fd=parent,
|
|
166
|
+
)
|
|
167
|
+
try:
|
|
168
|
+
with os.fdopen(descriptor, "wb") as stream:
|
|
169
|
+
stream.write(content)
|
|
170
|
+
stream.flush()
|
|
171
|
+
os.fsync(stream.fileno())
|
|
172
|
+
os.link(
|
|
173
|
+
temporary,
|
|
174
|
+
path.name,
|
|
175
|
+
src_dir_fd=parent,
|
|
176
|
+
dst_dir_fd=parent,
|
|
177
|
+
follow_symlinks=False,
|
|
178
|
+
)
|
|
179
|
+
os.fsync(parent)
|
|
180
|
+
finally:
|
|
181
|
+
os.unlink(temporary, dir_fd=parent)
|
|
182
|
+
os.fsync(parent)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _groups(value):
|
|
186
|
+
if type(value) is not dict or not value:
|
|
187
|
+
raise ValueError("expected groups must be a nonempty mapping")
|
|
188
|
+
if any(
|
|
189
|
+
not isinstance(key, str)
|
|
190
|
+
or not key.strip()
|
|
191
|
+
or type(count) is not int
|
|
192
|
+
or count < 1
|
|
193
|
+
for key, count in value.items()
|
|
194
|
+
):
|
|
195
|
+
raise ValueError(
|
|
196
|
+
"group counts must be positive integers with nonblank prompt IDs"
|
|
197
|
+
)
|
|
198
|
+
return dict(value)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _settings(format, run, reward_column):
|
|
202
|
+
if format not in {"trl", "verl"} or not isinstance(run, str) or not run.strip():
|
|
203
|
+
raise ValueError("watch requires a supported format and nonblank run")
|
|
204
|
+
if reward_column is not None and (
|
|
205
|
+
not isinstance(reward_column, str) or not reward_column.strip()
|
|
206
|
+
):
|
|
207
|
+
raise ValueError("reward column must be nonblank")
|
|
208
|
+
if format == "verl" and reward_column is not None:
|
|
209
|
+
raise ValueError("reward column is supported only for TRL")
|
|
210
|
+
return (reward_column or "reward") if format == "trl" else None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _pattern(format):
|
|
214
|
+
return adapters._TRL_FILE if format == "trl" else adapters._VERL_FILE
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _load(path, configuration):
|
|
218
|
+
return (
|
|
219
|
+
adapters.load_trl(
|
|
220
|
+
path, run=configuration["run"], reward_column=configuration["reward_column"]
|
|
221
|
+
)
|
|
222
|
+
if configuration["format"] == "trl"
|
|
223
|
+
else adapters.load_verl(path, run=configuration["run"])
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _population(rows, groups):
|
|
228
|
+
expected = _groups(groups)
|
|
229
|
+
if rows.empty or rows.groupby("prompt_id").size().to_dict() != expected:
|
|
230
|
+
raise ValueError("completed input does not match expected group counts")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _marker(value, configuration, name):
|
|
234
|
+
match = _pattern(configuration["format"]).fullmatch(name)
|
|
235
|
+
if (
|
|
236
|
+
set(value) != _MARKER_FIELDS
|
|
237
|
+
or type(value["schema_version"]) is not int
|
|
238
|
+
or value["schema_version"] != 1
|
|
239
|
+
or value["run"] != configuration["run"]
|
|
240
|
+
or value["format"] != configuration["format"]
|
|
241
|
+
or value["reward_column"] != configuration["reward_column"]
|
|
242
|
+
or type(value["step"]) is not int
|
|
243
|
+
or match is None
|
|
244
|
+
or value["step"] != int(match.group(1))
|
|
245
|
+
or value["file"] != name
|
|
246
|
+
or type(value["size_bytes"]) is not int
|
|
247
|
+
or value["size_bytes"] < 1
|
|
248
|
+
or not isinstance(value["sha256"], str)
|
|
249
|
+
or re.fullmatch(r"[a-f0-9]{64}", value["sha256"]) is None
|
|
250
|
+
):
|
|
251
|
+
raise ValueError("invalid completion marker or loader settings")
|
|
252
|
+
_groups(value["groups"])
|
|
253
|
+
return value
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _input_record(value, configuration, step):
|
|
257
|
+
fields = {
|
|
258
|
+
"file",
|
|
259
|
+
"step",
|
|
260
|
+
"sha256",
|
|
261
|
+
"size_bytes",
|
|
262
|
+
"live_stat",
|
|
263
|
+
"marker",
|
|
264
|
+
"marker_sha256",
|
|
265
|
+
"marker_size_bytes",
|
|
266
|
+
"marker_stat",
|
|
267
|
+
"groups",
|
|
268
|
+
"n_rollouts",
|
|
269
|
+
"n_prompts",
|
|
270
|
+
}
|
|
271
|
+
if type(value) is not dict or set(value) != fields:
|
|
272
|
+
raise ValueError("invalid watch input manifest")
|
|
273
|
+
if not isinstance(value["file"], str) or value["step"] != step:
|
|
274
|
+
raise ValueError("invalid watch input step or filename")
|
|
275
|
+
_marker(
|
|
276
|
+
{
|
|
277
|
+
"schema_version": 1,
|
|
278
|
+
**{key: configuration[key] for key in ("run", "format", "reward_column")},
|
|
279
|
+
**{
|
|
280
|
+
key: value[key]
|
|
281
|
+
for key in ("step", "file", "sha256", "size_bytes", "groups")
|
|
282
|
+
},
|
|
283
|
+
},
|
|
284
|
+
configuration,
|
|
285
|
+
value["file"],
|
|
286
|
+
)
|
|
287
|
+
if (
|
|
288
|
+
value["marker"] != value["file"] + _MARKER_SUFFIX
|
|
289
|
+
or not isinstance(value["marker_sha256"], str)
|
|
290
|
+
or re.fullmatch(r"[a-f0-9]{64}", value["marker_sha256"]) is None
|
|
291
|
+
or type(value["marker_size_bytes"]) is not int
|
|
292
|
+
or value["marker_size_bytes"] < 1
|
|
293
|
+
or type(value["n_rollouts"]) is not int
|
|
294
|
+
or value["n_rollouts"] != sum(value["groups"].values())
|
|
295
|
+
or type(value["n_prompts"]) is not int
|
|
296
|
+
or value["n_prompts"] != len(value["groups"])
|
|
297
|
+
):
|
|
298
|
+
raise ValueError("invalid watch input counts or completion marker")
|
|
299
|
+
for name, size in (
|
|
300
|
+
("live_stat", value["size_bytes"]),
|
|
301
|
+
("marker_stat", value["marker_size_bytes"]),
|
|
302
|
+
):
|
|
303
|
+
metadata = value[name]
|
|
304
|
+
if (
|
|
305
|
+
type(metadata) is not dict
|
|
306
|
+
or set(metadata)
|
|
307
|
+
!= {"device", "inode", "size_bytes", "mtime_ns", "ctime_ns"}
|
|
308
|
+
or any(type(item) is not int for item in metadata.values())
|
|
309
|
+
or metadata["size_bytes"] != size
|
|
310
|
+
):
|
|
311
|
+
raise ValueError("invalid watch input metadata")
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def seal_watch_step(file, *, format, run, expected_groups, reward_column=None):
|
|
315
|
+
"""Seal a closed file against independently supplied prompt completion counts."""
|
|
316
|
+
_platform()
|
|
317
|
+
reward_column = _settings(format, run, reward_column)
|
|
318
|
+
groups = _groups(expected_groups)
|
|
319
|
+
path = Path(os.path.abspath(file))
|
|
320
|
+
match = _pattern(format).fullmatch(path.name)
|
|
321
|
+
if match is None:
|
|
322
|
+
raise ValueError("unsupported rollout filename")
|
|
323
|
+
marker = path.with_name(path.name + _MARKER_SUFFIX)
|
|
324
|
+
with _open(path.parent, directory=True) as parent:
|
|
325
|
+
try:
|
|
326
|
+
os.stat(marker.name, dir_fd=parent, follow_symlinks=False)
|
|
327
|
+
except FileNotFoundError:
|
|
328
|
+
pass
|
|
329
|
+
else:
|
|
330
|
+
raise FileExistsError("completion marker already exists")
|
|
331
|
+
configuration = {"format": format, "run": run, "reward_column": reward_column}
|
|
332
|
+
with TemporaryDirectory(prefix="rollout-watch-seal-") as temporary:
|
|
333
|
+
copied = Path(temporary) / path.name
|
|
334
|
+
with copied.open("xb") as destination:
|
|
335
|
+
captured = _capture(path, destination)
|
|
336
|
+
rows = _load(copied, configuration)
|
|
337
|
+
_population(rows, groups)
|
|
338
|
+
with _open(path) as descriptor:
|
|
339
|
+
if _stat(os.fstat(descriptor)) != captured["stat"]:
|
|
340
|
+
raise ValueError("watch input changed before sealing")
|
|
341
|
+
value = {
|
|
342
|
+
"schema_version": 1,
|
|
343
|
+
**configuration,
|
|
344
|
+
"step": int(match.group(1)),
|
|
345
|
+
"file": path.name,
|
|
346
|
+
"size_bytes": captured["size_bytes"],
|
|
347
|
+
"sha256": captured["sha256"],
|
|
348
|
+
"groups": groups,
|
|
349
|
+
}
|
|
350
|
+
_publish(marker, _json(value).encode("utf-8"))
|
|
351
|
+
return marker
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@dataclass(frozen=True)
|
|
355
|
+
class Candidate:
|
|
356
|
+
file: Path
|
|
357
|
+
marker: Path
|
|
358
|
+
step: int
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
@dataclass
|
|
362
|
+
class Stage:
|
|
363
|
+
path: Path
|
|
364
|
+
new_file: Path
|
|
365
|
+
rows: pd.DataFrame
|
|
366
|
+
input_record: dict
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@dataclass(frozen=True)
|
|
370
|
+
class Record:
|
|
371
|
+
path: Path
|
|
372
|
+
manifest: dict
|
|
373
|
+
sha256: str
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _tree(path):
|
|
377
|
+
result = []
|
|
378
|
+
with _open(path, directory=True) as descriptor:
|
|
379
|
+
for name in sorted(os.listdir(descriptor)):
|
|
380
|
+
mode = os.stat(name, dir_fd=descriptor, follow_symlinks=False).st_mode
|
|
381
|
+
child = path / name
|
|
382
|
+
if stat.S_ISDIR(mode):
|
|
383
|
+
result.extend(_tree(child))
|
|
384
|
+
elif stat.S_ISREG(mode):
|
|
385
|
+
result.append(child)
|
|
386
|
+
else:
|
|
387
|
+
raise ValueError(
|
|
388
|
+
"state artifacts must be regular files without symbolic links"
|
|
389
|
+
)
|
|
390
|
+
return result
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _artifacts(path):
|
|
394
|
+
result = {}
|
|
395
|
+
for file in _tree(path):
|
|
396
|
+
if file == path / "manifest.json":
|
|
397
|
+
continue
|
|
398
|
+
captured = _capture(file)
|
|
399
|
+
result[file.relative_to(path).as_posix()] = {
|
|
400
|
+
key: captured[key] for key in ("sha256", "size_bytes")
|
|
401
|
+
}
|
|
402
|
+
return result
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
class WatchStore:
|
|
406
|
+
"""Own one locked state and commit complete prefixes by directory rename."""
|
|
407
|
+
|
|
408
|
+
def __init__(self, source, state, configuration):
|
|
409
|
+
self.source = Path(os.path.abspath(source))
|
|
410
|
+
self.state = Path(os.path.abspath(state))
|
|
411
|
+
self.configuration = _decode(_json(configuration))
|
|
412
|
+
self.configuration["reward_column"] = _settings(
|
|
413
|
+
self.configuration.get("format"),
|
|
414
|
+
self.configuration.get("run"),
|
|
415
|
+
self.configuration.get("reward_column"),
|
|
416
|
+
)
|
|
417
|
+
for name, minimum in (("start_step", 0), ("step_stride", 1)):
|
|
418
|
+
value = self.configuration.get(name)
|
|
419
|
+
if type(value) is not int or value < minimum:
|
|
420
|
+
raise ValueError(f"invalid watch {name}")
|
|
421
|
+
self.records = []
|
|
422
|
+
self.commits = self.state / "commits"
|
|
423
|
+
self._lock = None
|
|
424
|
+
self._failed = False
|
|
425
|
+
self._rows = None
|
|
426
|
+
|
|
427
|
+
@property
|
|
428
|
+
def next_step(self):
|
|
429
|
+
return (
|
|
430
|
+
self.configuration["start_step"]
|
|
431
|
+
+ len(self.records) * self.configuration["step_stride"]
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
def __enter__(self):
|
|
435
|
+
fcntl = _platform()
|
|
436
|
+
if (
|
|
437
|
+
self.source == self.state
|
|
438
|
+
or self.source in self.state.parents
|
|
439
|
+
or self.state in self.source.parents
|
|
440
|
+
):
|
|
441
|
+
raise ValueError("source and state directories must be disjoint")
|
|
442
|
+
with _open(self.source, directory=True) as descriptor:
|
|
443
|
+
source_stat = os.fstat(descriptor)
|
|
444
|
+
self.configuration["_source_identity"] = {
|
|
445
|
+
"path": str(self.source),
|
|
446
|
+
"device": source_stat.st_dev,
|
|
447
|
+
"inode": source_stat.st_ino,
|
|
448
|
+
}
|
|
449
|
+
self.configuration_sha256 = _sha(_json(self.configuration).encode("utf-8"))
|
|
450
|
+
try:
|
|
451
|
+
with _open(self.state.parent, directory=True) as parent:
|
|
452
|
+
try:
|
|
453
|
+
os.mkdir(self.state.name, mode=0o700, dir_fd=parent)
|
|
454
|
+
os.fsync(parent)
|
|
455
|
+
except FileExistsError:
|
|
456
|
+
pass
|
|
457
|
+
with _open(self.state, directory=True) as directory:
|
|
458
|
+
current = os.fstat(directory)
|
|
459
|
+
if (
|
|
460
|
+
current.st_uid != os.geteuid()
|
|
461
|
+
or stat.S_IMODE(current.st_mode) & 0o077
|
|
462
|
+
):
|
|
463
|
+
raise ValueError(
|
|
464
|
+
"watch state must be private and owned by the current user"
|
|
465
|
+
)
|
|
466
|
+
self._lock = os.open(
|
|
467
|
+
"lock",
|
|
468
|
+
os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
469
|
+
0o600,
|
|
470
|
+
dir_fd=directory,
|
|
471
|
+
)
|
|
472
|
+
lock_stat = os.fstat(self._lock)
|
|
473
|
+
if (
|
|
474
|
+
not stat.S_ISREG(lock_stat.st_mode)
|
|
475
|
+
or lock_stat.st_uid != os.geteuid()
|
|
476
|
+
):
|
|
477
|
+
raise ValueError("watch lock must be a regular owned file")
|
|
478
|
+
try:
|
|
479
|
+
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
480
|
+
except BlockingIOError:
|
|
481
|
+
raise ValueError(
|
|
482
|
+
"watch state is locked by another writer"
|
|
483
|
+
) from None
|
|
484
|
+
if _stat(
|
|
485
|
+
os.stat("lock", dir_fd=directory, follow_symlinks=False)
|
|
486
|
+
) != _stat(lock_stat):
|
|
487
|
+
raise ValueError("watch lock changed")
|
|
488
|
+
names = set(os.listdir(directory))
|
|
489
|
+
if names - {"lock", "commits", "configuration.json"}:
|
|
490
|
+
raise ValueError("unknown watch state files")
|
|
491
|
+
if "commits" not in names:
|
|
492
|
+
os.mkdir("commits", mode=0o700, dir_fd=directory)
|
|
493
|
+
os.fsync(directory)
|
|
494
|
+
configuration_file = self.state / "configuration.json"
|
|
495
|
+
if configuration_file.exists() or configuration_file.is_symlink():
|
|
496
|
+
if _decode(_read(configuration_file)) != self.configuration:
|
|
497
|
+
raise ValueError(
|
|
498
|
+
"watch configuration does not match initialized state"
|
|
499
|
+
)
|
|
500
|
+
else:
|
|
501
|
+
with _open(self.commits, directory=True) as descriptor:
|
|
502
|
+
if os.listdir(descriptor):
|
|
503
|
+
raise ValueError("watch state configuration is missing")
|
|
504
|
+
_publish(configuration_file, _json(self.configuration).encode("utf-8"))
|
|
505
|
+
self._root_ids = {self.source: (source_stat.st_dev, source_stat.st_ino)}
|
|
506
|
+
for path in (self.state, self.commits):
|
|
507
|
+
with _open(path, directory=True) as descriptor:
|
|
508
|
+
observed = os.fstat(descriptor)
|
|
509
|
+
self._root_ids[path] = (observed.st_dev, observed.st_ino)
|
|
510
|
+
self._recover()
|
|
511
|
+
return self
|
|
512
|
+
except BaseException:
|
|
513
|
+
self.__exit__(None, None, None)
|
|
514
|
+
raise
|
|
515
|
+
|
|
516
|
+
def __exit__(self, *args):
|
|
517
|
+
if self._lock is not None:
|
|
518
|
+
os.close(self._lock)
|
|
519
|
+
self._lock = None
|
|
520
|
+
|
|
521
|
+
def _ready(self):
|
|
522
|
+
if self._lock is None:
|
|
523
|
+
raise ValueError("watch store is not open")
|
|
524
|
+
if self._failed:
|
|
525
|
+
raise ValueError("watch publication failed; reopen and verify state")
|
|
526
|
+
if _sha(_json(self.configuration).encode("utf-8")) != self.configuration_sha256:
|
|
527
|
+
raise ValueError("watch configuration changed")
|
|
528
|
+
for path, expected in self._root_ids.items():
|
|
529
|
+
with _open(path, directory=True) as descriptor:
|
|
530
|
+
current = os.fstat(descriptor)
|
|
531
|
+
if (current.st_dev, current.st_ino) != expected:
|
|
532
|
+
raise ValueError("watch directory identity changed")
|
|
533
|
+
with _open(self.state / "lock") as descriptor:
|
|
534
|
+
current, held = os.fstat(descriptor), os.fstat(self._lock)
|
|
535
|
+
if (current.st_dev, current.st_ino) != (held.st_dev, held.st_ino):
|
|
536
|
+
raise ValueError("watch lock changed")
|
|
537
|
+
|
|
538
|
+
def _live(self, record, *, hashes=False):
|
|
539
|
+
for name, expected, digest in (
|
|
540
|
+
(record["file"], record["live_stat"], record["sha256"]),
|
|
541
|
+
(record["marker"], record["marker_stat"], record["marker_sha256"]),
|
|
542
|
+
):
|
|
543
|
+
path = self.source / name
|
|
544
|
+
with _open(path) as descriptor:
|
|
545
|
+
if _stat(os.fstat(descriptor)) != expected:
|
|
546
|
+
raise ValueError("committed watch input changed")
|
|
547
|
+
if hashes and _capture(path)["sha256"] != digest:
|
|
548
|
+
raise ValueError("committed watch input hash changed")
|
|
549
|
+
|
|
550
|
+
def _recover(self):
|
|
551
|
+
self.records = []
|
|
552
|
+
with _open(self.commits, directory=True) as descriptor:
|
|
553
|
+
names = sorted(os.listdir(descriptor))
|
|
554
|
+
for name in names:
|
|
555
|
+
mode = os.stat(name, dir_fd=descriptor, follow_symlinks=False).st_mode
|
|
556
|
+
if not stat.S_ISDIR(mode):
|
|
557
|
+
raise ValueError("invalid committed watch directory")
|
|
558
|
+
if name.startswith(".pending-"):
|
|
559
|
+
continue
|
|
560
|
+
if name != f"{len(self.records) + 1:012d}":
|
|
561
|
+
raise ValueError("noncontiguous watch commit chain")
|
|
562
|
+
path = self.commits / name
|
|
563
|
+
encoded = _read(path / "manifest.json")
|
|
564
|
+
manifest = _decode(encoded)
|
|
565
|
+
previous = self.records[-1].sha256 if self.records else None
|
|
566
|
+
if (
|
|
567
|
+
set(manifest) != _MANIFEST_FIELDS
|
|
568
|
+
or type(manifest["schema_version"]) is not int
|
|
569
|
+
or manifest["schema_version"] != 1
|
|
570
|
+
or type(manifest["sequence"]) is not int
|
|
571
|
+
or manifest["sequence"] != len(self.records) + 1
|
|
572
|
+
or type(manifest["step"]) is not int
|
|
573
|
+
or manifest["step"] != self.next_step
|
|
574
|
+
or manifest["previous_sha256"] != previous
|
|
575
|
+
or manifest["configuration"] != self.configuration
|
|
576
|
+
or manifest["configuration_sha256"] != self.configuration_sha256
|
|
577
|
+
or type(manifest["input"]) is not dict
|
|
578
|
+
or type(manifest["metadata"]) is not dict
|
|
579
|
+
):
|
|
580
|
+
raise ValueError("invalid watch manifest chain or configuration")
|
|
581
|
+
self._verify_artifacts(path, manifest["artifacts"], manifest["input"])
|
|
582
|
+
self._live(manifest["input"], hashes=True)
|
|
583
|
+
self.records.append(Record(path, manifest, _sha(encoded)))
|
|
584
|
+
if self.records:
|
|
585
|
+
# A surviving rename may precede an interrupted parent-directory fsync.
|
|
586
|
+
_sync_directory(self.commits)
|
|
587
|
+
self._rows = _load(self.records[-1].path / "source", self.configuration)
|
|
588
|
+
self.discover()
|
|
589
|
+
|
|
590
|
+
def _verify_artifacts(self, path, expected, incoming):
|
|
591
|
+
_input_record(incoming, self.configuration, self.next_step)
|
|
592
|
+
if type(expected) is not dict or _artifacts(path) != expected:
|
|
593
|
+
raise ValueError("watch artifact inventory or hash changed")
|
|
594
|
+
inputs = [record.manifest["input"] for record in self.records] + [incoming]
|
|
595
|
+
sources = {
|
|
596
|
+
"source/" + record["file"]: {
|
|
597
|
+
key: record[key] for key in ("sha256", "size_bytes")
|
|
598
|
+
}
|
|
599
|
+
for record in inputs
|
|
600
|
+
}
|
|
601
|
+
if {
|
|
602
|
+
name: digest
|
|
603
|
+
for name, digest in expected.items()
|
|
604
|
+
if name.startswith("source/")
|
|
605
|
+
} != sources:
|
|
606
|
+
raise ValueError(
|
|
607
|
+
"watch source artifact prefix does not match committed inputs"
|
|
608
|
+
)
|
|
609
|
+
marker = expected.get(incoming["marker"])
|
|
610
|
+
if marker != {
|
|
611
|
+
"sha256": incoming["marker_sha256"],
|
|
612
|
+
"size_bytes": incoming["marker_size_bytes"],
|
|
613
|
+
}:
|
|
614
|
+
raise ValueError("watch completion marker artifact changed")
|
|
615
|
+
completion = _marker(
|
|
616
|
+
_decode(_read(path / incoming["marker"])),
|
|
617
|
+
self.configuration,
|
|
618
|
+
incoming["file"],
|
|
619
|
+
)
|
|
620
|
+
if any(
|
|
621
|
+
completion[key] != incoming[key]
|
|
622
|
+
for key in ("file", "step", "sha256", "size_bytes", "groups")
|
|
623
|
+
):
|
|
624
|
+
raise ValueError(
|
|
625
|
+
"watch input manifest does not match its completion marker"
|
|
626
|
+
)
|
|
627
|
+
|
|
628
|
+
def discover(self):
|
|
629
|
+
"""Capture eligible names without allowing future failures to gate a prefix."""
|
|
630
|
+
self._ready()
|
|
631
|
+
for record in self.records:
|
|
632
|
+
self._live(record.manifest["input"])
|
|
633
|
+
pattern = _pattern(self.configuration["format"])
|
|
634
|
+
inventory = {}
|
|
635
|
+
with _open(self.source, directory=True) as descriptor:
|
|
636
|
+
for name in os.listdir(descriptor):
|
|
637
|
+
basename = name.removesuffix(_MARKER_SUFFIX)
|
|
638
|
+
match = pattern.fullmatch(basename)
|
|
639
|
+
if match is not None:
|
|
640
|
+
step = int(match.group(1))
|
|
641
|
+
if step >= self.configuration["start_step"]:
|
|
642
|
+
inventory.setdefault(step, set()).add(name)
|
|
643
|
+
accepted = {
|
|
644
|
+
record.manifest["step"]: record.manifest["input"]["file"]
|
|
645
|
+
for record in self.records
|
|
646
|
+
}
|
|
647
|
+
for step, names in sorted(inventory.items()):
|
|
648
|
+
if step >= self.next_step:
|
|
649
|
+
break
|
|
650
|
+
if (step - self.configuration["start_step"]) % self.configuration[
|
|
651
|
+
"step_stride"
|
|
652
|
+
]:
|
|
653
|
+
raise ValueError(
|
|
654
|
+
"input step does not match configured watch stride"
|
|
655
|
+
)
|
|
656
|
+
aliases = {name.removesuffix(_MARKER_SUFFIX) for name in names}
|
|
657
|
+
if aliases != {accepted.get(step)}:
|
|
658
|
+
raise ValueError("late input alias at finalized watch step")
|
|
659
|
+
candidates = []
|
|
660
|
+
expected = self.next_step
|
|
661
|
+
for step, names in sorted(inventory.items()):
|
|
662
|
+
if step < self.next_step:
|
|
663
|
+
continue
|
|
664
|
+
if step != expected:
|
|
665
|
+
break
|
|
666
|
+
try:
|
|
667
|
+
candidate = self._candidate(step, names, descriptor)
|
|
668
|
+
except ValueError:
|
|
669
|
+
if not candidates:
|
|
670
|
+
raise
|
|
671
|
+
break
|
|
672
|
+
if candidate is None:
|
|
673
|
+
break
|
|
674
|
+
candidates.append(candidate)
|
|
675
|
+
expected += self.configuration["step_stride"]
|
|
676
|
+
return candidates
|
|
677
|
+
|
|
678
|
+
def _candidate(self, step, names, descriptor):
|
|
679
|
+
aliases = {name.removesuffix(_MARKER_SUFFIX) for name in names}
|
|
680
|
+
if len(aliases) != 1:
|
|
681
|
+
raise ValueError("multiple input aliases for one watch step")
|
|
682
|
+
name = next(iter(aliases))
|
|
683
|
+
marker = name + _MARKER_SUFFIX
|
|
684
|
+
if marker not in names:
|
|
685
|
+
return None
|
|
686
|
+
if name not in names:
|
|
687
|
+
raise ValueError("published completion marker has missing input")
|
|
688
|
+
for entry in (name, marker):
|
|
689
|
+
try:
|
|
690
|
+
mode = os.stat(entry, dir_fd=descriptor, follow_symlinks=False).st_mode
|
|
691
|
+
except OSError as exc:
|
|
692
|
+
raise ValueError("watch input changed during discovery") from exc
|
|
693
|
+
if not stat.S_ISREG(mode):
|
|
694
|
+
raise ValueError(
|
|
695
|
+
"watch inputs must be regular files without symbolic links"
|
|
696
|
+
)
|
|
697
|
+
return Candidate(self.source / name, self.source / marker, step)
|
|
698
|
+
|
|
699
|
+
def stage(self, candidate):
|
|
700
|
+
candidates = self.discover()
|
|
701
|
+
if not candidates or candidate != candidates[0]:
|
|
702
|
+
raise ValueError("only the next expected watch step may be staged")
|
|
703
|
+
if self.records:
|
|
704
|
+
last = self.records[-1]
|
|
705
|
+
# Validate previous private bytes before reusing their inodes.
|
|
706
|
+
if _artifacts(last.path) != last.manifest["artifacts"]:
|
|
707
|
+
raise ValueError("watch artifact changed before staging")
|
|
708
|
+
directory = self.commits / (".pending-" + uuid4().hex)
|
|
709
|
+
directory.mkdir(mode=0o700)
|
|
710
|
+
snapshot = directory / "source"
|
|
711
|
+
snapshot.mkdir(mode=0o700)
|
|
712
|
+
if self.records:
|
|
713
|
+
for file in (self.records[-1].path / "source").iterdir():
|
|
714
|
+
os.link(file, snapshot / file.name, follow_symlinks=False)
|
|
715
|
+
marker_copy = directory / candidate.marker.name
|
|
716
|
+
with marker_copy.open("xb") as stream:
|
|
717
|
+
marker_info, content = _capture(candidate.marker, stream, content=True)
|
|
718
|
+
marker = _marker(_decode(content), self.configuration, candidate.file.name)
|
|
719
|
+
copied = snapshot / candidate.file.name
|
|
720
|
+
with copied.open("xb") as stream:
|
|
721
|
+
captured = _capture(candidate.file, stream)
|
|
722
|
+
if any(captured[key] != marker[key] for key in ("sha256", "size_bytes")):
|
|
723
|
+
raise ValueError("completed input hash or size does not match its marker")
|
|
724
|
+
rows = _load(copied, self.configuration)
|
|
725
|
+
_population(rows, marker["groups"])
|
|
726
|
+
if self._rows is not None:
|
|
727
|
+
validate_rollouts(pd.concat([self._rows, rows], ignore_index=True))
|
|
728
|
+
record = {
|
|
729
|
+
"file": candidate.file.name,
|
|
730
|
+
"step": candidate.step,
|
|
731
|
+
"sha256": captured["sha256"],
|
|
732
|
+
"size_bytes": captured["size_bytes"],
|
|
733
|
+
"live_stat": captured["stat"],
|
|
734
|
+
"marker": candidate.marker.name,
|
|
735
|
+
"marker_sha256": marker_info["sha256"],
|
|
736
|
+
"marker_size_bytes": marker_info["size_bytes"],
|
|
737
|
+
"marker_stat": marker_info["stat"],
|
|
738
|
+
"groups": marker["groups"],
|
|
739
|
+
"n_rollouts": len(rows),
|
|
740
|
+
"n_prompts": rows.prompt_id.nunique(),
|
|
741
|
+
}
|
|
742
|
+
self._live(record)
|
|
743
|
+
copied.chmod(0o400)
|
|
744
|
+
return Stage(directory, copied, rows, record)
|
|
745
|
+
|
|
746
|
+
def commit(self, stage, metadata):
|
|
747
|
+
self._ready()
|
|
748
|
+
if type(metadata) is not dict:
|
|
749
|
+
raise ValueError("watch metadata must be an object")
|
|
750
|
+
if (
|
|
751
|
+
stage.path.parent != self.commits
|
|
752
|
+
or not stage.path.name.startswith(".pending-")
|
|
753
|
+
or stage.input_record["step"] != self.next_step
|
|
754
|
+
):
|
|
755
|
+
raise ValueError("invalid watch stage or sequence")
|
|
756
|
+
self.discover()
|
|
757
|
+
self._live(stage.input_record)
|
|
758
|
+
sequence = len(self.records) + 1
|
|
759
|
+
artifacts = _artifacts(stage.path)
|
|
760
|
+
self._verify_artifacts(stage.path, artifacts, stage.input_record)
|
|
761
|
+
manifest = {
|
|
762
|
+
"schema_version": 1,
|
|
763
|
+
"sequence": sequence,
|
|
764
|
+
"step": self.next_step,
|
|
765
|
+
"previous_sha256": self.records[-1].sha256 if self.records else None,
|
|
766
|
+
"configuration": self.configuration,
|
|
767
|
+
"configuration_sha256": self.configuration_sha256,
|
|
768
|
+
"input": stage.input_record,
|
|
769
|
+
"artifacts": artifacts,
|
|
770
|
+
"metadata": metadata,
|
|
771
|
+
}
|
|
772
|
+
encoded = _json(manifest).encode("utf-8")
|
|
773
|
+
destination = self.commits / f"{sequence:012d}"
|
|
774
|
+
try:
|
|
775
|
+
_publish(stage.path / "manifest.json", encoded)
|
|
776
|
+
for file in _tree(stage.path):
|
|
777
|
+
with _open(file) as descriptor:
|
|
778
|
+
os.fsync(descriptor)
|
|
779
|
+
directories = {file.parent for file in _tree(stage.path)} | {stage.path}
|
|
780
|
+
for directory in sorted(
|
|
781
|
+
directories, key=lambda path: len(path.parts), reverse=True
|
|
782
|
+
):
|
|
783
|
+
_sync_directory(directory)
|
|
784
|
+
if destination.exists() or destination.is_symlink():
|
|
785
|
+
raise ValueError("watch commit destination already exists")
|
|
786
|
+
self.discover()
|
|
787
|
+
self._live(stage.input_record)
|
|
788
|
+
os.rename(stage.path, destination)
|
|
789
|
+
_sync_directory(self.commits)
|
|
790
|
+
except BaseException:
|
|
791
|
+
self._failed = True
|
|
792
|
+
raise
|
|
793
|
+
record = Record(destination, _decode(encoded), _sha(encoded))
|
|
794
|
+
self.records.append(record)
|
|
795
|
+
self._rows = (
|
|
796
|
+
stage.rows.copy()
|
|
797
|
+
if self._rows is None
|
|
798
|
+
else pd.concat([self._rows, stage.rows], ignore_index=True)
|
|
799
|
+
)
|
|
800
|
+
return record
|