rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/producer.py
ADDED
|
@@ -0,0 +1,663 @@
|
|
|
1
|
+
"""Bounded asynchronous publication of complete rollout steps on local POSIX storage."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import platform
|
|
10
|
+
import re
|
|
11
|
+
import stat
|
|
12
|
+
from concurrent.futures import Future
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from importlib.metadata import version
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from queue import Queue
|
|
17
|
+
from threading import Lock, Thread
|
|
18
|
+
|
|
19
|
+
import pandas as pd
|
|
20
|
+
|
|
21
|
+
from ._watch_storage import (
|
|
22
|
+
_capture,
|
|
23
|
+
_decode,
|
|
24
|
+
_groups,
|
|
25
|
+
_json,
|
|
26
|
+
_open,
|
|
27
|
+
_pattern,
|
|
28
|
+
_platform,
|
|
29
|
+
_population,
|
|
30
|
+
_publish,
|
|
31
|
+
_read,
|
|
32
|
+
_settings,
|
|
33
|
+
_sha,
|
|
34
|
+
_stat,
|
|
35
|
+
_sync_directory,
|
|
36
|
+
)
|
|
37
|
+
from .adapters import from_table
|
|
38
|
+
from .schema import validate_rollouts
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ProducerQueueFull(ValueError):
|
|
42
|
+
"""The complete payload was rejected before acceptance; nothing was dropped."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _producer_pins():
|
|
46
|
+
package = Path(__file__).parent
|
|
47
|
+
return {
|
|
48
|
+
"source": {
|
|
49
|
+
name: _sha(_read(package / name))
|
|
50
|
+
for name in (
|
|
51
|
+
"producer.py",
|
|
52
|
+
"integrations.py",
|
|
53
|
+
"adapters.py",
|
|
54
|
+
"schema.py",
|
|
55
|
+
"_watch_storage.py",
|
|
56
|
+
)
|
|
57
|
+
},
|
|
58
|
+
"runtime": {
|
|
59
|
+
"python": platform.python_version(),
|
|
60
|
+
"numpy": version("numpy"),
|
|
61
|
+
"pandas": version("pandas"),
|
|
62
|
+
"pyarrow": version("pyarrow"),
|
|
63
|
+
},
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class PendingReceipt:
|
|
68
|
+
"""Submission is pending until acknowledge(); result() also waits for publication."""
|
|
69
|
+
|
|
70
|
+
def __init__(self, step):
|
|
71
|
+
self.step = step
|
|
72
|
+
self._durable = Future()
|
|
73
|
+
self._published = Future()
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def durable(self):
|
|
77
|
+
return self._durable.done() and self._durable.exception() is None
|
|
78
|
+
|
|
79
|
+
def acknowledge(self, timeout=None):
|
|
80
|
+
"""Return only after the immutable spool record and directory are synced."""
|
|
81
|
+
return self._durable.result(timeout)
|
|
82
|
+
|
|
83
|
+
def result(self, timeout=None):
|
|
84
|
+
"""Return only after the source data and completion marker are synced."""
|
|
85
|
+
return self._published.result(timeout)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class _Submission:
|
|
90
|
+
payload: bytes
|
|
91
|
+
fingerprint: str
|
|
92
|
+
receipt: PendingReceipt
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _mkdir(path, *, private=False):
|
|
96
|
+
with _open(path.parent, directory=True) as parent:
|
|
97
|
+
try:
|
|
98
|
+
os.mkdir(path.name, mode=0o700, dir_fd=parent)
|
|
99
|
+
os.fsync(parent)
|
|
100
|
+
except FileExistsError:
|
|
101
|
+
pass
|
|
102
|
+
with _open(path, directory=True) as descriptor:
|
|
103
|
+
observed = os.fstat(descriptor)
|
|
104
|
+
if private and (
|
|
105
|
+
observed.st_uid != os.geteuid() or stat.S_IMODE(observed.st_mode) & 0o077
|
|
106
|
+
):
|
|
107
|
+
raise ValueError(
|
|
108
|
+
"producer spool must be private and owned by the current user"
|
|
109
|
+
)
|
|
110
|
+
return observed.st_dev, observed.st_ino
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _lock(path):
|
|
114
|
+
fcntl = _platform()
|
|
115
|
+
with _open(path.parent, directory=True) as parent:
|
|
116
|
+
descriptor = os.open(
|
|
117
|
+
path.name,
|
|
118
|
+
os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
119
|
+
0o600,
|
|
120
|
+
dir_fd=parent,
|
|
121
|
+
)
|
|
122
|
+
try:
|
|
123
|
+
observed = os.fstat(descriptor)
|
|
124
|
+
if not stat.S_ISREG(observed.st_mode) or observed.st_uid != os.geteuid():
|
|
125
|
+
raise ValueError("producer lock must be a regular owned file")
|
|
126
|
+
try:
|
|
127
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
128
|
+
except BlockingIOError:
|
|
129
|
+
raise ValueError(
|
|
130
|
+
"producer directory is locked by another writer"
|
|
131
|
+
) from None
|
|
132
|
+
if _stat(os.stat(path.name, dir_fd=parent, follow_symlinks=False)) != _stat(
|
|
133
|
+
observed
|
|
134
|
+
):
|
|
135
|
+
raise ValueError("producer lock changed")
|
|
136
|
+
return descriptor
|
|
137
|
+
except BaseException:
|
|
138
|
+
os.close(descriptor)
|
|
139
|
+
raise
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class AsyncRolloutProducer:
|
|
143
|
+
"""One writer with durable replay, immutable outputs, and bounded pending payloads.
|
|
144
|
+
|
|
145
|
+
submit() validates and copies canonical rows but performs no analysis. Native
|
|
146
|
+
encoding and filesystem publication run on one worker. Pending byte limits
|
|
147
|
+
count the canonical JSON snapshot; a native step is bounded by the same cap.
|
|
148
|
+
Durable records are retained for recovery, so total disk use grows with the run.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
def __init__(
|
|
152
|
+
self,
|
|
153
|
+
source,
|
|
154
|
+
spool,
|
|
155
|
+
*,
|
|
156
|
+
format,
|
|
157
|
+
run,
|
|
158
|
+
start_step,
|
|
159
|
+
step_stride=1,
|
|
160
|
+
max_pending=8,
|
|
161
|
+
max_pending_bytes=64 * 1024 * 1024,
|
|
162
|
+
):
|
|
163
|
+
_platform()
|
|
164
|
+
reward_column = _settings(format, run, None)
|
|
165
|
+
for name, value, minimum in (
|
|
166
|
+
("start_step", start_step, 0),
|
|
167
|
+
("step_stride", step_stride, 1),
|
|
168
|
+
("max_pending", max_pending, 1),
|
|
169
|
+
("max_pending_bytes", max_pending_bytes, 1),
|
|
170
|
+
):
|
|
171
|
+
if type(value) is not int or value < minimum:
|
|
172
|
+
raise ValueError(f"invalid producer {name}")
|
|
173
|
+
self.source = Path(os.path.abspath(source))
|
|
174
|
+
self.spool = Path(os.path.abspath(spool))
|
|
175
|
+
if (
|
|
176
|
+
self.source == self.spool
|
|
177
|
+
or self.source in self.spool.parents
|
|
178
|
+
or self.spool in self.source.parents
|
|
179
|
+
):
|
|
180
|
+
raise ValueError("producer source and spool directories must be disjoint")
|
|
181
|
+
self.configuration = {
|
|
182
|
+
"schema_version": 1,
|
|
183
|
+
"format": format,
|
|
184
|
+
"run": run,
|
|
185
|
+
"reward_column": reward_column,
|
|
186
|
+
"start_step": start_step,
|
|
187
|
+
"step_stride": step_stride,
|
|
188
|
+
"pins": _producer_pins(),
|
|
189
|
+
"limits": {
|
|
190
|
+
"max_pending": max_pending,
|
|
191
|
+
"max_pending_bytes": max_pending_bytes,
|
|
192
|
+
},
|
|
193
|
+
}
|
|
194
|
+
self.max_pending = max_pending
|
|
195
|
+
self.max_pending_bytes = max_pending_bytes
|
|
196
|
+
self._submissions = {}
|
|
197
|
+
self._records = {}
|
|
198
|
+
self._sample_ids = set()
|
|
199
|
+
self._prompt_content = {}
|
|
200
|
+
self._locks = {}
|
|
201
|
+
self._roots = {}
|
|
202
|
+
self._pending = self._pending_bytes = 0
|
|
203
|
+
self._mutex = Lock()
|
|
204
|
+
self._close_mutex = Lock()
|
|
205
|
+
self._queue = Queue(maxsize=max_pending)
|
|
206
|
+
self._thread = None
|
|
207
|
+
self._failure = None
|
|
208
|
+
self._closing = False
|
|
209
|
+
|
|
210
|
+
@property
|
|
211
|
+
def next_step(self):
|
|
212
|
+
return (
|
|
213
|
+
self.configuration["start_step"]
|
|
214
|
+
+ len(self._submissions) * self.configuration["step_stride"]
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
def __enter__(self):
|
|
218
|
+
if self._thread is not None or self._locks or self._closing:
|
|
219
|
+
raise ValueError("producer is already open")
|
|
220
|
+
try:
|
|
221
|
+
self._roots[self.source] = _mkdir(self.source)
|
|
222
|
+
self._locks[self.source / ".rolloutdiff-producer.lock"] = _lock(
|
|
223
|
+
self.source / ".rolloutdiff-producer.lock"
|
|
224
|
+
)
|
|
225
|
+
self._roots[self.spool] = _mkdir(self.spool, private=True)
|
|
226
|
+
self._locks[self.spool / "lock"] = _lock(self.spool / "lock")
|
|
227
|
+
self._roots[self.spool / "records"] = _mkdir(
|
|
228
|
+
self.spool / "records", private=True
|
|
229
|
+
)
|
|
230
|
+
self.configuration["source"] = {
|
|
231
|
+
"path": str(self.source),
|
|
232
|
+
"device": self._roots[self.source][0],
|
|
233
|
+
"inode": self._roots[self.source][1],
|
|
234
|
+
}
|
|
235
|
+
self._configuration_bytes = _json(self.configuration).encode()
|
|
236
|
+
path = self.spool / "configuration.json"
|
|
237
|
+
if path.exists() or path.is_symlink():
|
|
238
|
+
if _read(path) != self._configuration_bytes:
|
|
239
|
+
raise ValueError("producer configuration changed")
|
|
240
|
+
else:
|
|
241
|
+
if list((self.spool / "records").iterdir()):
|
|
242
|
+
raise ValueError(
|
|
243
|
+
"producer configuration missing from nonempty spool"
|
|
244
|
+
)
|
|
245
|
+
_publish(path, self._configuration_bytes)
|
|
246
|
+
self._ready()
|
|
247
|
+
self._recover()
|
|
248
|
+
self._thread = Thread(
|
|
249
|
+
target=self._work, name="rollout-producer", daemon=False
|
|
250
|
+
)
|
|
251
|
+
self._thread.start()
|
|
252
|
+
return self
|
|
253
|
+
except BaseException:
|
|
254
|
+
self._release()
|
|
255
|
+
raise
|
|
256
|
+
|
|
257
|
+
def _ready(self, *, draining=False):
|
|
258
|
+
if not self._locks or (self._closing and not draining):
|
|
259
|
+
raise ValueError("producer is not open")
|
|
260
|
+
if self._failure is not None:
|
|
261
|
+
raise ValueError(
|
|
262
|
+
"producer failed; close and reopen for recovery"
|
|
263
|
+
) from self._failure
|
|
264
|
+
if _json(self.configuration).encode() != self._configuration_bytes:
|
|
265
|
+
raise ValueError("producer configuration changed")
|
|
266
|
+
if _producer_pins() != self.configuration["pins"]:
|
|
267
|
+
raise ValueError("producer source or runtime pins changed")
|
|
268
|
+
if {
|
|
269
|
+
"max_pending": self.max_pending,
|
|
270
|
+
"max_pending_bytes": self.max_pending_bytes,
|
|
271
|
+
} != self.configuration["limits"]:
|
|
272
|
+
raise ValueError("producer buffer configuration changed")
|
|
273
|
+
for path, identity in self._roots.items():
|
|
274
|
+
with _open(path, directory=True) as descriptor:
|
|
275
|
+
observed = os.fstat(descriptor)
|
|
276
|
+
if (observed.st_dev, observed.st_ino) != identity:
|
|
277
|
+
raise ValueError("producer directory identity changed")
|
|
278
|
+
for path, descriptor in self._locks.items():
|
|
279
|
+
with _open(path) as current:
|
|
280
|
+
left, right = os.fstat(current), os.fstat(descriptor)
|
|
281
|
+
if (left.st_dev, left.st_ino) != (right.st_dev, right.st_ino):
|
|
282
|
+
raise ValueError("producer lock changed")
|
|
283
|
+
if _read(self.spool / "configuration.json") != self._configuration_bytes:
|
|
284
|
+
raise ValueError("producer disk configuration changed")
|
|
285
|
+
|
|
286
|
+
def _payload(self, step, rows, expected_groups):
|
|
287
|
+
if type(step) is not int or step < 0:
|
|
288
|
+
raise ValueError("producer step must be a nonnegative integer")
|
|
289
|
+
groups = _groups(expected_groups)
|
|
290
|
+
rows = validate_rollouts(
|
|
291
|
+
rows if isinstance(rows, pd.DataFrame) else pd.DataFrame(rows)
|
|
292
|
+
)
|
|
293
|
+
if (
|
|
294
|
+
rows.empty
|
|
295
|
+
or not rows.run.eq(self.configuration["run"]).all()
|
|
296
|
+
or not rows.step.eq(step).all()
|
|
297
|
+
):
|
|
298
|
+
raise ValueError("producer rows must match the run and step")
|
|
299
|
+
_population(rows, groups)
|
|
300
|
+
return _json(
|
|
301
|
+
{"step": step, "groups": groups, "rows": rows.to_dict("records")}
|
|
302
|
+
).encode()
|
|
303
|
+
|
|
304
|
+
def submit(self, step, rows, *, expected_groups):
|
|
305
|
+
"""Copy a complete step; queue overflow rejects it without advancing sequence."""
|
|
306
|
+
payload = self._payload(step, rows, expected_groups)
|
|
307
|
+
fingerprint = _sha(payload)
|
|
308
|
+
with self._mutex:
|
|
309
|
+
self._ready()
|
|
310
|
+
previous = self._submissions.get(step)
|
|
311
|
+
if previous is not None:
|
|
312
|
+
if previous.fingerprint != fingerprint:
|
|
313
|
+
raise ValueError("producer step conflicts with an accepted payload")
|
|
314
|
+
if (
|
|
315
|
+
previous.receipt._published.done()
|
|
316
|
+
and previous.receipt._published.exception() is None
|
|
317
|
+
):
|
|
318
|
+
self._verify_output(_decode(_read(self._records[step])))
|
|
319
|
+
return previous.receipt
|
|
320
|
+
if step != self.next_step:
|
|
321
|
+
raise ValueError(
|
|
322
|
+
f"producer expected step {self.next_step}, received {step}"
|
|
323
|
+
)
|
|
324
|
+
if (
|
|
325
|
+
self._pending >= self.max_pending
|
|
326
|
+
or self._pending_bytes + len(payload) > self.max_pending_bytes
|
|
327
|
+
):
|
|
328
|
+
raise ProducerQueueFull("producer queue is full; step was not accepted")
|
|
329
|
+
self._remember_identities(_decode(payload)["rows"])
|
|
330
|
+
item = _Submission(payload, fingerprint, PendingReceipt(step))
|
|
331
|
+
self._submissions[step] = item
|
|
332
|
+
self._pending += 1
|
|
333
|
+
self._pending_bytes += len(payload)
|
|
334
|
+
self._queue.put_nowait(item)
|
|
335
|
+
return item.receipt
|
|
336
|
+
|
|
337
|
+
def _remember_identities(self, rows):
|
|
338
|
+
ids = {row["sample_id"] for row in rows}
|
|
339
|
+
if self._sample_ids & ids:
|
|
340
|
+
raise ValueError("duplicate rollout identity across producer steps")
|
|
341
|
+
prompts = {
|
|
342
|
+
row["prompt_id"]: _sha(
|
|
343
|
+
_json(
|
|
344
|
+
{"prompt": row["prompt"], "messages": row["prompt_messages"]}
|
|
345
|
+
).encode()
|
|
346
|
+
)
|
|
347
|
+
for row in rows
|
|
348
|
+
}
|
|
349
|
+
if any(
|
|
350
|
+
identity in self._prompt_content
|
|
351
|
+
and self._prompt_content[identity] != digest
|
|
352
|
+
for identity, digest in prompts.items()
|
|
353
|
+
):
|
|
354
|
+
raise ValueError(
|
|
355
|
+
"prompt identity refers to changed content across producer steps"
|
|
356
|
+
)
|
|
357
|
+
self._sample_ids.update(ids)
|
|
358
|
+
self._prompt_content.update(prompts)
|
|
359
|
+
|
|
360
|
+
def receipt(self, step):
|
|
361
|
+
"""Return a known receipt, including recovered acknowledged steps."""
|
|
362
|
+
item = self._submissions.get(step)
|
|
363
|
+
return item.receipt if item is not None else None
|
|
364
|
+
|
|
365
|
+
def _encode(self, payload):
|
|
366
|
+
rows = []
|
|
367
|
+
for row in payload["rows"]:
|
|
368
|
+
prompt = (
|
|
369
|
+
row["prompt_messages"]
|
|
370
|
+
if row["prompt_messages"] is not None
|
|
371
|
+
else row["prompt"]
|
|
372
|
+
)
|
|
373
|
+
response = (
|
|
374
|
+
row["response_messages"]
|
|
375
|
+
if row["response_messages"] is not None
|
|
376
|
+
else row["response"]
|
|
377
|
+
)
|
|
378
|
+
common = {
|
|
379
|
+
key: row[key]
|
|
380
|
+
for key in ("step", "sample_id", "prompt_id", "tool_calls")
|
|
381
|
+
}
|
|
382
|
+
if self.configuration["format"] == "trl":
|
|
383
|
+
rows.append(
|
|
384
|
+
common
|
|
385
|
+
| {
|
|
386
|
+
"prompt": prompt,
|
|
387
|
+
"completion": response,
|
|
388
|
+
"reward": row["reward"],
|
|
389
|
+
}
|
|
390
|
+
)
|
|
391
|
+
else:
|
|
392
|
+
rows.append(
|
|
393
|
+
common
|
|
394
|
+
| {"input": prompt, "output": response, "score": row["reward"]}
|
|
395
|
+
)
|
|
396
|
+
table = pd.DataFrame(rows)
|
|
397
|
+
canonical = self._canonical_table(table)
|
|
398
|
+
if (
|
|
399
|
+
self._payload(payload["step"], canonical, payload["groups"])
|
|
400
|
+
!= _json(payload).encode()
|
|
401
|
+
):
|
|
402
|
+
raise ValueError(
|
|
403
|
+
"native format cannot represent the canonical text and messages without changes"
|
|
404
|
+
)
|
|
405
|
+
if self.configuration["format"] == "trl":
|
|
406
|
+
buffer = io.BytesIO()
|
|
407
|
+
try:
|
|
408
|
+
table.to_parquet(buffer, index=False)
|
|
409
|
+
except (ValueError, TypeError) as exc:
|
|
410
|
+
raise ValueError(
|
|
411
|
+
"TRL parquet cannot represent this canonical payload"
|
|
412
|
+
) from exc
|
|
413
|
+
return buffer.getvalue()
|
|
414
|
+
return "".join(
|
|
415
|
+
json.dumps(row, ensure_ascii=False, allow_nan=False, sort_keys=True) + "\n"
|
|
416
|
+
for row in rows
|
|
417
|
+
).encode()
|
|
418
|
+
|
|
419
|
+
def _canonical_table(self, table):
|
|
420
|
+
columns = (
|
|
421
|
+
{"response": "completion"}
|
|
422
|
+
if self.configuration["format"] == "trl"
|
|
423
|
+
else {
|
|
424
|
+
"prompt": "input",
|
|
425
|
+
"response": "output",
|
|
426
|
+
"reward": "score",
|
|
427
|
+
}
|
|
428
|
+
)
|
|
429
|
+
return from_table(table, run=self.configuration["run"], columns=columns)
|
|
430
|
+
|
|
431
|
+
def _decode_native(self, native):
|
|
432
|
+
table = (
|
|
433
|
+
pd.read_parquet(io.BytesIO(native))
|
|
434
|
+
if self.configuration["format"] == "trl"
|
|
435
|
+
else pd.DataFrame(
|
|
436
|
+
[json.loads(line) for line in native.decode().splitlines()]
|
|
437
|
+
)
|
|
438
|
+
)
|
|
439
|
+
return self._canonical_table(table)
|
|
440
|
+
|
|
441
|
+
def _persist(self, item):
|
|
442
|
+
self._ready(draining=True)
|
|
443
|
+
payload = _decode(item.payload)
|
|
444
|
+
native = self._encode(payload)
|
|
445
|
+
decoded = self._decode_native(native)
|
|
446
|
+
if (
|
|
447
|
+
_sha(self._payload(payload["step"], decoded, payload["groups"]))
|
|
448
|
+
!= item.fingerprint
|
|
449
|
+
):
|
|
450
|
+
raise ValueError(
|
|
451
|
+
"native format cannot round-trip the exact canonical payload"
|
|
452
|
+
)
|
|
453
|
+
if len(native) > self.max_pending_bytes:
|
|
454
|
+
raise ProducerQueueFull(
|
|
455
|
+
"encoded step exceeds producer byte limit; not acknowledged"
|
|
456
|
+
)
|
|
457
|
+
record = {
|
|
458
|
+
"schema_version": 1,
|
|
459
|
+
"step": payload["step"],
|
|
460
|
+
"groups": payload["groups"],
|
|
461
|
+
"payload_sha256": item.fingerprint,
|
|
462
|
+
"data_sha256": _sha(native),
|
|
463
|
+
"data": base64.b64encode(native).decode("ascii"),
|
|
464
|
+
}
|
|
465
|
+
path = self.spool / "records" / f"{payload['step']:012d}.json"
|
|
466
|
+
self._ready(draining=True)
|
|
467
|
+
_publish(path, _json(record).encode())
|
|
468
|
+
self._records[payload["step"]] = path
|
|
469
|
+
item.receipt._durable.set_result(
|
|
470
|
+
{"step": payload["step"], "spool": path, "sha256": item.fingerprint}
|
|
471
|
+
)
|
|
472
|
+
return record
|
|
473
|
+
|
|
474
|
+
def _paths(self, step):
|
|
475
|
+
basename = (
|
|
476
|
+
f"completions_{step}.parquet"
|
|
477
|
+
if self.configuration["format"] == "trl"
|
|
478
|
+
else f"{step}.jsonl"
|
|
479
|
+
)
|
|
480
|
+
data = self.source / basename
|
|
481
|
+
return data, self.source / (basename + ".complete.json")
|
|
482
|
+
|
|
483
|
+
def _marker(self, record, data):
|
|
484
|
+
path, _ = self._paths(record["step"])
|
|
485
|
+
return {
|
|
486
|
+
"schema_version": 1,
|
|
487
|
+
**{
|
|
488
|
+
key: self.configuration[key]
|
|
489
|
+
for key in ("run", "format", "reward_column")
|
|
490
|
+
},
|
|
491
|
+
"step": record["step"],
|
|
492
|
+
"file": path.name,
|
|
493
|
+
"size_bytes": len(data),
|
|
494
|
+
"sha256": record["data_sha256"],
|
|
495
|
+
"groups": record["groups"],
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
def _verify_output(self, record):
|
|
499
|
+
path, marker = self._paths(record["step"])
|
|
500
|
+
data = base64.b64decode(record["data"], validate=True)
|
|
501
|
+
captured = _capture(path)
|
|
502
|
+
if captured["sha256"] != record["data_sha256"] or captured["size_bytes"] != len(
|
|
503
|
+
data
|
|
504
|
+
):
|
|
505
|
+
raise ValueError("published producer source changed")
|
|
506
|
+
if _read(marker) != _json(self._marker(record, data)).encode():
|
|
507
|
+
raise ValueError("published producer marker changed")
|
|
508
|
+
|
|
509
|
+
def _publish_step(self, record):
|
|
510
|
+
self._ready(draining=True)
|
|
511
|
+
path, marker = self._paths(record["step"])
|
|
512
|
+
data = base64.b64decode(record["data"], validate=True)
|
|
513
|
+
if path.exists() or path.is_symlink():
|
|
514
|
+
captured = _capture(path)
|
|
515
|
+
if captured["sha256"] != record["data_sha256"] or captured[
|
|
516
|
+
"size_bytes"
|
|
517
|
+
] != len(data):
|
|
518
|
+
raise ValueError("producer source conflicts with durable payload")
|
|
519
|
+
elif marker.exists() or marker.is_symlink():
|
|
520
|
+
raise ValueError("published producer source changed or is missing")
|
|
521
|
+
else:
|
|
522
|
+
_publish(path, data)
|
|
523
|
+
encoded_marker = _json(self._marker(record, data)).encode()
|
|
524
|
+
self._ready(draining=True)
|
|
525
|
+
if marker.exists() or marker.is_symlink():
|
|
526
|
+
if _read(marker) != encoded_marker:
|
|
527
|
+
raise ValueError("producer marker conflicts with durable payload")
|
|
528
|
+
else:
|
|
529
|
+
_publish(marker, encoded_marker)
|
|
530
|
+
self._verify_output(record)
|
|
531
|
+
_sync_directory(self.source)
|
|
532
|
+
return {"step": record["step"], "data": path, "marker": marker}
|
|
533
|
+
|
|
534
|
+
def _recover(self):
|
|
535
|
+
files = []
|
|
536
|
+
for path in (self.spool / "records").iterdir():
|
|
537
|
+
if path.name.startswith(".watch-"):
|
|
538
|
+
continue
|
|
539
|
+
if re.fullmatch(r"\d{12,}\.json", path.name) is None:
|
|
540
|
+
raise ValueError("invalid producer spool record")
|
|
541
|
+
files.append(path)
|
|
542
|
+
for path in sorted(files, key=lambda item: int(item.stem)):
|
|
543
|
+
record = _decode(_read(path))
|
|
544
|
+
if (
|
|
545
|
+
set(record)
|
|
546
|
+
!= {
|
|
547
|
+
"schema_version",
|
|
548
|
+
"step",
|
|
549
|
+
"groups",
|
|
550
|
+
"payload_sha256",
|
|
551
|
+
"data_sha256",
|
|
552
|
+
"data",
|
|
553
|
+
}
|
|
554
|
+
or record["schema_version"] != 1
|
|
555
|
+
):
|
|
556
|
+
raise ValueError("invalid producer spool record")
|
|
557
|
+
step = record["step"]
|
|
558
|
+
if (
|
|
559
|
+
type(step) is not int
|
|
560
|
+
or step != self.next_step
|
|
561
|
+
or int(path.stem) != step
|
|
562
|
+
):
|
|
563
|
+
raise ValueError("producer spool sequence is incomplete")
|
|
564
|
+
try:
|
|
565
|
+
native = base64.b64decode(record["data"], validate=True)
|
|
566
|
+
if _sha(native) != record["data_sha256"]:
|
|
567
|
+
raise ValueError("producer spool data changed")
|
|
568
|
+
rows = self._decode_native(native)
|
|
569
|
+
payload = self._payload(step, rows, record["groups"])
|
|
570
|
+
if _sha(payload) != record["payload_sha256"]:
|
|
571
|
+
raise ValueError("producer spool payload changed")
|
|
572
|
+
except (KeyError, TypeError, UnicodeError) as exc:
|
|
573
|
+
raise ValueError("invalid producer spool payload") from exc
|
|
574
|
+
item = _Submission(b"", record["payload_sha256"], PendingReceipt(step))
|
|
575
|
+
self._remember_identities(rows.to_dict("records"))
|
|
576
|
+
self._submissions[step] = item
|
|
577
|
+
self._records[step] = path
|
|
578
|
+
item.receipt._durable.set_result(
|
|
579
|
+
{"step": step, "spool": path, "sha256": item.fingerprint}
|
|
580
|
+
)
|
|
581
|
+
item.receipt._published.set_result(self._publish_step(record))
|
|
582
|
+
expected = {path.name for step in self._records for path in self._paths(step)}
|
|
583
|
+
for path in self.source.iterdir():
|
|
584
|
+
name = path.name.removesuffix(".complete.json")
|
|
585
|
+
if (
|
|
586
|
+
_pattern(self.configuration["format"]).fullmatch(name)
|
|
587
|
+
and path.name not in expected
|
|
588
|
+
):
|
|
589
|
+
raise ValueError(
|
|
590
|
+
"producer source contains an unacknowledged or aliased step"
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
def _work(self):
|
|
594
|
+
while (item := self._queue.get()) is not None:
|
|
595
|
+
try:
|
|
596
|
+
if self._failure is not None:
|
|
597
|
+
raise ValueError(
|
|
598
|
+
"producer worker previously failed"
|
|
599
|
+
) from self._failure
|
|
600
|
+
record = self._persist(item)
|
|
601
|
+
item.receipt._published.set_result(self._publish_step(record))
|
|
602
|
+
except BaseException as exc: # noqa: BLE001 - Always settle worker receipts.
|
|
603
|
+
self._failure = exc
|
|
604
|
+
if not item.receipt._durable.done():
|
|
605
|
+
item.receipt._durable.set_exception(exc)
|
|
606
|
+
item.receipt._published.set_exception(exc)
|
|
607
|
+
finally:
|
|
608
|
+
with self._mutex:
|
|
609
|
+
self._pending -= 1
|
|
610
|
+
self._pending_bytes -= len(item.payload)
|
|
611
|
+
item.payload = b""
|
|
612
|
+
self._queue.task_done()
|
|
613
|
+
self._queue.task_done()
|
|
614
|
+
|
|
615
|
+
def bind(self, name, value):
|
|
616
|
+
"""Pin integration configuration while holding the producer's writer locks."""
|
|
617
|
+
self._ready()
|
|
618
|
+
if not isinstance(name, str) or re.fullmatch(r"[a-z][a-z0-9-]*", name) is None:
|
|
619
|
+
raise ValueError("invalid producer binding name")
|
|
620
|
+
path = self.spool / f"binding-{name}.json"
|
|
621
|
+
encoded = _json(value).encode()
|
|
622
|
+
if path.exists() or path.is_symlink():
|
|
623
|
+
if _read(path) != encoded:
|
|
624
|
+
raise ValueError("producer integration configuration changed")
|
|
625
|
+
else:
|
|
626
|
+
_publish(path, encoded)
|
|
627
|
+
|
|
628
|
+
def flush(self, timeout=None):
|
|
629
|
+
"""Wait for all accepted steps to publish; propagate worker failures."""
|
|
630
|
+
for item in list(self._submissions.values()):
|
|
631
|
+
item.receipt.result(timeout)
|
|
632
|
+
|
|
633
|
+
def _release(self):
|
|
634
|
+
for descriptor in self._locks.values():
|
|
635
|
+
os.close(descriptor)
|
|
636
|
+
self._locks.clear()
|
|
637
|
+
|
|
638
|
+
def close(self):
|
|
639
|
+
with self._close_mutex:
|
|
640
|
+
self._close()
|
|
641
|
+
|
|
642
|
+
def _close(self):
|
|
643
|
+
if self._thread is None:
|
|
644
|
+
self._release()
|
|
645
|
+
return
|
|
646
|
+
try:
|
|
647
|
+
with self._mutex:
|
|
648
|
+
self._closing = True
|
|
649
|
+
self._queue.put(None)
|
|
650
|
+
self._thread.join()
|
|
651
|
+
self.flush()
|
|
652
|
+
for path in self._records.values():
|
|
653
|
+
self._verify_output(_decode(_read(path)))
|
|
654
|
+
finally:
|
|
655
|
+
self._release()
|
|
656
|
+
self._thread = None
|
|
657
|
+
|
|
658
|
+
def __exit__(self, exc_type, exc, traceback):
|
|
659
|
+
try:
|
|
660
|
+
self.close()
|
|
661
|
+
except BaseException:
|
|
662
|
+
if exc_type is None:
|
|
663
|
+
raise
|