rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
"""Optional trainer boundaries and explicitly invoked fixed-prompt probes.
|
|
2
|
+
|
|
3
|
+
Source contracts: TRL 1.18.0 (480dde905c9137898819b27b6c2c6f9be9f34046),
|
|
4
|
+
Transformers 4.56.2 (cd74917ffc3e8f84e4a886052c5ab32b7ac623cc), and verl
|
|
5
|
+
0.9.1 (1876b06d0a3e4e71e06230be10af14492ca8a75b). No trainer is imported
|
|
6
|
+
until make_trl_callback() is called. These hooks never select or load a model.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import copy
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import pandas as pd
|
|
21
|
+
|
|
22
|
+
from ._watch_storage import _decode, _groups, _json, _publish, _read, _sha
|
|
23
|
+
from .adapters import _text, from_table
|
|
24
|
+
from .producer import _mkdir
|
|
25
|
+
|
|
26
|
+
_TRL_VERSIONS = {"trl": "1.18.0", "transformers": "4.56.2"}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def expected_prompt_groups(prompts, *, repeats=1):
|
|
30
|
+
"""Build independent group expectations from an intended exact prompt population."""
|
|
31
|
+
if (
|
|
32
|
+
type(repeats) is not int
|
|
33
|
+
or repeats < 1
|
|
34
|
+
or not isinstance(prompts, (list, tuple))
|
|
35
|
+
or not prompts
|
|
36
|
+
):
|
|
37
|
+
raise ValueError("probe prompts and repeat count must be nonempty and positive")
|
|
38
|
+
groups = Counter()
|
|
39
|
+
for value in prompts:
|
|
40
|
+
text, messages = _text(value)
|
|
41
|
+
identity = ["chat", messages] if messages is not None else ["text", text]
|
|
42
|
+
digest = hashlib.sha256(
|
|
43
|
+
json.dumps(
|
|
44
|
+
identity, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
|
45
|
+
).encode()
|
|
46
|
+
).hexdigest()
|
|
47
|
+
groups[digest] += repeats
|
|
48
|
+
return dict(groups)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _plan(expected_groups):
|
|
52
|
+
if not isinstance(expected_groups, Mapping):
|
|
53
|
+
raise ValueError("expected groups require an independent per-step plan")
|
|
54
|
+
planned = {}
|
|
55
|
+
for step, groups in expected_groups.items():
|
|
56
|
+
if type(step) is not int or step < 0:
|
|
57
|
+
raise ValueError("planned steps must be nonnegative integers")
|
|
58
|
+
planned[step] = _groups(groups)
|
|
59
|
+
return planned
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _plan_records(plan):
|
|
63
|
+
return [{"step": step, "groups": plan[step]} for step in sorted(plan)]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _rows(producer, step, values, source):
|
|
67
|
+
table = pd.DataFrame(copy.deepcopy(values))
|
|
68
|
+
if "run" in table and not table.run.eq(producer.configuration["run"]).all():
|
|
69
|
+
raise ValueError("completed rows contain a different run")
|
|
70
|
+
if "step" in table and not table.step.eq(step).all():
|
|
71
|
+
raise ValueError("completed rows contain a different step")
|
|
72
|
+
table["step"] = step
|
|
73
|
+
if "sample_id" not in table:
|
|
74
|
+
table["sample_id"] = [f"{source}:{step}:{index}" for index in range(len(table))]
|
|
75
|
+
return from_table(table, run=producer.configuration["run"], source=source)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _trl_configuration(trainer):
|
|
79
|
+
if (
|
|
80
|
+
type(trainer).__module__ != "trl.trainer.grpo_trainer"
|
|
81
|
+
or type(trainer).__name__ != "GRPOTrainer"
|
|
82
|
+
):
|
|
83
|
+
raise ValueError("TRL snapshot callback requires the pinned GRPOTrainer class")
|
|
84
|
+
args = trainer.args
|
|
85
|
+
integers = {
|
|
86
|
+
name: getattr(args, name, None)
|
|
87
|
+
for name in (
|
|
88
|
+
"gradient_accumulation_steps",
|
|
89
|
+
"steps_per_generation",
|
|
90
|
+
"num_iterations",
|
|
91
|
+
"per_device_train_batch_size",
|
|
92
|
+
"generation_batch_size",
|
|
93
|
+
"logging_steps",
|
|
94
|
+
)
|
|
95
|
+
}
|
|
96
|
+
if any(type(value) is not int or value < 1 for value in integers.values()):
|
|
97
|
+
raise ValueError(
|
|
98
|
+
"TRL snapshot configuration requires positive integer batch and logging settings"
|
|
99
|
+
)
|
|
100
|
+
world = trainer.accelerator.num_processes
|
|
101
|
+
if type(world) is not int or world < 1:
|
|
102
|
+
raise ValueError("invalid TRL process count")
|
|
103
|
+
if (
|
|
104
|
+
integers["num_iterations"] != 1
|
|
105
|
+
or trainer.num_iterations != 1
|
|
106
|
+
or integers["steps_per_generation"] != integers["gradient_accumulation_steps"]
|
|
107
|
+
or integers["generation_batch_size"]
|
|
108
|
+
!= integers["per_device_train_batch_size"]
|
|
109
|
+
* world
|
|
110
|
+
* integers["steps_per_generation"]
|
|
111
|
+
or args.logging_strategy != "steps"
|
|
112
|
+
or args.logging_first_step
|
|
113
|
+
or not args.dataloader_drop_last
|
|
114
|
+
or args.eval_strategy != "no"
|
|
115
|
+
or args.eval_on_start
|
|
116
|
+
or args.log_unique_prompts
|
|
117
|
+
or getattr(args, "use_transformers_continuous_batching", False)
|
|
118
|
+
or trainer.eval_dataset is not None
|
|
119
|
+
or trainer.rollout_func is not None
|
|
120
|
+
or trainer.environment_factories is not None
|
|
121
|
+
or trainer.tools
|
|
122
|
+
or trainer._is_vlm
|
|
123
|
+
):
|
|
124
|
+
raise ValueError(
|
|
125
|
+
"unsupported TRL logged-snapshot configuration; evaluation and reused/partial generations are excluded"
|
|
126
|
+
)
|
|
127
|
+
return integers | {
|
|
128
|
+
"world_size": world,
|
|
129
|
+
"versions": _TRL_VERSIONS,
|
|
130
|
+
"native_output": os.path.abspath(args.output_dir),
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class _TRLLoggedSnapshot:
|
|
135
|
+
def __init__(self, trainer, producer, expected_groups, reward_column):
|
|
136
|
+
if not isinstance(reward_column, str) or not reward_column.strip():
|
|
137
|
+
raise ValueError("select a named TRL reward column explicitly")
|
|
138
|
+
self.trainer = trainer
|
|
139
|
+
self.producer = producer
|
|
140
|
+
self.expected_groups = _plan(expected_groups)
|
|
141
|
+
self.reward_column = reward_column
|
|
142
|
+
self.configuration = _trl_configuration(trainer)
|
|
143
|
+
self._begin = None
|
|
144
|
+
self._completed = None
|
|
145
|
+
self._started = False
|
|
146
|
+
self._evaluation_seen = False
|
|
147
|
+
if producer is not None:
|
|
148
|
+
config = producer.configuration
|
|
149
|
+
native = Path(self.configuration["native_output"])
|
|
150
|
+
if (
|
|
151
|
+
producer.source == native
|
|
152
|
+
or producer.source in native.parents
|
|
153
|
+
or native in producer.source.parents
|
|
154
|
+
):
|
|
155
|
+
raise ValueError(
|
|
156
|
+
"TRL native and producer output directories must be disjoint"
|
|
157
|
+
)
|
|
158
|
+
stride = self.configuration["logging_steps"]
|
|
159
|
+
if (
|
|
160
|
+
config["format"] != "trl"
|
|
161
|
+
or config["step_stride"] != stride
|
|
162
|
+
or config["start_step"] < stride
|
|
163
|
+
or config["start_step"] % stride
|
|
164
|
+
):
|
|
165
|
+
raise ValueError(
|
|
166
|
+
"TRL producer schedule must match optimizer logging steps"
|
|
167
|
+
)
|
|
168
|
+
if any(
|
|
169
|
+
step % stride or step < config["start_step"]
|
|
170
|
+
for step in self.expected_groups
|
|
171
|
+
):
|
|
172
|
+
raise ValueError("TRL expected groups contain an off-schedule step")
|
|
173
|
+
|
|
174
|
+
def _check(self, args, state):
|
|
175
|
+
if args is not self.trainer.args or state is not self.trainer.state:
|
|
176
|
+
raise ValueError("callback is bound to a different trainer")
|
|
177
|
+
if _trl_configuration(self.trainer) != self.configuration:
|
|
178
|
+
raise ValueError("TRL snapshot configuration changed")
|
|
179
|
+
if self._evaluation_seen:
|
|
180
|
+
raise ValueError("evaluation mixed with TRL training snapshots")
|
|
181
|
+
writer = state.is_world_process_zero
|
|
182
|
+
if (
|
|
183
|
+
type(writer) is not bool
|
|
184
|
+
or writer != self.trainer.accelerator.is_main_process
|
|
185
|
+
):
|
|
186
|
+
raise ValueError("TRL writer rank is inconsistent")
|
|
187
|
+
if writer != (self.producer is not None):
|
|
188
|
+
raise ValueError("only the world-process-zero callback may own a producer")
|
|
189
|
+
return writer
|
|
190
|
+
|
|
191
|
+
def on_train_begin(self, args, state, control, **kwargs):
|
|
192
|
+
writer = self._check(args, state)
|
|
193
|
+
stride = self.configuration["logging_steps"]
|
|
194
|
+
if state.logging_steps != stride:
|
|
195
|
+
raise ValueError(
|
|
196
|
+
"TRL runtime logging cadence differs from the declared cadence"
|
|
197
|
+
)
|
|
198
|
+
if writer:
|
|
199
|
+
if (state.global_step // stride + 1) * stride != self.producer.next_step:
|
|
200
|
+
raise ValueError(
|
|
201
|
+
"TRL resume position is incompatible with the next expected producer step"
|
|
202
|
+
)
|
|
203
|
+
self.producer.bind(
|
|
204
|
+
"trl",
|
|
205
|
+
{
|
|
206
|
+
"configuration": self.configuration,
|
|
207
|
+
"reward_column": self.reward_column,
|
|
208
|
+
"expected_groups": _plan_records(self.expected_groups),
|
|
209
|
+
"scope": "latest-complete-generation-at-logged-optimizer-step",
|
|
210
|
+
},
|
|
211
|
+
)
|
|
212
|
+
self._begin = self._completed = None
|
|
213
|
+
self._started = True
|
|
214
|
+
|
|
215
|
+
def on_step_begin(self, args, state, control, **kwargs):
|
|
216
|
+
self._check(args, state)
|
|
217
|
+
if (
|
|
218
|
+
not self._started
|
|
219
|
+
or self._begin is not None
|
|
220
|
+
or self.trainer._step % self.configuration["steps_per_generation"]
|
|
221
|
+
):
|
|
222
|
+
raise ValueError(
|
|
223
|
+
"TRL generation step is not aligned with a witnessed optimizer step"
|
|
224
|
+
)
|
|
225
|
+
self._begin = (state.global_step, self.trainer._step)
|
|
226
|
+
self._completed = None
|
|
227
|
+
|
|
228
|
+
def on_step_end(self, args, state, control, **kwargs):
|
|
229
|
+
self._check(args, state)
|
|
230
|
+
if self._begin is None or state.global_step != self._begin[0] + 1:
|
|
231
|
+
raise ValueError("TRL optimizer step was not witnessed")
|
|
232
|
+
if (
|
|
233
|
+
self.trainer._step - self._begin[1]
|
|
234
|
+
!= self.configuration["gradient_accumulation_steps"]
|
|
235
|
+
):
|
|
236
|
+
raise ValueError(
|
|
237
|
+
"TRL short or unexpected gradient accumulation cannot establish a complete generation"
|
|
238
|
+
)
|
|
239
|
+
self._begin = None
|
|
240
|
+
self._completed = state.global_step
|
|
241
|
+
|
|
242
|
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
|
243
|
+
writer = self._check(args, state)
|
|
244
|
+
if not self.trainer.model.training or any(
|
|
245
|
+
key.startswith("eval_") for key in (logs or {})
|
|
246
|
+
):
|
|
247
|
+
self._evaluation_seen = True
|
|
248
|
+
raise ValueError("evaluation logs cannot be admitted as training snapshots")
|
|
249
|
+
if "loss" not in (logs or {}):
|
|
250
|
+
return
|
|
251
|
+
if (
|
|
252
|
+
self._completed != state.global_step
|
|
253
|
+
or state.global_step % self.configuration["logging_steps"]
|
|
254
|
+
):
|
|
255
|
+
raise ValueError(
|
|
256
|
+
"TRL loss log does not match a witnessed scheduled optimizer step"
|
|
257
|
+
)
|
|
258
|
+
if not writer:
|
|
259
|
+
return
|
|
260
|
+
groups = self.expected_groups.get(state.global_step)
|
|
261
|
+
if groups is None:
|
|
262
|
+
raise ValueError(
|
|
263
|
+
"independent expected groups are missing for this TRL logging step"
|
|
264
|
+
)
|
|
265
|
+
logs = self.trainer._logs
|
|
266
|
+
try:
|
|
267
|
+
prompts = copy.deepcopy(list(logs["prompt"]))
|
|
268
|
+
completions = copy.deepcopy(list(logs["completion"]))
|
|
269
|
+
rewards = list(logs["rewards"][self.reward_column])
|
|
270
|
+
except KeyError:
|
|
271
|
+
raise ValueError(
|
|
272
|
+
"TRL logged snapshot lacks the selected completion or reward payload"
|
|
273
|
+
) from None
|
|
274
|
+
size = self.configuration["generation_batch_size"]
|
|
275
|
+
if any(len(values) != size for values in (prompts, completions, rewards)):
|
|
276
|
+
raise ValueError("TRL logged generation population is incomplete")
|
|
277
|
+
values = [
|
|
278
|
+
{"prompt": prompt, "response": response, "reward": reward}
|
|
279
|
+
for prompt, response, reward in zip(
|
|
280
|
+
prompts, completions, rewards, strict=True
|
|
281
|
+
)
|
|
282
|
+
]
|
|
283
|
+
rows = _rows(self.producer, state.global_step, values, "trl-logged")
|
|
284
|
+
self.producer.submit(state.global_step, rows, expected_groups=groups)
|
|
285
|
+
|
|
286
|
+
def on_prediction_step(self, args, state, control, **kwargs):
|
|
287
|
+
self._evaluation_seen = True
|
|
288
|
+
raise ValueError(
|
|
289
|
+
"evaluation is unsupported while the TRL snapshot callback is attached"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
def on_evaluate(self, args, state, control, **kwargs):
|
|
293
|
+
self.on_prediction_step(args, state, control, **kwargs)
|
|
294
|
+
|
|
295
|
+
def on_train_end(self, args, state, control, **kwargs):
|
|
296
|
+
if self._check(args, state):
|
|
297
|
+
self.producer.flush()
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def make_trl_callback(trainer, producer, *, expected_groups, reward_column):
|
|
301
|
+
"""Bind a pinned GRPO logged-snapshot adapter to an existing trainer.
|
|
302
|
+
|
|
303
|
+
Only world rank zero receives an open producer; other ranks pass None. The
|
|
304
|
+
independent group plan refers to logged optimizer steps, not all generations.
|
|
305
|
+
Completions precede that optimizer boundary and do not prove checkpoint parity.
|
|
306
|
+
Native TRL parquet is written after on_log, so this hook copies in-memory rows
|
|
307
|
+
into its own producer directory. The producer must not share TRL's native output.
|
|
308
|
+
"""
|
|
309
|
+
try:
|
|
310
|
+
if any(version(name) != required for name, required in _TRL_VERSIONS.items()):
|
|
311
|
+
raise ValueError(
|
|
312
|
+
"TRL callback requires trl==1.18.0 and transformers==4.56.2"
|
|
313
|
+
)
|
|
314
|
+
from transformers import TrainerCallback
|
|
315
|
+
except (PackageNotFoundError, ImportError) as exc:
|
|
316
|
+
raise ImportError(
|
|
317
|
+
"TRL callback requires separately installed trl==1.18.0 and transformers==4.56.2"
|
|
318
|
+
) from exc
|
|
319
|
+
callback = type(
|
|
320
|
+
"TRLLoggedSnapshotCallback", (_TRLLoggedSnapshot, TrainerCallback), {}
|
|
321
|
+
)
|
|
322
|
+
return callback(trainer, producer, expected_groups, reward_column)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
class VerlCompletionLogger:
|
|
326
|
+
"""Publish verl 0.9.1 completed payloads from one designated writer.
|
|
327
|
+
|
|
328
|
+
Pass the native inputs/outputs/gts/scores/reward_extra_infos_dict/dump_path/
|
|
329
|
+
global_steps arguments. Ground truth and unrelated metadata are discarded.
|
|
330
|
+
For training use bind(trainer), which replaces _dump_generations before the
|
|
331
|
+
native unbounded executor. Keep the producer context around trainer.fit() so
|
|
332
|
+
final asynchronous failures propagate through close(). Validation dumps are
|
|
333
|
+
excluded. Calling this adapter from the native executor does not bound that
|
|
334
|
+
upstream queue and is not the supported training integration.
|
|
335
|
+
"""
|
|
336
|
+
|
|
337
|
+
def __init__(self, producer, *, expected_groups, rank=0, writer_rank=0):
|
|
338
|
+
if any(type(value) is not int or value < 0 for value in (rank, writer_rank)):
|
|
339
|
+
raise ValueError("verl ranks must be nonnegative integers")
|
|
340
|
+
self.writer = rank == writer_rank
|
|
341
|
+
if self.writer != (producer is not None):
|
|
342
|
+
raise ValueError("only the designated verl writer may own a producer")
|
|
343
|
+
self.producer = producer
|
|
344
|
+
self._bound_trainer = None
|
|
345
|
+
self.expected_groups = _plan(expected_groups)
|
|
346
|
+
if self.writer:
|
|
347
|
+
if producer.configuration["format"] != "verl":
|
|
348
|
+
raise ValueError("verl completion logger requires a verl producer")
|
|
349
|
+
producer.bind(
|
|
350
|
+
"verl",
|
|
351
|
+
{
|
|
352
|
+
"source_version": "0.9.1",
|
|
353
|
+
"writer_rank": writer_rank,
|
|
354
|
+
"expected_groups": _plan_records(self.expected_groups),
|
|
355
|
+
},
|
|
356
|
+
)
|
|
357
|
+
|
|
358
|
+
def _check_trainer(self, trainer):
|
|
359
|
+
if (
|
|
360
|
+
type(trainer).__module__ != "verl.trainer.ppo.ray_trainer"
|
|
361
|
+
or type(trainer).__name__ != "RayPPOTrainer"
|
|
362
|
+
):
|
|
363
|
+
raise ValueError("verl binding requires the pinned RayPPOTrainer class")
|
|
364
|
+
settings = trainer.config.trainer
|
|
365
|
+
rollout_path = settings.get("rollout_data_dir")
|
|
366
|
+
if (
|
|
367
|
+
not rollout_path
|
|
368
|
+
or Path(os.path.abspath(rollout_path)) != self.producer.source
|
|
369
|
+
or settings.get("validation_data_dir")
|
|
370
|
+
or trainer._dump_futures
|
|
371
|
+
):
|
|
372
|
+
raise ValueError(
|
|
373
|
+
"verl binding requires matching training output, no validation dump, and an empty native queue"
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
def bind(self, trainer):
|
|
377
|
+
"""Bypass verl's native executor; queue rejection reaches the training call."""
|
|
378
|
+
if not self.writer or self._bound_trainer is not None:
|
|
379
|
+
raise ValueError(
|
|
380
|
+
"verl logger binding requires one unbound designated writer"
|
|
381
|
+
)
|
|
382
|
+
try:
|
|
383
|
+
if version("verl") != "0.9.1":
|
|
384
|
+
raise ValueError("verl binding requires verl==0.9.1")
|
|
385
|
+
except PackageNotFoundError:
|
|
386
|
+
raise ImportError(
|
|
387
|
+
"verl binding requires separately installed verl==0.9.1"
|
|
388
|
+
) from None
|
|
389
|
+
self._check_trainer(trainer)
|
|
390
|
+
|
|
391
|
+
def dump(inputs, outputs, gts, scores, reward_extra_infos_dict, dump_path):
|
|
392
|
+
self._check_trainer(trainer)
|
|
393
|
+
return self(
|
|
394
|
+
inputs,
|
|
395
|
+
outputs,
|
|
396
|
+
gts,
|
|
397
|
+
scores,
|
|
398
|
+
reward_extra_infos_dict,
|
|
399
|
+
dump_path,
|
|
400
|
+
trainer.global_steps,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
self.producer.bind(
|
|
404
|
+
"verl-trainer", {"source_version": "0.9.1", "boundary": "_dump_generations"}
|
|
405
|
+
)
|
|
406
|
+
self._bound_trainer = trainer
|
|
407
|
+
trainer._dump_generations = dump
|
|
408
|
+
return self
|
|
409
|
+
|
|
410
|
+
def __call__(
|
|
411
|
+
self,
|
|
412
|
+
inputs,
|
|
413
|
+
outputs,
|
|
414
|
+
gts,
|
|
415
|
+
scores,
|
|
416
|
+
reward_extra_infos_dict,
|
|
417
|
+
dump_path,
|
|
418
|
+
global_steps,
|
|
419
|
+
):
|
|
420
|
+
if not self.writer:
|
|
421
|
+
return None
|
|
422
|
+
if Path(os.path.abspath(dump_path)) != self.producer.source:
|
|
423
|
+
raise ValueError("verl dump path differs from the locked producer output")
|
|
424
|
+
if len(inputs) != len(outputs) or len(inputs) != len(scores):
|
|
425
|
+
raise ValueError("verl completed payload columns have different lengths")
|
|
426
|
+
groups = self.expected_groups.get(global_steps)
|
|
427
|
+
if groups is None:
|
|
428
|
+
raise ValueError(
|
|
429
|
+
"independent expected groups are missing for this verl step"
|
|
430
|
+
)
|
|
431
|
+
values = [
|
|
432
|
+
{"prompt": prompt, "response": response, "reward": score}
|
|
433
|
+
for prompt, response, score in zip(inputs, outputs, scores, strict=True)
|
|
434
|
+
]
|
|
435
|
+
for name in ("sample_id", "prompt_id", "tool_calls"):
|
|
436
|
+
if name in reward_extra_infos_dict:
|
|
437
|
+
column = reward_extra_infos_dict[name]
|
|
438
|
+
if len(column) != len(values):
|
|
439
|
+
raise ValueError(
|
|
440
|
+
"verl identity or tool-call metadata has the wrong length"
|
|
441
|
+
)
|
|
442
|
+
for row, value in zip(values, column, strict=True):
|
|
443
|
+
row[name] = value
|
|
444
|
+
rows = _rows(self.producer, global_steps, values, "verl-logged")
|
|
445
|
+
return self.producer.submit(global_steps, rows, expected_groups=groups)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
class FixedProbeHook:
|
|
449
|
+
"""Invoke a caller-owned sampler explicitly, then asynchronously publish its rows.
|
|
450
|
+
|
|
451
|
+
Sampling is synchronous and its model/snapshot synchronization, scoring, RNG,
|
|
452
|
+
distributed execution, and cost belong to the caller. An attempt is persisted
|
|
453
|
+
before calling sample(prompts, repeats, step=..., snapshot_id=...). Returned rows
|
|
454
|
+
require prompt, response and finite reward. Interrupted or failed attempts are
|
|
455
|
+
spent; restart never invokes the sampler again automatically.
|
|
456
|
+
"""
|
|
457
|
+
|
|
458
|
+
def __init__(
|
|
459
|
+
self, producer, *, prompts, repeats, sample, training_run, training_source
|
|
460
|
+
):
|
|
461
|
+
training_source = Path(os.path.abspath(training_source))
|
|
462
|
+
if (
|
|
463
|
+
not isinstance(training_run, str)
|
|
464
|
+
or not training_run.strip()
|
|
465
|
+
or producer.configuration["run"] == training_run
|
|
466
|
+
or producer.source == training_source
|
|
467
|
+
or producer.source in training_source.parents
|
|
468
|
+
or training_source in producer.source.parents
|
|
469
|
+
):
|
|
470
|
+
raise ValueError(
|
|
471
|
+
"fixed probes require a separate run and disjoint training output"
|
|
472
|
+
)
|
|
473
|
+
if not callable(sample):
|
|
474
|
+
raise ValueError("fixed probe requires an explicit trusted sampler")
|
|
475
|
+
self.producer = producer
|
|
476
|
+
self.prompts = copy.deepcopy(prompts)
|
|
477
|
+
self.repeats = repeats
|
|
478
|
+
self.sample = sample
|
|
479
|
+
self.groups = expected_prompt_groups(self.prompts, repeats=repeats)
|
|
480
|
+
self._prompt_bytes = _json(
|
|
481
|
+
{"prompts": self.prompts, "repeats": repeats}
|
|
482
|
+
).encode()
|
|
483
|
+
self._fingerprint = _sha(self._prompt_bytes)
|
|
484
|
+
producer.bind(
|
|
485
|
+
"fixed-probe",
|
|
486
|
+
{
|
|
487
|
+
"prompts": self.prompts,
|
|
488
|
+
"repeats": repeats,
|
|
489
|
+
"groups": self.groups,
|
|
490
|
+
"prompt_sha256": self._fingerprint,
|
|
491
|
+
"training_run": training_run,
|
|
492
|
+
"training_source": str(training_source),
|
|
493
|
+
"start_step": producer.configuration["start_step"],
|
|
494
|
+
"step_stride": producer.configuration["step_stride"],
|
|
495
|
+
},
|
|
496
|
+
)
|
|
497
|
+
self.attempts = producer.spool / "probe-attempts"
|
|
498
|
+
producer._roots[self.attempts] = _mkdir(self.attempts, private=True)
|
|
499
|
+
|
|
500
|
+
def __call__(self, step, *, snapshot_id):
|
|
501
|
+
self.producer._ready()
|
|
502
|
+
if type(step) is not int or step < 0:
|
|
503
|
+
raise ValueError("probe step must be a nonnegative integer")
|
|
504
|
+
if not isinstance(snapshot_id, str) or not snapshot_id.strip():
|
|
505
|
+
raise ValueError("fixed probe requires an explicit model snapshot identity")
|
|
506
|
+
if (
|
|
507
|
+
_json({"prompts": self.prompts, "repeats": self.repeats}).encode()
|
|
508
|
+
!= self._prompt_bytes
|
|
509
|
+
or expected_prompt_groups(self.prompts, repeats=self.repeats) != self.groups
|
|
510
|
+
):
|
|
511
|
+
raise ValueError("fixed probe prompt configuration changed")
|
|
512
|
+
config = self.producer.configuration
|
|
513
|
+
if (
|
|
514
|
+
step < config["start_step"]
|
|
515
|
+
or (step - config["start_step"]) % config["step_stride"]
|
|
516
|
+
):
|
|
517
|
+
return None
|
|
518
|
+
attempt = {
|
|
519
|
+
"schema_version": 1,
|
|
520
|
+
"step": step,
|
|
521
|
+
"snapshot_id": snapshot_id,
|
|
522
|
+
"prompt_sha256": self._fingerprint,
|
|
523
|
+
}
|
|
524
|
+
path = self.attempts / f"{step:012d}.json"
|
|
525
|
+
if path.exists() or path.is_symlink():
|
|
526
|
+
if _decode(_read(path)) != attempt:
|
|
527
|
+
raise ValueError(
|
|
528
|
+
"fixed probe attempt conflicts with its snapshot identity"
|
|
529
|
+
)
|
|
530
|
+
existing = self.producer.receipt(step)
|
|
531
|
+
if existing is None:
|
|
532
|
+
raise ValueError(
|
|
533
|
+
"fixed probe attempt is spent or uncertain; automatic retry is forbidden"
|
|
534
|
+
)
|
|
535
|
+
return existing
|
|
536
|
+
if step != self.producer.next_step:
|
|
537
|
+
raise ValueError(f"fixed probe expected step {self.producer.next_step}")
|
|
538
|
+
_publish(path, _json(attempt).encode())
|
|
539
|
+
values = self.sample(
|
|
540
|
+
copy.deepcopy(self.prompts),
|
|
541
|
+
self.repeats,
|
|
542
|
+
step=step,
|
|
543
|
+
snapshot_id=snapshot_id,
|
|
544
|
+
)
|
|
545
|
+
rows = _rows(self.producer, step, values, "fixed-probe")
|
|
546
|
+
actual_prompts = [
|
|
547
|
+
row.prompt_messages if row.prompt_messages is not None else row.prompt
|
|
548
|
+
for row in rows.itertuples()
|
|
549
|
+
]
|
|
550
|
+
if expected_prompt_groups(actual_prompts) != self.groups:
|
|
551
|
+
raise ValueError("fixed probe returned different exact prompt content")
|
|
552
|
+
return self.producer.submit(step, rows, expected_groups=self.groups)
|