rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,552 @@
1
+ """Optional trainer boundaries and explicitly invoked fixed-prompt probes.
2
+
3
+ Source contracts: TRL 1.18.0 (480dde905c9137898819b27b6c2c6f9be9f34046),
4
+ Transformers 4.56.2 (cd74917ffc3e8f84e4a886052c5ab32b7ac623cc), and verl
5
+ 0.9.1 (1876b06d0a3e4e71e06230be10af14492ca8a75b). No trainer is imported
6
+ until make_trl_callback() is called. These hooks never select or load a model.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import copy
12
+ import hashlib
13
+ import json
14
+ import os
15
+ from collections import Counter
16
+ from collections.abc import Mapping
17
+ from importlib.metadata import PackageNotFoundError, version
18
+ from pathlib import Path
19
+
20
+ import pandas as pd
21
+
22
+ from ._watch_storage import _decode, _groups, _json, _publish, _read, _sha
23
+ from .adapters import _text, from_table
24
+ from .producer import _mkdir
25
+
26
+ _TRL_VERSIONS = {"trl": "1.18.0", "transformers": "4.56.2"}
27
+
28
+
29
+ def expected_prompt_groups(prompts, *, repeats=1):
30
+ """Build independent group expectations from an intended exact prompt population."""
31
+ if (
32
+ type(repeats) is not int
33
+ or repeats < 1
34
+ or not isinstance(prompts, (list, tuple))
35
+ or not prompts
36
+ ):
37
+ raise ValueError("probe prompts and repeat count must be nonempty and positive")
38
+ groups = Counter()
39
+ for value in prompts:
40
+ text, messages = _text(value)
41
+ identity = ["chat", messages] if messages is not None else ["text", text]
42
+ digest = hashlib.sha256(
43
+ json.dumps(
44
+ identity, ensure_ascii=False, sort_keys=True, separators=(",", ":")
45
+ ).encode()
46
+ ).hexdigest()
47
+ groups[digest] += repeats
48
+ return dict(groups)
49
+
50
+
51
+ def _plan(expected_groups):
52
+ if not isinstance(expected_groups, Mapping):
53
+ raise ValueError("expected groups require an independent per-step plan")
54
+ planned = {}
55
+ for step, groups in expected_groups.items():
56
+ if type(step) is not int or step < 0:
57
+ raise ValueError("planned steps must be nonnegative integers")
58
+ planned[step] = _groups(groups)
59
+ return planned
60
+
61
+
62
+ def _plan_records(plan):
63
+ return [{"step": step, "groups": plan[step]} for step in sorted(plan)]
64
+
65
+
66
+ def _rows(producer, step, values, source):
67
+ table = pd.DataFrame(copy.deepcopy(values))
68
+ if "run" in table and not table.run.eq(producer.configuration["run"]).all():
69
+ raise ValueError("completed rows contain a different run")
70
+ if "step" in table and not table.step.eq(step).all():
71
+ raise ValueError("completed rows contain a different step")
72
+ table["step"] = step
73
+ if "sample_id" not in table:
74
+ table["sample_id"] = [f"{source}:{step}:{index}" for index in range(len(table))]
75
+ return from_table(table, run=producer.configuration["run"], source=source)
76
+
77
+
78
+ def _trl_configuration(trainer):
79
+ if (
80
+ type(trainer).__module__ != "trl.trainer.grpo_trainer"
81
+ or type(trainer).__name__ != "GRPOTrainer"
82
+ ):
83
+ raise ValueError("TRL snapshot callback requires the pinned GRPOTrainer class")
84
+ args = trainer.args
85
+ integers = {
86
+ name: getattr(args, name, None)
87
+ for name in (
88
+ "gradient_accumulation_steps",
89
+ "steps_per_generation",
90
+ "num_iterations",
91
+ "per_device_train_batch_size",
92
+ "generation_batch_size",
93
+ "logging_steps",
94
+ )
95
+ }
96
+ if any(type(value) is not int or value < 1 for value in integers.values()):
97
+ raise ValueError(
98
+ "TRL snapshot configuration requires positive integer batch and logging settings"
99
+ )
100
+ world = trainer.accelerator.num_processes
101
+ if type(world) is not int or world < 1:
102
+ raise ValueError("invalid TRL process count")
103
+ if (
104
+ integers["num_iterations"] != 1
105
+ or trainer.num_iterations != 1
106
+ or integers["steps_per_generation"] != integers["gradient_accumulation_steps"]
107
+ or integers["generation_batch_size"]
108
+ != integers["per_device_train_batch_size"]
109
+ * world
110
+ * integers["steps_per_generation"]
111
+ or args.logging_strategy != "steps"
112
+ or args.logging_first_step
113
+ or not args.dataloader_drop_last
114
+ or args.eval_strategy != "no"
115
+ or args.eval_on_start
116
+ or args.log_unique_prompts
117
+ or getattr(args, "use_transformers_continuous_batching", False)
118
+ or trainer.eval_dataset is not None
119
+ or trainer.rollout_func is not None
120
+ or trainer.environment_factories is not None
121
+ or trainer.tools
122
+ or trainer._is_vlm
123
+ ):
124
+ raise ValueError(
125
+ "unsupported TRL logged-snapshot configuration; evaluation and reused/partial generations are excluded"
126
+ )
127
+ return integers | {
128
+ "world_size": world,
129
+ "versions": _TRL_VERSIONS,
130
+ "native_output": os.path.abspath(args.output_dir),
131
+ }
132
+
133
+
134
+ class _TRLLoggedSnapshot:
135
+ def __init__(self, trainer, producer, expected_groups, reward_column):
136
+ if not isinstance(reward_column, str) or not reward_column.strip():
137
+ raise ValueError("select a named TRL reward column explicitly")
138
+ self.trainer = trainer
139
+ self.producer = producer
140
+ self.expected_groups = _plan(expected_groups)
141
+ self.reward_column = reward_column
142
+ self.configuration = _trl_configuration(trainer)
143
+ self._begin = None
144
+ self._completed = None
145
+ self._started = False
146
+ self._evaluation_seen = False
147
+ if producer is not None:
148
+ config = producer.configuration
149
+ native = Path(self.configuration["native_output"])
150
+ if (
151
+ producer.source == native
152
+ or producer.source in native.parents
153
+ or native in producer.source.parents
154
+ ):
155
+ raise ValueError(
156
+ "TRL native and producer output directories must be disjoint"
157
+ )
158
+ stride = self.configuration["logging_steps"]
159
+ if (
160
+ config["format"] != "trl"
161
+ or config["step_stride"] != stride
162
+ or config["start_step"] < stride
163
+ or config["start_step"] % stride
164
+ ):
165
+ raise ValueError(
166
+ "TRL producer schedule must match optimizer logging steps"
167
+ )
168
+ if any(
169
+ step % stride or step < config["start_step"]
170
+ for step in self.expected_groups
171
+ ):
172
+ raise ValueError("TRL expected groups contain an off-schedule step")
173
+
174
+ def _check(self, args, state):
175
+ if args is not self.trainer.args or state is not self.trainer.state:
176
+ raise ValueError("callback is bound to a different trainer")
177
+ if _trl_configuration(self.trainer) != self.configuration:
178
+ raise ValueError("TRL snapshot configuration changed")
179
+ if self._evaluation_seen:
180
+ raise ValueError("evaluation mixed with TRL training snapshots")
181
+ writer = state.is_world_process_zero
182
+ if (
183
+ type(writer) is not bool
184
+ or writer != self.trainer.accelerator.is_main_process
185
+ ):
186
+ raise ValueError("TRL writer rank is inconsistent")
187
+ if writer != (self.producer is not None):
188
+ raise ValueError("only the world-process-zero callback may own a producer")
189
+ return writer
190
+
191
+ def on_train_begin(self, args, state, control, **kwargs):
192
+ writer = self._check(args, state)
193
+ stride = self.configuration["logging_steps"]
194
+ if state.logging_steps != stride:
195
+ raise ValueError(
196
+ "TRL runtime logging cadence differs from the declared cadence"
197
+ )
198
+ if writer:
199
+ if (state.global_step // stride + 1) * stride != self.producer.next_step:
200
+ raise ValueError(
201
+ "TRL resume position is incompatible with the next expected producer step"
202
+ )
203
+ self.producer.bind(
204
+ "trl",
205
+ {
206
+ "configuration": self.configuration,
207
+ "reward_column": self.reward_column,
208
+ "expected_groups": _plan_records(self.expected_groups),
209
+ "scope": "latest-complete-generation-at-logged-optimizer-step",
210
+ },
211
+ )
212
+ self._begin = self._completed = None
213
+ self._started = True
214
+
215
+ def on_step_begin(self, args, state, control, **kwargs):
216
+ self._check(args, state)
217
+ if (
218
+ not self._started
219
+ or self._begin is not None
220
+ or self.trainer._step % self.configuration["steps_per_generation"]
221
+ ):
222
+ raise ValueError(
223
+ "TRL generation step is not aligned with a witnessed optimizer step"
224
+ )
225
+ self._begin = (state.global_step, self.trainer._step)
226
+ self._completed = None
227
+
228
+ def on_step_end(self, args, state, control, **kwargs):
229
+ self._check(args, state)
230
+ if self._begin is None or state.global_step != self._begin[0] + 1:
231
+ raise ValueError("TRL optimizer step was not witnessed")
232
+ if (
233
+ self.trainer._step - self._begin[1]
234
+ != self.configuration["gradient_accumulation_steps"]
235
+ ):
236
+ raise ValueError(
237
+ "TRL short or unexpected gradient accumulation cannot establish a complete generation"
238
+ )
239
+ self._begin = None
240
+ self._completed = state.global_step
241
+
242
+ def on_log(self, args, state, control, logs=None, **kwargs):
243
+ writer = self._check(args, state)
244
+ if not self.trainer.model.training or any(
245
+ key.startswith("eval_") for key in (logs or {})
246
+ ):
247
+ self._evaluation_seen = True
248
+ raise ValueError("evaluation logs cannot be admitted as training snapshots")
249
+ if "loss" not in (logs or {}):
250
+ return
251
+ if (
252
+ self._completed != state.global_step
253
+ or state.global_step % self.configuration["logging_steps"]
254
+ ):
255
+ raise ValueError(
256
+ "TRL loss log does not match a witnessed scheduled optimizer step"
257
+ )
258
+ if not writer:
259
+ return
260
+ groups = self.expected_groups.get(state.global_step)
261
+ if groups is None:
262
+ raise ValueError(
263
+ "independent expected groups are missing for this TRL logging step"
264
+ )
265
+ logs = self.trainer._logs
266
+ try:
267
+ prompts = copy.deepcopy(list(logs["prompt"]))
268
+ completions = copy.deepcopy(list(logs["completion"]))
269
+ rewards = list(logs["rewards"][self.reward_column])
270
+ except KeyError:
271
+ raise ValueError(
272
+ "TRL logged snapshot lacks the selected completion or reward payload"
273
+ ) from None
274
+ size = self.configuration["generation_batch_size"]
275
+ if any(len(values) != size for values in (prompts, completions, rewards)):
276
+ raise ValueError("TRL logged generation population is incomplete")
277
+ values = [
278
+ {"prompt": prompt, "response": response, "reward": reward}
279
+ for prompt, response, reward in zip(
280
+ prompts, completions, rewards, strict=True
281
+ )
282
+ ]
283
+ rows = _rows(self.producer, state.global_step, values, "trl-logged")
284
+ self.producer.submit(state.global_step, rows, expected_groups=groups)
285
+
286
+ def on_prediction_step(self, args, state, control, **kwargs):
287
+ self._evaluation_seen = True
288
+ raise ValueError(
289
+ "evaluation is unsupported while the TRL snapshot callback is attached"
290
+ )
291
+
292
+ def on_evaluate(self, args, state, control, **kwargs):
293
+ self.on_prediction_step(args, state, control, **kwargs)
294
+
295
+ def on_train_end(self, args, state, control, **kwargs):
296
+ if self._check(args, state):
297
+ self.producer.flush()
298
+
299
+
300
+ def make_trl_callback(trainer, producer, *, expected_groups, reward_column):
301
+ """Bind a pinned GRPO logged-snapshot adapter to an existing trainer.
302
+
303
+ Only world rank zero receives an open producer; other ranks pass None. The
304
+ independent group plan refers to logged optimizer steps, not all generations.
305
+ Completions precede that optimizer boundary and do not prove checkpoint parity.
306
+ Native TRL parquet is written after on_log, so this hook copies in-memory rows
307
+ into its own producer directory. The producer must not share TRL's native output.
308
+ """
309
+ try:
310
+ if any(version(name) != required for name, required in _TRL_VERSIONS.items()):
311
+ raise ValueError(
312
+ "TRL callback requires trl==1.18.0 and transformers==4.56.2"
313
+ )
314
+ from transformers import TrainerCallback
315
+ except (PackageNotFoundError, ImportError) as exc:
316
+ raise ImportError(
317
+ "TRL callback requires separately installed trl==1.18.0 and transformers==4.56.2"
318
+ ) from exc
319
+ callback = type(
320
+ "TRLLoggedSnapshotCallback", (_TRLLoggedSnapshot, TrainerCallback), {}
321
+ )
322
+ return callback(trainer, producer, expected_groups, reward_column)
323
+
324
+
325
+ class VerlCompletionLogger:
326
+ """Publish verl 0.9.1 completed payloads from one designated writer.
327
+
328
+ Pass the native inputs/outputs/gts/scores/reward_extra_infos_dict/dump_path/
329
+ global_steps arguments. Ground truth and unrelated metadata are discarded.
330
+ For training use bind(trainer), which replaces _dump_generations before the
331
+ native unbounded executor. Keep the producer context around trainer.fit() so
332
+ final asynchronous failures propagate through close(). Validation dumps are
333
+ excluded. Calling this adapter from the native executor does not bound that
334
+ upstream queue and is not the supported training integration.
335
+ """
336
+
337
+ def __init__(self, producer, *, expected_groups, rank=0, writer_rank=0):
338
+ if any(type(value) is not int or value < 0 for value in (rank, writer_rank)):
339
+ raise ValueError("verl ranks must be nonnegative integers")
340
+ self.writer = rank == writer_rank
341
+ if self.writer != (producer is not None):
342
+ raise ValueError("only the designated verl writer may own a producer")
343
+ self.producer = producer
344
+ self._bound_trainer = None
345
+ self.expected_groups = _plan(expected_groups)
346
+ if self.writer:
347
+ if producer.configuration["format"] != "verl":
348
+ raise ValueError("verl completion logger requires a verl producer")
349
+ producer.bind(
350
+ "verl",
351
+ {
352
+ "source_version": "0.9.1",
353
+ "writer_rank": writer_rank,
354
+ "expected_groups": _plan_records(self.expected_groups),
355
+ },
356
+ )
357
+
358
+ def _check_trainer(self, trainer):
359
+ if (
360
+ type(trainer).__module__ != "verl.trainer.ppo.ray_trainer"
361
+ or type(trainer).__name__ != "RayPPOTrainer"
362
+ ):
363
+ raise ValueError("verl binding requires the pinned RayPPOTrainer class")
364
+ settings = trainer.config.trainer
365
+ rollout_path = settings.get("rollout_data_dir")
366
+ if (
367
+ not rollout_path
368
+ or Path(os.path.abspath(rollout_path)) != self.producer.source
369
+ or settings.get("validation_data_dir")
370
+ or trainer._dump_futures
371
+ ):
372
+ raise ValueError(
373
+ "verl binding requires matching training output, no validation dump, and an empty native queue"
374
+ )
375
+
376
+ def bind(self, trainer):
377
+ """Bypass verl's native executor; queue rejection reaches the training call."""
378
+ if not self.writer or self._bound_trainer is not None:
379
+ raise ValueError(
380
+ "verl logger binding requires one unbound designated writer"
381
+ )
382
+ try:
383
+ if version("verl") != "0.9.1":
384
+ raise ValueError("verl binding requires verl==0.9.1")
385
+ except PackageNotFoundError:
386
+ raise ImportError(
387
+ "verl binding requires separately installed verl==0.9.1"
388
+ ) from None
389
+ self._check_trainer(trainer)
390
+
391
+ def dump(inputs, outputs, gts, scores, reward_extra_infos_dict, dump_path):
392
+ self._check_trainer(trainer)
393
+ return self(
394
+ inputs,
395
+ outputs,
396
+ gts,
397
+ scores,
398
+ reward_extra_infos_dict,
399
+ dump_path,
400
+ trainer.global_steps,
401
+ )
402
+
403
+ self.producer.bind(
404
+ "verl-trainer", {"source_version": "0.9.1", "boundary": "_dump_generations"}
405
+ )
406
+ self._bound_trainer = trainer
407
+ trainer._dump_generations = dump
408
+ return self
409
+
410
+ def __call__(
411
+ self,
412
+ inputs,
413
+ outputs,
414
+ gts,
415
+ scores,
416
+ reward_extra_infos_dict,
417
+ dump_path,
418
+ global_steps,
419
+ ):
420
+ if not self.writer:
421
+ return None
422
+ if Path(os.path.abspath(dump_path)) != self.producer.source:
423
+ raise ValueError("verl dump path differs from the locked producer output")
424
+ if len(inputs) != len(outputs) or len(inputs) != len(scores):
425
+ raise ValueError("verl completed payload columns have different lengths")
426
+ groups = self.expected_groups.get(global_steps)
427
+ if groups is None:
428
+ raise ValueError(
429
+ "independent expected groups are missing for this verl step"
430
+ )
431
+ values = [
432
+ {"prompt": prompt, "response": response, "reward": score}
433
+ for prompt, response, score in zip(inputs, outputs, scores, strict=True)
434
+ ]
435
+ for name in ("sample_id", "prompt_id", "tool_calls"):
436
+ if name in reward_extra_infos_dict:
437
+ column = reward_extra_infos_dict[name]
438
+ if len(column) != len(values):
439
+ raise ValueError(
440
+ "verl identity or tool-call metadata has the wrong length"
441
+ )
442
+ for row, value in zip(values, column, strict=True):
443
+ row[name] = value
444
+ rows = _rows(self.producer, global_steps, values, "verl-logged")
445
+ return self.producer.submit(global_steps, rows, expected_groups=groups)
446
+
447
+
448
+ class FixedProbeHook:
449
+ """Invoke a caller-owned sampler explicitly, then asynchronously publish its rows.
450
+
451
+ Sampling is synchronous and its model/snapshot synchronization, scoring, RNG,
452
+ distributed execution, and cost belong to the caller. An attempt is persisted
453
+ before calling sample(prompts, repeats, step=..., snapshot_id=...). Returned rows
454
+ require prompt, response and finite reward. Interrupted or failed attempts are
455
+ spent; restart never invokes the sampler again automatically.
456
+ """
457
+
458
+ def __init__(
459
+ self, producer, *, prompts, repeats, sample, training_run, training_source
460
+ ):
461
+ training_source = Path(os.path.abspath(training_source))
462
+ if (
463
+ not isinstance(training_run, str)
464
+ or not training_run.strip()
465
+ or producer.configuration["run"] == training_run
466
+ or producer.source == training_source
467
+ or producer.source in training_source.parents
468
+ or training_source in producer.source.parents
469
+ ):
470
+ raise ValueError(
471
+ "fixed probes require a separate run and disjoint training output"
472
+ )
473
+ if not callable(sample):
474
+ raise ValueError("fixed probe requires an explicit trusted sampler")
475
+ self.producer = producer
476
+ self.prompts = copy.deepcopy(prompts)
477
+ self.repeats = repeats
478
+ self.sample = sample
479
+ self.groups = expected_prompt_groups(self.prompts, repeats=repeats)
480
+ self._prompt_bytes = _json(
481
+ {"prompts": self.prompts, "repeats": repeats}
482
+ ).encode()
483
+ self._fingerprint = _sha(self._prompt_bytes)
484
+ producer.bind(
485
+ "fixed-probe",
486
+ {
487
+ "prompts": self.prompts,
488
+ "repeats": repeats,
489
+ "groups": self.groups,
490
+ "prompt_sha256": self._fingerprint,
491
+ "training_run": training_run,
492
+ "training_source": str(training_source),
493
+ "start_step": producer.configuration["start_step"],
494
+ "step_stride": producer.configuration["step_stride"],
495
+ },
496
+ )
497
+ self.attempts = producer.spool / "probe-attempts"
498
+ producer._roots[self.attempts] = _mkdir(self.attempts, private=True)
499
+
500
+ def __call__(self, step, *, snapshot_id):
501
+ self.producer._ready()
502
+ if type(step) is not int or step < 0:
503
+ raise ValueError("probe step must be a nonnegative integer")
504
+ if not isinstance(snapshot_id, str) or not snapshot_id.strip():
505
+ raise ValueError("fixed probe requires an explicit model snapshot identity")
506
+ if (
507
+ _json({"prompts": self.prompts, "repeats": self.repeats}).encode()
508
+ != self._prompt_bytes
509
+ or expected_prompt_groups(self.prompts, repeats=self.repeats) != self.groups
510
+ ):
511
+ raise ValueError("fixed probe prompt configuration changed")
512
+ config = self.producer.configuration
513
+ if (
514
+ step < config["start_step"]
515
+ or (step - config["start_step"]) % config["step_stride"]
516
+ ):
517
+ return None
518
+ attempt = {
519
+ "schema_version": 1,
520
+ "step": step,
521
+ "snapshot_id": snapshot_id,
522
+ "prompt_sha256": self._fingerprint,
523
+ }
524
+ path = self.attempts / f"{step:012d}.json"
525
+ if path.exists() or path.is_symlink():
526
+ if _decode(_read(path)) != attempt:
527
+ raise ValueError(
528
+ "fixed probe attempt conflicts with its snapshot identity"
529
+ )
530
+ existing = self.producer.receipt(step)
531
+ if existing is None:
532
+ raise ValueError(
533
+ "fixed probe attempt is spent or uncertain; automatic retry is forbidden"
534
+ )
535
+ return existing
536
+ if step != self.producer.next_step:
537
+ raise ValueError(f"fixed probe expected step {self.producer.next_step}")
538
+ _publish(path, _json(attempt).encode())
539
+ values = self.sample(
540
+ copy.deepcopy(self.prompts),
541
+ self.repeats,
542
+ step=step,
543
+ snapshot_id=snapshot_id,
544
+ )
545
+ rows = _rows(self.producer, step, values, "fixed-probe")
546
+ actual_prompts = [
547
+ row.prompt_messages if row.prompt_messages is not None else row.prompt
548
+ for row in rows.itertuples()
549
+ ]
550
+ if expected_prompt_groups(actual_prompts) != self.groups:
551
+ raise ValueError("fixed probe returned different exact prompt content")
552
+ return self.producer.submit(step, rows, expected_groups=self.groups)