dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,534 @@
1
+ """Owned Opacus training for private application tasks.
2
+
3
+ Application factories and model code are trusted. Dense FP32 optimizers use
4
+ built-in presets or a group-bound trusted factory contract. Raw losses never leave
5
+ this adapter, including research mode.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import copy
11
+ import hashlib
12
+ from pathlib import Path
13
+ from typing import Any, cast
14
+
15
+ import numpy as np
16
+ import torch
17
+ from torch import nn
18
+ from torch.utils.data import Dataset
19
+
20
+ from dromeus.errors import PrivacyError
21
+ from dromeus.manifests.canonical import canonical_hash, canonical_json, save_safetensors
22
+ from dromeus.manifests.models import (
23
+ FactoryOptimizerSpec,
24
+ PrivacyPolicy,
25
+ PrivacyReservation,
26
+ PrivateOptimizerSpec,
27
+ TaskTrainingSpec,
28
+ )
29
+ from dromeus.training.base import EvaluationResult
30
+ from dromeus.training.optimizer_factory import (
31
+ SanitizedOptimizer,
32
+ build_private_optimizer,
33
+ validate_builtin_optimizer_checkpoint,
34
+ )
35
+ from dromeus.training.privacy import (
36
+ ReservationStore,
37
+ opacus_module,
38
+ privacy_epsilon,
39
+ reserve_horizon,
40
+ )
41
+ from dromeus.training.private_state import decode_state, encode_state
42
+ from dromeus.training.private_workload import prepare_private_workload
43
+ from dromeus.training.task import TrainingTask
44
+ from dromeus.training.trainer import PyTorchTrainer
45
+ from dromeus.training.versions import torch_release
46
+
47
+
48
+ def private_training_definition(specification: TaskTrainingSpec) -> str:
49
+ return canonical_json(specification).decode()
50
+
51
+
52
+ class PrivateTrainer(PyTorchTrainer):
53
+ """Retain private model/optimizer/loader and count complete logical mechanisms."""
54
+
55
+ def __init__(
56
+ self,
57
+ *,
58
+ task: TrainingTask,
59
+ dataset: Dataset[Any],
60
+ policy: PrivacyPolicy,
61
+ specification: TaskTrainingSpec,
62
+ run_id: str,
63
+ local_data_identity: str,
64
+ reservation_store: ReservationStore,
65
+ max_physical_batch_size: int,
66
+ research_seed: int | None = None,
67
+ ) -> None:
68
+ if specification.optimizer.torch_version != torch_release():
69
+ raise PrivacyError("private optimizer Torch version mismatch")
70
+ if max_physical_batch_size <= 0 or not local_data_identity.strip():
71
+ raise PrivacyError(
72
+ "physical batch limit and local data identity are required"
73
+ )
74
+ if (
75
+ specification.gradient_backend == "opacus_native_b1"
76
+ and max_physical_batch_size != 1
77
+ ):
78
+ raise PrivacyError("native B1 backend requires physical batch limit 1")
79
+ secure = policy.randomness_profile == "secure"
80
+ if secure and research_seed is not None:
81
+ raise PrivacyError("secure mode cannot accept a research seed")
82
+ if not secure and research_seed is None:
83
+ raise PrivacyError("public research mode requires an explicit local seed")
84
+ # Freeze identity before application model construction can run callbacks.
85
+ self._specification_hash = canonical_hash(specification)
86
+ self._workload = prepare_private_workload(
87
+ specification=specification, task=task, dataset=dataset
88
+ )
89
+ optimizer_registration = self._workload.optimizer_registration
90
+ self.policy = policy
91
+ self.specification = specification
92
+ self._physical_limit = max_physical_batch_size
93
+ self._failed = False
94
+ self._physical_steps = 0
95
+ self._logical_steps = 0
96
+ self._reservation_store = reservation_store
97
+ self._raw_model = self._workload.model
98
+ spec = specification.optimizer
99
+ self._optimizer_registration = optimizer_registration
100
+ if isinstance(spec, FactoryOptimizerSpec):
101
+ if optimizer_registration is None:
102
+ raise PrivacyError("factory optimizer requires a local registration")
103
+ self._base_optimizer = optimizer_registration.build(self._raw_model, spec)
104
+ else:
105
+ if optimizer_registration is not None:
106
+ raise PrivacyError(
107
+ "optimizer registration requires a factory specification"
108
+ )
109
+ self._base_optimizer = build_private_optimizer(self._raw_model, spec)
110
+ self._validate_optimizer()
111
+ self._initial_optimizer_groups: list[dict[str, Any]] = (
112
+ copy.deepcopy(self._base_optimizer.state_dict()["param_groups"])
113
+ if isinstance(spec, PrivateOptimizerSpec)
114
+ else []
115
+ )
116
+ self._initial_learning_rates = tuple(
117
+ float(g["lr"]) for g in self._base_optimizer.param_groups
118
+ )
119
+ self._sampling_generator = (
120
+ None if secure else torch.Generator().manual_seed(int(research_seed or 0))
121
+ )
122
+ device = next(self._raw_model.parameters()).device
123
+ self._noise_generator = (
124
+ None
125
+ if secure
126
+ else torch.Generator(device=device).manual_seed(int(research_seed or 0) + 1)
127
+ )
128
+ loader = self._workload.make_loader(self._sampling_generator)
129
+ self._population_size = len(cast(Any, loader.dataset))
130
+ try:
131
+ self._engine = opacus_module().PrivacyEngine(
132
+ accountant="prv", secure_mode=secure
133
+ )
134
+ self._private_model, self._optimizer, self._loader = (
135
+ self._engine.make_private(
136
+ module=self._raw_model,
137
+ optimizer=SanitizedOptimizer(self._base_optimizer)
138
+ if optimizer_registration
139
+ else self._base_optimizer,
140
+ data_loader=loader,
141
+ noise_multiplier=policy.noise_multiplier,
142
+ max_grad_norm=policy.max_grad_norm,
143
+ poisson_sampling=self._workload.poisson_sampling,
144
+ clipping="flat",
145
+ grad_sample_mode="hooks",
146
+ noise_generator=self._noise_generator,
147
+ )
148
+ )
149
+ self._workload.configure_private_model(self._private_model)
150
+ except Exception:
151
+ raise PrivacyError(
152
+ "private preparation failed; verify model and secure RNG installation"
153
+ ) from None
154
+ self._accountant_hook = self._optimizer.step_hook
155
+ self._noise_stream = self._optimizer.generator
156
+ self._sampling_stream = self._loader.batch_sampler.generator
157
+ self._sample_rate = 1 / len(self._loader)
158
+ self._reservation = PrivacyReservation(
159
+ run_id=run_id,
160
+ policy_hash=canonical_hash(policy),
161
+ sample_rate=self._sample_rate,
162
+ noise_multiplier=policy.noise_multiplier,
163
+ logical_steps=policy.max_logical_steps,
164
+ )
165
+ reserve_horizon(reservation_store, self._reservation, policy)
166
+ identity = hashlib.sha256(
167
+ canonical_json(policy)
168
+ + canonical_json(specification)
169
+ + local_data_identity.encode()
170
+ + str(
171
+ (self._population_size, self._sample_rate, max_physical_batch_size)
172
+ ).encode()
173
+ ).hexdigest()
174
+ super().__init__(
175
+ model=self._raw_model,
176
+ train_step=self._owned_step,
177
+ save_state=self._save_private_state,
178
+ load_state=self._load_private_state,
179
+ state_identity=identity,
180
+ evaluate=self._workload.evaluate,
181
+ exchange_names=self._workload.exchange_names,
182
+ )
183
+
184
+ def _validate_optimizer(self) -> None:
185
+ spec = self.specification.optimizer
186
+ actual = self._base_optimizer
187
+ if isinstance(spec, FactoryOptimizerSpec):
188
+ assert self._optimizer_registration is not None
189
+ self._optimizer_registration.check(
190
+ actual,
191
+ self._raw_model,
192
+ spec,
193
+ logical_steps=0,
194
+ horizon=self.policy.max_logical_steps,
195
+ schedule=self.specification.schedule,
196
+ )
197
+ self._parameter_ids = tuple(id(p) for p in self._raw_model.parameters())
198
+ return
199
+ expected = build_private_optimizer(self._raw_model, spec)
200
+ if type(actual) is not type(expected) or len(actual.param_groups) != 1:
201
+ raise PrivacyError("unsupported private optimizer class or groups")
202
+ group = actual.param_groups[0]
203
+ expected_group = expected.param_groups[0]
204
+ if set(group) != set(expected_group) or any(
205
+ group[key] != expected_group[key]
206
+ for key in expected_group
207
+ if key != "params"
208
+ ):
209
+ raise PrivacyError(
210
+ "private optimizer options differ from shared specification"
211
+ )
212
+ if [id(p) for p in group["params"]] != [
213
+ id(p) for p in self._raw_model.parameters() if p.requires_grad
214
+ ]:
215
+ raise PrivacyError(
216
+ "private optimizer must cover the complete ordered parameter group"
217
+ )
218
+ self._parameter_ids = tuple(id(p) for p in self._raw_model.parameters())
219
+ self._optimizer_options = {
220
+ k: v for k, v in group.items() if k not in {"params", "lr"}
221
+ }
222
+
223
+ def _check_execution(self) -> None:
224
+ self._reservation_store.check(self._reservation)
225
+ if self._failed:
226
+ raise PrivacyError("private trainer failed; start a new accounted run")
227
+ if canonical_hash(self.specification) != self._specification_hash:
228
+ raise PrivacyError("private optimizer specification changed")
229
+ group = self._base_optimizer.param_groups
230
+ spec = self.specification.optimizer
231
+ if isinstance(spec, FactoryOptimizerSpec):
232
+ assert self._optimizer_registration is not None
233
+ self._optimizer_registration.check(
234
+ self._base_optimizer,
235
+ self._raw_model,
236
+ spec,
237
+ logical_steps=self._logical_steps,
238
+ horizon=self.policy.max_logical_steps,
239
+ schedule=self.specification.schedule,
240
+ )
241
+ elif len(group) != 1 or tuple(id(p) for p in group[0]["params"]) != tuple(
242
+ id(p) for p in self._raw_model.parameters() if p.requires_grad
243
+ ):
244
+ raise PrivacyError("private optimizer parameter groups changed")
245
+ if (
246
+ isinstance(spec, PrivateOptimizerSpec)
247
+ and {k: v for k, v in group[0].items() if k not in {"params", "lr"}}
248
+ != self._optimizer_options
249
+ ):
250
+ raise PrivacyError("private optimizer options changed")
251
+ for current, initial in zip(group, self._initial_learning_rates, strict=True):
252
+ expected_lr = initial
253
+ if self.specification.schedule == "linear_decay":
254
+ expected_lr *= 1 - self._logical_steps / self.policy.max_logical_steps
255
+ if current["lr"] != expected_lr:
256
+ raise PrivacyError("private optimizer learning rate changed")
257
+ if (
258
+ self._optimizer.noise_multiplier != self.policy.noise_multiplier
259
+ or self._optimizer.max_grad_norm != self.policy.max_grad_norm
260
+ ):
261
+ raise PrivacyError("private mechanism changed")
262
+ if (
263
+ 1 / len(self._loader) != self._sample_rate
264
+ or len(self._loader.dataset) != self._population_size
265
+ or self._optimizer.expected_batch_size
266
+ != int(len(self._loader.dataset) * self._sample_rate)
267
+ ):
268
+ raise PrivacyError("private sampling changed")
269
+ if (
270
+ self._optimizer.step_hook is not self._accountant_hook
271
+ or self._optimizer.generator is not self._noise_stream
272
+ or self._loader.batch_sampler.generator is not self._sampling_stream
273
+ or self._loader.batch_sampler.sample_rate != self._sample_rate
274
+ or self._loader.batch_sampler.num_samples != len(self._loader.dataset)
275
+ or self._optimizer.secure_mode
276
+ != (self.policy.randomness_profile == "secure")
277
+ or self._optimizer.loss_reduction != "mean"
278
+ ):
279
+ raise PrivacyError(
280
+ "private randomness, sampling or accountant hook changed"
281
+ )
282
+ if tuple(id(p) for p in self._raw_model.parameters()) != self._parameter_ids:
283
+ raise PrivacyError("private model parameters changed")
284
+ self._workload.check()
285
+ if any(
286
+ s != self.policy.noise_multiplier or q != self._sample_rate
287
+ for s, q, _ in self.accounting_history
288
+ ):
289
+ raise PrivacyError("accountant mechanism changed")
290
+ if self._steps != self._logical_steps:
291
+ raise PrivacyError("trainer and logical step count disagree")
292
+ if sum(n for _, _, n in self.accounting_history) != self._logical_steps:
293
+ raise PrivacyError("accountant and logical step count disagree")
294
+
295
+ @property
296
+ def accounting_history(self) -> tuple[tuple[float, float, int], ...]:
297
+ return tuple(
298
+ (float(s), float(q), int(n)) for s, q, n in self._engine.accountant.history
299
+ )
300
+
301
+ @property
302
+ def physical_steps(self) -> int:
303
+ return self._physical_steps
304
+
305
+ @property
306
+ def sample_rate(self) -> float:
307
+ return self._sample_rate
308
+
309
+ def accounting(self) -> dict[str, float | int | str]:
310
+ return {
311
+ "privacy_epsilon": privacy_epsilon(self.accounting_history, self.policy),
312
+ "rdp_epsilon": privacy_epsilon(
313
+ self.accounting_history, self.policy, accountant="rdp"
314
+ ),
315
+ "delta": self.policy.delta,
316
+ "logical_steps": self._logical_steps,
317
+ "reserved_steps": self.policy.max_logical_steps,
318
+ "randomness_profile": self.policy.randomness_profile,
319
+ }
320
+
321
+ def train_local_steps(self, step_count: int) -> None:
322
+ self._check_execution()
323
+ if (
324
+ step_count <= 0
325
+ or self._logical_steps + step_count > self.policy.max_logical_steps
326
+ ):
327
+ raise PrivacyError("private horizon exhausted or invalid step count")
328
+ try:
329
+ super().train_local_steps(step_count)
330
+ except Exception:
331
+ self._failed = True
332
+ raise PrivacyError("private local training failed") from None
333
+
334
+ def _owned_step(self, _: nn.Module) -> None:
335
+ self._check_execution()
336
+ if self._logical_steps >= self.policy.max_logical_steps:
337
+ raise PrivacyError("private horizon exhausted")
338
+ self._private_model.train()
339
+ manager = opacus_module("opacus.utils.batch_memory_manager").BatchMemoryManager
340
+ # Each logical call consumes one independent Poisson draw. No partially
341
+ # consumed loader iterator survives a boundary or research checkpoint.
342
+ before = self._logical_steps
343
+ step = self._workload.begin_step()
344
+ with manager(
345
+ data_loader=self._loader,
346
+ max_physical_batch_size=self._physical_limit,
347
+ optimizer=self._optimizer,
348
+ ) as physical_loader:
349
+ for batch in physical_loader:
350
+ # Clear p.grad too: an empty Poisson draw runs no backward, and a
351
+ # zero-filled grad left behind would look like an unsampled one.
352
+ self._optimizer.zero_grad(set_to_none=True)
353
+ step.prepare_gradients(self._private_model, batch)
354
+ self._check_execution()
355
+ self._optimizer.step()
356
+ step.after_optimizer_step()
357
+ self._physical_steps += 1
358
+ completed = sum(n for _, _, n in self.accounting_history)
359
+ if completed == self._logical_steps + 1:
360
+ self._logical_steps = completed
361
+ if self.specification.schedule == "linear_decay":
362
+ for group, initial in zip(
363
+ self._base_optimizer.param_groups,
364
+ self._initial_learning_rates,
365
+ strict=True,
366
+ ):
367
+ group["lr"] = initial * (
368
+ 1 - completed / self.policy.max_logical_steps
369
+ )
370
+ self._optimizer.zero_grad(set_to_none=True)
371
+ break
372
+ if completed != self._logical_steps:
373
+ raise PrivacyError("invalid logical mechanism boundary")
374
+ if self._logical_steps != before + 1:
375
+ raise PrivacyError("private logical step did not complete")
376
+ return None
377
+
378
+ def weights(self) -> dict[str, np.ndarray]:
379
+ self._check_execution()
380
+ return super().weights()
381
+
382
+ def evaluate(self) -> EvaluationResult | None:
383
+ self._check_execution()
384
+ try:
385
+ return super().evaluate()
386
+ except Exception:
387
+ raise PrivacyError("public evaluation failed") from None
388
+
389
+ def export_model(self, path: Path) -> None:
390
+ """Release only sanitized model tensors; no training state or metadata."""
391
+ self._check_execution()
392
+ save_safetensors(self._workload.export_tensors(super().weights), str(path))
393
+
394
+ def _save_private_state(self) -> dict[str, np.ndarray]:
395
+ state: dict[str, Any] = {
396
+ "version": 1,
397
+ "specification": canonical_hash(self.specification),
398
+ "policy": canonical_hash(self.policy),
399
+ "sample_rate": self._sample_rate,
400
+ "optimizer": self._base_optimizer.state_dict(),
401
+ "accountant": list(self.accounting_history),
402
+ "logical_steps": self._logical_steps,
403
+ "physical_steps": self._physical_steps,
404
+ "sampling_rng": self._sampling_generator.get_state()
405
+ if self._sampling_generator is not None
406
+ else None,
407
+ "noise_rng": self._noise_generator.get_state()
408
+ if self._noise_generator is not None
409
+ else None,
410
+ }
411
+ return encode_state(state)
412
+
413
+ def _load_private_state(self, arrays: dict[str, np.ndarray]) -> None:
414
+ state = decode_state(arrays)
415
+ self._validate_restore(state)
416
+ # Restore through Opacus so a factory delegate rebinds replacement groups.
417
+ self._optimizer.load_state_dict(state["optimizer"])
418
+ self._engine.accountant.history = list(state["accountant"])
419
+ self._logical_steps = state["logical_steps"]
420
+ self._physical_steps = state["physical_steps"]
421
+ assert (
422
+ self._sampling_generator is not None and self._noise_generator is not None
423
+ )
424
+ self._sampling_generator.set_state(state["sampling_rng"])
425
+ self._noise_generator.set_state(state["noise_rng"])
426
+
427
+ def _validate_restore(self, state: Any) -> None:
428
+ expected = {
429
+ "version",
430
+ "specification",
431
+ "policy",
432
+ "sample_rate",
433
+ "optimizer",
434
+ "accountant",
435
+ "logical_steps",
436
+ "physical_steps",
437
+ "sampling_rng",
438
+ "noise_rng",
439
+ }
440
+ if (
441
+ not isinstance(state, dict)
442
+ or set(cast(dict[str, Any], state)) != expected
443
+ or state["version"] != 1
444
+ ):
445
+ raise PrivacyError("research checkpoint schema mismatch")
446
+ if (
447
+ state["specification"] != canonical_hash(self.specification)
448
+ or state["policy"] != canonical_hash(self.policy)
449
+ or state["sample_rate"] != self._sample_rate
450
+ ):
451
+ raise PrivacyError(
452
+ "research checkpoint mechanism or optimizer identity mismatch"
453
+ )
454
+ state = cast(dict[str, Any], state)
455
+ steps: Any = state["logical_steps"]
456
+ if (
457
+ type(steps) is not int
458
+ or not 0 <= steps <= self.policy.max_logical_steps
459
+ or type(state["physical_steps"]) is not int
460
+ or state["physical_steps"] < steps
461
+ ):
462
+ raise PrivacyError("invalid research checkpoint counters")
463
+ history = state["accountant"]
464
+ if (
465
+ any(
466
+ s != self.policy.noise_multiplier
467
+ or q != self._sample_rate
468
+ or type(n) is not int
469
+ or n <= 0
470
+ for s, q, n in history
471
+ )
472
+ or sum(n for _, _, n in history) != steps
473
+ ):
474
+ raise PrivacyError("research checkpoint accountant history mismatch")
475
+ spec = self.specification.optimizer
476
+ if isinstance(spec, FactoryOptimizerSpec):
477
+ assert self._optimizer_registration is not None
478
+ self._optimizer_registration.validate_checkpoint(
479
+ self._base_optimizer,
480
+ self._raw_model,
481
+ spec,
482
+ state["optimizer"],
483
+ logical_steps=steps,
484
+ horizon=self.policy.max_logical_steps,
485
+ schedule=self.specification.schedule,
486
+ )
487
+ else:
488
+ validate_builtin_optimizer_checkpoint(
489
+ state["optimizer"],
490
+ spec=spec,
491
+ parameters=tuple(
492
+ p for p in self._raw_model.parameters() if p.requires_grad
493
+ ),
494
+ initial_groups=self._initial_optimizer_groups,
495
+ logical_steps=steps,
496
+ horizon=self.policy.max_logical_steps,
497
+ schedule=self.specification.schedule,
498
+ )
499
+ for key, generator in (
500
+ ("sampling_rng", self._sampling_generator),
501
+ ("noise_rng", self._noise_generator),
502
+ ):
503
+ if (
504
+ generator is None
505
+ or not isinstance(state[key], torch.Tensor)
506
+ or state[key].dtype != torch.uint8
507
+ or state[key].shape != generator.get_state().shape
508
+ ):
509
+ raise PrivacyError("research checkpoint RNG mismatch")
510
+ probe = torch.Generator(device=generator.device)
511
+ probe.set_state(state[key])
512
+
513
+ def load_checkpoint_tensors(self, state: dict[str, np.ndarray]) -> None:
514
+ self._check_execution()
515
+ if self.policy.randomness_profile == "secure":
516
+ raise PrivacyError(
517
+ "secure private runs are fresh-start only; resume refused"
518
+ )
519
+ payload = {
520
+ k.removeprefix("application."): v
521
+ for k, v in state.items()
522
+ if k.startswith("application.")
523
+ }
524
+ decoded = decode_state(payload)
525
+ self._validate_restore(decoded)
526
+ if (
527
+ "meta.steps" not in state
528
+ or int(state["meta.steps"][0]) != decoded["logical_steps"]
529
+ ):
530
+ raise PrivacyError("checkpoint counters disagree")
531
+ if "meta.loss" not in state or state["meta.loss"].size:
532
+ raise PrivacyError("private checkpoint contains raw loss")
533
+ self._workload.validate_checkpoint(state)
534
+ super().load_checkpoint_tensors(state)