dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
"""Owned Opacus training for private application tasks.
|
|
2
|
+
|
|
3
|
+
Application factories and model code are trusted. Dense FP32 optimizers use
|
|
4
|
+
built-in presets or a group-bound trusted factory contract. Raw losses never leave
|
|
5
|
+
this adapter, including research mode.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import copy
|
|
11
|
+
import hashlib
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, cast
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import torch
|
|
17
|
+
from torch import nn
|
|
18
|
+
from torch.utils.data import Dataset
|
|
19
|
+
|
|
20
|
+
from dromeus.errors import PrivacyError
|
|
21
|
+
from dromeus.manifests.canonical import canonical_hash, canonical_json, save_safetensors
|
|
22
|
+
from dromeus.manifests.models import (
|
|
23
|
+
FactoryOptimizerSpec,
|
|
24
|
+
PrivacyPolicy,
|
|
25
|
+
PrivacyReservation,
|
|
26
|
+
PrivateOptimizerSpec,
|
|
27
|
+
TaskTrainingSpec,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.training.base import EvaluationResult
|
|
30
|
+
from dromeus.training.optimizer_factory import (
|
|
31
|
+
SanitizedOptimizer,
|
|
32
|
+
build_private_optimizer,
|
|
33
|
+
validate_builtin_optimizer_checkpoint,
|
|
34
|
+
)
|
|
35
|
+
from dromeus.training.privacy import (
|
|
36
|
+
ReservationStore,
|
|
37
|
+
opacus_module,
|
|
38
|
+
privacy_epsilon,
|
|
39
|
+
reserve_horizon,
|
|
40
|
+
)
|
|
41
|
+
from dromeus.training.private_state import decode_state, encode_state
|
|
42
|
+
from dromeus.training.private_workload import prepare_private_workload
|
|
43
|
+
from dromeus.training.task import TrainingTask
|
|
44
|
+
from dromeus.training.trainer import PyTorchTrainer
|
|
45
|
+
from dromeus.training.versions import torch_release
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def private_training_definition(specification: TaskTrainingSpec) -> str:
|
|
49
|
+
return canonical_json(specification).decode()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class PrivateTrainer(PyTorchTrainer):
|
|
53
|
+
"""Retain private model/optimizer/loader and count complete logical mechanisms."""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
*,
|
|
58
|
+
task: TrainingTask,
|
|
59
|
+
dataset: Dataset[Any],
|
|
60
|
+
policy: PrivacyPolicy,
|
|
61
|
+
specification: TaskTrainingSpec,
|
|
62
|
+
run_id: str,
|
|
63
|
+
local_data_identity: str,
|
|
64
|
+
reservation_store: ReservationStore,
|
|
65
|
+
max_physical_batch_size: int,
|
|
66
|
+
research_seed: int | None = None,
|
|
67
|
+
) -> None:
|
|
68
|
+
if specification.optimizer.torch_version != torch_release():
|
|
69
|
+
raise PrivacyError("private optimizer Torch version mismatch")
|
|
70
|
+
if max_physical_batch_size <= 0 or not local_data_identity.strip():
|
|
71
|
+
raise PrivacyError(
|
|
72
|
+
"physical batch limit and local data identity are required"
|
|
73
|
+
)
|
|
74
|
+
if (
|
|
75
|
+
specification.gradient_backend == "opacus_native_b1"
|
|
76
|
+
and max_physical_batch_size != 1
|
|
77
|
+
):
|
|
78
|
+
raise PrivacyError("native B1 backend requires physical batch limit 1")
|
|
79
|
+
secure = policy.randomness_profile == "secure"
|
|
80
|
+
if secure and research_seed is not None:
|
|
81
|
+
raise PrivacyError("secure mode cannot accept a research seed")
|
|
82
|
+
if not secure and research_seed is None:
|
|
83
|
+
raise PrivacyError("public research mode requires an explicit local seed")
|
|
84
|
+
# Freeze identity before application model construction can run callbacks.
|
|
85
|
+
self._specification_hash = canonical_hash(specification)
|
|
86
|
+
self._workload = prepare_private_workload(
|
|
87
|
+
specification=specification, task=task, dataset=dataset
|
|
88
|
+
)
|
|
89
|
+
optimizer_registration = self._workload.optimizer_registration
|
|
90
|
+
self.policy = policy
|
|
91
|
+
self.specification = specification
|
|
92
|
+
self._physical_limit = max_physical_batch_size
|
|
93
|
+
self._failed = False
|
|
94
|
+
self._physical_steps = 0
|
|
95
|
+
self._logical_steps = 0
|
|
96
|
+
self._reservation_store = reservation_store
|
|
97
|
+
self._raw_model = self._workload.model
|
|
98
|
+
spec = specification.optimizer
|
|
99
|
+
self._optimizer_registration = optimizer_registration
|
|
100
|
+
if isinstance(spec, FactoryOptimizerSpec):
|
|
101
|
+
if optimizer_registration is None:
|
|
102
|
+
raise PrivacyError("factory optimizer requires a local registration")
|
|
103
|
+
self._base_optimizer = optimizer_registration.build(self._raw_model, spec)
|
|
104
|
+
else:
|
|
105
|
+
if optimizer_registration is not None:
|
|
106
|
+
raise PrivacyError(
|
|
107
|
+
"optimizer registration requires a factory specification"
|
|
108
|
+
)
|
|
109
|
+
self._base_optimizer = build_private_optimizer(self._raw_model, spec)
|
|
110
|
+
self._validate_optimizer()
|
|
111
|
+
self._initial_optimizer_groups: list[dict[str, Any]] = (
|
|
112
|
+
copy.deepcopy(self._base_optimizer.state_dict()["param_groups"])
|
|
113
|
+
if isinstance(spec, PrivateOptimizerSpec)
|
|
114
|
+
else []
|
|
115
|
+
)
|
|
116
|
+
self._initial_learning_rates = tuple(
|
|
117
|
+
float(g["lr"]) for g in self._base_optimizer.param_groups
|
|
118
|
+
)
|
|
119
|
+
self._sampling_generator = (
|
|
120
|
+
None if secure else torch.Generator().manual_seed(int(research_seed or 0))
|
|
121
|
+
)
|
|
122
|
+
device = next(self._raw_model.parameters()).device
|
|
123
|
+
self._noise_generator = (
|
|
124
|
+
None
|
|
125
|
+
if secure
|
|
126
|
+
else torch.Generator(device=device).manual_seed(int(research_seed or 0) + 1)
|
|
127
|
+
)
|
|
128
|
+
loader = self._workload.make_loader(self._sampling_generator)
|
|
129
|
+
self._population_size = len(cast(Any, loader.dataset))
|
|
130
|
+
try:
|
|
131
|
+
self._engine = opacus_module().PrivacyEngine(
|
|
132
|
+
accountant="prv", secure_mode=secure
|
|
133
|
+
)
|
|
134
|
+
self._private_model, self._optimizer, self._loader = (
|
|
135
|
+
self._engine.make_private(
|
|
136
|
+
module=self._raw_model,
|
|
137
|
+
optimizer=SanitizedOptimizer(self._base_optimizer)
|
|
138
|
+
if optimizer_registration
|
|
139
|
+
else self._base_optimizer,
|
|
140
|
+
data_loader=loader,
|
|
141
|
+
noise_multiplier=policy.noise_multiplier,
|
|
142
|
+
max_grad_norm=policy.max_grad_norm,
|
|
143
|
+
poisson_sampling=self._workload.poisson_sampling,
|
|
144
|
+
clipping="flat",
|
|
145
|
+
grad_sample_mode="hooks",
|
|
146
|
+
noise_generator=self._noise_generator,
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
self._workload.configure_private_model(self._private_model)
|
|
150
|
+
except Exception:
|
|
151
|
+
raise PrivacyError(
|
|
152
|
+
"private preparation failed; verify model and secure RNG installation"
|
|
153
|
+
) from None
|
|
154
|
+
self._accountant_hook = self._optimizer.step_hook
|
|
155
|
+
self._noise_stream = self._optimizer.generator
|
|
156
|
+
self._sampling_stream = self._loader.batch_sampler.generator
|
|
157
|
+
self._sample_rate = 1 / len(self._loader)
|
|
158
|
+
self._reservation = PrivacyReservation(
|
|
159
|
+
run_id=run_id,
|
|
160
|
+
policy_hash=canonical_hash(policy),
|
|
161
|
+
sample_rate=self._sample_rate,
|
|
162
|
+
noise_multiplier=policy.noise_multiplier,
|
|
163
|
+
logical_steps=policy.max_logical_steps,
|
|
164
|
+
)
|
|
165
|
+
reserve_horizon(reservation_store, self._reservation, policy)
|
|
166
|
+
identity = hashlib.sha256(
|
|
167
|
+
canonical_json(policy)
|
|
168
|
+
+ canonical_json(specification)
|
|
169
|
+
+ local_data_identity.encode()
|
|
170
|
+
+ str(
|
|
171
|
+
(self._population_size, self._sample_rate, max_physical_batch_size)
|
|
172
|
+
).encode()
|
|
173
|
+
).hexdigest()
|
|
174
|
+
super().__init__(
|
|
175
|
+
model=self._raw_model,
|
|
176
|
+
train_step=self._owned_step,
|
|
177
|
+
save_state=self._save_private_state,
|
|
178
|
+
load_state=self._load_private_state,
|
|
179
|
+
state_identity=identity,
|
|
180
|
+
evaluate=self._workload.evaluate,
|
|
181
|
+
exchange_names=self._workload.exchange_names,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
def _validate_optimizer(self) -> None:
|
|
185
|
+
spec = self.specification.optimizer
|
|
186
|
+
actual = self._base_optimizer
|
|
187
|
+
if isinstance(spec, FactoryOptimizerSpec):
|
|
188
|
+
assert self._optimizer_registration is not None
|
|
189
|
+
self._optimizer_registration.check(
|
|
190
|
+
actual,
|
|
191
|
+
self._raw_model,
|
|
192
|
+
spec,
|
|
193
|
+
logical_steps=0,
|
|
194
|
+
horizon=self.policy.max_logical_steps,
|
|
195
|
+
schedule=self.specification.schedule,
|
|
196
|
+
)
|
|
197
|
+
self._parameter_ids = tuple(id(p) for p in self._raw_model.parameters())
|
|
198
|
+
return
|
|
199
|
+
expected = build_private_optimizer(self._raw_model, spec)
|
|
200
|
+
if type(actual) is not type(expected) or len(actual.param_groups) != 1:
|
|
201
|
+
raise PrivacyError("unsupported private optimizer class or groups")
|
|
202
|
+
group = actual.param_groups[0]
|
|
203
|
+
expected_group = expected.param_groups[0]
|
|
204
|
+
if set(group) != set(expected_group) or any(
|
|
205
|
+
group[key] != expected_group[key]
|
|
206
|
+
for key in expected_group
|
|
207
|
+
if key != "params"
|
|
208
|
+
):
|
|
209
|
+
raise PrivacyError(
|
|
210
|
+
"private optimizer options differ from shared specification"
|
|
211
|
+
)
|
|
212
|
+
if [id(p) for p in group["params"]] != [
|
|
213
|
+
id(p) for p in self._raw_model.parameters() if p.requires_grad
|
|
214
|
+
]:
|
|
215
|
+
raise PrivacyError(
|
|
216
|
+
"private optimizer must cover the complete ordered parameter group"
|
|
217
|
+
)
|
|
218
|
+
self._parameter_ids = tuple(id(p) for p in self._raw_model.parameters())
|
|
219
|
+
self._optimizer_options = {
|
|
220
|
+
k: v for k, v in group.items() if k not in {"params", "lr"}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
def _check_execution(self) -> None:
|
|
224
|
+
self._reservation_store.check(self._reservation)
|
|
225
|
+
if self._failed:
|
|
226
|
+
raise PrivacyError("private trainer failed; start a new accounted run")
|
|
227
|
+
if canonical_hash(self.specification) != self._specification_hash:
|
|
228
|
+
raise PrivacyError("private optimizer specification changed")
|
|
229
|
+
group = self._base_optimizer.param_groups
|
|
230
|
+
spec = self.specification.optimizer
|
|
231
|
+
if isinstance(spec, FactoryOptimizerSpec):
|
|
232
|
+
assert self._optimizer_registration is not None
|
|
233
|
+
self._optimizer_registration.check(
|
|
234
|
+
self._base_optimizer,
|
|
235
|
+
self._raw_model,
|
|
236
|
+
spec,
|
|
237
|
+
logical_steps=self._logical_steps,
|
|
238
|
+
horizon=self.policy.max_logical_steps,
|
|
239
|
+
schedule=self.specification.schedule,
|
|
240
|
+
)
|
|
241
|
+
elif len(group) != 1 or tuple(id(p) for p in group[0]["params"]) != tuple(
|
|
242
|
+
id(p) for p in self._raw_model.parameters() if p.requires_grad
|
|
243
|
+
):
|
|
244
|
+
raise PrivacyError("private optimizer parameter groups changed")
|
|
245
|
+
if (
|
|
246
|
+
isinstance(spec, PrivateOptimizerSpec)
|
|
247
|
+
and {k: v for k, v in group[0].items() if k not in {"params", "lr"}}
|
|
248
|
+
!= self._optimizer_options
|
|
249
|
+
):
|
|
250
|
+
raise PrivacyError("private optimizer options changed")
|
|
251
|
+
for current, initial in zip(group, self._initial_learning_rates, strict=True):
|
|
252
|
+
expected_lr = initial
|
|
253
|
+
if self.specification.schedule == "linear_decay":
|
|
254
|
+
expected_lr *= 1 - self._logical_steps / self.policy.max_logical_steps
|
|
255
|
+
if current["lr"] != expected_lr:
|
|
256
|
+
raise PrivacyError("private optimizer learning rate changed")
|
|
257
|
+
if (
|
|
258
|
+
self._optimizer.noise_multiplier != self.policy.noise_multiplier
|
|
259
|
+
or self._optimizer.max_grad_norm != self.policy.max_grad_norm
|
|
260
|
+
):
|
|
261
|
+
raise PrivacyError("private mechanism changed")
|
|
262
|
+
if (
|
|
263
|
+
1 / len(self._loader) != self._sample_rate
|
|
264
|
+
or len(self._loader.dataset) != self._population_size
|
|
265
|
+
or self._optimizer.expected_batch_size
|
|
266
|
+
!= int(len(self._loader.dataset) * self._sample_rate)
|
|
267
|
+
):
|
|
268
|
+
raise PrivacyError("private sampling changed")
|
|
269
|
+
if (
|
|
270
|
+
self._optimizer.step_hook is not self._accountant_hook
|
|
271
|
+
or self._optimizer.generator is not self._noise_stream
|
|
272
|
+
or self._loader.batch_sampler.generator is not self._sampling_stream
|
|
273
|
+
or self._loader.batch_sampler.sample_rate != self._sample_rate
|
|
274
|
+
or self._loader.batch_sampler.num_samples != len(self._loader.dataset)
|
|
275
|
+
or self._optimizer.secure_mode
|
|
276
|
+
!= (self.policy.randomness_profile == "secure")
|
|
277
|
+
or self._optimizer.loss_reduction != "mean"
|
|
278
|
+
):
|
|
279
|
+
raise PrivacyError(
|
|
280
|
+
"private randomness, sampling or accountant hook changed"
|
|
281
|
+
)
|
|
282
|
+
if tuple(id(p) for p in self._raw_model.parameters()) != self._parameter_ids:
|
|
283
|
+
raise PrivacyError("private model parameters changed")
|
|
284
|
+
self._workload.check()
|
|
285
|
+
if any(
|
|
286
|
+
s != self.policy.noise_multiplier or q != self._sample_rate
|
|
287
|
+
for s, q, _ in self.accounting_history
|
|
288
|
+
):
|
|
289
|
+
raise PrivacyError("accountant mechanism changed")
|
|
290
|
+
if self._steps != self._logical_steps:
|
|
291
|
+
raise PrivacyError("trainer and logical step count disagree")
|
|
292
|
+
if sum(n for _, _, n in self.accounting_history) != self._logical_steps:
|
|
293
|
+
raise PrivacyError("accountant and logical step count disagree")
|
|
294
|
+
|
|
295
|
+
@property
|
|
296
|
+
def accounting_history(self) -> tuple[tuple[float, float, int], ...]:
|
|
297
|
+
return tuple(
|
|
298
|
+
(float(s), float(q), int(n)) for s, q, n in self._engine.accountant.history
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
@property
|
|
302
|
+
def physical_steps(self) -> int:
|
|
303
|
+
return self._physical_steps
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def sample_rate(self) -> float:
|
|
307
|
+
return self._sample_rate
|
|
308
|
+
|
|
309
|
+
def accounting(self) -> dict[str, float | int | str]:
|
|
310
|
+
return {
|
|
311
|
+
"privacy_epsilon": privacy_epsilon(self.accounting_history, self.policy),
|
|
312
|
+
"rdp_epsilon": privacy_epsilon(
|
|
313
|
+
self.accounting_history, self.policy, accountant="rdp"
|
|
314
|
+
),
|
|
315
|
+
"delta": self.policy.delta,
|
|
316
|
+
"logical_steps": self._logical_steps,
|
|
317
|
+
"reserved_steps": self.policy.max_logical_steps,
|
|
318
|
+
"randomness_profile": self.policy.randomness_profile,
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
def train_local_steps(self, step_count: int) -> None:
|
|
322
|
+
self._check_execution()
|
|
323
|
+
if (
|
|
324
|
+
step_count <= 0
|
|
325
|
+
or self._logical_steps + step_count > self.policy.max_logical_steps
|
|
326
|
+
):
|
|
327
|
+
raise PrivacyError("private horizon exhausted or invalid step count")
|
|
328
|
+
try:
|
|
329
|
+
super().train_local_steps(step_count)
|
|
330
|
+
except Exception:
|
|
331
|
+
self._failed = True
|
|
332
|
+
raise PrivacyError("private local training failed") from None
|
|
333
|
+
|
|
334
|
+
def _owned_step(self, _: nn.Module) -> None:
|
|
335
|
+
self._check_execution()
|
|
336
|
+
if self._logical_steps >= self.policy.max_logical_steps:
|
|
337
|
+
raise PrivacyError("private horizon exhausted")
|
|
338
|
+
self._private_model.train()
|
|
339
|
+
manager = opacus_module("opacus.utils.batch_memory_manager").BatchMemoryManager
|
|
340
|
+
# Each logical call consumes one independent Poisson draw. No partially
|
|
341
|
+
# consumed loader iterator survives a boundary or research checkpoint.
|
|
342
|
+
before = self._logical_steps
|
|
343
|
+
step = self._workload.begin_step()
|
|
344
|
+
with manager(
|
|
345
|
+
data_loader=self._loader,
|
|
346
|
+
max_physical_batch_size=self._physical_limit,
|
|
347
|
+
optimizer=self._optimizer,
|
|
348
|
+
) as physical_loader:
|
|
349
|
+
for batch in physical_loader:
|
|
350
|
+
# Clear p.grad too: an empty Poisson draw runs no backward, and a
|
|
351
|
+
# zero-filled grad left behind would look like an unsampled one.
|
|
352
|
+
self._optimizer.zero_grad(set_to_none=True)
|
|
353
|
+
step.prepare_gradients(self._private_model, batch)
|
|
354
|
+
self._check_execution()
|
|
355
|
+
self._optimizer.step()
|
|
356
|
+
step.after_optimizer_step()
|
|
357
|
+
self._physical_steps += 1
|
|
358
|
+
completed = sum(n for _, _, n in self.accounting_history)
|
|
359
|
+
if completed == self._logical_steps + 1:
|
|
360
|
+
self._logical_steps = completed
|
|
361
|
+
if self.specification.schedule == "linear_decay":
|
|
362
|
+
for group, initial in zip(
|
|
363
|
+
self._base_optimizer.param_groups,
|
|
364
|
+
self._initial_learning_rates,
|
|
365
|
+
strict=True,
|
|
366
|
+
):
|
|
367
|
+
group["lr"] = initial * (
|
|
368
|
+
1 - completed / self.policy.max_logical_steps
|
|
369
|
+
)
|
|
370
|
+
self._optimizer.zero_grad(set_to_none=True)
|
|
371
|
+
break
|
|
372
|
+
if completed != self._logical_steps:
|
|
373
|
+
raise PrivacyError("invalid logical mechanism boundary")
|
|
374
|
+
if self._logical_steps != before + 1:
|
|
375
|
+
raise PrivacyError("private logical step did not complete")
|
|
376
|
+
return None
|
|
377
|
+
|
|
378
|
+
def weights(self) -> dict[str, np.ndarray]:
|
|
379
|
+
self._check_execution()
|
|
380
|
+
return super().weights()
|
|
381
|
+
|
|
382
|
+
def evaluate(self) -> EvaluationResult | None:
|
|
383
|
+
self._check_execution()
|
|
384
|
+
try:
|
|
385
|
+
return super().evaluate()
|
|
386
|
+
except Exception:
|
|
387
|
+
raise PrivacyError("public evaluation failed") from None
|
|
388
|
+
|
|
389
|
+
def export_model(self, path: Path) -> None:
|
|
390
|
+
"""Release only sanitized model tensors; no training state or metadata."""
|
|
391
|
+
self._check_execution()
|
|
392
|
+
save_safetensors(self._workload.export_tensors(super().weights), str(path))
|
|
393
|
+
|
|
394
|
+
def _save_private_state(self) -> dict[str, np.ndarray]:
|
|
395
|
+
state: dict[str, Any] = {
|
|
396
|
+
"version": 1,
|
|
397
|
+
"specification": canonical_hash(self.specification),
|
|
398
|
+
"policy": canonical_hash(self.policy),
|
|
399
|
+
"sample_rate": self._sample_rate,
|
|
400
|
+
"optimizer": self._base_optimizer.state_dict(),
|
|
401
|
+
"accountant": list(self.accounting_history),
|
|
402
|
+
"logical_steps": self._logical_steps,
|
|
403
|
+
"physical_steps": self._physical_steps,
|
|
404
|
+
"sampling_rng": self._sampling_generator.get_state()
|
|
405
|
+
if self._sampling_generator is not None
|
|
406
|
+
else None,
|
|
407
|
+
"noise_rng": self._noise_generator.get_state()
|
|
408
|
+
if self._noise_generator is not None
|
|
409
|
+
else None,
|
|
410
|
+
}
|
|
411
|
+
return encode_state(state)
|
|
412
|
+
|
|
413
|
+
def _load_private_state(self, arrays: dict[str, np.ndarray]) -> None:
|
|
414
|
+
state = decode_state(arrays)
|
|
415
|
+
self._validate_restore(state)
|
|
416
|
+
# Restore through Opacus so a factory delegate rebinds replacement groups.
|
|
417
|
+
self._optimizer.load_state_dict(state["optimizer"])
|
|
418
|
+
self._engine.accountant.history = list(state["accountant"])
|
|
419
|
+
self._logical_steps = state["logical_steps"]
|
|
420
|
+
self._physical_steps = state["physical_steps"]
|
|
421
|
+
assert (
|
|
422
|
+
self._sampling_generator is not None and self._noise_generator is not None
|
|
423
|
+
)
|
|
424
|
+
self._sampling_generator.set_state(state["sampling_rng"])
|
|
425
|
+
self._noise_generator.set_state(state["noise_rng"])
|
|
426
|
+
|
|
427
|
+
def _validate_restore(self, state: Any) -> None:
|
|
428
|
+
expected = {
|
|
429
|
+
"version",
|
|
430
|
+
"specification",
|
|
431
|
+
"policy",
|
|
432
|
+
"sample_rate",
|
|
433
|
+
"optimizer",
|
|
434
|
+
"accountant",
|
|
435
|
+
"logical_steps",
|
|
436
|
+
"physical_steps",
|
|
437
|
+
"sampling_rng",
|
|
438
|
+
"noise_rng",
|
|
439
|
+
}
|
|
440
|
+
if (
|
|
441
|
+
not isinstance(state, dict)
|
|
442
|
+
or set(cast(dict[str, Any], state)) != expected
|
|
443
|
+
or state["version"] != 1
|
|
444
|
+
):
|
|
445
|
+
raise PrivacyError("research checkpoint schema mismatch")
|
|
446
|
+
if (
|
|
447
|
+
state["specification"] != canonical_hash(self.specification)
|
|
448
|
+
or state["policy"] != canonical_hash(self.policy)
|
|
449
|
+
or state["sample_rate"] != self._sample_rate
|
|
450
|
+
):
|
|
451
|
+
raise PrivacyError(
|
|
452
|
+
"research checkpoint mechanism or optimizer identity mismatch"
|
|
453
|
+
)
|
|
454
|
+
state = cast(dict[str, Any], state)
|
|
455
|
+
steps: Any = state["logical_steps"]
|
|
456
|
+
if (
|
|
457
|
+
type(steps) is not int
|
|
458
|
+
or not 0 <= steps <= self.policy.max_logical_steps
|
|
459
|
+
or type(state["physical_steps"]) is not int
|
|
460
|
+
or state["physical_steps"] < steps
|
|
461
|
+
):
|
|
462
|
+
raise PrivacyError("invalid research checkpoint counters")
|
|
463
|
+
history = state["accountant"]
|
|
464
|
+
if (
|
|
465
|
+
any(
|
|
466
|
+
s != self.policy.noise_multiplier
|
|
467
|
+
or q != self._sample_rate
|
|
468
|
+
or type(n) is not int
|
|
469
|
+
or n <= 0
|
|
470
|
+
for s, q, n in history
|
|
471
|
+
)
|
|
472
|
+
or sum(n for _, _, n in history) != steps
|
|
473
|
+
):
|
|
474
|
+
raise PrivacyError("research checkpoint accountant history mismatch")
|
|
475
|
+
spec = self.specification.optimizer
|
|
476
|
+
if isinstance(spec, FactoryOptimizerSpec):
|
|
477
|
+
assert self._optimizer_registration is not None
|
|
478
|
+
self._optimizer_registration.validate_checkpoint(
|
|
479
|
+
self._base_optimizer,
|
|
480
|
+
self._raw_model,
|
|
481
|
+
spec,
|
|
482
|
+
state["optimizer"],
|
|
483
|
+
logical_steps=steps,
|
|
484
|
+
horizon=self.policy.max_logical_steps,
|
|
485
|
+
schedule=self.specification.schedule,
|
|
486
|
+
)
|
|
487
|
+
else:
|
|
488
|
+
validate_builtin_optimizer_checkpoint(
|
|
489
|
+
state["optimizer"],
|
|
490
|
+
spec=spec,
|
|
491
|
+
parameters=tuple(
|
|
492
|
+
p for p in self._raw_model.parameters() if p.requires_grad
|
|
493
|
+
),
|
|
494
|
+
initial_groups=self._initial_optimizer_groups,
|
|
495
|
+
logical_steps=steps,
|
|
496
|
+
horizon=self.policy.max_logical_steps,
|
|
497
|
+
schedule=self.specification.schedule,
|
|
498
|
+
)
|
|
499
|
+
for key, generator in (
|
|
500
|
+
("sampling_rng", self._sampling_generator),
|
|
501
|
+
("noise_rng", self._noise_generator),
|
|
502
|
+
):
|
|
503
|
+
if (
|
|
504
|
+
generator is None
|
|
505
|
+
or not isinstance(state[key], torch.Tensor)
|
|
506
|
+
or state[key].dtype != torch.uint8
|
|
507
|
+
or state[key].shape != generator.get_state().shape
|
|
508
|
+
):
|
|
509
|
+
raise PrivacyError("research checkpoint RNG mismatch")
|
|
510
|
+
probe = torch.Generator(device=generator.device)
|
|
511
|
+
probe.set_state(state[key])
|
|
512
|
+
|
|
513
|
+
def load_checkpoint_tensors(self, state: dict[str, np.ndarray]) -> None:
|
|
514
|
+
self._check_execution()
|
|
515
|
+
if self.policy.randomness_profile == "secure":
|
|
516
|
+
raise PrivacyError(
|
|
517
|
+
"secure private runs are fresh-start only; resume refused"
|
|
518
|
+
)
|
|
519
|
+
payload = {
|
|
520
|
+
k.removeprefix("application."): v
|
|
521
|
+
for k, v in state.items()
|
|
522
|
+
if k.startswith("application.")
|
|
523
|
+
}
|
|
524
|
+
decoded = decode_state(payload)
|
|
525
|
+
self._validate_restore(decoded)
|
|
526
|
+
if (
|
|
527
|
+
"meta.steps" not in state
|
|
528
|
+
or int(state["meta.steps"][0]) != decoded["logical_steps"]
|
|
529
|
+
):
|
|
530
|
+
raise PrivacyError("checkpoint counters disagree")
|
|
531
|
+
if "meta.loss" not in state or state["meta.loss"].size:
|
|
532
|
+
raise PrivacyError("private checkpoint contains raw loss")
|
|
533
|
+
self._workload.validate_checkpoint(state)
|
|
534
|
+
super().load_checkpoint_tensors(state)
|