dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
"""Owned ordinary execution of the same finite-record application task."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import math
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from typing import Any, cast
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import torch
|
|
13
|
+
from torch import nn
|
|
14
|
+
from torch.utils.data import DataLoader, Dataset, Sampler
|
|
15
|
+
|
|
16
|
+
from dromeus.errors import TaskError
|
|
17
|
+
from dromeus.manifests.canonical import canonical_hash, canonical_json
|
|
18
|
+
from dromeus.manifests.models import FactoryOptimizerSpec, TaskTrainingSpec
|
|
19
|
+
from dromeus.training.optimizer_factory import (
|
|
20
|
+
build_private_optimizer,
|
|
21
|
+
validate_builtin_optimizer_checkpoint,
|
|
22
|
+
)
|
|
23
|
+
from dromeus.training.private_data import dataset_size, make_task_loader
|
|
24
|
+
from dromeus.training.private_model import (
|
|
25
|
+
ModelStateGuard,
|
|
26
|
+
describe_private_model,
|
|
27
|
+
require_unchanged_public_state,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.training.private_state import decode_state, encode_state
|
|
30
|
+
from dromeus.training.task import TrainingTask, validate_record_losses
|
|
31
|
+
from dromeus.training.trainer import PyTorchTrainer
|
|
32
|
+
from dromeus.training.versions import torch_release
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _ShuffleBatches(Sampler[list[int]]):
|
|
36
|
+
"""Retain explicit epoch permutation/cursor instead of an opaque iterator."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, count: int, batch: int, generator: torch.Generator) -> None:
|
|
39
|
+
self.count, self.batch, self.generator = count, batch, generator
|
|
40
|
+
self.order = torch.empty(0, dtype=torch.int64)
|
|
41
|
+
self.position = 0
|
|
42
|
+
|
|
43
|
+
def __len__(self) -> int:
|
|
44
|
+
return math.ceil(self.count / self.batch)
|
|
45
|
+
|
|
46
|
+
def __iter__(self) -> Iterator[list[int]]:
|
|
47
|
+
if not len(self.order) or self.position == self.count:
|
|
48
|
+
self.order = torch.randperm(self.count, generator=self.generator)
|
|
49
|
+
self.position = 0
|
|
50
|
+
while self.position < self.count:
|
|
51
|
+
end = min(self.position + self.batch, self.count)
|
|
52
|
+
indices = cast(
|
|
53
|
+
list[int], cast(Any, self.order[self.position : end]).tolist()
|
|
54
|
+
)
|
|
55
|
+
self.position = end
|
|
56
|
+
yield indices
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class OrdinaryTaskTrainer(PyTorchTrainer):
|
|
60
|
+
"""The task interface with ordinary gradients and explicit sampling policy.
|
|
61
|
+
|
|
62
|
+
Shuffle execution uses only Torch. Poisson execution deliberately requires
|
|
63
|
+
the optional privacy dependency to match the private experiment sampler.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
*,
|
|
69
|
+
task: TrainingTask,
|
|
70
|
+
dataset: Dataset[Any],
|
|
71
|
+
specification: TaskTrainingSpec,
|
|
72
|
+
local_data_identity: str,
|
|
73
|
+
max_logical_steps: int,
|
|
74
|
+
research_seed: int,
|
|
75
|
+
max_physical_batch_size: int,
|
|
76
|
+
) -> None:
|
|
77
|
+
if (
|
|
78
|
+
not local_data_identity.strip()
|
|
79
|
+
or max_logical_steps <= 0
|
|
80
|
+
or max_physical_batch_size <= 0
|
|
81
|
+
or specification.optimizer.torch_version != torch_release()
|
|
82
|
+
):
|
|
83
|
+
raise TaskError(
|
|
84
|
+
"ordinary task requires valid identity, limits and Torch version"
|
|
85
|
+
)
|
|
86
|
+
task.check_definition(specification)
|
|
87
|
+
if (
|
|
88
|
+
task.public_evaluate is not None
|
|
89
|
+
and specification.evaluation_provenance != "public_fixed_holdout"
|
|
90
|
+
):
|
|
91
|
+
raise TaskError("ordinary task evaluation provenance is undeclared")
|
|
92
|
+
self.task, self.specification = task, specification
|
|
93
|
+
self._dataset = dataset
|
|
94
|
+
self._specification_hash = canonical_hash(specification)
|
|
95
|
+
self._horizon, self._physical_limit = max_logical_steps, max_physical_batch_size
|
|
96
|
+
self._logical_steps = self._physical_steps = 0
|
|
97
|
+
self._failed = False
|
|
98
|
+
self._population = dataset_size(self._dataset)
|
|
99
|
+
self._raw_model = task.model_factory()
|
|
100
|
+
report = describe_private_model(self._raw_model)
|
|
101
|
+
if (
|
|
102
|
+
report.trainable_parameter_names != specification.trainable_parameter_names
|
|
103
|
+
or report.parameter_aliases != specification.parameter_aliases
|
|
104
|
+
or report.public_buffer_hashes != specification.public_buffer_hashes
|
|
105
|
+
or report.frozen_parameter_hashes != specification.frozen_parameter_hashes
|
|
106
|
+
):
|
|
107
|
+
raise TaskError("ordinary model differs from shared task specification")
|
|
108
|
+
self._guard = ModelStateGuard(self._raw_model, report)
|
|
109
|
+
optimizer_spec = specification.optimizer
|
|
110
|
+
if isinstance(optimizer_spec, FactoryOptimizerSpec):
|
|
111
|
+
if task.optimizer_registration is None:
|
|
112
|
+
raise TaskError("factory optimizer requires its local registration")
|
|
113
|
+
self._optimizer = task.optimizer_registration.build(
|
|
114
|
+
self._raw_model, optimizer_spec
|
|
115
|
+
)
|
|
116
|
+
else:
|
|
117
|
+
if task.optimizer_registration is not None:
|
|
118
|
+
raise TaskError(
|
|
119
|
+
"optimizer registration requires factory specification"
|
|
120
|
+
)
|
|
121
|
+
self._optimizer = build_private_optimizer(self._raw_model, optimizer_spec)
|
|
122
|
+
self._parameters = tuple(
|
|
123
|
+
p for p in self._raw_model.parameters() if p.requires_grad
|
|
124
|
+
)
|
|
125
|
+
self._initial_groups = copy.deepcopy(
|
|
126
|
+
self._optimizer.state_dict()["param_groups"]
|
|
127
|
+
)
|
|
128
|
+
self._sampling_rng = torch.Generator().manual_seed(research_seed)
|
|
129
|
+
self._model_rng: torch.Tensor = (
|
|
130
|
+
torch.Generator().manual_seed(research_seed + 1).get_state()
|
|
131
|
+
)
|
|
132
|
+
device = next(self._raw_model.parameters()).device
|
|
133
|
+
self._cuda_rng: torch.Tensor | None = (
|
|
134
|
+
torch.Generator(device=device).manual_seed(research_seed + 2).get_state()
|
|
135
|
+
if device.type == "cuda"
|
|
136
|
+
else None
|
|
137
|
+
)
|
|
138
|
+
self._shuffle: _ShuffleBatches | None = None
|
|
139
|
+
if specification.sampling == "poisson":
|
|
140
|
+
self._loader = make_task_loader(
|
|
141
|
+
self._dataset,
|
|
142
|
+
task.batch_adapter,
|
|
143
|
+
batch_size=specification.batch_size,
|
|
144
|
+
generator=self._sampling_rng,
|
|
145
|
+
)
|
|
146
|
+
self._sample_rate = self._loader.sample_rate
|
|
147
|
+
else:
|
|
148
|
+
self._shuffle = _ShuffleBatches(
|
|
149
|
+
self._population, specification.batch_size, self._sampling_rng
|
|
150
|
+
)
|
|
151
|
+
self._loader = DataLoader( # pyright: ignore[reportUnknownMemberType]
|
|
152
|
+
self._dataset,
|
|
153
|
+
batch_sampler=self._shuffle,
|
|
154
|
+
collate_fn=task.batch_adapter.collate,
|
|
155
|
+
generator=self._sampling_rng,
|
|
156
|
+
num_workers=0,
|
|
157
|
+
)
|
|
158
|
+
self._sample_rate = None
|
|
159
|
+
identity = hashlib.sha256(
|
|
160
|
+
canonical_json(specification)
|
|
161
|
+
+ local_data_identity.encode()
|
|
162
|
+
+ str(
|
|
163
|
+
(self._population, max_logical_steps, max_physical_batch_size)
|
|
164
|
+
).encode()
|
|
165
|
+
).hexdigest()
|
|
166
|
+
super().__init__(
|
|
167
|
+
model=self._raw_model,
|
|
168
|
+
train_step=self._owned_step,
|
|
169
|
+
save_state=self._save_state,
|
|
170
|
+
load_state=self._load_state,
|
|
171
|
+
state_identity=identity,
|
|
172
|
+
evaluate=task.public_evaluate,
|
|
173
|
+
exchange_names=specification.trainable_parameter_names,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
@property
|
|
177
|
+
def sample_rate(self) -> float | None:
|
|
178
|
+
return self._sample_rate
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def physical_steps(self) -> int:
|
|
182
|
+
return self._physical_steps
|
|
183
|
+
|
|
184
|
+
def weights(self) -> dict[str, np.ndarray]:
|
|
185
|
+
self._check()
|
|
186
|
+
return super().weights()
|
|
187
|
+
|
|
188
|
+
def checkpoint_tensors(self) -> dict[str, np.ndarray]:
|
|
189
|
+
self._check()
|
|
190
|
+
return super().checkpoint_tensors()
|
|
191
|
+
|
|
192
|
+
def _check(self) -> None:
|
|
193
|
+
if self._failed:
|
|
194
|
+
raise TaskError("ordinary trainer failed")
|
|
195
|
+
if canonical_hash(self.specification) != self._specification_hash:
|
|
196
|
+
raise TaskError("ordinary task specification changed")
|
|
197
|
+
if dataset_size(self._dataset) != self._population:
|
|
198
|
+
raise TaskError("ordinary dataset population changed")
|
|
199
|
+
self._guard.check()
|
|
200
|
+
actual = tuple(
|
|
201
|
+
p for group in self._optimizer.param_groups for p in group["params"]
|
|
202
|
+
)
|
|
203
|
+
if len(actual) != len(self._parameters) or {id(p) for p in actual} != {
|
|
204
|
+
id(p) for p in self._parameters
|
|
205
|
+
}:
|
|
206
|
+
raise TaskError("ordinary optimizer parameter ownership changed")
|
|
207
|
+
self._validate_optimizer(self._optimizer.state_dict(), self._logical_steps)
|
|
208
|
+
|
|
209
|
+
def train_local_steps(self, step_count: int) -> None:
|
|
210
|
+
self._check()
|
|
211
|
+
if step_count <= 0 or self._logical_steps + step_count > self._horizon:
|
|
212
|
+
raise TaskError(
|
|
213
|
+
"ordinary training horizon exhausted or invalid step count"
|
|
214
|
+
)
|
|
215
|
+
try:
|
|
216
|
+
super().train_local_steps(step_count)
|
|
217
|
+
except Exception:
|
|
218
|
+
self._failed = True
|
|
219
|
+
raise
|
|
220
|
+
|
|
221
|
+
def _owned_step(self, _: nn.Module) -> float:
|
|
222
|
+
self._check()
|
|
223
|
+
device = next(self._raw_model.parameters()).device
|
|
224
|
+
devices = [device.index or 0] if device.type == "cuda" else []
|
|
225
|
+
with cast(Any, torch.random).fork_rng(devices=devices):
|
|
226
|
+
torch.set_rng_state(self._model_rng)
|
|
227
|
+
if self._cuda_rng is not None:
|
|
228
|
+
torch.cuda.set_rng_state(self._cuda_rng, device)
|
|
229
|
+
self._raw_model.train()
|
|
230
|
+
batch = next(iter(self._loader))
|
|
231
|
+
count = self.task.batch_adapter.size(batch)
|
|
232
|
+
normalizer = (
|
|
233
|
+
int(self._population * self._sample_rate)
|
|
234
|
+
if self._sample_rate is not None
|
|
235
|
+
else max(count, 1)
|
|
236
|
+
)
|
|
237
|
+
self._optimizer.zero_grad(set_to_none=True)
|
|
238
|
+
for parameter in self._parameters:
|
|
239
|
+
parameter.grad = torch.zeros_like(parameter)
|
|
240
|
+
total_loss = 0.0
|
|
241
|
+
for start in range(0, count, self._physical_limit):
|
|
242
|
+
physical = self.task.batch_adapter.slice(
|
|
243
|
+
batch, start, min(count, start + self._physical_limit)
|
|
244
|
+
)
|
|
245
|
+
physical = self.task.batch_adapter.to_device(physical, device)
|
|
246
|
+
values = validate_record_losses(
|
|
247
|
+
self.task.loss_per_record(self._raw_model, physical),
|
|
248
|
+
self.task.batch_adapter.size(physical),
|
|
249
|
+
)
|
|
250
|
+
total_loss += float(values.detach().sum())
|
|
251
|
+
cast(Any, values.sum() / normalizer).backward()
|
|
252
|
+
self._physical_steps += 1
|
|
253
|
+
if count == 0:
|
|
254
|
+
self._physical_steps += 1
|
|
255
|
+
self._guard.check()
|
|
256
|
+
self._optimizer.step()
|
|
257
|
+
self._logical_steps += 1
|
|
258
|
+
if self.specification.schedule == "linear_decay":
|
|
259
|
+
for group, initial in zip(
|
|
260
|
+
self._optimizer.param_groups, self._initial_groups, strict=True
|
|
261
|
+
):
|
|
262
|
+
group["lr"] = initial["lr"] * (
|
|
263
|
+
1 - self._logical_steps / self._horizon
|
|
264
|
+
)
|
|
265
|
+
self._optimizer.zero_grad(set_to_none=True)
|
|
266
|
+
self._guard.check()
|
|
267
|
+
self._model_rng = torch.get_rng_state().clone()
|
|
268
|
+
if devices:
|
|
269
|
+
self._cuda_rng = torch.cuda.get_rng_state(device).clone()
|
|
270
|
+
return total_loss / max(count, 1)
|
|
271
|
+
|
|
272
|
+
def _save_state(self) -> dict[str, np.ndarray]:
|
|
273
|
+
return encode_state(
|
|
274
|
+
{
|
|
275
|
+
"version": 1,
|
|
276
|
+
"specification": self._specification_hash,
|
|
277
|
+
"optimizer": self._optimizer.state_dict(),
|
|
278
|
+
"logical_steps": self._logical_steps,
|
|
279
|
+
"physical_steps": self._physical_steps,
|
|
280
|
+
"sampling_rng": self._sampling_rng.get_state(),
|
|
281
|
+
"model_rng": self._model_rng,
|
|
282
|
+
"cuda_rng": self._cuda_rng,
|
|
283
|
+
"shuffle_order": self._shuffle.order if self._shuffle else None,
|
|
284
|
+
"shuffle_position": self._shuffle.position if self._shuffle else None,
|
|
285
|
+
}
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
def _validate_optimizer(self, state: dict[str, Any], steps: int) -> None:
|
|
289
|
+
spec = self.specification.optimizer
|
|
290
|
+
if isinstance(spec, FactoryOptimizerSpec):
|
|
291
|
+
registration = self.task.optimizer_registration
|
|
292
|
+
assert registration is not None
|
|
293
|
+
registration.validate_checkpoint(
|
|
294
|
+
self._optimizer,
|
|
295
|
+
self._raw_model,
|
|
296
|
+
spec,
|
|
297
|
+
state,
|
|
298
|
+
logical_steps=steps,
|
|
299
|
+
horizon=self._horizon,
|
|
300
|
+
schedule=self.specification.schedule,
|
|
301
|
+
)
|
|
302
|
+
return
|
|
303
|
+
validate_builtin_optimizer_checkpoint(
|
|
304
|
+
state,
|
|
305
|
+
spec=spec,
|
|
306
|
+
parameters=self._parameters,
|
|
307
|
+
initial_groups=self._initial_groups,
|
|
308
|
+
logical_steps=steps,
|
|
309
|
+
horizon=self._horizon,
|
|
310
|
+
schedule=self.specification.schedule,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
def _load_state(self, arrays: dict[str, np.ndarray]) -> None:
|
|
314
|
+
state = decode_state(arrays)
|
|
315
|
+
expected = set(decode_state(self._save_state()))
|
|
316
|
+
if (
|
|
317
|
+
not isinstance(state, dict)
|
|
318
|
+
or set(cast(dict[str, Any], state)) != expected
|
|
319
|
+
or state["version"] != 1
|
|
320
|
+
):
|
|
321
|
+
raise TaskError("ordinary task checkpoint schema mismatch")
|
|
322
|
+
state = cast(dict[str, Any], state)
|
|
323
|
+
if state["specification"] != self._specification_hash:
|
|
324
|
+
raise TaskError("ordinary task checkpoint specification mismatch")
|
|
325
|
+
steps, physical = state["logical_steps"], state["physical_steps"]
|
|
326
|
+
if (
|
|
327
|
+
type(steps) is not int
|
|
328
|
+
or not 0 <= steps <= self._horizon
|
|
329
|
+
or type(physical) is not int
|
|
330
|
+
or physical < steps
|
|
331
|
+
):
|
|
332
|
+
raise TaskError("ordinary task checkpoint counters mismatch")
|
|
333
|
+
self._validate_optimizer(state["optimizer"], steps)
|
|
334
|
+
for key, current in (
|
|
335
|
+
("sampling_rng", self._sampling_rng.get_state()),
|
|
336
|
+
("model_rng", self._model_rng),
|
|
337
|
+
("cuda_rng", self._cuda_rng),
|
|
338
|
+
):
|
|
339
|
+
saved = state[key]
|
|
340
|
+
if current is None:
|
|
341
|
+
if saved is not None:
|
|
342
|
+
raise TaskError("ordinary task checkpoint RNG device mismatch")
|
|
343
|
+
elif (
|
|
344
|
+
not isinstance(saved, torch.Tensor)
|
|
345
|
+
or saved.dtype != torch.uint8
|
|
346
|
+
or saved.shape != current.shape
|
|
347
|
+
):
|
|
348
|
+
raise TaskError("ordinary task checkpoint RNG mismatch")
|
|
349
|
+
probe = torch.Generator()
|
|
350
|
+
probe.set_state(state["sampling_rng"])
|
|
351
|
+
probe.set_state(state["model_rng"])
|
|
352
|
+
if self._cuda_rng is not None:
|
|
353
|
+
cuda_probe = torch.Generator(device=self._parameters[0].device)
|
|
354
|
+
cuda_probe.set_state(state["cuda_rng"])
|
|
355
|
+
if self._shuffle is not None:
|
|
356
|
+
order, position = state["shuffle_order"], state["shuffle_position"]
|
|
357
|
+
if (
|
|
358
|
+
not isinstance(order, torch.Tensor)
|
|
359
|
+
or order.dtype != torch.int64
|
|
360
|
+
or order.ndim != 1
|
|
361
|
+
or type(position) is not int
|
|
362
|
+
):
|
|
363
|
+
raise TaskError("ordinary shuffle checkpoint mismatch")
|
|
364
|
+
if (not len(order) and position != 0) or (
|
|
365
|
+
len(order)
|
|
366
|
+
and (
|
|
367
|
+
len(order) != self._population
|
|
368
|
+
or not 0 <= position <= len(order)
|
|
369
|
+
or not torch.equal(
|
|
370
|
+
order.sort().values, torch.arange(self._population)
|
|
371
|
+
)
|
|
372
|
+
)
|
|
373
|
+
):
|
|
374
|
+
raise TaskError("ordinary shuffle permutation mismatch")
|
|
375
|
+
elif (
|
|
376
|
+
state["shuffle_order"] is not None or state["shuffle_position"] is not None
|
|
377
|
+
):
|
|
378
|
+
raise TaskError("ordinary sampler checkpoint mismatch")
|
|
379
|
+
self._optimizer.load_state_dict(state["optimizer"])
|
|
380
|
+
self._sampling_rng.set_state(state["sampling_rng"])
|
|
381
|
+
self._model_rng, self._cuda_rng = state["model_rng"], state["cuda_rng"]
|
|
382
|
+
if self._shuffle is not None:
|
|
383
|
+
self._shuffle.order = cast(torch.Tensor, state["shuffle_order"])
|
|
384
|
+
self._shuffle.position = cast(int, state["shuffle_position"])
|
|
385
|
+
self._logical_steps, self._physical_steps = steps, physical
|
|
386
|
+
|
|
387
|
+
def load_checkpoint_tensors(self, state: dict[str, np.ndarray]) -> None:
|
|
388
|
+
self._check()
|
|
389
|
+
arrays = {
|
|
390
|
+
k.removeprefix("application."): v
|
|
391
|
+
for k, v in state.items()
|
|
392
|
+
if k.startswith("application.")
|
|
393
|
+
}
|
|
394
|
+
decoded = decode_state(arrays)
|
|
395
|
+
if "meta.steps" not in state or int(state["meta.steps"][0]) != decoded.get(
|
|
396
|
+
"logical_steps"
|
|
397
|
+
):
|
|
398
|
+
raise TaskError("ordinary checkpoint counters disagree")
|
|
399
|
+
require_unchanged_public_state(
|
|
400
|
+
self._raw_model,
|
|
401
|
+
state,
|
|
402
|
+
{
|
|
403
|
+
**self.specification.public_buffer_hashes,
|
|
404
|
+
**self.specification.frozen_parameter_hashes,
|
|
405
|
+
},
|
|
406
|
+
)
|
|
407
|
+
super().load_checkpoint_tensors(state)
|
|
408
|
+
self._guard.check()
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Lazy optional Opacus boundary and mechanism accounting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import math
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from typing import Any, Protocol
|
|
10
|
+
|
|
11
|
+
from dromeus.errors import PrivacyError
|
|
12
|
+
from dromeus.manifests.models import PrivacyPolicy, PrivacyReservation
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ReservationStore(Protocol):
|
|
16
|
+
def reserve(
|
|
17
|
+
self,
|
|
18
|
+
reservation: PrivacyReservation,
|
|
19
|
+
*,
|
|
20
|
+
epsilon_limit: float | None,
|
|
21
|
+
forecast: Callable[[tuple[PrivacyReservation, ...]], float],
|
|
22
|
+
) -> None: ...
|
|
23
|
+
|
|
24
|
+
def check(self, reservation: PrivacyReservation) -> None: ...
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def opacus_module(name: str = "opacus") -> Any:
|
|
28
|
+
"""Keep untyped upstream API confined to the optional implementation."""
|
|
29
|
+
try:
|
|
30
|
+
module = importlib.import_module(name)
|
|
31
|
+
if importlib.import_module("opacus").__version__ != "1.6.0":
|
|
32
|
+
raise PrivacyError("private training requires Opacus 1.6.0")
|
|
33
|
+
return module
|
|
34
|
+
except ImportError:
|
|
35
|
+
raise PrivacyError(
|
|
36
|
+
"install dromeus[privacy] and secure RNG when required"
|
|
37
|
+
) from None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def privacy_epsilon(
|
|
41
|
+
history: tuple[tuple[float, float, int], ...],
|
|
42
|
+
policy: PrivacyPolicy,
|
|
43
|
+
*,
|
|
44
|
+
accountant: str = "prv",
|
|
45
|
+
) -> float:
|
|
46
|
+
if not history:
|
|
47
|
+
return 0.0
|
|
48
|
+
accountants = opacus_module("opacus.accountants")
|
|
49
|
+
engine = (
|
|
50
|
+
accountants.PRVAccountant()
|
|
51
|
+
if accountant == "prv"
|
|
52
|
+
else accountants.RDPAccountant()
|
|
53
|
+
)
|
|
54
|
+
query_history = list(history)
|
|
55
|
+
# Do not normalize unsupported counts into a different valid mechanism.
|
|
56
|
+
if all(steps > 0 for _, _, steps in history):
|
|
57
|
+
# Identical fixed Poisson-Gaussian mechanisms share one privacy-loss
|
|
58
|
+
# distribution. Compose their total multiplicity once in the query copy;
|
|
59
|
+
# live accountant histories and durable reservations retain their order.
|
|
60
|
+
counts: Counter[tuple[float, float]] = Counter()
|
|
61
|
+
for noise, sample_rate, steps in history:
|
|
62
|
+
counts[noise, sample_rate] += steps
|
|
63
|
+
query_history = [
|
|
64
|
+
(noise, sample_rate, steps)
|
|
65
|
+
for (noise, sample_rate), steps in counts.items()
|
|
66
|
+
]
|
|
67
|
+
engine.history = query_history
|
|
68
|
+
kwargs = (
|
|
69
|
+
{
|
|
70
|
+
"eps_error": policy.prv_epsilon_error,
|
|
71
|
+
"delta_error": policy.delta * policy.prv_delta_error_ratio,
|
|
72
|
+
}
|
|
73
|
+
if accountant == "prv"
|
|
74
|
+
else {}
|
|
75
|
+
)
|
|
76
|
+
value = float(engine.get_epsilon(delta=policy.delta, **kwargs))
|
|
77
|
+
if not math.isfinite(value) or value < 0:
|
|
78
|
+
raise PrivacyError("privacy accounting failed")
|
|
79
|
+
return value
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def reserve_horizon(
|
|
83
|
+
store: ReservationStore,
|
|
84
|
+
reservation: PrivacyReservation,
|
|
85
|
+
policy: PrivacyPolicy,
|
|
86
|
+
) -> None:
|
|
87
|
+
store.reserve(
|
|
88
|
+
reservation,
|
|
89
|
+
epsilon_limit=policy.epsilon_limit,
|
|
90
|
+
forecast=lambda records: privacy_epsilon(
|
|
91
|
+
tuple(
|
|
92
|
+
(r.noise_multiplier, r.sample_rate, r.logical_steps) for r in records
|
|
93
|
+
),
|
|
94
|
+
policy,
|
|
95
|
+
),
|
|
96
|
+
)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Owned Poisson sampling for finite application datasets without eager reads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Sized
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
|
|
9
|
+
import torch
|
|
10
|
+
from torch.utils.data import DataLoader, Dataset, IterableDataset, Sampler
|
|
11
|
+
|
|
12
|
+
from dromeus.errors import PrivacyError
|
|
13
|
+
from dromeus.training.privacy import opacus_module
|
|
14
|
+
from dromeus.training.task import TensorBatchAdapter
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def dataset_size(dataset: Dataset[Any]) -> int:
|
|
18
|
+
"""Only metadata is consulted; never read a record during preparation."""
|
|
19
|
+
if isinstance(dataset, IterableDataset) or not isinstance(dataset, Sized):
|
|
20
|
+
raise PrivacyError("private task requires a finite indexed dataset")
|
|
21
|
+
count = len(dataset)
|
|
22
|
+
if count <= 0:
|
|
23
|
+
raise PrivacyError("private task requires a nonempty fixed dataset")
|
|
24
|
+
return count
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _FixedPopulation(Dataset[Any]):
|
|
28
|
+
def __init__(self, dataset: Dataset[Any], count: int) -> None:
|
|
29
|
+
self.source = dataset
|
|
30
|
+
self.count = count
|
|
31
|
+
|
|
32
|
+
def __len__(self) -> int:
|
|
33
|
+
if dataset_size(self.source) != self.count:
|
|
34
|
+
raise PrivacyError("private dataset population changed")
|
|
35
|
+
return self.count
|
|
36
|
+
|
|
37
|
+
def __getitem__(self, index: int) -> Any:
|
|
38
|
+
if not 0 <= index < len(self):
|
|
39
|
+
raise IndexError("private record index out of range")
|
|
40
|
+
return self.source[index]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TaskDataLoader(DataLoader[Any]):
|
|
44
|
+
"""An owned sampler and collator; no user-provided sampling distribution."""
|
|
45
|
+
|
|
46
|
+
sample_rate: float
|
|
47
|
+
population_size: int
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
dataset: Dataset[Any],
|
|
52
|
+
adapter: TensorBatchAdapter,
|
|
53
|
+
*,
|
|
54
|
+
batch_size: int,
|
|
55
|
+
generator: torch.Generator | None,
|
|
56
|
+
) -> None:
|
|
57
|
+
if type(batch_size) is not int or batch_size <= 0:
|
|
58
|
+
raise PrivacyError("logical batch size must be a positive integer")
|
|
59
|
+
self.population_size = dataset_size(dataset)
|
|
60
|
+
steps = math.ceil(self.population_size / batch_size)
|
|
61
|
+
self.sample_rate = 1 / steps
|
|
62
|
+
sampler_type = opacus_module(
|
|
63
|
+
"opacus.utils.uniform_sampler"
|
|
64
|
+
).UniformWithReplacementSampler
|
|
65
|
+
sampler = sampler_type(
|
|
66
|
+
num_samples=self.population_size,
|
|
67
|
+
sample_rate=self.sample_rate,
|
|
68
|
+
generator=generator,
|
|
69
|
+
steps=steps,
|
|
70
|
+
)
|
|
71
|
+
super().__init__( # pyright: ignore[reportUnknownMemberType]
|
|
72
|
+
_FixedPopulation(dataset, self.population_size),
|
|
73
|
+
batch_sampler=cast(Sampler[list[int]], sampler),
|
|
74
|
+
collate_fn=adapter.collate,
|
|
75
|
+
generator=generator,
|
|
76
|
+
num_workers=0,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
def check_population(self) -> None:
|
|
80
|
+
if len(cast(Sized, self.dataset)) != self.population_size:
|
|
81
|
+
raise PrivacyError("private dataset population changed")
|
|
82
|
+
|
|
83
|
+
def __iter__(self) -> Any:
|
|
84
|
+
self.check_population()
|
|
85
|
+
return super().__iter__()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def make_task_loader(
|
|
89
|
+
dataset: Dataset[Any],
|
|
90
|
+
adapter: TensorBatchAdapter,
|
|
91
|
+
*,
|
|
92
|
+
batch_size: int,
|
|
93
|
+
generator: torch.Generator | None,
|
|
94
|
+
) -> TaskDataLoader:
|
|
95
|
+
"""Use with PrivacyEngine(poisson_sampling=False): sampling is already owned."""
|
|
96
|
+
return TaskDataLoader(dataset, adapter, batch_size=batch_size, generator=generator)
|