dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,408 @@
1
+ """Owned ordinary execution of the same finite-record application task."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ import hashlib
7
+ import math
8
+ from collections.abc import Iterator
9
+ from typing import Any, cast
10
+
11
+ import numpy as np
12
+ import torch
13
+ from torch import nn
14
+ from torch.utils.data import DataLoader, Dataset, Sampler
15
+
16
+ from dromeus.errors import TaskError
17
+ from dromeus.manifests.canonical import canonical_hash, canonical_json
18
+ from dromeus.manifests.models import FactoryOptimizerSpec, TaskTrainingSpec
19
+ from dromeus.training.optimizer_factory import (
20
+ build_private_optimizer,
21
+ validate_builtin_optimizer_checkpoint,
22
+ )
23
+ from dromeus.training.private_data import dataset_size, make_task_loader
24
+ from dromeus.training.private_model import (
25
+ ModelStateGuard,
26
+ describe_private_model,
27
+ require_unchanged_public_state,
28
+ )
29
+ from dromeus.training.private_state import decode_state, encode_state
30
+ from dromeus.training.task import TrainingTask, validate_record_losses
31
+ from dromeus.training.trainer import PyTorchTrainer
32
+ from dromeus.training.versions import torch_release
33
+
34
+
35
+ class _ShuffleBatches(Sampler[list[int]]):
36
+ """Retain explicit epoch permutation/cursor instead of an opaque iterator."""
37
+
38
+ def __init__(self, count: int, batch: int, generator: torch.Generator) -> None:
39
+ self.count, self.batch, self.generator = count, batch, generator
40
+ self.order = torch.empty(0, dtype=torch.int64)
41
+ self.position = 0
42
+
43
+ def __len__(self) -> int:
44
+ return math.ceil(self.count / self.batch)
45
+
46
+ def __iter__(self) -> Iterator[list[int]]:
47
+ if not len(self.order) or self.position == self.count:
48
+ self.order = torch.randperm(self.count, generator=self.generator)
49
+ self.position = 0
50
+ while self.position < self.count:
51
+ end = min(self.position + self.batch, self.count)
52
+ indices = cast(
53
+ list[int], cast(Any, self.order[self.position : end]).tolist()
54
+ )
55
+ self.position = end
56
+ yield indices
57
+
58
+
59
+ class OrdinaryTaskTrainer(PyTorchTrainer):
60
+ """The task interface with ordinary gradients and explicit sampling policy.
61
+
62
+ Shuffle execution uses only Torch. Poisson execution deliberately requires
63
+ the optional privacy dependency to match the private experiment sampler.
64
+ """
65
+
66
+ def __init__(
67
+ self,
68
+ *,
69
+ task: TrainingTask,
70
+ dataset: Dataset[Any],
71
+ specification: TaskTrainingSpec,
72
+ local_data_identity: str,
73
+ max_logical_steps: int,
74
+ research_seed: int,
75
+ max_physical_batch_size: int,
76
+ ) -> None:
77
+ if (
78
+ not local_data_identity.strip()
79
+ or max_logical_steps <= 0
80
+ or max_physical_batch_size <= 0
81
+ or specification.optimizer.torch_version != torch_release()
82
+ ):
83
+ raise TaskError(
84
+ "ordinary task requires valid identity, limits and Torch version"
85
+ )
86
+ task.check_definition(specification)
87
+ if (
88
+ task.public_evaluate is not None
89
+ and specification.evaluation_provenance != "public_fixed_holdout"
90
+ ):
91
+ raise TaskError("ordinary task evaluation provenance is undeclared")
92
+ self.task, self.specification = task, specification
93
+ self._dataset = dataset
94
+ self._specification_hash = canonical_hash(specification)
95
+ self._horizon, self._physical_limit = max_logical_steps, max_physical_batch_size
96
+ self._logical_steps = self._physical_steps = 0
97
+ self._failed = False
98
+ self._population = dataset_size(self._dataset)
99
+ self._raw_model = task.model_factory()
100
+ report = describe_private_model(self._raw_model)
101
+ if (
102
+ report.trainable_parameter_names != specification.trainable_parameter_names
103
+ or report.parameter_aliases != specification.parameter_aliases
104
+ or report.public_buffer_hashes != specification.public_buffer_hashes
105
+ or report.frozen_parameter_hashes != specification.frozen_parameter_hashes
106
+ ):
107
+ raise TaskError("ordinary model differs from shared task specification")
108
+ self._guard = ModelStateGuard(self._raw_model, report)
109
+ optimizer_spec = specification.optimizer
110
+ if isinstance(optimizer_spec, FactoryOptimizerSpec):
111
+ if task.optimizer_registration is None:
112
+ raise TaskError("factory optimizer requires its local registration")
113
+ self._optimizer = task.optimizer_registration.build(
114
+ self._raw_model, optimizer_spec
115
+ )
116
+ else:
117
+ if task.optimizer_registration is not None:
118
+ raise TaskError(
119
+ "optimizer registration requires factory specification"
120
+ )
121
+ self._optimizer = build_private_optimizer(self._raw_model, optimizer_spec)
122
+ self._parameters = tuple(
123
+ p for p in self._raw_model.parameters() if p.requires_grad
124
+ )
125
+ self._initial_groups = copy.deepcopy(
126
+ self._optimizer.state_dict()["param_groups"]
127
+ )
128
+ self._sampling_rng = torch.Generator().manual_seed(research_seed)
129
+ self._model_rng: torch.Tensor = (
130
+ torch.Generator().manual_seed(research_seed + 1).get_state()
131
+ )
132
+ device = next(self._raw_model.parameters()).device
133
+ self._cuda_rng: torch.Tensor | None = (
134
+ torch.Generator(device=device).manual_seed(research_seed + 2).get_state()
135
+ if device.type == "cuda"
136
+ else None
137
+ )
138
+ self._shuffle: _ShuffleBatches | None = None
139
+ if specification.sampling == "poisson":
140
+ self._loader = make_task_loader(
141
+ self._dataset,
142
+ task.batch_adapter,
143
+ batch_size=specification.batch_size,
144
+ generator=self._sampling_rng,
145
+ )
146
+ self._sample_rate = self._loader.sample_rate
147
+ else:
148
+ self._shuffle = _ShuffleBatches(
149
+ self._population, specification.batch_size, self._sampling_rng
150
+ )
151
+ self._loader = DataLoader( # pyright: ignore[reportUnknownMemberType]
152
+ self._dataset,
153
+ batch_sampler=self._shuffle,
154
+ collate_fn=task.batch_adapter.collate,
155
+ generator=self._sampling_rng,
156
+ num_workers=0,
157
+ )
158
+ self._sample_rate = None
159
+ identity = hashlib.sha256(
160
+ canonical_json(specification)
161
+ + local_data_identity.encode()
162
+ + str(
163
+ (self._population, max_logical_steps, max_physical_batch_size)
164
+ ).encode()
165
+ ).hexdigest()
166
+ super().__init__(
167
+ model=self._raw_model,
168
+ train_step=self._owned_step,
169
+ save_state=self._save_state,
170
+ load_state=self._load_state,
171
+ state_identity=identity,
172
+ evaluate=task.public_evaluate,
173
+ exchange_names=specification.trainable_parameter_names,
174
+ )
175
+
176
+ @property
177
+ def sample_rate(self) -> float | None:
178
+ return self._sample_rate
179
+
180
+ @property
181
+ def physical_steps(self) -> int:
182
+ return self._physical_steps
183
+
184
+ def weights(self) -> dict[str, np.ndarray]:
185
+ self._check()
186
+ return super().weights()
187
+
188
+ def checkpoint_tensors(self) -> dict[str, np.ndarray]:
189
+ self._check()
190
+ return super().checkpoint_tensors()
191
+
192
+ def _check(self) -> None:
193
+ if self._failed:
194
+ raise TaskError("ordinary trainer failed")
195
+ if canonical_hash(self.specification) != self._specification_hash:
196
+ raise TaskError("ordinary task specification changed")
197
+ if dataset_size(self._dataset) != self._population:
198
+ raise TaskError("ordinary dataset population changed")
199
+ self._guard.check()
200
+ actual = tuple(
201
+ p for group in self._optimizer.param_groups for p in group["params"]
202
+ )
203
+ if len(actual) != len(self._parameters) or {id(p) for p in actual} != {
204
+ id(p) for p in self._parameters
205
+ }:
206
+ raise TaskError("ordinary optimizer parameter ownership changed")
207
+ self._validate_optimizer(self._optimizer.state_dict(), self._logical_steps)
208
+
209
+ def train_local_steps(self, step_count: int) -> None:
210
+ self._check()
211
+ if step_count <= 0 or self._logical_steps + step_count > self._horizon:
212
+ raise TaskError(
213
+ "ordinary training horizon exhausted or invalid step count"
214
+ )
215
+ try:
216
+ super().train_local_steps(step_count)
217
+ except Exception:
218
+ self._failed = True
219
+ raise
220
+
221
+ def _owned_step(self, _: nn.Module) -> float:
222
+ self._check()
223
+ device = next(self._raw_model.parameters()).device
224
+ devices = [device.index or 0] if device.type == "cuda" else []
225
+ with cast(Any, torch.random).fork_rng(devices=devices):
226
+ torch.set_rng_state(self._model_rng)
227
+ if self._cuda_rng is not None:
228
+ torch.cuda.set_rng_state(self._cuda_rng, device)
229
+ self._raw_model.train()
230
+ batch = next(iter(self._loader))
231
+ count = self.task.batch_adapter.size(batch)
232
+ normalizer = (
233
+ int(self._population * self._sample_rate)
234
+ if self._sample_rate is not None
235
+ else max(count, 1)
236
+ )
237
+ self._optimizer.zero_grad(set_to_none=True)
238
+ for parameter in self._parameters:
239
+ parameter.grad = torch.zeros_like(parameter)
240
+ total_loss = 0.0
241
+ for start in range(0, count, self._physical_limit):
242
+ physical = self.task.batch_adapter.slice(
243
+ batch, start, min(count, start + self._physical_limit)
244
+ )
245
+ physical = self.task.batch_adapter.to_device(physical, device)
246
+ values = validate_record_losses(
247
+ self.task.loss_per_record(self._raw_model, physical),
248
+ self.task.batch_adapter.size(physical),
249
+ )
250
+ total_loss += float(values.detach().sum())
251
+ cast(Any, values.sum() / normalizer).backward()
252
+ self._physical_steps += 1
253
+ if count == 0:
254
+ self._physical_steps += 1
255
+ self._guard.check()
256
+ self._optimizer.step()
257
+ self._logical_steps += 1
258
+ if self.specification.schedule == "linear_decay":
259
+ for group, initial in zip(
260
+ self._optimizer.param_groups, self._initial_groups, strict=True
261
+ ):
262
+ group["lr"] = initial["lr"] * (
263
+ 1 - self._logical_steps / self._horizon
264
+ )
265
+ self._optimizer.zero_grad(set_to_none=True)
266
+ self._guard.check()
267
+ self._model_rng = torch.get_rng_state().clone()
268
+ if devices:
269
+ self._cuda_rng = torch.cuda.get_rng_state(device).clone()
270
+ return total_loss / max(count, 1)
271
+
272
+ def _save_state(self) -> dict[str, np.ndarray]:
273
+ return encode_state(
274
+ {
275
+ "version": 1,
276
+ "specification": self._specification_hash,
277
+ "optimizer": self._optimizer.state_dict(),
278
+ "logical_steps": self._logical_steps,
279
+ "physical_steps": self._physical_steps,
280
+ "sampling_rng": self._sampling_rng.get_state(),
281
+ "model_rng": self._model_rng,
282
+ "cuda_rng": self._cuda_rng,
283
+ "shuffle_order": self._shuffle.order if self._shuffle else None,
284
+ "shuffle_position": self._shuffle.position if self._shuffle else None,
285
+ }
286
+ )
287
+
288
+ def _validate_optimizer(self, state: dict[str, Any], steps: int) -> None:
289
+ spec = self.specification.optimizer
290
+ if isinstance(spec, FactoryOptimizerSpec):
291
+ registration = self.task.optimizer_registration
292
+ assert registration is not None
293
+ registration.validate_checkpoint(
294
+ self._optimizer,
295
+ self._raw_model,
296
+ spec,
297
+ state,
298
+ logical_steps=steps,
299
+ horizon=self._horizon,
300
+ schedule=self.specification.schedule,
301
+ )
302
+ return
303
+ validate_builtin_optimizer_checkpoint(
304
+ state,
305
+ spec=spec,
306
+ parameters=self._parameters,
307
+ initial_groups=self._initial_groups,
308
+ logical_steps=steps,
309
+ horizon=self._horizon,
310
+ schedule=self.specification.schedule,
311
+ )
312
+
313
+ def _load_state(self, arrays: dict[str, np.ndarray]) -> None:
314
+ state = decode_state(arrays)
315
+ expected = set(decode_state(self._save_state()))
316
+ if (
317
+ not isinstance(state, dict)
318
+ or set(cast(dict[str, Any], state)) != expected
319
+ or state["version"] != 1
320
+ ):
321
+ raise TaskError("ordinary task checkpoint schema mismatch")
322
+ state = cast(dict[str, Any], state)
323
+ if state["specification"] != self._specification_hash:
324
+ raise TaskError("ordinary task checkpoint specification mismatch")
325
+ steps, physical = state["logical_steps"], state["physical_steps"]
326
+ if (
327
+ type(steps) is not int
328
+ or not 0 <= steps <= self._horizon
329
+ or type(physical) is not int
330
+ or physical < steps
331
+ ):
332
+ raise TaskError("ordinary task checkpoint counters mismatch")
333
+ self._validate_optimizer(state["optimizer"], steps)
334
+ for key, current in (
335
+ ("sampling_rng", self._sampling_rng.get_state()),
336
+ ("model_rng", self._model_rng),
337
+ ("cuda_rng", self._cuda_rng),
338
+ ):
339
+ saved = state[key]
340
+ if current is None:
341
+ if saved is not None:
342
+ raise TaskError("ordinary task checkpoint RNG device mismatch")
343
+ elif (
344
+ not isinstance(saved, torch.Tensor)
345
+ or saved.dtype != torch.uint8
346
+ or saved.shape != current.shape
347
+ ):
348
+ raise TaskError("ordinary task checkpoint RNG mismatch")
349
+ probe = torch.Generator()
350
+ probe.set_state(state["sampling_rng"])
351
+ probe.set_state(state["model_rng"])
352
+ if self._cuda_rng is not None:
353
+ cuda_probe = torch.Generator(device=self._parameters[0].device)
354
+ cuda_probe.set_state(state["cuda_rng"])
355
+ if self._shuffle is not None:
356
+ order, position = state["shuffle_order"], state["shuffle_position"]
357
+ if (
358
+ not isinstance(order, torch.Tensor)
359
+ or order.dtype != torch.int64
360
+ or order.ndim != 1
361
+ or type(position) is not int
362
+ ):
363
+ raise TaskError("ordinary shuffle checkpoint mismatch")
364
+ if (not len(order) and position != 0) or (
365
+ len(order)
366
+ and (
367
+ len(order) != self._population
368
+ or not 0 <= position <= len(order)
369
+ or not torch.equal(
370
+ order.sort().values, torch.arange(self._population)
371
+ )
372
+ )
373
+ ):
374
+ raise TaskError("ordinary shuffle permutation mismatch")
375
+ elif (
376
+ state["shuffle_order"] is not None or state["shuffle_position"] is not None
377
+ ):
378
+ raise TaskError("ordinary sampler checkpoint mismatch")
379
+ self._optimizer.load_state_dict(state["optimizer"])
380
+ self._sampling_rng.set_state(state["sampling_rng"])
381
+ self._model_rng, self._cuda_rng = state["model_rng"], state["cuda_rng"]
382
+ if self._shuffle is not None:
383
+ self._shuffle.order = cast(torch.Tensor, state["shuffle_order"])
384
+ self._shuffle.position = cast(int, state["shuffle_position"])
385
+ self._logical_steps, self._physical_steps = steps, physical
386
+
387
+ def load_checkpoint_tensors(self, state: dict[str, np.ndarray]) -> None:
388
+ self._check()
389
+ arrays = {
390
+ k.removeprefix("application."): v
391
+ for k, v in state.items()
392
+ if k.startswith("application.")
393
+ }
394
+ decoded = decode_state(arrays)
395
+ if "meta.steps" not in state or int(state["meta.steps"][0]) != decoded.get(
396
+ "logical_steps"
397
+ ):
398
+ raise TaskError("ordinary checkpoint counters disagree")
399
+ require_unchanged_public_state(
400
+ self._raw_model,
401
+ state,
402
+ {
403
+ **self.specification.public_buffer_hashes,
404
+ **self.specification.frozen_parameter_hashes,
405
+ },
406
+ )
407
+ super().load_checkpoint_tensors(state)
408
+ self._guard.check()
@@ -0,0 +1,96 @@
1
+ """Lazy optional Opacus boundary and mechanism accounting."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib
6
+ import math
7
+ from collections import Counter
8
+ from collections.abc import Callable
9
+ from typing import Any, Protocol
10
+
11
+ from dromeus.errors import PrivacyError
12
+ from dromeus.manifests.models import PrivacyPolicy, PrivacyReservation
13
+
14
+
15
+ class ReservationStore(Protocol):
16
+ def reserve(
17
+ self,
18
+ reservation: PrivacyReservation,
19
+ *,
20
+ epsilon_limit: float | None,
21
+ forecast: Callable[[tuple[PrivacyReservation, ...]], float],
22
+ ) -> None: ...
23
+
24
+ def check(self, reservation: PrivacyReservation) -> None: ...
25
+
26
+
27
+ def opacus_module(name: str = "opacus") -> Any:
28
+ """Keep untyped upstream API confined to the optional implementation."""
29
+ try:
30
+ module = importlib.import_module(name)
31
+ if importlib.import_module("opacus").__version__ != "1.6.0":
32
+ raise PrivacyError("private training requires Opacus 1.6.0")
33
+ return module
34
+ except ImportError:
35
+ raise PrivacyError(
36
+ "install dromeus[privacy] and secure RNG when required"
37
+ ) from None
38
+
39
+
40
+ def privacy_epsilon(
41
+ history: tuple[tuple[float, float, int], ...],
42
+ policy: PrivacyPolicy,
43
+ *,
44
+ accountant: str = "prv",
45
+ ) -> float:
46
+ if not history:
47
+ return 0.0
48
+ accountants = opacus_module("opacus.accountants")
49
+ engine = (
50
+ accountants.PRVAccountant()
51
+ if accountant == "prv"
52
+ else accountants.RDPAccountant()
53
+ )
54
+ query_history = list(history)
55
+ # Do not normalize unsupported counts into a different valid mechanism.
56
+ if all(steps > 0 for _, _, steps in history):
57
+ # Identical fixed Poisson-Gaussian mechanisms share one privacy-loss
58
+ # distribution. Compose their total multiplicity once in the query copy;
59
+ # live accountant histories and durable reservations retain their order.
60
+ counts: Counter[tuple[float, float]] = Counter()
61
+ for noise, sample_rate, steps in history:
62
+ counts[noise, sample_rate] += steps
63
+ query_history = [
64
+ (noise, sample_rate, steps)
65
+ for (noise, sample_rate), steps in counts.items()
66
+ ]
67
+ engine.history = query_history
68
+ kwargs = (
69
+ {
70
+ "eps_error": policy.prv_epsilon_error,
71
+ "delta_error": policy.delta * policy.prv_delta_error_ratio,
72
+ }
73
+ if accountant == "prv"
74
+ else {}
75
+ )
76
+ value = float(engine.get_epsilon(delta=policy.delta, **kwargs))
77
+ if not math.isfinite(value) or value < 0:
78
+ raise PrivacyError("privacy accounting failed")
79
+ return value
80
+
81
+
82
+ def reserve_horizon(
83
+ store: ReservationStore,
84
+ reservation: PrivacyReservation,
85
+ policy: PrivacyPolicy,
86
+ ) -> None:
87
+ store.reserve(
88
+ reservation,
89
+ epsilon_limit=policy.epsilon_limit,
90
+ forecast=lambda records: privacy_epsilon(
91
+ tuple(
92
+ (r.noise_multiplier, r.sample_rate, r.logical_steps) for r in records
93
+ ),
94
+ policy,
95
+ ),
96
+ )
@@ -0,0 +1,96 @@
1
+ """Owned Poisson sampling for finite application datasets without eager reads."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Sized
7
+ from typing import Any, cast
8
+
9
+ import torch
10
+ from torch.utils.data import DataLoader, Dataset, IterableDataset, Sampler
11
+
12
+ from dromeus.errors import PrivacyError
13
+ from dromeus.training.privacy import opacus_module
14
+ from dromeus.training.task import TensorBatchAdapter
15
+
16
+
17
+ def dataset_size(dataset: Dataset[Any]) -> int:
18
+ """Only metadata is consulted; never read a record during preparation."""
19
+ if isinstance(dataset, IterableDataset) or not isinstance(dataset, Sized):
20
+ raise PrivacyError("private task requires a finite indexed dataset")
21
+ count = len(dataset)
22
+ if count <= 0:
23
+ raise PrivacyError("private task requires a nonempty fixed dataset")
24
+ return count
25
+
26
+
27
+ class _FixedPopulation(Dataset[Any]):
28
+ def __init__(self, dataset: Dataset[Any], count: int) -> None:
29
+ self.source = dataset
30
+ self.count = count
31
+
32
+ def __len__(self) -> int:
33
+ if dataset_size(self.source) != self.count:
34
+ raise PrivacyError("private dataset population changed")
35
+ return self.count
36
+
37
+ def __getitem__(self, index: int) -> Any:
38
+ if not 0 <= index < len(self):
39
+ raise IndexError("private record index out of range")
40
+ return self.source[index]
41
+
42
+
43
+ class TaskDataLoader(DataLoader[Any]):
44
+ """An owned sampler and collator; no user-provided sampling distribution."""
45
+
46
+ sample_rate: float
47
+ population_size: int
48
+
49
+ def __init__(
50
+ self,
51
+ dataset: Dataset[Any],
52
+ adapter: TensorBatchAdapter,
53
+ *,
54
+ batch_size: int,
55
+ generator: torch.Generator | None,
56
+ ) -> None:
57
+ if type(batch_size) is not int or batch_size <= 0:
58
+ raise PrivacyError("logical batch size must be a positive integer")
59
+ self.population_size = dataset_size(dataset)
60
+ steps = math.ceil(self.population_size / batch_size)
61
+ self.sample_rate = 1 / steps
62
+ sampler_type = opacus_module(
63
+ "opacus.utils.uniform_sampler"
64
+ ).UniformWithReplacementSampler
65
+ sampler = sampler_type(
66
+ num_samples=self.population_size,
67
+ sample_rate=self.sample_rate,
68
+ generator=generator,
69
+ steps=steps,
70
+ )
71
+ super().__init__( # pyright: ignore[reportUnknownMemberType]
72
+ _FixedPopulation(dataset, self.population_size),
73
+ batch_sampler=cast(Sampler[list[int]], sampler),
74
+ collate_fn=adapter.collate,
75
+ generator=generator,
76
+ num_workers=0,
77
+ )
78
+
79
+ def check_population(self) -> None:
80
+ if len(cast(Sized, self.dataset)) != self.population_size:
81
+ raise PrivacyError("private dataset population changed")
82
+
83
+ def __iter__(self) -> Any:
84
+ self.check_population()
85
+ return super().__iter__()
86
+
87
+
88
+ def make_task_loader(
89
+ dataset: Dataset[Any],
90
+ adapter: TensorBatchAdapter,
91
+ *,
92
+ batch_size: int,
93
+ generator: torch.Generator | None,
94
+ ) -> TaskDataLoader:
95
+ """Use with PrivacyEngine(poisson_sampling=False): sampling is already owned."""
96
+ return TaskDataLoader(dataset, adapter, batch_size=batch_size, generator=generator)