dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,217 @@
1
+ """Internal workload adapter for private application tasks.
2
+
3
+ The workload prepares gradients and enforces the model/task contract. The trainer
4
+ owns Opacus, optimizer invocation, accounting, reservations and logical steps.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Callable, Mapping
10
+ from typing import Any, Protocol, cast
11
+
12
+ import numpy as np
13
+ import torch
14
+ from torch import nn
15
+ from torch.utils.data import DataLoader, Dataset
16
+
17
+ from dromeus.errors import PrivacyError
18
+ from dromeus.manifests.models import (
19
+ TaskTrainingSpec,
20
+ )
21
+ from dromeus.training.optimizer_factory import OptimizerRegistration
22
+ from dromeus.training.private_data import make_task_loader
23
+ from dromeus.training.private_model import (
24
+ ModelStateGuard,
25
+ ensure_grad_samples,
26
+ registered_model_state,
27
+ require_unchanged_public_state,
28
+ validate_private_model,
29
+ )
30
+ from dromeus.training.task import TrainingTask, validate_record_losses
31
+
32
+ type PublicEvaluation = Callable[[nn.Module], Mapping[str, float]]
33
+ type WeightSnapshot = Callable[[], dict[str, np.ndarray]]
34
+
35
+
36
+ class PrivateWorkloadStep(Protocol):
37
+ """Workload checks scoped to one draw and all its physical batches."""
38
+
39
+ def prepare_gradients(self, private_model: nn.Module, batch: Any) -> None: ...
40
+
41
+ def after_optimizer_step(self) -> None: ...
42
+
43
+
44
+ class PrivateWorkload(Protocol):
45
+ model: nn.Module
46
+ evaluate: PublicEvaluation | None
47
+ optimizer_registration: OptimizerRegistration | None
48
+ exchange_names: tuple[str, ...] | None
49
+ poisson_sampling: bool
50
+
51
+ def make_loader(self, generator: torch.Generator | None) -> DataLoader[Any]: ...
52
+
53
+ def configure_private_model(self, private_model: Any) -> None: ...
54
+
55
+ def check(self) -> None: ...
56
+
57
+ def begin_step(self) -> PrivateWorkloadStep: ...
58
+
59
+ def export_tensors(self, weights: WeightSnapshot) -> dict[str, np.ndarray]: ...
60
+
61
+ def validate_checkpoint(self, state: dict[str, np.ndarray]) -> None: ...
62
+
63
+
64
+ def prepare_private_workload(
65
+ *, specification: TaskTrainingSpec, task: TrainingTask, dataset: Dataset[Any]
66
+ ) -> PrivateWorkload:
67
+ """Bind one application task to the group's private training specification."""
68
+ if specification.sampling != "poisson":
69
+ raise PrivacyError("private tasks require Poisson sampling")
70
+ task.check_definition(specification)
71
+ if (
72
+ task.public_evaluate is not None
73
+ and specification.evaluation_provenance != "public_fixed_holdout"
74
+ ):
75
+ raise PrivacyError("evaluation requires public fixed holdout provenance")
76
+ return _ApplicationWorkload(task, specification, dataset)
77
+
78
+
79
+ class _ApplicationWorkload:
80
+ """Generic task semantics and state checks, without a tensor-task fallback."""
81
+
82
+ poisson_sampling = False # TaskDataLoader already owns its Poisson sampler.
83
+
84
+ def __init__(
85
+ self,
86
+ task: TrainingTask,
87
+ specification: TaskTrainingSpec,
88
+ dataset: Dataset[Any],
89
+ ) -> None:
90
+ self._task, self._specification = task, specification
91
+ self._dataset = dataset
92
+ self._native_private_model: Any = None
93
+ self._native_sampler_token: Any = None
94
+ self.evaluate = task.public_evaluate
95
+ self.optimizer_registration = task.optimizer_registration
96
+ self.exchange_names: tuple[str, ...] | None = (
97
+ specification.trainable_parameter_names
98
+ )
99
+ self.model = task.model_factory()
100
+ report = validate_private_model(
101
+ self.model,
102
+ task.loss_per_record,
103
+ task.batch_adapter,
104
+ task.batch_adapter.example,
105
+ gradient_backend=specification.gradient_backend,
106
+ )
107
+ for name in (
108
+ "trainable_parameter_names",
109
+ "parameter_aliases",
110
+ "public_buffer_hashes",
111
+ "frozen_parameter_hashes",
112
+ "gradient_backend",
113
+ ):
114
+ if getattr(report, name) != getattr(specification, name):
115
+ raise PrivacyError(
116
+ f"model {name} differs from shared task specification"
117
+ )
118
+ self._guard = ModelStateGuard(self.model, report)
119
+
120
+ def make_loader(self, generator: torch.Generator | None) -> DataLoader[Any]:
121
+ return make_task_loader(
122
+ self._dataset,
123
+ self._task.batch_adapter,
124
+ batch_size=self._specification.batch_size,
125
+ generator=generator,
126
+ )
127
+
128
+ def configure_private_model(self, private_model: Any) -> None:
129
+ private_model.forbid_grad_accumulation()
130
+ if self._specification.gradient_backend == "opacus_native_b1":
131
+ from dromeus.training.native_gradients import install_native_b1_samplers
132
+
133
+ self._native_sampler_token = install_native_b1_samplers(private_model)
134
+ self._native_private_model = private_model
135
+
136
+ def check(self) -> None:
137
+ self._guard.check()
138
+ self._task.check_definition(self._specification)
139
+ if self._native_private_model is not None:
140
+ from dromeus.training.native_gradients import check_native_b1_samplers
141
+
142
+ check_native_b1_samplers(
143
+ self._native_private_model, self._native_sampler_token
144
+ )
145
+
146
+ def begin_step(self) -> PrivateWorkloadStep:
147
+ return _ApplicationStep(self.model, self._task, self._guard)
148
+
149
+ def export_tensors(self, weights: WeightSnapshot) -> dict[str, np.ndarray]:
150
+ return {
151
+ name: value.detach().cpu().numpy().copy()
152
+ for name, value in registered_model_state(self.model).items()
153
+ }
154
+
155
+ def validate_checkpoint(self, state: dict[str, np.ndarray]) -> None:
156
+ require_unchanged_public_state(
157
+ self.model,
158
+ state,
159
+ {
160
+ **self._specification.public_buffer_hashes,
161
+ **self._specification.frozen_parameter_hashes,
162
+ },
163
+ )
164
+
165
+
166
+ class _ApplicationStep:
167
+ """Capture guards before sampling and release them with the logical call."""
168
+
169
+ def __init__(
170
+ self, model: nn.Module, task: TrainingTask, guard: ModelStateGuard
171
+ ) -> None:
172
+ self._model, self._task, self._guard = model, task, guard
173
+ self._device = next(model.parameters()).device
174
+ self._cpu_rng = torch.get_rng_state()
175
+ self._cuda_rng = (
176
+ torch.cuda.get_rng_state(self._device)
177
+ if self._device.type == "cuda"
178
+ else None
179
+ )
180
+ self._expected_parameters = self._parameters()
181
+
182
+ def _parameters(self) -> dict[str, torch.Tensor]:
183
+ return {
184
+ name: p.detach().clone()
185
+ for name, p in self._model.named_parameters()
186
+ if p.requires_grad
187
+ }
188
+
189
+ def _parameters_changed(self) -> bool:
190
+ return any(
191
+ not torch.equal(p.detach(), self._expected_parameters[name])
192
+ for name, p in self._model.named_parameters()
193
+ if p.requires_grad
194
+ )
195
+
196
+ def prepare_gradients(self, private_model: nn.Module, batch: Any) -> None:
197
+ if self._parameters_changed():
198
+ raise PrivacyError("task data callback mutated model parameters")
199
+ device = next(self._model.parameters()).device
200
+ batch = self._task.batch_adapter.to_device(batch, device)
201
+ count = self._task.batch_adapter.size(batch)
202
+ if count:
203
+ losses = self._task.loss_per_record(private_model, batch)
204
+ validate_record_losses(losses, count)
205
+ cast(Any, losses.mean()).backward()
206
+ if self._parameters_changed():
207
+ raise PrivacyError("task forward/backward mutated model parameters")
208
+ ensure_grad_samples(self._model, count)
209
+ self._guard.check()
210
+
211
+ def after_optimizer_step(self) -> None:
212
+ self._expected_parameters = self._parameters()
213
+ if not torch.equal(self._cpu_rng, torch.get_rng_state()) or (
214
+ self._cuda_rng is not None
215
+ and not torch.equal(self._cuda_rng, torch.cuda.get_rng_state(self._device))
216
+ ):
217
+ raise PrivacyError("task data/model callbacks consumed undeclared RNG")
@@ -0,0 +1,57 @@
1
+ """Initial model-state checkpoint helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import cast
9
+
10
+ from safetensors.torch import (
11
+ save_file as _save_file, # pyright: ignore[reportUnknownVariableType]
12
+ )
13
+ from torch import nn
14
+
15
+ from dromeus.manifests.canonical import file_sha256
16
+ from dromeus.manifests.models import TensorSchema
17
+ from dromeus.training.model_state import floating_model_state, tensor_schema_for_model
18
+
19
+ _save_checkpoint = cast(Callable[..., None], _save_file)
20
+
21
+
22
+ @dataclass(frozen=True, slots=True)
23
+ class InitialCheckpoint:
24
+ """Canonical checkpoint handoff data for initiator formation."""
25
+
26
+ path: Path
27
+ tensor_schema: TensorSchema
28
+ sha256: str
29
+
30
+
31
+ def create_initial_checkpoint(
32
+ path: Path,
33
+ *,
34
+ model: nn.Module,
35
+ model_definition: str,
36
+ exchange_names: tuple[str, ...] | None = None,
37
+ ) -> InitialCheckpoint:
38
+ """Write and describe the canonical checkpoint before formation."""
39
+ path.parent.mkdir(parents=True, exist_ok=True)
40
+ _save_checkpoint(
41
+ {
42
+ name: value.detach().cpu().contiguous().clone()
43
+ for name, value in floating_model_state(model, exchange_names).items()
44
+ },
45
+ str(path),
46
+ metadata={"model_definition": model_definition},
47
+ )
48
+ return InitialCheckpoint(
49
+ path=path,
50
+ tensor_schema=tensor_schema_for_model(model, exchange_names),
51
+ sha256=checkpoint_hash(path),
52
+ )
53
+
54
+
55
+ def checkpoint_hash(path: Path) -> str:
56
+ """Return a checkpoint's SHA-256 digest."""
57
+ return file_sha256(path)