dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""Internal workload adapter for private application tasks.
|
|
2
|
+
|
|
3
|
+
The workload prepares gradients and enforces the model/task contract. The trainer
|
|
4
|
+
owns Opacus, optimizer invocation, accounting, reservations and logical steps.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable, Mapping
|
|
10
|
+
from typing import Any, Protocol, cast
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import torch
|
|
14
|
+
from torch import nn
|
|
15
|
+
from torch.utils.data import DataLoader, Dataset
|
|
16
|
+
|
|
17
|
+
from dromeus.errors import PrivacyError
|
|
18
|
+
from dromeus.manifests.models import (
|
|
19
|
+
TaskTrainingSpec,
|
|
20
|
+
)
|
|
21
|
+
from dromeus.training.optimizer_factory import OptimizerRegistration
|
|
22
|
+
from dromeus.training.private_data import make_task_loader
|
|
23
|
+
from dromeus.training.private_model import (
|
|
24
|
+
ModelStateGuard,
|
|
25
|
+
ensure_grad_samples,
|
|
26
|
+
registered_model_state,
|
|
27
|
+
require_unchanged_public_state,
|
|
28
|
+
validate_private_model,
|
|
29
|
+
)
|
|
30
|
+
from dromeus.training.task import TrainingTask, validate_record_losses
|
|
31
|
+
|
|
32
|
+
type PublicEvaluation = Callable[[nn.Module], Mapping[str, float]]
|
|
33
|
+
type WeightSnapshot = Callable[[], dict[str, np.ndarray]]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class PrivateWorkloadStep(Protocol):
|
|
37
|
+
"""Workload checks scoped to one draw and all its physical batches."""
|
|
38
|
+
|
|
39
|
+
def prepare_gradients(self, private_model: nn.Module, batch: Any) -> None: ...
|
|
40
|
+
|
|
41
|
+
def after_optimizer_step(self) -> None: ...
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class PrivateWorkload(Protocol):
|
|
45
|
+
model: nn.Module
|
|
46
|
+
evaluate: PublicEvaluation | None
|
|
47
|
+
optimizer_registration: OptimizerRegistration | None
|
|
48
|
+
exchange_names: tuple[str, ...] | None
|
|
49
|
+
poisson_sampling: bool
|
|
50
|
+
|
|
51
|
+
def make_loader(self, generator: torch.Generator | None) -> DataLoader[Any]: ...
|
|
52
|
+
|
|
53
|
+
def configure_private_model(self, private_model: Any) -> None: ...
|
|
54
|
+
|
|
55
|
+
def check(self) -> None: ...
|
|
56
|
+
|
|
57
|
+
def begin_step(self) -> PrivateWorkloadStep: ...
|
|
58
|
+
|
|
59
|
+
def export_tensors(self, weights: WeightSnapshot) -> dict[str, np.ndarray]: ...
|
|
60
|
+
|
|
61
|
+
def validate_checkpoint(self, state: dict[str, np.ndarray]) -> None: ...
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def prepare_private_workload(
|
|
65
|
+
*, specification: TaskTrainingSpec, task: TrainingTask, dataset: Dataset[Any]
|
|
66
|
+
) -> PrivateWorkload:
|
|
67
|
+
"""Bind one application task to the group's private training specification."""
|
|
68
|
+
if specification.sampling != "poisson":
|
|
69
|
+
raise PrivacyError("private tasks require Poisson sampling")
|
|
70
|
+
task.check_definition(specification)
|
|
71
|
+
if (
|
|
72
|
+
task.public_evaluate is not None
|
|
73
|
+
and specification.evaluation_provenance != "public_fixed_holdout"
|
|
74
|
+
):
|
|
75
|
+
raise PrivacyError("evaluation requires public fixed holdout provenance")
|
|
76
|
+
return _ApplicationWorkload(task, specification, dataset)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class _ApplicationWorkload:
|
|
80
|
+
"""Generic task semantics and state checks, without a tensor-task fallback."""
|
|
81
|
+
|
|
82
|
+
poisson_sampling = False # TaskDataLoader already owns its Poisson sampler.
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
task: TrainingTask,
|
|
87
|
+
specification: TaskTrainingSpec,
|
|
88
|
+
dataset: Dataset[Any],
|
|
89
|
+
) -> None:
|
|
90
|
+
self._task, self._specification = task, specification
|
|
91
|
+
self._dataset = dataset
|
|
92
|
+
self._native_private_model: Any = None
|
|
93
|
+
self._native_sampler_token: Any = None
|
|
94
|
+
self.evaluate = task.public_evaluate
|
|
95
|
+
self.optimizer_registration = task.optimizer_registration
|
|
96
|
+
self.exchange_names: tuple[str, ...] | None = (
|
|
97
|
+
specification.trainable_parameter_names
|
|
98
|
+
)
|
|
99
|
+
self.model = task.model_factory()
|
|
100
|
+
report = validate_private_model(
|
|
101
|
+
self.model,
|
|
102
|
+
task.loss_per_record,
|
|
103
|
+
task.batch_adapter,
|
|
104
|
+
task.batch_adapter.example,
|
|
105
|
+
gradient_backend=specification.gradient_backend,
|
|
106
|
+
)
|
|
107
|
+
for name in (
|
|
108
|
+
"trainable_parameter_names",
|
|
109
|
+
"parameter_aliases",
|
|
110
|
+
"public_buffer_hashes",
|
|
111
|
+
"frozen_parameter_hashes",
|
|
112
|
+
"gradient_backend",
|
|
113
|
+
):
|
|
114
|
+
if getattr(report, name) != getattr(specification, name):
|
|
115
|
+
raise PrivacyError(
|
|
116
|
+
f"model {name} differs from shared task specification"
|
|
117
|
+
)
|
|
118
|
+
self._guard = ModelStateGuard(self.model, report)
|
|
119
|
+
|
|
120
|
+
def make_loader(self, generator: torch.Generator | None) -> DataLoader[Any]:
|
|
121
|
+
return make_task_loader(
|
|
122
|
+
self._dataset,
|
|
123
|
+
self._task.batch_adapter,
|
|
124
|
+
batch_size=self._specification.batch_size,
|
|
125
|
+
generator=generator,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
def configure_private_model(self, private_model: Any) -> None:
|
|
129
|
+
private_model.forbid_grad_accumulation()
|
|
130
|
+
if self._specification.gradient_backend == "opacus_native_b1":
|
|
131
|
+
from dromeus.training.native_gradients import install_native_b1_samplers
|
|
132
|
+
|
|
133
|
+
self._native_sampler_token = install_native_b1_samplers(private_model)
|
|
134
|
+
self._native_private_model = private_model
|
|
135
|
+
|
|
136
|
+
def check(self) -> None:
|
|
137
|
+
self._guard.check()
|
|
138
|
+
self._task.check_definition(self._specification)
|
|
139
|
+
if self._native_private_model is not None:
|
|
140
|
+
from dromeus.training.native_gradients import check_native_b1_samplers
|
|
141
|
+
|
|
142
|
+
check_native_b1_samplers(
|
|
143
|
+
self._native_private_model, self._native_sampler_token
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def begin_step(self) -> PrivateWorkloadStep:
|
|
147
|
+
return _ApplicationStep(self.model, self._task, self._guard)
|
|
148
|
+
|
|
149
|
+
def export_tensors(self, weights: WeightSnapshot) -> dict[str, np.ndarray]:
|
|
150
|
+
return {
|
|
151
|
+
name: value.detach().cpu().numpy().copy()
|
|
152
|
+
for name, value in registered_model_state(self.model).items()
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
def validate_checkpoint(self, state: dict[str, np.ndarray]) -> None:
|
|
156
|
+
require_unchanged_public_state(
|
|
157
|
+
self.model,
|
|
158
|
+
state,
|
|
159
|
+
{
|
|
160
|
+
**self._specification.public_buffer_hashes,
|
|
161
|
+
**self._specification.frozen_parameter_hashes,
|
|
162
|
+
},
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class _ApplicationStep:
|
|
167
|
+
"""Capture guards before sampling and release them with the logical call."""
|
|
168
|
+
|
|
169
|
+
def __init__(
|
|
170
|
+
self, model: nn.Module, task: TrainingTask, guard: ModelStateGuard
|
|
171
|
+
) -> None:
|
|
172
|
+
self._model, self._task, self._guard = model, task, guard
|
|
173
|
+
self._device = next(model.parameters()).device
|
|
174
|
+
self._cpu_rng = torch.get_rng_state()
|
|
175
|
+
self._cuda_rng = (
|
|
176
|
+
torch.cuda.get_rng_state(self._device)
|
|
177
|
+
if self._device.type == "cuda"
|
|
178
|
+
else None
|
|
179
|
+
)
|
|
180
|
+
self._expected_parameters = self._parameters()
|
|
181
|
+
|
|
182
|
+
def _parameters(self) -> dict[str, torch.Tensor]:
|
|
183
|
+
return {
|
|
184
|
+
name: p.detach().clone()
|
|
185
|
+
for name, p in self._model.named_parameters()
|
|
186
|
+
if p.requires_grad
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
def _parameters_changed(self) -> bool:
|
|
190
|
+
return any(
|
|
191
|
+
not torch.equal(p.detach(), self._expected_parameters[name])
|
|
192
|
+
for name, p in self._model.named_parameters()
|
|
193
|
+
if p.requires_grad
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
def prepare_gradients(self, private_model: nn.Module, batch: Any) -> None:
|
|
197
|
+
if self._parameters_changed():
|
|
198
|
+
raise PrivacyError("task data callback mutated model parameters")
|
|
199
|
+
device = next(self._model.parameters()).device
|
|
200
|
+
batch = self._task.batch_adapter.to_device(batch, device)
|
|
201
|
+
count = self._task.batch_adapter.size(batch)
|
|
202
|
+
if count:
|
|
203
|
+
losses = self._task.loss_per_record(private_model, batch)
|
|
204
|
+
validate_record_losses(losses, count)
|
|
205
|
+
cast(Any, losses.mean()).backward()
|
|
206
|
+
if self._parameters_changed():
|
|
207
|
+
raise PrivacyError("task forward/backward mutated model parameters")
|
|
208
|
+
ensure_grad_samples(self._model, count)
|
|
209
|
+
self._guard.check()
|
|
210
|
+
|
|
211
|
+
def after_optimizer_step(self) -> None:
|
|
212
|
+
self._expected_parameters = self._parameters()
|
|
213
|
+
if not torch.equal(self._cpu_rng, torch.get_rng_state()) or (
|
|
214
|
+
self._cuda_rng is not None
|
|
215
|
+
and not torch.equal(self._cuda_rng, torch.cuda.get_rng_state(self._device))
|
|
216
|
+
):
|
|
217
|
+
raise PrivacyError("task data/model callbacks consumed undeclared RNG")
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Initial model-state checkpoint helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import cast
|
|
9
|
+
|
|
10
|
+
from safetensors.torch import (
|
|
11
|
+
save_file as _save_file, # pyright: ignore[reportUnknownVariableType]
|
|
12
|
+
)
|
|
13
|
+
from torch import nn
|
|
14
|
+
|
|
15
|
+
from dromeus.manifests.canonical import file_sha256
|
|
16
|
+
from dromeus.manifests.models import TensorSchema
|
|
17
|
+
from dromeus.training.model_state import floating_model_state, tensor_schema_for_model
|
|
18
|
+
|
|
19
|
+
_save_checkpoint = cast(Callable[..., None], _save_file)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True, slots=True)
|
|
23
|
+
class InitialCheckpoint:
|
|
24
|
+
"""Canonical checkpoint handoff data for initiator formation."""
|
|
25
|
+
|
|
26
|
+
path: Path
|
|
27
|
+
tensor_schema: TensorSchema
|
|
28
|
+
sha256: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def create_initial_checkpoint(
|
|
32
|
+
path: Path,
|
|
33
|
+
*,
|
|
34
|
+
model: nn.Module,
|
|
35
|
+
model_definition: str,
|
|
36
|
+
exchange_names: tuple[str, ...] | None = None,
|
|
37
|
+
) -> InitialCheckpoint:
|
|
38
|
+
"""Write and describe the canonical checkpoint before formation."""
|
|
39
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
_save_checkpoint(
|
|
41
|
+
{
|
|
42
|
+
name: value.detach().cpu().contiguous().clone()
|
|
43
|
+
for name, value in floating_model_state(model, exchange_names).items()
|
|
44
|
+
},
|
|
45
|
+
str(path),
|
|
46
|
+
metadata={"model_definition": model_definition},
|
|
47
|
+
)
|
|
48
|
+
return InitialCheckpoint(
|
|
49
|
+
path=path,
|
|
50
|
+
tensor_schema=tensor_schema_for_model(model, exchange_names),
|
|
51
|
+
sha256=checkpoint_hash(path),
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def checkpoint_hash(path: Path) -> str:
|
|
56
|
+
"""Return a checkpoint's SHA-256 digest."""
|
|
57
|
+
return file_sha256(path)
|