dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""Opt-in native FP32 per-record reductions for a public physical cap of one.
|
|
2
|
+
|
|
3
|
+
Selection belongs to the caller's frozen public configuration. This module does
|
|
4
|
+
not select a backend from observed record counts or alter coverage/DP guards.
|
|
5
|
+
Optional runtime dependencies load only when the backend is explicitly used.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib
|
|
11
|
+
import math
|
|
12
|
+
import weakref
|
|
13
|
+
from collections.abc import Callable, Mapping
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from types import MappingProxyType
|
|
16
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
17
|
+
|
|
18
|
+
from dromeus.errors import TaskError
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from torch import nn
|
|
22
|
+
|
|
23
|
+
NATIVE_B1_BACKEND = "opacus_native_b1"
|
|
24
|
+
Sampler = Callable[[Any, list[Any], Any], dict[Any, Any]]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _require(value: bool, message: str) -> None:
|
|
28
|
+
if not value:
|
|
29
|
+
raise TaskError(message)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _runtime() -> tuple[Any, Any]:
|
|
33
|
+
try:
|
|
34
|
+
torch = importlib.import_module("torch")
|
|
35
|
+
except ImportError:
|
|
36
|
+
raise TaskError("native B1 requires pinned Torch 2.13.0") from None
|
|
37
|
+
_require(
|
|
38
|
+
torch.__version__ in {"2.13.0", "2.13.0+cpu", "2.13.0+cu130"},
|
|
39
|
+
"native B1 requires Torch 2.13.0 CPU or cu130",
|
|
40
|
+
)
|
|
41
|
+
from dromeus.training.privacy import opacus_module
|
|
42
|
+
|
|
43
|
+
base = opacus_module("opacus.grad_sample").GradSampleModule
|
|
44
|
+
return torch, base
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _leaf_signature(module: Any, torch: Any) -> tuple[Any, ...]:
|
|
48
|
+
nn = torch.nn
|
|
49
|
+
kind = cast(type[Any], type(module))
|
|
50
|
+
_require(
|
|
51
|
+
kind in (nn.Linear, nn.LayerNorm, nn.Embedding),
|
|
52
|
+
"native B1 supports exact Linear/LayerNorm/Embedding leaves only",
|
|
53
|
+
)
|
|
54
|
+
expected: Any = getattr(kind, "forward")
|
|
55
|
+
namespace = {
|
|
56
|
+
nn.Linear: "torch.nn.modules.linear",
|
|
57
|
+
nn.LayerNorm: "torch.nn.modules.normalization",
|
|
58
|
+
nn.Embedding: "torch.nn.modules.sparse",
|
|
59
|
+
}[kind]
|
|
60
|
+
_require(
|
|
61
|
+
"forward" not in module.__dict__
|
|
62
|
+
and getattr(module.forward, "__func__", None) is expected
|
|
63
|
+
and expected.__module__ == namespace
|
|
64
|
+
and expected.__qualname__ == kind.__name__ + ".forward"
|
|
65
|
+
and getattr(module, "_compiled_call_impl", None) is None,
|
|
66
|
+
"native B1 forbids foreign leaf forward overrides",
|
|
67
|
+
)
|
|
68
|
+
parameters = tuple(module.named_parameters(recurse=False))
|
|
69
|
+
_require(
|
|
70
|
+
bool(parameters)
|
|
71
|
+
and all(p.dtype == torch.float32 for _, p in parameters)
|
|
72
|
+
and not tuple(module.buffers(recurse=False)),
|
|
73
|
+
"native B1 requires FP32 leaves without trainable-layer buffers",
|
|
74
|
+
)
|
|
75
|
+
if kind is nn.Linear:
|
|
76
|
+
options = (module.in_features, module.out_features, module.bias is not None)
|
|
77
|
+
elif kind is nn.LayerNorm:
|
|
78
|
+
_require(
|
|
79
|
+
math.isfinite(module.eps) and module.eps > 0,
|
|
80
|
+
"native B1 LayerNorm epsilon must be finite and positive",
|
|
81
|
+
)
|
|
82
|
+
options = (
|
|
83
|
+
tuple(module.normalized_shape),
|
|
84
|
+
module.eps,
|
|
85
|
+
module.elementwise_affine,
|
|
86
|
+
module.bias is not None,
|
|
87
|
+
)
|
|
88
|
+
else:
|
|
89
|
+
_require(
|
|
90
|
+
module.max_norm is None
|
|
91
|
+
and not module.sparse
|
|
92
|
+
and not module.scale_grad_by_freq,
|
|
93
|
+
"native B1 forbids mutating/sparse/frequency Embedding options",
|
|
94
|
+
)
|
|
95
|
+
options = (
|
|
96
|
+
module.num_embeddings,
|
|
97
|
+
module.embedding_dim,
|
|
98
|
+
module.padding_idx,
|
|
99
|
+
module.norm_type,
|
|
100
|
+
)
|
|
101
|
+
return (
|
|
102
|
+
kind,
|
|
103
|
+
id(module),
|
|
104
|
+
expected,
|
|
105
|
+
options,
|
|
106
|
+
tuple(
|
|
107
|
+
(n, id(p), tuple(p.shape), p.dtype, p.device, p.requires_grad)
|
|
108
|
+
for n, p in parameters
|
|
109
|
+
),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _model_signature(model: Any, torch: Any) -> tuple[Any, ...]:
|
|
114
|
+
_require(isinstance(model, torch.nn.Module), "native B1 requires a model module")
|
|
115
|
+
leaves: list[tuple[Any, ...]] = []
|
|
116
|
+
for path, module in model.named_modules():
|
|
117
|
+
if any(p.requires_grad for p in module.parameters(recurse=False)):
|
|
118
|
+
leaves.append((path, _leaf_signature(module, torch)))
|
|
119
|
+
_require(bool(leaves), "native B1 requires trainable supported leaves")
|
|
120
|
+
return tuple(leaves)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def validate_native_b1_model(raw_model: nn.Module) -> None:
|
|
124
|
+
"""Inspect native leaves without forward execution or record/weight copies."""
|
|
125
|
+
torch, _ = _runtime()
|
|
126
|
+
_model_signature(raw_model, torch)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _native_b1_sampler(
|
|
130
|
+
layer: Any, activations: list[Any], backprops: Any
|
|
131
|
+
) -> dict[Any, Any]:
|
|
132
|
+
torch, _ = _runtime()
|
|
133
|
+
_leaf_signature(layer, torch)
|
|
134
|
+
_require(
|
|
135
|
+
len(activations) == 1
|
|
136
|
+
and isinstance(backprops, torch.Tensor)
|
|
137
|
+
and backprops.ndim >= 1
|
|
138
|
+
and backprops.shape[0] == 1
|
|
139
|
+
and backprops.dtype == torch.float32,
|
|
140
|
+
"native B1 sampler requires exactly one nonempty physical record",
|
|
141
|
+
)
|
|
142
|
+
x = activations[0]
|
|
143
|
+
_require(
|
|
144
|
+
isinstance(x, torch.Tensor)
|
|
145
|
+
and x.ndim >= 1
|
|
146
|
+
and x.shape[0] == 1
|
|
147
|
+
and x.device == backprops.device,
|
|
148
|
+
"native B1 activation batch/device differs",
|
|
149
|
+
)
|
|
150
|
+
x = x.detach()
|
|
151
|
+
if type(layer) is torch.nn.Linear:
|
|
152
|
+
_require(x.dtype == torch.float32, "native B1 Linear activation must be FP32")
|
|
153
|
+
dy = backprops.detach()[0].reshape(-1, layer.out_features)
|
|
154
|
+
values = x[0].reshape(-1, layer.in_features)
|
|
155
|
+
result: dict[Any, Any] = {}
|
|
156
|
+
if layer.weight.requires_grad:
|
|
157
|
+
result[layer.weight] = dy.T.mm(values).unsqueeze(0)
|
|
158
|
+
if layer.bias is not None and layer.bias.requires_grad:
|
|
159
|
+
result[layer.bias] = dy.sum(0).unsqueeze(0)
|
|
160
|
+
return result
|
|
161
|
+
parameters = tuple(
|
|
162
|
+
(name, p)
|
|
163
|
+
for name, p in layer.named_parameters(recurse=False)
|
|
164
|
+
if p.requires_grad
|
|
165
|
+
)
|
|
166
|
+
with torch.enable_grad():
|
|
167
|
+
# These views share immutable weight storage, but own their autograd
|
|
168
|
+
# metadata. Functional calls trigger no module hooks or original .grad.
|
|
169
|
+
views = {
|
|
170
|
+
name: p.detach().requires_grad_(p.requires_grad)
|
|
171
|
+
for name, p in layer.named_parameters(recurse=False)
|
|
172
|
+
}
|
|
173
|
+
if type(layer) is torch.nn.LayerNorm:
|
|
174
|
+
_require(
|
|
175
|
+
x.dtype == torch.float32, "native B1 LayerNorm activation must be FP32"
|
|
176
|
+
)
|
|
177
|
+
output = torch.nn.functional.layer_norm(
|
|
178
|
+
x,
|
|
179
|
+
layer.normalized_shape,
|
|
180
|
+
views.get("weight"),
|
|
181
|
+
views.get("bias"),
|
|
182
|
+
layer.eps,
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
_require(
|
|
186
|
+
x.dtype in (torch.int32, torch.int64),
|
|
187
|
+
"native B1 Embedding indices must be integral",
|
|
188
|
+
)
|
|
189
|
+
output = torch.nn.functional.embedding(
|
|
190
|
+
x,
|
|
191
|
+
views["weight"],
|
|
192
|
+
layer.padding_idx,
|
|
193
|
+
None,
|
|
194
|
+
layer.norm_type,
|
|
195
|
+
False,
|
|
196
|
+
False,
|
|
197
|
+
)
|
|
198
|
+
gradients = torch.autograd.grad(
|
|
199
|
+
output,
|
|
200
|
+
tuple(views[name] for name, _ in parameters),
|
|
201
|
+
grad_outputs=backprops.detach(),
|
|
202
|
+
retain_graph=False,
|
|
203
|
+
create_graph=False,
|
|
204
|
+
allow_unused=False,
|
|
205
|
+
)
|
|
206
|
+
return {
|
|
207
|
+
parameter: gradient.detach().unsqueeze(0)
|
|
208
|
+
for (_, parameter), gradient in zip(parameters, gradients, strict=True)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@dataclass(frozen=True, slots=True, eq=False)
|
|
213
|
+
class NativeB1SamplerToken:
|
|
214
|
+
"""Immutable, tensor-free identity certificate for one wrapper instance."""
|
|
215
|
+
|
|
216
|
+
wrapper_ref: weakref.ReferenceType[Any] = field(repr=False)
|
|
217
|
+
model_ref: weakref.ReferenceType[Any] = field(repr=False)
|
|
218
|
+
mapping: Mapping[Any, Sampler] = field(repr=False)
|
|
219
|
+
global_mapping: Any = field(repr=False)
|
|
220
|
+
global_snapshot: Mapping[Any, Any] = field(repr=False)
|
|
221
|
+
model_signature: tuple[Any, ...] = field(repr=False)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _wrapper_model(wrapper: Any, torch: Any, base: Any) -> Any:
|
|
225
|
+
_require(
|
|
226
|
+
type(wrapper) is base
|
|
227
|
+
and wrapper.batch_first is True
|
|
228
|
+
and wrapper.loss_reduction == "mean"
|
|
229
|
+
and not wrapper.force_functorch,
|
|
230
|
+
"native B1 requires the standard batch-first mean Opacus wrapper",
|
|
231
|
+
)
|
|
232
|
+
model = getattr(wrapper, "_module", None)
|
|
233
|
+
_require(isinstance(model, torch.nn.Module), "native B1 wrapper model missing")
|
|
234
|
+
return model
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def install_native_b1_samplers(wrapper: Any) -> NativeB1SamplerToken:
|
|
238
|
+
"""Install once after caller selected this backend with public physical cap1."""
|
|
239
|
+
torch, base = _runtime()
|
|
240
|
+
model = _wrapper_model(wrapper, torch, base)
|
|
241
|
+
signature = _model_signature(model, torch)
|
|
242
|
+
_require(
|
|
243
|
+
"GRAD_SAMPLERS" not in wrapper.__dict__,
|
|
244
|
+
"native B1 refuses an existing instance sampler override",
|
|
245
|
+
)
|
|
246
|
+
global_mapping = base.GRAD_SAMPLERS
|
|
247
|
+
mapping: Mapping[Any, Sampler] = MappingProxyType(
|
|
248
|
+
{
|
|
249
|
+
leaf: _native_b1_sampler
|
|
250
|
+
for leaf in (torch.nn.Linear, torch.nn.LayerNorm, torch.nn.Embedding)
|
|
251
|
+
}
|
|
252
|
+
)
|
|
253
|
+
token = NativeB1SamplerToken(
|
|
254
|
+
weakref.ref(wrapper),
|
|
255
|
+
weakref.ref(model),
|
|
256
|
+
mapping,
|
|
257
|
+
global_mapping,
|
|
258
|
+
MappingProxyType(dict(global_mapping)),
|
|
259
|
+
signature,
|
|
260
|
+
)
|
|
261
|
+
wrapper.GRAD_SAMPLERS = mapping
|
|
262
|
+
check_native_b1_samplers(wrapper, token)
|
|
263
|
+
return token
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def check_native_b1_samplers(wrapper: Any, token: NativeB1SamplerToken) -> None:
|
|
267
|
+
"""Fail closed on wrapper, leaf, callback, or shared registry replacement."""
|
|
268
|
+
_require(type(token) is NativeB1SamplerToken, "native B1 identity token required")
|
|
269
|
+
torch, base = _runtime()
|
|
270
|
+
model = _wrapper_model(wrapper, torch, base)
|
|
271
|
+
_require(
|
|
272
|
+
token.wrapper_ref() is wrapper
|
|
273
|
+
and token.model_ref() is model
|
|
274
|
+
and wrapper.__dict__.get("GRAD_SAMPLERS") is token.mapping
|
|
275
|
+
and set(token.mapping)
|
|
276
|
+
== {torch.nn.Linear, torch.nn.LayerNorm, torch.nn.Embedding}
|
|
277
|
+
and all(fn is _native_b1_sampler for fn in token.mapping.values()),
|
|
278
|
+
"native B1 wrapper/sampler identity changed",
|
|
279
|
+
)
|
|
280
|
+
_require(
|
|
281
|
+
base.GRAD_SAMPLERS is token.global_mapping
|
|
282
|
+
and set(base.GRAD_SAMPLERS) == set(token.global_snapshot)
|
|
283
|
+
and all(base.GRAD_SAMPLERS[k] is v for k, v in token.global_snapshot.items()),
|
|
284
|
+
"native B1 global sampler registry changed",
|
|
285
|
+
)
|
|
286
|
+
_require(
|
|
287
|
+
_model_signature(model, torch) == token.model_signature,
|
|
288
|
+
"native B1 model/leaf configuration changed",
|
|
289
|
+
)
|