dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
"""Model capabilities and public-fixture checks for owned private execution.
|
|
2
|
+
|
|
3
|
+
Application code is trusted. These checks validate a concrete gradient backend
|
|
4
|
+
and immutable state contract, not arbitrary Python or every future input branch.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import copy
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any, Protocol, cast
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import torch
|
|
18
|
+
from torch import nn
|
|
19
|
+
|
|
20
|
+
from dromeus.errors import TaskError
|
|
21
|
+
from dromeus.manifests.models import GradientBackend
|
|
22
|
+
from dromeus.training.model_state import model_state_alias_groups
|
|
23
|
+
from dromeus.training.privacy import opacus_module
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ModelBatchAdapter(Protocol):
|
|
27
|
+
def size(self, batch: Any) -> int: ...
|
|
28
|
+
|
|
29
|
+
def slice(self, batch: Any, start: int, end: int) -> Any: ...
|
|
30
|
+
|
|
31
|
+
def to_device(self, batch: Any, device: torch.device) -> Any: ...
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class ModelCompatibility:
|
|
36
|
+
"""The public state and unique parameter semantics of one prepared model."""
|
|
37
|
+
|
|
38
|
+
trainable_parameter_names: tuple[str, ...]
|
|
39
|
+
parameter_aliases: tuple[tuple[str, ...], ...]
|
|
40
|
+
public_buffer_hashes: Mapping[str, str]
|
|
41
|
+
frozen_parameter_hashes: Mapping[str, str]
|
|
42
|
+
gradient_backend: GradientBackend = "opacus_hooks"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def tensor_digest(value: torch.Tensor) -> str:
|
|
46
|
+
"""Hash public tensor values without binding their physical device."""
|
|
47
|
+
tensor = value.detach().cpu().contiguous()
|
|
48
|
+
metadata = json.dumps([str(tensor.dtype), list(tensor.shape)]).encode()
|
|
49
|
+
raw = tensor.reshape(-1).view(torch.uint8).numpy().tobytes()
|
|
50
|
+
return hashlib.sha256(metadata + b"\x00" + raw).hexdigest()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def require_unchanged_public_state(
|
|
54
|
+
model: nn.Module,
|
|
55
|
+
state: Mapping[str, np.ndarray],
|
|
56
|
+
expected_hashes: Mapping[str, str],
|
|
57
|
+
) -> None:
|
|
58
|
+
"""Reject a checkpoint whose frozen parameters or public buffers changed."""
|
|
59
|
+
persistent = registered_model_state(model)
|
|
60
|
+
for name, expected in expected_hashes.items():
|
|
61
|
+
if name not in persistent:
|
|
62
|
+
continue
|
|
63
|
+
value = state.get(f"model.{name}")
|
|
64
|
+
if value is None or tensor_digest(torch.from_numpy(value.copy())) != expected: # pyright: ignore[reportUnknownMemberType]
|
|
65
|
+
raise TaskError("checkpoint changed immutable public model state")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def registered_model_state(model: nn.Module) -> dict[str, torch.Tensor]:
|
|
69
|
+
"""Return load_state_dict-compatible registered tensors, without callbacks.
|
|
70
|
+
|
|
71
|
+
Nonpersistent buffers stay local but are still hash-bound by compatibility.
|
|
72
|
+
Parameter aliases remain named so standard model loading preserves tying.
|
|
73
|
+
"""
|
|
74
|
+
state: dict[str, torch.Tensor] = dict(
|
|
75
|
+
model.named_parameters(remove_duplicate=False)
|
|
76
|
+
)
|
|
77
|
+
for path, module in cast(
|
|
78
|
+
Iterable[tuple[str, nn.Module]], model.named_modules(remove_duplicate=False)
|
|
79
|
+
):
|
|
80
|
+
omitted = cast(set[str], getattr(module, "_non_persistent_buffers_set"))
|
|
81
|
+
for name, value in module.named_buffers(recurse=False, remove_duplicate=False):
|
|
82
|
+
if name not in omitted:
|
|
83
|
+
state[f"{path}.{name}" if path else name] = value
|
|
84
|
+
return state
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def describe_private_model(
|
|
88
|
+
model: nn.Module, *, gradient_backend: GradientBackend = "opacus_hooks"
|
|
89
|
+
) -> ModelCompatibility:
|
|
90
|
+
"""Inspect structure without running the model or reading a local dataset."""
|
|
91
|
+
if gradient_backend not in ("opacus_hooks", "opacus_native_b1"):
|
|
92
|
+
raise TaskError("unsupported private gradient backend")
|
|
93
|
+
registered = registered_model_state(model)
|
|
94
|
+
state: Mapping[str, Any] = model.state_dict()
|
|
95
|
+
if set(state) != set(registered):
|
|
96
|
+
raise TaskError(
|
|
97
|
+
"private serialization must contain exactly registered parameters "
|
|
98
|
+
"and buffers"
|
|
99
|
+
)
|
|
100
|
+
for name, actual in state.items():
|
|
101
|
+
expected = registered[name]
|
|
102
|
+
if (
|
|
103
|
+
not isinstance(actual, torch.Tensor)
|
|
104
|
+
or actual.shape != expected.shape
|
|
105
|
+
or actual.dtype != expected.dtype
|
|
106
|
+
or actual.device != expected.device
|
|
107
|
+
or not torch.equal(actual, expected.detach())
|
|
108
|
+
):
|
|
109
|
+
raise TaskError(
|
|
110
|
+
f"{name}: serialized state differs from its registered tensor"
|
|
111
|
+
)
|
|
112
|
+
if len({p.device for p in model.parameters()}) != 1:
|
|
113
|
+
raise TaskError("private model parameters require one device")
|
|
114
|
+
names: list[str] = []
|
|
115
|
+
frozen: dict[str, str] = {}
|
|
116
|
+
for name, parameter in model.named_parameters():
|
|
117
|
+
if (
|
|
118
|
+
parameter.dtype != torch.float32
|
|
119
|
+
or parameter.is_sparse
|
|
120
|
+
or parameter.ndim == 0
|
|
121
|
+
or not bool(torch.isfinite(parameter).all())
|
|
122
|
+
):
|
|
123
|
+
raise TaskError(f"{name}: private parameters require finite dense FP32")
|
|
124
|
+
if parameter.requires_grad:
|
|
125
|
+
names.append(name)
|
|
126
|
+
else:
|
|
127
|
+
frozen[name] = tensor_digest(parameter)
|
|
128
|
+
if not names:
|
|
129
|
+
raise TaskError("private model requires trainable parameters")
|
|
130
|
+
aliases = model_state_alias_groups(model)
|
|
131
|
+
parameter_objects = {
|
|
132
|
+
name: id(parameter)
|
|
133
|
+
for name, parameter in model.named_parameters(remove_duplicate=False)
|
|
134
|
+
}
|
|
135
|
+
for group in aliases:
|
|
136
|
+
parameter_names = [name for name in group if name in parameter_objects]
|
|
137
|
+
if parameter_names and (
|
|
138
|
+
len(parameter_names) != len(group)
|
|
139
|
+
or len({parameter_objects[name] for name in parameter_names}) != 1
|
|
140
|
+
):
|
|
141
|
+
raise TaskError("parameter aliases must share one Parameter object")
|
|
142
|
+
return ModelCompatibility(
|
|
143
|
+
gradient_backend=gradient_backend,
|
|
144
|
+
trainable_parameter_names=tuple(names),
|
|
145
|
+
parameter_aliases=aliases,
|
|
146
|
+
public_buffer_hashes={
|
|
147
|
+
name: tensor_digest(value)
|
|
148
|
+
for name, value in model.named_buffers(remove_duplicate=False)
|
|
149
|
+
},
|
|
150
|
+
frozen_parameter_hashes=frozen,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _backend_structure(model: nn.Module) -> None:
|
|
155
|
+
samplers = opacus_module("opacus.grad_sample").GradSampleModule.GRAD_SAMPLERS
|
|
156
|
+
for path, module in cast(Iterable[tuple[str, nn.Module]], model.named_modules()):
|
|
157
|
+
label = path or "<root>"
|
|
158
|
+
if isinstance(
|
|
159
|
+
module, (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.SyncBatchNorm)
|
|
160
|
+
):
|
|
161
|
+
raise TaskError(f"{label}: batch-dependent normalization is unsupported")
|
|
162
|
+
if (
|
|
163
|
+
isinstance(
|
|
164
|
+
module,
|
|
165
|
+
(
|
|
166
|
+
nn.Dropout,
|
|
167
|
+
nn.Dropout1d,
|
|
168
|
+
nn.Dropout2d,
|
|
169
|
+
nn.Dropout3d,
|
|
170
|
+
nn.AlphaDropout,
|
|
171
|
+
nn.FeatureAlphaDropout,
|
|
172
|
+
),
|
|
173
|
+
)
|
|
174
|
+
and module.p
|
|
175
|
+
):
|
|
176
|
+
raise TaskError(f"{label}: stochastic layer requires a model RNG profile")
|
|
177
|
+
if isinstance(module, nn.RReLU):
|
|
178
|
+
raise TaskError(f"{label}: stochastic layer requires a model RNG profile")
|
|
179
|
+
if isinstance(module, nn.Embedding) and (
|
|
180
|
+
module.sparse or module.scale_grad_by_freq or module.max_norm is not None
|
|
181
|
+
):
|
|
182
|
+
raise TaskError(f"{label}: embedding gradient/state options unsupported")
|
|
183
|
+
direct = any(p.requires_grad for p in module.parameters(recurse=False))
|
|
184
|
+
if direct and type(module) not in samplers:
|
|
185
|
+
raise TaskError(f"{label}: no registered Opacus gradient sampler")
|
|
186
|
+
if direct and any(True for _ in module.buffers()):
|
|
187
|
+
raise TaskError(
|
|
188
|
+
f"{label}: Opacus hooks cannot handle trainable-layer buffers"
|
|
189
|
+
)
|
|
190
|
+
if (
|
|
191
|
+
getattr(module, "_forward_hooks")
|
|
192
|
+
or getattr(module, "_forward_pre_hooks")
|
|
193
|
+
or getattr(module, "_backward_hooks")
|
|
194
|
+
):
|
|
195
|
+
raise TaskError(
|
|
196
|
+
f"{label}: pre-existing model hooks need an explicit profile"
|
|
197
|
+
)
|
|
198
|
+
opacus_module("opacus.validators").ModuleValidator.validate(model, strict=True)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def ensure_grad_samples(model: nn.Module, batch_size: int) -> None:
|
|
202
|
+
"""Require one full gradient per record; unused parameters contribute zeros.
|
|
203
|
+
|
|
204
|
+
Invoke after backward and before the Opacus optimizer. Missing hook results
|
|
205
|
+
with an ordinary gradient or an incomplete backward cannot be guessed.
|
|
206
|
+
"""
|
|
207
|
+
if batch_size < 0:
|
|
208
|
+
raise TaskError("invalid record count")
|
|
209
|
+
for name, parameter in model.named_parameters():
|
|
210
|
+
if not parameter.requires_grad:
|
|
211
|
+
continue
|
|
212
|
+
if getattr(parameter, "_forward_counter", 0):
|
|
213
|
+
raise TaskError(f"{name}: unresolved per-record backward coverage")
|
|
214
|
+
sample = getattr(parameter, "grad_sample", None)
|
|
215
|
+
if sample is None:
|
|
216
|
+
if parameter.grad is not None:
|
|
217
|
+
raise TaskError(f"{name}: missing per-record gradient sampler result")
|
|
218
|
+
sample = parameter.new_zeros((batch_size, *parameter.shape))
|
|
219
|
+
cast(Any, parameter).grad_sample = sample
|
|
220
|
+
if (
|
|
221
|
+
not isinstance(sample, torch.Tensor)
|
|
222
|
+
or sample.shape != (batch_size, *parameter.shape)
|
|
223
|
+
or not bool(torch.isfinite(sample).all())
|
|
224
|
+
):
|
|
225
|
+
raise TaskError(f"{name}: invalid per-record gradient shape or values")
|
|
226
|
+
if (
|
|
227
|
+
batch_size
|
|
228
|
+
and parameter.grad is not None
|
|
229
|
+
and not torch.allclose(sample.mean(0), parameter.grad, rtol=1e-4, atol=1e-6)
|
|
230
|
+
):
|
|
231
|
+
raise TaskError(
|
|
232
|
+
f"{name}: per-record gradients do not cover aggregate gradient"
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _losses(
|
|
237
|
+
loss: Callable[[nn.Module, Any], torch.Tensor],
|
|
238
|
+
model: nn.Module,
|
|
239
|
+
batch: Any,
|
|
240
|
+
count: int,
|
|
241
|
+
) -> torch.Tensor:
|
|
242
|
+
result: Any = loss(model, batch)
|
|
243
|
+
if (
|
|
244
|
+
not isinstance(result, torch.Tensor)
|
|
245
|
+
or not result.is_floating_point()
|
|
246
|
+
or result.shape != (count,)
|
|
247
|
+
or not bool(torch.isfinite(result).all())
|
|
248
|
+
or not result.requires_grad
|
|
249
|
+
):
|
|
250
|
+
raise TaskError(
|
|
251
|
+
"loss callback must return finite differentiable record losses [B]"
|
|
252
|
+
)
|
|
253
|
+
return result
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _state_hashes(model: nn.Module) -> dict[str, str]:
|
|
257
|
+
return {name: tensor_digest(value) for name, value in model.state_dict().items()}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def validate_private_model(
|
|
261
|
+
model: nn.Module,
|
|
262
|
+
loss_per_record: Callable[[nn.Module, Any], torch.Tensor],
|
|
263
|
+
batch_adapter: ModelBatchAdapter,
|
|
264
|
+
public_batch: Any,
|
|
265
|
+
*,
|
|
266
|
+
gradient_backend: GradientBackend = "opacus_hooks",
|
|
267
|
+
) -> ModelCompatibility:
|
|
268
|
+
"""Check supported hooks against single-record autograd on PUBLIC fixtures.
|
|
269
|
+
|
|
270
|
+
The caller must supply synthetic/public representative inputs. No actual
|
|
271
|
+
training records are sampled. All probes use disposable model copies.
|
|
272
|
+
"""
|
|
273
|
+
report = describe_private_model(model, gradient_backend=gradient_backend)
|
|
274
|
+
if gradient_backend == "opacus_native_b1":
|
|
275
|
+
from dromeus.training.native_gradients import validate_native_b1_model
|
|
276
|
+
|
|
277
|
+
validate_native_b1_model(model)
|
|
278
|
+
probe = copy.deepcopy(model).train()
|
|
279
|
+
_backend_structure(probe)
|
|
280
|
+
device = next(probe.parameters()).device
|
|
281
|
+
batch = batch_adapter.to_device(copy.deepcopy(public_batch), device)
|
|
282
|
+
count = batch_adapter.size(batch)
|
|
283
|
+
if count < 2:
|
|
284
|
+
raise TaskError("public compatibility fixture requires at least two records")
|
|
285
|
+
devices = [device.index or 0] if device.type == "cuda" else []
|
|
286
|
+
with cast(Any, torch.random).fork_rng(devices=devices):
|
|
287
|
+
rng = torch.get_rng_state().clone()
|
|
288
|
+
cuda_rng = torch.cuda.get_rng_state(device).clone() if devices else None
|
|
289
|
+
original = _state_hashes(probe)
|
|
290
|
+
references: dict[str, list[torch.Tensor]] = {
|
|
291
|
+
name: [] for name in report.trainable_parameter_names
|
|
292
|
+
}
|
|
293
|
+
named = [(n, p) for n, p in probe.named_parameters() if p.requires_grad]
|
|
294
|
+
record_values: list[torch.Tensor] = []
|
|
295
|
+
for index in range(count):
|
|
296
|
+
record = batch_adapter.slice(batch, index, index + 1)
|
|
297
|
+
value = _losses(loss_per_record, probe, record, 1)
|
|
298
|
+
record_values.append(value.detach())
|
|
299
|
+
gradients = cast(
|
|
300
|
+
tuple[torch.Tensor | None, ...],
|
|
301
|
+
torch.autograd.grad(
|
|
302
|
+
value.sum(), [p for _, p in named], allow_unused=True
|
|
303
|
+
),
|
|
304
|
+
)
|
|
305
|
+
for (name, parameter), gradient in zip(named, gradients, strict=True):
|
|
306
|
+
references[name].append(
|
|
307
|
+
torch.zeros_like(parameter)
|
|
308
|
+
if gradient is None
|
|
309
|
+
else gradient.detach()
|
|
310
|
+
)
|
|
311
|
+
if _state_hashes(probe) != original:
|
|
312
|
+
raise TaskError("model/loss mutated public model state during forward")
|
|
313
|
+
hooked = copy.deepcopy(model).train()
|
|
314
|
+
wrapper: Any = opacus_module("opacus.grad_sample").GradSampleModule(
|
|
315
|
+
hooked, batch_first=True, loss_reduction="mean", strict=True
|
|
316
|
+
)
|
|
317
|
+
try:
|
|
318
|
+
values = _losses(loss_per_record, wrapper, batch, count)
|
|
319
|
+
if not torch.allclose(
|
|
320
|
+
values.detach(), torch.cat(record_values), rtol=1e-4, atol=1e-6
|
|
321
|
+
):
|
|
322
|
+
raise TaskError(
|
|
323
|
+
"loss depends on other records or changes with batch size"
|
|
324
|
+
)
|
|
325
|
+
cast(Any, values.mean()).backward()
|
|
326
|
+
ensure_grad_samples(hooked, count)
|
|
327
|
+
for name, parameter in hooked.named_parameters():
|
|
328
|
+
if not parameter.requires_grad:
|
|
329
|
+
continue
|
|
330
|
+
observed = cast(torch.Tensor, getattr(parameter, "grad_sample"))
|
|
331
|
+
expected = torch.stack(references[name])
|
|
332
|
+
if not torch.allclose(observed, expected, rtol=1e-4, atol=1e-6):
|
|
333
|
+
raise TaskError(
|
|
334
|
+
f"{name}: per-record gradient differs from autograd"
|
|
335
|
+
)
|
|
336
|
+
if gradient_backend == "opacus_native_b1":
|
|
337
|
+
from dromeus.training.native_gradients import (
|
|
338
|
+
check_native_b1_samplers,
|
|
339
|
+
install_native_b1_samplers,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
wrapper.zero_grad(set_to_none=True)
|
|
343
|
+
token = install_native_b1_samplers(wrapper)
|
|
344
|
+
for index in range(count):
|
|
345
|
+
wrapper.zero_grad(set_to_none=True)
|
|
346
|
+
record = batch_adapter.slice(batch, index, index + 1)
|
|
347
|
+
single = _losses(loss_per_record, wrapper, record, 1)
|
|
348
|
+
cast(Any, single.sum()).backward()
|
|
349
|
+
ensure_grad_samples(hooked, 1)
|
|
350
|
+
check_native_b1_samplers(wrapper, token)
|
|
351
|
+
for name, parameter in hooked.named_parameters():
|
|
352
|
+
if not parameter.requires_grad:
|
|
353
|
+
continue
|
|
354
|
+
sample = cast(torch.Tensor, getattr(parameter, "grad_sample"))
|
|
355
|
+
if not torch.allclose(
|
|
356
|
+
sample[0], references[name][index], rtol=1e-4, atol=1e-6
|
|
357
|
+
):
|
|
358
|
+
raise TaskError(
|
|
359
|
+
f"{name}: native B1 gradient differs from autograd"
|
|
360
|
+
)
|
|
361
|
+
if _state_hashes(hooked) != original:
|
|
362
|
+
raise TaskError("model/loss mutated public model state during forward")
|
|
363
|
+
finally:
|
|
364
|
+
wrapper.remove_hooks()
|
|
365
|
+
if not torch.equal(rng, torch.get_rng_state()) or (
|
|
366
|
+
cuda_rng is not None
|
|
367
|
+
and not torch.equal(cuda_rng, torch.cuda.get_rng_state(device))
|
|
368
|
+
):
|
|
369
|
+
raise TaskError(
|
|
370
|
+
"model/loss consumed global randomness without an RNG profile"
|
|
371
|
+
)
|
|
372
|
+
return report
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
class ModelStateGuard:
|
|
376
|
+
"""Preserve declared immutable public state and parameter objects during a run."""
|
|
377
|
+
|
|
378
|
+
def __init__(self, model: nn.Module, report: ModelCompatibility) -> None:
|
|
379
|
+
if (
|
|
380
|
+
describe_private_model(model, gradient_backend=report.gradient_backend)
|
|
381
|
+
!= report
|
|
382
|
+
):
|
|
383
|
+
raise TaskError("model differs from declared compatibility report")
|
|
384
|
+
self._model = model
|
|
385
|
+
self._report = copy.deepcopy(report)
|
|
386
|
+
self._parameters = tuple(
|
|
387
|
+
(name, id(p), p.requires_grad, tuple(p.shape), p.dtype, p.device)
|
|
388
|
+
for name, p in model.named_parameters()
|
|
389
|
+
)
|
|
390
|
+
self._buffers = tuple((name, id(b)) for name, b in model.named_buffers())
|
|
391
|
+
|
|
392
|
+
def check(self) -> None:
|
|
393
|
+
actual = tuple(
|
|
394
|
+
(name, id(p), p.requires_grad, tuple(p.shape), p.dtype, p.device)
|
|
395
|
+
for name, p in self._model.named_parameters()
|
|
396
|
+
)
|
|
397
|
+
if actual != self._parameters:
|
|
398
|
+
raise TaskError(
|
|
399
|
+
"private model parameter identity/schema/trainability changed"
|
|
400
|
+
)
|
|
401
|
+
if (
|
|
402
|
+
tuple((name, id(b)) for name, b in self._model.named_buffers())
|
|
403
|
+
!= self._buffers
|
|
404
|
+
):
|
|
405
|
+
raise TaskError("private model buffer identity changed")
|
|
406
|
+
if (
|
|
407
|
+
describe_private_model(
|
|
408
|
+
self._model, gradient_backend=self._report.gradient_backend
|
|
409
|
+
)
|
|
410
|
+
!= self._report
|
|
411
|
+
):
|
|
412
|
+
raise TaskError(
|
|
413
|
+
"private model aliases, public buffers or frozen parameters changed"
|
|
414
|
+
)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Bounded numeric-array encoding for owned research state, without pickle."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any, cast
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import torch
|
|
10
|
+
|
|
11
|
+
from dromeus.errors import PrivacyError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Keep recursive walkers at module scope: a recursive closure capturing arrays
|
|
15
|
+
# retains entire optimizer checkpoints until cyclic GC, even after owner release.
|
|
16
|
+
def _encode(item: Any, arrays: dict[str, np.ndarray]) -> Any:
|
|
17
|
+
if isinstance(item, torch.Tensor):
|
|
18
|
+
key = f"tensor.{len(arrays)}"
|
|
19
|
+
arrays[key] = item.detach().cpu().numpy().copy()
|
|
20
|
+
return {"tensor": key}
|
|
21
|
+
if isinstance(item, dict):
|
|
22
|
+
return {
|
|
23
|
+
"dict": [
|
|
24
|
+
[_encode(k, arrays), _encode(v, arrays)]
|
|
25
|
+
for k, v in cast(dict[Any, Any], item).items()
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
if isinstance(item, tuple):
|
|
29
|
+
return {"tuple": [_encode(x, arrays) for x in cast(list[Any], item)]}
|
|
30
|
+
if isinstance(item, list):
|
|
31
|
+
return {"list": [_encode(x, arrays) for x in cast(list[Any], item)]}
|
|
32
|
+
if item is None or type(item) in (str, bool, int, float):
|
|
33
|
+
return item
|
|
34
|
+
raise PrivacyError("unsupported research state value")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def encode_state(value: Any) -> dict[str, np.ndarray]:
|
|
38
|
+
arrays: dict[str, np.ndarray] = {}
|
|
39
|
+
|
|
40
|
+
payload = json.dumps(
|
|
41
|
+
_encode(value, arrays), allow_nan=False, separators=(",", ":")
|
|
42
|
+
).encode()
|
|
43
|
+
if len(payload) > 1024 * 1024:
|
|
44
|
+
raise PrivacyError("research state metadata exceeds bound")
|
|
45
|
+
arrays["structure"] = np.frombuffer(payload, dtype=np.uint8).copy()
|
|
46
|
+
return arrays
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _decode(
|
|
50
|
+
item: Any, arrays: dict[str, np.ndarray], used: set[str], depth: int = 0
|
|
51
|
+
) -> Any:
|
|
52
|
+
if depth > 32:
|
|
53
|
+
raise PrivacyError("research state nesting exceeds bound")
|
|
54
|
+
if isinstance(item, dict):
|
|
55
|
+
if len(cast(dict[Any, Any], item)) != 1:
|
|
56
|
+
raise PrivacyError("invalid research state tag")
|
|
57
|
+
tag, values = next(iter(cast(dict[Any, Any], item).items()))
|
|
58
|
+
if tag == "tensor":
|
|
59
|
+
if values not in arrays or values in used:
|
|
60
|
+
raise PrivacyError("invalid research state tensor")
|
|
61
|
+
used.add(values)
|
|
62
|
+
value = arrays[values]
|
|
63
|
+
if value.dtype.kind not in "biuf" or not np.isfinite(value).all():
|
|
64
|
+
raise PrivacyError("invalid research state array")
|
|
65
|
+
return torch.from_numpy(value.copy()) # pyright: ignore[reportUnknownMemberType]
|
|
66
|
+
if tag == "dict":
|
|
67
|
+
return {
|
|
68
|
+
_decode(k, arrays, used, depth + 1): _decode(v, arrays, used, depth + 1)
|
|
69
|
+
for k, v in values
|
|
70
|
+
}
|
|
71
|
+
if tag in ("tuple", "list"):
|
|
72
|
+
result = [_decode(x, arrays, used, depth + 1) for x in values]
|
|
73
|
+
return tuple(result) if tag == "tuple" else result
|
|
74
|
+
raise PrivacyError("invalid research state tag")
|
|
75
|
+
if item is None or type(item) in (str, bool, int, float):
|
|
76
|
+
return item
|
|
77
|
+
raise PrivacyError("invalid research state value")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def decode_state(arrays: dict[str, np.ndarray]) -> Any:
|
|
81
|
+
metadata = arrays.get("structure")
|
|
82
|
+
if (
|
|
83
|
+
metadata is None
|
|
84
|
+
or metadata.dtype != np.uint8
|
|
85
|
+
or metadata.ndim != 1
|
|
86
|
+
or metadata.size > 1024 * 1024
|
|
87
|
+
):
|
|
88
|
+
raise PrivacyError("invalid research state structure")
|
|
89
|
+
used = {"structure"}
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
result = _decode(json.loads(metadata.tobytes()), arrays, used)
|
|
93
|
+
except Exception:
|
|
94
|
+
raise PrivacyError("invalid research checkpoint encoding") from None
|
|
95
|
+
if set(arrays) != used:
|
|
96
|
+
raise PrivacyError("unexpected research checkpoint arrays")
|
|
97
|
+
return result
|