dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,414 @@
1
+ """Model capabilities and public-fixture checks for owned private execution.
2
+
3
+ Application code is trusted. These checks validate a concrete gradient backend
4
+ and immutable state contract, not arbitrary Python or every future input branch.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import copy
10
+ import hashlib
11
+ import json
12
+ from collections.abc import Callable, Iterable, Mapping
13
+ from dataclasses import dataclass
14
+ from typing import Any, Protocol, cast
15
+
16
+ import numpy as np
17
+ import torch
18
+ from torch import nn
19
+
20
+ from dromeus.errors import TaskError
21
+ from dromeus.manifests.models import GradientBackend
22
+ from dromeus.training.model_state import model_state_alias_groups
23
+ from dromeus.training.privacy import opacus_module
24
+
25
+
26
+ class ModelBatchAdapter(Protocol):
27
+ def size(self, batch: Any) -> int: ...
28
+
29
+ def slice(self, batch: Any, start: int, end: int) -> Any: ...
30
+
31
+ def to_device(self, batch: Any, device: torch.device) -> Any: ...
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class ModelCompatibility:
36
+ """The public state and unique parameter semantics of one prepared model."""
37
+
38
+ trainable_parameter_names: tuple[str, ...]
39
+ parameter_aliases: tuple[tuple[str, ...], ...]
40
+ public_buffer_hashes: Mapping[str, str]
41
+ frozen_parameter_hashes: Mapping[str, str]
42
+ gradient_backend: GradientBackend = "opacus_hooks"
43
+
44
+
45
+ def tensor_digest(value: torch.Tensor) -> str:
46
+ """Hash public tensor values without binding their physical device."""
47
+ tensor = value.detach().cpu().contiguous()
48
+ metadata = json.dumps([str(tensor.dtype), list(tensor.shape)]).encode()
49
+ raw = tensor.reshape(-1).view(torch.uint8).numpy().tobytes()
50
+ return hashlib.sha256(metadata + b"\x00" + raw).hexdigest()
51
+
52
+
53
+ def require_unchanged_public_state(
54
+ model: nn.Module,
55
+ state: Mapping[str, np.ndarray],
56
+ expected_hashes: Mapping[str, str],
57
+ ) -> None:
58
+ """Reject a checkpoint whose frozen parameters or public buffers changed."""
59
+ persistent = registered_model_state(model)
60
+ for name, expected in expected_hashes.items():
61
+ if name not in persistent:
62
+ continue
63
+ value = state.get(f"model.{name}")
64
+ if value is None or tensor_digest(torch.from_numpy(value.copy())) != expected: # pyright: ignore[reportUnknownMemberType]
65
+ raise TaskError("checkpoint changed immutable public model state")
66
+
67
+
68
+ def registered_model_state(model: nn.Module) -> dict[str, torch.Tensor]:
69
+ """Return load_state_dict-compatible registered tensors, without callbacks.
70
+
71
+ Nonpersistent buffers stay local but are still hash-bound by compatibility.
72
+ Parameter aliases remain named so standard model loading preserves tying.
73
+ """
74
+ state: dict[str, torch.Tensor] = dict(
75
+ model.named_parameters(remove_duplicate=False)
76
+ )
77
+ for path, module in cast(
78
+ Iterable[tuple[str, nn.Module]], model.named_modules(remove_duplicate=False)
79
+ ):
80
+ omitted = cast(set[str], getattr(module, "_non_persistent_buffers_set"))
81
+ for name, value in module.named_buffers(recurse=False, remove_duplicate=False):
82
+ if name not in omitted:
83
+ state[f"{path}.{name}" if path else name] = value
84
+ return state
85
+
86
+
87
+ def describe_private_model(
88
+ model: nn.Module, *, gradient_backend: GradientBackend = "opacus_hooks"
89
+ ) -> ModelCompatibility:
90
+ """Inspect structure without running the model or reading a local dataset."""
91
+ if gradient_backend not in ("opacus_hooks", "opacus_native_b1"):
92
+ raise TaskError("unsupported private gradient backend")
93
+ registered = registered_model_state(model)
94
+ state: Mapping[str, Any] = model.state_dict()
95
+ if set(state) != set(registered):
96
+ raise TaskError(
97
+ "private serialization must contain exactly registered parameters "
98
+ "and buffers"
99
+ )
100
+ for name, actual in state.items():
101
+ expected = registered[name]
102
+ if (
103
+ not isinstance(actual, torch.Tensor)
104
+ or actual.shape != expected.shape
105
+ or actual.dtype != expected.dtype
106
+ or actual.device != expected.device
107
+ or not torch.equal(actual, expected.detach())
108
+ ):
109
+ raise TaskError(
110
+ f"{name}: serialized state differs from its registered tensor"
111
+ )
112
+ if len({p.device for p in model.parameters()}) != 1:
113
+ raise TaskError("private model parameters require one device")
114
+ names: list[str] = []
115
+ frozen: dict[str, str] = {}
116
+ for name, parameter in model.named_parameters():
117
+ if (
118
+ parameter.dtype != torch.float32
119
+ or parameter.is_sparse
120
+ or parameter.ndim == 0
121
+ or not bool(torch.isfinite(parameter).all())
122
+ ):
123
+ raise TaskError(f"{name}: private parameters require finite dense FP32")
124
+ if parameter.requires_grad:
125
+ names.append(name)
126
+ else:
127
+ frozen[name] = tensor_digest(parameter)
128
+ if not names:
129
+ raise TaskError("private model requires trainable parameters")
130
+ aliases = model_state_alias_groups(model)
131
+ parameter_objects = {
132
+ name: id(parameter)
133
+ for name, parameter in model.named_parameters(remove_duplicate=False)
134
+ }
135
+ for group in aliases:
136
+ parameter_names = [name for name in group if name in parameter_objects]
137
+ if parameter_names and (
138
+ len(parameter_names) != len(group)
139
+ or len({parameter_objects[name] for name in parameter_names}) != 1
140
+ ):
141
+ raise TaskError("parameter aliases must share one Parameter object")
142
+ return ModelCompatibility(
143
+ gradient_backend=gradient_backend,
144
+ trainable_parameter_names=tuple(names),
145
+ parameter_aliases=aliases,
146
+ public_buffer_hashes={
147
+ name: tensor_digest(value)
148
+ for name, value in model.named_buffers(remove_duplicate=False)
149
+ },
150
+ frozen_parameter_hashes=frozen,
151
+ )
152
+
153
+
154
+ def _backend_structure(model: nn.Module) -> None:
155
+ samplers = opacus_module("opacus.grad_sample").GradSampleModule.GRAD_SAMPLERS
156
+ for path, module in cast(Iterable[tuple[str, nn.Module]], model.named_modules()):
157
+ label = path or "<root>"
158
+ if isinstance(
159
+ module, (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.SyncBatchNorm)
160
+ ):
161
+ raise TaskError(f"{label}: batch-dependent normalization is unsupported")
162
+ if (
163
+ isinstance(
164
+ module,
165
+ (
166
+ nn.Dropout,
167
+ nn.Dropout1d,
168
+ nn.Dropout2d,
169
+ nn.Dropout3d,
170
+ nn.AlphaDropout,
171
+ nn.FeatureAlphaDropout,
172
+ ),
173
+ )
174
+ and module.p
175
+ ):
176
+ raise TaskError(f"{label}: stochastic layer requires a model RNG profile")
177
+ if isinstance(module, nn.RReLU):
178
+ raise TaskError(f"{label}: stochastic layer requires a model RNG profile")
179
+ if isinstance(module, nn.Embedding) and (
180
+ module.sparse or module.scale_grad_by_freq or module.max_norm is not None
181
+ ):
182
+ raise TaskError(f"{label}: embedding gradient/state options unsupported")
183
+ direct = any(p.requires_grad for p in module.parameters(recurse=False))
184
+ if direct and type(module) not in samplers:
185
+ raise TaskError(f"{label}: no registered Opacus gradient sampler")
186
+ if direct and any(True for _ in module.buffers()):
187
+ raise TaskError(
188
+ f"{label}: Opacus hooks cannot handle trainable-layer buffers"
189
+ )
190
+ if (
191
+ getattr(module, "_forward_hooks")
192
+ or getattr(module, "_forward_pre_hooks")
193
+ or getattr(module, "_backward_hooks")
194
+ ):
195
+ raise TaskError(
196
+ f"{label}: pre-existing model hooks need an explicit profile"
197
+ )
198
+ opacus_module("opacus.validators").ModuleValidator.validate(model, strict=True)
199
+
200
+
201
+ def ensure_grad_samples(model: nn.Module, batch_size: int) -> None:
202
+ """Require one full gradient per record; unused parameters contribute zeros.
203
+
204
+ Invoke after backward and before the Opacus optimizer. Missing hook results
205
+ with an ordinary gradient or an incomplete backward cannot be guessed.
206
+ """
207
+ if batch_size < 0:
208
+ raise TaskError("invalid record count")
209
+ for name, parameter in model.named_parameters():
210
+ if not parameter.requires_grad:
211
+ continue
212
+ if getattr(parameter, "_forward_counter", 0):
213
+ raise TaskError(f"{name}: unresolved per-record backward coverage")
214
+ sample = getattr(parameter, "grad_sample", None)
215
+ if sample is None:
216
+ if parameter.grad is not None:
217
+ raise TaskError(f"{name}: missing per-record gradient sampler result")
218
+ sample = parameter.new_zeros((batch_size, *parameter.shape))
219
+ cast(Any, parameter).grad_sample = sample
220
+ if (
221
+ not isinstance(sample, torch.Tensor)
222
+ or sample.shape != (batch_size, *parameter.shape)
223
+ or not bool(torch.isfinite(sample).all())
224
+ ):
225
+ raise TaskError(f"{name}: invalid per-record gradient shape or values")
226
+ if (
227
+ batch_size
228
+ and parameter.grad is not None
229
+ and not torch.allclose(sample.mean(0), parameter.grad, rtol=1e-4, atol=1e-6)
230
+ ):
231
+ raise TaskError(
232
+ f"{name}: per-record gradients do not cover aggregate gradient"
233
+ )
234
+
235
+
236
+ def _losses(
237
+ loss: Callable[[nn.Module, Any], torch.Tensor],
238
+ model: nn.Module,
239
+ batch: Any,
240
+ count: int,
241
+ ) -> torch.Tensor:
242
+ result: Any = loss(model, batch)
243
+ if (
244
+ not isinstance(result, torch.Tensor)
245
+ or not result.is_floating_point()
246
+ or result.shape != (count,)
247
+ or not bool(torch.isfinite(result).all())
248
+ or not result.requires_grad
249
+ ):
250
+ raise TaskError(
251
+ "loss callback must return finite differentiable record losses [B]"
252
+ )
253
+ return result
254
+
255
+
256
+ def _state_hashes(model: nn.Module) -> dict[str, str]:
257
+ return {name: tensor_digest(value) for name, value in model.state_dict().items()}
258
+
259
+
260
+ def validate_private_model(
261
+ model: nn.Module,
262
+ loss_per_record: Callable[[nn.Module, Any], torch.Tensor],
263
+ batch_adapter: ModelBatchAdapter,
264
+ public_batch: Any,
265
+ *,
266
+ gradient_backend: GradientBackend = "opacus_hooks",
267
+ ) -> ModelCompatibility:
268
+ """Check supported hooks against single-record autograd on PUBLIC fixtures.
269
+
270
+ The caller must supply synthetic/public representative inputs. No actual
271
+ training records are sampled. All probes use disposable model copies.
272
+ """
273
+ report = describe_private_model(model, gradient_backend=gradient_backend)
274
+ if gradient_backend == "opacus_native_b1":
275
+ from dromeus.training.native_gradients import validate_native_b1_model
276
+
277
+ validate_native_b1_model(model)
278
+ probe = copy.deepcopy(model).train()
279
+ _backend_structure(probe)
280
+ device = next(probe.parameters()).device
281
+ batch = batch_adapter.to_device(copy.deepcopy(public_batch), device)
282
+ count = batch_adapter.size(batch)
283
+ if count < 2:
284
+ raise TaskError("public compatibility fixture requires at least two records")
285
+ devices = [device.index or 0] if device.type == "cuda" else []
286
+ with cast(Any, torch.random).fork_rng(devices=devices):
287
+ rng = torch.get_rng_state().clone()
288
+ cuda_rng = torch.cuda.get_rng_state(device).clone() if devices else None
289
+ original = _state_hashes(probe)
290
+ references: dict[str, list[torch.Tensor]] = {
291
+ name: [] for name in report.trainable_parameter_names
292
+ }
293
+ named = [(n, p) for n, p in probe.named_parameters() if p.requires_grad]
294
+ record_values: list[torch.Tensor] = []
295
+ for index in range(count):
296
+ record = batch_adapter.slice(batch, index, index + 1)
297
+ value = _losses(loss_per_record, probe, record, 1)
298
+ record_values.append(value.detach())
299
+ gradients = cast(
300
+ tuple[torch.Tensor | None, ...],
301
+ torch.autograd.grad(
302
+ value.sum(), [p for _, p in named], allow_unused=True
303
+ ),
304
+ )
305
+ for (name, parameter), gradient in zip(named, gradients, strict=True):
306
+ references[name].append(
307
+ torch.zeros_like(parameter)
308
+ if gradient is None
309
+ else gradient.detach()
310
+ )
311
+ if _state_hashes(probe) != original:
312
+ raise TaskError("model/loss mutated public model state during forward")
313
+ hooked = copy.deepcopy(model).train()
314
+ wrapper: Any = opacus_module("opacus.grad_sample").GradSampleModule(
315
+ hooked, batch_first=True, loss_reduction="mean", strict=True
316
+ )
317
+ try:
318
+ values = _losses(loss_per_record, wrapper, batch, count)
319
+ if not torch.allclose(
320
+ values.detach(), torch.cat(record_values), rtol=1e-4, atol=1e-6
321
+ ):
322
+ raise TaskError(
323
+ "loss depends on other records or changes with batch size"
324
+ )
325
+ cast(Any, values.mean()).backward()
326
+ ensure_grad_samples(hooked, count)
327
+ for name, parameter in hooked.named_parameters():
328
+ if not parameter.requires_grad:
329
+ continue
330
+ observed = cast(torch.Tensor, getattr(parameter, "grad_sample"))
331
+ expected = torch.stack(references[name])
332
+ if not torch.allclose(observed, expected, rtol=1e-4, atol=1e-6):
333
+ raise TaskError(
334
+ f"{name}: per-record gradient differs from autograd"
335
+ )
336
+ if gradient_backend == "opacus_native_b1":
337
+ from dromeus.training.native_gradients import (
338
+ check_native_b1_samplers,
339
+ install_native_b1_samplers,
340
+ )
341
+
342
+ wrapper.zero_grad(set_to_none=True)
343
+ token = install_native_b1_samplers(wrapper)
344
+ for index in range(count):
345
+ wrapper.zero_grad(set_to_none=True)
346
+ record = batch_adapter.slice(batch, index, index + 1)
347
+ single = _losses(loss_per_record, wrapper, record, 1)
348
+ cast(Any, single.sum()).backward()
349
+ ensure_grad_samples(hooked, 1)
350
+ check_native_b1_samplers(wrapper, token)
351
+ for name, parameter in hooked.named_parameters():
352
+ if not parameter.requires_grad:
353
+ continue
354
+ sample = cast(torch.Tensor, getattr(parameter, "grad_sample"))
355
+ if not torch.allclose(
356
+ sample[0], references[name][index], rtol=1e-4, atol=1e-6
357
+ ):
358
+ raise TaskError(
359
+ f"{name}: native B1 gradient differs from autograd"
360
+ )
361
+ if _state_hashes(hooked) != original:
362
+ raise TaskError("model/loss mutated public model state during forward")
363
+ finally:
364
+ wrapper.remove_hooks()
365
+ if not torch.equal(rng, torch.get_rng_state()) or (
366
+ cuda_rng is not None
367
+ and not torch.equal(cuda_rng, torch.cuda.get_rng_state(device))
368
+ ):
369
+ raise TaskError(
370
+ "model/loss consumed global randomness without an RNG profile"
371
+ )
372
+ return report
373
+
374
+
375
+ class ModelStateGuard:
376
+ """Preserve declared immutable public state and parameter objects during a run."""
377
+
378
+ def __init__(self, model: nn.Module, report: ModelCompatibility) -> None:
379
+ if (
380
+ describe_private_model(model, gradient_backend=report.gradient_backend)
381
+ != report
382
+ ):
383
+ raise TaskError("model differs from declared compatibility report")
384
+ self._model = model
385
+ self._report = copy.deepcopy(report)
386
+ self._parameters = tuple(
387
+ (name, id(p), p.requires_grad, tuple(p.shape), p.dtype, p.device)
388
+ for name, p in model.named_parameters()
389
+ )
390
+ self._buffers = tuple((name, id(b)) for name, b in model.named_buffers())
391
+
392
+ def check(self) -> None:
393
+ actual = tuple(
394
+ (name, id(p), p.requires_grad, tuple(p.shape), p.dtype, p.device)
395
+ for name, p in self._model.named_parameters()
396
+ )
397
+ if actual != self._parameters:
398
+ raise TaskError(
399
+ "private model parameter identity/schema/trainability changed"
400
+ )
401
+ if (
402
+ tuple((name, id(b)) for name, b in self._model.named_buffers())
403
+ != self._buffers
404
+ ):
405
+ raise TaskError("private model buffer identity changed")
406
+ if (
407
+ describe_private_model(
408
+ self._model, gradient_backend=self._report.gradient_backend
409
+ )
410
+ != self._report
411
+ ):
412
+ raise TaskError(
413
+ "private model aliases, public buffers or frozen parameters changed"
414
+ )
@@ -0,0 +1,97 @@
1
+ """Bounded numeric-array encoding for owned research state, without pickle."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import Any, cast
7
+
8
+ import numpy as np
9
+ import torch
10
+
11
+ from dromeus.errors import PrivacyError
12
+
13
+
14
+ # Keep recursive walkers at module scope: a recursive closure capturing arrays
15
+ # retains entire optimizer checkpoints until cyclic GC, even after owner release.
16
+ def _encode(item: Any, arrays: dict[str, np.ndarray]) -> Any:
17
+ if isinstance(item, torch.Tensor):
18
+ key = f"tensor.{len(arrays)}"
19
+ arrays[key] = item.detach().cpu().numpy().copy()
20
+ return {"tensor": key}
21
+ if isinstance(item, dict):
22
+ return {
23
+ "dict": [
24
+ [_encode(k, arrays), _encode(v, arrays)]
25
+ for k, v in cast(dict[Any, Any], item).items()
26
+ ]
27
+ }
28
+ if isinstance(item, tuple):
29
+ return {"tuple": [_encode(x, arrays) for x in cast(list[Any], item)]}
30
+ if isinstance(item, list):
31
+ return {"list": [_encode(x, arrays) for x in cast(list[Any], item)]}
32
+ if item is None or type(item) in (str, bool, int, float):
33
+ return item
34
+ raise PrivacyError("unsupported research state value")
35
+
36
+
37
+ def encode_state(value: Any) -> dict[str, np.ndarray]:
38
+ arrays: dict[str, np.ndarray] = {}
39
+
40
+ payload = json.dumps(
41
+ _encode(value, arrays), allow_nan=False, separators=(",", ":")
42
+ ).encode()
43
+ if len(payload) > 1024 * 1024:
44
+ raise PrivacyError("research state metadata exceeds bound")
45
+ arrays["structure"] = np.frombuffer(payload, dtype=np.uint8).copy()
46
+ return arrays
47
+
48
+
49
+ def _decode(
50
+ item: Any, arrays: dict[str, np.ndarray], used: set[str], depth: int = 0
51
+ ) -> Any:
52
+ if depth > 32:
53
+ raise PrivacyError("research state nesting exceeds bound")
54
+ if isinstance(item, dict):
55
+ if len(cast(dict[Any, Any], item)) != 1:
56
+ raise PrivacyError("invalid research state tag")
57
+ tag, values = next(iter(cast(dict[Any, Any], item).items()))
58
+ if tag == "tensor":
59
+ if values not in arrays or values in used:
60
+ raise PrivacyError("invalid research state tensor")
61
+ used.add(values)
62
+ value = arrays[values]
63
+ if value.dtype.kind not in "biuf" or not np.isfinite(value).all():
64
+ raise PrivacyError("invalid research state array")
65
+ return torch.from_numpy(value.copy()) # pyright: ignore[reportUnknownMemberType]
66
+ if tag == "dict":
67
+ return {
68
+ _decode(k, arrays, used, depth + 1): _decode(v, arrays, used, depth + 1)
69
+ for k, v in values
70
+ }
71
+ if tag in ("tuple", "list"):
72
+ result = [_decode(x, arrays, used, depth + 1) for x in values]
73
+ return tuple(result) if tag == "tuple" else result
74
+ raise PrivacyError("invalid research state tag")
75
+ if item is None or type(item) in (str, bool, int, float):
76
+ return item
77
+ raise PrivacyError("invalid research state value")
78
+
79
+
80
+ def decode_state(arrays: dict[str, np.ndarray]) -> Any:
81
+ metadata = arrays.get("structure")
82
+ if (
83
+ metadata is None
84
+ or metadata.dtype != np.uint8
85
+ or metadata.ndim != 1
86
+ or metadata.size > 1024 * 1024
87
+ ):
88
+ raise PrivacyError("invalid research state structure")
89
+ used = {"structure"}
90
+
91
+ try:
92
+ result = _decode(json.loads(metadata.tobytes()), arrays, used)
93
+ except Exception:
94
+ raise PrivacyError("invalid research checkpoint encoding") from None
95
+ if set(arrays) != used:
96
+ raise PrivacyError("unexpected research checkpoint arrays")
97
+ return result