dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,464 @@
1
+ """Application-owned task semantics shared by ordinary and private training.
2
+
3
+ Callbacks are trusted application code. They may construct the forward/loss
4
+ computation, but private execution owns sampling, backward and optimizer steps.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import hashlib
10
+ import importlib
11
+ import json
12
+ import re
13
+ from collections.abc import Callable, Mapping, Sequence
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+ from typing import Any, Literal, cast
17
+
18
+ import torch
19
+ from torch import nn
20
+
21
+ from dromeus.errors import TaskError
22
+ from dromeus.manifests.canonical import file_sha256
23
+ from dromeus.manifests.models import (
24
+ AccountedInitializationProvenance,
25
+ GradientBackend,
26
+ PrivateOptimizerSpec,
27
+ TaskTrainingSpec,
28
+ )
29
+ from dromeus.training.optimizer_factory import OptimizerRegistration
30
+ from dromeus.training.versions import torch_release
31
+
32
+ Batch = Any
33
+ CollateRecords = Callable[[Sequence[Any]], Batch]
34
+ RecordLoss = Callable[[nn.Module, Batch], torch.Tensor]
35
+ Initialization = Literal[
36
+ "public_untrained", "public_pretrained", "accounted_private_pretrained"
37
+ ]
38
+
39
+
40
+ def _collate_tensors(records: Sequence[Any]) -> Batch:
41
+ """Stack tensor records without changing tuple/list semantics."""
42
+ first = records[0]
43
+ if isinstance(first, torch.Tensor):
44
+ if not all(isinstance(item, torch.Tensor) for item in records):
45
+ raise TaskError("record tensor structure changed")
46
+ return torch.stack(cast(list[torch.Tensor], list(records)))
47
+ if type(first) is dict:
48
+ keys = tuple(cast(dict[str, Any], first))
49
+ if not all(type(item) is dict and tuple(item) == keys for item in records):
50
+ raise TaskError("record mapping structure changed")
51
+ return {key: _collate_tensors([item[key] for item in records]) for key in keys}
52
+ if type(first) in (tuple, list):
53
+ count = len(first)
54
+ if not all(
55
+ type(item) is type(first) and len(item) == count for item in records
56
+ ):
57
+ raise TaskError("record sequence structure changed")
58
+ values = [_collate_tensors([item[i] for item in records]) for i in range(count)]
59
+ return tuple(values) if type(first) is tuple else values
60
+ raise TaskError("records must be tensor trees; constants need a task adapter")
61
+
62
+
63
+ def _map_tensors(
64
+ value: Batch, operation: Callable[[torch.Tensor], torch.Tensor]
65
+ ) -> Batch:
66
+ if isinstance(value, torch.Tensor):
67
+ return operation(value)
68
+ if type(value) is dict:
69
+ return {
70
+ k: _map_tensors(v, operation)
71
+ for k, v in cast(dict[str, Any], value).items()
72
+ }
73
+ mapped = [_map_tensors(item, operation) for item in value]
74
+ return tuple(mapped) if type(value) is tuple else mapped
75
+
76
+
77
+ def _batch_tensors(value: Batch, *, path: str = "batch") -> list[torch.Tensor]:
78
+ if isinstance(value, torch.Tensor):
79
+ if value.ndim == 0:
80
+ raise TaskError(f"{path}: tensor requires record axis 0")
81
+ if value.layout != torch.strided or value.is_complex():
82
+ raise TaskError(f"{path}: dense real tensors required")
83
+ if not torch.isfinite(value).all():
84
+ raise TaskError(f"{path}: nonfinite batch tensor")
85
+ return [value]
86
+ if type(value) is dict:
87
+ mapping = cast(dict[Any, Any], value)
88
+ if not mapping or not all(type(k) is str for k in mapping):
89
+ raise TaskError(f"{path}: nonempty string-keyed mapping required")
90
+ leaves: list[torch.Tensor] = []
91
+ for key, item in mapping.items():
92
+ leaves.extend(_batch_tensors(item, path=f"{path}.{key}"))
93
+ return leaves
94
+ if type(value) in (tuple, list) and len(value):
95
+ leaves = []
96
+ for index, item in enumerate(value):
97
+ leaves.extend(_batch_tensors(item, path=f"{path}[{index}]"))
98
+ return leaves
99
+ raise TaskError(
100
+ f"{path}: only tensor trees supported; declare constants in task code"
101
+ )
102
+
103
+
104
+ def _schema(value: Batch) -> Any:
105
+ if isinstance(value, torch.Tensor):
106
+ return ("tensor", str(value.dtype), tuple(value.shape[1:]))
107
+ if type(value) is dict:
108
+ return (
109
+ "dict",
110
+ tuple((k, _schema(v)) for k, v in cast(dict[str, Any], value).items()),
111
+ )
112
+ return (type(value).__name__, tuple(_schema(item) for item in value))
113
+
114
+
115
+ @dataclass(frozen=True)
116
+ class TensorBatchAdapter:
117
+ """Fixed trailing shapes and record axis 0 for nested tensor batches.
118
+
119
+ ``example`` is an explicitly public/synthetic batch of at least two records.
120
+ It describes empty batches without touching the protected dataset. Collation
121
+ must preserve each selected record and may not use cross-record statistics.
122
+ Scalars and non-batched constants belong in trusted task code for this profile.
123
+ """
124
+
125
+ example: Batch
126
+ definition: str
127
+ collate_fn: CollateRecords = _collate_tensors
128
+ _schema: Any = field(init=False, repr=False)
129
+
130
+ def __post_init__(self) -> None:
131
+ if not self.definition.strip():
132
+ raise TaskError("batch definition is required")
133
+ tensors = _batch_tensors(self.example)
134
+ count = len(tensors[0])
135
+ if count < 2 or any(len(tensor) != count for tensor in tensors):
136
+ raise TaskError(
137
+ "public batch fixture requires at least two aligned records"
138
+ )
139
+ copied = _map_tensors(self.example, lambda t: t.detach().clone())
140
+ object.__setattr__(self, "example", copied)
141
+ object.__setattr__(self, "_schema", _schema(copied))
142
+
143
+ @property
144
+ def definition_hash(self) -> str:
145
+ """Bind declared semantics and tensor schema, never public probe values."""
146
+ payload = json.dumps(
147
+ {"definition": self.definition, "schema": self._schema},
148
+ sort_keys=True,
149
+ separators=(",", ":"),
150
+ )
151
+ return hashlib.sha256(payload.encode()).hexdigest()
152
+
153
+ def size(self, batch: Batch) -> int:
154
+ tensors = _batch_tensors(batch)
155
+ if _schema(batch) != self._schema:
156
+ raise TaskError("batch structure, dtype or trailing shape changed")
157
+ count = len(tensors[0])
158
+ if any(len(tensor) != count for tensor in tensors):
159
+ raise TaskError("batch tensors disagree on record axis")
160
+ return count
161
+
162
+ def collate(self, records: Sequence[Any]) -> Batch:
163
+ batch = self.collate_fn(records) if records else self.empty()
164
+ if self.size(batch) != len(records):
165
+ raise TaskError("collation changed the sampled record count")
166
+ return batch
167
+
168
+ def to_device(self, batch: Batch, device: torch.device | str) -> Batch:
169
+ self.size(batch)
170
+ return _map_tensors(batch, lambda t: t.to(device))
171
+
172
+ def slice(self, batch: Batch, start: int, end: int) -> Batch:
173
+ count = self.size(batch)
174
+ if not 0 <= start <= end <= count:
175
+ raise TaskError("invalid physical batch record range")
176
+ return _map_tensors(batch, lambda t: t[start:end])
177
+
178
+ def empty(self) -> Batch:
179
+ return _map_tensors(self.example, lambda t: t[:0].detach().clone())
180
+
181
+
182
+ def validate_record_losses(losses: object, count: int) -> torch.Tensor:
183
+ """Check the application returns one finite differentiable loss per record."""
184
+ if (
185
+ not isinstance(losses, torch.Tensor)
186
+ or losses.shape != (count,)
187
+ or not losses.is_floating_point()
188
+ or not torch.isfinite(losses).all()
189
+ or (count > 0 and not losses.requires_grad)
190
+ ):
191
+ raise TaskError(
192
+ "task loss requires finite differentiable per-record values [B]"
193
+ )
194
+ return losses
195
+
196
+
197
+ @dataclass(frozen=True)
198
+ class AccountedInitializationEvidence:
199
+ """Locally verified ancestry at the trusted application boundary.
200
+
201
+ ``verify_parent`` must verify the bound parent audit and every accounting
202
+ authority, including unchanged local lineage prefixes, and return True.
203
+ Core code verifies the actual factory tensors against the bound model file;
204
+ it cannot interpret application-specific audit formats or attest callbacks.
205
+ This admits weights only, never optimizer, RNG or checkpoint restoration.
206
+ """
207
+
208
+ provenance: AccountedInitializationProvenance
209
+ model_path: Path
210
+ verify_parent: Callable[[AccountedInitializationProvenance], bool]
211
+
212
+ def verify_source(self) -> None:
213
+ if self.verify_parent(self.provenance) is not True:
214
+ raise TaskError("accounted initialization ancestry was not verified")
215
+ if (
216
+ not self.model_path.is_file()
217
+ or self.model_path.is_symlink()
218
+ or file_sha256(self.model_path) != self.provenance.parent_model_sha256
219
+ ):
220
+ raise TaskError("accounted initialization model source differs")
221
+
222
+ def verify_model(self, model: nn.Module) -> None:
223
+ from dromeus.training.private_model import registered_model_state
224
+
225
+ safe_open: Any = importlib.import_module("safetensors").safe_open
226
+ if file_sha256(self.model_path) != self.provenance.parent_model_sha256:
227
+ raise TaskError(
228
+ "accounted initialization source changed during preparation"
229
+ )
230
+
231
+ def equal_chunks(actual: torch.Tensor, expected: torch.Tensor) -> bool:
232
+ # Bound device-to-host copies, including noncontiguous parameters.
233
+ if actual.numel() <= 262144:
234
+ value = actual.detach().cpu()
235
+ return bool(torch.isfinite(value).all()) and torch.equal(
236
+ value, expected
237
+ )
238
+ width = actual.numel() // actual.shape[0]
239
+ rows = max(1, 262144 // width)
240
+ for start in range(0, actual.shape[0], rows):
241
+ left, right = (
242
+ actual[start : start + rows],
243
+ expected[start : start + rows],
244
+ )
245
+ if left.numel() > 262144:
246
+ left, right = left.squeeze(0), right.squeeze(0)
247
+ if not equal_chunks(left, right):
248
+ return False
249
+ return True
250
+
251
+ state = registered_model_state(model)
252
+ with safe_open(self.model_path, framework="pt", device="cpu") as source:
253
+ if set(source.keys()) != set(state):
254
+ raise TaskError("accounted initialization tensor inventory differs")
255
+ for name, actual in state.items():
256
+ expected = cast(torch.Tensor, source.get_tensor(name))
257
+ if (
258
+ actual.dtype != expected.dtype
259
+ or actual.shape != expected.shape
260
+ or not equal_chunks(actual, expected)
261
+ ):
262
+ raise TaskError("accounted initialization model tensors differ")
263
+ # Detect source replacement or mutation during the factory invocation.
264
+ if file_sha256(self.model_path) != self.provenance.parent_model_sha256:
265
+ raise TaskError(
266
+ "accounted initialization source changed during preparation"
267
+ )
268
+
269
+
270
+ @dataclass(frozen=True)
271
+ class Optimizer:
272
+ """A built-in PyTorch optimizer, identical on every node.
273
+
274
+ Build one with `Optimizer.sgd`, `Optimizer.adam` or `Optimizer.adamw`. For any
275
+ other optimizer, pass an `OptimizerRegistration` to `TrainingTask` instead.
276
+ """
277
+
278
+ name: Literal["sgd", "adam", "adamw"]
279
+ learning_rate: float
280
+ weight_decay: float = 0.0
281
+ momentum: float = 0.0
282
+ dampening: float = 0.0
283
+ nesterov: bool = False
284
+ betas: tuple[float, float] = (0.9, 0.999)
285
+ eps: float = 1e-8
286
+
287
+ @classmethod
288
+ def sgd(
289
+ cls,
290
+ learning_rate: float,
291
+ *,
292
+ momentum: float = 0.0,
293
+ dampening: float = 0.0,
294
+ nesterov: bool = False,
295
+ weight_decay: float = 0.0,
296
+ ) -> Optimizer:
297
+ return cls(
298
+ "sgd",
299
+ learning_rate,
300
+ weight_decay=weight_decay,
301
+ momentum=momentum,
302
+ dampening=dampening,
303
+ nesterov=nesterov,
304
+ )
305
+
306
+ @classmethod
307
+ def adam(
308
+ cls,
309
+ learning_rate: float,
310
+ *,
311
+ betas: tuple[float, float] = (0.9, 0.999),
312
+ eps: float = 1e-8,
313
+ weight_decay: float = 0.0,
314
+ ) -> Optimizer:
315
+ return cls("adam", learning_rate, weight_decay, betas=betas, eps=eps)
316
+
317
+ @classmethod
318
+ def adamw(
319
+ cls,
320
+ learning_rate: float,
321
+ *,
322
+ betas: tuple[float, float] = (0.9, 0.999),
323
+ eps: float = 1e-8,
324
+ weight_decay: float = 0.01,
325
+ ) -> Optimizer:
326
+ return cls("adamw", learning_rate, weight_decay, betas=betas, eps=eps)
327
+
328
+ def specification(self, parameter_names: tuple[str, ...]) -> PrivateOptimizerSpec:
329
+ """Bind this optimizer to the model's trainable parameters and Torch."""
330
+ try:
331
+ return PrivateOptimizerSpec(
332
+ name=self.name,
333
+ torch_version=torch_release(),
334
+ parameter_names=parameter_names,
335
+ learning_rate=self.learning_rate,
336
+ weight_decay=self.weight_decay,
337
+ momentum=self.momentum,
338
+ dampening=self.dampening,
339
+ nesterov=self.nesterov,
340
+ betas=self.betas,
341
+ numerical_epsilon=self.eps,
342
+ )
343
+ except ValueError as error:
344
+ raise TaskError(f"invalid {self.name} optimizer: {error}") from error
345
+
346
+
347
+ @dataclass(frozen=True, kw_only=True)
348
+ class TrainingTask:
349
+ """What every node trains: the model, batches, loss and optimizer.
350
+
351
+ Every node builds an identical task. It holds shared definitions only; each
352
+ node's own records are passed separately when the node runs. The callbacks
353
+ are trusted code: they compute the forward pass and per-record loss, while
354
+ Dromeus owns sampling, backward and optimizer steps.
355
+ """
356
+
357
+ model_factory: Callable[[], nn.Module]
358
+ batch_adapter: TensorBatchAdapter
359
+ loss_per_record: RecordLoss
360
+ model_definition: str
361
+ task_definition: str
362
+ loss_definition: str
363
+ implementation_hash: str
364
+ optimizer: Optimizer | OptimizerRegistration
365
+ batch_size: int
366
+ record_unit: str = "example"
367
+ schedule: Literal["constant", "linear_decay"] = "constant"
368
+ initialization: Literal["public_untrained", "public_pretrained"] = (
369
+ "public_untrained"
370
+ )
371
+ public_evaluate: Callable[[nn.Module], Mapping[str, float]] | None = None
372
+ initialization_evidence: AccountedInitializationEvidence | None = None
373
+ gradient_backend: GradientBackend = "opacus_hooks"
374
+
375
+ def __post_init__(self) -> None:
376
+ if any(
377
+ not value.strip()
378
+ for value in (
379
+ self.model_definition,
380
+ self.task_definition,
381
+ self.loss_definition,
382
+ self.record_unit,
383
+ )
384
+ ):
385
+ raise TaskError(
386
+ "model, task, loss and protected record definitions required"
387
+ )
388
+ if re.fullmatch(r"[0-9a-f]{64}", self.implementation_hash) is None:
389
+ raise TaskError("task implementation requires a SHA-256 identity")
390
+ if self.batch_size <= 0:
391
+ raise TaskError("batch size must be positive")
392
+ evidence = self.initialization_evidence
393
+ if evidence is not None:
394
+ factory = self.model_factory
395
+
396
+ def verified_factory() -> nn.Module:
397
+ evidence.verify_source()
398
+ model = factory()
399
+ evidence.verify_model(model)
400
+ return model
401
+
402
+ object.__setattr__(self, "model_factory", verified_factory)
403
+
404
+ @property
405
+ def optimizer_registration(self) -> OptimizerRegistration | None:
406
+ """The custom optimizer registration, or None for a built-in optimizer."""
407
+ optimizer = self.optimizer
408
+ return optimizer if isinstance(optimizer, OptimizerRegistration) else None
409
+
410
+ @property
411
+ def initialization_mode(self) -> Initialization:
412
+ if self.initialization_evidence is not None:
413
+ return "accounted_private_pretrained"
414
+ return self.initialization
415
+
416
+ def check_initialization(
417
+ self,
418
+ initialization: str,
419
+ provenance: AccountedInitializationProvenance | None,
420
+ ) -> None:
421
+ evidence = self.initialization_evidence
422
+ if initialization == "accounted_private_pretrained":
423
+ if (
424
+ evidence is None
425
+ or provenance is None
426
+ or evidence.provenance != provenance
427
+ ):
428
+ raise TaskError(
429
+ "matching locally verified initialization evidence required"
430
+ )
431
+ elif evidence is not None or provenance is not None:
432
+ raise TaskError("accounted initialization cannot be declared public")
433
+
434
+ def check_definition(self, specification: TaskTrainingSpec) -> None:
435
+ """Bind trusted stateless task callbacks to the group's declared semantics.
436
+
437
+ Stateful task transforms need a separately versioned state contract.
438
+ Hash agreement does not attest the behavior of arbitrary Python code.
439
+ """
440
+ self.check_initialization(
441
+ specification.initialization, specification.initialization_provenance
442
+ )
443
+ expected = {
444
+ "task": (
445
+ hashlib.sha256(self.task_definition.encode()).hexdigest(),
446
+ specification.task_definition_hash,
447
+ ),
448
+ "implementation": (
449
+ self.implementation_hash,
450
+ specification.implementation_hash,
451
+ ),
452
+ "batch": (
453
+ self.batch_adapter.definition_hash,
454
+ specification.batch_definition_hash,
455
+ ),
456
+ "loss": (
457
+ hashlib.sha256(self.loss_definition.encode()).hexdigest(),
458
+ specification.loss_definition_hash,
459
+ ),
460
+ "record unit": (self.record_unit, specification.record_unit),
461
+ }
462
+ for name, (actual, declared) in expected.items():
463
+ if actual != declared:
464
+ raise TaskError(f"task {name} definition differs from shared contract")