dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/application.py ADDED
@@ -0,0 +1,523 @@
1
+ """Turn a training task into a shared draft and a node's prepared training."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import importlib.metadata
7
+ import inspect
8
+ import json
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from types import ModuleType
12
+ from typing import Any, Literal
13
+
14
+ import yaml
15
+ from pydantic import ValidationError
16
+ from torch.utils.data import Dataset
17
+
18
+ from dromeus.errors import ConfigError, DraftError, PrivacyError, TaskError
19
+ from dromeus.manifests.canonical import (
20
+ canonical_hash,
21
+ parse_draft_yaml,
22
+ validate_sealed_draft,
23
+ )
24
+ from dromeus.manifests.models import (
25
+ ApplicationTaskContract,
26
+ ArtifactCodec,
27
+ ConsensusSketchConfig,
28
+ DivergencePolicy,
29
+ DraftRunSpec,
30
+ EnvironmentFingerprint,
31
+ NoLoCoConfig,
32
+ PrivacyPolicy,
33
+ TaskTrainingSpec,
34
+ TensorSchema,
35
+ TransportLimits,
36
+ )
37
+ from dromeus.membership.formation import FormationResult
38
+ from dromeus.runtime import (
39
+ FailureConfig,
40
+ MetricsService,
41
+ TrainingConfig,
42
+ build_algorithm,
43
+ )
44
+ from dromeus.training.privacy import ReservationStore
45
+ from dromeus.training.private_model import describe_private_model
46
+ from dromeus.training.private_trainer import PrivateTrainer
47
+ from dromeus.training.state import InitialCheckpoint
48
+ from dromeus.training.task import Optimizer, TrainingTask
49
+ from dromeus.training.trainer import PyTorchTrainer
50
+ from dromeus.training.versions import torch_release
51
+
52
+ DEFAULT_TRANSPORT = TransportLimits(
53
+ max_payload_bytes=8 * 1024 * 1024,
54
+ max_retries=3,
55
+ retry_timeout_seconds=5.0,
56
+ chunk_size_bytes=64 * 1024,
57
+ window_size=2,
58
+ )
59
+
60
+
61
+ def definition_hash(definition: str) -> str:
62
+ """Identify an application-supplied semantic definition."""
63
+ return hashlib.sha256(definition.encode()).hexdigest()
64
+
65
+
66
+ def code_hash(*modules: ModuleType | Path) -> str:
67
+ """Identify the code that implements a task, for `implementation_hash`.
68
+
69
+ Hashes the source files of the given modules (or paths) in order. Every node
70
+ must install identical files. Agreement shows the files match; it does not
71
+ verify what the code does.
72
+ """
73
+ if not modules:
74
+ raise TaskError("code_hash needs at least one module or path")
75
+ digest = hashlib.sha256()
76
+ for module in modules:
77
+ if isinstance(module, Path):
78
+ path = module
79
+ else:
80
+ source = inspect.getsourcefile(module)
81
+ if source is None:
82
+ raise TaskError(f"module has no source file: {module.__name__}")
83
+ path = Path(source)
84
+ data = path.read_bytes()
85
+ digest.update(len(data).to_bytes(8, "big"))
86
+ digest.update(data)
87
+ return digest.hexdigest()
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class NoLoCo:
92
+ """NoLoCo outer-step settings shared by the group.
93
+
94
+ Set `top_k_fraction` to compress updates: the outer gradient sends its largest
95
+ entries as int8 and slow weights are sent as int8. None sends exact values.
96
+ """
97
+
98
+ alpha: float = 0.5
99
+ beta: float = 0.7
100
+ gamma: float = 0.7
101
+ top_k_fraction: float | None = None
102
+
103
+ def codecs(self) -> tuple[ArtifactCodec, ...]:
104
+ if self.top_k_fraction is None:
105
+ return (
106
+ ArtifactCodec(artifact_name="outer_gradient", codec_id="identity"),
107
+ ArtifactCodec(artifact_name="slow_weights", codec_id="identity"),
108
+ )
109
+ return (
110
+ ArtifactCodec(
111
+ artifact_name="outer_gradient",
112
+ codec_id="topk-bitmap-int8",
113
+ top_k_fraction=self.top_k_fraction,
114
+ lossy_allowed=True,
115
+ ),
116
+ ArtifactCodec(
117
+ artifact_name="slow_weights",
118
+ codec_id="dense-int8",
119
+ lossy_allowed=True,
120
+ ),
121
+ )
122
+
123
+
124
+ @dataclass(frozen=True)
125
+ class Privacy:
126
+ """Record-level differential privacy for every node's local training.
127
+
128
+ The privacy horizon is the run's total local steps (rounds x local steps).
129
+ Use `randomness="reproducible_public_benchmark"` only for seeded research
130
+ runs on public data; real private runs use the default `"secure"`.
131
+ """
132
+
133
+ max_grad_norm: float
134
+ noise_multiplier: float
135
+ delta: float
136
+ epsilon_limit: float | None = None
137
+ randomness: Literal["secure", "reproducible_public_benchmark"] = "secure"
138
+
139
+ def policy(self, max_logical_steps: int) -> PrivacyPolicy:
140
+ try:
141
+ return PrivacyPolicy(
142
+ max_grad_norm=self.max_grad_norm,
143
+ noise_multiplier=self.noise_multiplier,
144
+ delta=self.delta,
145
+ max_logical_steps=max_logical_steps,
146
+ epsilon_limit=self.epsilon_limit,
147
+ randomness_profile=self.randomness,
148
+ )
149
+ except ValidationError as error:
150
+ raise PrivacyError(f"invalid privacy settings: {error}") from error
151
+
152
+
153
+ @dataclass(frozen=True)
154
+ class LocalData:
155
+ """This node's own records and limits; never sent to peers.
156
+
157
+ `data_identity` names this exact copy of the records (for example a file
158
+ hash); checkpoints and privacy spending are bound to it. A private run needs
159
+ `ledger`, the node's durable privacy-spending record. `physical_batch_size`
160
+ caps how many records are processed at once. `research_seed` makes ordinary
161
+ or public-benchmark runs reproducible; secure private runs refuse it.
162
+ """
163
+
164
+ dataset: Dataset[Any]
165
+ data_identity: str
166
+ ledger: ReservationStore | None = None
167
+ physical_batch_size: int = 1
168
+ research_seed: int | None = None
169
+
170
+
171
+ def installed_versions() -> tuple[str, str]:
172
+ """Dromeus and PyTorch versions installed on this machine."""
173
+ return importlib.metadata.version("dromeus"), torch_release()
174
+
175
+
176
+ def describe_task(
177
+ task: TrainingTask, *, sampling: Literal["poisson", "shuffle"]
178
+ ) -> TaskTrainingSpec:
179
+ """The shared training specification for a task, without opening records."""
180
+ evidence = task.initialization_evidence
181
+ provenance = evidence.provenance if evidence is not None else None
182
+ task.check_initialization(task.initialization_mode, provenance)
183
+ model = task.model_factory()
184
+ report = describe_private_model(model, gradient_backend=task.gradient_backend)
185
+ optimizer = task.optimizer
186
+ if isinstance(optimizer, Optimizer):
187
+ selected = optimizer.specification(report.trainable_parameter_names)
188
+ else:
189
+ selected = optimizer.describe(model)
190
+ try:
191
+ return TaskTrainingSpec(
192
+ optimizer=selected,
193
+ batch_size=task.batch_size,
194
+ sampling=sampling,
195
+ schedule=task.schedule,
196
+ initialization=task.initialization_mode,
197
+ initialization_provenance=provenance,
198
+ evaluation_provenance="public_fixed_holdout"
199
+ if task.public_evaluate
200
+ else "disabled",
201
+ task_definition_hash=definition_hash(task.task_definition),
202
+ implementation_hash=task.implementation_hash,
203
+ batch_definition_hash=task.batch_adapter.definition_hash,
204
+ loss_definition_hash=definition_hash(task.loss_definition),
205
+ record_unit=task.record_unit,
206
+ gradient_backend=report.gradient_backend,
207
+ trainable_parameter_names=report.trainable_parameter_names,
208
+ parameter_aliases=report.parameter_aliases,
209
+ public_buffer_hashes=dict(report.public_buffer_hashes),
210
+ frozen_parameter_hashes=dict(report.frozen_parameter_hashes),
211
+ )
212
+ except ValidationError as error:
213
+ raise TaskError(f"task cannot be described: {error}") from error
214
+
215
+
216
+ def draft_for_task(
217
+ task: TrainingTask,
218
+ *,
219
+ run_id: str,
220
+ participants: int,
221
+ rounds: int,
222
+ local_steps: int,
223
+ axl_version: str,
224
+ noloco: NoLoCo | None = None,
225
+ privacy: Privacy | None = None,
226
+ transport: TransportLimits | None = None,
227
+ divergence: DivergencePolicy | None = None,
228
+ seed: int = 0,
229
+ ) -> DraftRunSpec:
230
+ """Build the shared draft for a task; every hash is computed here."""
231
+ noloco = noloco or NoLoCo()
232
+ dromeus_version, pytorch_version = installed_versions()
233
+ model_hash = definition_hash(task.model_definition)
234
+ training = describe_task(task, sampling="poisson" if privacy else "shuffle")
235
+ try:
236
+ return DraftRunSpec(
237
+ run_id=run_id,
238
+ model_id="application",
239
+ model_definition_hash=model_hash,
240
+ dataset=ApplicationTaskContract(
241
+ task_id="application",
242
+ definition_hash=training.task_definition_hash,
243
+ ),
244
+ expected_participant_count=participants,
245
+ environment=EnvironmentFingerprint(
246
+ dromeus_version=dromeus_version,
247
+ pytorch_version=pytorch_version,
248
+ axl_version=axl_version,
249
+ model_definition_hash=model_hash,
250
+ ),
251
+ local_steps=local_steps,
252
+ round_count=rounds,
253
+ peer_scheduler_seed=seed,
254
+ codec_id="safetensors",
255
+ transport=transport or DEFAULT_TRANSPORT,
256
+ consensus_sketch=ConsensusSketchConfig(seed=seed),
257
+ training=training,
258
+ algorithm_config=NoLoCoConfig(
259
+ alpha=noloco.alpha,
260
+ beta=noloco.beta,
261
+ gamma=noloco.gamma,
262
+ inner_steps=local_steps,
263
+ ),
264
+ artifact_codecs=noloco.codecs(),
265
+ privacy=privacy.policy(local_steps * rounds) if privacy else None,
266
+ divergence_detection=divergence,
267
+ )
268
+ except ValidationError as error:
269
+ raise DraftError(f"invalid draft settings: {error}") from error
270
+
271
+
272
+ def make_draft(
273
+ task: TrainingTask,
274
+ path: Path | str,
275
+ *,
276
+ run_id: str,
277
+ participants: int,
278
+ rounds: int,
279
+ local_steps: int,
280
+ axl_version: str,
281
+ noloco: NoLoCo | None = None,
282
+ privacy: Privacy | None = None,
283
+ transport: TransportLimits | None = None,
284
+ divergence: DivergencePolicy | None = None,
285
+ seed: int = 0,
286
+ ) -> str:
287
+ """Write the group's shared draft for `task` to `path`; return its hash.
288
+
289
+ Run once, by one person, then give the file to every node. Every hash and
290
+ the Dromeus and PyTorch versions are filled in from this machine.
291
+ """
292
+ draft = draft_for_task(
293
+ task,
294
+ run_id=run_id,
295
+ participants=participants,
296
+ rounds=rounds,
297
+ local_steps=local_steps,
298
+ axl_version=axl_version,
299
+ noloco=noloco,
300
+ privacy=privacy,
301
+ transport=transport,
302
+ divergence=divergence,
303
+ seed=seed,
304
+ )
305
+ text = yaml.safe_dump(
306
+ draft.model_dump(mode="json", exclude_none=True), sort_keys=False
307
+ )
308
+ Path(path).write_text(text, encoding="utf-8")
309
+ return canonical_hash(draft)
310
+
311
+
312
+ def load_draft(path: Path | str) -> DraftRunSpec:
313
+ """Read and validate a draft file."""
314
+ try:
315
+ return parse_draft_yaml(Path(path))
316
+ except OSError as error:
317
+ raise ConfigError(f"cannot read draft {path}: {error}") from error
318
+
319
+
320
+ def check_environment(draft: DraftRunSpec) -> None:
321
+ """Refuse a draft made for a different Dromeus or PyTorch version."""
322
+ dromeus_version, pytorch_version = installed_versions()
323
+ for name, declared, installed in (
324
+ ("Dromeus", draft.environment.dromeus_version, dromeus_version),
325
+ ("PyTorch", draft.environment.pytorch_version, pytorch_version),
326
+ ):
327
+ if declared != installed:
328
+ raise DraftError(
329
+ f"draft requires {name} {declared}; this machine has {installed}"
330
+ )
331
+
332
+
333
+ def require_task_matches(draft: DraftRunSpec, task: TrainingTask) -> None:
334
+ """Refuse a draft that was not made from this task."""
335
+ if definition_hash(task.model_definition) != draft.model_definition_hash:
336
+ raise DraftError("draft model definition differs from this task")
337
+ local = describe_task(task, sampling=draft.training.sampling)
338
+ if local != draft.training:
339
+ shared = draft.training.model_dump()
340
+ differing = sorted(
341
+ name for name, value in local.model_dump().items() if shared[name] != value
342
+ )
343
+ raise DraftError(
344
+ "draft training differs from this task: " + ", ".join(differing)
345
+ )
346
+
347
+
348
+ def check_draft(path: Path | str, task: TrainingTask) -> None:
349
+ """Check a draft file against this task and this machine before running.
350
+
351
+ Raises `DraftError` naming what differs. Every node should run this check;
352
+ `Node.run` performs it too.
353
+ """
354
+ draft = load_draft(path)
355
+ check_environment(draft)
356
+ require_task_matches(draft, task)
357
+
358
+
359
+ def prepare_task(
360
+ *,
361
+ draft: DraftRunSpec,
362
+ task: TrainingTask,
363
+ local: LocalData,
364
+ evaluation_interval: int = 1,
365
+ ) -> PreparedApplication:
366
+ """Prepare this node's training for the group's draft."""
367
+ require_task_matches(draft, task)
368
+ specification = draft.training
369
+ trainer: PyTorchTrainer
370
+ if draft.privacy is not None:
371
+ if local.ledger is None:
372
+ raise PrivacyError("private task requires a persistent privacy ledger")
373
+ trainer = PrivateTrainer(
374
+ task=task,
375
+ dataset=local.dataset,
376
+ policy=draft.privacy,
377
+ specification=specification,
378
+ run_id=draft.run_id,
379
+ local_data_identity=local.data_identity,
380
+ reservation_store=local.ledger,
381
+ max_physical_batch_size=local.physical_batch_size,
382
+ research_seed=local.research_seed,
383
+ )
384
+ else:
385
+ from dromeus.training.ordinary_task import OrdinaryTaskTrainer
386
+
387
+ trainer = OrdinaryTaskTrainer(
388
+ task=task,
389
+ dataset=local.dataset,
390
+ specification=specification,
391
+ local_data_identity=local.data_identity,
392
+ max_logical_steps=draft.local_steps * draft.round_count,
393
+ research_seed=local.research_seed if local.research_seed is not None else 0,
394
+ max_physical_batch_size=local.physical_batch_size,
395
+ )
396
+ return prepare_trainer(
397
+ draft=draft,
398
+ trainer=trainer,
399
+ model_definition=task.model_definition,
400
+ evaluation_interval=evaluation_interval,
401
+ )
402
+
403
+
404
+ @dataclass(frozen=True, slots=True)
405
+ class PreparedApplication:
406
+ """A validated application workload ready for fixed-group formation."""
407
+
408
+ _draft_hash: str
409
+ _trainer: PyTorchTrainer
410
+ _model_definition: str
411
+ _evaluation_interval: int
412
+
413
+ @property
414
+ def tensor_schema(self) -> TensorSchema:
415
+ return self._trainer.tensor_schema
416
+
417
+ def validate_draft(self, draft: DraftRunSpec) -> None:
418
+ if draft.privacy is not None:
419
+ if (
420
+ not isinstance(self._trainer, PrivateTrainer)
421
+ or self._trainer.policy != draft.privacy
422
+ or self._trainer.specification != draft.training
423
+ ):
424
+ raise PrivacyError("private preparation does not match draft")
425
+ self._trainer.weights()
426
+ if canonical_hash(draft) != self._draft_hash:
427
+ raise DraftError("prepared application draft does not match")
428
+
429
+ def create_initial_checkpoint(self, path: Path) -> InitialCheckpoint:
430
+ return self._trainer.create_initial_checkpoint(
431
+ path, model_definition=self._model_definition
432
+ )
433
+
434
+ @property
435
+ def private_trainer(self) -> PrivateTrainer | None:
436
+ return self._trainer if isinstance(self._trainer, PrivateTrainer) else None
437
+
438
+ def export_model(self, path: Path) -> None:
439
+ """Explicit model-only private export; training archives remain local."""
440
+ if not isinstance(self._trainer, PrivateTrainer):
441
+ raise PrivacyError("model-only private export requires a private adapter")
442
+ self._trainer.export_model(path)
443
+
444
+ def build_config(
445
+ self,
446
+ *,
447
+ result: FormationResult,
448
+ local_public_key: str,
449
+ run_root: Path,
450
+ metrics_publisher: MetricsService | None = None,
451
+ ) -> TrainingConfig:
452
+ validate_sealed_draft(result.manifest, expected_draft_hash=self._draft_hash)
453
+ if canonical_hash(result.manifest) != result.manifest_hash:
454
+ raise DraftError("formed manifest hash does not match its contents")
455
+ if result.manifest.tensor_schema != self.tensor_schema:
456
+ raise DraftError("formed tensor schema does not match application")
457
+ if local_public_key not in {
458
+ member.public_key for member in result.manifest.participants
459
+ }:
460
+ raise DraftError("local key is not a formed participant")
461
+ durable = FailureConfig.for_run_root(run_root)
462
+ return TrainingConfig(
463
+ algorithm=build_algorithm(manifest=result.manifest, trainer=self._trainer),
464
+ load_checkpoint=self._trainer.load_checkpoint,
465
+ run_store=durable.run_store,
466
+ artifact_root=durable.artifact_root,
467
+ metrics_publisher=metrics_publisher,
468
+ evaluation_interval=self._evaluation_interval,
469
+ )
470
+
471
+
472
+ def prepare_trainer(
473
+ *,
474
+ draft: DraftRunSpec,
475
+ trainer: PyTorchTrainer,
476
+ model_definition: str,
477
+ evaluation_interval: int = 1,
478
+ ) -> PreparedApplication:
479
+ """Bind a fresh trainer to the draft, before any AXL or formation work."""
480
+ if draft.privacy is not None:
481
+ if not isinstance(trainer, PrivateTrainer):
482
+ raise PrivacyError("private policy requires the owned private adapter")
483
+ if draft.privacy != trainer.policy or draft.training != trainer.specification:
484
+ raise PrivacyError("private trainer policy or specification mismatch")
485
+ elif isinstance(trainer, PrivateTrainer):
486
+ raise PrivacyError("private adapter requires a private manifest")
487
+ if definition_hash(model_definition) != draft.model_definition_hash:
488
+ raise DraftError("model definition does not match shared draft")
489
+ if evaluation_interval <= 0:
490
+ raise ConfigError("evaluation interval must be positive")
491
+ if trainer.completed_steps:
492
+ raise TaskError("prepare a fresh trainer before formation")
493
+ trainer.weights()
494
+ outer_policy = draft.model_dump(
495
+ mode="json",
496
+ include={
497
+ "algorithm_id",
498
+ "algorithm_config",
499
+ "artifact_codecs",
500
+ "codec_id",
501
+ "local_steps",
502
+ "round_count",
503
+ "privacy",
504
+ "training",
505
+ },
506
+ )
507
+ outer_identity = definition_hash(
508
+ json.dumps(outer_policy, sort_keys=True, separators=(",", ":"), allow_nan=False)
509
+ )
510
+ trainer.bind_contract(
511
+ ":".join(
512
+ (
513
+ "application-restore",
514
+ draft.model_definition_hash,
515
+ canonical_hash(draft.dataset),
516
+ canonical_hash(draft.training),
517
+ outer_identity,
518
+ )
519
+ )
520
+ )
521
+ return PreparedApplication(
522
+ canonical_hash(draft), trainer, model_definition, evaluation_interval
523
+ )