dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,157 @@
1
+ """Canonical JSON encoding, hashing, and draft YAML parsing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from collections.abc import Callable
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+ from typing import cast
11
+ from uuid import uuid4
12
+
13
+ import numpy as np
14
+ import yaml
15
+ from pydantic import BaseModel, ValidationError
16
+ from safetensors.numpy import (
17
+ load_file as _load_file, # pyright: ignore[reportUnknownVariableType]
18
+ )
19
+ from safetensors.numpy import (
20
+ save_file as _save_file, # pyright: ignore[reportUnknownVariableType]
21
+ )
22
+
23
+ from dromeus.errors import DraftError
24
+ from dromeus.manifests.models import (
25
+ DraftRunSpec,
26
+ OpaqueUpdateBundleMetadata,
27
+ SealedManifest,
28
+ SealedManifestExpectation,
29
+ Tensor,
30
+ TensorSchema,
31
+ )
32
+
33
+ _LoadSafetensors = Callable[[str], dict[str, np.ndarray]]
34
+ _SaveSafetensors = Callable[[dict[str, np.ndarray], str], None]
35
+ load_safetensors = cast(_LoadSafetensors, _load_file)
36
+ save_safetensors = cast(_SaveSafetensors, _save_file)
37
+
38
+
39
+ def canonical_json_bytes(value: object) -> bytes:
40
+ """Encode a JSON-compatible value as deterministic, strict UTF-8 JSON."""
41
+ return json.dumps(
42
+ value,
43
+ ensure_ascii=False,
44
+ allow_nan=False,
45
+ separators=(",", ":"),
46
+ sort_keys=True,
47
+ ).encode()
48
+
49
+
50
+ def canonical_json(model: BaseModel) -> bytes:
51
+ """Encode a validated model as deterministic UTF-8 JSON."""
52
+ return canonical_json_bytes(model.model_dump(mode="json", exclude_none=True))
53
+
54
+
55
+ def canonical_hash(model: BaseModel) -> str:
56
+ return hashlib.sha256(canonical_json(model)).hexdigest()
57
+
58
+
59
+ def validate_sealed_draft(
60
+ manifest: SealedManifest,
61
+ *,
62
+ expected_draft_hash: str | None = None,
63
+ ) -> None:
64
+ """Verify the actual shared fields, not just the manifest's claimed hash."""
65
+ draft = DraftRunSpec.model_validate(
66
+ manifest.model_dump(mode="python", include=set(DraftRunSpec.model_fields))
67
+ )
68
+ actual = canonical_hash(draft)
69
+ if actual != manifest.draft_hash:
70
+ raise DraftError("sealed draft fields do not match the claimed draft hash")
71
+ if expected_draft_hash is not None and actual != expected_draft_hash:
72
+ raise DraftError("sealed draft does not match the expected local draft")
73
+
74
+
75
+ def update_bundle_digest(metadata: OpaqueUpdateBundleMetadata) -> str:
76
+ """Hash bundle identity and canonically ordered artifact descriptors."""
77
+ ordered = tuple(sorted(metadata.artifacts, key=lambda artifact: artifact.name))
78
+ return canonical_hash(metadata.model_copy(update={"artifacts": ordered}))
79
+
80
+
81
+ def file_sha256(path: Path) -> str:
82
+ digest = hashlib.sha256()
83
+ with path.open("rb") as handle:
84
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
85
+ digest.update(chunk)
86
+ return digest.hexdigest()
87
+
88
+
89
+ @dataclass(frozen=True, slots=True)
90
+ class EncodedBundleMetadata:
91
+ path: Path
92
+ tensor_schema: TensorSchema
93
+
94
+
95
+ def materialize_bundle_metadata(
96
+ metadata: OpaqueUpdateBundleMetadata, root: Path
97
+ ) -> EncodedBundleMetadata:
98
+ """Atomically encode bundle metadata into a safetensors carrier artifact."""
99
+ root.mkdir(parents=True, exist_ok=True)
100
+ payload = canonical_json(metadata)
101
+ values = np.frombuffer(payload, dtype=np.uint8).view(np.int8).copy()
102
+ identifier = uuid4().hex
103
+ temp = root / f".{identifier}.tmp"
104
+ final = root / f"{identifier}.safetensors"
105
+ try:
106
+ save_safetensors({"metadata": values}, str(temp))
107
+ temp.replace(final)
108
+ except BaseException:
109
+ temp.unlink(missing_ok=True)
110
+ final.unlink(missing_ok=True)
111
+ raise
112
+ return EncodedBundleMetadata(
113
+ path=final,
114
+ tensor_schema=TensorSchema(
115
+ tensors=(Tensor(name="metadata", dtype="int8", shape=values.shape),)
116
+ ),
117
+ )
118
+
119
+
120
+ def parse_bundle_metadata(path: Path) -> OpaqueUpdateBundleMetadata:
121
+ """Decode and validate current bundle metadata from its carrier artifact."""
122
+ values = load_safetensors(str(path))
123
+ if set(values) != {"metadata"} or values["metadata"].dtype != np.dtype(np.int8):
124
+ raise DraftError("invalid update bundle metadata artifact")
125
+ payload = np.ascontiguousarray(values["metadata"]).view(np.uint8).tobytes()
126
+ return OpaqueUpdateBundleMetadata.model_validate_json(payload)
127
+
128
+
129
+ def parse_draft_yaml(source: str | bytes | Path) -> DraftRunSpec:
130
+ """Parse and validate a draft from YAML text or a file path.
131
+
132
+ Raises `DraftError` for malformed YAML or invalid draft fields.
133
+ """
134
+ text = source.read_text() if isinstance(source, Path) else source
135
+ try:
136
+ return DraftRunSpec.model_validate(yaml.safe_load(text))
137
+ except (yaml.YAMLError, ValidationError) as error:
138
+ raise DraftError(f"invalid draft: {error}") from error
139
+
140
+
141
+ def parse_sealed_json(data: str | bytes) -> SealedManifest:
142
+ return SealedManifest.model_validate_json(data)
143
+
144
+
145
+ def validate_sealed_expectation(
146
+ expectation: SealedManifestExpectation,
147
+ manifest: SealedManifest,
148
+ ) -> None:
149
+ """Reject a formed manifest that differs from machine-local preflight input."""
150
+ if manifest.draft_hash != expectation.draft_hash:
151
+ raise DraftError("sealed manifest draft does not match expectation")
152
+ if manifest.participants != expectation.participants:
153
+ raise DraftError("sealed manifest membership does not match expectation")
154
+ if manifest.initial_checkpoint_hash != expectation.initial_checkpoint_hash:
155
+ raise DraftError("sealed manifest checkpoint does not match expectation")
156
+ if canonical_hash(manifest.tensor_schema) != expectation.tensor_schema_hash:
157
+ raise DraftError("sealed manifest tensor schema does not match expectation")