dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Canonical JSON encoding, hashing, and draft YAML parsing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import cast
|
|
11
|
+
from uuid import uuid4
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import yaml
|
|
15
|
+
from pydantic import BaseModel, ValidationError
|
|
16
|
+
from safetensors.numpy import (
|
|
17
|
+
load_file as _load_file, # pyright: ignore[reportUnknownVariableType]
|
|
18
|
+
)
|
|
19
|
+
from safetensors.numpy import (
|
|
20
|
+
save_file as _save_file, # pyright: ignore[reportUnknownVariableType]
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
from dromeus.errors import DraftError
|
|
24
|
+
from dromeus.manifests.models import (
|
|
25
|
+
DraftRunSpec,
|
|
26
|
+
OpaqueUpdateBundleMetadata,
|
|
27
|
+
SealedManifest,
|
|
28
|
+
SealedManifestExpectation,
|
|
29
|
+
Tensor,
|
|
30
|
+
TensorSchema,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
_LoadSafetensors = Callable[[str], dict[str, np.ndarray]]
|
|
34
|
+
_SaveSafetensors = Callable[[dict[str, np.ndarray], str], None]
|
|
35
|
+
load_safetensors = cast(_LoadSafetensors, _load_file)
|
|
36
|
+
save_safetensors = cast(_SaveSafetensors, _save_file)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def canonical_json_bytes(value: object) -> bytes:
|
|
40
|
+
"""Encode a JSON-compatible value as deterministic, strict UTF-8 JSON."""
|
|
41
|
+
return json.dumps(
|
|
42
|
+
value,
|
|
43
|
+
ensure_ascii=False,
|
|
44
|
+
allow_nan=False,
|
|
45
|
+
separators=(",", ":"),
|
|
46
|
+
sort_keys=True,
|
|
47
|
+
).encode()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def canonical_json(model: BaseModel) -> bytes:
|
|
51
|
+
"""Encode a validated model as deterministic UTF-8 JSON."""
|
|
52
|
+
return canonical_json_bytes(model.model_dump(mode="json", exclude_none=True))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def canonical_hash(model: BaseModel) -> str:
|
|
56
|
+
return hashlib.sha256(canonical_json(model)).hexdigest()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def validate_sealed_draft(
|
|
60
|
+
manifest: SealedManifest,
|
|
61
|
+
*,
|
|
62
|
+
expected_draft_hash: str | None = None,
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Verify the actual shared fields, not just the manifest's claimed hash."""
|
|
65
|
+
draft = DraftRunSpec.model_validate(
|
|
66
|
+
manifest.model_dump(mode="python", include=set(DraftRunSpec.model_fields))
|
|
67
|
+
)
|
|
68
|
+
actual = canonical_hash(draft)
|
|
69
|
+
if actual != manifest.draft_hash:
|
|
70
|
+
raise DraftError("sealed draft fields do not match the claimed draft hash")
|
|
71
|
+
if expected_draft_hash is not None and actual != expected_draft_hash:
|
|
72
|
+
raise DraftError("sealed draft does not match the expected local draft")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def update_bundle_digest(metadata: OpaqueUpdateBundleMetadata) -> str:
|
|
76
|
+
"""Hash bundle identity and canonically ordered artifact descriptors."""
|
|
77
|
+
ordered = tuple(sorted(metadata.artifacts, key=lambda artifact: artifact.name))
|
|
78
|
+
return canonical_hash(metadata.model_copy(update={"artifacts": ordered}))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def file_sha256(path: Path) -> str:
|
|
82
|
+
digest = hashlib.sha256()
|
|
83
|
+
with path.open("rb") as handle:
|
|
84
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
85
|
+
digest.update(chunk)
|
|
86
|
+
return digest.hexdigest()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True, slots=True)
|
|
90
|
+
class EncodedBundleMetadata:
|
|
91
|
+
path: Path
|
|
92
|
+
tensor_schema: TensorSchema
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def materialize_bundle_metadata(
|
|
96
|
+
metadata: OpaqueUpdateBundleMetadata, root: Path
|
|
97
|
+
) -> EncodedBundleMetadata:
|
|
98
|
+
"""Atomically encode bundle metadata into a safetensors carrier artifact."""
|
|
99
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
100
|
+
payload = canonical_json(metadata)
|
|
101
|
+
values = np.frombuffer(payload, dtype=np.uint8).view(np.int8).copy()
|
|
102
|
+
identifier = uuid4().hex
|
|
103
|
+
temp = root / f".{identifier}.tmp"
|
|
104
|
+
final = root / f"{identifier}.safetensors"
|
|
105
|
+
try:
|
|
106
|
+
save_safetensors({"metadata": values}, str(temp))
|
|
107
|
+
temp.replace(final)
|
|
108
|
+
except BaseException:
|
|
109
|
+
temp.unlink(missing_ok=True)
|
|
110
|
+
final.unlink(missing_ok=True)
|
|
111
|
+
raise
|
|
112
|
+
return EncodedBundleMetadata(
|
|
113
|
+
path=final,
|
|
114
|
+
tensor_schema=TensorSchema(
|
|
115
|
+
tensors=(Tensor(name="metadata", dtype="int8", shape=values.shape),)
|
|
116
|
+
),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def parse_bundle_metadata(path: Path) -> OpaqueUpdateBundleMetadata:
|
|
121
|
+
"""Decode and validate current bundle metadata from its carrier artifact."""
|
|
122
|
+
values = load_safetensors(str(path))
|
|
123
|
+
if set(values) != {"metadata"} or values["metadata"].dtype != np.dtype(np.int8):
|
|
124
|
+
raise DraftError("invalid update bundle metadata artifact")
|
|
125
|
+
payload = np.ascontiguousarray(values["metadata"]).view(np.uint8).tobytes()
|
|
126
|
+
return OpaqueUpdateBundleMetadata.model_validate_json(payload)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def parse_draft_yaml(source: str | bytes | Path) -> DraftRunSpec:
|
|
130
|
+
"""Parse and validate a draft from YAML text or a file path.
|
|
131
|
+
|
|
132
|
+
Raises `DraftError` for malformed YAML or invalid draft fields.
|
|
133
|
+
"""
|
|
134
|
+
text = source.read_text() if isinstance(source, Path) else source
|
|
135
|
+
try:
|
|
136
|
+
return DraftRunSpec.model_validate(yaml.safe_load(text))
|
|
137
|
+
except (yaml.YAMLError, ValidationError) as error:
|
|
138
|
+
raise DraftError(f"invalid draft: {error}") from error
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def parse_sealed_json(data: str | bytes) -> SealedManifest:
|
|
142
|
+
return SealedManifest.model_validate_json(data)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def validate_sealed_expectation(
|
|
146
|
+
expectation: SealedManifestExpectation,
|
|
147
|
+
manifest: SealedManifest,
|
|
148
|
+
) -> None:
|
|
149
|
+
"""Reject a formed manifest that differs from machine-local preflight input."""
|
|
150
|
+
if manifest.draft_hash != expectation.draft_hash:
|
|
151
|
+
raise DraftError("sealed manifest draft does not match expectation")
|
|
152
|
+
if manifest.participants != expectation.participants:
|
|
153
|
+
raise DraftError("sealed manifest membership does not match expectation")
|
|
154
|
+
if manifest.initial_checkpoint_hash != expectation.initial_checkpoint_hash:
|
|
155
|
+
raise DraftError("sealed manifest checkpoint does not match expectation")
|
|
156
|
+
if canonical_hash(manifest.tensor_schema) != expectation.tensor_schema_hash:
|
|
157
|
+
raise DraftError("sealed manifest tensor schema does not match expectation")
|