dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Closed models carried by the Dromeus wire protocol."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
from typing import Annotated, Final, Literal, Self
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator
|
|
10
|
+
|
|
11
|
+
from dromeus.protocol.version import PROTOCOL_VERSION
|
|
12
|
+
|
|
13
|
+
Identifier = Annotated[
|
|
14
|
+
str, StringConstraints(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$")
|
|
15
|
+
]
|
|
16
|
+
RunId = Identifier
|
|
17
|
+
MessageId = Identifier
|
|
18
|
+
TransferId = Identifier
|
|
19
|
+
AlgorithmId = Identifier
|
|
20
|
+
RoundId = Annotated[int, Field(ge=0)]
|
|
21
|
+
Sha256 = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{64}$")]
|
|
22
|
+
PublicKey = Annotated[
|
|
23
|
+
str, StringConstraints(min_length=1, max_length=512, pattern=r"^\S+$")
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
MAGIC: Final = b"DRM1"
|
|
27
|
+
# Upper bound on the encoded size of a chunk message beyond its data bytes.
|
|
28
|
+
CHUNK_FRAMING_BYTES: Final = 512
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DomainModel(BaseModel):
|
|
32
|
+
"""Closed, immutable value object used at protocol seams."""
|
|
33
|
+
|
|
34
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Tensor(DomainModel):
|
|
38
|
+
name: Identifier
|
|
39
|
+
dtype: Literal["float16", "float32", "float64", "int8", "int32", "int64"]
|
|
40
|
+
shape: tuple[Annotated[int, Field(gt=0)], ...]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TensorSchema(DomainModel):
|
|
44
|
+
tensors: tuple[Tensor, ...] = Field(min_length=1)
|
|
45
|
+
|
|
46
|
+
@model_validator(mode="after")
|
|
47
|
+
def unique_names(self) -> Self:
|
|
48
|
+
names = [tensor.name for tensor in self.tensors]
|
|
49
|
+
if len(names) != len(set(names)):
|
|
50
|
+
raise ValueError("tensor names must be unique")
|
|
51
|
+
return self
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class MessageType(StrEnum):
|
|
55
|
+
JOIN_REQUEST = "JOIN_REQUEST"
|
|
56
|
+
JOIN_ACCEPTED = "JOIN_ACCEPTED"
|
|
57
|
+
MANIFEST_SEALED = "MANIFEST_SEALED"
|
|
58
|
+
READY = "READY"
|
|
59
|
+
START = "START"
|
|
60
|
+
START_ACK = "START_ACK"
|
|
61
|
+
RUN_FAILED = "RUN_FAILED"
|
|
62
|
+
RUN_COMPLETE = "RUN_COMPLETE"
|
|
63
|
+
TRANSFER_BEGIN = "TRANSFER_BEGIN"
|
|
64
|
+
CHUNK = "CHUNK"
|
|
65
|
+
CHUNK_ACK = "CHUNK_ACK"
|
|
66
|
+
UPDATE_READY = "UPDATE_READY"
|
|
67
|
+
ROUND_COMMITTED = "ROUND_COMMITTED"
|
|
68
|
+
CONSENSUS_SKETCH = "CONSENSUS_SKETCH"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Envelope(DomainModel):
|
|
72
|
+
magic: Literal[b"DRM1"] = MAGIC
|
|
73
|
+
protocol_version: Literal[1] = PROTOCOL_VERSION
|
|
74
|
+
message_type: MessageType
|
|
75
|
+
message_id: MessageId
|
|
76
|
+
run_id: RunId
|
|
77
|
+
manifest_hash: Sha256
|
|
78
|
+
sender_public_key: PublicKey
|
|
79
|
+
algorithm_id: AlgorithmId
|
|
80
|
+
round_id: RoundId | None = None
|
|
81
|
+
correlation_id: MessageId | None = None
|
|
82
|
+
payload_length: Annotated[int, Field(ge=0)]
|
|
83
|
+
payload_sha256: Sha256
|
|
84
|
+
payload: bytes
|
|
85
|
+
|
|
86
|
+
@model_validator(mode="after")
|
|
87
|
+
def valid_payload(self) -> Envelope:
|
|
88
|
+
if len(self.payload) != self.payload_length:
|
|
89
|
+
raise ValueError("payload length does not match envelope")
|
|
90
|
+
if hashlib.sha256(self.payload).hexdigest() != self.payload_sha256:
|
|
91
|
+
raise ValueError("payload checksum does not match envelope")
|
|
92
|
+
return self
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class JoinRequest(DomainModel):
|
|
96
|
+
draft_hash: Sha256
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class JoinAccepted(DomainModel):
|
|
100
|
+
draft_hash: Sha256
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class ReadyMessage(DomainModel):
|
|
104
|
+
manifest_hash: Sha256
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class StartMessage(DomainModel):
|
|
108
|
+
manifest_hash: Sha256
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TransferBegin(DomainModel):
|
|
112
|
+
transfer_id: TransferId
|
|
113
|
+
artifact_name: Identifier
|
|
114
|
+
total_size_bytes: int = Field(gt=0)
|
|
115
|
+
total_sha256: Sha256
|
|
116
|
+
chunk_count: int = Field(gt=0)
|
|
117
|
+
codec_id: Identifier
|
|
118
|
+
tensor_schema: TensorSchema
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class Chunk(DomainModel):
|
|
122
|
+
transfer_id: TransferId
|
|
123
|
+
chunk_index: int = Field(ge=0)
|
|
124
|
+
chunk_count: int = Field(gt=0)
|
|
125
|
+
chunk_sha256: Sha256
|
|
126
|
+
data: bytes
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class ChunkAck(DomainModel):
|
|
130
|
+
transfer_id: TransferId
|
|
131
|
+
chunk_index: int = Field(ge=0)
|
|
132
|
+
chunk_sha256: Sha256
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class PairCommitMessage(DomainModel):
|
|
136
|
+
round_id: RoundId
|
|
137
|
+
checksum: Sha256
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class RunFailedMessage(DomainModel):
|
|
141
|
+
round_id: RoundId
|
|
142
|
+
error_type: str
|
|
143
|
+
reason: str
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def create_envelope(
|
|
147
|
+
*,
|
|
148
|
+
message_type: MessageType,
|
|
149
|
+
message_id: MessageId,
|
|
150
|
+
run_id: RunId,
|
|
151
|
+
manifest_hash: Sha256,
|
|
152
|
+
sender_public_key: PublicKey,
|
|
153
|
+
algorithm_id: AlgorithmId,
|
|
154
|
+
payload: bytes,
|
|
155
|
+
round_id: RoundId | None = None,
|
|
156
|
+
correlation_id: MessageId | None = None,
|
|
157
|
+
) -> Envelope:
|
|
158
|
+
return Envelope(
|
|
159
|
+
message_type=message_type,
|
|
160
|
+
message_id=message_id,
|
|
161
|
+
run_id=run_id,
|
|
162
|
+
manifest_hash=manifest_hash,
|
|
163
|
+
sender_public_key=sender_public_key,
|
|
164
|
+
algorithm_id=algorithm_id,
|
|
165
|
+
round_id=round_id,
|
|
166
|
+
correlation_id=correlation_id,
|
|
167
|
+
payload_length=len(payload),
|
|
168
|
+
payload_sha256=hashlib.sha256(payload).hexdigest(),
|
|
169
|
+
payload=payload,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
__all__ = [
|
|
174
|
+
"CHUNK_FRAMING_BYTES",
|
|
175
|
+
"AlgorithmId",
|
|
176
|
+
"Chunk",
|
|
177
|
+
"ChunkAck",
|
|
178
|
+
"DomainModel",
|
|
179
|
+
"Envelope",
|
|
180
|
+
"Identifier",
|
|
181
|
+
"JoinAccepted",
|
|
182
|
+
"JoinRequest",
|
|
183
|
+
"MAGIC",
|
|
184
|
+
"MessageId",
|
|
185
|
+
"MessageType",
|
|
186
|
+
"PairCommitMessage",
|
|
187
|
+
"PublicKey",
|
|
188
|
+
"ReadyMessage",
|
|
189
|
+
"RoundId",
|
|
190
|
+
"RunFailedMessage",
|
|
191
|
+
"RunId",
|
|
192
|
+
"Sha256",
|
|
193
|
+
"StartMessage",
|
|
194
|
+
"Tensor",
|
|
195
|
+
"TensorSchema",
|
|
196
|
+
"TransferBegin",
|
|
197
|
+
"TransferId",
|
|
198
|
+
"create_envelope",
|
|
199
|
+
]
|
dromeus/py.typed
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|