dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,199 @@
1
+ """Closed models carried by the Dromeus wire protocol."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from enum import StrEnum
7
+ from typing import Annotated, Final, Literal, Self
8
+
9
+ from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator
10
+
11
+ from dromeus.protocol.version import PROTOCOL_VERSION
12
+
13
+ Identifier = Annotated[
14
+ str, StringConstraints(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$")
15
+ ]
16
+ RunId = Identifier
17
+ MessageId = Identifier
18
+ TransferId = Identifier
19
+ AlgorithmId = Identifier
20
+ RoundId = Annotated[int, Field(ge=0)]
21
+ Sha256 = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{64}$")]
22
+ PublicKey = Annotated[
23
+ str, StringConstraints(min_length=1, max_length=512, pattern=r"^\S+$")
24
+ ]
25
+
26
+ MAGIC: Final = b"DRM1"
27
+ # Upper bound on the encoded size of a chunk message beyond its data bytes.
28
+ CHUNK_FRAMING_BYTES: Final = 512
29
+
30
+
31
+ class DomainModel(BaseModel):
32
+ """Closed, immutable value object used at protocol seams."""
33
+
34
+ model_config = ConfigDict(extra="forbid", frozen=True)
35
+
36
+
37
+ class Tensor(DomainModel):
38
+ name: Identifier
39
+ dtype: Literal["float16", "float32", "float64", "int8", "int32", "int64"]
40
+ shape: tuple[Annotated[int, Field(gt=0)], ...]
41
+
42
+
43
+ class TensorSchema(DomainModel):
44
+ tensors: tuple[Tensor, ...] = Field(min_length=1)
45
+
46
+ @model_validator(mode="after")
47
+ def unique_names(self) -> Self:
48
+ names = [tensor.name for tensor in self.tensors]
49
+ if len(names) != len(set(names)):
50
+ raise ValueError("tensor names must be unique")
51
+ return self
52
+
53
+
54
+ class MessageType(StrEnum):
55
+ JOIN_REQUEST = "JOIN_REQUEST"
56
+ JOIN_ACCEPTED = "JOIN_ACCEPTED"
57
+ MANIFEST_SEALED = "MANIFEST_SEALED"
58
+ READY = "READY"
59
+ START = "START"
60
+ START_ACK = "START_ACK"
61
+ RUN_FAILED = "RUN_FAILED"
62
+ RUN_COMPLETE = "RUN_COMPLETE"
63
+ TRANSFER_BEGIN = "TRANSFER_BEGIN"
64
+ CHUNK = "CHUNK"
65
+ CHUNK_ACK = "CHUNK_ACK"
66
+ UPDATE_READY = "UPDATE_READY"
67
+ ROUND_COMMITTED = "ROUND_COMMITTED"
68
+ CONSENSUS_SKETCH = "CONSENSUS_SKETCH"
69
+
70
+
71
+ class Envelope(DomainModel):
72
+ magic: Literal[b"DRM1"] = MAGIC
73
+ protocol_version: Literal[1] = PROTOCOL_VERSION
74
+ message_type: MessageType
75
+ message_id: MessageId
76
+ run_id: RunId
77
+ manifest_hash: Sha256
78
+ sender_public_key: PublicKey
79
+ algorithm_id: AlgorithmId
80
+ round_id: RoundId | None = None
81
+ correlation_id: MessageId | None = None
82
+ payload_length: Annotated[int, Field(ge=0)]
83
+ payload_sha256: Sha256
84
+ payload: bytes
85
+
86
+ @model_validator(mode="after")
87
+ def valid_payload(self) -> Envelope:
88
+ if len(self.payload) != self.payload_length:
89
+ raise ValueError("payload length does not match envelope")
90
+ if hashlib.sha256(self.payload).hexdigest() != self.payload_sha256:
91
+ raise ValueError("payload checksum does not match envelope")
92
+ return self
93
+
94
+
95
+ class JoinRequest(DomainModel):
96
+ draft_hash: Sha256
97
+
98
+
99
+ class JoinAccepted(DomainModel):
100
+ draft_hash: Sha256
101
+
102
+
103
+ class ReadyMessage(DomainModel):
104
+ manifest_hash: Sha256
105
+
106
+
107
+ class StartMessage(DomainModel):
108
+ manifest_hash: Sha256
109
+
110
+
111
+ class TransferBegin(DomainModel):
112
+ transfer_id: TransferId
113
+ artifact_name: Identifier
114
+ total_size_bytes: int = Field(gt=0)
115
+ total_sha256: Sha256
116
+ chunk_count: int = Field(gt=0)
117
+ codec_id: Identifier
118
+ tensor_schema: TensorSchema
119
+
120
+
121
+ class Chunk(DomainModel):
122
+ transfer_id: TransferId
123
+ chunk_index: int = Field(ge=0)
124
+ chunk_count: int = Field(gt=0)
125
+ chunk_sha256: Sha256
126
+ data: bytes
127
+
128
+
129
+ class ChunkAck(DomainModel):
130
+ transfer_id: TransferId
131
+ chunk_index: int = Field(ge=0)
132
+ chunk_sha256: Sha256
133
+
134
+
135
+ class PairCommitMessage(DomainModel):
136
+ round_id: RoundId
137
+ checksum: Sha256
138
+
139
+
140
+ class RunFailedMessage(DomainModel):
141
+ round_id: RoundId
142
+ error_type: str
143
+ reason: str
144
+
145
+
146
+ def create_envelope(
147
+ *,
148
+ message_type: MessageType,
149
+ message_id: MessageId,
150
+ run_id: RunId,
151
+ manifest_hash: Sha256,
152
+ sender_public_key: PublicKey,
153
+ algorithm_id: AlgorithmId,
154
+ payload: bytes,
155
+ round_id: RoundId | None = None,
156
+ correlation_id: MessageId | None = None,
157
+ ) -> Envelope:
158
+ return Envelope(
159
+ message_type=message_type,
160
+ message_id=message_id,
161
+ run_id=run_id,
162
+ manifest_hash=manifest_hash,
163
+ sender_public_key=sender_public_key,
164
+ algorithm_id=algorithm_id,
165
+ round_id=round_id,
166
+ correlation_id=correlation_id,
167
+ payload_length=len(payload),
168
+ payload_sha256=hashlib.sha256(payload).hexdigest(),
169
+ payload=payload,
170
+ )
171
+
172
+
173
+ __all__ = [
174
+ "CHUNK_FRAMING_BYTES",
175
+ "AlgorithmId",
176
+ "Chunk",
177
+ "ChunkAck",
178
+ "DomainModel",
179
+ "Envelope",
180
+ "Identifier",
181
+ "JoinAccepted",
182
+ "JoinRequest",
183
+ "MAGIC",
184
+ "MessageId",
185
+ "MessageType",
186
+ "PairCommitMessage",
187
+ "PublicKey",
188
+ "ReadyMessage",
189
+ "RoundId",
190
+ "RunFailedMessage",
191
+ "RunId",
192
+ "Sha256",
193
+ "StartMessage",
194
+ "Tensor",
195
+ "TensorSchema",
196
+ "TransferBegin",
197
+ "TransferId",
198
+ "create_envelope",
199
+ ]
@@ -0,0 +1,5 @@
1
+ """Dromeus wire-protocol version."""
2
+
3
+ PROTOCOL_VERSION = 1
4
+
5
+ __all__ = ["PROTOCOL_VERSION"]
dromeus/py.typed ADDED
@@ -0,0 +1 @@
1
+