dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"""Adapt application-owned PyTorch training to Dromeus tensor exchange.
|
|
2
|
+
|
|
3
|
+
Callbacks own batching, optimizer, loss, scheduling, and random generators. State
|
|
4
|
+
callbacks must save/restore all of that state as numeric arrays, including a
|
|
5
|
+
loader position and RNG states when relevant. They never receive peer data.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Callable, Mapping
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import cast
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import torch
|
|
18
|
+
from torch import nn
|
|
19
|
+
|
|
20
|
+
from dromeus.errors import TaskError
|
|
21
|
+
from dromeus.manifests.canonical import load_safetensors
|
|
22
|
+
from dromeus.manifests.models import TensorSchema
|
|
23
|
+
from dromeus.training.base import EvaluationResult
|
|
24
|
+
from dromeus.training.model_state import (
|
|
25
|
+
floating_model_state,
|
|
26
|
+
model_state_alias_groups,
|
|
27
|
+
tensor_schema_for_model,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.training.state import InitialCheckpoint, create_initial_checkpoint
|
|
30
|
+
|
|
31
|
+
_from_numpy = cast(Callable[[np.ndarray], torch.Tensor], torch.from_numpy) # pyright: ignore[reportUnknownMemberType]
|
|
32
|
+
|
|
33
|
+
TrainStep = Callable[[nn.Module], float | None]
|
|
34
|
+
Evaluate = Callable[[nn.Module], Mapping[str, float]]
|
|
35
|
+
SaveState = Callable[[], dict[str, np.ndarray]]
|
|
36
|
+
LoadState = Callable[[dict[str, np.ndarray]], None]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _copy_state(state: Mapping[str, np.ndarray]) -> dict[str, np.ndarray]:
|
|
40
|
+
result: dict[str, np.ndarray] = {}
|
|
41
|
+
for name, value in state.items():
|
|
42
|
+
if not name or value.dtype.kind not in "biuf" or not np.isfinite(value).all():
|
|
43
|
+
raise TaskError("checkpoint state needs named finite numeric arrays")
|
|
44
|
+
result[name] = value.copy(order="C")
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class PyTorchTrainer:
|
|
49
|
+
"""A custom local step plus complete application state, with no model registry.
|
|
50
|
+
|
|
51
|
+
Exchange currently supports non-scalar FP32 floating model state. Integer
|
|
52
|
+
buffers stay local and are checkpointed. Callbacks must not replace the model
|
|
53
|
+
or mutate its schema. Each instance is used serially by one node's runtime.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
*,
|
|
59
|
+
model: nn.Module,
|
|
60
|
+
train_step: TrainStep,
|
|
61
|
+
save_state: SaveState,
|
|
62
|
+
load_state: LoadState,
|
|
63
|
+
state_identity: str,
|
|
64
|
+
evaluate: Evaluate | None = None,
|
|
65
|
+
exchange_names: tuple[str, ...] | None = None,
|
|
66
|
+
) -> None:
|
|
67
|
+
if not state_identity.strip():
|
|
68
|
+
raise TaskError(
|
|
69
|
+
"state identity must bind local data and training configuration"
|
|
70
|
+
)
|
|
71
|
+
self._model = model
|
|
72
|
+
self._exchange_names = exchange_names
|
|
73
|
+
self._train_step = train_step
|
|
74
|
+
self._evaluate = evaluate
|
|
75
|
+
self._save_state = save_state
|
|
76
|
+
self._load_state = load_state
|
|
77
|
+
self._identity = np.frombuffer(
|
|
78
|
+
hashlib.sha256(state_identity.encode()).digest(), dtype=np.uint8
|
|
79
|
+
).copy()
|
|
80
|
+
self._contract_identity: str | None = None
|
|
81
|
+
values = floating_model_state(model, exchange_names)
|
|
82
|
+
if not values:
|
|
83
|
+
raise TaskError("model must expose floating exchangeable state")
|
|
84
|
+
if any(
|
|
85
|
+
value.dtype != torch.float32 or value.ndim == 0 for value in values.values()
|
|
86
|
+
):
|
|
87
|
+
raise TaskError(
|
|
88
|
+
"exchange requires non-scalar FP32 state; use shape [1] for scalars"
|
|
89
|
+
)
|
|
90
|
+
self._schema = tensor_schema_for_model(model, exchange_names)
|
|
91
|
+
self._aliases = model_state_alias_groups(model)
|
|
92
|
+
self._steps = 0
|
|
93
|
+
self._local_loss: float | None = None
|
|
94
|
+
# Reject unusable state before the node starts AXL.
|
|
95
|
+
self.checkpoint_tensors()
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def tensor_schema(self) -> TensorSchema:
|
|
99
|
+
return self._schema
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def completed_steps(self) -> int:
|
|
103
|
+
return self._steps
|
|
104
|
+
|
|
105
|
+
def bind_contract(self, identity: str) -> None:
|
|
106
|
+
"""Bind durable state to shared application semantics before formation."""
|
|
107
|
+
if self._contract_identity is not None:
|
|
108
|
+
if identity != self._contract_identity:
|
|
109
|
+
raise TaskError("trainer is already bound to another application")
|
|
110
|
+
return
|
|
111
|
+
if self._steps:
|
|
112
|
+
raise TaskError("bind the application contract before training")
|
|
113
|
+
self._identity = np.frombuffer(
|
|
114
|
+
hashlib.sha256(self._identity.tobytes() + identity.encode()).digest(),
|
|
115
|
+
dtype=np.uint8,
|
|
116
|
+
).copy()
|
|
117
|
+
self._contract_identity = identity
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def local_loss(self) -> float | None:
|
|
121
|
+
return self._local_loss
|
|
122
|
+
|
|
123
|
+
def weights(self) -> dict[str, np.ndarray]:
|
|
124
|
+
self._validate_alias_layout()
|
|
125
|
+
if tensor_schema_for_model(self._model, self._exchange_names) != self._schema:
|
|
126
|
+
raise TaskError("application changed its model tensor schema")
|
|
127
|
+
return _copy_state(
|
|
128
|
+
{
|
|
129
|
+
name: value.detach().cpu().numpy()
|
|
130
|
+
for name, value in floating_model_state(
|
|
131
|
+
self._model, self._exchange_names
|
|
132
|
+
).items()
|
|
133
|
+
}
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
def load_weights(self, weights: dict[str, np.ndarray]) -> None:
|
|
137
|
+
current = self.weights()
|
|
138
|
+
candidate = self._validate_like(weights, current)
|
|
139
|
+
self._validate_alias_values(candidate)
|
|
140
|
+
with torch.no_grad():
|
|
141
|
+
for name, target in floating_model_state(
|
|
142
|
+
self._model, self._exchange_names
|
|
143
|
+
).items():
|
|
144
|
+
target.copy_(_from_numpy(candidate[name]).to(target.device))
|
|
145
|
+
|
|
146
|
+
def train_local_steps(self, step_count: int) -> None:
|
|
147
|
+
if step_count <= 0:
|
|
148
|
+
raise TaskError("local step count must be positive")
|
|
149
|
+
for _ in range(step_count):
|
|
150
|
+
loss = self._train_step(self._model)
|
|
151
|
+
if loss is not None and not math.isfinite(loss):
|
|
152
|
+
raise TaskError("application returned non-finite local loss")
|
|
153
|
+
self._local_loss = None if loss is None else float(loss)
|
|
154
|
+
self._steps += 1
|
|
155
|
+
self.weights() # Detect non-finite or structurally changed model state.
|
|
156
|
+
|
|
157
|
+
def evaluate(self) -> EvaluationResult | None:
|
|
158
|
+
if self._evaluate is None:
|
|
159
|
+
return None
|
|
160
|
+
modes = [(module, module.training) for module in self._model.modules()]
|
|
161
|
+
try:
|
|
162
|
+
self._model.eval()
|
|
163
|
+
with torch.no_grad():
|
|
164
|
+
return EvaluationResult(self._evaluate(self._model))
|
|
165
|
+
finally:
|
|
166
|
+
for module, training in modes:
|
|
167
|
+
module.training = training
|
|
168
|
+
|
|
169
|
+
def create_initial_checkpoint(
|
|
170
|
+
self, path: Path, *, model_definition: str
|
|
171
|
+
) -> InitialCheckpoint:
|
|
172
|
+
self.weights()
|
|
173
|
+
return create_initial_checkpoint(
|
|
174
|
+
path,
|
|
175
|
+
model=self._model,
|
|
176
|
+
model_definition=model_definition,
|
|
177
|
+
exchange_names=self._exchange_names,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
def load_checkpoint(self, path: Path) -> None:
|
|
181
|
+
"""Load group initial weights, retaining local application state."""
|
|
182
|
+
self.load_weights(load_safetensors(str(path)))
|
|
183
|
+
|
|
184
|
+
def checkpoint_tensors(self) -> dict[str, np.ndarray]:
|
|
185
|
+
self._validate_alias_layout()
|
|
186
|
+
state = {
|
|
187
|
+
f"model.{name}": value.detach().cpu().numpy().copy()
|
|
188
|
+
for name, value in self._model.state_dict().items()
|
|
189
|
+
}
|
|
190
|
+
state.update(
|
|
191
|
+
{
|
|
192
|
+
f"application.{name}": value
|
|
193
|
+
for name, value in _copy_state(self._save_state()).items()
|
|
194
|
+
}
|
|
195
|
+
)
|
|
196
|
+
state.update(
|
|
197
|
+
{
|
|
198
|
+
"meta.version": np.array([1], dtype=np.int64),
|
|
199
|
+
"meta.identity": self._identity.copy(),
|
|
200
|
+
"meta.steps": np.array([self._steps], dtype=np.int64),
|
|
201
|
+
"meta.loss": np.array(
|
|
202
|
+
[] if self._local_loss is None else [self._local_loss],
|
|
203
|
+
dtype=np.float64,
|
|
204
|
+
),
|
|
205
|
+
"meta.modes": np.array(
|
|
206
|
+
[module.training for module in self._model.modules()],
|
|
207
|
+
dtype=np.bool_,
|
|
208
|
+
),
|
|
209
|
+
}
|
|
210
|
+
)
|
|
211
|
+
return _copy_state(state)
|
|
212
|
+
|
|
213
|
+
def load_checkpoint_tensors(self, state: dict[str, np.ndarray]) -> None:
|
|
214
|
+
"""Validate model/meta before restoring callbacks; roll back on failure.
|
|
215
|
+
|
|
216
|
+
Application load_state must also accept its own save_state output. If an
|
|
217
|
+
application violates that contract, rollback failure is fatal to the run.
|
|
218
|
+
"""
|
|
219
|
+
candidate = _copy_state(state)
|
|
220
|
+
old = self.checkpoint_tensors()
|
|
221
|
+
model_keys = {name for name in old if name.startswith("model.")}
|
|
222
|
+
meta_keys = {name for name in old if name.startswith("meta.")}
|
|
223
|
+
if {
|
|
224
|
+
name for name in candidate if not name.startswith("application.")
|
|
225
|
+
} != model_keys | meta_keys:
|
|
226
|
+
raise TaskError("checkpoint model or metadata keys mismatch")
|
|
227
|
+
self._validate_like(
|
|
228
|
+
{k: candidate[k] for k in model_keys}, {k: old[k] for k in model_keys}
|
|
229
|
+
)
|
|
230
|
+
self._validate_alias_values(
|
|
231
|
+
{name.removeprefix("model."): candidate[name] for name in model_keys}
|
|
232
|
+
)
|
|
233
|
+
for key in meta_keys - {"meta.loss"}:
|
|
234
|
+
self._validate_like({key: candidate[key]}, {key: old[key]})
|
|
235
|
+
if not np.array_equal(
|
|
236
|
+
candidate["meta.version"], old["meta.version"]
|
|
237
|
+
) or not np.array_equal(candidate["meta.identity"], self._identity):
|
|
238
|
+
raise TaskError("checkpoint version or local state identity mismatch")
|
|
239
|
+
if candidate["meta.steps"][0] < 0:
|
|
240
|
+
raise TaskError("checkpoint step count must be non-negative")
|
|
241
|
+
loss = candidate["meta.loss"]
|
|
242
|
+
if loss.dtype != np.float64 or loss.shape not in ((0,), (1,)):
|
|
243
|
+
raise TaskError("invalid checkpoint loss")
|
|
244
|
+
try:
|
|
245
|
+
self._restore(candidate)
|
|
246
|
+
except Exception:
|
|
247
|
+
self._restore(old)
|
|
248
|
+
raise
|
|
249
|
+
|
|
250
|
+
def _restore(self, state: dict[str, np.ndarray]) -> None:
|
|
251
|
+
self._load_state(
|
|
252
|
+
{
|
|
253
|
+
name.removeprefix("application."): value.copy()
|
|
254
|
+
for name, value in state.items()
|
|
255
|
+
if name.startswith("application.")
|
|
256
|
+
}
|
|
257
|
+
)
|
|
258
|
+
with torch.no_grad():
|
|
259
|
+
for name, value in self._model.state_dict().items():
|
|
260
|
+
value.copy_(_from_numpy(state[f"model.{name}"]).to(value.device))
|
|
261
|
+
for module, mode in zip(
|
|
262
|
+
self._model.modules(), state["meta.modes"], strict=True
|
|
263
|
+
):
|
|
264
|
+
module.training = bool(mode)
|
|
265
|
+
self._steps = int(state["meta.steps"][0])
|
|
266
|
+
loss = state["meta.loss"]
|
|
267
|
+
self._local_loss = float(loss[0]) if loss.size else None
|
|
268
|
+
|
|
269
|
+
def _validate_alias_layout(self) -> None:
|
|
270
|
+
if model_state_alias_groups(self._model) != self._aliases:
|
|
271
|
+
raise TaskError("application changed its model alias layout")
|
|
272
|
+
|
|
273
|
+
def _validate_alias_values(self, values: Mapping[str, np.ndarray]) -> None:
|
|
274
|
+
for group in self._aliases:
|
|
275
|
+
present = [name for name in group if name in values]
|
|
276
|
+
if present and any(
|
|
277
|
+
not np.array_equal(values[present[0]], values[name])
|
|
278
|
+
for name in present[1:]
|
|
279
|
+
):
|
|
280
|
+
raise TaskError(f"conflicting values for tied model aliases: {group}")
|
|
281
|
+
|
|
282
|
+
@staticmethod
|
|
283
|
+
def _validate_like(
|
|
284
|
+
state: Mapping[str, np.ndarray], expected: Mapping[str, np.ndarray]
|
|
285
|
+
) -> dict[str, np.ndarray]:
|
|
286
|
+
if state.keys() != expected.keys():
|
|
287
|
+
raise TaskError("model state keys mismatch")
|
|
288
|
+
candidate = _copy_state(state)
|
|
289
|
+
if any(
|
|
290
|
+
candidate[name].shape != value.shape or candidate[name].dtype != value.dtype
|
|
291
|
+
for name, value in expected.items()
|
|
292
|
+
):
|
|
293
|
+
raise TaskError("model state shape or dtype mismatch")
|
|
294
|
+
return candidate
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""The PyTorch version a group agrees on."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def torch_release() -> str:
|
|
9
|
+
"""Installed PyTorch release without its build label.
|
|
10
|
+
|
|
11
|
+
Wheels for one release differ only in their local label: `2.13.0+cpu` on
|
|
12
|
+
Linux CPU, `2.13.0+cu130` on CUDA, `2.13.0` on macOS. Nodes on different
|
|
13
|
+
builds of one release train together, so drafts and checks use the release.
|
|
14
|
+
"""
|
|
15
|
+
return torch.__version__.split("+", 1)[0]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""AXL transport and reliable transfer."""
|
dromeus/transport/axl.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""AXL HTTP bridge adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import cast
|
|
10
|
+
from urllib.error import HTTPError, URLError
|
|
11
|
+
from urllib.request import Request, urlopen
|
|
12
|
+
|
|
13
|
+
from dromeus.manifests.models import PublicKey
|
|
14
|
+
from dromeus.protocol.codec import ProtocolDecodeError, decode_envelope_sender
|
|
15
|
+
from dromeus.transport.interface import ReceivedBytes, TransportError
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def matches_yggdrasil_sender(public_key: str, bridge_sender: str) -> bool:
|
|
19
|
+
try:
|
|
20
|
+
sender_value = int(bridge_sender, 16)
|
|
21
|
+
key_value = int(public_key, 16)
|
|
22
|
+
except ValueError:
|
|
23
|
+
return False
|
|
24
|
+
trailing_ones = 0
|
|
25
|
+
value = sender_value
|
|
26
|
+
while value & 1:
|
|
27
|
+
trailing_ones += 1
|
|
28
|
+
value >>= 1
|
|
29
|
+
unknown_mask = (1 << trailing_ones) - 1
|
|
30
|
+
return key_value | unknown_mask == sender_value
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class AXLBridgeConfig:
|
|
35
|
+
base_url: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class AXLTransport:
|
|
39
|
+
"""Async wrapper over the local AXL `/send`, `/recv`, and `/topology` bridge."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, config: AXLBridgeConfig) -> None:
|
|
42
|
+
self._config = config
|
|
43
|
+
self._cached_public_key: PublicKey | None = None
|
|
44
|
+
|
|
45
|
+
async def local_public_key(self) -> PublicKey:
|
|
46
|
+
if self._cached_public_key is None:
|
|
47
|
+
self._cached_public_key = await self._request_public_key()
|
|
48
|
+
return self._cached_public_key
|
|
49
|
+
|
|
50
|
+
async def send(self, destination: PublicKey, payload: bytes) -> None:
|
|
51
|
+
await self._post_send(destination, payload)
|
|
52
|
+
|
|
53
|
+
async def recv(self, timeout_seconds: float) -> ReceivedBytes | None:
|
|
54
|
+
return await self._get_recv(timeout_seconds)
|
|
55
|
+
|
|
56
|
+
async def topology(self) -> dict[str, object]:
|
|
57
|
+
"""Return one raw local AXL topology snapshot off the event loop."""
|
|
58
|
+
return await asyncio.to_thread(self._load_topology)
|
|
59
|
+
|
|
60
|
+
async def _request_public_key(self) -> PublicKey:
|
|
61
|
+
def load_topology() -> PublicKey:
|
|
62
|
+
payload = self._load_topology()
|
|
63
|
+
public_key = payload.get("our_public_key")
|
|
64
|
+
if not isinstance(public_key, str):
|
|
65
|
+
raise TransportError("AXL topology did not include our_public_key")
|
|
66
|
+
return public_key
|
|
67
|
+
|
|
68
|
+
return await asyncio.to_thread(load_topology)
|
|
69
|
+
|
|
70
|
+
def _load_topology(self) -> dict[str, object]:
|
|
71
|
+
request = Request(f"{self._config.base_url}/topology", method="GET")
|
|
72
|
+
try:
|
|
73
|
+
with urlopen(request, timeout=5.0) as response:
|
|
74
|
+
body = response.read()
|
|
75
|
+
except (HTTPError, URLError) as error:
|
|
76
|
+
raise TransportError("failed to query AXL topology") from error
|
|
77
|
+
payload = cast(object, json.loads(body))
|
|
78
|
+
if not isinstance(payload, dict):
|
|
79
|
+
raise TransportError("AXL topology response was not an object")
|
|
80
|
+
return cast(dict[str, object], payload)
|
|
81
|
+
|
|
82
|
+
def _resolve_sender(
|
|
83
|
+
self, bridge_sender: str, payload: bytes | None = None
|
|
84
|
+
) -> PublicKey:
|
|
85
|
+
claimed_sender = _payload_sender(payload)
|
|
86
|
+
if claimed_sender is not None and matches_yggdrasil_sender(
|
|
87
|
+
claimed_sender, bridge_sender
|
|
88
|
+
):
|
|
89
|
+
return claimed_sender
|
|
90
|
+
candidates: set[str] = set()
|
|
91
|
+
for attempt in range(20):
|
|
92
|
+
topology = self._load_topology()
|
|
93
|
+
candidates.clear()
|
|
94
|
+
for section in ("peers", "tree"):
|
|
95
|
+
entries = topology.get(section)
|
|
96
|
+
if not isinstance(entries, list):
|
|
97
|
+
continue
|
|
98
|
+
for entry in cast(list[object], entries):
|
|
99
|
+
if isinstance(entry, dict):
|
|
100
|
+
public_key = cast(dict[object, object], entry).get("public_key")
|
|
101
|
+
if isinstance(public_key, str):
|
|
102
|
+
candidates.add(public_key)
|
|
103
|
+
if bridge_sender in candidates:
|
|
104
|
+
return bridge_sender
|
|
105
|
+
matches = [
|
|
106
|
+
key
|
|
107
|
+
for key in candidates
|
|
108
|
+
if matches_yggdrasil_sender(key, bridge_sender)
|
|
109
|
+
]
|
|
110
|
+
if len(matches) == 1:
|
|
111
|
+
return matches[0]
|
|
112
|
+
if attempt < 19:
|
|
113
|
+
time.sleep(0.1)
|
|
114
|
+
raise TransportError(
|
|
115
|
+
"AXL sender header did not resolve uniquely "
|
|
116
|
+
f"from {len(candidates)} topology keys"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
async def _post_send(self, destination: PublicKey, payload: bytes) -> None:
|
|
120
|
+
def send_request() -> None:
|
|
121
|
+
request = Request(
|
|
122
|
+
f"{self._config.base_url}/send",
|
|
123
|
+
method="POST",
|
|
124
|
+
data=payload,
|
|
125
|
+
headers={"X-Destination-Peer-Id": destination},
|
|
126
|
+
)
|
|
127
|
+
try:
|
|
128
|
+
with urlopen(request, timeout=5.0):
|
|
129
|
+
return
|
|
130
|
+
except (HTTPError, URLError) as error:
|
|
131
|
+
raise TransportError("AXL send failed") from error
|
|
132
|
+
|
|
133
|
+
await asyncio.to_thread(send_request)
|
|
134
|
+
|
|
135
|
+
async def _get_recv(self, timeout_seconds: float) -> ReceivedBytes | None:
|
|
136
|
+
def recv_request() -> ReceivedBytes | None:
|
|
137
|
+
request = Request(f"{self._config.base_url}/recv", method="GET")
|
|
138
|
+
try:
|
|
139
|
+
with urlopen(request, timeout=timeout_seconds) as response:
|
|
140
|
+
if response.status == 204:
|
|
141
|
+
return None
|
|
142
|
+
bridge_sender = response.headers.get("X-From-Peer-Id")
|
|
143
|
+
if bridge_sender is None:
|
|
144
|
+
raise TransportError("AXL recv response missing sender header")
|
|
145
|
+
body = response.read()
|
|
146
|
+
return ReceivedBytes(
|
|
147
|
+
sender_public_key=self._resolve_sender(bridge_sender, body),
|
|
148
|
+
payload=body,
|
|
149
|
+
)
|
|
150
|
+
except HTTPError as error:
|
|
151
|
+
if error.code == 204:
|
|
152
|
+
return None
|
|
153
|
+
raise TransportError("AXL recv failed") from error
|
|
154
|
+
except TimeoutError:
|
|
155
|
+
return None
|
|
156
|
+
except URLError as error:
|
|
157
|
+
raise TransportError("AXL recv failed") from error
|
|
158
|
+
|
|
159
|
+
return await asyncio.to_thread(recv_request)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _payload_sender(payload: bytes | None) -> str | None:
|
|
163
|
+
if payload is None:
|
|
164
|
+
return None
|
|
165
|
+
try:
|
|
166
|
+
return decode_envelope_sender(payload, max_payload_bytes=len(payload))
|
|
167
|
+
except (ValueError, ProtocolDecodeError):
|
|
168
|
+
return None
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""One node's run identity, stamped onto every outbound envelope."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, replace
|
|
6
|
+
|
|
7
|
+
from dromeus.manifests.models import (
|
|
8
|
+
AlgorithmId,
|
|
9
|
+
MessageId,
|
|
10
|
+
PublicKey,
|
|
11
|
+
RoundId,
|
|
12
|
+
RunId,
|
|
13
|
+
Sha256,
|
|
14
|
+
TransportLimits,
|
|
15
|
+
)
|
|
16
|
+
from dromeus.protocol.codec import encode_envelope
|
|
17
|
+
from dromeus.protocol.models import MessageType, create_envelope
|
|
18
|
+
from dromeus.transport.outbound_scheduler import (
|
|
19
|
+
OutboundScheduler,
|
|
20
|
+
Priority,
|
|
21
|
+
SendTiming,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
UNSEALED_MANIFEST_HASH: Sha256 = "0" * 64
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class EnvelopeSender:
|
|
29
|
+
"""Send payloads as this node, in this run, under the manifest retry policy."""
|
|
30
|
+
|
|
31
|
+
scheduler: OutboundScheduler
|
|
32
|
+
local_public_key: PublicKey
|
|
33
|
+
run_id: RunId
|
|
34
|
+
manifest_hash: Sha256
|
|
35
|
+
algorithm_id: AlgorithmId
|
|
36
|
+
transport_limits: TransportLimits
|
|
37
|
+
|
|
38
|
+
def for_manifest(self, manifest_hash: Sha256) -> EnvelopeSender:
|
|
39
|
+
"""Return the same sender bound to a sealed manifest."""
|
|
40
|
+
return replace(self, manifest_hash=manifest_hash)
|
|
41
|
+
|
|
42
|
+
async def send(
|
|
43
|
+
self,
|
|
44
|
+
destination: PublicKey,
|
|
45
|
+
*,
|
|
46
|
+
message_type: MessageType,
|
|
47
|
+
message_id: MessageId,
|
|
48
|
+
payload: bytes,
|
|
49
|
+
priority: Priority,
|
|
50
|
+
round_id: RoundId | None = None,
|
|
51
|
+
correlation_id: MessageId | None = None,
|
|
52
|
+
) -> SendTiming:
|
|
53
|
+
envelope = create_envelope(
|
|
54
|
+
message_type=message_type,
|
|
55
|
+
message_id=message_id,
|
|
56
|
+
run_id=self.run_id,
|
|
57
|
+
manifest_hash=self.manifest_hash,
|
|
58
|
+
sender_public_key=self.local_public_key,
|
|
59
|
+
algorithm_id=self.algorithm_id,
|
|
60
|
+
round_id=round_id,
|
|
61
|
+
correlation_id=correlation_id,
|
|
62
|
+
payload=payload,
|
|
63
|
+
)
|
|
64
|
+
return await self.scheduler.send(
|
|
65
|
+
destination,
|
|
66
|
+
encode_envelope(envelope),
|
|
67
|
+
priority=priority,
|
|
68
|
+
retries=self.transport_limits.max_retries,
|
|
69
|
+
retry_delay_seconds=self.transport_limits.retry_timeout_seconds,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
__all__ = ["UNSEALED_MANIFEST_HASH", "EnvelopeSender"]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Transport interface shared by production and test adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Protocol
|
|
7
|
+
|
|
8
|
+
from dromeus.errors import RunError
|
|
9
|
+
from dromeus.manifests.models import PublicKey
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class TransportError(RunError):
|
|
13
|
+
"""The underlying byte transport failed."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class ReceivedBytes:
|
|
18
|
+
"""One authenticated inbound byte payload from the transport."""
|
|
19
|
+
|
|
20
|
+
sender_public_key: PublicKey
|
|
21
|
+
payload: bytes
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AsyncTransport(Protocol):
|
|
25
|
+
"""Small async seam used by the receiver and sender."""
|
|
26
|
+
|
|
27
|
+
async def local_public_key(self) -> PublicKey: ...
|
|
28
|
+
|
|
29
|
+
async def send(self, destination: PublicKey, payload: bytes) -> None: ...
|
|
30
|
+
|
|
31
|
+
async def recv(self, timeout_seconds: float) -> ReceivedBytes | None: ...
|