dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
"""Strict benchmark evidence records over mixed diagnostic JSONL logs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Literal, Self, cast
|
|
11
|
+
|
|
12
|
+
from pydantic import (
|
|
13
|
+
BaseModel,
|
|
14
|
+
ConfigDict,
|
|
15
|
+
Field,
|
|
16
|
+
TypeAdapter,
|
|
17
|
+
ValidationError,
|
|
18
|
+
field_validator,
|
|
19
|
+
model_validator,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
from dromeus.errors import RunError
|
|
23
|
+
from dromeus.manifests.canonical import canonical_json_bytes
|
|
24
|
+
from dromeus.manifests.models import (
|
|
25
|
+
Identifier,
|
|
26
|
+
MessageId,
|
|
27
|
+
PublicKey,
|
|
28
|
+
RoundId,
|
|
29
|
+
RunId,
|
|
30
|
+
Sha256,
|
|
31
|
+
TransferId,
|
|
32
|
+
)
|
|
33
|
+
from dromeus.telemetry.consensus import ConsensusObservation
|
|
34
|
+
from dromeus.telemetry.events import EventSink
|
|
35
|
+
|
|
36
|
+
EVIDENCE_VERSION = 3
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class EvidenceError(RunError, ValueError):
|
|
40
|
+
"""Official benchmark evidence is invalid, incompatible, or inconsistent."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _EvidenceBase(BaseModel):
|
|
44
|
+
model_config = ConfigDict(
|
|
45
|
+
extra="forbid",
|
|
46
|
+
frozen=True,
|
|
47
|
+
strict=True,
|
|
48
|
+
allow_inf_nan=False,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
evidence_version: Literal[3] = EVIDENCE_VERSION
|
|
52
|
+
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
53
|
+
run_id: RunId
|
|
54
|
+
manifest_hash: Sha256
|
|
55
|
+
node_id: PublicKey
|
|
56
|
+
|
|
57
|
+
@field_validator("timestamp")
|
|
58
|
+
@classmethod
|
|
59
|
+
def timestamp_has_timezone(cls, value: datetime) -> datetime:
|
|
60
|
+
if value.tzinfo is None or value.utcoffset() is None:
|
|
61
|
+
raise ValueError("evidence timestamp must include a timezone")
|
|
62
|
+
return value
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class RoundMetricsEvidence(_EvidenceBase):
|
|
66
|
+
"""One committed round's timing, local loss and named evaluation metrics."""
|
|
67
|
+
|
|
68
|
+
event: Literal["round_metrics"] = "round_metrics"
|
|
69
|
+
message_id: MessageId
|
|
70
|
+
transfer_id: TransferId | None = None
|
|
71
|
+
peer_id: PublicKey
|
|
72
|
+
round_id: RoundId
|
|
73
|
+
local_loss: float | None = None
|
|
74
|
+
evaluation_metrics: dict[str, float] = Field(default_factory=dict)
|
|
75
|
+
error_feedback_residual_l2_norm: float | None = Field(default=None, ge=0)
|
|
76
|
+
error_feedback_signal_l2_norm: float | None = Field(default=None, ge=0)
|
|
77
|
+
error_feedback_residual_to_signal_ratio: float | None = Field(default=None, ge=0)
|
|
78
|
+
local_compute_seconds: float = Field(ge=0)
|
|
79
|
+
peer_wait_seconds: float = Field(ge=0)
|
|
80
|
+
transfer_seconds: float = Field(ge=0)
|
|
81
|
+
mixing_seconds: float = Field(ge=0)
|
|
82
|
+
evaluation_seconds: float = Field(ge=0)
|
|
83
|
+
retries: int = Field(ge=0)
|
|
84
|
+
encoded_artifact_bytes: int | None = Field(default=None, gt=0)
|
|
85
|
+
|
|
86
|
+
@field_validator("evaluation_metrics")
|
|
87
|
+
@classmethod
|
|
88
|
+
def valid_names(cls, values: dict[str, float]) -> dict[str, float]:
|
|
89
|
+
if any(not name.strip() for name in values):
|
|
90
|
+
raise ValueError("metric names must not be blank")
|
|
91
|
+
return values
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ConsensusObservationEvidence(_EvidenceBase):
|
|
95
|
+
"""One complete round's group spread and scale, as measured by this node."""
|
|
96
|
+
|
|
97
|
+
event: Literal["consensus_observation"] = "consensus_observation"
|
|
98
|
+
message_id: MessageId
|
|
99
|
+
round_id: RoundId
|
|
100
|
+
normalized_rms: float = Field(ge=0)
|
|
101
|
+
absolute_rms_spread: float = Field(ge=0)
|
|
102
|
+
mean_sketch_norm: float = Field(ge=0)
|
|
103
|
+
denominator_degenerate: bool
|
|
104
|
+
sketch_count: int = Field(ge=1)
|
|
105
|
+
|
|
106
|
+
@model_validator(mode="after")
|
|
107
|
+
def valid_scale(self) -> Self:
|
|
108
|
+
ConsensusObservation(
|
|
109
|
+
round_id=self.round_id,
|
|
110
|
+
normalized_rms=self.normalized_rms,
|
|
111
|
+
absolute_rms_spread=self.absolute_rms_spread,
|
|
112
|
+
mean_sketch_norm=self.mean_sketch_norm,
|
|
113
|
+
denominator_degenerate=self.denominator_degenerate,
|
|
114
|
+
sketch_count=self.sketch_count,
|
|
115
|
+
)
|
|
116
|
+
return self
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class NodeConsensusDistanceEvidence(_EvidenceBase):
|
|
120
|
+
"""Local node-to-mean CountSketch distance, independent of detector policy."""
|
|
121
|
+
|
|
122
|
+
event: Literal["node_consensus_distance"] = "node_consensus_distance"
|
|
123
|
+
measurement_kind: Literal["countsketch"] = "countsketch"
|
|
124
|
+
message_id: MessageId
|
|
125
|
+
round_id: RoundId
|
|
126
|
+
absolute_distance: float = Field(ge=0)
|
|
127
|
+
normalized_distance: float = Field(ge=0)
|
|
128
|
+
group_normalized_rms: float = Field(ge=0)
|
|
129
|
+
mean_sketch_norm: float = Field(ge=0)
|
|
130
|
+
denominator_degenerate: bool
|
|
131
|
+
sketch_count: int = Field(ge=1)
|
|
132
|
+
|
|
133
|
+
@model_validator(mode="after")
|
|
134
|
+
def valid_scale(self) -> Self:
|
|
135
|
+
if self.denominator_degenerate != (self.mean_sketch_norm <= 1e-12):
|
|
136
|
+
raise ValueError("inconsistent denominator scale flag")
|
|
137
|
+
if not math.isclose(
|
|
138
|
+
self.normalized_distance,
|
|
139
|
+
self.absolute_distance / (self.mean_sketch_norm + 1e-12),
|
|
140
|
+
rel_tol=1e-9,
|
|
141
|
+
abs_tol=1e-12,
|
|
142
|
+
):
|
|
143
|
+
raise ValueError("inconsistent node distance normalization")
|
|
144
|
+
minimum_rms = self.normalized_distance / math.sqrt(self.sketch_count)
|
|
145
|
+
if self.group_normalized_rms < minimum_rms and not math.isclose(
|
|
146
|
+
self.group_normalized_rms, minimum_rms, rel_tol=1e-9, abs_tol=1e-12
|
|
147
|
+
):
|
|
148
|
+
raise ValueError("node distance exceeds group RMS bound")
|
|
149
|
+
return self
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
DetectorStateName = Literal[
|
|
153
|
+
"warming_up", "healthy", "suspect", "diverging", "insufficient_data"
|
|
154
|
+
]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class DivergenceStatusEvidence(_EvidenceBase):
|
|
158
|
+
event: Literal["divergence_status"] = "divergence_status"
|
|
159
|
+
policy_hash: Sha256
|
|
160
|
+
threshold_set_id: Identifier
|
|
161
|
+
transition_id: int = Field(gt=0)
|
|
162
|
+
previous_state: DetectorStateName
|
|
163
|
+
state: DetectorStateName
|
|
164
|
+
warning_active: bool
|
|
165
|
+
scale_warning: bool
|
|
166
|
+
reason: Literal[
|
|
167
|
+
"warmup",
|
|
168
|
+
"missing_rounds",
|
|
169
|
+
"denominator_degenerate",
|
|
170
|
+
"severe_distance",
|
|
171
|
+
"sustained_growth",
|
|
172
|
+
"recovered",
|
|
173
|
+
"awaiting_recovery",
|
|
174
|
+
"growth",
|
|
175
|
+
"stable_window",
|
|
176
|
+
"collecting_window",
|
|
177
|
+
]
|
|
178
|
+
source_round: int | None = Field(default=None, ge=0)
|
|
179
|
+
detection_round: int = Field(ge=0)
|
|
180
|
+
window_start: int | None = Field(default=None, ge=0)
|
|
181
|
+
window_end: int | None = Field(default=None, ge=0)
|
|
182
|
+
window_count: int = Field(ge=0, le=4096)
|
|
183
|
+
expected_participants: int = Field(ge=2)
|
|
184
|
+
observed_participants: int = Field(ge=0)
|
|
185
|
+
age_rounds: int = Field(ge=0)
|
|
186
|
+
missing_rounds: int = Field(ge=0)
|
|
187
|
+
measurement_channel: Literal["normalized", "absolute"]
|
|
188
|
+
earlier_median: float | None = Field(default=None, ge=0)
|
|
189
|
+
later_median: float | None = Field(default=None, ge=0)
|
|
190
|
+
log_slope: float | None = None
|
|
191
|
+
normalized_rms: float | None = Field(default=None, ge=0)
|
|
192
|
+
absolute_rms_spread: float | None = Field(default=None, ge=0)
|
|
193
|
+
mean_sketch_norm: float | None = Field(default=None, ge=0)
|
|
194
|
+
|
|
195
|
+
@model_validator(mode="after")
|
|
196
|
+
def valid_window(self) -> Self:
|
|
197
|
+
if self.observed_participants not in (0, self.expected_participants):
|
|
198
|
+
raise ValueError("detector evidence must have complete or missing coverage")
|
|
199
|
+
if self.source_round is not None and self.source_round > self.detection_round:
|
|
200
|
+
raise ValueError("source round cannot follow detection round")
|
|
201
|
+
if self.window_count:
|
|
202
|
+
if (
|
|
203
|
+
self.window_start is None
|
|
204
|
+
or self.window_end is None
|
|
205
|
+
or self.window_end - self.window_start + 1 != self.window_count
|
|
206
|
+
):
|
|
207
|
+
raise ValueError("detector window must be consecutive")
|
|
208
|
+
if self.source_round != self.window_end:
|
|
209
|
+
raise ValueError("detector window must end at source round")
|
|
210
|
+
elif self.window_start is not None or self.window_end is not None:
|
|
211
|
+
raise ValueError("empty detector window cannot have endpoints")
|
|
212
|
+
if self.state == "diverging" and not self.warning_active:
|
|
213
|
+
raise ValueError("diverging state needs an active warning")
|
|
214
|
+
return self
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class ConsensusSketchSentEvidence(_EvidenceBase):
|
|
218
|
+
event: Literal["consensus_sketch_sent"] = "consensus_sketch_sent"
|
|
219
|
+
message_id: MessageId
|
|
220
|
+
round_id: RoundId
|
|
221
|
+
payload_bytes: int = Field(ge=0)
|
|
222
|
+
recipient_count: int = Field(ge=1)
|
|
223
|
+
successful_recipient_count: int = Field(ge=0)
|
|
224
|
+
retry_count: int = Field(ge=0)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class TransferMessageSentEvidence(_EvidenceBase):
|
|
228
|
+
event: Literal["transfer_message_sent"] = "transfer_message_sent"
|
|
229
|
+
message_id: MessageId
|
|
230
|
+
transfer_id: TransferId | None = None
|
|
231
|
+
peer_id: PublicKey
|
|
232
|
+
round_id: RoundId | None = None
|
|
233
|
+
message_type: Identifier
|
|
234
|
+
payload_bytes: int = Field(ge=0)
|
|
235
|
+
queue_seconds: float = Field(ge=0)
|
|
236
|
+
send_seconds: float = Field(ge=0)
|
|
237
|
+
retry_count: int = Field(ge=0)
|
|
238
|
+
completion_seconds: float = Field(ge=0)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class RunFailedEvidence(_EvidenceBase):
|
|
242
|
+
event: Literal["run_failed"] = "run_failed"
|
|
243
|
+
message_id: MessageId
|
|
244
|
+
round_id: RoundId
|
|
245
|
+
error_type: Identifier
|
|
246
|
+
error: str = Field(min_length=1, max_length=1024)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
type EvidenceRecord = (
|
|
250
|
+
RoundMetricsEvidence
|
|
251
|
+
| ConsensusObservationEvidence
|
|
252
|
+
| NodeConsensusDistanceEvidence
|
|
253
|
+
| DivergenceStatusEvidence
|
|
254
|
+
| ConsensusSketchSentEvidence
|
|
255
|
+
| TransferMessageSentEvidence
|
|
256
|
+
| RunFailedEvidence
|
|
257
|
+
)
|
|
258
|
+
_EVIDENCE_ADAPTER: TypeAdapter[EvidenceRecord] = TypeAdapter(EvidenceRecord)
|
|
259
|
+
_EVIDENCE_EVENTS = frozenset(
|
|
260
|
+
{
|
|
261
|
+
"round_metrics",
|
|
262
|
+
"consensus_observation",
|
|
263
|
+
"node_consensus_distance",
|
|
264
|
+
"divergence_status",
|
|
265
|
+
"consensus_sketch_sent",
|
|
266
|
+
"transfer_message_sent",
|
|
267
|
+
"run_failed",
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@dataclass(frozen=True, slots=True)
|
|
273
|
+
class EvidenceLog:
|
|
274
|
+
"""One strictly decoded node evidence stream from a mixed JSONL log."""
|
|
275
|
+
|
|
276
|
+
path: Path
|
|
277
|
+
run_id: str
|
|
278
|
+
manifest_hash: str
|
|
279
|
+
node_id: str | None
|
|
280
|
+
records: tuple[EvidenceRecord, ...]
|
|
281
|
+
|
|
282
|
+
@classmethod
|
|
283
|
+
def open(
|
|
284
|
+
cls,
|
|
285
|
+
path: Path,
|
|
286
|
+
*,
|
|
287
|
+
run_id: str,
|
|
288
|
+
manifest_hash: str,
|
|
289
|
+
) -> EvidenceLog:
|
|
290
|
+
try:
|
|
291
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
292
|
+
except OSError as error:
|
|
293
|
+
raise EvidenceError(f"evidence log is unreadable: {path}") from error
|
|
294
|
+
records: list[EvidenceRecord] = []
|
|
295
|
+
for line_number, line in enumerate(lines, start=1):
|
|
296
|
+
try:
|
|
297
|
+
value = _json_object(line)
|
|
298
|
+
except EvidenceError as error:
|
|
299
|
+
raise EvidenceError(
|
|
300
|
+
f"invalid JSONL record at {path}:{line_number}"
|
|
301
|
+
) from error
|
|
302
|
+
event = value.get("event")
|
|
303
|
+
if "evidence_version" not in value and event not in _EVIDENCE_EVENTS:
|
|
304
|
+
continue
|
|
305
|
+
try:
|
|
306
|
+
record = decode_evidence(line)
|
|
307
|
+
except EvidenceError as error:
|
|
308
|
+
raise EvidenceError(
|
|
309
|
+
f"invalid evidence at {path}:{line_number}"
|
|
310
|
+
) from error
|
|
311
|
+
if record.run_id != run_id:
|
|
312
|
+
raise EvidenceError(f"evidence run id mismatch in {path}")
|
|
313
|
+
if record.manifest_hash != manifest_hash:
|
|
314
|
+
raise EvidenceError(f"evidence manifest hash mismatch in {path}")
|
|
315
|
+
records.append(record)
|
|
316
|
+
transition_ids = [
|
|
317
|
+
r.transition_id for r in records if isinstance(r, DivergenceStatusEvidence)
|
|
318
|
+
]
|
|
319
|
+
if len(transition_ids) != len(set(transition_ids)) or transition_ids != sorted(
|
|
320
|
+
transition_ids
|
|
321
|
+
):
|
|
322
|
+
raise EvidenceError("duplicate or unordered detector transition IDs")
|
|
323
|
+
policy_ids = {
|
|
324
|
+
(r.policy_hash, r.threshold_set_id)
|
|
325
|
+
for r in records
|
|
326
|
+
if isinstance(r, DivergenceStatusEvidence)
|
|
327
|
+
}
|
|
328
|
+
if len(policy_ids) > 1:
|
|
329
|
+
raise EvidenceError("detector evidence policy changed")
|
|
330
|
+
node_ids = {record.node_id for record in records}
|
|
331
|
+
if len(node_ids) > 1:
|
|
332
|
+
raise EvidenceError(f"evidence node id mismatch in {path}")
|
|
333
|
+
for record_type in (
|
|
334
|
+
RoundMetricsEvidence,
|
|
335
|
+
ConsensusObservationEvidence,
|
|
336
|
+
NodeConsensusDistanceEvidence,
|
|
337
|
+
ConsensusSketchSentEvidence,
|
|
338
|
+
RunFailedEvidence,
|
|
339
|
+
):
|
|
340
|
+
rounds = [
|
|
341
|
+
record.round_id for record in records if isinstance(record, record_type)
|
|
342
|
+
]
|
|
343
|
+
if len(rounds) != len(set(rounds)):
|
|
344
|
+
raise EvidenceError(f"duplicate evidence round in {path}")
|
|
345
|
+
return cls(
|
|
346
|
+
path=path,
|
|
347
|
+
run_id=run_id,
|
|
348
|
+
manifest_hash=manifest_hash,
|
|
349
|
+
node_id=next(iter(node_ids), None),
|
|
350
|
+
records=tuple(records),
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def evidence_record(record: EvidenceRecord) -> dict[str, object]:
|
|
355
|
+
"""Return one JSON-compatible strict evidence mapping."""
|
|
356
|
+
return cast(
|
|
357
|
+
dict[str, object],
|
|
358
|
+
record.model_dump(mode="json", exclude_none=True),
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def encode_evidence(record: EvidenceRecord) -> bytes:
|
|
363
|
+
"""Encode one evidence record as deterministic strict JSON."""
|
|
364
|
+
return canonical_json_bytes(evidence_record(record))
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def decode_evidence(data: bytes | str) -> EvidenceRecord:
|
|
368
|
+
"""Strictly decode one supported evidence record."""
|
|
369
|
+
value = _json_object(data)
|
|
370
|
+
version = value.get("evidence_version")
|
|
371
|
+
if version != EVIDENCE_VERSION:
|
|
372
|
+
raise EvidenceError(f"unsupported evidence version: {version}")
|
|
373
|
+
try:
|
|
374
|
+
return _EVIDENCE_ADAPTER.validate_json(
|
|
375
|
+
json.dumps(value, allow_nan=False, separators=(",", ":")),
|
|
376
|
+
strict=True,
|
|
377
|
+
)
|
|
378
|
+
except (TypeError, ValueError, ValidationError) as error:
|
|
379
|
+
raise EvidenceError("evidence record is invalid") from error
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def append_evidence(sink: EventSink | None, record: EvidenceRecord) -> bool:
|
|
383
|
+
"""Best-effort evidence write that never propagates sink failures."""
|
|
384
|
+
if sink is None:
|
|
385
|
+
return False
|
|
386
|
+
try:
|
|
387
|
+
sink.append(evidence_record(record))
|
|
388
|
+
except Exception:
|
|
389
|
+
return False
|
|
390
|
+
return True
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _json_object(data: bytes | str) -> dict[str, object]:
|
|
394
|
+
try:
|
|
395
|
+
value = cast(
|
|
396
|
+
object,
|
|
397
|
+
json.loads(data, parse_constant=_reject_json_constant),
|
|
398
|
+
)
|
|
399
|
+
except (TypeError, ValueError) as error:
|
|
400
|
+
raise EvidenceError("evidence JSON is invalid") from error
|
|
401
|
+
if not isinstance(value, dict):
|
|
402
|
+
raise EvidenceError("evidence record must be an object")
|
|
403
|
+
entries = cast(dict[object, object], value)
|
|
404
|
+
if not all(isinstance(key, str) for key in entries):
|
|
405
|
+
raise EvidenceError("evidence object keys must be strings")
|
|
406
|
+
return cast(dict[str, object], entries)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _reject_json_constant(value: str) -> None:
|
|
410
|
+
raise ValueError(f"invalid JSON constant: {value}")
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
__all__ = [
|
|
414
|
+
"EVIDENCE_VERSION",
|
|
415
|
+
"ConsensusObservationEvidence",
|
|
416
|
+
"ConsensusSketchSentEvidence",
|
|
417
|
+
"DivergenceStatusEvidence",
|
|
418
|
+
"EvidenceError",
|
|
419
|
+
"EvidenceLog",
|
|
420
|
+
"EvidenceRecord",
|
|
421
|
+
"NodeConsensusDistanceEvidence",
|
|
422
|
+
"RoundMetricsEvidence",
|
|
423
|
+
"RunFailedEvidence",
|
|
424
|
+
"TransferMessageSentEvidence",
|
|
425
|
+
"append_evidence",
|
|
426
|
+
"decode_evidence",
|
|
427
|
+
"encode_evidence",
|
|
428
|
+
"evidence_record",
|
|
429
|
+
]
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Bounded, append-only node metric publication."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import math
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from types import MappingProxyType
|
|
10
|
+
from typing import Protocol
|
|
11
|
+
|
|
12
|
+
from dromeus.telemetry.events import EventSink
|
|
13
|
+
from dromeus.telemetry.evidence import (
|
|
14
|
+
EvidenceRecord,
|
|
15
|
+
RoundMetricsEvidence,
|
|
16
|
+
RunFailedEvidence,
|
|
17
|
+
append_evidence,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True, slots=True)
|
|
22
|
+
class RoundTiming:
|
|
23
|
+
"""Timing and scalar metrics captured for one committed local round."""
|
|
24
|
+
|
|
25
|
+
round_id: int
|
|
26
|
+
peer_id: str
|
|
27
|
+
local_compute_seconds: float
|
|
28
|
+
peer_wait_seconds: float
|
|
29
|
+
transfer_seconds: float
|
|
30
|
+
mixing_seconds: float
|
|
31
|
+
evaluation_seconds: float
|
|
32
|
+
retries: int = 0
|
|
33
|
+
local_loss: float | None = None
|
|
34
|
+
error_feedback_residual_l2_norm: float | None = None
|
|
35
|
+
error_feedback_signal_l2_norm: float | None = None
|
|
36
|
+
error_feedback_residual_to_signal_ratio: float | None = None
|
|
37
|
+
evaluation_metrics: Mapping[str, float] | None = None
|
|
38
|
+
transfer_id: str | None = None
|
|
39
|
+
encoded_artifact_bytes: int | None = None
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
if self.round_id < 0:
|
|
43
|
+
raise ValueError("round_id must be non-negative")
|
|
44
|
+
if not self.peer_id:
|
|
45
|
+
raise ValueError("peer_id must not be empty")
|
|
46
|
+
durations = (
|
|
47
|
+
self.local_compute_seconds,
|
|
48
|
+
self.peer_wait_seconds,
|
|
49
|
+
self.transfer_seconds,
|
|
50
|
+
self.mixing_seconds,
|
|
51
|
+
self.evaluation_seconds,
|
|
52
|
+
)
|
|
53
|
+
if any(not math.isfinite(value) or value < 0 for value in durations):
|
|
54
|
+
raise ValueError("metric durations must be finite and non-negative")
|
|
55
|
+
if self.retries < 0:
|
|
56
|
+
raise ValueError("retries must be non-negative")
|
|
57
|
+
if self.encoded_artifact_bytes is not None and self.encoded_artifact_bytes <= 0:
|
|
58
|
+
raise ValueError("encoded artifact bytes must be positive")
|
|
59
|
+
if self.local_loss is not None and (not math.isfinite(self.local_loss)):
|
|
60
|
+
raise ValueError("local_loss must be finite")
|
|
61
|
+
observations = (
|
|
62
|
+
self.error_feedback_residual_l2_norm,
|
|
63
|
+
self.error_feedback_signal_l2_norm,
|
|
64
|
+
self.error_feedback_residual_to_signal_ratio,
|
|
65
|
+
)
|
|
66
|
+
if any(
|
|
67
|
+
value is not None and (not math.isfinite(value) or value < 0)
|
|
68
|
+
for value in observations
|
|
69
|
+
):
|
|
70
|
+
raise ValueError(
|
|
71
|
+
"error-feedback observations must be finite and non-negative"
|
|
72
|
+
)
|
|
73
|
+
if self.evaluation_metrics is not None:
|
|
74
|
+
if any(
|
|
75
|
+
not name.strip() or not math.isfinite(value)
|
|
76
|
+
for name, value in self.evaluation_metrics.items()
|
|
77
|
+
):
|
|
78
|
+
raise ValueError(
|
|
79
|
+
"task metrics require nonblank names and finite values"
|
|
80
|
+
)
|
|
81
|
+
object.__setattr__(
|
|
82
|
+
self,
|
|
83
|
+
"evaluation_metrics",
|
|
84
|
+
MappingProxyType(dict(self.evaluation_metrics)),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class MetricsPublisher(Protocol):
|
|
89
|
+
"""Protocol-like base for non-blocking metric publication."""
|
|
90
|
+
|
|
91
|
+
def submit(self, timing: RoundTiming) -> bool:
|
|
92
|
+
raise NotImplementedError
|
|
93
|
+
|
|
94
|
+
def submit_failure(self, *, round_id: int, error_type: str, reason: str) -> bool:
|
|
95
|
+
raise NotImplementedError
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class JsonlMetricsPublisher(MetricsPublisher):
|
|
99
|
+
"""Publish correlated metrics to one node's JSONL sink off the event loop."""
|
|
100
|
+
|
|
101
|
+
def __init__(
|
|
102
|
+
self,
|
|
103
|
+
*,
|
|
104
|
+
sink: EventSink,
|
|
105
|
+
run_id: str,
|
|
106
|
+
manifest_hash: str,
|
|
107
|
+
node_id: str,
|
|
108
|
+
max_queue_size: int = 256,
|
|
109
|
+
) -> None:
|
|
110
|
+
if not run_id or not manifest_hash or not node_id:
|
|
111
|
+
raise ValueError("metric context identifiers must not be empty")
|
|
112
|
+
if max_queue_size <= 0:
|
|
113
|
+
raise ValueError("max_queue_size must be positive")
|
|
114
|
+
self._sink = sink
|
|
115
|
+
self._run_id = run_id
|
|
116
|
+
self._manifest_hash = manifest_hash
|
|
117
|
+
self._node_id = node_id
|
|
118
|
+
self._queue: asyncio.Queue[EvidenceRecord] = asyncio.Queue(
|
|
119
|
+
maxsize=max_queue_size
|
|
120
|
+
)
|
|
121
|
+
self._stop = asyncio.Event()
|
|
122
|
+
self._task: asyncio.Task[None] | None = None
|
|
123
|
+
self._dropped = 0
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def pending(self) -> int:
|
|
127
|
+
return self._queue.qsize()
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def dropped(self) -> int:
|
|
131
|
+
return self._dropped
|
|
132
|
+
|
|
133
|
+
async def start(self) -> None:
|
|
134
|
+
if self._task is None:
|
|
135
|
+
self._task = asyncio.create_task(self._run(), name="dromeus-metrics")
|
|
136
|
+
|
|
137
|
+
async def stop(self) -> None:
|
|
138
|
+
if self._task is None:
|
|
139
|
+
return
|
|
140
|
+
self._stop.set()
|
|
141
|
+
await self._task
|
|
142
|
+
self._task = None
|
|
143
|
+
|
|
144
|
+
def submit(self, timing: RoundTiming) -> bool:
|
|
145
|
+
# Copy fields individually: mappingproxy is deliberately not pickleable.
|
|
146
|
+
payload: dict[str, object] = {
|
|
147
|
+
name: getattr(timing, name)
|
|
148
|
+
for name in timing.__dataclass_fields__
|
|
149
|
+
if name != "evaluation_metrics"
|
|
150
|
+
}
|
|
151
|
+
payload.update(
|
|
152
|
+
run_id=self._run_id,
|
|
153
|
+
manifest_hash=self._manifest_hash,
|
|
154
|
+
node_id=self._node_id,
|
|
155
|
+
message_id=f"metric-{self._node_id[:8]}-{timing.round_id}",
|
|
156
|
+
)
|
|
157
|
+
payload["evaluation_metrics"] = dict(timing.evaluation_metrics or {})
|
|
158
|
+
return self._enqueue(RoundMetricsEvidence.model_validate(payload))
|
|
159
|
+
|
|
160
|
+
def submit_failure(self, *, round_id: int, error_type: str, reason: str) -> bool:
|
|
161
|
+
return self._enqueue(
|
|
162
|
+
RunFailedEvidence(
|
|
163
|
+
run_id=self._run_id,
|
|
164
|
+
manifest_hash=self._manifest_hash,
|
|
165
|
+
node_id=self._node_id,
|
|
166
|
+
message_id=f"run-failure-{self._node_id[:8]}-{round_id}",
|
|
167
|
+
round_id=round_id,
|
|
168
|
+
error_type=error_type[:128],
|
|
169
|
+
error=reason[:1024] or "unknown failure",
|
|
170
|
+
)
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
def _enqueue(self, record: EvidenceRecord) -> bool:
|
|
174
|
+
if self._stop.is_set():
|
|
175
|
+
self._dropped += 1
|
|
176
|
+
return False
|
|
177
|
+
try:
|
|
178
|
+
self._queue.put_nowait(record)
|
|
179
|
+
except asyncio.QueueFull:
|
|
180
|
+
self._dropped += 1
|
|
181
|
+
return False
|
|
182
|
+
return True
|
|
183
|
+
|
|
184
|
+
async def _run(self) -> None:
|
|
185
|
+
while not self._stop.is_set() or not self._queue.empty():
|
|
186
|
+
try:
|
|
187
|
+
record = await asyncio.wait_for(self._queue.get(), timeout=0.1)
|
|
188
|
+
except TimeoutError:
|
|
189
|
+
continue
|
|
190
|
+
try:
|
|
191
|
+
written = await asyncio.to_thread(append_evidence, self._sink, record)
|
|
192
|
+
if not written:
|
|
193
|
+
self._dropped += 1
|
|
194
|
+
except Exception:
|
|
195
|
+
self._dropped += 1
|
|
196
|
+
finally:
|
|
197
|
+
self._queue.task_done()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
__all__ = ["JsonlMetricsPublisher", "MetricsPublisher", "RoundTiming"]
|