dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,429 @@
1
+ """Strict benchmark evidence records over mixed diagnostic JSONL logs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ from dataclasses import dataclass
8
+ from datetime import UTC, datetime
9
+ from pathlib import Path
10
+ from typing import Literal, Self, cast
11
+
12
+ from pydantic import (
13
+ BaseModel,
14
+ ConfigDict,
15
+ Field,
16
+ TypeAdapter,
17
+ ValidationError,
18
+ field_validator,
19
+ model_validator,
20
+ )
21
+
22
+ from dromeus.errors import RunError
23
+ from dromeus.manifests.canonical import canonical_json_bytes
24
+ from dromeus.manifests.models import (
25
+ Identifier,
26
+ MessageId,
27
+ PublicKey,
28
+ RoundId,
29
+ RunId,
30
+ Sha256,
31
+ TransferId,
32
+ )
33
+ from dromeus.telemetry.consensus import ConsensusObservation
34
+ from dromeus.telemetry.events import EventSink
35
+
36
+ EVIDENCE_VERSION = 3
37
+
38
+
39
+ class EvidenceError(RunError, ValueError):
40
+ """Official benchmark evidence is invalid, incompatible, or inconsistent."""
41
+
42
+
43
+ class _EvidenceBase(BaseModel):
44
+ model_config = ConfigDict(
45
+ extra="forbid",
46
+ frozen=True,
47
+ strict=True,
48
+ allow_inf_nan=False,
49
+ )
50
+
51
+ evidence_version: Literal[3] = EVIDENCE_VERSION
52
+ timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
53
+ run_id: RunId
54
+ manifest_hash: Sha256
55
+ node_id: PublicKey
56
+
57
+ @field_validator("timestamp")
58
+ @classmethod
59
+ def timestamp_has_timezone(cls, value: datetime) -> datetime:
60
+ if value.tzinfo is None or value.utcoffset() is None:
61
+ raise ValueError("evidence timestamp must include a timezone")
62
+ return value
63
+
64
+
65
+ class RoundMetricsEvidence(_EvidenceBase):
66
+ """One committed round's timing, local loss and named evaluation metrics."""
67
+
68
+ event: Literal["round_metrics"] = "round_metrics"
69
+ message_id: MessageId
70
+ transfer_id: TransferId | None = None
71
+ peer_id: PublicKey
72
+ round_id: RoundId
73
+ local_loss: float | None = None
74
+ evaluation_metrics: dict[str, float] = Field(default_factory=dict)
75
+ error_feedback_residual_l2_norm: float | None = Field(default=None, ge=0)
76
+ error_feedback_signal_l2_norm: float | None = Field(default=None, ge=0)
77
+ error_feedback_residual_to_signal_ratio: float | None = Field(default=None, ge=0)
78
+ local_compute_seconds: float = Field(ge=0)
79
+ peer_wait_seconds: float = Field(ge=0)
80
+ transfer_seconds: float = Field(ge=0)
81
+ mixing_seconds: float = Field(ge=0)
82
+ evaluation_seconds: float = Field(ge=0)
83
+ retries: int = Field(ge=0)
84
+ encoded_artifact_bytes: int | None = Field(default=None, gt=0)
85
+
86
+ @field_validator("evaluation_metrics")
87
+ @classmethod
88
+ def valid_names(cls, values: dict[str, float]) -> dict[str, float]:
89
+ if any(not name.strip() for name in values):
90
+ raise ValueError("metric names must not be blank")
91
+ return values
92
+
93
+
94
+ class ConsensusObservationEvidence(_EvidenceBase):
95
+ """One complete round's group spread and scale, as measured by this node."""
96
+
97
+ event: Literal["consensus_observation"] = "consensus_observation"
98
+ message_id: MessageId
99
+ round_id: RoundId
100
+ normalized_rms: float = Field(ge=0)
101
+ absolute_rms_spread: float = Field(ge=0)
102
+ mean_sketch_norm: float = Field(ge=0)
103
+ denominator_degenerate: bool
104
+ sketch_count: int = Field(ge=1)
105
+
106
+ @model_validator(mode="after")
107
+ def valid_scale(self) -> Self:
108
+ ConsensusObservation(
109
+ round_id=self.round_id,
110
+ normalized_rms=self.normalized_rms,
111
+ absolute_rms_spread=self.absolute_rms_spread,
112
+ mean_sketch_norm=self.mean_sketch_norm,
113
+ denominator_degenerate=self.denominator_degenerate,
114
+ sketch_count=self.sketch_count,
115
+ )
116
+ return self
117
+
118
+
119
+ class NodeConsensusDistanceEvidence(_EvidenceBase):
120
+ """Local node-to-mean CountSketch distance, independent of detector policy."""
121
+
122
+ event: Literal["node_consensus_distance"] = "node_consensus_distance"
123
+ measurement_kind: Literal["countsketch"] = "countsketch"
124
+ message_id: MessageId
125
+ round_id: RoundId
126
+ absolute_distance: float = Field(ge=0)
127
+ normalized_distance: float = Field(ge=0)
128
+ group_normalized_rms: float = Field(ge=0)
129
+ mean_sketch_norm: float = Field(ge=0)
130
+ denominator_degenerate: bool
131
+ sketch_count: int = Field(ge=1)
132
+
133
+ @model_validator(mode="after")
134
+ def valid_scale(self) -> Self:
135
+ if self.denominator_degenerate != (self.mean_sketch_norm <= 1e-12):
136
+ raise ValueError("inconsistent denominator scale flag")
137
+ if not math.isclose(
138
+ self.normalized_distance,
139
+ self.absolute_distance / (self.mean_sketch_norm + 1e-12),
140
+ rel_tol=1e-9,
141
+ abs_tol=1e-12,
142
+ ):
143
+ raise ValueError("inconsistent node distance normalization")
144
+ minimum_rms = self.normalized_distance / math.sqrt(self.sketch_count)
145
+ if self.group_normalized_rms < minimum_rms and not math.isclose(
146
+ self.group_normalized_rms, minimum_rms, rel_tol=1e-9, abs_tol=1e-12
147
+ ):
148
+ raise ValueError("node distance exceeds group RMS bound")
149
+ return self
150
+
151
+
152
+ DetectorStateName = Literal[
153
+ "warming_up", "healthy", "suspect", "diverging", "insufficient_data"
154
+ ]
155
+
156
+
157
+ class DivergenceStatusEvidence(_EvidenceBase):
158
+ event: Literal["divergence_status"] = "divergence_status"
159
+ policy_hash: Sha256
160
+ threshold_set_id: Identifier
161
+ transition_id: int = Field(gt=0)
162
+ previous_state: DetectorStateName
163
+ state: DetectorStateName
164
+ warning_active: bool
165
+ scale_warning: bool
166
+ reason: Literal[
167
+ "warmup",
168
+ "missing_rounds",
169
+ "denominator_degenerate",
170
+ "severe_distance",
171
+ "sustained_growth",
172
+ "recovered",
173
+ "awaiting_recovery",
174
+ "growth",
175
+ "stable_window",
176
+ "collecting_window",
177
+ ]
178
+ source_round: int | None = Field(default=None, ge=0)
179
+ detection_round: int = Field(ge=0)
180
+ window_start: int | None = Field(default=None, ge=0)
181
+ window_end: int | None = Field(default=None, ge=0)
182
+ window_count: int = Field(ge=0, le=4096)
183
+ expected_participants: int = Field(ge=2)
184
+ observed_participants: int = Field(ge=0)
185
+ age_rounds: int = Field(ge=0)
186
+ missing_rounds: int = Field(ge=0)
187
+ measurement_channel: Literal["normalized", "absolute"]
188
+ earlier_median: float | None = Field(default=None, ge=0)
189
+ later_median: float | None = Field(default=None, ge=0)
190
+ log_slope: float | None = None
191
+ normalized_rms: float | None = Field(default=None, ge=0)
192
+ absolute_rms_spread: float | None = Field(default=None, ge=0)
193
+ mean_sketch_norm: float | None = Field(default=None, ge=0)
194
+
195
+ @model_validator(mode="after")
196
+ def valid_window(self) -> Self:
197
+ if self.observed_participants not in (0, self.expected_participants):
198
+ raise ValueError("detector evidence must have complete or missing coverage")
199
+ if self.source_round is not None and self.source_round > self.detection_round:
200
+ raise ValueError("source round cannot follow detection round")
201
+ if self.window_count:
202
+ if (
203
+ self.window_start is None
204
+ or self.window_end is None
205
+ or self.window_end - self.window_start + 1 != self.window_count
206
+ ):
207
+ raise ValueError("detector window must be consecutive")
208
+ if self.source_round != self.window_end:
209
+ raise ValueError("detector window must end at source round")
210
+ elif self.window_start is not None or self.window_end is not None:
211
+ raise ValueError("empty detector window cannot have endpoints")
212
+ if self.state == "diverging" and not self.warning_active:
213
+ raise ValueError("diverging state needs an active warning")
214
+ return self
215
+
216
+
217
+ class ConsensusSketchSentEvidence(_EvidenceBase):
218
+ event: Literal["consensus_sketch_sent"] = "consensus_sketch_sent"
219
+ message_id: MessageId
220
+ round_id: RoundId
221
+ payload_bytes: int = Field(ge=0)
222
+ recipient_count: int = Field(ge=1)
223
+ successful_recipient_count: int = Field(ge=0)
224
+ retry_count: int = Field(ge=0)
225
+
226
+
227
+ class TransferMessageSentEvidence(_EvidenceBase):
228
+ event: Literal["transfer_message_sent"] = "transfer_message_sent"
229
+ message_id: MessageId
230
+ transfer_id: TransferId | None = None
231
+ peer_id: PublicKey
232
+ round_id: RoundId | None = None
233
+ message_type: Identifier
234
+ payload_bytes: int = Field(ge=0)
235
+ queue_seconds: float = Field(ge=0)
236
+ send_seconds: float = Field(ge=0)
237
+ retry_count: int = Field(ge=0)
238
+ completion_seconds: float = Field(ge=0)
239
+
240
+
241
+ class RunFailedEvidence(_EvidenceBase):
242
+ event: Literal["run_failed"] = "run_failed"
243
+ message_id: MessageId
244
+ round_id: RoundId
245
+ error_type: Identifier
246
+ error: str = Field(min_length=1, max_length=1024)
247
+
248
+
249
+ type EvidenceRecord = (
250
+ RoundMetricsEvidence
251
+ | ConsensusObservationEvidence
252
+ | NodeConsensusDistanceEvidence
253
+ | DivergenceStatusEvidence
254
+ | ConsensusSketchSentEvidence
255
+ | TransferMessageSentEvidence
256
+ | RunFailedEvidence
257
+ )
258
+ _EVIDENCE_ADAPTER: TypeAdapter[EvidenceRecord] = TypeAdapter(EvidenceRecord)
259
+ _EVIDENCE_EVENTS = frozenset(
260
+ {
261
+ "round_metrics",
262
+ "consensus_observation",
263
+ "node_consensus_distance",
264
+ "divergence_status",
265
+ "consensus_sketch_sent",
266
+ "transfer_message_sent",
267
+ "run_failed",
268
+ }
269
+ )
270
+
271
+
272
+ @dataclass(frozen=True, slots=True)
273
+ class EvidenceLog:
274
+ """One strictly decoded node evidence stream from a mixed JSONL log."""
275
+
276
+ path: Path
277
+ run_id: str
278
+ manifest_hash: str
279
+ node_id: str | None
280
+ records: tuple[EvidenceRecord, ...]
281
+
282
+ @classmethod
283
+ def open(
284
+ cls,
285
+ path: Path,
286
+ *,
287
+ run_id: str,
288
+ manifest_hash: str,
289
+ ) -> EvidenceLog:
290
+ try:
291
+ lines = path.read_text(encoding="utf-8").splitlines()
292
+ except OSError as error:
293
+ raise EvidenceError(f"evidence log is unreadable: {path}") from error
294
+ records: list[EvidenceRecord] = []
295
+ for line_number, line in enumerate(lines, start=1):
296
+ try:
297
+ value = _json_object(line)
298
+ except EvidenceError as error:
299
+ raise EvidenceError(
300
+ f"invalid JSONL record at {path}:{line_number}"
301
+ ) from error
302
+ event = value.get("event")
303
+ if "evidence_version" not in value and event not in _EVIDENCE_EVENTS:
304
+ continue
305
+ try:
306
+ record = decode_evidence(line)
307
+ except EvidenceError as error:
308
+ raise EvidenceError(
309
+ f"invalid evidence at {path}:{line_number}"
310
+ ) from error
311
+ if record.run_id != run_id:
312
+ raise EvidenceError(f"evidence run id mismatch in {path}")
313
+ if record.manifest_hash != manifest_hash:
314
+ raise EvidenceError(f"evidence manifest hash mismatch in {path}")
315
+ records.append(record)
316
+ transition_ids = [
317
+ r.transition_id for r in records if isinstance(r, DivergenceStatusEvidence)
318
+ ]
319
+ if len(transition_ids) != len(set(transition_ids)) or transition_ids != sorted(
320
+ transition_ids
321
+ ):
322
+ raise EvidenceError("duplicate or unordered detector transition IDs")
323
+ policy_ids = {
324
+ (r.policy_hash, r.threshold_set_id)
325
+ for r in records
326
+ if isinstance(r, DivergenceStatusEvidence)
327
+ }
328
+ if len(policy_ids) > 1:
329
+ raise EvidenceError("detector evidence policy changed")
330
+ node_ids = {record.node_id for record in records}
331
+ if len(node_ids) > 1:
332
+ raise EvidenceError(f"evidence node id mismatch in {path}")
333
+ for record_type in (
334
+ RoundMetricsEvidence,
335
+ ConsensusObservationEvidence,
336
+ NodeConsensusDistanceEvidence,
337
+ ConsensusSketchSentEvidence,
338
+ RunFailedEvidence,
339
+ ):
340
+ rounds = [
341
+ record.round_id for record in records if isinstance(record, record_type)
342
+ ]
343
+ if len(rounds) != len(set(rounds)):
344
+ raise EvidenceError(f"duplicate evidence round in {path}")
345
+ return cls(
346
+ path=path,
347
+ run_id=run_id,
348
+ manifest_hash=manifest_hash,
349
+ node_id=next(iter(node_ids), None),
350
+ records=tuple(records),
351
+ )
352
+
353
+
354
+ def evidence_record(record: EvidenceRecord) -> dict[str, object]:
355
+ """Return one JSON-compatible strict evidence mapping."""
356
+ return cast(
357
+ dict[str, object],
358
+ record.model_dump(mode="json", exclude_none=True),
359
+ )
360
+
361
+
362
+ def encode_evidence(record: EvidenceRecord) -> bytes:
363
+ """Encode one evidence record as deterministic strict JSON."""
364
+ return canonical_json_bytes(evidence_record(record))
365
+
366
+
367
+ def decode_evidence(data: bytes | str) -> EvidenceRecord:
368
+ """Strictly decode one supported evidence record."""
369
+ value = _json_object(data)
370
+ version = value.get("evidence_version")
371
+ if version != EVIDENCE_VERSION:
372
+ raise EvidenceError(f"unsupported evidence version: {version}")
373
+ try:
374
+ return _EVIDENCE_ADAPTER.validate_json(
375
+ json.dumps(value, allow_nan=False, separators=(",", ":")),
376
+ strict=True,
377
+ )
378
+ except (TypeError, ValueError, ValidationError) as error:
379
+ raise EvidenceError("evidence record is invalid") from error
380
+
381
+
382
+ def append_evidence(sink: EventSink | None, record: EvidenceRecord) -> bool:
383
+ """Best-effort evidence write that never propagates sink failures."""
384
+ if sink is None:
385
+ return False
386
+ try:
387
+ sink.append(evidence_record(record))
388
+ except Exception:
389
+ return False
390
+ return True
391
+
392
+
393
+ def _json_object(data: bytes | str) -> dict[str, object]:
394
+ try:
395
+ value = cast(
396
+ object,
397
+ json.loads(data, parse_constant=_reject_json_constant),
398
+ )
399
+ except (TypeError, ValueError) as error:
400
+ raise EvidenceError("evidence JSON is invalid") from error
401
+ if not isinstance(value, dict):
402
+ raise EvidenceError("evidence record must be an object")
403
+ entries = cast(dict[object, object], value)
404
+ if not all(isinstance(key, str) for key in entries):
405
+ raise EvidenceError("evidence object keys must be strings")
406
+ return cast(dict[str, object], entries)
407
+
408
+
409
+ def _reject_json_constant(value: str) -> None:
410
+ raise ValueError(f"invalid JSON constant: {value}")
411
+
412
+
413
+ __all__ = [
414
+ "EVIDENCE_VERSION",
415
+ "ConsensusObservationEvidence",
416
+ "ConsensusSketchSentEvidence",
417
+ "DivergenceStatusEvidence",
418
+ "EvidenceError",
419
+ "EvidenceLog",
420
+ "EvidenceRecord",
421
+ "NodeConsensusDistanceEvidence",
422
+ "RoundMetricsEvidence",
423
+ "RunFailedEvidence",
424
+ "TransferMessageSentEvidence",
425
+ "append_evidence",
426
+ "decode_evidence",
427
+ "encode_evidence",
428
+ "evidence_record",
429
+ ]
@@ -0,0 +1,200 @@
1
+ """Bounded, append-only node metric publication."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import math
7
+ from collections.abc import Mapping
8
+ from dataclasses import dataclass
9
+ from types import MappingProxyType
10
+ from typing import Protocol
11
+
12
+ from dromeus.telemetry.events import EventSink
13
+ from dromeus.telemetry.evidence import (
14
+ EvidenceRecord,
15
+ RoundMetricsEvidence,
16
+ RunFailedEvidence,
17
+ append_evidence,
18
+ )
19
+
20
+
21
+ @dataclass(frozen=True, slots=True)
22
+ class RoundTiming:
23
+ """Timing and scalar metrics captured for one committed local round."""
24
+
25
+ round_id: int
26
+ peer_id: str
27
+ local_compute_seconds: float
28
+ peer_wait_seconds: float
29
+ transfer_seconds: float
30
+ mixing_seconds: float
31
+ evaluation_seconds: float
32
+ retries: int = 0
33
+ local_loss: float | None = None
34
+ error_feedback_residual_l2_norm: float | None = None
35
+ error_feedback_signal_l2_norm: float | None = None
36
+ error_feedback_residual_to_signal_ratio: float | None = None
37
+ evaluation_metrics: Mapping[str, float] | None = None
38
+ transfer_id: str | None = None
39
+ encoded_artifact_bytes: int | None = None
40
+
41
+ def __post_init__(self) -> None:
42
+ if self.round_id < 0:
43
+ raise ValueError("round_id must be non-negative")
44
+ if not self.peer_id:
45
+ raise ValueError("peer_id must not be empty")
46
+ durations = (
47
+ self.local_compute_seconds,
48
+ self.peer_wait_seconds,
49
+ self.transfer_seconds,
50
+ self.mixing_seconds,
51
+ self.evaluation_seconds,
52
+ )
53
+ if any(not math.isfinite(value) or value < 0 for value in durations):
54
+ raise ValueError("metric durations must be finite and non-negative")
55
+ if self.retries < 0:
56
+ raise ValueError("retries must be non-negative")
57
+ if self.encoded_artifact_bytes is not None and self.encoded_artifact_bytes <= 0:
58
+ raise ValueError("encoded artifact bytes must be positive")
59
+ if self.local_loss is not None and (not math.isfinite(self.local_loss)):
60
+ raise ValueError("local_loss must be finite")
61
+ observations = (
62
+ self.error_feedback_residual_l2_norm,
63
+ self.error_feedback_signal_l2_norm,
64
+ self.error_feedback_residual_to_signal_ratio,
65
+ )
66
+ if any(
67
+ value is not None and (not math.isfinite(value) or value < 0)
68
+ for value in observations
69
+ ):
70
+ raise ValueError(
71
+ "error-feedback observations must be finite and non-negative"
72
+ )
73
+ if self.evaluation_metrics is not None:
74
+ if any(
75
+ not name.strip() or not math.isfinite(value)
76
+ for name, value in self.evaluation_metrics.items()
77
+ ):
78
+ raise ValueError(
79
+ "task metrics require nonblank names and finite values"
80
+ )
81
+ object.__setattr__(
82
+ self,
83
+ "evaluation_metrics",
84
+ MappingProxyType(dict(self.evaluation_metrics)),
85
+ )
86
+
87
+
88
+ class MetricsPublisher(Protocol):
89
+ """Protocol-like base for non-blocking metric publication."""
90
+
91
+ def submit(self, timing: RoundTiming) -> bool:
92
+ raise NotImplementedError
93
+
94
+ def submit_failure(self, *, round_id: int, error_type: str, reason: str) -> bool:
95
+ raise NotImplementedError
96
+
97
+
98
+ class JsonlMetricsPublisher(MetricsPublisher):
99
+ """Publish correlated metrics to one node's JSONL sink off the event loop."""
100
+
101
+ def __init__(
102
+ self,
103
+ *,
104
+ sink: EventSink,
105
+ run_id: str,
106
+ manifest_hash: str,
107
+ node_id: str,
108
+ max_queue_size: int = 256,
109
+ ) -> None:
110
+ if not run_id or not manifest_hash or not node_id:
111
+ raise ValueError("metric context identifiers must not be empty")
112
+ if max_queue_size <= 0:
113
+ raise ValueError("max_queue_size must be positive")
114
+ self._sink = sink
115
+ self._run_id = run_id
116
+ self._manifest_hash = manifest_hash
117
+ self._node_id = node_id
118
+ self._queue: asyncio.Queue[EvidenceRecord] = asyncio.Queue(
119
+ maxsize=max_queue_size
120
+ )
121
+ self._stop = asyncio.Event()
122
+ self._task: asyncio.Task[None] | None = None
123
+ self._dropped = 0
124
+
125
+ @property
126
+ def pending(self) -> int:
127
+ return self._queue.qsize()
128
+
129
+ @property
130
+ def dropped(self) -> int:
131
+ return self._dropped
132
+
133
+ async def start(self) -> None:
134
+ if self._task is None:
135
+ self._task = asyncio.create_task(self._run(), name="dromeus-metrics")
136
+
137
+ async def stop(self) -> None:
138
+ if self._task is None:
139
+ return
140
+ self._stop.set()
141
+ await self._task
142
+ self._task = None
143
+
144
+ def submit(self, timing: RoundTiming) -> bool:
145
+ # Copy fields individually: mappingproxy is deliberately not pickleable.
146
+ payload: dict[str, object] = {
147
+ name: getattr(timing, name)
148
+ for name in timing.__dataclass_fields__
149
+ if name != "evaluation_metrics"
150
+ }
151
+ payload.update(
152
+ run_id=self._run_id,
153
+ manifest_hash=self._manifest_hash,
154
+ node_id=self._node_id,
155
+ message_id=f"metric-{self._node_id[:8]}-{timing.round_id}",
156
+ )
157
+ payload["evaluation_metrics"] = dict(timing.evaluation_metrics or {})
158
+ return self._enqueue(RoundMetricsEvidence.model_validate(payload))
159
+
160
+ def submit_failure(self, *, round_id: int, error_type: str, reason: str) -> bool:
161
+ return self._enqueue(
162
+ RunFailedEvidence(
163
+ run_id=self._run_id,
164
+ manifest_hash=self._manifest_hash,
165
+ node_id=self._node_id,
166
+ message_id=f"run-failure-{self._node_id[:8]}-{round_id}",
167
+ round_id=round_id,
168
+ error_type=error_type[:128],
169
+ error=reason[:1024] or "unknown failure",
170
+ )
171
+ )
172
+
173
+ def _enqueue(self, record: EvidenceRecord) -> bool:
174
+ if self._stop.is_set():
175
+ self._dropped += 1
176
+ return False
177
+ try:
178
+ self._queue.put_nowait(record)
179
+ except asyncio.QueueFull:
180
+ self._dropped += 1
181
+ return False
182
+ return True
183
+
184
+ async def _run(self) -> None:
185
+ while not self._stop.is_set() or not self._queue.empty():
186
+ try:
187
+ record = await asyncio.wait_for(self._queue.get(), timeout=0.1)
188
+ except TimeoutError:
189
+ continue
190
+ try:
191
+ written = await asyncio.to_thread(append_evidence, self._sink, record)
192
+ if not written:
193
+ self._dropped += 1
194
+ except Exception:
195
+ self._dropped += 1
196
+ finally:
197
+ self._queue.task_done()
198
+
199
+
200
+ __all__ = ["JsonlMetricsPublisher", "MetricsPublisher", "RoundTiming"]