dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"""Bounded deterministic warnings from ordered, complete consensus observations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections import deque
|
|
7
|
+
from dataclasses import dataclass, replace
|
|
8
|
+
from enum import StrEnum
|
|
9
|
+
from statistics import median
|
|
10
|
+
from threading import RLock
|
|
11
|
+
|
|
12
|
+
from dromeus.manifests.models import DivergencePolicy
|
|
13
|
+
from dromeus.telemetry.consensus import ConsensusObservation
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DetectorState(StrEnum):
|
|
17
|
+
WARMING_UP = "warming_up"
|
|
18
|
+
HEALTHY = "healthy"
|
|
19
|
+
SUSPECT = "suspect"
|
|
20
|
+
DIVERGING = "diverging"
|
|
21
|
+
INSUFFICIENT_DATA = "insufficient_data"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True, slots=True)
|
|
25
|
+
class DetectorSnapshot:
|
|
26
|
+
state: DetectorState = DetectorState.WARMING_UP
|
|
27
|
+
warning_active: bool = False
|
|
28
|
+
scale_warning: bool = False
|
|
29
|
+
reason: str = "warmup"
|
|
30
|
+
source_round: int | None = None
|
|
31
|
+
detection_round: int = -1
|
|
32
|
+
window_start: int | None = None
|
|
33
|
+
window_end: int | None = None
|
|
34
|
+
window_count: int = 0
|
|
35
|
+
expected_participants: int = 0
|
|
36
|
+
observed_participants: int = 0
|
|
37
|
+
age_rounds: int = 0
|
|
38
|
+
missing_rounds: int = 0
|
|
39
|
+
measurement_channel: str = "normalized"
|
|
40
|
+
earlier_median: float | None = None
|
|
41
|
+
later_median: float | None = None
|
|
42
|
+
log_slope: float | None = None
|
|
43
|
+
normalized_rms: float | None = None
|
|
44
|
+
absolute_rms_spread: float | None = None
|
|
45
|
+
mean_sketch_norm: float | None = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True, slots=True)
|
|
49
|
+
class DetectorTransition:
|
|
50
|
+
sequence: int
|
|
51
|
+
previous_state: DetectorState
|
|
52
|
+
snapshot: DetectorSnapshot
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DivergenceDetector:
|
|
56
|
+
"""Thread-safe pure state machine. Observation and commit order may differ.
|
|
57
|
+
|
|
58
|
+
A watermark is a locally committed round. Missing rounds older than the lag
|
|
59
|
+
allowance finalize as gaps. Warnings survive gaps until valid recovery evidence.
|
|
60
|
+
Work and memory are bounded by window/lag sizes, including watermark jumps.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(self, policy: DivergencePolicy, *, participant_count: int) -> None:
|
|
64
|
+
if participant_count < 2:
|
|
65
|
+
raise ValueError("detector requires at least two participants")
|
|
66
|
+
self.policy = policy
|
|
67
|
+
self._participants = participant_count
|
|
68
|
+
self._lock = RLock()
|
|
69
|
+
self._pending: dict[int, ConsensusObservation] = {}
|
|
70
|
+
self._window: deque[ConsensusObservation] = deque(maxlen=policy.window_rounds)
|
|
71
|
+
self._next = 0
|
|
72
|
+
self._watermark = -1
|
|
73
|
+
self._last_valid = -1
|
|
74
|
+
self._growth = self._severe = self._recovery = 0
|
|
75
|
+
self._sequence = 0
|
|
76
|
+
self._ignored = 0
|
|
77
|
+
self._invalid = 0
|
|
78
|
+
self._snapshot = DetectorSnapshot(expected_participants=participant_count)
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def snapshot(self) -> DetectorSnapshot:
|
|
82
|
+
with self._lock:
|
|
83
|
+
return self._snapshot
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def buffered_count(self) -> int:
|
|
87
|
+
with self._lock:
|
|
88
|
+
return len(self._pending) + len(self._window)
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def ignored_observations(self) -> int:
|
|
92
|
+
return self._ignored
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def invalid_observations(self) -> int:
|
|
96
|
+
return self._invalid
|
|
97
|
+
|
|
98
|
+
def observe(
|
|
99
|
+
self, observation: ConsensusObservation
|
|
100
|
+
) -> tuple[DetectorTransition, ...]:
|
|
101
|
+
with self._lock:
|
|
102
|
+
if (
|
|
103
|
+
observation.sketch_count != self._participants
|
|
104
|
+
or observation.round_id < 0
|
|
105
|
+
or not math.isfinite(observation.normalized_rms)
|
|
106
|
+
or observation.normalized_rms < 0
|
|
107
|
+
):
|
|
108
|
+
self._invalid += 1
|
|
109
|
+
return ()
|
|
110
|
+
round_id = observation.round_id
|
|
111
|
+
if (
|
|
112
|
+
round_id < self._next
|
|
113
|
+
or round_id > self._watermark + self.policy.max_lag_rounds + 1
|
|
114
|
+
):
|
|
115
|
+
self._ignored += 1
|
|
116
|
+
return ()
|
|
117
|
+
existing = self._pending.get(round_id)
|
|
118
|
+
if existing is not None:
|
|
119
|
+
if existing != observation:
|
|
120
|
+
self._invalid += 1
|
|
121
|
+
else:
|
|
122
|
+
self._ignored += 1
|
|
123
|
+
return ()
|
|
124
|
+
self._pending[round_id] = observation
|
|
125
|
+
return self._drain()
|
|
126
|
+
|
|
127
|
+
def advance(self, committed_round: int) -> tuple[DetectorTransition, ...]:
|
|
128
|
+
if committed_round < 0:
|
|
129
|
+
raise ValueError("committed round must be nonnegative")
|
|
130
|
+
with self._lock:
|
|
131
|
+
if committed_round <= self._watermark:
|
|
132
|
+
return ()
|
|
133
|
+
self._watermark = committed_round
|
|
134
|
+
return self._drain()
|
|
135
|
+
|
|
136
|
+
def _emit(self, candidate: DetectorSnapshot) -> tuple[DetectorTransition, ...]:
|
|
137
|
+
old = self._snapshot
|
|
138
|
+
self._snapshot = candidate
|
|
139
|
+
if (old.state, old.warning_active, old.scale_warning) == (
|
|
140
|
+
candidate.state,
|
|
141
|
+
candidate.warning_active,
|
|
142
|
+
candidate.scale_warning,
|
|
143
|
+
):
|
|
144
|
+
return ()
|
|
145
|
+
self._sequence += 1
|
|
146
|
+
return (DetectorTransition(self._sequence, old.state, candidate),)
|
|
147
|
+
|
|
148
|
+
def _drain(self) -> tuple[DetectorTransition, ...]:
|
|
149
|
+
transitions: list[DetectorTransition] = []
|
|
150
|
+
while self._next <= self._watermark:
|
|
151
|
+
observation = self._pending.pop(self._next, None)
|
|
152
|
+
if observation is not None:
|
|
153
|
+
self._next += 1
|
|
154
|
+
transitions.extend(self._consume(observation))
|
|
155
|
+
continue
|
|
156
|
+
cutoff = self._watermark - self.policy.max_lag_rounds
|
|
157
|
+
if self._next >= cutoff:
|
|
158
|
+
break
|
|
159
|
+
next_pending = min(self._pending, default=cutoff)
|
|
160
|
+
end = min(cutoff, next_pending)
|
|
161
|
+
missing = end - self._next
|
|
162
|
+
self._next = end
|
|
163
|
+
self._window.clear()
|
|
164
|
+
self._growth = self._severe = self._recovery = 0
|
|
165
|
+
transitions.extend(
|
|
166
|
+
self._emit(
|
|
167
|
+
replace(
|
|
168
|
+
self._snapshot,
|
|
169
|
+
state=DetectorState.INSUFFICIENT_DATA,
|
|
170
|
+
reason="missing_rounds",
|
|
171
|
+
detection_round=self._watermark,
|
|
172
|
+
window_start=None,
|
|
173
|
+
window_end=None,
|
|
174
|
+
window_count=0,
|
|
175
|
+
observed_participants=0,
|
|
176
|
+
missing_rounds=self._snapshot.missing_rounds + missing,
|
|
177
|
+
age_rounds=self._watermark - self._last_valid,
|
|
178
|
+
earlier_median=None,
|
|
179
|
+
later_median=None,
|
|
180
|
+
log_slope=None,
|
|
181
|
+
)
|
|
182
|
+
)
|
|
183
|
+
)
|
|
184
|
+
self._snapshot = replace(
|
|
185
|
+
self._snapshot,
|
|
186
|
+
detection_round=self._watermark,
|
|
187
|
+
age_rounds=max(0, self._watermark - self._last_valid),
|
|
188
|
+
)
|
|
189
|
+
return tuple(transitions)
|
|
190
|
+
|
|
191
|
+
def _consume(self, point: ConsensusObservation) -> tuple[DetectorTransition, ...]:
|
|
192
|
+
p = self.policy
|
|
193
|
+
self._last_valid = point.round_id
|
|
194
|
+
candidate = replace(
|
|
195
|
+
self._snapshot,
|
|
196
|
+
source_round=point.round_id,
|
|
197
|
+
detection_round=self._watermark,
|
|
198
|
+
age_rounds=max(0, self._watermark - point.round_id),
|
|
199
|
+
observed_participants=point.sketch_count,
|
|
200
|
+
normalized_rms=point.normalized_rms,
|
|
201
|
+
absolute_rms_spread=point.absolute_rms_spread,
|
|
202
|
+
mean_sketch_norm=point.mean_sketch_norm,
|
|
203
|
+
scale_warning=point.denominator_degenerate,
|
|
204
|
+
)
|
|
205
|
+
if point.round_id < p.warmup_rounds:
|
|
206
|
+
return self._emit(replace(candidate, reason="warmup"))
|
|
207
|
+
self._window.append(point)
|
|
208
|
+
absolute = any(x.denominator_degenerate for x in self._window)
|
|
209
|
+
channel = "absolute" if absolute else "normalized"
|
|
210
|
+
if channel != self._snapshot.measurement_channel:
|
|
211
|
+
self._growth = self._severe = self._recovery = 0
|
|
212
|
+
floor = p.absolute_distance_floor if absolute else p.distance_floor
|
|
213
|
+
severe = p.absolute_severe_distance if absolute else p.severe_distance
|
|
214
|
+
recovery = p.absolute_recovery_distance if absolute else p.recovery_distance
|
|
215
|
+
candidate = replace(
|
|
216
|
+
candidate,
|
|
217
|
+
measurement_channel=channel,
|
|
218
|
+
window_start=self._window[0].round_id,
|
|
219
|
+
window_end=point.round_id,
|
|
220
|
+
window_count=len(self._window),
|
|
221
|
+
earlier_median=None,
|
|
222
|
+
later_median=None,
|
|
223
|
+
log_slope=None,
|
|
224
|
+
)
|
|
225
|
+
if floor is None or severe is None or recovery is None:
|
|
226
|
+
self._growth = self._severe = self._recovery = 0
|
|
227
|
+
return self._emit(
|
|
228
|
+
replace(
|
|
229
|
+
candidate,
|
|
230
|
+
state=DetectorState.INSUFFICIENT_DATA,
|
|
231
|
+
reason="denominator_degenerate",
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
values = [
|
|
235
|
+
x.absolute_rms_spread if absolute else x.normalized_rms
|
|
236
|
+
for x in self._window
|
|
237
|
+
]
|
|
238
|
+
self._severe = self._severe + 1 if values[-1] > severe else 0
|
|
239
|
+
full = len(values) == p.window_rounds
|
|
240
|
+
growth = False
|
|
241
|
+
if full:
|
|
242
|
+
half = len(values) // 2
|
|
243
|
+
early, late = median(values[:half]), median(values[half:])
|
|
244
|
+
# Centered rounds avoid cancellation for large absolute round IDs.
|
|
245
|
+
xs = [x.round_id - self._window[0].round_id for x in self._window]
|
|
246
|
+
ys = [math.log(max(x, floor)) for x in values]
|
|
247
|
+
xm, ym = sum(xs) / len(xs), sum(ys) / len(ys)
|
|
248
|
+
slope = sum((x - xm) * (y - ym) for x, y in zip(xs, ys, strict=True)) / sum(
|
|
249
|
+
(x - xm) ** 2 for x in xs
|
|
250
|
+
)
|
|
251
|
+
growth = (
|
|
252
|
+
late > floor
|
|
253
|
+
and late / max(early, floor) > p.growth_ratio
|
|
254
|
+
and slope > p.min_log_slope
|
|
255
|
+
)
|
|
256
|
+
candidate = replace(
|
|
257
|
+
candidate, earlier_median=early, later_median=late, log_slope=slope
|
|
258
|
+
)
|
|
259
|
+
self._growth = self._growth + 1 if growth else 0
|
|
260
|
+
if self._growth >= p.patience_windows or self._severe >= p.severe_patience:
|
|
261
|
+
self._recovery = 0
|
|
262
|
+
return self._emit(
|
|
263
|
+
replace(
|
|
264
|
+
candidate,
|
|
265
|
+
state=DetectorState.DIVERGING,
|
|
266
|
+
warning_active=True,
|
|
267
|
+
reason="severe_distance"
|
|
268
|
+
if self._severe >= p.severe_patience
|
|
269
|
+
else "sustained_growth",
|
|
270
|
+
)
|
|
271
|
+
)
|
|
272
|
+
if candidate.warning_active and not full and candidate.missing_rounds:
|
|
273
|
+
self._recovery = 0
|
|
274
|
+
return self._emit(
|
|
275
|
+
replace(
|
|
276
|
+
candidate,
|
|
277
|
+
state=DetectorState.INSUFFICIENT_DATA,
|
|
278
|
+
reason="collecting_window",
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
if candidate.warning_active:
|
|
282
|
+
self._recovery = (
|
|
283
|
+
self._recovery + 1
|
|
284
|
+
if full and max(values) < recovery and not growth
|
|
285
|
+
else 0
|
|
286
|
+
)
|
|
287
|
+
if self._recovery >= p.recovery_windows:
|
|
288
|
+
return self._emit(
|
|
289
|
+
replace(
|
|
290
|
+
candidate,
|
|
291
|
+
state=DetectorState.HEALTHY,
|
|
292
|
+
warning_active=False,
|
|
293
|
+
reason="recovered",
|
|
294
|
+
)
|
|
295
|
+
)
|
|
296
|
+
return self._emit(
|
|
297
|
+
replace(
|
|
298
|
+
candidate, state=DetectorState.DIVERGING, reason="awaiting_recovery"
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
if growth or self._severe:
|
|
302
|
+
return self._emit(
|
|
303
|
+
replace(
|
|
304
|
+
candidate,
|
|
305
|
+
state=DetectorState.SUSPECT,
|
|
306
|
+
reason="growth" if growth else "severe_distance",
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
if full:
|
|
310
|
+
return self._emit(
|
|
311
|
+
replace(candidate, state=DetectorState.HEALTHY, reason="stable_window")
|
|
312
|
+
)
|
|
313
|
+
return self._emit(
|
|
314
|
+
replace(
|
|
315
|
+
candidate,
|
|
316
|
+
state=DetectorState.WARMING_UP
|
|
317
|
+
if candidate.missing_rounds == 0
|
|
318
|
+
else DetectorState.INSUFFICIENT_DATA,
|
|
319
|
+
reason="collecting_window",
|
|
320
|
+
)
|
|
321
|
+
)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Compose pure detector state with bounded best-effort evidence delivery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import asdict
|
|
6
|
+
from threading import RLock
|
|
7
|
+
|
|
8
|
+
from dromeus.manifests.canonical import canonical_hash
|
|
9
|
+
from dromeus.manifests.models import DivergencePolicy
|
|
10
|
+
from dromeus.telemetry.consensus import ConsensusObservation
|
|
11
|
+
from dromeus.telemetry.divergence import (
|
|
12
|
+
DetectorSnapshot,
|
|
13
|
+
DetectorTransition,
|
|
14
|
+
DivergenceDetector,
|
|
15
|
+
)
|
|
16
|
+
from dromeus.telemetry.events import BoundedEventSink, EventSink
|
|
17
|
+
from dromeus.telemetry.evidence import (
|
|
18
|
+
DivergenceStatusEvidence,
|
|
19
|
+
EvidenceRecord,
|
|
20
|
+
append_evidence,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DivergenceMonitor:
|
|
25
|
+
"""A single daemon writer isolates even a blocked synchronous sink.
|
|
26
|
+
|
|
27
|
+
Closing is bounded. An in-flight blocking sink cannot be killed by Python;
|
|
28
|
+
it is counted as a drop on timeout and may finish later. No new writes follow
|
|
29
|
+
that timeout. Status never depends on successful evidence delivery.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
*,
|
|
35
|
+
policy: DivergencePolicy,
|
|
36
|
+
participant_count: int,
|
|
37
|
+
run_id: str,
|
|
38
|
+
manifest_hash: str,
|
|
39
|
+
node_id: str,
|
|
40
|
+
sink: EventSink | None,
|
|
41
|
+
queue_size: int = 64,
|
|
42
|
+
) -> None:
|
|
43
|
+
if queue_size <= 0:
|
|
44
|
+
raise ValueError("warning queue size must be positive")
|
|
45
|
+
self.detector = DivergenceDetector(policy, participant_count=participant_count)
|
|
46
|
+
self._identity = dict(
|
|
47
|
+
run_id=run_id, manifest_hash=manifest_hash, node_id=node_id
|
|
48
|
+
)
|
|
49
|
+
self._policy_hash = canonical_hash(policy)
|
|
50
|
+
self._owns_delivery = not isinstance(sink, BoundedEventSink)
|
|
51
|
+
self._delivery = (
|
|
52
|
+
sink
|
|
53
|
+
if isinstance(sink, BoundedEventSink)
|
|
54
|
+
else BoundedEventSink(sink, capacity=queue_size)
|
|
55
|
+
)
|
|
56
|
+
self._lock = RLock()
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def snapshot(self) -> DetectorSnapshot:
|
|
60
|
+
return self.detector.snapshot
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def dropped(self) -> int:
|
|
64
|
+
return self._delivery.dropped
|
|
65
|
+
|
|
66
|
+
def start(self) -> None:
|
|
67
|
+
self._delivery.start()
|
|
68
|
+
|
|
69
|
+
def _enqueue(self, record: EvidenceRecord) -> None:
|
|
70
|
+
append_evidence(self._delivery, record)
|
|
71
|
+
|
|
72
|
+
def observe(self, observation: ConsensusObservation) -> None:
|
|
73
|
+
"""Feed one observation; the runtime records the observation itself."""
|
|
74
|
+
with self._lock:
|
|
75
|
+
self._publish(self.detector.observe(observation))
|
|
76
|
+
|
|
77
|
+
def advance(self, committed_round: int) -> None:
|
|
78
|
+
with self._lock:
|
|
79
|
+
self._publish(self.detector.advance(committed_round))
|
|
80
|
+
|
|
81
|
+
def _publish(self, transitions: tuple[DetectorTransition, ...]) -> None:
|
|
82
|
+
for change in transitions:
|
|
83
|
+
value = {
|
|
84
|
+
**self._identity,
|
|
85
|
+
**asdict(change.snapshot),
|
|
86
|
+
"state": change.snapshot.state.value,
|
|
87
|
+
"previous_state": change.previous_state.value,
|
|
88
|
+
"transition_id": change.sequence,
|
|
89
|
+
"policy_hash": self._policy_hash,
|
|
90
|
+
"threshold_set_id": self.detector.policy.threshold_set_id,
|
|
91
|
+
}
|
|
92
|
+
self._enqueue(DivergenceStatusEvidence.model_validate(value))
|
|
93
|
+
|
|
94
|
+
def stop(self, *, timeout_seconds: float = 0.2) -> bool:
|
|
95
|
+
if self._owns_delivery:
|
|
96
|
+
return self._delivery.stop(timeout_seconds=timeout_seconds)
|
|
97
|
+
return True
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Structured event output shared by Dromeus runtime modules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from datetime import UTC, datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from queue import Empty, Full, Queue
|
|
12
|
+
from threading import Event, Lock, Thread
|
|
13
|
+
from typing import Protocol
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class EventSink(Protocol):
|
|
17
|
+
def append(self, record: Mapping[str, object]) -> None: ...
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def event_log_path(run_root: Path) -> Path:
|
|
21
|
+
"""Where a node writes its JSONL events and evidence under its run root."""
|
|
22
|
+
return run_root / "logs" / "dromeus.jsonl"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(slots=True)
|
|
26
|
+
class JsonlEventSink:
|
|
27
|
+
"""Thread-safe append-only JSONL sink for one node's telemetry."""
|
|
28
|
+
|
|
29
|
+
path: Path
|
|
30
|
+
_lock: Lock = field(default_factory=Lock, init=False, repr=False)
|
|
31
|
+
|
|
32
|
+
def __post_init__(self) -> None:
|
|
33
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
|
|
35
|
+
def append(self, record: Mapping[str, object]) -> None:
|
|
36
|
+
line = json.dumps(
|
|
37
|
+
dict(record),
|
|
38
|
+
allow_nan=False,
|
|
39
|
+
separators=(",", ":"),
|
|
40
|
+
sort_keys=True,
|
|
41
|
+
)
|
|
42
|
+
with self._lock, self.path.open("a", encoding="utf-8") as handle:
|
|
43
|
+
handle.write(f"{line}\n")
|
|
44
|
+
handle.flush()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def emit_event(
|
|
48
|
+
event: str,
|
|
49
|
+
*,
|
|
50
|
+
run_id: str | None = None,
|
|
51
|
+
manifest_hash: str | None = None,
|
|
52
|
+
node_id: str | None = None,
|
|
53
|
+
message_id: str | None = None,
|
|
54
|
+
transfer_id: str | None = None,
|
|
55
|
+
peer_id: str | None = None,
|
|
56
|
+
round_id: int | None = None,
|
|
57
|
+
sink: EventSink | None = None,
|
|
58
|
+
**fields: object,
|
|
59
|
+
) -> None:
|
|
60
|
+
"""Write one JSON event, retaining correlation IDs when supplied."""
|
|
61
|
+
record = event_record(
|
|
62
|
+
event,
|
|
63
|
+
run_id=run_id,
|
|
64
|
+
manifest_hash=manifest_hash,
|
|
65
|
+
node_id=node_id,
|
|
66
|
+
message_id=message_id,
|
|
67
|
+
transfer_id=transfer_id,
|
|
68
|
+
peer_id=peer_id,
|
|
69
|
+
round_id=round_id,
|
|
70
|
+
**fields,
|
|
71
|
+
)
|
|
72
|
+
if sink is not None:
|
|
73
|
+
try:
|
|
74
|
+
sink.append(record)
|
|
75
|
+
except Exception:
|
|
76
|
+
pass
|
|
77
|
+
return
|
|
78
|
+
try:
|
|
79
|
+
print(
|
|
80
|
+
json.dumps(record, separators=(",", ":"), sort_keys=True),
|
|
81
|
+
file=sys.stdout,
|
|
82
|
+
)
|
|
83
|
+
except Exception:
|
|
84
|
+
pass
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def event_record(
|
|
88
|
+
event: str,
|
|
89
|
+
*,
|
|
90
|
+
run_id: str | None = None,
|
|
91
|
+
manifest_hash: str | None = None,
|
|
92
|
+
node_id: str | None = None,
|
|
93
|
+
message_id: str | None = None,
|
|
94
|
+
transfer_id: str | None = None,
|
|
95
|
+
peer_id: str | None = None,
|
|
96
|
+
round_id: int | None = None,
|
|
97
|
+
**fields: object,
|
|
98
|
+
) -> dict[str, object]:
|
|
99
|
+
"""Build one deterministic structured event without writing it."""
|
|
100
|
+
record: dict[str, object] = {
|
|
101
|
+
"timestamp": datetime.now(UTC).isoformat(),
|
|
102
|
+
"event": event,
|
|
103
|
+
}
|
|
104
|
+
identifiers: Mapping[str, str | int | None] = {
|
|
105
|
+
"run_id": run_id,
|
|
106
|
+
"manifest_hash": manifest_hash,
|
|
107
|
+
"node_id": node_id,
|
|
108
|
+
"message_id": message_id,
|
|
109
|
+
"transfer_id": transfer_id,
|
|
110
|
+
"peer_id": peer_id,
|
|
111
|
+
"round_id": round_id,
|
|
112
|
+
}
|
|
113
|
+
record.update(
|
|
114
|
+
{key: value for key, value in identifiers.items() if value is not None}
|
|
115
|
+
)
|
|
116
|
+
record.update(fields)
|
|
117
|
+
return record
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
__all__ = ["EventSink", "JsonlEventSink", "emit_event", "event_record"]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class BoundedEventSink:
|
|
124
|
+
"""Nonblocking facade over a possibly blocking sink, with bounded shutdown.
|
|
125
|
+
|
|
126
|
+
A timed-out in-flight write may complete later; no further writes are started.
|
|
127
|
+
All users sharing the target sink must share this facade to avoid lock contention.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
def __init__(self, sink: EventSink | None, *, capacity: int = 64) -> None:
|
|
131
|
+
if capacity <= 0:
|
|
132
|
+
raise ValueError("event capacity must be positive")
|
|
133
|
+
self._sink = sink
|
|
134
|
+
self._queue: Queue[Mapping[str, object]] = Queue(maxsize=capacity)
|
|
135
|
+
self._count_lock = Lock()
|
|
136
|
+
self._stop = Event()
|
|
137
|
+
self._abort = Event()
|
|
138
|
+
self._thread: Thread | None = None
|
|
139
|
+
self._in_flight = False
|
|
140
|
+
self._dropped = 0
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def dropped(self) -> int:
|
|
144
|
+
with self._count_lock:
|
|
145
|
+
return self._dropped
|
|
146
|
+
|
|
147
|
+
def start(self) -> None:
|
|
148
|
+
with self._count_lock:
|
|
149
|
+
if self._thread is not None or self._stop.is_set():
|
|
150
|
+
return
|
|
151
|
+
self._thread = Thread(
|
|
152
|
+
target=self._run, name="dromeus-event-delivery", daemon=True
|
|
153
|
+
)
|
|
154
|
+
self._thread.start()
|
|
155
|
+
|
|
156
|
+
def append(self, record: Mapping[str, object]) -> None:
|
|
157
|
+
with self._count_lock:
|
|
158
|
+
if self._stop.is_set():
|
|
159
|
+
self._dropped += 1
|
|
160
|
+
return
|
|
161
|
+
try:
|
|
162
|
+
self._queue.put_nowait(dict(record))
|
|
163
|
+
except Full:
|
|
164
|
+
self._dropped += 1
|
|
165
|
+
|
|
166
|
+
def _run(self) -> None:
|
|
167
|
+
while not self._abort.is_set():
|
|
168
|
+
if self._stop.is_set() and self._queue.empty():
|
|
169
|
+
return
|
|
170
|
+
try:
|
|
171
|
+
record = self._queue.get(timeout=0.05)
|
|
172
|
+
except Empty:
|
|
173
|
+
continue
|
|
174
|
+
with self._count_lock:
|
|
175
|
+
if self._abort.is_set():
|
|
176
|
+
self._dropped += 1
|
|
177
|
+
self._queue.task_done()
|
|
178
|
+
return
|
|
179
|
+
self._in_flight = True
|
|
180
|
+
success = False
|
|
181
|
+
try:
|
|
182
|
+
if self._sink is not None:
|
|
183
|
+
self._sink.append(record)
|
|
184
|
+
success = True
|
|
185
|
+
except Exception:
|
|
186
|
+
pass
|
|
187
|
+
with self._count_lock:
|
|
188
|
+
if not success and not self._abort.is_set():
|
|
189
|
+
self._dropped += 1
|
|
190
|
+
self._in_flight = False
|
|
191
|
+
self._queue.task_done()
|
|
192
|
+
|
|
193
|
+
def stop(self, *, timeout_seconds: float = 0.2) -> bool:
|
|
194
|
+
if timeout_seconds <= 0:
|
|
195
|
+
raise ValueError("event stop timeout must be positive")
|
|
196
|
+
with self._count_lock:
|
|
197
|
+
self._stop.set()
|
|
198
|
+
thread = self._thread
|
|
199
|
+
if thread is not None:
|
|
200
|
+
thread.join(timeout_seconds)
|
|
201
|
+
if not thread.is_alive():
|
|
202
|
+
return True
|
|
203
|
+
with self._count_lock:
|
|
204
|
+
if not self._abort.is_set():
|
|
205
|
+
self._abort.set()
|
|
206
|
+
self._dropped += int(self._in_flight)
|
|
207
|
+
while True:
|
|
208
|
+
try:
|
|
209
|
+
self._queue.get_nowait()
|
|
210
|
+
self._queue.task_done()
|
|
211
|
+
self._dropped += 1
|
|
212
|
+
except Empty:
|
|
213
|
+
break
|
|
214
|
+
return thread is None or not thread.is_alive()
|