sensitivebox 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sensitivebox/__init__.py +10 -0
- sensitivebox/alerts/__init__.py +94 -0
- sensitivebox/alerts/config.py +85 -0
- sensitivebox/alerts/defaults.toml +126 -0
- sensitivebox/alerts/engine.py +624 -0
- sensitivebox/alerts/events.py +87 -0
- sensitivebox/alerts/globs.py +84 -0
- sensitivebox/alerts/model.py +215 -0
- sensitivebox/alerts/mute.py +55 -0
- sensitivebox/alerts/replay.py +99 -0
- sensitivebox/alerts/routing.py +247 -0
- sensitivebox/api/__init__.py +6 -0
- sensitivebox/api/app.py +225 -0
- sensitivebox/api/data.py +109 -0
- sensitivebox/api/host.py +27 -0
- sensitivebox/api/model.py +90 -0
- sensitivebox/api/security.py +89 -0
- sensitivebox/api/server.py +108 -0
- sensitivebox/api/state.py +220 -0
- sensitivebox/api/static/assets/archivo-latin-wdth-normal-DY7AcnAa.woff2 +0 -0
- sensitivebox/api/static/assets/ibm-plex-mono-medium-BpyVxOr_.woff2 +0 -0
- sensitivebox/api/static/assets/index-BH9EydfS.js +10 -0
- sensitivebox/api/static/assets/index-RMruxW54.css +1 -0
- sensitivebox/api/static/assets/mark-on-dark-CpSxozdM.png +0 -0
- sensitivebox/api/static/assets/mark-on-light-DgUAeHTp.png +0 -0
- sensitivebox/api/static/assets/mark-raised-Bg5z6PGs.png +0 -0
- sensitivebox/api/static/index.html +14 -0
- sensitivebox/browser.py +106 -0
- sensitivebox/cli.py +467 -0
- sensitivebox/collector.py +319 -0
- sensitivebox/completion.py +117 -0
- sensitivebox/daemon.py +181 -0
- sensitivebox/data/icons/128.png +0 -0
- sensitivebox/data/icons/256.png +0 -0
- sensitivebox/data/icons/512.png +0 -0
- sensitivebox/data/proart-p16.tar.gz +0 -0
- sensitivebox/data/sensitivebox.desktop.in +8 -0
- sensitivebox/data/sensitivebox.service.in +11 -0
- sensitivebox/doctor.py +322 -0
- sensitivebox/fixtures.py +177 -0
- sensitivebox/history.py +298 -0
- sensitivebox/hwmon/__init__.py +20 -0
- sensitivebox/hwmon/classify.py +74 -0
- sensitivebox/hwmon/discovery.py +139 -0
- sensitivebox/hwmon/ids.py +35 -0
- sensitivebox/hwmon/model.py +78 -0
- sensitivebox/hwmon/normalise.py +42 -0
- sensitivebox/hwmon/reader.py +65 -0
- sensitivebox/inspect.py +175 -0
- sensitivebox/live.py +56 -0
- sensitivebox/notify/__init__.py +7 -0
- sensitivebox/notify/base.py +60 -0
- sensitivebox/notify/desktop.py +84 -0
- sensitivebox/notify/log.py +30 -0
- sensitivebox/operations.py +44 -0
- sensitivebox/output.py +76 -0
- sensitivebox/process.py +25 -0
- sensitivebox/py.typed +0 -0
- sensitivebox/redact.py +85 -0
- sensitivebox/report.py +357 -0
- sensitivebox/runtime.py +313 -0
- sensitivebox/sample.py +84 -0
- sensitivebox/service.py +112 -0
- sensitivebox/simulate.py +220 -0
- sensitivebox/sources/__init__.py +83 -0
- sensitivebox/sources/apps.py +135 -0
- sensitivebox/sources/bluez.py +96 -0
- sensitivebox/sources/bluez_watch.py +214 -0
- sensitivebox/sources/changes.py +121 -0
- sensitivebox/sources/cpu.py +152 -0
- sensitivebox/sources/dbus.py +174 -0
- sensitivebox/sources/logind.py +88 -0
- sensitivebox/sources/memory.py +127 -0
- sensitivebox/sources/net.py +128 -0
- sensitivebox/sources/power_profiles.py +116 -0
- sensitivebox/sources/power_supply.py +70 -0
- sensitivebox/sources/psi.py +78 -0
- sensitivebox/sources/report.py +85 -0
- sensitivebox/sources/wifi.py +90 -0
- sensitivebox/sources/wifi_nm.py +269 -0
- sensitivebox/status.py +205 -0
- sensitivebox/store/__init__.py +5 -0
- sensitivebox/store/changes.py +58 -0
- sensitivebox/store/db.py +423 -0
- sensitivebox/store/events.py +337 -0
- sensitivebox/store/mutes.py +58 -0
- sensitivebox/store/quality.py +51 -0
- sensitivebox/store/read.py +40 -0
- sensitivebox/store/schema.py +194 -0
- sensitivebox/store/status.py +105 -0
- sensitivebox-0.1.0.dist-info/METADATA +424 -0
- sensitivebox-0.1.0.dist-info/RECORD +95 -0
- sensitivebox-0.1.0.dist-info/WHEEL +4 -0
- sensitivebox-0.1.0.dist-info/entry_points.txt +4 -0
- sensitivebox-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,624 @@
|
|
|
1
|
+
"""The alert engine: one small state machine per (rule, signal), driven by an injected clock.
|
|
2
|
+
|
|
3
|
+
Contract (docs/BIBLE.md, "Alerting"):
|
|
4
|
+
|
|
5
|
+
- Input per poll is ``value | Missing(reason)``. Missing is never 0, never above or below a
|
|
6
|
+
threshold, and never a recovery. It *pauses* the ``for`` and ``clear_for`` timers.
|
|
7
|
+
- States: Warming → OK → Pending → Firing → Resolved, plus Unknown (missing, no alert open).
|
|
8
|
+
- Loading, editing a rule and ``resume()`` put it in Warming: it can't fire on values until it
|
|
9
|
+
has seen a full ``for`` window of real readings (at least one). An open alert survives that
|
|
10
|
+
and is not notified again.
|
|
11
|
+
- ``suspend()`` (logind PrepareForSleep/PrepareForShutdown) stops evaluation and the
|
|
12
|
+
missing-data timers, so sleep and shutdown never fire anything.
|
|
13
|
+
- After ``missing_for`` without data, ``on_missing`` decides: ignore, alert (at
|
|
14
|
+
``missing_severity``) or close ("closed: no data").
|
|
15
|
+
|
|
16
|
+
The engine never notifies anyone: it returns transitions, and the notification pipeline (mute,
|
|
17
|
+
cooldown, notifiers) runs after it. Everything it is given (rules, limits, each poll's readings,
|
|
18
|
+
suspend and resume) is reported to an optional ``journal``, so the store can record it and
|
|
19
|
+
``replay`` can reproduce exactly what the engine did.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from enum import StrEnum
|
|
25
|
+
from typing import Protocol
|
|
26
|
+
|
|
27
|
+
import msgspec
|
|
28
|
+
|
|
29
|
+
from sensitivebox.alerts import globs
|
|
30
|
+
from sensitivebox.alerts.events import (
|
|
31
|
+
EngineSettings,
|
|
32
|
+
InputEvent,
|
|
33
|
+
Journal,
|
|
34
|
+
LimitsChanged,
|
|
35
|
+
OpenAlerts,
|
|
36
|
+
Restart,
|
|
37
|
+
Resume,
|
|
38
|
+
RulesLoaded,
|
|
39
|
+
Suspend,
|
|
40
|
+
Tick,
|
|
41
|
+
)
|
|
42
|
+
from sensitivebox.alerts.model import (
|
|
43
|
+
Missing,
|
|
44
|
+
Rule,
|
|
45
|
+
Severity,
|
|
46
|
+
State,
|
|
47
|
+
Value,
|
|
48
|
+
clear_on_breaching_side,
|
|
49
|
+
)
|
|
50
|
+
from sensitivebox.hwmon import Limits
|
|
51
|
+
|
|
52
|
+
CLOSED_NO_DATA = "closed: no data"
|
|
53
|
+
RECOVERED = "recovered"
|
|
54
|
+
DATA_BACK = "data back"
|
|
55
|
+
RULE_REMOVED = "rule removed"
|
|
56
|
+
RULE_CHANGED = "rule no longer matches"
|
|
57
|
+
GUARD_FALSE = "not applicable (guard)"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Clock(Protocol):
|
|
61
|
+
"""Seconds on a **monotonic** clock that stops during suspend (``time.monotonic`` on Linux).
|
|
62
|
+
Every timer runs on it, never on the wall clock (docs/BIBLE.md, "Time"). A step backwards
|
|
63
|
+
or a zero step counts as no time passing."""
|
|
64
|
+
|
|
65
|
+
def __call__(self) -> float: ...
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class ManualClock:
|
|
69
|
+
"""A clock that only moves when told to: for tests, replay and the simulator."""
|
|
70
|
+
|
|
71
|
+
def __init__(self, now: float = 0.0) -> None:
|
|
72
|
+
self.now = now
|
|
73
|
+
|
|
74
|
+
def __call__(self) -> float:
|
|
75
|
+
return self.now
|
|
76
|
+
|
|
77
|
+
def advance(self, seconds: float) -> None:
|
|
78
|
+
self.now += seconds
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class TransitionKind(StrEnum):
|
|
82
|
+
FIRING = "firing"
|
|
83
|
+
RESOLVED = "resolved"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True, slots=True)
|
|
87
|
+
class Transition:
|
|
88
|
+
"""An alert opening or closing: the only thing that can lead to a notification."""
|
|
89
|
+
|
|
90
|
+
at: float
|
|
91
|
+
rule_id: str
|
|
92
|
+
signal: str
|
|
93
|
+
kind: TransitionKind
|
|
94
|
+
severity: Severity
|
|
95
|
+
reason: str # "95 above 90 for 10 s", "recovered", "closed: no data", …
|
|
96
|
+
value: float | str | None
|
|
97
|
+
threshold: float | None
|
|
98
|
+
opened_at: float
|
|
99
|
+
no_data: bool = False # a missing-data alert (on_missing = "alert")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True, slots=True)
|
|
103
|
+
class Evaluation:
|
|
104
|
+
"""One rule evaluated against one signal at one poll. Stored in the 1 h ring (Prompt 3)
|
|
105
|
+
so ``doctor --rule`` can answer "why didn't it fire?"."""
|
|
106
|
+
|
|
107
|
+
at: float
|
|
108
|
+
rule_id: str
|
|
109
|
+
signal: str
|
|
110
|
+
value: float | str | None
|
|
111
|
+
missing: str | None # the reason, when the reading was Missing
|
|
112
|
+
state: State
|
|
113
|
+
reason: str
|
|
114
|
+
threshold: float | None
|
|
115
|
+
alert_severity: Severity | None = None
|
|
116
|
+
no_data: bool = False
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True, slots=True)
|
|
120
|
+
class Step:
|
|
121
|
+
evaluations: tuple[Evaluation, ...] = ()
|
|
122
|
+
transitions: tuple[Transition, ...] = ()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def format_duration(seconds: float) -> str:
|
|
126
|
+
if seconds < 60:
|
|
127
|
+
return f"{seconds:g} s"
|
|
128
|
+
total = round(seconds)
|
|
129
|
+
if total < 3600:
|
|
130
|
+
m, s = divmod(total, 60)
|
|
131
|
+
return f"{m} min {s} s" if s else f"{m} min"
|
|
132
|
+
h, rem = divmod(total, 3600)
|
|
133
|
+
m = rem // 60
|
|
134
|
+
return f"{h} h {m} min" if m else f"{h} h"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@dataclass(slots=True)
|
|
138
|
+
class _Alert:
|
|
139
|
+
opened_at: float
|
|
140
|
+
severity: Severity
|
|
141
|
+
no_data: bool
|
|
142
|
+
threshold: float | None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(slots=True)
|
|
146
|
+
class _Instance:
|
|
147
|
+
alert: _Alert | None = None
|
|
148
|
+
state: State = State.WARMING
|
|
149
|
+
warming: bool = True
|
|
150
|
+
warm_elapsed: float = 0.0
|
|
151
|
+
warm_count: int = 0
|
|
152
|
+
in_breach: bool = False # the last *real* reading breached
|
|
153
|
+
breach_elapsed: float = 0.0
|
|
154
|
+
in_clear: bool = False # the last real reading was past `clear`
|
|
155
|
+
clear_elapsed: float = 0.0
|
|
156
|
+
last_t: float | None = None
|
|
157
|
+
last_real: bool = False
|
|
158
|
+
missing_since: float | None = None
|
|
159
|
+
|
|
160
|
+
def restart(self) -> None:
|
|
161
|
+
"""Back to Warming with fresh timers; an open alert stays open."""
|
|
162
|
+
alert = self.alert
|
|
163
|
+
for name, default in _FRESH.items():
|
|
164
|
+
setattr(self, name, default)
|
|
165
|
+
self.alert = alert
|
|
166
|
+
self.state = State.WARMING
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
_FRESH = {
|
|
170
|
+
"warming": True,
|
|
171
|
+
"warm_elapsed": 0.0,
|
|
172
|
+
"warm_count": 0,
|
|
173
|
+
"in_breach": False,
|
|
174
|
+
"breach_elapsed": 0.0,
|
|
175
|
+
"in_clear": False,
|
|
176
|
+
"clear_elapsed": 0.0,
|
|
177
|
+
"last_t": None,
|
|
178
|
+
"last_real": False,
|
|
179
|
+
"missing_since": None,
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
@dataclass(slots=True)
|
|
184
|
+
class _Ctx:
|
|
185
|
+
"""Everything one evaluation needs to know about its rule and signal."""
|
|
186
|
+
|
|
187
|
+
rule: Rule
|
|
188
|
+
signal: str
|
|
189
|
+
threshold: float | None
|
|
190
|
+
clear: float | None
|
|
191
|
+
guard_false: bool = False
|
|
192
|
+
unavailable: str | None = None # why the thresholds can't be resolved, e.g. "no crit limit"
|
|
193
|
+
out: list[Transition] = field(default_factory=lambda: [])
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _later(last: float | None, now: float) -> float:
|
|
197
|
+
"""The clock never runs backwards for an instance: a backwards step is no time passing."""
|
|
198
|
+
return now if last is None else max(last, now)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _matches(rule: Rule, signal: str) -> bool:
|
|
202
|
+
return rule.matches(signal)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass(frozen=True, slots=True)
|
|
206
|
+
class EngineMemory:
|
|
207
|
+
rules: tuple[Rule, ...]
|
|
208
|
+
limits: dict[str, Limits]
|
|
209
|
+
known: tuple[str, ...]
|
|
210
|
+
instances: list[tuple[str, str, _Instance]]
|
|
211
|
+
suspended: bool
|
|
212
|
+
max_gap: float | None
|
|
213
|
+
version: int = 1
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class Engine:
|
|
217
|
+
def __init__(
|
|
218
|
+
self,
|
|
219
|
+
rules: Sequence[Rule],
|
|
220
|
+
*,
|
|
221
|
+
clock: Clock,
|
|
222
|
+
limits: Mapping[str, Limits] | None = None,
|
|
223
|
+
journal: Journal | None = None,
|
|
224
|
+
max_gap: float | None = None,
|
|
225
|
+
) -> None:
|
|
226
|
+
"""``max_gap``: a gap between real readings longer than this (the daemon passes 3x its
|
|
227
|
+
poll interval) counts as MISSING for that span, never as breach time."""
|
|
228
|
+
self._clock = clock
|
|
229
|
+
self._journal = journal
|
|
230
|
+
self._max_gap = max_gap
|
|
231
|
+
self._limits: dict[str, Limits] = dict(limits or {})
|
|
232
|
+
self._known: set[str] = set(self._limits)
|
|
233
|
+
self._rules: dict[str, Rule] = {}
|
|
234
|
+
self._instances: dict[tuple[str, str], _Instance] = {}
|
|
235
|
+
self._suspended = False
|
|
236
|
+
self._report(Restart(clock()))
|
|
237
|
+
self._report(EngineSettings(clock(), max_gap))
|
|
238
|
+
self._report(LimitsChanged(clock(), dict(self._limits)))
|
|
239
|
+
self.load(rules)
|
|
240
|
+
|
|
241
|
+
def checkpoint(self) -> str:
|
|
242
|
+
return msgspec.json.encode(
|
|
243
|
+
EngineMemory(
|
|
244
|
+
self.rules,
|
|
245
|
+
self._limits,
|
|
246
|
+
tuple(sorted(self._known)),
|
|
247
|
+
[(r, s, inst) for (r, s), inst in self._instances.items()],
|
|
248
|
+
self._suspended,
|
|
249
|
+
self._max_gap,
|
|
250
|
+
)
|
|
251
|
+
).decode()
|
|
252
|
+
|
|
253
|
+
def restore(self, payload: str) -> None:
|
|
254
|
+
memory = msgspec.json.decode(payload, type=EngineMemory)
|
|
255
|
+
if memory.version != 1:
|
|
256
|
+
raise ValueError("unsupported engine checkpoint")
|
|
257
|
+
self._rules = {r.id: r for r in memory.rules}
|
|
258
|
+
self._limits = memory.limits
|
|
259
|
+
self._known = set(memory.known)
|
|
260
|
+
self._instances = {(r, s): inst for r, s, inst in memory.instances}
|
|
261
|
+
self._suspended, self._max_gap = memory.suspended, memory.max_gap
|
|
262
|
+
|
|
263
|
+
def restore_alerts(self, alerts: Sequence[Transition]) -> None:
|
|
264
|
+
"""Carry same-boot open incidents into a fresh Warming session."""
|
|
265
|
+
self._report(OpenAlerts(self._clock(), msgspec.json.encode(list(alerts)).decode()))
|
|
266
|
+
for t in alerts:
|
|
267
|
+
rule = self._rules.get(t.rule_id)
|
|
268
|
+
if rule is not None and rule.matches(t.signal):
|
|
269
|
+
inst = _Instance()
|
|
270
|
+
inst.alert = _Alert(t.opened_at, t.severity, t.no_data, t.threshold)
|
|
271
|
+
self._instances[(t.rule_id, t.signal)] = inst
|
|
272
|
+
self._known.add(t.signal)
|
|
273
|
+
|
|
274
|
+
def _report(self, event: InputEvent) -> None:
|
|
275
|
+
if self._journal is not None:
|
|
276
|
+
self._journal(event)
|
|
277
|
+
|
|
278
|
+
# -- configuration --------------------------------------------------------------------------
|
|
279
|
+
|
|
280
|
+
def load(self, rules: Sequence[Rule]) -> tuple[Transition, ...]:
|
|
281
|
+
"""Replace the rule set. New and **edited** rules start in Warming; unchanged rules keep
|
|
282
|
+
their state. Alerts of removed rules close with the reason "rule removed"."""
|
|
283
|
+
self._report(RulesLoaded(self._clock(), tuple(rules)))
|
|
284
|
+
new = {rule.id: rule for rule in rules}
|
|
285
|
+
out: list[Transition] = []
|
|
286
|
+
for (rule_id, signal), inst in list(self._instances.items()):
|
|
287
|
+
old = self._rules[rule_id]
|
|
288
|
+
if rule_id not in new:
|
|
289
|
+
if inst.alert is not None:
|
|
290
|
+
out.append(self._close(old, signal, inst, RULE_REMOVED, None))
|
|
291
|
+
del self._instances[(rule_id, signal)]
|
|
292
|
+
elif new[rule_id] != old:
|
|
293
|
+
inst.restart()
|
|
294
|
+
self._rules = new
|
|
295
|
+
return tuple(out)
|
|
296
|
+
|
|
297
|
+
def set_limits(self, limits: Mapping[str, Limits]) -> None:
|
|
298
|
+
"""New chip limits after rediscovery (relative thresholds resolve against these)."""
|
|
299
|
+
self._limits = dict(limits)
|
|
300
|
+
self._known |= set(self._limits)
|
|
301
|
+
self._report(LimitsChanged(self._clock(), dict(self._limits)))
|
|
302
|
+
|
|
303
|
+
def configure(self, max_gap: float | None) -> None:
|
|
304
|
+
"""Change how time is treated (replay applies the recorded settings with this)."""
|
|
305
|
+
self._max_gap = max_gap
|
|
306
|
+
self._report(EngineSettings(self._clock(), max_gap))
|
|
307
|
+
|
|
308
|
+
@property
|
|
309
|
+
def rules(self) -> tuple[Rule, ...]:
|
|
310
|
+
return tuple(self._rules.values())
|
|
311
|
+
|
|
312
|
+
# -- sleep ----------------------------------------------------------------------------------
|
|
313
|
+
|
|
314
|
+
@property
|
|
315
|
+
def suspended(self) -> bool:
|
|
316
|
+
return self._suspended
|
|
317
|
+
|
|
318
|
+
def suspend(self) -> None:
|
|
319
|
+
"""PrepareForSleep(true) / PrepareForShutdown(true): stop evaluating entirely."""
|
|
320
|
+
self._report(Suspend(self._clock()))
|
|
321
|
+
self._suspended = True
|
|
322
|
+
|
|
323
|
+
def resume(self) -> None:
|
|
324
|
+
"""PrepareForSleep(false): every rule warms up again, missing-data timers from zero."""
|
|
325
|
+
self._report(Resume(self._clock()))
|
|
326
|
+
self._suspended = False
|
|
327
|
+
for inst in self._instances.values():
|
|
328
|
+
inst.restart()
|
|
329
|
+
|
|
330
|
+
# -- queries --------------------------------------------------------------------------------
|
|
331
|
+
|
|
332
|
+
def state(self, rule_id: str, signal: str) -> State:
|
|
333
|
+
inst = self._instances.get((rule_id, signal))
|
|
334
|
+
return inst.state if inst is not None else State.UNKNOWN
|
|
335
|
+
|
|
336
|
+
def states(self) -> dict[tuple[str, str], State]:
|
|
337
|
+
return {key: inst.state for key, inst in self._instances.items()}
|
|
338
|
+
|
|
339
|
+
def resolved_clear(self, rule_id: str, signal: str) -> float | None:
|
|
340
|
+
"""The effective clear for this rule/signal, including defaults and unavailable cases."""
|
|
341
|
+
return self._context(self._rules[rule_id], signal).clear
|
|
342
|
+
|
|
343
|
+
def unwatched(self, signals: Iterable[str]) -> list[str]:
|
|
344
|
+
"""Signals no rule watches: shown grey (Unknown), never green."""
|
|
345
|
+
return sorted(s for s in signals if not any(_matches(r, s) for r in self._rules.values()))
|
|
346
|
+
|
|
347
|
+
def problems(self) -> list[tuple[str, str, str]]:
|
|
348
|
+
"""``(rule id, signal, reason)`` for matching signals a rule can't apply to, e.g. a
|
|
349
|
+
relative threshold on a chip without that limit."""
|
|
350
|
+
out: list[tuple[str, str, str]] = []
|
|
351
|
+
for rule in self._rules.values():
|
|
352
|
+
for signal in sorted(self._known):
|
|
353
|
+
if _matches(rule, signal):
|
|
354
|
+
why = self._context(rule, signal).unavailable
|
|
355
|
+
if why is not None:
|
|
356
|
+
out.append((rule.id, signal, why))
|
|
357
|
+
return out
|
|
358
|
+
|
|
359
|
+
# -- evaluation -----------------------------------------------------------------------------
|
|
360
|
+
|
|
361
|
+
def evaluate(self, signals: Mapping[str, Value]) -> Step:
|
|
362
|
+
"""Evaluate every rule against this poll's readings. Does nothing while suspended."""
|
|
363
|
+
if self._suspended:
|
|
364
|
+
return Step()
|
|
365
|
+
now = self._clock()
|
|
366
|
+
self._report(Tick(now, dict(signals)))
|
|
367
|
+
self._known |= signals.keys()
|
|
368
|
+
evaluations: list[Evaluation] = []
|
|
369
|
+
transitions: list[Transition] = []
|
|
370
|
+
for rule in self._rules.values():
|
|
371
|
+
current = {s for s in signals if _matches(rule, s)}
|
|
372
|
+
tracked = {sig for (rid, sig) in self._instances if rid == rule.id}
|
|
373
|
+
for signal in sorted(current | tracked):
|
|
374
|
+
inst = self._instances.get((rule.id, signal))
|
|
375
|
+
if signal not in current and signal in tracked and not _matches(rule, signal):
|
|
376
|
+
# The rule was edited and no longer matches this signal.
|
|
377
|
+
if inst is not None and inst.alert is not None:
|
|
378
|
+
transitions.append(self._close(rule, signal, inst, RULE_CHANGED, None))
|
|
379
|
+
del self._instances[(rule.id, signal)]
|
|
380
|
+
continue
|
|
381
|
+
ctx = self._context(rule, signal)
|
|
382
|
+
if inst is None:
|
|
383
|
+
inst = self._instances[(rule.id, signal)] = _Instance()
|
|
384
|
+
reading = self._reading(rule, signal, signals, ctx)
|
|
385
|
+
if ctx.unavailable is not None and not ctx.guard_false:
|
|
386
|
+
# A threshold that can't be resolved is no reading, never a skipped poll:
|
|
387
|
+
# it is recorded, pauses the timers and applies on_missing (ADV-006).
|
|
388
|
+
reading = Missing(ctx.unavailable)
|
|
389
|
+
evaluations.append(self._step(ctx, inst, now, reading))
|
|
390
|
+
transitions += ctx.out
|
|
391
|
+
return Step(tuple(evaluations), tuple(transitions))
|
|
392
|
+
|
|
393
|
+
def _context(self, rule: Rule, signal: str) -> _Ctx:
|
|
394
|
+
if rule.when == "equals":
|
|
395
|
+
return _Ctx(rule, signal, None, None)
|
|
396
|
+
limits = self._limits.get(signal)
|
|
397
|
+
threshold = rule.threshold.resolve(limits)
|
|
398
|
+
clear = rule.clear_threshold.resolve(limits)
|
|
399
|
+
for spec, resolved in ((rule.threshold, threshold), (rule.clear_threshold, clear)):
|
|
400
|
+
if resolved is None:
|
|
401
|
+
return _Ctx(rule, signal, None, None, unavailable=f"no {spec.limit} limit")
|
|
402
|
+
# A fixed value with a relative clear (or the reverse) can only be checked here.
|
|
403
|
+
if (
|
|
404
|
+
threshold is not None
|
|
405
|
+
and clear is not None
|
|
406
|
+
and clear_on_breaching_side(rule.when, threshold, clear)
|
|
407
|
+
):
|
|
408
|
+
why = f"clear {clear:g} is on the breaching side of {threshold:g}"
|
|
409
|
+
return _Ctx(rule, signal, None, None, unavailable=why)
|
|
410
|
+
return _Ctx(rule, signal, threshold, clear)
|
|
411
|
+
|
|
412
|
+
def _reading(self, rule: Rule, signal: str, signals: Mapping[str, Value], ctx: _Ctx) -> Value:
|
|
413
|
+
"""This poll's reading for one signal, after the guard.
|
|
414
|
+
|
|
415
|
+
Precedence (docs/BIBLE.md, "Alerting"): a guard that is readable and false wins over a
|
|
416
|
+
MISSING primary reading (the rule counts as clear); otherwise a MISSING primary or guard
|
|
417
|
+
makes the reading MISSING. Named wildcards bind the guard to the sensor's device.
|
|
418
|
+
"""
|
|
419
|
+
reading = signals.get(signal, Missing("no reading"))
|
|
420
|
+
if rule.when != "equals" and isinstance(reading, str):
|
|
421
|
+
reading = Missing(f"not a number: {reading!r}")
|
|
422
|
+
guard = rule.only_if
|
|
423
|
+
if guard is None:
|
|
424
|
+
return reading
|
|
425
|
+
pattern = globs.fill(guard.signal, rule.bindings(signal) or {})
|
|
426
|
+
values = {s: v for s, v in signals.items() if globs.match(pattern, s) is not None}
|
|
427
|
+
if any(not isinstance(v, Missing) and v == guard.equals for v in values.values()):
|
|
428
|
+
return reading
|
|
429
|
+
if not values:
|
|
430
|
+
return Missing(f"guard {pattern}: no reading")
|
|
431
|
+
missing = [(s, v) for s, v in values.items() if isinstance(v, Missing)]
|
|
432
|
+
if missing:
|
|
433
|
+
s, v = missing[0]
|
|
434
|
+
return Missing(f"guard {s}: {v.reason}")
|
|
435
|
+
ctx.guard_false = True
|
|
436
|
+
return reading
|
|
437
|
+
|
|
438
|
+
def _open(
|
|
439
|
+
self,
|
|
440
|
+
ctx: _Ctx,
|
|
441
|
+
inst: _Instance,
|
|
442
|
+
now: float,
|
|
443
|
+
reason: str,
|
|
444
|
+
value: float | str | None,
|
|
445
|
+
*,
|
|
446
|
+
no_data: bool = False,
|
|
447
|
+
) -> None:
|
|
448
|
+
rule = ctx.rule
|
|
449
|
+
severity = rule.missing_severity if no_data else rule.severity
|
|
450
|
+
inst.alert = _Alert(now, severity, no_data, ctx.threshold)
|
|
451
|
+
ctx.out.append(
|
|
452
|
+
Transition(
|
|
453
|
+
at=now,
|
|
454
|
+
rule_id=rule.id,
|
|
455
|
+
signal=ctx.signal,
|
|
456
|
+
kind=TransitionKind.FIRING,
|
|
457
|
+
severity=severity,
|
|
458
|
+
reason=reason,
|
|
459
|
+
value=value,
|
|
460
|
+
threshold=ctx.threshold,
|
|
461
|
+
opened_at=now,
|
|
462
|
+
no_data=no_data,
|
|
463
|
+
)
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
def _close(
|
|
467
|
+
self, rule: Rule, signal: str, inst: _Instance, reason: str, value: float | str | None
|
|
468
|
+
) -> Transition:
|
|
469
|
+
alert = inst.alert
|
|
470
|
+
if alert is None:
|
|
471
|
+
raise RuntimeError(f"{rule.id} {signal}: no open alert to close")
|
|
472
|
+
inst.alert = None
|
|
473
|
+
return Transition(
|
|
474
|
+
at=self._clock(),
|
|
475
|
+
rule_id=rule.id,
|
|
476
|
+
signal=signal,
|
|
477
|
+
kind=TransitionKind.RESOLVED,
|
|
478
|
+
severity=alert.severity,
|
|
479
|
+
reason=reason,
|
|
480
|
+
value=value,
|
|
481
|
+
threshold=alert.threshold,
|
|
482
|
+
opened_at=alert.opened_at,
|
|
483
|
+
no_data=alert.no_data,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
def _record(
|
|
487
|
+
self, ctx: _Ctx, inst: _Instance, now: float, reading: Value, state: State, reason: str
|
|
488
|
+
) -> Evaluation:
|
|
489
|
+
inst.state = state
|
|
490
|
+
missing = reading.reason if isinstance(reading, Missing) else None
|
|
491
|
+
value = None if isinstance(reading, Missing) else reading
|
|
492
|
+
return Evaluation(
|
|
493
|
+
now,
|
|
494
|
+
ctx.rule.id,
|
|
495
|
+
ctx.signal,
|
|
496
|
+
value,
|
|
497
|
+
missing,
|
|
498
|
+
state,
|
|
499
|
+
reason,
|
|
500
|
+
ctx.threshold,
|
|
501
|
+
inst.alert.severity if inst.alert is not None else None,
|
|
502
|
+
inst.alert.no_data if inst.alert is not None else False,
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
def _step(self, ctx: _Ctx, inst: _Instance, now: float, reading: Value) -> Evaluation:
|
|
506
|
+
rule = ctx.rule
|
|
507
|
+
# A step backwards (or a zero step) is no time passing.
|
|
508
|
+
dt = max(now - inst.last_t, 0.0) if inst.last_real and inst.last_t is not None else 0.0
|
|
509
|
+
|
|
510
|
+
if isinstance(reading, Missing) and not ctx.guard_false:
|
|
511
|
+
return self._step_missing(ctx, inst, now, reading)
|
|
512
|
+
# A real reading, or a false guard (which counts as clear even if the reading is missing).
|
|
513
|
+
value = None if isinstance(reading, Missing) else reading
|
|
514
|
+
if self._max_gap is not None and dt > self._max_gap and inst.last_t is not None:
|
|
515
|
+
# Nobody looked during that gap: it is MISSING for that span, never breach time.
|
|
516
|
+
if inst.missing_since is None:
|
|
517
|
+
inst.missing_since = inst.last_t
|
|
518
|
+
dt = 0.0
|
|
519
|
+
|
|
520
|
+
# Real data again after a long gap: start over, like a resume.
|
|
521
|
+
if inst.missing_since is not None:
|
|
522
|
+
gap = now - inst.missing_since
|
|
523
|
+
inst.missing_since = None
|
|
524
|
+
if gap >= rule.missing_for_s:
|
|
525
|
+
inst.restart()
|
|
526
|
+
dt = 0.0
|
|
527
|
+
# Data is back: a "no data" alert closes now, before any warm-up, which only concerns
|
|
528
|
+
# alerts on values (ADV-015: after resume it stayed open for a whole warm-up window).
|
|
529
|
+
if inst.alert is not None and inst.alert.no_data:
|
|
530
|
+
ctx.out.append(self._close(rule, ctx.signal, inst, DATA_BACK, value))
|
|
531
|
+
inst.last_t, inst.last_real = _later(inst.last_t, now), True
|
|
532
|
+
|
|
533
|
+
breaching, past_clear = self._compare(ctx, reading)
|
|
534
|
+
inst.breach_elapsed = inst.breach_elapsed + dt if breaching and inst.in_breach else 0.0
|
|
535
|
+
inst.in_breach = breaching
|
|
536
|
+
inst.clear_elapsed = inst.clear_elapsed + dt if past_clear and inst.in_clear else 0.0
|
|
537
|
+
inst.in_clear = past_clear
|
|
538
|
+
|
|
539
|
+
if inst.warming:
|
|
540
|
+
window = 0.0 if rule.mode == "any" else rule.for_s
|
|
541
|
+
inst.warm_elapsed += dt
|
|
542
|
+
inst.warm_count += 1
|
|
543
|
+
if inst.warm_elapsed < window:
|
|
544
|
+
done = format_duration(inst.warm_elapsed)
|
|
545
|
+
why = f"warming: {done} of {format_duration(window)}"
|
|
546
|
+
return self._record(ctx, inst, now, reading, State.WARMING, why)
|
|
547
|
+
inst.warming = False
|
|
548
|
+
|
|
549
|
+
if inst.alert is not None:
|
|
550
|
+
if past_clear and inst.clear_elapsed >= rule.clear_for_s:
|
|
551
|
+
ctx.out.append(self._close(rule, ctx.signal, inst, RECOVERED, value))
|
|
552
|
+
return self._record(ctx, inst, now, reading, State.RESOLVED, RECOVERED)
|
|
553
|
+
else:
|
|
554
|
+
why = f"firing; clear for {format_duration(inst.clear_elapsed)} of " + (
|
|
555
|
+
format_duration(rule.clear_for_s)
|
|
556
|
+
)
|
|
557
|
+
return self._record(ctx, inst, now, reading, State.FIRING, why)
|
|
558
|
+
|
|
559
|
+
if (
|
|
560
|
+
breaching
|
|
561
|
+
and value is not None
|
|
562
|
+
and (rule.mode == "any" or inst.breach_elapsed >= rule.for_s)
|
|
563
|
+
):
|
|
564
|
+
reason = self._breach_text(ctx, value)
|
|
565
|
+
self._open(ctx, inst, now, reason, value)
|
|
566
|
+
return self._record(ctx, inst, now, reading, State.FIRING, reason)
|
|
567
|
+
if breaching:
|
|
568
|
+
why = f"breaching for {format_duration(inst.breach_elapsed)} of " + format_duration(
|
|
569
|
+
rule.for_s
|
|
570
|
+
)
|
|
571
|
+
return self._record(ctx, inst, now, reading, State.PENDING, why)
|
|
572
|
+
why = GUARD_FALSE if ctx.guard_false else "not breaching"
|
|
573
|
+
return self._record(ctx, inst, now, reading, State.OK, why)
|
|
574
|
+
|
|
575
|
+
def _step_missing(self, ctx: _Ctx, inst: _Instance, now: float, reading: Missing) -> Evaluation:
|
|
576
|
+
rule = ctx.rule
|
|
577
|
+
if inst.missing_since is None:
|
|
578
|
+
inst.missing_since = now
|
|
579
|
+
inst.last_t, inst.last_real = _later(inst.last_t, now), False
|
|
580
|
+
gap = now - inst.missing_since
|
|
581
|
+
if gap >= rule.missing_for_s:
|
|
582
|
+
if rule.on_missing == "close" and inst.alert is not None:
|
|
583
|
+
ctx.out.append(self._close(rule, ctx.signal, inst, CLOSED_NO_DATA, None))
|
|
584
|
+
return self._record(ctx, inst, now, reading, State.RESOLVED, CLOSED_NO_DATA)
|
|
585
|
+
if rule.on_missing == "alert" and inst.alert is None:
|
|
586
|
+
reason = f"no data for {format_duration(rule.missing_for_s)}: {reading.reason}"
|
|
587
|
+
self._open(ctx, inst, now, reason, None, no_data=True)
|
|
588
|
+
return self._record(ctx, inst, now, reading, State.FIRING, reason)
|
|
589
|
+
why = f"missing for {format_duration(gap)}: {reading.reason}"
|
|
590
|
+
if inst.alert is not None:
|
|
591
|
+
return self._record(ctx, inst, now, reading, State.FIRING, why)
|
|
592
|
+
# Missing for longer than missing_for is Unknown, whatever on_missing says and even if
|
|
593
|
+
# the rule never finished warming up: it is not still starting (bible "Unknown"; ADV-014).
|
|
594
|
+
warming = inst.warming and gap < rule.missing_for_s
|
|
595
|
+
state = State.WARMING if warming else State.UNKNOWN
|
|
596
|
+
return self._record(ctx, inst, now, reading, state, why)
|
|
597
|
+
|
|
598
|
+
@staticmethod
|
|
599
|
+
def _compare(ctx: _Ctx, reading: Value) -> tuple[bool, bool]:
|
|
600
|
+
"""``(breaching, past clear)``. A false guard counts as fully clear."""
|
|
601
|
+
if ctx.guard_false:
|
|
602
|
+
return False, True
|
|
603
|
+
rule = ctx.rule
|
|
604
|
+
if rule.when == "equals":
|
|
605
|
+
breaching = reading == rule.value
|
|
606
|
+
return breaching, not breaching
|
|
607
|
+
if isinstance(reading, Missing): # only reachable with a false guard, handled above
|
|
608
|
+
raise RuntimeError(f"{rule.id} {ctx.signal}: compared a missing reading")
|
|
609
|
+
value = float(reading)
|
|
610
|
+
if ctx.threshold is None or ctx.clear is None: # _context() never allows this
|
|
611
|
+
raise RuntimeError(f"{rule.id} {ctx.signal}: threshold unresolved")
|
|
612
|
+
if rule.when == "above":
|
|
613
|
+
return value > ctx.threshold, value <= ctx.clear
|
|
614
|
+
return value < ctx.threshold, value >= ctx.clear
|
|
615
|
+
|
|
616
|
+
@staticmethod
|
|
617
|
+
def _breach_text(ctx: _Ctx, reading: float | str) -> str:
|
|
618
|
+
rule = ctx.rule
|
|
619
|
+
if rule.when == "equals":
|
|
620
|
+
return f"equals {rule.value!r}"
|
|
621
|
+
text = f"{float(reading):g} {rule.when} {ctx.threshold:g}"
|
|
622
|
+
if rule.mode == "sustained" and rule.for_s > 0:
|
|
623
|
+
text += f" for {format_duration(rule.for_s)}"
|
|
624
|
+
return text
|