detectkit 0.8.1__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {detectkit-0.8.1/detectkit.egg-info → detectkit-0.9.0}/PKG-INFO +1 -1
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/__init__.py +1 -1
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/base.py +86 -14
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/email.py +1 -1
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/webhook.py +20 -8
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/_decision.py +24 -1
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/_recovery.py +5 -0
- detectkit-0.9.0/detectkit/cli/_output.py +50 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/commands/clean.py +32 -53
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/commands/test_alert.py +23 -2
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/commands/unlock.py +5 -14
- {detectkit-0.8.1 → detectkit-0.9.0/detectkit.egg-info}/PKG-INFO +1 -1
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit.egg-info/SOURCES.txt +1 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/LICENSE +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/MANIFEST.in +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/README.md +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/factory.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/mattermost.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/slack.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/channels/telegram.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/_base.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/_types.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/commands/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/commands/init.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/commands/run.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/cli/main.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/config/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/config/metric_config.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/config/profile.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/config/project_config.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/config/validator.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/core/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/core/interval.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/core/models.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/clickhouse_manager.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_alert_states.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_base.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_datapoints.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_detections.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_maintenance.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_metrics.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_schema.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/_tasks.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/internal_tables/manager.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/manager.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/database/tables.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/base.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/factory.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/seasonality.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/statistical/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/statistical/_windowed.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/statistical/iqr.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/statistical/mad.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/statistical/manual_bounds.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/detectors/statistical/zscore.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/loaders/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/loaders/metric_loader.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/loaders/query_template.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/error_dispatch.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/_alert_step.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/_base.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/_load_step.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/_types.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/orchestration/task_manager/manager.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/utils/__init__.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/utils/datetime_utils.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/utils/env_interpolation.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/utils/json_utils.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit/utils/stats.py +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit.egg-info/dependency_links.txt +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit.egg-info/entry_points.txt +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit.egg-info/requires.txt +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/detectkit.egg-info/top_level.txt +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/pyproject.toml +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/requirements.txt +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/setup.cfg +0 -0
- {detectkit-0.8.1 → detectkit-0.9.0}/setup.py +0 -0
|
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
|
|
|
4
4
|
A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "0.
|
|
7
|
+
__version__ = "0.9.0"
|
|
8
8
|
|
|
9
9
|
from detectkit.core.interval import Interval
|
|
10
10
|
from detectkit.core.models import ColumnDefinition, TableModel
|
|
@@ -36,6 +36,15 @@ class AlertData:
|
|
|
36
36
|
as ``{project_name}`` in templates and as a ``[name] `` prefix
|
|
37
37
|
in the default error title. Lets multiple projects share the
|
|
38
38
|
same alert channel without ambiguity.
|
|
39
|
+
|
|
40
|
+
Alert-rule fields (``min_detectors``, ``direction_policy``,
|
|
41
|
+
``consecutive_required``, ``detector_count``) describe *why the alert
|
|
42
|
+
fired* — the configured quorum/direction/consecutive thresholds plus
|
|
43
|
+
the observed number of agreeing detectors. They are filled by the
|
|
44
|
+
orchestrator from :class:`AlertConditions` and are deliberately kept
|
|
45
|
+
distinct from the observed ``direction``/``consecutive_count`` above so
|
|
46
|
+
templates can contrast "required vs actual". They default to ``None``
|
|
47
|
+
so direct-API callers (and non-anomaly alerts) still render cleanly.
|
|
39
48
|
"""
|
|
40
49
|
|
|
41
50
|
metric_name: str
|
|
@@ -58,6 +67,11 @@ class AlertData:
|
|
|
58
67
|
description: str | None = None
|
|
59
68
|
mentions: list[str] = field(default_factory=list)
|
|
60
69
|
project_name: str | None = None
|
|
70
|
+
# Alert rule (the parameters the alert fired with) — see class docstring.
|
|
71
|
+
min_detectors: int | None = None
|
|
72
|
+
direction_policy: str | None = None
|
|
73
|
+
consecutive_required: int | None = None
|
|
74
|
+
detector_count: int = 1
|
|
61
75
|
|
|
62
76
|
|
|
63
77
|
class BaseAlertChannel(ABC):
|
|
@@ -123,10 +137,17 @@ class BaseAlertChannel(ABC):
|
|
|
123
137
|
- {value} / {value_display}
|
|
124
138
|
- {confidence_lower}
|
|
125
139
|
- {confidence_upper}
|
|
140
|
+
- {confidence_interval} — "[lower, upper]" or "N/A"
|
|
141
|
+
- {expected_range} — one-sided aware: ">= lo", "<= hi",
|
|
142
|
+
"[lo, hi]" or "N/A" (renders one-sided detector bounds cleanly)
|
|
126
143
|
- {detector_name}
|
|
127
|
-
- {
|
|
144
|
+
- {detector_count} — observed detectors that agreed (the quorum)
|
|
145
|
+
- {direction} — observed/locked direction of the anomaly
|
|
146
|
+
- {direction_policy} — configured direction rule ("same"/"any"/...)
|
|
147
|
+
- {min_detectors} — configured quorum threshold (the rule)
|
|
148
|
+
- {consecutive_count} — observed consecutive points
|
|
149
|
+
- {consecutive_required} — configured consecutive threshold (rule)
|
|
128
150
|
- {severity}
|
|
129
|
-
- {consecutive_count}
|
|
130
151
|
- {status}
|
|
131
152
|
|
|
132
153
|
Args:
|
|
@@ -177,6 +198,40 @@ class BaseAlertChannel(ABC):
|
|
|
177
198
|
else:
|
|
178
199
|
confidence_str = "N/A"
|
|
179
200
|
|
|
201
|
+
# One-sided-aware expected range. A NaN/inf bound means "no bound on
|
|
202
|
+
# that side" (e.g. ManualBounds with only ``lower_bound`` set), so we
|
|
203
|
+
# render ">= lo" / "<= hi" instead of the confusing "[7.00, nan]".
|
|
204
|
+
def _bounded(b: Any) -> bool:
|
|
205
|
+
return b is not None and not (isinstance(b, float) and (math.isnan(b) or math.isinf(b)))
|
|
206
|
+
|
|
207
|
+
lo_ok = _bounded(alert_data.confidence_lower)
|
|
208
|
+
hi_ok = _bounded(alert_data.confidence_upper)
|
|
209
|
+
if lo_ok and hi_ok:
|
|
210
|
+
expected_range = (
|
|
211
|
+
f"[{alert_data.confidence_lower:.2f}, {alert_data.confidence_upper:.2f}]"
|
|
212
|
+
)
|
|
213
|
+
elif lo_ok:
|
|
214
|
+
expected_range = f">= {alert_data.confidence_lower:.2f}"
|
|
215
|
+
elif hi_ok:
|
|
216
|
+
expected_range = f"<= {alert_data.confidence_upper:.2f}"
|
|
217
|
+
else:
|
|
218
|
+
expected_range = "N/A"
|
|
219
|
+
|
|
220
|
+
# Alert-rule display values. The orchestrator fills these from the
|
|
221
|
+
# configured AlertConditions; for direct-API/non-anomaly callers that
|
|
222
|
+
# leave them unset we fall back to the observed counts so the default
|
|
223
|
+
# templates never render a bare "None".
|
|
224
|
+
detector_count = alert_data.detector_count
|
|
225
|
+
min_detectors = (
|
|
226
|
+
alert_data.min_detectors if alert_data.min_detectors is not None else detector_count
|
|
227
|
+
)
|
|
228
|
+
consecutive_required = (
|
|
229
|
+
alert_data.consecutive_required
|
|
230
|
+
if alert_data.consecutive_required is not None
|
|
231
|
+
else alert_data.consecutive_count
|
|
232
|
+
)
|
|
233
|
+
direction_policy = alert_data.direction_policy or alert_data.direction
|
|
234
|
+
|
|
180
235
|
# Display-safe value: stays usable even when value is None/NaN (no-data).
|
|
181
236
|
raw_value = alert_data.value
|
|
182
237
|
if raw_value is None or (isinstance(raw_value, float) and math.isnan(raw_value)):
|
|
@@ -221,11 +276,16 @@ class BaseAlertChannel(ABC):
|
|
|
221
276
|
confidence_lower=alert_data.confidence_lower,
|
|
222
277
|
confidence_upper=alert_data.confidence_upper,
|
|
223
278
|
confidence_interval=confidence_str,
|
|
279
|
+
expected_range=expected_range,
|
|
224
280
|
detector_name=alert_data.detector_name,
|
|
281
|
+
detector_count=detector_count,
|
|
225
282
|
detector_params=alert_data.detector_params,
|
|
226
283
|
direction=alert_data.direction,
|
|
284
|
+
direction_policy=direction_policy,
|
|
285
|
+
min_detectors=min_detectors,
|
|
227
286
|
severity=alert_data.severity,
|
|
228
287
|
consecutive_count=alert_data.consecutive_count,
|
|
288
|
+
consecutive_required=consecutive_required,
|
|
229
289
|
status=status,
|
|
230
290
|
error_type=alert_data.error_type or "",
|
|
231
291
|
error_message=alert_data.error_message or "",
|
|
@@ -312,12 +372,19 @@ class BaseAlertChannel(ABC):
|
|
|
312
372
|
Default template string
|
|
313
373
|
"""
|
|
314
374
|
return (
|
|
315
|
-
"
|
|
375
|
+
"⚠ Alert: {metric_name}\n"
|
|
316
376
|
"{description_line}"
|
|
317
|
-
"
|
|
318
|
-
"
|
|
319
|
-
"
|
|
320
|
-
"
|
|
377
|
+
"Quorum {detector_count}/{min_detectors} · "
|
|
378
|
+
"direction {direction} (policy {direction_policy}) · "
|
|
379
|
+
"consecutive {consecutive_count}/{consecutive_required}\n"
|
|
380
|
+
"Rule: min_detectors={min_detectors} · "
|
|
381
|
+
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
382
|
+
"\n"
|
|
383
|
+
"Latest point (evidence):\n"
|
|
384
|
+
"· Time: {timestamp}\n"
|
|
385
|
+
"· Value: {value_display} | Expected: {expected_range}\n"
|
|
386
|
+
"· Severity: {severity:.2f}\n"
|
|
387
|
+
"Detectors: {detector_name}\n"
|
|
321
388
|
"Parameters: {detector_params}"
|
|
322
389
|
"{mentions_line}"
|
|
323
390
|
)
|
|
@@ -330,12 +397,17 @@ class BaseAlertChannel(ABC):
|
|
|
330
397
|
Default recovery template string
|
|
331
398
|
"""
|
|
332
399
|
return (
|
|
333
|
-
"
|
|
400
|
+
"✅ Alert cleared: {metric_name}\n"
|
|
334
401
|
"{description_line}"
|
|
335
|
-
"
|
|
336
|
-
"
|
|
337
|
-
"
|
|
338
|
-
"
|
|
402
|
+
"The alert condition no longer holds — "
|
|
403
|
+
"the metric is back within expected bounds.\n"
|
|
404
|
+
"Rule: min_detectors={min_detectors} · "
|
|
405
|
+
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
406
|
+
"\n"
|
|
407
|
+
"Latest point:\n"
|
|
408
|
+
"· Time: {timestamp}\n"
|
|
409
|
+
"· Value: {value_display} | Expected: {expected_range}\n"
|
|
410
|
+
"Detectors: {detector_name}"
|
|
339
411
|
"{mentions_line}"
|
|
340
412
|
)
|
|
341
413
|
|
|
@@ -348,7 +420,7 @@ class BaseAlertChannel(ABC):
|
|
|
348
420
|
Returns:
|
|
349
421
|
Default title template string
|
|
350
422
|
"""
|
|
351
|
-
return "
|
|
423
|
+
return "⚠ Alert: {metric_name}"
|
|
352
424
|
|
|
353
425
|
def get_default_recovery_title_template(self) -> str:
|
|
354
426
|
"""
|
|
@@ -357,7 +429,7 @@ class BaseAlertChannel(ABC):
|
|
|
357
429
|
Returns:
|
|
358
430
|
Default recovery title template string
|
|
359
431
|
"""
|
|
360
|
-
return "
|
|
432
|
+
return "✅ Alert cleared: {metric_name}"
|
|
361
433
|
|
|
362
434
|
def get_default_no_data_template(self) -> str:
|
|
363
435
|
"""
|
|
@@ -54,7 +54,7 @@ class EmailChannel(BaseAlertChannel):
|
|
|
54
54
|
smtp_username: str | None = None,
|
|
55
55
|
smtp_password: str | None = None,
|
|
56
56
|
use_tls: bool = True,
|
|
57
|
-
subject_template: str = "
|
|
57
|
+
subject_template: str = "⚠ Alert: {metric_name}",
|
|
58
58
|
template: str | None = None,
|
|
59
59
|
**kwargs,
|
|
60
60
|
):
|
|
@@ -155,10 +155,17 @@ class WebhookChannel(BaseAlertChannel):
|
|
|
155
155
|
"""
|
|
156
156
|
return (
|
|
157
157
|
"{description_line}"
|
|
158
|
-
"
|
|
159
|
-
"
|
|
160
|
-
"
|
|
161
|
-
"
|
|
158
|
+
"Quorum {detector_count}/{min_detectors} · "
|
|
159
|
+
"direction {direction} (policy {direction_policy}) · "
|
|
160
|
+
"consecutive {consecutive_count}/{consecutive_required}\n"
|
|
161
|
+
"Rule: min_detectors={min_detectors} · "
|
|
162
|
+
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
163
|
+
"\n"
|
|
164
|
+
"Latest point (evidence):\n"
|
|
165
|
+
"· Time: {timestamp}\n"
|
|
166
|
+
"· Value: {value_display} | Expected: {expected_range}\n"
|
|
167
|
+
"· Severity: {severity:.2f}\n"
|
|
168
|
+
"Detectors: {detector_name}\n"
|
|
162
169
|
"Parameters: {detector_params}"
|
|
163
170
|
"{mentions_line}"
|
|
164
171
|
)
|
|
@@ -171,10 +178,15 @@ class WebhookChannel(BaseAlertChannel):
|
|
|
171
178
|
"""
|
|
172
179
|
return (
|
|
173
180
|
"{description_line}"
|
|
174
|
-
"
|
|
175
|
-
"
|
|
176
|
-
"
|
|
177
|
-
"
|
|
181
|
+
"The alert condition no longer holds — "
|
|
182
|
+
"the metric is back within expected bounds.\n"
|
|
183
|
+
"Rule: min_detectors={min_detectors} · "
|
|
184
|
+
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
185
|
+
"\n"
|
|
186
|
+
"Latest point:\n"
|
|
187
|
+
"· Time: {timestamp}\n"
|
|
188
|
+
"· Value: {value_display} | Expected: {expected_range}\n"
|
|
189
|
+
"Detectors: {detector_name}"
|
|
178
190
|
"{mentions_line}"
|
|
179
191
|
)
|
|
180
192
|
|
|
@@ -207,6 +207,23 @@ class _DecisionMixin(_OrchestratorBase):
|
|
|
207
207
|
detector_params = primary.detector_params
|
|
208
208
|
combined_metadata = primary.detection_metadata
|
|
209
209
|
|
|
210
|
+
# Observed direction shown in the message. For "same"/"up"/"down" the
|
|
211
|
+
# caller passes the locked/policy direction. For "any" it passes None
|
|
212
|
+
# because the quorum may combine directions — collapse to the shared
|
|
213
|
+
# side only when every quorum member agrees, otherwise label it
|
|
214
|
+
# "mixed" so the message never claims an agreement that did not happen
|
|
215
|
+
# (e.g. one up + one down satisfying min_detectors=2).
|
|
216
|
+
if direction:
|
|
217
|
+
observed_direction = direction
|
|
218
|
+
else:
|
|
219
|
+
quorum_dirs = {d.direction for d in anomalies if d.direction in ("up", "down")}
|
|
220
|
+
if len(quorum_dirs) == 1:
|
|
221
|
+
observed_direction = next(iter(quorum_dirs))
|
|
222
|
+
elif len(quorum_dirs) >= 2:
|
|
223
|
+
observed_direction = "mixed"
|
|
224
|
+
else:
|
|
225
|
+
observed_direction = primary.direction
|
|
226
|
+
|
|
210
227
|
return AlertData(
|
|
211
228
|
metric_name=self.metric_name,
|
|
212
229
|
timestamp=primary.timestamp,
|
|
@@ -216,12 +233,18 @@ class _DecisionMixin(_OrchestratorBase):
|
|
|
216
233
|
confidence_upper=primary.confidence_upper,
|
|
217
234
|
detector_name=detector_name,
|
|
218
235
|
detector_params=detector_params,
|
|
219
|
-
direction=
|
|
236
|
+
direction=observed_direction,
|
|
220
237
|
severity=max_severity,
|
|
221
238
|
detection_metadata=combined_metadata,
|
|
222
239
|
consecutive_count=consecutive_count,
|
|
223
240
|
description=self.description,
|
|
224
241
|
mentions=self.mentions,
|
|
242
|
+
# Alert rule the message foregrounds: configured thresholds plus
|
|
243
|
+
# the observed quorum size that satisfied them.
|
|
244
|
+
min_detectors=self.conditions.min_detectors,
|
|
245
|
+
direction_policy=self.conditions.direction,
|
|
246
|
+
consecutive_required=self.conditions.consecutive_anomalies,
|
|
247
|
+
detector_count=len(anomalies),
|
|
225
248
|
)
|
|
226
249
|
|
|
227
250
|
def should_alert_no_data(
|
|
@@ -176,4 +176,9 @@ class _RecoveryMixin(_OrchestratorBase):
|
|
|
176
176
|
is_recovery=True,
|
|
177
177
|
description=self.description,
|
|
178
178
|
mentions=self.mentions,
|
|
179
|
+
# Echo the rule that had fired so the recovery message names the
|
|
180
|
+
# same alert condition that just cleared.
|
|
181
|
+
min_detectors=self.conditions.min_detectors,
|
|
182
|
+
direction_policy=self.conditions.direction,
|
|
183
|
+
consecutive_required=self.conditions.consecutive_anomalies,
|
|
179
184
|
)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Shared CLI output helpers so every command renders in one house style.
|
|
2
|
+
|
|
3
|
+
Mirrors the load → detect → alert pipeline's tree look (``┌─ / │ / └─``) used by
|
|
4
|
+
``dtk run`` so the maintenance commands (``dtk clean``, ``dtk unlock``) match it
|
|
5
|
+
instead of each inventing its own formatting.
|
|
6
|
+
|
|
7
|
+
House conventions:
|
|
8
|
+
- A metric *with* something to report is a tree: a cyan-bold ``┌─ <name>``
|
|
9
|
+
header followed by one child line per item (``│ `` for all but the last,
|
|
10
|
+
``└─ `` for the last).
|
|
11
|
+
- A metric with *nothing* to do is a single ``•`` line.
|
|
12
|
+
- A per-metric error is a red ``✗`` line (to stderr).
|
|
13
|
+
- The final summary is a cyan-bold ``Done. …`` line.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import click
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def echo_tree(name: str, children: list[str], *, warnings: list[str] | None = None) -> None:
|
|
22
|
+
"""Print a ``┌─ name`` header with ``│``/``└─`` child lines.
|
|
23
|
+
|
|
24
|
+
``warnings`` (if any) are rendered as yellow ``│`` continuation lines above
|
|
25
|
+
the children. ``children`` must be non-empty (a metric with no items should
|
|
26
|
+
use :func:`echo_noop` instead).
|
|
27
|
+
"""
|
|
28
|
+
click.echo(click.style(f" ┌─ {name}", fg="cyan", bold=True))
|
|
29
|
+
for warning in warnings or []:
|
|
30
|
+
click.echo(click.style(f" │ ⚠ {warning}", fg="yellow", bold=True))
|
|
31
|
+
last = len(children) - 1
|
|
32
|
+
for i, child in enumerate(children):
|
|
33
|
+
prefix = " └─ " if i == last else " │ "
|
|
34
|
+
click.echo(f"{prefix}{child}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def echo_noop(name: str, reason: str) -> None:
|
|
38
|
+
"""A metric with nothing to do — a single ``•`` line."""
|
|
39
|
+
click.echo(f" • {name}: {reason}")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def echo_error(name: str, message: str) -> None:
|
|
43
|
+
"""A per-metric failure — a red ``✗`` line on stderr."""
|
|
44
|
+
click.echo(click.style(f" ✗ {name}: {message}", fg="red"), err=True)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def echo_done(summary: str) -> None:
|
|
48
|
+
"""The closing ``Done. …`` summary (cyan, bold), preceded by a blank line."""
|
|
49
|
+
click.echo()
|
|
50
|
+
click.echo(click.style(f"Done. {summary}", fg="cyan", bold=True))
|
|
@@ -27,6 +27,7 @@ from pathlib import Path
|
|
|
27
27
|
|
|
28
28
|
import click
|
|
29
29
|
|
|
30
|
+
from detectkit.cli._output import echo_done, echo_error, echo_noop, echo_tree
|
|
30
31
|
from detectkit.cli.commands.run import find_project_root, select_metrics
|
|
31
32
|
from detectkit.config.metric_config import MetricConfig
|
|
32
33
|
from detectkit.config.profile import ProfilesConfig
|
|
@@ -112,6 +113,8 @@ def _clean_drift(
|
|
|
112
113
|
total_det_groups = 0
|
|
113
114
|
total_alert_rows = 0
|
|
114
115
|
|
|
116
|
+
verb = "deleting" if execute else "would delete"
|
|
117
|
+
|
|
115
118
|
for _, config in metrics:
|
|
116
119
|
metric_name = config.name
|
|
117
120
|
try:
|
|
@@ -120,7 +123,7 @@ def _clean_drift(
|
|
|
120
123
|
db_detectors = internal_manager.list_detector_ids(metric_name)
|
|
121
124
|
db_alerts = internal_manager.list_alert_config_ids(metric_name)
|
|
122
125
|
except Exception as e:
|
|
123
|
-
|
|
126
|
+
echo_error(metric_name, f"error inspecting: {e}")
|
|
124
127
|
continue
|
|
125
128
|
|
|
126
129
|
orphan_detectors = {
|
|
@@ -129,55 +132,42 @@ def _clean_drift(
|
|
|
129
132
|
orphan_alerts = [a for a in db_alerts if a not in valid_alerts]
|
|
130
133
|
|
|
131
134
|
if not orphan_detectors and not orphan_alerts:
|
|
132
|
-
|
|
135
|
+
echo_noop(metric_name, "nothing stale")
|
|
133
136
|
continue
|
|
134
137
|
|
|
135
|
-
|
|
138
|
+
children = [
|
|
139
|
+
f"detector {det_id}: {verb} {count:,} detection row(s)"
|
|
140
|
+
for det_id, count in sorted(orphan_detectors.items())
|
|
141
|
+
] + [
|
|
142
|
+
f"alert_config {alert_id}: {verb} stale alert state"
|
|
143
|
+
for alert_id in sorted(orphan_alerts)
|
|
144
|
+
]
|
|
136
145
|
|
|
137
146
|
# An empty valid set means EVERY stored row is "orphaned" — usually a
|
|
138
147
|
# config mid-edit, not an intent to wipe the metric. Flag it loudly.
|
|
148
|
+
warnings = []
|
|
139
149
|
if orphan_detectors and not valid_detectors:
|
|
140
|
-
|
|
141
|
-
click.style(
|
|
142
|
-
" ⚠ config defines no detectors — ALL detections below would be removed",
|
|
143
|
-
fg="yellow",
|
|
144
|
-
bold=True,
|
|
145
|
-
)
|
|
146
|
-
)
|
|
150
|
+
warnings.append("config defines no detectors — ALL detections below would be removed")
|
|
147
151
|
if orphan_alerts and not valid_alerts:
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
fg="yellow",
|
|
152
|
-
bold=True,
|
|
153
|
-
)
|
|
154
|
-
)
|
|
152
|
+
warnings.append("config defines no alerting — ALL alert states below would be removed")
|
|
153
|
+
|
|
154
|
+
echo_tree(metric_name, children, warnings=warnings)
|
|
155
155
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
verb = "deleting" if execute else "would delete"
|
|
159
|
-
click.echo(f" detector {det_id}: {verb} {count:,} detection row(s)")
|
|
160
|
-
if execute:
|
|
156
|
+
if execute:
|
|
157
|
+
for det_id in orphan_detectors:
|
|
161
158
|
internal_manager.delete_detections(
|
|
162
159
|
metric_name=metric_name, detector_id=det_id, mutations_sync=True
|
|
163
160
|
)
|
|
164
|
-
|
|
165
|
-
for alert_id in sorted(orphan_alerts):
|
|
166
|
-
total_alert_rows += 1
|
|
167
|
-
verb = "deleting" if execute else "would delete"
|
|
168
|
-
click.echo(f" alert_config {alert_id}: {verb} stale alert state")
|
|
169
|
-
if execute:
|
|
161
|
+
for alert_id in orphan_alerts:
|
|
170
162
|
internal_manager.delete_alert_state(metric_name, alert_id)
|
|
171
163
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
bold=True,
|
|
180
|
-
)
|
|
164
|
+
total_det_groups += len(orphan_detectors)
|
|
165
|
+
total_alert_rows += len(orphan_alerts)
|
|
166
|
+
|
|
167
|
+
verb_done = "Removed" if execute else "Would remove"
|
|
168
|
+
echo_done(
|
|
169
|
+
f"{verb_done} {total_det_groups} detector group(s) "
|
|
170
|
+
f"and {total_alert_rows} alert-state row(s)."
|
|
181
171
|
)
|
|
182
172
|
if not execute and (total_det_groups or total_alert_rows):
|
|
183
173
|
click.echo("Re-run with --execute to apply.")
|
|
@@ -222,15 +212,10 @@ def _clean_orphaned_metrics(
|
|
|
222
212
|
try:
|
|
223
213
|
counts = internal_manager.count_metric_rows(name)
|
|
224
214
|
except Exception as e:
|
|
225
|
-
|
|
215
|
+
echo_error(name, f"error counting rows: {e}")
|
|
226
216
|
continue
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
detail = ", ".join(f"{table}={count:,}" for table, count in counts.items() if count)
|
|
230
|
-
click.echo(
|
|
231
|
-
click.style(f" {name}: {verb} {total:,} row(s)", bold=True)
|
|
232
|
-
+ (f" [{detail}]" if detail else "")
|
|
233
|
-
)
|
|
217
|
+
children = [f"{table}: {count:,} row(s)" for table, count in counts.items() if count]
|
|
218
|
+
echo_tree(name, children or ["(no rows)"])
|
|
234
219
|
|
|
235
220
|
if not execute:
|
|
236
221
|
click.echo()
|
|
@@ -264,16 +249,10 @@ def _clean_orphaned_metrics(
|
|
|
264
249
|
try:
|
|
265
250
|
internal_manager.purge_metric(name)
|
|
266
251
|
purged += 1
|
|
267
|
-
click.echo(click.style(f" ✓ {name}: purged", fg="green"))
|
|
268
252
|
except Exception as e:
|
|
269
|
-
|
|
253
|
+
echo_error(name, f"error purging: {e}")
|
|
270
254
|
|
|
271
|
-
|
|
272
|
-
click.echo(
|
|
273
|
-
click.style(
|
|
274
|
-
f"Done. Purged {purged} of {len(orphans)} orphaned metric(s).", fg="cyan", bold=True
|
|
275
|
-
)
|
|
276
|
-
)
|
|
255
|
+
echo_done(f"Purged {purged} of {len(orphans)} orphaned metric(s).")
|
|
277
256
|
|
|
278
257
|
|
|
279
258
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
|
@@ -44,6 +44,23 @@ def create_mock_alert_data(
|
|
|
44
44
|
# different teams). Pull them from the specific config we're testing.
|
|
45
45
|
mentions = list(alerting_config.mentions) if alerting_config else []
|
|
46
46
|
|
|
47
|
+
# Preview the alert with the rule it would actually fire on (min_detectors
|
|
48
|
+
# / direction / consecutive) so the test message matches the alert-centric
|
|
49
|
+
# default layout. Observed counts are set to satisfy the rule, as a real
|
|
50
|
+
# firing would.
|
|
51
|
+
min_detectors = getattr(alerting_config, "min_detectors", 1) or 1
|
|
52
|
+
direction_policy = getattr(alerting_config, "direction", "same") or "same"
|
|
53
|
+
consecutive_required = getattr(alerting_config, "consecutive_anomalies", 1) or 1
|
|
54
|
+
# Observed direction for the preview: a concrete side for up/down/same; for
|
|
55
|
+
# an "any" quorum of 2+ detectors show "mixed" (its whole point is that
|
|
56
|
+
# cross-direction anomalies combine), mirroring the real engine output.
|
|
57
|
+
if direction_policy in ("up", "down"):
|
|
58
|
+
observed_direction = direction_policy
|
|
59
|
+
elif direction_policy == "any" and min_detectors >= 2:
|
|
60
|
+
observed_direction = "mixed"
|
|
61
|
+
else:
|
|
62
|
+
observed_direction = "up"
|
|
63
|
+
|
|
47
64
|
# Create realistic mock data
|
|
48
65
|
return AlertData(
|
|
49
66
|
metric_name=metric_config.name,
|
|
@@ -54,7 +71,7 @@ def create_mock_alert_data(
|
|
|
54
71
|
confidence_upper=0.6234,
|
|
55
72
|
detector_name="MADDetector:threshold=3.0",
|
|
56
73
|
detector_params='{"threshold": 3.0, "window_size": 8640}',
|
|
57
|
-
direction=
|
|
74
|
+
direction=observed_direction,
|
|
58
75
|
severity=4.52,
|
|
59
76
|
detection_metadata={
|
|
60
77
|
"global_median": 0.5123,
|
|
@@ -68,8 +85,12 @@ def create_mock_alert_data(
|
|
|
68
85
|
}
|
|
69
86
|
],
|
|
70
87
|
},
|
|
71
|
-
consecutive_count=
|
|
88
|
+
consecutive_count=consecutive_required,
|
|
72
89
|
mentions=mentions,
|
|
90
|
+
min_detectors=min_detectors,
|
|
91
|
+
direction_policy=direction_policy,
|
|
92
|
+
consecutive_required=consecutive_required,
|
|
93
|
+
detector_count=min_detectors,
|
|
73
94
|
)
|
|
74
95
|
|
|
75
96
|
|
|
@@ -8,6 +8,7 @@ auto-expires after its timeout, but this command clears it immediately.
|
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
10
|
|
|
11
|
+
from detectkit.cli._output import echo_done, echo_error, echo_noop, echo_tree
|
|
11
12
|
from detectkit.cli.commands.run import find_project_root, select_metrics
|
|
12
13
|
from detectkit.config.profile import ProfilesConfig
|
|
13
14
|
from detectkit.database.internal_tables import InternalTablesManager
|
|
@@ -83,23 +84,13 @@ def run_unlock(select: str, profile: str | None):
|
|
|
83
84
|
try:
|
|
84
85
|
was_locked = internal_manager.clear_lock(metric_name)
|
|
85
86
|
except Exception as e:
|
|
86
|
-
|
|
87
|
-
click.style(f" ✗ {metric_name}: error clearing lock: {e}", fg="red"),
|
|
88
|
-
err=True,
|
|
89
|
-
)
|
|
87
|
+
echo_error(metric_name, f"error clearing lock: {e}")
|
|
90
88
|
continue
|
|
91
89
|
|
|
92
90
|
if was_locked:
|
|
93
91
|
cleared += 1
|
|
94
|
-
|
|
92
|
+
echo_tree(metric_name, ["lock cleared"])
|
|
95
93
|
else:
|
|
96
|
-
|
|
94
|
+
echo_noop(metric_name, "no active lock")
|
|
97
95
|
|
|
98
|
-
|
|
99
|
-
click.echo(
|
|
100
|
-
click.style(
|
|
101
|
-
f"Done. Cleared {cleared} lock(s) of {len(metrics)} metric(s).",
|
|
102
|
-
fg="cyan",
|
|
103
|
-
bold=True,
|
|
104
|
-
)
|
|
105
|
-
)
|
|
96
|
+
echo_done(f"Cleared {cleared} lock(s) of {len(metrics)} metric(s).")
|
|
@@ -29,6 +29,7 @@ detectkit/alerting/orchestrator/_recovery.py
|
|
|
29
29
|
detectkit/alerting/orchestrator/_types.py
|
|
30
30
|
detectkit/alerting/orchestrator/orchestrator.py
|
|
31
31
|
detectkit/cli/__init__.py
|
|
32
|
+
detectkit/cli/_output.py
|
|
32
33
|
detectkit/cli/main.py
|
|
33
34
|
detectkit/cli/commands/__init__.py
|
|
34
35
|
detectkit/cli/commands/clean.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|