detectkit 0.51.0__tar.gz → 0.52.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {detectkit-0.51.0/detectkit.egg-info → detectkit-0.52.1}/PKG-INFO +3 -3
- {detectkit-0.51.0 → detectkit-0.52.1}/README.md +2 -2
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/__init__.py +1 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/base.py +110 -6
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/email.py +3 -5
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/telegram.py +2 -10
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/webhook.py +3 -9
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_decision.py +121 -7
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_recovery.py +36 -4
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_replay.py +35 -8
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_types.py +37 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/_types.py +5 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/autotuner.py +5 -8
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/crossval.py +16 -4
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/detector_select.py +4 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/scoring.py +88 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/alerting.md +57 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/autotune.md +9 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/cli.md +2 -2
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/detectors.md +62 -9
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/overview.md +5 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/skills/dtk-autotune/SKILL.md +7 -3
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/skills/dtk-new-metric/SKILL.md +11 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/skills/dtk-tune/SKILL.md +12 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/main.py +2 -1
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/config/metric_config.py +83 -3
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/factory.py +2 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/statistical/__init__.py +8 -1
- detectkit-0.52.1/detectkit/detectors/statistical/autoreg.py +411 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/_alert_step.py +6 -6
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/_types.py +7 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/reporting/builder.py +27 -10
- {detectkit-0.51.0 → detectkit-0.52.1/detectkit.egg-info}/PKG-INFO +3 -3
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit.egg-info/SOURCES.txt +1 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/LICENSE +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/MANIFEST.in +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/branding.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/factory.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/mattermost.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/channels/slack.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_base.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/_base.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/config_emitter.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/distribution.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/grid_search.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/labels.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/result.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/runner.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/seasonality_search.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/settings.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/autotune/window_select.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/_output.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/CLAUDE.section.md +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/metrics.md +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/rules/project.md +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/skills/dtk-feedback/SKILL.md +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/assets/claude/skills/dtk-setup-project/SKILL.md +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/autotune.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/clean.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/init.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/init_claude.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/osi.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/run.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/test_alert.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/tune.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/ui.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/cli/commands/unlock.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/config/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/config/metric_io.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/config/profile.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/config/project_config.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/config/validator.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/core/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/core/interval.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/core/models.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/_sql_manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/clickhouse_manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_alert_states.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_autotune_runs.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_base.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_datapoints.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_detections.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_maintenance.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_metrics.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_schema.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/_tasks.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/internal_tables/manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/mysql_manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/postgres_manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/database/tables.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/base.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/seasonality.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/statistical/_windowed.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/statistical/iqr.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/statistical/mad.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/statistical/manual_bounds.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/detectors/statistical/zscore.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/loaders/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/loaders/metric_loader.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/loaders/query_template.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/error_dispatch.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/_base.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/_load_step.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/orchestration/task_manager/manager.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/reporting/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/reporting/assets/report.js +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/reporting/html_report.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/semantic/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/semantic/errors.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/semantic/exporter.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/semantic/importer.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/semantic/osi_model.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/semantic/query_gen.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/tuning/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/tuning/assets/tune.js +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/tuning/config_writer.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/tuning/html.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/tuning/payload.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/tuning/server.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/assets/ui.js +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/html.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/jobs.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/metric_files.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/overview.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/ui/server.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/utils/__init__.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/utils/datetime_utils.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/utils/env_interpolation.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/utils/json_utils.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit/utils/stats.py +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit.egg-info/dependency_links.txt +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit.egg-info/entry_points.txt +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit.egg-info/requires.txt +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/detectkit.egg-info/top_level.txt +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/pyproject.toml +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/requirements.txt +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/setup.cfg +0 -0
- {detectkit-0.51.0 → detectkit-0.52.1}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: detectkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.52.1
|
|
4
4
|
Summary: Metric monitoring with automatic anomaly detection
|
|
5
5
|
Author: detectkit team
|
|
6
6
|
License: MIT
|
|
@@ -91,11 +91,11 @@ Dynamic: license-file
|
|
|
91
91
|
## Features
|
|
92
92
|
|
|
93
93
|
- **Pure numpy arrays** — no pandas dependency in core logic
|
|
94
|
-
- **Statistical detectors** — Z-Score, MAD, IQR, Manual Bounds
|
|
94
|
+
- **Statistical detectors** — Z-Score, MAD, IQR, Manual Bounds, and a prediction-based Autoreg (AR) detector for fast-moving, non-seasonal metrics
|
|
95
95
|
- **Trend & seasonality handling** — seasonality grouping, recency weighting (`half_life`), robust linear detrending for slowly drifting metrics
|
|
96
96
|
- **Multi-channel alerting** — Mattermost, Slack, Telegram, Email, Webhook
|
|
97
97
|
- **@mentions** — tag users/groups in alerts, each channel formats natively
|
|
98
|
-
- **Alert lifecycle** — consecutive anomalies, cooldown, recovery notifications, no-data alerts
|
|
98
|
+
- **Alert lifecycle** — consecutive anomalies, fraction-of-window rule (`anomaly_window` + `min_anomaly_share`), cooldown, recovery notifications, no-data alerts
|
|
99
99
|
- **Project-level error alerts** — catch DB outages and pipeline crashes once per run
|
|
100
100
|
- **Database agnostic** — ClickHouse, PostgreSQL, MySQL
|
|
101
101
|
- **Idempotent** — resume from interruptions, no duplicate processing
|
|
@@ -18,11 +18,11 @@
|
|
|
18
18
|
## Features
|
|
19
19
|
|
|
20
20
|
- **Pure numpy arrays** — no pandas dependency in core logic
|
|
21
|
-
- **Statistical detectors** — Z-Score, MAD, IQR, Manual Bounds
|
|
21
|
+
- **Statistical detectors** — Z-Score, MAD, IQR, Manual Bounds, and a prediction-based Autoreg (AR) detector for fast-moving, non-seasonal metrics
|
|
22
22
|
- **Trend & seasonality handling** — seasonality grouping, recency weighting (`half_life`), robust linear detrending for slowly drifting metrics
|
|
23
23
|
- **Multi-channel alerting** — Mattermost, Slack, Telegram, Email, Webhook
|
|
24
24
|
- **@mentions** — tag users/groups in alerts, each channel formats natively
|
|
25
|
-
- **Alert lifecycle** — consecutive anomalies, cooldown, recovery notifications, no-data alerts
|
|
25
|
+
- **Alert lifecycle** — consecutive anomalies, fraction-of-window rule (`anomaly_window` + `min_anomaly_share`), cooldown, recovery notifications, no-data alerts
|
|
26
26
|
- **Project-level error alerts** — catch DB outages and pipeline crashes once per run
|
|
27
27
|
- **Database agnostic** — ClickHouse, PostgreSQL, MySQL
|
|
28
28
|
- **Idempotent** — resume from interruptions, no duplicate processing
|
|
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
|
|
|
4
4
|
A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "0.
|
|
7
|
+
__version__ = "0.52.1"
|
|
8
8
|
|
|
9
9
|
from detectkit.core.interval import Interval
|
|
10
10
|
from detectkit.core.models import ColumnDefinition, TableModel
|
|
@@ -111,6 +111,49 @@ class AlertData:
|
|
|
111
111
|
interval_seconds: int | None = None
|
|
112
112
|
onset_timestamp: Any | None = None
|
|
113
113
|
streak_capped: bool = False
|
|
114
|
+
# Fraction alert rule (``anomaly_window`` + ``min_anomaly_share``).
|
|
115
|
+
# ``window_points`` / ``min_anomaly_share`` carry the CONFIGURED rule (set on
|
|
116
|
+
# every alert of a share-configured config so the rule chip can name it);
|
|
117
|
+
# ``window_matched`` + ``fired_by_share`` are set only when the fraction rule
|
|
118
|
+
# (not the consecutive rule) actually fired, switching the lead sentence to
|
|
119
|
+
# the window story. All default off, so existing alerts render byte-identical.
|
|
120
|
+
window_points: int | None = None
|
|
121
|
+
window_matched: int | None = None
|
|
122
|
+
min_anomaly_share: float | None = None
|
|
123
|
+
fired_by_share: bool = False
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def format_rule_display(
|
|
127
|
+
*,
|
|
128
|
+
min_detectors: Any,
|
|
129
|
+
direction_policy: Any,
|
|
130
|
+
consecutive_required: Any,
|
|
131
|
+
window_points: int | None = None,
|
|
132
|
+
min_anomaly_share: float | None = None,
|
|
133
|
+
fired_by_share: bool = False,
|
|
134
|
+
interval_seconds: int | None = None,
|
|
135
|
+
) -> str:
|
|
136
|
+
"""The alert-rule chip content, shared by every rendering surface.
|
|
137
|
+
|
|
138
|
+
Legacy configs (no fraction rule) render the exact historical chip —
|
|
139
|
+
``min_detectors=… · direction=… · consecutive=…`` — byte-identical. A
|
|
140
|
+
share-configured config names both OR-ed rules; a share-FIRED alert leads
|
|
141
|
+
with the share rule (the one that actually tripped).
|
|
142
|
+
"""
|
|
143
|
+
from detectkit.utils.datetime_utils import format_duration
|
|
144
|
+
|
|
145
|
+
base = f"min_detectors={min_detectors} · direction={direction_policy}"
|
|
146
|
+
if window_points and min_anomaly_share is not None:
|
|
147
|
+
pct = f"{min_anomaly_share * 100:g}%"
|
|
148
|
+
if interval_seconds:
|
|
149
|
+
span = format_duration(window_points * interval_seconds)
|
|
150
|
+
else:
|
|
151
|
+
span = f"{window_points} points"
|
|
152
|
+
share_rule = f"share>={pct} over {span}"
|
|
153
|
+
if fired_by_share:
|
|
154
|
+
return f"{base} · {share_rule}"
|
|
155
|
+
return f"{base} · consecutive={consecutive_required} (or {share_rule})"
|
|
156
|
+
return f"{base} · consecutive={consecutive_required}"
|
|
114
157
|
|
|
115
158
|
|
|
116
159
|
class BaseAlertChannel(ABC):
|
|
@@ -187,6 +230,10 @@ class BaseAlertChannel(ABC):
|
|
|
187
230
|
- {consecutive_count} — true consecutive streak length (resolved at
|
|
188
231
|
fire time, not capped at the rule's threshold)
|
|
189
232
|
- {consecutive_required} — configured consecutive threshold (rule)
|
|
233
|
+
- {rule_display} — the full alert-rule chip content (consecutive
|
|
234
|
+
and/or the fraction rule, whichever is configured/fired)
|
|
235
|
+
- {window_points} / {window_matched} — fraction-rule window size and
|
|
236
|
+
matched count (empty when the fraction rule isn't configured/fired)
|
|
190
237
|
- {interval_display} — metric interval as a string (e.g. "10min")
|
|
191
238
|
- {duration_display} — how long the streak/incident lasted
|
|
192
239
|
(e.g. "2h 30m"; "over …" when it predates the lookback window)
|
|
@@ -349,7 +396,42 @@ class BaseAlertChannel(ABC):
|
|
|
349
396
|
capped = alert_data.streak_capped
|
|
350
397
|
interval_display = str(Interval(interval_seconds)) if interval_seconds else ""
|
|
351
398
|
|
|
352
|
-
if
|
|
399
|
+
if (
|
|
400
|
+
alert_data.fired_by_share
|
|
401
|
+
and alert_data.window_points
|
|
402
|
+
and alert_data.window_matched is not None
|
|
403
|
+
):
|
|
404
|
+
# Fraction-rule story: a share alert has scattered matches, not a
|
|
405
|
+
# contiguous streak, so the lead reports the window share instead
|
|
406
|
+
# of a consecutive-run duration.
|
|
407
|
+
window = alert_data.window_points
|
|
408
|
+
matched = alert_data.window_matched
|
|
409
|
+
observed_pct = f"{matched / window * 100:.0f}%"
|
|
410
|
+
threshold_pct = (
|
|
411
|
+
f"{alert_data.min_anomaly_share * 100:g}%"
|
|
412
|
+
if alert_data.min_anomaly_share is not None
|
|
413
|
+
else ""
|
|
414
|
+
)
|
|
415
|
+
streak_display = f"{matched}/{window}"
|
|
416
|
+
started_display = onset_str
|
|
417
|
+
if interval_seconds:
|
|
418
|
+
duration_display = format_duration(window * interval_seconds)
|
|
419
|
+
anomaly_lead = (
|
|
420
|
+
f"{matched} of the last {window} {interval_display} intervals were "
|
|
421
|
+
f"anomalous ({observed_pct}) — at or above the {threshold_pct} share "
|
|
422
|
+
f"threshold over {duration_display}."
|
|
423
|
+
)
|
|
424
|
+
else:
|
|
425
|
+
duration_display = ""
|
|
426
|
+
anomaly_lead = (
|
|
427
|
+
f"{matched} of the last {window} intervals were anomalous "
|
|
428
|
+
f"({observed_pct}) — at or above the {threshold_pct} share threshold."
|
|
429
|
+
)
|
|
430
|
+
recovery_lead = (
|
|
431
|
+
"The alert condition no longer holds — the metric is back within "
|
|
432
|
+
"expected bounds."
|
|
433
|
+
)
|
|
434
|
+
elif interval_seconds and streak >= 1:
|
|
353
435
|
duration_display = format_duration(streak * interval_seconds)
|
|
354
436
|
if capped:
|
|
355
437
|
duration_display = f"over {duration_display}"
|
|
@@ -387,7 +469,14 @@ class BaseAlertChannel(ABC):
|
|
|
387
469
|
# in (direct-API callers). The firing message doesn't show it: there it
|
|
388
470
|
# coincides with the latest point the alert is firing on.
|
|
389
471
|
fired_display = ""
|
|
390
|
-
if
|
|
472
|
+
if (
|
|
473
|
+
alert_data.onset_timestamp is not None
|
|
474
|
+
and interval_seconds
|
|
475
|
+
and not capped
|
|
476
|
+
# A share-fired alert's onset is the first matched point in the
|
|
477
|
+
# window; onset + consecutive math doesn't describe its fire time.
|
|
478
|
+
and not alert_data.fired_by_share
|
|
479
|
+
):
|
|
391
480
|
req = (
|
|
392
481
|
consecutive_required if (consecutive_required and consecutive_required >= 1) else 1
|
|
393
482
|
)
|
|
@@ -451,6 +540,20 @@ class BaseAlertChannel(ABC):
|
|
|
451
540
|
project_name = alert_data.project_name or ""
|
|
452
541
|
project_name_prefix = f"[{alert_data.project_name}] " if alert_data.project_name else ""
|
|
453
542
|
|
|
543
|
+
# Alert-rule chip content — the single shared string every surface
|
|
544
|
+
# renders (default templates, webhook attachment, Telegram HTML, email).
|
|
545
|
+
# Built from the SAME fallback-resolved values as the individual keys,
|
|
546
|
+
# so legacy configs render the historical chip byte-identically.
|
|
547
|
+
rule_display = format_rule_display(
|
|
548
|
+
min_detectors=min_detectors,
|
|
549
|
+
direction_policy=direction_policy,
|
|
550
|
+
consecutive_required=consecutive_required,
|
|
551
|
+
window_points=alert_data.window_points,
|
|
552
|
+
min_anomaly_share=alert_data.min_anomaly_share,
|
|
553
|
+
fired_by_share=alert_data.fired_by_share,
|
|
554
|
+
interval_seconds=interval_seconds,
|
|
555
|
+
)
|
|
556
|
+
|
|
454
557
|
# Status keyword
|
|
455
558
|
if alert_data.is_error:
|
|
456
559
|
status = "ERROR"
|
|
@@ -482,6 +585,9 @@ class BaseAlertChannel(ABC):
|
|
|
482
585
|
"severity": alert_data.severity,
|
|
483
586
|
"consecutive_count": alert_data.consecutive_count,
|
|
484
587
|
"consecutive_required": consecutive_required,
|
|
588
|
+
"rule_display": rule_display,
|
|
589
|
+
"window_points": alert_data.window_points or "",
|
|
590
|
+
"window_matched": alert_data.window_matched or "",
|
|
485
591
|
"interval_display": interval_display,
|
|
486
592
|
"duration_display": duration_display,
|
|
487
593
|
"streak_display": streak_display,
|
|
@@ -617,8 +723,7 @@ class BaseAlertChannel(ABC):
|
|
|
617
723
|
"🔴 {project_name_prefix}Alert: {metric_name}\n"
|
|
618
724
|
"{description_line}"
|
|
619
725
|
"{anomaly_lead}\n"
|
|
620
|
-
"Rule:
|
|
621
|
-
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
726
|
+
"Rule: {rule_display}\n"
|
|
622
727
|
"\n"
|
|
623
728
|
"Value: {value_display} | Expected: {expected_range}\n"
|
|
624
729
|
"Quorum: {detector_count}/{min_detectors} · {direction}\n"
|
|
@@ -642,8 +747,7 @@ class BaseAlertChannel(ABC):
|
|
|
642
747
|
"🟢 {project_name_prefix}Alert cleared: {metric_name}\n"
|
|
643
748
|
"{description_line}"
|
|
644
749
|
"{recovery_lead}\n"
|
|
645
|
-
"Rule:
|
|
646
|
-
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
750
|
+
"Rule: {rule_display}\n"
|
|
647
751
|
"\n"
|
|
648
752
|
"Value: {value_display} | Expected: {expected_range}\n"
|
|
649
753
|
"{window_line}"
|
|
@@ -385,11 +385,9 @@ class EmailChannel(BaseAlertChannel):
|
|
|
385
385
|
Mirrors the inline-code "Rule chip" the webhook/Telegram channels render,
|
|
386
386
|
so the same firing rule reads the same way in every channel.
|
|
387
387
|
"""
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
f"consecutive={ctx['consecutive_required']}"
|
|
392
|
-
)
|
|
388
|
+
# Keep the middot as the ``·`` entity the chip has always used,
|
|
389
|
+
# so legacy (consecutive-only) configs render byte-identical HTML.
|
|
390
|
+
expr = html.escape(str(ctx["rule_display"])).replace("·", "·")
|
|
393
391
|
return (
|
|
394
392
|
f'<tr><td style="padding:0 24px 14px 24px;font-family:{_SANS};font-size:13px;'
|
|
395
393
|
f'color:{_MUTED};mso-line-height-rule:exactly;line-height:22px;">'
|
|
@@ -165,11 +165,7 @@ class TelegramChannel(BaseAlertChannel):
|
|
|
165
165
|
# Description (how long it's been going on) leads; the Rule chip sits
|
|
166
166
|
# right above the evidence it explains.
|
|
167
167
|
lines.append(esc(ctx["anomaly_lead"]))
|
|
168
|
-
lines.append(
|
|
169
|
-
f"<b>Rule</b> <code>min_detectors={ctx['min_detectors']} · "
|
|
170
|
-
f"direction={esc(ctx['direction_policy'])} · "
|
|
171
|
-
f"consecutive={ctx['consecutive_required']}</code>"
|
|
172
|
-
)
|
|
168
|
+
lines.append(f"<b>Rule</b> <code>{esc(ctx['rule_display'])}</code>")
|
|
173
169
|
lines.append("")
|
|
174
170
|
lines.append(
|
|
175
171
|
f"• Value: <code>{esc(ctx['value_display'])}</code> · "
|
|
@@ -193,11 +189,7 @@ class TelegramChannel(BaseAlertChannel):
|
|
|
193
189
|
lines.append(f"• Parameters: <code>{esc(params)}</code>")
|
|
194
190
|
elif kind == "recovery":
|
|
195
191
|
lines.append(esc(ctx["recovery_lead"]))
|
|
196
|
-
lines.append(
|
|
197
|
-
f"<b>Rule</b> <code>min_detectors={ctx['min_detectors']} · "
|
|
198
|
-
f"direction={esc(ctx['direction_policy'])} · "
|
|
199
|
-
f"consecutive={ctx['consecutive_required']}</code>"
|
|
200
|
-
)
|
|
192
|
+
lines.append(f"<b>Rule</b> <code>{esc(ctx['rule_display'])}</code>")
|
|
201
193
|
lines.append("")
|
|
202
194
|
lines.append(
|
|
203
195
|
f"• Value: <code>{esc(ctx['value_display'])}</code> · "
|
|
@@ -291,11 +291,7 @@ class WebhookChannel(BaseAlertChannel):
|
|
|
291
291
|
# inline-code chip so it reads as "this is the config that fired" at a
|
|
292
292
|
# glance. Backticks render identically on Slack and Mattermost; the bold
|
|
293
293
|
# label is platform-aware (see ``_bold``). Leads the body, above the fold.
|
|
294
|
-
rule_chip = f"{self._bold('Rule')} " + code(
|
|
295
|
-
f"min_detectors={ctx['min_detectors']} · "
|
|
296
|
-
f"direction={ctx['direction_policy']} · "
|
|
297
|
-
f"consecutive={ctx['consecutive_required']}"
|
|
298
|
-
)
|
|
294
|
+
rule_chip = f"{self._bold('Rule')} " + code(ctx["rule_display"])
|
|
299
295
|
|
|
300
296
|
# Links — compact clickable labels (never raw URL strings: a Grafana URL
|
|
301
297
|
# can be a paragraph long once it carries variables), rendered in the
|
|
@@ -458,8 +454,7 @@ class WebhookChannel(BaseAlertChannel):
|
|
|
458
454
|
return (
|
|
459
455
|
"{description_line}"
|
|
460
456
|
"{anomaly_lead}\n"
|
|
461
|
-
"Rule:
|
|
462
|
-
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
457
|
+
"Rule: {rule_display}\n"
|
|
463
458
|
"\n"
|
|
464
459
|
"Value: {value_display} | Expected: {expected_range}\n"
|
|
465
460
|
"Quorum: {detector_count}/{min_detectors} · {direction}\n"
|
|
@@ -479,8 +474,7 @@ class WebhookChannel(BaseAlertChannel):
|
|
|
479
474
|
return (
|
|
480
475
|
"{description_line}"
|
|
481
476
|
"{recovery_lead}\n"
|
|
482
|
-
"Rule:
|
|
483
|
-
"direction={direction_policy} · consecutive={consecutive_required}\n"
|
|
477
|
+
"Rule: {rule_display}\n"
|
|
484
478
|
"\n"
|
|
485
479
|
"Value: {value_display} | Expected: {expected_range}\n"
|
|
486
480
|
"{window_line}"
|
|
@@ -68,14 +68,96 @@ class _DecisionMixin(_OrchestratorBase):
|
|
|
68
68
|
consecutive, latest_quorum, direction = self._count_consecutive_anomalies(
|
|
69
69
|
detections_by_time, timestamps_sorted
|
|
70
70
|
)
|
|
71
|
-
if
|
|
72
|
-
|
|
71
|
+
if latest_quorum and consecutive >= self.conditions.consecutive_anomalies:
|
|
72
|
+
# The decision is made; now resolve the *true* streak length/onset
|
|
73
|
+
# for the message (the shallow alert window caps ``consecutive`` at
|
|
74
|
+
# the rule threshold, which can't answer "how long has this been
|
|
75
|
+
# going on").
|
|
76
|
+
streak, onset_ts, capped = self._resolve_streak(latest_quorum[0].timestamp)
|
|
77
|
+
return True, self._build_alert_data(latest_quorum, streak, direction, onset_ts, capped)
|
|
78
|
+
|
|
79
|
+
# The consecutive rule didn't fire; try the fraction rule (OR-ed).
|
|
80
|
+
share_fire = self._share_fire(detections_by_time, timestamps_sorted)
|
|
81
|
+
if share_fire is not None:
|
|
82
|
+
matched, onset_ts, quorum, share_direction = share_fire
|
|
83
|
+
return True, self._build_alert_data(
|
|
84
|
+
quorum,
|
|
85
|
+
matched,
|
|
86
|
+
share_direction,
|
|
87
|
+
onset_ts,
|
|
88
|
+
False,
|
|
89
|
+
window_matched=matched,
|
|
90
|
+
fired_by_share=True,
|
|
91
|
+
)
|
|
92
|
+
return False, None
|
|
73
93
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
94
|
+
def _share_fire(
|
|
95
|
+
self,
|
|
96
|
+
detections_by_time: dict[np.datetime64, list[DetectionRecord]],
|
|
97
|
+
timestamps_sorted: Sequence[np.datetime64],
|
|
98
|
+
) -> tuple[int, np.datetime64, list[DetectionRecord], str | None] | None:
|
|
99
|
+
"""Evaluate the fraction rule at the latest point.
|
|
100
|
+
|
|
101
|
+
Fires when the latest point itself meets the direction-aware quorum
|
|
102
|
+
(so an alert never fires on a stale window whose newest point is
|
|
103
|
+
already clean) AND at least ``min_anomaly_share`` of the trailing
|
|
104
|
+
``window_points`` grid slots meet the quorum. Grid slots with no
|
|
105
|
+
detections (gaps, outages) count in the denominator but never in the
|
|
106
|
+
numerator — missing data makes the rule harder to fire, not easier.
|
|
107
|
+
|
|
108
|
+
Returns ``(matched, onset_ts, latest_quorum, direction)`` or ``None``.
|
|
109
|
+
For ``"same"`` the direction is locked from the latest quorum, exactly
|
|
110
|
+
like the consecutive walk.
|
|
111
|
+
"""
|
|
112
|
+
window = self.conditions.window_points or 0
|
|
113
|
+
share_required = self.conditions.min_anomaly_share
|
|
114
|
+
if window <= 0 or share_required is None or not timestamps_sorted:
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
latest_ts = timestamps_sorted[0]
|
|
118
|
+
latest_anomalies = [d for d in detections_by_time[latest_ts] if d.is_anomaly]
|
|
119
|
+
quorum, direction = self._quorum_at(latest_anomalies, None)
|
|
120
|
+
if quorum is None:
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
locked = direction if self.conditions.direction == "same" else None
|
|
124
|
+
matched, _total, onset_ts = self._window_share_counts(detections_by_time, latest_ts, locked)
|
|
125
|
+
if matched / window < share_required:
|
|
126
|
+
return None
|
|
127
|
+
return matched, onset_ts, quorum, direction
|
|
128
|
+
|
|
129
|
+
def _window_share_counts(
|
|
130
|
+
self,
|
|
131
|
+
detections_by_time: dict[np.datetime64, list[DetectionRecord]],
|
|
132
|
+
latest_ts: np.datetime64,
|
|
133
|
+
locked_direction: str | None,
|
|
134
|
+
) -> tuple[int, int, np.datetime64]:
|
|
135
|
+
"""Count quorum-meeting grid slots in the window ending at *latest_ts*.
|
|
136
|
+
|
|
137
|
+
Walks the ``window_points`` grid slots ``latest_ts, latest_ts - step,
|
|
138
|
+
…`` looking each up in *detections_by_time* (an absent slot counts only
|
|
139
|
+
in the denominator). Returns ``(matched, window_points, onset_ts)``
|
|
140
|
+
where ``onset_ts`` is the oldest matching slot (the incident onset as
|
|
141
|
+
far as the window can see; falls back to *latest_ts* when only the
|
|
142
|
+
latest matches).
|
|
143
|
+
"""
|
|
144
|
+
window = self.conditions.window_points or 0
|
|
145
|
+
step = np.timedelta64(self.interval.seconds, "s")
|
|
146
|
+
matched = 0
|
|
147
|
+
onset_ts = latest_ts
|
|
148
|
+
for k in range(window):
|
|
149
|
+
ts = latest_ts - step * k
|
|
150
|
+
records = detections_by_time.get(ts)
|
|
151
|
+
if not records:
|
|
152
|
+
continue
|
|
153
|
+
anomalies = [d for d in records if d.is_anomaly]
|
|
154
|
+
if not anomalies:
|
|
155
|
+
continue
|
|
156
|
+
quorum, _ = self._quorum_at(anomalies, locked_direction)
|
|
157
|
+
if quorum is not None:
|
|
158
|
+
matched += 1
|
|
159
|
+
onset_ts = ts
|
|
160
|
+
return matched, window, onset_ts
|
|
79
161
|
|
|
80
162
|
def _resolve_streak(self, latest_ts: np.datetime64) -> tuple[int, np.datetime64, bool]:
|
|
81
163
|
"""Resolve the full anomalous run ending at *latest_ts*.
|
|
@@ -164,6 +246,29 @@ class _DecisionMixin(_OrchestratorBase):
|
|
|
164
246
|
return anomalies, None
|
|
165
247
|
return None, None
|
|
166
248
|
|
|
249
|
+
def _share_still_elevated(
|
|
250
|
+
self,
|
|
251
|
+
detections_by_time: dict[np.datetime64, list[DetectionRecord]],
|
|
252
|
+
latest_ts: np.datetime64,
|
|
253
|
+
locked_direction: str | None,
|
|
254
|
+
) -> bool:
|
|
255
|
+
"""Fraction-rule recovery hysteresis.
|
|
256
|
+
|
|
257
|
+
``True`` while the window share stays at or above **half** the firing
|
|
258
|
+
threshold — a share hovering right at the threshold would otherwise
|
|
259
|
+
flap alert/recover on every boundary crossing. Always ``False`` when
|
|
260
|
+
the fraction rule isn't configured, so consecutive-only configs
|
|
261
|
+
recover exactly as before.
|
|
262
|
+
"""
|
|
263
|
+
window = self.conditions.window_points or 0
|
|
264
|
+
share_required = self.conditions.min_anomaly_share
|
|
265
|
+
if window <= 0 or share_required is None:
|
|
266
|
+
return False
|
|
267
|
+
matched, total, _onset = self._window_share_counts(
|
|
268
|
+
detections_by_time, latest_ts, locked_direction
|
|
269
|
+
)
|
|
270
|
+
return total > 0 and matched / total >= share_required / 2.0
|
|
271
|
+
|
|
167
272
|
def _count_consecutive_anomalies(
|
|
168
273
|
self,
|
|
169
274
|
detections_by_time: dict[np.datetime64, list[DetectionRecord]],
|
|
@@ -227,6 +332,8 @@ class _DecisionMixin(_OrchestratorBase):
|
|
|
227
332
|
direction: str | None,
|
|
228
333
|
onset_timestamp: np.datetime64 | None = None,
|
|
229
334
|
streak_capped: bool = False,
|
|
335
|
+
window_matched: int | None = None,
|
|
336
|
+
fired_by_share: bool = False,
|
|
230
337
|
) -> AlertData:
|
|
231
338
|
primary = self._primary_record(anomalies)
|
|
232
339
|
|
|
@@ -296,6 +403,13 @@ class _DecisionMixin(_OrchestratorBase):
|
|
|
296
403
|
interval_seconds=self.interval.seconds,
|
|
297
404
|
onset_timestamp=onset_timestamp,
|
|
298
405
|
streak_capped=streak_capped,
|
|
406
|
+
# Fraction rule: the configured window/share ride on every alert of
|
|
407
|
+
# a share-configured config (so the rule chip can name both rules);
|
|
408
|
+
# the matched count + flag only on a share-fired one.
|
|
409
|
+
window_points=self.conditions.window_points,
|
|
410
|
+
min_anomaly_share=self.conditions.min_anomaly_share,
|
|
411
|
+
window_matched=window_matched,
|
|
412
|
+
fired_by_share=fired_by_share,
|
|
299
413
|
)
|
|
300
414
|
|
|
301
415
|
def should_alert_no_data(
|
|
@@ -58,8 +58,9 @@ class _RecoveryMixin(_OrchestratorBase):
|
|
|
58
58
|
return False
|
|
59
59
|
|
|
60
60
|
last_point = self.get_last_complete_point()
|
|
61
|
-
# +5
|
|
62
|
-
|
|
61
|
+
# The wider of the two rules' windows, +5 safety margin so we don't
|
|
62
|
+
# truncate the consecutive/fraction window.
|
|
63
|
+
num_points = self.conditions.lookback_points + 5
|
|
63
64
|
|
|
64
65
|
recent_detections = self.internal.get_recent_detections(
|
|
65
66
|
metric_name=self.metric_name,
|
|
@@ -78,20 +79,48 @@ class _RecoveryMixin(_OrchestratorBase):
|
|
|
78
79
|
latest_anomalies = [d for d in detections_by_time[timestamps_sorted[0]] if d.is_anomaly]
|
|
79
80
|
|
|
80
81
|
direction_condition = self.conditions.direction
|
|
82
|
+
locked_direction: str | None = None
|
|
81
83
|
if direction_condition == "down":
|
|
82
84
|
blocking = [d for d in latest_anomalies if d.direction == "down"]
|
|
85
|
+
locked_direction = "down"
|
|
83
86
|
elif direction_condition == "up":
|
|
84
87
|
blocking = [d for d in latest_anomalies if d.direction == "up"]
|
|
88
|
+
locked_direction = "up"
|
|
85
89
|
elif direction_condition == "same":
|
|
86
90
|
trigger_direction = self._get_alert_trigger_direction(last_alert_timestamp)
|
|
87
91
|
if trigger_direction is None:
|
|
88
92
|
blocking = latest_anomalies # conservative fallback
|
|
89
93
|
else:
|
|
90
94
|
blocking = [d for d in latest_anomalies if d.direction == trigger_direction]
|
|
95
|
+
locked_direction = trigger_direction
|
|
91
96
|
else: # "any" / unknown — preserve historical behaviour
|
|
92
97
|
blocking = latest_anomalies
|
|
93
98
|
|
|
94
|
-
|
|
99
|
+
if blocking:
|
|
100
|
+
return False
|
|
101
|
+
# Fraction-rule hysteresis: with the share rule configured, a clean
|
|
102
|
+
# latest point isn't enough — the window share must also drop below
|
|
103
|
+
# half the firing threshold, or the alert would flap around it.
|
|
104
|
+
# The share is computed over an UNFILTERED fetch: ``recent_detections``
|
|
105
|
+
# above is ``created_after``-filtered (only rows persisted after the
|
|
106
|
+
# alert), which is right for the freshness check but would make almost
|
|
107
|
+
# every window slot look empty and defeat the hysteresis — the window
|
|
108
|
+
# walk needs the full stored history, whenever it was written.
|
|
109
|
+
# ``_share_still_elevated`` lives in _DecisionMixin; both mixins compose
|
|
110
|
+
# into AlertOrchestrator so the call resolves at runtime.
|
|
111
|
+
if self.conditions.min_anomaly_share is None or not self.conditions.window_points:
|
|
112
|
+
return True
|
|
113
|
+
window_rows = self.internal.get_recent_detections(
|
|
114
|
+
metric_name=self.metric_name,
|
|
115
|
+
last_point=last_point,
|
|
116
|
+
num_points=num_points,
|
|
117
|
+
)
|
|
118
|
+
window_records = hydrate_detection_records(window_rows)
|
|
119
|
+
if not window_records:
|
|
120
|
+
return True
|
|
121
|
+
window_by_time = self._group_by_timestamp(window_records)
|
|
122
|
+
latest_window_ts = max(window_by_time.keys())
|
|
123
|
+
return not self._share_still_elevated(window_by_time, latest_window_ts, locked_direction)
|
|
95
124
|
|
|
96
125
|
def _get_alert_trigger_direction(self, last_alert_timestamp: datetime) -> str | None:
|
|
97
126
|
"""Return the direction of the anomaly that triggered the last alert.
|
|
@@ -194,10 +223,13 @@ class _RecoveryMixin(_OrchestratorBase):
|
|
|
194
223
|
project_name=self.project_name,
|
|
195
224
|
help_url=self.help_url,
|
|
196
225
|
# Echo the rule that had fired so the recovery message names the
|
|
197
|
-
# same alert condition that just cleared
|
|
226
|
+
# same alert condition that just cleared (including the fraction
|
|
227
|
+
# rule when configured, so fire and recovery render one chip).
|
|
198
228
|
min_detectors=self.conditions.min_detectors,
|
|
199
229
|
direction_policy=self.conditions.direction,
|
|
200
230
|
consecutive_required=self.conditions.consecutive_anomalies,
|
|
231
|
+
window_points=self.conditions.window_points,
|
|
232
|
+
min_anomaly_share=self.conditions.min_anomaly_share,
|
|
201
233
|
# Incident timing for the "Incident lasted …" line.
|
|
202
234
|
interval_seconds=self.interval.seconds,
|
|
203
235
|
onset_timestamp=onset_ts,
|
|
@@ -114,16 +114,34 @@ class _ReplayMixin(_OrchestratorBase):
|
|
|
114
114
|
consecutive, latest_quorum, direction = self._count_consecutive_anomalies(
|
|
115
115
|
causal, ts_desc
|
|
116
116
|
)
|
|
117
|
-
|
|
118
|
-
latest_quorum is not None
|
|
119
|
-
|
|
120
|
-
|
|
117
|
+
fired_consecutive = (
|
|
118
|
+
latest_quorum is not None and consecutive >= self.conditions.consecutive_anomalies
|
|
119
|
+
)
|
|
120
|
+
# The fraction rule is OR-ed, exactly like the live path — only
|
|
121
|
+
# evaluated when the consecutive rule didn't fire (and only when
|
|
122
|
+
# configured; ``_share_fire`` returns None otherwise).
|
|
123
|
+
share_fire = None if fired_consecutive else self._share_fire(causal, ts_desc)
|
|
124
|
+
fired = (fired_consecutive or share_fire is not None) and not self._replay_in_cooldown(
|
|
125
|
+
t, sim_last_alert, sim_last_recovery
|
|
121
126
|
)
|
|
122
127
|
|
|
123
128
|
if fired:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
129
|
+
if fired_consecutive:
|
|
130
|
+
assert latest_quorum is not None # narrowed by ``fired_consecutive``
|
|
131
|
+
streak, onset, capped = self._replay_streak(causal, ts_desc)
|
|
132
|
+
ad = self._build_alert_data(latest_quorum, streak, direction, onset, capped)
|
|
133
|
+
else:
|
|
134
|
+
assert share_fire is not None # narrowed by ``fired``
|
|
135
|
+
matched, onset, quorum, share_direction = share_fire
|
|
136
|
+
ad = self._build_alert_data(
|
|
137
|
+
quorum,
|
|
138
|
+
matched,
|
|
139
|
+
share_direction,
|
|
140
|
+
onset,
|
|
141
|
+
False,
|
|
142
|
+
window_matched=matched,
|
|
143
|
+
fired_by_share=True,
|
|
144
|
+
)
|
|
127
145
|
events.append(ReplayedEvent("anomaly", t, ad))
|
|
128
146
|
sim_last_alert = t
|
|
129
147
|
elif (
|
|
@@ -209,20 +227,29 @@ class _ReplayMixin(_OrchestratorBase):
|
|
|
209
227
|
latest_anomalies = [d for d in causal[latest_ts] if d.is_anomaly]
|
|
210
228
|
|
|
211
229
|
policy = self.conditions.direction
|
|
230
|
+
locked_direction: str | None = None
|
|
212
231
|
if policy == "down":
|
|
213
232
|
blocking = [d for d in latest_anomalies if d.direction == "down"]
|
|
233
|
+
locked_direction = "down"
|
|
214
234
|
elif policy == "up":
|
|
215
235
|
blocking = [d for d in latest_anomalies if d.direction == "up"]
|
|
236
|
+
locked_direction = "up"
|
|
216
237
|
elif policy == "same":
|
|
217
238
|
trigger_direction = self._replay_trigger_direction(causal, sim_last_alert)
|
|
218
239
|
if trigger_direction is None:
|
|
219
240
|
blocking = latest_anomalies # conservative fallback
|
|
220
241
|
else:
|
|
221
242
|
blocking = [d for d in latest_anomalies if d.direction == trigger_direction]
|
|
243
|
+
locked_direction = trigger_direction
|
|
222
244
|
else: # "any" / unknown — preserve historical behaviour
|
|
223
245
|
blocking = latest_anomalies
|
|
224
246
|
|
|
225
|
-
|
|
247
|
+
if blocking:
|
|
248
|
+
return False
|
|
249
|
+
# Fraction-rule hysteresis (in-memory analog of the live check): a share
|
|
250
|
+
# hovering at the threshold would flap alert/recover, so recovery also
|
|
251
|
+
# requires the window share to drop below half the firing threshold.
|
|
252
|
+
return not self._share_still_elevated(causal, latest_ts, locked_direction)
|
|
226
253
|
|
|
227
254
|
def _replay_trigger_direction(
|
|
228
255
|
self,
|
|
@@ -23,7 +23,7 @@ def _parse_detection_metadata(metadata: Any) -> dict:
|
|
|
23
23
|
return {}
|
|
24
24
|
if isinstance(metadata, dict):
|
|
25
25
|
return metadata
|
|
26
|
-
if isinstance(metadata,
|
|
26
|
+
if isinstance(metadata, bytes | bytearray):
|
|
27
27
|
try:
|
|
28
28
|
metadata = metadata.decode("utf-8")
|
|
29
29
|
except Exception:
|
|
@@ -64,11 +64,47 @@ class AlertConditions:
|
|
|
64
64
|
|
|
65
65
|
Defaults mirror :class:`detectkit.config.metric_config.AlertConfig`
|
|
66
66
|
so direct API users get the same behavior as YAML users.
|
|
67
|
+
|
|
68
|
+
``window_points`` / ``min_anomaly_share`` carry the optional fraction rule
|
|
69
|
+
(``AlertConfig.anomaly_window`` resolved to grid points): fire when the
|
|
70
|
+
share of quorum-meeting points over the trailing window reaches the
|
|
71
|
+
threshold AND the latest point itself meets the quorum. OR-ed with the
|
|
72
|
+
consecutive rule; both unset (the default) leaves behavior unchanged.
|
|
67
73
|
"""
|
|
68
74
|
|
|
69
75
|
min_detectors: int = 1
|
|
70
76
|
direction: str = "same" # "any", "same", "up", "down"
|
|
71
77
|
consecutive_anomalies: int = 3
|
|
78
|
+
window_points: int | None = None
|
|
79
|
+
min_anomaly_share: float | None = None
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def lookback_points(self) -> int:
|
|
83
|
+
"""Detection points the decision needs: the wider of the two rules."""
|
|
84
|
+
return max(self.consecutive_anomalies, self.window_points or 0)
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_alert_config(cls, alert_config: Any, interval_seconds: int) -> AlertConditions:
|
|
88
|
+
"""Build conditions from an ``AlertConfig``, resolving the window to points.
|
|
89
|
+
|
|
90
|
+
The single seam shared by the live alert step and the replay callers, so
|
|
91
|
+
the fraction-rule resolution (duration → grid points) can never drift
|
|
92
|
+
between them. ``getattr`` keeps duck-typed test stubs working.
|
|
93
|
+
"""
|
|
94
|
+
from detectkit.core.interval import Interval
|
|
95
|
+
|
|
96
|
+
window_points: int | None = None
|
|
97
|
+
window = getattr(alert_config, "anomaly_window", None)
|
|
98
|
+
share = getattr(alert_config, "min_anomaly_share", None)
|
|
99
|
+
if window is not None and share is not None and interval_seconds > 0:
|
|
100
|
+
window_points = max(1, Interval(window).seconds // interval_seconds)
|
|
101
|
+
return cls(
|
|
102
|
+
min_detectors=alert_config.min_detectors,
|
|
103
|
+
direction=alert_config.direction,
|
|
104
|
+
consecutive_anomalies=alert_config.consecutive_anomalies,
|
|
105
|
+
window_points=window_points,
|
|
106
|
+
min_anomaly_share=share if window_points is not None else None,
|
|
107
|
+
)
|
|
72
108
|
|
|
73
109
|
|
|
74
110
|
@dataclass
|