phlo-alerting 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_alerting-0.1.0/PKG-INFO +17 -0
- phlo_alerting-0.1.0/README.md +73 -0
- phlo_alerting-0.1.0/pyproject.toml +43 -0
- phlo_alerting-0.1.0/setup.cfg +4 -0
- phlo_alerting-0.1.0/src/phlo_alerting/__init__.py +18 -0
- phlo_alerting-0.1.0/src/phlo_alerting/cli_alerts.py +141 -0
- phlo_alerting-0.1.0/src/phlo_alerting/cli_plugin.py +21 -0
- phlo_alerting-0.1.0/src/phlo_alerting/destinations/__init__.py +1 -0
- phlo_alerting-0.1.0/src/phlo_alerting/destinations/email.py +166 -0
- phlo_alerting-0.1.0/src/phlo_alerting/destinations/pagerduty.py +80 -0
- phlo_alerting-0.1.0/src/phlo_alerting/destinations/slack.py +107 -0
- phlo_alerting-0.1.0/src/phlo_alerting/hooks_plugin.py +111 -0
- phlo_alerting-0.1.0/src/phlo_alerting/manager.py +181 -0
- phlo_alerting-0.1.0/src/phlo_alerting/settings.py +35 -0
- phlo_alerting-0.1.0/src/phlo_alerting.egg-info/PKG-INFO +17 -0
- phlo_alerting-0.1.0/src/phlo_alerting.egg-info/SOURCES.txt +20 -0
- phlo_alerting-0.1.0/src/phlo_alerting.egg-info/dependency_links.txt +1 -0
- phlo_alerting-0.1.0/src/phlo_alerting.egg-info/entry_points.txt +5 -0
- phlo_alerting-0.1.0/src/phlo_alerting.egg-info/requires.txt +8 -0
- phlo_alerting-0.1.0/src/phlo_alerting.egg-info/top_level.txt +1 -0
- phlo_alerting-0.1.0/tests/test_alerting.py +92 -0
- phlo_alerting-0.1.0/tests/test_integration_alerting.py +167 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phlo-alerting
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Alerting destinations and hooks for Phlo
|
|
5
|
+
Author-email: Phlo Team <team@phlo.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/plain
|
|
9
|
+
Requires-Dist: phlo>=0.1.0
|
|
10
|
+
Requires-Dist: click>=8.3.0
|
|
11
|
+
Requires-Dist: rich>=14.2.0
|
|
12
|
+
Requires-Dist: requests>=2.32.5
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
15
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
16
|
+
|
|
17
|
+
Alerting support for Phlo.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# phlo-alerting
|
|
2
|
+
|
|
3
|
+
Alert routing and notification plugin for Phlo.
|
|
4
|
+
|
|
5
|
+
## Description
|
|
6
|
+
|
|
7
|
+
Routes alerts from quality check failures, telemetry events, and pipeline errors to configured destinations (Slack, PagerDuty, Email).
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install phlo-alerting
|
|
13
|
+
# or
|
|
14
|
+
phlo plugin install alerting
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Configuration
|
|
18
|
+
|
|
19
|
+
| Variable | Default | Description |
|
|
20
|
+
| -------------------------------- | ------- | --------------------------------------- |
|
|
21
|
+
| `PHLO_ALERT_SLACK_WEBHOOK` | - | Slack incoming webhook URL |
|
|
22
|
+
| `PHLO_ALERT_SLACK_CHANNEL` | - | Default Slack channel |
|
|
23
|
+
| `PHLO_ALERT_PAGERDUTY_KEY` | - | PagerDuty Events API v2 key |
|
|
24
|
+
| `PHLO_ALERT_EMAIL_SMTP_HOST` | - | SMTP server hostname |
|
|
25
|
+
| `PHLO_ALERT_EMAIL_SMTP_PORT` | `587` | SMTP server port |
|
|
26
|
+
| `PHLO_ALERT_EMAIL_SMTP_USER` | - | SMTP username |
|
|
27
|
+
| `PHLO_ALERT_EMAIL_SMTP_PASSWORD` | - | SMTP password |
|
|
28
|
+
| `PHLO_ALERT_EMAIL_RECIPIENTS` | `[]` | Comma-separated list: `a@x.com,b@y.com` |
|
|
29
|
+
|
|
30
|
+
## Auto-Configuration
|
|
31
|
+
|
|
32
|
+
This package is **fully auto-configured**:
|
|
33
|
+
|
|
34
|
+
| Feature | How It Works |
|
|
35
|
+
| --------------------- | --------------------------------------------------------------- |
|
|
36
|
+
| **Hook Registration** | Automatically registers as a hook plugin via entry points |
|
|
37
|
+
| **Event Handling** | Listens for `quality.result` and `telemetry.*` events |
|
|
38
|
+
| **Severity Mapping** | Maps event severities to alert levels (critical, warning, info) |
|
|
39
|
+
|
|
40
|
+
### Supported Events
|
|
41
|
+
|
|
42
|
+
- `quality.result` - Receives quality check results and sends alerts on failures
|
|
43
|
+
- `telemetry.*` - Receives telemetry events and forwards to configured destinations
|
|
44
|
+
|
|
45
|
+
## Usage
|
|
46
|
+
|
|
47
|
+
### CLI Commands
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
# Send a test alert
|
|
51
|
+
phlo alerts test --destination slack
|
|
52
|
+
|
|
53
|
+
# List configured destinations
|
|
54
|
+
phlo alerts list-destinations
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Programmatic
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from phlo_alerting.manager import AlertManager
|
|
61
|
+
|
|
62
|
+
manager = AlertManager()
|
|
63
|
+
manager.send_alert(
|
|
64
|
+
title="Quality Check Failed",
|
|
65
|
+
message="Null check failed on table users",
|
|
66
|
+
severity="critical"
|
|
67
|
+
)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Entry Points
|
|
71
|
+
|
|
72
|
+
- `phlo.plugins.cli` - Provides `alerts` CLI command
|
|
73
|
+
- `phlo.plugins.hooks` - Registers `AlertingHookPlugin` for event handling
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "phlo-alerting"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Alerting destinations and hooks for Phlo"
|
|
9
|
+
readme = {text = "Alerting support for Phlo.", content-type = "text/plain"}
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [
|
|
12
|
+
{name = "Phlo Team", email = "team@phlo.dev"},
|
|
13
|
+
]
|
|
14
|
+
license = {text = "MIT"}
|
|
15
|
+
dependencies = [
|
|
16
|
+
"phlo>=0.1.0",
|
|
17
|
+
"click>=8.3.0",
|
|
18
|
+
"rich>=14.2.0",
|
|
19
|
+
"requests>=2.32.5",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
dev = [
|
|
24
|
+
"pytest>=7.0",
|
|
25
|
+
"ruff>=0.1.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.entry-points."phlo.plugins.cli"]
|
|
29
|
+
alerts = "phlo_alerting.cli_plugin:AlertingCliPlugin"
|
|
30
|
+
|
|
31
|
+
[project.entry-points."phlo.plugins.hooks"]
|
|
32
|
+
alerting = "phlo_alerting.hooks_plugin:AlertingHookPlugin"
|
|
33
|
+
|
|
34
|
+
[tool.setuptools]
|
|
35
|
+
package-dir = {"" = "src"}
|
|
36
|
+
include-package-data = true
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.packages.find]
|
|
39
|
+
where = ["src"]
|
|
40
|
+
|
|
41
|
+
[tool.ruff]
|
|
42
|
+
line-length = 100
|
|
43
|
+
target-version = "py311"
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Alerting integration for Phlo pipelines."""
|
|
2
|
+
|
|
3
|
+
from phlo_alerting.manager import (
|
|
4
|
+
Alert,
|
|
5
|
+
AlertManager,
|
|
6
|
+
AlertSeverity,
|
|
7
|
+
get_alert_manager,
|
|
8
|
+
)
|
|
9
|
+
from phlo_alerting.settings import AlertingSettings, get_settings
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"AlertManager",
|
|
13
|
+
"Alert",
|
|
14
|
+
"AlertSeverity",
|
|
15
|
+
"AlertingSettings",
|
|
16
|
+
"get_alert_manager",
|
|
17
|
+
"get_settings",
|
|
18
|
+
]
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""CLI commands for alert management."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from phlo_alerting import AlertSeverity
|
|
12
|
+
from phlo_alerting.manager import get_alert_manager
|
|
13
|
+
|
|
14
|
+
console = Console()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.group(name="alerts")
|
|
18
|
+
def alerts_group():
|
|
19
|
+
"""Alert management and configuration."""
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@alerts_group.command(name="test")
|
|
24
|
+
@click.option(
|
|
25
|
+
"--severity",
|
|
26
|
+
type=click.Choice(["info", "warning", "error", "critical"]),
|
|
27
|
+
default="warning",
|
|
28
|
+
help="Alert severity",
|
|
29
|
+
)
|
|
30
|
+
@click.option(
|
|
31
|
+
"--destination",
|
|
32
|
+
type=str,
|
|
33
|
+
default=None,
|
|
34
|
+
help="Specific destination to test (default: all)",
|
|
35
|
+
)
|
|
36
|
+
def test_alerts(severity: str, destination: Optional[str]) -> None:
|
|
37
|
+
"""
|
|
38
|
+
Send a test alert to configured destinations.
|
|
39
|
+
|
|
40
|
+
Useful for verifying alert configuration.
|
|
41
|
+
"""
|
|
42
|
+
from phlo_alerting import Alert
|
|
43
|
+
|
|
44
|
+
manager = get_alert_manager()
|
|
45
|
+
|
|
46
|
+
if not manager.destinations:
|
|
47
|
+
console.print(
|
|
48
|
+
"[red]✗[/red] No alert destinations configured. "
|
|
49
|
+
"Set PHLO_ALERT_SLACK_WEBHOOK, PHLO_ALERT_PAGERDUTY_KEY, or PHLO_ALERT_EMAIL_* environment variables."
|
|
50
|
+
)
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
# Create test alert
|
|
54
|
+
alert = Alert(
|
|
55
|
+
title="🧪 Phlo Test Alert",
|
|
56
|
+
message="This is a test alert from the Phlo CLI. If you see this, alerts are working!",
|
|
57
|
+
severity=AlertSeverity(severity),
|
|
58
|
+
asset_name="phlo_test",
|
|
59
|
+
run_id="test_run_123",
|
|
60
|
+
error_message=None,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Send to specific or all destinations
|
|
64
|
+
destinations = [destination] if destination else None
|
|
65
|
+
|
|
66
|
+
if manager.send(alert, destinations=destinations):
|
|
67
|
+
console.print(
|
|
68
|
+
"[green]✓[/green] Test alert sent successfully! "
|
|
69
|
+
"Check your configured alert destinations."
|
|
70
|
+
)
|
|
71
|
+
else:
|
|
72
|
+
console.print("[red]✗[/red] Failed to send test alert.")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@alerts_group.command(name="list")
|
|
76
|
+
def list_destinations() -> None:
|
|
77
|
+
"""List configured alert destinations."""
|
|
78
|
+
manager = get_alert_manager()
|
|
79
|
+
|
|
80
|
+
if not manager.destinations:
|
|
81
|
+
console.print(
|
|
82
|
+
"[yellow]⚠[/yellow] No alert destinations configured.\n"
|
|
83
|
+
"To enable alerts, set environment variables:"
|
|
84
|
+
)
|
|
85
|
+
console.print(
|
|
86
|
+
"""
|
|
87
|
+
PHLO_ALERT_SLACK_WEBHOOK=https://hooks.slack.com/services/...
|
|
88
|
+
PHLO_ALERT_SLACK_CHANNEL=#alerts (optional)
|
|
89
|
+
PHLO_ALERT_PAGERDUTY_KEY=...
|
|
90
|
+
PHLO_ALERT_EMAIL_SMTP_HOST=smtp.example.com
|
|
91
|
+
PHLO_ALERT_EMAIL_SMTP_PORT=587 (optional, default: 587)
|
|
92
|
+
PHLO_ALERT_EMAIL_SMTP_USER=user@example.com
|
|
93
|
+
PHLO_ALERT_EMAIL_SMTP_PASSWORD=password
|
|
94
|
+
PHLO_ALERT_EMAIL_RECIPIENTS=team@example.com,admin@example.com
|
|
95
|
+
"""
|
|
96
|
+
)
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
table = Table(title="Configured Alert Destinations")
|
|
100
|
+
table.add_column("Name", style="cyan")
|
|
101
|
+
table.add_column("Type", style="magenta")
|
|
102
|
+
table.add_column("Status", style="green")
|
|
103
|
+
|
|
104
|
+
for name, destination in manager.destinations.items():
|
|
105
|
+
dest_type = destination.__class__.__name__
|
|
106
|
+
status = "✓ Ready"
|
|
107
|
+
|
|
108
|
+
table.add_row(name, dest_type, status)
|
|
109
|
+
|
|
110
|
+
console.print(table)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@alerts_group.command(name="status")
|
|
114
|
+
def check_status() -> None:
|
|
115
|
+
"""Check alert system status."""
|
|
116
|
+
manager = get_alert_manager()
|
|
117
|
+
|
|
118
|
+
console.print("[bold]Alert System Status[/bold]\n")
|
|
119
|
+
|
|
120
|
+
# Check destinations
|
|
121
|
+
console.print(f"Configured Destinations: {len(manager.destinations)}")
|
|
122
|
+
for name in manager.destinations:
|
|
123
|
+
console.print(f" • {name}")
|
|
124
|
+
|
|
125
|
+
if not manager.destinations:
|
|
126
|
+
console.print(" [yellow]None configured[/yellow]")
|
|
127
|
+
|
|
128
|
+
# Show statistics
|
|
129
|
+
console.print(f"\nRecent Alerts Sent: {len(manager._sent_alerts)}")
|
|
130
|
+
console.print(f"Deduplication Window: {manager._dedup_window_minutes} minutes")
|
|
131
|
+
|
|
132
|
+
# Show configuration guidance
|
|
133
|
+
if len(manager.destinations) == 0:
|
|
134
|
+
console.print("\n[bold]Next Steps[/bold]")
|
|
135
|
+
console.print("1. Configure at least one alert destination via environment variables")
|
|
136
|
+
console.print("2. Run [cyan]phlo alerts test[/cyan] to verify configuration")
|
|
137
|
+
console.print("3. Alerts will automatically trigger on run failures")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
if __name__ == "__main__":
|
|
141
|
+
alerts_group()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""CLI plugin for alerting commands."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
|
|
7
|
+
from phlo.plugins.base import CliCommandPlugin, PluginMetadata
|
|
8
|
+
from phlo_alerting.cli_alerts import alerts_group
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AlertingCliPlugin(CliCommandPlugin):
|
|
12
|
+
@property
|
|
13
|
+
def metadata(self) -> PluginMetadata:
|
|
14
|
+
return PluginMetadata(
|
|
15
|
+
name="alerts",
|
|
16
|
+
version="0.1.0",
|
|
17
|
+
description="Alerting CLI commands",
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
def get_cli_commands(self) -> list[click.Command]:
|
|
21
|
+
return [alerts_group]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Alert destination implementations."""
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""Email alert destination."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import smtplib
|
|
6
|
+
from email.mime.multipart import MIMEMultipart
|
|
7
|
+
from email.mime.text import MIMEText
|
|
8
|
+
from typing import Optional
|
|
9
|
+
|
|
10
|
+
from phlo.logging import get_logger
|
|
11
|
+
from phlo_alerting.manager import Alert, AlertDestination, AlertSeverity
|
|
12
|
+
|
|
13
|
+
logger = get_logger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class EmailAlertDestination(AlertDestination):
|
|
17
|
+
"""Send alerts via email."""
|
|
18
|
+
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
smtp_host: str,
|
|
22
|
+
smtp_port: int = 587,
|
|
23
|
+
smtp_user: Optional[str] = None,
|
|
24
|
+
smtp_password: Optional[str] = None,
|
|
25
|
+
recipients: Optional[list[str]] = None,
|
|
26
|
+
):
|
|
27
|
+
"""
|
|
28
|
+
Initialize email destination.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
smtp_host: SMTP server hostname
|
|
32
|
+
smtp_port: SMTP server port (default: 587)
|
|
33
|
+
smtp_user: SMTP username (optional)
|
|
34
|
+
smtp_password: SMTP password (optional)
|
|
35
|
+
recipients: List of email addresses to send to
|
|
36
|
+
"""
|
|
37
|
+
self.smtp_host = smtp_host
|
|
38
|
+
self.smtp_port = smtp_port
|
|
39
|
+
self.smtp_user = smtp_user
|
|
40
|
+
self.smtp_password = smtp_password
|
|
41
|
+
self.recipients = recipients or []
|
|
42
|
+
|
|
43
|
+
def send(self, alert: Alert) -> bool:
|
|
44
|
+
"""Send alert via email."""
|
|
45
|
+
if not self.recipients:
|
|
46
|
+
logger.warning("No email recipients configured")
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
# Build email
|
|
51
|
+
msg = MIMEMultipart("alternative")
|
|
52
|
+
msg["Subject"] = f"[{alert.severity.value.upper()}] {alert.title}"
|
|
53
|
+
msg["From"] = self.smtp_user or "phlo@example.com"
|
|
54
|
+
msg["To"] = ", ".join(self.recipients)
|
|
55
|
+
|
|
56
|
+
# Build plain text and HTML versions
|
|
57
|
+
text_content = self._build_text(alert)
|
|
58
|
+
html_content = self._build_html(alert)
|
|
59
|
+
|
|
60
|
+
msg.attach(MIMEText(text_content, "plain"))
|
|
61
|
+
msg.attach(MIMEText(html_content, "html"))
|
|
62
|
+
|
|
63
|
+
# Send email
|
|
64
|
+
with smtplib.SMTP(self.smtp_host, self.smtp_port) as server:
|
|
65
|
+
server.starttls()
|
|
66
|
+
if self.smtp_user and self.smtp_password:
|
|
67
|
+
server.login(self.smtp_user, self.smtp_password)
|
|
68
|
+
server.sendmail(msg["From"], self.recipients, msg.as_string())
|
|
69
|
+
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
except Exception as e:
|
|
73
|
+
logger.exception(f"Failed to send email alert: {e}")
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
def _build_text(self, alert: Alert) -> str:
|
|
77
|
+
"""Build plain text email content."""
|
|
78
|
+
content = f"""
|
|
79
|
+
Phlo Alert Notification
|
|
80
|
+
=======================
|
|
81
|
+
|
|
82
|
+
Title: {alert.title}
|
|
83
|
+
Severity: {alert.severity.value.upper()}
|
|
84
|
+
Time: {alert.timestamp.isoformat() if alert.timestamp else "N/A"}
|
|
85
|
+
|
|
86
|
+
Message:
|
|
87
|
+
{alert.message}
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
if alert.asset_name:
|
|
91
|
+
content += f"\nAsset: {alert.asset_name}"
|
|
92
|
+
|
|
93
|
+
if alert.run_id:
|
|
94
|
+
content += f"\nRun ID: {alert.run_id}"
|
|
95
|
+
|
|
96
|
+
if alert.error_message:
|
|
97
|
+
content += f"\n\nError Details:\n{alert.error_message}"
|
|
98
|
+
|
|
99
|
+
return content
|
|
100
|
+
|
|
101
|
+
def _build_html(self, alert: Alert) -> str:
|
|
102
|
+
"""Build HTML email content."""
|
|
103
|
+
severity_color = {
|
|
104
|
+
AlertSeverity.INFO: "#36a64f",
|
|
105
|
+
AlertSeverity.WARNING: "#ff9900",
|
|
106
|
+
AlertSeverity.ERROR: "#ff3333",
|
|
107
|
+
AlertSeverity.CRITICAL: "#cc0000",
|
|
108
|
+
}.get(alert.severity, "#999999")
|
|
109
|
+
|
|
110
|
+
html = f"""
|
|
111
|
+
<html>
|
|
112
|
+
<body style="font-family: Arial, sans-serif;">
|
|
113
|
+
<div style="border-left: 4px solid {severity_color}; padding: 15px; background: #f9f9f9; margin: 10px 0;">
|
|
114
|
+
<h2 style="margin-top: 0; color: {severity_color};">{alert.title}</h2>
|
|
115
|
+
|
|
116
|
+
<table style="width: 100%; margin: 15px 0;">
|
|
117
|
+
<tr>
|
|
118
|
+
<td style="font-weight: bold; width: 120px;">Severity:</td>
|
|
119
|
+
<td style="color: {severity_color}; font-weight: bold;">{alert.severity.value.upper()}</td>
|
|
120
|
+
</tr>
|
|
121
|
+
<tr>
|
|
122
|
+
<td style="font-weight: bold;">Time:</td>
|
|
123
|
+
<td>{alert.timestamp.isoformat() if alert.timestamp else "N/A"}</td>
|
|
124
|
+
</tr>
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
if alert.asset_name:
|
|
128
|
+
html += f"""
|
|
129
|
+
<tr>
|
|
130
|
+
<td style="font-weight: bold;">Asset:</td>
|
|
131
|
+
<td>{alert.asset_name}</td>
|
|
132
|
+
</tr>
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
if alert.run_id:
|
|
136
|
+
html += f"""
|
|
137
|
+
<tr>
|
|
138
|
+
<td style="font-weight: bold;">Run ID:</td>
|
|
139
|
+
<td><code>{alert.run_id}</code></td>
|
|
140
|
+
</tr>
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
html += """
|
|
144
|
+
</table>
|
|
145
|
+
|
|
146
|
+
<div style="margin: 15px 0;">
|
|
147
|
+
<h3>Message</h3>
|
|
148
|
+
<p>{}</p>
|
|
149
|
+
</div>
|
|
150
|
+
""".format(alert.message)
|
|
151
|
+
|
|
152
|
+
if alert.error_message:
|
|
153
|
+
html += f"""
|
|
154
|
+
<div style="background: #f0f0f0; padding: 10px; border-radius: 3px; margin: 15px 0;">
|
|
155
|
+
<h3>Error Details</h3>
|
|
156
|
+
<pre style="margin: 0; font-size: 12px;">{alert.error_message}</pre>
|
|
157
|
+
</div>
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
html += """
|
|
161
|
+
</div>
|
|
162
|
+
</body>
|
|
163
|
+
</html>
|
|
164
|
+
"""
|
|
165
|
+
|
|
166
|
+
return html
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""PagerDuty alert destination."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from phlo.logging import get_logger
|
|
8
|
+
from phlo_alerting.manager import Alert, AlertDestination, AlertSeverity
|
|
9
|
+
|
|
10
|
+
logger = get_logger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class PagerDutyAlertDestination(AlertDestination):
|
|
14
|
+
"""Send alerts to PagerDuty via Events API."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, integration_key: str):
|
|
17
|
+
"""
|
|
18
|
+
Initialize PagerDuty destination.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
integration_key: PagerDuty Events API v2 integration key
|
|
22
|
+
"""
|
|
23
|
+
self.integration_key = integration_key
|
|
24
|
+
self.api_url = "https://events.pagerduty.com/v2/enqueue"
|
|
25
|
+
|
|
26
|
+
def send(self, alert: Alert) -> bool:
|
|
27
|
+
"""Send alert to PagerDuty."""
|
|
28
|
+
try:
|
|
29
|
+
payload = self._build_payload(alert)
|
|
30
|
+
response = requests.post(self.api_url, json=payload, timeout=10)
|
|
31
|
+
return response.status_code == 202 # Accepted
|
|
32
|
+
except Exception as e:
|
|
33
|
+
logger.exception(f"Failed to send PagerDuty alert: {e}")
|
|
34
|
+
return False
|
|
35
|
+
|
|
36
|
+
def _build_payload(self, alert: Alert) -> dict:
|
|
37
|
+
"""Build PagerDuty event payload."""
|
|
38
|
+
# Map severity to PagerDuty severity
|
|
39
|
+
severity_map = {
|
|
40
|
+
AlertSeverity.INFO: "info",
|
|
41
|
+
AlertSeverity.WARNING: "warning",
|
|
42
|
+
AlertSeverity.ERROR: "error",
|
|
43
|
+
AlertSeverity.CRITICAL: "critical",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
pd_severity = severity_map.get(alert.severity, "error")
|
|
47
|
+
|
|
48
|
+
# Build custom details
|
|
49
|
+
custom_details = {
|
|
50
|
+
"severity": alert.severity.value,
|
|
51
|
+
"message": alert.message,
|
|
52
|
+
"timestamp": alert.timestamp.isoformat() if alert.timestamp else None,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if alert.asset_name:
|
|
56
|
+
custom_details["asset"] = alert.asset_name
|
|
57
|
+
|
|
58
|
+
if alert.run_id:
|
|
59
|
+
custom_details["run_id"] = alert.run_id
|
|
60
|
+
|
|
61
|
+
if alert.error_message:
|
|
62
|
+
custom_details["error"] = alert.error_message
|
|
63
|
+
|
|
64
|
+
# Generate dedup key for alert grouping
|
|
65
|
+
dedup_key = f"phlo-{alert.asset_name or 'unknown'}-{alert.run_id or 'unknown'}"
|
|
66
|
+
|
|
67
|
+
payload = {
|
|
68
|
+
"routing_key": self.integration_key,
|
|
69
|
+
"event_action": "trigger",
|
|
70
|
+
"dedup_key": dedup_key,
|
|
71
|
+
"payload": {
|
|
72
|
+
"summary": alert.title,
|
|
73
|
+
"severity": pd_severity,
|
|
74
|
+
"source": "Phlo",
|
|
75
|
+
"timestamp": alert.timestamp.isoformat() if alert.timestamp else None,
|
|
76
|
+
"custom_details": custom_details,
|
|
77
|
+
},
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return payload
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Slack alert destination."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
from phlo.logging import get_logger
|
|
10
|
+
from phlo_alerting.manager import Alert, AlertDestination, AlertSeverity
|
|
11
|
+
|
|
12
|
+
logger = get_logger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SlackAlertDestination(AlertDestination):
|
|
16
|
+
"""Send alerts to Slack via webhook."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, webhook_url: str, channel: Optional[str] = None):
|
|
19
|
+
"""
|
|
20
|
+
Initialize Slack destination.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
webhook_url: Slack incoming webhook URL
|
|
24
|
+
channel: Optional channel to override (e.g., #alerts)
|
|
25
|
+
"""
|
|
26
|
+
self.webhook_url = webhook_url
|
|
27
|
+
self.channel = channel
|
|
28
|
+
|
|
29
|
+
def send(self, alert: Alert) -> bool:
|
|
30
|
+
"""Send alert to Slack."""
|
|
31
|
+
try:
|
|
32
|
+
payload = self._build_payload(alert)
|
|
33
|
+
response = requests.post(self.webhook_url, json=payload, timeout=10)
|
|
34
|
+
return response.status_code == 200
|
|
35
|
+
except Exception as e:
|
|
36
|
+
logger.exception(f"Failed to send Slack alert: {e}")
|
|
37
|
+
return False
|
|
38
|
+
|
|
39
|
+
def _build_payload(self, alert: Alert) -> dict:
|
|
40
|
+
"""Build Slack message payload."""
|
|
41
|
+
# Color based on severity
|
|
42
|
+
severity_colors = {
|
|
43
|
+
AlertSeverity.INFO: "#36a64f", # Green
|
|
44
|
+
AlertSeverity.WARNING: "#ff9900", # Orange
|
|
45
|
+
AlertSeverity.ERROR: "#ff3333", # Red
|
|
46
|
+
AlertSeverity.CRITICAL: "#cc0000", # Dark red
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
color = severity_colors.get(alert.severity, "#999999")
|
|
50
|
+
|
|
51
|
+
# Build message blocks
|
|
52
|
+
fields: list[dict[str, object]] = [
|
|
53
|
+
{
|
|
54
|
+
"title": "Severity",
|
|
55
|
+
"value": alert.severity.value.upper(),
|
|
56
|
+
"short": True,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"title": "Time",
|
|
60
|
+
"value": alert.timestamp.isoformat() if alert.timestamp else "N/A",
|
|
61
|
+
"short": True,
|
|
62
|
+
},
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
if alert.asset_name:
|
|
66
|
+
fields.append(
|
|
67
|
+
{
|
|
68
|
+
"title": "Asset",
|
|
69
|
+
"value": alert.asset_name,
|
|
70
|
+
"short": True,
|
|
71
|
+
}
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
if alert.run_id:
|
|
75
|
+
fields.append(
|
|
76
|
+
{
|
|
77
|
+
"title": "Run ID",
|
|
78
|
+
"value": alert.run_id[:8],
|
|
79
|
+
"short": True,
|
|
80
|
+
}
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
if alert.error_message:
|
|
84
|
+
fields.append(
|
|
85
|
+
{
|
|
86
|
+
"title": "Error",
|
|
87
|
+
"value": f"```{alert.error_message[:500]}```",
|
|
88
|
+
"short": False,
|
|
89
|
+
}
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Build attachment
|
|
93
|
+
attachment: dict[str, object] = {
|
|
94
|
+
"color": color,
|
|
95
|
+
"title": alert.title,
|
|
96
|
+
"text": alert.message,
|
|
97
|
+
"fields": fields,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
payload = {
|
|
101
|
+
"attachments": [attachment],
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if self.channel:
|
|
105
|
+
payload["channel"] = self.channel
|
|
106
|
+
|
|
107
|
+
return payload
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Hook plugin for alerting on quality and telemetry events."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from phlo.hooks import QualityResultEvent, TelemetryEvent
|
|
8
|
+
from phlo.plugins.base import PluginMetadata
|
|
9
|
+
from phlo.plugins.hooks import HookFilter, HookPlugin, HookRegistration
|
|
10
|
+
|
|
11
|
+
from phlo_alerting.manager import Alert, AlertSeverity, get_alert_manager
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class AlertingHookPlugin(HookPlugin):
|
|
15
|
+
"""Emit alerts based on quality and telemetry events."""
|
|
16
|
+
|
|
17
|
+
@property
|
|
18
|
+
def metadata(self) -> PluginMetadata:
|
|
19
|
+
"""Metadata for the alerting hook plugin."""
|
|
20
|
+
|
|
21
|
+
return PluginMetadata(
|
|
22
|
+
name="alerting",
|
|
23
|
+
version="0.1.0",
|
|
24
|
+
description="Alerting hooks for quality and telemetry events",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
def get_hooks(self) -> list[HookRegistration]:
|
|
28
|
+
"""Register quality and telemetry hook handlers."""
|
|
29
|
+
|
|
30
|
+
return [
|
|
31
|
+
HookRegistration(
|
|
32
|
+
hook_name="alerting_quality",
|
|
33
|
+
handler=self._handle_quality,
|
|
34
|
+
filters=HookFilter(event_types={"quality.result"}),
|
|
35
|
+
),
|
|
36
|
+
HookRegistration(
|
|
37
|
+
hook_name="alerting_telemetry",
|
|
38
|
+
handler=self._handle_telemetry,
|
|
39
|
+
filters=HookFilter(event_types={"telemetry.log", "telemetry.metric"}),
|
|
40
|
+
),
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
def _handle_quality(self, event: Any) -> None:
|
|
44
|
+
"""Send an alert for failed quality checks."""
|
|
45
|
+
|
|
46
|
+
if not isinstance(event, QualityResultEvent):
|
|
47
|
+
return
|
|
48
|
+
if event.passed:
|
|
49
|
+
return
|
|
50
|
+
severity = _map_quality_severity(event.severity)
|
|
51
|
+
message = _format_quality_message(event)
|
|
52
|
+
alert = Alert(
|
|
53
|
+
title=f"Quality check failed: {event.check_name}",
|
|
54
|
+
message=message,
|
|
55
|
+
severity=severity,
|
|
56
|
+
asset_name=event.asset_key,
|
|
57
|
+
)
|
|
58
|
+
get_alert_manager().send(alert)
|
|
59
|
+
|
|
60
|
+
def _handle_telemetry(self, event: Any) -> None:
|
|
61
|
+
"""Send an alert for error-level telemetry events."""
|
|
62
|
+
|
|
63
|
+
if not isinstance(event, TelemetryEvent):
|
|
64
|
+
return
|
|
65
|
+
if not event.level or event.level.lower() not in {"error", "critical"}:
|
|
66
|
+
return
|
|
67
|
+
alert = Alert(
|
|
68
|
+
title=f"Telemetry {event.level} event: {event.name}",
|
|
69
|
+
message=str(event.payload or event.value or ""),
|
|
70
|
+
severity=_map_telemetry_severity(event.level),
|
|
71
|
+
asset_name=event.tags.get("asset"),
|
|
72
|
+
)
|
|
73
|
+
get_alert_manager().send(alert)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _map_quality_severity(severity: str | None) -> AlertSeverity:
|
|
77
|
+
"""Map quality severity strings to alert severities."""
|
|
78
|
+
|
|
79
|
+
if not severity:
|
|
80
|
+
return AlertSeverity.ERROR
|
|
81
|
+
value = severity.upper()
|
|
82
|
+
if value == "WARN":
|
|
83
|
+
return AlertSeverity.WARNING
|
|
84
|
+
if value in {"CRITICAL", "FATAL"}:
|
|
85
|
+
return AlertSeverity.CRITICAL
|
|
86
|
+
return AlertSeverity.ERROR
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _map_telemetry_severity(level: str) -> AlertSeverity:
|
|
90
|
+
"""Map telemetry levels to alert severities."""
|
|
91
|
+
|
|
92
|
+
value = level.lower()
|
|
93
|
+
if value == "critical":
|
|
94
|
+
return AlertSeverity.CRITICAL
|
|
95
|
+
return AlertSeverity.ERROR
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _format_quality_message(event: QualityResultEvent) -> str:
|
|
99
|
+
"""Format a human-readable quality failure message."""
|
|
100
|
+
|
|
101
|
+
parts = [
|
|
102
|
+
f"Asset: {event.asset_key}",
|
|
103
|
+
f"Check: {event.check_name}",
|
|
104
|
+
]
|
|
105
|
+
if event.partition_key:
|
|
106
|
+
parts.append(f"Partition: {event.partition_key}")
|
|
107
|
+
if event.metadata.get("error"):
|
|
108
|
+
parts.append(f"Error: {event.metadata['error']}")
|
|
109
|
+
if event.metadata.get("failure_message"):
|
|
110
|
+
parts.append(f"Details: {event.metadata['failure_message']}")
|
|
111
|
+
return "\n".join(parts)
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Alert manager for sending notifications to multiple destinations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from enum import Enum
|
|
8
|
+
from typing import Optional
|
|
9
|
+
|
|
10
|
+
from phlo.logging import get_logger
|
|
11
|
+
|
|
12
|
+
logger = get_logger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class AlertSeverity(str, Enum):
|
|
16
|
+
"""Alert severity levels."""
|
|
17
|
+
|
|
18
|
+
INFO = "info"
|
|
19
|
+
WARNING = "warning"
|
|
20
|
+
ERROR = "error"
|
|
21
|
+
CRITICAL = "critical"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(slots=True)
|
|
25
|
+
class Alert:
|
|
26
|
+
"""Alert payload."""
|
|
27
|
+
|
|
28
|
+
title: str
|
|
29
|
+
message: str
|
|
30
|
+
severity: AlertSeverity = AlertSeverity.ERROR
|
|
31
|
+
asset_name: Optional[str] = None
|
|
32
|
+
run_id: Optional[str] = None
|
|
33
|
+
error_message: Optional[str] = None
|
|
34
|
+
timestamp: Optional[datetime] = None
|
|
35
|
+
|
|
36
|
+
def __post_init__(self) -> None:
|
|
37
|
+
"""Set default timestamp."""
|
|
38
|
+
if self.timestamp is None:
|
|
39
|
+
self.timestamp = datetime.now(timezone.utc)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class AlertDestination:
|
|
43
|
+
"""Base class for alert destinations."""
|
|
44
|
+
|
|
45
|
+
def send(self, alert: Alert) -> bool:
|
|
46
|
+
"""
|
|
47
|
+
Send an alert.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
alert: Alert to send
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
True if sent successfully, False otherwise
|
|
54
|
+
"""
|
|
55
|
+
raise NotImplementedError
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class AlertManager:
|
|
59
|
+
"""Manages alert destinations and deduplication."""
|
|
60
|
+
|
|
61
|
+
def __init__(self):
|
|
62
|
+
"""Initialize alert manager."""
|
|
63
|
+
self.destinations: dict[str, AlertDestination] = {}
|
|
64
|
+
self._sent_alerts: set[str] = set() # For deduplication
|
|
65
|
+
self._dedup_window_minutes = 60
|
|
66
|
+
|
|
67
|
+
def register_destination(self, name: str, destination: AlertDestination) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Register an alert destination.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
name: Name of the destination
|
|
73
|
+
destination: AlertDestination instance
|
|
74
|
+
"""
|
|
75
|
+
self.destinations[name] = destination
|
|
76
|
+
logger.info(f"Registered alert destination: {name}")
|
|
77
|
+
|
|
78
|
+
def send(self, alert: Alert, destinations: Optional[list[str]] = None) -> bool:
|
|
79
|
+
"""
|
|
80
|
+
Send an alert to registered destinations.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
alert: Alert to send
|
|
84
|
+
destinations: Specific destinations to use (None = all)
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
True if sent to at least one destination successfully
|
|
88
|
+
"""
|
|
89
|
+
# Check for duplicates
|
|
90
|
+
alert_key = self._get_alert_key(alert)
|
|
91
|
+
if self._is_duplicate(alert_key):
|
|
92
|
+
logger.debug(f"Skipping duplicate alert: {alert_key}")
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
# Determine which destinations to use
|
|
96
|
+
targets = destinations or list(self.destinations.keys())
|
|
97
|
+
|
|
98
|
+
# Send to each destination
|
|
99
|
+
sent = False
|
|
100
|
+
for dest_name in targets:
|
|
101
|
+
if dest_name not in self.destinations:
|
|
102
|
+
logger.warning(f"Unknown destination: {dest_name}")
|
|
103
|
+
continue
|
|
104
|
+
|
|
105
|
+
try:
|
|
106
|
+
dest = self.destinations[dest_name]
|
|
107
|
+
if dest.send(alert):
|
|
108
|
+
sent = True
|
|
109
|
+
logger.info(f"Sent alert to {dest_name}: {alert.title}")
|
|
110
|
+
except Exception as e:
|
|
111
|
+
logger.exception(f"Failed to send alert to {dest_name}: {e}")
|
|
112
|
+
|
|
113
|
+
# Mark as sent
|
|
114
|
+
if sent:
|
|
115
|
+
self._sent_alerts.add(alert_key)
|
|
116
|
+
|
|
117
|
+
return sent
|
|
118
|
+
|
|
119
|
+
def _get_alert_key(self, alert: Alert) -> str:
|
|
120
|
+
"""Generate deduplication key for an alert."""
|
|
121
|
+
return f"{alert.asset_name}:{alert.error_message}:{alert.severity.value}"
|
|
122
|
+
|
|
123
|
+
def _is_duplicate(self, key: str) -> bool:
|
|
124
|
+
"""Check if alert is a duplicate."""
|
|
125
|
+
return key in self._sent_alerts
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# Global alert manager instance
|
|
129
|
+
_alert_manager: Optional[AlertManager] = None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def get_alert_manager() -> AlertManager:
|
|
133
|
+
"""Get or create global alert manager."""
|
|
134
|
+
global _alert_manager
|
|
135
|
+
if _alert_manager is None:
|
|
136
|
+
_alert_manager = AlertManager()
|
|
137
|
+
_register_default_destinations(_alert_manager)
|
|
138
|
+
return _alert_manager
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _register_default_destinations(manager: AlertManager) -> None:
|
|
142
|
+
"""Register default alert destinations from config."""
|
|
143
|
+
from phlo_alerting.destinations.email import EmailAlertDestination
|
|
144
|
+
from phlo_alerting.destinations.pagerduty import PagerDutyAlertDestination
|
|
145
|
+
from phlo_alerting.destinations.slack import SlackAlertDestination
|
|
146
|
+
from phlo_alerting.settings import get_settings
|
|
147
|
+
|
|
148
|
+
config = get_settings()
|
|
149
|
+
|
|
150
|
+
# Register Slack if configured
|
|
151
|
+
if config.phlo_alert_slack_webhook:
|
|
152
|
+
try:
|
|
153
|
+
slack = SlackAlertDestination(
|
|
154
|
+
webhook_url=config.phlo_alert_slack_webhook,
|
|
155
|
+
channel=config.phlo_alert_slack_channel,
|
|
156
|
+
)
|
|
157
|
+
manager.register_destination("slack", slack)
|
|
158
|
+
except Exception as e:
|
|
159
|
+
logger.warning(f"Failed to register Slack destination: {e}")
|
|
160
|
+
|
|
161
|
+
# Register PagerDuty if configured
|
|
162
|
+
if config.phlo_alert_pagerduty_key:
|
|
163
|
+
try:
|
|
164
|
+
pagerduty = PagerDutyAlertDestination(integration_key=config.phlo_alert_pagerduty_key)
|
|
165
|
+
manager.register_destination("pagerduty", pagerduty)
|
|
166
|
+
except Exception as e:
|
|
167
|
+
logger.warning(f"Failed to register PagerDuty destination: {e}")
|
|
168
|
+
|
|
169
|
+
# Register Email if configured
|
|
170
|
+
if config.phlo_alert_email_smtp_host:
|
|
171
|
+
try:
|
|
172
|
+
email = EmailAlertDestination(
|
|
173
|
+
smtp_host=config.phlo_alert_email_smtp_host,
|
|
174
|
+
smtp_port=config.phlo_alert_email_smtp_port,
|
|
175
|
+
smtp_user=config.phlo_alert_email_smtp_user,
|
|
176
|
+
smtp_password=config.phlo_alert_email_smtp_password,
|
|
177
|
+
recipients=config.phlo_alert_email_recipients,
|
|
178
|
+
)
|
|
179
|
+
manager.register_destination("email", email)
|
|
180
|
+
except Exception as e:
|
|
181
|
+
logger.warning(f"Failed to register Email destination: {e}")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Alerting settings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from functools import lru_cache
|
|
6
|
+
|
|
7
|
+
from pydantic import Field
|
|
8
|
+
|
|
9
|
+
from phlo.config.base import BaseConfig
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AlertingSettings(BaseConfig):
|
|
13
|
+
"""Alert integration configuration (Slack, PagerDuty, Email)."""
|
|
14
|
+
|
|
15
|
+
phlo_alert_slack_webhook: str | None = Field(
|
|
16
|
+
default=None, description="Slack incoming webhook URL"
|
|
17
|
+
)
|
|
18
|
+
phlo_alert_slack_channel: str | None = Field(
|
|
19
|
+
default=None, description="Default Slack channel for alerts"
|
|
20
|
+
)
|
|
21
|
+
phlo_alert_pagerduty_key: str | None = Field(
|
|
22
|
+
default=None, description="PagerDuty Events API v2 integration key"
|
|
23
|
+
)
|
|
24
|
+
phlo_alert_email_smtp_host: str | None = Field(default=None, description="SMTP server hostname")
|
|
25
|
+
phlo_alert_email_smtp_port: int = Field(default=587, description="SMTP server port")
|
|
26
|
+
phlo_alert_email_smtp_user: str | None = Field(default=None, description="SMTP username")
|
|
27
|
+
phlo_alert_email_smtp_password: str | None = Field(default=None, description="SMTP password")
|
|
28
|
+
phlo_alert_email_recipients: list[str] = Field(
|
|
29
|
+
default_factory=list, description="Email recipients for alerts"
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@lru_cache(maxsize=1)
|
|
34
|
+
def get_settings() -> AlertingSettings:
|
|
35
|
+
return AlertingSettings()
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phlo-alerting
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Alerting destinations and hooks for Phlo
|
|
5
|
+
Author-email: Phlo Team <team@phlo.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/plain
|
|
9
|
+
Requires-Dist: phlo>=0.1.0
|
|
10
|
+
Requires-Dist: click>=8.3.0
|
|
11
|
+
Requires-Dist: rich>=14.2.0
|
|
12
|
+
Requires-Dist: requests>=2.32.5
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
15
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
16
|
+
|
|
17
|
+
Alerting support for Phlo.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/phlo_alerting/__init__.py
|
|
4
|
+
src/phlo_alerting/cli_alerts.py
|
|
5
|
+
src/phlo_alerting/cli_plugin.py
|
|
6
|
+
src/phlo_alerting/hooks_plugin.py
|
|
7
|
+
src/phlo_alerting/manager.py
|
|
8
|
+
src/phlo_alerting/settings.py
|
|
9
|
+
src/phlo_alerting.egg-info/PKG-INFO
|
|
10
|
+
src/phlo_alerting.egg-info/SOURCES.txt
|
|
11
|
+
src/phlo_alerting.egg-info/dependency_links.txt
|
|
12
|
+
src/phlo_alerting.egg-info/entry_points.txt
|
|
13
|
+
src/phlo_alerting.egg-info/requires.txt
|
|
14
|
+
src/phlo_alerting.egg-info/top_level.txt
|
|
15
|
+
src/phlo_alerting/destinations/__init__.py
|
|
16
|
+
src/phlo_alerting/destinations/email.py
|
|
17
|
+
src/phlo_alerting/destinations/pagerduty.py
|
|
18
|
+
src/phlo_alerting/destinations/slack.py
|
|
19
|
+
tests/test_alerting.py
|
|
20
|
+
tests/test_integration_alerting.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
phlo_alerting
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Unit tests for phlo-alerting."""
|
|
2
|
+
|
|
3
|
+
from phlo_alerting.manager import Alert, AlertManager, AlertDestination, AlertSeverity
|
|
4
|
+
from phlo_alerting.hooks_plugin import (
|
|
5
|
+
AlertingHookPlugin,
|
|
6
|
+
_map_quality_severity,
|
|
7
|
+
_map_telemetry_severity,
|
|
8
|
+
)
|
|
9
|
+
from phlo_alerting.settings import AlertingSettings
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class MockDestination(AlertDestination):
|
|
13
|
+
def __init__(self):
|
|
14
|
+
self.alerts: list[Alert] = []
|
|
15
|
+
|
|
16
|
+
def send(self, alert: Alert) -> bool:
|
|
17
|
+
self.alerts.append(alert)
|
|
18
|
+
return True
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_alert_defaults():
|
|
22
|
+
alert = Alert(title="Test", message="hello")
|
|
23
|
+
assert alert.severity == AlertSeverity.ERROR
|
|
24
|
+
assert alert.timestamp is not None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_alert_severity_values():
|
|
28
|
+
assert AlertSeverity.INFO.value == "info"
|
|
29
|
+
assert AlertSeverity.CRITICAL.value == "critical"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_alert_manager_register_and_send():
|
|
33
|
+
manager = AlertManager()
|
|
34
|
+
dest = MockDestination()
|
|
35
|
+
manager.register_destination("mock", dest)
|
|
36
|
+
|
|
37
|
+
alert = Alert(title="T", message="M", asset_name="a1", error_message="e1")
|
|
38
|
+
assert manager.send(alert) is True
|
|
39
|
+
assert len(dest.alerts) == 1
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_alert_manager_deduplication():
|
|
43
|
+
manager = AlertManager()
|
|
44
|
+
dest = MockDestination()
|
|
45
|
+
manager.register_destination("mock", dest)
|
|
46
|
+
|
|
47
|
+
alert = Alert(title="T", message="M", asset_name="a1", error_message="e1")
|
|
48
|
+
manager.send(alert)
|
|
49
|
+
assert manager.send(alert) is False
|
|
50
|
+
assert len(dest.alerts) == 1
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_alert_manager_targeted_destinations():
|
|
54
|
+
manager = AlertManager()
|
|
55
|
+
d1 = MockDestination()
|
|
56
|
+
d2 = MockDestination()
|
|
57
|
+
manager.register_destination("d1", d1)
|
|
58
|
+
manager.register_destination("d2", d2)
|
|
59
|
+
|
|
60
|
+
alert = Alert(title="T", message="M")
|
|
61
|
+
manager.send(alert, destinations=["d1"])
|
|
62
|
+
assert len(d1.alerts) == 1
|
|
63
|
+
assert len(d2.alerts) == 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_map_quality_severity():
|
|
67
|
+
assert _map_quality_severity(None) == AlertSeverity.ERROR
|
|
68
|
+
assert _map_quality_severity("WARN") == AlertSeverity.WARNING
|
|
69
|
+
assert _map_quality_severity("CRITICAL") == AlertSeverity.CRITICAL
|
|
70
|
+
assert _map_quality_severity("FATAL") == AlertSeverity.CRITICAL
|
|
71
|
+
assert _map_quality_severity("error") == AlertSeverity.ERROR
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_map_telemetry_severity():
|
|
75
|
+
assert _map_telemetry_severity("critical") == AlertSeverity.CRITICAL
|
|
76
|
+
assert _map_telemetry_severity("error") == AlertSeverity.ERROR
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_alerting_hooks_plugin_registrations():
|
|
80
|
+
plugin = AlertingHookPlugin()
|
|
81
|
+
hooks = plugin.get_hooks()
|
|
82
|
+
assert len(hooks) == 2
|
|
83
|
+
names = {h.hook_name for h in hooks}
|
|
84
|
+
assert "alerting_quality" in names
|
|
85
|
+
assert "alerting_telemetry" in names
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_alerting_settings_defaults():
|
|
89
|
+
settings = AlertingSettings()
|
|
90
|
+
assert settings.phlo_alert_email_smtp_port == 587
|
|
91
|
+
assert settings.phlo_alert_email_recipients == []
|
|
92
|
+
assert settings.phlo_alert_slack_webhook is None
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Integration tests for phlo-alerting."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
pytestmark = pytest.mark.integration
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_alert_manager_initializes():
|
|
9
|
+
"""Test that AlertManager initializes correctly."""
|
|
10
|
+
from phlo_alerting import AlertManager, get_alert_manager
|
|
11
|
+
|
|
12
|
+
manager = get_alert_manager()
|
|
13
|
+
assert isinstance(manager, AlertManager)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_alert_payload_defaults():
|
|
17
|
+
"""Test that Alert has correct default values."""
|
|
18
|
+
from phlo_alerting import Alert, AlertSeverity
|
|
19
|
+
|
|
20
|
+
alert = Alert(title="Test", message="hello")
|
|
21
|
+
assert alert.severity == AlertSeverity.ERROR
|
|
22
|
+
assert alert.timestamp is not None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_alert_severity_enum():
|
|
26
|
+
"""Test AlertSeverity enum values."""
|
|
27
|
+
from phlo_alerting import AlertSeverity
|
|
28
|
+
|
|
29
|
+
assert AlertSeverity.INFO.value == "info"
|
|
30
|
+
assert AlertSeverity.WARNING.value == "warning"
|
|
31
|
+
assert AlertSeverity.ERROR.value == "error"
|
|
32
|
+
assert AlertSeverity.CRITICAL.value == "critical"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_alert_destination_registration():
|
|
36
|
+
"""Test custom destination registration."""
|
|
37
|
+
from phlo_alerting.manager import AlertManager, AlertDestination, Alert
|
|
38
|
+
|
|
39
|
+
class MockDestination(AlertDestination):
|
|
40
|
+
def __init__(self):
|
|
41
|
+
self.alerts = []
|
|
42
|
+
|
|
43
|
+
def send(self, alert: Alert) -> bool:
|
|
44
|
+
self.alerts.append(alert)
|
|
45
|
+
return True
|
|
46
|
+
|
|
47
|
+
manager = AlertManager()
|
|
48
|
+
mock_dest = MockDestination()
|
|
49
|
+
manager.register_destination("mock", mock_dest)
|
|
50
|
+
|
|
51
|
+
assert "mock" in manager.destinations
|
|
52
|
+
assert manager.destinations["mock"] is mock_dest
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_alert_sending_to_destination():
|
|
56
|
+
"""Test sending alerts to registered destinations."""
|
|
57
|
+
from phlo_alerting.manager import AlertManager, AlertDestination, Alert, AlertSeverity
|
|
58
|
+
|
|
59
|
+
class MockDestination(AlertDestination):
|
|
60
|
+
def __init__(self):
|
|
61
|
+
self.alerts = []
|
|
62
|
+
|
|
63
|
+
def send(self, alert: Alert) -> bool:
|
|
64
|
+
self.alerts.append(alert)
|
|
65
|
+
return True
|
|
66
|
+
|
|
67
|
+
manager = AlertManager()
|
|
68
|
+
mock_dest = MockDestination()
|
|
69
|
+
manager.register_destination("mock", mock_dest)
|
|
70
|
+
|
|
71
|
+
alert = Alert(
|
|
72
|
+
title="Test Alert",
|
|
73
|
+
message="This is a test",
|
|
74
|
+
severity=AlertSeverity.WARNING,
|
|
75
|
+
asset_name="test_asset",
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
result = manager.send(alert)
|
|
79
|
+
|
|
80
|
+
assert result is True
|
|
81
|
+
assert len(mock_dest.alerts) == 1
|
|
82
|
+
assert mock_dest.alerts[0].title == "Test Alert"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_alert_deduplication():
|
|
86
|
+
"""Test that duplicate alerts are not sent."""
|
|
87
|
+
from phlo_alerting.manager import AlertManager, AlertDestination, Alert, AlertSeverity
|
|
88
|
+
|
|
89
|
+
class MockDestination(AlertDestination):
|
|
90
|
+
def __init__(self):
|
|
91
|
+
self.alerts = []
|
|
92
|
+
|
|
93
|
+
def send(self, alert: Alert) -> bool:
|
|
94
|
+
self.alerts.append(alert)
|
|
95
|
+
return True
|
|
96
|
+
|
|
97
|
+
manager = AlertManager()
|
|
98
|
+
mock_dest = MockDestination()
|
|
99
|
+
manager.register_destination("mock", mock_dest)
|
|
100
|
+
|
|
101
|
+
alert = Alert(
|
|
102
|
+
title="Duplicate Test",
|
|
103
|
+
message="Same alert",
|
|
104
|
+
severity=AlertSeverity.ERROR,
|
|
105
|
+
asset_name="dup_asset",
|
|
106
|
+
error_message="same error",
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# First send should succeed
|
|
110
|
+
result1 = manager.send(alert)
|
|
111
|
+
assert result1 is True
|
|
112
|
+
assert len(mock_dest.alerts) == 1
|
|
113
|
+
|
|
114
|
+
# Second send (duplicate) should be skipped
|
|
115
|
+
result2 = manager.send(alert)
|
|
116
|
+
assert result2 is False
|
|
117
|
+
assert len(mock_dest.alerts) == 1 # Still only 1
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_alert_to_specific_destination():
|
|
121
|
+
"""Test sending to specific destinations only."""
|
|
122
|
+
from phlo_alerting.manager import AlertManager, AlertDestination, Alert
|
|
123
|
+
|
|
124
|
+
class MockDestination(AlertDestination):
|
|
125
|
+
def __init__(self, name):
|
|
126
|
+
self.name = name
|
|
127
|
+
self.alerts = []
|
|
128
|
+
|
|
129
|
+
def send(self, alert: Alert) -> bool:
|
|
130
|
+
self.alerts.append(alert)
|
|
131
|
+
return True
|
|
132
|
+
|
|
133
|
+
manager = AlertManager()
|
|
134
|
+
dest1 = MockDestination("dest1")
|
|
135
|
+
dest2 = MockDestination("dest2")
|
|
136
|
+
manager.register_destination("dest1", dest1)
|
|
137
|
+
manager.register_destination("dest2", dest2)
|
|
138
|
+
|
|
139
|
+
alert = Alert(title="Specific Test", message="Only to dest1")
|
|
140
|
+
|
|
141
|
+
# Send only to dest1
|
|
142
|
+
result = manager.send(alert, destinations=["dest1"])
|
|
143
|
+
|
|
144
|
+
assert result is True
|
|
145
|
+
assert len(dest1.alerts) == 1
|
|
146
|
+
assert len(dest2.alerts) == 0
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_alerting_hooks_plugin_exists():
|
|
150
|
+
"""Test that hooks plugin is properly defined."""
|
|
151
|
+
from phlo_alerting.hooks_plugin import AlertingHookPlugin
|
|
152
|
+
|
|
153
|
+
plugin = AlertingHookPlugin()
|
|
154
|
+
assert plugin is not None
|
|
155
|
+
# Check it has required hook method(s)
|
|
156
|
+
assert hasattr(plugin, "get_hooks")
|
|
157
|
+
assert hasattr(plugin, "metadata")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def test_alerting_exports():
|
|
161
|
+
"""Test that phlo-alerting exports required classes."""
|
|
162
|
+
import phlo_alerting
|
|
163
|
+
|
|
164
|
+
assert hasattr(phlo_alerting, "Alert")
|
|
165
|
+
assert hasattr(phlo_alerting, "AlertManager")
|
|
166
|
+
assert hasattr(phlo_alerting, "AlertSeverity")
|
|
167
|
+
assert hasattr(phlo_alerting, "get_alert_manager")
|