nthlayer-workers 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nthlayer_workers/__init__.py +5 -0
- nthlayer_workers/cli.py +234 -0
- nthlayer_workers/correlate/__init__.py +1 -0
- nthlayer_workers/correlate/cli.py +847 -0
- nthlayer_workers/correlate/config.py +111 -0
- nthlayer_workers/correlate/correlation/__init__.py +1 -0
- nthlayer_workers/correlate/correlation/changes.py +87 -0
- nthlayer_workers/correlate/correlation/dedup.py +62 -0
- nthlayer_workers/correlate/correlation/engine.py +244 -0
- nthlayer_workers/correlate/correlation/temporal.py +79 -0
- nthlayer_workers/correlate/correlation/topology.py +104 -0
- nthlayer_workers/correlate/ingestion/__init__.py +1 -0
- nthlayer_workers/correlate/ingestion/protocol.py +10 -0
- nthlayer_workers/correlate/ingestion/severity.py +18 -0
- nthlayer_workers/correlate/ingestion/webhook.py +197 -0
- nthlayer_workers/correlate/notifications.py +85 -0
- nthlayer_workers/correlate/prometheus.py +234 -0
- nthlayer_workers/correlate/reasoning.py +375 -0
- nthlayer_workers/correlate/session.py +189 -0
- nthlayer_workers/correlate/snapshot/__init__.py +1 -0
- nthlayer_workers/correlate/snapshot/generator.py +170 -0
- nthlayer_workers/correlate/snapshot/model.py +177 -0
- nthlayer_workers/correlate/snapshot/token.py +14 -0
- nthlayer_workers/correlate/state.py +88 -0
- nthlayer_workers/correlate/store/__init__.py +5 -0
- nthlayer_workers/correlate/store/protocol.py +48 -0
- nthlayer_workers/correlate/store/sqlite.py +443 -0
- nthlayer_workers/correlate/summary.py +180 -0
- nthlayer_workers/correlate/traces/__init__.py +1 -0
- nthlayer_workers/correlate/traces/protocol.py +120 -0
- nthlayer_workers/correlate/traces/tempo.py +667 -0
- nthlayer_workers/correlate/traces/topology.py +39 -0
- nthlayer_workers/correlate/types.py +77 -0
- nthlayer_workers/correlate/worker.py +630 -0
- nthlayer_workers/learn/__init__.py +5 -0
- nthlayer_workers/learn/__main__.py +5 -0
- nthlayer_workers/learn/cli.py +164 -0
- nthlayer_workers/learn/retrospective.py +381 -0
- nthlayer_workers/learn/trends.py +102 -0
- nthlayer_workers/learn/worker.py +366 -0
- nthlayer_workers/measure/__init__.py +3 -0
- nthlayer_workers/measure/__main__.py +5 -0
- nthlayer_workers/measure/_parsing.py +15 -0
- nthlayer_workers/measure/adapters/__init__.py +0 -0
- nthlayer_workers/measure/adapters/_util.py +24 -0
- nthlayer_workers/measure/adapters/devin.py +119 -0
- nthlayer_workers/measure/adapters/gastown.py +88 -0
- nthlayer_workers/measure/adapters/prometheus.py +277 -0
- nthlayer_workers/measure/adapters/protocol.py +20 -0
- nthlayer_workers/measure/adapters/webhook.py +161 -0
- nthlayer_workers/measure/api/__init__.py +0 -0
- nthlayer_workers/measure/api/normalise.py +50 -0
- nthlayer_workers/measure/api/queue.py +243 -0
- nthlayer_workers/measure/api/response.py +51 -0
- nthlayer_workers/measure/api/server.py +504 -0
- nthlayer_workers/measure/calibration/__init__.py +0 -0
- nthlayer_workers/measure/calibration/loop.py +62 -0
- nthlayer_workers/measure/calibration/slos.py +212 -0
- nthlayer_workers/measure/calibration/verdict_calibration.py +31 -0
- nthlayer_workers/measure/cli.py +753 -0
- nthlayer_workers/measure/config.py +191 -0
- nthlayer_workers/measure/detection/__init__.py +6 -0
- nthlayer_workers/measure/detection/detector.py +82 -0
- nthlayer_workers/measure/detection/protocol.py +29 -0
- nthlayer_workers/measure/governance/__init__.py +0 -0
- nthlayer_workers/measure/governance/engine.py +163 -0
- nthlayer_workers/measure/manifest.py +77 -0
- nthlayer_workers/measure/notifications.py +53 -0
- nthlayer_workers/measure/pipeline/__init__.py +0 -0
- nthlayer_workers/measure/pipeline/evaluator.py +155 -0
- nthlayer_workers/measure/pipeline/router.py +160 -0
- nthlayer_workers/measure/store/__init__.py +0 -0
- nthlayer_workers/measure/store/protocol.py +38 -0
- nthlayer_workers/measure/store/sqlite.py +276 -0
- nthlayer_workers/measure/telemetry.py +116 -0
- nthlayer_workers/measure/tiering/__init__.py +0 -0
- nthlayer_workers/measure/tiering/classifier.py +58 -0
- nthlayer_workers/measure/tiering/promotion.py +118 -0
- nthlayer_workers/measure/trends/__init__.py +0 -0
- nthlayer_workers/measure/trends/tracker.py +72 -0
- nthlayer_workers/measure/types.py +75 -0
- nthlayer_workers/measure/worker.py +439 -0
- nthlayer_workers/observe/__init__.py +25 -0
- nthlayer_workers/observe/__main__.py +5 -0
- nthlayer_workers/observe/api/__init__.py +1 -0
- nthlayer_workers/observe/assessment.py +95 -0
- nthlayer_workers/observe/cli.py +737 -0
- nthlayer_workers/observe/config.py +11 -0
- nthlayer_workers/observe/db/__init__.py +1 -0
- nthlayer_workers/observe/decision_records.py +220 -0
- nthlayer_workers/observe/dependencies/__init__.py +18 -0
- nthlayer_workers/observe/dependencies/discovery.py +294 -0
- nthlayer_workers/observe/dependencies/providers/__init__.py +48 -0
- nthlayer_workers/observe/dependencies/providers/backstage.py +467 -0
- nthlayer_workers/observe/dependencies/providers/base.py +76 -0
- nthlayer_workers/observe/dependencies/providers/consul.py +518 -0
- nthlayer_workers/observe/dependencies/providers/etcd.py +360 -0
- nthlayer_workers/observe/dependencies/providers/kubernetes.py +682 -0
- nthlayer_workers/observe/dependencies/providers/prometheus.py +368 -0
- nthlayer_workers/observe/dependencies/providers/zookeeper.py +399 -0
- nthlayer_workers/observe/deployments/__init__.py +1 -0
- nthlayer_workers/observe/discovery/__init__.py +14 -0
- nthlayer_workers/observe/discovery/classifier.py +66 -0
- nthlayer_workers/observe/discovery/client.py +189 -0
- nthlayer_workers/observe/discovery/models.py +53 -0
- nthlayer_workers/observe/drift/__init__.py +26 -0
- nthlayer_workers/observe/drift/analyzer.py +383 -0
- nthlayer_workers/observe/drift/models.py +174 -0
- nthlayer_workers/observe/drift/patterns.py +88 -0
- nthlayer_workers/observe/explanation.py +118 -0
- nthlayer_workers/observe/gate/__init__.py +39 -0
- nthlayer_workers/observe/gate/conditions.py +92 -0
- nthlayer_workers/observe/gate/correlator.py +154 -0
- nthlayer_workers/observe/gate/evaluator.py +192 -0
- nthlayer_workers/observe/gate/policies.py +226 -0
- nthlayer_workers/observe/gate_adapter.py +40 -0
- nthlayer_workers/observe/incident.py +36 -0
- nthlayer_workers/observe/portfolio/__init__.py +17 -0
- nthlayer_workers/observe/portfolio/aggregator.py +168 -0
- nthlayer_workers/observe/portfolio/scorer.py +13 -0
- nthlayer_workers/observe/slo/__init__.py +19 -0
- nthlayer_workers/observe/slo/collector.py +235 -0
- nthlayer_workers/observe/slo/spec_loader.py +40 -0
- nthlayer_workers/observe/sqlite_store.py +152 -0
- nthlayer_workers/observe/store.py +92 -0
- nthlayer_workers/observe/verification/__init__.py +22 -0
- nthlayer_workers/observe/verification/exporter_guidance.py +146 -0
- nthlayer_workers/observe/verification/extractor.py +127 -0
- nthlayer_workers/observe/verification/models.py +101 -0
- nthlayer_workers/observe/verification/verifier.py +111 -0
- nthlayer_workers/observe/worker.py +332 -0
- nthlayer_workers/respond/__init__.py +2 -0
- nthlayer_workers/respond/__main__.py +4 -0
- nthlayer_workers/respond/agents/__init__.py +0 -0
- nthlayer_workers/respond/agents/base.py +556 -0
- nthlayer_workers/respond/agents/communication.py +115 -0
- nthlayer_workers/respond/agents/investigation.py +124 -0
- nthlayer_workers/respond/agents/remediation.py +219 -0
- nthlayer_workers/respond/agents/triage.py +132 -0
- nthlayer_workers/respond/cli.py +772 -0
- nthlayer_workers/respond/config.py +135 -0
- nthlayer_workers/respond/context_store.py +256 -0
- nthlayer_workers/respond/coordinator.py +487 -0
- nthlayer_workers/respond/metrics.py +104 -0
- nthlayer_workers/respond/notification_backends/__init__.py +1 -0
- nthlayer_workers/respond/notification_backends/ntfy_backend.py +158 -0
- nthlayer_workers/respond/notification_backends/protocol.py +59 -0
- nthlayer_workers/respond/notification_backends/slack_backend.py +203 -0
- nthlayer_workers/respond/notification_backends/stdout_backend.py +56 -0
- nthlayer_workers/respond/notifications.py +247 -0
- nthlayer_workers/respond/oncall/__init__.py +1 -0
- nthlayer_workers/respond/oncall/escalation.py +103 -0
- nthlayer_workers/respond/oncall/runner.py +193 -0
- nthlayer_workers/respond/oncall/schedule.py +243 -0
- nthlayer_workers/respond/safe_actions/__init__.py +0 -0
- nthlayer_workers/respond/safe_actions/actions.py +139 -0
- nthlayer_workers/respond/safe_actions/registry.py +171 -0
- nthlayer_workers/respond/safe_actions/webhook.py +194 -0
- nthlayer_workers/respond/server.py +357 -0
- nthlayer_workers/respond/sre/__init__.py +1 -0
- nthlayer_workers/respond/sre/brief.py +175 -0
- nthlayer_workers/respond/sre/delegation.py +101 -0
- nthlayer_workers/respond/sre/post_incident.py +146 -0
- nthlayer_workers/respond/sre/shift_report.py +129 -0
- nthlayer_workers/respond/sre/suppression.py +91 -0
- nthlayer_workers/respond/types.py +109 -0
- nthlayer_workers/respond/verdict_submission.py +56 -0
- nthlayer_workers/respond/worker.py +533 -0
- nthlayer_workers/respond/worker_helpers.py +140 -0
- nthlayer_workers/runner.py +198 -0
- nthlayer_workers-1.0.0.dist-info/METADATA +19 -0
- nthlayer_workers-1.0.0.dist-info/RECORD +175 -0
- nthlayer_workers-1.0.0.dist-info/WHEEL +5 -0
- nthlayer_workers-1.0.0.dist-info/entry_points.txt +2 -0
- nthlayer_workers-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""ntfy notification backend — DND-override push notifications."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import structlog
|
|
10
|
+
|
|
11
|
+
from nthlayer_workers.respond.notification_backends.protocol import (
|
|
12
|
+
NotificationPayload,
|
|
13
|
+
NotificationResult,
|
|
14
|
+
)
|
|
15
|
+
from nthlayer_workers.respond.oncall.schedule import RosterMember
|
|
16
|
+
|
|
17
|
+
logger = structlog.get_logger(__name__)
|
|
18
|
+
|
|
19
|
+
# Map incident severity to ntfy priority.
|
|
20
|
+
# P1 = max (overrides DND), P2 = urgent, P3 = high, P4 = default.
|
|
21
|
+
# Unknown severities default to "high" / "warning" (conservative).
|
|
22
|
+
PRIORITY_MAP = {1: "max", 2: "urgent", 3: "high", 4: "default"}
|
|
23
|
+
_DEFAULT_PRIORITY = "high"
|
|
24
|
+
|
|
25
|
+
# ntfy tags (emoji) by severity
|
|
26
|
+
TAGS_MAP = {
|
|
27
|
+
1: "rotating_light,fire",
|
|
28
|
+
2: "rotating_light,warning",
|
|
29
|
+
3: "warning",
|
|
30
|
+
4: "information_source",
|
|
31
|
+
}
|
|
32
|
+
_DEFAULT_TAGS = "warning"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class NtfyNotificationBackend:
|
|
36
|
+
"""ntfy push notification delivery.
|
|
37
|
+
|
|
38
|
+
Sends high-priority notifications that override Do Not Disturb.
|
|
39
|
+
Each roster member has a personal ntfy topic. The ntfy server
|
|
40
|
+
can be self-hosted or use ntfy.sh.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
server_url: str | None = None,
|
|
46
|
+
client: Any | None = None,
|
|
47
|
+
webhook_base_url: str | None = None,
|
|
48
|
+
timeout: float = 10.0,
|
|
49
|
+
) -> None:
|
|
50
|
+
self.server_url = (
|
|
51
|
+
server_url or os.environ.get("NTFY_SERVER_URL", "https://ntfy.sh")
|
|
52
|
+
)
|
|
53
|
+
self._webhook_base_url = webhook_base_url or os.environ.get(
|
|
54
|
+
"NTHLAYER_WEBHOOK_URL", "http://localhost:8090"
|
|
55
|
+
)
|
|
56
|
+
if client is not None:
|
|
57
|
+
self._client = client
|
|
58
|
+
self._owns_client = False
|
|
59
|
+
else:
|
|
60
|
+
import httpx
|
|
61
|
+
|
|
62
|
+
headers = {}
|
|
63
|
+
auth_token = os.environ.get("NTFY_AUTH_TOKEN")
|
|
64
|
+
if auth_token:
|
|
65
|
+
headers["Authorization"] = f"Bearer {auth_token}"
|
|
66
|
+
self._client = httpx.AsyncClient(headers=headers, timeout=timeout)
|
|
67
|
+
self._owns_client = True
|
|
68
|
+
|
|
69
|
+
async def close(self) -> None:
|
|
70
|
+
"""Release the httpx client if we own it."""
|
|
71
|
+
if self._owns_client:
|
|
72
|
+
await self._client.aclose()
|
|
73
|
+
|
|
74
|
+
async def send(
|
|
75
|
+
self, recipient: RosterMember, payload: NotificationPayload
|
|
76
|
+
) -> NotificationResult:
|
|
77
|
+
if not recipient.ntfy_topic: # None or empty string
|
|
78
|
+
return NotificationResult(
|
|
79
|
+
delivered=False,
|
|
80
|
+
channel="ntfy",
|
|
81
|
+
recipient=recipient.name,
|
|
82
|
+
timestamp=datetime.now(timezone.utc),
|
|
83
|
+
message_id=None,
|
|
84
|
+
error="No ntfy_topic configured for this user",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
priority = PRIORITY_MAP.get(payload.severity, _DEFAULT_PRIORITY)
|
|
88
|
+
title = f"{payload.incident_id}: {payload.title}"
|
|
89
|
+
body = payload.summary
|
|
90
|
+
if payload.root_cause:
|
|
91
|
+
body += f"\nRoot cause: {payload.root_cause}"
|
|
92
|
+
|
|
93
|
+
headers: dict[str, str] = {
|
|
94
|
+
"Title": title,
|
|
95
|
+
"Priority": priority,
|
|
96
|
+
"Tags": TAGS_MAP.get(payload.severity, _DEFAULT_TAGS),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if payload.actions_url:
|
|
100
|
+
headers["Click"] = payload.actions_url
|
|
101
|
+
|
|
102
|
+
if payload.requires_ack:
|
|
103
|
+
ack_url = (
|
|
104
|
+
f"{self._webhook_base_url}/api/v1/incidents/"
|
|
105
|
+
f"{payload.incident_id}/ack"
|
|
106
|
+
)
|
|
107
|
+
headers["Actions"] = (
|
|
108
|
+
f"http, Acknowledge, {ack_url}, method=POST, clear=true"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
response = await self._client.post(
|
|
113
|
+
f"{self.server_url}/{recipient.ntfy_topic}",
|
|
114
|
+
content=body,
|
|
115
|
+
headers=headers,
|
|
116
|
+
)
|
|
117
|
+
response.raise_for_status()
|
|
118
|
+
try:
|
|
119
|
+
data = response.json()
|
|
120
|
+
except Exception:
|
|
121
|
+
data = {}
|
|
122
|
+
|
|
123
|
+
logger.debug(
|
|
124
|
+
"ntfy_sent",
|
|
125
|
+
recipient=recipient.name,
|
|
126
|
+
topic=recipient.ntfy_topic,
|
|
127
|
+
priority=priority,
|
|
128
|
+
)
|
|
129
|
+
return NotificationResult(
|
|
130
|
+
delivered=True,
|
|
131
|
+
channel="ntfy",
|
|
132
|
+
recipient=recipient.name,
|
|
133
|
+
timestamp=datetime.now(timezone.utc),
|
|
134
|
+
message_id=data.get("id"),
|
|
135
|
+
error=None,
|
|
136
|
+
)
|
|
137
|
+
except Exception as exc:
|
|
138
|
+
logger.warning(
|
|
139
|
+
"ntfy_send_failed",
|
|
140
|
+
recipient=recipient.name,
|
|
141
|
+
error=str(exc),
|
|
142
|
+
)
|
|
143
|
+
return NotificationResult(
|
|
144
|
+
delivered=False,
|
|
145
|
+
channel="ntfy",
|
|
146
|
+
recipient=recipient.name,
|
|
147
|
+
timestamp=datetime.now(timezone.utc),
|
|
148
|
+
message_id=None,
|
|
149
|
+
error=str(exc),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
async def health_check(self) -> bool:
|
|
153
|
+
"""Check if the ntfy server is reachable."""
|
|
154
|
+
try:
|
|
155
|
+
response = await self._client.get(f"{self.server_url}/v1/health")
|
|
156
|
+
return response.status_code == 200
|
|
157
|
+
except Exception:
|
|
158
|
+
return False
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Notification backend protocol and shared dataclasses.
|
|
2
|
+
|
|
3
|
+
Every notification channel (Slack, ntfy, Twilio, PagerDuty, stdout)
|
|
4
|
+
implements the ``NotificationBackend`` protocol. The escalation engine
|
|
5
|
+
dispatches to backends without knowing delivery details.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from nthlayer_workers.respond.oncall.schedule import RosterMember
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class NotificationPayload:
|
|
20
|
+
"""What to tell the human."""
|
|
21
|
+
|
|
22
|
+
incident_id: str
|
|
23
|
+
severity: int # 1 = P1 (critical), 2 = P2 (major), 3 = P3 (minor), 4 = P4 (info)
|
|
24
|
+
title: str
|
|
25
|
+
summary: str
|
|
26
|
+
root_cause: str | None
|
|
27
|
+
blast_radius: list[str]
|
|
28
|
+
actions_url: str | None
|
|
29
|
+
escalation_step: int
|
|
30
|
+
requires_ack: bool
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class NotificationResult:
|
|
35
|
+
"""What happened when we tried to notify."""
|
|
36
|
+
|
|
37
|
+
delivered: bool
|
|
38
|
+
channel: str # "slack_dm" | "ntfy" | "phone" | "pagerduty" | "stdout"
|
|
39
|
+
recipient: str
|
|
40
|
+
timestamp: datetime
|
|
41
|
+
message_id: str | None
|
|
42
|
+
error: str | None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@runtime_checkable
|
|
46
|
+
class NotificationBackend(Protocol):
|
|
47
|
+
"""Protocol for all notification delivery mechanisms.
|
|
48
|
+
|
|
49
|
+
Each backend handles one delivery channel. Adding a new channel
|
|
50
|
+
is one file implementing this protocol.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
async def send(
|
|
54
|
+
self,
|
|
55
|
+
recipient: RosterMember,
|
|
56
|
+
payload: NotificationPayload,
|
|
57
|
+
) -> NotificationResult: ...
|
|
58
|
+
|
|
59
|
+
async def health_check(self) -> bool: ...
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Slack notification backend — DM and channel notifications with Block Kit."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import structlog
|
|
9
|
+
|
|
10
|
+
from nthlayer_workers.respond.notification_backends.protocol import (
|
|
11
|
+
NotificationPayload,
|
|
12
|
+
NotificationResult,
|
|
13
|
+
)
|
|
14
|
+
from nthlayer_workers.respond.oncall.schedule import RosterMember
|
|
15
|
+
|
|
16
|
+
logger = structlog.get_logger(__name__)
|
|
17
|
+
|
|
18
|
+
# red_circle, orange_circle, yellow_circle, blue_circle
|
|
19
|
+
SEVERITY_EMOJI = {1: "\U0001f534", 2: "\U0001f7e0", 3: "\U0001f7e1", 4: "\U0001f535"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SlackNotificationBackend:
|
|
23
|
+
"""Slack notification delivery via Web API.
|
|
24
|
+
|
|
25
|
+
Two modes:
|
|
26
|
+
- ``send()``: DM to a specific user via their ``slack_id``
|
|
27
|
+
- ``send_to_channel()``: post to a Slack channel with @here
|
|
28
|
+
|
|
29
|
+
Messages include interactive Acknowledge/Escalate buttons via
|
|
30
|
+
Slack Block Kit when ``payload.requires_ack`` is True.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, client: Any) -> None: # Any = SlackWebClient or compatible
|
|
34
|
+
self._client = client
|
|
35
|
+
|
|
36
|
+
async def send(
|
|
37
|
+
self, recipient: RosterMember, payload: NotificationPayload
|
|
38
|
+
) -> NotificationResult:
|
|
39
|
+
"""Send a DM to the recipient."""
|
|
40
|
+
blocks = _build_incident_blocks(payload)
|
|
41
|
+
fallback = f"{SEVERITY_EMOJI.get(payload.severity, '')} {payload.title}"
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
message_ts = await self._client.post_message(
|
|
45
|
+
channel=recipient.slack_id,
|
|
46
|
+
blocks=blocks,
|
|
47
|
+
text=fallback,
|
|
48
|
+
)
|
|
49
|
+
logger.debug(
|
|
50
|
+
"slack_dm_sent",
|
|
51
|
+
recipient=recipient.name,
|
|
52
|
+
incident_id=payload.incident_id,
|
|
53
|
+
)
|
|
54
|
+
return NotificationResult(
|
|
55
|
+
delivered=True,
|
|
56
|
+
channel="slack_dm",
|
|
57
|
+
recipient=recipient.name,
|
|
58
|
+
timestamp=datetime.now(timezone.utc),
|
|
59
|
+
message_id=message_ts,
|
|
60
|
+
error=None,
|
|
61
|
+
)
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
logger.warning(
|
|
64
|
+
"slack_dm_failed",
|
|
65
|
+
recipient=recipient.name,
|
|
66
|
+
error=str(exc),
|
|
67
|
+
)
|
|
68
|
+
return NotificationResult(
|
|
69
|
+
delivered=False,
|
|
70
|
+
channel="slack_dm",
|
|
71
|
+
recipient=recipient.name,
|
|
72
|
+
timestamp=datetime.now(timezone.utc),
|
|
73
|
+
message_id=None,
|
|
74
|
+
error=str(exc),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
async def send_to_channel(
|
|
78
|
+
self, channel: str, payload: NotificationPayload
|
|
79
|
+
) -> NotificationResult:
|
|
80
|
+
"""Post to a Slack channel with @here."""
|
|
81
|
+
blocks = _build_incident_blocks(payload, include_at_here=True)
|
|
82
|
+
fallback = f"<!here> {SEVERITY_EMOJI.get(payload.severity, '')} {payload.title}"
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
message_ts = await self._client.post_message(
|
|
86
|
+
channel=channel,
|
|
87
|
+
blocks=blocks,
|
|
88
|
+
text=fallback,
|
|
89
|
+
)
|
|
90
|
+
logger.debug(
|
|
91
|
+
"slack_channel_sent",
|
|
92
|
+
channel=channel,
|
|
93
|
+
incident_id=payload.incident_id,
|
|
94
|
+
)
|
|
95
|
+
return NotificationResult(
|
|
96
|
+
delivered=True,
|
|
97
|
+
channel="slack_channel",
|
|
98
|
+
recipient=channel,
|
|
99
|
+
timestamp=datetime.now(timezone.utc),
|
|
100
|
+
message_id=message_ts,
|
|
101
|
+
error=None,
|
|
102
|
+
)
|
|
103
|
+
except Exception as exc:
|
|
104
|
+
logger.warning(
|
|
105
|
+
"slack_channel_failed",
|
|
106
|
+
channel=channel,
|
|
107
|
+
error=str(exc),
|
|
108
|
+
)
|
|
109
|
+
return NotificationResult(
|
|
110
|
+
delivered=False,
|
|
111
|
+
channel="slack_channel",
|
|
112
|
+
recipient=channel,
|
|
113
|
+
timestamp=datetime.now(timezone.utc),
|
|
114
|
+
message_id=None,
|
|
115
|
+
error=str(exc),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
async def health_check(self) -> bool:
|
|
119
|
+
"""Check if the Slack client is usable."""
|
|
120
|
+
return self._client is not None and bool(
|
|
121
|
+
getattr(self._client, "bot_token", True)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _build_incident_blocks(
|
|
126
|
+
payload: NotificationPayload, *, include_at_here: bool = False
|
|
127
|
+
) -> list[dict[str, Any]]:
|
|
128
|
+
"""Build Slack Block Kit blocks for incident notification."""
|
|
129
|
+
emoji = SEVERITY_EMOJI.get(payload.severity, "")
|
|
130
|
+
|
|
131
|
+
header_text = f"{emoji} {payload.incident_id}: {payload.title}"[:150]
|
|
132
|
+
|
|
133
|
+
blocks: list[dict[str, Any]] = [
|
|
134
|
+
{
|
|
135
|
+
"type": "header",
|
|
136
|
+
"text": {
|
|
137
|
+
"type": "plain_text",
|
|
138
|
+
"text": header_text,
|
|
139
|
+
},
|
|
140
|
+
},
|
|
141
|
+
]
|
|
142
|
+
|
|
143
|
+
if include_at_here:
|
|
144
|
+
blocks.append(
|
|
145
|
+
{
|
|
146
|
+
"type": "section",
|
|
147
|
+
"text": {"type": "mrkdwn", "text": "<!here>"},
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
blocks.append(
|
|
152
|
+
{
|
|
153
|
+
"type": "section",
|
|
154
|
+
"text": {
|
|
155
|
+
"type": "mrkdwn",
|
|
156
|
+
"text": payload.summary,
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
if payload.root_cause:
|
|
162
|
+
blocks.append(
|
|
163
|
+
{
|
|
164
|
+
"type": "section",
|
|
165
|
+
"text": {
|
|
166
|
+
"type": "mrkdwn",
|
|
167
|
+
"text": f"*Root cause:* {payload.root_cause}",
|
|
168
|
+
},
|
|
169
|
+
}
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
if payload.blast_radius:
|
|
173
|
+
services = ", ".join(f"`{s}`" for s in payload.blast_radius)
|
|
174
|
+
blocks.append(
|
|
175
|
+
{
|
|
176
|
+
"type": "section",
|
|
177
|
+
"text": {
|
|
178
|
+
"type": "mrkdwn",
|
|
179
|
+
"text": f"*Blast radius:* {services}",
|
|
180
|
+
},
|
|
181
|
+
}
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
if payload.requires_ack:
|
|
185
|
+
buttons: list[dict[str, Any]] = [
|
|
186
|
+
{
|
|
187
|
+
"type": "button",
|
|
188
|
+
"text": {"type": "plain_text", "text": "Acknowledge"},
|
|
189
|
+
"style": "primary",
|
|
190
|
+
"action_id": "incident_ack",
|
|
191
|
+
"value": payload.incident_id,
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
"type": "button",
|
|
195
|
+
"text": {"type": "plain_text", "text": "Escalate"},
|
|
196
|
+
"style": "danger",
|
|
197
|
+
"action_id": "incident_escalate",
|
|
198
|
+
"value": payload.incident_id,
|
|
199
|
+
},
|
|
200
|
+
]
|
|
201
|
+
blocks.append({"type": "actions", "elements": buttons})
|
|
202
|
+
|
|
203
|
+
return blocks
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Stdout notification backend — for testing and local development."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
|
|
7
|
+
import structlog
|
|
8
|
+
|
|
9
|
+
from nthlayer_workers.respond.notification_backends.protocol import (
|
|
10
|
+
NotificationPayload,
|
|
11
|
+
NotificationResult,
|
|
12
|
+
)
|
|
13
|
+
from nthlayer_workers.respond.oncall.schedule import RosterMember
|
|
14
|
+
|
|
15
|
+
logger = structlog.get_logger(__name__)
|
|
16
|
+
|
|
17
|
+
SEVERITY_LABEL = {1: "P1 CRITICAL", 2: "P2 MAJOR", 3: "P3 MINOR", 4: "P4 INFO"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class StdoutNotificationBackend:
|
|
21
|
+
"""Print notifications to stdout. For testing and local development."""
|
|
22
|
+
|
|
23
|
+
async def send(
|
|
24
|
+
self, recipient: RosterMember, payload: NotificationPayload
|
|
25
|
+
) -> NotificationResult:
|
|
26
|
+
label = SEVERITY_LABEL.get(payload.severity, f"P{payload.severity}")
|
|
27
|
+
lines = [
|
|
28
|
+
f"{'=' * 60}",
|
|
29
|
+
f"NOTIFICATION -> {recipient.name}",
|
|
30
|
+
f" Incident: {payload.incident_id}",
|
|
31
|
+
f" Severity: {label}",
|
|
32
|
+
f" Title: {payload.title}",
|
|
33
|
+
f" Summary: {payload.summary}",
|
|
34
|
+
]
|
|
35
|
+
if payload.root_cause:
|
|
36
|
+
lines.append(f" Root cause: {payload.root_cause}")
|
|
37
|
+
if payload.blast_radius:
|
|
38
|
+
lines.append(f" Blast radius: {', '.join(payload.blast_radius)}")
|
|
39
|
+
lines.append(f" Escalation step: {payload.escalation_step}")
|
|
40
|
+
if payload.requires_ack:
|
|
41
|
+
lines.append(" Requires acknowledgment: yes")
|
|
42
|
+
lines.append(f"{'=' * 60}")
|
|
43
|
+
|
|
44
|
+
print("\n".join(lines)) # noqa: T201 — CLI entrypoint output
|
|
45
|
+
|
|
46
|
+
return NotificationResult(
|
|
47
|
+
delivered=True,
|
|
48
|
+
channel="stdout",
|
|
49
|
+
recipient=recipient.name,
|
|
50
|
+
timestamp=datetime.now(timezone.utc),
|
|
51
|
+
message_id=None,
|
|
52
|
+
error=None,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
async def health_check(self) -> bool:
|
|
56
|
+
return True
|