truthound-dashboard 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- truthound_dashboard/api/alerts.py +75 -86
- truthound_dashboard/api/anomaly.py +7 -13
- truthound_dashboard/api/cross_alerts.py +38 -52
- truthound_dashboard/api/drift.py +49 -59
- truthound_dashboard/api/drift_monitor.py +234 -79
- truthound_dashboard/api/enterprise_sampling.py +498 -0
- truthound_dashboard/api/history.py +57 -5
- truthound_dashboard/api/lineage.py +3 -48
- truthound_dashboard/api/maintenance.py +104 -49
- truthound_dashboard/api/mask.py +1 -2
- truthound_dashboard/api/middleware.py +2 -1
- truthound_dashboard/api/model_monitoring.py +435 -311
- truthound_dashboard/api/notifications.py +227 -191
- truthound_dashboard/api/notifications_advanced.py +21 -20
- truthound_dashboard/api/observability.py +586 -0
- truthound_dashboard/api/plugins.py +2 -433
- truthound_dashboard/api/profile.py +199 -37
- truthound_dashboard/api/quality_reporter.py +701 -0
- truthound_dashboard/api/reports.py +7 -16
- truthound_dashboard/api/router.py +66 -0
- truthound_dashboard/api/rule_suggestions.py +5 -5
- truthound_dashboard/api/scan.py +17 -19
- truthound_dashboard/api/schedules.py +85 -50
- truthound_dashboard/api/schema_evolution.py +6 -6
- truthound_dashboard/api/schema_watcher.py +667 -0
- truthound_dashboard/api/sources.py +98 -27
- truthound_dashboard/api/tiering.py +1323 -0
- truthound_dashboard/api/triggers.py +14 -11
- truthound_dashboard/api/validations.py +12 -11
- truthound_dashboard/api/versioning.py +1 -6
- truthound_dashboard/core/__init__.py +129 -3
- truthound_dashboard/core/actions/__init__.py +62 -0
- truthound_dashboard/core/actions/custom.py +426 -0
- truthound_dashboard/core/actions/notifications.py +910 -0
- truthound_dashboard/core/actions/storage.py +472 -0
- truthound_dashboard/core/actions/webhook.py +281 -0
- truthound_dashboard/core/anomaly.py +262 -67
- truthound_dashboard/core/anomaly_explainer.py +4 -3
- truthound_dashboard/core/backends/__init__.py +67 -0
- truthound_dashboard/core/backends/base.py +299 -0
- truthound_dashboard/core/backends/errors.py +191 -0
- truthound_dashboard/core/backends/factory.py +423 -0
- truthound_dashboard/core/backends/mock_backend.py +451 -0
- truthound_dashboard/core/backends/truthound_backend.py +718 -0
- truthound_dashboard/core/checkpoint/__init__.py +87 -0
- truthound_dashboard/core/checkpoint/adapters.py +814 -0
- truthound_dashboard/core/checkpoint/checkpoint.py +491 -0
- truthound_dashboard/core/checkpoint/runner.py +270 -0
- truthound_dashboard/core/connections.py +437 -10
- truthound_dashboard/core/converters/__init__.py +14 -0
- truthound_dashboard/core/converters/truthound.py +620 -0
- truthound_dashboard/core/cross_alerts.py +540 -320
- truthound_dashboard/core/datasource_factory.py +1672 -0
- truthound_dashboard/core/drift_monitor.py +216 -20
- truthound_dashboard/core/enterprise_sampling.py +1291 -0
- truthound_dashboard/core/interfaces/__init__.py +225 -0
- truthound_dashboard/core/interfaces/actions.py +652 -0
- truthound_dashboard/core/interfaces/base.py +247 -0
- truthound_dashboard/core/interfaces/checkpoint.py +676 -0
- truthound_dashboard/core/interfaces/protocols.py +664 -0
- truthound_dashboard/core/interfaces/reporters.py +650 -0
- truthound_dashboard/core/interfaces/routing.py +646 -0
- truthound_dashboard/core/interfaces/triggers.py +619 -0
- truthound_dashboard/core/lineage.py +407 -71
- truthound_dashboard/core/model_monitoring.py +431 -3
- truthound_dashboard/core/notifications/base.py +4 -0
- truthound_dashboard/core/notifications/channels.py +501 -1203
- truthound_dashboard/core/notifications/deduplication/__init__.py +81 -115
- truthound_dashboard/core/notifications/deduplication/service.py +131 -348
- truthound_dashboard/core/notifications/dispatcher.py +202 -11
- truthound_dashboard/core/notifications/escalation/__init__.py +119 -106
- truthound_dashboard/core/notifications/escalation/engine.py +168 -358
- truthound_dashboard/core/notifications/routing/__init__.py +88 -128
- truthound_dashboard/core/notifications/routing/engine.py +90 -317
- truthound_dashboard/core/notifications/stats_aggregator.py +246 -1
- truthound_dashboard/core/notifications/throttling/__init__.py +67 -50
- truthound_dashboard/core/notifications/throttling/builder.py +117 -255
- truthound_dashboard/core/notifications/truthound_adapter.py +842 -0
- truthound_dashboard/core/phase5/collaboration.py +1 -1
- truthound_dashboard/core/plugins/lifecycle/__init__.py +0 -13
- truthound_dashboard/core/quality_reporter.py +1359 -0
- truthound_dashboard/core/report_history.py +0 -6
- truthound_dashboard/core/reporters/__init__.py +175 -14
- truthound_dashboard/core/reporters/adapters.py +943 -0
- truthound_dashboard/core/reporters/base.py +0 -3
- truthound_dashboard/core/reporters/builtin/__init__.py +18 -0
- truthound_dashboard/core/reporters/builtin/csv_reporter.py +111 -0
- truthound_dashboard/core/reporters/builtin/html_reporter.py +270 -0
- truthound_dashboard/core/reporters/builtin/json_reporter.py +127 -0
- truthound_dashboard/core/reporters/compat.py +266 -0
- truthound_dashboard/core/reporters/csv_reporter.py +2 -35
- truthound_dashboard/core/reporters/factory.py +526 -0
- truthound_dashboard/core/reporters/interfaces.py +745 -0
- truthound_dashboard/core/reporters/registry.py +1 -10
- truthound_dashboard/core/scheduler.py +165 -0
- truthound_dashboard/core/schema_evolution.py +3 -3
- truthound_dashboard/core/schema_watcher.py +1528 -0
- truthound_dashboard/core/services.py +595 -76
- truthound_dashboard/core/store_manager.py +810 -0
- truthound_dashboard/core/streaming_anomaly.py +169 -4
- truthound_dashboard/core/tiering.py +1309 -0
- truthound_dashboard/core/triggers/evaluators.py +178 -8
- truthound_dashboard/core/truthound_adapter.py +2620 -197
- truthound_dashboard/core/unified_alerts.py +23 -20
- truthound_dashboard/db/__init__.py +8 -0
- truthound_dashboard/db/database.py +8 -2
- truthound_dashboard/db/models.py +944 -25
- truthound_dashboard/db/repository.py +2 -0
- truthound_dashboard/main.py +11 -0
- truthound_dashboard/schemas/__init__.py +177 -16
- truthound_dashboard/schemas/base.py +44 -23
- truthound_dashboard/schemas/collaboration.py +19 -6
- truthound_dashboard/schemas/cross_alerts.py +19 -3
- truthound_dashboard/schemas/drift.py +61 -55
- truthound_dashboard/schemas/drift_monitor.py +67 -23
- truthound_dashboard/schemas/enterprise_sampling.py +653 -0
- truthound_dashboard/schemas/lineage.py +0 -33
- truthound_dashboard/schemas/mask.py +10 -8
- truthound_dashboard/schemas/model_monitoring.py +89 -10
- truthound_dashboard/schemas/notifications_advanced.py +13 -0
- truthound_dashboard/schemas/observability.py +453 -0
- truthound_dashboard/schemas/plugins.py +0 -280
- truthound_dashboard/schemas/profile.py +154 -247
- truthound_dashboard/schemas/quality_reporter.py +403 -0
- truthound_dashboard/schemas/reports.py +2 -2
- truthound_dashboard/schemas/rule_suggestion.py +8 -1
- truthound_dashboard/schemas/scan.py +4 -24
- truthound_dashboard/schemas/schedule.py +11 -3
- truthound_dashboard/schemas/schema_watcher.py +727 -0
- truthound_dashboard/schemas/source.py +17 -2
- truthound_dashboard/schemas/tiering.py +822 -0
- truthound_dashboard/schemas/triggers.py +16 -0
- truthound_dashboard/schemas/unified_alerts.py +7 -0
- truthound_dashboard/schemas/validation.py +0 -13
- truthound_dashboard/schemas/validators/base.py +41 -21
- truthound_dashboard/schemas/validators/business_rule_validators.py +244 -0
- truthound_dashboard/schemas/validators/localization_validators.py +273 -0
- truthound_dashboard/schemas/validators/ml_feature_validators.py +308 -0
- truthound_dashboard/schemas/validators/profiling_validators.py +275 -0
- truthound_dashboard/schemas/validators/referential_validators.py +312 -0
- truthound_dashboard/schemas/validators/registry.py +93 -8
- truthound_dashboard/schemas/validators/timeseries_validators.py +389 -0
- truthound_dashboard/schemas/versioning.py +1 -6
- truthound_dashboard/static/index.html +2 -2
- truthound_dashboard-1.5.0.dist-info/METADATA +309 -0
- {truthound_dashboard-1.4.4.dist-info → truthound_dashboard-1.5.0.dist-info}/RECORD +149 -148
- truthound_dashboard/core/plugins/hooks/__init__.py +0 -63
- truthound_dashboard/core/plugins/hooks/decorators.py +0 -367
- truthound_dashboard/core/plugins/hooks/manager.py +0 -403
- truthound_dashboard/core/plugins/hooks/protocols.py +0 -265
- truthound_dashboard/core/plugins/lifecycle/hot_reload.py +0 -584
- truthound_dashboard/core/reporters/junit_reporter.py +0 -233
- truthound_dashboard/core/reporters/markdown_reporter.py +0 -207
- truthound_dashboard/core/reporters/pdf_reporter.py +0 -209
- truthound_dashboard/static/assets/_baseUniq-BcrSP13d.js +0 -1
- truthound_dashboard/static/assets/arc-DlYjKwIL.js +0 -1
- truthound_dashboard/static/assets/architectureDiagram-VXUJARFQ-Bb2drbQM.js +0 -36
- truthound_dashboard/static/assets/blockDiagram-VD42YOAC-BlsPG1CH.js +0 -122
- truthound_dashboard/static/assets/c4Diagram-YG6GDRKO-B9JdUoaC.js +0 -10
- truthound_dashboard/static/assets/channel-Q6mHF1Hd.js +0 -1
- truthound_dashboard/static/assets/chunk-4BX2VUAB-DmyoPVuJ.js +0 -1
- truthound_dashboard/static/assets/chunk-55IACEB6-Bcz6Siv8.js +0 -1
- truthound_dashboard/static/assets/chunk-B4BG7PRW-Br3G5Rum.js +0 -165
- truthound_dashboard/static/assets/chunk-DI55MBZ5-DuM9c23u.js +0 -220
- truthound_dashboard/static/assets/chunk-FMBD7UC4-DNU-5mvT.js +0 -15
- truthound_dashboard/static/assets/chunk-QN33PNHL-Im2yNcmS.js +0 -1
- truthound_dashboard/static/assets/chunk-QZHKN3VN-kZr8XFm1.js +0 -1
- truthound_dashboard/static/assets/chunk-TZMSLE5B-Q__360q_.js +0 -1
- truthound_dashboard/static/assets/classDiagram-2ON5EDUG-vtixxUyK.js +0 -1
- truthound_dashboard/static/assets/classDiagram-v2-WZHVMYZB-vtixxUyK.js +0 -1
- truthound_dashboard/static/assets/clone-BOt2LwD0.js +0 -1
- truthound_dashboard/static/assets/cose-bilkent-S5V4N54A-CBDw6iac.js +0 -1
- truthound_dashboard/static/assets/dagre-6UL2VRFP-XdKqmmY9.js +0 -4
- truthound_dashboard/static/assets/diagram-PSM6KHXK-DAZ8nx9V.js +0 -24
- truthound_dashboard/static/assets/diagram-QEK2KX5R-BRvDTbGD.js +0 -43
- truthound_dashboard/static/assets/diagram-S2PKOQOG-bQcczUkl.js +0 -24
- truthound_dashboard/static/assets/erDiagram-Q2GNP2WA-DPje7VMN.js +0 -60
- truthound_dashboard/static/assets/flowDiagram-NV44I4VS-B7BVtFVS.js +0 -162
- truthound_dashboard/static/assets/ganttDiagram-JELNMOA3-D6WKSS7U.js +0 -267
- truthound_dashboard/static/assets/gitGraphDiagram-NY62KEGX-D3vtVd3y.js +0 -65
- truthound_dashboard/static/assets/graph-BKgNKZVp.js +0 -1
- truthound_dashboard/static/assets/index-C6JSrkHo.css +0 -1
- truthound_dashboard/static/assets/index-DkU82VsU.js +0 -1800
- truthound_dashboard/static/assets/infoDiagram-WHAUD3N6-DnNCT429.js +0 -2
- truthound_dashboard/static/assets/journeyDiagram-XKPGCS4Q-DGiMozqS.js +0 -139
- truthound_dashboard/static/assets/kanban-definition-3W4ZIXB7-BV2gUgli.js +0 -89
- truthound_dashboard/static/assets/katex-Cu_Erd72.js +0 -261
- truthound_dashboard/static/assets/layout-DI2MfQ5G.js +0 -1
- truthound_dashboard/static/assets/min-DYdgXVcT.js +0 -1
- truthound_dashboard/static/assets/mindmap-definition-VGOIOE7T-C7x4ruxz.js +0 -68
- truthound_dashboard/static/assets/pieDiagram-ADFJNKIX-CAJaAB9f.js +0 -30
- truthound_dashboard/static/assets/quadrantDiagram-AYHSOK5B-DeqwDI46.js +0 -7
- truthound_dashboard/static/assets/requirementDiagram-UZGBJVZJ-e3XDpZIM.js +0 -64
- truthound_dashboard/static/assets/sankeyDiagram-TZEHDZUN-CNnAv5Ux.js +0 -10
- truthound_dashboard/static/assets/sequenceDiagram-WL72ISMW-Dsne-Of3.js +0 -145
- truthound_dashboard/static/assets/stateDiagram-FKZM4ZOC-Ee0sQXyb.js +0 -1
- truthound_dashboard/static/assets/stateDiagram-v2-4FDKWEC3-B26KqW_W.js +0 -1
- truthound_dashboard/static/assets/timeline-definition-IT6M3QCI-DZYi2yl3.js +0 -61
- truthound_dashboard/static/assets/treemap-KMMF4GRG-CY3f8In2.js +0 -128
- truthound_dashboard/static/assets/unmerged_dictionaries-Dd7xcPWG.js +0 -1
- truthound_dashboard/static/assets/xychartDiagram-PRI3JC2R-CS7fydZZ.js +0 -7
- truthound_dashboard-1.4.4.dist-info/METADATA +0 -507
- {truthound_dashboard-1.4.4.dist-info → truthound_dashboard-1.5.0.dist-info}/WHEEL +0 -0
- {truthound_dashboard-1.4.4.dist-info → truthound_dashboard-1.5.0.dist-info}/entry_points.txt +0 -0
- {truthound_dashboard-1.4.4.dist-info → truthound_dashboard-1.5.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,429 +1,239 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Dashboard-specific escalation service using truthound.
|
|
2
2
|
|
|
3
|
-
This module provides
|
|
4
|
-
|
|
5
|
-
acknowledging, and resolving incidents.
|
|
3
|
+
This module provides adapters that integrate truthound's escalation
|
|
4
|
+
system with the Dashboard's database configuration.
|
|
6
5
|
"""
|
|
7
6
|
|
|
8
7
|
from __future__ import annotations
|
|
9
8
|
|
|
10
9
|
import logging
|
|
11
|
-
from
|
|
12
|
-
from datetime import datetime, timedelta
|
|
13
|
-
from typing import Any, Callable
|
|
10
|
+
from typing import Any
|
|
14
11
|
|
|
15
|
-
from .
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
from truthound.checkpoint.escalation import (
|
|
13
|
+
EscalationEngine,
|
|
14
|
+
EscalationEngineConfig,
|
|
18
15
|
EscalationPolicy,
|
|
19
|
-
|
|
16
|
+
EscalationLevel,
|
|
20
17
|
EscalationTarget,
|
|
18
|
+
EscalationTrigger,
|
|
19
|
+
EscalationState,
|
|
20
|
+
TargetType,
|
|
21
|
+
InMemoryEscalationStore,
|
|
21
22
|
)
|
|
22
|
-
from .state_machine import EscalationStateMachine
|
|
23
|
-
from .stores import BaseEscalationStore, InMemoryEscalationStore
|
|
24
23
|
|
|
25
24
|
logger = logging.getLogger(__name__)
|
|
26
25
|
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
"""Configuration for the escalation engine.
|
|
31
|
-
|
|
32
|
-
Attributes:
|
|
33
|
-
check_interval_seconds: How often to check for pending escalations.
|
|
34
|
-
max_retries: Maximum notification retries per level.
|
|
35
|
-
default_delay_minutes: Default delay between levels if not specified.
|
|
36
|
-
"""
|
|
37
|
-
|
|
38
|
-
check_interval_seconds: int = 60
|
|
39
|
-
max_retries: int = 3
|
|
40
|
-
default_delay_minutes: int = 15
|
|
41
|
-
|
|
27
|
+
class DashboardEscalationService:
|
|
28
|
+
"""Dashboard-specific escalation service.
|
|
42
29
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
Orchestrates the complete escalation lifecycle:
|
|
47
|
-
1. Triggering new incidents
|
|
48
|
-
2. Escalating to next levels based on time
|
|
49
|
-
3. Acknowledging incidents
|
|
50
|
-
4. Resolving incidents
|
|
51
|
-
5. Auto-resolving on success
|
|
52
|
-
|
|
53
|
-
The engine can be used standalone or integrated with
|
|
54
|
-
APScheduler for periodic escalation checks.
|
|
30
|
+
Wraps truthound's EscalationEngine and provides integration
|
|
31
|
+
with the Dashboard's database configuration.
|
|
55
32
|
|
|
56
33
|
Example:
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
on_notify=send_notification,
|
|
60
|
-
)
|
|
34
|
+
service = DashboardEscalationService()
|
|
35
|
+
await service.start()
|
|
61
36
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
incident_ref="validation-123",
|
|
37
|
+
await service.trigger(
|
|
38
|
+
incident_id="incident-123",
|
|
39
|
+
policy_name="critical_alerts",
|
|
66
40
|
context={"severity": "critical"},
|
|
67
41
|
)
|
|
68
|
-
|
|
69
|
-
# Acknowledge
|
|
70
|
-
await engine.acknowledge("incident-id", actor="user@example.com")
|
|
71
|
-
|
|
72
|
-
# Resolve
|
|
73
|
-
await engine.resolve("incident-id", actor="user@example.com")
|
|
74
42
|
"""
|
|
75
43
|
|
|
76
|
-
def __init__(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
44
|
+
def __init__(self) -> None:
|
|
45
|
+
"""Initialize the service."""
|
|
46
|
+
self._engine: EscalationEngine | None = None
|
|
47
|
+
self._policies: dict[str, EscalationPolicy] = {}
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def engine(self) -> EscalationEngine:
|
|
51
|
+
"""Get the underlying truthound engine."""
|
|
52
|
+
if self._engine is None:
|
|
53
|
+
config = EscalationEngineConfig(
|
|
54
|
+
check_interval_seconds=60,
|
|
55
|
+
max_retries=3,
|
|
56
|
+
)
|
|
57
|
+
self._engine = EscalationEngine(
|
|
58
|
+
config=config,
|
|
59
|
+
store=InMemoryEscalationStore(),
|
|
60
|
+
)
|
|
61
|
+
return self._engine
|
|
62
|
+
|
|
63
|
+
async def start(self) -> None:
|
|
64
|
+
"""Start the escalation engine."""
|
|
65
|
+
await self.engine.start()
|
|
66
|
+
|
|
67
|
+
async def stop(self) -> None:
|
|
68
|
+
"""Stop the escalation engine."""
|
|
69
|
+
await self.engine.stop()
|
|
70
|
+
|
|
71
|
+
def register_policy(self, policy: EscalationPolicy) -> None:
|
|
72
|
+
"""Register an escalation policy."""
|
|
73
|
+
self._policies[policy.name] = policy
|
|
74
|
+
self.engine.register_policy(policy)
|
|
93
75
|
|
|
94
76
|
async def trigger(
|
|
95
77
|
self,
|
|
96
|
-
|
|
97
|
-
|
|
78
|
+
incident_id: str,
|
|
79
|
+
policy_name: str,
|
|
98
80
|
context: dict[str, Any] | None = None,
|
|
99
|
-
) ->
|
|
100
|
-
"""Trigger
|
|
101
|
-
|
|
102
|
-
Creates a new incident and starts the escalation process.
|
|
103
|
-
If an incident with the same ref already exists and is not
|
|
104
|
-
resolved, returns the existing incident.
|
|
81
|
+
) -> Any:
|
|
82
|
+
"""Trigger an escalation for an incident.
|
|
105
83
|
|
|
106
84
|
Args:
|
|
107
|
-
|
|
108
|
-
|
|
85
|
+
incident_id: Unique incident identifier.
|
|
86
|
+
policy_name: Name of the escalation policy.
|
|
109
87
|
context: Context data for the incident.
|
|
110
88
|
|
|
111
89
|
Returns:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
Raises:
|
|
115
|
-
ValueError: If policy not found.
|
|
90
|
+
Escalation record.
|
|
116
91
|
"""
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
logger.debug(f"Incident {incident_ref} already exists in state {existing.state}")
|
|
121
|
-
return existing
|
|
122
|
-
|
|
123
|
-
# Get policy
|
|
124
|
-
policy = self.store.get_policy(policy_id)
|
|
125
|
-
if not policy:
|
|
126
|
-
raise ValueError(f"Escalation policy not found: {policy_id}")
|
|
127
|
-
|
|
128
|
-
if not policy.is_active:
|
|
129
|
-
raise ValueError(f"Escalation policy is not active: {policy_id}")
|
|
130
|
-
|
|
131
|
-
# Create incident
|
|
132
|
-
incident = EscalationIncident(
|
|
133
|
-
policy_id=policy_id,
|
|
134
|
-
incident_ref=incident_ref,
|
|
92
|
+
return await self.engine.trigger(
|
|
93
|
+
incident_id=incident_id,
|
|
94
|
+
policy_name=policy_name,
|
|
135
95
|
context=context or {},
|
|
136
96
|
)
|
|
137
97
|
|
|
138
|
-
# Trigger state transition
|
|
139
|
-
incident = self.state_machine.trigger(incident)
|
|
140
|
-
|
|
141
|
-
# Set next escalation time
|
|
142
|
-
first_level = policy.get_level(1)
|
|
143
|
-
if first_level:
|
|
144
|
-
delay = first_level.delay_minutes
|
|
145
|
-
if delay > 0:
|
|
146
|
-
incident.next_escalation_at = datetime.utcnow() + timedelta(minutes=delay)
|
|
147
|
-
else:
|
|
148
|
-
incident.next_escalation_at = datetime.utcnow()
|
|
149
|
-
|
|
150
|
-
# Save incident
|
|
151
|
-
self.store.save_incident(incident)
|
|
152
|
-
|
|
153
|
-
# Notify first level
|
|
154
|
-
await self._notify_level(incident, policy, first_level)
|
|
155
|
-
|
|
156
|
-
logger.info(f"Triggered escalation for {incident_ref}")
|
|
157
|
-
return incident
|
|
158
|
-
|
|
159
|
-
async def escalate(self, incident_id: str) -> EscalationIncident:
|
|
160
|
-
"""Escalate incident to the next level.
|
|
161
|
-
|
|
162
|
-
Args:
|
|
163
|
-
incident_id: ID of the incident.
|
|
164
|
-
|
|
165
|
-
Returns:
|
|
166
|
-
Updated incident.
|
|
167
|
-
|
|
168
|
-
Raises:
|
|
169
|
-
ValueError: If incident not found or can't escalate.
|
|
170
|
-
"""
|
|
171
|
-
incident = self.store.get_incident(incident_id)
|
|
172
|
-
if not incident:
|
|
173
|
-
raise ValueError(f"Incident not found: {incident_id}")
|
|
174
|
-
|
|
175
|
-
if incident.state == EscalationState.RESOLVED:
|
|
176
|
-
raise ValueError("Cannot escalate resolved incident")
|
|
177
|
-
|
|
178
|
-
policy = self.store.get_policy(incident.policy_id)
|
|
179
|
-
if not policy:
|
|
180
|
-
raise ValueError(f"Policy not found: {incident.policy_id}")
|
|
181
|
-
|
|
182
|
-
# Check max escalations
|
|
183
|
-
if incident.escalation_count >= policy.max_escalations:
|
|
184
|
-
logger.warning(f"Incident {incident_id} reached max escalations")
|
|
185
|
-
return incident
|
|
186
|
-
|
|
187
|
-
# Get next level
|
|
188
|
-
next_level = policy.get_next_level(incident.current_level)
|
|
189
|
-
if not next_level:
|
|
190
|
-
logger.info(f"Incident {incident_id} at max level {incident.current_level}")
|
|
191
|
-
return incident
|
|
192
|
-
|
|
193
|
-
# Escalate
|
|
194
|
-
incident = self.state_machine.escalate(
|
|
195
|
-
incident,
|
|
196
|
-
to_level=next_level.level,
|
|
197
|
-
message=f"Escalating to level {next_level.level}",
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
# Set next escalation time
|
|
201
|
-
further_level = policy.get_next_level(next_level.level)
|
|
202
|
-
if further_level:
|
|
203
|
-
delay = further_level.delay_minutes
|
|
204
|
-
incident.next_escalation_at = datetime.utcnow() + timedelta(minutes=delay)
|
|
205
|
-
else:
|
|
206
|
-
incident.next_escalation_at = None
|
|
207
|
-
|
|
208
|
-
# Save
|
|
209
|
-
self.store.save_incident(incident)
|
|
210
|
-
|
|
211
|
-
# Notify
|
|
212
|
-
await self._notify_level(incident, policy, next_level)
|
|
213
|
-
|
|
214
|
-
logger.info(f"Escalated {incident_id} to level {next_level.level}")
|
|
215
|
-
return incident
|
|
216
|
-
|
|
217
98
|
async def acknowledge(
|
|
218
99
|
self,
|
|
219
100
|
incident_id: str,
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
Pauses further escalation until either resolved or
|
|
226
|
-
escalation time is reached.
|
|
101
|
+
*,
|
|
102
|
+
responder: str | None = None,
|
|
103
|
+
note: str | None = None,
|
|
104
|
+
) -> bool:
|
|
105
|
+
"""Acknowledge an escalation.
|
|
227
106
|
|
|
228
107
|
Args:
|
|
229
|
-
incident_id:
|
|
230
|
-
|
|
231
|
-
|
|
108
|
+
incident_id: Incident identifier.
|
|
109
|
+
responder: Name/ID of responder.
|
|
110
|
+
note: Optional note.
|
|
232
111
|
|
|
233
112
|
Returns:
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
Raises:
|
|
237
|
-
ValueError: If incident not found or can't acknowledge.
|
|
113
|
+
True if acknowledged successfully.
|
|
238
114
|
"""
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
# Check if can acknowledge
|
|
244
|
-
if not self.state_machine.can_transition(incident, EscalationState.ACKNOWLEDGED):
|
|
245
|
-
raise ValueError(f"Cannot acknowledge incident in state {incident.state}")
|
|
246
|
-
|
|
247
|
-
# Acknowledge
|
|
248
|
-
incident = self.state_machine.acknowledge(
|
|
249
|
-
incident,
|
|
250
|
-
actor=actor,
|
|
251
|
-
message=message or f"Acknowledged by {actor}",
|
|
115
|
+
return await self.engine.acknowledge(
|
|
116
|
+
incident_id=incident_id,
|
|
117
|
+
responder=responder,
|
|
118
|
+
note=note,
|
|
252
119
|
)
|
|
253
120
|
|
|
254
|
-
# Save
|
|
255
|
-
self.store.save_incident(incident)
|
|
256
|
-
|
|
257
|
-
logger.info(f"Incident {incident_id} acknowledged by {actor}")
|
|
258
|
-
return incident
|
|
259
|
-
|
|
260
121
|
async def resolve(
|
|
261
122
|
self,
|
|
262
123
|
incident_id: str,
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
) ->
|
|
267
|
-
"""Resolve an
|
|
124
|
+
*,
|
|
125
|
+
responder: str | None = None,
|
|
126
|
+
note: str | None = None,
|
|
127
|
+
) -> bool:
|
|
128
|
+
"""Resolve an escalation.
|
|
268
129
|
|
|
269
130
|
Args:
|
|
270
|
-
incident_id:
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
auto: Whether this is auto-resolution.
|
|
131
|
+
incident_id: Incident identifier.
|
|
132
|
+
responder: Name/ID of responder.
|
|
133
|
+
note: Optional resolution note.
|
|
274
134
|
|
|
275
135
|
Returns:
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
Raises:
|
|
279
|
-
ValueError: If incident not found or can't resolve.
|
|
136
|
+
True if resolved successfully.
|
|
280
137
|
"""
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
# Check if can resolve
|
|
286
|
-
if not self.state_machine.can_transition(incident, EscalationState.RESOLVED):
|
|
287
|
-
raise ValueError(f"Cannot resolve incident in state {incident.state}")
|
|
288
|
-
|
|
289
|
-
# Resolve
|
|
290
|
-
incident = self.state_machine.resolve(
|
|
291
|
-
incident,
|
|
292
|
-
actor=actor,
|
|
293
|
-
message=message,
|
|
294
|
-
auto=auto,
|
|
138
|
+
return await self.engine.resolve(
|
|
139
|
+
incident_id=incident_id,
|
|
140
|
+
responder=responder,
|
|
141
|
+
note=note,
|
|
295
142
|
)
|
|
296
143
|
|
|
297
|
-
|
|
298
|
-
self.store.save_incident(incident)
|
|
299
|
-
|
|
300
|
-
log_msg = f"Incident {incident_id} resolved"
|
|
301
|
-
if auto:
|
|
302
|
-
log_msg += " (auto)"
|
|
303
|
-
elif actor:
|
|
304
|
-
log_msg += f" by {actor}"
|
|
305
|
-
logger.info(log_msg)
|
|
306
|
-
|
|
307
|
-
return incident
|
|
308
|
-
|
|
309
|
-
async def auto_resolve_by_ref(
|
|
144
|
+
async def cancel(
|
|
310
145
|
self,
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
Called when validation passes to auto-resolve associated
|
|
317
|
-
incidents (if policy allows).
|
|
146
|
+
incident_id: str,
|
|
147
|
+
*,
|
|
148
|
+
reason: str | None = None,
|
|
149
|
+
) -> bool:
|
|
150
|
+
"""Cancel an escalation.
|
|
318
151
|
|
|
319
152
|
Args:
|
|
320
|
-
|
|
321
|
-
|
|
153
|
+
incident_id: Incident identifier.
|
|
154
|
+
reason: Optional cancellation reason.
|
|
322
155
|
|
|
323
156
|
Returns:
|
|
324
|
-
|
|
157
|
+
True if cancelled successfully.
|
|
325
158
|
"""
|
|
326
|
-
|
|
327
|
-
if not incident:
|
|
328
|
-
return None
|
|
329
|
-
|
|
330
|
-
if incident.state == EscalationState.RESOLVED:
|
|
331
|
-
return incident
|
|
332
|
-
|
|
333
|
-
# Check policy allows auto-resolve
|
|
334
|
-
policy = self.store.get_policy(incident.policy_id)
|
|
335
|
-
if not policy or not policy.auto_resolve_on_success:
|
|
336
|
-
return None
|
|
337
|
-
|
|
338
|
-
return await self.resolve(
|
|
339
|
-
incident.id,
|
|
340
|
-
message=message,
|
|
341
|
-
auto=True,
|
|
342
|
-
)
|
|
343
|
-
|
|
344
|
-
async def check_and_escalate(self) -> int:
|
|
345
|
-
"""Check for and process pending escalations.
|
|
159
|
+
return await self.engine.cancel(incident_id=incident_id, reason=reason)
|
|
346
160
|
|
|
347
|
-
|
|
348
|
-
|
|
161
|
+
async def get_active_escalations(self) -> list[Any]:
|
|
162
|
+
"""Get all active escalations.
|
|
349
163
|
|
|
350
164
|
Returns:
|
|
351
|
-
|
|
165
|
+
List of active escalation records.
|
|
352
166
|
"""
|
|
353
|
-
|
|
354
|
-
escalated = 0
|
|
167
|
+
return await self.engine.list_active()
|
|
355
168
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
escalated += 1
|
|
360
|
-
except Exception as e:
|
|
361
|
-
logger.error(f"Failed to escalate {incident.id}: {e}")
|
|
169
|
+
def get_policy(self, name: str) -> EscalationPolicy | None:
|
|
170
|
+
"""Get a registered policy by name."""
|
|
171
|
+
return self._policies.get(name)
|
|
362
172
|
|
|
363
|
-
return escalated
|
|
364
|
-
|
|
365
|
-
async def _notify_level(
|
|
366
|
-
self,
|
|
367
|
-
incident: EscalationIncident,
|
|
368
|
-
policy: EscalationPolicy,
|
|
369
|
-
level: EscalationLevel | None,
|
|
370
|
-
) -> None:
|
|
371
|
-
"""Send notifications for an escalation level.
|
|
372
173
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
policy: The policy.
|
|
376
|
-
level: The level to notify.
|
|
377
|
-
"""
|
|
378
|
-
if not level or not self.on_notify:
|
|
379
|
-
return
|
|
380
|
-
|
|
381
|
-
for target in level.targets:
|
|
382
|
-
try:
|
|
383
|
-
await self.on_notify(incident, level, target)
|
|
384
|
-
except Exception as e:
|
|
385
|
-
logger.error(
|
|
386
|
-
f"Failed to notify {target.identifier} for incident {incident.id}: {e}"
|
|
387
|
-
)
|
|
388
|
-
|
|
389
|
-
def get_incident(self, incident_id: str) -> EscalationIncident | None:
|
|
390
|
-
"""Get incident by ID."""
|
|
391
|
-
return self.store.get_incident(incident_id)
|
|
392
|
-
|
|
393
|
-
def get_incident_by_ref(self, incident_ref: str) -> EscalationIncident | None:
|
|
394
|
-
"""Get incident by reference."""
|
|
395
|
-
return self.store.get_incident_by_ref(incident_ref)
|
|
396
|
-
|
|
397
|
-
def list_active_incidents(self) -> list[EscalationIncident]:
|
|
398
|
-
"""List all active (non-resolved) incidents."""
|
|
399
|
-
return self.store.list_incidents(
|
|
400
|
-
states=[
|
|
401
|
-
EscalationState.PENDING,
|
|
402
|
-
EscalationState.TRIGGERED,
|
|
403
|
-
EscalationState.ACKNOWLEDGED,
|
|
404
|
-
EscalationState.ESCALATED,
|
|
405
|
-
]
|
|
406
|
-
)
|
|
174
|
+
def create_policy_from_db(db_config: dict[str, Any]) -> EscalationPolicy:
|
|
175
|
+
"""Create an EscalationPolicy from database configuration.
|
|
407
176
|
|
|
408
|
-
|
|
409
|
-
|
|
177
|
+
Args:
|
|
178
|
+
db_config: Configuration dictionary from database.
|
|
410
179
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
"
|
|
423
|
-
"
|
|
424
|
-
"
|
|
425
|
-
1 for i in all_incidents
|
|
426
|
-
if i.state != EscalationState.RESOLVED
|
|
427
|
-
),
|
|
428
|
-
"total_policies": len(self.store.list_policies(active_only=False)),
|
|
180
|
+
Returns:
|
|
181
|
+
EscalationPolicy for truthound's engine.
|
|
182
|
+
"""
|
|
183
|
+
# Build targets
|
|
184
|
+
def build_target(target_config: dict[str, Any]) -> EscalationTarget:
|
|
185
|
+
target_type_map = {
|
|
186
|
+
"user": TargetType.USER,
|
|
187
|
+
"team": TargetType.TEAM,
|
|
188
|
+
"channel": TargetType.CHANNEL,
|
|
189
|
+
"schedule": TargetType.SCHEDULE,
|
|
190
|
+
"webhook": TargetType.WEBHOOK,
|
|
191
|
+
"email": TargetType.EMAIL,
|
|
192
|
+
"phone": TargetType.PHONE,
|
|
193
|
+
"custom": TargetType.CUSTOM,
|
|
429
194
|
}
|
|
195
|
+
return EscalationTarget(
|
|
196
|
+
type=target_type_map.get(
|
|
197
|
+
target_config.get("type", "user").lower(),
|
|
198
|
+
TargetType.USER,
|
|
199
|
+
),
|
|
200
|
+
identifier=target_config.get("identifier", ""),
|
|
201
|
+
display_name=target_config.get("display_name"),
|
|
202
|
+
channel=target_config.get("channel"),
|
|
203
|
+
config=target_config.get("config", {}),
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
# Build levels
|
|
207
|
+
levels = []
|
|
208
|
+
for level_config in db_config.get("levels", []):
|
|
209
|
+
level = EscalationLevel(
|
|
210
|
+
level=level_config.get("level", 1),
|
|
211
|
+
delay_minutes=level_config.get("delay_minutes", 0),
|
|
212
|
+
targets=[build_target(t) for t in level_config.get("targets", [])],
|
|
213
|
+
repeat_count=level_config.get("repeat_count", 1),
|
|
214
|
+
repeat_interval_minutes=level_config.get("repeat_interval_minutes", 5),
|
|
215
|
+
)
|
|
216
|
+
levels.append(level)
|
|
217
|
+
|
|
218
|
+
# Build triggers
|
|
219
|
+
trigger_map = {
|
|
220
|
+
"unacknowledged": EscalationTrigger.UNACKNOWLEDGED,
|
|
221
|
+
"severity": EscalationTrigger.SEVERITY,
|
|
222
|
+
"manual": EscalationTrigger.MANUAL,
|
|
223
|
+
"failure": EscalationTrigger.FAILURE,
|
|
224
|
+
"sla_breach": EscalationTrigger.SLA_BREACH,
|
|
225
|
+
}
|
|
226
|
+
triggers = [
|
|
227
|
+
trigger_map.get(t.lower(), EscalationTrigger.UNACKNOWLEDGED)
|
|
228
|
+
for t in db_config.get("triggers", ["unacknowledged"])
|
|
229
|
+
]
|
|
230
|
+
|
|
231
|
+
return EscalationPolicy(
|
|
232
|
+
name=db_config.get("name", "default"),
|
|
233
|
+
levels=levels,
|
|
234
|
+
triggers=triggers,
|
|
235
|
+
description=db_config.get("description"),
|
|
236
|
+
severity_filter=db_config.get("severity_filter"),
|
|
237
|
+
auto_resolve_on_success=db_config.get("auto_resolve_on_success", True),
|
|
238
|
+
metadata=db_config.get("metadata", {}),
|
|
239
|
+
)
|