truthound-dashboard 1.4.4__py3-none-any.whl → 1.5.1__py3-none-any.whl

truthound_dashboard/core/notifications/dispatcher.py

@@ -3,13 +3,19 @@
 The dispatcher coordinates between events, rules, and channels to
 deliver notifications based on configured triggers.
 
-Architecture:
-    Event -> Dispatcher -> Rules -> Channels -> Delivery
+Architecture (with truthound integration):
+    Event -> Dispatcher -> Truthound Adapter (Routing/Dedup/Throttle) -> Channels -> Delivery -> Escalation
+
+The dispatcher now integrates with truthound library for:
+- Routing: ActionRouter with 11+ rule types
+- Deduplication: NotificationDeduplicator with time windows
+- Throttling: NotificationThrottler with rate limits
+- Escalation: EscalationEngine with multi-level policies
 
 Example:
     dispatcher = get_dispatcher()
 
-    # Notify about validation failure
+    # Notify about validation failure (with truthound processing)
     await dispatcher.notify_validation_failed(
         source_id="source-123",
         source_name="My Source",
@@ -24,7 +30,7 @@ from __future__ import annotations
 
 import logging
 from collections.abc import Sequence
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -50,6 +56,9 @@ from .events import (
     ValidationFailedEvent,
 )
 
+if TYPE_CHECKING:
+    from .truthound_adapter import TruthoundNotificationAdapter
+
 logger = logging.getLogger(__name__)
 
 
@@ -58,10 +67,12 @@ class NotificationDispatcher:
 
     The dispatcher:
     1. Receives notification events
-    2. Matches events against active rules
-    3. Resolves target channels from matching rules
-    4. Delivers notifications through channels
-    5. Logs delivery results
+    2. Routes through truthound ActionRouter (if enabled)
+    3. Checks deduplication via truthound NotificationDeduplicator
+    4. Checks throttling via truthound NotificationThrottler
+    5. Delivers notifications through channels
+    6. Logs delivery results
+    7. Triggers escalation via truthound EscalationEngine (for critical events)
 
     Usage:
         dispatcher = NotificationDispatcher(session)
@@ -69,17 +80,37 @@ class NotificationDispatcher:
         # Send test notification
         results = await dispatcher.test_channel(channel_id)
 
-        # Notify about events
+        # Notify about events (with truthound processing)
         await dispatcher.notify_validation_failed(...)
+
+        # Disable truthound for direct sending
+        dispatcher = NotificationDispatcher(session, use_truthound=False)
     """
 
-    def __init__(self, session: AsyncSession) -> None:
+    def __init__(
+        self,
+        session: AsyncSession,
+        use_truthound: bool = True,
+    ) -> None:
         """Initialize the dispatcher.
 
         Args:
             session: Database session for accessing rules and channels.
+            use_truthound: Whether to use truthound library for routing,
+                deduplication, throttling, and escalation. Default True.
         """
         self.session = session
+        self.use_truthound = use_truthound
+        self._truthound_adapter: TruthoundNotificationAdapter | None = None
+
+    async def _get_truthound_adapter(self) -> TruthoundNotificationAdapter:
+        """Get or create the truthound adapter lazily."""
+        if self._truthound_adapter is None:
+            from .truthound_adapter import TruthoundNotificationAdapter
+
+            self._truthound_adapter = TruthoundNotificationAdapter(self.session)
+            await self._truthound_adapter.initialize()
+        return self._truthound_adapter
 
     async def dispatch(
         self,
@@ -91,7 +122,15 @@ class NotificationDispatcher:
         """Dispatch a notification event to matching channels.
 
         If channel_ids is provided, sends directly to those channels.
-        Otherwise, matches event against rules to find target channels.
+        Otherwise, matches event against rules (or truthound routes) to find
+        target channels.
+
+        When truthound is enabled:
+        1. Routes through ActionRouter to get matched channels
+        2. Checks deduplication to suppress duplicate notifications
+        3. Checks throttling to enforce rate limits
+        4. Sends to channels that pass all checks
+        5. Triggers escalation for high severity unacknowledged events
 
         Args:
             event: The notification event to dispatch.
@@ -101,6 +140,18 @@ class NotificationDispatcher:
         Returns:
             List of delivery results for each channel.
         """
+        if self.use_truthound and not channel_ids:
+            return await self._dispatch_via_truthound(event, rule_id)
+
+        return await self._dispatch_legacy(event, channel_ids, rule_id)
+
+    async def _dispatch_legacy(
+        self,
+        event: NotificationEvent,
+        channel_ids: list[str] | None = None,
+        rule_id: str | None = None,
+    ) -> list[NotificationResult]:
+        """Legacy dispatch without truthound integration."""
         # Get target channels
         if channel_ids:
             channels = await self._get_channels_by_ids(channel_ids)
@@ -119,6 +170,146 @@ class NotificationDispatcher:
 
         return results
 
+    async def _dispatch_via_truthound(
+        self,
+        event: NotificationEvent,
+        rule_id: str | None = None,
+    ) -> list[NotificationResult]:
+        """Dispatch using truthound library for routing, dedup, throttle, escalation.
+
+        This method integrates with truthound's checkpoint modules:
+        - ActionRouter: Matches events to channels via configurable rules
+        - NotificationDeduplicator: Suppresses duplicate notifications
+        - NotificationThrottler: Enforces rate limits per channel
+        - EscalationEngine: Escalates unacknowledged critical events
+        """
+        adapter = await self._get_truthound_adapter()
+
+        # Step 1: Route event to channels via truthound ActionRouter
+        matched_channel_ids = await adapter.match_routes(event)
+
+        # Fall back to legacy rule matching if no routes configured
+        if not matched_channel_ids:
+            channels = await self._get_channels_for_event(event)
+            matched_channel_ids = [ch.id for ch in channels]
+
+        if not matched_channel_ids:
+            logger.debug(f"No channels matched for event: {event.event_type}")
+            return []
+
+        # Get channel models
+        channels = await self._get_channels_by_ids(matched_channel_ids)
+        if not channels:
+            return []
+
+        results: list[NotificationResult] = []
+
+        for channel_model in channels:
+            channel_id = channel_model.id
+
+            # Step 2: Check deduplication via truthound NotificationDeduplicator
+            if await adapter.is_duplicate(event, channel_id):
+                logger.debug(
+                    f"Notification deduplicated for channel {channel_id}: {event.event_type}"
+                )
+                results.append(
+                    NotificationResult(
+                        success=True,
+                        channel_id=channel_id,
+                        channel_type=channel_model.type,
+                        message="",
+                        error=None,
+                        suppressed=True,
+                        suppression_reason="duplicate",
+                    )
+                )
+                continue
+
+            # Step 3: Check throttling via truthound NotificationThrottler
+            if await adapter.is_throttled(channel_id):
+                logger.debug(
+                    f"Notification throttled for channel {channel_id}: {event.event_type}"
+                )
+                results.append(
+                    NotificationResult(
+                        success=True,
+                        channel_id=channel_id,
+                        channel_type=channel_model.type,
+                        message="",
+                        error=None,
+                        suppressed=True,
+                        suppression_reason="throttled",
+                    )
+                )
+                continue
+
+            # Step 4: Send notification
+            result = await self._send_to_channel(channel_model, event, rule_id)
+            results.append(result)
+
+            # Step 5: Mark as sent for deduplication tracking
+            if result.success:
+                await adapter.mark_notification_sent(event, channel_id)
+
+        # Step 6: Check escalation for high severity events
+        await self._check_escalation(event, adapter)
+
+        return results
+
+    async def _check_escalation(
+        self,
+        event: NotificationEvent,
+        adapter: TruthoundNotificationAdapter,
+    ) -> None:
+        """Check and trigger escalation for high severity events.
+
+        Escalation is triggered for:
+        - Critical validation failures
+        - High severity drift detection
+        - Schedule failures
+
+        The escalation engine will notify secondary/tertiary channels
+        if the primary notification is not acknowledged within timeout.
+        """
+        severity = self._get_event_severity(event)
+
+        if severity in ("critical", "high"):
+            try:
+                escalation_id = await adapter.trigger_escalation(
+                    event,
+                    policy_name=f"{severity}_alert",
+                )
+                if escalation_id:
+                    logger.info(
+                        f"Escalation triggered for {event.event_type}: {escalation_id}"
+                    )
+            except Exception as e:
+                logger.warning(f"Failed to trigger escalation: {e}")
+
+    def _get_event_severity(self, event: NotificationEvent) -> str:
+        """Determine severity level of an event."""
+        if isinstance(event, ValidationFailedEvent):
+            if event.has_critical:
+                return "critical"
+            if event.has_high:
+                return "high"
+            return "medium"
+
+        if isinstance(event, DriftDetectedEvent):
+            if event.has_high_drift:
+                return "high"
+            return "medium"
+
+        if isinstance(event, ScheduleFailedEvent):
+            return "high"
+
+        if isinstance(event, SchemaChangedEvent):
+            if event.has_breaking_changes:
+                return "high"
+            return "low"
+
+        return "low"
+
     async def _get_channels_by_ids(
         self, channel_ids: list[str]
     ) -> Sequence[NotificationChannel]:

truthound_dashboard/core/notifications/escalation/__init__.py

@@ -1,149 +1,162 @@
-"""Escalation engine for notification management.
+"""Escalation engine using truthound.checkpoint.escalation.
 
 This module provides a multi-level escalation system for managing
-alerts that require attention at different organizational levels.
+alerts using truthound's escalation infrastructure.
 
-Features:
-    - Multi-level escalation policies
-    - State machine for incident tracking
-    - Configurable delay between escalation levels
-    - Support for user, group, and on-call targets
-    - Auto-resolution on success
+Key Components from truthound.checkpoint.escalation:
+    - EscalationEngine: Main escalation engine
+    - EscalationEngineConfig: Engine configuration
+    - EscalationPolicy: Policy definition
+    - EscalationLevel: Level definition
+    - EscalationTarget: Target definition
+    - EscalationRecord: Escalation state record
 
-State Machine:
-    PENDING -> TRIGGERED -> ESCALATED -> RESOLVED
-                  |             ^
-                  v             |
-             ACKNOWLEDGED ------+
+Target Types from truthound:
+    - USER: Individual user
+    - TEAM: Team
+    - CHANNEL: Channel (Slack, etc.)
+    - SCHEDULE: On-call schedule
+    - WEBHOOK: Webhook URL
+    - EMAIL: Email
+    - PHONE: Phone
+    - CUSTOM: Custom
+
+Escalation States from truthound:
+    - PENDING: Initial state, waiting to start
+    - ACTIVE: Currently notifying at level
+    - ESCALATING: Escalating to next level
+    - ACKNOWLEDGED: Responder acknowledged
+    - RESOLVED: Issue resolved
+    - CANCELLED: Manually cancelled
+    - TIMED_OUT: Max escalation reached
+    - FAILED: System error
+
+Storage Backends from truthound:
+    - InMemoryEscalationStore: Single-process storage
+    - RedisEscalationStore: Distributed storage
+    - SQLiteEscalationStore: Persistent storage
 
 Example:
-    from truthound_dashboard.core.notifications.escalation import (
+    from truthound.checkpoint.escalation import (
         EscalationEngine,
+        EscalationEngineConfig,
         EscalationPolicy,
         EscalationLevel,
         EscalationTarget,
-        TargetType,
+        EscalationTrigger,
     )
 
+    # Define policy
     policy = EscalationPolicy(
         name="critical_alerts",
         levels=[
             EscalationLevel(
                 level=1,
                 delay_minutes=0,
-                targets=[
-                    EscalationTarget(type=TargetType.USER, identifier="team-lead", channel="slack")
-                ],
+                targets=[EscalationTarget.user("team-lead", "Team Lead")],
+                repeat_count=2,
+                repeat_interval_minutes=5,
             ),
             EscalationLevel(
                 level=2,
                 delay_minutes=15,
-                targets=[
-                    EscalationTarget(type=TargetType.GROUP, identifier="managers", channel="pagerduty")
-                ],
+                targets=[EscalationTarget.user("manager", "Manager")],
             ),
         ],
+        triggers=[EscalationTrigger.UNACKNOWLEDGED],
+        severity_filter=["critical", "high"],
     )
 
-    engine = EscalationEngine(policy=policy)
-    await engine.trigger("incident-123", context={"severity": "critical"})
+    # Create engine
+    config = EscalationEngineConfig(store_type="memory")
+    engine = EscalationEngine(config)
+    engine.register_policy(policy)
+
+    # Trigger escalation
+    await engine.start()
+    result = await engine.trigger(
+        incident_id="incident-123",
+        context={"severity": "critical"},
+        policy_name="critical_alerts",
+    )
 """
 
-from .engine import EscalationEngine, EscalationEngineConfig
-from .models import (
-    EscalationEvent,
-    EscalationIncident,
-    EscalationLevel,
+# Re-export from truthound.checkpoint.escalation
+from truthound.checkpoint.escalation import (
+    EscalationEngine,
+    EscalationEngineConfig,
     EscalationPolicy,
-    EscalationState,
+    EscalationLevel,
     EscalationTarget,
-    StateTransition,
+    EscalationRecord,
+    EscalationTrigger,
+    EscalationState,
     TargetType,
+    EscalationPolicyManager,
+    EscalationPolicyConfig,
 )
-from .backends import (
-    BackendType,
-    InMemorySchedulerBackend,
-    JobData,
-    JobExecutionResult,
-    JobState,
-    MisfirePolicy,
-    SchedulerBackend,
-    SchedulerBackendConfig,
-    SQLAlchemySchedulerBackend,
-    create_scheduler_backend,
+
+# Storage backends
+from truthound.checkpoint.escalation import (
+    InMemoryEscalationStore,
+    create_store,
 )
-from .scheduler import (
-    DefaultEscalationHandler,
-    EscalationHandler,
-    EscalationResult,
-    EscalationSchedulerConfig,
-    EscalationSchedulerService,
-    EscalationStrategy,
-    ImmediateEscalationStrategy,
-    LoggingEscalationHandler,
-    TimeBasedEscalationStrategy,
-    get_escalation_scheduler,
-    reset_escalation_scheduler,
-    start_escalation_scheduler,
-    stop_escalation_scheduler,
+
+# Redis/SQLite stores (optional)
+try:
+    from truthound.checkpoint.escalation import RedisEscalationStore
+    REDIS_AVAILABLE = True
+except ImportError:
+    RedisEscalationStore = None  # type: ignore
+    REDIS_AVAILABLE = False
+
+try:
+    from truthound.checkpoint.escalation import SQLiteEscalationStore
+    SQLITE_AVAILABLE = True
+except ImportError:
+    SQLiteEscalationStore = None  # type: ignore
+    SQLITE_AVAILABLE = False
+
+# Routing integration
+from truthound.checkpoint.escalation import (
+    EscalationRule,
+    EscalationRuleConfig,
+    EscalationAction,
+    create_escalation_route,
 )
-from .state_machine import EscalationStateMachine
-from .stores import (
-    BaseEscalationStore,
-    EscalationMetrics,
-    EscalationStoreType,
-    InMemoryEscalationStore,
-    RedisEscalationStore,
-    SQLiteEscalationStore,
-    create_escalation_store,
+
+# Dashboard-specific adapters
+from .engine import (
+    DashboardEscalationService,
+    create_policy_from_db,
 )
 
 __all__ = [
-    # Models
+    # truthound core
+    "EscalationEngine",
+    "EscalationEngineConfig",
+    "EscalationPolicy",
+    "EscalationLevel",
+    "EscalationTarget",
+    "EscalationRecord",
+    "EscalationTrigger",
     "EscalationState",
     "TargetType",
-    "EscalationTarget",
-    "EscalationLevel",
-    "EscalationPolicy",
-    "EscalationIncident",
-    "EscalationEvent",
-    "StateTransition",
-    # State Machine
-    "EscalationStateMachine",
-    # Stores
-    "BaseEscalationStore",
-    "EscalationMetrics",
-    "EscalationStoreType",
+    "EscalationPolicyManager",
+    "EscalationPolicyConfig",
+    # Storage
     "InMemoryEscalationStore",
+    "create_store",
     "RedisEscalationStore",
     "SQLiteEscalationStore",
-    "create_escalation_store",
-    # Engine
-    "EscalationEngine",
-    "EscalationEngineConfig",
-    # Scheduler Backends
-    "BackendType",
-    "JobState",
-    "MisfirePolicy",
-    "SchedulerBackendConfig",
-    "JobData",
-    "JobExecutionResult",
-    "SchedulerBackend",
-    "InMemorySchedulerBackend",
-    "SQLAlchemySchedulerBackend",
-    "create_scheduler_backend",
-    # Scheduler
-    "EscalationSchedulerService",
-    "EscalationSchedulerConfig",
-    "EscalationHandler",
-    "EscalationResult",
-    "EscalationStrategy",
-    "DefaultEscalationHandler",
-    "LoggingEscalationHandler",
-    "TimeBasedEscalationStrategy",
-    "ImmediateEscalationStrategy",
-    "get_escalation_scheduler",
-    "reset_escalation_scheduler",
-    "start_escalation_scheduler",
-    "stop_escalation_scheduler",
+    "REDIS_AVAILABLE",
+    "SQLITE_AVAILABLE",
+    # Routing integration
+    "EscalationRule",
+    "EscalationRuleConfig",
+    "EscalationAction",
+    "create_escalation_route",
+    # Dashboard adapters
+    "DashboardEscalationService",
+    "create_policy_from_db",
 ]