smartify-ai 0.1.0__py3-none-any.whl

smartify/notifications/webhook.py

@@ -0,0 +1,556 @@
+"""Webhook notification system for Smartify grid events.
+
+Sends HTTP POST notifications for grid lifecycle events:
+- run_started: Grid execution has begun
+- run_completed: Grid finished successfully
+- run_failed: Grid execution failed
+- run_paused: Grid was paused
+- run_stopped: Grid was stopped
+- approval_needed: Human approval required
+- breaker_tripped: A guardrail breaker was triggered
+- node_completed: Individual node finished (optional, high volume)
+
+Webhooks are configured per-grid in the GridSpec, with global defaults
+available at the server level.
+"""
+
+import asyncio
+import hashlib
+import hmac
+import json
+import logging
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Callable, Dict, List, Optional
+from uuid import uuid4
+
+import httpx
+
+
+logger = logging.getLogger(__name__)
+
+
+class EventType(str, Enum):
+    """Types of events that can trigger webhooks."""
+    RUN_STARTED = "run_started"
+    RUN_COMPLETED = "run_completed"
+    RUN_FAILED = "run_failed"
+    RUN_PAUSED = "run_paused"
+    RUN_STOPPED = "run_stopped"
+    APPROVAL_NEEDED = "approval_needed"
+    BREAKER_TRIPPED = "breaker_tripped"
+    NODE_COMPLETED = "node_completed"
+
+
+@dataclass
+class WebhookEvent:
+    """A webhook event to be delivered."""
+    id: str
+    type: EventType
+    grid_id: str
+    timestamp: datetime
+    data: Dict[str, Any]
+    
+    # Delivery tracking
+    attempts: int = 0
+    last_attempt: Optional[datetime] = None
+    last_error: Optional[str] = None
+    delivered: bool = False
+    
+    def to_payload(self) -> Dict[str, Any]:
+        """Convert to webhook payload."""
+        return {
+            "event_id": self.id,
+            "event_type": self.type.value,
+            "grid_id": self.grid_id,
+            "timestamp": self.timestamp.isoformat(),
+            "data": self.data,
+        }
+
+
+@dataclass
+class WebhookConfig:
+    """Configuration for a webhook endpoint."""
+    url: str
+    events: List[EventType] = field(default_factory=lambda: list(EventType))
+    
+    # Authentication
+    secret: Optional[str] = None  # For HMAC signature
+    headers: Dict[str, str] = field(default_factory=dict)
+    
+    # Retry configuration
+    max_retries: int = 3
+    retry_delay_seconds: float = 5.0
+    retry_backoff_multiplier: float = 2.0
+    timeout_seconds: float = 30.0
+    
+    # Filtering
+    enabled: bool = True
+    
+    def accepts_event(self, event_type: EventType) -> bool:
+        """Check if this webhook accepts the given event type."""
+        return self.enabled and event_type in self.events
+
+
+@dataclass 
+class DeliveryResult:
+    """Result of a webhook delivery attempt."""
+    success: bool
+    status_code: Optional[int] = None
+    response_body: Optional[str] = None
+    error: Optional[str] = None
+    duration_ms: float = 0.0
+
+
+class WebhookNotifier:
+    """Manages webhook notifications for grid events.
+    
+    Usage:
+        notifier = WebhookNotifier()
+        
+        # Add webhook endpoints
+        notifier.add_webhook(WebhookConfig(
+            url="https://example.com/webhooks/smartify",
+            events=[EventType.RUN_COMPLETED, EventType.RUN_FAILED],
+            secret="my-secret-key",
+        ))
+        
+        # Send events
+        await notifier.notify(
+            event_type=EventType.RUN_COMPLETED,
+            grid_id="my-grid",
+            data={"outputs": {...}, "duration_seconds": 45.2}
+        )
+    """
+    
+    def __init__(
+        self,
+        default_timeout: float = 30.0,
+        max_concurrent_deliveries: int = 10,
+    ):
+        self.webhooks: List[WebhookConfig] = []
+        self.default_timeout = default_timeout
+        self.max_concurrent = max_concurrent_deliveries
+        
+        # Event history for debugging/auditing
+        self.event_history: List[WebhookEvent] = []
+        self.max_history_size: int = 1000
+        
+        # Delivery semaphore to limit concurrency
+        self._semaphore = asyncio.Semaphore(max_concurrent_deliveries)
+        
+        # Background retry queue
+        self._retry_queue: List[tuple[WebhookEvent, WebhookConfig]] = []
+        self._retry_task: Optional[asyncio.Task] = None
+    
+    def add_webhook(self, config: WebhookConfig) -> None:
+        """Add a webhook endpoint."""
+        self.webhooks.append(config)
+        logger.info(f"Added webhook: {config.url} for events: {[e.value for e in config.events]}")
+    
+    def remove_webhook(self, url: str) -> bool:
+        """Remove a webhook by URL. Returns True if found and removed."""
+        for i, webhook in enumerate(self.webhooks):
+            if webhook.url == url:
+                self.webhooks.pop(i)
+                logger.info(f"Removed webhook: {url}")
+                return True
+        return False
+    
+    def add_webhooks_from_grid(self, grid_spec: Any) -> None:
+        """Add webhooks configured in a grid specification.
+        
+        Looks for webhooks in grid.notifications.webhooks
+        """
+        if not hasattr(grid_spec, 'notifications') or not grid_spec.notifications:
+            return
+        
+        notifications = grid_spec.notifications
+        if not hasattr(notifications, 'webhooks') or not notifications.webhooks:
+            return
+        
+        for webhook_config in notifications.webhooks:
+            # Convert from grid spec format
+            events = []
+            if hasattr(webhook_config, 'events') and webhook_config.events:
+                for event_name in webhook_config.events:
+                    try:
+                        events.append(EventType(event_name))
+                    except ValueError:
+                        logger.warning(f"Unknown event type: {event_name}")
+            else:
+                events = list(EventType)  # All events by default
+            
+            config = WebhookConfig(
+                url=webhook_config.url,
+                events=events,
+                secret=getattr(webhook_config, 'secret', None),
+                headers=getattr(webhook_config, 'headers', {}) or {},
+                max_retries=getattr(webhook_config, 'maxRetries', 3),
+                timeout_seconds=getattr(webhook_config, 'timeout', 30.0),
+                enabled=getattr(webhook_config, 'enabled', True),
+            )
+            self.add_webhook(config)
+    
+    async def notify(
+        self,
+        event_type: EventType,
+        grid_id: str,
+        data: Dict[str, Any],
+        wait_for_delivery: bool = False,
+    ) -> WebhookEvent:
+        """Send a notification to all configured webhooks.
+        
+        Args:
+            event_type: Type of event
+            grid_id: ID of the grid this event relates to
+            data: Event-specific data payload
+            wait_for_delivery: If True, wait for all deliveries to complete
+            
+        Returns:
+            The created WebhookEvent
+        """
+        event = WebhookEvent(
+            id=f"evt-{uuid4().hex[:12]}",
+            type=event_type,
+            grid_id=grid_id,
+            timestamp=datetime.now(),
+            data=data,
+        )
+        
+        # Track in history
+        self._add_to_history(event)
+        
+        # Find webhooks that accept this event
+        matching_webhooks = [w for w in self.webhooks if w.accepts_event(event_type)]
+        
+        if not matching_webhooks:
+            logger.debug(f"No webhooks configured for event {event_type.value}")
+            return event
+        
+        logger.info(f"Sending {event_type.value} event to {len(matching_webhooks)} webhook(s)")
+        
+        # Create delivery tasks
+        tasks = [
+            self._deliver_with_retry(event, webhook)
+            for webhook in matching_webhooks
+        ]
+        
+        if wait_for_delivery:
+            await asyncio.gather(*tasks, return_exceptions=True)
+        else:
+            # Fire and forget, but track tasks
+            for task in tasks:
+                asyncio.create_task(task)
+        
+        return event
+    
+    async def notify_run_started(
+        self,
+        grid_id: str,
+        grid_name: str,
+        inputs: Dict[str, Any],
+        **kwargs,
+    ) -> WebhookEvent:
+        """Convenience method for run_started events."""
+        return await self.notify(
+            EventType.RUN_STARTED,
+            grid_id,
+            {
+                "grid_name": grid_name,
+                "inputs": inputs,
+                "started_at": datetime.now().isoformat(),
+                **kwargs,
+            },
+        )
+    
+    async def notify_run_completed(
+        self,
+        grid_id: str,
+        outputs: Dict[str, Any],
+        duration_seconds: float,
+        total_tokens: int,
+        total_cost: float,
+        **kwargs,
+    ) -> WebhookEvent:
+        """Convenience method for run_completed events."""
+        return await self.notify(
+            EventType.RUN_COMPLETED,
+            grid_id,
+            {
+                "outputs": outputs,
+                "duration_seconds": duration_seconds,
+                "total_tokens": total_tokens,
+                "total_cost": total_cost,
+                "completed_at": datetime.now().isoformat(),
+                **kwargs,
+            },
+        )
+    
+    async def notify_run_failed(
+        self,
+        grid_id: str,
+        error: str,
+        node_id: Optional[str] = None,
+        duration_seconds: Optional[float] = None,
+        **kwargs,
+    ) -> WebhookEvent:
+        """Convenience method for run_failed events."""
+        return await self.notify(
+            EventType.RUN_FAILED,
+            grid_id,
+            {
+                "error": error,
+                "failed_node": node_id,
+                "duration_seconds": duration_seconds,
+                "failed_at": datetime.now().isoformat(),
+                **kwargs,
+            },
+        )
+    
+    async def notify_breaker_tripped(
+        self,
+        grid_id: str,
+        breaker_type: str,
+        current_value: float,
+        limit: float,
+        action: str,
+        node_id: Optional[str] = None,
+        **kwargs,
+    ) -> WebhookEvent:
+        """Convenience method for breaker_tripped events."""
+        return await self.notify(
+            EventType.BREAKER_TRIPPED,
+            grid_id,
+            {
+                "breaker_type": breaker_type,
+                "current_value": current_value,
+                "limit": limit,
+                "action": action,
+                "node_id": node_id,
+                "tripped_at": datetime.now().isoformat(),
+                **kwargs,
+            },
+        )
+    
+    async def notify_approval_needed(
+        self,
+        grid_id: str,
+        node_id: str,
+        approval_id: str,
+        prompt: str,
+        context: Dict[str, Any],
+        callback_url: str,
+        expires_at: datetime,
+        **kwargs,
+    ) -> WebhookEvent:
+        """Convenience method for approval_needed events."""
+        return await self.notify(
+            EventType.APPROVAL_NEEDED,
+            grid_id,
+            {
+                "node_id": node_id,
+                "approval_id": approval_id,
+                "prompt": prompt,
+                "context": context,
+                "callback_url": callback_url,
+                "expires_at": expires_at.isoformat(),
+                **kwargs,
+            },
+        )
+    
+    async def _deliver_with_retry(
+        self,
+        event: WebhookEvent,
+        config: WebhookConfig,
+    ) -> DeliveryResult:
+        """Deliver event to webhook with retry logic."""
+        async with self._semaphore:
+            delay = config.retry_delay_seconds
+            
+            for attempt in range(config.max_retries + 1):
+                event.attempts += 1
+                event.last_attempt = datetime.now()
+                
+                result = await self._deliver(event, config)
+                
+                if result.success:
+                    event.delivered = True
+                    logger.info(
+                        f"Delivered event {event.id} to {config.url} "
+                        f"(attempt {attempt + 1}, {result.duration_ms:.0f}ms)"
+                    )
+                    return result
+                
+                event.last_error = result.error
+                
+                if attempt < config.max_retries:
+                    logger.warning(
+                        f"Webhook delivery failed for {event.id} to {config.url}: "
+                        f"{result.error}. Retrying in {delay}s..."
+                    )
+                    await asyncio.sleep(delay)
+                    delay *= config.retry_backoff_multiplier
+                else:
+                    logger.error(
+                        f"Webhook delivery failed permanently for {event.id} to {config.url} "
+                        f"after {config.max_retries + 1} attempts: {result.error}"
+                    )
+            
+            return result
+    
+    async def _deliver(
+        self,
+        event: WebhookEvent,
+        config: WebhookConfig,
+    ) -> DeliveryResult:
+        """Deliver a single webhook request."""
+        start_time = time.monotonic()
+        payload = event.to_payload()
+        body = json.dumps(payload)
+        
+        # Build headers
+        headers = {
+            "Content-Type": "application/json",
+            "User-Agent": "Smartify-Webhook/1.0",
+            "X-Smartify-Event": event.type.value,
+            "X-Smartify-Event-ID": event.id,
+            "X-Smartify-Grid-ID": event.grid_id,
+            "X-Smartify-Timestamp": str(int(event.timestamp.timestamp())),
+            **config.headers,
+        }
+        
+        # Add HMAC signature if secret configured
+        if config.secret:
+            signature = self._compute_signature(body, config.secret)
+            headers["X-Smartify-Signature"] = f"sha256={signature}"
+        
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    config.url,
+                    content=body,
+                    headers=headers,
+                    timeout=config.timeout_seconds,
+                )
+                
+                duration_ms = (time.monotonic() - start_time) * 1000
+                
+                # Accept 2xx as success
+                if 200 <= response.status_code < 300:
+                    return DeliveryResult(
+                        success=True,
+                        status_code=response.status_code,
+                        response_body=response.text[:1000],  # Truncate
+                        duration_ms=duration_ms,
+                    )
+                else:
+                    return DeliveryResult(
+                        success=False,
+                        status_code=response.status_code,
+                        response_body=response.text[:1000],
+                        error=f"HTTP {response.status_code}: {response.text[:200]}",
+                        duration_ms=duration_ms,
+                    )
+                    
+        except httpx.TimeoutException:
+            duration_ms = (time.monotonic() - start_time) * 1000
+            return DeliveryResult(
+                success=False,
+                error=f"Timeout after {config.timeout_seconds}s",
+                duration_ms=duration_ms,
+            )
+        except httpx.RequestError as e:
+            duration_ms = (time.monotonic() - start_time) * 1000
+            return DeliveryResult(
+                success=False,
+                error=f"Request error: {str(e)}",
+                duration_ms=duration_ms,
+            )
+        except Exception as e:
+            duration_ms = (time.monotonic() - start_time) * 1000
+            return DeliveryResult(
+                success=False,
+                error=f"Unexpected error: {str(e)}",
+                duration_ms=duration_ms,
+            )
+    
+    def _compute_signature(self, body: str, secret: str) -> str:
+        """Compute HMAC-SHA256 signature for webhook payload."""
+        return hmac.new(
+            secret.encode('utf-8'),
+            body.encode('utf-8'),
+            hashlib.sha256,
+        ).hexdigest()
+    
+    def _add_to_history(self, event: WebhookEvent) -> None:
+        """Add event to history, trimming if needed."""
+        self.event_history.append(event)
+        if len(self.event_history) > self.max_history_size:
+            self.event_history = self.event_history[-self.max_history_size:]
+    
+    def get_recent_events(
+        self,
+        limit: int = 50,
+        event_type: Optional[EventType] = None,
+        grid_id: Optional[str] = None,
+    ) -> List[WebhookEvent]:
+        """Get recent events from history with optional filtering."""
+        events = self.event_history
+        
+        if event_type:
+            events = [e for e in events if e.type == event_type]
+        
+        if grid_id:
+            events = [e for e in events if e.grid_id == grid_id]
+        
+        return events[-limit:]
+    
+    def get_failed_deliveries(self) -> List[WebhookEvent]:
+        """Get events that failed to deliver."""
+        return [e for e in self.event_history if not e.delivered and e.attempts > 0]
+
+
+# Singleton for global access
+_webhook_notifier: Optional[WebhookNotifier] = None
+
+
+def get_webhook_notifier() -> WebhookNotifier:
+    """Get the global webhook notifier instance."""
+    global _webhook_notifier
+    if _webhook_notifier is None:
+        _webhook_notifier = WebhookNotifier()
+    return _webhook_notifier
+
+
+def configure_webhook_notifier(
+    webhooks: Optional[List[Dict[str, Any]]] = None,
+    default_timeout: float = 30.0,
+) -> WebhookNotifier:
+    """Configure the global webhook notifier."""
+    global _webhook_notifier
+    
+    _webhook_notifier = WebhookNotifier(default_timeout=default_timeout)
+    
+    if webhooks:
+        for wh in webhooks:
+            events = []
+            for event_name in wh.get('events', []):
+                try:
+                    events.append(EventType(event_name))
+                except ValueError:
+                    pass
+            
+            config = WebhookConfig(
+                url=wh['url'],
+                events=events or list(EventType),
+                secret=wh.get('secret'),
+                headers=wh.get('headers', {}),
+                max_retries=wh.get('maxRetries', 3),
+                timeout_seconds=wh.get('timeout', 30.0),
+            )
+            _webhook_notifier.add_webhook(config)
+    
+    return _webhook_notifier

smartify/state/__init__.py

@@ -0,0 +1,46 @@
+"""State persistence for Smartify."""
+
+from smartify.state.store import (
+    RunStatus,
+    RunRecord,
+    NodeOutput,
+    StateStore,
+    InMemoryStore,
+    SQLiteStore,
+    get_default_store,
+    set_default_store,
+)
+from smartify.state.checkpoint import (
+    CheckpointStore,
+    CheckpointStatus,
+    Checkpoint,
+    WebhookRetryJob,
+    WebhookDeliveryStatus,
+    get_checkpoint_store,
+)
+from smartify.state.resume import (
+    ResumeManager,
+    create_checkpointed_orchestrator,
+)
+
+__all__ = [
+    # Store
+    "RunStatus",
+    "RunRecord",
+    "NodeOutput",
+    "StateStore",
+    "InMemoryStore",
+    "SQLiteStore",
+    "get_default_store",
+    "set_default_store",
+    # Checkpoint
+    "CheckpointStore",
+    "CheckpointStatus",
+    "Checkpoint",
+    "WebhookRetryJob",
+    "WebhookDeliveryStatus",
+    "get_checkpoint_store",
+    # Resume
+    "ResumeManager",
+    "create_checkpointed_orchestrator",
+]