puffinflow 2.dev0__py3-none-any.whl

puffinflow/core/observability/alerting.py

@@ -0,0 +1,73 @@
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Optional
+
+import aiohttp
+
+from .config import AlertingConfig
+from .interfaces import AlertingProvider, AlertSeverity
+
+
+@dataclass
+class Alert:
+    """Alert data structure"""
+
+    message: str
+    severity: AlertSeverity
+    attributes: dict[str, Any]
+    timestamp: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        if self.timestamp is None:
+            self.timestamp = datetime.now()
+        else:
+            pass
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "message": self.message,
+            "severity": self.severity.value,
+            "attributes": self.attributes,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+        }
+
+
+class WebhookAlerting(AlertingProvider):
+    """Webhook-based alerting"""
+
+    def __init__(self, config: AlertingConfig):
+        self.config = config
+
+    async def send_alert(
+        self,
+        message: str,
+        severity: AlertSeverity,
+        attributes: Optional[dict[str, Any]] = None,
+    ) -> None:
+        """Send alert via webhooks"""
+        if not self.config.enabled or not self.config.webhook_urls:
+            return
+
+        alert = Alert(message, severity, attributes or {})
+        payload = {"alert": alert.to_dict()}
+
+        tasks = []
+        for webhook_url in self.config.webhook_urls:
+            task = asyncio.create_task(self._send_webhook(webhook_url, payload))
+            tasks.append(task)
+
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _send_webhook(self, url: str, payload: dict[str, Any]) -> None:
+        """Send single webhook"""
+        try:
+            async with (
+                aiohttp.ClientSession() as session,
+                session.post(url, json=payload, timeout=30) as response,
+            ):
+                if response.status >= 400:
+                    print(f"Webhook failed: {response.status}")
+        except Exception as e:
+            print(f"Failed to send webhook to {url}: {e}")

puffinflow/core/observability/config.py

@@ -0,0 +1,127 @@
+import os
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class TracingConfig:
+    """Tracing configuration"""
+
+    enabled: bool = True
+    service_name: str = "puffinflow"
+    service_version: str = "1.0.0"
+    sample_rate: float = 1.0
+    otlp_endpoint: Optional[str] = None
+    jaeger_endpoint: Optional[str] = None
+    console_enabled: bool = False
+
+    @classmethod
+    def from_env(cls) -> "TracingConfig":
+        return cls(
+            enabled=os.getenv("TRACING_ENABLED", "true").lower() == "true",
+            service_name=os.getenv("SERVICE_NAME", "puffinflow"),
+            service_version=os.getenv("SERVICE_VERSION", "1.0.0"),
+            sample_rate=float(os.getenv("TRACE_SAMPLE_RATE", "1.0")),
+            otlp_endpoint=os.getenv("OTLP_ENDPOINT"),
+            jaeger_endpoint=os.getenv("JAEGER_ENDPOINT"),
+            console_enabled=os.getenv("TRACE_CONSOLE", "false").lower() == "true",
+        )
+
+
+@dataclass
+class MetricsConfig:
+    """Metrics configuration"""
+
+    enabled: bool = True
+    namespace: str = "puffinflow"
+    prometheus_port: int = 9090
+    prometheus_path: str = "/metrics"
+    collection_interval: float = 15.0
+    cardinality_limit: int = 10000
+
+    @classmethod
+    def from_env(cls) -> "MetricsConfig":
+        return cls(
+            enabled=os.getenv("METRICS_ENABLED", "true").lower() == "true",
+            namespace=os.getenv("METRICS_NAMESPACE", "puffinflow"),
+            prometheus_port=int(os.getenv("METRICS_PORT", "9090")),
+            prometheus_path=os.getenv("METRICS_PATH", "/metrics"),
+            collection_interval=float(os.getenv("METRICS_INTERVAL", "15.0")),
+            cardinality_limit=int(os.getenv("METRICS_CARDINALITY_LIMIT", "10000")),
+        )
+
+
+@dataclass
+class AlertingConfig:
+    """Alerting configuration"""
+
+    enabled: bool = True
+    evaluation_interval: float = 30.0
+    webhook_urls: list[str] = field(default_factory=list)
+    email_recipients: list[str] = field(default_factory=list)
+    slack_webhook_url: Optional[str] = None
+
+    @classmethod
+    def from_env(cls) -> "AlertingConfig":
+        webhook_urls = (
+            os.getenv("ALERT_WEBHOOK_URLS", "").split(",")
+            if os.getenv("ALERT_WEBHOOK_URLS")
+            else []
+        )
+        email_recipients = (
+            os.getenv("ALERT_EMAIL_RECIPIENTS", "").split(",")
+            if os.getenv("ALERT_EMAIL_RECIPIENTS")
+            else []
+        )
+
+        return cls(
+            enabled=os.getenv("ALERTING_ENABLED", "true").lower() == "true",
+            evaluation_interval=float(os.getenv("ALERT_EVALUATION_INTERVAL", "30.0")),
+            webhook_urls=[url.strip() for url in webhook_urls if url.strip()],
+            email_recipients=[
+                email.strip() for email in email_recipients if email.strip()
+            ],
+            slack_webhook_url=os.getenv("ALERT_SLACK_WEBHOOK"),
+        )
+
+
+@dataclass
+class EventsConfig:
+    """Events configuration"""
+
+    enabled: bool = True
+    buffer_size: int = 1000
+    batch_size: int = 100
+    flush_interval: float = 5.0
+
+    @classmethod
+    def from_env(cls) -> "EventsConfig":
+        return cls(
+            enabled=os.getenv("EVENTS_ENABLED", "true").lower() == "true",
+            buffer_size=int(os.getenv("EVENT_BUFFER_SIZE", "1000")),
+            batch_size=int(os.getenv("EVENT_BATCH_SIZE", "100")),
+            flush_interval=float(os.getenv("EVENT_FLUSH_INTERVAL", "5.0")),
+        )
+
+
+@dataclass
+class ObservabilityConfig:
+    """Complete observability configuration"""
+
+    enabled: bool = True
+    environment: str = "development"
+    tracing: TracingConfig = field(default_factory=TracingConfig)
+    metrics: MetricsConfig = field(default_factory=MetricsConfig)
+    alerting: AlertingConfig = field(default_factory=AlertingConfig)
+    events: EventsConfig = field(default_factory=EventsConfig)
+
+    @classmethod
+    def from_env(cls) -> "ObservabilityConfig":
+        return cls(
+            enabled=os.getenv("OBSERVABILITY_ENABLED", "true").lower() == "true",
+            environment=os.getenv("ENVIRONMENT", "development"),
+            tracing=TracingConfig.from_env(),
+            metrics=MetricsConfig.from_env(),
+            alerting=AlertingConfig.from_env(),
+            events=EventsConfig.from_env(),
+        )

puffinflow/core/observability/context.py

@@ -0,0 +1,88 @@
+import time
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Any, Optional
+
+from ..agent.context import Context
+from .core import ObservabilityManager
+from .interfaces import ObservabilityEvent, SpanType
+
+
+class ObservableContext(Context):
+    """Context with observability integration"""
+
+    def __init__(
+        self,
+        shared_state: dict[str, Any],
+        observability: Optional[ObservabilityManager] = None,
+    ):
+        super().__init__(shared_state)
+        self._observability = observability
+
+    @contextmanager
+    def trace(self, name: str, **attributes: Any) -> Generator[Any, None, None]:
+        """Create trace span with context"""
+        if not self._observability or not self._observability.tracing:
+            yield None
+            return
+
+        # Add context attributes
+        context_attrs = {
+            "workflow_id": self.get_variable("workflow_id"),
+            "agent_name": self.get_variable("agent_name"),
+            "state_name": self.get_variable("current_state"),
+            **attributes,
+        }
+
+        with self._observability.tracing.span(
+            name, SpanType.BUSINESS, **context_attrs
+        ) as span:
+            yield span
+
+    def metric(self, name: str, value: float, **labels: Any) -> None:
+        """Record metric with context"""
+        if not self._observability or not self._observability.metrics:
+            return
+
+        context_labels = {
+            "workflow_id": self.get_variable("workflow_id", "unknown"),
+            "agent_name": self.get_variable("agent_name", "unknown"),
+            **labels,
+        }
+
+        histogram = self._observability.histogram(
+            name, labels=list(context_labels.keys())
+        )
+        if histogram:
+            histogram.record(value, **context_labels)
+
+    def log(self, level: str, message: str, **kwargs: Any) -> None:
+        """Log with observability"""
+        if not self._observability or not self._observability.events:
+            return
+
+        event = ObservabilityEvent(
+            timestamp=datetime.fromtimestamp(time.time()),
+            event_type="log",
+            source="context",
+            level=level.upper(),
+            message=message,
+            attributes={
+                "workflow_id": self.get_variable("workflow_id"),
+                "agent_name": self.get_variable("agent_name"),
+                "state_name": self.get_variable("current_state"),
+                **kwargs,
+            },
+        )
+
+        import asyncio
+
+        try:
+            loop = asyncio.get_event_loop()
+            # Create task but don't store reference as it's fire-and-forget
+            task = loop.create_task(self._observability.events.process_event(event))
+            if task:
+                task.add_done_callback(lambda t: None)  # Prevent warnings
+        except RuntimeError:
+            pass

puffinflow/core/observability/core.py

@@ -0,0 +1,147 @@
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any, Optional
+
+from .alerting import WebhookAlerting
+from .config import ObservabilityConfig
+from .events import BufferedEventProcessor
+from .interfaces import (
+    AlertingProvider,
+    EventProcessor,
+    MetricsProvider,
+    TracingProvider,
+)
+from .metrics import PrometheusMetricsProvider
+from .tracing import OpenTelemetryTracingProvider
+
+
+class ObservabilityManager:
+    """Main observability coordinator"""
+
+    def __init__(self, config: Optional[ObservabilityConfig] = None):
+        self.config = config or ObservabilityConfig()
+        self._initialized = False
+        self._lock = threading.Lock()
+
+        # Initialize providers
+        self._tracing: Optional[TracingProvider] = None
+        self._metrics: Optional[MetricsProvider] = None
+        self._alerting: Optional[AlertingProvider] = None
+        self._events: Optional[EventProcessor] = None
+
+    @property
+    def tracing(self) -> Optional[TracingProvider]:
+        return self._tracing
+
+    @property
+    def metrics(self) -> Optional[MetricsProvider]:
+        return self._metrics
+
+    @property
+    def alerting(self) -> Optional[AlertingProvider]:
+        return self._alerting
+
+    @property
+    def events(self) -> Optional[EventProcessor]:
+        return self._events
+
+    async def initialize(self) -> None:
+        """Initialize all components"""
+        with self._lock:
+            if self._initialized:
+                return
+
+            # Initialize tracing
+            if self.config.tracing.enabled:
+                self._tracing = OpenTelemetryTracingProvider(self.config.tracing)
+
+            # Initialize metrics
+            if self.config.metrics.enabled:
+                self._metrics = PrometheusMetricsProvider(self.config.metrics)
+
+            # Initialize alerting
+            if self.config.alerting.enabled:
+                self._alerting = WebhookAlerting(self.config.alerting)
+
+            # Initialize events
+            if self.config.events.enabled:
+                self._events = BufferedEventProcessor(self.config.events)
+                await self._events.initialize()
+
+            self._initialized = True
+
+    async def shutdown(self) -> None:
+        """Shutdown all components"""
+        if self._events and hasattr(self._events, "shutdown"):
+            await self._events.shutdown()
+        self._initialized = False
+
+    # Convenience methods
+    @contextmanager
+    def trace(self, name: str, **attributes: Any) -> Iterator[Any]:
+        """Create trace span"""
+        if self._tracing and hasattr(self._tracing, "span"):
+            with self._tracing.span(name, **attributes) as span:
+                yield span
+        else:
+            yield None
+
+    def counter(
+        self, name: str, description: str = "", labels: Optional[list[str]] = None
+    ) -> Any:
+        """Create counter metric"""
+        if self._metrics:
+            return self._metrics.counter(name, description, labels)
+        return None
+
+    def gauge(
+        self, name: str, description: str = "", labels: Optional[list[str]] = None
+    ) -> Any:
+        """Create gauge metric"""
+        if self._metrics:
+            return self._metrics.gauge(name, description, labels)
+        return None
+
+    def histogram(
+        self, name: str, description: str = "", labels: Optional[list[str]] = None
+    ) -> Any:
+        """Create histogram metric"""
+        if self._metrics:
+            return self._metrics.histogram(name, description, labels)
+        return None
+
+    async def alert(
+        self, message: str, severity: str = "warning", **attributes: Any
+    ) -> None:
+        """Send alert"""
+        if self._alerting:
+            from .interfaces import AlertSeverity
+
+            sev = AlertSeverity(severity.lower())
+            await self._alerting.send_alert(message, sev, attributes)
+
+
+# Global instance management
+_global_observability: Optional[ObservabilityManager] = None
+_lock = threading.Lock()
+
+
+def get_observability() -> ObservabilityManager:
+    """Get global observability instance"""
+    global _global_observability
+    with _lock:
+        if _global_observability is None:
+            _global_observability = ObservabilityManager()
+        return _global_observability
+
+
+async def setup_observability(
+    config: Optional[ObservabilityConfig] = None,
+) -> ObservabilityManager:
+    """Setup and initialize observability"""
+    global _global_observability
+    with _lock:
+        _global_observability = ObservabilityManager(config)
+    await _global_observability.initialize()
+    return _global_observability

puffinflow/core/observability/decorators.py

@@ -0,0 +1,105 @@
+import asyncio
+import functools
+import time
+from typing import Any, Callable, Optional
+
+from .core import get_observability
+from .interfaces import SpanType
+
+
+def observe(
+    name: Optional[str] = None,
+    span_type: SpanType = SpanType.BUSINESS,
+    **span_attributes: Any,
+) -> Callable[[Callable], Callable]:
+    """Observability decorator"""
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+            observability = get_observability()
+            operation_name = name or f"{func.__module__}.{func.__name__}"
+
+            if observability.tracing and hasattr(observability.tracing, "span"):
+                with observability.tracing.span(
+                    operation_name, span_type, function=func.__name__, **span_attributes
+                ) as span:
+                    start_time = time.time()
+                    try:
+                        result = await func(*args, **kwargs)
+                        duration = time.time() - start_time
+
+                        if span:
+                            span.set_attribute("function.duration", duration)
+                            span.set_status("ok")
+
+                        return result
+                    except Exception as e:
+                        if span:
+                            span.record_exception(e)
+                        raise
+            else:
+                return await func(*args, **kwargs)
+
+        @functools.wraps(func)
+        def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+            observability = get_observability()
+            operation_name = name or f"{func.__module__}.{func.__name__}"
+
+            if observability.tracing and hasattr(observability.tracing, "span"):
+                with observability.tracing.span(
+                    operation_name, span_type, **span_attributes
+                ) as span:
+                    try:
+                        result = func(*args, **kwargs)
+                        if span:
+                            span.set_status("ok")
+                        return result
+                    except Exception as e:
+                        if span:
+                            span.record_exception(e)
+                        raise
+            else:
+                return func(*args, **kwargs)
+
+        return async_wrapper if asyncio.iscoroutinefunction(func) else sync_wrapper
+
+    return decorator
+
+
+def trace_state(
+    span_type: SpanType = SpanType.STATE, **span_attributes: Any
+) -> Callable[[Callable], Callable]:
+    """Decorator for tracing PuffinFlow states"""
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        async def wrapper(context: Any, *args: Any, **kwargs: Any) -> Any:
+            observability = get_observability()
+
+            if observability.tracing and hasattr(observability.tracing, "span"):
+                attrs = {
+                    "state.name": func.__name__,
+                    "workflow.id": context.get_variable("workflow_id"),
+                    "agent.name": context.get_variable("agent_name"),
+                    **span_attributes,
+                }
+
+                with observability.tracing.span(
+                    f"state.{func.__name__}", span_type, **attrs
+                ) as span:
+                    try:
+                        result = await func(context, *args, **kwargs)
+                        if span:
+                            span.set_status("ok")
+                        return result
+                    except Exception as e:
+                        if span:
+                            span.record_exception(e)
+                        raise
+            else:
+                return await func(context, *args, **kwargs)
+
+        return wrapper
+
+    return decorator

puffinflow/core/observability/events.py

@@ -0,0 +1,71 @@
+import asyncio
+import contextlib
+from collections import deque
+from typing import Any, Callable, Optional
+
+from .config import EventsConfig
+from .interfaces import EventProcessor, ObservabilityEvent
+
+
+class BufferedEventProcessor(EventProcessor):
+    """Buffered event processor"""
+
+    def __init__(self, config: EventsConfig):
+        self.config = config
+        self.buffer: Any = deque(maxlen=config.buffer_size)
+        self.subscribers: list[Callable[[ObservabilityEvent], None]] = []
+        self._task: Optional[Any] = None
+        self._shutdown = False
+
+    async def initialize(self) -> None:
+        """Initialize event processor"""
+        if self.config.enabled:
+            self._task = asyncio.create_task(self._process_loop())
+
+    async def shutdown(self) -> None:
+        """Shutdown event processor"""
+        self._shutdown = True
+        if self._task:
+            self._task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._task
+
+    async def process_event(self, event: ObservabilityEvent) -> None:
+        """Add event to buffer"""
+        if self.config.enabled:
+            self.buffer.append(event)
+
+    def subscribe(self, callback: Callable[[ObservabilityEvent], None]) -> None:
+        """Subscribe to events"""
+        self.subscribers.append(callback)
+
+    async def _process_loop(self) -> None:
+        """Process events from buffer"""
+        while not self._shutdown:
+            try:
+                events_to_process = []
+
+                # Collect batch
+                for _ in range(min(self.config.batch_size, len(self.buffer))):
+                    if self.buffer:
+                        events_to_process.append(self.buffer.popleft())
+
+                # Process events
+                for event in events_to_process:
+                    for subscriber in self.subscribers:
+                        try:
+                            if asyncio.iscoroutinefunction(subscriber):
+                                await subscriber(event)
+                            else:
+                                subscriber(event)
+                        except Exception as e:
+                            print(f"Event processing error: {e}")
+
+                # Wait for next batch
+                await asyncio.sleep(self.config.flush_interval)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                print(f"Event processing loop error: {e}")
+                await asyncio.sleep(1)