flowyml 1.1.0__py3-none-any.whl

flowyml/monitoring/data.py

@@ -0,0 +1,102 @@
+"""Data monitoring and drift detection."""
+
+from typing import Any
+import numpy as np
+
+
+def compute_stats(data: list | np.ndarray) -> dict[str, float]:
+    """Compute basic statistics for a dataset.
+
+    Args:
+        data: Input data (list or numpy array)
+
+    Returns:
+        Dictionary of statistics
+    """
+    if isinstance(data, list):
+        data = np.array(data)
+
+    if len(data) == 0:
+        return {}
+
+    stats = {
+        "count": float(len(data)),
+        "mean": float(np.mean(data)),
+        "std": float(np.std(data)),
+        "min": float(np.min(data)),
+        "max": float(np.max(data)),
+        "median": float(np.median(data)),
+        "q25": float(np.percentile(data, 25)),
+        "q75": float(np.percentile(data, 75)),
+    }
+
+    return stats
+
+
+def calculate_psi(expected: np.ndarray, actual: np.ndarray, buckets: int = 10) -> float:
+    """Calculate Population Stability Index (PSI) to detect drift.
+
+    Args:
+        expected: Reference distribution
+        actual: Current distribution
+        buckets: Number of buckets for histogram
+
+    Returns:
+        PSI value
+    """
+
+    def scale_range(input_array, min_val, max_val):
+        input_array += -(np.min(input_array))
+        input_array /= np.max(input_array) / (max_val - min_val)
+        input_array += min_val
+        return input_array
+
+    breakpoints = np.arange(0, buckets + 1) / (buckets) * 100
+    breakpoints = np.percentile(expected, breakpoints)
+
+    expected_percents = np.histogram(expected, breakpoints)[0] / len(expected)
+    actual_percents = np.histogram(actual, breakpoints)[0] / len(actual)
+
+    def sub_psi(e_perc, a_perc):
+        if a_perc == 0:
+            a_perc = 0.0001
+        if e_perc == 0:
+            e_perc = 0.0001
+
+        value = (e_perc - a_perc) * np.log(e_perc / a_perc)
+        return value
+
+    psi_value = np.sum([sub_psi(expected_percents[i], actual_percents[i]) for i in range(0, len(expected_percents))])
+
+    return psi_value
+
+
+def detect_drift(
+    reference_data: list | np.ndarray,
+    current_data: list | np.ndarray,
+    threshold: float = 0.1,
+) -> dict[str, Any]:
+    """Detect data drift between reference and current data.
+
+    Args:
+        reference_data: Reference dataset (e.g. training data)
+        current_data: Current dataset (e.g. inference data)
+        threshold: PSI threshold for drift warning
+
+    Returns:
+        Drift detection result
+    """
+    if isinstance(reference_data, list):
+        reference_data = np.array(reference_data)
+    if isinstance(current_data, list):
+        current_data = np.array(current_data)
+
+    psi = calculate_psi(reference_data, current_data)
+
+    return {
+        "drift_detected": psi > threshold,
+        "psi": psi,
+        "threshold": threshold,
+        "reference_stats": compute_stats(reference_data),
+        "current_stats": compute_stats(current_data),
+    }

flowyml/monitoring/llm.py

@@ -0,0 +1,160 @@
+"""LLM Monitoring and Observability module."""
+
+import time
+import uuid
+from dataclasses import dataclass, field, asdict
+from typing import Any
+import functools
+
+
+@dataclass
+class LLMEvent:
+    """Event representing an LLM interaction."""
+
+    event_id: str
+    trace_id: str
+    parent_id: str | None
+    event_type: str  # 'llm', 'tool', 'chain', 'agent'
+    name: str
+    inputs: dict[str, Any]
+    outputs: dict[str, Any] | None = None
+    start_time: float = field(default_factory=time.time)
+    end_time: float | None = None
+    duration: float | None = None
+    status: str = "running"  # 'running', 'success', 'error'
+    error: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    # Token usage and cost
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    cost: float = 0.0
+    model: str | None = None
+
+    def end(self, outputs: dict[str, Any] | None = None, error: str | None = None) -> None:
+        """End the event."""
+        self.end_time = time.time()
+        self.duration = self.end_time - self.start_time
+        self.outputs = outputs
+        if error:
+            self.status = "error"
+            self.error = str(error)
+        else:
+            self.status = "success"
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return asdict(self)
+
+
+class LLMTracer:
+    """Tracer for LLM calls."""
+
+    def __init__(self):
+        self.current_trace_id: str | None = None
+        self.event_stack: list[LLMEvent] = []
+        self._metadata_store = None
+
+    @property
+    def metadata_store(self):
+        if self._metadata_store is None:
+            from flowyml.storage.metadata import SQLiteMetadataStore
+
+            self._metadata_store = SQLiteMetadataStore()
+        return self._metadata_store
+
+    def start_trace(self, name: str = "root") -> str:
+        """Start a new trace."""
+        self.current_trace_id = str(uuid.uuid4())
+        return self.current_trace_id
+
+    def start_event(
+        self,
+        name: str,
+        event_type: str,
+        inputs: dict[str, Any],
+        metadata: dict[str, Any] | None = None,
+        parent_id: str | None = None,
+    ) -> LLMEvent:
+        """Start a new event."""
+        if not self.current_trace_id:
+            self.start_trace()
+
+        event = LLMEvent(
+            event_id=str(uuid.uuid4()),
+            trace_id=self.current_trace_id,
+            parent_id=parent_id or (self.event_stack[-1].event_id if self.event_stack else None),
+            event_type=event_type,
+            name=name,
+            inputs=inputs,
+            metadata=metadata or {},
+        )
+        self.event_stack.append(event)
+        return event
+
+    def end_event(
+        self,
+        outputs: dict[str, Any] | None = None,
+        error: str | None = None,
+        metrics: dict[str, Any] | None = None,
+    ):
+        """End the current event."""
+        if not self.event_stack:
+            return None
+
+        event = self.event_stack.pop()
+        event.end(outputs, error)
+
+        if metrics:
+            event.prompt_tokens = metrics.get("prompt_tokens", 0)
+            event.completion_tokens = metrics.get("completion_tokens", 0)
+            event.total_tokens = metrics.get("total_tokens", 0)
+            event.cost = metrics.get("cost", 0.0)
+            event.model = metrics.get("model")
+
+        # Save to storage
+        self.metadata_store.save_trace_event(event.to_dict())
+
+        return event
+
+
+# Global tracer instance
+tracer = LLMTracer()
+
+
+def trace_llm(name: str = None, event_type: str = "llm"):
+    """Decorator to trace LLM calls."""
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            event_name = name or func.__name__
+
+            # Capture inputs
+            inputs = {
+                "args": [str(a) for a in args],
+                "kwargs": {k: str(v) for k, v in kwargs.items()},
+            }
+
+            tracer.start_event(event_name, event_type, inputs)
+
+            try:
+                result = func(*args, **kwargs)
+
+                # Try to extract metrics if result has them (e.g. OpenAI response)
+                metrics = {}
+                if hasattr(result, "usage"):  # OpenAI style
+                    metrics["prompt_tokens"] = getattr(result.usage, "prompt_tokens", 0)
+                    metrics["completion_tokens"] = getattr(result.usage, "completion_tokens", 0)
+                    metrics["total_tokens"] = getattr(result.usage, "total_tokens", 0)
+
+                tracer.end_event(outputs={"result": str(result)}, metrics=metrics)
+                return result
+            except Exception as e:
+                tracer.end_event(error=str(e))
+                raise e
+
+        return wrapper
+
+    return decorator

flowyml/monitoring/monitor.py

@@ -0,0 +1,57 @@
+from flowyml.monitoring.alerts import alert_manager, AlertLevel
+
+
+class Monitor:
+    """Base class for monitors."""
+
+    def __init__(self, name: str):
+        self.name = name
+
+    def check(self) -> bool:
+        """Perform check. Return True if healthy."""
+        raise NotImplementedError
+
+
+class SystemMonitor(Monitor):
+    """Monitors system resources."""
+
+    def check(self) -> bool:
+        try:
+            import psutil
+        except ImportError:
+            return True  # Skip if psutil not installed
+
+        cpu = psutil.cpu_percent()
+        mem = psutil.virtual_memory().percent
+
+        if cpu > 90:
+            alert_manager.send_alert(
+                "High CPU Usage",
+                f"CPU usage is at {cpu}%",
+                AlertLevel.WARNING,
+            )
+            return False
+
+        if mem > 90:
+            alert_manager.send_alert(
+                "High Memory Usage",
+                f"Memory usage is at {mem}%",
+                AlertLevel.WARNING,
+            )
+            return False
+
+        return True
+
+
+class PipelineMonitor(Monitor):
+    """Monitors pipeline execution health."""
+
+    def __init__(self, pipeline_name: str):
+        super().__init__(f"pipeline-{pipeline_name}")
+        self.pipeline_name = pipeline_name
+        self.failed_runs_threshold = 3
+
+    def check(self) -> bool:
+        # Logic to check recent runs from metadata store
+        # For now, placeholder
+        return True

flowyml/monitoring/notifications.py

@@ -0,0 +1,246 @@
+"""Notification system for pipeline events."""
+
+import os
+from typing import Any
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime
+import contextlib
+
+
+@dataclass
+class Notification:
+    """A notification about a pipeline event."""
+
+    title: str
+    message: str
+    level: str  # 'info', 'warning', 'error', 'success'
+    timestamp: datetime
+    metadata: dict[str, Any]
+
+
+class NotificationChannel(ABC):
+    """Base class for notification channels."""
+
+    @abstractmethod
+    def send(self, notification: Notification) -> bool:
+        """Send a notification."""
+        pass
+
+
+class ConsoleNotifier(NotificationChannel):
+    """Print notifications to console."""
+
+    def send(self, notification: Notification) -> bool:
+        {
+            "info": "ℹ️",
+            "warning": "⚠️",
+            "error": "❌",
+            "success": "✅",
+        }.get(notification.level, "📢")
+
+        return True
+
+
+class SlackNotifier(NotificationChannel):
+    """Send notifications to Slack."""
+
+    def __init__(self, webhook_url: str | None = None):
+        self.webhook_url = webhook_url or os.getenv("SLACK_WEBHOOK_URL")
+
+    def send(self, notification: Notification) -> bool:
+        if not self.webhook_url:
+            return False
+
+        try:
+            import requests
+
+            color = {
+                "info": "#36a64f",
+                "warning": "#ff9900",
+                "error": "#ff0000",
+                "success": "#00ff00",
+            }.get(notification.level, "#cccccc")
+
+            payload = {
+                "attachments": [
+                    {
+                        "color": color,
+                        "title": notification.title,
+                        "text": notification.message,
+                        "footer": "flowyml",
+                        "ts": int(notification.timestamp.timestamp()),
+                    },
+                ],
+            }
+
+            response = requests.post(self.webhook_url, json=payload)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+
+class EmailNotifier(NotificationChannel):
+    """Send notifications via email."""
+
+    def __init__(
+        self,
+        smtp_host: str | None = None,
+        smtp_port: int = 587,
+        username: str | None = None,
+        password: str | None = None,
+        from_addr: str | None = None,
+        to_addrs: list[str] | None = None,
+    ):
+        self.smtp_host = smtp_host or os.getenv("SMTP_HOST")
+        self.smtp_port = smtp_port
+        self.username = username or os.getenv("SMTP_USERNAME")
+        self.password = password or os.getenv("SMTP_PASSWORD")
+        self.from_addr = from_addr or os.getenv("SMTP_FROM")
+        self.to_addrs = to_addrs or []
+
+    def send(self, notification: Notification) -> bool:
+        if not all([self.smtp_host, self.username, self.password, self.from_addr]):
+            return False
+
+        try:
+            import smtplib
+            from email.mime.text import MIMEText
+            from email.mime.multipart import MIMEMultipart
+
+            msg = MIMEMultipart()
+            msg["From"] = self.from_addr
+            msg["To"] = ", ".join(self.to_addrs)
+            msg["Subject"] = notification.title
+
+            body = f"{notification.message}\n\nTime: {notification.timestamp}"
+            msg.attach(MIMEText(body, "plain"))
+
+            with smtplib.SMTP(self.smtp_host, self.smtp_port) as server:
+                server.starttls()
+                server.login(self.username, self.password)
+                server.send_message(msg)
+
+            return True
+        except Exception:
+            return False
+
+
+class NotificationManager:
+    """Manage notifications across channels.
+
+    Examples:
+        >>> notifier = NotificationManager()
+        >>> notifier.add_channel(ConsoleNotifier())
+        >>> notifier.add_channel(SlackNotifier())
+        >>> # Send notification
+        >>> notifier.notify(title="Pipeline Failed", message="Training pipeline failed at step 3", level="error")
+    """
+
+    def __init__(self):
+        self.channels: list[NotificationChannel] = []
+        self.enabled = True
+
+    def add_channel(self, channel: NotificationChannel) -> None:
+        """Add a notification channel."""
+        self.channels.append(channel)
+
+    def remove_channel(self, channel: NotificationChannel) -> None:
+        """Remove a notification channel."""
+        if channel in self.channels:
+            self.channels.remove(channel)
+
+    def notify(
+        self,
+        title: str,
+        message: str,
+        level: str = "info",
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Send a notification to all channels."""
+        if not self.enabled:
+            return
+
+        notification = Notification(
+            title=title,
+            message=message,
+            level=level,
+            timestamp=datetime.now(),
+            metadata=metadata or {},
+        )
+
+        for channel in self.channels:
+            with contextlib.suppress(Exception):
+                channel.send(notification)
+
+    def on_pipeline_start(self, pipeline_name: str, run_id: str) -> None:
+        """Notify when pipeline starts."""
+        self.notify(
+            title="Pipeline Started",
+            message=f"Pipeline '{pipeline_name}' started (Run: {run_id})",
+            level="info",
+            metadata={"pipeline": pipeline_name, "run_id": run_id},
+        )
+
+    def on_pipeline_success(self, pipeline_name: str, run_id: str, duration: float) -> None:
+        """Notify when pipeline succeeds."""
+        self.notify(
+            title="Pipeline Completed",
+            message=f"Pipeline '{pipeline_name}' completed successfully in {duration:.2f}s",
+            level="success",
+            metadata={"pipeline": pipeline_name, "run_id": run_id, "duration": duration},
+        )
+
+    def on_pipeline_failure(self, pipeline_name: str, run_id: str, error: str) -> None:
+        """Notify when pipeline fails."""
+        self.notify(
+            title="Pipeline Failed",
+            message=f"Pipeline '{pipeline_name}' failed: {error}",
+            level="error",
+            metadata={"pipeline": pipeline_name, "run_id": run_id, "error": error},
+        )
+
+    def on_drift_detected(self, feature: str, psi: float) -> None:
+        """Notify when data drift is detected."""
+        self.notify(
+            title="Data Drift Detected",
+            message=f"Drift detected in feature '{feature}' (PSI: {psi:.4f})",
+            level="warning",
+            metadata={"feature": feature, "psi": psi},
+        )
+
+
+# Global notification manager
+_global_notifier = NotificationManager()
+
+
+def get_notifier() -> NotificationManager:
+    """Get the global notification manager."""
+    return _global_notifier
+
+
+def configure_notifications(
+    console: bool = True,
+    slack_webhook: str | None = None,
+    email_config: dict[str, Any] | None = None,
+) -> None:
+    """Configure notifications.
+
+    Args:
+        console: Enable console notifications
+        slack_webhook: Slack webhook URL
+        email_config: Email configuration dict
+    """
+    notifier = get_notifier()
+
+    # Clear existing channels
+    notifier.channels = []
+
+    if console:
+        notifier.add_channel(ConsoleNotifier())
+
+    if slack_webhook:
+        notifier.add_channel(SlackNotifier(slack_webhook))
+
+    if email_config:
+        notifier.add_channel(EmailNotifier(**email_config))

flowyml/registry/__init__.py

@@ -0,0 +1,5 @@
+"""Model registry for versioning and deployment."""
+
+from flowyml.registry.model_registry import ModelRegistry, ModelVersion, ModelStage
+
+__all__ = ["ModelRegistry", "ModelVersion", "ModelStage"]