adamops 0.1.0__py3-none-any.whl

adamops/monitoring/drift.py

@@ -0,0 +1,212 @@
+"""
+AdamOps Drift Detection Module
+
+Detect data drift and concept drift in production.
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+import numpy as np
+import pandas as pd
+from scipy import stats
+from sklearn.model_selection import train_test_split
+
+from adamops.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class DriftDetector:
+    """Detect data and concept drift."""
+    
+    def __init__(self, reference_data: pd.DataFrame, threshold: float = 0.05):
+        """
+        Initialize drift detector.
+        
+        Args:
+            reference_data: Reference/training data.
+            threshold: P-value threshold for drift detection.
+        """
+        self.reference = reference_data
+        self.threshold = threshold
+        self._compute_reference_stats()
+    
+    def _compute_reference_stats(self):
+        """Compute statistics for reference data."""
+        self.ref_stats = {}
+        
+        for col in self.reference.columns:
+            if pd.api.types.is_numeric_dtype(self.reference[col]):
+                self.ref_stats[col] = {
+                    "type": "numeric",
+                    "mean": self.reference[col].mean(),
+                    "std": self.reference[col].std(),
+                    "min": self.reference[col].min(),
+                    "max": self.reference[col].max(),
+                    "values": self.reference[col].dropna().values,
+                }
+            else:
+                value_counts = self.reference[col].value_counts(normalize=True)
+                self.ref_stats[col] = {
+                    "type": "categorical",
+                    "distribution": value_counts.to_dict(),
+                    "values": self.reference[col].dropna().values,
+                }
+    
+    def detect_drift(self, current_data: pd.DataFrame) -> Dict[str, Any]:
+        """
+        Detect drift between reference and current data.
+        
+        Returns:
+            Dict with drift detection results.
+        """
+        results = {"drift_detected": False, "columns": {}, "summary": {}}
+        drift_count = 0
+        
+        for col in self.reference.columns:
+            if col not in current_data.columns:
+                continue
+            
+            col_result = self._detect_column_drift(col, current_data[col])
+            results["columns"][col] = col_result
+            
+            if col_result["drift_detected"]:
+                drift_count += 1
+        
+        results["drift_detected"] = drift_count > 0
+        results["summary"] = {
+            "total_columns": len(self.reference.columns),
+            "drifted_columns": drift_count,
+            "drift_ratio": drift_count / len(self.reference.columns),
+        }
+        
+        if results["drift_detected"]:
+            logger.warning(f"Drift detected in {drift_count} columns")
+        
+        return results
+    
+    def _detect_column_drift(self, col: str, current: pd.Series) -> Dict:
+        """Detect drift for a single column."""
+        ref_stats = self.ref_stats.get(col)
+        if ref_stats is None:
+            return {"drift_detected": False, "reason": "unknown_column"}
+        
+        current_values = current.dropna().values
+        
+        if ref_stats["type"] == "numeric":
+            # Kolmogorov-Smirnov test
+            stat, pvalue = stats.ks_2samp(ref_stats["values"], current_values)
+            drift_detected = pvalue < self.threshold
+            
+            return {
+                "drift_detected": drift_detected,
+                "test": "ks_test",
+                "statistic": float(stat),
+                "p_value": float(pvalue),
+                "ref_mean": ref_stats["mean"],
+                "current_mean": float(current.mean()),
+            }
+        else:
+            # Chi-square test for categorical
+            current_dist = current.value_counts(normalize=True)
+            
+            # Align distributions
+            all_categories = set(ref_stats["distribution"].keys()) | set(current_dist.index)
+            ref_freq = [ref_stats["distribution"].get(c, 0.001) for c in all_categories]
+            cur_freq = [current_dist.get(c, 0.001) for c in all_categories]
+            
+            # Normalize
+            ref_freq = np.array(ref_freq) / sum(ref_freq)
+            cur_freq = np.array(cur_freq) / sum(cur_freq)
+            
+            # Chi-square
+            stat, pvalue = stats.chisquare(cur_freq, ref_freq)
+            drift_detected = pvalue < self.threshold
+            
+            return {
+                "drift_detected": drift_detected,
+                "test": "chi_square",
+                "statistic": float(stat),
+                "p_value": float(pvalue),
+            }
+    
+    def get_drift_report(self, current_data: pd.DataFrame) -> str:
+        """Generate human-readable drift report."""
+        results = self.detect_drift(current_data)
+        
+        lines = [
+            "=" * 50,
+            "DRIFT DETECTION REPORT",
+            "=" * 50,
+            f"Status: {'DRIFT DETECTED' if results['drift_detected'] else 'NO DRIFT'}",
+            f"Columns with drift: {results['summary']['drifted_columns']}/{results['summary']['total_columns']}",
+            "",
+        ]
+        
+        if results["drift_detected"]:
+            lines.append("Drifted Columns:")
+            for col, info in results["columns"].items():
+                if info["drift_detected"]:
+                    lines.append(f"  - {col}: p-value={info['p_value']:.4f} ({info['test']})")
+        
+        return "\n".join(lines)
+
+
+class PSI:
+    """Population Stability Index calculator."""
+    
+    @staticmethod
+    def calculate(reference: np.ndarray, current: np.ndarray, bins: int = 10) -> float:
+        """
+        Calculate PSI between reference and current distributions.
+        
+        PSI < 0.1: No significant change
+        0.1 < PSI < 0.2: Moderate change
+        PSI > 0.2: Significant change
+        """
+        # Create bins from reference
+        _, bin_edges = np.histogram(reference, bins=bins)
+        
+        # Get distributions
+        ref_counts, _ = np.histogram(reference, bins=bin_edges)
+        cur_counts, _ = np.histogram(current, bins=bin_edges)
+        
+        # Convert to percentages
+        ref_pct = ref_counts / len(reference)
+        cur_pct = cur_counts / len(current)
+        
+        # Avoid division by zero
+        ref_pct = np.where(ref_pct == 0, 0.0001, ref_pct)
+        cur_pct = np.where(cur_pct == 0, 0.0001, cur_pct)
+        
+        # Calculate PSI
+        psi = np.sum((cur_pct - ref_pct) * np.log(cur_pct / ref_pct))
+        
+        return float(psi)
+    
+    @staticmethod
+    def interpret(psi: float) -> str:
+        """Interpret PSI value."""
+        if psi < 0.1:
+            return "No significant change"
+        elif psi < 0.2:
+            return "Moderate change - monitor closely"
+        else:
+            return "Significant change - investigate"
+
+
+def detect_drift(
+    reference: pd.DataFrame, current: pd.DataFrame, threshold: float = 0.05
+) -> Dict:
+    """Detect drift between reference and current data."""
+    detector = DriftDetector(reference, threshold)
+    return detector.detect_drift(current)
+
+
+def calculate_psi(reference: np.ndarray, current: np.ndarray, bins: int = 10) -> Dict:
+    """Calculate PSI with interpretation."""
+    psi = PSI.calculate(reference, current, bins)
+    return {
+        "psi": psi,
+        "interpretation": PSI.interpret(psi),
+        "significant": psi > 0.2,
+    }

adamops/monitoring/performance.py

@@ -0,0 +1,195 @@
+"""
+AdamOps Performance Monitoring Module
+
+Track model performance over time.
+"""
+
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from adamops.utils.logging import get_logger
+from adamops.utils.helpers import ensure_dir
+from adamops.evaluation.metrics import evaluate
+
+logger = get_logger(__name__)
+
+
+class PerformanceMonitor:
+    """Monitor model performance over time."""
+    
+    def __init__(self, model_name: str, storage_path: Optional[str] = None):
+        """
+        Initialize performance monitor.
+        
+        Args:
+            model_name: Name of the model being monitored.
+            storage_path: Path to store performance logs.
+        """
+        self.model_name = model_name
+        self.storage_path = Path(storage_path or f".adamops_monitor/{model_name}")
+        ensure_dir(self.storage_path)
+        
+        self.metrics_file = self.storage_path / "metrics.json"
+        self.predictions_file = self.storage_path / "predictions.json"
+        
+        self._load_history()
+    
+    def _load_history(self):
+        """Load existing metrics history."""
+        if self.metrics_file.exists():
+            with open(self.metrics_file) as f:
+                self.metrics_history = json.load(f)
+        else:
+            self.metrics_history = []
+    
+    def _save_history(self):
+        """Save metrics history."""
+        with open(self.metrics_file, 'w') as f:
+            json.dump(self.metrics_history, f, indent=2, default=str)
+    
+    def log_metrics(
+        self, metrics: Dict[str, float], 
+        timestamp: Optional[str] = None,
+        metadata: Optional[Dict] = None
+    ):
+        """
+        Log performance metrics.
+        
+        Args:
+            metrics: Dict of metric name to value.
+            timestamp: Timestamp (uses current if None).
+            metadata: Additional metadata.
+        """
+        entry = {
+            "timestamp": timestamp or datetime.now().isoformat(),
+            "metrics": metrics,
+            "metadata": metadata or {},
+        }
+        
+        self.metrics_history.append(entry)
+        self._save_history()
+        
+        logger.info(f"Logged metrics for {self.model_name}: {metrics}")
+    
+    def log_prediction(
+        self, y_true: np.ndarray, y_pred: np.ndarray,
+        task: str = "classification", y_prob: Optional[np.ndarray] = None
+    ):
+        """Log prediction and compute metrics."""
+        metrics = evaluate(y_true, y_pred, task, y_prob)
+        self.log_metrics(metrics, metadata={"task": task, "n_samples": len(y_true)})
+        return metrics
+    
+    def get_history(self, n_latest: Optional[int] = None) -> List[Dict]:
+        """Get metrics history."""
+        if n_latest:
+            return self.metrics_history[-n_latest:]
+        return self.metrics_history
+    
+    def get_metric_trend(self, metric: str) -> pd.DataFrame:
+        """Get trend for a specific metric."""
+        data = []
+        for entry in self.metrics_history:
+            if metric in entry["metrics"]:
+                data.append({
+                    "timestamp": entry["timestamp"],
+                    "value": entry["metrics"][metric],
+                })
+        
+        return pd.DataFrame(data)
+    
+    def detect_degradation(
+        self, metric: str, threshold: float = 0.1, window: int = 5
+    ) -> Dict:
+        """
+        Detect performance degradation.
+        
+        Args:
+            metric: Metric to monitor.
+            threshold: Relative change threshold.
+            window: Number of recent entries to compare.
+        
+        Returns:
+            Dict with degradation info.
+        """
+        trend = self.get_metric_trend(metric)
+        
+        if len(trend) < window + 1:
+            return {"degraded": False, "reason": "insufficient_data"}
+        
+        baseline = trend["value"].iloc[:-window].mean()
+        recent = trend["value"].iloc[-window:].mean()
+        
+        change = (baseline - recent) / baseline if baseline != 0 else 0
+        degraded = change > threshold
+        
+        result = {
+            "degraded": degraded,
+            "metric": metric,
+            "baseline": baseline,
+            "recent": recent,
+            "change_pct": change * 100,
+            "threshold_pct": threshold * 100,
+        }
+        
+        if degraded:
+            logger.warning(f"Performance degradation detected for {metric}: {change*100:.1f}% drop")
+        
+        return result
+    
+    def summary(self) -> Dict:
+        """Get monitoring summary."""
+        if not self.metrics_history:
+            return {"model": self.model_name, "entries": 0}
+        
+        latest = self.metrics_history[-1]
+        
+        return {
+            "model": self.model_name,
+            "entries": len(self.metrics_history),
+            "latest_timestamp": latest["timestamp"],
+            "latest_metrics": latest["metrics"],
+        }
+
+
+class LatencyMonitor:
+    """Monitor prediction latency."""
+    
+    def __init__(self, model_name: str):
+        self.model_name = model_name
+        self.latencies = []
+    
+    def record(self, latency_ms: float):
+        """Record a latency measurement."""
+        self.latencies.append({
+            "timestamp": datetime.now().isoformat(),
+            "latency_ms": latency_ms,
+        })
+    
+    def get_stats(self) -> Dict:
+        """Get latency statistics."""
+        if not self.latencies:
+            return {}
+        
+        values = [l["latency_ms"] for l in self.latencies]
+        
+        return {
+            "count": len(values),
+            "mean_ms": np.mean(values),
+            "std_ms": np.std(values),
+            "p50_ms": np.percentile(values, 50),
+            "p95_ms": np.percentile(values, 95),
+            "p99_ms": np.percentile(values, 99),
+            "min_ms": np.min(values),
+            "max_ms": np.max(values),
+        }
+
+
+def create_monitor(model_name: str, storage_path: Optional[str] = None) -> PerformanceMonitor:
+    """Create a performance monitor."""
+    return PerformanceMonitor(model_name, storage_path)

adamops/pipelines/__init__.py

@@ -0,0 +1,15 @@
+"""
+AdamOps Pipelines Module
+
+Provides pipeline orchestration capabilities:
+- workflows: Define end-to-end ML workflows as DAGs
+- orchestrators: Schedule and run pipelines
+"""
+
+from adamops.pipelines import workflows
+from adamops.pipelines import orchestrators
+
+__all__ = [
+    "workflows",
+    "orchestrators",
+]

adamops/pipelines/orchestrators.py

@@ -0,0 +1,183 @@
+"""
+AdamOps Orchestrators Module
+
+Schedule and run pipelines.
+"""
+
+from typing import Any, Callable, Dict, List, Optional
+from datetime import datetime, timedelta
+import threading
+import time
+import json
+from pathlib import Path
+
+from adamops.utils.logging import get_logger
+from adamops.utils.helpers import ensure_dir
+from adamops.pipelines.workflows import Workflow, TaskStatus
+
+logger = get_logger(__name__)
+
+
+class PipelineRun:
+    """Represents a single pipeline run."""
+    
+    def __init__(self, workflow: Workflow, run_id: str):
+        self.workflow = workflow
+        self.run_id = run_id
+        self.start_time: Optional[datetime] = None
+        self.end_time: Optional[datetime] = None
+        self.status = TaskStatus.PENDING
+        self.result: Optional[Dict] = None
+        self.error: Optional[str] = None
+    
+    @property
+    def duration(self) -> Optional[float]:
+        if self.start_time and self.end_time:
+            return (self.end_time - self.start_time).total_seconds()
+        return None
+    
+    def to_dict(self) -> Dict:
+        return {
+            "run_id": self.run_id,
+            "workflow": self.workflow.name,
+            "status": self.status.value,
+            "start_time": self.start_time.isoformat() if self.start_time else None,
+            "end_time": self.end_time.isoformat() if self.end_time else None,
+            "duration": self.duration,
+            "error": self.error,
+        }
+
+
+class Orchestrator:
+    """Orchestrate and schedule pipeline runs."""
+    
+    def __init__(self, storage_path: Optional[str] = None):
+        self.storage_path = Path(storage_path or ".adamops_runs")
+        ensure_dir(self.storage_path)
+        
+        self.workflows: Dict[str, Workflow] = {}
+        self.runs: Dict[str, PipelineRun] = {}
+        self.schedules: Dict[str, Dict] = {}
+        self._scheduler_thread: Optional[threading.Thread] = None
+        self._running = False
+    
+    def register(self, workflow: Workflow):
+        """Register a workflow."""
+        self.workflows[workflow.name] = workflow
+        logger.info(f"Registered workflow: {workflow.name}")
+    
+    def run(self, workflow_name: str, context: Optional[Dict] = None) -> PipelineRun:
+        """Run a workflow immediately."""
+        if workflow_name not in self.workflows:
+            raise ValueError(f"Unknown workflow: {workflow_name}")
+        
+        workflow = self.workflows[workflow_name]
+        run_id = f"{workflow_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        
+        run = PipelineRun(workflow, run_id)
+        self.runs[run_id] = run
+        
+        run.start_time = datetime.now()
+        run.status = TaskStatus.RUNNING
+        
+        try:
+            run.result = workflow.run(context or {})
+            run.status = TaskStatus.COMPLETED
+        except Exception as e:
+            run.status = TaskStatus.FAILED
+            run.error = str(e)
+        finally:
+            run.end_time = datetime.now()
+            self._save_run(run)
+        
+        return run
+    
+    def run_async(self, workflow_name: str, context: Optional[Dict] = None) -> str:
+        """Run a workflow asynchronously."""
+        run_id = f"{workflow_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        
+        def _run():
+            self.run(workflow_name, context)
+        
+        thread = threading.Thread(target=_run, name=run_id)
+        thread.start()
+        
+        return run_id
+    
+    def schedule(self, workflow_name: str, interval_seconds: int,
+                 context: Optional[Dict] = None):
+        """Schedule a workflow to run at intervals."""
+        self.schedules[workflow_name] = {
+            "interval": interval_seconds,
+            "context": context or {},
+            "last_run": None,
+            "next_run": datetime.now(),
+        }
+        logger.info(f"Scheduled {workflow_name} every {interval_seconds}s")
+    
+    def start_scheduler(self):
+        """Start the background scheduler."""
+        if self._running:
+            return
+        
+        self._running = True
+        
+        def _scheduler_loop():
+            while self._running:
+                now = datetime.now()
+                
+                for name, schedule in self.schedules.items():
+                    if now >= schedule["next_run"]:
+                        try:
+                            self.run(name, schedule["context"])
+                        except Exception as e:
+                            logger.error(f"Scheduled run failed: {e}")
+                        
+                        schedule["last_run"] = now
+                        schedule["next_run"] = now + timedelta(seconds=schedule["interval"])
+                
+                time.sleep(1)
+        
+        self._scheduler_thread = threading.Thread(target=_scheduler_loop, daemon=True)
+        self._scheduler_thread.start()
+        logger.info("Started scheduler")
+    
+    def stop_scheduler(self):
+        """Stop the background scheduler."""
+        self._running = False
+        if self._scheduler_thread:
+            self._scheduler_thread.join(timeout=5)
+        logger.info("Stopped scheduler")
+    
+    def _save_run(self, run: PipelineRun):
+        """Save run to storage."""
+        run_file = self.storage_path / f"{run.run_id}.json"
+        with open(run_file, 'w') as f:
+            json.dump(run.to_dict(), f, indent=2)
+    
+    def get_runs(self, workflow_name: Optional[str] = None, limit: int = 10) -> List[Dict]:
+        """Get recent runs."""
+        runs = list(self.runs.values())
+        
+        if workflow_name:
+            runs = [r for r in runs if r.workflow.name == workflow_name]
+        
+        runs.sort(key=lambda r: r.start_time or datetime.min, reverse=True)
+        return [r.to_dict() for r in runs[:limit]]
+    
+    def get_run(self, run_id: str) -> Optional[Dict]:
+        """Get a specific run."""
+        if run_id in self.runs:
+            return self.runs[run_id].to_dict()
+        return None
+
+
+# Global orchestrator
+_orchestrator: Optional[Orchestrator] = None
+
+def get_orchestrator() -> Orchestrator:
+    """Get global orchestrator."""
+    global _orchestrator
+    if _orchestrator is None:
+        _orchestrator = Orchestrator()
+    return _orchestrator