churnkit 0.75.0a1__py3-none-any.whl

customer_retention/stages/modeling/model_comparator.py

@@ -0,0 +1,149 @@
+"""Model comparison and selection for customer retention prediction."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    f1_score,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+)
+
+from customer_retention.core.compat import DataFrame, Series
+
+
+@dataclass
+class ModelMetrics:
+    pr_auc: float
+    roc_auc: float
+    f1: float
+    precision: float
+    recall: float
+    accuracy: float
+    train_test_gap: Optional[float] = None
+    cv_std: Optional[float] = None
+
+
+@dataclass
+class ComparisonResult:
+    model_metrics: Dict[str, ModelMetrics]
+    ranking: List[str]
+    best_model_name: str
+    comparison_table: DataFrame
+    selection_reason: str
+
+
+class ModelComparator:
+    def __init__(
+        self,
+        primary_metric: str = "pr_auc",
+        weights: Optional[Dict[str, float]] = None,
+    ):
+        self.primary_metric = primary_metric
+        self.weights = weights or {
+            "pr_auc": 0.40,
+            "generalization_gap": 0.20,
+            "cv_stability": 0.15,
+            "business_cost": 0.15,
+            "training_time": 0.05,
+            "interpretability": 0.05,
+        }
+
+    def compare(
+        self,
+        models: Dict[str, Any],
+        X_test: DataFrame,
+        y_test: Series,
+        X_train: Optional[DataFrame] = None,
+        y_train: Optional[Series] = None,
+    ) -> ComparisonResult:
+        model_metrics = {}
+
+        for name, model in models.items():
+            metrics = self._evaluate_model(model, X_test, y_test, X_train, y_train)
+            model_metrics[name] = metrics
+
+        ranking = self._rank_models(model_metrics)
+        best_model_name = ranking[0]
+        comparison_table = self._build_comparison_table(model_metrics, ranking)
+        selection_reason = self._generate_selection_reason(best_model_name, model_metrics)
+
+        return ComparisonResult(
+            model_metrics=model_metrics,
+            ranking=ranking,
+            best_model_name=best_model_name,
+            comparison_table=comparison_table,
+            selection_reason=selection_reason,
+        )
+
+    def _evaluate_model(
+        self,
+        model,
+        X_test: DataFrame,
+        y_test: Series,
+        X_train: Optional[DataFrame],
+        y_train: Optional[Series],
+    ) -> ModelMetrics:
+        y_pred = model.predict(X_test)
+        y_proba = model.predict_proba(X_test)[:, 1]
+
+        pr_auc = average_precision_score(y_test, y_proba)
+        roc_auc = roc_auc_score(y_test, y_proba)
+
+        train_test_gap = None
+        if X_train is not None and y_train is not None:
+            y_train_proba = model.predict_proba(X_train)[:, 1]
+            train_pr_auc = average_precision_score(y_train, y_train_proba)
+            train_test_gap = train_pr_auc - pr_auc
+
+        return ModelMetrics(
+            pr_auc=pr_auc,
+            roc_auc=roc_auc,
+            f1=f1_score(y_test, y_pred, zero_division=0),
+            precision=precision_score(y_test, y_pred, zero_division=0),
+            recall=recall_score(y_test, y_pred, zero_division=0),
+            accuracy=accuracy_score(y_test, y_pred),
+            train_test_gap=train_test_gap,
+        )
+
+    def _rank_models(self, model_metrics: Dict[str, ModelMetrics]) -> List[str]:
+        scores = {}
+        for name, metrics in model_metrics.items():
+            scores[name] = getattr(metrics, self.primary_metric)
+
+        return sorted(scores.keys(), key=lambda x: scores[x], reverse=True)
+
+    def _build_comparison_table(
+        self,
+        model_metrics: Dict[str, ModelMetrics],
+        ranking: List[str],
+    ) -> DataFrame:
+        rows = []
+        for name in ranking:
+            metrics = model_metrics[name]
+            rows.append({
+                "model": name,
+                "pr_auc": metrics.pr_auc,
+                "roc_auc": metrics.roc_auc,
+                "f1": metrics.f1,
+                "precision": metrics.precision,
+                "recall": metrics.recall,
+                "accuracy": metrics.accuracy,
+                "train_test_gap": metrics.train_test_gap,
+            })
+
+        return DataFrame(rows).set_index("model")
+
+    def _generate_selection_reason(
+        self,
+        best_model_name: str,
+        model_metrics: Dict[str, ModelMetrics],
+    ) -> str:
+        metrics = model_metrics[best_model_name]
+        return (
+            f"Selected {best_model_name} based on highest {self.primary_metric} "
+            f"({getattr(metrics, self.primary_metric):.4f})"
+        )

customer_retention/stages/modeling/model_evaluator.py

@@ -0,0 +1,138 @@
+"""Model evaluation metrics for customer retention prediction."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+import numpy as np
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    balanced_accuracy_score,
+    brier_score_loss,
+    classification_report,
+    confusion_matrix,
+    f1_score,
+    log_loss,
+    precision_recall_curve,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+    roc_curve,
+)
+
+from customer_retention.core.compat import DataFrame, Series
+
+
+@dataclass
+class EvaluationResult:
+    metrics: Dict[str, float]
+    confusion_matrix: np.ndarray
+    classification_report: Dict[str, Any]
+    curves: Dict[str, Dict[str, np.ndarray]]
+    threshold: float
+    predictions: np.ndarray
+    probabilities: np.ndarray
+    dataset_name: Optional[str] = None
+
+
+class ModelEvaluator:
+    def __init__(self, threshold: float = 0.5, positive_class: int = 1):
+        self.threshold = threshold
+        self.positive_class = positive_class
+
+    def evaluate(
+        self,
+        model,
+        X: DataFrame,
+        y: Series,
+        dataset_name: Optional[str] = None,
+    ) -> EvaluationResult:
+        probabilities = model.predict_proba(X)[:, self.positive_class]
+        predictions = (probabilities >= self.threshold).astype(int)
+
+        metrics = self._compute_metrics(y, predictions, probabilities)
+        cm = confusion_matrix(y, predictions)
+        report = classification_report(y, predictions, output_dict=True)
+        curves = self._compute_curves(y, probabilities)
+
+        return EvaluationResult(
+            metrics=metrics,
+            confusion_matrix=cm,
+            classification_report=report,
+            curves=curves,
+            threshold=self.threshold,
+            predictions=predictions,
+            probabilities=probabilities,
+            dataset_name=dataset_name,
+        )
+
+    def _compute_metrics(
+        self,
+        y_true: Series,
+        y_pred: np.ndarray,
+        y_proba: np.ndarray,
+    ) -> Dict[str, float]:
+        metrics = {
+            "accuracy": accuracy_score(y_true, y_pred),
+            "balanced_accuracy": balanced_accuracy_score(y_true, y_pred),
+            "precision": precision_score(y_true, y_pred, zero_division=0),
+            "recall": recall_score(y_true, y_pred, zero_division=0),
+            "f1": f1_score(y_true, y_pred, zero_division=0),
+            "roc_auc": roc_auc_score(y_true, y_proba),
+            "pr_auc": average_precision_score(y_true, y_proba),
+            "average_precision": average_precision_score(y_true, y_proba),
+            "brier_score": brier_score_loss(y_true, y_proba),
+            "log_loss": log_loss(y_true, y_proba),
+        }
+
+        lift_gain = self._compute_lift_gain(y_true, y_proba)
+        metrics.update(lift_gain)
+
+        return metrics
+
+    def _compute_curves(
+        self,
+        y_true: Series,
+        y_proba: np.ndarray,
+    ) -> Dict[str, Dict[str, np.ndarray]]:
+        fpr, tpr, roc_thresholds = roc_curve(y_true, y_proba)
+        precision, recall, pr_thresholds = precision_recall_curve(y_true, y_proba)
+
+        return {
+            "roc_curve": {
+                "fpr": fpr,
+                "tpr": tpr,
+                "thresholds": roc_thresholds,
+            },
+            "pr_curve": {
+                "precision": precision,
+                "recall": recall,
+                "thresholds": pr_thresholds,
+            },
+        }
+
+    def _compute_lift_gain(
+        self,
+        y_true: Series,
+        y_proba: np.ndarray,
+    ) -> Dict[str, float]:
+        y_true = np.array(y_true)
+        sorted_indices = np.argsort(y_proba)[::-1]
+        y_sorted = y_true[sorted_indices]
+
+        n_total = len(y_true)
+        n_positive = y_true.sum()
+        baseline_rate = n_positive / n_total
+
+        metrics = {}
+        for k in [10, 20]:
+            top_k_idx = int(n_total * k / 100)
+            top_k_positive = y_sorted[:top_k_idx].sum()
+
+            lift = (top_k_positive / top_k_idx) / baseline_rate if top_k_idx > 0 else 0
+            gain = top_k_positive / n_positive if n_positive > 0 else 0
+
+            metrics[f"lift_at_{k}"] = lift
+            metrics[f"gain_at_{k}"] = gain
+
+        return metrics

customer_retention/stages/modeling/threshold_optimizer.py

@@ -0,0 +1,131 @@
+"""Threshold optimization for classification models."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, Optional
+
+import numpy as np
+from sklearn.metrics import confusion_matrix, f1_score, fbeta_score, precision_score, recall_score
+
+from customer_retention.core.compat import DataFrame, Series
+
+
+class OptimizationObjective(Enum):
+    MIN_COST = "min_cost"
+    MAX_F1 = "max_f1"
+    MAX_F2 = "max_f2"
+    TARGET_RECALL = "target_recall"
+    TARGET_PRECISION = "target_precision"
+
+
+@dataclass
+class ThresholdResult:
+    optimal_threshold: float
+    threshold_metrics: Dict[str, float]
+    cost_at_threshold: Optional[float]
+    comparison_default: Dict[str, Any]
+
+
+class ThresholdOptimizer:
+    def __init__(
+        self,
+        objective: OptimizationObjective = OptimizationObjective.MAX_F1,
+        cost_fn: float = 100,
+        cost_fp: float = 10,
+        target_recall: Optional[float] = None,
+        target_precision: Optional[float] = None,
+        threshold_step: float = 0.01,
+    ):
+        self.objective = objective
+        self.cost_fn = cost_fn
+        self.cost_fp = cost_fp
+        self.target_recall = target_recall
+        self.target_precision = target_precision
+        self.threshold_step = threshold_step
+
+    def optimize(self, model, X: DataFrame, y: Series) -> ThresholdResult:
+        probabilities = model.predict_proba(X)[:, 1]
+        thresholds = np.arange(0.01, 1.0, self.threshold_step)
+
+        best_threshold = 0.5
+        best_score = float("-inf") if self.objective != OptimizationObjective.MIN_COST else float("inf")
+
+        for threshold in thresholds:
+            predictions = (probabilities >= threshold).astype(int)
+            score = self._calculate_score(y, predictions, probabilities, threshold)
+
+            if self._is_better_score(score, best_score):
+                best_score = score
+                best_threshold = threshold
+
+        optimal_predictions = (probabilities >= best_threshold).astype(int)
+        threshold_metrics = self._calculate_metrics(y, optimal_predictions)
+        cost_at_threshold = self._calculate_cost(y, optimal_predictions)
+        comparison_default = self._compare_with_default(y, probabilities, best_threshold)
+
+        return ThresholdResult(
+            optimal_threshold=best_threshold,
+            threshold_metrics=threshold_metrics,
+            cost_at_threshold=cost_at_threshold,
+            comparison_default=comparison_default,
+        )
+
+    def _calculate_score(self, y_true, y_pred, y_proba, threshold) -> float:
+        if self.objective == OptimizationObjective.MIN_COST:
+            return self._calculate_cost(y_true, y_pred)
+
+        if self.objective == OptimizationObjective.MAX_F1:
+            return f1_score(y_true, y_pred, zero_division=0)
+
+        if self.objective == OptimizationObjective.MAX_F2:
+            return fbeta_score(y_true, y_pred, beta=2, zero_division=0)
+
+        if self.objective == OptimizationObjective.TARGET_RECALL:
+            recall = recall_score(y_true, y_pred, zero_division=0)
+            if recall >= self.target_recall:
+                return precision_score(y_true, y_pred, zero_division=0)
+            return float("-inf")
+
+        if self.objective == OptimizationObjective.TARGET_PRECISION:
+            precision = precision_score(y_true, y_pred, zero_division=0)
+            if precision >= self.target_precision:
+                return recall_score(y_true, y_pred, zero_division=0)
+            return float("-inf")
+
+        return f1_score(y_true, y_pred, zero_division=0)
+
+    def _is_better_score(self, score: float, best_score: float) -> bool:
+        if self.objective == OptimizationObjective.MIN_COST:
+            return score < best_score
+        return score > best_score
+
+    def _calculate_cost(self, y_true, y_pred) -> float:
+        cm = confusion_matrix(y_true, y_pred)
+        tn, fp, fn, tp = cm.ravel()
+        return fn * self.cost_fn + fp * self.cost_fp
+
+    def _calculate_metrics(self, y_true, y_pred) -> Dict[str, float]:
+        return {
+            "precision": precision_score(y_true, y_pred, zero_division=0),
+            "recall": recall_score(y_true, y_pred, zero_division=0),
+            "f1": f1_score(y_true, y_pred, zero_division=0),
+            "f2": fbeta_score(y_true, y_pred, beta=2, zero_division=0),
+        }
+
+    def _compare_with_default(
+        self,
+        y_true: Series,
+        y_proba: np.ndarray,
+        optimal_threshold: float,
+    ) -> Dict[str, Any]:
+        default_threshold = 0.5
+        default_preds = (y_proba >= default_threshold).astype(int)
+        optimal_preds = (y_proba >= optimal_threshold).astype(int)
+
+        return {
+            "default_threshold": default_threshold,
+            "default_f1": f1_score(y_true, default_preds, zero_division=0),
+            "default_cost": self._calculate_cost(y_true, default_preds),
+            "optimal_f1": f1_score(y_true, optimal_preds, zero_division=0),
+            "optimal_cost": self._calculate_cost(y_true, optimal_preds),
+        }

customer_retention/stages/monitoring/__init__.py

@@ -0,0 +1,37 @@
+from customer_retention.core.components.enums import Severity
+
+from .alert_manager import (
+    Alert,
+    AlertChannel,
+    AlertCondition,
+    AlertConfig,
+    AlertLevel,
+    AlertManager,
+    AlertResult,
+    EmailSender,
+    SlackSender,
+)
+from .drift_detector import DriftConfig, DriftDetector, DriftResult, DriftType, FeatureDriftResult, TargetDriftResult
+from .performance_monitor import (
+    CalibrationCurve,
+    DistributionAnalysis,
+    DistributionComparison,
+    MonitoringConfig,
+    PerformanceMonitor,
+    PerformanceResult,
+    PerformanceStatus,
+    ProportionAnalysis,
+    ProxyMetrics,
+    TrendReport,
+)
+
+__all__ = [
+    "Severity",
+    "DriftDetector", "DriftType", "DriftResult",
+    "DriftConfig", "FeatureDriftResult", "TargetDriftResult",
+    "PerformanceMonitor", "PerformanceResult", "PerformanceStatus",
+    "ProxyMetrics", "MonitoringConfig", "CalibrationCurve", "DistributionAnalysis",
+    "ProportionAnalysis", "DistributionComparison", "TrendReport",
+    "AlertManager", "Alert", "AlertLevel", "AlertChannel",
+    "AlertConfig", "AlertCondition", "AlertResult", "EmailSender", "SlackSender"
+]