churnkit 0.75.0a1__py3-none-any.whl

customer_retention/integrations/iteration/feedback_collector.py

@@ -0,0 +1,184 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import yaml
+
+
+@dataclass
+class ModelFeedback:
+    iteration_id: str
+    model_type: str
+    metrics: Dict[str, float]
+    feature_importances: Dict[str, float]
+    confusion_matrix: Optional[List[List[int]]] = None
+    error_analysis: Optional[Dict[str, Any]] = None
+    collected_at: datetime = field(default_factory=datetime.now)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "iteration_id": self.iteration_id,
+            "model_type": self.model_type,
+            "metrics": self.metrics,
+            "feature_importances": self.feature_importances,
+            "confusion_matrix": self.confusion_matrix,
+            "error_analysis": self.error_analysis,
+            "collected_at": self.collected_at.isoformat()
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ModelFeedback":
+        return cls(
+            iteration_id=data["iteration_id"],
+            model_type=data["model_type"],
+            metrics=data["metrics"],
+            feature_importances=data["feature_importances"],
+            confusion_matrix=data.get("confusion_matrix"),
+            error_analysis=data.get("error_analysis"),
+            collected_at=datetime.fromisoformat(data["collected_at"]) if "collected_at" in data else datetime.now()
+        )
+
+
+@dataclass
+class FeatureInsight:
+    feature_name: str
+    importance_rank: int
+    importance_score: float
+    recommendation_to_drop: bool
+    recommendation_to_engineer: Optional[str] = None
+
+
+class ModelFeedbackCollector:
+    def __init__(self, drop_threshold: float = 0.01):
+        self.drop_threshold = drop_threshold
+
+    def create_from_sklearn(self, model, iteration_id: str,
+                            feature_names: List[str],
+                            metrics: Dict[str, float]) -> ModelFeedback:
+        model_type = type(model).__name__
+
+        feature_importances = {}
+        if hasattr(model, 'feature_importances_'):
+            for name, imp in zip(feature_names, model.feature_importances_):
+                feature_importances[name] = float(imp)
+        elif hasattr(model, 'coef_'):
+            import numpy as np
+            coefs = np.abs(model.coef_).flatten()
+            for name, coef in zip(feature_names, coefs):
+                feature_importances[name] = float(coef)
+
+        return ModelFeedback(
+            iteration_id=iteration_id,
+            model_type=model_type,
+            metrics=metrics,
+            feature_importances=feature_importances
+        )
+
+    def analyze_feature_importance(self, feedback: ModelFeedback) -> List[FeatureInsight]:
+        sorted_features = sorted(
+            feedback.feature_importances.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )
+
+        insights = []
+        for rank, (feature, score) in enumerate(sorted_features, 1):
+            should_drop = score < self.drop_threshold
+            recommendation = None
+            if should_drop:
+                recommendation = f"Consider removing {feature} (importance: {score:.4f})"
+
+            insights.append(FeatureInsight(
+                feature_name=feature,
+                importance_rank=rank,
+                importance_score=score,
+                recommendation_to_drop=should_drop,
+                recommendation_to_engineer=recommendation
+            ))
+
+        return insights
+
+    def suggest_next_actions(self, feedback: ModelFeedback) -> List[str]:
+        actions = []
+
+        roc_auc = feedback.metrics.get("roc_auc", 0)
+        pr_auc = feedback.metrics.get("pr_auc", 0)
+
+        if roc_auc >= 0.90 or pr_auc >= 0.85:
+            actions.append(f"Model performance is excellent (ROC-AUC: {roc_auc:.2f}). Consider deploying.")
+        elif roc_auc >= 0.80:
+            actions.append(f"Model performance is good (ROC-AUC: {roc_auc:.2f}). Consider feature engineering for improvement.")
+        else:
+            actions.append(f"Model performance needs improvement (ROC-AUC: {roc_auc:.2f}). Review feature engineering and data quality.")
+
+        low_importance = [
+            name for name, score in feedback.feature_importances.items()
+            if score < self.drop_threshold
+        ]
+        if low_importance:
+            actions.append(f"Consider dropping {len(low_importance)} low-importance features: {', '.join(low_importance[:5])}")
+
+        if len(feedback.feature_importances) < 5:
+            actions.append("Feature set is small. Consider engineering additional features.")
+
+        top_features = self.get_top_features(feedback, n=3)
+        if top_features:
+            top_names = [f[0] for f in top_features]
+            actions.append(f"Top performing features: {', '.join(top_names)}. Consider creating derived features from these.")
+
+        return actions
+
+    def compare_feedback(self, previous: ModelFeedback,
+                        current: ModelFeedback) -> Dict[str, Any]:
+        metric_improvements = {}
+        for metric in current.metrics:
+            if metric in previous.metrics:
+                metric_improvements[metric] = current.metrics[metric] - previous.metrics[metric]
+
+        avg_improvement = sum(metric_improvements.values()) / len(metric_improvements) if metric_improvements else 0
+
+        if avg_improvement > 0.02:
+            trend = "improved"
+        elif avg_improvement < -0.02:
+            trend = "degraded"
+        else:
+            trend = "stable"
+
+        feature_changes = {}
+        for feature, score in current.feature_importances.items():
+            if feature in previous.feature_importances:
+                feature_changes[feature] = score - previous.feature_importances[feature]
+
+        return {
+            "metric_improvements": metric_improvements,
+            "overall_trend": trend,
+            "feature_importance_changes": feature_changes,
+            "previous_iteration": previous.iteration_id,
+            "current_iteration": current.iteration_id
+        }
+
+    def get_top_features(self, feedback: ModelFeedback,
+                        n: int = 5) -> List[Tuple[str, float]]:
+        sorted_features = sorted(
+            feedback.feature_importances.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )
+        return sorted_features[:n]
+
+    def get_low_importance_features(self, feedback: ModelFeedback) -> List[str]:
+        return [
+            name for name, score in feedback.feature_importances.items()
+            if score < self.drop_threshold
+        ]
+
+    def save_feedback(self, feedback: ModelFeedback, path: str) -> None:
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "w") as f:
+            yaml.dump(feedback.to_dict(), f, default_flow_style=False, sort_keys=False)
+
+    def load_feedback(self, path: str) -> ModelFeedback:
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+        return ModelFeedback.from_dict(data)

customer_retention/integrations/iteration/orchestrator.py

@@ -0,0 +1,168 @@
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from .context import IterationContext, IterationContextManager, IterationStatus, IterationTrigger
+from .feedback_collector import ModelFeedback, ModelFeedbackCollector
+from .recommendation_tracker import RecommendationTracker, TrackedRecommendation
+from .signals import SignalAggregator
+
+
+class IterationOrchestrator:
+    def __init__(self, findings_dir: str,
+                 signal_aggregator: Optional[SignalAggregator] = None):
+        self.findings_dir = Path(findings_dir)
+        self.iterations_dir = self.findings_dir / "iterations"
+        self.recommendations_dir = self.findings_dir / "recommendations"
+        self.feedback_dir = self.findings_dir / "feedback"
+
+        self.iterations_dir.mkdir(parents=True, exist_ok=True)
+        self.recommendations_dir.mkdir(parents=True, exist_ok=True)
+        self.feedback_dir.mkdir(parents=True, exist_ok=True)
+
+        self._context_manager = IterationContextManager(str(self.iterations_dir))
+        self._recommendation_tracker = RecommendationTracker(
+            str(self.recommendations_dir / "tracked.yaml")
+        )
+        self._feedback_collector = ModelFeedbackCollector()
+        self._signal_aggregator = signal_aggregator or SignalAggregator()
+
+        self._current_context: Optional[IterationContext] = None
+        self._latest_feedback: Optional[ModelFeedback] = None
+
+    def start_new_iteration(self, trigger: IterationTrigger) -> IterationContext:
+        ctx = IterationContext.create_new(str(self.findings_dir), trigger)
+        self._current_context = ctx
+        self._save_current_context()
+        self._context_manager.set_current(ctx.iteration_id)
+        return ctx
+
+    def start_child_iteration(self, trigger: IterationTrigger) -> IterationContext:
+        if self._current_context is None:
+            return self.start_new_iteration(trigger)
+
+        child = self._current_context.create_child(trigger)
+        self._current_context = child
+        self._save_current_context()
+        self._context_manager.set_current(child.iteration_id)
+        return child
+
+    def get_current_iteration(self) -> Optional[IterationContext]:
+        if self._current_context is not None:
+            return self._current_context
+        return self._context_manager.get_current()
+
+    def update_status(self, status: IterationStatus) -> None:
+        if self._current_context:
+            self._current_context.update_status(status)
+            self._save_current_context()
+
+    def record_model_metrics(self, metrics: Dict[str, float],
+                            artifact_path: Optional[str] = None) -> None:
+        if self._current_context:
+            self._current_context.set_model_metrics(metrics, artifact_path)
+            self._save_current_context()
+
+    def get_recommendation_tracker(self) -> RecommendationTracker:
+        return self._recommendation_tracker
+
+    def track_recommendation(self, recommendation: TrackedRecommendation) -> None:
+        self._recommendation_tracker.add(recommendation)
+
+    def apply_recommendation(self, recommendation_id: str) -> None:
+        if self._current_context:
+            self._recommendation_tracker.mark_applied(
+                recommendation_id, self._current_context.iteration_id
+            )
+            self._current_context.add_applied_recommendation(recommendation_id)
+            self._save_current_context()
+
+    def skip_recommendation(self, recommendation_id: str, reason: str) -> None:
+        self._recommendation_tracker.mark_skipped(recommendation_id, reason)
+        if self._current_context:
+            self._current_context.add_skipped_recommendation(recommendation_id)
+            self._save_current_context()
+
+    def collect_feedback(self, feedback: ModelFeedback) -> None:
+        self._latest_feedback = feedback
+        feedback_path = self.feedback_dir / f"feedback_{feedback.iteration_id}.yaml"
+        self._feedback_collector.save_feedback(feedback, str(feedback_path))
+
+    def get_latest_feedback(self) -> Optional[ModelFeedback]:
+        return self._latest_feedback
+
+    def check_for_iteration_triggers(self) -> Tuple[bool, Optional[IterationTrigger]]:
+        return self._signal_aggregator.should_trigger_iteration()
+
+    def trigger_manual_iteration(self, reason: str) -> None:
+        self._signal_aggregator.add_manual_signal(reason, {})
+
+    def prepare_iteration_from_feedback(self, feedback: ModelFeedback,
+                                        trigger: IterationTrigger) -> IterationContext:
+        new_ctx = self.start_child_iteration(trigger)
+
+        insights = self._feedback_collector.analyze_feature_importance(feedback)
+        [i.feature_name for i in insights if i.recommendation_to_drop]
+
+        return new_ctx
+
+    def get_refined_recommendations(self, findings, feedback: ModelFeedback) -> Dict[str, Any]:
+        insights = self._feedback_collector.analyze_feature_importance(feedback)
+
+        features_to_drop = [i.feature_name for i in insights if i.recommendation_to_drop]
+        top_features = self._feedback_collector.get_top_features(feedback, n=5)
+
+        refined = {
+            "features_to_drop": features_to_drop,
+            "top_features": [f[0] for f in top_features],
+            "feature_insights": [
+                {
+                    "name": i.feature_name,
+                    "rank": i.importance_rank,
+                    "score": i.importance_score,
+                    "drop": i.recommendation_to_drop
+                }
+                for i in insights
+            ],
+            "next_actions": self._feedback_collector.suggest_next_actions(feedback)
+        }
+
+        return refined
+
+    def get_iteration_history(self) -> List[IterationContext]:
+        if self._current_context:
+            return self._context_manager.get_iteration_history(
+                self._current_context.iteration_id
+            )
+        return self._context_manager.list_iterations()
+
+    def compare_iterations(self, iteration_id_1: str,
+                          iteration_id_2: str) -> Dict[str, Any]:
+        ctx1 = self._context_manager.get_by_id(iteration_id_1)
+        ctx2 = self._context_manager.get_by_id(iteration_id_2)
+
+        if ctx1 is None or ctx2 is None:
+            return {"error": "Iteration not found"}
+
+        return ctx2.compare(ctx1)
+
+    def save_state(self) -> None:
+        if self._current_context:
+            self._save_current_context()
+        self._recommendation_tracker.save()
+
+    def load_state(self) -> None:
+        self._recommendation_tracker.load()
+
+        current = self._context_manager.get_current()
+        if current:
+            self._current_context = current
+
+        feedback_files = sorted(self.feedback_dir.glob("feedback_*.yaml"))
+        if feedback_files:
+            self._latest_feedback = self._feedback_collector.load_feedback(
+                str(feedback_files[-1])
+            )
+
+    def _save_current_context(self) -> None:
+        if self._current_context:
+            self._context_manager.save_iteration(self._current_context)