churnkit 0.75.0a1__py3-none-any.whl

customer_retention/stages/features/feature_manifest.py

@@ -0,0 +1,340 @@
+"""
+Feature manifest and versioning for customer retention analysis.
+
+This module provides classes for tracking feature sets, manifests,
+and registry for version management.
+"""
+
+import hashlib
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from customer_retention.core.compat import DataFrame, pd
+
+
+@dataclass
+class FeatureManifest:
+    """
+    Manifest tracking the composition and provenance of a feature set.
+
+    Attributes
+    ----------
+    manifest_id : str
+        Unique identifier for this manifest.
+    created_at : datetime
+        When the manifest was created.
+    created_by : str, optional
+        Who created the manifest.
+    feature_table : str, optional
+        Source table name.
+    feature_table_version : int, optional
+        Delta version number if applicable.
+    features_included : List[str]
+        List of feature names included.
+    features_excluded : List[str]
+        Excluded features and reasons.
+    row_count : int
+        Number of rows in the dataset.
+    column_count : int
+        Number of feature columns.
+    checksum : str
+        Data integrity hash.
+    """
+    manifest_id: str
+    created_at: datetime
+    features_included: List[str]
+    row_count: int
+    column_count: int
+    checksum: str
+    created_by: Optional[str] = None
+    feature_table: Optional[str] = None
+    feature_table_version: Optional[int] = None
+    features_excluded: List[str] = field(default_factory=list)
+    feature_transformations: Dict[str, str] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_dataframe(
+        cls,
+        df: DataFrame,
+        feature_columns: List[str],
+        entity_column: Optional[str] = None,
+        created_by: Optional[str] = None,
+        feature_table: Optional[str] = None,
+    ) -> "FeatureManifest":
+        """
+        Create a manifest from a DataFrame.
+
+        Parameters
+        ----------
+        df : DataFrame
+            Source DataFrame.
+        feature_columns : List[str]
+            List of feature column names.
+        entity_column : str, optional
+            Entity/ID column name.
+        created_by : str, optional
+            Creator name.
+        feature_table : str, optional
+            Source table name.
+
+        Returns
+        -------
+        FeatureManifest
+            New manifest instance.
+        """
+        # Generate unique ID
+        manifest_id = str(uuid.uuid4())
+
+        # Compute checksum from feature data
+        feature_data = df[feature_columns].values
+        checksum = cls._compute_checksum(feature_data)
+
+        return cls(
+            manifest_id=manifest_id,
+            created_at=datetime.now(),
+            created_by=created_by,
+            feature_table=feature_table,
+            features_included=feature_columns.copy(),
+            row_count=len(df),
+            column_count=len(feature_columns),
+            checksum=checksum,
+        )
+
+    @staticmethod
+    def _compute_checksum(data) -> str:
+        """Compute MD5 checksum of data."""
+        # Convert to bytes and hash
+        data_bytes = pd.util.hash_pandas_object(
+            pd.DataFrame(data), index=False
+        ).values.tobytes()
+        return hashlib.md5(data_bytes).hexdigest()
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert manifest to dictionary."""
+        return {
+            "manifest_id": self.manifest_id,
+            "created_at": self.created_at.isoformat(),
+            "created_by": self.created_by,
+            "feature_table": self.feature_table,
+            "feature_table_version": self.feature_table_version,
+            "features_included": self.features_included,
+            "features_excluded": self.features_excluded,
+            "feature_transformations": self.feature_transformations,
+            "row_count": self.row_count,
+            "column_count": self.column_count,
+            "checksum": self.checksum,
+            "metadata": self.metadata,
+        }
+
+
+@dataclass
+class FeatureSet:
+    """
+    Named, versioned collection of features.
+
+    Attributes
+    ----------
+    name : str
+        Feature set name.
+    version : str
+        Version identifier (semver format).
+    description : str
+        Purpose of this feature set.
+    features_included : List[str]
+        Selected features.
+    features_excluded : List[str]
+        Dropped features.
+    exclusion_reasons : Dict[str, str]
+        Why each was dropped.
+    created_at : datetime
+        Creation timestamp.
+    created_by : str, optional
+        Creator.
+    parent_feature_set : str, optional
+        If derived from another set.
+    metadata : Dict
+        Additional info.
+    """
+    name: str
+    version: str
+    description: str
+    features_included: List[str]
+    features_excluded: List[str] = field(default_factory=list)
+    exclusion_reasons: Dict[str, str] = field(default_factory=dict)
+    created_at: datetime = field(default_factory=datetime.now)
+    created_by: Optional[str] = None
+    parent_feature_set: Optional[str] = None
+    feature_table: Optional[str] = None
+    feature_table_version: Optional[int] = None
+    transformations: Dict[str, str] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert feature set to dictionary."""
+        return {
+            "name": self.name,
+            "version": self.version,
+            "description": self.description,
+            "features_included": self.features_included,
+            "features_excluded": self.features_excluded,
+            "exclusion_reasons": self.exclusion_reasons,
+            "created_at": self.created_at.isoformat(),
+            "created_by": self.created_by,
+            "parent_feature_set": self.parent_feature_set,
+            "feature_table": self.feature_table,
+            "feature_table_version": self.feature_table_version,
+            "transformations": self.transformations,
+            "metadata": self.metadata,
+        }
+
+
+class FeatureSetRegistry:
+    """
+    Registry for managing feature sets.
+
+    Provides methods for registering, retrieving, and comparing
+    feature sets.
+    """
+
+    def __init__(self):
+        self._registry: Dict[str, Dict[str, FeatureSet]] = {}
+
+    def register(self, feature_set: FeatureSet) -> None:
+        """
+        Register a new feature set.
+
+        Parameters
+        ----------
+        feature_set : FeatureSet
+            Feature set to register.
+
+        Raises
+        ------
+        ValueError
+            If feature set with same name and version exists.
+        """
+        name = feature_set.name
+        version = feature_set.version
+
+        if name not in self._registry:
+            self._registry[name] = {}
+
+        if version in self._registry[name]:
+            raise ValueError(
+                f"Feature set '{name}' version '{version}' already registered."
+            )
+
+        self._registry[name][version] = feature_set
+
+    def get(
+        self,
+        name: str,
+        version: str
+    ) -> Optional[FeatureSet]:
+        """
+        Get a feature set by name and version.
+
+        Parameters
+        ----------
+        name : str
+            Feature set name.
+        version : str
+            Version string.
+
+        Returns
+        -------
+        FeatureSet or None
+            The feature set, or None if not found.
+        """
+        if name not in self._registry:
+            return None
+        return self._registry[name].get(version)
+
+    def get_latest(self, name: str) -> Optional[FeatureSet]:
+        """
+        Get the latest version of a feature set.
+
+        Parameters
+        ----------
+        name : str
+            Feature set name.
+
+        Returns
+        -------
+        FeatureSet or None
+            Latest version, or None if not found.
+        """
+        if name not in self._registry:
+            return None
+
+        versions = list(self._registry[name].keys())
+        if not versions:
+            return None
+
+        # Sort versions (assumes semver-like format)
+        versions.sort(key=lambda v: [int(x) for x in v.split(".")])
+        return self._registry[name][versions[-1]]
+
+    def list_all(self) -> List[FeatureSet]:
+        """
+        List all registered feature sets.
+
+        Returns
+        -------
+        List[FeatureSet]
+            All feature sets.
+        """
+        result = []
+        for versions in self._registry.values():
+            result.extend(versions.values())
+        return result
+
+    def list_versions(self, name: str) -> List[str]:
+        """
+        List all versions of a feature set.
+
+        Parameters
+        ----------
+        name : str
+            Feature set name.
+
+        Returns
+        -------
+        List[str]
+            Available versions.
+        """
+        if name not in self._registry:
+            return []
+        return list(self._registry[name].keys())
+
+    def compare(
+        self,
+        set1: FeatureSet,
+        set2: FeatureSet
+    ) -> Dict[str, List[str]]:
+        """
+        Compare two feature sets.
+
+        Parameters
+        ----------
+        set1 : FeatureSet
+            First feature set.
+        set2 : FeatureSet
+            Second feature set.
+
+        Returns
+        -------
+        Dict[str, List[str]]
+            Dictionary with 'added', 'removed', and 'unchanged' keys.
+        """
+        features1 = set(set1.features_included)
+        features2 = set(set2.features_included)
+
+        return {
+            "added": list(features2 - features1),
+            "removed": list(features1 - features2),
+            "unchanged": list(features1 & features2),
+        }

customer_retention/stages/features/feature_selector.py

@@ -0,0 +1,239 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+import numpy as np
+
+from customer_retention.core.compat import DataFrame, is_numeric_dtype, pd
+
+if TYPE_CHECKING:
+    from customer_retention.analysis.auto_explorer.findings import FeatureAvailabilityMetadata
+
+
+class SelectionMethod(Enum):
+    VARIANCE = "VARIANCE"
+    CORRELATION = "CORRELATION"
+    MUTUAL_INFO = "MUTUAL_INFO"
+    IMPORTANCE = "IMPORTANCE"
+    RECURSIVE = "RECURSIVE"
+    L1_SELECTION = "L1_SELECTION"
+
+
+@dataclass
+class FeatureSelectionResult:
+    df: DataFrame
+    selected_features: List[str]
+    dropped_features: List[str]
+    drop_reasons: Dict[str, str]
+    method_used: SelectionMethod
+    importance_scores: Optional[Dict[str, float]] = None
+
+
+@dataclass
+class AvailabilityRecommendation:
+    column: str
+    issue_type: str
+    coverage_pct: float
+    first_valid_date: Optional[str]
+    last_valid_date: Optional[str]
+    options: List[Dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "column": self.column,
+            "issue_type": self.issue_type,
+            "coverage_pct": self.coverage_pct,
+            "first_valid_date": self.first_valid_date,
+            "last_valid_date": self.last_valid_date,
+            "options": self.options,
+        }
+
+
+class FeatureSelector:
+    def __init__(self, method: SelectionMethod = SelectionMethod.VARIANCE, variance_threshold: float = 0.01, correlation_threshold: float = 0.95, target_column: Optional[str] = None, preserve_features: Optional[List[str]] = None, max_features: Optional[int] = None, apply_correlation_filter: bool = False):
+        self.method = method
+        self.variance_threshold = variance_threshold
+        self.correlation_threshold = correlation_threshold
+        self.target_column = target_column
+        self.preserve_features = preserve_features or []
+        self.max_features = max_features
+        self.apply_correlation_filter = apply_correlation_filter
+
+        self.selected_features: List[str] = []
+        self.dropped_features: List[str] = []
+        self.drop_reasons: Dict[str, str] = {}
+        self._is_fitted = False
+
+    def fit(self, df: DataFrame) -> "FeatureSelector":
+        feature_cols = [c for c in df.columns if c != self.target_column]
+
+        self.selected_features = feature_cols.copy()
+        self.dropped_features = []
+        self.drop_reasons = {}
+
+        if self.method == SelectionMethod.VARIANCE:
+            self._apply_variance_selection(df, feature_cols)
+        elif self.method == SelectionMethod.CORRELATION:
+            self._apply_correlation_selection(df, feature_cols)
+
+        if self.apply_correlation_filter and self.method != SelectionMethod.CORRELATION:
+            self._apply_correlation_selection(df, self.selected_features.copy())
+
+        if self.max_features and len(self.selected_features) > self.max_features:
+            feature_df = df[self.selected_features]
+            variances = feature_df.var().sort_values(ascending=False)
+            to_keep = variances.head(self.max_features).index.tolist()
+            to_drop = [f for f in self.selected_features if f not in to_keep]
+            for feature in to_drop:
+                if feature not in self.preserve_features:
+                    self.selected_features.remove(feature)
+                    self.dropped_features.append(feature)
+                    self.drop_reasons[feature] = "max_features limit"
+
+        self._is_fitted = True
+        return self
+
+    def transform(self, df: DataFrame) -> FeatureSelectionResult:
+        if not self._is_fitted:
+            raise ValueError("Selector not fitted. Call fit() first.")
+
+        cols_to_keep = self.selected_features.copy()
+        if self.target_column and self.target_column in df.columns:
+            cols_to_keep.append(self.target_column)
+
+        cols_to_keep = [c for c in cols_to_keep if c in df.columns]
+        result_df = df[cols_to_keep].copy()
+
+        return FeatureSelectionResult(
+            df=result_df,
+            selected_features=self.selected_features.copy(),
+            dropped_features=self.dropped_features.copy(),
+            drop_reasons=self.drop_reasons.copy(),
+            method_used=self.method,
+        )
+
+    def fit_transform(self, df: DataFrame) -> FeatureSelectionResult:
+        self.fit(df)
+        return self.transform(df)
+
+    def _apply_variance_selection(self, df: DataFrame, features: List[str]) -> None:
+        for feature in features:
+            if feature in self.preserve_features:
+                continue
+
+            series = df[feature]
+            if not is_numeric_dtype(series):
+                continue
+
+            variance = series.var()
+            if pd.isna(variance) or variance < self.variance_threshold:
+                if feature in self.selected_features:
+                    self.selected_features.remove(feature)
+                    self.dropped_features.append(feature)
+                    self.drop_reasons[feature] = f"low variance ({variance:.6f})"
+
+    def _apply_correlation_selection(self, df: DataFrame, features: List[str]) -> None:
+        numeric_features = [f for f in features if f in df.columns and is_numeric_dtype(df[f]) and f in self.selected_features]
+
+        if len(numeric_features) < 2:
+            return
+
+        corr_matrix = df[numeric_features].corr().abs()
+
+        upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))
+
+        to_drop = set()
+        for column in upper.columns:
+            correlated = upper.index[upper[column] > self.correlation_threshold].tolist()
+            for corr_feature in correlated:
+                if corr_feature in self.preserve_features:
+                    if column not in self.preserve_features:
+                        to_drop.add(column)
+                elif column in self.preserve_features:
+                    to_drop.add(corr_feature)
+                else:
+                    var1 = df[column].var()
+                    var2 = df[corr_feature].var()
+                    if var1 >= var2:
+                        to_drop.add(corr_feature)
+                    else:
+                        to_drop.add(column)
+
+        for feature in to_drop:
+            if feature in self.selected_features:
+                self.selected_features.remove(feature)
+                self.dropped_features.append(feature)
+                self.drop_reasons[feature] = f"high correlation (> {self.correlation_threshold})"
+
+    def get_availability_recommendations(self, availability: Optional["FeatureAvailabilityMetadata"]) -> List[AvailabilityRecommendation]:
+        if availability is None:
+            return []
+        recommendations: List[AvailabilityRecommendation] = []
+        problem_columns = availability.new_tracking + availability.retired_tracking + availability.partial_window
+        for col in problem_columns:
+            feat_info = availability.features.get(col)
+            if feat_info is None:
+                continue
+            recommendations.append(AvailabilityRecommendation(
+                column=col,
+                issue_type=feat_info.availability_type,
+                coverage_pct=feat_info.coverage_pct,
+                first_valid_date=feat_info.first_valid_date,
+                last_valid_date=feat_info.last_valid_date,
+                options=self._build_availability_options(col, feat_info.availability_type, feat_info.first_valid_date, feat_info.last_valid_date, feat_info.coverage_pct),
+            ))
+        return recommendations
+
+    def _build_availability_options(self, col: str, issue_type: str, first_date: Optional[str], last_date: Optional[str], coverage_pct: float) -> List[Dict[str, Any]]:
+        options: List[Dict[str, Any]] = []
+        options.append({
+            "type": "remove",
+            "description": f"Remove '{col}' from feature selection (recommended for most cases)",
+            "preserves_data": False,
+            "recommended": True,
+        })
+        options.append({
+            "type": "add_indicator",
+            "description": f"Create '{col}_available' indicator column to flag valid observations",
+            "preserves_data": True,
+        })
+        if issue_type == "new_tracking":
+            options.append({
+                "type": "filter_window",
+                "description": f"Filter training data to start from {first_date}",
+                "preserves_data": True,
+            })
+            options.append({
+                "type": "segment_by_cohort",
+                "description": f"Train separate models: pre-{first_date} cohort (without feature) vs post-{first_date} cohort (with feature)",
+                "preserves_data": True,
+            })
+        elif issue_type == "retired":
+            options.append({
+                "type": "filter_window",
+                "description": f"Filter test/scoring data to end at {last_date}",
+                "preserves_data": True,
+            })
+            options.append({
+                "type": "segment_by_cohort",
+                "description": "Use feature only for historical scoring; train fallback model without it for future predictions",
+                "preserves_data": True,
+            })
+        elif issue_type == "partial_window":
+            options.append({
+                "type": "filter_window",
+                "description": f"Use data only within {first_date} to {last_date}",
+                "preserves_data": True,
+            })
+            options.append({
+                "type": "segment_by_availability",
+                "description": "Train separate models: one using this feature (within window), one without (outside window)",
+                "preserves_data": True,
+            })
+        if coverage_pct >= 30:
+            options.append({
+                "type": "impute",
+                "description": f"Impute missing values (median/mode) - {coverage_pct:.0f}% coverage may be sufficient",
+                "preserves_data": True,
+            })
+        return options