featcopilot 0.1.0__py3-none-any.whl

featcopilot/selection/importance.py

@@ -0,0 +1,161 @@
+"""Model-based feature importance selection."""
+
+from typing import Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from featcopilot.core.base import BaseSelector
+
+
+class ImportanceSelector(BaseSelector):
+    """
+    Feature selector based on model importance scores.
+
+    Uses tree-based models to evaluate feature importance.
+
+    Parameters
+    ----------
+    model : str, default='random_forest'
+        Model to use ('random_forest', 'gradient_boosting', 'xgboost')
+    max_features : int, optional
+        Maximum features to select
+    threshold : float, optional
+        Minimum importance threshold
+
+    Examples
+    --------
+    >>> selector = ImportanceSelector(model='random_forest', max_features=50)
+    >>> X_selected = selector.fit_transform(X, y)
+    """
+
+    def __init__(
+        self,
+        model: str = "random_forest",
+        max_features: Optional[int] = None,
+        threshold: Optional[float] = None,
+        n_estimators: int = 100,
+        verbose: bool = False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.model_type = model
+        self.max_features = max_features
+        self.threshold = threshold
+        self.n_estimators = n_estimators
+        self.verbose = verbose
+        self._model = None
+
+    def fit(
+        self, X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray], **kwargs
+    ) -> "ImportanceSelector":
+        """
+        Fit selector using a tree model.
+
+        Parameters
+        ----------
+        X : DataFrame or ndarray
+            Input features
+        y : Series or ndarray
+            Target variable
+
+        Returns
+        -------
+        self : ImportanceSelector
+        """
+        X = self._validate_input(X)
+        y = np.array(y)
+
+        # Determine task type
+        unique_y = len(np.unique(y))
+        is_classification = unique_y < 20 and not np.issubdtype(y.dtype, np.floating)
+
+        # Create model
+        self._model = self._create_model(is_classification)
+
+        # Fit model
+        X_array = X.fillna(0).values
+        self._model.fit(X_array, y)
+
+        # Get importances
+        importances = self._model.feature_importances_
+        self._feature_scores = dict(zip(X.columns, importances))
+
+        # Select features
+        self._select_features()
+
+        self._is_fitted = True
+        return self
+
+    def _create_model(self, is_classification: bool):
+        """Create the appropriate model."""
+        if self.model_type == "random_forest":
+            if is_classification:
+                from sklearn.ensemble import RandomForestClassifier
+
+                return RandomForestClassifier(n_estimators=self.n_estimators, random_state=42, n_jobs=-1)
+            else:
+                from sklearn.ensemble import RandomForestRegressor
+
+                return RandomForestRegressor(n_estimators=self.n_estimators, random_state=42, n_jobs=-1)
+
+        elif self.model_type == "gradient_boosting":
+            if is_classification:
+                from sklearn.ensemble import GradientBoostingClassifier
+
+                return GradientBoostingClassifier(n_estimators=self.n_estimators, random_state=42)
+            else:
+                from sklearn.ensemble import GradientBoostingRegressor
+
+                return GradientBoostingRegressor(n_estimators=self.n_estimators, random_state=42)
+
+        elif self.model_type == "xgboost":
+            try:
+                import xgboost as xgb
+
+                if is_classification:
+                    return xgb.XGBClassifier(n_estimators=self.n_estimators, random_state=42, n_jobs=-1)
+                else:
+                    return xgb.XGBRegressor(n_estimators=self.n_estimators, random_state=42, n_jobs=-1)
+            except ImportError:
+                if self.verbose:
+                    print("XGBoost not available, falling back to RandomForest")
+                return self._create_model_fallback(is_classification)
+
+        else:
+            raise ValueError(f"Unknown model type: {self.model_type}")
+
+    def _create_model_fallback(self, is_classification: bool):
+        """Fallback to RandomForest."""
+        if is_classification:
+            from sklearn.ensemble import RandomForestClassifier
+
+            return RandomForestClassifier(n_estimators=self.n_estimators, random_state=42)
+        else:
+            from sklearn.ensemble import RandomForestRegressor
+
+            return RandomForestRegressor(n_estimators=self.n_estimators, random_state=42)
+
+    def _select_features(self) -> None:
+        """Select features based on importance."""
+        sorted_features = sorted(self._feature_scores.items(), key=lambda x: x[1], reverse=True)
+
+        if self.threshold is not None:
+            sorted_features = [(name, score) for name, score in sorted_features if score >= self.threshold]
+
+        if self.max_features is not None:
+            sorted_features = sorted_features[: self.max_features]
+
+        self._selected_features = [name for name, _ in sorted_features]
+
+        if self.verbose:
+            print(f"ImportanceSelector: Selected {len(self._selected_features)} features")
+
+    def transform(self, X: Union[pd.DataFrame, np.ndarray], **kwargs) -> pd.DataFrame:
+        """Select features from data."""
+        if not self._is_fitted:
+            raise RuntimeError("Selector must be fitted before transform")
+
+        X = self._validate_input(X)
+        available = [f for f in self._selected_features if f in X.columns]
+        return X[available]

featcopilot/selection/redundancy.py

@@ -0,0 +1,156 @@
+"""Redundancy elimination through correlation analysis."""
+
+from typing import Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from featcopilot.core.base import BaseSelector
+
+
+class RedundancyEliminator(BaseSelector):
+    """
+    Eliminate redundant features based on correlation.
+
+    Removes highly correlated features, keeping the one with
+    higher importance (if provided) or the first one.
+
+    Parameters
+    ----------
+    correlation_threshold : float, default=0.95
+        Correlation threshold for redundancy
+    method : str, default='pearson'
+        Correlation method ('pearson', 'spearman', 'kendall')
+
+    Examples
+    --------
+    >>> eliminator = RedundancyEliminator(correlation_threshold=0.95)
+    >>> X_reduced = eliminator.fit_transform(X, y)
+    """
+
+    def __init__(
+        self,
+        correlation_threshold: float = 0.95,
+        method: str = "pearson",
+        importance_scores: Optional[dict[str, float]] = None,
+        verbose: bool = False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.correlation_threshold = correlation_threshold
+        self.method = method
+        self.importance_scores = importance_scores or {}
+        self.verbose = verbose
+        self._correlation_matrix: Optional[pd.DataFrame] = None
+
+    def fit_transform(
+        self,
+        X: Union[pd.DataFrame, np.ndarray],
+        y: Optional[Union[pd.Series, np.ndarray]] = None,
+        **kwargs,
+    ) -> pd.DataFrame:
+        """Fit and transform in one step (y is optional for this selector)."""
+        return self.fit(X, y, **kwargs).transform(X, **kwargs)
+
+    def fit(
+        self,
+        X: Union[pd.DataFrame, np.ndarray],
+        y: Optional[Union[pd.Series, np.ndarray]] = None,
+        importance_scores: Optional[dict[str, float]] = None,
+        **kwargs,
+    ) -> "RedundancyEliminator":
+        """
+        Fit eliminator by computing correlations.
+
+        Parameters
+        ----------
+        X : DataFrame or ndarray
+            Input features
+        y : Series or ndarray, optional
+            Target variable (unused)
+        importance_scores : dict, optional
+            Pre-computed importance scores
+
+        Returns
+        -------
+        self : RedundancyEliminator
+        """
+        X = self._validate_input(X)
+
+        if importance_scores:
+            self.importance_scores = importance_scores
+
+        # Compute correlation matrix
+        numeric_cols = X.select_dtypes(include=[np.number]).columns
+        self._correlation_matrix = X[numeric_cols].corr(method=self.method)
+
+        # Find redundant features
+        self._find_redundant_features(numeric_cols)
+
+        self._is_fitted = True
+        return self
+
+    def _find_redundant_features(self, columns: list[str]) -> None:
+        """Identify and mark redundant features for removal."""
+        to_remove: set[str] = set()
+        checked_pairs: set[tuple] = set()
+
+        for i, col1 in enumerate(columns):
+            if col1 in to_remove:
+                continue
+
+            for col2 in columns[i + 1 :]:
+                if col2 in to_remove:
+                    continue
+
+                pair = tuple(sorted([col1, col2]))
+                if pair in checked_pairs:
+                    continue
+                checked_pairs.add(pair)
+
+                # Get correlation
+                corr = abs(self._correlation_matrix.loc[col1, col2])
+
+                if corr >= self.correlation_threshold:
+                    # Decide which to remove based on importance
+                    imp1 = self.importance_scores.get(col1, 0)
+                    imp2 = self.importance_scores.get(col2, 0)
+
+                    if imp1 >= imp2:
+                        to_remove.add(col2)
+                        if self.verbose:
+                            print(f"Removing {col2} (corr={corr:.3f} with {col1})")
+                    else:
+                        to_remove.add(col1)
+                        if self.verbose:
+                            print(f"Removing {col1} (corr={corr:.3f} with {col2})")
+                        break  # col1 is removed, move to next
+
+        # Selected features are those not removed
+        self._selected_features = [c for c in columns if c not in to_remove]
+        self._removed_features = list(to_remove)
+
+        if self.verbose:
+            print(f"RedundancyEliminator: Removed {len(to_remove)} redundant features")
+
+    def transform(self, X: Union[pd.DataFrame, np.ndarray], **kwargs) -> pd.DataFrame:
+        """Remove redundant features."""
+        if not self._is_fitted:
+            raise RuntimeError("Eliminator must be fitted before transform")
+
+        X = self._validate_input(X)
+
+        # Keep selected features plus any non-numeric columns
+        non_numeric = X.select_dtypes(exclude=[np.number]).columns.tolist()
+        keep_cols = [c for c in self._selected_features if c in X.columns]
+        keep_cols.extend([c for c in non_numeric if c not in keep_cols])
+
+        return X[keep_cols]
+
+    def get_removed_features(self) -> list[str]:
+        """Get list of removed redundant features."""
+        return getattr(self, "_removed_features", [])
+
+    def get_correlation_matrix(self) -> Optional[pd.DataFrame]:
+        """Get the computed correlation matrix."""
+        return self._correlation_matrix

featcopilot/selection/statistical.py

@@ -0,0 +1,199 @@
+"""Statistical feature selection methods."""
+
+from typing import Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from featcopilot.core.base import BaseSelector
+
+
+class StatisticalSelector(BaseSelector):
+    """
+    Feature selector based on statistical tests.
+
+    Uses statistical tests to evaluate feature relevance:
+    - Mutual information
+    - Chi-square test (categorical)
+    - F-test (ANOVA)
+    - Correlation with target
+
+    Parameters
+    ----------
+    method : str, default='mutual_info'
+        Selection method ('mutual_info', 'f_test', 'chi2', 'correlation')
+    max_features : int, optional
+        Maximum features to select
+    threshold : float, optional
+        Minimum score threshold
+
+    Examples
+    --------
+    >>> selector = StatisticalSelector(method='mutual_info', max_features=50)
+    >>> X_selected = selector.fit_transform(X, y)
+    """
+
+    METHODS = ["mutual_info", "f_test", "chi2", "correlation"]
+
+    def __init__(
+        self,
+        method: str = "mutual_info",
+        max_features: Optional[int] = None,
+        threshold: Optional[float] = None,
+        verbose: bool = False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        if method not in self.METHODS:
+            raise ValueError(f"Method must be one of {self.METHODS}")
+
+        self.method = method
+        self.max_features = max_features
+        self.threshold = threshold
+        self.verbose = verbose
+
+    def fit(
+        self, X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray], **kwargs
+    ) -> "StatisticalSelector":
+        """
+        Fit selector to compute feature scores.
+
+        Parameters
+        ----------
+        X : DataFrame or ndarray
+            Input features
+        y : Series or ndarray
+            Target variable
+
+        Returns
+        -------
+        self : StatisticalSelector
+        """
+        X = self._validate_input(X)
+        y = np.array(y)
+
+        # Compute scores based on method
+        if self.method == "mutual_info":
+            scores = self._compute_mutual_info(X, y)
+        elif self.method == "f_test":
+            scores = self._compute_f_test(X, y)
+        elif self.method == "chi2":
+            scores = self._compute_chi2(X, y)
+        elif self.method == "correlation":
+            scores = self._compute_correlation(X, y)
+        else:
+            raise ValueError(f"Unknown method: {self.method}")
+
+        self._feature_scores = dict(zip(X.columns, scores))
+
+        # Select features
+        self._select_features()
+
+        self._is_fitted = True
+        return self
+
+    def _compute_mutual_info(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
+        """Compute mutual information scores."""
+        from sklearn.feature_selection import mutual_info_classif, mutual_info_regression
+
+        # Determine if classification or regression
+        unique_y = len(np.unique(y))
+        is_classification = unique_y < 20 and y.dtype in [np.int32, np.int64, "object"]
+
+        X_array = X.fillna(0).values
+
+        if is_classification:
+            scores = mutual_info_classif(X_array, y, random_state=42)
+        else:
+            scores = mutual_info_regression(X_array, y, random_state=42)
+
+        return scores
+
+    def _compute_f_test(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
+        """Compute F-test scores."""
+        from sklearn.feature_selection import f_classif, f_regression
+
+        unique_y = len(np.unique(y))
+        is_classification = unique_y < 20
+
+        X_array = X.fillna(0).values
+
+        if is_classification:
+            scores, _ = f_classif(X_array, y)
+        else:
+            scores, _ = f_regression(X_array, y)
+
+        # Handle NaN scores
+        scores = np.nan_to_num(scores, 0)
+        return scores
+
+    def _compute_chi2(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
+        """Compute chi-square scores (for non-negative features)."""
+        from sklearn.feature_selection import chi2
+
+        X_array = X.fillna(0).values
+
+        # Chi2 requires non-negative values
+        X_positive = X_array - X_array.min(axis=0) + 1e-8
+
+        try:
+            scores, _ = chi2(X_positive, y)
+            scores = np.nan_to_num(scores, 0)
+        except Exception:
+            # Fallback to mutual information
+            scores = self._compute_mutual_info(X, y)
+
+        return scores
+
+    def _compute_correlation(self, X: pd.DataFrame, y: np.ndarray) -> np.ndarray:
+        """Compute absolute correlation with target."""
+        scores = []
+        for col in X.columns:
+            try:
+                corr = np.abs(np.corrcoef(X[col].fillna(0).values, y)[0, 1])
+                scores.append(corr if not np.isnan(corr) else 0)
+            except Exception:
+                scores.append(0)
+
+        return np.array(scores)
+
+    def _select_features(self) -> None:
+        """Select features based on scores."""
+        # Sort features by score
+        sorted_features = sorted(self._feature_scores.items(), key=lambda x: x[1], reverse=True)
+
+        # Apply threshold
+        if self.threshold is not None:
+            sorted_features = [(name, score) for name, score in sorted_features if score >= self.threshold]
+
+        # Apply max_features limit
+        if self.max_features is not None:
+            sorted_features = sorted_features[: self.max_features]
+
+        self._selected_features = [name for name, _ in sorted_features]
+
+        if self.verbose:
+            print(f"StatisticalSelector: Selected {len(self._selected_features)} features")
+
+    def transform(self, X: Union[pd.DataFrame, np.ndarray], **kwargs) -> pd.DataFrame:
+        """
+        Select features from data.
+
+        Parameters
+        ----------
+        X : DataFrame or ndarray
+            Input features
+
+        Returns
+        -------
+        X_selected : DataFrame
+            Data with only selected features
+        """
+        if not self._is_fitted:
+            raise RuntimeError("Selector must be fitted before transform")
+
+        X = self._validate_input(X)
+
+        # Keep only selected features that exist in X
+        available = [f for f in self._selected_features if f in X.columns]
+        return X[available]

featcopilot/selection/unified.py

@@ -0,0 +1,172 @@
+"""Unified feature selector combining multiple methods."""
+
+from typing import Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from featcopilot.core.base import BaseSelector
+from featcopilot.selection.importance import ImportanceSelector
+from featcopilot.selection.redundancy import RedundancyEliminator
+from featcopilot.selection.statistical import StatisticalSelector
+
+
+class FeatureSelector(BaseSelector):
+    """
+    Unified feature selector combining multiple selection methods.
+
+    Combines statistical tests, model importance, and redundancy
+    elimination for comprehensive feature selection.
+
+    Parameters
+    ----------
+    methods : list, default=['mutual_info', 'importance']
+        Selection methods to use
+    max_features : int, optional
+        Maximum features to select
+    correlation_threshold : float, default=0.95
+        Threshold for redundancy elimination
+
+    Examples
+    --------
+    >>> selector = FeatureSelector(
+    ...     methods=['mutual_info', 'importance', 'correlation'],
+    ...     max_features=50,
+    ...     correlation_threshold=0.95
+    ... )
+    >>> X_selected = selector.fit_transform(X, y)
+    """
+
+    def __init__(
+        self,
+        methods: Optional[list[str]] = None,
+        max_features: Optional[int] = None,
+        correlation_threshold: float = 0.95,
+        combination: str = "union",
+        verbose: bool = False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.methods = methods or ["mutual_info", "importance"]
+        self.max_features = max_features
+        self.correlation_threshold = correlation_threshold
+        self.combination = combination  # 'union' or 'intersection'
+        self.verbose = verbose
+        self._selectors: dict[str, BaseSelector] = {}
+        self._method_scores: dict[str, dict[str, float]] = {}
+
+    def fit(self, X: Union[pd.DataFrame, np.ndarray], y: Union[pd.Series, np.ndarray], **kwargs) -> "FeatureSelector":
+        """
+        Fit all selection methods.
+
+        Parameters
+        ----------
+        X : DataFrame or ndarray
+            Input features
+        y : Series or ndarray
+            Target variable
+
+        Returns
+        -------
+        self : FeatureSelector
+        """
+        X = self._validate_input(X)
+        y = np.array(y)
+
+        # Initialize and fit each selector
+        for method in self.methods:
+            selector = self._create_selector(method)
+            selector.fit(X, y)
+            self._selectors[method] = selector
+            self._method_scores[method] = selector.get_feature_scores()
+
+        # Combine scores from all methods
+        self._combine_scores(X.columns.tolist())
+
+        # Apply redundancy elimination
+        if self.correlation_threshold < 1.0:
+            eliminator = RedundancyEliminator(
+                correlation_threshold=self.correlation_threshold,
+                importance_scores=self._feature_scores,
+                verbose=self.verbose,
+            )
+            eliminator.fit(X)
+            non_redundant = set(eliminator.get_selected_features())
+            self._feature_scores = {k: v for k, v in self._feature_scores.items() if k in non_redundant}
+
+        # Final selection
+        self._final_selection()
+
+        self._is_fitted = True
+        return self
+
+    def _create_selector(self, method: str) -> BaseSelector:
+        """Create selector for a given method."""
+        if method == "mutual_info":
+            return StatisticalSelector(method="mutual_info", verbose=self.verbose)
+        elif method == "f_test":
+            return StatisticalSelector(method="f_test", verbose=self.verbose)
+        elif method == "chi2":
+            return StatisticalSelector(method="chi2", verbose=self.verbose)
+        elif method == "correlation":
+            return StatisticalSelector(method="correlation", verbose=self.verbose)
+        elif method == "importance":
+            return ImportanceSelector(model="random_forest", verbose=self.verbose)
+        elif method == "xgboost":
+            return ImportanceSelector(model="xgboost", verbose=self.verbose)
+        else:
+            raise ValueError(f"Unknown selection method: {method}")
+
+    def _combine_scores(self, columns: list[str]) -> None:
+        """Combine scores from multiple methods."""
+        combined = {}
+
+        for col in columns:
+            scores = []
+            for _, method_scores in self._method_scores.items():
+                if col in method_scores:
+                    # Normalize score to 0-1 range
+                    all_scores = list(method_scores.values())
+                    max_score = max(all_scores) if all_scores else 1
+                    if max_score > 0:
+                        normalized = method_scores[col] / max_score
+                    else:
+                        normalized = 0
+                    scores.append(normalized)
+
+            # Average normalized scores
+            if scores:
+                combined[col] = np.mean(scores)
+            else:
+                combined[col] = 0
+
+        self._feature_scores = combined
+
+    def _final_selection(self) -> None:
+        """Make final feature selection."""
+        sorted_features = sorted(self._feature_scores.items(), key=lambda x: x[1], reverse=True)
+
+        if self.max_features is not None:
+            sorted_features = sorted_features[: self.max_features]
+
+        self._selected_features = [name for name, _ in sorted_features]
+
+        if self.verbose:
+            print(f"FeatureSelector: Selected {len(self._selected_features)} features")
+
+    def transform(self, X: Union[pd.DataFrame, np.ndarray], **kwargs) -> pd.DataFrame:
+        """Select features from data."""
+        if not self._is_fitted:
+            raise RuntimeError("Selector must be fitted before transform")
+
+        X = self._validate_input(X)
+        available = [f for f in self._selected_features if f in X.columns]
+        return X[available]
+
+    def get_method_scores(self) -> dict[str, dict[str, float]]:
+        """Get scores from each individual method."""
+        return self._method_scores
+
+    def get_ranking(self) -> list[tuple]:
+        """Get feature ranking as list of (name, score) tuples."""
+        return sorted(self._feature_scores.items(), key=lambda x: x[1], reverse=True)

featcopilot/transformers/__init__.py

@@ -0,0 +1,11 @@
+"""Scikit-learn compatible transformers."""
+
+from featcopilot.transformers.sklearn_compat import (
+    AutoFeatureEngineer,
+    FeatureEngineerTransformer,
+)
+
+__all__ = [
+    "AutoFeatureEngineer",
+    "FeatureEngineerTransformer",
+]