classifier-toolkit 0.1.0__py3-none-any.whl

classifier_toolkit/feature_selection/embedded_methods/elastic_net.py

@@ -0,0 +1,178 @@
+from typing import List, Optional
+
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator
+from sklearn.linear_model import ElasticNetCV, LogisticRegression
+from sklearn.model_selection import cross_val_score
+
+from classifier_toolkit.feature_selection.base import (
+    BaseFeatureSelector,
+    FeatureSelectionError,
+)
+
+
+class ElasticNetSelector(BaseFeatureSelector):
+    """
+    Feature selector based on Elastic Net regularization.
+
+    This class implements feature selection using Elastic Net, which combines
+    L1 and L2 regularization.
+
+    Parameters
+    ----------
+    estimator : Optional[BaseEstimator], optional
+        The estimator to use, by default None.
+    l1_ratio : float, optional
+        The ElasticNet mixing parameter, by default 0.5.
+    max_iter : int, optional
+        Maximum number of iterations, by default 10000.
+    **kwargs
+        Additional keyword arguments to be passed to the BaseFeatureSelector.
+
+    Attributes
+    ----------
+    l1_ratio : float
+        The ElasticNet mixing parameter.
+    max_iter : int
+        Maximum number of iterations.
+    model : Optional[ElasticNetCV]
+        The fitted ElasticNetCV model.
+    feature_importances_ : pd.Series
+        The feature importances (coefficients) from the fitted model.
+    selected_features_ : List[str]
+        The list of selected features.
+    """
+
+    def __init__(
+        self,
+        estimator: Optional[BaseEstimator] = None,
+        l1_ratio: float = 0.5,
+        max_iter: int = 10000,
+        **kwargs,
+    ) -> None:
+        # Use LogisticRegression as the default estimator if none is provided
+        if estimator is None:
+            estimator = LogisticRegression(random_state=42)
+        super().__init__(estimator=estimator, **kwargs)
+        self.l1_ratio = l1_ratio
+        self.max_iter = max_iter
+        self.model: Optional[ElasticNetCV] = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "ElasticNetSelector":
+        """
+        Fit the ElasticNetSelector to the data.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+
+        Returns
+        -------
+        ElasticNetSelector
+            The fitted ElasticNetSelector instance.
+
+        Raises
+        ------
+        ValueError
+            If model fitting fails or coefficients are not available.
+        """
+        self.model = ElasticNetCV(
+            l1_ratio=self.l1_ratio, cv=self.cv, max_iter=self.max_iter
+        ).fit(X, y)  # type: ignore
+        if self.model is not None and hasattr(self.model, "coef_"):
+            self.feature_importances_ = pd.Series(
+                abs(self.model.coef_), index=X.columns
+            )
+            if self.feature_importances_ is not None:
+                self.selected_features_ = self.feature_importances_[
+                    self.feature_importances_ > 0
+                ].index.tolist()
+            else:
+                raise ValueError("Feature importances are None after fitting.")
+        else:
+            raise ValueError("Model fitting failed or coefficients are not available.")
+        return self
+
+    def get_feature_importances(self) -> pd.Series:
+        """
+        Get the feature importances (coefficients) from the fitted model.
+
+        Returns
+        -------
+        pd.Series
+            The feature importances.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If the selector has not been fitted yet.
+        """
+        if self.feature_importances_ is None:
+            raise FeatureSelectionError(
+                "Selector has not been fitted yet. Call 'fit' first."
+            )
+        return self.feature_importances_
+
+    def _get_score(self, X: pd.DataFrame, y: pd.Series, features: List[int]) -> float:
+        """
+        Calculate the cross-validation score for a subset of features.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+        features : List[int]
+            The indices of features to use.
+
+        Returns
+        -------
+        float
+            The mean cross-validation score.
+
+        Raises
+        ------
+        ValueError
+            If the estimator is not set.
+        """
+        if self.estimator is None:
+            raise ValueError("Estimator is not set. Please provide an estimator.")
+
+        X_subset = X.iloc[:, features]
+        scores = cross_val_score(
+            self.estimator,
+            X_subset,
+            y,
+            scoring=self.scorer,
+            cv=self.cv,
+            n_jobs=-1,
+            error_score="raise",
+        )
+        return float(np.mean(scores))
+
+    def return_feature_importances(self):
+        """
+        Print the feature importances (coefficients) in descending order of absolute value.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If feature importances are not available (i.e., fit hasn't been called).
+        """
+        if self.feature_importances_ is None:
+            raise FeatureSelectionError(
+                "Feature importances are not available. Call 'fit' first."
+            )
+
+        sorted_importances = sorted(
+            zip(self.feature_importances_.index, self.feature_importances_),
+            key=lambda x: abs(x[1]),  # Sort by absolute value of coefficients
+            reverse=True,
+        )
+
+        return sorted_importances

classifier_toolkit/feature_selection/meta_selector.py

@@ -0,0 +1,329 @@
+from typing import List, Literal, Optional, Union
+
+import pandas as pd
+from tqdm import tqdm
+
+from classifier_toolkit.feature_selection.base import (
+    BaseFeatureSelector,
+    FeatureSelectionError,
+)
+from classifier_toolkit.feature_selection.wrapper_methods.rfe import (
+    EnsembleFeatureSelector,
+)
+from classifier_toolkit.feature_selection.wrapper_methods.sequential_selection import (
+    SequentialSelector,
+)
+
+
+class MetaSelector(BaseFeatureSelector):
+    """
+    A meta-selector that combines multiple feature selection methods.
+
+    This class allows for the combination of multiple feature selection methods
+    using different voting strategies.
+
+    Parameters
+    ----------
+    methods : Optional[List[BaseFeatureSelector]], optional
+        List of feature selection methods to be combined, by default None.
+    voting : str, optional
+        The voting strategy to use for combining methods, by default "majority".
+        Options are "majority", "union", or "intersection".
+    n_features_to_select : Optional[int], optional
+        The number of features to select, by default None.
+    scoring : str, optional
+        The scoring metric to use, by default "accuracy".
+    cv : int, optional
+        The number of cross-validation folds, by default 5.
+    verbose : int, optional
+        Verbosity level, by default 0.
+
+    Attributes
+    ----------
+    methods : List[BaseFeatureSelector]
+        The list of feature selection methods.
+    voting : str
+        The voting strategy used.
+    feature_importances_ : pd.Series
+        The combined feature importances.
+    selected_features_ : List[str]
+        The list of selected features.
+
+    Raises
+    ------
+    ValueError
+        If any of the provided methods is not an instance of BaseFeatureSelector.
+    """
+
+    def __init__(
+        self,
+        methods: Optional[List[BaseFeatureSelector]] = None,
+        voting: Literal["majority", "union", "intersection"] = "majority",
+        n_features_to_select: int = 1,
+        scoring: Literal[
+            "accuracy", "f1", "precision", "recall", "roc_auc"
+        ] = "accuracy",
+        cv: int = 5,
+        verbose: int = 0,
+    ):
+        super().__init__(
+            estimator=None,  # MetaSelector doesn't need its own estimator
+            n_features_to_select=n_features_to_select,
+            scoring=scoring,
+            cv=cv,
+            verbose=verbose,
+        )
+        self.methods = methods or [
+            EnsembleFeatureSelector(
+                estimator="random_forest",
+                n_features_to_select=self.n_features_to_select,
+            ),
+            SequentialSelector(
+                method="forward",
+                estimator_name="lightgbm",
+                estimator_params={"n_estimators": 100},
+                n_features_to_select=self.n_features_to_select,
+            ),
+        ]
+        if not all(isinstance(method, BaseFeatureSelector) for method in self.methods):
+            raise ValueError("All methods must be instances of BaseFeatureSelector")
+        self.voting = voting
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "MetaSelector":
+        """
+        Fit the MetaSelector to the data.
+
+        This method fits all the individual feature selection methods and then
+        combines their results based on the specified voting strategy.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+
+        Returns
+        -------
+        MetaSelector
+            The fitted MetaSelector instance.
+
+        Raises
+        ------
+        ValueError
+            If an invalid voting strategy is specified.
+        """
+        # Create a progress bar
+        pbar = tqdm(total=len(self.methods), desc="Fitting methods")
+
+        for i, method in enumerate(self.methods):
+            print(
+                f"\nFitting method {i+1}/{len(self.methods)}: {type(method).__name__}"
+            )
+            method.fit(X, y)
+            # Update the progress bar
+            pbar.update(1)
+            pbar.set_description(f"Fitted {type(method).__name__}")
+
+        # Close the progress bar
+        pbar.close()
+
+        self._combine_feature_importances()
+
+        if self.voting == "majority":
+            self._majority_voting()
+        elif self.voting == "union":
+            self._union_voting()
+        elif self.voting == "intersection":
+            self._intersection_voting()
+        else:
+            raise ValueError("Voting must be 'majority', 'union', or 'intersection'")
+
+        return self
+
+    def _combine_feature_importances(self):
+        all_importances = [method.get_feature_importances() for method in self.methods]
+        combined_importances = pd.concat(all_importances, axis=1)
+
+        if self.voting == "majority":
+            self.feature_importances_ = combined_importances.mean(axis=1)
+        elif self.voting == "union" or self.voting == "intersection":
+            self.feature_importances_ = combined_importances.max(axis=1)
+
+        self.feature_importances_ = self.feature_importances_.sort_values(
+            ascending=False
+        )
+
+    def print_feature_importances(self):
+        if self.feature_importances_ is None:
+            raise FeatureSelectionError(
+                "Feature importances are not available. Call 'fit' first."
+            )
+
+        print("Combined Feature Importances (in descending order):")
+        for feature, importance in self.feature_importances_.items():
+            print(f"('{feature}', {importance:.6f})")
+
+    def get_feature_importances(self) -> pd.Series:
+        """
+        Get the combined feature importances.
+
+        Returns
+        -------
+        pd.Series
+            The combined feature importances.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If the selector has not been fitted yet.
+        """
+        if self.feature_importances_ is None:
+            raise FeatureSelectionError(
+                "Selector has not been fitted yet. Call 'fit' first."
+            )
+        return self.feature_importances_
+
+    def _majority_voting(self):
+        all_importances = [method.get_feature_importances() for method in self.methods]
+        combined_importances = pd.concat(all_importances, axis=1)
+        threshold = len(self.methods) / 2
+        feature_counts = (combined_importances > 0).sum(axis=1)
+        self.selected_features_ = list(feature_counts[feature_counts > threshold].index)
+        self.feature_importances_ = combined_importances.loc[
+            self.selected_features_
+        ].mean(axis=1)
+
+    def _union_voting(self):
+        all_selected = [set(method.selected_features_) for method in self.methods]
+        union_features = set.union(*all_selected)
+        self.selected_features_ = list(union_features)
+        all_importances = [method.get_feature_importances() for method in self.methods]
+        combined_importances = pd.concat(all_importances, axis=1)
+        self.feature_importances_ = combined_importances.loc[
+            self.selected_features_
+        ].mean(axis=1)
+
+    def _intersection_voting(self):
+        all_selected = [set(method.selected_features_) for method in self.methods]
+        intersection_features = set.intersection(*all_selected)
+        self.selected_features_ = list(intersection_features)
+        all_importances = [method.get_feature_importances() for method in self.methods]
+        combined_importances = pd.concat(all_importances, axis=1)
+        self.feature_importances_ = combined_importances.loc[
+            self.selected_features_
+        ].mean(axis=1)
+
+    def print_results(self):
+        """
+        Print the results of the feature selection process.
+
+        This method prints the results of individual methods, the combined results
+        based on voting, and the final feature importances.
+        """
+        print("Individual method results:")
+        for method in self.methods:
+            print(f"{type(method).__name__}: {method.selected_features_}")
+
+        print("\nCombined results based on voting:")
+        print(f"Voting method: {self.voting}")
+        print(f"Selected features: {self.selected_features_}")
+
+        print("\nFinal feature importances:")
+        for feature, importance in self.feature_importances_.sort_values(
+            ascending=False
+        ).items():
+            print(f"('{feature}', {importance:.6f})")
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """
+        Transform the input data by selecting the chosen features.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+
+        Returns
+        -------
+        pd.DataFrame
+            The transformed data containing only the selected features.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If the selector has not been fitted yet.
+        """
+        if self.selected_features_ is None:
+            raise FeatureSelectionError(
+                "Selector has not been fitted yet. Call 'fit' first."
+            )
+        return X[self.selected_features_]
+
+    def fit_transform(self, X: pd.DataFrame, y: pd.Series) -> pd.DataFrame:
+        """
+        Fit the MetaSelector to the data and then transform it.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+
+        Returns
+        -------
+        pd.DataFrame
+            The transformed data containing only the selected features.
+        """
+        return self.fit(X, y).transform(X)
+
+    def get_support(self, indices: bool = False) -> Union[List[bool], List[int]]:
+        """
+        Get a boolean mask or integer index array indicating the selected features.
+
+        Parameters
+        ----------
+        indices : bool, optional
+            If True, returns an integer index array, otherwise returns a boolean mask.
+
+        Returns
+        -------
+        Union[List[bool], List[int]]
+            Boolean mask or integer index array indicating the selected features.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If the selector has not been fitted yet.
+        """
+        if self.selected_features_ is None:
+            raise FeatureSelectionError(
+                "Selector has not been fitted yet. Call 'fit' first."
+            )
+        assert self.feature_importances_ is not None, ValueError(
+            'Feature importances are not available. Call "fit" first.'
+        )
+        mask = [
+            feature in self.selected_features_
+            for feature in self.feature_importances_.index
+        ]
+        if indices:
+            return [i for i, m in enumerate(mask) if m]
+        return mask
+
+    def get_individual_results(self):
+        """
+        Get the results of individual feature selection methods.
+
+        Returns
+        -------
+        dict
+            A dictionary containing the selected features for each method.
+        """
+        return {
+            type(method).__name__: method.selected_features_ for method in self.methods
+        }
+
+    def _get_score(self, X: pd.DataFrame, y: pd.Series, features: List[int]) -> float:
+        raise NotImplementedError

classifier_toolkit/feature_selection/utils/__init__.py

@@ -0,0 +1,17 @@
+from classifier_toolkit.feature_selection.utils.plottings import (
+    plot_feature_importances,
+    plot_rfecv_results,
+)
+from classifier_toolkit.feature_selection.utils.scoring import (
+    false_positive_rate,
+    get_scorer,
+    true_positive_rate,
+)
+
+__all__ = [
+    "plot_feature_importances",
+    "plot_rfecv_results",
+    "get_scorer",
+    "false_positive_rate",
+    "true_positive_rate",
+]

classifier_toolkit/feature_selection/utils/plottings.py

@@ -0,0 +1,65 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+from classifier_toolkit.feature_selection.base import BaseFeatureSelector
+
+
+def plot_feature_importances(selector: BaseFeatureSelector) -> None:
+    importances = selector.get_feature_importances().sort_values(ascending=True)
+
+    plt.figure(figsize=(10, max(8, len(importances) * 0.3)))
+    y_pos = np.arange(len(importances))
+    plt.barh(y_pos, np.array(importances.values))
+    plt.yticks(y_pos, importances.index.astype(str).tolist())
+    plt.xlabel("Importance")
+    plt.title("Feature Importances")
+    plt.tight_layout()
+    plt.show()
+
+
+def plot_rfecv_results(selector: BaseFeatureSelector) -> None:
+    assert hasattr(
+        selector, "rfecv"
+    ), "This plot is only available for RFECV-based selectors."
+    assert (
+        selector.rfecv is not None
+    ), "RFECV results are not available. Make sure to fit the selector first."
+    assert hasattr(
+        selector.rfecv, "cv_results_"
+    ), "The selector's RFECV doesn't have cv_results_. Make sure to fit the selector first."
+
+    n_features_total = len(selector.get_feature_importances())
+
+    grid_scores = selector.rfecv.cv_results_["mean_test_score"]
+    std_errors = selector.rfecv.cv_results_["std_test_score"]
+
+    n_features = np.arange(1, n_features_total + 1)
+
+    plt.figure(figsize=(12, 6))
+    plt.title("Feature Selection using RFECV")
+    plt.xlabel("Number of features selected")
+    plt.ylabel("Cross-validation score")
+    plt.plot(n_features, grid_scores)
+    plt.fill_between(
+        n_features, grid_scores - std_errors, grid_scores + std_errors, alpha=0.2
+    )
+    plt.axvline(
+        selector.rfecv.n_features_,
+        color="red",
+        linestyle="--",
+        label=f"Optimal number of features ({selector.rfecv.n_features_})",
+    )
+    plt.legend()
+    plt.tight_layout()
+    plt.show()
+
+    # Plot feature importances
+    importances = selector.get_feature_importances().sort_values(ascending=False)
+    plt.figure(figsize=(12, 8))
+    plt.title("Feature Importances")
+    importances.plot(kind="bar")
+    plt.xlabel("Features")
+    plt.ylabel("Importance")
+    plt.xticks(rotation=90)
+    plt.tight_layout()
+    plt.show()

classifier_toolkit/feature_selection/utils/scoring.py

@@ -0,0 +1,86 @@
+from typing import Callable, Dict, Union
+
+from sklearn.metrics import (
+    accuracy_score,
+    confusion_matrix,
+    f1_score,
+    make_scorer,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+)
+
+# Define a dictionary of scoring functions
+SCORING_FUNCTIONS: Dict[str, Callable] = {
+    "accuracy": accuracy_score,
+    "f1": f1_score,
+    "precision": precision_score,
+    "recall": recall_score,
+    "roc_auc": roc_auc_score,
+    # add prauc to the scoring functions, and make it default in the Literal call
+}
+
+
+def get_scorer(metric: Union[str, Callable]) -> Callable:
+    """
+    Get a scorer function for the given metric.
+
+    Args:
+        metric (str or callable): The metric to use for scoring.
+                                  If string, must be one of the keys in SCORING_FUNCTIONS.
+                                  If callable, should be a custom scoring function.
+
+    Returns:
+        A scorer function that can be used with scikit-learn's cross-validation and model selection tools.
+
+    Raises:
+        ValueError: If the metric is not recognized.
+    """
+    if isinstance(metric, str):
+        if metric not in SCORING_FUNCTIONS:
+            raise ValueError(
+                f"Unrecognized metric: {metric}. Available metrics are: {', '.join(SCORING_FUNCTIONS.keys())}"
+            )
+        return make_scorer(SCORING_FUNCTIONS[metric])
+    elif callable(metric):
+        return make_scorer(metric)
+    else:
+        raise ValueError("metric must be either a string or a callable")
+
+
+def false_positive_rate(y_true: int, y_pred: int) -> float:
+    """
+    Calculate the false positive rate.
+
+    Args:
+        y_true: True labels
+        y_pred: Predicted labels
+
+    Returns:
+        False positive rate
+    """
+    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+    return fp / (fp + tn)
+
+
+def true_positive_rate(y_true: int, y_pred: int):
+    """
+    Calculate the true positive rate (also known as recall or sensitivity).
+
+    Args:
+        y_true: True labels
+        y_pred: Predicted labels
+
+    Returns:
+        True positive rate
+    """
+    return recall_score(y_true, y_pred)
+
+
+# Add FPR and TPR to the scoring functions
+SCORING_FUNCTIONS["fpr"] = false_positive_rate
+SCORING_FUNCTIONS["tpr"] = true_positive_rate
+
+# Create scorers for FPR and TPR
+fpr_scorer = make_scorer(false_positive_rate, greater_is_better=False)
+tpr_scorer = make_scorer(true_positive_rate, greater_is_better=True)

classifier_toolkit/feature_selection/wrapper_methods/__init__.py

@@ -0,0 +1,11 @@
+from classifier_toolkit.feature_selection.wrapper_methods.rfe import (
+    EnsembleFeatureSelector,
+)
+from classifier_toolkit.feature_selection.wrapper_methods.sequential_selection import (
+    SequentialSelector,
+)
+
+__all__ = [
+    "SequentialSelector",
+    "EnsembleFeatureSelector",
+]