classifier-toolkit 0.1.0__py3-none-any.whl

classifier_toolkit/feature_selection/wrapper_methods/sequential_selection.py

@@ -0,0 +1,566 @@
+from typing import Dict, List, Literal, Optional, Set, Union
+
+import numpy as np
+import pandas as pd
+from lightgbm import LGBMClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import get_scorer
+from sklearn.model_selection import cross_val_score, train_test_split
+from xgboost import XGBClassifier
+
+from classifier_toolkit.feature_selection.base import (
+    BaseFeatureSelector,
+    FeatureSelectionError,
+)
+
+
+class SequentialSelector(BaseFeatureSelector):
+    """
+    Sequential feature selector using various estimators.
+
+    This class implements forward, backward, and bidirectional feature selection
+    using different estimators.
+
+    Parameters
+    ----------
+    estimator_params : Optional[Dict[str, float]]
+        Parameters for the estimator.
+    estimator_name : {'random_forest', 'lightgbm', 'xgboost', 'logistic_regression'}, optional
+        The name of the estimator to use, by default 'random_forest'.
+    n_features_to_select : int, optional
+        The number of features to select, by default 1.
+    scoring : {'accuracy', 'f1', 'precision', 'recall', 'roc_auc'}, optional
+        The scoring metric to use, by default 'roc_auc'.
+    cv : int, optional
+        The number of cross-validation folds, by default 5.
+    method : {'forward', 'backward', 'bidirectional'}, optional
+        The selection method to use, by default 'forward'.
+    verbose : int, optional
+        Verbosity level, by default 0.
+    tolerance : float, optional
+        Tolerance for early stopping, by default 1e-4.
+    n_jobs : int, optional
+        The number of jobs to run in parallel, by default -1.
+    early_stopping_rounds : int, optional
+        The number of rounds for early stopping, by default 5.
+
+    Attributes
+    ----------
+    estimator_params : Dict[str, float]
+        Parameters for the estimator.
+    method : str
+        The selection method used.
+    feature_scores_ : Dict[str, float]
+        Scores for each feature.
+    tolerance : float
+        Tolerance for early stopping.
+    n_jobs : int
+        The number of jobs to run in parallel.
+    early_stopping_rounds : int
+        The number of rounds for early stopping.
+    estimator_name : str
+        The name of the estimator used.
+    estimator : object
+        The estimator object.
+    """
+
+    def __init__(
+        self,
+        estimator_params: Optional[Dict[str, Union[int, float, str]]],
+        estimator_name: Literal[
+            "random_forest", "lightgbm", "xgboost", "logistic_regression"
+        ] = "random_forest",
+        n_features_to_select: int = 1,
+        scoring: Literal[
+            "accuracy", "f1", "precision", "recall", "roc_auc"
+        ] = "roc_auc",
+        cv: int = 5,
+        method: Literal["forward", "backward", "bidirectional"] = "forward",
+        verbose: int = 0,
+        tolerance: float = 1e-4,
+        n_jobs: int = -1,
+        early_stopping_rounds: int = 5,
+    ) -> None:
+        super().__init__(
+            estimator=None,
+            n_features_to_select=n_features_to_select,
+            scoring=scoring,
+            cv=cv,
+            verbose=verbose,
+        )
+        self.estimator_params = estimator_params or {}
+        self.method = method
+        self.feature_scores_: Dict[str, float] = {}
+        self.tolerance = tolerance
+        self.n_jobs = n_jobs
+        self.early_stopping_rounds = early_stopping_rounds
+        self.estimator_name = estimator_name
+        self.estimator = self.initialize_estimator()  # type: ignore
+
+        # Handle the scoring parameter
+        self.scorer = get_scorer(scoring)
+
+    def initialize_estimator(self):
+        """
+        Initialize the estimator based on the provided name and parameters.
+
+        Returns
+        -------
+        object
+            The initialized estimator.
+
+        Raises
+        ------
+        ValueError
+            If the estimator name is not recognized.
+        """
+        if self.estimator_name == "random_forest":
+            return RandomForestClassifier(**self.estimator_params)
+        elif self.estimator_name == "lightgbm":
+            return LGBMClassifier(**self.estimator_params)
+        elif self.estimator_name == "xgboost":
+            return XGBClassifier(**self.estimator_params)
+        elif self.estimator_name == "logistic_regression":
+            return LogisticRegression(**self.estimator_params)
+        else:
+            raise ValueError(
+                "Estimator must be 'random_forest', 'lightgbm', 'xgboost', or 'logistic_regression'"
+            )
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "SequentialSelector":
+        """
+        Fit the SequentialSelector to the data.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+
+        Returns
+        -------
+        SequentialSelector
+            The fitted SequentialSelector instance.
+
+        Raises
+        ------
+        ValueError
+            If the selection method is not recognized.
+        """
+        self.feature_names_ = X.columns.tolist()
+
+        if self.method == "forward":
+            self._forward_selection(X, y)
+        elif self.method == "backward":
+            self._backward_selection(X, y)
+        elif self.method == "bidirectional":
+            self._bidirectional_selection(X, y)
+        else:
+            raise ValueError("Method must be 'forward', 'backward', or 'bidirectional'")
+
+        self.feature_importances_ = self.get_feature_importances()
+        return self
+
+    def _get_score(self, X: pd.DataFrame, y: pd.Series, features: List[int]) -> float:
+        """
+        Calculate the cross-validation score for a subset of features.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+        features : List[int]
+            The indices of features to use.
+
+        Returns
+        -------
+        float
+            The mean cross-validation score.
+
+        Raises
+        ------
+        ValueError
+            If the estimator is not set.
+        """
+        X_subset = X.iloc[:, features]
+
+        if self.estimator is not None:
+            scores = cross_val_score(
+                self.estimator, X_subset, y, scoring=self.scorer, cv=self.cv, n_jobs=-1
+            )
+            return float(np.mean(scores))
+        else:
+            raise ValueError("Estimator is not set.")
+
+    def _get_score_fast(self, X_train, X_val, y_train, y_val, features: List[int]):
+        """
+        Calculate the score for a subset of features using a fast method.
+
+        Parameters
+        ----------
+        X_train : pd.DataFrame
+            The training input features.
+        X_val : pd.DataFrame
+            The validation input features.
+        y_train : pd.Series
+            The training target variable.
+        y_val : pd.Series
+            The validation target variable.
+        features : List[int]
+            The indices of features to use.
+
+        Returns
+        -------
+        float
+            The score for the given features.
+
+        Raises
+        ------
+        ValueError
+            If the estimator is not set.
+        """
+        X_train_subset = X_train.iloc[:, features]
+        X_val_subset = X_val.iloc[:, features]
+
+        if self.estimator is not None:
+            self.estimator.fit(X_train_subset, y_train)  # type: ignore
+            return self.scorer(self.estimator, X_val_subset, y_val)
+        else:
+            raise ValueError("Estimator is not set.")
+
+    def _forward_selection(self, X: pd.DataFrame, y: pd.Series) -> None:
+        """
+        Perform forward feature selection.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+        """
+        selected: List[int] = []
+        remaining = list(range(X.shape[1]))
+        feature_scores: Dict[int, float] = {}
+
+        X_train, X_val, y_train, y_val = train_test_split(
+            X, y, test_size=0.2, random_state=42
+        )
+
+        max_features = (
+            self.n_features_to_select
+            if self.n_features_to_select is not None
+            else X.shape[1]
+        )
+
+        while len(selected) < max_features:
+            scores = []
+            for feature in remaining:
+                temp_selected = [*selected, feature]
+                score = self._get_score_fast(
+                    X_train, X_val, y_train, y_val, temp_selected
+                )
+                scores.append((score, feature))
+
+            best_score, best_feature = max(scores)
+            selected.append(best_feature)
+            remaining.remove(best_feature)
+            feature_scores[best_feature] = best_score
+
+            print(
+                f"Selected feature: {X.columns[best_feature]}, Score: {best_score:.6f}"
+            )
+
+            if self.n_features_to_select is None and len(selected) == X.shape[1]:
+                break
+
+        self.selected_features_ = [X.columns[i] for i in selected]
+        self.feature_scores_ = {X.columns[k]: v for k, v in feature_scores.items()}
+
+    def _backward_selection(self, X: pd.DataFrame, y: pd.Series) -> None:
+        """
+        Perform backward feature selection.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+        """
+        selected = list(range(X.shape[1]))
+        feature_scores: Dict[int, float] = {
+            feature: 0.0 for feature in selected
+        }  # Initialize with floats
+
+        X_train, X_val, y_train, y_val = train_test_split(
+            X, y, test_size=0.2, random_state=42
+        )
+
+        min_features = (
+            self.n_features_to_select if self.n_features_to_select is not None else 1
+        )
+
+        while len(selected) > min_features:
+            scores = []
+            for feature in selected:
+                temp_selected = [f for f in selected if f != feature]
+                score = self._get_score_fast(
+                    X_train, X_val, y_train, y_val, temp_selected
+                )
+                scores.append((score, feature))
+
+            best_score, worst_feature = max(scores)
+            selected.remove(worst_feature)
+            feature_scores[worst_feature] = best_score
+
+            print(
+                f"Removed feature: {X.columns[worst_feature]}, Score: {best_score:.6f}"
+            )
+
+            if self.n_features_to_select is None and len(selected) == 1:
+                break
+
+        self.selected_features_ = [X.columns[i] for i in selected]
+        self.feature_scores_ = {X.columns[k]: v for k, v in feature_scores.items()}
+
+    def _bidirectional_selection(self, X: pd.DataFrame, y: pd.Series) -> None:
+        """
+        Perform bidirectional feature selection.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+        y : pd.Series
+            The target variable.
+        """
+        selected: List[int] = []
+        remaining: List[int] = list(range(X.shape[1]))
+        feature_scores: Dict[int, List[float]] = {
+            feature: [] for feature in range(X.shape[1])
+        }
+        recently_added: Set[int] = set()
+        recently_removed: Set[int] = set()
+
+        X_train, X_val, y_train, y_val = train_test_split(
+            X, y, test_size=0.2, random_state=42
+        )
+
+        # Start with the best single feature
+        initial_scores = [
+            (self._get_score_fast(X_train, X_val, y_train, y_val, [f]), f)
+            for f in remaining
+        ]
+        best_score, best_feature = max(initial_scores)
+        selected.append(best_feature)
+        remaining.remove(best_feature)
+        feature_scores[best_feature].append(best_score)
+        recently_added.add(best_feature)
+
+        print(f"Initial feature: {X.columns[best_feature]}, Score: {best_score:.6f}")
+
+        max_iterations = (
+            X.shape[1] * 2
+            if self.n_features_to_select is None
+            else self.n_features_to_select * 2
+        )
+        iteration = 0
+
+        while iteration < max_iterations:
+            iteration += 1
+            # Forward step
+            forward_scores = [
+                (
+                    self._get_score_fast(
+                        X_train,
+                        X_val,
+                        y_train,
+                        y_val,
+                        [*selected, f],
+                    ),
+                    f,
+                )
+                for f in remaining
+                if f not in recently_removed
+            ]
+            best_forward_score, best_forward_feature = (
+                max(forward_scores) if forward_scores else (-np.inf, None)
+            )
+
+            # Backward step (if there's more than one feature)
+            backward_scores = []
+            if len(selected) > 1:
+                backward_scores = [
+                    (
+                        self._get_score_fast(
+                            X_train,
+                            X_val,
+                            y_train,
+                            y_val,
+                            [f for f in selected if f != feat],
+                        ),
+                        feat,
+                    )
+                    for feat in selected
+                    if feat not in recently_added
+                ]
+            best_backward_score, best_backward_feature = (
+                max(backward_scores) if backward_scores else (-np.inf, None)
+            )
+
+            # Update feature scores
+            for score, feature in forward_scores:
+                feature_scores[feature].append(score)
+            for score, feature in backward_scores:
+                feature_scores[feature].append(score)
+
+            # Decide whether to add or remove a feature
+            if (
+                best_forward_score > best_backward_score
+                and best_forward_feature is not None
+            ):
+                selected.append(best_forward_feature)
+                remaining.remove(best_forward_feature)
+                recently_added.add(best_forward_feature)
+                recently_removed.clear()
+                print(
+                    f"Added feature: {X.columns[best_forward_feature]}, Score: {best_forward_score:.6f}"
+                )
+            elif best_backward_feature is not None:
+                selected.remove(best_backward_feature)
+                remaining.append(best_backward_feature)
+                recently_removed.add(best_backward_feature)
+                recently_added.clear()
+                print(
+                    f"Removed feature: {X.columns[best_backward_feature]}, Score: {best_backward_score:.6f}"
+                )
+            else:
+                print("No improvement possible. Stopping.")
+                break
+
+            # Clear recent sets after a few iterations to allow reconsideration
+            if len(selected) % 5 == 0:
+                recently_added.clear()
+                recently_removed.clear()
+
+            print(f"Iteration {iteration}, Selected features: {len(selected)}")
+
+            # Check if we've reached the desired number of features
+            if len(selected) == self.n_features_to_select:
+                print(
+                    f"Reached desired number of features ({self.n_features_to_select}). Stopping."
+                )
+                break
+
+            # Check if we've reached the maximum number of iterations
+            if iteration >= max_iterations:
+                print(
+                    f"Reached maximum number of iterations ({max_iterations}). Stopping."
+                )
+                break
+
+        self.selected_features_ = [
+            X.columns[i] for i in selected[: self.n_features_to_select]
+        ]
+
+        # Calculate average scores for all features
+        if self.feature_scores_ is None:
+            self.feature_scores_ = {}
+
+        for feature_idx, scores in feature_scores.items():
+            feature_name = X.columns[feature_idx]
+            self.feature_scores_[feature_name] = (
+                float(np.mean(scores)) if scores else 0.0
+            )
+
+        print("\nFeature selection completed.")
+        print(f"Total iterations: {iteration}")
+        print(f"Selected features: {len(self.selected_features_)}")
+        print("Feature importances (average scores):")
+        for feature, score in sorted(  # type: ignore
+            self.feature_scores_.items(), key=lambda x: x[1], reverse=True
+        ):
+            print(f"{feature}: {score:.6f}")
+
+    def get_feature_importances(self) -> pd.Series:
+        """
+        Get the feature importances (scores) from the fitted model.
+
+        Returns
+        -------
+        pd.Series
+            The feature importances.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If the selector has not been fitted yet.
+        """
+        if self.feature_scores_ is None:
+            raise FeatureSelectionError(
+                "Selector has not been fitted yet. Call 'fit' first."
+            )
+
+        importances = pd.Series(self.feature_scores_)
+
+        # Normalize importances
+        if importances.sum() > 0:
+            importances = importances / importances.sum()
+        else:
+            print("Warning: Sum of importances is zero. Skipping normalization.")
+
+        return importances
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """
+        Transform the input data by selecting the chosen features.
+
+        Parameters
+        ----------
+        X : pd.DataFrame
+            The input features.
+
+        Returns
+        -------
+        pd.DataFrame
+            The transformed data containing only the selected features.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If the selector has not been fitted yet.
+        """
+        if self.selected_features_ is None:
+            raise FeatureSelectionError(
+                "Selector has not been fitted yet. Call 'fit' first."
+            )
+        return X[self.selected_features_]
+
+    def print_feature_importances(self):
+        """
+        Print the feature importances (scores) in descending order.
+
+        Raises
+        ------
+        FeatureSelectionError
+            If feature importances are not available (i.e., fit hasn't been called).
+        """
+        if self.feature_importances_ is None:
+            raise FeatureSelectionError(
+                "Feature importances are not available. Call 'fit' first."
+            )
+
+        sorted_importances = sorted(
+            zip(self.feature_importances_.index, self.feature_importances_),
+            key=lambda x: x[1],
+            reverse=True,
+        )
+
+        print("Feature importances (in descending order):")
+        for feature, importance in sorted_importances:
+            print(f"('{feature}', {importance:.6f})")

classifier_toolkit/model_fitting/model_search.py

@@ -0,0 +1,3 @@
+"""
+import optuna
+"""

classifier_toolkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,99 @@
+Metadata-Version: 2.1
+Name: classifier-toolkit
+Version: 0.1.0
+Summary: 
+Author: gauthier.marquand
+Author-email: gauthier.marquand@qonto.com
+Requires-Python: >=3.9,<3.11
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Requires-Dist: catboost (>=1.2.2,<2.0.0)
+Requires-Dist: category-encoders (>=2.6.3,<3.0.0)
+Requires-Dist: colorama (>=0.4.6,<0.5.0)
+Requires-Dist: graphviz (>=0.20.3,<0.21.0)
+Requires-Dist: ipywidgets (>=8.1.5,<9.0.0)
+Requires-Dist: lightgbm (>=4.5.0,<5.0.0)
+Requires-Dist: matplotlib (>=3.9.2,<4.0.0)
+Requires-Dist: nbformat (>=5.10.4,<6.0.0)
+Requires-Dist: numpy (<1.26)
+Requires-Dist: optuna (>=3.5.0,<4.0.0)
+Requires-Dist: pandas (>=2.2.0,<3.0.0)
+Requires-Dist: polars (>=1.2.1,<2.0.0)
+Requires-Dist: pyarrow (>=17.0.0,<18.0.0)
+Requires-Dist: pydantic-settings (>=2.3.4,<3.0.0)
+Requires-Dist: pydot (>=3.0.1,<4.0.0)
+Requires-Dist: pylint (>=3.2.6,<4.0.0)
+Requires-Dist: scikit-learn (>=1.4.0,<2.0.0)
+Requires-Dist: scipy (<1.12)
+Requires-Dist: seaborn (>=0.10.1)
+Requires-Dist: shap (>=0.44.1,<0.45.0)
+Requires-Dist: snowflake (>=0.10.0,<0.11.0)
+Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: xgboost (>=2.1.1,<3.0.0)
+Requires-Dist: ydata-profiling (>=4.6.4,<5.0.0)
+Description-Content-Type: text/markdown
+
+# Classifier Toolkit
+
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+[![Linting - Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Code style - Black](https://img.shields.io/badge/Code%20Style-Black-000000.svg)](https://github.com/psf/black)
+
+This is a new project.
+
+-----
+
+## Table of Content
+
+<!-- [[_TOC_]] -->
+
+1. [Installation](#installation)
+2. [Usage](#usage)
+3. [Modules Overview](#modules-overview)
+4. [Future Work](#future-work)
+
+### Installation
+
+This library is published in the PyPI directory. To install, users can run pip install 'classifier_toolkit' command.
+
+### Usage
+
+This library automates binary classification tasks in the finance domain, specifically for default and fraud labeling. It includes several packages designed to address the main steps in any machine learning/data science task:
+
+1. EDA: which is accessible by EDA_Toolkit. This package provides the EDA and feature engineering functionality alongside with all the necessary visualizations.
+2. Feature Selection: To be implemented.
+3. Model fitting and hyperparameter tuning: To be implemented.
+4. Evaluation and reporting: To be implemented.
+
+In the future, the package architectures will be included here. However, for now please consult the docstrings in the specific methods in the relevant modules.
+
+**Note**: that this library does not contain data wrangling steps (although it contains feature engineering), it's an intermediate step between EDA and feature engineering where users should fix any data quality related issues. Therefore, conducting the EDA is crucial to mitigate any issues before moving onto the feature engineering and the subsequent steps.
+
+### Modules Overview
+
+- **EDA Toolkit**: This module includes classes and methods for performing comprehensive exploratory data analysis. It provides automated warnings for data quality issues, univariate and bivariate analysis, and various data visualizations to help understand the dataset.
+
+- **Univariate Analysis**: This class focuses on the analysis of individual variables. It includes methods for calculating statistical measures, visualizing distributions, and assessing relationships between variables and a target through techniques like Cramer's V and Information Value. This helps in understanding the significance and distribution of each feature independently.
+
+- **Bivariate Analysis**: This class deals with the analysis of two variables to understand their relationship. It includes functionalities for generating correlation heatmaps, performing ANOVA tests between numerical and categorical variables, and computing pairwise Cramer's V for categorical features. This aids in identifying patterns and correlations between pairs of variables, which is crucial for feature selection and engineering.
+
+- **Feature Engineering**: This module assists in transforming features, handling missing values, encoding categorical variables, and more. It aims to enhance the dataset's quality for better model performance.
+
+- **Visualizations**: This module offers a wide range of plotting capabilities to visually analyze data distributions, relationships, and other crucial aspects of the dataset.
+
+- **Automated Warnings**: A utility to automatically check the dataset for common issues such as missing or duplicate values, outliers, and more, providing warnings to guide data cleaning efforts.
+
+- **Feature Selection**: This module provides various feature selection techniques:
+  - **Embedded Methods**: Includes ElasticNet for regularization-based feature selection.
+  - **Wrapper Methods**: 
+    - Recursive Feature Elimination (RFE) with support for various ensemble methods (Random Forest, XGBoost, LightGBM, CatBoost).
+    - Sequential Feature Selection (forward, backward, floating, and bidirectional).
+  - **Meta Selector**: Combines multiple feature selection methods to provide a robust selection.
+  - **Utility Functions**: Includes scoring functions and plotting utilities for feature importance visualization.
+
+### Future Work
+The next planned improvements and additions to the library include:
+* Adding model fitting and hyperparameter tuning functionalities.
+* Developing comprehensive evaluation and reporting tools to assist with model assessment.
+* Expanding documentation to include architecture diagrams and detailed usage examples.