classifier-toolkit 0.1.3__tar.gz → 0.2.0__tar.gz

classifier_toolkit-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Olinda SAS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{classifier_toolkit-0.1.3 → classifier_toolkit-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: classifier-toolkit
-Version: 0.1.3
+Version: 0.2.0
 Summary: 
 Author: gauthier.marquand
 Author-email: gauthier.marquand@qonto.com

{classifier_toolkit-0.1.3 → classifier_toolkit-0.2.0}/classifier_toolkit/eda/eda_toolkit.py

@@ -1,4 +1,3 @@
-from datetime import date, datetime
 from pathlib import Path
 from typing import Any, Dict, List, Literal, Optional, Union
 
@@ -35,8 +34,8 @@ class EDAToolkit:
                 "Please make sure to specify the path in the constants file."
             )
         else:
-            path_to_save = f"{path}/df_{fname}.csv"
-            interm.to_csv(path_to_save, index=False)
+            path_to_save = f"{path}/df_{fname}.parquet"
+            interm.to_parquet(path_to_save, index=False)
 
     def __init__(
         self,
@@ -88,6 +87,9 @@ class EDAToolkit:
         # Initialize an empty list for transformations
         self.transformations: TransformationType = []
 
+        # Create the empty transformer object for normality
+        self.normality_transformer = None
+
         # Initialize the warning system
         default_warning_system = get_default_warnings(
             df=self.dataframe,
@@ -153,6 +155,17 @@ class EDAToolkit:
                 ]
         return self.transformations
 
+    def get_normality_transformer(self):
+        """
+        Get the normality transformer object.
+
+        Returns
+        -------
+        Any
+            The normality transformer object.
+        """
+        return self.normality_transformer
+
     def register_custom_warning(self, name: str, warning_obj: EDAWarning):
         """
         Register a custom warning function.
@@ -206,12 +219,20 @@ class EDAToolkit:
         self.first_glance.super_vision()
         self.first_glance.check_duplicates()
 
-    def plot_target_evolution(self):
+    def plot_target_evolution(self, time_column: str, evolving_numeric: str):
         """
         Plot the target balance and evolution over time.
+
+        Parameters
+        ----------
+        time_column : str
+            The name of the time column.
+        evolving_numeric : str
+            The name of the numerical column to plot that we want to see the evolution of
+            which helps us understanding the behavior of the target column.
         """
         self.visualizations.plot_target_balance()
-        self.visualizations.plot_target_evolution("scheduled_at", "min_balance_3m")
+        self.visualizations.plot_target_evolution(time_column, evolving_numeric)
 
     def plot_numerical_distributions(self, sub_plots: bool = False):
         """
@@ -232,7 +253,7 @@ class EDAToolkit:
         """
         self.visualizations.plot_categorical_value_counts()
         self.visualizations.visualize_high_cardinality(
-            excluded_vars=self.exclusion, id_column=self.id_column
+            excluded_columns=self.exclusion, id_column=self.id_column
         )
         self.visualizations.visualize_low_cardinality()
 
@@ -272,74 +293,33 @@ class EDAToolkit:
             time_column, number_of_partitions
         )
 
-    def print_psi(
+    def plot_psi(
         self,
-        buckets: int,
-        bucket_type: Literal["quantile", "fixed_width"],
-        time_column: str,
-        time_expected: Union[str, datetime, date],
-        end_date: Union[str, datetime, date],
-        start_date: Union[str, datetime, date],
-        test_data: Optional[pd.DataFrame] = None,
+        train: pd.DataFrame,
+        test: pd.DataFrame,
+        feature_cols: List[str],
+        buckets: int = 10,
+        bucket_type: Literal["quantile", "fixed_width"] = "quantile",
     ):
         """
         Print Population Stability Index (PSI) for numerical features.
 
         Parameters
         ----------
+        train : pd.DataFrame
+            Training dataset.
+        test : pd.DataFrame
+            Testing dataset.
+        feature_cols : List[str]
+            List of feature columns to analyze.
         buckets : int
-            Number of buckets for binning.
+            Number of buckets to use for binning. Default is 10.
         bucket_type : Literal["quantile", "fixed_width"]
-            The type of binning to use.
-        time_column : str
-            Name of the time column.
-        time_expected : Union[str, datetime, date]
-            Expected time for PSI calculation.
-        end_date : Union[str, datetime, date]
-            End date for PSI calculation.
-        start_date : Union[str, datetime, date]
-            Start date for PSI calculation.
-        test_data : Optional[pd.DataFrame], optional
-            Test dataset for PSI calculation, by default None.
-        """
-        print("Calculating PSI with the following parameters:")
-        print(f"Time column: {time_column}")
-        print(f"Start date: {start_date}")
-        print(f"End date: {end_date}")
-        print(f"Expected time (split point): {time_expected}")
-        print(f"Number of buckets: {buckets}")
-        print(f"Bucket type: {bucket_type}")
-        print(f"Using separate test data: {'Yes' if test_data is not None else 'No'}")
-        print("=" * 60)
-
-        # Print data info before PSI calculation
-        print(f"Data shape before PSI calculation: {self.dataframe.shape}")
-        print(
-            f"Date range in data: {self.dataframe[time_column].min()} to {self.dataframe[time_column].max()}"
-        )
-
-        if test_data is not None:
-            print(f"Test data shape: {test_data.shape}")
-            print(
-                f"Date range in test data: {test_data[time_column].min()} to {test_data[time_column].max()}"
-            )
-
-        self.univariate_analysis.compute_psi(
-            time_column,
-            time_expected,
-            start_date,
-            end_date,
-            test_data,
-            buckets,
-            bucket_type,
-        )
+            Type of bucketing to use. Default is "quantile".
+        """
 
-        # Print data info after PSI calculation
-        print(
-            f"Data shape after PSI calculation: {self.univariate_analysis.data.shape}"
-        )
-        print(
-            f"Date range in filtered data: {self.univariate_analysis.data[time_column].min()} to {self.univariate_analysis.data[time_column].max()}"
+        self.univariate_analysis.visualize_psi_for_features(
+            train, test, feature_cols, buckets, bucket_type
         )
 
     def plot_num_cat_anova(self):
@@ -377,7 +357,7 @@ class EDAToolkit:
         """
         _, filtered_cramers = self.univariate_analysis.plot_cramers_v(
             self.numerical_columns,
-            excluded_vars=self.exclusion,
+            excluded_columns=self.exclusion,
             filter_threshold=threshold_c,
         )
 
@@ -417,7 +397,8 @@ class EDAToolkit:
             transformations=transforms,
         )
 
-        self.feature_engineering.diagnose_normality_transform()
+        _, transformer = self.feature_engineering.diagnose_normality_transform()
+        self.normality_transformer = transformer
 
     def normalize_numericals(self, to_transformed):
         """
@@ -513,6 +494,21 @@ class EDAToolkit:
         """
         self.visualizations.plot_numerical_target_frequency_line(scale=scaling)
 
+    def plot_boxplots(self):
+        """
+        Plot boxplots for numerical columns.
+
+        Parameters
+        ----------
+        num_rows : int
+            Number of rows to display in the output.
+        scale : bool
+            If True, apply scaling to the columns before plotting.
+        log_scale : bool
+            If True, apply log scaling to the columns before plotting.
+        """
+        self.visualizations.plot_boxplots()
+
     def print_crosstab_freqs(self, num_rows):
         """
         Print cross-tabulation frequencies for categorical columns.

{classifier_toolkit-0.1.3 → classifier_toolkit-0.2.0}/classifier_toolkit/eda/feature_engineering.py

@@ -1,5 +1,5 @@
 import math
-from typing import List, Literal, NamedTuple, Optional
+from typing import List, Literal, NamedTuple, Optional, Union
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -228,7 +228,7 @@ class FeatureEngineering:
         method: Literal["power", "standard"] = "power",
         alpha: float = 0.05,
         test_mode: bool = False,
-    ) -> dict:
+    ) -> tuple[dict, Union[PowerTransformer, StandardScaler]]:
         transformed_data = self.data.copy()
         results = {}
         for column in self.numerical_columns:
@@ -332,7 +332,7 @@ class FeatureEngineering:
         self._log_transformation(
             "normality_transform", {"method": method, "alpha": alpha}
         )
-        return results
+        return results, transformer
 
     def apply_normality_transformations(
         self, columns_to_keep: List[str]
@@ -364,9 +364,11 @@ class FeatureEngineering:
                 raise ValueError(f"Column {column} not found in transformed data")
 
         self.data = updated_data
-        self._log_transformation(
-            "apply_transformations", {"columns_to_keep": columns_to_keep}
-        )
+
+        for transform in self.transformations:
+            if transform["method"] == "normality_transform":
+                transform["selected_columns"] = columns_to_keep
+
         return self.data
 
     def handle_high_cardinality(
@@ -935,63 +937,156 @@ class FeatureEngineering:
     def encoding_categorical(
         self,
         method: Literal["one_hot", "distribution", "woe", "catboost"] = "one_hot",
+        fit: bool = True,
+        data: Optional[pd.DataFrame] = None,
+        verbose: bool = False,
     ) -> pd.DataFrame:
         """
         Perform encoding for the specified categorical columns using the chosen method.
 
-        The columns to be encoded are specified during the initialization of the FeatureEngineering class.
-
         Parameters
         ----------
         method : {'one_hot', 'distribution', 'woe', 'catboost'}, optional
             The encoding method to use, by default 'one_hot'.
+        fit : bool, optional
+            Whether to fit the encoder (True) or just transform (False), by default True.
+        data : Optional[pd.DataFrame], optional
+            The data to encode. If None, uses self.data, by default None.
+
+        Returns
+        -------
+        pd.DataFrame
+            DataFrame with encoded columns.
         """
         if self.encode_list is None:
             print("No columns specified for encoding.")
+            return self.data if data is None else data
 
-        missing_columns = [
-            col for col in self.encode_list if col not in self.data.columns
-        ]
+        data = self.data.copy() if data is None else data.copy()
+
+        missing_columns = [col for col in self.encode_list if col not in data.columns]
         if missing_columns:
             raise ValueError(
                 f"The following columns are not in the DataFrame: {missing_columns}"
             )
 
-        data = self.data.copy()
-
         if method == "one_hot":
-            data = self._encoding_categorical_one_hot(data)
-            print(f"One-hot encoding performed on columns: {self.encode_list}")
-            # Get the names of the new encoded columns
-            new_columns = [
-                col for col in data.columns if col.startswith(tuple(self.encode_list))
-            ]
-            print("\nHead of encoded columns:")
-            print(data[new_columns].head())
+            if fit:
+                print("Fitting OneHotEncoder...") if verbose else None
+                self.encoder = OneHotEncoder(
+                    sparse_output=False, drop="first", handle_unknown="ignore"
+                )
+                encoded_array = self.encoder.fit_transform(data[self.encode_list])
+            else:
+                print(
+                    "Transforming data with OneHotEncoder fit before..."
+                ) if verbose else None
+                encoded_array = self.encoder.transform(data[self.encode_list])
+
+            encoded_df = pd.DataFrame(
+                encoded_array,  # type: ignore
+                columns=self.encoder.get_feature_names_out(self.encode_list),
+                index=data.index,
+            )  # type: ignore
+
+            data = data.drop(columns=self.encode_list)
+            data = pd.concat([data, encoded_df], axis=1)
+
         elif method == "distribution":
-            data = self._encoding_categorical_distribution(data)
-            print(f"Distribution encoding performed on columns: {self.encode_list}")
-            print("\nHead of encoded columns:")
-            print(data[self.encode_list].head())
+            if fit:
+                print("Fitting distribution maps...") if verbose else None
+                self.distribution_maps = {
+                    col: data[col].value_counts(normalize=True)
+                    for col in self.encode_list
+                }
+                for column in self.encode_list:
+                    data[column] = (
+                        data[column].map(self.distribution_maps[column]).fillna(0)
+                    )
+            else:
+                print(
+                    "Transforming data with distribution maps fit before..."
+                ) if verbose else None
+                if self.distribution_maps:
+                    for column in self.encode_list:
+                        data[column] = (
+                            data[column].map(self.distribution_maps[column]).fillna(0)
+                        )
+                else:
+                    raise ValueError(
+                        "Distribution maps not found. Please fit the encoder first."
+                    )
+
         elif method == "woe":
-            data = self._encoding_categorical_woe(data)
-            print(f"WOE encoding performed on columns: {self.encode_list}")
-            print("\nHead of encoded columns:")
-            print(data[self.encode_list].head())
+            if fit:
+                print("Fitting WOE maps...") if verbose else None
+                self.woe_maps = {}
+                for column in self.encode_list:
+                    target = data[self.target_column]
+                    df = pd.DataFrame({column: data[column], "target": target})
+                    total_good = df["target"].sum()
+                    total_bad = df["target"].count() - total_good
+                    grouped = df.groupby(column, observed=False)["target"].agg(
+                        ["sum", "count"]
+                    )
+                    grouped["good"] = grouped["sum"]
+                    grouped["bad"] = grouped["count"] - grouped["sum"]
+                    epsilon = 1e-10
+                    grouped["good_pct"] = (grouped["good"] + epsilon) / (
+                        total_good + epsilon
+                    )
+                    grouped["bad_pct"] = (grouped["bad"] + epsilon) / (
+                        total_bad + epsilon
+                    )
+                    grouped["woe"] = np.log(grouped["good_pct"] / grouped["bad_pct"])
+                    self.woe_maps[column] = grouped["woe"].to_dict()
+
+                for column in self.encode_list:
+                    data[column] = (
+                        data[column].astype(str).map(self.woe_maps[column]).fillna(0)
+                    )
+            else:
+                print(
+                    "Transforming data with WOE maps fit before..."
+                ) if verbose else None
+                if self.woe_maps:
+                    for column in self.encode_list:
+                        data[column] = (
+                            data[column]
+                            .astype(str)
+                            .map(self.woe_maps[column])
+                            .fillna(0)
+                        )
+                else:
+                    raise ValueError(
+                        "WOE maps not found. Please fit the encoder first."
+                    )
+
         elif method == "catboost":
-            data = self._encoding_categorical_catboost(data)
-            print(f"CatBoost encoding performed on columns: {self.encode_list}")
-            print("\nHead of encoded columns:")
-            print(data[self.encode_list].head())
+            if fit:
+                print("Fitting CatBoostEncoder...") if verbose else None
+                self.encoder = CatBoostEncoder(cols=self.encode_list)
+                data[self.encode_list] = self.encoder.fit_transform(
+                    data[self.encode_list], data[self.target_column]
+                )
+            else:
+                print(
+                    "Transforming data with CatBoostEncoder fit before..."
+                ) if verbose else None
+                data[self.encode_list] = self.encoder.transform(data[self.encode_list])
+
         else:
             raise ValueError(
                 f"Encoding method '{method}' is not supported. Choose from 'one_hot', 'distribution', 'woe', or 'catboost'."
             )
 
-        self.data = data
-        self.data.info()
-        self._log_transformation("encoding_categorical", {"method": method})
-        return self.data
+        if fit:
+            self._log_transformation(
+                "encoding_categorical",
+                {"method": method, "selected_columns": self.encode_list},
+            )
+
+        return data
 
     ####### FEATURE ENGINEERING: OPTIONAL FUNCTIONS #######
     def search_pca_components(self):
@@ -1267,44 +1362,67 @@ class FeatureEngineering:
 
 
 '''
-    def apply_winsorization(
-        self, selected_columns: List[str], percentile: float = 0.05
+    def encoding_categorical(
+        self,
+        method: Literal["one_hot", "distribution", "woe", "catboost"] = "one_hot",
     ) -> pd.DataFrame:
         """
-        Apply winsorization to numerical columns to limit extreme values.
+        Perform encoding for the specified categorical columns using the chosen method.
+
+        The columns to be encoded are specified during the initialization of the FeatureEngineering class.
 
         Parameters
         ----------
-        selected_columns : List[str]
-            List of columns to apply winsorization to.
-        percentile : float, optional
-            The percentage of data to be considered as extreme values on each side, by default 0.05.
-
-        Returns
-        -------
-        pd.DataFrame
-            DataFrame with winsorized columns.
+        method : {'one_hot', 'distribution', 'woe', 'catboost'}, optional
+            The encoding method to use, by default 'one_hot'.
         """
-        if len(selected_columns) == 0:
-            selected_columns = self.numerical_columns
+        if self.encode_list is None:
+            print("No columns specified for encoding.")
 
-        for column in selected_columns:
-            if column == self.target_column:
-                print(f"{column} is the target column. Skipping winsorization.")
-                continue
+        missing_columns = [
+            col for col in self.encode_list if col not in self.data.columns
+        ]
+        if missing_columns:
+            raise ValueError(
+                f"The following columns are not in the DataFrame: {missing_columns}"
+            )
 
-            lower_bound = np.percentile(self.data[column], percentile * 100)
-            upper_bound = np.percentile(self.data[column], (1 - percentile) * 100)
-            self.data[column] = np.clip(self.data[column], lower_bound, upper_bound)
+        data = self.data.copy()
 
-        print(f"\nWinsorization applied with {percentile * 100}% capping on each side.")
-        print(self.data[selected_columns].describe())
+        if method == "one_hot":
+            data = self._encoding_categorical_one_hot(data)
+            print(f"One-hot encoding performed on columns: {self.encode_list}")
+            # Get the names of the new encoded columns
+            new_columns = [
+                col for col in data.columns if col.startswith(tuple(self.encode_list))
+            ]
+            print("\nHead of encoded columns:")
+            print(data[new_columns].head())
+        elif method == "distribution":
+            data = self._encoding_categorical_distribution(data)
+            print(f"Distribution encoding performed on columns: {self.encode_list}")
+            print("\nHead of encoded columns:")
+            print(data[self.encode_list].head())
+        elif method == "woe":
+            data = self._encoding_categorical_woe(data)
+            print(f"WOE encoding performed on columns: {self.encode_list}")
+            print("\nHead of encoded columns:")
+            print(data[self.encode_list].head())
+        elif method == "catboost":
+            data = self._encoding_categorical_catboost(data)
+            print(f"CatBoost encoding performed on columns: {self.encode_list}")
+            print("\nHead of encoded columns:")
+            print(data[self.encode_list].head())
+        else:
+            raise ValueError(
+                f"Encoding method '{method}' is not supported. Choose from 'one_hot', 'distribution', 'woe', or 'catboost'."
+            )
 
+        self.data = data
+        self.data.info()
         self._log_transformation(
-            "winsorization",
-            {"selected_columns": selected_columns, "percentile": percentile},
+            "encoding_categorical",
+            {"method": method, "selected_columns": self.encode_list},
         )
         return self.data
-
-    percentile_capping = apply_winsorization
 '''

{classifier_toolkit-0.1.3 → classifier_toolkit-0.2.0}/classifier_toolkit/eda/first_glance.py

@@ -112,7 +112,7 @@ values in more detail, please call the see_duplicates method
         top_n : int, optional
             Number of top values to display, by default 10.
         """
-        for col in self.columns:
+        for col in self.categorical_columns:
             print(Fore.GREEN + f"\nColumn: {col}")
             value_counts = self.data[col].value_counts().head(top_n)
             for value, count in value_counts.items():