lecrapaud 0.19.0__py3-none-any.whl → 0.22.6__py3-none-any.whl

lecrapaud/feature_engineering.py

@@ -47,30 +47,17 @@ Development
 import pandas as pd
 import numpy as np
 from itertools import product
-import joblib
-import os
-
-from sklearn.compose import ColumnTransformer
-from sklearn.decomposition import PCA
-from sklearn.impute import SimpleImputer
-from sklearn.preprocessing import StandardScaler
-from sklearn.pipeline import Pipeline
-from category_encoders import BinaryEncoder, CountEncoder
-from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
-from sklearn.model_selection import train_test_split
 
 from lecrapaud.integrations.openai_integration import (
     truncate_text,
     get_openai_embeddings,
 )
-from lecrapaud.feature_selection import get_features_by_types
 from lecrapaud.utils import logger
-from lecrapaud.db import Target, Feature, Experiment
-from lecrapaud.config import PYTHON_ENV
+from lecrapaud.mixins import LeCrapaudEstimatorMixin
 
 
 # main function
-class FeatureEngineeringEngine:
+class FeatureEngineering(LeCrapaudEstimatorMixin):
     """
     Feature engineering pipeline
 
@@ -86,24 +73,39 @@ class FeatureEngineeringEngine:
 
     def __init__(
         self,
-        data: pd.DataFrame,
-        columns_drop: list[str] = [],
-        columns_boolean: list[str] = [],
-        columns_date: list[str] = [],
-        columns_te_groupby: list[str] = [],
-        columns_te_target: list[str] = [],
+        experiment=None,
         for_training: bool = True,
         **kwargs,
     ):
-        self.data = data
-        self.columns_drop = columns_drop
-        self.columns_boolean = columns_boolean
-        self.columns_date = columns_date
-        self.columns_te_groupby = columns_te_groupby
-        self.columns_te_target = columns_te_target
-        self.for_training = for_training
-
-    def run(self) -> pd.DataFrame:
+        # The mixin will automatically set all experiment.context parameters as attributes
+        # and kwargs will override them if provided
+        super().__init__(experiment=experiment, for_training=for_training, **kwargs)
+        
+        # Set defaults for required parameters if not provided
+        if not hasattr(self, 'columns_drop'):
+            self.columns_drop = []
+        if not hasattr(self, 'columns_boolean'):
+            self.columns_boolean = []
+        if not hasattr(self, 'columns_date'):
+            self.columns_date = []
+        if not hasattr(self, 'columns_te_groupby'):
+            self.columns_te_groupby = []
+        if not hasattr(self, 'columns_te_target'):
+            self.columns_te_target = []
+
+    def fit(self, X, y=None):
+        """
+        Fit the feature engineering estimator.
+        
+        Args:
+            X (pd.DataFrame): Input data
+            y: Target values (ignored)
+        
+        Returns:
+            Transformed data (for compatibility with existing code)
+        """
+        self.data = X.copy()
+        
         # drop columns
         self.data = self.data.drop(columns=self.columns_drop, errors="ignore")
 
@@ -126,6 +128,17 @@ class FeatureEngineeringEngine:
         # Cyclic encode dates
         self.data = self.cyclic_encode_date()
 
+        self._set_fitted()
+        return self
+
+    def get_data(self):
+        """
+        Get the transformed data after feature engineering.
+        
+        Returns:
+            pd.DataFrame: The transformed data with engineered features
+        """
+        self._check_is_fitted()
         return self.data
 
     def cyclic_encode_date(self) -> pd.DataFrame:
@@ -220,7 +233,7 @@ class FeatureEngineeringEngine:
         Returns:
             pd.DataFrame: Original dataframe with new encoded columns added
         """
-        # TODO: target encoding needs to be fit / transform based at inference time.
+        # TODO: target encoding needs to be fit / transform based at transform time.
         df: pd.DataFrame = self.data
         columns_te_groupby: list[list[str]] = self.columns_te_groupby
         columns_te_target: list[str] = self.columns_te_target
@@ -299,7 +312,7 @@ class FeatureEngineeringEngine:
             non_numeric_cols = [col for col in missing_cols if col not in numeric_cols]
 
             logger.warning(
-                f"Missing values found in inference data."
+                f"Missing values found in transform data."
                 f"Filling with 0 for numeric columns: {numeric_cols}, "
                 f"and 'unknown' for non-numeric columns: {non_numeric_cols}"
             )
@@ -310,649 +323,7 @@ class FeatureEngineeringEngine:
         return df
 
 
-class PreprocessFeature:
-
-    def __init__(
-        self,
-        data: pd.DataFrame,
-        experiment,
-        time_series: bool = False,
-        date_column: str | None = None,
-        group_column: str | None = None,
-        val_size: float = 0.2,
-        test_size: float = 0.2,
-        columns_pca: list[str] = [],
-        pca_temporal: list[dict[str, list[str]]] = [],
-        pca_cross_sectional: list[dict[str, list[str]]] = [],
-        columns_onehot: list[str] = [],
-        columns_binary: list[str] = [],
-        columns_ordinal: list[str] = [],
-        columns_frequency: list[str] = [],
-        target_numbers: list = [],
-        target_clf: list = [],
-        **kwargs,
-    ):
-        self.data = data
-        self.data.columns = self.data.columns.str.upper()
-
-        self.experiment = experiment
-        self.columns_pca = [col.upper() for col in columns_pca]
-        self.pca_temporal = pca_temporal
-        self.pca_cross_sectional = pca_cross_sectional
-        self.columns_onehot = [col.upper() for col in columns_onehot]
-        self.columns_binary = [col.upper() for col in columns_binary]
-        self.columns_ordinal = [col.upper() for col in columns_ordinal]
-        self.columns_frequency = [col.upper() for col in columns_frequency]
-        self.target_numbers = target_numbers
-        self.target_clf = target_clf
-
-        self.time_series = time_series
-        self.date_column = date_column
-        self.group_column = group_column
-        self.val_size = val_size
-        self.test_size = test_size
-
-        self.experiment_dir = self.experiment.path
-        self.experiment_id = self.experiment.id
-        self.data_dir = f"{self.experiment_dir}/data"
-        self.preprocessing_dir = f"{self.experiment_dir}/preprocessing"
-
-    def run(self):
-        # Split
-        train, val, test = (
-            self.train_val_test_split_time_series()
-            if self.time_series
-            else self.train_val_test_split(
-                stratify_col=f"TARGET_{self.target_numbers[0]}"
-            )
-        )  # TODO: only stratifying first target for now
-
-        # PCA
-        train, pcas = self.add_pca_features(train)
-        val, _ = self.add_pca_features(val, pcas=pcas)
-        test, _ = self.add_pca_features(test, pcas=pcas)
-
-        joblib.dump(pcas, f"{self.preprocessing_dir}/pcas.pkl")
-
-        train, pcas_cross_sectional = self.add_pca_feature_cross_sectional(train)
-        val, _ = self.add_pca_feature_cross_sectional(val, pcas=pcas_cross_sectional)
-        test, _ = self.add_pca_feature_cross_sectional(test, pcas=pcas_cross_sectional)
-
-        joblib.dump(
-            pcas_cross_sectional, f"{self.preprocessing_dir}/pcas_cross_sectional.pkl"
-        )
-
-        train, pcas_temporal = self.add_pca_feature_temporal(train)
-        val, _ = self.add_pca_feature_temporal(val, pcas=pcas_temporal)
-        test, _ = self.add_pca_feature_temporal(test, pcas=pcas_temporal)
-
-        joblib.dump(pcas_temporal, f"{self.preprocessing_dir}/pcas_temporal.pkl")
-
-        # Save all features before encoding
-        joblib.dump(
-            list(train.columns),
-            f"{self.preprocessing_dir}/all_features_before_encoding.pkl",
-        )
-
-        # Encoding
-        train, transformer = self.encode_categorical_features(train)
-        val, _ = self.encode_categorical_features(
-            val,
-            transformer=transformer,
-        )
-        test, _ = self.encode_categorical_features(
-            test,
-            transformer=transformer,
-        )
-
-        joblib.dump(self.data, f"{self.data_dir}/full.pkl")
-        joblib.dump(transformer, f"{self.preprocessing_dir}/column_transformer.pkl")
-        summary = summarize_dataframe(train)
-        summary.to_csv(f"{self.experiment_dir}/feature_summary.csv", index=False)
-
-        # Save all features before selection
-        joblib.dump(
-            list(train.columns),
-            f"{self.preprocessing_dir}/all_features_before_selection.pkl",
-        )
-
-        return train, val, test
-
-    def inference(self):
-        data = self.data
-
-        # PCA
-        if os.path.exists(f"{self.preprocessing_dir}/pcas.pkl"):
-            pcas = joblib.load(f"{self.preprocessing_dir}/pcas.pkl")
-            data, _ = self.add_pca_features(data, pcas=pcas)
-
-        if os.path.exists(f"{self.preprocessing_dir}/pcas_cross_sectional.pkl"):
-            pcas_cross_sectional = joblib.load(
-                f"{self.preprocessing_dir}/pcas_cross_sectional.pkl"
-            )
-            data, _ = self.add_pca_feature_cross_sectional(
-                data, pcas=pcas_cross_sectional
-            )
-
-        if os.path.exists(f"{self.preprocessing_dir}/pcas_temporal.pkl"):
-            pcas_temporal = joblib.load(f"{self.preprocessing_dir}/pcas_temporal.pkl")
-            data, _ = self.add_pca_feature_temporal(data, pcas=pcas_temporal)
-
-        # Encoding
-        transformer = joblib.load(f"{self.preprocessing_dir}/column_transformer.pkl")
-        data, _ = self.encode_categorical_features(
-            data,
-            transformer=transformer,
-        )
-        return data
-
-    def train_val_test_split_time_series(self):
-        df: pd.DataFrame = self.data
-        date_column: str = self.date_column
-        group_column: str = self.group_column
-        val_size: float = self.val_size
-        test_size: float = self.test_size
-
-        if not date_column:
-            ValueError("Please specify a date_column for time series")
-
-        if group_column:
-            df.sort_values([date_column, group_column], inplace=True)
-        else:
-            df.sort_values(date_column, inplace=True)
-
-        dates = df[date_column].unique()
-
-        val_first_id = int(len(dates) * (1 - val_size - test_size)) + 1
-        test_first_id = int(len(dates) * (1 - test_size)) + 1
-
-        train = df[df[date_column].isin(dates[:val_first_id])]
-        val = df[df[date_column].isin(dates[val_first_id:test_first_id])]
-        test = df[df[date_column].isin(dates[test_first_id:])]
-
-        dates = {}
-        for name, data in zip(["train", "val", "test"], [train, val, test]):
-            dates[f"{name}_start_date"] = (
-                data[date_column].map(pd.Timestamp.fromordinal).iat[0]
-            )
-            dates[f"{name}_end_date"] = (
-                data[date_column].map(pd.Timestamp.fromordinal).iat[-1]
-            )
-
-            logger.info(
-                f"{data.shape} {name} data from {dates[f"{name}_start_date"].strftime('%d/%m/%Y')} to {dates[f"{name}_end_date"].strftime('%d/%m/%Y')}"
-            )
-
-        Experiment.upsert(
-            match_fields=["id"],
-            id=self.experiment_id,
-            train_size=len(train),
-            val_size=len(val),
-            test_size=len(test),
-            **dates,
-        )
-        return (
-            train.reset_index(drop=True),
-            val.reset_index(drop=True),
-            test.reset_index(drop=True),
-        )
-
-    def train_val_test_split(
-        self,
-        random_state: int = 42,
-        stratify_col: str | None = None,
-    ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-        """
-        Splits a DataFrame into train, validation, and test sets.
-
-        Parameters:
-            df (pd.DataFrame): The full experiment
-            val_size (float): Proportion of validation set (default 0.1)
-            test_size (float): Proportion of test set (default 0.1)
-            random_state (int): Random seed for reproducibility
-            stratify_col (str | None): Optional column to stratify on (for classification tasks)
-
-        Returns:
-            Tuple of (train_df, val_df, test_df)
-        """
-        df: pd.DataFrame = self.data
-        val_size: float = self.val_size
-        test_size: float = self.test_size
-
-        stratify_vals = df[stratify_col] if stratify_col else None
-
-        # First split: train + (val + test)
-        train, temp = train_test_split(
-            df,
-            test_size=val_size + test_size,
-            random_state=random_state,
-            stratify=stratify_vals,
-        )
-
-        # Adjust stratify target for val/test split
-        stratify_temp = temp[stratify_col] if stratify_col else None
-
-        # Compute val and test sizes relative to temp
-        val_ratio = val_size / (val_size + test_size)
-
-        val, test = train_test_split(
-            temp,
-            test_size=1 - val_ratio,
-            random_state=random_state,
-            stratify=stratify_temp,
-        )
-
-        for name, data in zip(["train", "val", "test"], [train, val, test]):
-            logger.info(f"{data.shape} {name} data")
-
-        Experiment.upsert(
-            match_fields=["id"],
-            id=self.experiment_id,
-            train_size=len(train),
-            val_size=len(val),
-            test_size=len(test),
-        )
-        return (
-            train.reset_index(drop=True),
-            val.reset_index(drop=True),
-            test.reset_index(drop=True),
-        )
-
-    # embedding and pca
-    def add_pca_features(
-        self, df: pd.DataFrame, n_components: int = 5, pcas=None
-    ) -> tuple[pd.DataFrame, dict]:
-        """
-        Adds PCA components as new columns to a DataFrame from a column containing numpy arrays.
-        NEED TRAIN/TEST SPLIT BEFORE APPLYING - LIKE ENCODING CATEGORICAL VARIABLES
-
-        Parameters:
-            df (pd.DataFrame): Input DataFrame
-            column (str): Name of the column containing np.ndarray
-            n_components (int): Number of PCA components to keep
-
-        Returns:
-            pd.DataFrame: DataFrame with new PCA columns added
-        """
-        columns: list[str] = self.columns_pca
-
-        pcas_dict = {}
-        for column in columns:
-            # Convert text to embeddings if necessary
-            if not isinstance(df[column].iloc[0], (np.ndarray, list)):
-                sentences = df[column].astype(str).tolist()
-                logger.info(
-                    f"Total sentences to embed for column {column}: {len(sentences)}"
-                )
-
-                # Truncate each sentence
-                truncate_sentences = [truncate_text(sentence) for sentence in sentences]
-
-                # embedding
-                embedding_matrix = get_openai_embeddings(truncate_sentences)
-            else:
-                logger.info(f"Column {column} is already embeddings")
-                # Stack the vectors into a 2D array
-                embedding_matrix = np.vstack(df[column].values)
-
-            # Apply PCA
-            if pcas:
-                pca = pcas[column]
-                pca_features = pca.transform(embedding_matrix)
-            else:
-                pca = PCA(n_components=n_components)
-                pca_features = pca.fit_transform(embedding_matrix)
-
-            # Add PCA columns
-            for i in range(n_components):
-                df[f"{column}_pca_{i+1}"] = pca_features[:, i]
-
-            # Drop the original column
-            df.drop(column, axis=1, inplace=True)
-            pcas_dict.update({column: pca})
-
-        return df, pcas_dict
-
-    def add_pca_feature_cross_sectional(
-        self,
-        df: pd.DataFrame,
-        *,
-        n_components: int = 5,
-        pcas: dict[str, Pipeline] | None = None,  # si fourni: transform only
-        impute_strategy: str = "median",
-        standardize: bool = True,
-    ) -> tuple[pd.DataFrame, dict[str, Pipeline]]:
-        """
-        Construit un pivot (index=index_col, columns=columns_col, values=value_col),
-        fit (ou réutilise) un Pipeline Imputer(+Scaler)+PCA, puis merge les scores
-        (par index_col) dans df. Renvoie (df_avec_features, pipe).
-        """
-
-        pcas_dict = {}
-
-        for pca_cross_sectional in self.pca_cross_sectional:
-            name, index_col, columns_col, value_col = (
-                pca_cross_sectional[k] for k in ("name", "index", "columns", "value")
-            )
-            prefix = f"CS_PC_{name}"
-
-            pivot = df.pivot_table(
-                index=index_col, columns=columns_col, values=value_col
-            ).sort_index()
-
-            # Pipeline à réutiliser entre train et test
-            if pcas is None:
-                steps = [("imputer", SimpleImputer(strategy=impute_strategy))]
-                if standardize:
-                    steps.append(
-                        ("scaler", StandardScaler(with_mean=True, with_std=True))
-                    )
-                pca = PCA(n_components=n_components, random_state=0)
-                steps.append(("pca", pca))
-                pipe = Pipeline(steps)
-                pipe.fit(pivot)  # <- fit sur TRAIN uniquement
-            else:
-                pipe = pcas[name]  # <- TEST : on réutilise le pipe existant
-
-            scores = pipe.transform(pivot)  # shape: (n_index, n_components)
-            cols = [f"{prefix}_{i}" for i in range(n_components)]
-            scores_df = pd.DataFrame(scores, index=pivot.index, columns=cols)
-
-            df = df.merge(scores_df.reset_index(), on=index_col, how="left")
-            pcas_dict.update({name: pipe})
-
-        return df, pcas_dict
-
-    # ----------------- 2) PCA TEMPORELLE (liste de colonnes lags) ----------------
-    def add_pca_feature_temporal(
-        self,
-        df: pd.DataFrame,
-        *,
-        n_components: int = 5,
-        pcas: dict[str, Pipeline] | None = None,  # si fourni: transform only
-        impute_strategy: (
-            str | None
-        ) = None,  # None = on exige toutes les colonnes présentes
-        standardize: bool = True,
-    ) -> tuple[pd.DataFrame, dict[str, Pipeline]]:
-        """
-        Applique une PCA sur une matrice (rows = lignes df, cols = lags).
-        Fit le Pipeline sur TRAIN si pcas=None; sinon, utilise pcas et fait transform.
-        Ajoute les colonnes f"{prefix}_{i}" dans df. Renvoie (df, pipe).
-        """
-        pcas_dict = {}
-
-        for pca_temporal in self.pca_temporal:
-            name, cols = (pca_temporal[k] for k in ("name", "columns"))
-            prefix = f"TMP_PC_{name}"
-
-            # Masque des lignes utilisables
-            if impute_strategy is None:
-                mask = (
-                    df[cols].notna().all(axis=1)
-                )  # on n'impute pas → lignes complètes
-                X_fit = df.loc[mask, cols]
-            else:
-                mask = df[cols].notna().any(axis=1)  # on imputera → au moins une valeur
-                X_fit = df.loc[mask, cols]
-
-            # Pipeline
-            if pcas is None:
-                steps = []
-                if impute_strategy is not None:
-                    steps.append(("imputer", SimpleImputer(strategy=impute_strategy)))
-                if standardize:
-                    steps.append(
-                        ("scaler", StandardScaler(with_mean=True, with_std=True))
-                    )
-                pca = PCA(n_components=n_components, random_state=0)
-                steps.append(("pca", pca))
-                pipe = Pipeline(steps)
-                if not X_fit.empty:
-                    pipe.fit(X_fit)  # <- fit sur TRAIN uniquement
-            else:
-                pipe = pcas[name]  # <- TEST
-
-            # Transform uniquement sur lignes valides (mask)
-            if not df.loc[mask, cols].empty:
-                Z = pipe.transform(df.loc[mask, cols])
-                for i in range(n_components):
-                    df.loc[mask, f"{prefix}_{i}"] = Z[:, i]
-            else:
-                # crée les colonnes vides si aucune ligne valide (cohérence de schéma)
-                for i in range(n_components):
-                    df[f"{prefix}_{i}"] = pd.NA
-
-            pcas_dict.update({name: pipe})
-
-        return df, pcas_dict
-
-    # encoding categorical features
-    def encode_categorical_features(
-        self,
-        df: pd.DataFrame,
-        transformer: ColumnTransformer | None = None,
-    ) -> tuple[pd.DataFrame, ColumnTransformer]:
-        """
-        Encodes categorical columns using one-hot, binary, ordinal, and frequency encoding.
-
-        Parameters:
-            df (pd.DataFrame): Input DataFrame
-            columns_onehot (list[str]) Creates one binary column per category forLow-cardinality categorical features
-            columns_binary (list[str]) Converts categories into binary and splits bits across columns for Mid-to-high cardinality (e.g., 10–100 unique values)
-            columns_ordinal (list[str]) Assigns integer ranks to categories When order matters (e.g., low < medium < high)
-            columns_frequency (list[str]) Replaces each category with its frequency count, normalized to proportion. High-cardinality features with meaning in frequency
-            transformer (ColumnTransformer, optional): if provided, applies transform only
-
-        Returns:
-            tuple: (transformed DataFrame, ColumnTransformer)
-        """
-        columns_onehot: list[str] = self.columns_onehot
-        columns_binary: list[str] = self.columns_binary
-        columns_ordinal: list[str] = self.columns_ordinal
-        columns_frequency: list[str] = self.columns_frequency
-
-        X = df.loc[:, ~df.columns.str.contains("^TARGET_")]
-        y = df.loc[:, df.columns.str.contains("^TARGET_")]
-        save_in_db = False
-
-        all_columns = (
-            columns_onehot + columns_binary + columns_ordinal + columns_frequency
-        )
-
-        if transformer:
-            transformed = transformer.transform(X)
-        else:
-            transformer = ColumnTransformer(
-                transformers=[
-                    (
-                        "onehot",
-                        OneHotEncoder(handle_unknown="ignore", sparse_output=False),
-                        columns_onehot,
-                    ),
-                    (
-                        "ordinal",
-                        OrdinalEncoder(
-                            handle_unknown="use_encoded_value", unknown_value=-1
-                        ),
-                        columns_ordinal,
-                    ),
-                    ("binary", BinaryEncoder(handle_unknown="value"), columns_binary),
-                    ("freq", CountEncoder(normalize=True), columns_frequency),
-                ],
-                remainder="passthrough",
-            )
-            transformed = transformer.fit_transform(X)
-            save_in_db = True
-
-        # Build output column names
-        column_names = []
-
-        if columns_onehot:
-            column_names.extend(
-                transformer.named_transformers_["onehot"]
-                .get_feature_names_out(columns_onehot)
-                .tolist()
-            )
-
-        if columns_ordinal:
-            column_names.extend(columns_ordinal)
-
-        if columns_binary:
-            column_names.extend(
-                transformer.named_transformers_["binary"]
-                .get_feature_names_out(columns_binary)
-                .tolist()
-            )
-
-        if columns_frequency:
-            column_names.extend(columns_frequency)
-
-        # Add passthrough (non-encoded) columns
-        passthrough_columns = [col for col in X.columns if col not in all_columns]
-        column_names.extend(passthrough_columns)
-
-        X_transformed = pd.DataFrame(transformed, columns=column_names, index=df.index)
-
-        # Try to convert columns to best possible dtypes
-        X_transformed = X_transformed.convert_dtypes()
-
-        # Insert features in db
-        if save_in_db:
-            # Get feature types from transformed data
-            categorical_features, numerical_features = get_features_by_types(
-                X_transformed
-            )
-
-            # Get column names from DataFrames
-            cat_feature_names = categorical_features.columns.tolist()
-            num_feature_names = numerical_features.columns.tolist()
-
-            # Combine all feature names and their types
-            all_feature_names = cat_feature_names + num_feature_names
-            all_feature_types = ["categorical"] * len(cat_feature_names) + [
-                "numerical"
-            ] * len(num_feature_names)
-
-            # Upsert features in bulk if we have any features
-            if all_feature_names:
-                Feature.upsert_bulk(
-                    match_fields=["name"],
-                    name=all_feature_names,
-                    type=all_feature_types,
-                )
-
-            # Upsert targets in bulk
-            target_names = y.columns.tolist()
-            target_types = [
-                (
-                    "classification"
-                    if int(target.split("_")[1]) in self.target_clf
-                    else "regression"
-                )
-                for target in target_names
-            ]
-
-            Target.upsert_bulk(
-                match_fields=["name"], name=target_names, type=target_types
-            )
-
-            # Get all the upserted objects
-            targets = Target.filter(name__in=target_names)
-
-            # Update experiment with targets
-            experiment = Experiment.get(self.experiment_id)
-            if experiment:
-                experiment.targets = targets
-                experiment.save()
-
-        return pd.concat([X_transformed, y], axis=1), transformer
-
-
 # analysis & utils
-def summarize_dataframe(
-    df: pd.DataFrame, sample_categorical_threshold: int = 15
-) -> pd.DataFrame:
-    summary = []
-
-    def is_hashable_series(series: pd.Series) -> bool:
-        try:
-            _ = series.dropna().unique()
-            return True
-        except TypeError:
-            return False
-
-    df = convert_object_columns_that_are_numeric(df)
-    df = df.convert_dtypes()
-
-    for col in df.columns:
-        total_missing = df[col].isna().sum()
-        col_data = df[col].dropna()
-        dtype = col_data.dtype
-
-        if col_data.empty:
-            summary.append(
-                {
-                    "Column": col,
-                    "Dtype": dtype,
-                    "Type": "unknown",
-                    "Detail": "No non-null values",
-                    "Missing": total_missing,
-                }
-            )
-            continue
-
-        # Case 1: Numeric columns
-        if pd.api.types.is_numeric_dtype(col_data):
-            unique_vals = col_data.nunique()
-
-            if set(col_data.unique()).issubset({0, 1}):
-                col_type = "binary-categorical"
-                detail = "0/1 values only"
-            elif (
-                pd.api.types.is_integer_dtype(col_data)
-                and unique_vals <= sample_categorical_threshold
-            ):
-                col_type = "multi-categorical"
-                top_vals = col_data.value_counts().head(10)
-                detail = ", ".join(f"{k} ({v})" for k, v in top_vals.items())
-            else:
-                col_type = "numeric"
-                q = col_data.quantile([0, 0.25, 0.5, 0.75, 1])
-                detail = (
-                    f"Min: {q.iloc[0]:.2f}, Q1: {q.iloc[1]:.2f}, Median: {q.iloc[2]:.2f}, "
-                    f"Q3: {q.iloc[3]:.2f}, Max: {q.iloc[4]:.2f}"
-                )
-
-        # Case 2: Object or other hashable columns
-        elif is_hashable_series(col_data):
-            unique_vals = col_data.nunique()
-            if unique_vals <= sample_categorical_threshold:
-                col_type = "object-categorical"
-                top_vals = col_data.value_counts().head(10)
-                detail = ", ".join(f"{k} ({v})" for k, v in top_vals.items())
-            else:
-                col_type = "high-cardinality-categorical"
-                detail = f"{unique_vals} unique values"
-
-        # Case 3: Unusable columns
-        else:
-            col_type = "non-hashable"
-            detail = f"Non-hashable type: {type(col_data.iloc[0])}"
-
-        summary.append(
-            {
-                "Column": col,
-                "Dtype": dtype,
-                "Type": col_type,
-                "Detail": detail,
-                "Missing": total_missing,
-            }
-        )
-
-    return pd.DataFrame(summary)
-
-
 def convert_object_columns_that_are_numeric(df: pd.DataFrame) -> list:
     """
     Detect object columns that can be safely converted to numeric (float or int).