lecrapaud 0.16.7__tar.gz → 0.17.0__tar.gz

{lecrapaud-0.16.7 → lecrapaud-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lecrapaud
-Version: 0.16.7
+Version: 0.17.0
 Summary: Framework for machine and deep learning, with regression, classification and time series analysis
 License: Apache License
 Author: Pierre H. Gallet

{lecrapaud-0.16.7 → lecrapaud-0.17.0}/lecrapaud/api.py

@@ -102,46 +102,49 @@ class LeCrapaud:
 
     def compare_experiment_scores(self, name: str):
         """Compare scores of experiments with matching names.
-        
+
         Args:
             name (str): Name or partial name of experiments to compare
-            
+
         Returns:
             dict: Dictionary containing experiment names as keys and their scores as values
         """
         from lecrapaud.db import SessionLocal
         from sqlalchemy.orm import joinedload
-        
+
         db = SessionLocal()
         try:
             # Get all experiments with the given name pattern
             experiments = (
                 db.query(Experiment)
-                .options(joinedload(Experiment.model_selections)
-                        .joinedload(ModelSelection.scores))
+                .options(
+                    joinedload(Experiment.model_selections).joinedload(
+                        ModelSelection.scores
+                    )
+                )
                 .filter(Experiment.name.ilike(f"%{name}%"))
                 .all()
             )
-            
+
             if not experiments:
                 return {"error": f"No experiments found with name containing '{name}'"}
-                
+
             comparison = {}
-            
+
             for exp in experiments:
                 scores = {
                     "rmse": exp.avg_rmse,
                     "logloss": exp.avg_logloss,
                     "accuracy": None,
                     "f1": None,
-                    "roc_auc": None
+                    "roc_auc": None,
                 }
-                
+
                 # Get classification metrics from the first model selection with scores
                 for model_sel in exp.model_selections:
                     if model_sel.scores:
                         for score in model_sel.scores:
-                            if score.type == 'validation':  # Use validation scores
+                            if score.type == "validation":  # Use validation scores
                                 if score.accuracy is not None:
                                     scores["accuracy"] = score.accuracy
                                 if score.f1 is not None:
@@ -149,16 +152,16 @@ class LeCrapaud:
                                 if score.roc_auc is not None:
                                     scores["roc_auc"] = score.roc_auc
                                 break
-                
+
                 comparison[exp.name] = scores
-                
+
             return comparison
-            
+
         except Exception as e:
             return {"error": f"Error comparing experiment scores: {str(e)}"}
         finally:
             db.close()
-    
+
     def list_experiments(
         self, name: str = None, limit: int = 1000
     ) -> list["ExperimentEngine"]:
@@ -348,6 +351,8 @@ class ExperimentEngine:
             val_size=self.val_size,
             test_size=self.test_size,
             columns_pca=self.columns_pca,
+            pca_temporal=self.pca_temporal,
+            pca_cross_sectional=self.pca_cross_sectional,
             columns_onehot=self.columns_onehot,
             columns_binary=self.columns_binary,
             columns_frequency=self.columns_frequency,

{lecrapaud-0.16.7 → lecrapaud-0.17.0}/lecrapaud/feature_engineering.py

@@ -52,6 +52,9 @@ import os
 
 from sklearn.compose import ColumnTransformer
 from sklearn.decomposition import PCA
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import StandardScaler
+from sklearn.pipeline import Pipeline
 from category_encoders import BinaryEncoder, CountEncoder
 from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
 from sklearn.model_selection import train_test_split
@@ -316,6 +319,8 @@ class PreprocessFeature:
         val_size: float = 0.2,
         test_size: float = 0.2,
         columns_pca: list[str] = [],
+        pca_temporal: dict[str, list[str]] = {},
+        pca_cross_sectional: dict[str, list[str]] = {},
         columns_onehot: list[str] = [],
         columns_binary: list[str] = [],
         columns_ordinal: list[str] = [],
@@ -329,6 +334,8 @@ class PreprocessFeature:
 
         self.experiment = experiment
         self.columns_pca = [col.upper() for col in columns_pca]
+        self.pca_temporal = pca_temporal
+        self.pca_cross_sectional = pca_cross_sectional
         self.columns_onehot = [col.upper() for col in columns_onehot]
         self.columns_binary = [col.upper() for col in columns_binary]
         self.columns_ordinal = [col.upper() for col in columns_ordinal]
@@ -364,6 +371,20 @@ class PreprocessFeature:
 
         joblib.dump(pcas, f"{self.preprocessing_dir}/pcas.pkl")
 
+        train, pcas_cross_sectional = self.add_pca_feature_cross_sectional(train)
+        val, _ = self.add_pca_feature_cross_sectional(val, pcas=pcas_cross_sectional)
+        test, _ = self.add_pca_feature_cross_sectional(test, pcas=pcas_cross_sectional)
+
+        joblib.dump(
+            pcas_cross_sectional, f"{self.preprocessing_dir}/pcas_cross_sectional.pkl"
+        )
+
+        train, pcas_temporal = self.add_pca_feature_temporal(train)
+        val, _ = self.add_pca_feature_temporal(val, pcas=pcas_temporal)
+        test, _ = self.add_pca_feature_temporal(test, pcas=pcas_temporal)
+
+        joblib.dump(pcas_temporal, f"{self.preprocessing_dir}/pcas_temporal.pkl")
+
         # Save all features before encoding
         joblib.dump(
             list(train.columns),
@@ -402,6 +423,18 @@ class PreprocessFeature:
             pcas = joblib.load(f"{self.preprocessing_dir}/pcas.pkl")
             data, _ = self.add_pca_features(data, pcas=pcas)
 
+        if os.path.exists(f"{self.preprocessing_dir}/pcas_cross_sectional.pkl"):
+            pcas_cross_sectional = joblib.load(
+                f"{self.preprocessing_dir}/pcas_cross_sectional.pkl"
+            )
+            data, _ = self.add_pca_feature_cross_sectional(
+                data, pcas=pcas_cross_sectional
+            )
+
+        if os.path.exists(f"{self.preprocessing_dir}/pcas_temporal.pkl"):
+            pcas_temporal = joblib.load(f"{self.preprocessing_dir}/pcas_temporal.pkl")
+            data, _ = self.add_pca_feature_temporal(data, pcas=pcas_temporal)
+
         # Encoding
         transformer = joblib.load(f"{self.preprocessing_dir}/column_transformer.pkl")
         data, _ = self.encode_categorical_features(
@@ -577,6 +610,120 @@ class PreprocessFeature:
 
         return df, pcas_dict
 
+    def add_pca_feature_cross_sectional(
+        self,
+        df: pd.DataFrame,
+        *,
+        n_components: int = 5,
+        pcas: dict[str, Pipeline] | None = None,  # si fourni: transform only
+        impute_strategy: str = "median",
+        standardize: bool = True,
+    ) -> tuple[pd.DataFrame, dict[str, Pipeline]]:
+        """
+        Construit un pivot (index=index_col, columns=columns_col, values=value_col),
+        fit (ou réutilise) un Pipeline Imputer(+Scaler)+PCA, puis merge les scores
+        (par index_col) dans df. Renvoie (df_avec_features, pipe).
+        """
+
+        pcas_dict = {}
+
+        for pca_cross_sectional in self.pca_cross_sectional:
+            name, index_col, columns_col, value_col = (
+                pca_cross_sectional[k] for k in ("name", "index", "columns", "value")
+            )
+            prefix = f"CS_PC_{name}"
+
+            pivot = df.pivot_table(
+                index=index_col, columns=columns_col, values=value_col
+            ).sort_index()
+
+            # Pipeline à réutiliser entre train et test
+            if pcas is None:
+                steps = [("imputer", SimpleImputer(strategy=impute_strategy))]
+                if standardize:
+                    steps.append(
+                        ("scaler", StandardScaler(with_mean=True, with_std=True))
+                    )
+                pca = PCA(n_components=n_components, random_state=0)
+                steps.append(("pca", pca))
+                pipe = Pipeline(steps)
+                pipe.fit(pivot)  # <- fit sur TRAIN uniquement
+            else:
+                pipe = pcas[name]  # <- TEST : on réutilise le pipe existant
+
+            scores = pipe.transform(pivot)  # shape: (n_index, n_components)
+            cols = [f"{prefix}_{i}" for i in range(n_components)]
+            scores_df = pd.DataFrame(scores, index=pivot.index, columns=cols)
+
+            df = df.merge(scores_df.reset_index(), on=index_col, how="left")
+            pcas_dict.update({name: pipe})
+
+        return df, pcas_dict
+
+    # ----------------- 2) PCA TEMPORELLE (liste de colonnes lags) ----------------
+    def add_pca_feature_temporal(
+        self,
+        df: pd.DataFrame,
+        *,
+        n_components: int = 5,
+        pcas: dict[str, Pipeline] | None = None,  # si fourni: transform only
+        impute_strategy: (
+            str | None
+        ) = None,  # None = on exige toutes les colonnes présentes
+        standardize: bool = True,
+    ) -> tuple[pd.DataFrame, dict[str, Pipeline]]:
+        """
+        Applique une PCA sur une matrice (rows = lignes df, cols = lags).
+        Fit le Pipeline sur TRAIN si pcas=None; sinon, utilise pcas et fait transform.
+        Ajoute les colonnes f"{prefix}_{i}" dans df. Renvoie (df, pipe).
+        """
+        pcas_dict = {}
+
+        for pca_temporal in self.pca_temporal:
+            name, cols = (pca_temporal[k] for k in ("name", "columns"))
+            prefix = f"TMP_PC_{name}"
+
+            # Masque des lignes utilisables
+            if impute_strategy is None:
+                mask = (
+                    df[cols].notna().all(axis=1)
+                )  # on n'impute pas → lignes complètes
+                X_fit = df.loc[mask, cols]
+            else:
+                mask = df[cols].notna().any(axis=1)  # on imputera → au moins une valeur
+                X_fit = df.loc[mask, cols]
+
+            # Pipeline
+            if pcas is None:
+                steps = []
+                if impute_strategy is not None:
+                    steps.append(("imputer", SimpleImputer(strategy=impute_strategy)))
+                if standardize:
+                    steps.append(
+                        ("scaler", StandardScaler(with_mean=True, with_std=True))
+                    )
+                pca = PCA(n_components=n_components, random_state=0)
+                steps.append(("pca", pca))
+                pipe = Pipeline(steps)
+                if not X_fit.empty:
+                    pipe.fit(X_fit)  # <- fit sur TRAIN uniquement
+            else:
+                pipe = pcas[name]  # <- TEST
+
+            # Transform uniquement sur lignes valides (mask)
+            if not df.loc[mask, cols].empty:
+                Z = pipe.transform(df.loc[mask, cols])
+                for i in range(n_components):
+                    df.loc[mask, f"{prefix}_{i}"] = Z[:, i]
+            else:
+                # crée les colonnes vides si aucune ligne valide (cohérence de schéma)
+                for i in range(n_components):
+                    df[f"{prefix}_{i}"] = pd.NA
+
+            pcas_dict.update({name: pipe})
+
+        return df, pcas_dict
+
     # encoding categorical features
     def encode_categorical_features(
         self,

{lecrapaud-0.16.7 → lecrapaud-0.17.0}/lecrapaud/model_selection.py

@@ -1781,7 +1781,17 @@ def find_best_threshold(
                     logger.warning(
                         f"[Class {cls}] No threshold with precision ≥ {target_value}"
                     )
-                    best_idx = int(np.argmax(precision))  # fallback
+                    # fallback: meilleure precision parmi ceux avec recall>0
+                    cand = np.where(recall > 0)[0]
+                    if cand.size:
+                        best_idx = cand[int(np.argmax(precision[cand]))]
+                        logger.warning(
+                            f"[Class {cls}] Fallback to best precision with recall>0: "
+                            f"idx={best_idx}, precision={precision[best_idx]:.4f}, recall={recall[best_idx]:.4f}"
+                        )
+                    else:
+                        logger.error(f"[Class {cls}] No threshold achieves recall>0.")
+                        best_idx = None
 
             elif metric == "f1":
                 valid_indices = [i for i, val in enumerate(f1) if val >= target_value]
@@ -1795,6 +1805,15 @@ def find_best_threshold(
         else:
             best_idx = int(np.argmax(values))  # no constraint, get best value
 
+        if best_idx is None:
+            results[cls_str] = {
+                "threshold": None,
+                "precision": None,
+                "recall": None,
+                "f1": None,
+            }
+            continue
+
         results[cls_str] = {
             "threshold": float(thresholds[best_idx]),
             "precision": float(precision[best_idx]),

{lecrapaud-0.16.7 → lecrapaud-0.17.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lecrapaud"
-version = "0.16.7"
+version = "0.17.0"
 description = "Framework for machine and deep learning, with regression, classification and time series analysis"
 authors = [
     {name = "Pierre H. Gallet"}