lecrapaud 0.19.3__py3-none-any.whl → 0.20.1__py3-none-any.whl

lecrapaud/feature_engineering.py

@@ -87,21 +87,20 @@ class FeatureEngineeringEngine:
     def __init__(
         self,
         data: pd.DataFrame,
-        columns_drop: list[str] = [],
-        columns_boolean: list[str] = [],
-        columns_date: list[str] = [],
-        columns_te_groupby: list[str] = [],
-        columns_te_target: list[str] = [],
+        experiment,
         for_training: bool = True,
         **kwargs,
     ):
         self.data = data
-        self.columns_drop = columns_drop
-        self.columns_boolean = columns_boolean
-        self.columns_date = columns_date
-        self.columns_te_groupby = columns_te_groupby
-        self.columns_te_target = columns_te_target
+        self.experiment = experiment
         self.for_training = for_training
+        
+        # Get all parameters from experiment context
+        self.columns_drop = self.experiment.context.get("columns_drop", [])
+        self.columns_boolean = self.experiment.context.get("columns_boolean", [])
+        self.columns_date = self.experiment.context.get("columns_date", [])
+        self.columns_te_groupby = self.experiment.context.get("columns_te_groupby", [])
+        self.columns_te_target = self.experiment.context.get("columns_te_target", [])
 
     def run(self) -> pd.DataFrame:
         # drop columns
@@ -316,41 +315,30 @@ class PreprocessFeature:
         self,
         data: pd.DataFrame,
         experiment,
-        time_series: bool = False,
-        date_column: str | None = None,
-        group_column: str | None = None,
-        val_size: float = 0.2,
-        test_size: float = 0.2,
-        columns_pca: list[str] = [],
-        pca_temporal: list[dict[str, list[str]]] = [],
-        pca_cross_sectional: list[dict[str, list[str]]] = [],
-        columns_onehot: list[str] = [],
-        columns_binary: list[str] = [],
-        columns_ordinal: list[str] = [],
-        columns_frequency: list[str] = [],
-        target_numbers: list = [],
-        target_clf: list = [],
         **kwargs,
     ):
         self.data = data
         self.data.columns = self.data.columns.str.upper()
-
         self.experiment = experiment
-        self.columns_pca = [col.upper() for col in columns_pca]
-        self.pca_temporal = pca_temporal
-        self.pca_cross_sectional = pca_cross_sectional
-        self.columns_onehot = [col.upper() for col in columns_onehot]
-        self.columns_binary = [col.upper() for col in columns_binary]
-        self.columns_ordinal = [col.upper() for col in columns_ordinal]
-        self.columns_frequency = [col.upper() for col in columns_frequency]
-        self.target_numbers = target_numbers
-        self.target_clf = target_clf
-
-        self.time_series = time_series
-        self.date_column = date_column
-        self.group_column = group_column
-        self.val_size = val_size
-        self.test_size = test_size
+
+        # Get all parameters from experiment context
+        context = self.experiment.context
+        self.time_series = context.get("time_series", False)
+        self.date_column = context.get("date_column", None)
+        self.group_column = context.get("group_column", None)
+        self.val_size = context.get("val_size", 0.2)
+        self.test_size = context.get("test_size", 0.2)
+        self.target_numbers = context.get("target_numbers", [])
+        self.target_clf = context.get("target_clf", [])
+        
+        # Handle list parameters with uppercase conversion
+        self.columns_pca = [col.upper() for col in context.get("columns_pca", [])]
+        self.pca_temporal = context.get("pca_temporal", [])
+        self.pca_cross_sectional = context.get("pca_cross_sectional", [])
+        self.columns_onehot = [col.upper() for col in context.get("columns_onehot", [])]
+        self.columns_binary = [col.upper() for col in context.get("columns_binary", [])]
+        self.columns_ordinal = [col.upper() for col in context.get("columns_ordinal", [])]
+        self.columns_frequency = [col.upper() for col in context.get("columns_frequency", [])]
 
         self.experiment_dir = self.experiment.path
         self.experiment_id = self.experiment.id
@@ -483,8 +471,8 @@ class PreprocessFeature:
                 f"{data.shape} {name} data from {dates[f"{name}_start_date"].strftime('%d/%m/%Y')} to {dates[f"{name}_end_date"].strftime('%d/%m/%Y')}"
             )
 
-        Experiment.upsert(
-            match_fields=["id"],
+        # Update existing experiment with sizes and dates
+        Experiment.update(
             id=self.experiment_id,
             train_size=len(train),
             val_size=len(val),
@@ -545,8 +533,8 @@ class PreprocessFeature:
         for name, data in zip(["train", "val", "test"], [train, val, test]):
             logger.info(f"{data.shape} {name} data")
 
-        Experiment.upsert(
-            match_fields=["id"],
+        # Update existing experiment with sizes
+        Experiment.update(
             id=self.experiment_id,
             train_size=len(train),
             val_size=len(val),
@@ -838,8 +826,7 @@ class PreprocessFeature:
 
             # Upsert features in bulk if we have any features
             if all_feature_names:
-                Feature.upsert_bulk(
-                    match_fields=["name"],
+                Feature.bulk_upsert(
                     name=all_feature_names,
                     type=all_feature_types,
                 )
@@ -855,9 +842,7 @@ class PreprocessFeature:
                 for target in target_names
             ]
 
-            Target.upsert_bulk(
-                match_fields=["name"], name=target_names, type=target_types
-            )
+            Target.bulk_upsert(name=target_names, type=target_types)
 
             # Get all the upserted objects
             targets = Target.filter(name__in=target_names)

lecrapaud/feature_selection.py

@@ -73,18 +73,21 @@ def load_train_data(experiment_dir):
 
 
 class FeatureSelectionEngine:
-    def __init__(self, train, experiment, target_number, target_clf, **kwargs):
+    def __init__(self, train, experiment, target_number, **kwargs):
         self.experiment = experiment
         self.train = train
         self.target_number = target_number
-        self.target_clf = target_clf
+        
+        # Get all parameters from experiment context
+        self.target_clf = self.experiment.context.get("target_clf", [])
+        self.max_p_value_categorical = self.experiment.context.get("max_p_value_categorical", 0.05)
+        self.percentile = self.experiment.context.get("percentile", 20)
+        self.corr_threshold = self.experiment.context.get("corr_threshold", 80)
+        self.max_features = self.experiment.context.get("max_features", 50)
 
         self.target_type = (
             "classification" if self.target_number in self.target_clf else "regression"
         )
-        self.percentile = self.experiment.percentile
-        self.corr_threshold = self.experiment.corr_threshold
-        self.max_features = self.experiment.max_features
 
         self.experiment_dir = self.experiment.path
         self.experiment_id = self.experiment.id
@@ -115,7 +118,6 @@ class FeatureSelectionEngine:
         max_features = self.max_features
 
         feature_selection = FeatureSelection.upsert(
-            match_fields=["target_id", "experiment_id"],
             target_id=target.id,
             experiment_id=self.experiment_id,
         )
@@ -275,6 +277,38 @@ class FeatureSelectionEngine:
         features_selected.drop_duplicates("features", inplace=True)
 
         features_selected_list = features_selected["features"].values.tolist()
+        
+        # Save ensemble features before correlation (aggregated features)
+        logger.info("Saving ensemble features before correlation...")
+        all_features_in_data = self.X.columns.tolist()
+        ensemble_rows = []
+        
+        # Add global rank for selected features
+        features_selected_with_global_rank = features_selected.copy()
+        features_selected_with_global_rank["global_rank"] = range(1, len(features_selected_with_global_rank) + 1)
+        
+        for feature in all_features_in_data:
+            feature_id = feature_map.get(feature)
+            if feature_id:
+                is_selected = feature in features_selected_list
+                global_rank = None
+                if is_selected:
+                    global_rank = features_selected_with_global_rank[
+                        features_selected_with_global_rank["features"] == feature
+                    ]["global_rank"].values[0]
+                
+                ensemble_rows.append({
+                    "feature_selection_id": feature_selection.id,
+                    "feature_id": feature_id,
+                    "method": "ensemble",
+                    "score": None,
+                    "pvalue": None,
+                    "support": 2 if is_selected else 0,  # 2 = in aggregated features
+                    "rank": global_rank,
+                    "training_time": 0,
+                })
+        
+        FeatureSelectionRank.bulk_upsert(rows=ensemble_rows)
 
         # analysis 1
         features_selected_by_every_methods = set(results[0]["features"].values.tolist())
@@ -303,12 +337,46 @@ class FeatureSelectionEngine:
             header=True,
             index_label="ID",
         )
+        
+        # Update support for features after correlation removal (before max)
+        logger.info("Updating ensemble features after correlation removal...")
+        for row in ensemble_rows:
+            feature = Feature.get(row["feature_id"]).name
+            if feature in features:
+                row["support"] = 1  # 1 = survived correlation removal
+        
         features = features[:max_features]
 
         # adding categorical features selected
         features += (
             categorical_features_selected if target_type == "classification" else []
         )
+        
+        # Final update for features after max limitation (final selection)
+        logger.info("Finalizing ensemble features with categorical features...")
+        for row in ensemble_rows:
+            feature = Feature.get(row["feature_id"]).name
+            if feature in features and row["support"] == 1:
+                row["support"] = 2  # 2 = in final selection
+        
+        # Add categorical features to ensemble if not already present
+        if target_type == "classification":
+            for cat_feature in categorical_features_selected:
+                feature_id = feature_map.get(cat_feature)
+                if feature_id and not any(row["feature_id"] == feature_id for row in ensemble_rows):
+                    ensemble_rows.append({
+                        "feature_selection_id": feature_selection.id,
+                        "feature_id": feature_id,
+                        "method": "ensemble",
+                        "score": None,
+                        "pvalue": None,
+                        "support": 2,  # 2 = in final selection (categorical)
+                        "rank": None,  # No rank for categorical features added at the end
+                        "training_time": 0,
+                    })
+        
+        # Re-save all ensemble data with updated support values
+        FeatureSelectionRank.bulk_upsert(rows=ensemble_rows)
         logger.debug(
             f"Final pre-selection: {len(features)} features below {corr_threshold}% out of {len(features_selected_list)} features, and rejected {len(features_correlated)} features, {100*len(features)/len(features_selected_list):.2f}% features selected"
         )
@@ -441,13 +509,18 @@ class FeatureSelectionEngine:
         feat_scores["features"] = X.columns
         feat_scores["rank"] = feat_scores["score"].rank(method="first", ascending=False)
         feat_scores["method"] = "Chi2"
+        
+        # Apply both percentile and p-value filtering
+        # Keep features that satisfy BOTH conditions: within percentile AND p-value < threshold
+        feat_scores["support"] = feat_scores["support"] & (feat_scores["pvalue"] <= self.max_p_value_categorical)
+        
         feat_scores.sort_values("rank", ascending=True, inplace=True)
         stop = time.time()
         training_time = timedelta(seconds=(stop - start)).total_seconds()
         feat_scores["training_time"] = training_time
 
         logger.debug(
-            f"Chi2 evaluation selected {feat_scores['support'].sum()} features in {training_time:.2f} seconds"
+            f"Chi2 evaluation selected {feat_scores['support'].sum()} features in {training_time:.2f} seconds (percentile={percentile}%, p-value<={self.max_p_value_categorical})"
         )
 
         feat_scores.to_csv(
@@ -804,33 +877,28 @@ class PreprocessModel:
         val,
         test,
         experiment,
-        target_numbers,
-        target_clf,
-        models_idx,
-        time_series,
-        max_timesteps,
-        group_column,
-        date_column,
         **kwargs,
     ):
         self.train = train
         self.val = val
         self.test = test
         self.experiment = experiment
-        self.target_numbers = target_numbers
-        self.target_clf = target_clf
-        self.models_idx = models_idx
-        self.time_series = time_series
-        self.max_timesteps = max_timesteps
-        self.group_column = group_column
-        self.date_column = date_column
+        
+        # Get all parameters from experiment context
+        self.target_numbers = self.experiment.context.get("target_numbers", [])
+        self.target_clf = self.experiment.context.get("target_clf", [])
+        self.models_idx = self.experiment.context.get("models_idx", [])
+        self.time_series = self.experiment.context.get("time_series", False)
+        self.max_timesteps = self.experiment.context.get("max_timesteps", 120)
+        self.group_column = self.experiment.context.get("group_column", None)
+        self.date_column = self.experiment.context.get("date_column", None)
 
         self.experiment_dir = experiment.path
         self.data_dir = f"{self.experiment_dir}/data"
         self.preprocessing_dir = f"{self.experiment_dir}/preprocessing"
 
         self.all_features = experiment.get_all_features(
-            date_column=date_column, group_column=group_column
+            date_column=self.date_column, group_column=self.group_column
         )
 
     def run(self):