lecrapaud 0.8.1__py3-none-any.whl → 0.8.3__py3-none-any.whl

lecrapaud/feature_selection.py

@@ -135,10 +135,9 @@ class FeatureSelectionEngine:
         logger.info(f"Starting feature selection for TARGET_{target_number}...")
         clean_directory(self.fs_dir_target)
 
-        # Let's start by removing extremly correlated features
-        # This is needed to reduce nb of feature but also for methods such as anova or chi2 that requires independent features
-        # TODO: we could also remove low variance features
-        self.X = self.remove_constant_columns()
+        # Let's start by removing very low variance feature and extremly correlated features
+        # This is needed to reduce nb of feature but also for methods such as anova or chi2 that requires independent, non constant, non full 0 features
+        self.X = self.remove_low_variance_columns()
         features_uncorrelated, features_correlated = self.remove_correlated_features(
             90, vizualize=False
         )
@@ -353,34 +352,31 @@ class FeatureSelectionEngine:
     # Remove correlation
     # ------------------
 
-    def remove_constant_columns(
-        self,
-        threshold: float = 0.99,
-    ) -> pd.DataFrame:
+    def remove_low_variance_columns(self, threshold: float = 1e-10) -> pd.DataFrame:
         """
-            Removes constant or almost constant columns from a DataFrame.
+        Removes columns with very low variance (including constant columns).
 
         Parameters:
-            threshold (float): Max proportion for a single value (default 0.99 = 99%).
+            threshold (float): Minimum variance required to keep a column.
+                            Default is 1e-10 to eliminate near-constant features.
 
         Returns:
-            pd.DataFrame: Cleaned DataFrame.
+            pd.DataFrame: Cleaned DataFrame without low-variance columns.
         """
+        X = self.X
 
-        to_drop = []
-
-        for col in self.data.columns:
-            value_counts = self.data[col].value_counts(dropna=False, normalize=True)
-            if value_counts.empty:
-                to_drop.append(col)
-            elif value_counts.iloc[0] >= threshold:
-                to_drop.append(col)
+        low_var_cols = [
+            col
+            for col in X.columns
+            if pd.api.types.is_numeric_dtype(X[col])
+            and np.nanvar(X[col].values) < threshold
+        ]
 
-        if to_drop:
-            logger.info(f"🔍 Removed {len(to_drop)} constant/almost constant columns:")
-            logger.info(to_drop)
+        if low_var_cols:
+            logger.info(f"🧹 Removed {len(low_var_cols)} low-variance columns:")
+            logger.info(low_var_cols)
 
-        return self.data.drop(columns=to_drop, errors="ignore")
+        return X.drop(columns=low_var_cols, errors="ignore")
 
     def remove_correlated_features(self, corr_threshold: int, vizualize: bool = False):
         X = self.X

{lecrapaud-0.8.1.dist-info → lecrapaud-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lecrapaud
-Version: 0.8.1
+Version: 0.8.3
 Summary: Framework for machine and deep learning, with regression, classification and time series analysis
 License: Apache License
 Author: Pierre H. Gallet

{lecrapaud-0.8.1.dist-info → lecrapaud-0.8.3.dist-info}/RECORD

@@ -24,7 +24,7 @@ lecrapaud/db/session.py,sha256=K9dTyXmO-aF_2r9RRHsDsbW9_zLNDaOcchXlpiv7cSo,2719
 lecrapaud/directories.py,sha256=t4PrnJR48MpDfBOTYTyGlDVMUr39mcaj7uCPTaocBRw,725
 lecrapaud/experiment.py,sha256=NwwGDZqUyvvRu5EDK3_Oh0_kF29bNIPDawVFFpzFvZM,2350
 lecrapaud/feature_engineering.py,sha256=U3YOftZBB3PEqGbu2aFY_3B3Ks9Hiu04UxixOkBz0UU,31168
-lecrapaud/feature_selection.py,sha256=v4azPcLs-xutvjlIWdTTgJu9vnT5Gpmg9Tb3hEeDqms,43681
+lecrapaud/feature_selection.py,sha256=CUekyBet6AehobFJV_RWmKCOjMHKWadcXJUDb6FbRZM,43671
 lecrapaud/integrations/openai_integration.py,sha256=hHLF3fk5Bps8KNbNrEL3NUFa945jwClE6LrLpuMZOd4,7459
 lecrapaud/jobs/__init__.py,sha256=ZkrsyTOR21c_wN7RY8jPhm8jCrL1oCEtTsf3VFIlQiE,292
 lecrapaud/jobs/config.py,sha256=AmO0j3RFjx8H66dfKw_7vnshaOJb9Ox5BAZ9cwwLFMY,377
@@ -37,7 +37,7 @@ lecrapaud/speed_tests/test-gpu-resnet.ipynb,sha256=27Vu7nYwujYeh3fOxBNCnKJn3MXNP
 lecrapaud/speed_tests/test-gpu-transformers.ipynb,sha256=k6MBSs_Um1h4PykvE-LTBcdpbWLbIFST_xl_AFW2jgI,8444
 lecrapaud/speed_tests/tests.ipynb,sha256=RjI7LDHSsbadUkea_hT14sD7ivljtIQk4NB5McXJ1bE,3835
 lecrapaud/utils.py,sha256=Dy2uhK9cslzoqRHiIE6MdkxjNJWjmKmzGr6i7EYM28A,8106
-lecrapaud-0.8.1.dist-info/LICENSE,sha256=MImCryu0AnqhJE_uAZD-PIDKXDKb8sT7v0i1NOYeHTM,11350
-lecrapaud-0.8.1.dist-info/METADATA,sha256=-N9LIA9ueqU7d4-AlJ0Evl2KBxiZYrzGBz5oMZ9MQnY,11623
-lecrapaud-0.8.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lecrapaud-0.8.1.dist-info/RECORD,,
+lecrapaud-0.8.3.dist-info/LICENSE,sha256=MImCryu0AnqhJE_uAZD-PIDKXDKb8sT7v0i1NOYeHTM,11350
+lecrapaud-0.8.3.dist-info/METADATA,sha256=w6Nq7AP4_mybzw_dTveQm6PbAddeWMtYswdNe2qoVxc,11623
+lecrapaud-0.8.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lecrapaud-0.8.3.dist-info/RECORD,,