dragon-ml-toolbox 3.2.1__py3-none-any.whl → 3.3.0__py3-none-any.whl

{dragon_ml_toolbox-3.2.1.dist-info → dragon_ml_toolbox-3.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 3.2.1
+Version: 3.3.0
 Summary: A collection of tools for data science and machine learning projects.
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-3.2.1.dist-info → dragon_ml_toolbox-3.3.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
-dragon_ml_toolbox-3.2.1.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
-dragon_ml_toolbox-3.2.1.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=6cfpIeQ6D4Mcs10nkogQrkVyq1T7i2qXjjNHFoUMOyE,1892
-ml_tools/ETL_engineering.py,sha256=Y1EW_OeUiJju-BA31tnmetV_09hDLvgrE2Rk31TnDOU,37270
+dragon_ml_toolbox-3.3.0.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
+dragon_ml_toolbox-3.3.0.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=6cfpIeQ6D4Mcs10nkogQrkVyq1T7i2qXjjNHFoUMOyE,1892
+ml_tools/ETL_engineering.py,sha256=TL5sz7ac4uwa993Uy6Ex11-9vN_t5IaD-Y9UjyI88_o,39030
 ml_tools/GUI_tools.py,sha256=uFx6zIrQZzDPSTtOSHz8ptz-fxZiQz-lXHcrqwuYV_E,20385
 ml_tools/MICE_imputation.py,sha256=ed-YeQkEAeHxTNkWIHs09T4YeYNF0aqAnrUTcdIEp9E,11372
 ml_tools/ML_callbacks.py,sha256=gHZk-lyzAax6iEtG26zHuoobdAZCFJ6BmI6pWoXkOrw,13189
@@ -13,13 +13,13 @@ ml_tools/VIF_factor.py,sha256=5GVAldH69Vkei3WRUZN1uPBMzGoOOeEOA-bgmZXbbUw,10301
 ml_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ml_tools/_particle_swarm_optimization.py,sha256=b_eNNkA89Y40hj76KauivT8KLScH1B9wF2IXptOqkOw,22220
 ml_tools/_pytorch_models.py,sha256=bpWZsrSwCvHJQkR6UfoPpElsMv9AvmiNErNHC8NYB_I,10132
-ml_tools/data_exploration.py,sha256=bOcCoQLeDPFJ7nB5Fsi16lzB22cG-c-mxObMsTetgS4,23655
+ml_tools/data_exploration.py,sha256=IkpOyIRPKdu4qKeUdvvMvNPelSVWegNEKMqX3IInmpw,25003
 ml_tools/datasetmaster.py,sha256=N-uwfzWnl_qnoAqjbfS98I1pVNra5u6rhKLdWbFIReA,30122
 ml_tools/ensemble_learning.py,sha256=PPtBBLgLvaYOdY-MlcjXuxWWXf3JQavLNEysFgzjc_s,37470
 ml_tools/handle_excel.py,sha256=lwds7rDLlGSCWiWGI7xNg-Z7kxAepogp0lstSFa0590,12949
 ml_tools/logger.py,sha256=jC4Q2OqmDm8ZO9VpuZqBSWdXryqaJvLscqVJ6caNMOk,6009
 ml_tools/utilities.py,sha256=opNR-ACH6BnLkWAKcb19ef5tFxfx22TI6E2o0RYwiGA,21021
-dragon_ml_toolbox-3.2.1.dist-info/METADATA,sha256=qPGzA1lRzS1MC5RGxI1uuUQyGyOC1r6tw87xkwe2Cyw,3273
-dragon_ml_toolbox-3.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dragon_ml_toolbox-3.2.1.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
-dragon_ml_toolbox-3.2.1.dist-info/RECORD,,
+dragon_ml_toolbox-3.3.0.dist-info/METADATA,sha256=rIwDGwx-RQ1XvVe0TLA8YkQIs71W7MoQJHfx28l580M,3273
+dragon_ml_toolbox-3.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dragon_ml_toolbox-3.3.0.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
+dragon_ml_toolbox-3.3.0.dist-info/RECORD,,

ml_tools/ETL_engineering.py

@@ -12,6 +12,7 @@ __all__ = [
     "TransformationRecipe",
     "DataProcessor",
     "BinaryTransformer",
+    "MultiBinaryDummifier",
     "KeywordDummifier",
     "NumberExtractor",
     "MultiNumberExtractor",
@@ -400,12 +401,72 @@ class BinaryTransformer:
             return (~contains_keyword).cast(pl.UInt8)
 
 
+class MultiBinaryDummifier:
+    """
+    A one-to-many transformer that creates multiple binary columns from a single
+    text column based on a list of keywords.
+
+    For each keyword provided, this transformer generates a corresponding column
+    with a value of 1 if the keyword is present in the input string, and 0 otherwise.
+    It is designed to be used within the DataProcessor pipeline.
+
+    Args:
+        keywords (List[str]):
+            A list of strings, where each string is a keyword to search for. A separate
+            binary column will be created for each keyword.
+        case_insensitive (bool):
+            If True, keyword matching ignores case. Defaults to True.
+    """
+    def __init__(self, keywords: List[str], case_insensitive: bool = True):
+        if not isinstance(keywords, list) or not all(isinstance(k, str) for k in keywords):
+            raise TypeError("The 'keywords' argument must be a list of strings.")
+        if not keywords:
+            raise ValueError("The 'keywords' list cannot be empty.")
+
+        self.keywords = keywords
+        self.case_insensitive = case_insensitive
+
+    def __call__(self, column: pl.Series) -> pl.DataFrame:
+        """
+        Executes the dummification logic.
+
+        Args:
+            column (pl.Series): The input Polars Series to transform.
+
+        Returns:
+            pl.DataFrame: A DataFrame where each column corresponds to a keyword.
+        """
+        # Ensure the input is treated as a string, preserving nulls
+        str_column = column.cast(pl.Utf8)
+        
+        output_expressions = []
+        for i, keyword in enumerate(self.keywords):
+            # Escape keyword to treat it as a literal, not a regex pattern
+            base_pattern = re.escape(keyword)
+
+            # Add case-insensitivity flag `(?i)` if needed
+            pattern = f"(?i){base_pattern}" if self.case_insensitive else base_pattern
+
+            # Create the binary expression
+            expr = (
+                pl.when(str_column.is_null())
+                .then(None) # Propagate nulls from original column
+                .when(str_column.str.contains(pattern))
+                .then(pl.lit(1, dtype=pl.UInt8))
+                .otherwise(pl.lit(0, dtype=pl.UInt8))
+                .alias(f"col_{i}") # Generic name for DataProcessor
+            )
+            output_expressions.append(expr)
+
+        return pl.select(output_expressions)
+
+
 class KeywordDummifier:
     """
     A configurable transformer that creates one-hot encoded columns based on
     keyword matching in a Polars Series.
 
-    Instantiate this class with keyword configurations. The instance can be used as a 'transform' callable compatible with the `TransformationRecipe`.
+    Operates on a "first match wins" principle.
 
     Args:
         group_names (List[str]): 
@@ -417,17 +478,14 @@ class KeywordDummifier:
             `group_name` at the same index and contains the keywords to search for.
         case_insensitive (bool):
             If True, keyword matching ignores case.
-        drop_empty (bool):
-            If True, columns that contain no positive matches (all zeros) will be dropped from the final output.
     """
-    def __init__(self, group_names: List[str], group_keywords: List[List[str]], case_insensitive: bool = True, drop_empty: bool = True):
+    def __init__(self, group_names: List[str], group_keywords: List[List[str]], case_insensitive: bool = True):
         if len(group_names) != len(group_keywords):
             raise ValueError("Initialization failed: 'group_names' and 'group_keywords' must have the same length.")
         
         self.group_names = group_names
         self.group_keywords = group_keywords
         self.case_insensitive = case_insensitive
-        self.drop_empty = drop_empty
 
     def __call__(self, column: pl.Series) -> pl.DataFrame:
         """
@@ -474,16 +532,7 @@ class KeywordDummifier:
                 # If a group had no matches, create a column of zeros
                 final_columns.append(pl.lit(0, dtype=pl.UInt8).alias(name))
 
-        # First, create a full DataFrame with all potential columns
-        result_df = pl.DataFrame(final_columns)
-
-        # If drop_empty is True, filter out all-zero columns
-        if self.drop_empty:
-            # A column is kept if its sum is greater than 0
-            cols_to_keep = [col for col in result_df if col.sum() > 0]
-            return result_df.select(cols_to_keep)
-        
-        return result_df
+        return pl.DataFrame(final_columns)
 
 
 class NumberExtractor:

ml_tools/data_exploration.py

@@ -15,6 +15,7 @@ import re
 # Keep track of all available tools, show using `info()`
 __all__ = [
     "summarize_dataframe",
+    "drop_zero_only_columns",
     "drop_rows_with_missing_data",
     "split_features_targets", 
     "show_null_columns",
@@ -61,6 +62,47 @@ def summarize_dataframe(df: pd.DataFrame, round_digits: int = 2):
     return summary
 
 
+def drop_zero_only_columns(df: pd.DataFrame, verbose: bool=True) -> pd.DataFrame:
+    """
+    Removes columns from a pandas DataFrame that contain only zeros and null/NaN values.
+
+    This utility is useful for cleaning data after dummification steps that may result in empty columns.
+
+    Args:
+        df (pd.DataFrame): 
+            The pandas DataFrame to clean.
+
+    Returns:
+        pd.DataFrame: 
+            A new DataFrame with the empty columns removed.
+    """
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("Input must be a pandas DataFrame.")
+    
+    original_columns = set(df.columns)
+    
+    cols_to_keep = []
+    for col_name in df.columns:
+        column = df[col_name]
+        
+        # Keep any column that is not numeric by default
+        if not is_numeric_dtype(column):
+            cols_to_keep.append(col_name)
+            continue
+
+        # For numeric columns, check if there's at least one non-zero value.
+        if (column != 0).any():
+            cols_to_keep.append(col_name)
+    
+    dropped_columns = original_columns - set(cols_to_keep)      
+    if dropped_columns and verbose:
+        print(f"Dropped {len(dropped_columns)} columns:")
+        for dropped_column in dropped_columns:
+            print(f"    {dropped_column}")
+    
+    return df[cols_to_keep]
+
+
 def drop_rows_with_missing_data(df: pd.DataFrame, targets: Optional[list[str]], threshold: float = 0.7) -> pd.DataFrame:
     """
     Drops rows from the DataFrame using a two-stage strategy: