dragon-ml-toolbox 9.0.0__tar.gz → 9.2.0__tar.gz

{dragon_ml_toolbox-9.0.0/dragon_ml_toolbox.egg-info → dragon_ml_toolbox-9.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 9.0.0
+Version: 9.2.0
 Summary: A collection of tools for data science and machine learning projects.
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-9.0.0 → dragon_ml_toolbox-9.2.0/dragon_ml_toolbox.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 9.0.0
+Version: 9.2.0
 Summary: A collection of tools for data science and machine learning projects.
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-9.0.0 → dragon_ml_toolbox-9.2.0}/ml_tools/ETL_engineering.py

@@ -4,6 +4,7 @@ import re
 from pathlib import Path
 from typing import Literal, Union, Optional, Any, Callable, List, Dict, Tuple
 from .path_manager import sanitize_filename, make_fullpath
+from .utilities import save_dataframe, load_dataframe
 from ._script_info import _script_info
 from ._logger import _LOGGER
 
@@ -62,7 +63,7 @@ def save_unique_values(csv_path: Union[str, Path], output_dir: Union[str, Path])
         
     # --- 3. Process Each Column ---
     for i, column_name in enumerate(df.columns):
-        _LOGGER.info(f"Processing column: '{column_name}'...")
+        # _LOGGER.info(f"Processing column: '{column_name}'...")
 
         # --- Get unique values AS IS ---
         try:
@@ -96,7 +97,7 @@ def save_unique_values(csv_path: Union[str, Path], output_dir: Union[str, Path])
         except IOError:
             _LOGGER.exception(f"Error writing to file {file_path}.")
         else:
-            _LOGGER.info(f"Successfully saved {len(sorted_uniques)} unique values to '{file_path}'")
+            _LOGGER.info(f"Successfully saved {len(sorted_uniques)} unique values from '{column_name}'.")
             
     _LOGGER.info("Process complete.")
 
@@ -190,12 +191,13 @@ class DataFrameCleaner:
 
         self.cleaners = cleaners
 
-    def clean(self, df: pl.DataFrame) -> pl.DataFrame:
+    def clean(self, df: pl.DataFrame, clone_df: bool=True) -> pl.DataFrame:
         """
         Applies all defined cleaning rules to the Polars DataFrame.
 
         Args:
             df (pl.DataFrame): The Polars DataFrame to clean.
+            clone_df (bool): Whether to work on a clone to prevent undesired changes.
 
         Returns:
             pl.DataFrame: A new, cleaned Polars DataFrame.
@@ -214,7 +216,10 @@ class DataFrameCleaner:
                 print(f"\t- {miss_col}")
             raise ValueError()
 
-        df_cleaned = df.clone()
+        if clone_df:
+            df_cleaned = df.clone()
+        else:
+            df_cleaned = df
         
         # Build and apply a series of expressions for each column
         for cleaner in self.cleaners:
@@ -226,7 +231,14 @@ class DataFrameCleaner:
             # Sequentially chain 'replace_all' expressions for each rule
             for pattern, replacement in cleaner.rules.items():
                 final_pattern = f"(?i){pattern}" if cleaner.case_insensitive else pattern
-                col_expr = col_expr.str.replace_all(final_pattern, replacement)
+                
+                if replacement is None:
+                    # If replacement is None, use a when/then expression to set matching values to null
+                    col_expr = pl.when(col_expr.str.contains(final_pattern)) \
+                                .then(None) \
+                                .otherwise(col_expr)
+                else:
+                    col_expr = col_expr.str.replace_all(final_pattern, replacement)
             
             # Execute the expression chain for the column
             df_cleaned = df_cleaned.with_columns(col_expr.alias(col_name))
@@ -234,6 +246,33 @@ class DataFrameCleaner:
         _LOGGER.info(f"Cleaned {len(self.cleaners)} columns.")
             
         return df_cleaned
+    
+    def load_clean_save(self, input_filepath: Union[str,Path], output_filepath: Union[str,Path]):
+        """
+        This convenience method encapsulates the entire cleaning process into a
+        single call. It loads a DataFrame from a specified file, applies all
+        cleaning rules configured in the `DataFrameCleaner` instance, and saves
+        the resulting cleaned DataFrame to a new file.
+
+        The method ensures that all data is loaded as string types to prevent
+        unintended type inference issues before cleaning operations are applied.
+
+        Args:
+            input_filepath (Union[str, Path]):
+                The path to the input data file.
+            output_filepath (Union[str, Path]):
+                The full path, where the cleaned data file will be saved.
+        """
+        df, _ = load_dataframe(df_path=input_filepath, kind="polars", all_strings=True)
+        
+        df_clean = self.clean(df=df, clone_df=False)
+        
+        if isinstance(output_filepath, str):
+            output_filepath = make_fullpath(input_path=output_filepath, enforce="file")
+        
+        save_dataframe(df=df_clean, save_dir=output_filepath.parent, filename=output_filepath.name)
+        
+        return None
 
 
 ############ TRANSFORM MAIN ####################

{dragon_ml_toolbox-9.0.0 → dragon_ml_toolbox-9.2.0}/ml_tools/handle_excel.py

@@ -167,49 +167,63 @@ def validate_excel_schema(
     strict: bool = False
 ) -> None:
     """
-    Validates that each Excel file in a directory conforms to the expected column schema.
+    Validates that each Excel file in a directory conforms to the expected column schema. Only the first worksheet of each file is analyzed.
     
     Parameters:
         target_dir (str | Path): Path to the directory containing Excel files.
         expected_columns (list[str]): List of expected column names.
         strict (bool): If True, columns must match exactly (names and order).
                       If False, columns must contain at least all expected names.
-
-    Returns:
-        List[str]: List of file paths that failed the schema validation.
     """
-    invalid_files: list[Path] = []
+    invalid_files: dict[str, str] = {}
     expected_set = set(expected_columns)
     
     target_path = make_fullpath(target_dir)
-    
     excel_paths = find_excel_files(target_path)
     
     for file in excel_paths:
         try:
+            # Using first worksheet
             wb = load_workbook(file, read_only=True)
-            ws = wb.active  # Only check the first worksheet
+            ws = wb.active
 
             header = [cell.value for cell in next(ws.iter_rows(max_row=1))] # type: ignore
 
+            # Change 2: Detailed reason-finding logic
             if strict:
                 if header != expected_columns:
-                    invalid_files.append(file)
+                    header_set = set(header)
+                    reason_parts = []
+                    missing = sorted(list(expected_set - header_set)) # type: ignore
+                    extra = sorted(list(header_set - expected_set)) # type: ignore
+                    
+                    if missing:
+                        reason_parts.append(f"Missing: {missing}")
+                    if extra:
+                        reason_parts.append(f"Extra: {extra}")
+                    if not missing and not extra:
+                        reason_parts.append("Incorrect column order")
+                    
+                    invalid_files[file.name] = ". ".join(reason_parts)
             else:
                 header_set = set(header)
                 if not expected_set.issubset(header_set):
-                    invalid_files.append(file)
+                    missing_cols = sorted(list(expected_set - header_set)) # type: ignore
+                    reason = f"Missing required columns: {missing_cols}"
+                    invalid_files[file.name] = reason
 
         except Exception as e:
             _LOGGER.error(f"Error processing '{file}': {e}")
-            invalid_files.append(file)
+            invalid_files[file.name] = f"File could not be read. Error: {e}"
     
     valid_excel_number = len(excel_paths) - len(invalid_files)
     _LOGGER.info(f"{valid_excel_number} out of {len(excel_paths)} excel files conform to the schema.")
+    
+    # Change 3: Updated print loop to show the reason
     if invalid_files:
         _LOGGER.warning(f"{len(invalid_files)} excel files are invalid:")
-        for in_file in invalid_files:
-            print(f"  - {in_file.name}")
+        for file_name, reason in invalid_files.items():
+            print(f"  - {file_name}: {reason}")
 
     return None
 

{dragon_ml_toolbox-9.0.0 → dragon_ml_toolbox-9.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dragon-ml-toolbox"
-version = "9.0.0"
+version = "9.2.0"
 description = "A collection of tools for data science and machine learning projects."
 authors = [
     { name = "Karl Loza", email = "luigiloza@gmail.com" }