dragon-ml-toolbox 10.0.1__py3-none-any.whl → 10.1.1__py3-none-any.whl

{dragon_ml_toolbox-10.0.1.dist-info → dragon_ml_toolbox-10.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 10.0.1
+Version: 10.1.1
 Summary: A collection of tools for data science and machine learning projects.
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-10.0.1.dist-info → dragon_ml_toolbox-10.1.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-dragon_ml_toolbox-10.0.1.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
-dragon_ml_toolbox-10.0.1.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
-ml_tools/ETL_cleaning.py,sha256=g_6BH0amK4aQwX8aEM2z4JYyskjbSg5ktu8n0YbrM3w,14905
+dragon_ml_toolbox-10.1.1.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
+dragon_ml_toolbox-10.1.1.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
+ml_tools/ETL_cleaning.py,sha256=i-hrafaAivg8wprcCmwHA5MkXFsUmHNR9RRGbIyw4ZE,15981
 ml_tools/ETL_engineering.py,sha256=sgpIhlFIeId4eSJ-a33MnVuPNXs50msxFWa8-kw2hOI,36369
 ml_tools/GUI_tools.py,sha256=kEQWg-bog3pB5tI22gMGKWaCGHnz9TB2Lvvfhf5F2CI,45412
 ml_tools/MICE_imputation.py,sha256=kVSythWfxJFR4-2mtcYCWQaQ1Oz5yyx_SJu5gjnS7H8,11670
@@ -30,7 +30,7 @@ ml_tools/keys.py,sha256=HtPG8-MWh89C32A7eIlfuuA-DLwkxGkoDfwR2TGN9CQ,1074
 ml_tools/optimization_tools.py,sha256=P3I6lIpvZ8Xf2kX5FvvBKBmrK2pB6idBpkTzfUJxTeE,5073
 ml_tools/path_manager.py,sha256=TJgoqMAryc5F0dal8W_zvJgE1TpOzlskIyYJk614WW4,13809
 ml_tools/utilities.py,sha256=SVMaSDigh6SUoAeig2_sXLLIj5w5mUs5KuVWpHvFDec,19816
-dragon_ml_toolbox-10.0.1.dist-info/METADATA,sha256=aWKOlsr3Ru2rUeadnl_uhKNbjFTPTtYDHv4zg7kcM9c,6968
-dragon_ml_toolbox-10.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dragon_ml_toolbox-10.0.1.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
-dragon_ml_toolbox-10.0.1.dist-info/RECORD,,
+dragon_ml_toolbox-10.1.1.dist-info/METADATA,sha256=wJ2byoP5azuIBrLRpUUQ96DkDAQuxVtgf2lFPafBUUQ,6968
+dragon_ml_toolbox-10.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dragon_ml_toolbox-10.1.1.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
+dragon_ml_toolbox-10.1.1.dist-info/RECORD,,

ml_tools/ETL_cleaning.py

@@ -1,8 +1,7 @@
 import polars as pl
 import pandas as pd
-import re
 from pathlib import Path
-from typing import Literal, Union, Optional, Any, Callable, List, Dict, Tuple
+from typing import Union, List, Dict
 from .path_manager import sanitize_filename, make_fullpath
 from .utilities import save_dataframe, load_dataframe
 from ._script_info import _script_info
@@ -18,7 +17,7 @@ __all__ = [
 
 
 ################ Unique Values per column #################
-def save_unique_values(csv_path: Union[str, Path], output_dir: Union[str, Path]) -> None:
+def save_unique_values(csv_path: Union[str, Path], output_dir: Union[str, Path], verbose: bool=False) -> None:
     """
     Loads a CSV file, then analyzes it and saves the unique non-null values
     from each column into a separate text file exactly as they appear.
@@ -51,6 +50,7 @@ def save_unique_values(csv_path: Union[str, Path], output_dir: Union[str, Path])
         _LOGGER.info(f"Data loaded from '{csv_path}'")
         
     # --- 3. Process Each Column ---
+    counter = 0
     for i, column_name in enumerate(df.columns):
         # _LOGGER.info(f"Processing column: '{column_name}'...")
 
@@ -86,9 +86,11 @@ def save_unique_values(csv_path: Union[str, Path], output_dir: Union[str, Path])
         except IOError:
             _LOGGER.exception(f"Error writing to file {file_path}.")
         else:
-            _LOGGER.info(f"Successfully saved {len(sorted_uniques)} unique values from '{column_name}'.")
-            
-    _LOGGER.info("Process complete.")
+            if verbose:
+                _LOGGER.info(f"Successfully saved {len(sorted_uniques)} unique values from '{column_name}'.")
+            counter += 1
+
+    _LOGGER.info(f"{counter} files of unique values created.")
 
 
 ########## Basic df cleaner #############
@@ -131,9 +133,30 @@ def basic_clean(input_filepath: Union[str,Path], output_filepath: Union[str,Path
         r'\p{C}+': '',
         
         # Full-width to half-width
+        # Numbers
+        '０': '0', '１': '1', '２': '2', '３': '3', '４': '4',
+        '５': '5', '６': '6', '７': '7', '８': '8', '９': '9',
+        # Superscripts & Subscripts
+        '¹': '1', '²': '2', '³': '3', '⁴': '4', '⁵': '5',
+        '⁶': '6', '⁷': '7', '⁸': '8', '⁹': '9', '⁰': '0',
+        '₁': '1', '₂': '2', '₃': '3', '₄': '4', '₅': '5',
+        '₆': '6', '₇': '7', '₈': '8', '₉': '9', '₀': '0',
+        # Uppercase Alphabet
+        'Ａ': 'A', 'Ｂ': 'B', 'Ｃ': 'C', 'Ｄ': 'D', 'Ｅ': 'E', 'Ｆ': 'F',
+        'Ｇ': 'G', 'Ｈ': 'H', 'Ｉ': 'I', 'Ｊ': 'J', 'Ｋ': 'K', 'Ｌ': 'L',
+        'Ｍ': 'M', 'Ｎ': 'N', 'Ｏ': 'O', 'Ｐ': 'P', 'Ｑ': 'Q', 'Ｒ': 'R',
+        'Ｓ': 'S', 'Ｔ': 'T', 'Ｕ': 'U', 'Ｖ': 'V', 'Ｗ': 'W', 'Ｘ': 'X',
+        'Ｙ': 'Y', 'Ｚ': 'Z',
+        # Lowercase Alphabet
+        'ａ': 'a', 'ｂ': 'b', 'ｃ': 'c', 'ｄ': 'd', 'ｅ': 'e', 'ｆ': 'f',
+        'ｇ': 'g', 'ｈ': 'h', 'ｉ': 'i', 'ｊ': 'j', 'ｋ': 'k', 'ｌ': 'l',
+        'ｍ': 'm', 'ｎ': 'n', 'ｏ': 'o', 'ｐ': 'p', 'ｑ': 'q', 'ｒ': 'r',
+        'ｓ': 's', 'ｔ': 't', 'ｕ': 'u', 'ｖ': 'v', 'ｗ': 'w', 'ｘ': 'x',
+        'ｙ': 'y', 'ｚ': 'z',
+        # Punctuation
         '》': '>', '《': '<', '：': ':', '，': ',', '。': '.', '；': ';', '【': '[', '】': ']',
-        '（': '(', '）': ')', '？': '?', '！': '!', '～': '~', '＠': '@', '＃': '#',
-        '＄': '$', '％': '%', '＾': '^', '＆': '&', '＊': '*', '＼': '\\', '｜': '|',
+        '（': '(', '）': ')', '？': '?', '！': '!', '～': '~', '＠': '@', '＃': '#', '＋': '+', '－': '-',
+        '＄': '$', '％': '%', '＾': '^', '＆': '&', '＊': '*', '＼': '\\', '｜': '|', '、':',', '≈':'=',
         
         # Others
         '©': '',
@@ -148,9 +171,9 @@ def basic_clean(input_filepath: Union[str,Path], output_filepath: Union[str,Path
         # Typographical standardization
         # Unify various dashes and hyphens to a standard hyphen-minus
         r'[—–―]': '-',
-        # Unify various quote types to standard single quotes
+        # Unify various quote types to standard quotes
         r'[“”]': "'",
-        r'[‘’]': "'",
+        r'[‘’′]': "'",
 
         # 2. Internal Whitespace Consolidation
         # Collapse any sequence of whitespace chars (including non-breaking spaces) to a single space
@@ -162,7 +185,7 @@ def basic_clean(input_filepath: Union[str,Path], output_filepath: Union[str,Path
         
         # 4. Textual Null Standardization (New Step)
         # Convert common null-like text to actual nulls. (?i) makes it case-insensitive.
-        r'^(N/A|NA|NULL|NONE|NIL|)$': None,
+        r'^(N/A|无|NA|NULL|NONE|NIL|)$': None,
 
         # 5. Final Nullification of Empty Strings
         # After all cleaning, if a string is now empty, convert it to a null
@@ -238,14 +261,6 @@ class ColumnCleaner:
             _LOGGER.error("The 'rules' argument must be a dictionary.")
             raise TypeError()
 
-        # Validate each regex pattern for correctness
-        for pattern in rules.keys():
-            try:
-                re.compile(pattern)
-            except re.error:
-                _LOGGER.error(f"Invalid regex pattern '{pattern}'.")
-                raise
-
         self.column_name = column_name
         self.rules = rules
         self.case_insensitive = case_insensitive