dragon-ml-toolbox 10.2.0__py3-none-any.whl → 10.2.1__py3-none-any.whl

{dragon_ml_toolbox-10.2.0.dist-info → dragon_ml_toolbox-10.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 10.2.0
+Version: 10.2.1
 Summary: A collection of tools for data science and machine learning projects.
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-10.2.0.dist-info → dragon_ml_toolbox-10.2.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-dragon_ml_toolbox-10.2.0.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
-dragon_ml_toolbox-10.2.0.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
-ml_tools/ETL_cleaning.py,sha256=-hxvnJYkGcBAR2eattOcgfGqPxM3TIORC6pCNvwDsf4,19113
+dragon_ml_toolbox-10.2.1.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
+dragon_ml_toolbox-10.2.1.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
+ml_tools/ETL_cleaning.py,sha256=lSP5q6-ukGhJBPV8dlsqJvPXAzj4du_0J-SbtEd0Pjg,19292
 ml_tools/ETL_engineering.py,sha256=sgpIhlFIeId4eSJ-a33MnVuPNXs50msxFWa8-kw2hOI,36369
 ml_tools/GUI_tools.py,sha256=kEQWg-bog3pB5tI22gMGKWaCGHnz9TB2Lvvfhf5F2CI,45412
 ml_tools/MICE_imputation.py,sha256=kVSythWfxJFR4-2mtcYCWQaQ1Oz5yyx_SJu5gjnS7H8,11670
@@ -30,7 +30,7 @@ ml_tools/keys.py,sha256=HtPG8-MWh89C32A7eIlfuuA-DLwkxGkoDfwR2TGN9CQ,1074
 ml_tools/optimization_tools.py,sha256=P3I6lIpvZ8Xf2kX5FvvBKBmrK2pB6idBpkTzfUJxTeE,5073
 ml_tools/path_manager.py,sha256=TJgoqMAryc5F0dal8W_zvJgE1TpOzlskIyYJk614WW4,13809
 ml_tools/utilities.py,sha256=SVMaSDigh6SUoAeig2_sXLLIj5w5mUs5KuVWpHvFDec,19816
-dragon_ml_toolbox-10.2.0.dist-info/METADATA,sha256=nJ-15xA7A7FgzYDRSi6xjhBmn32Fz57TEn2Wqg5hZRg,6968
-dragon_ml_toolbox-10.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dragon_ml_toolbox-10.2.0.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
-dragon_ml_toolbox-10.2.0.dist-info/RECORD,,
+dragon_ml_toolbox-10.2.1.dist-info/METADATA,sha256=zBNbArgUJkZxhs6hnjwgZX9rrjIKPKLRKxSKlPVqvLE,6968
+dragon_ml_toolbox-10.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dragon_ml_toolbox-10.2.1.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
+dragon_ml_toolbox-10.2.1.dist-info/RECORD,,

ml_tools/ETL_cleaning.py

@@ -126,27 +126,37 @@ def _cleaner_core(df_in: pl.DataFrame) -> pl.DataFrame:
         'ｓ': 's', 'ｔ': 't', 'ｕ': 'u', 'ｖ': 'v', 'ｗ': 'w', 'ｘ': 'x',
         'ｙ': 'y', 'ｚ': 'z',
         # Punctuation
-        '》': '>', '《': '<', '：': ':', '，': ',', '。': '.', '；': ';', '【': '[', '】': ']', 
+        '》': '>', '《': '<', '：': ':', '。': '.', '；': ';', '【': '[', '】': ']', 
         '（': '(', '）': ')', '？': '?', '！': '!', '～': '~', '＠': '@', '＃': '#', '＋': '+', '－': '-',
-        '＄': '$', '％': '%', '＾': '^', '＆': '&', '＊': '*', '＼': '\\', '｜': '|', '、':',', '≈':'=',
+        '＄': '$', '％': '%', '＾': '^', '＆': '&', '＊': '*', '＼': '-', '｜': '|', '≈':'=',
+        
+        # Commas (avoid commas in entries)
+        '，': ';',
+        ',': ';',
+        '、':';',
         
         # Others
         '©': '',
         '®': '',
         '™': '',
+        r'[°˚]': '',
         
-        # Collapse repeating punctuation
-        r'\.{2,}': '.',      # Replace two or more dots with a single dot
-        r'\?{2,}': '?',      # Replace two or more question marks with a single question mark
-        r'!{2,}': '!',      # Replace two or more exclamation marks with a single one
+        # Replace special characters in entries
+        r'\\': '-',
+        '/': '-',
         
         # Typographical standardization
-        # Unify various dashes and hyphens to a standard hyphen-minus
+        # Unify various dashes and hyphens to a standard hyphen
         r'[—–―]': '-',
         r'−': '-',
-        # Unify various quote types to standard quotes
-        r'[“”]': "'",
-        r'[‘’′]': "'",
+        # remove various quote types
+        r'[“”"]': '',
+        r"[‘’′']": '',
+        
+        # Collapse repeating punctuation
+        r'\.{2,}': '.',      # Replace two or more dots with a single dot
+        r'\?{2,}': '?',      # Replace two or more question marks with a single question mark
+        r'!{2,}': '!',      # Replace two or more exclamation marks with a single one
 
         # 2. Internal Whitespace Consolidation
         # Collapse any sequence of whitespace chars (including non-breaking spaces) to a single space