datamarket 0.9.30__py3-none-any.whl → 0.9.31__py3-none-any.whl

datamarket/utils/strings/normalization.py

@@ -3,7 +3,7 @@
 
 import unicodedata
 from enum import Enum, auto
-from typing import Any
+from typing import Any, Optional, Set
 
 import numpy as np
 from inflection import camelize, parameterize, titleize, underscore
@@ -23,6 +23,8 @@ class NormalizationMode(Enum):
 
 class NamingConvention(Enum):
     NONE = auto()  # no style change
+    LOWER = auto()  # lowercase
+    UPPER = auto()  # UPPERCASE
     CONSTANT = auto()  # CONSTANT_CASE (uppercase, underscores)
     SNAKE = auto()  # snake_case (lowercase, underscores)
     CAMEL = auto()  # camelCase (capitalize words except first one, no spaces)
@@ -35,20 +37,52 @@ class NamingConvention(Enum):
 # FUNCTIONS
 
 
-def transliterate_symbols(s: str) -> str:
+def get_unidecoded_text(input_text: str, allowed_chars: Set[str], apply_lowercase: bool = False) -> str:
+    """
+    Processes a string by unidecoding characters, optionally lowercasing them,
+    while preserving a specified set of allowed characters.
+
+    Args:
+        input_text: The string to process.
+        allowed_chars: A set of characters to preserve in their original form.
+        apply_lowercase: Whether to convert unidecoded characters to lowercase. Defaults to False.
+
+    Returns:
+        The processed string.
+    """
+    chars_list: list[str] = []
+    for char_original in input_text:
+        if char_original in allowed_chars:
+            chars_list.append(char_original)
+        else:
+            decoded_segment = unidecode(char_original)
+            for dc in decoded_segment:  # unidecode can return multiple chars
+                if apply_lowercase:
+                    chars_list.append(dc.lower())
+                else:
+                    chars_list.append(dc)
+    return "".join(chars_list)
+
+
+def transliterate_symbols(s: str, allowed_symbols_set: Optional[Set[str]] = None) -> str:
     """
     Translates Unicode symbols (category S*) in the input string to their lowercase Unicode names,
-    with spaces replaced by underscores. Other characters remain unchanged.
+    with spaces replaced by underscores. Other characters, or characters in allowed_symbols_set, remain unchanged.
 
     Args:
         s: The input string.
+        allowed_symbols_set: A set of characters to preserve without transliteration.
 
     Returns:
-        The string with symbols transliterated.
+        The string with symbols transliterated or preserved.
     """
+    if allowed_symbols_set is None:
+        allowed_symbols_set = set()
     out: list[str] = []
     for c in s:
-        if unicodedata.category(c).startswith("S"):
+        if c in allowed_symbols_set:
+            out.append(c)
+        elif unicodedata.category(c).startswith("S"):
             name = unicodedata.name(c, "")
             if name:
                 out.append(name.lower().replace(" ", "_"))
@@ -58,21 +92,32 @@ def transliterate_symbols(s: str) -> str:
 
 
 def normalize(
-    s: Any, mode: NormalizationMode = NormalizationMode.BASIC, naming: NamingConvention = NamingConvention.NONE
+    s: Any,
+    mode: NormalizationMode = NormalizationMode.BASIC,
+    naming: NamingConvention = NamingConvention.LOWER,
+    allowed_symbols: Optional[str] = None,
 ) -> str:
     """
     Normalizes and applies a naming convention to the input.
 
     Handles None and NaN values by returning an empty string. Converts non-string inputs to strings.
 
-    Normalization is applied according to `mode`:
-    - NONE: Returns the input as a string without any normalization.
-    - BASIC: Removes accents, converts punctuation and spaces to single spaces, and preserves alphanumeric characters.
+    Normalization (controlled by `mode`) occurs first, followed by naming convention application.
+    - NONE: Returns the input as a string without any normalization. Case is preserved.
+    - BASIC: Removes accents (via unidecode). Punctuation and spaces typically become single spaces between tokens.
+             Case is preserved from the unidecode step by default.
     - SYMBOLS: Translates only Unicode symbols (category S*) to their lowercase Unicode names with underscores.
-    - FULL: Applies both BASIC and SYMBOLS normalization.
+               Other characters are preserved, including their case.
+    - FULL: Applies unidecode (case-preserved by default) and then SYMBOLS-like transliteration for S* category
+            characters not otherwise handled.
+
+    The `allowed_symbols` parameter can be used to specify characters that should be preserved in their original form
+    throughout the normalization process. These characters will not be unidecoded or transliterated by the symbol logic.
 
-    After normalization, a naming convention is applied according to `naming`:
-    - NONE: Returns the normalized text.
+    After normalization, a naming convention (controlled by `naming`) is applied:
+    - NONE: Returns the normalized text, preserving its case from the normalization step.
+    - LOWER: Converts the normalized text to lowercase. (Default)
+    - UPPER: Converts the normalized text to UPPERCASE.
     - CONSTANT: Converts to CONSTANT_CASE (uppercase with underscores).
     - SNAKE: Converts to snake_case (lowercase with underscores).
     - CAMEL: Converts to camelCase (lowercase first word, capitalize subsequent words, no spaces).
@@ -83,7 +128,8 @@ def normalize(
     Args:
         s: The input value to normalize and format. Can be any type.
         mode: The normalization mode to apply. Defaults to NormalizationMode.BASIC.
-        naming: The naming convention to apply. Defaults to NamingConvention.NONE.
+        naming: The naming convention to apply. Defaults to NamingConvention.LOWER.
+        allowed_symbols: A string of characters to preserve during normalization.
 
     Returns:
         The normalized and formatted string.
@@ -94,6 +140,8 @@ def normalize(
     if isinstance(naming, str):
         naming = NamingConvention[naming]
 
+    _allowed_symbols_set: Set[str] = set(allowed_symbols) if allowed_symbols else set()
+
     # Handling null values
     if s is None or (isinstance(s, float) and np.isnan(s)):
         normalized = ""
@@ -101,41 +149,52 @@ def normalize(
         return str(s)
     else:
         text = prettify(strip_html(str(s), True))
+
         if mode is NormalizationMode.NONE:
             normalized = text
         elif mode is NormalizationMode.SYMBOLS:
-            normalized = transliterate_symbols(text)
+            normalized = transliterate_symbols(text, _allowed_symbols_set)
         else:
-            # BASIC and FULL: remove accents and lowercase
-            normalized = unidecode(text).lower()
+            # BASIC and FULL modes
+            intermediate_text = get_unidecoded_text(text, _allowed_symbols_set)
+
+            # Now, tokenize the intermediate_text for BASIC and FULL
             tokens: list[str] = []
-            current: list[str] = []
+            current_token_chars: list[str] = []
 
-            def flush_current():
-                nonlocal current
-                if current:
-                    tokens.append("".join(current))
-                    current.clear()
+            def flush_current_token():
+                nonlocal current_token_chars
+                if current_token_chars:
+                    tokens.append("".join(current_token_chars))
+                    current_token_chars.clear()
 
-            for c in normalized:
+            for c in intermediate_text:
                 cat = unicodedata.category(c)
-                if c.isalnum():
-                    current.append(c)
+                if c in _allowed_symbols_set:  # Allowed symbols are part of tokens
+                    current_token_chars.append(c)
+                elif c.isalnum():
+                    current_token_chars.append(c)
                 elif mode is NormalizationMode.FULL and cat.startswith("S"):
-                    flush_current()
+                    # Transliterate S* category symbols not in allowed_symbols
+                    flush_current_token()
                     name = unicodedata.name(c, "")
                     if name:
                         tokens.append(name.lower().replace(" ", "_"))
                 elif cat.startswith("P") or c.isspace():
-                    flush_current()
-                # other characters ignored
+                    # Punctuation (not allowed) or space acts as a separator
+                    flush_current_token()
+                # Other characters are ignored
 
-            flush_current()
+            flush_current_token()
             normalized = " ".join(tokens)
 
     # Apply naming convention
     if naming is NamingConvention.NONE:
         return normalized
+    if naming is NamingConvention.LOWER:
+        return normalized.lower()
+    if naming is NamingConvention.UPPER:
+        return normalized.upper()
     if naming is NamingConvention.PARAM:
         return parameterize(normalized)
     if naming is NamingConvention.TITLE:

{datamarket-0.9.30.dist-info → datamarket-0.9.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamarket
-Version: 0.9.30
+Version: 0.9.31
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 Home-page: https://datamarket.es
 License: GPL-3.0-or-later

{datamarket-0.9.30.dist-info → datamarket-0.9.31.dist-info}/RECORD

@@ -17,11 +17,11 @@ datamarket/utils/main.py,sha256=j8wnAxeLvijdRU9M4V6HunWH7vgWWHP4u4xamzkWcUU,7009
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/strings/__init__.py,sha256=RmyN3hKGXmUym8w5tn28yWkw2uM-b5OvntB4D0lU1eo,84
-datamarket/utils/strings/normalization.py,sha256=337M2UPwEETvhVTOnP4w_igTXpHUHoaD8e7x_-L-Bpk,5654
+datamarket/utils/strings/normalization.py,sha256=z2dDXFVQ-nVqPDRR1T4HWmELpJuXUVt_P6leHqPVheY,8666
 datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.30.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.30.dist-info/METADATA,sha256=zzhHMrHhBf_CfBLwjj4melul8sCkcO8np-nmay0jKOQ,6871
-datamarket-0.9.30.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-datamarket-0.9.30.dist-info/RECORD,,
+datamarket-0.9.31.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.31.dist-info/METADATA,sha256=bM8NgTxQZXA9bSE5vsraTMx_TZTuSsC9VuTU1fyhalQ,6871
+datamarket-0.9.31.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+datamarket-0.9.31.dist-info/RECORD,,