datamarket 0.9.29__tar.gz → 0.9.31__tar.gz

{datamarket-0.9.29 → datamarket-0.9.31}/PKG-INFO

@@ -1,17 +1,17 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: datamarket
-Version: 0.9.29
+Version: 0.9.31
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
+Home-page: https://datamarket.es
 License: GPL-3.0-or-later
 Author: DataMarket
 Author-email: techsupport@datamarket.es
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.12,<3.13
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: alchemy
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
@@ -40,6 +40,7 @@ Provides-Extra: openpyxl
 Provides-Extra: pandas
 Provides-Extra: pandera
 Provides-Extra: peerdb
+Provides-Extra: pii
 Provides-Extra: pillow
 Provides-Extra: playwright
 Provides-Extra: playwright-stealth
@@ -96,6 +97,8 @@ Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
 Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
 Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
+Requires-Dist: presidio-analyzer[phonenumbers] (>=2.0.0,<3.0.0) ; extra == "pii"
+Requires-Dist: presidio-anonymizer (>=2.0.0,<3.0.0) ; extra == "pii"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
 Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
 Requires-Dist: pycountry (>=24.0.0,<25.0.0)
@@ -111,14 +114,16 @@ Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
 Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
 Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
+Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
+Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
 Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
 Requires-Dist: tenacity (>=9.0.0,<10.0.0)
+Requires-Dist: tf-playwright-stealth (>=1.0.0,<2.0.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
 Requires-Dist: typer (>=0.15.0,<0.16.0)
 Requires-Dist: unidecode (>=1.0.0,<2.0.0)
 Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
 Project-URL: Documentation, https://github.com/Data-Market/datamarket
-Project-URL: Homepage, https://datamarket.es
 Project-URL: Repository, https://github.com/Data-Market/datamarket
 Description-Content-Type: text/markdown
 

{datamarket-0.9.29 → datamarket-0.9.31}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.29"
+version = "0.9.31"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -15,7 +15,7 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.12"
+python = ">=3.12,<3.13"
 typer = "~0.15.0"
 psycopg2-binary = "^2.0.0"
 requests = "^2.0.0"
@@ -76,6 +76,12 @@ openpyxl = { version = "^3.0.0", optional = true }
 httpx = { extras = ["http2"], version = "~0.28.0", optional = true }
 SQLAlchemy = { version = "^2.0.0", optional = true }
 camoufox = { extras = ["geoip"], version = "~0.4.11", optional = true }
+presidio-analyzer = { version = "^2.0.0", optional = true, extras = [
+    "phonenumbers",
+] }
+presidio-anonymizer = { version = "^2.0.0", optional = true }
+spacy = { version = "^3.0.0", optional = true }
+spacy-langdetect = { version = "~0.1.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -128,6 +134,9 @@ peerdb = ["boto3", "clickhouse-driver"]
 proxy = ["stem"]
 alchemy = ["SQLAlchemy"]
 
+# Other groups
+pii = ["presidio-analyzer", "presidio-anonymizer", "spacy", "spacy-langdetect"]
+
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

datamarket-0.9.31/src/datamarket/utils/strings/__init__.py

@@ -0,0 +1,2 @@
+from .normalization import *  # noqa: F403
+from .obfuscation import *  # noqa: F403

datamarket-0.9.31/src/datamarket/utils/strings/normalization.py

@@ -0,0 +1,211 @@
+########################################################################################################################
+# IMPORTS
+
+import unicodedata
+from enum import Enum, auto
+from typing import Any, Optional, Set
+
+import numpy as np
+from inflection import camelize, parameterize, titleize, underscore
+from string_utils import prettify, strip_html
+from unidecode import unidecode
+
+########################################################################################################################
+# CLASSES
+
+
+class NormalizationMode(Enum):
+    NONE = auto()
+    BASIC = auto()  # removes accents and converts punctuation to spaces
+    SYMBOLS = auto()  # translates only symbols to Unicode name
+    FULL = auto()  # BASIC + SYMBOLS
+
+
+class NamingConvention(Enum):
+    NONE = auto()  # no style change
+    LOWER = auto()  # lowercase
+    UPPER = auto()  # UPPERCASE
+    CONSTANT = auto()  # CONSTANT_CASE (uppercase, underscores)
+    SNAKE = auto()  # snake_case (lowercase, underscores)
+    CAMEL = auto()  # camelCase (capitalize words except first one, no spaces)
+    PASCAL = auto()  # PascalCase (capitalize words including first one, no spaces)
+    PARAM = auto()  # parameterize (hyphens)
+    TITLE = auto()  # titleize (capitalize words)
+
+
+########################################################################################################################
+# FUNCTIONS
+
+
+def get_unidecoded_text(input_text: str, allowed_chars: Set[str], apply_lowercase: bool = False) -> str:
+    """
+    Processes a string by unidecoding characters, optionally lowercasing them,
+    while preserving a specified set of allowed characters.
+
+    Args:
+        input_text: The string to process.
+        allowed_chars: A set of characters to preserve in their original form.
+        apply_lowercase: Whether to convert unidecoded characters to lowercase. Defaults to False.
+
+    Returns:
+        The processed string.
+    """
+    chars_list: list[str] = []
+    for char_original in input_text:
+        if char_original in allowed_chars:
+            chars_list.append(char_original)
+        else:
+            decoded_segment = unidecode(char_original)
+            for dc in decoded_segment:  # unidecode can return multiple chars
+                if apply_lowercase:
+                    chars_list.append(dc.lower())
+                else:
+                    chars_list.append(dc)
+    return "".join(chars_list)
+
+
+def transliterate_symbols(s: str, allowed_symbols_set: Optional[Set[str]] = None) -> str:
+    """
+    Translates Unicode symbols (category S*) in the input string to their lowercase Unicode names,
+    with spaces replaced by underscores. Other characters, or characters in allowed_symbols_set, remain unchanged.
+
+    Args:
+        s: The input string.
+        allowed_symbols_set: A set of characters to preserve without transliteration.
+
+    Returns:
+        The string with symbols transliterated or preserved.
+    """
+    if allowed_symbols_set is None:
+        allowed_symbols_set = set()
+    out: list[str] = []
+    for c in s:
+        if c in allowed_symbols_set:
+            out.append(c)
+        elif unicodedata.category(c).startswith("S"):
+            name = unicodedata.name(c, "")
+            if name:
+                out.append(name.lower().replace(" ", "_"))
+        else:
+            out.append(c)
+    return "".join(out)
+
+
+def normalize(
+    s: Any,
+    mode: NormalizationMode = NormalizationMode.BASIC,
+    naming: NamingConvention = NamingConvention.LOWER,
+    allowed_symbols: Optional[str] = None,
+) -> str:
+    """
+    Normalizes and applies a naming convention to the input.
+
+    Handles None and NaN values by returning an empty string. Converts non-string inputs to strings.
+
+    Normalization (controlled by `mode`) occurs first, followed by naming convention application.
+    - NONE: Returns the input as a string without any normalization. Case is preserved.
+    - BASIC: Removes accents (via unidecode). Punctuation and spaces typically become single spaces between tokens.
+             Case is preserved from the unidecode step by default.
+    - SYMBOLS: Translates only Unicode symbols (category S*) to their lowercase Unicode names with underscores.
+               Other characters are preserved, including their case.
+    - FULL: Applies unidecode (case-preserved by default) and then SYMBOLS-like transliteration for S* category
+            characters not otherwise handled.
+
+    The `allowed_symbols` parameter can be used to specify characters that should be preserved in their original form
+    throughout the normalization process. These characters will not be unidecoded or transliterated by the symbol logic.
+
+    After normalization, a naming convention (controlled by `naming`) is applied:
+    - NONE: Returns the normalized text, preserving its case from the normalization step.
+    - LOWER: Converts the normalized text to lowercase. (Default)
+    - UPPER: Converts the normalized text to UPPERCASE.
+    - CONSTANT: Converts to CONSTANT_CASE (uppercase with underscores).
+    - SNAKE: Converts to snake_case (lowercase with underscores).
+    - CAMEL: Converts to camelCase (lowercase first word, capitalize subsequent words, no spaces).
+    - PASCAL: Converts to PascalCase (capitalize all words, no spaces).
+    - PARAM: Converts to parameterize (lowercase with hyphens).
+    - TITLE: Converts to Title Case (capitalize each word).
+
+    Args:
+        s: The input value to normalize and format. Can be any type.
+        mode: The normalization mode to apply. Defaults to NormalizationMode.BASIC.
+        naming: The naming convention to apply. Defaults to NamingConvention.LOWER.
+        allowed_symbols: A string of characters to preserve during normalization.
+
+    Returns:
+        The normalized and formatted string.
+    """
+    # Parameter mapping
+    if isinstance(mode, str):
+        mode = NormalizationMode[mode]
+    if isinstance(naming, str):
+        naming = NamingConvention[naming]
+
+    _allowed_symbols_set: Set[str] = set(allowed_symbols) if allowed_symbols else set()
+
+    # Handling null values
+    if s is None or (isinstance(s, float) and np.isnan(s)):
+        normalized = ""
+    elif not isinstance(s, str):
+        return str(s)
+    else:
+        text = prettify(strip_html(str(s), True))
+
+        if mode is NormalizationMode.NONE:
+            normalized = text
+        elif mode is NormalizationMode.SYMBOLS:
+            normalized = transliterate_symbols(text, _allowed_symbols_set)
+        else:
+            # BASIC and FULL modes
+            intermediate_text = get_unidecoded_text(text, _allowed_symbols_set)
+
+            # Now, tokenize the intermediate_text for BASIC and FULL
+            tokens: list[str] = []
+            current_token_chars: list[str] = []
+
+            def flush_current_token():
+                nonlocal current_token_chars
+                if current_token_chars:
+                    tokens.append("".join(current_token_chars))
+                    current_token_chars.clear()
+
+            for c in intermediate_text:
+                cat = unicodedata.category(c)
+                if c in _allowed_symbols_set:  # Allowed symbols are part of tokens
+                    current_token_chars.append(c)
+                elif c.isalnum():
+                    current_token_chars.append(c)
+                elif mode is NormalizationMode.FULL and cat.startswith("S"):
+                    # Transliterate S* category symbols not in allowed_symbols
+                    flush_current_token()
+                    name = unicodedata.name(c, "")
+                    if name:
+                        tokens.append(name.lower().replace(" ", "_"))
+                elif cat.startswith("P") or c.isspace():
+                    # Punctuation (not allowed) or space acts as a separator
+                    flush_current_token()
+                # Other characters are ignored
+
+            flush_current_token()
+            normalized = " ".join(tokens)
+
+    # Apply naming convention
+    if naming is NamingConvention.NONE:
+        return normalized
+    if naming is NamingConvention.LOWER:
+        return normalized.lower()
+    if naming is NamingConvention.UPPER:
+        return normalized.upper()
+    if naming is NamingConvention.PARAM:
+        return parameterize(normalized)
+    if naming is NamingConvention.TITLE:
+        return titleize(normalized)
+
+    underscored = underscore(parameterize(normalized))
+    if naming is NamingConvention.CONSTANT:
+        return underscored.upper()
+    if naming is NamingConvention.CAMEL:
+        return camelize(underscored, False)
+    if naming is NamingConvention.PASCAL:
+        return camelize(underscored)
+
+    return underscored

datamarket-0.9.31/src/datamarket/utils/strings/obfuscation.py

@@ -0,0 +1,153 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+import warnings
+from typing import List, Optional
+
+
+class PiiDependenciesMissingError(ImportError):
+    pass
+
+
+class SpacyModelNotFoundError(ImportError):
+    pass
+
+
+try:
+    import phonenumbers
+    import spacy
+    from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
+    from presidio_analyzer.nlp_engine import NlpEngineProvider
+    from presidio_analyzer.predefined_recognizers import PhoneRecognizer
+    from presidio_anonymizer import AnonymizerEngine
+    from spacy.language import Language
+    from spacy_langdetect import LanguageDetector
+except ImportError as e:
+    raise PiiDependenciesMissingError(
+        "One or more PII anonymization dependencies are missing. "
+        "Please install them by running: pip install datamarket[pii]\n"
+        f"Original error: {e}"
+    ) from e
+
+
+########################################################################################################################
+# SETTINGS
+
+logger = logging.getLogger()
+logging.getLogger("presidio-analyzer").setLevel(logging.ERROR)
+
+warnings.filterwarnings(
+    "ignore",
+    message=r"\[W108\]",
+    category=UserWarning,
+    module="spacy.pipeline.lemmatizer",
+)
+
+
+@Language.factory("language_detector")
+def get_lang_detector(nlp, name):
+    return LanguageDetector()
+
+
+########################################################################################################################
+# CLASSES
+
+
+class PiiAnonymizer:
+    SUPPORTED_LANG = ["es", "en"]
+
+    def __init__(self):
+        # Check for required spaCy models
+        required_models = {
+            "en_core_web_md": "python -m spacy download en_core_web_md",
+            "es_core_news_md": "python -m spacy download es_core_news_md",
+        }
+        missing_models_instructions = []
+        for model_name, install_command in required_models.items():
+            if not spacy.util.is_package(model_name):
+                missing_models_instructions.append(
+                    f"Model '{model_name}' not found. Please install it by running: {install_command}"
+                )
+
+        if missing_models_instructions:
+            raise SpacyModelNotFoundError("\n".join(missing_models_instructions))
+
+        self.anonymizer = AnonymizerEngine()
+        self.analyzer = self._load_analyzer_engine()
+
+        self.nlp = self._nlp()
+
+    def _nlp(self) -> Language:
+        analyzer_en_model = self.analyzer.nlp_engine.nlp.get("en")
+        shared_vocab = analyzer_en_model.vocab
+        nlp = spacy.blank("en", vocab=shared_vocab)
+
+        if nlp.has_factory("sentencizer"):
+            nlp.add_pipe("sentencizer")
+
+        if nlp.has_factory("language_detector"):
+            nlp.add_pipe("language_detector", last=True)
+
+        return nlp
+
+    @staticmethod
+    def _nlp_config():
+        return {
+            "nlp_engine_name": "spacy",
+            "models": [
+                {"lang_code": "es", "model_name": "es_core_news_md"},
+                {"lang_code": "en", "model_name": "en_core_web_md"},
+            ],
+        }
+
+    def _load_analyzer_engine(self) -> AnalyzerEngine:
+        provider = NlpEngineProvider(nlp_configuration=PiiAnonymizer._nlp_config())
+        nlp_engine = provider.create_engine()
+        phone_recognizer_es = PhoneRecognizer(
+            supported_language="es",
+            supported_regions=phonenumbers.SUPPORTED_REGIONS,
+            context=["teléfono", "móvil", "número"],
+        )
+        registry = RecognizerRegistry(supported_languages=self.SUPPORTED_LANG)
+        registry.load_predefined_recognizers(nlp_engine=nlp_engine, languages=self.SUPPORTED_LANG)
+        registry.add_recognizer(phone_recognizer_es)
+
+        analyzer = AnalyzerEngine(
+            registry=registry,
+            nlp_engine=nlp_engine,
+            supported_languages=self.SUPPORTED_LANG,
+        )
+        return analyzer
+
+    def detect_lang(self, text: str) -> str:
+        if hasattr(self, "nlp") and self.nlp:
+            with self.nlp.select_pipes(enable=["tokenizer", "sentencizer", "language_detector"]):
+                doc = self.nlp(text)
+            return doc._.language["language"]
+        else:
+            logger.error("Language detection NLP model not initialized. Cannot detect language.")
+            return "unknown"
+
+    def anonymize_text(
+        self,
+        text: str,
+        entities: Optional[List[str]] = None,
+        lang: str = "unknown",
+    ) -> str:
+        if lang == "unknown":
+            lang = self.detect_lang(text)
+            if lang not in self.SUPPORTED_LANG:
+                logger.warning(f"Support for language {lang} is not implemented yet! Fail safe to empty string.")
+                return ""
+        elif lang not in self.SUPPORTED_LANG:
+            logger.warning(f"Support for language {lang} is not implemented yet! Fail safe to empty string.")
+            return ""
+
+        analyzer_result = self.analyzer.analyze(
+            text=text,
+            entities=entities,
+            language=lang,
+        )
+        anonymizer_result = self.anonymizer.anonymize(text=text, analyzer_results=analyzer_result)
+        return anonymizer_result.text

datamarket-0.9.29/src/datamarket/utils/strings.py

@@ -1,152 +0,0 @@
-########################################################################################################################
-# IMPORTS
-
-from enum import Enum, auto
-from typing import Any
-import unicodedata
-
-import numpy as np
-from unidecode import unidecode
-from inflection import parameterize, underscore, titleize, camelize
-from string_utils import prettify, strip_html
-
-########################################################################################################################
-# CLASSES
-
-
-class NormalizationMode(Enum):
-    NONE = auto()
-    BASIC = auto()  # removes accents and converts punctuation to spaces
-    SYMBOLS = auto()  # translates only symbols to Unicode name
-    FULL = auto()  # BASIC + SYMBOLS
-
-
-class NamingConvention(Enum):
-    NONE = auto()  # no style change
-    CONSTANT = auto()  # CONSTANT_CASE (uppercase, underscores)
-    SNAKE = auto()  # snake_case (lowercase, underscores)
-    CAMEL = auto()  # camelCase (capitalize words except first one, no spaces)
-    PASCAL = auto()  # PascalCase (capitalize words including first one, no spaces)
-    PARAM = auto()  # parameterize (hyphens)
-    TITLE = auto()  # titleize (capitalize words)
-
-
-########################################################################################################################
-# FUNCTIONS
-
-
-def transliterate_symbols(s: str) -> str:
-    """
-    Translates Unicode symbols (category S*) in the input string to their lowercase Unicode names,
-    with spaces replaced by underscores. Other characters remain unchanged.
-
-    Args:
-        s: The input string.
-
-    Returns:
-        The string with symbols transliterated.
-    """
-    out: list[str] = []
-    for c in s:
-        if unicodedata.category(c).startswith("S"):
-            name = unicodedata.name(c, "")
-            if name:
-                out.append(name.lower().replace(" ", "_"))
-        else:
-            out.append(c)
-    return "".join(out)
-
-
-def normalize(
-    s: Any, mode: NormalizationMode = NormalizationMode.BASIC, naming: NamingConvention = NamingConvention.NONE
-) -> str:
-    """
-    Normalizes and applies a naming convention to the input.
-
-    Handles None and NaN values by returning an empty string. Converts non-string inputs to strings.
-
-    Normalization is applied according to `mode`:
-    - NONE: Returns the input as a string without any normalization.
-    - BASIC: Removes accents, converts punctuation and spaces to single spaces, and preserves alphanumeric characters.
-    - SYMBOLS: Translates only Unicode symbols (category S*) to their lowercase Unicode names with underscores.
-    - FULL: Applies both BASIC and SYMBOLS normalization.
-
-    After normalization, a naming convention is applied according to `naming`:
-    - NONE: Returns the normalized text.
-    - CONSTANT: Converts to CONSTANT_CASE (uppercase with underscores).
-    - SNAKE: Converts to snake_case (lowercase with underscores).
-    - CAMEL: Converts to camelCase (lowercase first word, capitalize subsequent words, no spaces).
-    - PASCAL: Converts to PascalCase (capitalize all words, no spaces).
-    - PARAM: Converts to parameterize (lowercase with hyphens).
-    - TITLE: Converts to Title Case (capitalize each word).
-
-    Args:
-        s: The input value to normalize and format. Can be any type.
-        mode: The normalization mode to apply. Defaults to NormalizationMode.BASIC.
-        naming: The naming convention to apply. Defaults to NamingConvention.NONE.
-
-    Returns:
-        The normalized and formatted string.
-    """
-    # Parameter mapping
-    if isinstance(mode, str):
-        mode = NormalizationMode[mode]
-    if isinstance(naming, str):
-        naming = NamingConvention[naming]
-
-    # Handling null values
-    if s is None or (isinstance(s, float) and np.isnan(s)):
-        normalized = ""
-    elif not isinstance(s, str):
-        return str(s)
-    else:
-        text = prettify(strip_html(str(s), True))
-        if mode is NormalizationMode.NONE:
-            normalized = text
-        elif mode is NormalizationMode.SYMBOLS:
-            normalized = transliterate_symbols(text)
-        else:
-            # BASIC and FULL: remove accents and lowercase
-            normalized = unidecode(text).lower()
-            tokens: list[str] = []
-            current: list[str] = []
-
-            def flush_current():
-                nonlocal current
-                if current:
-                    tokens.append("".join(current))
-                    current.clear()
-
-            for c in normalized:
-                cat = unicodedata.category(c)
-                if c.isalnum():
-                    current.append(c)
-                elif mode is NormalizationMode.FULL and cat.startswith("S"):
-                    flush_current()
-                    name = unicodedata.name(c, "")
-                    if name:
-                        tokens.append(name.lower().replace(" ", "_"))
-                elif cat.startswith("P") or c.isspace():
-                    flush_current()
-                # other characters ignored
-
-            flush_current()
-            normalized = " ".join(tokens)
-
-    # Apply naming convention
-    if naming is NamingConvention.NONE:
-        return normalized
-    if naming is NamingConvention.PARAM:
-        return parameterize(normalized)
-    if naming is NamingConvention.TITLE:
-        return titleize(normalized)
-
-    underscored = underscore(parameterize(normalized))
-    if naming is NamingConvention.CONSTANT:
-        return underscored.upper()
-    if naming is NamingConvention.CAMEL:
-        return camelize(underscored, False)
-    if naming is NamingConvention.PASCAL:
-        return camelize(underscored)
-
-    return underscored