datamarket 0.9.29__py3-none-any.whl → 0.9.30__py3-none-any.whl

datamarket/utils/strings/__init__.py

@@ -0,0 +1,2 @@
+from .normalization import *  # noqa: F403
+from .obfuscation import *  # noqa: F403

datamarket/utils/{strings.py → strings/normalization.py}

@@ -1,14 +1,14 @@
 ########################################################################################################################
 # IMPORTS
 
+import unicodedata
 from enum import Enum, auto
 from typing import Any
-import unicodedata
 
 import numpy as np
-from unidecode import unidecode
-from inflection import parameterize, underscore, titleize, camelize
+from inflection import camelize, parameterize, titleize, underscore
 from string_utils import prettify, strip_html
+from unidecode import unidecode
 
 ########################################################################################################################
 # CLASSES

datamarket/utils/strings/obfuscation.py

@@ -0,0 +1,153 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+import warnings
+from typing import List, Optional
+
+
+class PiiDependenciesMissingError(ImportError):
+    pass
+
+
+class SpacyModelNotFoundError(ImportError):
+    pass
+
+
+try:
+    import phonenumbers
+    import spacy
+    from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
+    from presidio_analyzer.nlp_engine import NlpEngineProvider
+    from presidio_analyzer.predefined_recognizers import PhoneRecognizer
+    from presidio_anonymizer import AnonymizerEngine
+    from spacy.language import Language
+    from spacy_langdetect import LanguageDetector
+except ImportError as e:
+    raise PiiDependenciesMissingError(
+        "One or more PII anonymization dependencies are missing. "
+        "Please install them by running: pip install datamarket[pii]\n"
+        f"Original error: {e}"
+    ) from e
+
+
+########################################################################################################################
+# SETTINGS
+
+logger = logging.getLogger()
+logging.getLogger("presidio-analyzer").setLevel(logging.ERROR)
+
+warnings.filterwarnings(
+    "ignore",
+    message=r"\[W108\]",
+    category=UserWarning,
+    module="spacy.pipeline.lemmatizer",
+)
+
+
+@Language.factory("language_detector")
+def get_lang_detector(nlp, name):
+    return LanguageDetector()
+
+
+########################################################################################################################
+# CLASSES
+
+
+class PiiAnonymizer:
+    SUPPORTED_LANG = ["es", "en"]
+
+    def __init__(self):
+        # Check for required spaCy models
+        required_models = {
+            "en_core_web_md": "python -m spacy download en_core_web_md",
+            "es_core_news_md": "python -m spacy download es_core_news_md",
+        }
+        missing_models_instructions = []
+        for model_name, install_command in required_models.items():
+            if not spacy.util.is_package(model_name):
+                missing_models_instructions.append(
+                    f"Model '{model_name}' not found. Please install it by running: {install_command}"
+                )
+
+        if missing_models_instructions:
+            raise SpacyModelNotFoundError("\n".join(missing_models_instructions))
+
+        self.anonymizer = AnonymizerEngine()
+        self.analyzer = self._load_analyzer_engine()
+
+        self.nlp = self._nlp()
+
+    def _nlp(self) -> Language:
+        analyzer_en_model = self.analyzer.nlp_engine.nlp.get("en")
+        shared_vocab = analyzer_en_model.vocab
+        nlp = spacy.blank("en", vocab=shared_vocab)
+
+        if nlp.has_factory("sentencizer"):
+            nlp.add_pipe("sentencizer")
+
+        if nlp.has_factory("language_detector"):
+            nlp.add_pipe("language_detector", last=True)
+
+        return nlp
+
+    @staticmethod
+    def _nlp_config():
+        return {
+            "nlp_engine_name": "spacy",
+            "models": [
+                {"lang_code": "es", "model_name": "es_core_news_md"},
+                {"lang_code": "en", "model_name": "en_core_web_md"},
+            ],
+        }
+
+    def _load_analyzer_engine(self) -> AnalyzerEngine:
+        provider = NlpEngineProvider(nlp_configuration=PiiAnonymizer._nlp_config())
+        nlp_engine = provider.create_engine()
+        phone_recognizer_es = PhoneRecognizer(
+            supported_language="es",
+            supported_regions=phonenumbers.SUPPORTED_REGIONS,
+            context=["teléfono", "móvil", "número"],
+        )
+        registry = RecognizerRegistry(supported_languages=self.SUPPORTED_LANG)
+        registry.load_predefined_recognizers(nlp_engine=nlp_engine, languages=self.SUPPORTED_LANG)
+        registry.add_recognizer(phone_recognizer_es)
+
+        analyzer = AnalyzerEngine(
+            registry=registry,
+            nlp_engine=nlp_engine,
+            supported_languages=self.SUPPORTED_LANG,
+        )
+        return analyzer
+
+    def detect_lang(self, text: str) -> str:
+        if hasattr(self, "nlp") and self.nlp:
+            with self.nlp.select_pipes(enable=["tokenizer", "sentencizer", "language_detector"]):
+                doc = self.nlp(text)
+            return doc._.language["language"]
+        else:
+            logger.error("Language detection NLP model not initialized. Cannot detect language.")
+            return "unknown"
+
+    def anonymize_text(
+        self,
+        text: str,
+        entities: Optional[List[str]] = None,
+        lang: str = "unknown",
+    ) -> str:
+        if lang == "unknown":
+            lang = self.detect_lang(text)
+            if lang not in self.SUPPORTED_LANG:
+                logger.warning(f"Support for language {lang} is not implemented yet! Fail safe to empty string.")
+                return ""
+        elif lang not in self.SUPPORTED_LANG:
+            logger.warning(f"Support for language {lang} is not implemented yet! Fail safe to empty string.")
+            return ""
+
+        analyzer_result = self.analyzer.analyze(
+            text=text,
+            entities=entities,
+            language=lang,
+        )
+        anonymizer_result = self.anonymizer.anonymize(text=text, analyzer_results=analyzer_result)
+        return anonymizer_result.text

{datamarket-0.9.29.dist-info → datamarket-0.9.30.dist-info}/METADATA

@@ -1,17 +1,17 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: datamarket
-Version: 0.9.29
+Version: 0.9.30
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
+Home-page: https://datamarket.es
 License: GPL-3.0-or-later
 Author: DataMarket
 Author-email: techsupport@datamarket.es
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.12,<3.13
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: alchemy
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
@@ -40,6 +40,7 @@ Provides-Extra: openpyxl
 Provides-Extra: pandas
 Provides-Extra: pandera
 Provides-Extra: peerdb
+Provides-Extra: pii
 Provides-Extra: pillow
 Provides-Extra: playwright
 Provides-Extra: playwright-stealth
@@ -96,6 +97,8 @@ Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
 Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
 Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
+Requires-Dist: presidio-analyzer[phonenumbers] (>=2.0.0,<3.0.0) ; extra == "pii"
+Requires-Dist: presidio-anonymizer (>=2.0.0,<3.0.0) ; extra == "pii"
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
 Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
 Requires-Dist: pycountry (>=24.0.0,<25.0.0)
@@ -111,14 +114,16 @@ Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
 Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
 Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
+Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
+Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
 Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
 Requires-Dist: tenacity (>=9.0.0,<10.0.0)
+Requires-Dist: tf-playwright-stealth (>=1.0.0,<2.0.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
 Requires-Dist: typer (>=0.15.0,<0.16.0)
 Requires-Dist: unidecode (>=1.0.0,<2.0.0)
 Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
 Project-URL: Documentation, https://github.com/Data-Market/datamarket
-Project-URL: Homepage, https://datamarket.es
 Project-URL: Repository, https://github.com/Data-Market/datamarket
 Description-Content-Type: text/markdown
 

{datamarket-0.9.29.dist-info → datamarket-0.9.30.dist-info}/RECORD

@@ -16,10 +16,12 @@ datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,6
 datamarket/utils/main.py,sha256=j8wnAxeLvijdRU9M4V6HunWH7vgWWHP4u4xamzkWcUU,7009
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
-datamarket/utils/strings.py,sha256=rEX9NeBG4C7RECgT0EQebgoFoxgZMy9-7EcBSxgBANU,5654
+datamarket/utils/strings/__init__.py,sha256=RmyN3hKGXmUym8w5tn28yWkw2uM-b5OvntB4D0lU1eo,84
+datamarket/utils/strings/normalization.py,sha256=337M2UPwEETvhVTOnP4w_igTXpHUHoaD8e7x_-L-Bpk,5654
+datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.29.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.29.dist-info/METADATA,sha256=h4DuPT0ToLAN6vSLidYyriB9gtKjjaDPcf2MWH5fm44,6588
-datamarket-0.9.29.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.9.29.dist-info/RECORD,,
+datamarket-0.9.30.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.30.dist-info/METADATA,sha256=zzhHMrHhBf_CfBLwjj4melul8sCkcO8np-nmay0jKOQ,6871
+datamarket-0.9.30.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+datamarket-0.9.30.dist-info/RECORD,,

{datamarket-0.9.29.dist-info → datamarket-0.9.30.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any