datamarket 0.7.98__py3-none-any.whl → 0.7.100__py3-none-any.whl

datamarket/params/nominatim.py

@@ -412,7 +412,7 @@ COUNTRY_PARSING_RULES = {
         
         "zip_search_pattern": re.compile(r"\b\d{5}\b"), 
         
-        "phone_validate_pattern": re.compile(r"^(\+?34)?[6|7]\d{8}$")
+        "phone_validate_pattern": re.compile(r"^(\+?34)?[67]\d{8}$")
     },
     "pt": {
         "zip_validate_pattern": re.compile(r"^\d{4}[- ]{0,1}\d{3}$|^\d{4}$"),

datamarket/utils/playwright/sync_api.py

@@ -53,12 +53,14 @@ def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, add_
 class PlaywrightCrawler:
     """A robust, proxy-enabled Playwright crawler with captcha bypass and retry logic."""
 
-    def __init__(self, proxy_interface: ProxyInterface):
+    def __init__(self, proxy_interface: Optional[ProxyInterface] = None):
         """
-        Initializes the crawler with a proxy interface.
+        Initializes the crawler.
 
         Args:
-            proxy_interface (ProxyInterface): An object to fetch proxy credentials.
+            proxy_interface (Optional[ProxyInterface], optional): Provider used to fetch
+                proxy credentials. Defaults to None. When None, no proxy is configured and
+                the browser will run without a proxy.
         """
         self.proxy_interface = proxy_interface
         self.pw: Optional[Camoufox] = None
@@ -81,6 +83,25 @@ class PlaywrightCrawler:
         if self.pw:
             self.pw.__exit__(exc_type, exc_val, exc_tb)
 
+    def _build_proxy_config(self) -> Optional[dict]:
+        """Builds the proxy configuration dictionary.
+
+        Returns:
+            Optional[dict]: Proxy configuration if a proxy_interface is provided; otherwise None.
+        """
+        if not self.proxy_interface:
+            logger.info("Starting browser without proxy.")
+            return None
+
+        host, port, user, pwd = self.proxy_interface.get_proxies(raw=True, use_auth=True)
+        proxy_url = f"http://{host}:{port}"
+        proxy_cfg: dict = {"server": proxy_url}
+        if user and pwd:
+            proxy_cfg.update({"username": user, "password": pwd})
+
+        logger.info(f"Starting browser with proxy: {proxy_url}")
+        return proxy_cfg
+
     @retry(
         wait=wait_exponential(exp_base=2, multiplier=3, max=90),
         stop=stop_after_delay(timedelta(minutes=10)),
@@ -88,16 +109,20 @@ class PlaywrightCrawler:
         reraise=True,
     )
     def init_context(self) -> Self:
-        """Initializes a new browser instance and context with a fresh proxy."""
-        try:
-            host, port, user, pwd = self.proxy_interface.get_proxies(raw=True, use_auth=True)
-            proxy_url = f"http://{host}:{port}"
-            proxy_cfg = {"server": proxy_url}
+        """
+        Initializes a new browser instance and context.
+
+        Behavior:
+        - If a proxy_interface is provided, fetches fresh proxy credentials and starts
+          the browser using that proxy.
+        - If proxy_interface is None, starts the browser without any proxy.
 
-            if user and pwd:
-                proxy_cfg.update({"username": user, "password": pwd})
+        Returns:
+            Self: The crawler instance with active browser, context, and page.
+        """
+        try:
+            proxy_cfg: Optional[dict] = self._build_proxy_config()
 
-            logger.info(f"Starting browser with proxy: {proxy_url}")
             self.pw = Camoufox(headless=True, geoip=True, humanize=True, proxy=proxy_cfg)
             self.browser = self.pw.__enter__()
             self.context = self.browser.new_context()
@@ -146,4 +171,4 @@ class PlaywrightCrawler:
         if not self.page:
             logger.info("Browser context not found, initializing now...")
             self.init_context()
-        return self._goto_with_retry(url)
+        return self._goto_with_retry(url)

datamarket/utils/strings/standardization.py

@@ -8,45 +8,6 @@ from ...params.nominatim import COUNTRY_PARSING_RULES
 ########################################################################################################################
 # FUNCTIONS
 
-
-def _standardize_es_phone_number(number: str) -> str | None:
-    """Standardize phone numbers from Spain using regex validation.
-
-    Args:
-        number (str): cleaned, digits-only phone number
-
-    Returns:
-        str | None: standardized 9-digit phone number
-    """
-    # Get the validation regex from params
-    pattern = COUNTRY_PARSING_RULES["es"]["phone_validate_pattern"]
-
-    # Validate and extract in one step
-    match = pattern.match(number)
-
-    # Return the captured group (the 9-digit number)
-    return match.group(1) if match else None
-
-
-def _standardize_pt_phone_number(number: str) -> str | None:
-    """Standardize phone numbers from Portugal using regex validation.
-
-    Args:
-        number (str): cleaned, digits-only phone number
-
-    Returns:
-        str | None: standardized 9-digit phone number
-    """
-    # Get the validation regex from params
-    pattern = COUNTRY_PARSING_RULES["pt"]["phone_validate_pattern"]
-
-    # Validate and extract in one step
-    match = pattern.match(number)
-
-    # Return the captured group (the 9-digit number)
-    return match.group(1) if match else None
-
-
 def parse_phone_number(number: str, country_code: Literal["es", "pt"]) -> str | None:
     """Clean and standardize phone number from a certain country_code
 
@@ -61,9 +22,17 @@ def parse_phone_number(number: str, country_code: Literal["es", "pt"]) -> str |
         str | None: standardized phone number
     """
     clean_number = re.sub(r"\D", "", number)
-    if country_code == "es":
-        return _standardize_es_phone_number(clean_number)
-    elif country_code == "pt":
-        return _standardize_pt_phone_number(clean_number)
+    if country_code in {"es", "pt"}:
+        # Get the validation regex from params
+        pattern = COUNTRY_PARSING_RULES[country_code]["phone_validate_pattern"]
+
+        # Validate and extract in one step
+        if len(clean_number) >= 9: # Check if the cleaned number has at least 9 digits
+            match = pattern.match(clean_number)
+
+            # Return the captured group (the 9-digit number)
+            return match.group(0)[-9:] if match else None
+        else:
+            return None # Or handle the case where the number is too short
     else:
         raise ValueError(f"Country code ({country_code}) is not currently supported")

{datamarket-0.7.98.dist-info → datamarket-0.7.100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.98
+Version: 0.7.100
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.98.dist-info → datamarket-0.7.100.dist-info}/RECORD

@@ -12,7 +12,7 @@ datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspX
 datamarket/interfaces/proxy.py,sha256=Uu-dHvpQOLNBZPGHAanLXnKT1789ArcHfOw8exECt34,5398
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/params/nominatim.py,sha256=Xl0mBls_Yz7y8nU6-boNTMDYsxvJRIFpZyMT2gLOqvs,11968
+datamarket/params/nominatim.py,sha256=RnmYXGoJQCijOsuCavCYcxw98WvOd_vOMK4KaraI0RU,11967
 datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
@@ -20,16 +20,16 @@ datamarket/utils/main.py,sha256=KYHjDOps6_Q3TFV_Jj7MLj-L9Evx05AXELCvp06BARU,5857
 datamarket/utils/nominatim.py,sha256=IxexKY2KOlDhiKtzsqQfoVUjJXPxJl7tn3iHUaQKg08,5795
 datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/utils/playwright/async_api.py,sha256=UbA2D4ScBtYeMfrRjly4RO-s8wXIub9c05J1eoOCpsQ,5782
-datamarket/utils/playwright/sync_api.py,sha256=Tw_-KLB3vipFuEQwcX8iCbj7giCzcwXB-bhl_ncR-2Q,5542
+datamarket/utils/playwright/sync_api.py,sha256=eXaZsd7xgWSYJtZv6EAstjSbS2bl9OYlkwMBfqqTbFY,6434
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/strings/__init__.py,sha256=b6TYOT9v7y9ID-lDyZk4E8BH2uIPbsF2ZSLGjCQ1MCQ,43
 datamarket/utils/strings/normalization.py,sha256=UBluU6ABY6aCpnd02F7L7HcivVSisRJ9IUXdj9D1MyE,9050
 datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnmH3FPlgUUjM,5246
-datamarket/utils/strings/standardization.py,sha256=xl4I6F3brDFdRWKy7jKnOIIEo2YsqcBcPa2p5TJGRC0,2236
+datamarket/utils/strings/standardization.py,sha256=c8CAG6HI3AfK0hB3A3IGwsbnQebZ6R3PrA5PELHRXM0,1492
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.98.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.98.dist-info/METADATA,sha256=f8YsBdGrtjmUOroA-qvv1bEpnt1LSfrCxqt8z6kF_tk,7381
-datamarket-0.7.98.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.98.dist-info/RECORD,,
+datamarket-0.7.100.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.100.dist-info/METADATA,sha256=ZzGfCV51bIyPYJVdCSfJDdX8YuC9_BjKR1VCoRtd6yI,7382
+datamarket-0.7.100.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.100.dist-info/RECORD,,