datamarket 0.7.103__py3-none-any.whl → 0.7.104__py3-none-any.whl

datamarket/exceptions/__init__.py

@@ -1 +1 @@
-from .main import *  # noqa: F403
+from .main import *  # noqa: F403

datamarket/exceptions/main.py

@@ -21,9 +21,11 @@ class BadRequestError(Exception):
     def __init__(self, message="Bad request!"):
         self.message = message
         super().__init__(self.message)
-        
+
+
 class ManagedHTTPError(Exception):
     """Signal that this HTTP status was handled and should not be retried."""
+
     def __init__(self, response: requests.Response, *, url: str | None = None, message: str | None = None):
         self.response = response
         self.request = getattr(response, "request", None)
@@ -31,3 +33,15 @@ class ManagedHTTPError(Exception):
         self.url = url or (self.request.url if self.request is not None else None)
         self.message = message
         super().__init__(message or f"HTTP {self.status_code} for {self.url}")
+
+
+class NoWorkingProxiesError(Exception):
+    def __init__(self, message="No working proxies available"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class EnsureNewIPTimeoutError(Exception):
+    def __init__(self, message="Timed out waiting for new IP"):
+        self.message = message
+        super().__init__(self.message)

datamarket/interfaces/alchemy.py

@@ -3,6 +3,7 @@
 
 import logging
 from collections.abc import MutableMapping
+from enum import Enum, auto
 from typing import Any, Iterator, List, Optional, Type, TypeVar, Union
 from urllib.parse import quote_plus
 
@@ -12,7 +13,6 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import DeclarativeMeta
 from sqlalchemy.orm import Session, sessionmaker
 from sqlalchemy.sql.expression import ClauseElement
-from enum import Enum, auto
 
 ########################################################################################################################
 # CLASSES
@@ -198,29 +198,28 @@ class AlchemyInterface:
     @staticmethod
     def _log_integrity_error(ex: IntegrityError, alchemy_obj, action="insert"):
         """
-            Compact, readable IntegrityError logger using SQLSTATE codes.
-            Consult https://www.postgresql.org/docs/current/errcodes-appendix.html for details.
+        Compact, readable IntegrityError logger using SQLSTATE codes.
+        Consult https://www.postgresql.org/docs/current/errcodes-appendix.html for details.
         """
 
         PG_ERROR_LABELS = {
-        "23000": "Integrity constraint violation",
-        "23001": "Restrict violation",
-        "23502": "NOT NULL violation",
-        "23503": "Foreign key violation",
-        "23505": "Unique violation",
-        "23514": "Check constraint violation",
-        "23P01": "Exclusion constraint violation",
+            "23000": "Integrity constraint violation",
+            "23001": "Restrict violation",
+            "23502": "NOT NULL violation",
+            "23503": "Foreign key violation",
+            "23505": "Unique violation",
+            "23514": "Check constraint violation",
+            "23P01": "Exclusion constraint violation",
         }
         code = getattr(ex.orig, "pgcode", None)
         label = PG_ERROR_LABELS.get(code, "Integrity error (unspecified)")
 
         # Log one clean message with trace + the raw DB message separately
-        if code == "23505": # A simple info log for unique violations
+        if code == "23505":  # A simple info log for unique violations
             logger.info(f"{label} trying to {action} {alchemy_obj}")
         else:
             logger.error(f"{label} trying to {action} {alchemy_obj}\nPostgreSQL message: {ex.orig}")
 
-
     def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")

datamarket/interfaces/aws.py

@@ -3,8 +3,9 @@
 
 import io
 import logging
+from typing import Any, Dict, List, Optional
+
 import boto3
-from typing import Optional, List, Dict, Any
 
 ########################################################################################################################
 # CLASSES

datamarket/interfaces/azure.py

@@ -29,17 +29,15 @@ class AzureBlobInterface:
                     {
                         "profile": profile_name,
                         "container_name": container_name,
-                        "session": BlobServiceClient.from_connection_string(
-                            connection_string
-                        ).get_container_client(container_name),
+                        "session": BlobServiceClient.from_connection_string(connection_string).get_container_client(
+                            container_name
+                        ),
                     }
                 )
 
         if not self.profiles:
             logger.warning("No Azure profiles found in config file")
-        self.current_profile: Optional[Dict[str, Any]] = (
-            self.profiles[0] if self.profiles else None
-        )
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
 
     def switch_profile(self, profile_name: str) -> None:
         for profile in self.profiles:
@@ -109,14 +107,10 @@ class AzureBlobInterface:
             if blob_client.exists():
                 properties = blob_client.get_blob_properties()
                 if properties.size > 100:  # Check if size is greater than 100 bytes
-                    logger.debug(
-                        f"Blob '{remote_path}' exists and is not empty (size: {properties.size})."
-                    )
+                    logger.debug(f"Blob '{remote_path}' exists and is not empty (size: {properties.size}).")
                     return True
                 else:
-                    logger.debug(
-                        f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes."
-                    )
+                    logger.debug(f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes.")
                     return False
             else:
                 logger.debug(f"Blob '{remote_path}' does not exist.")

datamarket/interfaces/ftp.py

@@ -20,22 +20,14 @@ class FTPInterface:
             if section.startswith("ftp:"):
                 profile_name = section.split(":", 1)[1]
                 ftps = self.config[section]["ftps"].lower() == "true"
-                ftp_conn = (
-                    FTP_TLS(self.config[section]["server"])
-                    if ftps
-                    else FTP(self.config[section]["server"])
-                )  # noqa: S321
-                ftp_conn.login(
-                    self.config[section]["username"], self.config[section]["password"]
-                )
+                ftp_conn = FTP_TLS(self.config[section]["server"]) if ftps else FTP(self.config[section]["server"])  # noqa: S321
+                ftp_conn.login(self.config[section]["username"], self.config[section]["password"])
                 self.profiles.append({"profile": profile_name, "session": ftp_conn})
 
         if not self.profiles:
             logger.warning("no ftp section in config")
 
-        self.current_profile: Optional[Dict[str, Any]] = (
-            self.profiles[0] if self.profiles else None
-        )
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
         self.ftp = self.current_profile["session"] if self.current_profile else None
 
     def switch_profile(self, profile_name: str) -> None:

datamarket/interfaces/nominatim.py

@@ -11,8 +11,8 @@ from geopy.distance import geodesic
 from jellyfish import jaro_winkler_similarity
 
 from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES
-from ..utils.strings import normalize
 from ..utils.nominatim import standardize_admin_division
+from ..utils.strings import normalize
 
 ########################################################################################################################
 # PARAMETERS
@@ -335,16 +335,14 @@ class Nominatim:
             selected_province_from_postcode,
             selected_state,
         )
-        
+
         # Standardize
         final_result["province"] = standardize_admin_division(
-            name=final_result["province"], 
-            level="province",
-            country_code=final_result["country_code"])
+            name=final_result["province"], level="province", country_code=final_result["country_code"]
+        )
         final_result["state"] = standardize_admin_division(
-            name=final_result["state"], 
-            level="state",
-            country_code=final_result["country_code"])
+            name=final_result["state"], level="state", country_code=final_result["country_code"]
+        )
         return final_result
 
 
@@ -358,4 +356,4 @@ class NominatimInterface(Nominatim):
 
             super().__init__(self.nominatim_endpoint, self.geonames_endpoint)
         else:
-            logger.warning("no osm section in config")
+            logger.warning("no osm section in config")

datamarket/interfaces/proxy.py

@@ -1,10 +1,14 @@
 import logging
-import time
 import random
+import time
+
 import requests
+import tenacity
 from stem import Signal
 from stem.control import Controller
 
+from datamarket.exceptions import EnsureNewIPTimeoutError, NoWorkingProxiesError
+
 logger = logging.getLogger(__name__)
 logging.getLogger("stem").setLevel(logging.WARNING)
 
@@ -18,7 +22,7 @@ class ProxyInterface:
 
     def __init__(self, config):
         self._load_from_config(config)
-        self.current_index = random.randrange(len(self.entries)) if self.entries else 0
+        self.current_index = random.randrange(len(self.entries)) if self.entries else 0  # noqa: S311
         self._health = {}  # {entry: {"ok": bool, "last_checked": time.time(), "last_error": str}}
 
     def _load_from_config(self, cfg):
@@ -66,6 +70,10 @@ class ProxyInterface:
         health_check=True,
         check_timeout=5,
         cooldown_seconds=600,
+        ensure_new_ip=False,
+        ensure_new_ip_timeout=600,
+        ensure_new_ip_interval=5,
+        max_retry_seconds=600,
     ):
         """
         Return parsed proxy URLs or raw entry tuple for a working proxy.
@@ -77,6 +85,10 @@ class ProxyInterface:
         :param health_check: perform health checks to ensure proxy is working if True
         :param check_timeout: timeout in seconds for health check requests
         :param cooldown_seconds: how long to cache health status before re-checking
+        :param ensure_new_ip: if True and only one proxy available, wait until IP changes before returning
+        :param ensure_new_ip_timeout: max seconds to wait for IP change when ensure_new_ip=True
+        :param ensure_new_ip_interval: seconds between IP checks when ensure_new_ip=True
+        :param max_retry_seconds: max seconds to retry finding working proxies (0 to disable)
         """
         # Tor handling (skip health check for tor)
         if use_tor:
@@ -87,7 +99,14 @@ class ProxyInterface:
         # Get a working entry (with health checks if enabled)
         if health_check:
             host, port, user, password = self._get_working_entry(
-                use_auth=use_auth, randomize=randomize, check_timeout=check_timeout, cooldown_seconds=cooldown_seconds
+                use_auth=use_auth,
+                randomize=randomize,
+                check_timeout=check_timeout,
+                cooldown_seconds=cooldown_seconds,
+                ensure_new_ip=ensure_new_ip,
+                ensure_new_ip_timeout=ensure_new_ip_timeout,
+                ensure_new_ip_interval=ensure_new_ip_interval,
+                max_retry_seconds=max_retry_seconds,
             )
         else:
             # Legacy behavior: no health check
@@ -110,7 +129,7 @@ class ProxyInterface:
     def get_next(self, use_auth=False):
         # Round-robin selection, optionally filtering out authenticated proxies
         if not self.entries:
-            raise RuntimeError("No proxies available")
+            raise NoWorkingProxiesError("No proxies available")
 
         pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
         if not pool:
@@ -130,13 +149,13 @@ class ProxyInterface:
     def get_random(self, use_auth=False):
         # Random selection, optionally filtering out authenticated proxies
         if not self.entries:
-            raise RuntimeError("No proxies available")
+            raise NoWorkingProxiesError("No proxies available")
 
         pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
         if not pool:
             pool = self.entries
 
-        entry = random.choice(pool)
+        entry = random.choice(pool)  # noqa: S311
         # Update index to after selected entry for round-robin continuity
         try:
             pos = self.entries.index(entry)
@@ -146,9 +165,10 @@ class ProxyInterface:
 
         return entry
 
-    def check_current_ip(self):
+    def check_current_ip(self, proxies=None):
         try:
-            resp = requests.get(self.CHECK_IP_URL, proxies={"http": self.proxies["http"]})
+            proxies_arg = proxies or {"http": self.proxies["http"]}
+            resp = requests.get(self.CHECK_IP_URL, proxies=proxies_arg, timeout=30)
             return resp.json().get("YourFuckingIPAddress")
         except Exception as ex:
             logger.error(ex)
@@ -170,12 +190,13 @@ class ProxyInterface:
             logger.error("Failed to renew Tor IP")
             logger.error(ex)
 
-    def mark_entry_status(self, entry, ok, error=None):
+    def mark_entry_status(self, entry, ok, error=None, last_ip=None):
         """Update health cache for an entry."""
         self._health[entry] = {
             "ok": ok,
             "last_checked": time.time(),
             "last_error": error,
+            "last_ip": last_ip,
         }
 
     def is_entry_alive(self, entry, timeout=5):
@@ -188,25 +209,84 @@ class ProxyInterface:
             }
             resp = requests.get(self.CHECK_IP_URL, proxies=proxies, timeout=timeout)
             ok = resp.status_code == 200
-            self.mark_entry_status(entry, ok)
+            last_ip = resp.json().get("YourFuckingIPAddress") if ok else None
+            self.mark_entry_status(entry, ok, last_ip=last_ip)
             return ok
         except Exception as ex:
             self.mark_entry_status(entry, False, str(ex))
             return False
 
-    def _get_working_entry(self, use_auth=False, randomize=False, check_timeout=5, cooldown_seconds=60):
+    def _get_working_entry(
+        self,
+        use_auth=False,
+        randomize=False,
+        check_timeout=5,
+        cooldown_seconds=60,
+        ensure_new_ip=False,
+        ensure_new_ip_timeout=600,
+        ensure_new_ip_interval=5,
+        max_retry_seconds=600,
+    ):
         """Get a working proxy entry, performing health checks as needed."""
-        if not self.entries:
-            raise RuntimeError("No proxies available")
+        pool = self._build_pool(use_auth)
+        candidates = self._get_candidates(pool, randomize)
+
+        def _find_working_entry():
+            if not self.entries:
+                raise NoWorkingProxiesError("No proxies available")
+            return self._find_working_entry_once(candidates, check_timeout, cooldown_seconds)
+
+        if max_retry_seconds > 0:
+            retrying = tenacity.Retrying(
+                stop=tenacity.stop_after_delay(max_retry_seconds),
+                reraise=True,
+            )
+            entry = retrying(_find_working_entry)
+        else:
+            entry = _find_working_entry()
+
+        if ensure_new_ip and len(pool) == 1:
+            logger.info(f"ensure_new_ip=True and single proxy, waiting for IP change: {entry[0]}:{entry[1]}")
+            baseline = self._health.get(entry, {}).get("last_ip")
+            if not baseline:
+                if not self.is_entry_alive(entry, timeout=check_timeout):
+                    raise NoWorkingProxiesError("Proxy became unavailable during ensure_new_ip")
+                baseline = self._health.get(entry, {}).get("last_ip")
+            entry = self._wait_for_new_ip(entry, baseline, ensure_new_ip_timeout, ensure_new_ip_interval, check_timeout)
+
+        return entry
+
+    def _get_round_robin_candidates(self, pool):
+        """Get candidates in round-robin order starting from current_index."""
+        candidates = []
+        start_idx = self.current_index
+        for i in range(len(self.entries)):
+            idx = (start_idx + i) % len(self.entries)
+            entry = self.entries[idx]
+            if entry in pool:
+                candidates.append(entry)
+        # Update current_index for next call
+        if candidates:
+            try:
+                pos = self.entries.index(candidates[0])
+                self.current_index = (pos + 1) % len(self.entries)
+            except ValueError:
+                pass
+        return candidates
 
-        # Build candidate list respecting use_auth and randomize/round-robin
+    def _build_pool(self, use_auth):
         pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
         if not pool:
             pool = self.entries
+        return pool
 
-        candidates = pool[:] if randomize else self._get_round_robin_candidates(pool)
+    def _get_candidates(self, pool, randomize):
+        if randomize:
+            return pool[:]
+        else:
+            return self._get_round_robin_candidates(pool)
 
-        # First pass: check cache and health
+    def _find_working_entry_once(self, candidates, check_timeout, cooldown_seconds):
         for entry in candidates:
             health = self._health.get(entry, {})
             last_checked = health.get("last_checked", 0)
@@ -214,42 +294,42 @@ class ProxyInterface:
             now = time.time()
 
             if ok and (now - last_checked) < cooldown_seconds:
-                # Cached as working and recent
                 logger.debug(f"Using cached working proxy: {entry[0]}:{entry[1]}")
                 return entry
             elif not ok and (now - last_checked) < cooldown_seconds:
-                # Cached as failed and recent, skip
                 continue
             else:
-                # Not cached or expired, check now
                 logger.debug(f"Checking proxy health: {entry[0]}:{entry[1]}")
                 if self.is_entry_alive(entry, timeout=check_timeout):
                     return entry
 
-        # Second pass: force fresh check for all candidates (in case cache skipped everything)
         logger.warning("No cached working proxies, forcing fresh checks")
         for entry in candidates:
             logger.debug(f"Force-checking proxy: {entry[0]}:{entry[1]}")
             if self.is_entry_alive(entry, timeout=check_timeout):
                 return entry
 
-        # No working proxies found
-        raise RuntimeError("No working proxies available")
+        raise NoWorkingProxiesError("No working proxies available")
 
-    def _get_round_robin_candidates(self, pool):
-        """Get candidates in round-robin order starting from current_index."""
-        candidates = []
-        start_idx = self.current_index
-        for i in range(len(self.entries)):
-            idx = (start_idx + i) % len(self.entries)
-            entry = self.entries[idx]
-            if entry in pool:
-                candidates.append(entry)
-        # Update current_index for next call
-        if candidates:
+    def _wait_for_new_ip(self, entry, baseline, timeout, interval, check_timeout):
+        start = time.time()
+        while time.time() - start < timeout:
+            host, port, user, pwd = entry
+            proxies_map = {
+                "http": self.get_proxy_url(host, port, user, pwd, "http"),
+                "https": self.get_proxy_url(host, port, user, pwd, "http"),
+            }
             try:
-                pos = self.entries.index(candidates[0])
-                self.current_index = (pos + 1) % len(self.entries)
-            except ValueError:
-                pass
-        return candidates
+                resp = requests.get(self.CHECK_IP_URL, proxies=proxies_map, timeout=check_timeout)
+                current_ip = resp.json().get("YourFuckingIPAddress")
+            except Exception:
+                current_ip = None
+
+            if current_ip and current_ip != baseline:
+                self.mark_entry_status(entry, True, last_ip=current_ip)
+                logger.info(f"IP changed from {baseline} to {current_ip}")
+                return entry
+
+            time.sleep(interval)
+
+        raise EnsureNewIPTimeoutError(f"Timed out waiting for new IP after {timeout}s")

datamarket/interfaces/tinybird.py

@@ -43,9 +43,7 @@ class TinybirdInterface:
         }
 
     def __prepare_json_row(self, obj_dict):
-        return json.dumps(
-            self.__dict_lists_to_string(obj_dict), default=self.__converter
-        )
+        return json.dumps(self.__dict_lists_to_string(obj_dict), default=self.__converter)
 
     @staticmethod
     def __handle_api_response(json_response):
@@ -53,13 +51,9 @@ class TinybirdInterface:
         quarantined_rows = json_response["quarantined_rows"]
 
         if quarantined_rows > 0:
-            logger.error(
-                f"wrong insertion of {quarantined_rows} records to Tinybird API..."
-            )
+            logger.error(f"wrong insertion of {quarantined_rows} records to Tinybird API...")
         else:
-            logger.info(
-                f"successfully inserted {successful_rows} records to Tinybird API!"
-            )
+            logger.info(f"successfully inserted {successful_rows} records to Tinybird API!")
 
         return successful_rows, quarantined_rows
 
@@ -72,9 +66,7 @@ class TinybirdInterface:
         return self.__insert_data_to_endpoint(self.__prepare_json_row(obj_dict))
 
     def insert_batch_to_api(self, batch):
-        return self.__insert_data_to_endpoint(
-            "\n".join([self.__prepare_json_row(x) for x in batch])
-        )
+        return self.__insert_data_to_endpoint("\n".join([self.__prepare_json_row(x) for x in batch]))
 
     def insert_pandas_df_to_api(self, df):
         return self.__insert_data_to_endpoint(df.to_json(orient="records", lines=True))

datamarket/params/nominatim.py

@@ -1,6 +1,7 @@
-from unidecode import unidecode
 import re
 
+from unidecode import unidecode
+
 CITY_TO_PROVINCE = {"Madrid": "Madrid"}
 
 POSTCODES = {
@@ -396,31 +397,27 @@ _NORMALIZED_PROVINCE_CACHE = {}
 for country, provinces in PROVINCE_TO_POSTCODE.items():
     # Get the original keys (e.g., "A Coruña", "Álava")
     original_keys = list(provinces.keys())
-    
+
     # Create the normalized list (e.g., "a coruna", "alava")
     normalized_choices = [unidecode(p).lower() for p in original_keys]
-    
+
     _NORMALIZED_PROVINCE_CACHE[country] = {
-        "choices": normalized_choices, # The list for rapidfuzz to search in
-        "keys": original_keys          # The list to find the name by index
+        "choices": normalized_choices,  # The list for rapidfuzz to search in
+        "keys": original_keys,  # The list to find the name by index
     }
 
 # Source: https://github.com/ariankoochak/regex-patterns-of-all-countries
 COUNTRY_PARSING_RULES = {
     "es": {
         "zip_validate_pattern": re.compile(r"^\d{5}$"),
-        
-        "zip_search_pattern": re.compile(r"\b\d{5}\b"), 
-        
-        "phone_validate_pattern": re.compile(r"^(\+?34)?[67]\d{8}$")
+        "zip_search_pattern": re.compile(r"\b\d{5}\b"),
+        "phone_validate_pattern": re.compile(r"^(\+?34)?[67]\d{8}$"),
     },
     "pt": {
         "zip_validate_pattern": re.compile(r"^\d{4}[- ]{0,1}\d{3}$|^\d{4}$"),
-        
         "zip_search_pattern": re.compile(r"\b\d{4}[- ]?\d{3}\b|\b\d{4}\b"),
-        
-        "phone_validate_pattern": re.compile(r"^(\+?351)?9[1236]\d{7}$")
-    }
+        "phone_validate_pattern": re.compile(r"^(\+?351)?9[1236]\d{7}$"),
+    },
 }
 
 # Cutoff score for rapidfuzz in the name standardization function

datamarket/utils/airflow.py

@@ -3,20 +3,23 @@
 
 import re
 import unicodedata
+
 import inflection
 
 ########################################################################################################################
 # FUNCTIONS
 
+
 def process_task_name(task_id):
-    task_id = ''.join(
-        f"_{unicodedata.name(c)}_" if not c.isalnum() else c for c in task_id 
-        if c.isalnum() or (unicodedata.category(c) not in ('Cc', 'Cf', 'Cs', 'Co', 'Cn'))
+    task_id = "".join(
+        f"_{unicodedata.name(c)}_" if not c.isalnum() else c
+        for c in task_id
+        if c.isalnum() or (unicodedata.category(c) not in ("Cc", "Cf", "Cs", "Co", "Cn"))
     )
-    task_id = inflection.parameterize(task_id, separator='_')
+    task_id = inflection.parameterize(task_id, separator="_")
     task_id = task_id.lower()
-    task_id = task_id.strip('_')
-    task_id = re.sub(r'_+', '_', task_id)
+    task_id = task_id.strip("_")
+    task_id = re.sub(r"_+", "_", task_id)
     if task_id[0].isdigit():
-        task_id = 'task_' + task_id
+        task_id = "task_" + task_id
     return task_id

datamarket/utils/alchemy.py

@@ -8,6 +8,7 @@ from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
 
+
 class View(Base):
     __abstract__ = True
     is_view = True
@@ -19,4 +20,4 @@ class View(Base):
         """
         conn.execute(f"""
             CREATE OR REPLACE VIEW {cls.__tablename__} AS {query}
-        """)
+        """)

datamarket/utils/main.py

@@ -3,19 +3,19 @@
 
 import asyncio
 import configparser
-from datetime import timedelta
 import logging
 import random
 import re
 import shlex
 import subprocess
 import time
+from datetime import timedelta
 from typing import Sequence
-from babel.numbers import parse_decimal
 
-from bs4 import BeautifulSoup
 import pendulum
 import requests
+from babel.numbers import parse_decimal
+from bs4 import BeautifulSoup
 from requests.exceptions import ProxyError
 from tenacity import (
     before_sleep_log,
@@ -29,7 +29,7 @@ from tenacity import (
 
 from datamarket.exceptions.main import ManagedHTTPError
 
-from ..exceptions import RedirectionDetectedError, NotFoundError, BadRequestError
+from ..exceptions import BadRequestError, NotFoundError, RedirectionDetectedError
 from ..interfaces.proxy import ProxyInterface
 
 ########################################################################################################################

datamarket/utils/nominatim.py

@@ -2,16 +2,18 @@
 # IMPORTS
 
 from typing import Literal, Optional
+
 from rapidfuzz import fuzz, process
 from unidecode import unidecode
+
 from ..params.nominatim import (
+    _NORMALIZED_PROVINCE_CACHE,
+    COUNTRY_PARSING_RULES,
     POSTCODE_TO_STATES,
     PROVINCE_TO_POSTCODE,
     PROVINCES,
     STANDARD_THRESHOLD,
     STATES,
-    _NORMALIZED_PROVINCE_CACHE,
-    COUNTRY_PARSING_RULES
 )
 from .strings import normalize
 
@@ -32,9 +34,7 @@ def standardize_admin_division(
         return None
 
     country_code = country_code.lower()
-    mapping = (
-        STATES.get(country_code) if level == "state" else PROVINCES.get(country_code)
-    )
+    mapping = STATES.get(country_code) if level == "state" else PROVINCES.get(country_code)
 
     if not mapping:  # If country is not standardized, return raw name
         return name
@@ -103,7 +103,7 @@ def _province_postcode_match(
         normalized_address,
         cache["choices"],  # <-- Uses pre-computed list
         scorer=fuzz.partial_ratio,
-        score_cutoff=100
+        score_cutoff=100,
     )
 
     if not result:
@@ -119,11 +119,8 @@ def _province_postcode_match(
     province_map = PROVINCE_TO_POSTCODE.get(country_code, {})
     postcode_prefix = province_map[original_province]
 
-    return (
-        postcode_prefix + zip_code[1:]
-        if len(zip_code) == 4
-        else zip_code
-    )
+    return postcode_prefix + zip_code[1:] if len(zip_code) == 4 else zip_code
+
 
 def _parse_es_zip_code(
     zip_code: str,
@@ -131,16 +128,16 @@ def _parse_es_zip_code(
     opt_address: str | None,
 ) -> str:
     """parse spain zip code"""
-    
+
     # Get the validation regex from params
-    validate_regex = COUNTRY_PARSING_RULES['es']['zip_validate_pattern']
-    
+    validate_regex = COUNTRY_PARSING_RULES["es"]["zip_validate_pattern"]
+
     if validate_regex.match(zip_code):
         return zip_code
     else:
         # Use search regex from params
-        pattern = COUNTRY_PARSING_RULES['es']['zip_search_pattern']
-        
+        pattern = COUNTRY_PARSING_RULES["es"]["zip_search_pattern"]
+
         match = pattern.search(address)
         if match:
             return match.group()
@@ -148,7 +145,7 @@ def _parse_es_zip_code(
             match = pattern.search(opt_address)
             if match:
                 return match.group()
-            
+
         province_match = _province_postcode_match(address, zip_code, country_code="es")
         return province_match or zip_code
 
@@ -161,18 +158,18 @@ def _parse_pt_zip_code(
     """parse portugal zip code"""
 
     # Get the validation regex from params
-    validate_regex = COUNTRY_PARSING_RULES['pt']['zip_validate_pattern']
+    validate_regex = COUNTRY_PARSING_RULES["pt"]["zip_validate_pattern"]
 
     if validate_regex.match(zip_code):
         return zip_code
     else:
         # Use search regex from params
-        pattern = COUNTRY_PARSING_RULES['pt']['zip_search_pattern']
-        
+        pattern = COUNTRY_PARSING_RULES["pt"]["zip_search_pattern"]
+
         match = pattern.search(address)
         if match is None and opt_address:
             match = pattern.search(opt_address)
-            
+
         return match.group() if match else zip_code
 
 

datamarket/utils/playwright/async_api.py

@@ -13,8 +13,12 @@ from camoufox.async_api import AsyncCamoufox as Camoufox
 from playwright.async_api import (
     Browser,
     BrowserContext,
-    Error as PlaywrightError,
     Page,
+)
+from playwright.async_api import (
+    Error as PlaywrightError,
+)
+from playwright.async_api import (
     TimeoutError as PlaywrightTimeoutError,
 )
 from tenacity import (
@@ -27,7 +31,6 @@ from tenacity import (
 
 from datamarket.interfaces.proxy import ProxyInterface
 
-
 ########################################################################################################################
 # SETUP LOGGER
 

datamarket/utils/playwright/sync_api.py

@@ -12,8 +12,12 @@ from camoufox import Camoufox
 from playwright.sync_api import (
     Browser,
     BrowserContext,
-    Error as PlaywrightError,
     Page,
+)
+from playwright.sync_api import (
+    Error as PlaywrightError,
+)
+from playwright.sync_api import (
     TimeoutError as PlaywrightTimeoutError,
 )
 from tenacity import (
@@ -23,6 +27,7 @@ from tenacity import (
     stop_after_delay,
     wait_exponential,
 )
+
 from datamarket.interfaces.proxy import ProxyInterface
 
 ########################################################################################################################
@@ -171,4 +176,4 @@ class PlaywrightCrawler:
         if not self.page:
             logger.info("Browser context not found, initializing now...")
             self.init_context()
-        return self._goto_with_retry(url)
+        return self._goto_with_retry(url)

datamarket/utils/selenium.py

@@ -17,9 +17,7 @@ logger = logging.getLogger(__name__)
 
 
 def get_chromedriver_version():
-    return int(
-        run_bash_command("/usr/bin/google-chrome --version").split(" ")[2].split(".")[0]
-    )
+    return int(run_bash_command("/usr/bin/google-chrome --version").split(" ")[2].split(".")[0])
 
 
 def get_driver(chrome_options=None, **kwargs):
@@ -38,23 +36,19 @@ def get_driver(chrome_options=None, **kwargs):
 
 def wait(driver, css_selector, timeout=30):
     logger.info(f"waiting for {css_selector}...")
-    return WebDriverWait(driver, timeout).until(
-        EC.visibility_of_element_located(("css selector", css_selector))
-    )
+    return WebDriverWait(driver, timeout).until(EC.visibility_of_element_located(("css selector", css_selector)))
 
 
 def wait_and_click(driver, css_selector, timeout=30):
     logger.info(f"clicking on {css_selector}...")
-    WebDriverWait(driver, timeout).until(
-        EC.element_to_be_clickable(("css selector", css_selector))
-    ).click()
+    WebDriverWait(driver, timeout).until(EC.element_to_be_clickable(("css selector", css_selector))).click()
 
 
 def wait_and_fill(driver, css_selector, text_to_fill, timeout=30):
     logger.info(f"sending text to {css_selector}...")
-    WebDriverWait(driver, timeout).until(
-        EC.presence_of_element_located(("css selector", css_selector))
-    ).send_keys(text_to_fill)
+    WebDriverWait(driver, timeout).until(EC.presence_of_element_located(("css selector", css_selector))).send_keys(
+        text_to_fill
+    )
 
 
 def scroll(driver, css_selector):

datamarket/utils/strings/normalization.py

@@ -1,9 +1,9 @@
 ########################################################################################################################
 # IMPORTS
-import re
 import unicodedata
 from enum import Enum, auto
 from typing import Any, Optional, Set, Union
+
 import numpy as np
 from inflection import camelize, parameterize, titleize, underscore
 from string_utils import prettify, strip_html
@@ -36,9 +36,7 @@ class NamingConvention(Enum):
 # FUNCTIONS
 
 
-def get_unidecoded_text(
-    input_text: str, allowed_chars: Set[str], apply_lowercase: bool = False
-) -> str:
+def get_unidecoded_text(input_text: str, allowed_chars: Set[str], apply_lowercase: bool = False) -> str:
     """
     Processes a string by unidecoding characters, optionally lowercasing them,
     while preserving a specified set of allowed characters.
@@ -65,9 +63,7 @@ def get_unidecoded_text(
     return "".join(chars_list)
 
 
-def transliterate_symbols(
-    s: str, allowed_symbols_set: Optional[Set[str]] = None
-) -> str:
+def transliterate_symbols(s: str, allowed_symbols_set: Optional[Set[str]] = None) -> str:
     """
     Translates Unicode symbols (category S*) in the input string to their lowercase Unicode names,
     with spaces replaced by underscores. Other characters, or characters in allowed_symbols_set, remain unchanged.
@@ -182,9 +178,7 @@ def normalize(
 
             for c in intermediate_text:
                 cat = unicodedata.category(c)
-                if (
-                    c in _allowed_symbols_set or c.isalnum()
-                ):  # Allowed symbols are part of tokens
+                if c in _allowed_symbols_set or c.isalnum():  # Allowed symbols are part of tokens
                     current_token_chars.append(c)
                 elif mode is NormalizationMode.FULL and cat.startswith("S"):
                     # Transliterate S* category symbols not in allowed_symbols
@@ -220,4 +214,4 @@ def normalize(
     if naming is NamingConvention.PASCAL:
         return camelize(underscored)
 
-    return underscored
+    return underscored

datamarket/utils/strings/standardization.py

@@ -3,11 +3,13 @@
 
 import re
 from typing import Literal
+
 from ...params.nominatim import COUNTRY_PARSING_RULES
 
 ########################################################################################################################
 # FUNCTIONS
 
+
 def parse_phone_number(number: str, country_code: Literal["es", "pt"]) -> str | None:
     """Clean and standardize phone number from a certain country_code
 
@@ -27,12 +29,12 @@ def parse_phone_number(number: str, country_code: Literal["es", "pt"]) -> str |
         pattern = COUNTRY_PARSING_RULES[country_code]["phone_validate_pattern"]
 
         # Validate and extract in one step
-        if len(clean_number) >= 9: # Check if the cleaned number has at least 9 digits
+        if len(clean_number) >= 9:  # Check if the cleaned number has at least 9 digits
             match = pattern.match(clean_number)
 
             # Return the captured group (the 9-digit number)
             return match.group(0)[-9:] if match else None
         else:
-            return None # Or handle the case where the number is too short
+            return None  # Or handle the case where the number is too short
     else:
         raise ValueError(f"Country code ({country_code}) is not currently supported")

datamarket/utils/typer.py

@@ -9,6 +9,7 @@ from typing_extensions import Annotated
 ########################################################################################################################
 # TYPES
 
+
 class Dict(dict):
     def __init__(self, value: str):
         super().__init__(json.loads(value))
@@ -25,4 +26,4 @@ def parse_json_dict(value: str) -> Dict:
 
 
 DictArg = Annotated[Dict, typer.Argument(parser=parse_json_dict)]
-DictOpt = Annotated[Dict, typer.Option(parser=parse_json_dict)]
+DictOpt = Annotated[Dict, typer.Option(parser=parse_json_dict)]

{datamarket-0.7.103.dist-info → datamarket-0.7.104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.103
+Version: 0.7.104
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

datamarket-0.7.104.dist-info/RECORD

@@ -0,0 +1,35 @@
+datamarket/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamarket/exceptions/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
+datamarket/exceptions/main.py,sha256=sJDsspp3e0gwK2GmVnIR9NEW6lEFVcsq8ONpfJ0NESE,1553
+datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamarket/interfaces/alchemy.py,sha256=2EZp7fn7-o8yL918dnqIYZ-gO7FUXGH8d8TzQFa7XRI,15769
+datamarket/interfaces/aws.py,sha256=4HEN_VfQuEEvDnksRYlcMBUdKbgJXWBkLnymKpyRtrs,4781
+datamarket/interfaces/azure.py,sha256=GXmdwG2MvWNGGyHhAxOhKmQw7bU7RyA5iffdPC2qzmk,4821
+datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjHk,4840
+datamarket/interfaces/ftp.py,sha256=LH3Oz19k_xUNhzDXcrq5Ofb4c3uiph5pWUqpgiaDvHI,2671
+datamarket/interfaces/nominatim.py,sha256=xizT94tVum7QPppfDgI5sEhx1mAXT-SM3JyPl8CDxxU,15148
+datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspXzA,23640
+datamarket/interfaces/proxy.py,sha256=vI_CGkTAnCz27gF_a-8zEON3a5vxvzmWAwOO2jeRJXk,13065
+datamarket/interfaces/tinybird.py,sha256=cNG-kAPTdQn2inlNX9LPf-VVdtnLud947ApLVO40Now,2594
+datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamarket/params/nominatim.py,sha256=S9TEB4FxmffvFyK9KffWl20TfXzWX69IAdbEehKar1I,11920
+datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
+datamarket/utils/airflow.py,sha256=Tc8vFB85NGJn0vgEkvT_yGMbn_NmW0OAJa9fy1qKocQ,804
+datamarket/utils/alchemy.py,sha256=B-6cdMiEStzD4JKhi7Xpk7pVs7eUcdT_fHqpfm2ToNc,637
+datamarket/utils/main.py,sha256=M9W_j4zkDxtAOa-4XxYxjDtFZ618tV6QAbHngnO6ViQ,8001
+datamarket/utils/nominatim.py,sha256=HUJfR86lw68PzaLfhZOCIT5YlopDvRSbwEY2JCf0WyI,5704
+datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamarket/utils/playwright/async_api.py,sha256=i6dxeCBJm1fWKeCotlSEuwmXIUyJD80lHVC8wgdjIAE,5855
+datamarket/utils/playwright/sync_api.py,sha256=lK0kjK8ERPMUHMvHr6jmLk0FEFPtdzZ2NTdOEYPh600,6508
+datamarket/utils/selenium.py,sha256=Fc2BJzTH7_xIqjBP9LbZODF69RSH4fF8LhD5WuGdlZ0,2457
+datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
+datamarket/utils/strings/__init__.py,sha256=b6TYOT9v7y9ID-lDyZk4E8BH2uIPbsF2ZSLGjCQ1MCQ,43
+datamarket/utils/strings/normalization.py,sha256=tlZHq8h9AtcANkaJ2AOrR6UD5yKShn1cLldfFfFQgTA,8990
+datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnmH3FPlgUUjM,5246
+datamarket/utils/strings/standardization.py,sha256=j_NbT-O1XnxDvDhct8panfkrfAC8R5OX6XM5fYBZ4RU,1496
+datamarket/utils/typer.py,sha256=geWuwMwGQjBQhxo27hX0vEAeRl1j1TS0u2oFVfpAs5I,816
+datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
+datamarket-0.7.104.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.104.dist-info/METADATA,sha256=K_hWvJnTd-1UesiRW-vKhl6eoNBKa3lR6u0k_bnDaHc,7382
+datamarket-0.7.104.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.104.dist-info/RECORD,,

datamarket-0.7.103.dist-info/RECORD

@@ -1,35 +0,0 @@
-datamarket/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/exceptions/__init__.py,sha256=-Vu-RZNKjW6fYCLqbUJTkKNuHeA8Yi_gyR50oZNaA_8,33
-datamarket/exceptions/main.py,sha256=SuP-ZKZIxJYdnOpNb63Y7BpYGRhLl-4JIyTEqgUoWV4,1205
-datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=i2lKLLLy3-jpbzV3-jxfRCXTy7jRoTsNU3063pmSonk,15749
-datamarket/interfaces/aws.py,sha256=co5JkC3iFIp-0FqdYX4eKy3_m71LhZKuJoW6kXwEImc,4780
-datamarket/interfaces/azure.py,sha256=PnPlo95skYiq63qYa4QDvEnVYi2JblPmMSfbTsmXhFs,4937
-datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjHk,4840
-datamarket/interfaces/ftp.py,sha256=K219-PP21EhQo1A1LkvRLahlrw2-pf4svBN0LogZaJE,2813
-datamarket/interfaces/nominatim.py,sha256=TjS9O2U446XuPUzfP65NwDSG-RDNqmYb6-NKikM-34w,15187
-datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspXzA,23640
-datamarket/interfaces/proxy.py,sha256=YNPNDFd2xTF-P5MITRHxGCLiXD8Fal4HK0yN8KhuYgI,9738
-datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
-datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/params/nominatim.py,sha256=RnmYXGoJQCijOsuCavCYcxw98WvOd_vOMK4KaraI0RU,11967
-datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
-datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
-datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=OORsHggUqa2lKj5AG5LTPzEvXfAtx3ry4rSaAwkuS38,8001
-datamarket/utils/nominatim.py,sha256=IxexKY2KOlDhiKtzsqQfoVUjJXPxJl7tn3iHUaQKg08,5795
-datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/utils/playwright/async_api.py,sha256=UbA2D4ScBtYeMfrRjly4RO-s8wXIub9c05J1eoOCpsQ,5782
-datamarket/utils/playwright/sync_api.py,sha256=eXaZsd7xgWSYJtZv6EAstjSbS2bl9OYlkwMBfqqTbFY,6434
-datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
-datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
-datamarket/utils/strings/__init__.py,sha256=b6TYOT9v7y9ID-lDyZk4E8BH2uIPbsF2ZSLGjCQ1MCQ,43
-datamarket/utils/strings/normalization.py,sha256=UBluU6ABY6aCpnd02F7L7HcivVSisRJ9IUXdj9D1MyE,9050
-datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnmH3FPlgUUjM,5246
-datamarket/utils/strings/standardization.py,sha256=c8CAG6HI3AfK0hB3A3IGwsbnQebZ6R3PrA5PELHRXM0,1492
-datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.103.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.103.dist-info/METADATA,sha256=XAS_V3qRLGddVGC6sRhxavgwa4EdzONa1B-YWnvcMK0,7382
-datamarket-0.7.103.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.103.dist-info/RECORD,,