datamarket 0.7.89__py3-none-any.whl → 0.7.125__py3-none-any.whl

datamarket/utils/main.py

@@ -9,22 +9,12 @@ import re
 import shlex
 import subprocess
 import time
+from datetime import timedelta
+from typing import Sequence, overload
 
-from bs4 import BeautifulSoup
 import pendulum
-import requests
-from requests.exceptions import ProxyError
-from tenacity import (
-    before_sleep_log,
-    retry,
-    retry_if_exception_type,
-    retry_if_not_exception_type,
-    stop_after_attempt,
-    stop_after_delay,
-    wait_exponential,
-)
-
-from ..exceptions import RedirectionDetectedError, NotFoundError
+from babel.numbers import parse_decimal
+
 from ..interfaces.proxy import ProxyInterface
 
 ########################################################################################################################
@@ -49,15 +39,63 @@ def set_logger(level):
     log.addHandler(ch)
 
 
-def ban_sleep(max_time, min_time=0):
-    sleep_time = int(random.uniform(min_time, max_time))
-    logger.info(f"sleeping for {sleep_time} seconds...")
+@overload
+def ban_sleep(max_time: float) -> None: ...
+
+
+@overload
+def ban_sleep(min_time: float, max_time: float) -> None: ...
+
+
+def ban_sleep(x: float, y: float | None = None) -> None:
+    """
+    Sleep for a random number of seconds.
+
+    Usage:
+        ban_sleep(5)          -> sleeps ~N(5, 2.5²) seconds, truncated to >= 0
+        ban_sleep(3, 7)       -> sleeps uniformly between 3 and 7 seconds
+        ban_sleep(7, 3)       -> same as above (order doesn't matter)
+    """
+    if y is None:
+        mean = float(x)
+        std_dev = mean / 2.0
+        sleep_time = random.gauss(mean, std_dev)  # noqa: S311
+        sleep_time = max(0.0, sleep_time)
+    else:
+        x, y = sorted([float(x), float(y)])
+        sleep_time = random.uniform(x, y)  # noqa: S311
+
+    logger.info(f"sleeping for {sleep_time:.2f} seconds...")
     time.sleep(sleep_time)
 
 
-async def ban_sleep_async(max_time, min_time=0):
-    sleep_time = int(random.uniform(min_time, max_time))  # noqa: S311
-    logger.info(f"sleeping for {sleep_time} seconds...")
+@overload
+async def ban_sleep_async(seconds: float) -> None: ...
+
+
+@overload
+async def ban_sleep_async(min_time: float, max_time: float) -> None: ...
+
+
+async def ban_sleep_async(min_time: float, max_time: float | None = None) -> None:
+    """
+    Asynchronous sleep for a random number of seconds.
+
+    Usage:
+        await ban_sleep_async(5)          # sleeps ~N(5, (5/2)²) seconds, truncated to >= 0
+        await ban_sleep_async(3, 7)       # sleeps uniformly between 3 and 7 seconds
+        await ban_sleep_async(7, 3)       # same as above (order doesn't matter)
+    """
+    if max_time is None:
+        mean = float(min_time)
+        std_dev = mean / 2.0
+        sleep_time = random.gauss(mean, std_dev)  # noqa: S311
+        sleep_time = max(0.0, sleep_time)
+    else:
+        min_time, max_time = sorted([float(min_time), float(max_time)])
+        sleep_time = random.uniform(min_time, max_time)  # noqa: S311
+
+    logger.info(f"sleeping for {sleep_time:.2f} seconds...")
     await asyncio.sleep(sleep_time)
 
 
@@ -89,6 +127,19 @@ def text_to_int(text):
         return num
 
 
+def text_to_float(text: str | None, locale: str = "es_ES") -> float | None:
+    if not text:
+        return None
+    match = re.search(r"\d(?:[\d\s.,]*\d)?", text)
+    if not match:
+        return None
+    number_str = match.group(0).replace(" ", "")
+    try:
+        return float(parse_decimal(number_str, locale=locale))
+    except Exception:
+        return None
+
+
 def sleep_out_interval(from_h, to_h, tz="Europe/Madrid", seconds=1800):
     while pendulum.now(tz=tz).hour >= to_h or pendulum.now(tz=tz).hour < from_h:
         logger.warning("time to sleep and not scrape anything...")
@@ -116,13 +167,6 @@ def parse_field(dict_struct, field_path, format_method=None):
     return format_method(field_value) if format_method else field_value
 
 
-@retry(
-    retry=retry_if_not_exception_type((NotFoundError, RedirectionDetectedError, ProxyError)),
-    wait=wait_exponential(exp_base=3, multiplier=3, max=60),
-    stop=stop_after_attempt(5),
-    before_sleep=before_sleep_log(logger, logging.WARNING),
-    reraise=True,
-)
 def get_data(
     url: str,
     method: str = "GET",
@@ -130,45 +174,49 @@ def get_data(
     sleep: tuple = (6, 3),
     proxy_interface: ProxyInterface = None,
     use_auth_proxies: bool = False,
-    max_proxy_delay: int = 1800,
+    max_proxy_delay: timedelta = timedelta(minutes=10),
+    ignored_status_codes: Sequence[int] = (),
     **kwargs,
 ):
-    retry_type = retry_if_exception_type(ProxyError)
-    wait = wait_exponential(exp_base=3, multiplier=3, max=60)
-    stop = stop_after_delay(max_proxy_delay)
-    before_sleep = before_sleep_log(logger, logging.WARNING)
-
-    @retry(retry=retry_type, wait=wait, stop=stop, before_sleep=before_sleep, reraise=True)
-    def _fetch_with_proxy_retry(url, method, proxy_interface, use_auth, **params):
-        logger.info(f"Fetching data from {url} ...")
-        proxy_cfg = None
-        if proxy_interface:
-            host, port, user, pwd = proxy_interface.get_proxies(raw=True, use_auth=use_auth)
-            if host and port:
-                proxy_url = f"http://{host}:{port}"
-                proxy_auth_url = f"http://{user}:{pwd}@{host}:{port}"
-                proxy_cfg = {"http": proxy_url, "https": proxy_url}
-                if user and pwd:
-                    proxy_cfg = {"http": proxy_auth_url, "https": proxy_auth_url}
-                logger.info(f"Using proxy: {proxy_url}")
-        response = getattr(requests, method.lower())(url, proxies=proxy_cfg, **params)
-        return response
-
-    params = {"timeout": 30} | kwargs
-    r = _fetch_with_proxy_retry(url, method, proxy_interface, use_auth_proxies, **params)
-
-    ban_sleep(*sleep)
-
-    if r.status_code == 404:
-        raise NotFoundError(f"404 Not Found error for {url}")
-    r.raise_for_status()
-    r.encoding = "utf-8"
-
-    if output == "json":
-        return r.json()
-    elif output == "text":
-        return r.text
-    elif output == "soup":
-        return BeautifulSoup(r.content, "html.parser")
-    elif output == "response":
-        return r
+    """
+    Fetches data from a given URL using HTTP requests, with support for proxy configuration, retries, and flexible output formats.
+
+    Args:
+        url (str): The target URL to fetch data from.
+        method (str, optional): HTTP method to use (e.g., 'GET', 'POST'). Defaults to 'GET'.
+        output (str, optional): Output format ('json', 'text', 'soup', 'response'). Defaults to 'json'.
+        sleep (tuple, optional): Tuple specifying max and min sleep times (seconds) after request. Defaults to (6, 3).
+        use_auth_proxies (bool, optional): Whether to use authenticated proxies. Defaults to False.
+        max_proxy_delay (timedelta, optional): Maximum delay for proxy retry logic. Defaults to 10 minutes.
+        ignored_status_codes (Sequence[int], optional): Status codes to ignore and return response for. Defaults to ().
+        **kwargs: Additional arguments passed to the requests method (timeout defaults to 30 seconds if not specified).
+
+    Returns:
+        Depends on the 'output' argument:
+            - 'json': Parsed JSON response.
+            - 'text': Response text.
+            - 'soup': BeautifulSoup-parsed HTML.
+            - 'response': Raw requests.Response object.
+
+    Raises:
+        IgnoredHTTPError: If a response status code is in `ignored_status_codes`.
+        NotFoundError: If a 404 or 410 status code is returned and not in `ignored_status_codes`.
+        BadRequestError: If a 400 status code is returned and not in `ignored_status_codes`.
+        EmptyResponseError: If the response has no content.
+        ProxyError: On proxy-related errors.
+        requests.HTTPError: For other HTTP errors if not ignored.
+    """
+
+    from .requests import RequestsClient
+
+    client = RequestsClient(proxy_interface)
+    return client.get_data(
+        url=url,
+        method=method,
+        output=output,
+        sleep=sleep,
+        use_auth_proxies=use_auth_proxies,
+        max_proxy_delay=max_proxy_delay,
+        ignored_status_codes=ignored_status_codes,
+        **kwargs,
+    )

datamarket/utils/nominatim.py

@@ -0,0 +1,201 @@
+########################################################################################################################
+# IMPORTS
+
+from typing import Literal, Optional
+
+from rapidfuzz import fuzz, process
+from unidecode import unidecode
+
+from ..params.nominatim import (
+    _NORMALIZED_PROVINCE_CACHE,
+    COUNTRY_PARSING_RULES,
+    POSTCODE_TO_STATES,
+    PROVINCE_TO_POSTCODE,
+    PROVINCES,
+    STANDARD_THRESHOLD,
+    STATES,
+)
+from .strings import normalize
+
+########################################################################################################################
+# FUNCTIONS
+
+
+def standardize_admin_division(
+    name: str,
+    level: Literal["province", "state"] = "province",
+    country_code: str = "es",
+) -> Optional[str]:
+    """
+    Normalize and standardize administrative divisions of a given country using RapidFuzz.
+    Uses normalized dict keys for comparison and returns dict values with the official names.
+    """
+    if not name:
+        return None
+
+    country_code = country_code.lower()
+    mapping = STATES.get(country_code) if level == "state" else PROVINCES.get(country_code)
+
+    if not mapping:  # If country is not standardized, return raw name
+        return name
+
+    normalized_name = normalize(name)  # Essential for rapidfuzz to work well
+    result = process.extractOne(
+        normalized_name,
+        mapping.keys(),  # Compare with the normalized names in the dict
+        scorer=fuzz.WRatio,
+        score_cutoff=STANDARD_THRESHOLD,
+    )
+
+    if not result:
+        return None
+
+    best_key, score, _ = result
+
+    # Return the standardized name corresponding to the normalized name
+    return mapping[best_key]
+
+
+def parse_state(
+    zip_code: str,
+    country_code: str,
+) -> str | None:
+    """Given a zip code and a country code, returns the state in which the zip code is located
+
+    Args:
+        zip_code (str)
+        country_code (str)
+
+    Returns:
+        str | None: state if coincidence found, else None
+    """
+    country_postcodes = POSTCODE_TO_STATES.get(country_code, {})
+    state = country_postcodes.get(zip_code[:2], None)
+    return state
+
+
+def _province_postcode_match(
+    address: str,
+    zip_code: str,
+    country_code: str,
+) -> str | None:
+    """
+    Match and return province with the start of all of its zip codes
+    using a pre-computed cache and rapidfuzz for efficient matching.
+
+    Args:
+        address (str)
+        zip_code (str)
+        country_code (str)
+
+    Returns:
+        str | None:
+    """
+    # Get the pre-computed cache for the country
+    cache = _NORMALIZED_PROVINCE_CACHE.get(country_code)
+    if not cache:
+        return None  # Country not configured
+
+    normalized_address = unidecode(address).lower()
+
+    # Use the cached 'choices' list for the search
+    result = process.extractOne(
+        normalized_address,
+        cache["choices"],  # <-- Uses pre-computed list
+        scorer=fuzz.partial_ratio,
+        score_cutoff=100,
+    )
+
+    if not result:
+        return None  # No exact substring match found
+
+    # We only need the index from the result
+    _, _, index = result
+
+    # Get the original province name from the cached 'keys' list
+    original_province = cache["keys"][index]  # <-- Uses pre-computed list
+
+    # Get the postcode prefix from the original map
+    province_map = PROVINCE_TO_POSTCODE.get(country_code, {})
+    postcode_prefix = province_map[original_province]
+
+    return postcode_prefix + zip_code[1:] if len(zip_code) == 4 else zip_code
+
+
+def _parse_es_zip_code(
+    zip_code: str,
+    address: str,
+    opt_address: str | None,
+) -> str:
+    """parse spain zip code"""
+
+    # Get the validation regex from params
+    validate_regex = COUNTRY_PARSING_RULES["es"]["zip_validate_pattern"]
+
+    if validate_regex.match(zip_code):
+        return zip_code
+    else:
+        # Use search regex from params
+        pattern = COUNTRY_PARSING_RULES["es"]["zip_search_pattern"]
+
+        match = pattern.search(address)
+        if match:
+            return match.group()
+        if opt_address:
+            match = pattern.search(opt_address)
+            if match:
+                return match.group()
+
+        province_match = _province_postcode_match(address, zip_code, country_code="es")
+        return province_match or zip_code
+
+
+def _parse_pt_zip_code(
+    zip_code: str,
+    address: str,
+    opt_address: str | None,
+) -> str:
+    """parse portugal zip code"""
+
+    # Get the validation regex from params
+    validate_regex = COUNTRY_PARSING_RULES["pt"]["zip_validate_pattern"]
+
+    if validate_regex.match(zip_code):
+        return zip_code
+    else:
+        # Use search regex from params
+        pattern = COUNTRY_PARSING_RULES["pt"]["zip_search_pattern"]
+
+        match = pattern.search(address)
+        if match is None and opt_address:
+            match = pattern.search(opt_address)
+
+        return match.group() if match else zip_code
+
+
+def parse_zip_code(
+    address: str,
+    zip_code: str,
+    country_code: str,
+    opt_address: str | None = None,
+) -> str | None:
+    """Parse and standardize zip code
+
+    Args:
+        address (str): written address
+        zip_code (str)
+        country_code (str):
+        opt_address (str | None, optional): optional extra address, usually None. Defaults to None.
+
+    Raises:
+        ValueError: when parsing zip code is not supported for the passed country_code
+
+    Returns:
+        str | None
+    """
+    if country_code == "es":
+        return _parse_es_zip_code(zip_code, address, opt_address)
+    elif country_code == "pt":
+        return _parse_pt_zip_code(zip_code, address, opt_address)
+    else:
+        raise ValueError(f"Country code ({country_code}) is not currently supported")

datamarket/utils/playwright/async_api.py

@@ -2,31 +2,35 @@
 # IMPORTS
 
 import asyncio
+import json
 import logging
 from datetime import timedelta
 from random import randint
 from types import TracebackType
-from typing import Optional, Self
+from typing import Optional, Self, Sequence
 
-# 'BdbQuit' import is removed as it's no longer used
+from bs4 import BeautifulSoup
 from camoufox.async_api import AsyncCamoufox as Camoufox
+from playwright.async_api import Browser, BrowserContext, Page, Response
 from playwright.async_api import (
-    Browser,
-    BrowserContext,
     Error as PlaywrightError,
-    Page,
-    TimeoutError as PlaywrightTimeoutError,
 )
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from requests.exceptions import HTTPError, ProxyError
 from tenacity import (
     before_sleep_log,
     retry,
     retry_if_exception_type,
+    retry_if_not_exception_type,
+    stop_after_attempt,
     stop_after_delay,
     wait_exponential,
 )
 
+from datamarket.exceptions import BadRequestError, EmptyResponseError, NotFoundError, RedirectionDetectedError
+from datamarket.exceptions.main import IgnoredHTTPError
 from datamarket.interfaces.proxy import ProxyInterface
-
+from datamarket.utils.main import ban_sleep_async
 
 ########################################################################################################################
 # SETUP LOGGER
@@ -39,15 +43,15 @@ logger = logging.getLogger(__name__)
 
 async def human_type(page: Page, text: str, delay: int = 100):
     for char in text:
-        await page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))
+        await page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
 
 
 async def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, add_sleep: bool = True) -> None:
     """Asynchronously presses a key with a random delay, optionally sleeping between presses."""
     for _ in range(count):
-        await page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))
+        await page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
         if add_sleep:
-            await asyncio.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)
+            await asyncio.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311
 
 
 ########################################################################################################################
@@ -57,12 +61,16 @@ async def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100
 class PlaywrightCrawler:
     """An robust, proxy-enabled asynchronous Playwright crawler with captcha bypass and retry logic."""
 
-    def __init__(self, proxy_interface: ProxyInterface):
+    _REDIRECT_STATUS_CODES = set(range(300, 309))
+
+    def __init__(self, proxy_interface: Optional[ProxyInterface] = None):
         """
-        Initializes the async crawler with a proxy interface.
+        Initializes the async crawler.
 
         Args:
-            proxy_interface (ProxyInterface): An async-compatible object to fetch proxy credentials.
+            proxy_interface (Optional[ProxyInterface], optional): Provider used to fetch
+                proxy credentials. Defaults to None. When None, no proxy is configured and
+                the browser will run without a proxy.
         """
         self.proxy_interface = proxy_interface
         self.pw: Optional[Camoufox] = None
@@ -85,6 +93,25 @@ class PlaywrightCrawler:
         if self.pw:
             await self.pw.__aexit__(exc_type, exc_val, exc_tb)
 
+    async def _build_proxy_config(self) -> Optional[dict]:
+        """Builds the proxy configuration dictionary.
+
+        Returns:
+            Optional[dict]: Proxy configuration if a proxy_interface is provided; otherwise None.
+        """
+        if not self.proxy_interface:
+            logger.info("Starting browser without proxy.")
+            return None
+
+        host, port, user, pwd = await asyncio.to_thread(self.proxy_interface.get_proxies, raw=True, use_auth=True)
+        proxy_url = f"http://{host}:{port}"
+        proxy_cfg: dict = {"server": proxy_url}
+        if user and pwd:
+            proxy_cfg.update({"username": user, "password": pwd})
+
+        logger.info(f"Starting browser with proxy: {proxy_url}")
+        return proxy_cfg
+
     @retry(
         wait=wait_exponential(exp_base=2, multiplier=3, max=90),
         stop=stop_after_delay(timedelta(minutes=10)),
@@ -92,17 +119,20 @@ class PlaywrightCrawler:
         reraise=True,
     )
     async def init_context(self) -> Self:
-        """Initializes a new async browser instance and context with a fresh proxy."""
-        try:
-            # Correctly wrap the blocking I/O call
-            host, port, user, pwd = await asyncio.to_thread(self.proxy_interface.get_proxies, raw=True, use_auth=True)
-            proxy_url = f"http://{host}:{port}"
-            proxy_cfg = {"server": proxy_url}
+        """
+        Initializes a new async browser instance and context.
 
-            if user and pwd:
-                proxy_cfg.update({"username": user, "password": pwd})
+        Behavior:
+        - If a proxy_interface is provided, fetches fresh proxy credentials and starts
+          the browser using that proxy.
+        - If proxy_interface is None, starts the browser without any proxy.
+
+        Returns:
+            Self: The crawler instance with active browser, context, and page.
+        """
+        try:
+            proxy_cfg: Optional[dict] = await self._build_proxy_config()
 
-            logger.info(f"Starting browser with proxy: {proxy_url}")
             self.pw = Camoufox(headless=True, geoip=True, humanize=True, proxy=proxy_cfg)
             self.browser = await self.pw.__aenter__()
             self.context = await self.browser.new_context()
@@ -128,7 +158,7 @@ class PlaywrightCrawler:
         before_sleep=before_sleep_log(logger, logging.INFO),
         reraise=True,
     )
-    async def _goto_with_retry(self, url: str) -> Page:
+    async def _goto_with_retry(self, url: str, timeout: int = 30_000) -> Response:
         """
         Asynchronously navigates to a URL with retries for common Playwright errors.
         Restarts the browser context on repeated failures.
@@ -137,11 +167,12 @@ class PlaywrightCrawler:
             logger.warning("Page is not available or closed. Restarting context.")
             await self.restart_context()
 
-        assert self.page is not None
-        await self.page.goto(url, timeout=30000, wait_until="domcontentloaded")
-        return self.page
+        response = await self.page.goto(url, timeout=timeout, wait_until="domcontentloaded")
+        return response
 
-    async def goto(self, url: str) -> Page:
+    async def goto(
+        self, url: str, max_proxy_delay: timedelta = timedelta(minutes=10), timeout: int = 30_000
+    ) -> Response:
         """
         Ensures the browser is initialized and navigates to the given URL.
         Public wrapper for the internal retry-enabled navigation method.
@@ -149,4 +180,95 @@ class PlaywrightCrawler:
         if not self.page:
             logger.info("Browser context not found, initializing now...")
             await self.init_context()
-        return await self._goto_with_retry(url)
+        return await self._goto_with_retry.retry_with(stop=stop_after_delay(max_proxy_delay))(self, url, timeout)
+
+    def _handle_http_error(self, status_code: int, url: str, response, allow_redirects: bool = True) -> None:
+        """
+        Handle HTTP errors with special handling for redirects.
+
+        Args:
+            status_code: HTTP status code
+            url: Request URL
+            response: Response object
+
+        Raises:
+            RedirectionDetectedError: If a redirect status is received
+            NotFoundError: For 404/410 errors
+            BadRequestError: For 400 errors
+            HTTPError: For other non-2xx status codes
+        """
+        # Check for redirect status codes
+        if not allow_redirects and response.request.redirected_from:  # noqa: F841
+            raise RedirectionDetectedError(
+                message=f"HTTP redirect detected from {response.request.redirected_from.url} to {response.request.redirected_from.redirected_to.url}",
+                response=response,
+            )
+
+        # Standard error handlers
+        error_handlers = {
+            404: lambda: NotFoundError(message=f"404 Not Found error for {url}", response=response),
+            410: lambda: NotFoundError(message=f"410 Gone error for {url}", response=response),
+            400: lambda: BadRequestError(message=f"400 Bad Request error for {url}", response=response),
+        }
+
+        if status_code in error_handlers:
+            raise error_handlers[status_code]()
+
+        # Raise for any other non-2xx status
+        if status_code >= 400:
+            raise HTTPError(f"Navigation failed: {status_code} - {url}", response=response)
+
+    @retry(
+        retry=retry_if_not_exception_type(
+            (IgnoredHTTPError, NotFoundError, BadRequestError, RedirectionDetectedError, ProxyError)
+        ),
+        wait=wait_exponential(exp_base=3, multiplier=3, max=60),
+        stop=stop_after_attempt(5),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
+        reraise=False,
+        retry_error_callback=lambda rs: None,
+    )
+    async def get_data(
+        self,
+        url: str,
+        output: str = "json",
+        sleep: tuple = (6, 3),
+        max_proxy_delay: timedelta = timedelta(minutes=10),
+        ignored_status_codes: Sequence[int] = (),
+        timeout: int = 30_000,
+        **kwargs,
+    ):
+        """
+        Asynchronously crawls a given URL using Playwright and attempts to parse its body content.
+        Maintains full retry structure and output versatility.
+        """
+
+        params = kwargs.copy()
+
+        allow_redirects = params.get("allow_redirects", True)
+
+        logger.info(f"Fetching data from {url} ...")
+        r = await self.goto(url, max_proxy_delay, timeout)
+        await ban_sleep_async(*sleep)
+        body_content = await self.page.eval_on_selector("body", "body => body.innerText")
+
+        if r.status in ignored_status_codes:
+            raise IgnoredHTTPError(message=f"Status {r.status} in ignored_status_codes for URL {url}", response=r)
+
+        # Handle HTTP errors with redirect detection
+        self._handle_http_error(r.status, url, r, allow_redirects)
+
+        if not body_content:
+            raise EmptyResponseError(message=f"Empty body received from {url} (status {r.status})", response=r)
+
+        output_format = {
+            "json": lambda: json.loads(body_content),
+            "text": lambda: body_content,
+            "soup": lambda: BeautifulSoup(body_content, "html.parser"),
+            "response": lambda: r,
+        }
+
+        if output in output_format:
+            return output_format[output]()
+
+        raise ValueError(f"Unsupported output format: {output}")