datamarket 0.7.101__py3-none-any.whl → 0.7.103__py3-none-any.whl

datamarket/exceptions/main.py

@@ -2,6 +2,9 @@
 # CLASSES
 
 
+import requests
+
+
 class RedirectionDetectedError(Exception):
     def __init__(self, message="Redirection detected!"):
         self.message = message
@@ -18,3 +21,13 @@ class BadRequestError(Exception):
     def __init__(self, message="Bad request!"):
         self.message = message
         super().__init__(self.message)
+        
+class ManagedHTTPError(Exception):
+    """Signal that this HTTP status was handled and should not be retried."""
+    def __init__(self, response: requests.Response, *, url: str | None = None, message: str | None = None):
+        self.response = response
+        self.request = getattr(response, "request", None)
+        self.status_code = getattr(response, "status_code", None)
+        self.url = url or (self.request.url if self.request is not None else None)
+        self.message = message
+        super().__init__(message or f"HTTP {self.status_code} for {self.url}")

datamarket/interfaces/proxy.py

@@ -19,6 +19,7 @@ class ProxyInterface:
     def __init__(self, config):
         self._load_from_config(config)
         self.current_index = random.randrange(len(self.entries)) if self.entries else 0
+        self._health = {}  # {entry: {"ok": bool, "last_checked": time.time(), "last_error": str}}
 
     def _load_from_config(self, cfg):
         # Tor password (optional)
@@ -55,23 +56,42 @@ class ProxyInterface:
         auth = f"{user}:{password}@" if user and password else ""
         return f"{schema}://{auth}{host}:{port}"
 
-    def get_proxies(self, use_tor=False, randomize=False, raw=False, use_auth=False, use_socks=False):
+    def get_proxies(
+        self,
+        use_tor=False,
+        randomize=False,
+        raw=False,
+        use_auth=False,
+        use_socks=False,
+        health_check=True,
+        check_timeout=5,
+        cooldown_seconds=600,
+    ):
         """
-        Return parsed proxy URLs or raw entry tuple.
+        Return parsed proxy URLs or raw entry tuple for a working proxy.
 
         :param use_tor: route via local Tor SOCKS5 if True
         :param randomize: select a random proxy if True, otherwise round-robin
         :param raw: return raw (host, port, user, password) tuple if True
         :param use_auth: include proxies that require authentication if True; otherwise only credential-free
+        :param health_check: perform health checks to ensure proxy is working if True
+        :param check_timeout: timeout in seconds for health check requests
+        :param cooldown_seconds: how long to cache health status before re-checking
         """
-        # Tor handling
+        # Tor handling (skip health check for tor)
         if use_tor:
             if raw:
                 return ("127.0.0.1", "9050", None, None)
             return {"socks5": self.get_proxy_url("127.0.0.1", 9050, schema="socks5")}
 
-        # Select entry based on strategy and auth preference
-        host, port, user, password = self.get_random(use_auth) if randomize else self.get_next(use_auth)
+        # Get a working entry (with health checks if enabled)
+        if health_check:
+            host, port, user, password = self._get_working_entry(
+                use_auth=use_auth, randomize=randomize, check_timeout=check_timeout, cooldown_seconds=cooldown_seconds
+            )
+        else:
+            # Legacy behavior: no health check
+            host, port, user, password = self.get_random(use_auth) if randomize else self.get_next(use_auth)
 
         if raw:
             return host, port, user, password
@@ -149,3 +169,87 @@ class ProxyInterface:
         except Exception as ex:
             logger.error("Failed to renew Tor IP")
             logger.error(ex)
+
+    def mark_entry_status(self, entry, ok, error=None):
+        """Update health cache for an entry."""
+        self._health[entry] = {
+            "ok": ok,
+            "last_checked": time.time(),
+            "last_error": error,
+        }
+
+    def is_entry_alive(self, entry, timeout=5):
+        """Check if a proxy entry is working by making a test request."""
+        host, port, user, pwd = entry
+        try:
+            proxies = {
+                "http": self.get_proxy_url(host, port, user, pwd, "http"),
+                "https": self.get_proxy_url(host, port, user, pwd, "http"),
+            }
+            resp = requests.get(self.CHECK_IP_URL, proxies=proxies, timeout=timeout)
+            ok = resp.status_code == 200
+            self.mark_entry_status(entry, ok)
+            return ok
+        except Exception as ex:
+            self.mark_entry_status(entry, False, str(ex))
+            return False
+
+    def _get_working_entry(self, use_auth=False, randomize=False, check_timeout=5, cooldown_seconds=60):
+        """Get a working proxy entry, performing health checks as needed."""
+        if not self.entries:
+            raise RuntimeError("No proxies available")
+
+        # Build candidate list respecting use_auth and randomize/round-robin
+        pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
+        if not pool:
+            pool = self.entries
+
+        candidates = pool[:] if randomize else self._get_round_robin_candidates(pool)
+
+        # First pass: check cache and health
+        for entry in candidates:
+            health = self._health.get(entry, {})
+            last_checked = health.get("last_checked", 0)
+            ok = health.get("ok", False)
+            now = time.time()
+
+            if ok and (now - last_checked) < cooldown_seconds:
+                # Cached as working and recent
+                logger.debug(f"Using cached working proxy: {entry[0]}:{entry[1]}")
+                return entry
+            elif not ok and (now - last_checked) < cooldown_seconds:
+                # Cached as failed and recent, skip
+                continue
+            else:
+                # Not cached or expired, check now
+                logger.debug(f"Checking proxy health: {entry[0]}:{entry[1]}")
+                if self.is_entry_alive(entry, timeout=check_timeout):
+                    return entry
+
+        # Second pass: force fresh check for all candidates (in case cache skipped everything)
+        logger.warning("No cached working proxies, forcing fresh checks")
+        for entry in candidates:
+            logger.debug(f"Force-checking proxy: {entry[0]}:{entry[1]}")
+            if self.is_entry_alive(entry, timeout=check_timeout):
+                return entry
+
+        # No working proxies found
+        raise RuntimeError("No working proxies available")
+
+    def _get_round_robin_candidates(self, pool):
+        """Get candidates in round-robin order starting from current_index."""
+        candidates = []
+        start_idx = self.current_index
+        for i in range(len(self.entries)):
+            idx = (start_idx + i) % len(self.entries)
+            entry = self.entries[idx]
+            if entry in pool:
+                candidates.append(entry)
+        # Update current_index for next call
+        if candidates:
+            try:
+                pos = self.entries.index(candidates[0])
+                self.current_index = (pos + 1) % len(self.entries)
+            except ValueError:
+                pass
+        return candidates

datamarket/utils/main.py

@@ -10,6 +10,7 @@ import re
 import shlex
 import subprocess
 import time
+from typing import Sequence
 from babel.numbers import parse_decimal
 
 from bs4 import BeautifulSoup
@@ -26,6 +27,8 @@ from tenacity import (
     wait_exponential,
 )
 
+from datamarket.exceptions.main import ManagedHTTPError
+
 from ..exceptions import RedirectionDetectedError, NotFoundError, BadRequestError
 from ..interfaces.proxy import ProxyInterface
 
@@ -132,7 +135,9 @@ def parse_field(dict_struct, field_path, format_method=None):
 
 
 @retry(
-    retry=retry_if_not_exception_type((NotFoundError, BadRequestError, RedirectionDetectedError, ProxyError)),
+    retry=retry_if_not_exception_type(
+        (NotFoundError, BadRequestError, RedirectionDetectedError, ProxyError, ManagedHTTPError)
+    ),
     wait=wait_exponential(exp_base=3, multiplier=3, max=60),
     stop=stop_after_attempt(5),
     before_sleep=before_sleep_log(logger, logging.WARNING),
@@ -146,8 +151,37 @@ def get_data(
     proxy_interface: ProxyInterface = None,
     use_auth_proxies: bool = False,
     max_proxy_delay: timedelta = timedelta(minutes=10),
+    ignored_status_codes: Sequence[int] = (),
     **kwargs,
 ):
+    """
+    Fetches data from a given URL using HTTP requests, with support for proxy configuration, retries, and flexible output formats.
+
+    Args:
+        url (str): The target URL to fetch data from.
+        method (str, optional): HTTP method to use (e.g., 'GET', 'POST'). Defaults to 'GET'.
+        output (str, optional): Output format ('json', 'text', 'soup', 'response'). Defaults to 'json'.
+        sleep (tuple, optional): Tuple specifying max and min sleep times (seconds) after request. Defaults to (6, 3).
+        proxy_interface (ProxyInterface, optional): Proxy provider. If None, no proxy is used. Defaults to None.
+        use_auth_proxies (bool, optional): Whether to use authenticated proxies. Defaults to False.
+        max_proxy_delay (timedelta, optional): Maximum delay for proxy retry logic. Defaults to 10 minutes.
+        ignored_status_codes (Sequence[int], optional): Status codes to ignore and return response for. Defaults to ().
+        **kwargs: Additional arguments passed to the requests method.
+
+    Returns:
+        Depends on the 'output' argument:
+            - 'json': Parsed JSON response.
+            - 'text': Response text.
+            - 'soup': BeautifulSoup-parsed HTML.
+            - 'response': Raw requests.Response object.
+
+    Raises:
+        ManagedHTTPError: If a response status code is in `ignored_status_codes`.
+        NotFoundError: If a 404 status code is returned.
+        BadRequestError: If a 400 status code is returned.
+        RedirectionDetectedError, ProxyError: On specific error conditions.
+        requests.HTTPError: For other HTTP errors if not ignored.
+    """
     retry_type = retry_if_exception_type(ProxyError)
     wait = wait_exponential(exp_base=3, multiplier=3, max=60)
     stop = stop_after_delay(max_proxy_delay)
@@ -174,11 +208,15 @@ def get_data(
 
     ban_sleep(*sleep)
 
+    if r.status_code in ignored_status_codes:
+        raise ManagedHTTPError(r, url=url, message=f"Status {r.status_code} in ignored_status_codes for URL {url}")
     if r.status_code == 404:
         raise NotFoundError(f"404 Not Found error for {url}")
     if r.status_code == 400:
         raise BadRequestError(f"400 Bad Request error for {url}")
+
     r.raise_for_status()
+
     r.encoding = "utf-8"
 
     if output == "json":

{datamarket-0.7.101.dist-info → datamarket-0.7.103.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.101
+Version: 0.7.103
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.101.dist-info → datamarket-0.7.103.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/exceptions/__init__.py,sha256=-Vu-RZNKjW6fYCLqbUJTkKNuHeA8Yi_gyR50oZNaA_8,33
-datamarket/exceptions/main.py,sha256=AlPOzLoJT14QSNT7hHvvDBxfNew50evFUgCZ9FxF9lA,607
+datamarket/exceptions/main.py,sha256=SuP-ZKZIxJYdnOpNb63Y7BpYGRhLl-4JIyTEqgUoWV4,1205
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=i2lKLLLy3-jpbzV3-jxfRCXTy7jRoTsNU3063pmSonk,15749
 datamarket/interfaces/aws.py,sha256=co5JkC3iFIp-0FqdYX4eKy3_m71LhZKuJoW6kXwEImc,4780
@@ -9,14 +9,14 @@ datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjH
 datamarket/interfaces/ftp.py,sha256=K219-PP21EhQo1A1LkvRLahlrw2-pf4svBN0LogZaJE,2813
 datamarket/interfaces/nominatim.py,sha256=TjS9O2U446XuPUzfP65NwDSG-RDNqmYb6-NKikM-34w,15187
 datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspXzA,23640
-datamarket/interfaces/proxy.py,sha256=Uu-dHvpQOLNBZPGHAanLXnKT1789ArcHfOw8exECt34,5398
+datamarket/interfaces/proxy.py,sha256=YNPNDFd2xTF-P5MITRHxGCLiXD8Fal4HK0yN8KhuYgI,9738
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/params/nominatim.py,sha256=RnmYXGoJQCijOsuCavCYcxw98WvOd_vOMK4KaraI0RU,11967
 datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=1F-Klw7GX59BkRvWbTl3j7gX6MneZwlI3KOTmJK4P8Q,6040
+datamarket/utils/main.py,sha256=OORsHggUqa2lKj5AG5LTPzEvXfAtx3ry4rSaAwkuS38,8001
 datamarket/utils/nominatim.py,sha256=IxexKY2KOlDhiKtzsqQfoVUjJXPxJl7tn3iHUaQKg08,5795
 datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/utils/playwright/async_api.py,sha256=UbA2D4ScBtYeMfrRjly4RO-s8wXIub9c05J1eoOCpsQ,5782
@@ -29,7 +29,7 @@ datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnm
 datamarket/utils/strings/standardization.py,sha256=c8CAG6HI3AfK0hB3A3IGwsbnQebZ6R3PrA5PELHRXM0,1492
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.101.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.101.dist-info/METADATA,sha256=1UzLpLNWiHpv7qTEzfxCwvfUfZVzAMAYa5hAihtNdiQ,7382
-datamarket-0.7.101.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.101.dist-info/RECORD,,
+datamarket-0.7.103.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.103.dist-info/METADATA,sha256=XAS_V3qRLGddVGC6sRhxavgwa4EdzONa1B-YWnvcMK0,7382
+datamarket-0.7.103.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.103.dist-info/RECORD,,