datamarket 0.7.102__py3-none-any.whl → 0.7.103__py3-none-any.whl

datamarket/interfaces/proxy.py

@@ -19,6 +19,7 @@ class ProxyInterface:
     def __init__(self, config):
         self._load_from_config(config)
         self.current_index = random.randrange(len(self.entries)) if self.entries else 0
+        self._health = {}  # {entry: {"ok": bool, "last_checked": time.time(), "last_error": str}}
 
     def _load_from_config(self, cfg):
         # Tor password (optional)
@@ -55,23 +56,42 @@ class ProxyInterface:
         auth = f"{user}:{password}@" if user and password else ""
         return f"{schema}://{auth}{host}:{port}"
 
-    def get_proxies(self, use_tor=False, randomize=False, raw=False, use_auth=False, use_socks=False):
+    def get_proxies(
+        self,
+        use_tor=False,
+        randomize=False,
+        raw=False,
+        use_auth=False,
+        use_socks=False,
+        health_check=True,
+        check_timeout=5,
+        cooldown_seconds=600,
+    ):
         """
-        Return parsed proxy URLs or raw entry tuple.
+        Return parsed proxy URLs or raw entry tuple for a working proxy.
 
         :param use_tor: route via local Tor SOCKS5 if True
         :param randomize: select a random proxy if True, otherwise round-robin
         :param raw: return raw (host, port, user, password) tuple if True
         :param use_auth: include proxies that require authentication if True; otherwise only credential-free
+        :param health_check: perform health checks to ensure proxy is working if True
+        :param check_timeout: timeout in seconds for health check requests
+        :param cooldown_seconds: how long to cache health status before re-checking
         """
-        # Tor handling
+        # Tor handling (skip health check for tor)
         if use_tor:
             if raw:
                 return ("127.0.0.1", "9050", None, None)
             return {"socks5": self.get_proxy_url("127.0.0.1", 9050, schema="socks5")}
 
-        # Select entry based on strategy and auth preference
-        host, port, user, password = self.get_random(use_auth) if randomize else self.get_next(use_auth)
+        # Get a working entry (with health checks if enabled)
+        if health_check:
+            host, port, user, password = self._get_working_entry(
+                use_auth=use_auth, randomize=randomize, check_timeout=check_timeout, cooldown_seconds=cooldown_seconds
+            )
+        else:
+            # Legacy behavior: no health check
+            host, port, user, password = self.get_random(use_auth) if randomize else self.get_next(use_auth)
 
         if raw:
             return host, port, user, password
@@ -149,3 +169,87 @@ class ProxyInterface:
         except Exception as ex:
             logger.error("Failed to renew Tor IP")
             logger.error(ex)
+
+    def mark_entry_status(self, entry, ok, error=None):
+        """Update health cache for an entry."""
+        self._health[entry] = {
+            "ok": ok,
+            "last_checked": time.time(),
+            "last_error": error,
+        }
+
+    def is_entry_alive(self, entry, timeout=5):
+        """Check if a proxy entry is working by making a test request."""
+        host, port, user, pwd = entry
+        try:
+            proxies = {
+                "http": self.get_proxy_url(host, port, user, pwd, "http"),
+                "https": self.get_proxy_url(host, port, user, pwd, "http"),
+            }
+            resp = requests.get(self.CHECK_IP_URL, proxies=proxies, timeout=timeout)
+            ok = resp.status_code == 200
+            self.mark_entry_status(entry, ok)
+            return ok
+        except Exception as ex:
+            self.mark_entry_status(entry, False, str(ex))
+            return False
+
+    def _get_working_entry(self, use_auth=False, randomize=False, check_timeout=5, cooldown_seconds=60):
+        """Get a working proxy entry, performing health checks as needed."""
+        if not self.entries:
+            raise RuntimeError("No proxies available")
+
+        # Build candidate list respecting use_auth and randomize/round-robin
+        pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
+        if not pool:
+            pool = self.entries
+
+        candidates = pool[:] if randomize else self._get_round_robin_candidates(pool)
+
+        # First pass: check cache and health
+        for entry in candidates:
+            health = self._health.get(entry, {})
+            last_checked = health.get("last_checked", 0)
+            ok = health.get("ok", False)
+            now = time.time()
+
+            if ok and (now - last_checked) < cooldown_seconds:
+                # Cached as working and recent
+                logger.debug(f"Using cached working proxy: {entry[0]}:{entry[1]}")
+                return entry
+            elif not ok and (now - last_checked) < cooldown_seconds:
+                # Cached as failed and recent, skip
+                continue
+            else:
+                # Not cached or expired, check now
+                logger.debug(f"Checking proxy health: {entry[0]}:{entry[1]}")
+                if self.is_entry_alive(entry, timeout=check_timeout):
+                    return entry
+
+        # Second pass: force fresh check for all candidates (in case cache skipped everything)
+        logger.warning("No cached working proxies, forcing fresh checks")
+        for entry in candidates:
+            logger.debug(f"Force-checking proxy: {entry[0]}:{entry[1]}")
+            if self.is_entry_alive(entry, timeout=check_timeout):
+                return entry
+
+        # No working proxies found
+        raise RuntimeError("No working proxies available")
+
+    def _get_round_robin_candidates(self, pool):
+        """Get candidates in round-robin order starting from current_index."""
+        candidates = []
+        start_idx = self.current_index
+        for i in range(len(self.entries)):
+            idx = (start_idx + i) % len(self.entries)
+            entry = self.entries[idx]
+            if entry in pool:
+                candidates.append(entry)
+        # Update current_index for next call
+        if candidates:
+            try:
+                pos = self.entries.index(candidates[0])
+                self.current_index = (pos + 1) % len(self.entries)
+            except ValueError:
+                pass
+        return candidates

{datamarket-0.7.102.dist-info → datamarket-0.7.103.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.102
+Version: 0.7.103
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.102.dist-info → datamarket-0.7.103.dist-info}/RECORD

@@ -9,7 +9,7 @@ datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjH
 datamarket/interfaces/ftp.py,sha256=K219-PP21EhQo1A1LkvRLahlrw2-pf4svBN0LogZaJE,2813
 datamarket/interfaces/nominatim.py,sha256=TjS9O2U446XuPUzfP65NwDSG-RDNqmYb6-NKikM-34w,15187
 datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspXzA,23640
-datamarket/interfaces/proxy.py,sha256=Uu-dHvpQOLNBZPGHAanLXnKT1789ArcHfOw8exECt34,5398
+datamarket/interfaces/proxy.py,sha256=YNPNDFd2xTF-P5MITRHxGCLiXD8Fal4HK0yN8KhuYgI,9738
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/params/nominatim.py,sha256=RnmYXGoJQCijOsuCavCYcxw98WvOd_vOMK4KaraI0RU,11967
@@ -29,7 +29,7 @@ datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnm
 datamarket/utils/strings/standardization.py,sha256=c8CAG6HI3AfK0hB3A3IGwsbnQebZ6R3PrA5PELHRXM0,1492
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.102.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.102.dist-info/METADATA,sha256=FQcOMGhkANO_QeIbb9ISfm_MwcRCEo3TqCeVV5PONnI,7382
-datamarket-0.7.102.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.102.dist-info/RECORD,,
+datamarket-0.7.103.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.103.dist-info/METADATA,sha256=XAS_V3qRLGddVGC6sRhxavgwa4EdzONa1B-YWnvcMK0,7382
+datamarket-0.7.103.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.103.dist-info/RECORD,,