datamarket 0.7.106__tar.gz → 0.7.107__tar.gz

{datamarket-0.7.106 → datamarket-0.7.107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.106
+Version: 0.7.107
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.106 → datamarket-0.7.107}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.7.106"
+version = "0.7.107"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.7.106 → datamarket-0.7.107}/src/datamarket/interfaces/proxy.py

@@ -1,6 +1,7 @@
 import logging
 import random
 import time
+from datetime import timedelta
 
 import requests
 import tenacity
@@ -12,6 +13,9 @@ from datamarket.exceptions import EnsureNewIPTimeoutError, NoWorkingProxiesError
 logger = logging.getLogger(__name__)
 logging.getLogger("stem").setLevel(logging.WARNING)
 
+PROXY_ROTATION_INTERVAL = timedelta(minutes=10)
+PROXY_ROTATION_TIMEOUT_SECONDS = int(PROXY_ROTATION_INTERVAL.total_seconds())
+
 
 class ProxyInterface:
     """
@@ -24,6 +28,10 @@ class ProxyInterface:
         self._load_from_config(config)
         self.current_index = -2  # -2 means no selection made yet, -1 means Tor selected
         self._health = {}  # {entry: {"ok": bool, "last_checked": time.time(), "last_error": str}}
+        self._traversal_queue = []  # Queue of indices left to test in current traversal
+        self._traversal_start = None  # Timestamp when current traversal started
+        self._last_ip_wait = {}  # {entry: (timestamp, traversal_cycle)} - last time we attempted to wait for IP change
+        self._traversal_cycle = 0  # Counter of full traversals of the queue
 
     def _load_from_config(self, cfg):
         # Tor password (optional)
@@ -67,13 +75,9 @@ class ProxyInterface:
         raw=False,
         use_auth=False,
         use_socks=False,
-        health_check=True,
         check_timeout=5,
-        cooldown_seconds=600,
-        ensure_new_ip=False,
-        ensure_new_ip_timeout=600,
-        ensure_new_ip_interval=5,
-        max_retry_seconds=600,
+        cooldown_seconds=30,
+        proxy_rotation_interval=PROXY_ROTATION_INTERVAL,
     ):
         """
         Return parsed proxy URLs or raw entry tuple for a working proxy.
@@ -82,13 +86,9 @@ class ProxyInterface:
         :param randomize: select a random proxy if True, otherwise round-robin
         :param raw: return raw (host, port, user, password) tuple if True
         :param use_auth: include proxies that require authentication if True; otherwise only credential-free
-        :param health_check: perform health checks to ensure proxy is working if True
         :param check_timeout: timeout in seconds for health check requests
         :param cooldown_seconds: how long to cache health status before re-checking
-        :param ensure_new_ip: if True and only one proxy available, wait until IP changes before returning
-        :param ensure_new_ip_timeout: max seconds to wait for IP change when ensure_new_ip=True
-        :param ensure_new_ip_interval: seconds between IP checks when ensure_new_ip=True
-        :param max_retry_seconds: max seconds to retry finding working proxies (0 to disable)
+        :param proxy_rotation_interval: max time to retry finding working proxies (timedelta or seconds, 0 to disable)
         """
         # Tor handling (skip health check for tor)
         if use_tor:
@@ -98,20 +98,13 @@ class ProxyInterface:
             return {"socks5": self.get_proxy_url("127.0.0.1", 9050, schema="socks5")}
 
         # Get a working entry (with health checks if enabled)
-        if health_check:
-            host, port, user, password = self._get_working_entry(
-                use_auth=use_auth,
-                randomize=randomize,
-                check_timeout=check_timeout,
-                cooldown_seconds=cooldown_seconds,
-                ensure_new_ip=ensure_new_ip,
-                ensure_new_ip_timeout=ensure_new_ip_timeout,
-                ensure_new_ip_interval=ensure_new_ip_interval,
-                max_retry_seconds=max_retry_seconds,
-            )
-        else:
-            # Legacy behavior: no health check
-            host, port, user, password = self.get_random(use_auth) if randomize else self.get_next(use_auth)
+        host, port, user, password = self._get_working_entry(
+            use_auth=use_auth,
+            randomize=randomize,
+            check_timeout=check_timeout,
+            cooldown_seconds=cooldown_seconds,
+            proxy_rotation_interval=proxy_rotation_interval,
+        )
 
         if raw:
             return host, port, user, password
@@ -127,49 +120,6 @@ class ProxyInterface:
                 "https": self.get_proxy_url(host, port, user, password, "http"),
             }
 
-    def get_next(self, use_auth=False):
-        # Round-robin selection, optionally filtering out authenticated proxies
-        if not self.entries:
-            raise NoWorkingProxiesError("No proxies available")
-
-        pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
-        if not pool:
-            pool = self.entries
-
-        # Start from the next index after current_index
-        start_idx = (self.current_index + 1) % len(self.entries) if self.current_index >= 0 else 0
-
-        # Find next in pool starting from start_idx
-        for i in range(len(self.entries)):
-            idx = (start_idx + i) % len(self.entries)
-            entry = self.entries[idx]
-            if entry in pool:
-                self.current_index = idx  # Update to selected index
-                return entry
-
-        # Fallback to first entry
-        self.current_index = 0
-        return self.entries[0]
-
-    def get_random(self, use_auth=False):
-        # Random selection, optionally filtering out authenticated proxies
-        if not self.entries:
-            raise NoWorkingProxiesError("No proxies available")
-
-        pool = self.entries if use_auth else [e for e in self.entries if not e[2] and not e[3]]
-        if not pool:
-            pool = self.entries
-
-        entry = random.choice(pool)  # noqa: S311
-        # Update index to selected entry
-        try:
-            pos = self.entries.index(entry)
-            self.current_index = pos
-        except ValueError:
-            pass
-
-        return entry
-
     def check_current_ip(self, proxies=None):
         try:
             proxies_arg = proxies or {"http": self.proxies["http"]}
@@ -195,9 +145,9 @@ class ProxyInterface:
             logger.error("Failed to renew Tor IP")
             logger.error(ex)
 
-    def wait_for_new_ip(self, timeout=600, interval=5, check_timeout=5):
+    def wait_for_new_ip(self, timeout=PROXY_ROTATION_TIMEOUT_SECONDS, interval=30, check_timeout=5):
         """
-        Wait for the IP address of the currently selected proxy to change.
+        Ensures that the IP address of the selected proxy differs from any other proxy chosen within the proxy rotation interval.
 
         :param timeout: Max seconds to wait for IP change
         :param interval: Seconds between IP checks
@@ -224,24 +174,36 @@ class ProxyInterface:
             else:
                 raise RuntimeError("Could not select a proxy for IP waiting")
 
-        # Auto-detect baseline IP
-        host, port, user, pwd = entry
-        proxies_map = {
-            "http": self.get_proxy_url(host, port, user, pwd, "http"),
-            "https": self.get_proxy_url(host, port, user, pwd, "http"),
-        }
-        try:
-            resp = requests.get(self.CHECK_IP_URL, proxies=proxies_map, timeout=check_timeout)
-            baseline = resp.json().get("YourFuckingIPAddress")
-            if not baseline:
-                raise RuntimeError(f"Could not determine baseline IP for entry {host}:{port}")
-            logger.debug(f"Auto-detected baseline IP: {baseline}")
-        except Exception as ex:
-            raise RuntimeError(f"Could not determine baseline IP for entry {host}:{port}: {ex}") from ex
+        # Check if we should skip waiting based on global cooldown and traversal cycle
+        now = time.time()
+        interval_seconds = PROXY_ROTATION_INTERVAL.total_seconds()
+        for last_wait in self._last_ip_wait.values():
+            last_ts, last_cycle = last_wait
+
+            time_recent = last_ts is not None and (now - last_ts) <= interval_seconds
+            no_full_rotation = self._traversal_cycle <= last_cycle
+
+            # Skip only if both conditions are true: recent wait AND no full traversal cycle since
+            if time_recent and no_full_rotation:
+                logger.debug(
+                    "Skipping wait_for_new_ip: last wait %.1fs ago and no full traversal since (last_cycle=%s current=%s)",
+                    now - last_ts,
+                    last_cycle,
+                    self._traversal_cycle,
+                )
+                return
 
+        # Mark we are now attempting to wait for this entry
+        self._last_ip_wait[entry] = (now, self._traversal_cycle)
+
+        # Try to use cached baseline IP from health check
+        health = self._health.get(entry, {})
+        baseline = health.get("last_ip")
+        if baseline is None:
+            raise RuntimeError(f"Could not determine baseline IP for entry {entry[0]}:{entry[1]}")
         return self._wait_for_new_ip(entry, baseline, timeout, interval, check_timeout)
 
-    def mark_entry_status(self, entry, ok, error=None, last_ip=None):
+    def _mark_entry_status(self, entry, ok, error=None, last_ip=None):
         """Update health cache for an entry."""
         self._health[entry] = {
             "ok": ok,
@@ -250,7 +212,7 @@ class ProxyInterface:
             "last_ip": last_ip,
         }
 
-    def is_entry_alive(self, entry, timeout=5):
+    def _is_entry_alive(self, entry, timeout=5):
         """Check if a proxy entry is working by making a test request."""
         host, port, user, pwd = entry
         try:
@@ -261,10 +223,10 @@ class ProxyInterface:
             resp = requests.get(self.CHECK_IP_URL, proxies=proxies, timeout=timeout)
             ok = resp.status_code == 200
             last_ip = resp.json().get("YourFuckingIPAddress") if ok else None
-            self.mark_entry_status(entry, ok, last_ip=last_ip)
+            self._mark_entry_status(entry, ok, last_ip=last_ip)
             return ok
         except Exception as ex:
-            self.mark_entry_status(entry, False, str(ex))
+            self._mark_entry_status(entry, False, str(ex))
             return False
 
     def _get_working_entry(
@@ -272,49 +234,28 @@ class ProxyInterface:
         use_auth=False,
         randomize=False,
         check_timeout=5,
-        cooldown_seconds=60,
-        ensure_new_ip=False,
-        ensure_new_ip_timeout=600,
-        ensure_new_ip_interval=5,
-        max_retry_seconds=600,
+        cooldown_seconds=30,
+        proxy_rotation_interval=PROXY_ROTATION_INTERVAL,
     ):
         """Get a working proxy entry, performing health checks as needed."""
         pool = self._build_pool(use_auth)
-        candidates = self._get_candidates(pool, randomize)
-
-        # Capture baseline before any health checks that might set it
-        baseline_before = None
-        if len(candidates) == 1:
-            idx = candidates[0]
-            entry = self.entries[idx]
-            baseline_before = self._health.get(entry, {}).get("last_ip")
+        self._refresh_traversal_queue(pool, randomize)
 
         def _find_working_entry():
             if not self.entries:
                 raise NoWorkingProxiesError("No proxies available")
-            return self._find_working_entry_once(candidates, check_timeout, cooldown_seconds)
+            return self._find_working_entry_once(check_timeout, cooldown_seconds)
 
-        if max_retry_seconds > 0:
+        # Handle both timedelta and numeric seconds for backward compatibility
+        if proxy_rotation_interval:
             retrying = tenacity.Retrying(
-                stop=tenacity.stop_after_delay(max_retry_seconds),
+                stop=tenacity.stop_after_delay(proxy_rotation_interval),
                 reraise=True,
             )
             entry = retrying(_find_working_entry)
         else:
             entry = _find_working_entry()
 
-        if ensure_new_ip and len(pool) == 1:
-            baseline = self._health.get(entry, {}).get("last_ip")
-            if not baseline_before:
-                # First time seeing this proxy: it was already checked in _find_working_entry, so return immediately
-                logger.debug("No baseline IP found; returning first-working proxy without waiting for IP change")
-            else:
-                # There is a baseline: wait for the IP to change
-                logger.info(f"ensure_new_ip=True and single proxy, waiting for IP change: {entry[0]}:{entry[1]}")
-                entry = self._wait_for_new_ip(
-                    entry, baseline, ensure_new_ip_timeout, ensure_new_ip_interval, check_timeout
-                )
-
         return entry
 
     def _get_round_robin_candidates(self, pool):
@@ -334,15 +275,29 @@ class ProxyInterface:
             pool = self.entries
         return pool
 
-    def _get_candidates(self, pool, randomize):
-        if randomize:
-            candidates = [idx for idx, entry in enumerate(self.entries) if entry in pool]
-            return random.sample(candidates, k=len(candidates))
-        else:
-            return self._get_round_robin_candidates(pool)
+    def _refresh_traversal_queue(self, pool, randomize):
+        # Build current pool indices
+        current_pool_indices = [idx for idx, entry in enumerate(self.entries) if entry in pool]
 
-    def _find_working_entry_once(self, candidates, check_timeout, cooldown_seconds):
-        for idx in candidates:
+        # Check if we need to refill the traversal queue
+        if not self._traversal_queue and current_pool_indices:
+            if randomize:
+                self._traversal_queue = current_pool_indices.copy()
+                random.shuffle(self._traversal_queue)
+            else:
+                # Round-robin: start from next after current_index
+                self._traversal_queue = []
+                start_idx = (self.current_index + 1) % len(self.entries) if self.current_index >= 0 else 0
+                for i in range(len(self.entries)):
+                    idx = (start_idx + i) % len(self.entries)
+                    if idx in current_pool_indices:
+                        self._traversal_queue.append(idx)
+            self._traversal_start = time.time()
+            self._traversal_cycle += 1
+
+    def _find_working_entry_once(self, check_timeout, cooldown_seconds):
+        # Consume from traversal queue for cached checks
+        for idx in self._traversal_queue:
             entry = self.entries[idx]
             health = self._health.get(entry, {})
             last_checked = health.get("last_checked", 0)
@@ -352,23 +307,17 @@ class ProxyInterface:
             if ok and (now - last_checked) < cooldown_seconds:
                 logger.debug(f"Using cached working proxy: {entry[0]}:{entry[1]}")
                 self.current_index = idx
+                self._traversal_queue.remove(idx)
                 return entry
             elif not ok and (now - last_checked) < cooldown_seconds:
                 continue
             else:
                 logger.debug(f"Checking proxy health: {entry[0]}:{entry[1]}")
-                if self.is_entry_alive(entry, timeout=check_timeout):
+                if self._is_entry_alive(entry, timeout=check_timeout):
                     self.current_index = idx
+                    self._traversal_queue.remove(idx)
                     return entry
 
-        logger.warning("No cached working proxies, forcing fresh checks")
-        for idx in candidates:
-            entry = self.entries[idx]
-            logger.debug(f"Force-checking proxy: {entry[0]}:{entry[1]}")
-            if self.is_entry_alive(entry, timeout=check_timeout):
-                self.current_index = idx
-                return entry
-
         raise NoWorkingProxiesError("No working proxies available")
 
     def _wait_for_new_ip(self, entry, baseline, timeout, interval, check_timeout):
@@ -386,9 +335,9 @@ class ProxyInterface:
                 current_ip = None
 
             if current_ip and current_ip != baseline:
-                self.mark_entry_status(entry, True, last_ip=current_ip)
+                self._mark_entry_status(entry, True, last_ip=current_ip)
                 logger.info(f"IP changed from {baseline} to {current_ip}")
-                return entry
+                return
 
             time.sleep(interval)