datamarket 0.7.104__tar.gz → 0.7.106__tar.gz

{datamarket-0.7.104 → datamarket-0.7.106}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.104
+Version: 0.7.106
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.7.104 → datamarket-0.7.106}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.7.104"
+version = "0.7.106"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.7.104 → datamarket-0.7.106}/src/datamarket/interfaces/proxy.py

@@ -22,7 +22,7 @@ class ProxyInterface:
 
     def __init__(self, config):
         self._load_from_config(config)
-        self.current_index = random.randrange(len(self.entries)) if self.entries else 0  # noqa: S311
+        self.current_index = -2  # -2 means no selection made yet, -1 means Tor selected
         self._health = {}  # {entry: {"ok": bool, "last_checked": time.time(), "last_error": str}}
 
     def _load_from_config(self, cfg):
@@ -92,6 +92,7 @@ class ProxyInterface:
         """
         # Tor handling (skip health check for tor)
         if use_tor:
+            self.current_index = -1  # Indicate Tor is selected
             if raw:
                 return ("127.0.0.1", "9050", None, None)
             return {"socks5": self.get_proxy_url("127.0.0.1", 9050, schema="socks5")}
@@ -135,15 +136,19 @@ class ProxyInterface:
         if not pool:
             pool = self.entries
 
-        # Find next in pool using current_index
-        for _ in range(len(self.entries)):
-            idx = self.current_index
-            self.current_index = (self.current_index + 1) % len(self.entries)
+        # Start from the next index after current_index
+        start_idx = (self.current_index + 1) % len(self.entries) if self.current_index >= 0 else 0
+
+        # Find next in pool starting from start_idx
+        for i in range(len(self.entries)):
+            idx = (start_idx + i) % len(self.entries)
             entry = self.entries[idx]
             if entry in pool:
+                self.current_index = idx  # Update to selected index
                 return entry
 
         # Fallback to first entry
+        self.current_index = 0
         return self.entries[0]
 
     def get_random(self, use_auth=False):
@@ -156,10 +161,10 @@ class ProxyInterface:
             pool = self.entries
 
         entry = random.choice(pool)  # noqa: S311
-        # Update index to after selected entry for round-robin continuity
+        # Update index to selected entry
         try:
             pos = self.entries.index(entry)
-            self.current_index = (pos + 1) % len(self.entries)
+            self.current_index = pos
         except ValueError:
             pass
 
@@ -179,17 +184,63 @@ class ProxyInterface:
             return
 
         try:
-            logger.info(f"Current IP: {self.check_current_ip()}")
+            logger.debug(f"Current IP: {self.check_current_ip()}")
             with Controller.from_port(port=9051) as controller:
                 controller.authenticate(password=self.tor_password)
                 controller.signal(Signal.NEWNYM)
 
             time.sleep(5)
-            logger.info(f"New IP: {self.check_current_ip()}")
+            logger.debug(f"New IP: {self.check_current_ip()}")
         except Exception as ex:
             logger.error("Failed to renew Tor IP")
             logger.error(ex)
 
+    def wait_for_new_ip(self, timeout=600, interval=5, check_timeout=5):
+        """
+        Wait for the IP address of the currently selected proxy to change.
+
+        :param timeout: Max seconds to wait for IP change
+        :param interval: Seconds between IP checks
+        :param check_timeout: Timeout for individual IP check requests
+        :return: The selected entry (unchanged)
+        :raises RuntimeError: If no proxy is available or baseline cannot be determined
+        :raises EnsureNewIPTimeoutError: If IP doesn't change within timeout
+        """
+        # Use currently selected proxy
+        if self.current_index == -1:
+            # Tor is selected
+            entry = ("127.0.0.1", "9050", None, None)
+        elif self.current_index >= 0 and self.current_index < len(self.entries):
+            # current_index points to the selected entry
+            entry = self.entries[self.current_index]
+        else:
+            # No valid selection, select one
+            logger.debug("No proxy currently selected, selecting one for IP waiting")
+            self.get_proxies(raw=True)
+            if self.current_index == -1:
+                entry = ("127.0.0.1", "9050", None, None)
+            elif self.current_index >= 0 and self.current_index < len(self.entries):
+                entry = self.entries[self.current_index]
+            else:
+                raise RuntimeError("Could not select a proxy for IP waiting")
+
+        # Auto-detect baseline IP
+        host, port, user, pwd = entry
+        proxies_map = {
+            "http": self.get_proxy_url(host, port, user, pwd, "http"),
+            "https": self.get_proxy_url(host, port, user, pwd, "http"),
+        }
+        try:
+            resp = requests.get(self.CHECK_IP_URL, proxies=proxies_map, timeout=check_timeout)
+            baseline = resp.json().get("YourFuckingIPAddress")
+            if not baseline:
+                raise RuntimeError(f"Could not determine baseline IP for entry {host}:{port}")
+            logger.debug(f"Auto-detected baseline IP: {baseline}")
+        except Exception as ex:
+            raise RuntimeError(f"Could not determine baseline IP for entry {host}:{port}: {ex}") from ex
+
+        return self._wait_for_new_ip(entry, baseline, timeout, interval, check_timeout)
+
     def mark_entry_status(self, entry, ok, error=None, last_ip=None):
         """Update health cache for an entry."""
         self._health[entry] = {
@@ -231,6 +282,13 @@ class ProxyInterface:
         pool = self._build_pool(use_auth)
         candidates = self._get_candidates(pool, randomize)
 
+        # Capture baseline before any health checks that might set it
+        baseline_before = None
+        if len(candidates) == 1:
+            idx = candidates[0]
+            entry = self.entries[idx]
+            baseline_before = self._health.get(entry, {}).get("last_ip")
+
         def _find_working_entry():
             if not self.entries:
                 raise NoWorkingProxiesError("No proxies available")
@@ -246,32 +304,28 @@ class ProxyInterface:
             entry = _find_working_entry()
 
         if ensure_new_ip and len(pool) == 1:
-            logger.info(f"ensure_new_ip=True and single proxy, waiting for IP change: {entry[0]}:{entry[1]}")
             baseline = self._health.get(entry, {}).get("last_ip")
-            if not baseline:
-                if not self.is_entry_alive(entry, timeout=check_timeout):
-                    raise NoWorkingProxiesError("Proxy became unavailable during ensure_new_ip")
-                baseline = self._health.get(entry, {}).get("last_ip")
-            entry = self._wait_for_new_ip(entry, baseline, ensure_new_ip_timeout, ensure_new_ip_interval, check_timeout)
+            if not baseline_before:
+                # First time seeing this proxy: it was already checked in _find_working_entry, so return immediately
+                logger.debug("No baseline IP found; returning first-working proxy without waiting for IP change")
+            else:
+                # There is a baseline: wait for the IP to change
+                logger.info(f"ensure_new_ip=True and single proxy, waiting for IP change: {entry[0]}:{entry[1]}")
+                entry = self._wait_for_new_ip(
+                    entry, baseline, ensure_new_ip_timeout, ensure_new_ip_interval, check_timeout
+                )
 
         return entry
 
     def _get_round_robin_candidates(self, pool):
         """Get candidates in round-robin order starting from current_index."""
         candidates = []
-        start_idx = self.current_index
+        start_idx = (self.current_index + 1) % len(self.entries) if self.current_index >= 0 else 0
         for i in range(len(self.entries)):
             idx = (start_idx + i) % len(self.entries)
             entry = self.entries[idx]
             if entry in pool:
-                candidates.append(entry)
-        # Update current_index for next call
-        if candidates:
-            try:
-                pos = self.entries.index(candidates[0])
-                self.current_index = (pos + 1) % len(self.entries)
-            except ValueError:
-                pass
+                candidates.append(idx)
         return candidates
 
     def _build_pool(self, use_auth):
@@ -282,12 +336,14 @@ class ProxyInterface:
 
     def _get_candidates(self, pool, randomize):
         if randomize:
-            return pool[:]
+            candidates = [idx for idx, entry in enumerate(self.entries) if entry in pool]
+            return random.sample(candidates, k=len(candidates))
         else:
             return self._get_round_robin_candidates(pool)
 
     def _find_working_entry_once(self, candidates, check_timeout, cooldown_seconds):
-        for entry in candidates:
+        for idx in candidates:
+            entry = self.entries[idx]
             health = self._health.get(entry, {})
             last_checked = health.get("last_checked", 0)
             ok = health.get("ok", False)
@@ -295,18 +351,22 @@ class ProxyInterface:
 
             if ok and (now - last_checked) < cooldown_seconds:
                 logger.debug(f"Using cached working proxy: {entry[0]}:{entry[1]}")
+                self.current_index = idx
                 return entry
             elif not ok and (now - last_checked) < cooldown_seconds:
                 continue
             else:
                 logger.debug(f"Checking proxy health: {entry[0]}:{entry[1]}")
                 if self.is_entry_alive(entry, timeout=check_timeout):
+                    self.current_index = idx
                     return entry
 
         logger.warning("No cached working proxies, forcing fresh checks")
-        for entry in candidates:
+        for idx in candidates:
+            entry = self.entries[idx]
             logger.debug(f"Force-checking proxy: {entry[0]}:{entry[1]}")
             if self.is_entry_alive(entry, timeout=check_timeout):
+                self.current_index = idx
                 return entry
 
         raise NoWorkingProxiesError("No working proxies available")