datamarket 0.7.107__py3-none-any.whl → 0.7.109__py3-none-any.whl

datamarket/interfaces/proxy.py

@@ -1,7 +1,11 @@
+########################################################################################################################
+# IMPORTS
+
 import logging
 import random
 import time
 from datetime import timedelta
+from functools import partial
 
 import requests
 import tenacity
@@ -10,12 +14,18 @@ from stem.control import Controller
 
 from datamarket.exceptions import EnsureNewIPTimeoutError, NoWorkingProxiesError
 
+########################################################################################################################
+# SETUP
+
 logger = logging.getLogger(__name__)
 logging.getLogger("stem").setLevel(logging.WARNING)
 
 PROXY_ROTATION_INTERVAL = timedelta(minutes=10)
 PROXY_ROTATION_TIMEOUT_SECONDS = int(PROXY_ROTATION_INTERVAL.total_seconds())
 
+########################################################################################################################
+# CLASSES
+
 
 class ProxyInterface:
     """
@@ -237,26 +247,36 @@ class ProxyInterface:
         cooldown_seconds=30,
         proxy_rotation_interval=PROXY_ROTATION_INTERVAL,
     ):
-        """Get a working proxy entry, performing health checks as needed."""
+        """Get a working proxy entry, performing health checks as needed.
+
+        - Fails fast if there are no entries.
+        - Optionally retries for up to `proxy_rotation_interval`,
+        refreshing the traversal queue before each attempt.
+        """
+
+        if not self.entries:
+            raise NoWorkingProxiesError("No proxies available")
+
         pool = self._build_pool(use_auth)
         self._refresh_traversal_queue(pool, randomize)
 
-        def _find_working_entry():
-            if not self.entries:
-                raise NoWorkingProxiesError("No proxies available")
-            return self._find_working_entry_once(check_timeout, cooldown_seconds)
-
-        # Handle both timedelta and numeric seconds for backward compatibility
-        if proxy_rotation_interval:
-            retrying = tenacity.Retrying(
-                stop=tenacity.stop_after_delay(proxy_rotation_interval),
-                reraise=True,
-            )
-            entry = retrying(_find_working_entry)
-        else:
-            entry = _find_working_entry()
+        find_once = partial(self._find_working_entry_once, check_timeout, cooldown_seconds)
+
+        if not proxy_rotation_interval:
+            return find_once()
 
-        return entry
+        def before_sleep(retry_state):
+            tenacity.before_sleep_log(logger, logging.INFO)(retry_state)
+            self._refresh_traversal_queue(pool, randomize)
+
+        retrying = tenacity.Retrying(
+            wait=tenacity.wait_fixed(cooldown_seconds),
+            stop=tenacity.stop_after_delay(proxy_rotation_interval),
+            before_sleep=before_sleep,
+            retry=tenacity.retry_if_exception_type(NoWorkingProxiesError),
+            reraise=True,
+        )
+        return retrying(find_once)
 
     def _get_round_robin_candidates(self, pool):
         """Get candidates in round-robin order starting from current_index."""
@@ -318,9 +338,11 @@ class ProxyInterface:
                     self._traversal_queue.remove(idx)
                     return entry
 
+        self._traversal_queue = []
         raise NoWorkingProxiesError("No working proxies available")
 
     def _wait_for_new_ip(self, entry, baseline, timeout, interval, check_timeout):
+        logger.info("Refreshing proxy IP...")
         start = time.time()
         while time.time() - start < timeout:
             host, port, user, pwd = entry

{datamarket-0.7.107.dist-info → datamarket-0.7.109.dist-info}/METADATA

@@ -1,16 +1,17 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.107
+Version: 0.7.109
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
 Author-email: techsupport@datamarket.es
-Requires-Python: >=3.12,<3.13
+Requires-Python: >=3.12,<4.0
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
 Provides-Extra: boto3

{datamarket-0.7.107.dist-info → datamarket-0.7.109.dist-info}/RECORD

@@ -9,7 +9,7 @@ datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjH
 datamarket/interfaces/ftp.py,sha256=LH3Oz19k_xUNhzDXcrq5Ofb4c3uiph5pWUqpgiaDvHI,2671
 datamarket/interfaces/nominatim.py,sha256=xizT94tVum7QPppfDgI5sEhx1mAXT-SM3JyPl8CDxxU,15148
 datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspXzA,23640
-datamarket/interfaces/proxy.py,sha256=Hxwmkii3wP3oplg4yMhr_NF6ru8tmz7z8jHld4SfPRw,14325
+datamarket/interfaces/proxy.py,sha256=64r5os8yIRpG_f4HxdGXLIRguzVEErG-GRAF8vcH58Y,15171
 datamarket/interfaces/tinybird.py,sha256=cNG-kAPTdQn2inlNX9LPf-VVdtnLud947ApLVO40Now,2594
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/params/nominatim.py,sha256=S9TEB4FxmffvFyK9KffWl20TfXzWX69IAdbEehKar1I,11920
@@ -29,7 +29,7 @@ datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnm
 datamarket/utils/strings/standardization.py,sha256=j_NbT-O1XnxDvDhct8panfkrfAC8R5OX6XM5fYBZ4RU,1496
 datamarket/utils/typer.py,sha256=geWuwMwGQjBQhxo27hX0vEAeRl1j1TS0u2oFVfpAs5I,816
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.107.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.107.dist-info/METADATA,sha256=F0zxym6rN2EWf3jh_1ilKdrIXMqdd_D4py1MzuYzuLA,7382
-datamarket-0.7.107.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.107.dist-info/RECORD,,
+datamarket-0.7.109.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.109.dist-info/METADATA,sha256=Q01pb6jVLuJFXAYqT_2qNgUKBUlbu89RI_Fj4aIL_I4,7432
+datamarket-0.7.109.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.109.dist-info/RECORD,,