reqstorm 2.4.0__tar.gz → 2.4.1__tar.gz

{reqstorm-2.4.0/reqstorm.egg-info → reqstorm-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reqstorm
-Version: 2.4.0
+Version: 2.4.1
 Summary: Send large numbers of HTTP requests concurrently with asyncio, with retries, timeouts and per-request results.
 Author-email: Melih Colpan <colpanmelih@gmail.com>
 License-Expression: MIT

{reqstorm-2.4.0 → reqstorm-2.4.1}/reqstorm/__init__.py

@@ -33,7 +33,7 @@ from ._schema import Field, Schema, SchemaError, extract, infer_schema
 from ._sync import fetch_all_sync, fetch_to_db_sync, fetch_to_file_sync, stream_sync
 from ._template import from_template, read_csv, read_sql
 
-__version__ = "2.4.0"
+__version__ = "2.4.1"
 
 __all__ = [
     "Attempt",

{reqstorm-2.4.0 → reqstorm-2.4.1}/reqstorm/_client.py

@@ -35,7 +35,7 @@ from multidict import CIMultiDict, CIMultiDictProxy
 from ._adaptive import AdaptiveRateLimiter
 from ._limits import HostRateLimiter, RateLimit, host_key
 from ._observe import LogLevel, LogTarget, ProgressTarget, Run, describe
-from ._socks import SocksSessions, check_available, is_socks
+from ._socks import SocksSessions, check_available, is_proxy_error, is_socks
 
 __all__ = ["Attempt", "HTTPStatusError", "Request", "Result", "Results", "fetch_all", "stream"]
 
@@ -284,7 +284,7 @@ def _resolve(request: Union[str, Request], options: _Options) -> Request:
 def _is_retryable_error(error: BaseException) -> bool:
     if isinstance(error, (aiohttp.InvalidURL, ValueError)):
         return False
-    return isinstance(error, (aiohttp.ClientError, asyncio.TimeoutError))
+    return isinstance(error, (aiohttp.ClientError, asyncio.TimeoutError)) or is_proxy_error(error)
 
 
 def _should_retry(result: Result, retry_statuses: frozenset) -> bool:

{reqstorm-2.4.0 → reqstorm-2.4.1}/reqstorm/_socks.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import importlib
 from typing import Any, Dict, Optional, Tuple
 from urllib.parse import unquote, urlsplit
 
@@ -15,6 +16,24 @@ def is_socks(proxy: Optional[str]) -> bool:
     return proxy is not None and urlsplit(proxy).scheme.lower() in SCHEMES
 
 
+def is_proxy_error(error: BaseException) -> bool:
+    """A failure of a SOCKS proxy: unreachable, refused, timed out or a bad reply.
+
+    These are worth retrying: the next attempt may use another proxy from the pool, or the
+    proxy may be back.
+    """
+    kinds: Tuple[type, ...] = ()
+    # aiohttp-socks raises its own classes; python-socks has classes of the same names
+    for module in ("aiohttp_socks", "python_socks"):
+        try:
+            package = importlib.import_module(module)
+        except ImportError:
+            continue
+        names = ("ProxyError", "ProxyConnectionError", "ProxyTimeoutError")
+        kinds += tuple(getattr(package, name) for name in names if hasattr(package, name))
+    return bool(kinds) and isinstance(error, kinds)
+
+
 def check_available() -> None:
     try:
         import aiohttp_socks  # noqa: F401

{reqstorm-2.4.0 → reqstorm-2.4.1/reqstorm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reqstorm
-Version: 2.4.0
+Version: 2.4.1
 Summary: Send large numbers of HTTP requests concurrently with asyncio, with retries, timeouts and per-request results.
 Author-email: Melih Colpan <colpanmelih@gmail.com>
 License-Expression: MIT

{reqstorm-2.4.0 → reqstorm-2.4.1}/tests/test_socks_backoff.py

@@ -142,3 +142,23 @@ async def test_max_backoff_limits_the_real_wait(server):
 async def test_negative_backoff_is_rejected(server):
     with pytest.raises(ValueError, match="backoff"):
         await reqstorm.fetch_all([server + "/ok"], max_backoff=-1)
+
+
+async def test_unreachable_socks_proxy_is_retried_with_the_next_one(server, socks_proxy):
+    dead = "socks5://127.0.0.1:9"
+    results = await reqstorm.fetch_all(
+        [server + "/ok"], proxy=[dead, socks_proxy.url()], retries=1, backoff=0
+    )
+    assert results[0].text() == "ok" and results[0].attempts == 2
+    assert results[0].history[0].error is not None and len(socks_proxy.requests) == 1
+
+
+async def test_proxy_failures_count_as_retryable_errors():
+    import aiohttp_socks
+    import python_socks
+
+    from reqstorm._client import _is_retryable_error
+
+    for package in (aiohttp_socks, python_socks):
+        for name in ("ProxyError", "ProxyConnectionError", "ProxyTimeoutError"):
+            assert _is_retryable_error(getattr(package, name)("x")), (package.__name__, name)