reqstorm 2.4.0__tar.gz → 2.4.2__tar.gz

{reqstorm-2.4.0/reqstorm.egg-info → reqstorm-2.4.2}/PKG-INFO

@@ -1,17 +1,22 @@
 Metadata-Version: 2.4
 Name: reqstorm
-Version: 2.4.0
-Summary: Send large numbers of HTTP requests concurrently with asyncio, with retries, timeouts and per-request results.
+Version: 2.4.2
+Summary: Send thousands of HTTP requests from Python or the command line, with per-host rate limits, retries, pagination, progress and logs, and results written to files or databases.
 Author-email: Melih Colpan <colpanmelih@gmail.com>
 License-Expression: MIT
 Project-URL: Homepage, https://reqstorm.github.io
+Project-URL: Documentation, https://reqstorm.github.io
 Project-URL: Source, https://github.com/melihcolpan/reqstorm
 Project-URL: Changelog, https://github.com/melihcolpan/reqstorm/blob/master/CHANGELOG.md
 Project-URL: Issues, https://github.com/melihcolpan/reqstorm/issues
-Keywords: asyncio,aiohttp,http,requests,concurrent,bulk,rate-limit,retry,crawler,scraping
+Keywords: http,http-client,asyncio,aiohttp,bulk-requests,batch,concurrency,rate-limit,retry,backoff,pagination,api-client,cli,socks,jsonl,sqlite,postgresql
 Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
 Classifier: Framework :: AsyncIO
+Classifier: Framework :: Pydantic :: 2
 Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -20,7 +25,10 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
 Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -144,11 +152,35 @@ reqstorm does all of that for you, with one call.
 
 ## Installation
 
+**In your Python code:**
+
 ```console
 $ python -m pip install reqstorm
 ```
 
-reqstorm supports Python 3.9 to 3.13 on Linux, macOS and Windows. Its only dependency is [aiohttp](https://docs.aiohttp.org). For PostgreSQL or MySQL output, install the driver you already use (`psycopg`, `psycopg2`, `pymysql`, `mysqlclient` or `mysql-connector-python`). To use Pydantic models as schemas, install `reqstorm[pydantic]`; for SOCKS proxies, `reqstorm[socks]`.
+**As a command** (`reqstorm urls.txt -o out.jsonl`), pick what you already use:
+
+| You have | Install with |
+|---|---|
+| Homebrew (macOS, Linux) | `brew install melihcolpan/tap/reqstorm` |
+| pipx | `pipx install "reqstorm[socks]"` |
+| uv | `uv tool install "reqstorm[socks]"` |
+| Docker | nothing to install, see below |
+
+```console
+$ docker run --rm -v "$PWD:/data" \
+    ghcr.io/melihcolpan/reqstorm \
+    urls.txt -o results.jsonl
+```
+
+Python 3.9 to 3.13 on Linux, macOS and Windows. The only required dependency is [aiohttp](https://docs.aiohttp.org). Optional extras:
+
+| Extra | Adds |
+|---|---|
+| `reqstorm[socks]` | SOCKS4/5 proxies |
+| `reqstorm[pydantic]` | Pydantic models as schemas |
+
+For PostgreSQL or MySQL output, install the driver you already use (`psycopg`, `psycopg2`, `pymysql`, `mysqlclient` or `mysql-connector-python`).
 
 ## Quick start
 

{reqstorm-2.4.0 → reqstorm-2.4.2}/README.md

@@ -99,11 +99,35 @@ reqstorm does all of that for you, with one call.
 
 ## Installation
 
+**In your Python code:**
+
 ```console
 $ python -m pip install reqstorm
 ```
 
-reqstorm supports Python 3.9 to 3.13 on Linux, macOS and Windows. Its only dependency is [aiohttp](https://docs.aiohttp.org). For PostgreSQL or MySQL output, install the driver you already use (`psycopg`, `psycopg2`, `pymysql`, `mysqlclient` or `mysql-connector-python`). To use Pydantic models as schemas, install `reqstorm[pydantic]`; for SOCKS proxies, `reqstorm[socks]`.
+**As a command** (`reqstorm urls.txt -o out.jsonl`), pick what you already use:
+
+| You have | Install with |
+|---|---|
+| Homebrew (macOS, Linux) | `brew install melihcolpan/tap/reqstorm` |
+| pipx | `pipx install "reqstorm[socks]"` |
+| uv | `uv tool install "reqstorm[socks]"` |
+| Docker | nothing to install, see below |
+
+```console
+$ docker run --rm -v "$PWD:/data" \
+    ghcr.io/melihcolpan/reqstorm \
+    urls.txt -o results.jsonl
+```
+
+Python 3.9 to 3.13 on Linux, macOS and Windows. The only required dependency is [aiohttp](https://docs.aiohttp.org). Optional extras:
+
+| Extra | Adds |
+|---|---|
+| `reqstorm[socks]` | SOCKS4/5 proxies |
+| `reqstorm[pydantic]` | Pydantic models as schemas |
+
+For PostgreSQL or MySQL output, install the driver you already use (`psycopg`, `psycopg2`, `pymysql`, `mysqlclient` or `mysql-connector-python`).
 
 ## Quick start
 

{reqstorm-2.4.0 → reqstorm-2.4.2}/pyproject.toml

@@ -5,18 +5,23 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "reqstorm"
 dynamic = ["version"]
-description = "Send large numbers of HTTP requests concurrently with asyncio, with retries, timeouts and per-request results."
+description = "Send thousands of HTTP requests from Python or the command line, with per-host rate limits, retries, pagination, progress and logs, and results written to files or databases."
 readme = "README.md"
 license = "MIT"
 license-files = ["LICENSE"]
 authors = [{ name = "Melih Colpan", email = "colpanmelih@gmail.com" }]
 requires-python = ">=3.9"
 dependencies = ["aiohttp>=3.9,<4"]
-keywords = ["asyncio", "aiohttp", "http", "requests", "concurrent", "bulk", "rate-limit", "retry", "crawler", "scraping"]
+keywords = ["http", "http-client", "asyncio", "aiohttp", "bulk-requests", "batch", "concurrency", "rate-limit",
+            "retry", "backoff", "pagination", "api-client", "cli", "socks", "jsonl", "sqlite", "postgresql"]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
     "Framework :: AsyncIO",
+    "Framework :: Pydantic :: 2",
     "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: System Administrators",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
@@ -25,7 +30,10 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
     "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Utilities",
     "Typing :: Typed",
 ]
 
@@ -40,6 +48,7 @@ reqstorm = "reqstorm._cli:main"
 
 [project.urls]
 Homepage = "https://reqstorm.github.io"
+Documentation = "https://reqstorm.github.io"
 Source = "https://github.com/melihcolpan/reqstorm"
 Changelog = "https://github.com/melihcolpan/reqstorm/blob/master/CHANGELOG.md"
 Issues = "https://github.com/melihcolpan/reqstorm/issues"

{reqstorm-2.4.0 → reqstorm-2.4.2}/reqstorm/__init__.py

@@ -33,7 +33,7 @@ from ._schema import Field, Schema, SchemaError, extract, infer_schema
 from ._sync import fetch_all_sync, fetch_to_db_sync, fetch_to_file_sync, stream_sync
 from ._template import from_template, read_csv, read_sql
 
-__version__ = "2.4.0"
+__version__ = "2.4.2"
 
 __all__ = [
     "Attempt",

{reqstorm-2.4.0 → reqstorm-2.4.2}/reqstorm/_client.py

@@ -35,7 +35,7 @@ from multidict import CIMultiDict, CIMultiDictProxy
 from ._adaptive import AdaptiveRateLimiter
 from ._limits import HostRateLimiter, RateLimit, host_key
 from ._observe import LogLevel, LogTarget, ProgressTarget, Run, describe
-from ._socks import SocksSessions, check_available, is_socks
+from ._socks import SocksSessions, check_available, is_proxy_error, is_socks
 
 __all__ = ["Attempt", "HTTPStatusError", "Request", "Result", "Results", "fetch_all", "stream"]
 
@@ -284,7 +284,7 @@ def _resolve(request: Union[str, Request], options: _Options) -> Request:
 def _is_retryable_error(error: BaseException) -> bool:
     if isinstance(error, (aiohttp.InvalidURL, ValueError)):
         return False
-    return isinstance(error, (aiohttp.ClientError, asyncio.TimeoutError))
+    return isinstance(error, (aiohttp.ClientError, asyncio.TimeoutError)) or is_proxy_error(error)
 
 
 def _should_retry(result: Result, retry_statuses: frozenset) -> bool:

{reqstorm-2.4.0 → reqstorm-2.4.2}/reqstorm/_socks.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import importlib
 from typing import Any, Dict, Optional, Tuple
 from urllib.parse import unquote, urlsplit
 
@@ -15,6 +16,24 @@ def is_socks(proxy: Optional[str]) -> bool:
     return proxy is not None and urlsplit(proxy).scheme.lower() in SCHEMES
 
 
+def is_proxy_error(error: BaseException) -> bool:
+    """A failure of a SOCKS proxy: unreachable, refused, timed out or a bad reply.
+
+    These are worth retrying: the next attempt may use another proxy from the pool, or the
+    proxy may be back.
+    """
+    kinds: Tuple[type, ...] = ()
+    # aiohttp-socks raises its own classes; python-socks has classes of the same names
+    for module in ("aiohttp_socks", "python_socks"):
+        try:
+            package = importlib.import_module(module)
+        except ImportError:
+            continue
+        names = ("ProxyError", "ProxyConnectionError", "ProxyTimeoutError")
+        kinds += tuple(getattr(package, name) for name in names if hasattr(package, name))
+    return bool(kinds) and isinstance(error, kinds)
+
+
 def check_available() -> None:
     try:
         import aiohttp_socks  # noqa: F401

{reqstorm-2.4.0 → reqstorm-2.4.2/reqstorm.egg-info}/PKG-INFO

@@ -1,17 +1,22 @@
 Metadata-Version: 2.4
 Name: reqstorm
-Version: 2.4.0
-Summary: Send large numbers of HTTP requests concurrently with asyncio, with retries, timeouts and per-request results.
+Version: 2.4.2
+Summary: Send thousands of HTTP requests from Python or the command line, with per-host rate limits, retries, pagination, progress and logs, and results written to files or databases.
 Author-email: Melih Colpan <colpanmelih@gmail.com>
 License-Expression: MIT
 Project-URL: Homepage, https://reqstorm.github.io
+Project-URL: Documentation, https://reqstorm.github.io
 Project-URL: Source, https://github.com/melihcolpan/reqstorm
 Project-URL: Changelog, https://github.com/melihcolpan/reqstorm/blob/master/CHANGELOG.md
 Project-URL: Issues, https://github.com/melihcolpan/reqstorm/issues
-Keywords: asyncio,aiohttp,http,requests,concurrent,bulk,rate-limit,retry,crawler,scraping
+Keywords: http,http-client,asyncio,aiohttp,bulk-requests,batch,concurrency,rate-limit,retry,backoff,pagination,api-client,cli,socks,jsonl,sqlite,postgresql
 Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
 Classifier: Framework :: AsyncIO
+Classifier: Framework :: Pydantic :: 2
 Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -20,7 +25,10 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
 Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -144,11 +152,35 @@ reqstorm does all of that for you, with one call.
 
 ## Installation
 
+**In your Python code:**
+
 ```console
 $ python -m pip install reqstorm
 ```
 
-reqstorm supports Python 3.9 to 3.13 on Linux, macOS and Windows. Its only dependency is [aiohttp](https://docs.aiohttp.org). For PostgreSQL or MySQL output, install the driver you already use (`psycopg`, `psycopg2`, `pymysql`, `mysqlclient` or `mysql-connector-python`). To use Pydantic models as schemas, install `reqstorm[pydantic]`; for SOCKS proxies, `reqstorm[socks]`.
+**As a command** (`reqstorm urls.txt -o out.jsonl`), pick what you already use:
+
+| You have | Install with |
+|---|---|
+| Homebrew (macOS, Linux) | `brew install melihcolpan/tap/reqstorm` |
+| pipx | `pipx install "reqstorm[socks]"` |
+| uv | `uv tool install "reqstorm[socks]"` |
+| Docker | nothing to install, see below |
+
+```console
+$ docker run --rm -v "$PWD:/data" \
+    ghcr.io/melihcolpan/reqstorm \
+    urls.txt -o results.jsonl
+```
+
+Python 3.9 to 3.13 on Linux, macOS and Windows. The only required dependency is [aiohttp](https://docs.aiohttp.org). Optional extras:
+
+| Extra | Adds |
+|---|---|
+| `reqstorm[socks]` | SOCKS4/5 proxies |
+| `reqstorm[pydantic]` | Pydantic models as schemas |
+
+For PostgreSQL or MySQL output, install the driver you already use (`psycopg`, `psycopg2`, `pymysql`, `mysqlclient` or `mysql-connector-python`).
 
 ## Quick start
 

{reqstorm-2.4.0 → reqstorm-2.4.2}/tests/test_socks_backoff.py

@@ -142,3 +142,23 @@ async def test_max_backoff_limits_the_real_wait(server):
 async def test_negative_backoff_is_rejected(server):
     with pytest.raises(ValueError, match="backoff"):
         await reqstorm.fetch_all([server + "/ok"], max_backoff=-1)
+
+
+async def test_unreachable_socks_proxy_is_retried_with_the_next_one(server, socks_proxy):
+    dead = "socks5://127.0.0.1:9"
+    results = await reqstorm.fetch_all(
+        [server + "/ok"], proxy=[dead, socks_proxy.url()], retries=1, backoff=0
+    )
+    assert results[0].text() == "ok" and results[0].attempts == 2
+    assert results[0].history[0].error is not None and len(socks_proxy.requests) == 1
+
+
+async def test_proxy_failures_count_as_retryable_errors():
+    import aiohttp_socks
+    import python_socks
+
+    from reqstorm._client import _is_retryable_error
+
+    for package in (aiohttp_socks, python_socks):
+        for name in ("ProxyError", "ProxyConnectionError", "ProxyTimeoutError"):
+            assert _is_retryable_error(getattr(package, name)("x")), (package.__name__, name)