proxyspin 0.1.0__tar.gz

proxyspin-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.egg-info/
+dist/
+.venv/

proxyspin-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GProxy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

proxyspin-0.1.0/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: proxyspin
+Version: 0.1.0
+Summary: Rotating proxy pool for Scrapy, Playwright and requests — health tracking, ban detection, sticky sessions. Zero dependencies.
+Project-URL: Homepage, https://github.com/gproxynet/proxyspin
+Project-URL: Issues, https://github.com/gproxynet/proxyspin/issues
+Author-email: GProxy <support@gproxy.net>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: anti-ban,playwright,proxy,proxy-pool,proxy-rotation,requests,rotating-proxies,scrapy,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Scrapy
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Provides-Extra: playwright
+Requires-Dist: playwright>=1.30; extra == 'playwright'
+Provides-Extra: requests
+Requires-Dist: requests>=2.25; extra == 'requests'
+Provides-Extra: scrapy
+Requires-Dist: scrapy>=2.5; extra == 'scrapy'
+Description-Content-Type: text/markdown
+
+# proxyspin
+
+Rotating proxy pool for **Scrapy**, **Playwright** and **requests** — with health tracking, ban detection, sticky sessions and a built-in list checker. Zero required dependencies, pure stdlib.
+
+```
+pip install proxyspin
+```
+
+Why another one? The classic `scrapy-rotating-proxies` hasn't been updated in years and is Scrapy-only. `proxyspin` is one small pool you can share across Scrapy spiders, Playwright contexts and plain `requests` code, with the same health model everywhere.
+
+## Features
+
+- **One pool, three integrations** — Scrapy middleware, Playwright helper, `requests` session.
+- **Rotation strategies** — `round_robin`, `random`, `sticky` (same proxy per domain/account/worker until it dies).
+- **Health model** — a proxy failing N times in a row is benched with exponential backoff and returns automatically; a success resets its streak.
+- **Ban detection** — configurable HTTP codes (403/407/429 by default) count as proxy failures and trigger a retry through the next proxy.
+- **Any list format** — `scheme://user:pass@host:port`, `host:port`, `host:port:user:pass`, `user:pass@host:port`; load from a list, a file or a provider URL.
+- **CLI checker** — `proxyspin check proxies.txt` tests the whole list concurrently and can write the alive ones out.
+
+## Quickstart
+
+### The pool
+
+```python
+from proxyspin import ProxyPool
+
+pool = ProxyPool.from_file("proxies.txt", strategy="round_robin")
+# or inline:
+pool = ProxyPool(["http://user:pass@gate1.example.com:8000", "10.0.0.2:8000"])
+# or straight from your provider's export endpoint:
+pool = ProxyPool.from_url("https://provider.example.com/api/my-list.txt")
+
+proxy = pool.get()          # -> Proxy; proxy.url is ready to use
+pool.mark_failed(proxy)     # bench it after repeated failures
+pool.mark_ok(proxy)         # reset its failure streak
+```
+
+### Scrapy
+
+```python
+# settings.py
+DOWNLOADER_MIDDLEWARES = {
+    "proxyspin.scrapy_middleware.ProxySpinMiddleware": 610,
+}
+PROXYSPIN_FILE = "proxies.txt"
+PROXYSPIN_STRATEGY = "sticky"          # one proxy per target host
+PROXYSPIN_BAN_CODES = [403, 429]       # these responses rotate the proxy
+PROXYSPIN_MAX_RETRIES = 3
+```
+
+Per-request overrides: set `request.meta["proxy"]` to pin a proxy, `meta["proxyspin_disabled"] = True` to skip proxying, `meta["proxyspin_key"]` to control stickiness.
+
+### Playwright
+
+```python
+from playwright.sync_api import sync_playwright
+from proxyspin import ProxyPool
+from proxyspin.playwright_helper import proxy_settings
+
+pool = ProxyPool.from_file("proxies.txt", strategy="sticky")
+
+with sync_playwright() as p:
+    browser = p.chromium.launch()
+    for account in accounts:
+        context = browser.new_context(proxy=proxy_settings(pool, key=account.id))
+        # each account keeps its own IP for the whole session
+```
+
+### requests
+
+```python
+from proxyspin import ProxyPool
+from proxyspin.requests_adapter import RotatingSession
+
+session = RotatingSession(ProxyPool.from_file("proxies.txt"))
+print(session.get("https://httpbin.org/ip").json())   # new IP per call
+```
+
+### Check a list
+
+```
+$ proxyspin check proxies.txt --workers 100 --alive-out alive.txt
+OK   45.155.10.4:8000        612 ms  HTTP 200
+DEAD 91.10.77.2:3128                 TimeoutError
+...
+118/200 alive
+wrote 118 proxies to alive.txt
+```
+
+## Providers
+
+`proxyspin` works with any proxy source: your own servers, free lists, or commercial providers. Example with [GProxy](https://gproxy.net/?utm_source=github&utm_medium=readme&utm_campaign=proxyspin) residential/mobile gateways (rotation happens server-side, so a pool of one entry per gateway is enough — use `sticky` if you need session pinning):
+
+```python
+pool = ProxyPool([
+    "http://USER:PASS@gate.gproxy.net:8000",   # residential, rotating
+])
+```
+
+Any other provider works the same way — put its gateway or IP list into the pool.
+
+## Health model in one paragraph
+
+Every proxy starts healthy. `mark_failed` increments its failure streak; when the streak reaches `max_failures` (default 2) the proxy is benched for `cooldown * 2**overshoot` seconds (default base 60 s, capped at 1 h), then automatically rejoins rotation. `mark_ok` resets the streak. The Scrapy middleware and `RotatingSession` call these for you based on exceptions and ban codes; with Playwright you call them yourself since only your code knows what a "ban" looks like for your flow.
+
+## License
+
+MIT

proxyspin-0.1.0/README.md

@@ -0,0 +1,108 @@
+# proxyspin
+
+Rotating proxy pool for **Scrapy**, **Playwright** and **requests** — with health tracking, ban detection, sticky sessions and a built-in list checker. Zero required dependencies, pure stdlib.
+
+```
+pip install proxyspin
+```
+
+Why another one? The classic `scrapy-rotating-proxies` hasn't been updated in years and is Scrapy-only. `proxyspin` is one small pool you can share across Scrapy spiders, Playwright contexts and plain `requests` code, with the same health model everywhere.
+
+## Features
+
+- **One pool, three integrations** — Scrapy middleware, Playwright helper, `requests` session.
+- **Rotation strategies** — `round_robin`, `random`, `sticky` (same proxy per domain/account/worker until it dies).
+- **Health model** — a proxy failing N times in a row is benched with exponential backoff and returns automatically; a success resets its streak.
+- **Ban detection** — configurable HTTP codes (403/407/429 by default) count as proxy failures and trigger a retry through the next proxy.
+- **Any list format** — `scheme://user:pass@host:port`, `host:port`, `host:port:user:pass`, `user:pass@host:port`; load from a list, a file or a provider URL.
+- **CLI checker** — `proxyspin check proxies.txt` tests the whole list concurrently and can write the alive ones out.
+
+## Quickstart
+
+### The pool
+
+```python
+from proxyspin import ProxyPool
+
+pool = ProxyPool.from_file("proxies.txt", strategy="round_robin")
+# or inline:
+pool = ProxyPool(["http://user:pass@gate1.example.com:8000", "10.0.0.2:8000"])
+# or straight from your provider's export endpoint:
+pool = ProxyPool.from_url("https://provider.example.com/api/my-list.txt")
+
+proxy = pool.get()          # -> Proxy; proxy.url is ready to use
+pool.mark_failed(proxy)     # bench it after repeated failures
+pool.mark_ok(proxy)         # reset its failure streak
+```
+
+### Scrapy
+
+```python
+# settings.py
+DOWNLOADER_MIDDLEWARES = {
+    "proxyspin.scrapy_middleware.ProxySpinMiddleware": 610,
+}
+PROXYSPIN_FILE = "proxies.txt"
+PROXYSPIN_STRATEGY = "sticky"          # one proxy per target host
+PROXYSPIN_BAN_CODES = [403, 429]       # these responses rotate the proxy
+PROXYSPIN_MAX_RETRIES = 3
+```
+
+Per-request overrides: set `request.meta["proxy"]` to pin a proxy, `meta["proxyspin_disabled"] = True` to skip proxying, `meta["proxyspin_key"]` to control stickiness.
+
+### Playwright
+
+```python
+from playwright.sync_api import sync_playwright
+from proxyspin import ProxyPool
+from proxyspin.playwright_helper import proxy_settings
+
+pool = ProxyPool.from_file("proxies.txt", strategy="sticky")
+
+with sync_playwright() as p:
+    browser = p.chromium.launch()
+    for account in accounts:
+        context = browser.new_context(proxy=proxy_settings(pool, key=account.id))
+        # each account keeps its own IP for the whole session
+```
+
+### requests
+
+```python
+from proxyspin import ProxyPool
+from proxyspin.requests_adapter import RotatingSession
+
+session = RotatingSession(ProxyPool.from_file("proxies.txt"))
+print(session.get("https://httpbin.org/ip").json())   # new IP per call
+```
+
+### Check a list
+
+```
+$ proxyspin check proxies.txt --workers 100 --alive-out alive.txt
+OK   45.155.10.4:8000        612 ms  HTTP 200
+DEAD 91.10.77.2:3128                 TimeoutError
+...
+118/200 alive
+wrote 118 proxies to alive.txt
+```
+
+## Providers
+
+`proxyspin` works with any proxy source: your own servers, free lists, or commercial providers. Example with [GProxy](https://gproxy.net/?utm_source=github&utm_medium=readme&utm_campaign=proxyspin) residential/mobile gateways (rotation happens server-side, so a pool of one entry per gateway is enough — use `sticky` if you need session pinning):
+
+```python
+pool = ProxyPool([
+    "http://USER:PASS@gate.gproxy.net:8000",   # residential, rotating
+])
+```
+
+Any other provider works the same way — put its gateway or IP list into the pool.
+
+## Health model in one paragraph
+
+Every proxy starts healthy. `mark_failed` increments its failure streak; when the streak reaches `max_failures` (default 2) the proxy is benched for `cooldown * 2**overshoot` seconds (default base 60 s, capped at 1 h), then automatically rejoins rotation. `mark_ok` resets the streak. The Scrapy middleware and `RotatingSession` call these for you based on exceptions and ban codes; with Playwright you call them yourself since only your code knows what a "ban" looks like for your flow.
+
+## License
+
+MIT

proxyspin-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "proxyspin"
+version = "0.1.0"
+description = "Rotating proxy pool for Scrapy, Playwright and requests — health tracking, ban detection, sticky sessions. Zero dependencies."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [{ name = "GProxy", email = "support@gproxy.net" }]
+keywords = [
+  "proxy", "rotating-proxies", "scrapy", "playwright", "requests",
+  "web-scraping", "proxy-rotation", "proxy-pool", "anti-ban",
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+  "Framework :: Scrapy",
+]
+
+[project.optional-dependencies]
+requests = ["requests>=2.25"]
+scrapy = ["scrapy>=2.5"]
+playwright = ["playwright>=1.30"]
+
+[project.scripts]
+proxyspin = "proxyspin.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/gproxynet/proxyspin"
+Issues = "https://github.com/gproxynet/proxyspin/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/proxyspin"]

proxyspin-0.1.0/src/proxyspin/__init__.py

@@ -0,0 +1,7 @@
+"""proxyspin — rotating proxy pool for Scrapy, Playwright and requests."""
+
+from .pool import NoHealthyProxies, ProxyPool
+from .proxy import Proxy, parse_proxy
+
+__version__ = "0.1.0"
+__all__ = ["Proxy", "ProxyPool", "NoHealthyProxies", "parse_proxy", "__version__"]

proxyspin-0.1.0/src/proxyspin/cli.py

@@ -0,0 +1,94 @@
+"""``proxyspin check`` — fast concurrent liveness check for a proxy list.
+
+Zero dependencies: stdlib only (http/https proxies; socks lines are skipped).
+
+::
+
+    proxyspin check proxies.txt
+    proxyspin check proxies.txt --url https://example.com --timeout 5 --workers 100
+    proxyspin check proxies.txt --alive-out alive.txt
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from .proxy import Proxy, parse_proxy
+
+DEFAULT_TEST_URL = "http://httpbin.org/ip"
+
+
+def check_one(proxy: Proxy, url: str, timeout: float) -> tuple[Proxy, float | None, str]:
+    """Return (proxy, latency_seconds_or_None, detail)."""
+    handler = urllib.request.ProxyHandler({"http": proxy.url, "https": proxy.url})
+    opener = urllib.request.build_opener(handler)
+    opener.addheaders = [("User-Agent", "proxyspin-check")]
+    start = time.monotonic()
+    try:
+        with opener.open(url, timeout=timeout) as resp:
+            resp.read(256)
+            return proxy, time.monotonic() - start, f"HTTP {resp.status}"
+    except urllib.error.HTTPError as exc:
+        return proxy, None, f"HTTP {exc.code}"
+    except Exception as exc:
+        return proxy, None, exc.__class__.__name__
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="proxyspin", description=__doc__)
+    sub = parser.add_subparsers(dest="command", required=True)
+    check = sub.add_parser("check", help="check which proxies in a list are alive")
+    check.add_argument("file", help="proxy list, one per line (any common format)")
+    check.add_argument("--url", default=DEFAULT_TEST_URL, help="URL fetched through each proxy")
+    check.add_argument("--timeout", type=float, default=10.0)
+    check.add_argument("--workers", type=int, default=50)
+    check.add_argument("--alive-out", help="write working proxies to this file")
+    args = parser.parse_args(argv)
+
+    proxies: list[Proxy] = []
+    for line in Path(args.file).read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        try:
+            proxy = parse_proxy(line)
+        except ValueError as exc:
+            print(f"  skip: {exc}", file=sys.stderr)
+            continue
+        if proxy.scheme.startswith("socks"):
+            print(f"  skip (socks not supported by checker): {proxy.address}", file=sys.stderr)
+            continue
+        proxies.append(proxy)
+
+    if not proxies:
+        print("no proxies to check", file=sys.stderr)
+        return 2
+
+    alive: list[tuple[Proxy, float]] = []
+    with ThreadPoolExecutor(max_workers=args.workers) as pool:
+        futures = [pool.submit(check_one, p, args.url, args.timeout) for p in proxies]
+        for future in as_completed(futures):
+            proxy, latency, detail = future.result()
+            if latency is not None:
+                alive.append((proxy, latency))
+                print(f"OK   {proxy.address:<21} {latency * 1000:6.0f} ms  {detail}")
+            else:
+                print(f"DEAD {proxy.address:<21}          {detail}")
+
+    print(f"\n{len(alive)}/{len(proxies)} alive")
+    if args.alive_out:
+        alive.sort(key=lambda item: item[1])
+        Path(args.alive_out).write_text(
+            "\n".join(proxy.url for proxy, _ in alive) + ("\n" if alive else "")
+        )
+        print(f"wrote {len(alive)} proxies to {args.alive_out}")
+    return 0 if alive else 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

proxyspin-0.1.0/src/proxyspin/playwright_helper.py

@@ -0,0 +1,38 @@
+"""Playwright integration: per-context rotating proxies.
+
+Playwright accepts a proxy per browser launch or per context. The natural
+rotation unit is the context::
+
+    from playwright.sync_api import sync_playwright
+    from proxyspin import ProxyPool
+    from proxyspin.playwright_helper import proxy_settings
+
+    pool = ProxyPool.from_file("proxies.txt", strategy="sticky")
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch()
+        context = browser.new_context(proxy=proxy_settings(pool, key="job-42"))
+        page = context.new_page()
+        page.goto("https://example.com")
+
+Async API is identical — ``proxy_settings`` is plain data, no I/O.
+"""
+from __future__ import annotations
+
+from .pool import ProxyPool
+from .proxy import Proxy
+
+
+def proxy_settings(pool_or_proxy: ProxyPool | Proxy, key: str | None = None) -> dict:
+    """Build the ``proxy`` dict Playwright expects, from a pool or a proxy.
+
+    Credentials are passed via the dedicated fields (Playwright ignores
+    userinfo embedded in the server URL).
+    """
+    proxy = pool_or_proxy.get(key=key) if isinstance(pool_or_proxy, ProxyPool) else pool_or_proxy
+    settings: dict = {"server": f"{proxy.scheme}://{proxy.host}:{proxy.port}"}
+    if proxy.username is not None:
+        settings["username"] = proxy.username
+    if proxy.password is not None:
+        settings["password"] = proxy.password
+    return settings

proxyspin-0.1.0/src/proxyspin/pool.py

@@ -0,0 +1,167 @@
+"""Proxy pool: sources, rotation strategies and health tracking."""
+from __future__ import annotations
+
+import random
+import threading
+import time
+import urllib.request
+from collections.abc import Iterable
+from pathlib import Path
+
+from .proxy import Proxy, ProxyState, parse_proxy
+
+STRATEGIES = ("round_robin", "random", "sticky")
+
+
+class NoHealthyProxies(RuntimeError):
+    """Every proxy in the pool is currently banned."""
+
+
+class ProxyPool:
+    """Thread-safe rotating proxy pool with failure-based cooldowns.
+
+    Rotation strategies:
+        round_robin  cycle through healthy proxies in order (default)
+        random       pick a healthy proxy at random
+        sticky       keep returning the same proxy for a given ``key``
+                     (e.g. a target domain) until it goes unhealthy
+
+    Health model: a proxy that fails ``max_failures`` times in a row is
+    benched for ``cooldown * 2**(streak-1)`` seconds (capped at
+    ``max_cooldown``), then automatically returns to rotation. A success
+    resets its failure counter.
+    """
+
+    def __init__(
+        self,
+        proxies: Iterable[Proxy | str] = (),
+        *,
+        strategy: str = "round_robin",
+        max_failures: int = 2,
+        cooldown: float = 60.0,
+        max_cooldown: float = 3600.0,
+        default_scheme: str = "http",
+    ) -> None:
+        if strategy not in STRATEGIES:
+            raise ValueError(f"unknown strategy {strategy!r}, expected one of {STRATEGIES}")
+        self.strategy = strategy
+        self.max_failures = max_failures
+        self.cooldown = cooldown
+        self.max_cooldown = max_cooldown
+        self.default_scheme = default_scheme
+        self._lock = threading.Lock()
+        self._states: dict[Proxy, ProxyState] = {}
+        self._order: list[Proxy] = []
+        self._rr_index = 0
+        self._sticky: dict[str, Proxy] = {}
+        self.extend(proxies)
+
+    # ------------------------------------------------------------- sources
+    @classmethod
+    def from_file(cls, path: str | Path, **kwargs) -> "ProxyPool":
+        """One proxy per line; blank lines and ``#`` comments are skipped."""
+        lines = Path(path).read_text().splitlines()
+        return cls([l for l in lines if l.strip() and not l.lstrip().startswith("#")], **kwargs)
+
+    @classmethod
+    def from_url(cls, url: str, *, timeout: float = 15.0, **kwargs) -> "ProxyPool":
+        """Fetch a plain-text proxy list (one per line) from a URL.
+
+        Handy for provider endpoints that export your current proxy list.
+        """
+        with urllib.request.urlopen(url, timeout=timeout) as resp:
+            body = resp.read().decode("utf-8", "replace")
+        lines = [l for l in body.splitlines() if l.strip() and not l.lstrip().startswith("#")]
+        return cls(lines, **kwargs)
+
+    def extend(self, proxies: Iterable[Proxy | str]) -> None:
+        with self._lock:
+            for item in proxies:
+                proxy = item if isinstance(item, Proxy) else parse_proxy(item, self.default_scheme)
+                if proxy not in self._states:
+                    self._states[proxy] = ProxyState(proxy)
+                    self._order.append(proxy)
+
+    # ------------------------------------------------------------ rotation
+    def get(self, key: str | None = None) -> Proxy:
+        """Return the next healthy proxy according to the pool strategy.
+
+        ``key`` is only used by the ``sticky`` strategy (any hashable label:
+        target domain, account id, worker name...).
+        """
+        with self._lock:
+            healthy = self._healthy_locked()
+            if not healthy:
+                raise NoHealthyProxies(
+                    f"all {len(self._order)} proxies are cooling down; "
+                    "retry later or add more proxies"
+                )
+            if self.strategy == "sticky" and key is not None:
+                current = self._sticky.get(key)
+                if current is not None and current in healthy:
+                    return current
+                choice = random.choice(healthy)
+                self._sticky[key] = choice
+                return choice
+            if self.strategy == "random":
+                return random.choice(healthy)
+            self._rr_index = (self._rr_index + 1) % len(healthy)
+            return healthy[self._rr_index]
+
+    # -------------------------------------------------------------- health
+    def mark_ok(self, proxy: Proxy) -> None:
+        with self._lock:
+            state = self._states.get(proxy)
+            if state is not None:
+                state.successes += 1
+                state.failures = 0
+                state.banned_until = 0.0
+
+    def mark_failed(self, proxy: Proxy) -> None:
+        with self._lock:
+            state = self._states.get(proxy)
+            if state is None:
+                return
+            state.failures += 1
+            if state.failures >= self.max_failures:
+                streak = state.failures - self.max_failures
+                delay = min(self.cooldown * (2**streak), self.max_cooldown)
+                state.banned_until = time.monotonic() + delay
+                for key, sticky_proxy in list(self._sticky.items()):
+                    if sticky_proxy == proxy:
+                        del self._sticky[key]
+
+    def remove(self, proxy: Proxy) -> None:
+        with self._lock:
+            self._states.pop(proxy, None)
+            if proxy in self._order:
+                self._order.remove(proxy)
+            for key, sticky_proxy in list(self._sticky.items()):
+                if sticky_proxy == proxy:
+                    del self._sticky[key]
+
+    # --------------------------------------------------------------- stats
+    def _healthy_locked(self) -> list[Proxy]:
+        now = time.monotonic()
+        return [p for p in self._order if self._states[p].banned_until <= now]
+
+    @property
+    def healthy_count(self) -> int:
+        with self._lock:
+            return len(self._healthy_locked())
+
+    def __len__(self) -> int:
+        return len(self._order)
+
+    def stats(self) -> dict[str, dict]:
+        """Snapshot of per-proxy health, keyed by ``host:port``."""
+        now = time.monotonic()
+        with self._lock:
+            return {
+                p.address: {
+                    "successes": s.successes,
+                    "failures": s.failures,
+                    "banned_for": max(0.0, round(s.banned_until - now, 1)),
+                }
+                for p, s in self._states.items()
+            }

proxyspin-0.1.0/src/proxyspin/proxy.py

@@ -0,0 +1,97 @@
+"""Proxy model and parsing of common list formats."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from urllib.parse import quote, unquote, urlsplit
+
+SUPPORTED_SCHEMES = ("http", "https", "socks4", "socks5", "socks5h")
+
+
+@dataclass(frozen=True)
+class Proxy:
+    host: str
+    port: int
+    scheme: str = "http"
+    username: str | None = None
+    password: str | None = None
+
+    @property
+    def url(self) -> str:
+        """Full proxy URL, credentials included if present."""
+        auth = ""
+        if self.username is not None:
+            auth = quote(self.username, safe="")
+            if self.password is not None:
+                auth += ":" + quote(self.password, safe="")
+            auth += "@"
+        return f"{self.scheme}://{auth}{self.host}:{self.port}"
+
+    @property
+    def address(self) -> str:
+        return f"{self.host}:{self.port}"
+
+    def __str__(self) -> str:  # pragma: no cover - convenience
+        return self.url
+
+
+def parse_proxy(line: str, default_scheme: str = "http") -> Proxy:
+    """Parse one proxy from any of the common formats.
+
+    Accepted:
+        scheme://user:pass@host:port
+        scheme://host:port
+        user:pass@host:port
+        host:port
+        host:port:user:pass
+    """
+    line = line.strip()
+    if not line:
+        raise ValueError("empty proxy line")
+
+    if "://" in line:
+        parts = urlsplit(line)
+        if parts.scheme not in SUPPORTED_SCHEMES:
+            raise ValueError(f"unsupported proxy scheme: {parts.scheme!r}")
+        if not parts.hostname or not parts.port:
+            raise ValueError(f"proxy needs host and port: {line!r}")
+        return Proxy(
+            host=parts.hostname,
+            port=parts.port,
+            scheme=parts.scheme,
+            username=unquote(parts.username) if parts.username else None,
+            password=unquote(parts.password) if parts.password else None,
+        )
+
+    if "@" in line:
+        creds, _, hostport = line.rpartition("@")
+        user, _, pwd = creds.partition(":")
+        host, _, port = hostport.rpartition(":")
+        _validate(host, port, line)
+        return Proxy(host, int(port), default_scheme, user, pwd or None)
+
+    pieces = line.split(":")
+    if len(pieces) == 2:
+        host, port = pieces
+        _validate(host, port, line)
+        return Proxy(host, int(port), default_scheme)
+    if len(pieces) == 4:
+        host, port, user, pwd = pieces
+        _validate(host, port, line)
+        return Proxy(host, int(port), default_scheme, user, pwd)
+    raise ValueError(f"cannot parse proxy: {line!r}")
+
+
+def _validate(host: str, port: str, line: str) -> None:
+    if not host or not port.isdigit() or not 0 < int(port) < 65536:
+        raise ValueError(f"cannot parse proxy: {line!r}")
+
+
+@dataclass
+class ProxyState:
+    """Mutable health bookkeeping attached to a proxy inside a pool."""
+
+    proxy: Proxy
+    failures: int = 0
+    successes: int = 0
+    banned_until: float = 0.0
+    extra: dict = field(default_factory=dict)

proxyspin-0.1.0/src/proxyspin/requests_adapter.py

@@ -0,0 +1,63 @@
+"""requests integration: a Session that rotates proxies per request.
+
+::
+
+    from proxyspin import ProxyPool
+    from proxyspin.requests_adapter import RotatingSession
+
+    pool = ProxyPool.from_file("proxies.txt")
+    session = RotatingSession(pool)
+    r = session.get("https://httpbin.org/ip")   # each call uses the next proxy
+
+Failures (connect errors, or responses with a status in ``ban_codes``)
+mark the proxy in the pool and the call is retried through another proxy
+up to ``max_retries`` times before the last error is raised.
+"""
+from __future__ import annotations
+
+try:
+    import requests
+except ImportError as exc:  # pragma: no cover
+    raise ImportError("proxyspin.requests_adapter needs requests: pip install proxyspin[requests]") from exc
+
+from .pool import ProxyPool
+
+DEFAULT_BAN_CODES = frozenset({403, 407, 429})
+
+
+class RotatingSession(requests.Session):
+    def __init__(
+        self,
+        pool: ProxyPool,
+        *,
+        max_retries: int = 3,
+        ban_codes: frozenset[int] = DEFAULT_BAN_CODES,
+    ) -> None:
+        super().__init__()
+        self.pool = pool
+        self.max_retries = max_retries
+        self.ban_codes = ban_codes
+
+    def request(self, method, url, **kwargs):  # type: ignore[override]
+        if "proxies" in kwargs:  # caller pinned a proxy — don't interfere
+            return super().request(method, url, **kwargs)
+        sticky_key = kwargs.pop("proxyspin_key", None)
+        last_exc: Exception | None = None
+        for _ in range(self.max_retries + 1):
+            proxy = self.pool.get(key=sticky_key)
+            kwargs["proxies"] = {"http": proxy.url, "https": proxy.url}
+            try:
+                response = super().request(method, url, **kwargs)
+            except (requests.ConnectionError, requests.Timeout) as exc:
+                self.pool.mark_failed(proxy)
+                last_exc = exc
+                continue
+            if response.status_code in self.ban_codes:
+                self.pool.mark_failed(proxy)
+                last_response = response
+                continue
+            self.pool.mark_ok(proxy)
+            return response
+        if last_exc is not None:
+            raise last_exc
+        return last_response

proxyspin-0.1.0/src/proxyspin/scrapy_middleware.py

@@ -0,0 +1,127 @@
+"""Scrapy downloader middleware with rotation, ban detection and retries.
+
+Enable in ``settings.py``::
+
+    DOWNLOADER_MIDDLEWARES = {
+        "proxyspin.scrapy_middleware.ProxySpinMiddleware": 610,
+    }
+
+    PROXYSPIN_LIST = ["http://user:pass@gate.example.com:8000"]
+    # or PROXYSPIN_FILE = "proxies.txt"
+    # or PROXYSPIN_URL = "https://provider.example.com/api/list"
+
+Optional settings (defaults shown)::
+
+    PROXYSPIN_STRATEGY = "round_robin"      # round_robin | random | sticky
+    PROXYSPIN_BAN_CODES = [403, 407, 429]   # responses treated as proxy bans
+    PROXYSPIN_MAX_RETRIES = 3               # per-request proxy switches
+    PROXYSPIN_MAX_FAILURES = 2              # pool: failures before cooldown
+    PROXYSPIN_COOLDOWN = 60.0               # pool: base cooldown, seconds
+
+Per-request control via ``request.meta``:
+
+    ``proxy``               set explicitly to bypass the pool
+    ``proxyspin_disabled``  truthy to skip proxying this request
+    ``proxyspin_key``       sticky key (defaults to the request host)
+"""
+from __future__ import annotations
+
+import logging
+
+from .pool import NoHealthyProxies, ProxyPool
+from .proxy import parse_proxy
+
+logger = logging.getLogger(__name__)
+
+
+class ProxySpinMiddleware:
+    def __init__(self, pool: ProxyPool, ban_codes: frozenset[int], max_retries: int) -> None:
+        self.pool = pool
+        self.ban_codes = ban_codes
+        self.max_retries = max_retries
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        settings = crawler.settings
+        kwargs = {
+            "strategy": settings.get("PROXYSPIN_STRATEGY", "round_robin"),
+            "max_failures": settings.getint("PROXYSPIN_MAX_FAILURES", 2),
+            "cooldown": settings.getfloat("PROXYSPIN_COOLDOWN", 60.0),
+        }
+        if settings.get("PROXYSPIN_URL"):
+            pool = ProxyPool.from_url(settings.get("PROXYSPIN_URL"), **kwargs)
+        elif settings.get("PROXYSPIN_FILE"):
+            pool = ProxyPool.from_file(settings.get("PROXYSPIN_FILE"), **kwargs)
+        else:
+            proxies = settings.getlist("PROXYSPIN_LIST")
+            if not proxies:
+                from scrapy.exceptions import NotConfigured
+
+                raise NotConfigured(
+                    "set PROXYSPIN_LIST, PROXYSPIN_FILE or PROXYSPIN_URL"
+                )
+            pool = ProxyPool(proxies, **kwargs)
+        logger.info("proxyspin: loaded %d proxies", len(pool))
+        ban_codes = frozenset(
+            int(c) for c in settings.getlist("PROXYSPIN_BAN_CODES", [403, 407, 429])
+        )
+        return cls(pool, ban_codes, settings.getint("PROXYSPIN_MAX_RETRIES", 3))
+
+    # ----------------------------------------------------------- lifecycle
+    def process_request(self, request, spider):
+        if request.meta.get("proxyspin_disabled"):
+            return None
+        if "proxy" in request.meta and not request.meta.get("_proxyspin_managed"):
+            return None  # user pinned a proxy explicitly — leave it alone
+        key = request.meta.get("proxyspin_key") or _host(request.url)
+        proxy = self.pool.get(key=key)
+        request.meta["proxy"] = proxy.url
+        request.meta["_proxyspin_managed"] = True
+        return None
+
+    def process_response(self, request, response, spider):
+        proxy_url = request.meta.get("proxy")
+        if not request.meta.get("_proxyspin_managed") or not proxy_url:
+            return response
+        if response.status in self.ban_codes:
+            self.pool.mark_failed(parse_proxy(proxy_url))
+            retried = self._retry(request, spider, reason=f"HTTP {response.status}")
+            if retried is not None:
+                return retried
+        else:
+            self.pool.mark_ok(parse_proxy(proxy_url))
+        return response
+
+    def process_exception(self, request, exception, spider):
+        proxy_url = request.meta.get("proxy")
+        if not request.meta.get("_proxyspin_managed") or not proxy_url:
+            return None
+        self.pool.mark_failed(parse_proxy(proxy_url))
+        return self._retry(request, spider, reason=repr(exception))
+
+    # ------------------------------------------------------------- helpers
+    def _retry(self, request, spider, *, reason: str):
+        retries = request.meta.get("_proxyspin_retries", 0)
+        if retries >= self.max_retries:
+            logger.warning(
+                "proxyspin: giving up on %s after %d proxy switches (%s)",
+                request.url, retries, reason,
+            )
+            return None
+        try:
+            proxy = self.pool.get(key=request.meta.get("proxyspin_key") or _host(request.url))
+        except NoHealthyProxies:
+            logger.warning("proxyspin: no healthy proxies left for %s", request.url)
+            return None
+        retry = request.copy()
+        retry.meta["proxy"] = proxy.url
+        retry.meta["_proxyspin_retries"] = retries + 1
+        retry.dont_filter = True
+        logger.debug("proxyspin: retrying %s via %s (%s)", request.url, proxy.address, reason)
+        return retry
+
+
+def _host(url: str) -> str:
+    from urllib.parse import urlsplit
+
+    return urlsplit(url).hostname or url

proxyspin-0.1.0/tests/test_proxyspin.py

@@ -0,0 +1,110 @@
+import time
+import unittest
+
+from proxyspin import NoHealthyProxies, Proxy, ProxyPool, parse_proxy
+
+
+class TestParse(unittest.TestCase):
+    def test_url_with_auth(self):
+        p = parse_proxy("http://user:p%40ss@10.0.0.1:8000")
+        self.assertEqual((p.host, p.port, p.username, p.password), ("10.0.0.1", 8000, "user", "p@ss"))
+        self.assertEqual(p.url, "http://user:p%40ss@10.0.0.1:8000")
+
+    def test_socks(self):
+        p = parse_proxy("socks5://10.0.0.1:1080")
+        self.assertEqual(p.scheme, "socks5")
+
+    def test_host_port(self):
+        p = parse_proxy("10.0.0.1:8000")
+        self.assertEqual(p, Proxy("10.0.0.1", 8000))
+
+    def test_host_port_user_pass(self):
+        p = parse_proxy("10.0.0.1:8000:alice:secret")
+        self.assertEqual(p.username, "alice")
+        self.assertEqual(p.password, "secret")
+
+    def test_user_pass_at_host_port(self):
+        p = parse_proxy("alice:secret@10.0.0.1:8000")
+        self.assertEqual((p.username, p.password, p.port), ("alice", "secret", 8000))
+
+    def test_invalid(self):
+        for bad in ("", "hostonly", "10.0.0.1:notaport", "ftp://10.0.0.1:21", "a:1:b:c:d"):
+            with self.assertRaises(ValueError, msg=bad):
+                parse_proxy(bad)
+
+
+class TestPool(unittest.TestCase):
+    def proxies(self, n=3):
+        return [f"10.0.0.{i}:8000" for i in range(1, n + 1)]
+
+    def test_round_robin_cycles(self):
+        pool = ProxyPool(self.proxies())
+        seen = {pool.get().address for _ in range(3)}
+        self.assertEqual(len(seen), 3)
+
+    def test_dedup(self):
+        pool = ProxyPool(["10.0.0.1:8000", "http://10.0.0.1:8000"])
+        self.assertEqual(len(pool), 1)
+
+    def test_cooldown_and_recovery(self):
+        pool = ProxyPool(self.proxies(2), max_failures=1, cooldown=0.05)
+        bad = pool.get()
+        pool.mark_failed(bad)
+        self.assertEqual(pool.healthy_count, 1)
+        for _ in range(5):
+            self.assertNotEqual(pool.get(), bad)
+        time.sleep(0.06)
+        self.assertEqual(pool.healthy_count, 2)
+
+    def test_success_resets_failures(self):
+        pool = ProxyPool(self.proxies(1), max_failures=2)
+        p = pool.get()
+        pool.mark_failed(p)
+        pool.mark_ok(p)
+        pool.mark_failed(p)
+        self.assertEqual(pool.healthy_count, 1)  # streak broken, not banned
+
+    def test_all_banned_raises(self):
+        pool = ProxyPool(self.proxies(2), max_failures=1, cooldown=60)
+        for _ in range(2):
+            pool.mark_failed(pool.get())
+        with self.assertRaises(NoHealthyProxies):
+            pool.get()
+
+    def test_sticky_keeps_proxy_per_key(self):
+        pool = ProxyPool(self.proxies(3), strategy="sticky")
+        first = pool.get(key="acct-1")
+        for _ in range(5):
+            self.assertEqual(pool.get(key="acct-1"), first)
+
+    def test_sticky_reassigns_after_ban(self):
+        pool = ProxyPool(self.proxies(2), strategy="sticky", max_failures=1, cooldown=60)
+        first = pool.get(key="acct-1")
+        pool.mark_failed(first)
+        self.assertNotEqual(pool.get(key="acct-1"), first)
+
+    def test_stats(self):
+        pool = ProxyPool(self.proxies(1))
+        p = pool.get()
+        pool.mark_ok(p)
+        self.assertEqual(pool.stats()[p.address]["successes"], 1)
+
+
+class TestPlaywrightHelper(unittest.TestCase):
+    def test_settings_with_auth(self):
+        from proxyspin.playwright_helper import proxy_settings
+
+        pool = ProxyPool(["http://u:p@10.0.0.1:8000"])
+        settings = proxy_settings(pool)
+        self.assertEqual(
+            settings, {"server": "http://10.0.0.1:8000", "username": "u", "password": "p"}
+        )
+
+    def test_settings_bare_proxy(self):
+        from proxyspin.playwright_helper import proxy_settings
+
+        self.assertEqual(proxy_settings(Proxy("10.0.0.1", 8000)), {"server": "http://10.0.0.1:8000"})
+
+
+if __name__ == "__main__":
+    unittest.main()