proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/judges.py

@@ -0,0 +1,159 @@
+"""Check targets ("judges"): services that return nothing but the sender's IP as text.
+
+Every proxy has to fetch its exit IP from a check target. If the target goes down or throttles,
+every proxy looks dead – and the source statistics would learn the wrong thing from it. So:
+
+- test every target directly before the run and take the best reachable one
+- check regularly during the run (JudgeWatch) and switch if it goes down
+
+Targets behind Cloudflare are off limits: many "proxies" in the lists are Cloudflare addresses that
+simply answer a request to a Cloudflare site themselves and would pass as working that way.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import socket
+import time
+from dataclasses import dataclass
+from typing import Awaitable, Callable, List, Optional
+
+from .netio import read_response
+
+
+@dataclass(frozen=True)
+class Judge:
+    host: str
+    path: str = "/"
+    port: int = 80
+
+    @property
+    def authority(self) -> str:
+        """host[:port] for Host headers and URLs."""
+        return self.host if self.port == 80 else f"{self.host}:{self.port}"
+
+
+# order = suitability. Measured with the same 300 most recently working proxies (September 2026):
+# amazonaws 255, ifconfig.me 230, ipinfo.io 223, wtfismyip 213, ident.me 199 hits.
+# The direct latency says little about it – some services simply reject proxies more often.
+JUDGES = (
+    Judge("checkip.amazonaws.com"),   # AWS
+    Judge("ifconfig.me", "/ip"),      # Google Cloud
+    Judge("ipinfo.io", "/ip"),        # Google Cloud
+    Judge("wtfismyip.com", "/text"),
+    Judge("ident.me"),                # Hetzner
+)
+DEFAULT_JUDGE = JUDGES[0]
+
+# https://www.cloudflare.com/ips-v4/
+CLOUDFLARE_V4 = tuple(ipaddress.IPv4Network(n) for n in (
+    "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22", "141.101.64.0/18",
+    "108.162.192.0/18", "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22", "198.41.128.0/17",
+    "162.158.0.0/15", "104.16.0.0/13", "104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
+))
+
+
+def behind_cloudflare(ip: str) -> bool:
+    addr = ipaddress.IPv4Address(ip)
+    return any(addr in net for net in CLOUDFLARE_V4)
+
+
+@dataclass(frozen=True)
+class JudgeProbe:
+    judge: Judge
+    ip: str
+    latency: int  # ms
+    seen_ip: str = ""  # what the target saw as our IP (fallback in case get_own_ips fails)
+
+
+async def probe_judge(judge: Judge, timeout: float = 5.0) -> Optional[JudgeProbe]:
+    """Query the target directly (without a proxy). None = unreachable, wrong answer or Cloudflare."""
+    loop = asyncio.get_running_loop()
+    try:
+        infos = await asyncio.wait_for(loop.getaddrinfo(judge.host, judge.port, family=socket.AF_INET), timeout)
+    except Exception:
+        return None
+    addresses = list(dict.fromkeys(info[4][0] for info in infos))
+    if any(behind_cloudflare(ip) for ip in addresses):
+        return None  # being behind Cloudflare even partially is enough to exclude it
+    for ip in addresses:  # first working address – exactly that one is used by the checker later
+        start = time.perf_counter()
+        seen = await _ask(ip, judge, timeout)
+        if seen:
+            return JudgeProbe(judge, ip, round((time.perf_counter() - start) * 1000), seen)
+    return None
+
+
+async def _ask(ip: str, judge: Judge, timeout: float) -> str:
+    """Ask the target via exactly this address -> the IP it sees ('' on error)."""
+    request = (f"GET {judge.path} HTTP/1.1\r\nHost: {judge.authority}\r\nUser-Agent: Mozilla/5.0\r\n"
+               f"Connection: close\r\n\r\n").encode()
+    try:
+        reader, writer = await asyncio.wait_for(asyncio.open_connection(ip, judge.port), timeout)
+        try:
+            writer.write(request)
+            await writer.drain()
+            status, _, body = await asyncio.wait_for(read_response(reader), timeout)
+        finally:
+            writer.close()
+        seen = body.strip().decode("ascii", "ignore")
+        # directly this may also return IPv6 (e.g. iCloud Private Relay) – through proxies we address
+        # the target by its IPv4 address and then get the IPv4 exit IP
+        ipaddress.ip_address(seen)
+    except Exception:  # timeout, connection error, no IP in the body – all mean "unusable right now"
+        return ""
+    return seen if status == 200 else ""
+
+
+async def rank_judges(judges=JUDGES, timeout: float = 5.0, probe=probe_judge) -> List[JudgeProbe]:
+    """Test every target in parallel; the reachable ones in order of suitability (see JUDGES)."""
+    results = await asyncio.gather(*(probe(j, timeout) for j in judges))
+    return [r for r in results if r]
+
+
+class JudgeWatch:
+    """Keeps an eye on the check target during the run and switches if it goes down.
+
+    on_ok is called after every successful probe, on_switch(old, new) on a switch –
+    that way the caller can treat the checks since the last good probe as suspicious.
+    """
+
+    def __init__(self, ranked: List[JudgeProbe], switch: Callable[[JudgeProbe], None],
+                 interval: float = 15.0, failures_before_switch: int = 2,
+                 probe: Callable[..., Awaitable[Optional[JudgeProbe]]] = probe_judge):
+        self.current = ranked[0]
+        self.reserve = list(ranked[1:])
+        self.switch = switch
+        self.interval = interval
+        self.failures_before_switch = failures_before_switch
+        self.probe = probe
+        self.failures = 0
+        self.switches: List[str] = []
+        self.on_ok: Callable[[], None] = lambda: None
+        self.on_switch: Callable[[JudgeProbe, JudgeProbe], None] = lambda old, new: None
+
+    async def run(self) -> None:
+        while True:
+            await asyncio.sleep(self.interval)
+            await self.check_once()
+
+    async def check_once(self) -> None:
+        if await self.probe(self.current.judge):
+            self.failures = 0
+            self.on_ok()
+            return
+        self.failures += 1
+        if self.failures < self.failures_before_switch:
+            return
+        for candidate in list(self.reserve):
+            fresh = await self.probe(candidate.judge)
+            if fresh:
+                old = self.current
+                self.reserve.remove(candidate)
+                self.reserve.append(old)  # maybe it comes back later
+                self.current, self.failures = fresh, 0
+                self.switch(fresh)
+                self.switches.append(f"{old.judge.host} → {fresh.judge.host}")
+                self.on_switch(old, fresh)
+                return

proxyscraper/mcp_entry.py

@@ -0,0 +1,34 @@
+"""`proxy-scraper-mcp`: starts the MCP server over stdio – or explains what's missing.
+
+A separate module on purpose: it has to run without the MCP SDK, to say how to install it.
+stdout belongs to the MCP protocol: every message here goes to stderr.
+"""
+
+import sys
+
+INSTALL_HINT = ('The MCP server needs the MCP SDK: pip install "proxy-scraper-cli[mcp]"  '
+                '(or: pipx inject proxy-scraper-cli mcp)')
+
+
+def _load_server():
+    from . import mcp_server
+    return mcp_server
+
+
+def main() -> int:
+    if sys.version_info < (3, 10):
+        print(f"The MCP server needs Python 3.10 or newer (this is {sys.version_info[0]}.{sys.version_info[1]}). "
+              "The rest of proxy-scraper works on 3.9.", file=sys.stderr)
+        return 1
+    try:
+        server = _load_server()
+    except ImportError as e:
+        if e.name and e.name.split(".")[0] in ("mcp", "pydantic"):
+            print(INSTALL_HINT, file=sys.stderr)
+            return 1
+        raise
+    return server.serve()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

proxyscraper/mcp_server.py

@@ -0,0 +1,220 @@
+"""MCP server: working free proxies for AI agents, and pages fetched through them.
+
+Three tools – the logic lives in agent.py, this module describes it for agents (names, parameter docs,
+structured results, annotations) and turns AgentError into tool errors the agent can act on.
+Start it with `proxy-scraper-mcp` (stdio); needs `pip install "proxy-scraper-cli[mcp]"` and Python 3.10+.
+"""
+
+import contextlib
+import logging
+import sys
+import time
+from typing import Annotated, List, Literal, Optional
+
+from mcp.server.mcpserver import Context, MCPServer
+from mcp.server.mcpserver.exceptions import ToolError
+from mcp.types import ToolAnnotations
+from pydantic import BaseModel, Field
+
+from . import __version__, agent
+
+REPO = "https://github.com/maximilianfeix/proxy-scraper"
+
+INSTRUCTIONS = """\
+Free HTTP, SOCKS4 and SOCKS5 proxies that passed real checks: a protocol handshake, the same exit IP on two \
+unrelated sites (drops honeypots), a byte-for-byte content check (drops proxies that inject scripts) and, for \
+HTTPS, a verified TLS handshake.
+
+- get_proxies: instant, from a list re-checked every hour. Start here.
+- check_proxies: checks from this machine's network, so the results work from here right now. Takes 30-90 s.
+- fetch_url: loads a page through a verified proxy and switches proxies by itself if one fails.
+
+Free proxies are run by strangers: never send passwords, cookies, API keys or personal data through them, \
+and expect some of them to stop working at any time."""
+
+Protocol = Literal["any", "http", "socks4", "socks5"]
+Countries = Annotated[List[str], Field(description="Two-letter country codes of the exit IP, e.g. ['DE', 'NL']. "
+                                                   "Empty = any country.")]
+
+
+class Proxy(BaseModel):
+    url: str = Field(description="Ready to use, e.g. socks5://1.2.3.4:1080 – curl, requests, httpx and "
+                                 "browsers take it as it is")
+    protocol: str = Field(description="http, socks4 or socks5")
+    address: str = Field(description="ip:port")
+    country: Optional[str] = Field(description="Two-letter code of the exit IP's country")
+    latency_ms: Optional[int] = Field(description="Response time in the last check")
+    https: Optional[bool] = Field(description="Tunnels HTTPS with a verified TLS handshake")
+    anonymity: Optional[str] = Field(description="elite (target sees no proxy), anonymous or transparent")
+    provider: Optional[str] = Field(description="Network the exit IP belongs to")
+    datacenter: Optional[bool] = Field(description="Exit IP in a datacenter – some sites block those")
+    blocklisted: Optional[bool] = Field(description="Exit IP on the SpamCop blocklist – expect captchas")
+    up_for_hours: Optional[int] = Field(description="How long it has worked without a gap (live list only)")
+
+
+class ProxyList(BaseModel):
+    proxies: List[Proxy] = Field(description="Fastest first")
+    matched: int = Field(description="How many proxies in the list pass the filters (limit cuts the rest)")
+    list_updated: Optional[str] = Field(description="When the list was last checked (ISO 8601, UTC)")
+    list_age_minutes: Optional[int] = Field(description="Minutes since that check")
+    note: Optional[str] = Field(description="What to do if fewer proxies matched than asked for")
+
+
+class FreshProxies(BaseModel):
+    proxies: List[Proxy] = Field(description="Working from this machine right now, fastest first")
+    mode: str = Field(description="live = the hourly list checked again, full = all 700+ sources")
+    took_seconds: float
+    note: Optional[str] = Field(description="What to do if none were found")
+
+
+class Page(BaseModel):
+    url: str
+    final_url: Optional[str] = Field(description="After redirects")
+    status: int = Field(description="HTTP status of the target, e.g. 200, 403, 404")
+    content_type: Optional[str]
+    via: Optional[str] = Field(description="The proxy that delivered the page")
+    proxy_country: Optional[str] = Field(description="Country the target saw the request from")
+    elapsed_ms: int
+    bytes: int = Field(description="Size of the response body")
+    text: str = Field(description="Readable text (HTML is turned into plain text unless raw_html is set)")
+    truncated: bool = Field(description="True if the text was cut at max_chars")
+    note: Optional[str]
+
+
+def _fail(e: agent.AgentError):
+    raise ToolError(str(e)) from None
+
+
+def build_server(live: Optional[agent.LiveSource] = None, fetcher: Optional[agent.PageFetcher] = None) -> MCPServer:
+    live = live or agent.LiveSource()
+    fetcher = fetcher or agent.PageFetcher(live)
+
+    @contextlib.asynccontextmanager
+    async def lifespan(_server):
+        try:
+            yield {}
+        finally:
+            await fetcher.close()  # the internal rotating server
+
+    server = MCPServer("proxy-scraper", title="proxy-scraper – free proxies that actually work",
+                       version=__version__, instructions=INSTRUCTIONS, website_url=REPO, lifespan=lifespan)
+
+    @server.tool(title="Get working proxies",
+                 annotations=ToolAnnotations(read_only_hint=True, idempotent_hint=True, open_world_hint=True))
+    async def get_proxies(
+        protocol: Annotated[Protocol, Field(description="Proxy protocol. socks5 works for any TCP traffic, http is "
+                                                        "what most HTTP clients expect.")] = "any",
+        countries: Countries = [],  # noqa: B006 – pydantic copies defaults, and a list reads best in the schema
+        https_only: Annotated[bool, Field(description="Only proxies that tunnel HTTPS with verified TLS. "
+                                                      "Needed for https:// sites.")] = False,
+        elite_only: Annotated[bool, Field(description="Only proxies the target can't recognize as proxies")] = False,
+        exclude_datacenter: Annotated[bool, Field(description="Skip exits in datacenters (blocked by many "
+                                                              "sites)")] = False,
+        exclude_blocklisted: Annotated[bool, Field(description="Skip exits on the SpamCop blocklist (fewer "
+                                                               "captchas)")] = False,
+        stable_only: Annotated[bool, Field(description="Only proxies that have worked for 24 h or more")] = False,
+        max_latency_ms: Annotated[int, Field(ge=0, description="Maximum response time, 0 = any")] = 0,
+        limit: Annotated[int, Field(ge=1, le=100, description="How many to return (fastest first)")] = 10,
+    ) -> ProxyList:
+        """Get working free proxies right now, fastest first.
+
+        Instant: comes from a public list that is re-checked every hour from GitHub's servers, so a proxy may have
+        died since, and some behave differently from this machine's network. Use check_proxies when they must
+        work from here right now. Every result has a ready-to-use url like socks5://1.2.3.4:1080."""
+        try:
+            data = await live.get()
+            rows = agent.select(data.rows, protocol, countries, https_only, elite_only, exclude_datacenter,
+                                exclude_blocklisted, stable_only, max_latency_ms, data.run_hours)
+        except agent.AgentError as e:
+            _fail(e)
+        return ProxyList(proxies=[Proxy(**agent.describe(r, data.run_hours)) for r in rows[:limit]],
+                         matched=len(rows), list_updated=data.updated or None, list_age_minutes=data.age_minutes(),
+                         note=agent.shortage_note(data.rows, limit, len(rows), countries))
+
+    @server.tool(title="Check proxies from this machine",
+                 annotations=ToolAnnotations(read_only_hint=False, destructive_hint=False, idempotent_hint=False,
+                                             open_world_hint=True))
+    async def check_proxies(
+        ctx: Context,
+        want: Annotated[int, Field(ge=1, le=100, description="Stop once this many work")] = 10,
+        protocol: Annotated[Protocol, Field(description="Proxy protocol to look for")] = "any",
+        countries: Countries = [],  # noqa: B006
+        https_only: Annotated[bool, Field(description="Only proxies that tunnel HTTPS with verified TLS")] = False,
+        elite_only: Annotated[bool, Field(description="Only proxies the target can't recognize as proxies")] = False,
+        exclude_datacenter: Annotated[bool, Field(description="Skip exits in datacenters")] = False,
+        exclude_blocklisted: Annotated[bool, Field(description="Skip exits on the SpamCop blocklist")] = False,
+        max_latency_ms: Annotated[int, Field(ge=0, description="Maximum response time, 0 = any")] = 0,
+        mode: Annotated[Literal["live", "full"], Field(description="live: check the hourly list again from here "
+                                                                   "(30-90 s). full: collect from all 700+ "
+                                                                   "sources (a few minutes)")] = "live",
+    ) -> FreshProxies:
+        """Find proxies that work from this machine's network right now.
+
+        Runs the same checks as the public list – handshake, honeypot test, byte-for-byte content check, verified
+        TLS – but from here, so the results were verified seconds ago from the network your code runs on. Slower
+        than get_proxies: 30-90 s in live mode, and progress is reported while it runs. It also updates the local
+        statistics proxy-scraper learns from."""
+        started = time.monotonic()
+
+        async def progress(elapsed: float, message: str):
+            await ctx.report_progress(elapsed, None, message)
+
+        try:
+            found = await agent.check_fresh(want, protocol, countries, https_only, elite_only, exclude_datacenter,
+                                            exclude_blocklisted, max_latency_ms, mode, progress)
+        except agent.AgentError as e:
+            _fail(e)
+        note = None if found else ("Nothing passed from this machine. A firewall may block proxy ports – "
+                                   "try get_proxies, or mode='full'.")
+        return FreshProxies(proxies=[Proxy(**p) for p in found], mode=mode,
+                            took_seconds=round(time.monotonic() - started, 1), note=note)
+
+    @server.tool(title="Fetch a page through a proxy",
+                 annotations=ToolAnnotations(read_only_hint=True, idempotent_hint=False, open_world_hint=True))
+    async def fetch_url(
+        ctx: Context,
+        url: Annotated[str, Field(description="Full http:// or https:// URL of a public site")],
+        country: Annotated[str, Field(description="Two-letter code: load the page as seen from this country. "
+                                                  "Empty = any country.")] = "",
+        protocol: Annotated[Protocol, Field(description="Only use proxies of this protocol")] = "any",
+        max_chars: Annotated[int, Field(ge=100, le=200_000, description="Cut the text after this many "
+                                                                        "characters")] = 20_000,
+        raw_html: Annotated[bool, Field(description="Return the HTML as it is instead of readable "
+                                                    "text")] = False,
+    ) -> Page:
+        """Load a web page through a verified free proxy and return it as readable text.
+
+        If a proxy fails, the next one is tried by itself (up to 8), redirects are followed. HTTPS only goes
+        through proxies that passed a verified-TLS test, so the page can't be changed on the way. Local and
+        private addresses are refused, also when a redirect points there. Useful to see a site from another country or
+        when a site blocks this machine. GET only; don't use it for logins or anything with personal data."""
+        async def progress(elapsed: float, message: str):
+            await ctx.report_progress(elapsed, None, message)
+
+        try:
+            page = await fetcher.fetch(url, protocol=protocol, country=country, max_chars=max_chars,
+                                       raw_html=raw_html, progress=progress)
+        except agent.AgentError as e:
+            _fail(e)
+        return Page(**page)
+
+    return server
+
+
+def serve() -> int:
+    # stdout carries the protocol: the terminal UI and every log line go to stderr
+    from rich.console import Console
+
+    from . import paths
+    from .ui import widgets
+    # clients start servers in the user's project folder – results/ doesn't belong there
+    paths.RESULTS_DIR = paths.DATA_DIR / "results"
+    widgets.console = Console(stderr=True, highlight=False)
+    logging.basicConfig(stream=sys.stderr, level=logging.WARNING)
+    build_server().run("stdio")
+    return 0
+
+
+if __name__ == "__main__":  # python -m proxyscraper.mcp_server: same checks as the proxy-scraper-mcp command
+    from .mcp_entry import main
+    sys.exit(main())

proxyscraper/netio.py

@@ -0,0 +1,167 @@
+"""Minimal HTTP client on plain asyncio + ssl – for sources, APIs and check targets."""
+
+from __future__ import annotations
+
+import asyncio
+import ssl
+from functools import lru_cache
+from typing import Dict, Optional, Set, Tuple
+from urllib.parse import urljoin, urlsplit
+
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/128.0 Safari/537.36"
+)
+MAX_BODY = 64 * 1024 * 1024
+
+# hosts whose certificate couldn't be verified (typically: a firewall with TLS inspection like Sophos)
+INSECURE_HOSTS: Set[str] = set()
+
+
+@lru_cache(maxsize=None)
+def ssl_context() -> ssl.SSLContext:
+    # build once instead of per connection – otherwise loading the CA file blocks the event loop every time
+    try:
+        import certifi  # python.org Python on macOS often has no system certificates
+
+        return ssl.create_default_context(cafile=certifi.where())
+    except ImportError:
+        return ssl.create_default_context()
+
+
+@lru_cache(maxsize=None)
+def _insecure_ssl_context() -> ssl.SSLContext:
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+CONDITIONAL_HEADERS = {"If-None-Match", "If-Modified-Since"}
+
+
+async def http_request(
+    url: str,
+    timeout: float = 15.0,
+    headers: Optional[Dict[str, str]] = None,
+    method: str = "GET",
+    body: Optional[bytes] = None,
+    max_redirects: int = 3,
+    insecure_fallback: bool = True,
+) -> Tuple[int, Dict[bytes, bytes], bytes]:
+    """One HTTP/1.1 request -> (status, headers, body). Follows redirects for GET.
+
+    If certificate verification fails, it is only retried unverified for GETs without a body and without
+    custom headers – so for public proxy lists, never with API tokens or sent data. The only headers
+    allowed are If-None-Match / If-Modified-Since (ETag cache), which carry no secret.
+    The lists are public, and every proxy from them gets checked on its own anyway.
+    """
+    extra = "".join(f"{k}: {v}\r\n" for k, v in (headers or {}).items())
+    if body is not None:
+        extra += f"Content-Length: {len(body)}\r\n"
+    for _ in range(max_redirects + 1):
+        u = urlsplit(url)
+        https = u.scheme == "https"
+        port = u.port or (443 if https else 80)
+        path = (u.path or "/") + (f"?{u.query}" if u.query else "")
+        # unverified only for GETs without a body and without custom headers (except the conditional ETag headers,
+        # they carry no secret). Never tokens or sent data over an unverified connection.
+        insecure_ok = insecure_fallback and method == "GET" and body is None and \
+            set(headers or ()) <= CONDITIONAL_HEADERS
+        reader, writer = await _connect(u.hostname, port, https, allow_insecure=insecure_ok, timeout=timeout)
+        try:
+            writer.write(
+                f"{method} {path} HTTP/1.1\r\nHost: {u.hostname}\r\nUser-Agent: {USER_AGENT}\r\n"
+                f"Accept: */*\r\nAccept-Encoding: identity\r\n{extra}Connection: close\r\n\r\n".encode()
+                + (body or b"")
+            )
+            await writer.drain()
+            status, resp_headers, resp_body = await asyncio.wait_for(read_response(reader), timeout)
+        finally:
+            writer.close()
+
+        if method == "GET" and status in (301, 302, 303, 307, 308) and b"location" in resp_headers:
+            url = urljoin(url, resp_headers[b"location"].decode())
+            continue
+        return status, resp_headers, resp_body
+    raise ConnectionError("too many redirects")
+
+
+async def http_get(
+    url: str, timeout: float = 15.0, max_redirects: int = 3, headers: Optional[Dict[str, str]] = None
+) -> bytes:
+    """GET that raises an exception for anything but 200."""
+    status, _, body = await http_request(url, timeout, headers, max_redirects=max_redirects)
+    if status != 200:
+        raise ConnectionError(f"HTTP {status}")
+    return body
+
+
+async def _connect(host: str, port: int, https: bool, allow_insecure: bool, timeout: float):
+    if not https:
+        return await asyncio.wait_for(asyncio.open_connection(host, port), timeout)
+    ctx = _insecure_ssl_context() if allow_insecure and host in INSECURE_HOSTS else ssl_context()
+    try:
+        return await asyncio.wait_for(asyncio.open_connection(host, port, ssl=ctx, server_hostname=host), timeout)
+    except ssl.SSLCertVerificationError:
+        if not allow_insecure:
+            raise
+        INSECURE_HOSTS.add(host)
+        return await asyncio.wait_for(
+            asyncio.open_connection(host, port, ssl=_insecure_ssl_context(), server_hostname=host), timeout
+        )
+
+
+async def read_response(reader) -> Tuple[int, Dict[bytes, bytes], bytes]:
+    head = await reader.readuntil(b"\r\n\r\n")
+    lines = head[:-4].split(b"\r\n")
+    parts = lines[0].split()
+    status = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 0
+    headers: Dict[bytes, bytes] = {}
+    for line in lines[1:]:
+        k, _, v = line.partition(b":")
+        headers[k.strip().lower()] = v.strip()
+
+    # respect Content-Length: servers with keep-alive (e.g. checkip.amazonaws.com) don't close
+    # the connection otherwise, and reading to the end would run into the timeout.
+    if headers.get(b"transfer-encoding", b"").lower() == b"chunked":
+        body = dechunk(await read_all(reader))
+    elif headers.get(b"content-length", b"").isdigit():
+        length = int(headers[b"content-length"])
+        if length > MAX_BODY:
+            raise ConnectionError("response too large")
+        body = await reader.readexactly(length)
+    else:
+        body = await read_all(reader)
+    return status, headers, body
+
+
+async def read_all(reader, limit: int = MAX_BODY) -> bytes:
+    # some servers (e.g. Cloudflare) end with RST instead of FIN – keep the data already read
+    buf = bytearray()
+    while len(buf) < limit:
+        try:
+            chunk = await reader.read(65536)
+        except ConnectionResetError:
+            break
+        if not chunk:
+            break
+        buf += chunk
+    return bytes(buf)
+
+
+def dechunk(data: bytes) -> bytes:
+    out, i = bytearray(), 0
+    while i < len(data):
+        j = data.find(b"\r\n", i)
+        if j < 0:
+            break
+        try:
+            size = int(data[i:j].split(b";")[0], 16)
+        except ValueError:
+            break
+        if size == 0:
+            break
+        out += data[j + 2 : j + 2 + size]
+        i = j + 2 + size + 2
+    return bytes(out)