proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/server/core.py

@@ -0,0 +1,518 @@
+"""Local rotating proxy server (--serve).
+
+Accepts HTTP proxy requests (CONNECT for HTTPS and plain HTTP requests) and sends every
+connection through one of the proxies that were found. Fast, reliable proxies are preferred;
+if one fails, the next one is tried automatically, and one that fails several times in a row
+drops out of the rotation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Deque, Dict, Optional, Set, Tuple
+
+from ..handshake import parse_endpoint, with_proxy_auth
+from .http import (
+    TLS_HANDSHAKE,
+    ResponseScreen,
+    forward_request,
+    origin_request,
+    parse_request_head,
+    plausible_answer,
+    request_body_length,
+)
+from .pool import ANY, ProxyPool, Selection
+from .socks import SOCKS5_VERSION, Socks5Refused, socks5_accept, socks5_reply
+from .status import (
+    METRICS_PATH,
+    METRICS_TYPE,
+    STATUS_PATH,
+    STATUS_PREFIX,
+    metrics_text,
+    password_ok,
+    selection_from_headers,
+    status_json,
+)
+from .upstream import TargetError, Unsupported, UpstreamError, open_upstream
+
+MAX_ATTEMPTS = 3            # this many proxies per request before the client gets an error
+FIRST_CHUNK_WAIT = 5.0      # how long to wait for the client's first packet in the tunnel
+MAX_REPLAY_BODY = 1024 * 1024  # request bodies up to this size are buffered for switching proxies
+HTTP_ESTABLISHED = b"HTTP/1.1 200 Connection established\r\n\r\n"
+HEAD_LIMIT = 64 * 1024
+
+
+@dataclass
+class RequestLog:
+    client: str
+    target: str
+    via: str
+    ok: bool
+    ms: int
+    attempts: int
+
+
+@dataclass
+class ServerStats:
+    requests: int = 0
+    ok: int = 0
+    failed: int = 0
+    active: int = 0
+    bytes_up: int = 0
+    bytes_down: int = 0
+    recent: Deque[RequestLog] = field(default_factory=lambda: deque(maxlen=10))
+    started: float = field(default_factory=time.perf_counter)
+
+
+class RotatingServer:
+    def __init__(self, pool: ProxyPool, host: str = "127.0.0.1", port: int = 8899, timeout: float = 10.0,
+                 password: str = "", max_attempts: int = MAX_ATTEMPTS, public_targets_only: bool = False):
+        self.pool = pool
+        # names this machine resolves for SOCKS4 upstreams must not lead to private addresses (DNS rebinding)
+        self.public_targets_only = public_targets_only
+        self.max_attempts = max_attempts  # proxies per request before the client gets a 502
+        self.password = password  # empty = no authentication (fine on 127.0.0.1)
+        self.host = host
+        self.port = port
+        self.timeout = timeout
+        self.stats = ServerStats()
+        self.revived = 0  # proxies that passed the recheck after being disabled
+        self.refilled = 0  # new proxies added by --serve-refill
+        self.last_refill: Optional[float] = None  # time.time() of the last finished refill
+        self._server: Optional[asyncio.AbstractServer] = None
+        self._handlers: Set[asyncio.Task] = set()  # open client connections, ended by close()
+        self._sessions: Dict[str, Set[asyncio.Task]] = {}  # session name -> its connections (abort_session)
+
+    async def start(self) -> None:
+        self._server = await asyncio.start_server(self._handle, self.host, self.port, limit=HEAD_LIMIT)
+        self.port = self._server.sockets[0].getsockname()[1]
+
+    async def close(self) -> None:
+        """Stop listening and end open connections – a tunnel whose target never hangs up would otherwise keep
+        running, and from Python 3.12 on wait_closed() waits for every connection."""
+        if self._server:
+            self._server.close()
+            for task in list(self._handlers):
+                task.cancel()
+            await asyncio.gather(*self._handlers, return_exceptions=True)
+            await self._server.wait_closed()
+
+    def abort_session(self, session: str) -> int:
+        """End every connection of a session (username session-NAME). The client sees its connection closed – the
+        one way to wake a thread that is blocked reading it on every platform. -> connections ended."""
+        tasks = self._sessions.pop(session, set())
+        for task in tasks:
+            task.cancel()
+        return len(tasks)
+
+    def _forget(self, session: str, task: asyncio.Task) -> None:
+        tasks = self._sessions.get(session)
+        if tasks is not None:
+            tasks.discard(task)
+            if not tasks:
+                del self._sessions[session]
+
+    async def _handle(self, reader, writer) -> None:
+        task = asyncio.current_task()
+        if task is not None:
+            self._handlers.add(task)
+            task.add_done_callback(self._handlers.discard)
+        peer = writer.get_extra_info("peername") or ("?", 0)
+        client = f"{peer[0]}:{peer[1]}"
+        self.stats.active += 1
+        try:
+            try:
+                # SOCKS5 and HTTP on the same port: SOCKS5 always starts with 0x05, HTTP with a letter
+                first = await asyncio.wait_for(reader.readexactly(1), self.timeout)
+                if first == SOCKS5_VERSION:
+                    await self._serve_socks5(reader, writer, client)
+                    return
+                head = first + await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), self.timeout)
+                if head.startswith(STATUS_PREFIX):
+                    await self._serve_status(writer, head)
+                    return
+                method, host, port, path, headers = parse_request_head(head)
+                if not password_ok(headers, self.password):
+                    writer.write(b"HTTP/1.1 407 Proxy Authentication Required\r\n"
+                                 b'Proxy-Authenticate: Basic realm="proxy-scraper"\r\n'
+                                 b"Content-Length: 0\r\nConnection: close\r\n\r\n")
+                    return
+            except (ValueError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, asyncio.TimeoutError,
+                    UnicodeDecodeError):
+                writer.write(b"HTTP/1.1 400 Bad Request\r\nX-Proxy-Scraper: bad-request\r\n"
+                             b"Content-Length: 0\r\nConnection: close\r\n\r\n")
+                return
+            selection = selection_from_headers(headers)
+            if selection.session and task is not None:
+                self._sessions.setdefault(selection.session, set()).add(task)
+                task.add_done_callback(lambda t, name=selection.session: self._forget(name, t))
+            await self._serve_request(reader, writer, client, method, host, port, path, headers, selection)
+        except (ConnectionError, OSError, asyncio.IncompleteReadError):
+            pass  # client hung up (also in the middle of a request body)
+        finally:
+            self.stats.active -= 1
+            writer.close()
+
+    async def _serve_request(self, reader, writer, client, method, host, port, path, headers,
+                             selection: Selection = ANY) -> None:
+        self.stats.requests += 1
+        started = time.perf_counter()
+        if method == "CONNECT":
+            served = await self._serve_tunnel(reader, writer, client, host, port, started, selection)
+        else:
+            served = await self._serve_http(reader, writer, client, method, host, port, path, headers, started,
+                                            selection)
+        if not served:
+            self.stats.failed += 1
+
+    async def _serve_tunnel(self, reader, writer, client, host, port, started, selection: Selection = ANY,
+                            established: bytes = HTTP_ESTABLISHED, refuse=None) -> bool:
+        """CONNECT: open a tunnel first, then "200" to the client – if none works, it gets a 502.
+
+        After that a proxy only counts as successful once it answers. The client's first packet (for HTTPS
+        the start of the TLS handshake) is buffered and goes to the next proxy unnoticed if needed.
+        """
+        tried: Set[str] = set()
+        # CONNECT is almost always TLS (also on ports like 8443) – the first ClientHello should only go through
+        # proxies that passed the HTTPS test; without any, pick() falls back to all of them
+        opened = await self._open_next(tried, host, port, tls=True, tunnel=True, selection=selection)
+        if opened is None:
+            self._log(client, host, port, None, False, started, len(tried))
+            await (refuse or self._bad_gateway)(writer)
+            return False
+        writer.write(established)
+        await writer.drain()
+        first_out = await self._first_client_chunk(reader)
+        tls = first_out[:1] == TLS_HANDSHAKE
+
+        while opened is not None:
+            entry, up_reader, up_writer = opened
+            first_in = await self._exchange(up_reader, up_writer, first_out)
+            if first_in is not None:
+                await self._relay(reader, writer, client, host, port, started, len(tried), entry,
+                                  up_reader, up_writer, first_out, first_in)
+                return True
+            self._give_up(entry, up_writer)
+            opened = await self._open_next(tried, host, port, tls=tls, tunnel=True, selection=selection)
+        self._log(client, host, port, None, False, started, len(tried))
+        return False  # after "200 Connection established" all that's left is to close the connection
+
+    async def _serve_http(self, reader, writer, client, method, host, port, path, headers, started,
+                          selection: Selection = ANY) -> bool:
+        """Plain HTTP request. HTTP upstreams get it as a classic proxy request (without CONNECT),
+        SOCKS upstreams in the form for the target server. Small bodies are buffered for a switch."""
+        body, replayable = await self._read_body(reader, headers)
+        tried: Set[str] = set()
+        while True:
+            opened = await self._open_next(tried, host, port, tls=False, tunnel=False, selection=selection)
+            if opened is None:
+                break
+            entry, up_reader, up_writer = opened
+            if entry.result.ptype == "http":
+                head = with_proxy_auth(forward_request(method, path, host, port, headers),
+                                       parse_endpoint(entry.result.proxy))
+            else:
+                head = origin_request(method, path, host, port, headers)
+            first_out = head + body
+            if not replayable:
+                # large or streamed body: no switch possible – send the head, pass the rest through
+                up_writer.write(first_out)
+                await up_writer.drain()
+                return await self._relay(reader, writer, client, host, port, started, len(tried), entry,
+                                         up_reader, up_writer, first_out, b"", upload=request_body_length(headers))
+            first_in = await self._exchange(up_reader, up_writer, first_out)
+            if first_in is not None:
+                await self._relay(reader, writer, client, host, port, started, len(tried), entry,
+                                  up_reader, up_writer, first_out, first_in, upload=0)
+                return True
+            self._give_up(entry, up_writer)
+        self._log(client, host, port, None, False, started, len(tried))
+        await self._bad_gateway(writer)
+        return False
+
+    async def _open_next(self, tried: Set[str], host: str, port: int, tls: bool, tunnel: bool,
+                         selection: Selection = ANY):
+        """Open the next proxy from the pool (at most max_attempts per request). None = none left.
+
+        A proxy that is reachable but can't reach the target is only blamed once another proxy reaches it:
+        if every attempt fails that way, the target is the problem and nobody in the pool gets disabled."""
+        refused = []  # proxies that answered but couldn't open the connection to the target
+        while len(tried) < self.max_attempts:
+            entry = self.pool.pick(tried, tls=tls, selection=selection, target=f"{host}:{port}")
+            if entry is None:
+                return None
+            tried.add(entry.result.key)
+            # count as busy while connecting already – otherwise concurrent requests would see the fastest
+            # proxy as free (important for --rotate fastest); released in _give_up or at the end of _relay
+            entry.active += 1
+            try:
+                up_reader, up_writer = await open_upstream(entry, host, port, self.timeout, tunnel=tunnel,
+                                                           public_only=self.public_targets_only)
+            except Unsupported:
+                entry.active -= 1  # this type can't do this target – not a failure of the proxy
+                continue
+            except TargetError:
+                entry.active -= 1
+                refused.append(entry)
+                continue
+            except UpstreamError:
+                entry.active -= 1
+                self.pool.report(entry, False)
+                continue
+            for other in refused:  # the target is reachable after all – those proxies were the problem
+                self.pool.report(other, False)
+            return entry, up_reader, up_writer
+        return None
+
+    async def _exchange(self, up_reader, up_writer, first_out: bytes) -> Optional[bytes]:
+        """Send the first packet, wait for the first response. None = this proxy is no good right now."""
+        try:
+            if first_out:
+                up_writer.write(first_out)
+                await up_writer.drain()
+            first_in = await asyncio.wait_for(up_reader.read(65536), self.timeout)
+            if first_in and first_out[:1] != TLS_HANDSHAKE:
+                first_in = await self._screen_first_answer(up_reader, first_in)
+        except (OSError, asyncio.TimeoutError):
+            return None
+        if not first_in or not plausible_answer(first_out, first_in):
+            return None
+        return first_in
+
+    async def _screen_first_answer(self, up_reader, first_in: bytes) -> bytes:
+        """Read up to the final status line (past 100 Continue and the like). b"" = the proxy wants a login."""
+        screen = ResponseScreen()
+        out, verdict = screen.feed(first_in)
+        while verdict is None:
+            more = await asyncio.wait_for(up_reader.read(65536), self.timeout)
+            if not more:
+                return b""  # hung up before a final response came – that's not a success
+            data, verdict = screen.feed(more)
+            out += data
+        return b"" if verdict == "407" else out
+
+    async def _relay(self, reader, writer, client, host, port, started, attempts, entry,
+                     up_reader, up_writer, first_out: bytes, first_in: bytes, upload=None) -> bool:
+        """Pass both directions through. Success only counts once the upstream has answered – for
+        streamed bodies (first_in empty) that means once any data comes back at all.
+
+        upload: for plain HTTP only the rest of this request's body may go up (a byte count or a ChunkedEnd).
+        Whatever the client sends after it – a second keep-alive request with its Proxy-Authorization,
+        for example – never reaches the free proxy. None = tunnel, everything goes through."""
+        if first_in:
+            self._account(client, host, port, entry, True, started, attempts)
+        try:
+            self.stats.bytes_up += len(first_out)
+            self.stats.bytes_down += len(first_in)
+            if first_in:
+                writer.write(first_in)
+                await writer.drain()
+            _, received = await asyncio.gather(
+                self._pipe(reader, up_writer, up=True) if upload is None
+                else self._send_body(reader, up_writer, upload),
+                # without a first response up front (streamed body) watch for a 407 from the proxy here
+                self._pipe(up_reader, writer, up=False, reject_proxy_auth=not first_in),
+            )
+        finally:
+            entry.active -= 1
+            up_writer.close()
+        if not first_in:
+            self._account(client, host, port, entry, received > 0, started, attempts)
+        return bool(first_in) or received > 0
+
+    def _give_up(self, entry, up_writer) -> None:
+        """This proxy didn't deliver: count it as a failure, close the connection, release the reservation."""
+        entry.active -= 1
+        self.pool.report(entry, False)
+        up_writer.close()
+
+    def _account(self, client, host, port, entry, ok: bool, started, attempts) -> None:
+        self.pool.report(entry, ok)
+        if ok:
+            self.stats.ok += 1
+        self._log(client, host, port, entry, ok, started, attempts)
+
+    async def _read_body(self, reader, headers) -> Tuple[bytes, bool]:
+        """Read the request body if it's small enough to buffer -> (body, repeatable?)."""
+        values = {name.lower(): value for name, value in headers}
+        if b"chunked" in values.get(b"transfer-encoding", b"").lower():
+            return b"", False
+        if b"100-continue" in values.get(b"expect", b"").lower():
+            return b"", False  # the client only sends the body after "100 Continue" from the target
+        length = values.get(b"content-length", b"0").strip()
+        if not length.isdigit() or int(length) > MAX_REPLAY_BODY:
+            return b"", False
+        return (await reader.readexactly(int(length)) if int(length) else b""), True
+
+    async def _first_client_chunk(self, reader) -> bytes:
+        """The client's first packet in the tunnel (for HTTPS: TLS ClientHello). Empty if the server is
+        supposed to talk first (e.g. SSH) – then we wait for the other side right away."""
+        try:
+            data = await asyncio.wait_for(reader.read(65536), FIRST_CHUNK_WAIT)
+            if data[:1] == TLS_HANDSHAKE:
+                data = await self._complete_tls_record(reader, data)
+            return data
+        except asyncio.TimeoutError:
+            return b""
+
+    async def _complete_tls_record(self, reader, data: bytes) -> bytes:
+        """Collect the first TLS record completely – it can be spread over several TCP packets, and when
+        switching proxies not just a fragment of the ClientHello should go out. If the rest is slow, what
+        arrived so far goes out anyway – dropping it would break the handshake for good."""
+        while True:
+            # first the 5-byte header, then the length is known (at most one buffer full)
+            needed = 5 if len(data) < 5 else min(5 + int.from_bytes(data[3:5], "big"), 65536)
+            if len(data) >= needed:
+                return data
+            try:
+                more = await asyncio.wait_for(reader.read(needed - len(data)), FIRST_CHUNK_WAIT)
+            except asyncio.TimeoutError:
+                return data
+            if not more:
+                return data
+            data += more
+
+    async def _bad_gateway(self, writer) -> None:
+        # the marker tells clients like agent.PageFetcher that no proxy got through – the target never answered
+        writer.write(b"HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain; charset=utf-8\r\n"
+                     b"X-Proxy-Scraper: no-proxy-answered\r\n"
+                     b"Connection: close\r\n\r\nNo proxy from the pool answered.\n")
+        await writer.drain()
+
+    async def _send_body(self, reader, writer, body) -> int:
+        """Pass exactly the rest of one request body upstream, then stop reading from the client."""
+        total = 0
+        with contextlib.suppress(ConnectionError, OSError, ValueError):
+            while body:
+                data = await reader.read(min(body, 65536) if isinstance(body, int) else 65536)
+                if not data:
+                    break
+                if isinstance(body, int):
+                    body -= len(data)
+                else:
+                    end = body.feed(data)
+                    if end is not None:
+                        data, body = data[:end], 0
+                total += len(data)
+                self.stats.bytes_up += len(data)
+                writer.write(data)
+                await writer.drain()
+        return total
+
+    async def _pipe(self, reader, writer, up: bool, reject_proxy_auth: bool = False) -> int:
+        """Pass data through until one side stops; returns the number of bytes.
+
+        reject_proxy_auth: if the final response is a 407 (also after 100 Continue), the client gets
+        a 502 instead and -1 is returned (the proxy counts as failed)."""
+        total = 0
+        screen = ResponseScreen() if reject_proxy_auth else None
+        try:
+            while True:
+                data = await reader.read(65536)
+                if not data:
+                    if screen:  # hung up before a final response came – a 502 for the client
+                        await self._bad_gateway(writer)
+                        return -1
+                    break
+                if screen:
+                    data, verdict = screen.feed(data)
+                    if verdict == "407":
+                        await self._bad_gateway(writer)
+                        return -1
+                    if verdict == "ok":
+                        screen = None
+                    if not data:
+                        continue
+                total += len(data)
+                if up:
+                    self.stats.bytes_up += len(data)
+                else:
+                    self.stats.bytes_down += len(data)
+                writer.write(data)
+                await writer.drain()
+        except (ConnectionError, OSError, asyncio.TimeoutError):
+            # one side hung up – that normally ends the relay just fine. But if no final response came yet
+            # (the upstream aborts with an RST, for example, because our body sat unread in its buffer),
+            # that's just as much a failure as a clean hang-up.
+            if screen:
+                with contextlib.suppress(ConnectionError, OSError):
+                    await self._bad_gateway(writer)
+                return -1
+        finally:
+            try:
+                writer.write_eof()
+            except (OSError, RuntimeError, AttributeError):
+                writer.close()
+        return total
+
+    def _log(self, client, host, port, entry, ok, started, attempts) -> None:
+        via = f"{entry.result.ptype}://{entry.result.proxy}" if entry else "–"
+        target = host if port in (80, 443) else f"{host}:{port}"
+        self.stats.recent.append(RequestLog(client, target, via, ok, round((time.perf_counter() - started) * 1000),
+                                            attempts))
+
+    # ------------------------------------------------------------------ SOCKS5, status, refreshing
+
+    async def _serve_socks5(self, reader, writer, client) -> None:
+        """SOCKS5 on the same port. Authentication optional – the user name carries the same wishes as with
+        HTTP ("country-de-session-abc"). After that everything works like CONNECT, including switching on errors."""
+        try:
+            host, port, username = await asyncio.wait_for(socks5_accept(reader, writer, self.password), self.timeout)
+        except (Socks5Refused, asyncio.IncompleteReadError, asyncio.TimeoutError, ValueError, UnicodeError):
+            return  # refused, hung up half-way, too slow or a broken address – close the connection
+        self.stats.requests += 1
+        started = time.perf_counter()
+
+        async def refuse(w) -> None:
+            w.write(socks5_reply(0x04))  # host unreachable – none of the proxies got through
+            await w.drain()
+
+        served = await self._serve_tunnel(reader, writer, client, host, port, started,
+                                          Selection.from_username(username), socks5_reply(0x00), refuse)
+        if not served:
+            self.stats.failed += 1
+
+    async def _serve_status(self, writer, head: bytes) -> None:
+        target = head.split(b" ", 2)[1].split(b"?", 1)[0]
+        kind = b"application/json"
+        headers = [tuple(part.strip() for part in line.split(b":", 1)) for line in head.split(b"\r\n")[1:]
+                   if b":" in line]
+        if not password_ok(headers, self.password, b"authorization"):
+            writer.write(b"HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm=\"proxy-scraper\"\r\n"
+                         b"Content-Length: 0\r\nConnection: close\r\n\r\n")
+            await writer.drain()
+            return
+        if target == STATUS_PATH:
+            status, body = b"200 OK", status_json(self).encode()
+        elif target == METRICS_PATH:
+            status, body, kind = b"200 OK", metrics_text(self).encode(), METRICS_TYPE
+        else:  # only exactly these paths – typos shouldn't silently return the status
+            status, body = b"404 Not Found", b'{"error": "unknown path, try /__proxy-scraper/status or /metrics"}'
+        writer.write(b"HTTP/1.1 " + status + b"\r\nContent-Type: " + kind + b"\r\nCache-Control: no-store\r\n"
+                     b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(body) + body)
+        await writer.drain()
+
+    async def keep_fresh(self, recheck, interval: float = 300.0) -> None:
+        """Recheck disabled proxies regularly and bring them back when they work again.
+
+        `recheck(result)` is the normal check (True = works). Runs until the task is cancelled."""
+        while True:
+            await asyncio.sleep(interval)
+            await self.refresh_once(recheck)
+
+    async def refresh_once(self, recheck) -> int:
+        dead = [e for e in self.pool.entries if e.disabled]
+        if not dead:
+            return 0
+        results = await asyncio.gather(*(recheck(e.result) for e in dead), return_exceptions=True)
+        revived = 0
+        for entry, ok in zip(dead, results):
+            if ok is True:
+                self.pool.revive(entry)
+                revived += 1
+        self.revived += revived
+        return revived

proxyscraper/server/http.py

@@ -0,0 +1,164 @@
+"""HTTP on the client and the upstream side: take requests apart and rebuild them, check responses."""
+
+from __future__ import annotations
+
+import re
+from typing import List, Tuple
+
+TLS_HANDSHAKE, TLS_ALERT = b"\x16", b"\x15"  # first byte of a TLS record
+HOP_BY_HOP = {b"proxy-connection", b"connection", b"keep-alive", b"proxy-authorization", b"te", b"upgrade"}
+PROXY_AUTH_REQUIRED = re.compile(rb"HTTP/1\.[01] 407\b")
+STATUS_LEN = len(b"HTTP/1.1 407 ")
+STATUS_RE = re.compile(rb"HTTP/1\.[01] (\d{3})[ \r\n]")  # exactly three digits – "4070" is not a 407
+SCREEN_LIMIT = 16384
+
+
+def parse_request_head(head: bytes) -> Tuple[str, str, int, bytes, List[Tuple[bytes, bytes]]]:
+    """-> (method, host, port, path, headers). Supports CONNECT host:port and absolute URLs."""
+    lines = head.split(b"\r\n")
+    method, target, _version = lines[0].split(b" ", 2)
+    headers = []
+    for line in lines[1:]:
+        if line:
+            name, _, value = line.partition(b":")
+            headers.append((name.strip(), value.strip()))
+    if method == b"CONNECT":
+        host, _, port = target.decode("ascii").rpartition(":")
+        return "CONNECT", host.strip("[]"), _valid_port(port), b"", headers
+    if not target.lower().startswith(b"http://"):
+        raise ValueError("only absolute http:// URLs or CONNECT")
+    rest = target[7:]
+    hostport, slash, path = rest.partition(b"/")
+    authority = hostport.decode("ascii")
+    if authority.startswith("["):  # IPv6 literal: http://[2606:4700::1]:8080/
+        host, _, after = authority[1:].partition("]")
+        port = after[1:] if after.startswith(":") else ""
+    else:
+        host, _, port = authority.partition(":")
+    return method.decode("ascii"), host, _valid_port(port or "80"), b"/" + path if slash else b"/", headers
+
+
+def _valid_port(text: str) -> int:
+    port = int(text)
+    if not 0 < port < 65536:
+        raise ValueError(f"invalid port {port}")
+    return port
+
+
+def origin_request(method: str, path: bytes, host: str, port: int, headers: List[Tuple[bytes, bytes]]) -> bytes:
+    """Request for the target server: path instead of an absolute URL, no proxy headers, one request per connection."""
+    lines = [f"{method} ".encode() + path + b" HTTP/1.1"]
+    if not any(name.lower() == b"host" for name, _ in headers):
+        lines.append(b"Host: " + (host if port == 80 else f"{host}:{port}").encode())
+    lines += [name + b": " + value for name, value in headers if name.lower() not in HOP_BY_HOP]
+    lines.append(b"Connection: close")
+    return b"\r\n".join(lines) + b"\r\n\r\n"
+
+
+def forward_request(method: str, path: bytes, host: str, port: int, headers: List[Tuple[bytes, bytes]]) -> bytes:
+    """Request to an HTTP upstream proxy: absolute URL as sent by the client, just without proxy headers."""
+    authority = host if port == 80 else f"{host}:{port}"
+    request = origin_request(method, path, host, port, headers)
+    rest = request.split(b"\r\n", 1)[1]  # the first line is replaced by the absolute URL
+    return f"{method} http://{authority}".encode() + path + b" HTTP/1.1\r\n" + rest
+
+
+class ChunkedEnd:
+    """Finds where a chunked request body ends, fed piece by piece. feed() returns how many bytes of this
+    piece still belong to the body once the end is in it, otherwise None. Garbage raises ValueError."""
+
+    MAX_LINE = 4096
+
+    def __init__(self):
+        self.line = b""
+        self.left = 0  # bytes of chunk data (plus its CRLF) still to come
+        self.trailer = False
+
+    def feed(self, data: bytes):
+        i = 0
+        while i < len(data):
+            if self.left:
+                take = min(self.left, len(data) - i)
+                self.left -= take
+                i += take
+                continue
+            j = data.find(b"\n", i)
+            if j < 0:
+                self.line += data[i:]
+                if len(self.line) > self.MAX_LINE:
+                    raise ValueError("chunk line too long")
+                return None
+            line = (self.line + data[i:j]).rstrip(b"\r")
+            self.line = b""
+            i = j + 1
+            if self.trailer:
+                if not line:
+                    return i
+                continue
+            size = int(line.split(b";", 1)[0].strip(), 16)
+            if size < 0:
+                raise ValueError("negative chunk size")
+            if size:
+                self.left = size + 2
+            else:
+                self.trailer = True
+        return None
+
+
+def request_body_length(headers: List[Tuple[bytes, bytes]]):
+    """How much body follows the head: a byte count, or a ChunkedEnd for chunked bodies."""
+    values = {name.lower(): value for name, value in headers}
+    if b"chunked" in values.get(b"transfer-encoding", b"").lower():
+        return ChunkedEnd()
+    length = values.get(b"content-length", b"0").strip()
+    return int(length) if length.isdigit() else 0
+
+
+class ResponseScreen:
+    """Checks the start of an upstream response until the final status line is there.
+
+    Interim responses (100 Continue and the like) go to the client right away; if a 407 follows,
+    the client should never see it. feed() returns what may already go on, and the decision:
+    None = still open, "ok" = just pass everything else through, "407" = the proxy wants a login
+    (or sends an implausibly long interim response – both mean: don't use this upstream)."""
+
+    def __init__(self):
+        self.buf = b""
+
+    def feed(self, data: bytes):
+        self.buf += data
+        out = b""
+        while True:
+            head = self.buf
+            if len(head) < STATUS_LEN and b"\n" not in head and head[:5] == b"HTTP/"[:len(head[:5])]:
+                return out, None  # status line still incomplete
+            m = STATUS_RE.match(head)
+            if not m:
+                return out + self._flush(), "ok"  # not an HTTP response (e.g. a tunnel) – nothing to check
+            code = m.group(1)
+            if code == b"407":
+                return out, "407"
+            if not code.startswith(b"1") or code == b"101":  # 101 Switching Protocols is final
+                return out + self._flush(), "ok"
+            end = head.find(b"\r\n\r\n")
+            if end < 0:
+                if len(head) > SCREEN_LIMIT:
+                    return out, "407"  # huge interim response – better count it as failed than let it through blindly
+                return out, None  # interim response not complete yet
+            out += head[:end + 4]
+            self.buf = head[end + 4:]
+            if not self.buf:
+                return out, None
+
+    def _flush(self) -> bytes:
+        data, self.buf = self.buf, b""
+        return data
+
+
+def plausible_answer(first_out: bytes, first_in: bytes) -> bool:
+    """Does the first response fit the request? If the client starts with a TLS handshake (0x16), the
+    other side has to speak TLS too – some proxies send an HTTP error page inside the tunnel instead.
+    A 407 always comes from the proxy itself (login missing or wrong), never from the target site."""
+    if first_out[:1] == TLS_HANDSHAKE:
+        return first_in[:1] in (TLS_HANDSHAKE, TLS_ALERT)
+    return not PROXY_AUTH_REQUIRED.match(first_in)