pybrws 0.1.0__py3-none-any.whl

pybrws/__init__.py

@@ -0,0 +1,3 @@
+"""pybrws - a from-scratch browser-fingerprinted TLS client."""
+
+__version__ = "0.1.0"

pybrws/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

pybrws/certs.py

@@ -0,0 +1,275 @@
+"""Certificate handling: parsing, decompression, chain and signature checks.
+
+Path validation is delegated to `cryptography.x509.verification`. Hand-rolling
+it would mean reimplementing name constraints, policy handling and revocation
+semantics, which is the kind of code that is wrong in ways nobody notices
+until it matters.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import ssl
+import warnings
+
+from cryptography import x509
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec, ed25519, padding, rsa
+from cryptography.x509.verification import PolicyBuilder, Store
+
+from . import hosts
+from .parser import _Reader
+
+# RFC 8879
+COMPRESSION_ZLIB = 1
+COMPRESSION_BROTLI = 2
+COMPRESSION_ZSTD = 3
+
+# The string a TLS 1.3 server signs, RFC 8446 section 4.4.3.
+SIGNATURE_CONTEXT = b" " * 64 + b"TLS 1.3, server CertificateVerify" + b"\x00"
+
+RSA_PSS = {
+    0x0804: hashes.SHA256,
+    0x0805: hashes.SHA384,
+    0x0806: hashes.SHA512,
+}
+ECDSA = {
+    0x0403: hashes.SHA256,
+    0x0503: hashes.SHA384,
+    0x0603: hashes.SHA512,
+}
+RSA_PKCS1 = {
+    0x0401: hashes.SHA256,
+    0x0501: hashes.SHA384,
+    0x0601: hashes.SHA512,
+}
+ED25519 = 0x0807
+
+
+class CertificateError(Exception):
+    pass
+
+
+def decompress_certificate(message: bytes) -> bytes:
+    """Turn an RFC 8879 CompressedCertificate body into a Certificate body.
+
+    The output is bounded by the declared length *while* decompressing.
+    Checking it afterwards would be too late: this message arrives before the
+    server has proved anything, and a kilobyte of brotli can expand to a
+    gigabyte. RFC 8879 section 4 asks for exactly this bound. The declared
+    length is a uint24, so its own ceiling of 2**24 - 1 holds by construction.
+    """
+    reader = _Reader(message, "CompressedCertificate")
+    algorithm = reader.uint(2)
+    expected = reader.uint(3)
+    compressed = reader.vector(3)
+    # One byte past the declared length is enough to know it was a lie.
+    limit = expected + 1
+
+    try:
+        if algorithm == COMPRESSION_ZLIB:
+            import zlib
+
+            plain = zlib.decompressobj().decompress(compressed, limit)
+        elif algorithm == COMPRESSION_BROTLI:
+            import brotli
+
+            # Bounded to within one internal block of the limit, which is
+            # enough: the length check below rejects anything past it.
+            plain = brotli.Decompressor().process(
+                compressed, output_buffer_limit=limit)
+        elif algorithm == COMPRESSION_ZSTD:
+            try:
+                from compression import zstd
+            except ImportError:
+                import io
+
+                import zstandard
+
+                with zstandard.ZstdDecompressor().stream_reader(
+                        io.BytesIO(compressed)) as source:
+                    plain = source.read(limit)
+            else:
+                plain = zstd.ZstdDecompressor().decompress(
+                    compressed, max_length=limit)
+        else:
+            raise CertificateError(
+                "unknown certificate compression " + str(algorithm))
+    except CertificateError:
+        raise
+    except Exception as exc:
+        raise CertificateError(
+            "certificate decompression failed: " + str(exc)) from exc
+
+    if len(plain) != expected:
+        raise CertificateError(
+            "decompressed to %d bytes, the server said %d" % (len(plain), expected)
+        )
+    return plain
+
+
+def parse_certificate(body: bytes) -> list[bytes]:
+    """Return the DER certificates in a Certificate message body, leaf first."""
+    reader = _Reader(body, "Certificate")
+    reader.vector(1)  # certificate_request_context, empty for a server
+    entries = _Reader(reader.vector(3), "certificate_list")
+    chain = []
+    while entries.remaining:
+        chain.append(entries.vector(3))
+        entries.vector(2)  # per-certificate extensions
+    if not chain:
+        raise CertificateError("the server sent an empty certificate list")
+    return chain
+
+
+def _parse_anchors(ders) -> list:
+    anchors = []
+    for der in ders:
+        try:
+            anchors.append(x509.load_der_x509_certificate(der))
+        except Exception:
+            # Bundles carry the occasional malformed anchor. One
+            # unparseable certificate should not cost the other few hundred.
+            continue
+    return anchors
+
+
+def _platform_roots() -> list[bytes]:
+    context = ssl.create_default_context()
+    return list(context.get_ca_certs(binary_form=True))
+
+
+def _mozilla_roots() -> list[bytes]:
+    """The DER of each certificate in the certifi bundle, which is Mozilla's
+    root program. A bundle that cannot be read counts as having none."""
+    import certifi
+
+    try:
+        with open(certifi.where(), encoding="ascii") as bundle:
+            text = bundle.read()
+    except (OSError, UnicodeDecodeError):
+        return []
+    begin, end = "-----BEGIN CERTIFICATE-----", "-----END CERTIFICATE-----"
+    ders = []
+    for block in text.split(end)[:-1]:
+        # The bundle puts a comment above each certificate.
+        _, found, body = block.partition(begin)
+        if not found:
+            continue
+        try:
+            ders.append(ssl.PEM_cert_to_DER_cert(begin + body + end + "\n"))
+        except ValueError:
+            continue
+    return ders
+
+
+def system_store() -> Store:
+    """The platform's trust anchors alone, as Python's ssl module sees them.
+
+    Raises:
+        CertificateError: if the platform has none.
+    """
+    warnings.filterwarnings("ignore", message="Parsed a serial number")
+    anchors = _parse_anchors(_platform_roots())
+    if not anchors:
+        raise CertificateError("no trust anchors available from the system store")
+    return Store(anchors)
+
+
+def default_store() -> Store:
+    """The roots a connection is verified against by default: Mozilla's
+    (the certifi bundle) plus whatever the platform adds, an enterprise CA
+    for example. That is close to Firefox's mix: Firefox adds only the roots
+    an administrator or user installed on the system, not the platform's
+    whole root program.
+
+    A certificate that is in both counts once. A source that cannot be read
+    counts as having none.
+
+    Raises:
+        CertificateError: if both sources yield no anchors.
+    """
+    warnings.filterwarnings("ignore", message="Parsed a serial number")
+    seen = set()
+    unique = []
+    for der in _mozilla_roots() + _platform_roots():
+        if der not in seen:
+            seen.add(der)
+            unique.append(der)
+    anchors = _parse_anchors(unique)
+    if not anchors:
+        raise CertificateError("no trust anchors available from certifi or the system store")
+    return Store(anchors)
+
+
+def verify_chain(chain: list[bytes], hostname: str, store: Store | None = None):
+    """Validate the chain for `hostname`, returning the leaf certificate.
+
+    Raises:
+        CertificateError: if the chain does not validate.
+    """
+    certificates = [x509.load_der_x509_certificate(der) for der in chain]
+    leaf, intermediates = certificates[0], certificates[1:]
+    if hosts.is_ip_literal(hostname):
+        # An address is matched against an IP address SAN, never a DNS one.
+        subject = x509.IPAddress(ipaddress.ip_address(hostname.strip("[]")))
+    else:
+        subject = x509.DNSName(hostname)
+    verifier = (
+        PolicyBuilder()
+        .store(store if store is not None else default_store())
+        .build_server_verifier(subject)
+    )
+    try:
+        verifier.verify(leaf, intermediates)
+    except Exception as exc:
+        raise CertificateError("certificate chain rejected: " + str(exc)) from exc
+    return leaf
+
+
+def verify_signature(leaf, scheme: int, signature: bytes, transcript_hash: bytes) -> None:
+    """Check the CertificateVerify signature over the handshake transcript.
+
+    This is what proves the peer holds the key in the certificate, rather
+    than merely having a copy of someone else's chain.
+    """
+    signed = SIGNATURE_CONTEXT + transcript_hash
+    key = leaf.public_key()
+    try:
+        if scheme in RSA_PSS:
+            algorithm = RSA_PSS[scheme]()
+            key.verify(signature, signed,
+                       padding.PSS(mgf=padding.MGF1(algorithm),
+                                   salt_length=algorithm.digest_size),
+                       algorithm)
+        elif scheme in ECDSA:
+            key.verify(signature, signed, ec.ECDSA(ECDSA[scheme]()))
+        elif scheme == ED25519:
+            key.verify(signature, signed)
+        elif scheme in RSA_PKCS1:
+            # Not permitted for TLS 1.3 CertificateVerify, but accepting it
+            # silently would hide a server doing the wrong thing.
+            raise CertificateError(
+                "server signed with rsa_pkcs1, which TLS 1.3 forbids here"
+            )
+        else:
+            raise CertificateError("unsupported signature scheme 0x%04x" % scheme)
+    except InvalidSignature as exc:
+        raise CertificateError("CertificateVerify signature is invalid") from exc
+    except CertificateError:
+        raise
+    except Exception as exc:
+        raise CertificateError("CertificateVerify could not be checked: "
+                               + str(exc)) from exc
+
+
+def expected_key_type(leaf) -> str:
+    key = leaf.public_key()
+    if isinstance(key, rsa.RSAPublicKey):
+        return "rsa"
+    if isinstance(key, ec.EllipticCurvePublicKey):
+        return "ecdsa"
+    if isinstance(key, ed25519.Ed25519PublicKey):
+        return "ed25519"
+    return type(key).__name__

pybrws/cli.py

@@ -0,0 +1,303 @@
+"""Command line front end, shaped like curl."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from urllib.parse import urljoin
+
+from . import profiles
+from . import headers as header_tools
+from .client import (REDIRECT_STATUSES, TooManyRedirects, UnsupportedUrl,
+                     Unmeasured, redirect_headers, request)
+from .message import DEFAULT_MAX_BODY, ContentEncodingError, Response
+from .certs import CertificateError
+from .h3 import Http3Error
+from .handshake import HandshakeError, TruncatedStream
+from .http1 import Http1Error
+from .http2 import Http2Error
+from .proxies import ProxyError, parse_proxy
+from .quic_impersonate import NotInstalled
+from .records import TlsAlert, UnexpectedRecord
+from .tls12 import Tls12Error
+
+# What the library raises for a server, a response, a profile or an input it
+# will not take: reported in one line, as curl does, not as a traceback.
+REPORTED = (UnsupportedUrl, TooManyRedirects, TlsAlert, ProxyError, CertificateError,
+            Unmeasured, ContentEncodingError, Http1Error, Http2Error, Http3Error,
+            HandshakeError, Tls12Error, UnexpectedRecord, TruncatedStream,
+            UnicodeError, profiles.ProfileError, NotInstalled)
+
+
+# `pybrws <name> ...` runs one of the method's tools; each module has
+# main(argv) -> int. `capture` takes a second word, the kind of capture.
+SUBCOMMANDS = {
+    "read-quic": "pybrws.tools.read_quic",
+    "ja4": "pybrws.tools.ja4",
+    "verify": "pybrws.tools.verify",
+    "probe": "pybrws.tools.probe_fingerprint",
+    "ab": "pybrws.tools.ab_test",
+    "isolate": "pybrws.tools.isolate",
+    "waf": "pybrws.tools.detect_waf",
+}
+CAPTURE_KINDS = {
+    "hello": "pybrws.tools.catch_hello",
+    "request": "pybrws.tools.catch_request",
+    "h2": "pybrws.tools.catch_h2",
+    "quic": "pybrws.tools.catch_quic",
+    "resumed": "pybrws.tools.catch_resumed",
+}
+# The optional dependency to name when an import of it fails.
+EXTRA_FOR = {"httpx": "measure", "aioquic": "http3"}
+
+
+def missing_extra(error: ModuleNotFoundError, what: str):
+    """SystemExit naming the extra to install, or None for any other module."""
+    extra = EXTRA_FOR.get((error.name or "").split(".")[0])
+    if extra is None:
+        return None
+    return SystemExit("pybrws %s needs %s: pip install 'pybrws[%s]'"
+                      % (what, error.name.split(".")[0], extra))
+
+
+def run_tool(module_name: str, argv: list[str], what: str) -> int:
+    import importlib
+
+    try:
+        return importlib.import_module(module_name).main(argv)
+    except ModuleNotFoundError as error:
+        exit_ = missing_extra(error, what)
+        if exit_ is None:
+            raise
+        raise exit_ from None
+
+
+def dispatch(argv: list[str]) -> int | None:
+    """Run a subcommand if `argv` names one; None means it is a fetch."""
+    if not argv:
+        return None
+    if argv[0] in SUBCOMMANDS:
+        return run_tool(SUBCOMMANDS[argv[0]], argv[1:], argv[0])
+    if argv[0] == "capture":
+        kinds = ", ".join(CAPTURE_KINDS)
+        if len(argv) < 2 or argv[1] in ("-h", "--help"):
+            print("usage: pybrws capture KIND [options]\n\nKIND is one of: " + kinds)
+            return 0
+        if argv[1] not in CAPTURE_KINDS:
+            raise SystemExit("pybrws capture: unknown kind %r; one of: %s"
+                             % (argv[1], kinds))
+        return run_tool(CAPTURE_KINDS[argv[1]], argv[2:], "capture " + argv[1])
+    return None
+
+
+def parse_header(raw: str) -> tuple[str, str]:
+    name, separator, value = raw.partition(":")
+    if not separator:
+        raise argparse.ArgumentTypeError("expected 'Name: value', got " + repr(raw))
+    return name.strip(), value.strip()
+
+
+def resolve_profile(name: str, http3: bool):
+    """The profile for the transport actually being used."""
+    try:
+        return profiles.load(name, transport="quic" if http3 else "tcp")
+    except profiles.WrongTransport as error:
+        # Only this front end has --http3, so only it can name the flag.
+        raise SystemExit(str(error) + ("; add --http3" if error.has == "quic"
+                                       else "; drop --http3"))
+    except profiles.ProfileError as error:
+        raise SystemExit(str(error))
+
+
+def fetch_http3(args, profile, overrides) -> Response:
+    """One HTTP/3 fetch, following redirects by hand.
+
+    `pybrws.h3` returns the response it gets and does not follow anything,
+    so the loop lives here. Without it `-L` would silently mean nothing on
+    this path while meaning something on the other -- the same arm
+    answering a different question, which is how a comparison between the
+    two transports goes wrong.
+    """
+    from . import h3
+
+    url = args.url
+    history: list[Response] = []
+    budget = args.max_redirs if args.location else 0
+    for hop in range(budget + 1):
+        # The method and body asked for: h3 refuses what it has no
+        # capture for, where this used to send a GET whatever -X and -d said.
+        response = h3.request(url, method=args.request.upper(), data=args.data,
+                              timeout=args.timeout, headers=overrides or None,
+                              profile=profile, verify=not args.insecure,
+                              max_body_size=getattr(args, "max_body_size",
+                                                    DEFAULT_MAX_BODY))
+        response.url = url
+        location = response.header("location")
+        # The statuses client.request follows, so both arms agree on what
+        # a redirect is; a 304 with a Location header is not one.
+        if response.status not in REDIRECT_STATUSES or not location:
+            break
+        if hop == budget:
+            if budget == 0:
+                break   # --no-location: the redirect is the answer, as over TCP
+            raise TooManyRedirects("still redirecting after %d hops, last %r"
+                                   % (budget, urljoin(url, location)))
+        history.append(response)
+        next_url = urljoin(url, location)
+        overrides = redirect_headers(overrides, url, next_url)
+        url = next_url
+    response.history = history
+    return response
+
+
+def describe_request(profile, args) -> str:
+    lines = ["> " + args.url,
+             "> transport: " + ("quic/h3" if args.http3 else "tcp"),
+             "> profile: " + profile.NAME]
+    if args.proxy:
+        # str(Proxy) omits the password: -v output gets pasted into issues.
+        lines.append("> proxy: " + str(parse_proxy(args.proxy)))
+    for name, value in args.header:
+        lines.append("> override: " + name + ": " + value)
+    return "\n".join(lines)
+
+
+def render_head(response: Response) -> str:
+    version = response.http_version or "h2"
+    lines = [version + " " + str(response.status)]
+    lines += [name + ": " + value for name, value in response.headers]
+    return "\n".join(lines)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pybrws",
+        description="Fetch an https URL with a browser's protocol fingerprint.",
+        epilog="subcommands (pybrws <name> --help): capture {%s}, %s"
+               % (",".join(CAPTURE_KINDS), ", ".join(SUBCOMMANDS)),
+    )
+    parser.add_argument("url", help="the https URL to fetch")
+    parser.add_argument("-X", "--request", default=None, metavar="METHOD",
+                        help="HTTP method; defaults to GET, or POST with -d")
+    parser.add_argument("-d", "--data", metavar="BODY",
+                        help="request body; implies POST unless -X says otherwise")
+    parser.add_argument("-H", "--header", action="append", default=[],
+                        type=parse_header, metavar="'Name: value'",
+                        help="replace or add a request header, repeatable")
+    parser.add_argument("-o", "--output", metavar="FILE",
+                        help="write the body to FILE instead of stdout")
+    parser.add_argument("-i", "--include", action="store_true",
+                        help="print the response head before the body")
+    parser.add_argument("-I", "--head-only", action="store_true",
+                        help="print the response head and not the body")
+    # curl needs -L; a browser always follows, and this tool imitates a
+    # browser. --no-location restores curl's behaviour.
+    parser.add_argument("-L", "--location", action="store_true", default=True,
+                        help="follow redirects (the default)")
+    parser.add_argument("--no-location", dest="location", action="store_false",
+                        help="return the redirect instead of following it")
+    parser.add_argument("--max-redirs", type=int, default=5, metavar="N")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="describe the request on stderr")
+    parser.add_argument("-s", "--silent", action="store_true",
+                        help="suppress the status line on stderr")
+    parser.add_argument("-x", "--proxy", metavar="URL",
+                        help="http://, socks5:// or socks5h://, with optional "
+                             "user:pass@")
+    parser.add_argument("-k", "--insecure", action="store_true",
+                        help="skip certificate verification, as curl's -k does")
+    parser.add_argument("--profile", default=None, metavar="NAME",
+                        help="firefox157 or chrome155 over TCP, "
+                             "firefox157_quic with --http3; or a .py "
+                             "profile file")
+    parser.add_argument("--http3", action="store_true",
+                        help="fetch over HTTP/3 instead of TCP. A separate "
+                             "profile and a separate code path, not a "
+                             "negotiation")
+    parser.add_argument("--timeout", type=float, default=20.0, metavar="SECONDS")
+    parser.add_argument("--max-filesize", type=int, default=None, metavar="BYTES",
+                        help="the most a response body may be, received or "
+                             "decoded (default 128 MiB); 0 lifts the cap")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv[1:]
+    done = dispatch(list(argv))
+    if done is not None:
+        return done
+    args = build_parser().parse_args(argv)
+    if args.request is None:
+        args.request = "POST" if args.data else "GET"
+    if args.max_filesize is None:
+        args.max_body_size = DEFAULT_MAX_BODY
+    else:
+        args.max_body_size = args.max_filesize or None   # 0: no cap
+    overrides = list(args.header)
+    if args.data and not header_tools.has(overrides, "content-type"):
+        # curl's default for -d, and a server will otherwise guess.
+        overrides.append(("Content-Type", "application/x-www-form-urlencoded"))
+
+    if args.profile is None:
+        args.profile = "firefox157_quic" if args.http3 else "firefox157"
+    profile = resolve_profile(args.profile, args.http3)
+    if args.http3 and args.proxy:
+        raise SystemExit("--proxy does not apply to --http3: QUIC is UDP and "
+                         "this client has no UDP proxy support")
+
+    try:
+        if args.verbose:
+            # Inside the try: describing the proxy parses it, and a proxy
+            # that does not parse is the ProxyError the request would raise.
+            print(describe_request(profile, args), file=sys.stderr)
+        if args.http3:
+            response = fetch_http3(args, profile, overrides)
+        else:
+            response = request(
+                args.url,
+                method=args.request.upper(),
+                headers=overrides or None,
+                data=args.data,
+                profile=profile,
+                timeout=args.timeout,
+                follow_redirects=args.max_redirs if args.location else 0,
+                proxy=args.proxy,
+                verify=not args.insecure,
+                max_body_size=args.max_body_size,
+            )
+    except ModuleNotFoundError as exc:
+        exit_ = missing_extra(exc, "--http3")
+        if exit_ is None:
+            raise
+        raise exit_ from None
+    except REPORTED as exc:
+        raise SystemExit(type(exc).__name__ + ": " + str(exc))
+    except OSError as exc:
+        raise SystemExit("connection failed: " + str(exc))
+
+    if not args.silent:
+        for hop in response.history:
+            print("* " + str(hop.status) + " -> " + (hop.header("location") or ""),
+                  file=sys.stderr)
+        print("* " + str(response.status) + " " + (response.http_version or "h2")
+              + " " + str(len(response.body)) + " bytes", file=sys.stderr)
+
+    if args.include or args.head_only:
+        print(render_head(response))
+        if not args.head_only:
+            print()
+
+    if not args.head_only:
+        if args.output:
+            Path(args.output).write_bytes(response.body)
+        else:
+            sys.stdout.buffer.write(response.body)
+            sys.stdout.buffer.flush()
+
+    return 0 if response.status < 400 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())