mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/acquisition/url_policy.py

@@ -0,0 +1,530 @@
+"""Fail-closed public HTTPS target validation and DNS pinning."""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import socket
+import unicodedata
+from dataclasses import dataclass
+from typing import Protocol
+from urllib.parse import SplitResult, urlsplit, urlunsplit
+
+import idna
+
+from mostlyright.data_harness.canonical import canonical_sha256
+
+MAX_URL_LENGTH = 2_048
+MAX_DNS_ANSWERS = 32
+DEFAULT_ALLOWED_PORTS = frozenset({443})
+
+_METADATA_HOSTS = frozenset(
+    {
+        "metadata",
+        "metadata.google.internal",
+        "metadata.goog",
+        "instance-data",
+        "169.254.169.254",
+        "100.100.100.200",
+    }
+)
+_FORBIDDEN_NETWORKS = tuple(
+    ipaddress.ip_network(value)
+    for value in (
+        "0.0.0.0/8",
+        "10.0.0.0/8",
+        "100.64.0.0/10",
+        "127.0.0.0/8",
+        "169.254.0.0/16",
+        "172.16.0.0/12",
+        "192.0.0.0/24",
+        "192.0.2.0/24",
+        "192.168.0.0/16",
+        "198.18.0.0/15",
+        "198.51.100.0/24",
+        "203.0.113.0/24",
+        "224.0.0.0/4",
+        "240.0.0.0/4",
+        "::/128",
+        "::1/128",
+        "::ffff:0:0/96",
+        "64:ff9b::/96",
+        "100::/64",
+        "2001:db8::/32",
+        "fc00::/7",
+        "fe80::/10",
+        "ff00::/8",
+    )
+)
+# The URL Standard's *forbidden domain code points*: the forbidden host code points, plus the C0
+# controls, plus U+0025. A host parser rejects a domain holding one before it reads anything else,
+# so no URL parser keeps any of them inside a machine name.
+#
+# Spelled out rather than probed, for the same reason `_SPECIAL_SCHEMES` is: the list is normative
+# and closed, and `urlsplit` -- the only parser available to probe locally -- is a generic RFC 3986
+# splitter that keeps most of them. Measured against node v24.13.0 over every code point the
+# catalog scanner's reader-terminator class matches, this agrees with the parser on all 33.
+_FORBIDDEN_DOMAIN_CODE_POINTS = frozenset(
+    "".join(chr(code) for code in range(0x20)) + " #%/:<>?@[\\]^|"
+)
+_DECIMAL_OR_OBFUSCATED_IP = re.compile(
+    r"^(?:0[xX][0-9A-Fa-f]+|0[0-7]+|[0-9]+)(?:\.(?:0[xX][0-9A-Fa-f]+|0[0-7]+|[0-9]+)){0,3}$"
+)
+
+
+class AcquisitionSecurityError(ValueError):
+    """Stable acquisition policy failure."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{detail} [{code}]")
+
+
+class Resolver(Protocol):
+    def resolve(self, hostname: str, port: int) -> tuple[str, ...]:
+        """Return every address candidate that may be used for this connection."""
+
+
+@dataclass(frozen=True)
+class SystemResolver:
+    """System DNS resolver with a bounded, deduplicated answer set."""
+
+    def resolve(self, hostname: str, port: int) -> tuple[str, ...]:
+        try:
+            results = socket.getaddrinfo(
+                hostname,
+                port,
+                family=socket.AF_UNSPEC,
+                type=socket.SOCK_STREAM,
+                proto=socket.IPPROTO_TCP,
+            )
+        except socket.gaierror as exc:
+            raise AcquisitionSecurityError(
+                "DNS_FAILURE",
+                f"public host resolution failed: {exc.errno}",
+            ) from None
+        addresses = tuple(dict.fromkeys(item[4][0] for item in results))
+        if not addresses:
+            raise AcquisitionSecurityError("DNS_EMPTY", "public host returned no addresses")
+        if len(addresses) > MAX_DNS_ANSWERS:
+            raise AcquisitionSecurityError(
+                "DNS_ANSWER_LIMIT",
+                f"public host returned more than {MAX_DNS_ANSWERS} addresses",
+            )
+        return addresses
+
+
+@dataclass(frozen=True)
+class EgressPolicy:
+    """Coordinator-owned public-host allowlist."""
+
+    allowed_hostnames: tuple[str, ...]
+    allowed_suffixes: tuple[str, ...] = ()
+    allowed_ports: frozenset[int] = DEFAULT_ALLOWED_PORTS
+
+    def __post_init__(self) -> None:
+        if not self.allowed_hostnames and not self.allowed_suffixes:
+            raise AcquisitionSecurityError(
+                "EGRESS_ALLOWLIST_EMPTY",
+                "at least one exact hostname or suffix is required",
+            )
+        for value in (*self.allowed_hostnames, *self.allowed_suffixes):
+            normalized = _normalize_hostname(value.lstrip("."))
+            if normalized != value.lstrip(".").rstrip(".").lower():
+                raise AcquisitionSecurityError(
+                    "EGRESS_HOST_CANONICAL",
+                    "egress host entries must use canonical ASCII spelling",
+                )
+        if not self.allowed_ports or any(
+            type(port) is not int or port < 1 or port > 65_535 for port in self.allowed_ports
+        ):
+            raise AcquisitionSecurityError(
+                "EGRESS_PORTS",
+                "allowed ports must be a non-empty set of valid integers",
+            )
+
+    def permits(self, hostname: str, port: int) -> bool:
+        if port not in self.allowed_ports:
+            return False
+        if hostname in self.allowed_hostnames:
+            return True
+        return any(
+            hostname != suffix and hostname.endswith("." + suffix)
+            for suffix in self.allowed_suffixes
+        )
+
+
+@dataclass(frozen=True)
+class ValidatedTarget:
+    """One URL bound to an exact DNS-validation epoch."""
+
+    original_url: str
+    normalized_url: str
+    hostname: str
+    port: int
+    approved_ips: tuple[str, ...]
+    resolution_digest: str
+
+    def __post_init__(self) -> None:
+        if not self.approved_ips:
+            raise AcquisitionSecurityError("DNS_EMPTY", "target has no approved peer addresses")
+        expected = canonical_sha256(
+            {
+                "hostname": self.hostname,
+                "port": self.port,
+                "approved_ips": list(self.approved_ips),
+            }
+        )
+        if expected != self.resolution_digest:
+            raise AcquisitionSecurityError(
+                "RESOLUTION_DIGEST",
+                "target resolution digest does not match its approved peer set",
+            )
+
+    @property
+    def request_target(self) -> str:
+        parsed = urlsplit(self.normalized_url)
+        return urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
+
+
+def validate_public_https_url(
+    url: str,
+    *,
+    resolver: Resolver,
+    egress_policy: EgressPolicy,
+) -> ValidatedTarget:
+    """Validate a public HTTPS URL and bind every DNS answer to the result."""
+
+    if not isinstance(url, str) or not url or len(url) > MAX_URL_LENGTH:
+        raise AcquisitionSecurityError(
+            "URL_LENGTH",
+            f"URL must contain 1..{MAX_URL_LENGTH} characters",
+        )
+    if any(0xD800 <= ord(char) <= 0xDFFF for char in url):
+        raise AcquisitionSecurityError("URL_UNICODE", "URL contains a non-Unicode-scalar value")
+    if any(ord(char) < 0x20 or ord(char) == 0x7F for char in url):
+        raise AcquisitionSecurityError("URL_CONTROL", "URL contains a control character")
+    try:
+        parsed = urlsplit(url)
+        port = parsed.port
+    except ValueError:
+        raise AcquisitionSecurityError("URL_PORT", "URL has an invalid port") from None
+    _validate_url_structure(parsed)
+    hostname = canonical_public_hostname(parsed)
+    effective_port = port if port is not None else 443
+    if not egress_policy.permits(hostname, effective_port):
+        raise AcquisitionSecurityError(
+            "EGRESS_DENIED",
+            "target hostname or port is not coordinator-allowlisted",
+        )
+    if hostname in _METADATA_HOSTS or hostname.endswith(".internal"):
+        raise AcquisitionSecurityError("METADATA_DESTINATION", "metadata/internal hosts are denied")
+
+    literal = _parse_ip_literal(hostname)
+    if literal is not None:
+        addresses = (literal.compressed,)
+    else:
+        if _looks_like_obfuscated_ip(hostname):
+            raise AcquisitionSecurityError(
+                "OBFUSCATED_IP",
+                "non-canonical numeric IP hostnames are denied",
+            )
+        addresses = resolver.resolve(hostname, effective_port)
+    approved: list[str] = []
+    for raw in addresses:
+        try:
+            address = ipaddress.ip_address(raw)
+        except ValueError:
+            raise AcquisitionSecurityError(
+                "DNS_ADDRESS",
+                "resolver returned a non-IP address",
+            ) from None
+        _require_globally_routable(address)
+        approved.append(address.compressed)
+    approved_ips = tuple(sorted(dict.fromkeys(approved)))
+    if not approved_ips:
+        raise AcquisitionSecurityError("DNS_EMPTY", "target has no approved public addresses")
+    if len(approved_ips) > MAX_DNS_ANSWERS:
+        raise AcquisitionSecurityError(
+            "DNS_ANSWER_LIMIT",
+            f"target has more than {MAX_DNS_ANSWERS} approved addresses",
+        )
+
+    netloc = hostname
+    if ":" in hostname and literal is not None:
+        netloc = f"[{hostname}]"
+    if effective_port != 443:
+        netloc = f"{netloc}:{effective_port}"
+    normalized_url = urlunsplit(
+        (
+            "https",
+            netloc,
+            parsed.path or "/",
+            parsed.query,
+            "",
+        )
+    )
+    resolution_digest = canonical_sha256(
+        {
+            "hostname": hostname,
+            "port": effective_port,
+            "approved_ips": list(approved_ips),
+        }
+    )
+    return ValidatedTarget(
+        original_url=url,
+        normalized_url=normalized_url,
+        hostname=hostname,
+        port=effective_port,
+        approved_ips=approved_ips,
+        resolution_digest=resolution_digest,
+    )
+
+
+def require_approved_peer(target: ValidatedTarget, connected_peer: str) -> str:
+    """Verify the actual connected peer belongs to the exact validated epoch."""
+
+    try:
+        peer = ipaddress.ip_address(connected_peer).compressed
+    except ValueError:
+        raise AcquisitionSecurityError(
+            "PEER_ADDRESS",
+            "transport returned a non-IP peer address",
+        ) from None
+    if peer not in target.approved_ips:
+        raise AcquisitionSecurityError(
+            "PEER_MISMATCH",
+            "connected peer was not part of the approved DNS result",
+        )
+    return peer
+
+
+def canonical_public_hostname(parsed: SplitResult) -> str:
+    """The one canonical ASCII machine name ``parsed`` names, or a typed refusal.
+
+    This is the repository's single answer to "which host is this string about", and it is a
+    function rather than three checks written out at each call site because it has to be *one*
+    answer. ``urlsplit`` is not that answer on its own: it is a generic RFC 3986 splitter, and the
+    parsers that actually resolve these strings -- every browser, every JavaScript consumer, every
+    modern HTTP client -- implement the WHATWG URL Standard instead. Where the two disagree,
+    ``urlsplit`` hands back a *string* that no parser would resolve as a machine name, and a rule
+    that consults an allowlist about that string is asking about something that is not a host.
+
+    Four refusals, in this order, and each one is a place the two readings diverge:
+
+    * a port ``urlsplit`` will not read. ``.hostname`` is computed by splitting everything after
+      the last ``:`` off the netloc and never looking at it, so
+      ``https://allowed.example:S3cretToken/x`` answers ``allowed.example`` while ``parsed.port``
+      raises. Whatever is written where a port goes reaches the caller's artifact verbatim -- a
+      literal credential, a private machine name -- and no rule downstream of ``.hostname`` can
+      see it. ``validate_public_https_url`` was safe only because it happens to evaluate
+      ``parsed.port`` inside a ``try`` of its own before it gets here; the catalog door does not
+      call that function, so the shared answer has to carry the question rather than each caller
+      duplicating it. Asking here applies the rule to every caller.
+    * a backslash anywhere in the authority. ``urlsplit`` keeps it; for a special scheme the URL
+      Standard treats ``\\`` exactly as ``/``, so the authority *ends* there. The whole run
+      ``a.private.example\\allowed.example`` is one host to ``urlsplit`` and the host
+      ``a.private.example`` to everything that resolves it.
+    * a percent sign in the host. ``urlsplit`` keeps it; a WHATWG parser refuses the URL outright.
+    * anything ``_normalize_hostname`` cannot read as a hostname: the URL Standard's *domain to
+      ASCII* (:func:`_domain_to_ascii`), then per-label ``[a-z0-9-]``, at most 63 characters per
+      label and 253 overall, with IP literals passed through for the caller to judge.
+
+    The return value is the *normalised* name, and callers must consult their allowlist about that
+    and not about what they were handed. Normalisation is not decoration here: it is what makes the
+    name this repository checks the same name a browser resolves, so ``a.example<U+00AD>b.example``
+    is checked as the single name a browser goes to rather than as two names glued by a character
+    nobody can see.
+
+    And normalisation is not a licence to *invent* a name. A spelling no URL parser can read is
+    refused rather than reduced to whatever a lenient codec leaves behind:
+    ``a.private.example<U+200D>allowed.example`` is not a machine name, ``new URL()`` throws on it,
+    and answering ``a.private.exampleallowed.example`` -- a name under an allowlisted suffix --
+    would be this function inventing the very thing it exists to prevent. Python's ``idna`` codec
+    deletes that joiner, so this function instead delegates to :func:`_domain_to_ascii`, which
+    applies the standard's derived code-point classes rather than an incident-driven list.
+    """
+
+    try:
+        _ = parsed.port
+    except ValueError:
+        raise AcquisitionSecurityError("URL_PORT", "URL has an invalid port") from None
+    if "\\" in parsed.netloc:
+        raise AcquisitionSecurityError("URL_BACKSLASH", "URL authority backslashes are forbidden")
+    hostname = parsed.hostname or ""
+    if "%" in hostname:
+        raise AcquisitionSecurityError(
+            "URL_HOST_ENCODING",
+            "percent-encoded hostnames are forbidden",
+        )
+    return _normalize_hostname(hostname)
+
+
+def _validate_url_structure(parsed: SplitResult) -> None:
+    if parsed.scheme != "https":
+        raise AcquisitionSecurityError("URL_SCHEME", "only HTTPS URLs are allowed")
+    if not parsed.netloc or parsed.hostname is None:
+        raise AcquisitionSecurityError("URL_HOST", "URL must contain a hostname")
+    if parsed.username is not None or parsed.password is not None:
+        raise AcquisitionSecurityError("URL_USERINFO", "URL user information is forbidden")
+    if parsed.fragment:
+        raise AcquisitionSecurityError("URL_FRAGMENT", "URL fragments are forbidden")
+
+
+def _domain_to_ascii(raw: str) -> str:
+    """``raw`` as the URL Standard's *domain to ASCII* reads it, or a typed refusal.
+
+    Python's built-in ``idna`` **codec** is not this operation and using it here failed open. It
+    implements IDNA2003 nameprep, whose "commonly mapped to nothing" table silently *deletes* code
+    points that the URL Standard's UTS46 processing refuses outright. The two answers differ in the
+    direction that leaks: the codec deletes the invisible character, the glued name ends in an
+    allowlisted suffix, the allowlist says public, and the private machine name is written into the
+    shared catalog verbatim -- while no URL parser can read the string at all. Swept over the glue
+    code points this module was attacked with, node v24.13.0 throws ``Invalid URL`` on
+    ``https://internal-wiki.corp.example.com<GLUE>bulks-faostat.fao.org/x`` for U+200C, U+200D,
+    U+200E, U+200F, U+061C, U+E0000 and U+E0001 through U+E0004, every one of which the codec
+    reduced to a name under ``fao.org``.
+
+    A name no parser can read is a refusal and never a reduction; that is what
+    :func:`canonical_public_hostname` promises its callers. The question is therefore asked of a
+    UTS46 implementation with the standard's own validity checks
+    (non-transitional processing, ``UseSTD3ASCIIRules=false``, and the joiner and code-point
+    classes UTS46 derives from the Unicode tables) rather than of a list of the code points that
+    happened to be found.
+
+    **That was measured over 33 hand-picked code points and the promise is about all of them.**
+    Swept instead over the whole code-point space, the pinned ``idna`` table admitted three names
+    node v24.13.0 refuses outright -- U+A7D2, U+A7D4 and U+A7F1, the last of which UTS46 *maps* to
+    ``s`` -- each of them reducing ``internal-wiki.corp.example.com<GLUE>bulks-faostat.fao.org`` to
+    a name under an allowlisted suffix, which is the exact failure this function exists to stop.
+    The three are not the point and are not fixed as three. They are all *unassigned* in this
+    interpreter's Unicode data and assigned in the newer Unicode the installed ``idna`` was built
+    from, and UTS46 marks an unassigned code point ``disallowed``. So the refusal below is that
+    rule, asked of :mod:`unicodedata` -- the same source the catalog scanner asks for NFKC -- and
+    it holds for code points added to the table ahead of the interpreter rather than for
+    the three that happened to exist on the day. It costs refusals for the 38 further unassigned
+    code points node currently reads; none of them is a name the public DNS resolves, and refusing
+    a name whose spelling this interpreter cannot classify is the direction this module refuses in.
+
+    The residual set is still governed by a table (``idna``) pinned independently of the ICU in
+    every consumer that will read ``catalog.json``, so
+    ``tests/h3/test_url_policy.py::test_the_whole_code_point_space_is_pinned_against_the_installed_table``
+    sweeps the entire space and pins the answer, and a dependency bump that moves it fails there
+    instead of moving silently.
+
+    A pure-ASCII name is its own domain-to-ASCII form apart from case, so it takes the same path it
+    always did and the hostname grammar in :func:`_normalize_hostname` judges it. UTS46 changes and
+    deletes nothing in ASCII once STD3 rules are off, so this short-circuit is the standard's
+    answer and not an exemption from it -- and it is what keeps IP literals, and the underscore and
+    hyphen shapes this module already states as false refusals, exactly where they were.
+    """
+
+    if raw.isascii():
+        return raw.lower()
+    if any(unicodedata.category(character) == "Cn" for character in raw):
+        raise AcquisitionSecurityError(
+            "URL_HOST",
+            "hostname is spelled with a code point this Unicode data does not assign",
+        )
+    try:
+        encoded = idna.encode(raw, uts46=True, std3_rules=False, transitional=False)
+    except UnicodeError:
+        raise AcquisitionSecurityError(
+            "URL_HOST",
+            "hostname is not a domain a URL parser can read",
+        ) from None
+    return encoded.decode("ascii").lower()
+
+
+def a_domain_may_contain(character: str) -> bool:
+    """Whether a URL parser goes on reading a machine name when it meets ``character``.
+
+    This is the URL Standard's *forbidden domain code point* set and nothing else. That is the
+    whole of the standard's character-level answer: the host parser accumulates code points until
+    it meets one of those, and only then runs *domain to ASCII* over the **whole name** it
+    collected. Validity is a property of the name, never of a character, and the two questions
+    are answered in that order by every parser that will read ``catalog.json``.
+
+    It exists because reader's punctuation and a parser's alphabet are different sets and the
+    catalog scanner had been using one for the other. ``"`` and ``'`` end a link for anyone reading
+    a sentence; to every URL parser they are ordinary host characters, so
+    ``https://allowed.example"internal.corp/x`` names ``allowed.example"internal.corp`` and
+    resolves. A space does not: it is a forbidden domain code point, and the same string written
+    with one is refused by every parser. A rule that treats those two the same way is wrong about
+    one of them, and which one it is wrong about decides whether the mistake leaks.
+
+    This function intentionally does not run ``_domain_to_ascii(f"a{character}b")``. Domain-to-
+    ASCII is a whole-name operation, while both callers use ``False`` to **stop looking**: the
+    catalog scanner reads it as "the machine name ends here", and where the name has not started
+    yet that means "this is prose, report no link at all". A per-character domain-to-ASCII probe
+    diverges from consumers' ICU tables by construction, and every divergence points at silence.
+    Measured over the whole code-point space, such a probe rejects 974,396 code points that are not
+    forbidden domain code points, while node v24.13.0 reads a machine name from
+    ``https:private.example<C>/x`` for **16,604** of them -- U+00A9 and U+00B0 among them -- so a
+    private host and literal credential could otherwise enter the shared catalog with no link
+    reported at all.
+
+    So this answers the alphabet question, exactly, and callers that need to know whether the name
+    is one this repository will admit must ask for the name. :func:`canonical_public_hostname` is
+    that question and it refuses what it cannot read, which is the direction a shared artifact can
+    afford to be wrong in. An underscore is kept inside a machine name by every parser and refused
+    by :func:`_normalize_hostname`; that stays a stated false refusal and it stays over there.
+
+    ``:`` is answered ``False`` here because it is a forbidden domain code point; an authority may
+    still carry one, introducing the port, which is a fact about authorities rather than about
+    domains and belongs to the caller that is looking at an authority.
+    """
+
+    return character not in _FORBIDDEN_DOMAIN_CODE_POINTS
+
+
+def _normalize_hostname(hostname: str) -> str:
+    raw = hostname.rstrip(".")
+    if not raw or len(raw) > 253:
+        raise AcquisitionSecurityError("URL_HOST", "hostname length is invalid")
+    normalized = _domain_to_ascii(raw)
+    labels = normalized.split(".")
+    if any(
+        not label
+        or len(label) > 63
+        or label.startswith("-")
+        or label.endswith("-")
+        or not re.fullmatch(r"[a-z0-9-]+", label)
+        for label in labels
+    ):
+        if _parse_ip_literal(normalized) is None:
+            raise AcquisitionSecurityError("URL_HOST", "hostname labels are invalid")
+    return normalized
+
+
+def _parse_ip_literal(hostname: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
+    try:
+        return ipaddress.ip_address(hostname)
+    except ValueError:
+        return None
+
+
+def _looks_like_obfuscated_ip(hostname: str) -> bool:
+    if _DECIMAL_OR_OBFUSCATED_IP.fullmatch(hostname):
+        return _parse_ip_literal(hostname) is None
+    return False
+
+
+def _require_globally_routable(
+    address: ipaddress.IPv4Address | ipaddress.IPv6Address,
+) -> None:
+    if (
+        not address.is_global
+        or address.is_private
+        or address.is_loopback
+        or address.is_link_local
+        or address.is_multicast
+        or address.is_reserved
+        or address.is_unspecified
+        or any(address in network for network in _FORBIDDEN_NETWORKS)
+    ):
+        raise AcquisitionSecurityError(
+            "NON_PUBLIC_DESTINATION",
+            "loopback/private/link-local/metadata/reserved destinations are denied",
+        )