mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/admission.py

@@ -0,0 +1,477 @@
+"""Admission rules that keep the shared catalog public by construction.
+
+The catalog is shared across tenants. Harvested bytes therefore pass the repository's bounded
+public-content classifier, and every typed evidence locator is parsed and checked against the
+coordinator's public-host policy before an entry can be sealed.
+
+Prose has a stricter, structural contract: it carries no locators. If any conservative form of a
+prose string contains a URL-scheme shape, the entry is refused and the locator must move to a typed
+citation field. Over-detection costs one refused entry; a parser/scanner disagreement cannot admit
+tenant-private text because no prose locator is admitted at all.
+
+The walk remains derived from ``CatalogEntry.to_dict()`` -- the exact shape written to
+``catalog.json`` -- so every serialized prose field is covered without enumerating field names.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import unicodedata
+from collections.abc import Iterator
+from typing import Any
+from urllib.parse import SplitResult, unquote, urlsplit
+
+from mostlyright.data_harness.acquisition.url_policy import (
+    AcquisitionSecurityError,
+    EgressPolicy,
+    canonical_public_hostname,
+)
+from mostlyright.data_harness.sources.catalog.contracts import CatalogEntry
+from mostlyright.data_harness.sources.contracts import SourceContractError
+from mostlyright.data_harness.sources.governance import (
+    ClassificationResult,
+    classify_and_require_admissible,
+)
+
+# Four MiB. Deliberately four times tighter than the classifier's own 16 MiB scan cap: harvested
+# metadata documents are small, and the tight cap is what stops an SDMX-scale structure response
+# (the observed Eurostat one is 37,166,239 bytes) from ever reaching a scan.
+MAX_CATALOG_FACT_BYTES = 4 * 1024 * 1024
+
+CATALOG_CLASSIFICATION = "public"
+
+# Names that are never a public source, regardless of what DNS would say about them.
+_FORBIDDEN_HOST_SUFFIXES = (".local", ".internal", ".localdomain")
+_FORBIDDEN_HOST_NAMES = frozenset({"localhost"})
+
+_CITATION_PORT = 443
+
+# Special schemes can introduce a URL directly at their colon; other schemes can introduce an
+# authority when followed by a slash or backslash. These deliberately broad patterns answer only
+# whether prose can carry a locator. They never try to find its end or decide which host it names.
+_SPECIAL_SCHEMES = ("file", "ftp", "http", "https", "ws", "wss")
+_SPECIAL_SCHEME_COLON = re.compile(
+    r"(?:" + "|".join(_SPECIAL_SCHEMES) + r"):",
+    re.IGNORECASE,
+)
+_ANY_SCHEME_AUTHORITY = re.compile(r"[A-Za-z][A-Za-z0-9+.\-]*:[/\\]")
+_SCHEME_RELATIVE_AUTHORITY = re.compile(r"[/\\]{2}")
+
+# Python 3.13 ships Unicode 15.1 data while the WHATWG consumer oracle already uses Unicode 16.
+# Unicode 16 assigned the contiguous OUTLINED LATIN CAPITAL LETTER A..Z and OUTLINED DIGIT
+# ZERO..NINE block at U+1CCD6..U+1CCF9, with compatibility decompositions to ASCII. Keep that
+# complete delta explicit instead of branching on ``unicodedata.unidata_version``. Translation is
+# idempotent when the runtime NFKC implementation already normalizes the block.
+_OUTLINED_ALPHANUMERICS = "".join(chr(codepoint) for codepoint in range(0x1CCD6, 0x1CCFA))
+_OUTLINED_ALPHANUMERIC_ASCII = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+_UNICODE_16_NFKC_DELTA = str.maketrans(
+    _OUTLINED_ALPHANUMERICS,
+    _OUTLINED_ALPHANUMERIC_ASCII,
+)
+
+
+def _consumer_nfkc(text: str) -> str:
+    """NFKC plus the pinned Unicode 16 delta used by current URL consumers."""
+
+    return unicodedata.normalize("NFKC", text).translate(_UNICODE_16_NFKC_DELTA)
+
+
+def _characters_a_url_parser_deletes() -> str:
+    """Return C0 characters that the active URL parser removes from a hostname.
+
+    Deriving the set from the parser avoids a duplicate parser table.
+    """
+
+    deleted: list[str] = []
+    for code in range(0x21):
+        character = chr(code)
+        try:
+            reads = urlsplit(f"https://a{character}b/").hostname
+        except ValueError:
+            continue
+        if reads == "ab":
+            deleted.append(character)
+    return "".join(deleted)
+
+
+# Deletion can assemble a scheme token that was not contiguous in the written form, so the prose
+# predicate checks the parser-deleted form as well.
+_PARSER_DELETES = _characters_a_url_parser_deletes()
+
+
+def _the_deletions_a_url_parser_makes(text: str) -> tuple[str, frozenset[int]]:
+    """``text`` with the deleted characters gone, and the positions they were deleted from.
+
+    One pass that reports its own seams, because the rest of this module has to be able to tell a
+    character that was *written* next to another one from a character a deletion pulled up against
+    it. A seam at index ``i`` of the returned text means at least one deleted character stood
+    between ``text[i - 1]`` and ``text[i]`` as the author wrote them.
+
+    A deleted character can appear before a scheme token or inside the scheme name, as in
+    ``ht<LF>tps:``. The written form and the deletion-normalized form cannot independently preserve
+    that boundary; the seam does so for every parser-deleted character and position.
+    """
+
+    kept: list[str] = []
+    seams: set[int] = set()
+    for character in text:
+        if character in _PARSER_DELETES:
+            seams.add(len(kept))
+        else:
+            kept.append(character)
+    return "".join(kept), frozenset(seams)
+
+
+class CatalogPrivacyError(SourceContractError):
+    """A refusal taken to keep tenant-private or non-public content out of the catalog.
+
+    It subclasses ``SourceContractError`` so existing fail-closed handling still catches it, and
+    is its own type so a privacy refusal is never mistaken for an ordinary validation failure in
+    a log or a test.
+    """
+
+
+def admit_public_fact_bytes(
+    content: bytes,
+    *,
+    lawful_basis_declared: bool = True,
+    declared_classification: str = CATALOG_CLASSIFICATION,
+) -> ClassificationResult:
+    """Admit bounded public bytes as catalog facts, or refuse them with a typed error.
+
+    ``declared_classification`` exists only so a caller that passes the wrong thing gets a loud
+    refusal instead of quietly widening the catalog. There is no value of it that admits
+    non-public content.
+    """
+
+    if not isinstance(content, bytes):
+        raise SourceContractError(
+            "TYPE",
+            "catalog_fact.content",
+            "catalog fact content must be bytes",
+        )
+    if len(content) > MAX_CATALOG_FACT_BYTES:
+        raise SourceContractError(
+            "CATALOG_FACT_LIMIT",
+            "catalog_fact.content",
+            f"catalog fact input exceeds {MAX_CATALOG_FACT_BYTES} bytes",
+        )
+    if declared_classification != CATALOG_CLASSIFICATION:
+        raise CatalogPrivacyError(
+            "CATALOG_NON_PUBLIC_CLASSIFICATION",
+            "catalog_fact.declared_classification",
+            "the shared catalog admits public content and nothing else",
+        )
+    return classify_and_require_admissible(
+        content,
+        declared_classification=CATALOG_CLASSIFICATION,
+        lawful_basis_declared=lawful_basis_declared,
+    )
+
+
+def entry_locators(entry: CatalogEntry) -> tuple[tuple[str, str], ...]:
+    """Every typed evidence locator on this entry that has to name a public host.
+
+    Derived from the entry rather than written out, so this is what the check enumerates *and*
+    what a test can assert the check covers. A hand-written list is checkable only against another
+    hand-written list, which proves the two lists agree and nothing about the object.
+
+    An ``EvidenceReference.uri`` is a locator by type: every one of them, wherever it sits on the
+    entry, is walked here. ``rights.human_decision_reference`` is free text, not a typed locator;
+    opaque ticket IDs remain prose and a URL-shaped value is refused by the prose contract.
+    """
+
+    if not isinstance(entry, CatalogEntry):
+        raise SourceContractError("TYPE", "entry", "must be a CatalogEntry")
+
+    found: list[tuple[str, str]] = []
+    for path, references in (
+        ("catalog_entry.evidence", entry.evidence),
+        ("catalog_entry.rights.evidence", entry.rights.evidence),
+        ("catalog_entry.observation.evidence", entry.observation.evidence),
+    ):
+        found.extend(
+            (f"{path}[{index}].uri", reference.uri) for index, reference in enumerate(references)
+        )
+    return tuple(found)
+
+
+class _SchemeFormLimit(ValueError):
+    """The conservative transform closure exceeded its fail-closed work budget."""
+
+
+_MAX_SCHEME_FORMS = 64
+_MAX_SCHEME_FORM_CHARACTERS = 256 * 1024
+_FORM_LIMIT_TOKEN = "an encoded locator shape"
+_UNASSIGNED_CODEPOINT_TOKEN = "a runtime-unassigned Unicode code point"
+
+
+def _scheme_forms(text: str) -> tuple[str, ...]:
+    """The bounded fixed-point closure of forms a URL consumer may expose.
+
+    Percent-decoding, Unicode compatibility normalisation, and parser deletions do not commute.
+    Applying each operation only to the written string misses compositions such as a full-width
+    percent sign that becomes an escape only after NFKC. The worklist therefore applies every
+    transform to every newly discovered form until no form changes.
+
+    Catalog strings are already bounded by their contracts. The independent form-count and
+    aggregate-character budgets make that bound explicit here too; exceeding either budget is a
+    fail-closed prose refusal, never permission to stop looking.
+    """
+
+    if not isinstance(text, str):
+        raise SourceContractError("TYPE", "catalog_entry.prose", "must be text")
+
+    forms: list[str] = [text]
+    seen = {text}
+    total_characters = len(text)
+    position = 0
+    while position < len(forms):
+        current = forms[position]
+        position += 1
+        deleted, _ = _the_deletions_a_url_parser_makes(current)
+        for transformed in (
+            unquote(current),
+            _consumer_nfkc(current),
+            deleted,
+        ):
+            if transformed in seen:
+                continue
+            if (
+                len(forms) >= _MAX_SCHEME_FORMS
+                or total_characters + len(transformed) > _MAX_SCHEME_FORM_CHARACTERS
+            ):
+                raise _SchemeFormLimit
+            seen.add(transformed)
+            forms.append(transformed)
+            total_characters += len(transformed)
+    return tuple(forms)
+
+
+def _scheme_token_a_parser_could_read(
+    text: str,
+    *,
+    include_scheme_relative: bool = False,
+) -> str | None:
+    """Return one conservative refusal token from any transformed form of the text.
+
+    Unassigned code points have no stable compatibility normalization. Refuse them rather than
+    admit a locator under a different Unicode version.
+    """
+
+    try:
+        forms = _scheme_forms(text)
+    except _SchemeFormLimit:
+        return _FORM_LIMIT_TOKEN
+
+    for form in forms:
+        if unassigned := next(
+            (character for character in form if unicodedata.category(character) == "Cn"),
+            None,
+        ):
+            return f"{_UNASSIGNED_CODEPOINT_TOKEN} U+{ord(unassigned):04X}"
+        patterns = (_SPECIAL_SCHEME_COLON, _ANY_SCHEME_AUTHORITY)
+        if include_scheme_relative:
+            patterns += (_SCHEME_RELATIVE_AUTHORITY,)
+        matches = tuple(
+            match for pattern in patterns if (match := pattern.search(form)) is not None
+        )
+        if matches:
+            return min(matches, key=lambda match: match.start()).group(0)
+    return None
+
+
+def _a_parser_could_read_a_scheme_from(text: str) -> bool:
+    """Return whether conservative prose admission finds a scheme or unknown normalization."""
+
+    return _scheme_token_a_parser_could_read(text) is not None
+
+
+def entry_prose_carrying_a_locator(entry: CatalogEntry) -> tuple[tuple[str, str], ...]:
+    """Every path and scheme token whose prose must be refused.
+
+    The walk is over the serialized entry rather than a field list. Typed locator values are
+    excluded as whole values and checked by _require_public_citation; their path, query, and
+    fragment components receive this same scheme-shape rule there so a nested locator cannot move
+    from prose into a citation to evade it.
+    """
+
+    if not isinstance(entry, CatalogEntry):
+        raise SourceContractError("TYPE", "entry", "must be a CatalogEntry")
+    locators = set(entry_locators(entry))
+    found: list[tuple[str, str]] = []
+    for path, text in _strings_in(entry.to_dict(), "catalog_entry"):
+        if (path, text) in locators:
+            continue
+        if (token := _scheme_token_a_parser_could_read(text)) is not None:
+            found.append((path, token))
+    return tuple(found)
+
+
+def _strings_in(node: Any, path: str) -> Iterator[tuple[str, str]]:
+    """Every string key and value under ``node``, with its canonical path.
+
+    Keys are visited in sorted order so the reported path order is the order the canonical JSON
+    writer would use, and so a refusal names the same field on every run.
+    """
+
+    if isinstance(node, dict):
+        for key in sorted(node):
+            # Catalog contract keys are closed, but they are serialized bytes too. Scanning
+            # them keeps this walk sound for dynamic-key mappings.
+            yield f"{path}.<key>", key
+            yield from _strings_in(node[key], f"{path}.{key}")
+    elif isinstance(node, list):
+        for index, item in enumerate(node):
+            yield from _strings_in(item, f"{path}[{index}]")
+    elif isinstance(node, str):
+        yield path, node
+
+
+def require_public_source_evidence(entry: CatalogEntry, *, egress_policy: EgressPolicy) -> None:
+    """Refuse entries with unsafe typed locators or any locator-shaped prose.
+
+    Typed locators must be HTTPS, host-check clean, and contain no nested locator in a path, query,
+    or fragment. Every other serialized string -- including mapping keys -- is prose and is
+    refused when the conservative transform closure exposes a scheme shape. There is no prose
+    host allowlist: links belong in typed citation fields.
+    """
+
+    if not isinstance(egress_policy, EgressPolicy):
+        raise SourceContractError("TYPE", "egress_policy", "must be an EgressPolicy")
+    for path, uri in entry_locators(entry):
+        _require_public_citation(uri, path, egress_policy=egress_policy)
+    if carrying := entry_prose_carrying_a_locator(entry):
+        path, token = carrying[0]
+        raise CatalogPrivacyError(
+            "CATALOG_PROSE_CARRIES_LOCATOR",
+            path,
+            f"catalog prose carries unsafe locator material {token!r}; put locators in a typed "
+            "citation field and remove runtime-unassigned characters",
+        )
+
+
+def _require_public_citation(uri: str, path: str, *, egress_policy: EgressPolicy) -> None:
+    try:
+        parsed = urlsplit(uri)
+        hostname = parsed.hostname
+    except ValueError:
+        raise CatalogPrivacyError(
+            "CATALOG_EVIDENCE_NOT_PUBLIC",
+            path,
+            "citation is not a parsable URL",
+        ) from None
+    if parsed.scheme != "https":
+        raise CatalogPrivacyError(
+            "CATALOG_EVIDENCE_NOT_PUBLIC",
+            path,
+            "citations must be HTTPS",
+        )
+    if not hostname:
+        raise CatalogPrivacyError(
+            "CATALOG_EVIDENCE_NOT_PUBLIC",
+            path,
+            "citations must name a host",
+        )
+    _require_no_nested_locator(parsed, path)
+    _require_public_host(
+        parsed,
+        path,
+        egress_policy=egress_policy,
+        not_public="CATALOG_EVIDENCE_NOT_PUBLIC",
+        not_allowlisted="CATALOG_EVIDENCE_NOT_ALLOWLISTED",
+        subject="this citation",
+    )
+
+
+def _require_no_nested_locator(parsed: SplitResult, path: str) -> None:
+    """Refuse a locator hidden in any non-authority component of a typed locator."""
+
+    for component_name, component in (
+        ("path", parsed.path),
+        ("query", parsed.query),
+        ("fragment", parsed.fragment),
+    ):
+        if (
+            token := _scheme_token_a_parser_could_read(
+                component,
+                include_scheme_relative=True,
+            )
+        ) is not None:
+            raise CatalogPrivacyError(
+                "CATALOG_EVIDENCE_CARRIES_NESTED_LOCATOR",
+                path,
+                f"citation {component_name} carries unsafe locator material {token!r}; a typed "
+                "citation names exactly one stable locator",
+            )
+
+
+def _require_public_host(
+    parsed: SplitResult,
+    path: str,
+    *,
+    egress_policy: EgressPolicy,
+    not_public: str,
+    not_allowlisted: str,
+    subject: str,
+) -> None:
+    """Apply the canonical public-host rule to the complete parsed locator.
+
+    Passing the complete locator lets ``canonical_public_hostname`` reject credentials and
+    authority delimiters before it normalizes the hostname through IDNA. The allowlist is applied
+    only to that canonical name, so every typed citation and serialized locator follows the same
+    consumer-compatible rule.
+
+    It costs false refusals, and they are stated on the customer surface rather than discovered:
+    a label over 63 characters, an empty label (``https://.example.org``, ``https://a..b``), a
+    label carrying an underscore or a leading or trailing hyphen, and a name over 253 characters
+    are all refused. None of them is a name the public DNS resolves; the escape in every case is
+    to write the source's real machine name.
+    """
+
+    try:
+        host = canonical_public_hostname(parsed)
+    except AcquisitionSecurityError:
+        raise CatalogPrivacyError(
+            not_public,
+            path,
+            f"{subject} does not spell a machine name a URL parser would resolve, so this module "
+            "cannot say which host it names. A catalog names public hosts, and a name it cannot "
+            "read is not one",
+        ) from None
+    # An IP literal is never a catalog link. Refusing the whole shape covers loopback, the
+    # private ranges and the cloud metadata address in one rule rather than enumerating them.
+    try:
+        ipaddress.ip_address(host)
+    except ValueError:
+        pass
+    else:
+        raise CatalogPrivacyError(
+            not_public,
+            path,
+            f"{subject} names an address; a catalog names public hosts, never addresses",
+        )
+    if "." not in host or host in _FORBIDDEN_HOST_NAMES or host.endswith(_FORBIDDEN_HOST_SUFFIXES):
+        raise CatalogPrivacyError(
+            not_public,
+            path,
+            f"{subject} names a single-label host or an internal namespace",
+        )
+    if not egress_policy.permits(host, _CITATION_PORT):
+        raise CatalogPrivacyError(
+            not_allowlisted,
+            path,
+            f"{subject} names a host the coordinator has not allowlisted",
+        )
+    # Last, so a link that is wrong about its host as well as carrying userinfo is still refused
+    # for its host: which host a link names is the more useful thing for a refusal to say, and
+    # `https://public.example@internal.corp/x` is a host problem wearing a userinfo disguise.
+    if parsed.username is not None or parsed.password is not None:
+        raise CatalogPrivacyError(
+            not_public,
+            path,
+            f"{subject} carries user information before its host. A public source is addressed "
+            "without credentials, and whatever is written there -- a token, a service account, an "
+            "internal name -- reaches every tenant verbatim in the shared catalog",
+        )