mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_crawler.py

@@ -0,0 +1,1115 @@
+"""Networkless normalizer child for hosted public connectors.
+
+Despite the name, nothing here crawls. This is the process behind the mr-data-crawler
+console script, and it holds no network authority. The mr-data-crawler-job coordinator
+fetches the response and passes it on a file descriptor, then spawns this process under a
+seccomp filter that denies socket and connect. This module authenticates that framed input
+and normalizes OpenLigaDB match JSON into the fixed CSV column set.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import io
+import json
+import os
+import re
+import sys
+from dataclasses import asdict
+from datetime import datetime
+from typing import Any, Protocol
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.acquisition.parsing import (
+    ParseLimits,
+    parse_csv_evidence,
+    parse_tabular_bytes,
+    row_digest_for,
+)
+from mostlyright.data_harness.acquisition.url_policy import canonical_public_hostname
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.formats import DATA_FORMATS
+from mostlyright.data_harness.hosted_crawler_protocol import (
+    CRAWLER_REQUEST_SCHEMA_V3,
+    CRAWLER_RESULT_SCHEMA_V1,
+    CRAWLER_RESULT_SCHEMA_V3,
+    MAX_CRAWLER_REQUEST_BYTES,
+    MAX_CRAWLER_RESULT_BYTES,
+    MAX_NORMALIZED_SOURCE_BYTES,
+    OPENLIGADB_EGRESS_POLICY_ATTESTATION,
+    OPENLIGADB_QUERY_ALLOWLIST_DETAIL,
+    OPENLIGADB_QUERY_FIELDS_DETAIL,
+    PUBLIC_HTTPS_EGRESS_POLICY_ATTESTATION,
+    HostedCrawlerProtocolError,
+    HostedCrawlerRequest,
+    HostedCrawlerResult,
+    parse_crawler_request,
+)
+from mostlyright.data_harness.local_contracts import ContractError, validate_source_locator
+from mostlyright.data_harness.readers.contracts import ReaderPin
+from mostlyright.data_harness.readers.registry import TOOLBOX
+from mostlyright.data_harness.readers.samples import DecodedFacts, warm_up
+
+CRAWLER_EGRESS_POLICY_ENV = "MOSTLYRIGHT_CRAWLER_EGRESS_POLICY_ATTESTATION"
+# The public-HTTPS fetch ceiling, equal to the hosted contract's max_source_bytes maximum and
+# stated as max_response_bytes inside PUBLIC_HTTPS_EGRESS_POLICY_ATTESTATION. The OpenLigaDB
+# retriever keeps its own reviewed 8 MiB cap, stated in its policy attestation.
+MAX_RAW_RESPONSE_BYTES = 268_435_456
+MAX_NORMALIZATION_HEADER_BYTES = 128 * 1024
+MAX_NORMALIZATION_INPUT_BYTES = 4 + MAX_NORMALIZATION_HEADER_BYTES + MAX_RAW_RESPONSE_BYTES
+NORMALIZATION_INPUT_SCHEMA = "mostlyright-hosted-crawler-normalization-input.v1"
+WARM_UP_RESULT_SCHEMA = "mostlyright-hosted-crawler-reader-warm-up.v1"
+MAX_ROWS = 20_000
+MAX_TEXT_BYTES = 512
+MAX_SAFE_INTEGER = 9_007_199_254_740_991
+
+_COLUMNS = (
+    "match_id",
+    "match_datetime_utc",
+    "last_update_datetime",
+    "league_shortcut",
+    "league_season",
+    "group_order_id",
+    "team1_id",
+    "team1_name",
+    "team2_id",
+    "team2_name",
+    "match_finished",
+    "score1",
+    "score2",
+)
+_QUERY_FIELDS = frozenset({"league_shortcut", "league_season"})
+_PUBLIC_QUERY_FIELDS = frozenset(
+    {"url", "data_format", "filename", "reader_pin", "resource_caps", "limits"}
+)
+_PUBLIC_LIMIT_FIELDS = frozenset(
+    {"max_source_bytes", "max_normalized_bytes", "max_rows", "max_columns"}
+)
+_PUBLIC_RESOURCE_CAP_FIELDS = frozenset(
+    {"max_declared_cells", "max_container_members", "max_nesting_depth"}
+)
+_LEAGUE = re.compile(r"^[a-z0-9][a-z0-9-]{0,15}$")
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+_UTC_TIMESTAMP = re.compile(
+    r"^[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])"
+    r"T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:\.[0-9]{1,6})?Z$"
+)
+_FORBIDDEN_ENVIRONMENT = re.compile(
+    r"(?:^|_)(?:STUDIO|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTHORIZATION|"
+    r"GOOGLE_APPLICATION_CREDENTIALS|HOME)(?:_|$)",
+    re.IGNORECASE,
+)
+
+
+class HostedCrawlerError(RuntimeError):
+    """One safe fixed-crawler execution invariant failed."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{detail} [{code}]")
+
+
+class RetrievedSource(Protocol):
+    source_url: str
+    final_url: str
+    media_type: str
+    content: bytes
+    content_sha256: str
+    transport_evidence_digest: str
+    hops: tuple[Any, ...]
+
+
+def validate_crawler_request(
+    request: HostedCrawlerRequest,
+    *,
+    expected_egress_policy_attestation: str,
+) -> tuple[str, int]:
+    """Validate the exact deployed adapter and return its bounded query.
+
+    expected_egress_policy_attestation is the attestation the calling process trusts. Only
+    crawler_main supplies an independent value, read from the environment. Every other call
+    site passes the request's own attestation, so for those the first check cannot fail and
+    the binding rests on the OPENLIGADB_EGRESS_POLICY_ATTESTATION comparison after it.
+    """
+
+    if request.egress_policy_attestation != expected_egress_policy_attestation:
+        raise HostedCrawlerError(
+            "CRAWLER_POLICY_MISMATCH",
+            "crawler request differs from the deployed egress-policy attestation",
+        )
+    expected_policy = {
+        "external.openligadb": OPENLIGADB_EGRESS_POLICY_ATTESTATION,
+        "public.https": PUBLIC_HTTPS_EGRESS_POLICY_ATTESTATION,
+    }.get(request.adapter_id)
+    if expected_egress_policy_attestation != expected_policy:
+        raise HostedCrawlerError(
+            "CRAWLER_POLICY_MISMATCH",
+            "deployed egress-policy attestation is not the reviewed adapter policy",
+        )
+    if request.adapter_version != "1.0.0" or expected_policy is None:
+        raise HostedCrawlerError(
+            "CRAWLER_ADAPTER_UNAVAILABLE",
+            "crawler request adapter coordinate is not image-allowlisted",
+        )
+    if request.adapter_id == "public.https":
+        _public_https_query(request.query)
+        return "public.https", 1
+    league, season = _query(request.query)
+    return league, season
+
+
+def encode_normalization_input(
+    request: HostedCrawlerRequest,
+    retrieved: RetrievedSource,
+) -> bytes:
+    """Frame trusted retrieval evidence and raw bytes for the networkless child."""
+
+    coordinate = validate_crawler_request(
+        request,
+        expected_egress_policy_attestation=request.egress_policy_attestation,
+    )
+    if request.adapter_id == "public.https":
+        public = _public_https_query(request.query)
+        expected_url = public["url"]
+        expected_media_types = _public_expected_media_types(public)
+        final_host = _canonical_url_host(retrieved.final_url)
+        expected_host = _canonical_url_host(expected_url)
+        evidence_valid = (
+            retrieved.source_url == expected_url
+            and final_host == expected_host
+            and retrieved.media_type in expected_media_types
+            and 1 <= len(retrieved.content) <= public["limits"]["max_source_bytes"]
+        )
+    else:
+        league, season = coordinate
+        expected_url = f"https://api.openligadb.de/getmatchdata/{league}/{season}"
+        evidence_valid = (
+            retrieved.source_url == expected_url
+            and retrieved.final_url == expected_url
+            and retrieved.media_type == "application/json"
+        )
+    if not evidence_valid or (
+        not 1 <= len(retrieved.content) <= MAX_RAW_RESPONSE_BYTES
+        or sha256_bytes(retrieved.content) != retrieved.content_sha256
+        or _DIGEST.fullmatch(retrieved.transport_evidence_digest) is None
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_RETRIEVAL_EVIDENCE_INVALID",
+            "trusted retrieval evidence is invalid or outside the reviewed source authority",
+        )
+    if not retrieved.hops:
+        raise HostedCrawlerError(
+            "CRAWLER_RETRIEVAL_EVIDENCE_INVALID",
+            "trusted retrieval evidence omitted its terminal HTTP hop",
+        )
+    terminal = retrieved.hops[-1]
+    if (
+        type(terminal.status) is not int
+        or not 200 <= terminal.status <= 299
+        or (terminal.etag is not None and not isinstance(terminal.etag, str))
+        or (terminal.last_modified is not None and not isinstance(terminal.last_modified, str))
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_RETRIEVAL_EVIDENCE_INVALID",
+            "trusted retrieval evidence has invalid terminal HTTP facts",
+        )
+    header = canonical_json_bytes(
+        {
+            "schema_version": NORMALIZATION_INPUT_SCHEMA,
+            "request": request.to_dict(),
+            "source_url": retrieved.source_url,
+            "final_url": retrieved.final_url,
+            "media_type": retrieved.media_type,
+            "raw_content_sha256": retrieved.content_sha256,
+            "raw_size_bytes": len(retrieved.content),
+            "transport_evidence_digest": retrieved.transport_evidence_digest,
+            "http_status_code": terminal.status,
+            "etag": terminal.etag,
+            "last_modified": terminal.last_modified,
+        }
+    )
+    if len(header) > MAX_NORMALIZATION_HEADER_BYTES:
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "crawler normalization header exceeds its byte bound",
+        )
+    return len(header).to_bytes(4, "big") + header + retrieved.content
+
+
+def parse_normalization_input(
+    content: bytes,
+) -> tuple[HostedCrawlerRequest, bytes, str, str, str, str, int, str | None, str | None]:
+    """Parse and authenticate the exact coordinator-to-normalizer frame.
+
+    The returned tuple is, in order: the request, the raw response bytes, the source URL,
+    the media type, the raw-content digest, the transport-evidence digest, the terminal HTTP
+    status, the raw ETag, and the raw Last-Modified value.
+    """
+
+    if not 5 <= len(content) <= MAX_NORMALIZATION_INPUT_BYTES:
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "crawler normalization input exceeds its byte bound",
+        )
+    header_size = int.from_bytes(content[:4], "big")
+    if not 1 <= header_size <= MAX_NORMALIZATION_HEADER_BYTES or 4 + header_size >= len(content):
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "crawler normalization header framing is invalid",
+        )
+    header_bytes = content[4 : 4 + header_size]
+    raw = content[4 + header_size :]
+    header = parse_canonical_json(header_bytes)
+    expected_fields = {
+        "schema_version",
+        "request",
+        "source_url",
+        "final_url",
+        "media_type",
+        "raw_content_sha256",
+        "raw_size_bytes",
+        "transport_evidence_digest",
+        "http_status_code",
+        "etag",
+        "last_modified",
+    }
+    if not isinstance(header, dict) or set(header) != expected_fields:
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "crawler normalization header fields are not exact",
+        )
+    request = parse_crawler_request(header["request"])
+    coordinate = validate_crawler_request(
+        request,
+        expected_egress_policy_attestation=request.egress_policy_attestation,
+    )
+    if request.adapter_id == "public.https":
+        public = _public_https_query(request.query)
+        requested_url = public["url"]
+        route_valid = (
+            header["source_url"] == requested_url
+            and _canonical_url_host(header["final_url"]) == _canonical_url_host(requested_url)
+            and header["media_type"] in _public_expected_media_types(public)
+            and len(raw) <= public["limits"]["max_source_bytes"]
+        )
+    else:
+        league, season = coordinate
+        requested_url = f"https://api.openligadb.de/getmatchdata/{league}/{season}"
+        route_valid = (
+            header["source_url"] == requested_url
+            and header["final_url"] == requested_url
+            and header["media_type"] == "application/json"
+            # Defense in depth: the reviewed OpenLigaDB policy caps one response at 8 MiB,
+            # so its normalization frame never carries more, whatever the public ceiling is.
+            and len(raw) <= 8 * 1024 * 1024
+        )
+    if (
+        header["schema_version"] != NORMALIZATION_INPUT_SCHEMA
+        or not route_valid
+        or type(header["raw_size_bytes"]) is not int
+        or header["raw_size_bytes"] != len(raw)
+        or not 1 <= len(raw) <= MAX_RAW_RESPONSE_BYTES
+        or not isinstance(header["raw_content_sha256"], str)
+        or _DIGEST.fullmatch(header["raw_content_sha256"]) is None
+        or header["raw_content_sha256"] != sha256_bytes(raw)
+        or not isinstance(header["transport_evidence_digest"], str)
+        or _DIGEST.fullmatch(header["transport_evidence_digest"]) is None
+        or type(header["http_status_code"]) is not int
+        or not 200 <= header["http_status_code"] <= 299
+        or (
+            header["etag"] is not None
+            and (not isinstance(header["etag"], str) or len(header["etag"]) > 1_024)
+        )
+        or (
+            header["last_modified"] is not None
+            and (
+                not isinstance(header["last_modified"], str) or len(header["last_modified"]) > 1_024
+            )
+        )
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "crawler normalization evidence binding is invalid",
+        )
+    return (
+        request,
+        raw,
+        header["final_url"],
+        header["media_type"],
+        header["raw_content_sha256"],
+        header["transport_evidence_digest"],
+        header["http_status_code"],
+        header["etag"],
+        header["last_modified"],
+    )
+
+
+def normalize_crawler_response(
+    request: HostedCrawlerRequest,
+    *,
+    raw: bytes,
+    source_url: str,
+    media_type: str,
+    raw_digest: str,
+    transport_evidence_digest: str,
+    http_status_code: int,
+    etag: str | None,
+    last_modified: str | None,
+) -> HostedCrawlerResult:
+    """Strictly normalize one already-fetched response without network authority."""
+
+    coordinate = validate_crawler_request(
+        request,
+        expected_egress_policy_attestation=request.egress_policy_attestation,
+    )
+    if request.adapter_id == "public.https":
+        return _normalize_public_https_response(
+            request,
+            raw=raw,
+            final_url=source_url,
+            media_type=media_type,
+            raw_digest=raw_digest,
+            transport_evidence_digest=transport_evidence_digest,
+            http_status_code=http_status_code,
+            etag=etag,
+            last_modified=last_modified,
+        )
+    league, season = coordinate
+    requested_url = f"https://api.openligadb.de/getmatchdata/{league}/{season}"
+    if (
+        source_url != requested_url
+        or media_type != "application/json"
+        or raw_digest != sha256_bytes(raw)
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "crawler normalization evidence binding is invalid",
+        )
+    rows, historical_start, historical_end = _normalize(
+        raw,
+        expected_league=league,
+        expected_season=season,
+    )
+    normalized = _encode_csv(rows)
+    normalized_digest = sha256_bytes(normalized)
+    receipt = {
+        "schema_version": "hosted-openligadb-acquisition-receipt.v1",
+        "source_id": request.source_id,
+        "adapter_id": request.adapter_id,
+        "adapter_version": request.adapter_version,
+        "request_digest": request.digest,
+        "query": {
+            "league_shortcut": league,
+            "league_season": season,
+        },
+        "source_url": requested_url,
+        "final_url": source_url,
+        "media_type": media_type,
+        "acquired_at": request.requested_at,
+        "raw_content_sha256": raw_digest,
+        "raw_size_bytes": len(raw),
+        "normalized_content_sha256": normalized_digest,
+        "normalized_size_bytes": len(normalized),
+        "row_count": len(rows),
+        "column_names": list(_COLUMNS),
+        "transport_evidence_digest": transport_evidence_digest,
+        "egress_policy_attestation": request.egress_policy_attestation,
+    }
+    observation = {
+        "schema_version": "hosted-openligadb-source-observation.v1",
+        "source_id": request.source_id,
+        "adapter_id": request.adapter_id,
+        "adapter_version": request.adapter_version,
+        "request_digest": request.digest,
+        "observed_at": request.requested_at,
+        "available_at": request.requested_at,
+        "historical_start": historical_start,
+        "historical_end": historical_end,
+        "event_time_field": "match_datetime_utc",
+        "live_status": "live",
+        "raw_content_sha256": raw_digest,
+        "normalized_content_sha256": normalized_digest,
+        "row_count": len(rows),
+    }
+    return HostedCrawlerResult(
+        request_digest=request.digest,
+        source_id=request.source_id,
+        adapter_id=request.adapter_id,
+        adapter_version=request.adapter_version,
+        normalized_content=normalized,
+        acquisition_receipt=receipt,
+        observation=observation,
+        next_watermark=None,
+    )
+
+
+def _canonical_url_host(value: Any) -> str:
+    if not isinstance(value, str) or not value:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS URL is invalid")
+    try:
+        validate_source_locator(value, "https_url", "crawler.query.url")
+        parsed = urlsplit(value)
+        port = parsed.port
+        host = canonical_public_hostname(parsed)
+    except (ContractError, TypeError, ValueError) as error:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS URL is invalid") from error
+    if parsed.scheme != "https" or parsed.username is not None or parsed.password is not None:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS URL is invalid")
+    if port not in {None, 443} or parsed.fragment:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS URL is invalid")
+    return host
+
+
+def _public_https_query(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict) or set(value) != _PUBLIC_QUERY_FIELDS:
+        raise HostedCrawlerError(
+            "CRAWLER_QUERY_INVALID",
+            "public HTTPS query fields are not exact",
+        )
+    url = value["url"]
+    if not isinstance(url, str) or not 1 <= len(url) <= 2_048:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS URL is invalid")
+    _canonical_url_host(url)
+    data_format = value["data_format"]
+    filename = value["filename"]
+    if data_format not in DATA_FORMATS:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS format is unsupported")
+    if (
+        not isinstance(filename, str)
+        or not 1 <= len(filename.encode("utf-8")) <= 255
+        or "/" in filename
+        or "\\" in filename
+        or any(ord(character) < 0x20 for character in filename)
+    ):
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS filename is invalid")
+    limits = value["limits"]
+    if not isinstance(limits, dict) or set(limits) != _PUBLIC_LIMIT_FIELDS:
+        raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS limits are not exact")
+    bounds = {
+        "max_source_bytes": (1, MAX_RAW_RESPONSE_BYTES),
+        "max_normalized_bytes": (1, MAX_NORMALIZED_SOURCE_BYTES),
+        "max_rows": (1, 10_000_000),
+        "max_columns": (1, 1_024),
+    }
+    for name, (minimum, maximum) in bounds.items():
+        item = limits[name]
+        if type(item) is not int or not minimum <= item <= maximum:
+            raise HostedCrawlerError("CRAWLER_QUERY_INVALID", f"public HTTPS {name} is invalid")
+    reader = value["reader_pin"]
+    resource_caps = value["resource_caps"]
+    if resource_caps is not None:
+        if not isinstance(resource_caps, dict) or set(resource_caps) != _PUBLIC_RESOURCE_CAP_FIELDS:
+            raise HostedCrawlerError(
+                "CRAWLER_QUERY_INVALID", "public HTTPS Reader resource caps are not exact"
+            )
+        for name in sorted(_PUBLIC_RESOURCE_CAP_FIELDS):
+            item = resource_caps[name]
+            if type(item) is not int or item < 1:
+                raise HostedCrawlerError("CRAWLER_QUERY_INVALID", f"public HTTPS {name} is invalid")
+        if resource_caps["max_nesting_depth"] != 1:
+            raise HostedCrawlerError(
+                "CRAWLER_QUERY_INVALID", "public HTTPS max_nesting_depth must be exactly 1"
+            )
+    if reader is not None:
+        if not isinstance(reader, dict) or set(reader) != {
+            "family_id",
+            "family_version",
+            "decode_options",
+        }:
+            raise HostedCrawlerError("CRAWLER_QUERY_INVALID", "public HTTPS Reader pin is invalid")
+        try:
+            pin = ReaderPin(reader["family_id"], reader["family_version"], reader["decode_options"])
+            family = TOOLBOX.resolve(pin.family_id, pin.family_version)
+            admitted = family.validate_options(pin.decode_options)
+        except (TypeError, ValueError) as error:
+            raise HostedCrawlerError(
+                getattr(error, "code", "CRAWLER_READER_INVALID"),
+                "public HTTPS Reader pin is not certified",
+            ) from error
+        reader = {
+            "family_id": pin.family_id,
+            "family_version": pin.family_version,
+            "decode_options": dict(admitted),
+        }
+    elif resource_caps is not None:
+        raise HostedCrawlerError(
+            "CRAWLER_QUERY_INVALID", "public HTTPS resource caps require a Reader pin"
+        )
+    return {
+        "url": url,
+        "data_format": data_format,
+        "filename": filename,
+        "reader_pin": reader,
+        "resource_caps": None if resource_caps is None else dict(resource_caps),
+        "limits": dict(limits),
+    }
+
+
+def _public_expected_media_types(query: dict[str, Any]) -> tuple[str, ...]:
+    reader = query["reader_pin"]
+    if reader is None:
+        from mostlyright.data_harness.formats import FORMAT_MEDIA_TYPES
+
+        return FORMAT_MEDIA_TYPES[query["data_format"]]
+    family = TOOLBOX.resolve(reader["family_id"], reader["family_version"])
+    return family.accepted_media_types
+
+
+def _normalize_public_https_response(
+    request: HostedCrawlerRequest,
+    *,
+    raw: bytes,
+    final_url: str,
+    media_type: str,
+    raw_digest: str,
+    transport_evidence_digest: str,
+    http_status_code: int,
+    etag: str | None,
+    last_modified: str | None,
+) -> HostedCrawlerResult:
+    query = _public_https_query(request.query)
+    if (
+        raw_digest != sha256_bytes(raw)
+        or not 200 <= http_status_code <= 299
+        or _canonical_url_host(final_url) != _canonical_url_host(query["url"])
+        or media_type not in _public_expected_media_types(query)
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_NORMALIZATION_INPUT_INVALID",
+            "public HTTPS retrieval binding is invalid",
+        )
+    reader = query["reader_pin"]
+    if reader is None:
+        normalized = raw
+        normalized_format = query["data_format"]
+        normalized_media_type = media_type
+        normalized_filename = query["filename"]
+        family_id = family_version = options_digest = None
+        decode_flags: list[str] = []
+        reader_budgets = None
+    else:
+        pin = ReaderPin(reader["family_id"], reader["family_version"], reader["decode_options"])
+        family = TOOLBOX.resolve(pin.family_id, pin.family_version)
+        budget_caps = {
+            "max_input_bytes": query["limits"]["max_source_bytes"],
+            "max_output_bytes": query["limits"]["max_normalized_bytes"],
+            "max_rows": query["limits"]["max_rows"],
+            "max_columns": query["limits"]["max_columns"],
+            **(query["resource_caps"] or {}),
+        }
+        budgets = family.default_budgets.narrowed_by(budget_caps)
+        reader_budgets = asdict(budgets)
+        decoded = family.decode(raw, pin, budgets)
+        normalized = decoded.content
+        normalized_format = decoded.data_format
+        normalized_media_type = decoded.media_type
+        normalized_filename = decoded.filename
+        family_id = pin.family_id
+        family_version = pin.family_version
+        options_digest = pin.options_digest
+        decode_flags = list(decoded.flags)
+    parse_limits = ParseLimits(
+        # The parsed bytes are the normalized decode output, so its budget is the
+        # normalized-output budget, not the fetched-byte budget.
+        max_input_bytes=query["limits"]["max_normalized_bytes"],
+        max_uncompressed_bytes=max(128 * 1024 * 1024, query["limits"]["max_normalized_bytes"]),
+        max_rows=query["limits"]["max_rows"],
+        max_columns=query["limits"]["max_columns"],
+        # The cell budget follows the query's own geometry rather than the ParseLimits
+        # default: an EPA year is ~9.4M rows x 24 columns and must pass when its stated
+        # row and column limits admit it.
+        max_total_cells=min(
+            1_000_000_000,
+            query["limits"]["max_rows"] * query["limits"]["max_columns"],
+        ),
+    )
+    if normalized_format == "csv":
+        # The verification pass streams: a year-scale table re-checked as rows, never held
+        # as one.  The evidence values are the drained parse's own, pinned equal by test.
+        evidence = parse_csv_evidence(
+            normalized,
+            media_type=normalized_media_type,
+            filename=normalized_filename,
+            limits=parse_limits,
+        )
+        parsed_columns = evidence.columns
+        parsed_row_count = evidence.row_count
+        parsed_schema_digest = evidence.schema_digest
+        row_digest = evidence.row_digest
+    else:
+        parsed = parse_tabular_bytes(
+            normalized,
+            data_format=normalized_format,
+            media_type=normalized_media_type,
+            filename=normalized_filename,
+            limits=parse_limits,
+        )
+        parsed_columns = parsed.columns
+        parsed_row_count = len(parsed.rows)
+        parsed_schema_digest = parsed.schema_digest
+        row_digest = row_digest_for(parsed)
+    normalized_digest = sha256_bytes(normalized)
+    etag_digest = None if etag is None else sha256_bytes(etag.encode("utf-8"))
+    last_modified_digest = (
+        None if last_modified is None else sha256_bytes(last_modified.encode("utf-8"))
+    )
+    enhanced_cadence_evidence = request.schema_version == CRAWLER_REQUEST_SCHEMA_V3
+    receipt = {
+        "schema_version": (
+            "hosted-public-https-acquisition-receipt.v3"
+            if enhanced_cadence_evidence
+            else "hosted-public-https-acquisition-receipt.v1"
+        ),
+        "source_id": request.source_id,
+        "adapter_id": request.adapter_id,
+        "adapter_version": request.adapter_version,
+        "request_digest": request.digest,
+        "source_url": query["url"],
+        "final_url": final_url,
+        "fetched_media_type": media_type,
+        "normalized_media_type": normalized_media_type,
+        "normalized_data_format": normalized_format,
+        "normalized_filename": normalized_filename,
+        "acquired_at": request.requested_at,
+        "raw_content_sha256": raw_digest,
+        "raw_size_bytes": len(raw),
+        "normalized_content_sha256": normalized_digest,
+        "normalized_size_bytes": len(normalized),
+        "row_count": parsed_row_count,
+        "column_names": list(parsed_columns),
+        "parsed_schema_digest": parsed_schema_digest,
+        "family_id": family_id,
+        "family_version": family_version,
+        "decode_options_digest": options_digest,
+        "decode_flags": decode_flags,
+        "resource_caps": query["resource_caps"],
+        "reader_budgets": reader_budgets,
+        "transport_evidence_digest": transport_evidence_digest,
+        "egress_policy_attestation": request.egress_policy_attestation,
+        "clean_room": {
+            "host_platform": "linux-amd64",
+            "parser_network_syscalls": "denied",
+            "parser_process_creation": "denied",
+            "no_new_privs": True,
+        },
+    }
+    if enhanced_cadence_evidence:
+        receipt.update(
+            row_digest=row_digest,
+            http_status_code=http_status_code,
+            etag_digest=etag_digest,
+            last_modified_digest=last_modified_digest,
+        )
+    observation = {
+        "schema_version": (
+            "hosted-public-https-source-observation.v3"
+            if enhanced_cadence_evidence
+            else "hosted-public-https-source-observation.v1"
+        ),
+        "source_id": request.source_id,
+        "request_digest": request.digest,
+        "observed_at": request.requested_at,
+        "raw_content_sha256": raw_digest,
+        "normalized_content_sha256": normalized_digest,
+        "row_count": parsed_row_count,
+    }
+    if enhanced_cadence_evidence:
+        observation.update(
+            row_digest=row_digest,
+            http_status_code=http_status_code,
+            etag_digest=etag_digest,
+            last_modified_digest=last_modified_digest,
+        )
+    return HostedCrawlerResult(
+        request_digest=request.digest,
+        source_id=request.source_id,
+        adapter_id=request.adapter_id,
+        adapter_version=request.adapter_version,
+        normalized_content=normalized,
+        acquisition_receipt=receipt,
+        observation=observation,
+        next_watermark=None,
+        schema_version=(
+            CRAWLER_RESULT_SCHEMA_V3 if enhanced_cadence_evidence else CRAWLER_RESULT_SCHEMA_V1
+        ),
+    )
+
+
+def warm_up_public_https_reader(request: HostedCrawlerRequest) -> None:
+    """Re-open the packaged known sample on the exact hosted Reader execution path."""
+
+    query = _public_https_query(request.query)
+    reader = query["reader_pin"]
+    if reader is None:
+        return
+    pin = ReaderPin(reader["family_id"], reader["family_version"], reader["decode_options"])
+
+    def decode(sample: Any) -> DecodedFacts:
+        sample_family = TOOLBOX.resolve(sample.family_id, sample.family_version)
+        decoded = sample_family.decode(sample.content, sample.pin, sample_family.default_budgets)
+        parsed = parse_tabular_bytes(
+            decoded.content,
+            data_format=decoded.data_format,
+            media_type=decoded.media_type,
+            filename=decoded.filename,
+            limits=ParseLimits(
+                max_input_bytes=sample_family.default_budgets.max_output_bytes,
+                max_uncompressed_bytes=sample_family.default_budgets.max_output_bytes,
+                max_rows=sample_family.default_budgets.max_rows,
+                max_columns=sample_family.default_budgets.max_columns,
+            ),
+        )
+        return DecodedFacts(
+            sha256_bytes(decoded.content),
+            len(parsed.rows),
+            tuple(parsed.columns),
+            tuple(decoded.flags),
+        )
+
+    warm_up(pin.family_id, pin.family_version, decode=decode)
+
+
+def _query(value: Any) -> tuple[str, int]:
+    if not isinstance(value, dict) or set(value) != _QUERY_FIELDS:
+        raise HostedCrawlerError(
+            "CRAWLER_QUERY_INVALID",
+            OPENLIGADB_QUERY_FIELDS_DETAIL,
+        )
+    league = value["league_shortcut"]
+    season = value["league_season"]
+    if (
+        not isinstance(league, str)
+        or _LEAGUE.fullmatch(league) is None
+        or type(season) is not int
+        or not 2000 <= season <= 2100
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_QUERY_INVALID",
+            OPENLIGADB_QUERY_ALLOWLIST_DETAIL,
+        )
+    return league, season
+
+
+def _normalize(
+    raw: bytes,
+    *,
+    expected_league: str,
+    expected_season: int,
+) -> tuple[list[dict[str, str]], str, str]:
+    def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError("duplicate JSON object key")
+            result[key] = value
+        return result
+
+    try:
+        value = json.loads(
+            raw.decode("utf-8", errors="strict"),
+            object_pairs_hook=unique_object,
+            parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("non-finite number")),
+        )
+    except (UnicodeError, ValueError, json.JSONDecodeError) as error:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "OpenLigaDB returned invalid strict JSON",
+        ) from error
+    if not isinstance(value, list) or not 1 <= len(value) <= MAX_ROWS:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "OpenLigaDB response row inventory is empty or exceeds the bound",
+        )
+    rows = [_normalize_match(item) for item in value]
+    rows.sort(key=lambda item: int(item["match_id"]))
+    if len({item["match_id"] for item in rows}) != len(rows) or any(
+        item["league_shortcut"] != expected_league or item["league_season"] != str(expected_season)
+        for item in rows
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "OpenLigaDB response identity or league/season binding mismatches",
+        )
+    event_times = [item["match_datetime_utc"] for item in rows]
+    return rows, min(event_times), max(event_times)
+
+
+def _normalize_match(value: Any) -> dict[str, str]:
+    if not isinstance(value, dict):
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "OpenLigaDB match rows must be objects",
+        )
+    match_id = _integer(value.get("matchID"), "matchID", minimum=1)
+    event_time = _utc(value.get("matchDateTimeUTC"), "matchDateTimeUTC")
+    last_update = _text(value.get("lastUpdateDateTime"), "lastUpdateDateTime")
+    league = _text(value.get("leagueShortcut"), "leagueShortcut")
+    season = _integer(value.get("leagueSeason"), "leagueSeason", minimum=2000)
+    group = _object(value.get("group"), "group")
+    team1 = _object(value.get("team1"), "team1")
+    team2 = _object(value.get("team2"), "team2")
+    finished = value.get("matchIsFinished")
+    if type(finished) is not bool:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "OpenLigaDB matchIsFinished must be boolean",
+        )
+    score1, score2 = _score(value.get("matchResults"), finished=finished)
+    return {
+        "match_id": str(match_id),
+        "match_datetime_utc": event_time,
+        "last_update_datetime": last_update,
+        "league_shortcut": league,
+        "league_season": str(season),
+        "group_order_id": str(_integer(group.get("groupOrderID"), "group.groupOrderID", minimum=1)),
+        "team1_id": str(_integer(team1.get("teamId"), "team1.teamId", minimum=1)),
+        "team1_name": _text(team1.get("teamName"), "team1.teamName"),
+        "team2_id": str(_integer(team2.get("teamId"), "team2.teamId", minimum=1)),
+        "team2_name": _text(team2.get("teamName"), "team2.teamName"),
+        "match_finished": "true" if finished else "false",
+        "score1": "" if score1 is None else str(score1),
+        "score2": "" if score2 is None else str(score2),
+    }
+
+
+def _score(value: Any, *, finished: bool) -> tuple[int | None, int | None]:
+    if not isinstance(value, list) or len(value) > 32:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "OpenLigaDB matchResults must be a bounded array",
+        )
+    if not value:
+        if finished:
+            raise HostedCrawlerError(
+                "CRAWLER_RESPONSE_INVALID",
+                "finished OpenLigaDB match has no final result",
+            )
+        return None, None
+    results = []
+    for item in value:
+        result = _object(item, "matchResults[]")
+        results.append(
+            (
+                _integer(
+                    result.get("resultTypeID"),
+                    "matchResults[].resultTypeID",
+                    minimum=1,
+                ),
+                _integer(
+                    result.get("resultOrderID"),
+                    "matchResults[].resultOrderID",
+                    minimum=1,
+                ),
+                _integer(
+                    result.get("resultID"),
+                    "matchResults[].resultID",
+                    minimum=1,
+                ),
+                _integer(
+                    result.get("pointsTeam1"),
+                    "matchResults[].pointsTeam1",
+                    minimum=0,
+                ),
+                _integer(
+                    result.get("pointsTeam2"),
+                    "matchResults[].pointsTeam2",
+                    minimum=0,
+                ),
+            )
+        )
+    finals = [item for item in results if item[0] == 2]
+    selected = max(finals or results, key=lambda item: (item[1], item[2]))
+    return selected[3], selected[4]
+
+
+def _object(value: Any, field: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            f"OpenLigaDB {field} must be an object",
+        )
+    return value
+
+
+def _integer(value: Any, field: str, *, minimum: int) -> int:
+    if type(value) is not int or not minimum <= value <= MAX_SAFE_INTEGER:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            f"OpenLigaDB {field} is outside its integer bound",
+        )
+    return value
+
+
+def _text(value: Any, field: str) -> str:
+    if (
+        not isinstance(value, str)
+        or not 1 <= len(value.encode("utf-8")) <= MAX_TEXT_BYTES
+        or any(character in value for character in ("\x00", "\r", "\n"))
+    ):
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            f"OpenLigaDB {field} is invalid or exceeds its text bound",
+        )
+    return value
+
+
+def _utc(value: Any, field: str) -> str:
+    if not isinstance(value, str) or _UTC_TIMESTAMP.fullmatch(value) is None:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            f"OpenLigaDB {field} must be an explicit UTC timestamp",
+        )
+    try:
+        datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
+    except ValueError as error:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            f"OpenLigaDB {field} is not a real UTC timestamp",
+        ) from error
+    return value
+
+
+def _encode_csv(rows: list[dict[str, str]]) -> bytes:
+    stream = io.StringIO(newline="")
+    writer = csv.DictWriter(
+        stream,
+        fieldnames=list(_COLUMNS),
+        lineterminator="\n",
+        extrasaction="raise",
+    )
+    writer.writeheader()
+    writer.writerows(rows)
+    content = stream.getvalue().encode("utf-8")
+    # The reviewed OpenLigaDB bound, pinned to its own 8 MiB literal so the shared
+    # public-HTTPS ceiling cannot widen this fixed adapter.
+    if not 1 <= len(content) <= 8 * 1024 * 1024:
+        raise HostedCrawlerError(
+            "CRAWLER_RESPONSE_INVALID",
+            "normalized OpenLigaDB CSV exceeds the source byte bound",
+        )
+    return content
+
+
+def _read_fd_once(descriptor: int, *, maximum: int) -> bytes:
+    chunks: list[bytes] = []
+    total = 0
+    while True:
+        chunk = os.read(descriptor, min(65_536, maximum + 1 - total))
+        if not chunk:
+            break
+        total += len(chunk)
+        if total > maximum:
+            raise HostedCrawlerError(
+                "CRAWLER_PROTOCOL_LIMIT",
+                "crawler request exceeds its byte bound",
+            )
+        chunks.append(chunk)
+    os.close(descriptor)
+    return b"".join(chunks)
+
+
+def _write_fd_once(descriptor: int, content: bytes) -> None:
+    if len(content) > MAX_CRAWLER_RESULT_BYTES:
+        raise HostedCrawlerError(
+            "CRAWLER_PROTOCOL_LIMIT",
+            "crawler result exceeds its byte bound",
+        )
+    view = memoryview(content)
+    while view:
+        written = os.write(descriptor, view)
+        view = view[written:]
+    os.close(descriptor)
+
+
+def _require_secret_minimized_environment() -> None:
+    forbidden = sorted(key for key in os.environ if _FORBIDDEN_ENVIRONMENT.search(key))
+    if forbidden:
+        raise HostedCrawlerError(
+            "CRAWLER_ENVIRONMENT_FORBIDDEN",
+            "crawler environment contains a credential, Studio, token, or home variable",
+        )
+
+
+def crawler_main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="mr-data-crawler")
+    parser.add_argument("--input-fd", type=int, default=0)
+    parser.add_argument("--result-fd", type=int, default=1)
+    parser.add_argument("--warm-up", action="store_true")
+    args = parser.parse_args(argv)
+    try:
+        _require_secret_minimized_environment()
+        attestation = os.environ.get(CRAWLER_EGRESS_POLICY_ENV)
+        if attestation is None:
+            raise HostedCrawlerError(
+                "CRAWLER_POLICY_REQUIRED",
+                "crawler egress-policy attestation is unavailable",
+            )
+        maximum = MAX_CRAWLER_REQUEST_BYTES if args.warm_up else MAX_NORMALIZATION_INPUT_BYTES
+        content = _read_fd_once(args.input_fd, maximum=maximum)
+        if args.warm_up:
+            request = parse_crawler_request(parse_canonical_json(content))
+            validate_crawler_request(
+                request,
+                expected_egress_policy_attestation=attestation,
+            )
+            warm_up_public_https_reader(request)
+            _write_fd_once(
+                args.result_fd,
+                canonical_json_bytes(
+                    {
+                        "schema_version": WARM_UP_RESULT_SCHEMA,
+                        "request_digest": request.digest,
+                        "status": "passed",
+                    }
+                ),
+            )
+            return 0
+        (
+            request,
+            raw,
+            source_url,
+            media_type,
+            raw_digest,
+            transport_evidence_digest,
+            http_status_code,
+            etag,
+            last_modified,
+        ) = parse_normalization_input(content)
+        validate_crawler_request(
+            request,
+            expected_egress_policy_attestation=attestation,
+        )
+        result = normalize_crawler_response(
+            request,
+            raw=raw,
+            source_url=source_url,
+            media_type=media_type,
+            raw_digest=raw_digest,
+            transport_evidence_digest=transport_evidence_digest,
+            http_status_code=http_status_code,
+            etag=etag,
+            last_modified=last_modified,
+        )
+        _write_fd_once(args.result_fd, canonical_json_bytes(result.to_dict()))
+        return 0
+    except (
+        CanonicalJSONError,
+        HostedCrawlerError,
+        HostedCrawlerProtocolError,
+        ValueError,
+        TypeError,
+    ) as error:
+        code = getattr(error, "code", "CRAWLER_FAILED")
+        sys.stderr.write(canonical_json_bytes({"status": "failed", "code": code}).decode() + "\n")
+        return 1
+
+
+if __name__ == "__main__":  # pragma: no cover - exercised through the executable boundary.
+    raise SystemExit(crawler_main())