mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_crawler_protocol.py

@@ -0,0 +1,676 @@
+"""Exact public-crawler request/result protocol shared with hosted workers."""
+
+from __future__ import annotations
+
+import base64
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Protocol
+
+from mostlyright.data_harness.canonical import (
+    canonical_json_bytes,
+    canonical_sha256,
+    sha256_bytes,
+)
+
+CRAWLER_REQUEST_SCHEMA_V1 = "mostlyright-hosted-crawler-request.v1"
+CRAWLER_REQUEST_SCHEMA_V3 = "mostlyright-hosted-crawler-request.v3"
+CRAWLER_REQUEST_SCHEMA = CRAWLER_REQUEST_SCHEMA_V1
+CRAWLER_RESULT_SCHEMA_V1 = "mostlyright-hosted-crawler-result.v1"
+CRAWLER_RESULT_SCHEMA_V3 = "mostlyright-hosted-crawler-result.v3"
+# Compatibility export for consumers that only understand the original OpenLigaDB/V1 result.
+CRAWLER_RESULT_SCHEMA = CRAWLER_RESULT_SCHEMA_V1
+CRAWLER_FAILURE_SCHEMA = "mostlyright-hosted-crawler-failure.v1"
+CRAWLER_RESULT_MEDIA_TYPE = "application/vnd.mostlyright.hosted-crawler-result+json"
+MAX_CRAWLER_REQUEST_BYTES = 64 * 1024
+MAX_CRAWLER_RESULT_BYTES = 6_442_450_944
+MAX_NORMALIZED_SOURCE_BYTES = 4_294_967_296
+MAX_CRAWLER_BASE64URL_CHUNK_CHARS = 768 * 1024
+OPENLIGADB_EGRESS_POLICY_ATTESTATION = canonical_sha256(
+    {
+        "schema_version": "mostlyright-hosted-crawler-egress-policy.v1",
+        "adapter_id": "external.openligadb",
+        "adapter_version": "1.0.0",
+        "credential_free": True,
+        "allowed_hostnames": ["api.openligadb.de"],
+        "allowed_ports": [443],
+        "allowed_path_template": "/getmatchdata/{league_shortcut}/{league_season}",
+        "redirects": 0,
+        "ambient_proxy": False,
+        "source_retriever_private_metadata_internal_networks": "denied",
+        "wrapper_metadata_identity": "fixed-link-local-google-oidc-exact-session-bootstrap",
+        "wrapper_studio_authority": "bootstrap-only-no-attempt-token",
+        "wrapper_dumpability": "disabled-before-workload-identity",
+        "parser_network_syscalls": "linux-amd64-no-new-privs-seccomp-deny",
+        "parser_x32_syscalls": "denied-before-native-allowlist",
+        "parser_process_creation_and_parent_inspection": "denied",
+        "parser_network_fds": "closed-before-exec",
+        "isolation": "separate-crawler-job-and-networkless-parser-child",
+        "max_response_bytes": 8 * 1024 * 1024,
+        "max_requests": 1,
+        "connect_timeout_milliseconds": 5_000,
+        "read_timeout_milliseconds": 15_000,
+        "total_timeout_milliseconds": 20_000,
+        "max_concurrency": 2,
+        "min_interval_milliseconds": 1_000,
+    }
+)
+# The exact attested public-HTTPS egress policy. Named, rather than inlined into the
+# digest call below, so a test can hold each attested figure against the constant that
+# enforces it in hosted_crawler_fetch without restating the whole policy: the attestation
+# must never claim an envelope the fetch path does not run under.
+PUBLIC_HTTPS_EGRESS_POLICY: dict[str, Any] = {
+    "schema_version": "mostlyright-hosted-crawler-egress-policy.v1",
+    "adapter_id": "public.https",
+    "adapter_version": "1.0.0",
+    "credential_free": True,
+    "allowed_hostnames": "exact canonical hostname from the Studio-owned request",
+    "allowed_ports": [443],
+    "redirects": "at most 3; every hop stays on the exact hostname",
+    "ambient_proxy": False,
+    "dns_policy": "public addresses only; rebinding refused on every connection",
+    "source_retriever_private_metadata_internal_networks": "denied",
+    "wrapper_metadata_identity": "fixed-link-local-google-oidc-exact-session-bootstrap",
+    "wrapper_studio_authority": "bootstrap-only-no-user-or-attempt-token",
+    "wrapper_dumpability": "disabled-before-workload-identity",
+    "parser_network_syscalls": "linux-amd64-no-new-privs-seccomp-deny",
+    "parser_x32_syscalls": "denied-before-native-allowlist",
+    "parser_process_creation_and_parent_inspection": "denied",
+    "parser_network_fds": "closed-before-exec",
+    "isolation": "separate-crawler-job-and-networkless-parser-child",
+    # The enforced public-HTTPS fetch ceiling (hosted_crawler.MAX_RAW_RESPONSE_BYTES),
+    # equal to the hosted contract's max_source_bytes maximum. Decode output is bounded
+    # separately by MAX_NORMALIZED_SOURCE_BYTES.
+    "max_response_bytes": 268_435_456,
+    "max_requests": 4,
+    "connect_timeout_milliseconds": 5_000,
+    # Large public archives (EPA AirData) stall tens of seconds before the first
+    # byte for cloud egress addresses; the total budget, not the per-read stall,
+    # is the meaningful ceiling for a bulk fetch.
+    "read_timeout_milliseconds": 120_000,
+    "total_timeout_milliseconds": 600_000,
+    "max_concurrency": 2,
+    "min_interval_milliseconds": 1_000,
+    # A bulk object whose 0-0 probe names a total above threshold_bytes is fetched as
+    # consecutive range_bytes slices, at most `parallelism` in flight, each slice under
+    # its own tight stall timeouts with up to range_attempts fresh attempts.  The
+    # total_timeout_milliseconds budget above bounds the ranged fetch at attempt
+    # admission: no new range attempt starts after it, and one already-admitted attempt
+    # may still run to range_total_timeout_milliseconds, so the wall-clock worst case is
+    # the sum of the two.  range_validator_pinning means the ranged path runs only when
+    # the probe names an ETag or Last-Modified validator, and every range must answer
+    # with exactly the probed validators -- an origin that updates the object mid-fetch
+    # is refused rather than assembled across editions, and a validator-less origin keeps
+    # the single-shot path.  Origins that ignore or refuse the range probe keep the
+    # single-shot behavior the rest of this policy describes, with the probe's own
+    # transfer sealed into the result's transport evidence.
+    "ranged_fetch": {
+        "threshold_bytes": 33_554_432,
+        "range_bytes": 16_777_216,
+        "parallelism": 2,
+        "range_read_timeout_milliseconds": 30_000,
+        "range_total_timeout_milliseconds": 120_000,
+        "range_attempts": 4,
+        "range_validator_pinning": True,
+    },
+}
+PUBLIC_HTTPS_EGRESS_POLICY_ATTESTATION = canonical_sha256(PUBLIC_HTTPS_EGRESS_POLICY)
+
+# The OpenLigaDB query is validated twice against the same reviewed shape: once producer-side
+# in ``hosted_worker`` before the intent leaves, and once crawler-side in ``hosted_crawler``
+# before any egress. The two refusals keep their own codes (SOURCE_ADAPTER_QUERY_INVALID and
+# CRAWLER_QUERY_INVALID) because they name different boundaries, but the detail text is shared
+# so the two sides cannot drift into describing the same rejected query differently.
+OPENLIGADB_QUERY_FIELDS_DETAIL = (
+    "OpenLigaDB query must contain only league_shortcut and league_season"
+)
+OPENLIGADB_QUERY_ALLOWLIST_DETAIL = "OpenLigaDB league or season is outside the reviewed allowlist"
+
+_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+_PREFIXED_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
+_BASE64URL_CHUNK = re.compile(r"^[A-Za-z0-9_-]+$")
+_TIMESTAMP = re.compile(
+    r"^[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])"
+    r"T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:\.[0-9]{1,6})?Z$"
+)
+HOSTED_CRAWLER_FAILURE_CODES = frozenset(
+    {
+        "HTTP_NOT_FOUND",
+        "HTTP_STATUS",
+        "MEDIA_TYPE",
+        "BYTE_LIMIT",
+        "TOTAL_TIMEOUT",
+        "CONNECT_TIMEOUT",
+        "READ_TIMEOUT",
+        "CONNECT_FAILED",
+        "TLS_FAILED",
+        "DNS_FAILED",
+        "REDIRECT_LIMIT",
+        "REDIRECT_POLICY",
+        "CRAWLER_ENDPOINT_MISMATCH",
+        "CRAWLER_MEDIA_TYPE_MISMATCH",
+        "CRAWLER_RESPONSE_INVALID",
+        "CRAWLER_NORMALIZATION_INPUT_INVALID",
+        "CRAWLER_MEMORY_BOUNDARY",
+        "READER_FAMILY_UNKNOWN",
+        "READER_DEPENDENCY_UNAVAILABLE",
+        "READER_CONTRACT",
+        "READER_OPTIONS",
+        "READER_ADMISSION",
+        "READER_BUDGET",
+        "READER_MEMBER_MISSING",
+        "READER_MEMBER_AMBIGUOUS",
+        "READER_MEMBER_TYPE",
+        "READER_NESTED_CONTAINER",
+        "READER_SHEET_MISSING",
+        "READER_RANGE",
+        "READER_GRID_UNKNOWN",
+        "READER_PACKING_UNKNOWN",
+        "READER_DECODE",
+        "READER_OUTPUT",
+        "READER_WARMUP",
+    }
+)
+
+
+class HostedCrawlerProtocolError(ValueError):
+    """One closed crawler request/result invariant failed."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{detail} [{code}]")
+
+
+def _exact(value: Any, fields: set[str], label: str) -> dict[str, Any]:
+    if not isinstance(value, dict) or set(value) != fields:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_FIELDS",
+            f"{label} fields are not exact",
+        )
+    return value
+
+
+def _identifier(value: Any, label: str) -> str:
+    if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_IDENTIFIER",
+            f"{label} is not a canonical identifier",
+        )
+    return value
+
+
+def _timestamp(value: Any, label: str) -> str:
+    if not isinstance(value, str) or _TIMESTAMP.fullmatch(value) is None:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_TIMESTAMP",
+            f"{label} is not a canonical UTC timestamp",
+        )
+    try:
+        parsed = datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
+    except ValueError as error:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_TIMESTAMP",
+            f"{label} is not a real UTC timestamp",
+        ) from error
+    if parsed.tzinfo is None or parsed.astimezone(UTC).utcoffset().total_seconds() != 0:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_TIMESTAMP",
+            f"{label} is not a canonical UTC timestamp",
+        )
+    return value
+
+
+def _digest(value: Any, label: str, *, prefixed: bool = False) -> str:
+    pattern = _PREFIXED_DIGEST if prefixed else _DIGEST
+    if not isinstance(value, str) or pattern.fullmatch(value) is None:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_DIGEST",
+            f"{label} is not a canonical SHA-256 digest",
+        )
+    return value
+
+
+def _json_object(value: Any, label: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_JSON",
+            f"{label} must be an object",
+        )
+    try:
+        encoded = canonical_json_bytes(value)
+    except (TypeError, ValueError) as error:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_JSON",
+            f"{label} contains unsupported JSON values",
+        ) from error
+    if len(encoded) > MAX_CRAWLER_REQUEST_BYTES:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_LIMIT",
+            f"{label} exceeds its byte bound",
+        )
+    return dict(value)
+
+
+@dataclass(frozen=True)
+class HostedCrawlerRequest:
+    request_id: str
+    source_id: str
+    adapter_id: str
+    adapter_version: str
+    requested_at: str
+    query: dict[str, Any]
+    egress_policy_attestation: str
+    schema_version: str | None = None
+
+    def __post_init__(self) -> None:
+        _identifier(self.request_id, "request_id")
+        _identifier(self.source_id, "source_id")
+        _identifier(self.adapter_id, "adapter_id")
+        if self.adapter_version != "1.0.0" or self.adapter_id not in {
+            "external.openligadb",
+            "public.https",
+        }:
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_VERSION",
+                "adapter coordinate is not a reviewed hosted-crawler version",
+            )
+        _timestamp(self.requested_at, "requested_at")
+        object.__setattr__(self, "query", _json_object(self.query, "query"))
+        _digest(self.egress_policy_attestation, "egress_policy_attestation")
+        schema_version = self.schema_version or CRAWLER_REQUEST_SCHEMA_V1
+        if schema_version not in {CRAWLER_REQUEST_SCHEMA_V1, CRAWLER_REQUEST_SCHEMA_V3} or (
+            schema_version == CRAWLER_REQUEST_SCHEMA_V3 and self.adapter_id != "public.https"
+        ):
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_VERSION",
+                "crawler request schema is unsupported for its adapter",
+            )
+        object.__setattr__(self, "schema_version", schema_version)
+        if len(canonical_json_bytes(self.to_dict())) > MAX_CRAWLER_REQUEST_BYTES:
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_LIMIT",
+                "crawler request exceeds its byte bound",
+            )
+
+    @property
+    def digest(self) -> str:
+        return sha256_bytes(canonical_json_bytes(self.to_dict()))
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "request_id": self.request_id,
+            "source_id": self.source_id,
+            "adapter_id": self.adapter_id,
+            "adapter_version": self.adapter_version,
+            "requested_at": self.requested_at,
+            "query": dict(self.query),
+            "egress_policy_attestation": self.egress_policy_attestation,
+        }
+
+
+@dataclass(frozen=True)
+class HostedCrawlerFailure:
+    request_digest: str
+    source_id: str
+    adapter_id: str
+    adapter_version: str
+    failure_code: str
+    retryable: bool = False
+
+    def __post_init__(self) -> None:
+        _digest(self.request_digest, "request_digest")
+        _identifier(self.source_id, "source_id")
+        if (self.adapter_id, self.adapter_version) not in {
+            ("external.openligadb", "1.0.0"),
+            ("public.https", "1.0.0"),
+        }:
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_VERSION", "failure adapter coordinate is not reviewed"
+            )
+        if self.failure_code not in HOSTED_CRAWLER_FAILURE_CODES or self.retryable is not False:
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_FAILURE", "crawler failure code is not safely transportable"
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": CRAWLER_FAILURE_SCHEMA,
+            "request_digest": self.request_digest,
+            "source_id": self.source_id,
+            "adapter_id": self.adapter_id,
+            "adapter_version": self.adapter_version,
+            "failure_code": self.failure_code,
+            "retryable": self.retryable,
+        }
+
+
+def parse_crawler_failure(value: Any) -> HostedCrawlerFailure:
+    document = _exact(
+        value,
+        {
+            "schema_version",
+            "request_digest",
+            "source_id",
+            "adapter_id",
+            "adapter_version",
+            "failure_code",
+            "retryable",
+        },
+        "crawler failure",
+    )
+    if document["schema_version"] != CRAWLER_FAILURE_SCHEMA:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_SCHEMA", "crawler failure schema version is unsupported"
+        )
+    return HostedCrawlerFailure(
+        request_digest=document["request_digest"],
+        source_id=document["source_id"],
+        adapter_id=document["adapter_id"],
+        adapter_version=document["adapter_version"],
+        failure_code=document["failure_code"],
+        retryable=document["retryable"],
+    )
+
+
+def parse_crawler_request(value: Any) -> HostedCrawlerRequest:
+    document = _exact(
+        value,
+        {
+            "schema_version",
+            "request_id",
+            "source_id",
+            "adapter_id",
+            "adapter_version",
+            "requested_at",
+            "query",
+            "egress_policy_attestation",
+        },
+        "crawler request",
+    )
+    if document["schema_version"] not in {
+        CRAWLER_REQUEST_SCHEMA_V1,
+        CRAWLER_REQUEST_SCHEMA_V3,
+    } or (
+        document["schema_version"] == CRAWLER_REQUEST_SCHEMA_V3
+        and document["adapter_id"] != "public.https"
+    ):
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_VERSION",
+            "crawler request schema is unsupported",
+        )
+    return HostedCrawlerRequest(
+        request_id=document["request_id"],
+        source_id=document["source_id"],
+        adapter_id=document["adapter_id"],
+        adapter_version=document["adapter_version"],
+        requested_at=document["requested_at"],
+        query=document["query"],
+        egress_policy_attestation=document["egress_policy_attestation"],
+        schema_version=document["schema_version"],
+    )
+
+
+@dataclass(frozen=True)
+class HostedCrawlerIntent:
+    """Producer request fields that Studio expands into server-owned crawler authority."""
+
+    source_id: str
+    adapter_id: str
+    adapter_version: str
+    query: dict[str, Any]
+    egress_policy_attestation: str
+
+    def __post_init__(self) -> None:
+        _identifier(self.source_id, "source_id")
+        _identifier(self.adapter_id, "adapter_id")
+        if self.adapter_version != "1.0.0" or self.adapter_id not in {
+            "external.openligadb",
+            "public.https",
+        }:
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_VERSION",
+                "intent adapter coordinate is unsupported",
+            )
+        object.__setattr__(self, "query", _json_object(self.query, "query"))
+        _digest(self.egress_policy_attestation, "egress_policy_attestation")
+
+
+@dataclass(frozen=True)
+class HostedCrawlerResult:
+    request_digest: str
+    source_id: str
+    adapter_id: str
+    adapter_version: str
+    normalized_content: bytes
+    acquisition_receipt: dict[str, Any]
+    observation: dict[str, Any]
+    next_watermark: str | None
+    schema_version: str | None = None
+
+    def __post_init__(self) -> None:
+        _digest(self.request_digest, "request_digest")
+        _identifier(self.source_id, "source_id")
+        _identifier(self.adapter_id, "adapter_id")
+        if self.adapter_version != "1.0.0" or self.adapter_id not in {
+            "external.openligadb",
+            "public.https",
+        }:
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_VERSION",
+                "result adapter coordinate is unsupported",
+            )
+        if (
+            not isinstance(self.normalized_content, bytes)
+            or not 1 <= len(self.normalized_content) <= MAX_NORMALIZED_SOURCE_BYTES
+        ):
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_LIMIT",
+                "normalized crawler content is empty or exceeds its byte bound",
+            )
+        object.__setattr__(
+            self,
+            "acquisition_receipt",
+            _json_object(self.acquisition_receipt, "acquisition_receipt"),
+        )
+        object.__setattr__(
+            self,
+            "observation",
+            _json_object(self.observation, "observation"),
+        )
+        if self.next_watermark is not None:
+            _timestamp(self.next_watermark, "next_watermark")
+        schema_version = self.schema_version or CRAWLER_RESULT_SCHEMA_V1
+        if schema_version not in {CRAWLER_RESULT_SCHEMA_V1, CRAWLER_RESULT_SCHEMA_V3} or (
+            schema_version == CRAWLER_RESULT_SCHEMA_V3 and self.adapter_id != "public.https"
+        ):
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_VERSION",
+                "crawler result schema is unsupported for its adapter",
+            )
+        object.__setattr__(self, "schema_version", schema_version)
+
+    @property
+    def normalized_content_digest(self) -> str:
+        return "sha256:" + sha256_bytes(self.normalized_content)
+
+    @property
+    def result_digest(self) -> str:
+        return sha256_bytes(canonical_json_bytes(self._preimage()))
+
+    @property
+    def normalized_content_base64url_chunks(self) -> list[str]:
+        encoded = base64.urlsafe_b64encode(self.normalized_content).decode("ascii").rstrip("=")
+        return [
+            encoded[index : index + MAX_CRAWLER_BASE64URL_CHUNK_CHARS]
+            for index in range(0, len(encoded), MAX_CRAWLER_BASE64URL_CHUNK_CHARS)
+        ]
+
+    def _preimage(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "request_digest": self.request_digest,
+            "source_id": self.source_id,
+            "adapter_id": self.adapter_id,
+            "adapter_version": self.adapter_version,
+            "normalized_content_base64url_chunks": (self.normalized_content_base64url_chunks),
+            "normalized_content_digest": self.normalized_content_digest,
+            "normalized_size_bytes": len(self.normalized_content),
+            "acquisition_receipt": dict(self.acquisition_receipt),
+            "observation": dict(self.observation),
+            "next_watermark": self.next_watermark,
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        return {**self._preimage(), "result_digest": self.result_digest}
+
+
+def parse_crawler_result(value: Any) -> HostedCrawlerResult:
+    document = _exact(
+        value,
+        {
+            "schema_version",
+            "request_digest",
+            "source_id",
+            "adapter_id",
+            "adapter_version",
+            "normalized_content_base64url_chunks",
+            "normalized_content_digest",
+            "normalized_size_bytes",
+            "acquisition_receipt",
+            "observation",
+            "next_watermark",
+            "result_digest",
+        },
+        "crawler result",
+    )
+    if document["schema_version"] not in {
+        CRAWLER_RESULT_SCHEMA_V1,
+        CRAWLER_RESULT_SCHEMA_V3,
+    } or (
+        document["schema_version"] == CRAWLER_RESULT_SCHEMA_V3
+        and document["adapter_id"] != "public.https"
+    ):
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_VERSION",
+            "crawler result schema is unsupported",
+        )
+    chunks = document["normalized_content_base64url_chunks"]
+    complete_groups, remainder = divmod(MAX_NORMALIZED_SOURCE_BYTES, 3)
+    maximum_encoded_chars = 4 * complete_groups + (0 if remainder == 0 else remainder + 1)
+    maximum_chunks = (
+        maximum_encoded_chars + MAX_CRAWLER_BASE64URL_CHUNK_CHARS - 1
+    ) // MAX_CRAWLER_BASE64URL_CHUNK_CHARS
+    if (
+        not isinstance(chunks, list)
+        or not 1 <= len(chunks) <= maximum_chunks
+        or any(
+            not isinstance(chunk, str)
+            or not 1 <= len(chunk) <= MAX_CRAWLER_BASE64URL_CHUNK_CHARS
+            or _BASE64URL_CHUNK.fullmatch(chunk) is None
+            for chunk in chunks
+        )
+        or any(len(chunk) != MAX_CRAWLER_BASE64URL_CHUNK_CHARS for chunk in chunks[:-1])
+    ):
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_LIMIT",
+            "normalized content chunk inventory is invalid or exceeds its bound",
+        )
+    encoded = "".join(chunks)
+    if len(encoded) > maximum_encoded_chars:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_LIMIT",
+            "normalized content encoding exceeds its byte bound",
+        )
+    try:
+        content = base64.b64decode(
+            encoded + "=" * (-len(encoded) % 4),
+            altchars=b"-_",
+            validate=True,
+        )
+    except (ValueError, TypeError) as error:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_BASE64",
+            "normalized content is not canonical base64url",
+        ) from error
+    if base64.urlsafe_b64encode(content).decode("ascii").rstrip("=") != encoded:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_BASE64",
+            "normalized content is not canonical base64url",
+        )
+    result = HostedCrawlerResult(
+        request_digest=document["request_digest"],
+        source_id=document["source_id"],
+        adapter_id=document["adapter_id"],
+        adapter_version=document["adapter_version"],
+        normalized_content=content,
+        acquisition_receipt=document["acquisition_receipt"],
+        observation=document["observation"],
+        next_watermark=document["next_watermark"],
+        schema_version=document["schema_version"],
+    )
+    if (
+        type(document["normalized_size_bytes"]) is not int
+        or document["normalized_content_digest"] != result.normalized_content_digest
+        or document["normalized_size_bytes"] != len(content)
+        or document["result_digest"] != result.result_digest
+    ):
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_BINDING",
+            "crawler result digest or size binding mismatches",
+        )
+    if len(canonical_json_bytes(document)) > MAX_CRAWLER_RESULT_BYTES:
+        raise HostedCrawlerProtocolError(
+            "CRAWLER_PROTOCOL_LIMIT",
+            "crawler result exceeds its byte bound",
+        )
+    return result
+
+
+@dataclass(frozen=True)
+class HostedCrawlerExchange:
+    """Studio-returned server request plus its one-shot isolated crawler result."""
+
+    request: HostedCrawlerRequest
+    result: HostedCrawlerResult
+
+    def __post_init__(self) -> None:
+        expected_result_schema = (
+            CRAWLER_RESULT_SCHEMA_V3
+            if self.request.schema_version == CRAWLER_REQUEST_SCHEMA_V3
+            else CRAWLER_RESULT_SCHEMA_V1
+        )
+        if (
+            self.result.request_digest != self.request.digest
+            or self.result.source_id != self.request.source_id
+            or self.result.adapter_id != self.request.adapter_id
+            or self.result.adapter_version != self.request.adapter_version
+            or self.result.schema_version != expected_result_schema
+        ):
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_BINDING",
+                "crawler result differs from its server-owned request authority",
+            )
+
+    def validate_intent(self, intent: HostedCrawlerIntent) -> None:
+        if (
+            self.request.source_id != intent.source_id
+            or self.request.adapter_id != intent.adapter_id
+            or self.request.adapter_version != intent.adapter_version
+            or self.request.query != intent.query
+            or self.request.egress_policy_attestation != intent.egress_policy_attestation
+        ):
+            raise HostedCrawlerProtocolError(
+                "CRAWLER_PROTOCOL_BINDING",
+                "Studio crawler request differs from the producer's frozen intent",
+            )
+
+
+class HostedCrawlerClient(Protocol):
+    """Producer-side create/poll/download abstraction for one isolated crawler result."""
+
+    def acquire(self, intent: HostedCrawlerIntent) -> HostedCrawlerExchange:
+        """Create/poll/download one server-owned request and its exact claimed result."""