mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_crawler_job.py

@@ -0,0 +1,1277 @@
+"""One-shot Cloud Run wrapper for broker bootstrap, trusted crawling, and signed GCS PUT."""
+
+from __future__ import annotations
+
+import argparse
+import http.client
+import os
+import re
+import shutil
+import ssl
+import subprocess
+import sys
+from dataclasses import asdict, dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, NoReturn, Protocol
+from urllib.error import HTTPError
+from urllib.parse import parse_qsl, urlencode, urlsplit
+from urllib.request import (
+    HTTPRedirectHandler,
+    HTTPSHandler,
+    ProxyHandler,
+    Request,
+    build_opener,
+)
+from uuid import UUID
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+    parse_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.hosted_crawler import (
+    CRAWLER_EGRESS_POLICY_ENV,
+    WARM_UP_RESULT_SCHEMA,
+    encode_normalization_input,
+)
+from mostlyright.data_harness.hosted_crawler_fetch import fetch_crawler_source
+from mostlyright.data_harness.hosted_crawler_protocol import (
+    CRAWLER_REQUEST_SCHEMA_V3,
+    CRAWLER_RESULT_MEDIA_TYPE,
+    CRAWLER_RESULT_SCHEMA_V1,
+    CRAWLER_RESULT_SCHEMA_V3,
+    HOSTED_CRAWLER_FAILURE_CODES,
+    MAX_CRAWLER_REQUEST_BYTES,
+    MAX_CRAWLER_RESULT_BYTES,
+    HostedCrawlerExchange,
+    HostedCrawlerFailure,
+    HostedCrawlerProtocolError,
+    HostedCrawlerRequest,
+    parse_crawler_request,
+    parse_crawler_result,
+)
+from mostlyright.data_harness.linux_process_boundary import (
+    _disable_process_dumpability,
+    _install_networkless_seccomp,
+)
+
+UPLOAD_CAPABILITY_SCHEMA = "mostlyright-crawler-upload-capability.v1"
+BOOTSTRAP_RESPONSE_SCHEMA = "mostlyright-crawler-bootstrap.v1"
+UPLOAD_MEDIA_TYPE = CRAWLER_RESULT_MEDIA_TYPE
+RESULTS_BUCKET_ENV = "MOSTLYRIGHT_CRAWLER_RESULTS_BUCKET"
+BOOTSTRAP_AUDIENCE_BASE_ENV = "MOSTLYRIGHT_CRAWLER_BOOTSTRAP_AUDIENCE_BASE"
+BOOTSTRAP_URL_BASE_ENV = "MOSTLYRIGHT_CRAWLER_BOOTSTRAP_URL_BASE"
+CLOUD_RUN_EXECUTION_ENV = "CLOUD_RUN_EXECUTION"
+RESULT_OBJECT_PREFIX = "crawler-quarantine/v2/"
+MAX_UPLOAD_CAPABILITY_BYTES = 32 * 1024
+MAX_BOOTSTRAP_RESPONSE_BYTES = MAX_CRAWLER_REQUEST_BYTES + MAX_UPLOAD_CAPABILITY_BYTES + 16 * 1024
+MAX_METADATA_IDENTITY_TOKEN_BYTES = 16 * 1024
+# The only failures a wrapper can know before Studio has supplied the sealed
+# request and single-use upload capability.  Keep this closed in lock-step with
+# Studio's ``PRE_BOOTSTRAP_FAILURE_CODES``; ordinary Courier failures travel in
+# the signed result bundle and must never acquire this control-plane path.
+PRE_BOOTSTRAP_FAILURE_CODES = frozenset(
+    {
+        "CRAWLER_BOOTSTRAP_REDEMPTION_FAILED",
+        "CRAWLER_BOOTSTRAP_RESPONSE_INVALID",
+        "CRAWLER_BOOTSTRAP_RESPONSE_TOO_LARGE",
+    }
+)
+# Sized for an EPA-scale year: the streamed decode, the streamed verification pass, and
+# the chunked result encode are each one pure-Python walk over ~200M cells, which a live
+# run spends 12-15 minutes on.  The old 600 was calibrated to decode alone and killed a
+# healthy child mid-verification the first time one survived that far.
+CHILD_TIMEOUT_SECONDS = 1800.0
+# Sized for EPA-scale archive decode under the raised Courier caps: the pure-Python
+# row materialization of a ~200M-cell year file peaks at a measured 24.5 GiB, so the
+# boundary is 32 GiB, equal to the Studio job's container limit by the exact-value check.
+READER_MEMORY_CGROUP_MAX_BYTES = 34_359_738_368
+UPLOAD_TIMEOUT_SECONDS = 600.0
+# The networkless child runs under a seccomp filter that denies clone/clone3 as
+# process-escape syscalls, so native libraries must not spawn worker threads.
+# numpy's BLAS spawns its pool at import and would be killed mid-import without
+# these pins; a single-threaded child is also the deterministic one. Arrow's
+# prefixed jemalloc additionally announces its denied background thread on
+# stderr, and the wrapper's contract treats any child stderr as failure, so
+# its background thread is disabled outright.
+CRAWLER_CHILD_THREAD_PINS = (
+    ("JE_ARROW_MALLOC_CONF", "background_thread:false"),
+    ("MKL_NUM_THREADS", "1"),
+    ("NUMEXPR_NUM_THREADS", "1"),
+    ("OMP_NUM_THREADS", "1"),
+    ("OPENBLAS_NUM_THREADS", "1"),
+)
+BOOTSTRAP_TIMEOUT_SECONDS = 10.0
+METADATA_IDENTITY_URL = (
+    "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/identity"
+)
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+_BUCKET = re.compile(r"^[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]$")
+_OBJECT_SUFFIX = re.compile(r"^[0-9a-f]{64}$")
+_JWT = re.compile(rb"^[A-Za-z0-9_-]{1,8192}\.[A-Za-z0-9_-]{1,8192}\.[A-Za-z0-9_-]{1,8192}$")
+_CLOUD_RUN_EXECUTION = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$")
+_TIMESTAMP = re.compile(
+    r"^[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])"
+    r"T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:\.[0-9]{1,6})?Z$"
+)
+_SIGNED_QUERY_FIELDS = {
+    "X-Goog-Algorithm",
+    "X-Goog-Credential",
+    "X-Goog-Date",
+    "X-Goog-Expires",
+    "X-Goog-Signature",
+    "X-Goog-SignedHeaders",
+}
+_SIGNED_HEADERS = ";".join(
+    sorted(
+        (
+            "content-type",
+            "host",
+            "x-goog-content-length-range",
+            "x-goog-content-sha256",
+            "x-goog-if-generation-match",
+            "x-goog-meta-mostlyright-crawler-session",
+            "x-goog-meta-mostlyright-egress-policy",
+            "x-goog-meta-mostlyright-request-digest",
+        )
+    )
+)
+
+
+class HostedCrawlerJobError(RuntimeError):
+    """Fail-closed wrapper error that never includes signed URLs or argv values."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{detail} [{code}]")
+
+
+@dataclass(frozen=True)
+class UploadCapability:
+    crawler_session_id: str
+    request_digest: str
+    upload_url: str
+    required_headers: tuple[tuple[str, str], ...]
+    expires_at: str
+
+
+@dataclass(frozen=True)
+class CrawlerBootstrap:
+    request_bytes: bytes
+    upload_capability: UploadCapability
+
+
+class CrawlerBootstrapTransport(Protocol):
+    """Minimal no-proxy, no-redirect transport used for one bootstrap redemption."""
+
+    def request(
+        self,
+        *,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        body: bytes | None,
+        maximum_response_bytes: int,
+    ) -> tuple[int, bytearray]: ...
+
+
+class _NoRedirect(HTTPRedirectHandler):
+    def redirect_request(  # type: ignore[override]
+        self,
+        req: Request,
+        fp: Any,
+        code: int,
+        msg: str,
+        headers: Any,
+        newurl: str,
+    ) -> None:
+        return None
+
+
+class UrlLibCrawlerBootstrapTransport:
+    """Bounded transport with ambient proxy use and redirects disabled."""
+
+    def __init__(self, *, timeout: float = BOOTSTRAP_TIMEOUT_SECONDS) -> None:
+        self._timeout = timeout
+        self._opener = build_opener(
+            ProxyHandler({}),
+            HTTPSHandler(context=ssl.create_default_context()),
+            _NoRedirect(),
+        )
+
+    def request(
+        self,
+        *,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        body: bytes | None,
+        maximum_response_bytes: int,
+    ) -> tuple[int, bytearray]:
+        request = Request(url, data=body, headers=headers, method=method)
+        try:
+            response = self._opener.open(request, timeout=self._timeout)
+        except HTTPError as error:
+            response = error
+        try:
+            content_length = response.headers.get("Content-Length")
+            if (
+                content_length is not None
+                and content_length.isdigit()
+                and int(content_length) > maximum_response_bytes
+            ):
+                _fail("CRAWLER_BOOTSTRAP_RESPONSE_TOO_LARGE", "bootstrap response is too large")
+            content = bytearray(response.read(maximum_response_bytes + 1))
+            if len(content) > maximum_response_bytes:
+                _zero(content)
+                _fail("CRAWLER_BOOTSTRAP_RESPONSE_TOO_LARGE", "bootstrap response is too large")
+            return int(response.status), content
+        finally:
+            response.close()
+
+
+def _fail(code: str, detail: str) -> NoReturn:
+    raise HostedCrawlerJobError(code, detail)
+
+
+def _zero(value: bytearray | None) -> None:
+    if value is not None:
+        value[:] = b"\x00" * len(value)
+
+
+def _canonical_timestamp(value: Any, label: str) -> str:
+    if not isinstance(value, str) or _TIMESTAMP.fullmatch(value) is None:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", f"{label} is not canonical UTC")
+    try:
+        parsed = datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
+    except ValueError:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", f"{label} is not a real UTC time")
+    if parsed.tzinfo is None or parsed.utcoffset() is None:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", f"{label} is not canonical UTC")
+    return value
+
+
+def _canonical_uuid(value: Any, label: str) -> str:
+    try:
+        parsed = UUID(value)
+    except (TypeError, ValueError, AttributeError):
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", f"{label} is not a canonical UUID")
+    if str(parsed) != value:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", f"{label} is not a canonical UUID")
+    return value
+
+
+def _bootstrap_base(value: Any, *, label: str) -> str:
+    if not isinstance(value, str) or not 1 <= len(value) <= 2048:
+        _fail("CRAWLER_BOOTSTRAP_CONFIGURATION_INVALID", f"{label} is unavailable")
+    parsed = urlsplit(value)
+    try:
+        port = parsed.port
+    except ValueError:
+        _fail("CRAWLER_BOOTSTRAP_CONFIGURATION_INVALID", f"{label} is invalid")
+    if (
+        parsed.scheme != "https"
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or port is not None
+        or parsed.query
+        or parsed.fragment
+        or not parsed.path.startswith("/")
+        or parsed.path.endswith("/")
+        or "%" in parsed.path
+        or "\\" in parsed.path
+        or "//" in parsed.path
+        or "/./" in parsed.path
+        or "/../" in parsed.path
+        or value != f"https://{parsed.hostname}{parsed.path}"
+    ):
+        _fail("CRAWLER_BOOTSTRAP_CONFIGURATION_INVALID", f"{label} is invalid")
+    return value
+
+
+def _crawler_session_id(value: Any) -> str:
+    try:
+        parsed = UUID(value)
+    except (TypeError, ValueError, AttributeError):
+        _fail("CRAWLER_JOB_ARGUMENT_INVALID", "crawler session identifier is invalid")
+    if str(parsed) != value:
+        _fail("CRAWLER_JOB_ARGUMENT_INVALID", "crawler session identifier is invalid")
+    return value
+
+
+def _cloud_run_execution(value: Any) -> str:
+    """Accept only Cloud Run's canonical injected execution-name token."""
+
+    if not isinstance(value, str) or _CLOUD_RUN_EXECUTION.fullmatch(value) is None:
+        _fail("CRAWLER_BOOTSTRAP_CONFIGURATION_INVALID", "crawler execution coordinate is invalid")
+    return value
+
+
+def _metadata_identity_token(
+    *,
+    audience: str,
+    transport: CrawlerBootstrapTransport,
+) -> bytearray:
+    metadata_url = f"{METADATA_IDENTITY_URL}?{urlencode({'audience': audience, 'format': 'full'})}"
+    try:
+        status, token = transport.request(
+            method="GET",
+            url=metadata_url,
+            headers={"Metadata-Flavor": "Google"},
+            body=None,
+            maximum_response_bytes=MAX_METADATA_IDENTITY_TOKEN_BYTES,
+        )
+    except HostedCrawlerJobError:
+        raise
+    except Exception:
+        _fail(
+            "CRAWLER_WORKLOAD_IDENTITY_UNAVAILABLE",
+            "crawler workload identity is unavailable",
+        )
+    if not isinstance(token, bytearray):
+        token = bytearray(token)
+    if status != 200 or _JWT.fullmatch(token) is None:
+        _zero(token)
+        _fail(
+            "CRAWLER_WORKLOAD_IDENTITY_UNAVAILABLE",
+            "crawler workload identity is unavailable",
+        )
+    return token
+
+
+def _redeem_crawler_bootstrap(
+    *,
+    crawler_session_id: str,
+    bucket: str,
+    transport: CrawlerBootstrapTransport | None = None,
+    now: datetime | None = None,
+) -> CrawlerBootstrap:
+    """Redeem one workload-bound bootstrap without putting authority on argv or env."""
+
+    try:
+        _disable_process_dumpability()
+    except OSError:
+        _fail(
+            "CRAWLER_PROCESS_HARDENING_UNAVAILABLE",
+            "crawler process hardening is unavailable",
+        )
+    session_id = _crawler_session_id(crawler_session_id)
+    audience_base = _bootstrap_base(
+        os.environ.get(BOOTSTRAP_AUDIENCE_BASE_ENV),
+        label="bootstrap audience base",
+    )
+    url_base = _bootstrap_base(
+        os.environ.get(BOOTSTRAP_URL_BASE_ENV),
+        label="bootstrap URL base",
+    )
+    audience = f"{audience_base}/{session_id}:redeem"
+    redemption_url = f"{url_base}/{session_id}:redeem"
+    cloud_run_execution = _cloud_run_execution(os.environ.get(CLOUD_RUN_EXECUTION_ENV))
+    client = transport or UrlLibCrawlerBootstrapTransport()
+    token: bytearray | None = None
+    response: bytearray | None = None
+    authorization_headers: dict[str, str] = {}
+    token_text = ""
+    try:
+        token = _metadata_identity_token(audience=audience, transport=client)
+        token_text = token.decode("ascii", errors="strict")
+        authorization_headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {token_text}",
+            "X-Mostlyright-Cloud-Run-Execution": cloud_run_execution,
+        }
+        try:
+            status, response = client.request(
+                method="POST",
+                url=redemption_url,
+                headers=authorization_headers,
+                body=None,
+                maximum_response_bytes=MAX_BOOTSTRAP_RESPONSE_BYTES,
+            )
+        except HostedCrawlerJobError:
+            raise
+        except Exception:
+            _fail(
+                "CRAWLER_BOOTSTRAP_REDEMPTION_FAILED",
+                "crawler bootstrap redemption failed",
+            )
+        if not isinstance(response, bytearray):
+            response = bytearray(response)
+        if status != 200 or not 1 <= len(response) <= MAX_BOOTSTRAP_RESPONSE_BYTES:
+            _fail(
+                "CRAWLER_BOOTSTRAP_REDEMPTION_FAILED",
+                "crawler bootstrap redemption failed",
+            )
+        value = parse_json(response)
+        expected_fields = {
+            "schema_version",
+            "crawler_session_id",
+            "request",
+            "upload_capability",
+        }
+        if (
+            not isinstance(value, dict)
+            or set(value) != expected_fields
+            or value["schema_version"] != BOOTSTRAP_RESPONSE_SCHEMA
+            or value["crawler_session_id"] != session_id
+        ):
+            _fail(
+                "CRAWLER_BOOTSTRAP_RESPONSE_INVALID",
+                "crawler bootstrap response fields or session binding are invalid",
+            )
+        request = parse_crawler_request(value["request"])
+        request_bytes = canonical_json_bytes(request.to_dict())
+        if not 1 <= len(request_bytes) <= MAX_CRAWLER_REQUEST_BYTES:
+            _fail(
+                "CRAWLER_BOOTSTRAP_RESPONSE_INVALID",
+                "crawler request exceeds its fixed bound",
+            )
+        capability = parse_upload_capability(
+            value["upload_capability"],
+            request_digest=request.digest,
+            egress_policy_attestation=request.egress_policy_attestation,
+            bucket=bucket,
+            now=now,
+        )
+        if capability.crawler_session_id != session_id:
+            _fail(
+                "CRAWLER_BOOTSTRAP_RESPONSE_INVALID",
+                "crawler upload capability session binding is invalid",
+            )
+        return CrawlerBootstrap(
+            request_bytes=request_bytes,
+            upload_capability=capability,
+        )
+    except (UnicodeDecodeError, CanonicalJSONError, HostedCrawlerProtocolError):
+        _fail(
+            "CRAWLER_BOOTSTRAP_RESPONSE_INVALID",
+            "crawler bootstrap response is invalid",
+        )
+    finally:
+        authorization_headers.clear()
+        token_text = ""
+        _zero(token)
+        _zero(response)
+
+
+def _report_crawler_bootstrap_failure(
+    *,
+    crawler_session_id: str,
+    failure_code: str,
+    transport: CrawlerBootstrapTransport | None = None,
+) -> bool:
+    """Best-effort one-way report for a failure before upload authority exists.
+
+    The failed job still exits non-zero if this report cannot be delivered.  It
+    obtains an independent workload token for the exact ``:fail`` route, sends
+    only an allowlisted code in canonical bytes, and accepts only an empty 204
+    acknowledgement.  No response diagnostic, request, or storage capability
+    can cross back into the crawler through this path.
+    """
+
+    if failure_code not in PRE_BOOTSTRAP_FAILURE_CODES:
+        return False
+    token: bytearray | None = None
+    response: bytearray | None = None
+    authorization_headers: dict[str, str] = {}
+    token_text = ""
+    try:
+        session_id = _crawler_session_id(crawler_session_id)
+        audience_base = _bootstrap_base(
+            os.environ.get(BOOTSTRAP_AUDIENCE_BASE_ENV),
+            label="bootstrap audience base",
+        )
+        url_base = _bootstrap_base(
+            os.environ.get(BOOTSTRAP_URL_BASE_ENV),
+            label="bootstrap URL base",
+        )
+        audience = f"{audience_base}/{session_id}:fail"
+        failure_url = f"{url_base}/{session_id}:fail"
+        report = canonical_json_bytes({"failure_code": failure_code})
+        client = transport or UrlLibCrawlerBootstrapTransport()
+        token = _metadata_identity_token(audience=audience, transport=client)
+        token_text = token.decode("ascii", errors="strict")
+        authorization_headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {token_text}",
+            "Content-Type": "application/json",
+        }
+        status, response = client.request(
+            method="POST",
+            url=failure_url,
+            headers=authorization_headers,
+            body=report,
+            maximum_response_bytes=0,
+        )
+        if not isinstance(response, bytearray):
+            response = bytearray(response)
+        return status == 204 and not response
+    except (HostedCrawlerJobError, UnicodeDecodeError, CanonicalJSONError, ValueError):
+        return False
+    except Exception:
+        return False
+    finally:
+        authorization_headers.clear()
+        token_text = ""
+        _zero(token)
+        _zero(response)
+
+
+def _required_headers(
+    value: Any,
+    *,
+    crawler_session_id: str,
+    request_digest: str,
+    egress_policy_attestation: str,
+) -> tuple[tuple[str, str], ...]:
+    expected = (
+        ("Content-Type", UPLOAD_MEDIA_TYPE),
+        ("x-goog-content-length-range", f"1,{MAX_CRAWLER_RESULT_BYTES}"),
+        ("x-goog-content-sha256", "UNSIGNED-PAYLOAD"),
+        ("x-goog-if-generation-match", "0"),
+        ("x-goog-meta-mostlyright-crawler-session", crawler_session_id),
+        ("x-goog-meta-mostlyright-request-digest", request_digest),
+        ("x-goog-meta-mostlyright-egress-policy", egress_policy_attestation),
+    )
+    if not isinstance(value, list) or len(value) != len(expected):
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "required upload header inventory is not exact",
+        )
+    parsed: list[tuple[str, str]] = []
+    for item in value:
+        if not isinstance(item, dict) or set(item) != {"name", "value"}:
+            _fail(
+                "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+                "required upload header entry is not exact",
+            )
+        name = item["name"]
+        header_value = item["value"]
+        if (
+            not isinstance(name, str)
+            or not isinstance(header_value, str)
+            or "\r" in name
+            or "\n" in name
+            or "\r" in header_value
+            or "\n" in header_value
+        ):
+            _fail(
+                "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+                "required upload header entry is invalid",
+            )
+        parsed.append((name, header_value))
+    if tuple(parsed) != expected:
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "required upload headers differ from the fixed authority binding",
+        )
+    return tuple(parsed)
+
+
+def _validate_upload_url(value: Any, *, bucket: str) -> str:
+    if not isinstance(value, str) or not 1 <= len(value) <= 16 * 1024:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", "upload URL is invalid")
+    parsed = urlsplit(value)
+    try:
+        port = parsed.port
+    except ValueError:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", "upload URL port is invalid")
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname != "storage.googleapis.com"
+        or parsed.username is not None
+        or parsed.password is not None
+        or port is not None
+        or parsed.fragment
+    ):
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "upload URL is outside the exact GCS authority",
+        )
+    path_prefix = f"/{bucket}/{RESULT_OBJECT_PREFIX}"
+    if (
+        not parsed.path.startswith(path_prefix)
+        or parsed.path.count("/") != path_prefix.count("/")
+        or _OBJECT_SUFFIX.fullmatch(parsed.path.removeprefix(path_prefix)) is None
+        or "%" in parsed.path
+        or "\\" in parsed.path
+        or "//" in parsed.path
+        or "/./" in parsed.path
+        or "/../" in parsed.path
+    ):
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "upload URL object path is outside the quarantine prefix",
+        )
+    try:
+        query = parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True)
+    except ValueError:
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "upload URL signed query inventory is invalid",
+        )
+    if (
+        len(query) != len(_SIGNED_QUERY_FIELDS)
+        or {name for name, _ in query} != _SIGNED_QUERY_FIELDS
+        or any(not item for _, item in query)
+    ):
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "upload URL signed query inventory is not exact",
+        )
+    values = dict(query)
+    if (
+        values["X-Goog-Algorithm"] != "GOOG4-RSA-SHA256"
+        or values["X-Goog-SignedHeaders"] != _SIGNED_HEADERS
+        or not values["X-Goog-Expires"].isdigit()
+        # Bounded by the Studio upload window, which covers the child's whole execution
+        # budget rather than an interactive session: the capability has to stay
+        # presentable through a 15-20 minute year-scale crawl.
+        or not 1 <= int(values["X-Goog-Expires"]) <= 2100
+        or re.fullmatch(
+            r"(?:[0-9a-f]{2}){64,512}",
+            values["X-Goog-Signature"],
+        )
+        is None
+    ):
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "upload URL signature parameters are outside the fixed policy",
+        )
+    return value
+
+
+def parse_upload_capability(
+    value: Any,
+    *,
+    request_digest: str,
+    egress_policy_attestation: str,
+    bucket: str,
+    now: datetime | None = None,
+) -> UploadCapability:
+    fields = {
+        "schema_version",
+        "crawler_session_id",
+        "request_digest",
+        "method",
+        "upload_url",
+        "required_headers",
+        "max_result_bytes",
+        "expires_at",
+        "redirect_policy",
+        "single_use",
+    }
+    if not isinstance(value, dict) or set(value) != fields:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_INVALID", "upload capability fields are not exact")
+    if (
+        value["schema_version"] != UPLOAD_CAPABILITY_SCHEMA
+        or value["method"] != "PUT"
+        or type(value["max_result_bytes"]) is not int
+        or value["max_result_bytes"] != MAX_CRAWLER_RESULT_BYTES
+        or value["redirect_policy"] != "deny"
+        or value["single_use"] is not True
+        or not isinstance(value["request_digest"], str)
+        or _DIGEST.fullmatch(value["request_digest"]) is None
+        or value["request_digest"] != request_digest
+    ):
+        _fail(
+            "CRAWLER_UPLOAD_CAPABILITY_INVALID",
+            "upload capability constants or request binding mismatch",
+        )
+    crawler_session_id = _canonical_uuid(
+        value["crawler_session_id"],
+        "crawler_session_id",
+    )
+    expires_at = _canonical_timestamp(value["expires_at"], "expires_at")
+    expiry = datetime.fromisoformat(expires_at.removesuffix("Z") + "+00:00")
+    current = now or datetime.now(UTC)
+    if current.tzinfo is None or current.astimezone(UTC) >= expiry:
+        _fail("CRAWLER_UPLOAD_CAPABILITY_EXPIRED", "upload capability has expired")
+    upload_url = _validate_upload_url(value["upload_url"], bucket=bucket)
+    headers = _required_headers(
+        value["required_headers"],
+        crawler_session_id=crawler_session_id,
+        request_digest=request_digest,
+        egress_policy_attestation=egress_policy_attestation,
+    )
+    return UploadCapability(
+        crawler_session_id=crawler_session_id,
+        request_digest=request_digest,
+        upload_url=upload_url,
+        required_headers=headers,
+        expires_at=expires_at,
+    )
+
+
+def _run_crawler_child(
+    request: HostedCrawlerRequest,
+    normalization_input: bytes,
+) -> bytes:
+    crawler = shutil.which("mr-data-crawler")
+    if crawler is None or not os.path.isabs(crawler):
+        _fail("CRAWLER_EXECUTABLE_UNAVAILABLE", "fixed crawler executable is unavailable")
+    completed = subprocess.run(
+        [crawler, "--input-fd", "0", "--result-fd", "1"],
+        input=normalization_input,
+        capture_output=True,
+        env={
+            CRAWLER_EGRESS_POLICY_ENV: request.egress_policy_attestation,
+            **dict(CRAWLER_CHILD_THREAD_PINS),
+        },
+        close_fds=True,
+        preexec_fn=_install_networkless_seccomp,
+        check=False,
+        timeout=CHILD_TIMEOUT_SECONDS,
+    )
+    if completed.returncode != 0:
+        _fail(_child_failure_code(completed.stderr), "fixed crawler execution failed")
+    if completed.stderr or not 1 <= len(completed.stdout) <= MAX_CRAWLER_RESULT_BYTES:
+        _fail("CRAWLER_CHILD_INVALID", "fixed crawler output is invalid")
+    return completed.stdout
+
+
+def _run_crawler_warm_up(request: HostedCrawlerRequest) -> None:
+    """Run the mandatory known-sample check before any public source byte is fetched."""
+
+    crawler = shutil.which("mr-data-crawler")
+    if crawler is None or not os.path.isabs(crawler):
+        _fail("CRAWLER_EXECUTABLE_UNAVAILABLE", "fixed crawler executable is unavailable")
+    completed = subprocess.run(
+        [crawler, "--warm-up", "--input-fd", "0", "--result-fd", "1"],
+        input=canonical_json_bytes(request.to_dict()),
+        capture_output=True,
+        env={
+            CRAWLER_EGRESS_POLICY_ENV: request.egress_policy_attestation,
+            **dict(CRAWLER_CHILD_THREAD_PINS),
+        },
+        close_fds=True,
+        preexec_fn=_install_networkless_seccomp,
+        check=False,
+        timeout=CHILD_TIMEOUT_SECONDS,
+    )
+    if completed.returncode != 0:
+        code = _child_failure_code(completed.stderr)
+        _fail(code if code == "READER_WARMUP" else "READER_WARMUP", "hosted Reader warm-up failed")
+    try:
+        result = parse_canonical_json(completed.stdout)
+    except (CanonicalJSONError, TypeError, ValueError):
+        _fail("READER_WARMUP", "hosted Reader warm-up result is invalid")
+    if completed.stderr or result != {
+        "schema_version": WARM_UP_RESULT_SCHEMA,
+        "request_digest": request.digest,
+        "status": "passed",
+    }:
+        _fail("READER_WARMUP", "hosted Reader warm-up result is invalid")
+
+
+def _require_reader_memory_boundary() -> None:
+    """Bind hosted Reader execution to the immutable Cloud Run cgroup ceiling.
+
+    Cloud Run's job sandbox mounts a hybrid cgroup layout: the kernel enforces the
+    container memory limit through the v1 controller at memory/memory.limit_in_bytes,
+    and the v2 unified file memory.max is absent. Both spellings name the same
+    kernel-enforced ceiling, so either satisfies the boundary; the required value is
+    exact in both cases.
+    """
+
+    value = None
+    for candidate in (
+        Path("/sys/fs/cgroup/memory.max"),
+        Path("/sys/fs/cgroup/memory/memory.limit_in_bytes"),
+    ):
+        try:
+            value = candidate.read_text(encoding="ascii").strip()
+        except (OSError, UnicodeError):
+            continue
+        break
+    if value is None:
+        _fail("CRAWLER_MEMORY_BOUNDARY", "hosted Reader memory cgroup is unavailable")
+    if value != str(READER_MEMORY_CGROUP_MAX_BYTES):
+        _fail("CRAWLER_MEMORY_BOUNDARY", "hosted Reader memory cgroup is not exact")
+
+
+def _child_failure_code(stderr: bytes) -> str:
+    """Admit only a closed stable child code; never propagate child detail or source text."""
+
+    if not 1 <= len(stderr) <= 512:
+        return "CRAWLER_CHILD_FAILED"
+    try:
+        value = parse_canonical_json(stderr.strip())
+    except (CanonicalJSONError, TypeError, ValueError):
+        return "CRAWLER_CHILD_FAILED"
+    if (
+        isinstance(value, dict)
+        and set(value) == {"status", "code"}
+        and value.get("status") == "failed"
+        and value.get("code") in HOSTED_CRAWLER_FAILURE_CODES
+    ):
+        return str(value["code"])
+    return "CRAWLER_CHILD_FAILED"
+
+
+def _validate_result_against_retrieval(
+    *,
+    request: HostedCrawlerRequest,
+    retrieved: Any,
+    result: Any,
+) -> None:
+    """Bind the crawler child's result to the exact bytes and evidence fetched by this job."""
+
+    if (request.adapter_id, request.adapter_version) == ("public.https", "1.0.0"):
+        _validate_public_https_result(request=request, retrieved=retrieved, result=result)
+        return
+    if (request.adapter_id, request.adapter_version) != ("external.openligadb", "1.0.0"):
+        _fail(
+            "CRAWLER_CHILD_INVALID",
+            "crawler result uses an unsupported adapter coordinate",
+        )
+
+    receipt = result.acquisition_receipt
+    observation = result.observation
+    if not isinstance(receipt, dict) or not isinstance(observation, dict):
+        _fail(
+            "CRAWLER_CHILD_INVALID",
+            "crawler result does not bind the trusted retrieval evidence",
+        )
+    receipt_fields = {
+        "schema_version",
+        "source_id",
+        "adapter_id",
+        "adapter_version",
+        "request_digest",
+        "query",
+        "source_url",
+        "final_url",
+        "media_type",
+        "acquired_at",
+        "raw_content_sha256",
+        "raw_size_bytes",
+        "normalized_content_sha256",
+        "normalized_size_bytes",
+        "row_count",
+        "column_names",
+        "transport_evidence_digest",
+        "egress_policy_attestation",
+    }
+    observation_fields = {
+        "schema_version",
+        "source_id",
+        "adapter_id",
+        "adapter_version",
+        "request_digest",
+        "observed_at",
+        "available_at",
+        "historical_start",
+        "historical_end",
+        "event_time_field",
+        "live_status",
+        "raw_content_sha256",
+        "normalized_content_sha256",
+        "row_count",
+    }
+    normalized_digest = sha256_bytes(result.normalized_content)
+    identity = (
+        receipt.get("source_id"),
+        receipt.get("adapter_id"),
+        receipt.get("adapter_version"),
+        receipt.get("request_digest"),
+    )
+    expected_identity = (
+        request.source_id,
+        request.adapter_id,
+        request.adapter_version,
+        request.digest,
+    )
+    if (
+        set(receipt) != receipt_fields
+        or set(observation) != observation_fields
+        or identity != expected_identity
+        or (
+            observation.get("source_id"),
+            observation.get("adapter_id"),
+            observation.get("adapter_version"),
+            observation.get("request_digest"),
+        )
+        != expected_identity
+        or receipt.get("query") != request.query
+        or receipt.get("source_url") != retrieved.source_url
+        or receipt.get("final_url") != retrieved.final_url
+        or receipt.get("media_type") != retrieved.media_type
+        or receipt.get("acquired_at") != request.requested_at
+        or receipt.get("raw_content_sha256") != retrieved.content_sha256
+        or receipt.get("raw_size_bytes") != len(retrieved.content)
+        or receipt.get("transport_evidence_digest") != retrieved.transport_evidence_digest
+        or receipt.get("egress_policy_attestation") != request.egress_policy_attestation
+        or receipt.get("normalized_content_sha256") != normalized_digest
+        or receipt.get("normalized_size_bytes") != len(result.normalized_content)
+        or observation.get("observed_at") != request.requested_at
+        or observation.get("available_at") != request.requested_at
+        or observation.get("event_time_field") != "match_datetime_utc"
+        or observation.get("live_status") != "live"
+        or observation.get("raw_content_sha256") != retrieved.content_sha256
+        or observation.get("normalized_content_sha256") != normalized_digest
+        or observation.get("row_count") != receipt.get("row_count")
+    ):
+        _fail(
+            "CRAWLER_CHILD_INVALID",
+            "crawler result does not bind the trusted retrieval evidence",
+        )
+
+
+def _validate_public_https_result(
+    *,
+    request: HostedCrawlerRequest,
+    retrieved: Any,
+    result: Any,
+) -> None:
+    """Bind the public-HTTPS normalizer result to the trusted wrapper retrieval."""
+
+    receipt = result.acquisition_receipt
+    observation = result.observation
+    enhanced_cadence_evidence = request.schema_version == CRAWLER_REQUEST_SCHEMA_V3
+    receipt_fields = {
+        "schema_version",
+        "source_id",
+        "adapter_id",
+        "adapter_version",
+        "request_digest",
+        "source_url",
+        "final_url",
+        "fetched_media_type",
+        "normalized_media_type",
+        "normalized_data_format",
+        "normalized_filename",
+        "acquired_at",
+        "raw_content_sha256",
+        "raw_size_bytes",
+        "normalized_content_sha256",
+        "normalized_size_bytes",
+        "row_count",
+        "column_names",
+        "parsed_schema_digest",
+        "family_id",
+        "family_version",
+        "decode_options_digest",
+        "decode_flags",
+        "resource_caps",
+        "reader_budgets",
+        "transport_evidence_digest",
+        "egress_policy_attestation",
+        "clean_room",
+    }
+    if enhanced_cadence_evidence:
+        receipt_fields |= {
+            "row_digest",
+            "http_status_code",
+            "etag_digest",
+            "last_modified_digest",
+        }
+    observation_fields = {
+        "schema_version",
+        "source_id",
+        "request_digest",
+        "observed_at",
+        "raw_content_sha256",
+        "normalized_content_sha256",
+        "row_count",
+    }
+    if enhanced_cadence_evidence:
+        observation_fields |= {
+            "row_digest",
+            "http_status_code",
+            "etag_digest",
+            "last_modified_digest",
+        }
+    if (
+        not isinstance(receipt, dict)
+        or set(receipt) != receipt_fields
+        or not isinstance(observation, dict)
+        or set(observation) != observation_fields
+    ):
+        _fail(
+            "CRAWLER_CHILD_INVALID",
+            "public HTTPS result evidence fields are not exact",
+        )
+
+    query = request.query
+    reader = query.get("reader_pin")
+    if reader is None:
+        reader_identity = (None, None, None)
+        expected_resource_caps = None
+        expected_reader_budgets = None
+        direct_identity = (
+            query.get("data_format"),
+            query.get("filename"),
+            retrieved.media_type,
+        )
+    else:
+        from mostlyright.data_harness.readers.contracts import ReaderPin
+
+        try:
+            pin = ReaderPin(
+                reader["family_id"],
+                reader["family_version"],
+                reader["decode_options"],
+            )
+        except (KeyError, TypeError, ValueError):
+            _fail("CRAWLER_CHILD_INVALID", "public HTTPS Reader identity is invalid")
+        reader_identity = (pin.family_id, pin.family_version, pin.options_digest)
+        resource_caps = query.get("resource_caps")
+        try:
+            from mostlyright.data_harness.readers.registry import TOOLBOX
+
+            family = TOOLBOX.resolve(pin.family_id, pin.family_version)
+            expected_reader_budgets = asdict(
+                family.default_budgets.narrowed_by(
+                    {
+                        "max_input_bytes": query["limits"]["max_source_bytes"],
+                        "max_output_bytes": query["limits"]["max_normalized_bytes"],
+                        "max_rows": query["limits"]["max_rows"],
+                        "max_columns": query["limits"]["max_columns"],
+                        **(resource_caps or {}),
+                    }
+                )
+            )
+        except (KeyError, TypeError, ValueError):
+            _fail("CRAWLER_CHILD_INVALID", "public HTTPS Reader budgets are invalid")
+        expected_resource_caps = resource_caps
+        direct_identity = (
+            receipt.get("normalized_data_format"),
+            receipt.get("normalized_filename"),
+            receipt.get("normalized_media_type"),
+        )
+
+    normalized_digest = sha256_bytes(result.normalized_content)
+    terminal = retrieved.hops[-1]
+    expected_etag_digest = (
+        None if terminal.etag is None else sha256_bytes(terminal.etag.encode("utf-8"))
+    )
+    expected_last_modified_digest = (
+        None
+        if terminal.last_modified is None
+        else sha256_bytes(terminal.last_modified.encode("utf-8"))
+    )
+    row_count = receipt.get("row_count")
+    column_names = receipt.get("column_names")
+    limits = query.get("limits", {})
+    expected_identity = (
+        request.source_id,
+        request.adapter_id,
+        request.adapter_version,
+        request.digest,
+    )
+    if (
+        (
+            receipt.get("source_id"),
+            receipt.get("adapter_id"),
+            receipt.get("adapter_version"),
+            receipt.get("request_digest"),
+        )
+        != expected_identity
+        or receipt.get("schema_version")
+        != (
+            "hosted-public-https-acquisition-receipt.v3"
+            if enhanced_cadence_evidence
+            else "hosted-public-https-acquisition-receipt.v1"
+        )
+        or observation.get("schema_version")
+        != (
+            "hosted-public-https-source-observation.v3"
+            if enhanced_cadence_evidence
+            else "hosted-public-https-source-observation.v1"
+        )
+        or result.schema_version
+        != (CRAWLER_RESULT_SCHEMA_V3 if enhanced_cadence_evidence else CRAWLER_RESULT_SCHEMA_V1)
+        or observation.get("source_id") != request.source_id
+        or observation.get("request_digest") != request.digest
+        or receipt.get("source_url") != query.get("url")
+        or receipt.get("source_url") != retrieved.source_url
+        or receipt.get("final_url") != retrieved.final_url
+        or receipt.get("fetched_media_type") != retrieved.media_type
+        or receipt.get("acquired_at") != request.requested_at
+        or receipt.get("raw_content_sha256") != retrieved.content_sha256
+        or receipt.get("raw_size_bytes") != len(retrieved.content)
+        or receipt.get("transport_evidence_digest") != retrieved.transport_evidence_digest
+        or receipt.get("egress_policy_attestation") != request.egress_policy_attestation
+        or receipt.get("normalized_content_sha256") != normalized_digest
+        or receipt.get("normalized_size_bytes") != len(result.normalized_content)
+        or (
+            receipt.get("family_id"),
+            receipt.get("family_version"),
+            receipt.get("decode_options_digest"),
+        )
+        != reader_identity
+        or (
+            receipt.get("normalized_data_format"),
+            receipt.get("normalized_filename"),
+            receipt.get("normalized_media_type"),
+        )
+        != direct_identity
+        or not isinstance(receipt.get("decode_flags"), list)
+        or receipt.get("resource_caps") != expected_resource_caps
+        or receipt.get("reader_budgets") != expected_reader_budgets
+        or (reader is None and receipt.get("decode_flags") != [])
+        or type(row_count) is not int
+        or not 0 <= row_count <= limits.get("max_rows", -1)
+        or not isinstance(column_names, list)
+        or not 1 <= len(column_names) <= limits.get("max_columns", -1)
+        or any(not isinstance(name, str) or not name for name in column_names)
+        or not isinstance(receipt.get("parsed_schema_digest"), str)
+        or _DIGEST.fullmatch(receipt["parsed_schema_digest"]) is None
+        or receipt.get("clean_room")
+        != {
+            "host_platform": "linux-amd64",
+            "parser_network_syscalls": "denied",
+            "parser_process_creation": "denied",
+            "no_new_privs": True,
+        }
+        or observation.get("observed_at") != request.requested_at
+        or observation.get("raw_content_sha256") != retrieved.content_sha256
+        or observation.get("normalized_content_sha256") != normalized_digest
+        or observation.get("row_count") != row_count
+        or (
+            enhanced_cadence_evidence
+            and (
+                receipt.get("http_status_code") != terminal.status
+                or receipt.get("etag_digest") != expected_etag_digest
+                or receipt.get("last_modified_digest") != expected_last_modified_digest
+                or not isinstance(receipt.get("row_digest"), str)
+                or _DIGEST.fullmatch(receipt["row_digest"]) is None
+                or observation.get("row_digest") != receipt["row_digest"]
+                or observation.get("http_status_code") != terminal.status
+                or observation.get("etag_digest") != expected_etag_digest
+                or observation.get("last_modified_digest") != expected_last_modified_digest
+            )
+        )
+        or result.next_watermark is not None
+    ):
+        _fail(
+            "CRAWLER_CHILD_INVALID",
+            "public HTTPS result does not bind the trusted retrieval evidence",
+        )
+
+
+def _upload_result(capability: UploadCapability, content: bytes) -> None:
+    parsed = urlsplit(capability.upload_url)
+    target = parsed.path + "?" + parsed.query
+    connection = http.client.HTTPSConnection(
+        "storage.googleapis.com",
+        443,
+        timeout=UPLOAD_TIMEOUT_SECONDS,
+        context=ssl.create_default_context(),
+    )
+    try:
+        connection.putrequest("PUT", target, skip_host=False, skip_accept_encoding=True)
+        connection.putheader("Content-Length", str(len(content)))
+        for name, value in capability.required_headers:
+            connection.putheader(name, value)
+        connection.endheaders(content)
+        response = connection.getresponse()
+        response_body = response.read(64 * 1024 + 1)
+        if response.status not in {200, 201} or len(response_body) > 64 * 1024:
+            _fail("CRAWLER_UPLOAD_FAILED", "quarantine result upload failed")
+    except (OSError, ssl.SSLError, http.client.HTTPException):
+        _fail("CRAWLER_UPLOAD_FAILED", "quarantine result upload failed")
+    finally:
+        connection.close()
+
+
+def _scrub_argv(argv: list[str] | None) -> None:
+    target = sys.argv if argv is None else argv
+    for index in range(1, len(target)):
+        target[index] = "[redacted]"
+
+
+def crawler_job_main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="mr-data-crawler-job")
+    parser.add_argument("--crawler-session-id", required=True)
+    arguments = parser.parse_args(argv)
+    crawler_session_id = arguments.crawler_session_id
+    arguments.crawler_session_id = "[redacted]"
+    _scrub_argv(argv)
+    bootstrap: CrawlerBootstrap | None = None
+    request: HostedCrawlerRequest | None = None
+    try:
+        bucket = os.environ.get(RESULTS_BUCKET_ENV)
+        if not isinstance(bucket, str) or _BUCKET.fullmatch(bucket) is None:
+            _fail(
+                "CRAWLER_RESULTS_BUCKET_REQUIRED",
+                "crawler quarantine bucket pin is unavailable",
+            )
+        bootstrap = _redeem_crawler_bootstrap(
+            crawler_session_id=crawler_session_id,
+            bucket=bucket,
+        )
+        request = parse_crawler_request(parse_canonical_json(bootstrap.request_bytes))
+        if request.adapter_id == "public.https" and request.query.get("reader_pin") is not None:
+            _require_reader_memory_boundary()
+            _run_crawler_warm_up(request)
+        retrieved = fetch_crawler_source(
+            request,
+            expected_egress_policy_attestation=request.egress_policy_attestation,
+        )
+        normalization_input = encode_normalization_input(request, retrieved)
+        result_bytes = _run_crawler_child(request, normalization_input)
+        result = parse_crawler_result(parse_canonical_json(result_bytes))
+        # Constructed for its __post_init__ check, not for the value: the result must
+        # carry this request's digest, source id, adapter id, and adapter version.
+        HostedCrawlerExchange(request=request, result=result)
+        _validate_result_against_retrieval(
+            request=request,
+            retrieved=retrieved,
+            result=result,
+        )
+        if canonical_json_bytes(result.to_dict()) != result_bytes:
+            _fail("CRAWLER_CHILD_INVALID", "crawler result bytes are not exact canonical JSON")
+        _upload_result(bootstrap.upload_capability, result_bytes)
+        return 0
+    except (
+        CanonicalJSONError,
+        HostedCrawlerJobError,
+        HostedCrawlerProtocolError,
+        TypeError,
+        ValueError,
+        subprocess.SubprocessError,
+    ) as error:
+        code = getattr(error, "code", "CRAWLER_JOB_FAILED")
+        if bootstrap is None and code in PRE_BOOTSTRAP_FAILURE_CODES:
+            # There is no upload capability before a successful bootstrap, so
+            # this narrow redacted report is the only way Studio can release
+            # capacity and expose the terminal cause before clients time out.
+            _report_crawler_bootstrap_failure(
+                crawler_session_id=crawler_session_id,
+                failure_code=code,
+            )
+        if bootstrap is not None and request is not None and code in HOSTED_CRAWLER_FAILURE_CODES:
+            failure = HostedCrawlerFailure(
+                request_digest=request.digest,
+                source_id=request.source_id,
+                adapter_id=request.adapter_id,
+                adapter_version=request.adapter_version,
+                failure_code=code,
+            )
+            try:
+                _upload_result(
+                    bootstrap.upload_capability,
+                    canonical_json_bytes(failure.to_dict()),
+                )
+            except HostedCrawlerJobError:
+                pass
+            else:
+                sys.stderr.write(
+                    canonical_json_bytes({"status": "reported", "code": code}).decode()
+                )
+                sys.stderr.write("\n")
+                return 0
+        sys.stderr.write(canonical_json_bytes({"status": "failed", "code": code}).decode())
+        sys.stderr.write("\n")
+        return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(crawler_job_main())