mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_bootstrap.py

@@ -0,0 +1,972 @@
+"""Workload-identity bootstrap for hosted producer and verifier workers.
+
+The launcher is the only process that sees the ambient workload identity response. It exchanges
+that identity for one attempt-scoped Studio bootstrap envelope, validates the complete envelope,
+places each worker input in a separate anonymous descriptor, sanitizes descriptor inheritance and
+the environment, and then replaces itself with the fixed worker entrypoint.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import re
+import ssl
+import stat
+import sys
+import tempfile
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, NoReturn, Protocol
+from urllib.error import HTTPError
+from urllib.parse import urlencode, urlsplit
+from urllib.request import (
+    HTTPRedirectHandler,
+    HTTPSHandler,
+    ProxyHandler,
+    Request,
+    build_opener,
+)
+from uuid import UUID
+
+from mostlyright.data_harness import pipeline
+from mostlyright.data_harness.canonical import canonical_json_bytes
+from mostlyright.data_harness.governors import (
+    CapExceeded,
+    CapLedger,
+    CapStoreError,
+    FileCapStore,
+    Reservation,
+)
+from mostlyright.data_harness.hosted_worker import (
+    MAX_CAPABILITY_BYTES,
+    MAX_DESCRIPTOR_BYTES,
+    MAX_SIGNING_KEY_BYTES,
+    HostedWorkerError,
+    _strict_json,
+    parse_capability,
+    parse_producer_job,
+    parse_signing_key,
+    parse_verifier_job,
+)
+from mostlyright.data_harness.local_contracts import GraphTablePlan
+
+# The workers have accepted V3 jobs since the Dataset/Table cutover.  Bootstrap must speak the
+# matching envelope too: accepting a V2 response here would otherwise let a legacy coordinator
+# re-introduce the child-only identity that the job parser correctly rejects.
+BOOTSTRAP_REQUEST_SCHEMA = "mostlyright-worker-bootstrap-request.v3"
+BOOTSTRAP_RESPONSE_SCHEMA = "mostlyright-worker-bootstrap-response.v3"
+BOOTSTRAP_EXCHANGE_PATH = "/_internal/worker-bootstrap:exchange"
+METADATA_IDENTITY_URL = (
+    "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/identity"
+)
+MAX_IDENTITY_BYTES = 16 * 1024
+MAX_BOOTSTRAP_RESPONSE_BYTES = 768 * 1024
+BOOTSTRAP_TIMEOUT_SECONDS = 10.0
+WORKER_BIN_DIRECTORY = Path("/opt/worker/bin")
+_JWT = re.compile(r"^[A-Za-z0-9_-]{1,8192}\.[A-Za-z0-9_-]{1,8192}\.[A-Za-z0-9_-]{1,8192}$")
+_DISPATCH_QUERY = re.compile(
+    r"^dispatch_id=([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$"
+)
+_POLICY_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
+_IMAGE_DIGEST = re.compile(r"^[a-z0-9][a-z0-9._/-]{0,254}@sha256:[0-9a-f]{64}$")
+_POLICY_SELECTION_ENV = "MOSTLYRIGHT_WORKER_POLICY_SELECTION_JSON"
+_POLICY_BINDING_SCHEMA = "mostlyright-worker-policy-binding.v2"
+
+#: Where the durable cap counters live. See :func:`admission_ledger` for what setting it and
+#: leaving it unset each actually enforce.
+CAP_STORE_ENV = "MOSTLYRIGHT_CAP_STORE"
+
+#: What this deployment declares one run costs, in cost units (one cent each). See
+#: :func:`declared_run_cost_units`: it is the operator's worst-case budget for a run, not a
+#: measurement, and leaving it unset means the monthly spend ceiling cannot trip.
+RUN_COST_UNITS_ENV = "MOSTLYRIGHT_RUN_COST_UNITS"
+
+
+class BootstrapTransport(Protocol):
+    """Minimal redirect-free, proxy-free bootstrap transport."""
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str],
+        body: bytes | None,
+        maximum: int,
+    ) -> tuple[int, bytes]: ...
+
+
+class _NoRedirect(HTTPRedirectHandler):
+    def redirect_request(
+        self,
+        request: Request,
+        file_pointer: Any,
+        code: int,
+        message: str,
+        headers: Any,
+        new_url: str,
+    ) -> None:
+        return None
+
+
+class UrlLibBootstrapTransport:
+    """Small stdlib transport with ambient proxies and redirects disabled."""
+
+    def __init__(self) -> None:
+        self._opener = build_opener(
+            ProxyHandler({}),
+            HTTPSHandler(context=ssl.create_default_context()),
+            _NoRedirect(),
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str],
+        body: bytes | None,
+        maximum: int,
+    ) -> tuple[int, bytes]:
+        request = Request(url, data=body, headers=dict(headers), method=method)
+        try:
+            response = self._opener.open(request, timeout=BOOTSTRAP_TIMEOUT_SECONDS)
+        except HTTPError as error:
+            response = error
+        with response:
+            status = int(response.status)
+            content_length = response.headers.get("Content-Length")
+            if content_length is not None:
+                try:
+                    declared = int(content_length)
+                except ValueError as error:
+                    raise HostedWorkerError(
+                        "BOOTSTRAP_TRANSPORT_INVALID",
+                        "bootstrap response has an invalid content length",
+                    ) from error
+                if declared < 0 or declared > maximum:
+                    raise HostedWorkerError(
+                        "BOOTSTRAP_TOO_LARGE",
+                        "bootstrap response exceeds its byte limit",
+                    )
+            raw = response.read(maximum + 1)
+        if len(raw) > maximum:
+            raise HostedWorkerError(
+                "BOOTSTRAP_TOO_LARGE",
+                "bootstrap response exceeds its byte limit",
+            )
+        return status, raw
+
+
+@dataclass(frozen=True)
+class BootstrapConfig:
+    exchange_url: str
+    audience: str
+    role: str
+    worker_policy_binding: WorkerPolicyBinding | None = None
+
+
+def _policy_digest(value: Any, label: str) -> str:
+    if (
+        not isinstance(value, str)
+        or not _POLICY_DIGEST.fullmatch(value)
+        or value == "sha256:" + "0" * 64
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            f"{label} is not a SHA-256 digest",
+        )
+    return value
+
+
+def _image_digest(value: Any, label: str) -> str:
+    if (
+        not isinstance(value, str)
+        or not _IMAGE_DIGEST.fullmatch(value)
+        or value.endswith("@sha256:" + "0" * 64)
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            f"{label} is not immutable",
+        )
+    return value
+
+
+def parse_worker_policy_binding(value: Any) -> WorkerPolicyBinding:
+    """Parse the exact deploy-owned V2 policy-set binding and recompute both digests."""
+
+    expected = {
+        "schema_version",
+        "producer_image_digest",
+        "verifier_image_digest",
+        "validation_policy_digests",
+        "policy_set_digest",
+        "release_revision_coordinate",
+        "selection_id",
+    }
+    if not isinstance(value, dict) or set(value) != expected:
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            "worker policy binding fields are not exact",
+        )
+    if value["schema_version"] != _POLICY_BINDING_SCHEMA:
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            "worker policy binding schema is invalid",
+        )
+    digests = value["validation_policy_digests"]
+    if (
+        not isinstance(digests, dict)
+        or set(digests) != {"table", "graph"}
+        or any(not isinstance(mode, str) for mode in digests)
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            "worker policy digest set must contain exactly table and graph",
+        )
+    policies = {
+        mode: _policy_digest(digests[mode], f"validation_policy_digests.{mode}")
+        for mode in ("table", "graph")
+    }
+    policy_set_digest = _policy_digest(value["policy_set_digest"], "policy_set_digest")
+    if "sha256:" + hashlib.sha256(canonical_json_bytes(policies)).hexdigest() != policy_set_digest:
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            "policy set digest does not bind the exact table and graph digests",
+        )
+    selection_id = value["selection_id"]
+    if (
+        not isinstance(selection_id, str)
+        or not re.fullmatch(r"[0-9a-f]{64}", selection_id)
+        or selection_id == "0" * 64
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            "selection ID is invalid",
+        )
+    binding = WorkerPolicyBinding(
+        producer_image_digest=_image_digest(
+            value["producer_image_digest"], "producer_image_digest"
+        ),
+        verifier_image_digest=_image_digest(
+            value["verifier_image_digest"], "verifier_image_digest"
+        ),
+        validation_policy_digests=policies,
+        policy_set_digest=policy_set_digest,
+        release_revision_coordinate=_policy_digest(
+            value["release_revision_coordinate"], "release_revision_coordinate"
+        ),
+        selection_id=selection_id,
+    )
+    unsigned = binding.to_dict()
+    unsigned.pop("selection_id")
+    if hashlib.sha256(canonical_json_bytes(unsigned)).hexdigest() != selection_id:
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_INVALID",
+            "selection ID does not bind the exact worker policy selection",
+        )
+    return binding
+
+
+def _local_validation_policy_digests() -> dict[str, str]:
+    return {
+        "table": "sha256:" + pipeline.validation_policy_digest(graph=False),
+        "graph": "sha256:" + pipeline.validation_policy_digest(graph=True),
+    }
+
+
+def _require_local_policy_binding(binding: WorkerPolicyBinding) -> None:
+    local = _local_validation_policy_digests()
+    if binding.validation_policy_digests != local:
+        deployed = canonical_json_bytes(binding.validation_policy_digests).decode("utf-8")
+        computed = canonical_json_bytes(local).decode("utf-8")
+        raise HostedWorkerError(
+            "WORKER_POLICY_RUNTIME_MISMATCH",
+            "deployed validation policy set "
+            f"{deployed} differs from this worker runtime {computed}; "
+            "publish and deploy worker images whose receipt carries the runtime policy set",
+        )
+
+
+def _binding_from_environment(environment: Mapping[str, str]) -> WorkerPolicyBinding:
+    raw = environment.get(_POLICY_SELECTION_ENV)
+    if not isinstance(raw, str):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISSING",
+            "worker policy selection is not configured in this worker template",
+        )
+    return parse_worker_policy_binding(_strict_json(raw.encode("utf-8"), "worker policy selection"))
+
+
+@dataclass(frozen=True)
+class WorkerPolicyBinding:
+    producer_image_digest: str
+    verifier_image_digest: str
+    validation_policy_digests: dict[str, str]
+    policy_set_digest: str
+    release_revision_coordinate: str
+    selection_id: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": _POLICY_BINDING_SCHEMA,
+            "producer_image_digest": self.producer_image_digest,
+            "verifier_image_digest": self.verifier_image_digest,
+            "validation_policy_digests": dict(self.validation_policy_digests),
+            "policy_set_digest": self.policy_set_digest,
+            "release_revision_coordinate": self.release_revision_coordinate,
+            "selection_id": self.selection_id,
+        }
+
+
+@dataclass
+class _AnonymousPayload:
+    descriptor: int
+    handle: Any
+
+    def close(self) -> None:
+        self.handle.close()
+
+
+def _https_url(value: str, label: str) -> str:
+    if not isinstance(value, str) or not 1 <= len(value) <= 2048 or value.endswith("/"):
+        raise HostedWorkerError("BOOTSTRAP_CONFIG_INVALID", f"{label} is invalid")
+    parsed = urlsplit(value)
+    if (
+        parsed.scheme != "https"
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.fragment
+    ):
+        raise HostedWorkerError(
+            "BOOTSTRAP_CONFIG_INVALID",
+            f"{label} must be an absolute HTTPS URL without user information or fragment",
+        )
+    return value
+
+
+def _v3_exchange_url(value: str) -> str:
+    _https_url(value, "exchange_url")
+    parsed = urlsplit(value)
+    matched = _DISPATCH_QUERY.fullmatch(parsed.query)
+    if parsed.path != BOOTSTRAP_EXCHANGE_PATH or matched is None:
+        raise HostedWorkerError(
+            "BOOTSTRAP_CONFIG_INVALID",
+            "exchange_url must use the exact bootstrap callback path "
+            f"{BOOTSTRAP_EXCHANGE_PATH} with one canonical UUIDv5 dispatch_id",
+        )
+    dispatch_id = matched.group(1)
+    try:
+        parsed_dispatch_id = UUID(dispatch_id)
+    except ValueError as error:
+        raise HostedWorkerError(
+            "BOOTSTRAP_CONFIG_INVALID",
+            "exchange_url dispatch_id must be a canonical UUIDv5",
+        ) from error
+    if parsed_dispatch_id.version != 5 or str(parsed_dispatch_id) != dispatch_id:
+        raise HostedWorkerError(
+            "BOOTSTRAP_CONFIG_INVALID",
+            "exchange_url dispatch_id must be a canonical UUIDv5",
+        )
+    return value
+
+
+def _metadata_identity(
+    transport: BootstrapTransport,
+    audience: str,
+) -> bytearray:
+    query = urlencode({"audience": audience, "format": "full"})
+    status, raw = transport.request(
+        "GET",
+        f"{METADATA_IDENTITY_URL}?{query}",
+        {"Metadata-Flavor": "Google"},
+        None,
+        MAX_IDENTITY_BYTES,
+    )
+    if status != 200:
+        raise HostedWorkerError(
+            "WORKLOAD_IDENTITY_UNAVAILABLE",
+            f"metadata identity endpoint returned HTTP {status}",
+        )
+    try:
+        token = raw.decode("ascii")
+    except UnicodeDecodeError as error:
+        raise HostedWorkerError(
+            "WORKLOAD_IDENTITY_INVALID",
+            "metadata identity response is not ASCII",
+        ) from error
+    if not _JWT.fullmatch(token):
+        raise HostedWorkerError(
+            "WORKLOAD_IDENTITY_INVALID",
+            "metadata identity response is not a compact JWT",
+        )
+    return bytearray(raw)
+
+
+def _exchange(
+    config: BootstrapConfig,
+    transport: BootstrapTransport,
+    identity: bytearray,
+) -> dict[str, Any]:
+    body = canonical_json_bytes(
+        {
+            "schema_version": BOOTSTRAP_REQUEST_SCHEMA,
+            "role": config.role,
+        }
+    )
+    status, raw = transport.request(
+        "POST",
+        config.exchange_url,
+        {
+            "Accept": "application/json",
+            "Authorization": "Bearer " + identity.decode("ascii"),
+            "Content-Type": "application/json",
+        },
+        body,
+        MAX_BOOTSTRAP_RESPONSE_BYTES,
+    )
+    if status != 200:
+        raise HostedWorkerError(
+            "BOOTSTRAP_EXCHANGE_FAILED",
+            f"Studio bootstrap exchange returned HTTP {status}",
+        )
+    value = _strict_json(raw, "bootstrap response")
+    if not isinstance(value, dict):
+        raise HostedWorkerError(
+            "BOOTSTRAP_RESPONSE_INVALID",
+            "bootstrap response must be an object",
+        )
+    return value
+
+
+def _require_worker_policy_preflight(
+    *,
+    job_wire: Any,
+    job: Any,
+    response_preflight: Any,
+    binding: WorkerPolicyBinding,
+) -> None:
+    """Bind the deployed selection, Studio response, and parsed recipe mode before execution."""
+
+    if not isinstance(job_wire, dict):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISMATCH",
+            "bootstrap job is not an object",
+        )
+    job_preflight = job_wire.get("worker_policy_preflight")
+    if not isinstance(job_preflight, dict) or response_preflight != job_preflight:
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISMATCH",
+            "Studio bootstrap response and job do not carry one exact policy preflight",
+        )
+    expected = {
+        "schema_version",
+        "workspace_id",
+        "dataset_id",
+        "table_plan_id",
+        "table_recipe_id",
+        "recipe_version",
+        "recipe_digest",
+        "builder_image_digest",
+        "verifier_image_digest",
+        "policy_mode",
+        "validation_policy_digest",
+        "validation_policy_digests",
+        "policy_set_digest",
+        "selection_fence",
+        "selection_id",
+    }
+    fields = set(job_preflight)
+    if (
+        fields not in (expected, expected | {"table_id"})
+        or job_preflight.get("schema_version") != "3.0.0"
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISMATCH",
+            "worker policy preflight fields are not exact",
+        )
+    mode = "graph" if isinstance(job.documents.recipe.transform_plan, GraphTablePlan) else "table"
+    expected_scope = {
+        "workspace_id": str(job.workspace_id),
+        "dataset_id": str(job.dataset_id),
+        "table_plan_id": str(job.table_plan_id),
+        "table_recipe_id": job.table_recipe_id,
+        "recipe_version": job.recipe_version,
+        "recipe_digest": job.documents.recipe_digest,
+    }
+    if any(job_preflight.get(name) != value for name, value in expected_scope.items()) or (
+        "table_id" in job_preflight and job_preflight["table_id"] != str(job.table_id)
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISMATCH",
+            "worker policy preflight scope differs from the bootstrap job",
+        )
+    static = {
+        "builder_image_digest": binding.producer_image_digest,
+        "verifier_image_digest": binding.verifier_image_digest,
+        "validation_policy_digests": binding.validation_policy_digests,
+        "policy_set_digest": binding.policy_set_digest,
+        "selection_fence": binding.release_revision_coordinate,
+        "selection_id": binding.selection_id,
+        "policy_mode": mode,
+        "validation_policy_digest": binding.validation_policy_digests[mode],
+    }
+    mismatched = [name for name, value in static.items() if job_preflight.get(name) != value]
+    if mismatched:
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISMATCH",
+            "worker policy preflight differs from the deployed selection: " + ",".join(mismatched),
+        )
+
+
+def _validate_bootstrap_response(
+    value: dict[str, Any],
+    role: str,
+    *,
+    worker_policy_binding: WorkerPolicyBinding | None = None,
+) -> tuple[dict[str, bytes], Any]:
+    """Return the worker payload bytes and the parsed job the validation already produced.
+
+    The parsed job is returned rather than discarded because the admission gate needs the tenant
+    coordinate out of it, and reparsing to get it would mean two parses that could disagree.
+    """
+
+    common = {"schema_version", "role", "job", "capability"}
+    expected = common if role == "producer" else common | {"signing_key"}
+    if worker_policy_binding is not None:
+        expected = expected | {"worker_policy_preflight"}
+    if set(value) != expected:
+        raise HostedWorkerError(
+            "BOOTSTRAP_RESPONSE_INVALID",
+            "bootstrap response fields are not exact for the requested role",
+        )
+    if value["schema_version"] != BOOTSTRAP_RESPONSE_SCHEMA or value["role"] != role:
+        raise HostedWorkerError(
+            "BOOTSTRAP_RESPONSE_INVALID",
+            "bootstrap response schema or role mismatches",
+        )
+
+    job_wire = value["job"]
+    if not isinstance(job_wire, dict):
+        raise HostedWorkerError(
+            "BOOTSTRAP_RESPONSE_INVALID",
+            "bootstrap job must be an object",
+        )
+    job = dict(job_wire)
+    if worker_policy_binding is not None:
+        job.pop("worker_policy_preflight", None)
+    capability = value["capability"]
+    if not isinstance(capability, str):
+        raise HostedWorkerError(
+            "BOOTSTRAP_RESPONSE_INVALID",
+            "bootstrap capability must be a string",
+        )
+    capability_raw = capability.encode("ascii", errors="strict")
+    if len(capability_raw) > MAX_CAPABILITY_BYTES:
+        raise HostedWorkerError("BOOTSTRAP_TOO_LARGE", "capability exceeds its byte limit")
+    parse_capability(capability_raw)
+    job_raw = canonical_json_bytes(job)
+    if len(job_raw) > MAX_DESCRIPTOR_BYTES:
+        raise HostedWorkerError("BOOTSTRAP_TOO_LARGE", "worker job exceeds its byte limit")
+    if role == "producer":
+        parsed_job = parse_producer_job(job)
+        if worker_policy_binding is not None:
+            _require_worker_policy_preflight(
+                job_wire=job_wire,
+                job=parsed_job,
+                response_preflight=value["worker_policy_preflight"],
+                binding=worker_policy_binding,
+            )
+        return {"job": job_raw, "capability": capability_raw}, parsed_job
+
+    parsed_job = parse_verifier_job(job)
+    if worker_policy_binding is not None:
+        _require_worker_policy_preflight(
+            job_wire=job_wire,
+            job=parsed_job,
+            response_preflight=value["worker_policy_preflight"],
+            binding=worker_policy_binding,
+        )
+    signing_key = value["signing_key"]
+    parse_signing_key(signing_key)
+    signing_raw = canonical_json_bytes(signing_key)
+    if len(signing_raw) > MAX_SIGNING_KEY_BYTES:
+        raise HostedWorkerError("BOOTSTRAP_TOO_LARGE", "signing key exceeds its byte limit")
+    return {
+        "job": job_raw,
+        "capability": capability_raw,
+        "signing_key": signing_raw,
+    }, parsed_job
+
+
+def _anonymous_payload(raw: bytes, label: str) -> _AnonymousPayload:
+    if hasattr(os, "memfd_create"):
+        descriptor = os.memfd_create(  # type: ignore[attr-defined]
+            f"mostlyright-{label}",
+            flags=getattr(os, "MFD_CLOEXEC", 0),
+        )
+        handle = os.fdopen(descriptor, "w+b", closefd=True)
+    else:
+        # macOS development fallback. TemporaryFile is opened mode 0600 and immediately unlinked;
+        # production Linux images use memfd_create and never materialize the payload on disk.
+        handle = tempfile.TemporaryFile(mode="w+b", prefix=f"mostlyright-{label}-")
+        descriptor = handle.fileno()
+    handle.write(raw)
+    handle.flush()
+    handle.seek(0)
+    os.set_inheritable(descriptor, True)
+    return _AnonymousPayload(descriptor=descriptor, handle=handle)
+
+
+def _validate_worker_executable(
+    executable: Path,
+    role: str,
+    *,
+    chain_root: Path = Path("/"),
+    trusted_uid: int = 0,
+) -> str:
+    """Validate one fixed worker executable without following mutable path indirection."""
+
+    name = f"mr-data-{role}"
+    if role not in {"producer", "verifier"} or executable.name != name:
+        raise HostedWorkerError(
+            "WORKER_EXECUTABLE_UNAVAILABLE",
+            f"fixed {name} entrypoint is invalid",
+        )
+    if (
+        not executable.is_absolute()
+        or not chain_root.is_absolute()
+        or any(part in {".", ".."} for part in executable.parts)
+    ):
+        raise HostedWorkerError(
+            "WORKER_EXECUTABLE_UNAVAILABLE",
+            f"fixed {name} entrypoint path is invalid",
+        )
+    try:
+        relative = executable.relative_to(chain_root)
+    except ValueError as error:
+        raise HostedWorkerError(
+            "WORKER_EXECUTABLE_UNAVAILABLE",
+            f"fixed {name} entrypoint escapes its trusted path chain",
+        ) from error
+
+    current = chain_root
+    components = (
+        current,
+        *(current / Path(*relative.parts[:index]) for index in range(1, len(relative.parts) + 1)),
+    )
+    for index, component in enumerate(components):
+        try:
+            metadata = component.lstat()
+        except OSError as error:
+            raise HostedWorkerError(
+                "WORKER_EXECUTABLE_UNAVAILABLE",
+                f"fixed {name} entrypoint is not installed",
+            ) from error
+        is_last = index == len(components) - 1
+        if stat.S_ISLNK(metadata.st_mode):
+            raise HostedWorkerError(
+                "WORKER_EXECUTABLE_UNSAFE",
+                f"fixed {name} entrypoint path contains a symbolic link",
+            )
+        if metadata.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
+            raise HostedWorkerError(
+                "WORKER_EXECUTABLE_UNSAFE",
+                f"fixed {name} entrypoint path is group or world writable",
+            )
+        if metadata.st_uid != trusted_uid:
+            raise HostedWorkerError(
+                "WORKER_EXECUTABLE_UNSAFE",
+                f"fixed {name} entrypoint path is not owned by the trusted uid",
+            )
+        if (is_last and not stat.S_ISREG(metadata.st_mode)) or (
+            not is_last and not stat.S_ISDIR(metadata.st_mode)
+        ):
+            raise HostedWorkerError(
+                "WORKER_EXECUTABLE_UNSAFE",
+                f"fixed {name} entrypoint path has an invalid file type",
+            )
+    if not os.access(executable, os.X_OK):
+        raise HostedWorkerError(
+            "WORKER_EXECUTABLE_UNAVAILABLE",
+            f"fixed {name} entrypoint is not executable",
+        )
+    return str(executable)
+
+
+def _worker_executable(role: str) -> str:
+    """Resolve only the immutable container installation coordinate."""
+
+    return _validate_worker_executable(WORKER_BIN_DIRECTORY / f"mr-data-{role}", role)
+
+
+def _worker_argv(
+    executable: str, role: str, payloads: Mapping[str, _AnonymousPayload]
+) -> list[str]:
+    arguments = [
+        executable,
+        "--job-fd",
+        str(payloads["job"].descriptor),
+        "--capability-fd",
+        str(payloads["capability"].descriptor),
+    ]
+    if role == "verifier":
+        arguments.extend(
+            [
+                "--signing-key-fd",
+                str(payloads["signing_key"].descriptor),
+            ]
+        )
+    return arguments
+
+
+def _restrict_inherited_descriptors(allowed: set[int]) -> list[int]:
+    directory = next(
+        (Path(candidate) for candidate in ("/proc/self/fd", "/dev/fd") if Path(candidate).is_dir()),
+        None,
+    )
+    if directory is None:
+        raise HostedWorkerError(
+            "DESCRIPTOR_SANITIZATION_UNAVAILABLE",
+            "cannot enumerate descriptors before worker process replacement",
+        )
+    changed: list[int] = []
+    for entry in directory.iterdir():
+        try:
+            descriptor = int(entry.name)
+        except ValueError:
+            continue
+        if descriptor <= 2 or descriptor in allowed:
+            continue
+        try:
+            if os.get_inheritable(descriptor):
+                os.set_inheritable(descriptor, False)
+                changed.append(descriptor)
+        except OSError:
+            # The enumeration directory itself, or another concurrently closed descriptor, can
+            # disappear between listing and inspection. It cannot survive exec in that state.
+            continue
+    return changed
+
+
+def _restore_inherited_descriptors(descriptors: Sequence[int]) -> None:
+    for descriptor in descriptors:
+        try:
+            os.set_inheritable(descriptor, True)
+        except OSError:
+            continue
+
+
+def admission_ledger() -> CapLedger:
+    """Build the workspace admission ledger.
+
+    ``MOSTLYRIGHT_CAP_STORE`` must name shared POSIX storage for monthly limits to persist across
+    job executions. The store requires advisory locking and atomic rename and must be initialized
+    before use. Without it, counters are process-local. Admission charges the run before execution.
+
+    Spend uses ``MOSTLYRIGHT_RUN_COST_UNITS``. The ledger uses ``CapPolicy`` defaults because this
+    entry point receives no per-workspace policy. It does not persist concurrency reservations.
+    """
+
+    configured = os.environ.get(CAP_STORE_ENV)
+    if configured:
+        return CapLedger(store=FileCapStore(Path(configured)), subject="workspace")
+    return CapLedger(subject="workspace")
+
+
+def _tenant_coordinate(workspace_id: Any) -> str:
+    """Render a workspace id into the 16-lowercase-hex coordinate the cap ledger counts by.
+
+    The tenant coordinate on this path is ``workspace_id``, taken from the validated bootstrap job.
+    It is deliberately NOT an API key id: ``key_id`` in this repository names the attestation
+    signing key (``hosted_worker.SigningKey.key_id``), which is a different concept, and the
+    customer API key never reaches this process at all (see ``key_seam.CONSUMER_CALL_SITE``).
+
+    The ledger's coordinate shape is cloud's ``hashKeyId`` shape — the first 8 bytes of a SHA-256
+    digest, hex-encoded — and that shape check is what makes it impossible to file a raw credential
+    where an identifier belongs. The same derivation is applied to the workspace id here, so the
+    check still holds. The honest consequence: these counters are per workspace, so they do not
+    line up with cloud's ``usage_events.key_id`` rows, which are per API key. This ledger bounds a
+    workspace's runs; it is not the billing attribution record.
+    """
+
+    return hashlib.sha256(str(workspace_id).encode("utf-8")).hexdigest()[:16]
+
+
+def declared_run_cost_units() -> int:
+    """Return the per-run spend this deployment declares, in cost units. Zero when undeclared.
+
+    What this number is, stated plainly because the alternative reading is flattering and wrong:
+    it is a **declared budget, not a measurement**. This seam cannot observe what a run costs. It
+    hands the process to the worker with ``execve`` and never regains control, so there is no
+    moment here at which a real bill exists to record. What it can do is charge the monthly spend
+    ledger the amount the operator says one run is worth at worst, which is how admission-time
+    spend control works everywhere: reserve the worst case, admit only if it fits.
+
+    ``MOSTLYRIGHT_RUN_COST_UNITS`` is therefore set on the worker job specification beside
+    ``MOSTLYRIGHT_CAP_STORE``, and the two have the same character: they are not "on" and "slightly
+    less on".
+
+    - **Unset** — every run is charged zero, so the monthly spend ceiling in
+      :class:`governors.CapPolicy` cannot trip no matter how many runs are admitted. Spend is
+      bounded only by the cloud project's own budget ceiling. This is a real gap, not a
+      degraded-but-safe default. What still holds is the monthly run allowance, and only where
+      ``MOSTLYRIGHT_CAP_STORE`` names shared storage (:func:`admission_ledger`); the concurrency
+      cap does not bind in this topology at all. With neither variable set, no cap here bounds
+      anything across executions.
+    - **Set to a whole number of cost units** — each admitted run reserves that much against the
+      monthly ceiling before it starts and commits the same amount when it does, so the ceiling
+      binds at ``max_monthly_cost_units / declared`` runs.
+
+    A value that cannot be read as a whole number of cost units refuses the run rather than
+    defaulting to zero, for the same reason an unreadable cap store refuses it: "this budget is
+    misconfigured" and "this run is free" must never be the same answer.
+    """
+
+    configured = os.environ.get(RUN_COST_UNITS_ENV)
+    if configured is None or configured == "":
+        return 0
+    # str.isdigit is the check rather than int(): it refuses the signs, the underscores, the
+    # surrounding whitespace and the unicode digits that int() would otherwise accept and turn
+    # into a number nobody wrote.
+    if not configured.isascii() or not configured.isdigit():
+        raise HostedWorkerError(
+            "ADMISSION_REFUSED",
+            f"{RUN_COST_UNITS_ENV} must be a whole number of cost units, zero or more",
+        )
+    return int(configured)
+
+
+def _reserve_run_slot(ledger: CapLedger, job: Any, cost_units: int) -> Reservation:
+    """Hold a slot for this run, or refuse the run before anything is allocated for it."""
+
+    coordinate = _tenant_coordinate(getattr(job, "workspace_id", None))
+    try:
+        return ledger.reserve(
+            coordinate,
+            now=datetime.now(UTC),
+            estimated_cost_units=cost_units,
+        )
+    except CapExceeded as error:
+        # A cap breach is the product working: it carries the cap that was reached and the two
+        # numbers behind it, in plain language, and no credential.
+        raise HostedWorkerError("ADMISSION_REFUSED", str(error)) from error
+    except CapStoreError as error:
+        # Fail closed. Counters that cannot be read are not counters that say zero.
+        raise HostedWorkerError("ADMISSION_REFUSED", str(error)) from error
+
+
+def launch(
+    config: BootstrapConfig,
+    *,
+    transport: BootstrapTransport | None = None,
+    execve: Callable[[str, Sequence[str], Mapping[str, str]], Any] = os.execve,
+    ledger: CapLedger | None = None,
+    _test_executable_resolver: Callable[[str], str] | None = None,
+) -> NoReturn:
+    """Exchange ambient identity, admit the run against its caps, and replace this process.
+
+    ``ledger`` is a test seam and nothing else: when it is None the real admission ledger is built
+    here. The gate itself is unconditional. There is no flag that turns it off, because a cap a
+    caller can switch off is not a cap.
+    """
+
+    if config.role not in {"producer", "verifier"}:
+        raise HostedWorkerError("BOOTSTRAP_CONFIG_INVALID", "role must be producer or verifier")
+    _v3_exchange_url(config.exchange_url)
+    _https_url(config.audience, "audience")
+    if config.worker_policy_binding is not None:
+        _require_local_policy_binding(config.worker_policy_binding)
+    client = transport or UrlLibBootstrapTransport()
+    identity = _metadata_identity(client, config.audience)
+    try:
+        response = _exchange(config, client, identity)
+    finally:
+        identity[:] = b"\x00" * len(identity)
+    raw_payloads, job = _validate_bootstrap_response(
+        response,
+        config.role,
+        worker_policy_binding=config.worker_policy_binding,
+    )
+
+    # The gate sits here, at the first moment the tenant coordinate is known and before anything
+    # is allocated for the run: a refused run creates no descriptor, holds no memory, and above
+    # all never reaches execve.
+    if ledger is None:
+        try:
+            ledger = admission_ledger()
+        except CapStoreError as error:
+            raise HostedWorkerError("ADMISSION_REFUSED", str(error)) from error
+    cost_units = declared_run_cost_units()
+    reservation = _reserve_run_slot(ledger, job, cost_units)
+
+    descriptors: dict[str, _AnonymousPayload] = {}
+    # Everything from here to the commit runs inside the reservation's block, so any fault —
+    # a descriptor that cannot be created, an entrypoint that fails validation — gives the slot
+    # back instead of leaving it held by a run that never started.
+    with reservation:
+        try:
+            for name, raw in raw_payloads.items():
+                mutable = bytearray(raw)
+                try:
+                    descriptors[name] = _anonymous_payload(bytes(mutable), name)
+                finally:
+                    mutable[:] = b"\x00" * len(mutable)
+            # Production has no executable-resolution injection seam. Tests may supply an explicit
+            # trusted stand-in because the fixed /opt/worker installation is created only in images.
+            executable = (
+                _worker_executable(config.role)
+                if _test_executable_resolver is None
+                else _test_executable_resolver(config.role)
+            )
+            arguments = _worker_argv(executable, config.role, descriptors)
+            environment = {
+                "PATH": "/opt/worker/bin:/usr/local/bin:/usr/bin:/bin",
+                "PYTHONDONTWRITEBYTECODE": "1",
+                "PYTHONUNBUFFERED": "1",
+            }
+            allowed = {payload.descriptor for payload in descriptors.values()}
+            changed = _restrict_inherited_descriptors(allowed)
+            try:
+                # Committed before the call, never after: execve replaces this process image and
+                # does not return, so an "after" that would record the run does not exist. A
+                # commit written below the call would never run and every admitted run would hold
+                # its slot forever. The committed amount is the declared budget rather than a
+                # measured bill for the same reason: after this line there is no process left here
+                # to measure anything with. See declared_run_cost_units.
+                reservation.commit(actual_cost_units=cost_units)
+                execve(executable, arguments, environment)
+            finally:
+                _restore_inherited_descriptors(changed)
+        finally:
+            for payload in descriptors.values():
+                payload.close()
+    raise HostedWorkerError("WORKER_EXEC_FAILED", "worker process replacement returned")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--exchange-url", required=True)
+    parser.add_argument("--audience", required=True)
+    parser.add_argument("--role", required=True, choices=("producer", "verifier"))
+    arguments = parser.parse_args(argv)
+    try:
+        launch(
+            BootstrapConfig(
+                exchange_url=arguments.exchange_url,
+                audience=arguments.audience,
+                role=arguments.role,
+                worker_policy_binding=_binding_from_environment(os.environ),
+            )
+        )
+    except HostedWorkerError as error:
+        print(f"{error.code}: {error.detail}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())