mostlyright-data 0.9.6__tar.gz → 0.9.7__tar.gz

{mostlyright_data-0.9.6 → mostlyright_data-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.9.6
+Version: 0.9.7
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.9.6 → mostlyright_data-0.9.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.9.6"
+version = "0.9.7"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.9.6 → mostlyright_data-0.9.7}/src/mostlyright/data_harness/acquisition/sandbox.py

@@ -59,6 +59,7 @@ from mostlyright.data_harness.acquisition.url_policy import (
     AcquisitionSecurityError,
     EgressPolicy,
     SystemResolver,
+    canonical_peer_order,
     validate_public_https_url,
 )
 from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
@@ -3011,7 +3012,10 @@ def _parse_approved_endpoints(
                 )
             normalized.append(address.compressed)
         key = (hostname, port)
-        if key in endpoints or tuple(sorted(normalized)) != addresses:
+        # The SAME order `url_policy.canonical_peer_order` writes, read from that function rather
+        # than restated: the parent decides connection order there, and a second sorting rule here
+        # would refuse the very sets the parent now produces (#351).
+        if key in endpoints or canonical_peer_order(normalized) != addresses:
             raise AcquisitionSecurityError(
                 "SANDBOX_EGRESS",
                 "approved endpoint set is duplicate or non-canonical",

{mostlyright_data-0.9.6 → mostlyright_data-0.9.7}/src/mostlyright/data_harness/acquisition/url_policy.py

@@ -6,6 +6,7 @@ import ipaddress
 import re
 import socket
 import unicodedata
+from collections.abc import Iterable
 from dataclasses import dataclass
 from typing import Protocol
 from urllib.parse import SplitResult, urlsplit, urlunsplit
@@ -188,6 +189,31 @@ class ValidatedTarget:
         return urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
 
 
+def canonical_peer_order(addresses: Iterable[str]) -> tuple[str, ...]:
+    """The one order an approved peer set is ever written in: IPv4 first, then lexicographic.
+
+    ⚠ THIS IS A CONNECTION ORDER, NOT ONLY A CANONICAL FORM. A fetch spends a bounded request
+    budget (``max_requests``, four) walking this tuple, so whatever sorts first is what an
+    IPv4-only deployment -- every Cloud Run job -- actually dials. Plain lexicographic sorting put
+    ``2606:…`` ahead of ``3.x``/``34.x``/``52.x``, which is precisely the A-record shape every
+    Fastly and CloudFront host has, so an IPv6-unroutable worker burned its whole budget on AAAA
+    candidates and died ``REQUEST_LIMIT`` before reaching a routable address (#351).
+
+    Family first is therefore the rule, and it is a rule rather than a preference expressed in the
+    resolver: ordering is decided HERE, and ``sandbox._parse_approved_endpoints`` refuses an
+    endpoint set that is not in this exact order, so the two must move together. Within a family
+    the order stays lexicographic on the compressed text, which keeps the set canonical -- the
+    resolution digest is computed over it -- and keeps it independent of resolver answer order.
+    """
+
+    return tuple(
+        sorted(
+            dict.fromkeys(addresses),
+            key=lambda address: (0 if ipaddress.ip_address(address).version == 4 else 1, address),
+        )
+    )
+
+
 def approved_public_addresses(
     hostname: str,
     port: int,
@@ -233,7 +259,7 @@ def approved_public_addresses(
             ) from None
         _require_globally_routable(address)
         approved.append(address.compressed)
-    approved_ips = tuple(sorted(dict.fromkeys(approved)))
+    approved_ips = canonical_peer_order(dict.fromkeys(approved))
     if not approved_ips:
         raise AcquisitionSecurityError("DNS_EMPTY", "target has no approved public addresses")
     if len(approved_ips) > MAX_DNS_ANSWERS:

{mostlyright_data-0.9.6 → mostlyright_data-0.9.7}/src/mostlyright/data_harness/hosted_bootstrap.py

@@ -22,7 +22,7 @@ from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, NoReturn, Protocol
 from urllib.error import HTTPError
-from urllib.parse import urlencode, urlsplit
+from urllib.parse import urlencode, urlsplit, urlunsplit
 from urllib.request import (
     HTTPRedirectHandler,
     HTTPSHandler,
@@ -42,6 +42,7 @@ from mostlyright.data_harness.governors import (
     Reservation,
 )
 from mostlyright.data_harness.hosted_worker import (
+    CRAWLER_RESULTS_BUCKET_ENV,
     MAX_CAPABILITY_BYTES,
     MAX_DESCRIPTOR_BYTES,
     MAX_SIGNING_KEY_BYTES,
@@ -62,6 +63,7 @@ BOOTSTRAP_REQUEST_SCHEMA = "mostlyright-worker-bootstrap-request.v3"
 BOOTSTRAP_RESPONSE_SCHEMA = "mostlyright-worker-bootstrap-response.v3"
 GENERIC_PROBE_BOOTSTRAP_RESPONSE_SCHEMA = "mostlyright-worker-bootstrap-response.v5"
 BOOTSTRAP_EXCHANGE_PATH = "/_internal/worker-bootstrap:exchange"
+BOOTSTRAP_FAILURE_PATH = "/_internal/worker-bootstrap:fail"
 GENERIC_PROBE_BOOTSTRAP_EXCHANGE_PATH = "/_internal/generic-api-probe-bootstrap:exchange"
 METADATA_IDENTITY_URL = (
     "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/identity"
@@ -69,6 +71,21 @@ METADATA_IDENTITY_URL = (
 MAX_IDENTITY_BYTES = 16 * 1024
 MAX_BOOTSTRAP_RESPONSE_BYTES = 768 * 1024
 BOOTSTRAP_TIMEOUT_SECONDS = 10.0
+#: Exchange refusals worth waiting out. Everything else in 4xx is Studio's final answer for this
+#: dispatch, so the attempt is dead and something has to say so before this process exits.
+RETRYABLE_EXCHANGE_STATUSES = frozenset({408, 425, 429})
+#: The closed set of reasons a bootstrap can report before it holds a job, one per refusal it can
+#: be told. Keep it closed in lock-step with Studio's accepted set: a code Studio does not know is
+#: refused, the report is lost, and the run waits for the stuck-run poller instead.
+EXCHANGE_REFUSAL_CODES = {
+    400: "BOOTSTRAP_EXCHANGE_REJECTED",
+    401: "BOOTSTRAP_IDENTITY_REJECTED",
+    403: "BOOTSTRAP_IDENTITY_REJECTED",
+    404: "BOOTSTRAP_DISPATCH_UNCLAIMABLE",
+    409: "BOOTSTRAP_DISPATCH_UNCLAIMABLE",
+    410: "BOOTSTRAP_DISPATCH_UNCLAIMABLE",
+    422: "BOOTSTRAP_EXCHANGE_REJECTED",
+}
 WORKER_BIN_DIRECTORY = Path("/opt/worker/bin")
 _JWT = re.compile(r"^[A-Za-z0-9_-]{1,8192}\.[A-Za-z0-9_-]{1,8192}\.[A-Za-z0-9_-]{1,8192}$")
 _DISPATCH_QUERY = re.compile(
@@ -284,6 +301,14 @@ def _local_validation_policy_digests() -> dict[str, str]:
 
 
 def _require_local_policy_binding(binding: WorkerPolicyBinding) -> None:
+    """Fail a misdeployed image before it spends an exchange. Not the authority on a run.
+
+    The run's own authority is the sealed preflight, which ``_require_worker_policy_preflight``
+    holds against this same runtime. This one only catches an image whose deployment record and
+    whose code disagree, which no run can be sealed correctly against, and it catches it one
+    round trip earlier.
+    """
+
     local = _local_validation_policy_digests()
     if binding.validation_policy_digests != local:
         deployed = canonical_json_bytes(binding.validation_policy_digests).decode("utf-8")
@@ -434,6 +459,46 @@ def _metadata_identity(
     return bytearray(raw)
 
 
+def _report_attempt_failure(
+    config: BootstrapConfig,
+    transport: BootstrapTransport,
+    identity: bytearray,
+    status: int,
+) -> bool:
+    """Tell Studio, best effort, that this attempt will never start. Return whether it landed.
+
+    A refused bootstrap is the one failure the attempt itself cannot report: the process holds no
+    job, no capability and no attempt id, so nothing downstream of it will ever call ``fail``. Left
+    silent, the run sits in "running" until a human notices. This is the cheapest way to close that
+    gap, not the guarantee -- Studio's stuck-run poller is what actually guarantees it, so a report
+    that cannot be delivered changes nothing about how this process exits.
+
+    It reuses the attempt's own identity token, sends only the allowlisted refusal code in
+    canonical bytes on the ``:fail`` sibling of the exchange route, and accepts only an empty 204.
+    Nothing a compromised Studio could answer here can cross back into the worker.
+    """
+
+    code = EXCHANGE_REFUSAL_CODES.get(status)
+    if code is None:
+        return False
+    parsed = urlsplit(config.exchange_url)
+    try:
+        reported, raw = transport.request(
+            "POST",
+            urlunsplit((parsed.scheme, parsed.netloc, BOOTSTRAP_FAILURE_PATH, parsed.query, "")),
+            {
+                "Accept": "application/json",
+                "Authorization": "Bearer " + identity.decode("ascii"),
+                "Content-Type": "application/json",
+            },
+            canonical_json_bytes({"failure_code": code}),
+            0,
+        )
+    except Exception:
+        return False
+    return reported == 204 and not raw
+
+
 def _exchange(
     config: BootstrapConfig,
     transport: BootstrapTransport,
@@ -457,6 +522,14 @@ def _exchange(
         MAX_BOOTSTRAP_RESPONSE_BYTES,
     )
     if status != 200:
+        # The generic API probe is excluded on purpose: it has no run and no attempt behind it, so
+        # there is no state anywhere that a report would move off "running".
+        if (
+            config.role != "generic_api"
+            and 400 <= status < 500
+            and status not in RETRYABLE_EXCHANGE_STATUSES
+        ):
+            _report_attempt_failure(config, transport, identity, status)
         raise HostedWorkerError(
             "BOOTSTRAP_EXCHANGE_FAILED",
             f"Studio bootstrap exchange returned HTTP {status}",
@@ -475,9 +548,19 @@ def _require_worker_policy_preflight(
     job_wire: Any,
     job: Any,
     response_preflight: Any,
-    binding: WorkerPolicyBinding,
 ) -> None:
-    """Bind the deployed selection, Studio response, and parsed recipe mode before execution."""
+    """Bind the Studio response, the sealed preflight and the parsed recipe before execution.
+
+    The sealed preflight is the run's single fleet authority. The image digests, selection id and
+    release fence it names are deliberately NOT re-compared against a second copy carried in this
+    worker's environment: that mirror could only ever say "the fleet moved on since this run was
+    sealed", which is not a reason to refuse a run whose tooling was chosen for it. Nine hosted
+    builds died as WORKER_POLICY_BINDING_MISMATCH saying exactly that.
+
+    What survives is everything the traveler can be held to without a second copy: both preflight
+    copies agree, the field set is exact, it names this job's scope and this recipe's mode, its
+    policy set is internally consistent, and that policy set is the one this runtime implements.
+    """
 
     if not isinstance(job_wire, dict):
         raise HostedWorkerError(
@@ -524,6 +607,7 @@ def _require_worker_policy_preflight(
         "table_recipe_id": job.table_recipe_id,
         "recipe_version": job.recipe_version,
         "recipe_digest": job.documents.recipe_digest,
+        "policy_mode": mode,
     }
     if any(job_preflight.get(name) != value for name, value in expected_scope.items()) or (
         "table_id" in job_preflight and job_preflight["table_id"] != str(job.table_id)
@@ -532,21 +616,41 @@ def _require_worker_policy_preflight(
             "WORKER_POLICY_BINDING_MISMATCH",
             "worker policy preflight scope differs from the bootstrap job",
         )
-    static = {
-        "builder_image_digest": binding.producer_image_digest,
-        "verifier_image_digest": binding.verifier_image_digest,
-        "validation_policy_digests": binding.validation_policy_digests,
-        "policy_set_digest": binding.policy_set_digest,
-        "selection_fence": binding.release_revision_coordinate,
-        "selection_id": binding.selection_id,
-        "policy_mode": mode,
-        "validation_policy_digest": binding.validation_policy_digests[mode],
-    }
-    mismatched = [name for name, value in static.items() if job_preflight.get(name) != value]
-    if mismatched:
+    digests = job_preflight["validation_policy_digests"]
+    # Shape first, and before anything renders it. The refusals below quote the sealed set back as
+    # canonical JSON, and canonical JSON is a narrower dialect than the one the response was parsed
+    # in -- a non-integer number inside this field would raise out of the renderer, past the coded
+    # refusal it was rendering, and reach the operator as a traceback with no code on it.
+    if (
+        not isinstance(digests, dict)
+        or set(digests) != {"table", "graph"}
+        or any(not isinstance(digest, str) for digest in digests.values())
+    ):
+        raise HostedWorkerError(
+            "WORKER_POLICY_BINDING_MISMATCH",
+            "worker policy preflight validation policy set is not one table and graph digest pair",
+        )
+    # The runtime is the authority on which validation policy this image implements, so the sealed
+    # set is checked against the code rather than against a deployment variable claiming to
+    # describe the code. A run sealed under a policy this image does not implement would attest to
+    # a check it never ran.
+    local = _local_validation_policy_digests()
+    if digests != local:
+        sealed = canonical_json_bytes(digests).decode("utf-8")
+        computed = canonical_json_bytes(local).decode("utf-8")
+        raise HostedWorkerError(
+            "WORKER_POLICY_RUNTIME_MISMATCH",
+            f"sealed validation policy set {sealed} differs from this worker runtime {computed}; "
+            "launch this run on the worker image its preflight sealed",
+        )
+    if (
+        job_preflight["validation_policy_digest"] != digests[mode]
+        or job_preflight["policy_set_digest"]
+        != "sha256:" + hashlib.sha256(canonical_json_bytes(digests)).hexdigest()
+    ):
         raise HostedWorkerError(
             "WORKER_POLICY_BINDING_MISMATCH",
-            "worker policy preflight differs from the deployed selection: " + ",".join(mismatched),
+            "worker policy preflight does not carry one self-consistent policy set",
         )
 
 
@@ -611,7 +715,6 @@ def _validate_bootstrap_response(
                 job_wire=job_wire,
                 job=parsed_job,
                 response_preflight=value["worker_policy_preflight"],
-                binding=worker_policy_binding,
             )
         return {"job": job_raw, "capability": capability_raw}, parsed_job
 
@@ -625,7 +728,6 @@ def _validate_bootstrap_response(
             job_wire=job_wire,
             job=parsed_job,
             response_preflight=value["worker_policy_preflight"],
-            binding=worker_policy_binding,
         )
     signing_key = value["signing_key"]
     parse_signing_key(signing_key)
@@ -977,6 +1079,19 @@ def launch(
                 "PYTHONDONTWRITEBYTECODE": "1",
                 "PYTHONUNBUFFERED": "1",
             }
+            # The hygiene exec strips the inherited environment so no credential can leak into
+            # the worker -- and it must keep doing so. But the crawler-results bucket is a public
+            # deployment coordinate, not a secret: it names which GCS bucket a result download's
+            # storage authority may claim, the worker's download check refuses every GCS-served
+            # crawl without it, and the deploy pins it on the job for exactly that check. It is
+            # the ONE inherited name that crosses; anything else stays stripped.
+            # It is the FALLBACK, not the source: a job that carries `results_bucket` travels with
+            # its own pin and the worker prefers that one. This pass-through covers dispatch
+            # payloads minted before the traveler did; it retires once every dispatch mints
+            # the field.
+            results_bucket = os.environ.get(CRAWLER_RESULTS_BUCKET_ENV)
+            if results_bucket:
+                environment[CRAWLER_RESULTS_BUCKET_ENV] = results_bucket
             allowed = {payload.descriptor for payload in descriptors.values()}
             changed = _restrict_inherited_descriptors(allowed)
             try:
@@ -997,21 +1112,47 @@ def launch(
 
 
 def main(argv: Sequence[str] | None = None) -> int:
+    """Parse one Cloud Run Job dispatch and launch the bootstrap it names.
+
+    Two dispatch spellings reach this executable, because two Studio enqueuers write them and
+    each names the coordinate it carries.  A Builder or verifier task carries
+    ``--exchange-url ...?dispatch_id=<uuid> --role {producer,verifier}``.  A standalone generic
+    API probe carries ``--generic-api-probe-bootstrap-url ...?probe_id=<uuid>`` and no ``--role``
+    at all: the flag names the probe exchange, so the role is not a second thing to state and
+    Studio's ``GoogleCloudTasksGenericApiProbeEnqueuer`` does not state it.
+
+    Accepting only the first spelling is what made the hosted keyed-source probe lane
+    unreachable: argparse rejected the deployed probe task's argv outright (``unrecognized
+    arguments`` plus a missing ``--role``) and the Job exited 2 before a single line of bootstrap
+    code ran.  ``tests/test_hosted_bootstrap.py`` holds this parser against the argv Studio's own
+    ``dispatch.py`` emits, so the two cannot drift apart again silently.
+    """
+
     parser = argparse.ArgumentParser()
-    parser.add_argument("--exchange-url", required=True)
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument("--exchange-url")
+    source.add_argument("--generic-api-probe-bootstrap-url")
     parser.add_argument("--audience", required=True)
-    parser.add_argument("--role", required=True, choices=("producer", "verifier", "generic_api"))
+    parser.add_argument("--role", choices=("producer", "verifier", "generic_api"))
     arguments = parser.parse_args(argv)
+    if arguments.generic_api_probe_bootstrap_url is not None:
+        if arguments.role not in {None, "generic_api"}:
+            parser.error("--generic-api-probe-bootstrap-url admits only --role generic_api")
+        exchange_url = arguments.generic_api_probe_bootstrap_url
+        role = "generic_api"
+    else:
+        if arguments.role is None:
+            parser.error("--exchange-url requires --role")
+        exchange_url = arguments.exchange_url
+        role = arguments.role
     try:
         launch(
             BootstrapConfig(
-                exchange_url=arguments.exchange_url,
+                exchange_url=exchange_url,
                 audience=arguments.audience,
-                role=arguments.role,
+                role=role,
                 worker_policy_binding=(
-                    None
-                    if arguments.role == "generic_api"
-                    else _binding_from_environment(os.environ)
+                    None if role == "generic_api" else _binding_from_environment(os.environ)
                 ),
             )
         )

{mostlyright_data-0.9.6 → mostlyright_data-0.9.7}/src/mostlyright/data_harness/hosted_crawler_fetch.py

@@ -139,7 +139,15 @@ def _public_https_retriever(
             max_concurrency=2,
             min_interval_seconds=1.0,
             allowed_media_types=_public_expected_media_types(request.query),
-            user_agent="MostlyRightDataHarness-PublicHttps/1.0",
+            # The contact URL is load-bearing, not decoration: Akamai-fronted
+            # providers (stlouisfed.org, measured 2026-08-27 in harness#430)
+            # silently tarpit a bare unknown product token - RST on h2, a
+            # 15s+ hang on http/1.1 that surfaces as the acquisition timeout -
+            # while the same token WITH the crawler-convention "(+URL)"
+            # segment answers in half a second from the same egress IP. An
+            # honest identity that names where to reach us is also simply the
+            # right thing for a public crawler to send.
+            user_agent="MostlyRightDataHarness-PublicHttps/1.0 (+https://mostlyright.md/crawler)",
         ),
         limiter=_PUBLIC_HTTPS_LIMITER,
     )