mostlyright-data 0.9.4__tar.gz → 0.9.5__tar.gz

{mostlyright_data-0.9.4 → mostlyright_data-0.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.9.4
+Version: 0.9.5
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.9.4 → mostlyright_data-0.9.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.9.4"
+version = "0.9.5"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.9.4 → mostlyright_data-0.9.5}/src/mostlyright/data_harness/acquisition/url_policy.py

@@ -188,6 +188,62 @@ class ValidatedTarget:
         return urlunsplit(("", "", parsed.path or "/", parsed.query, ""))
 
 
+def approved_public_addresses(
+    hostname: str,
+    port: int,
+    *,
+    resolver: Resolver | None = None,
+) -> tuple[str, ...]:
+    """Return every address a hostname answers with, once each has been proved publicly routable.
+
+    The part of :func:`validate_public_https_url` that is about *where a name leads* rather than
+    about a URL or an allowlist, so that a transport which is not fetching a URL can hold the same
+    rule without holding a second copy of it.  A client that hands a hostname to the operating
+    system and lets it connect has delegated the decision to DNS: a document naming
+    ``stream.example.test`` reaches whatever that name resolves to, and inside a network that is
+    an internal service that never expected a stranger.
+
+    Every answer is checked, and one bad answer refuses the whole resolution.  Approving the public
+    members of a mixed set would be deciding *after* a resolver has already demonstrated that it
+    can put an address of its choosing in front of the caller.
+
+    A caller that connects to one of these addresses -- rather than handing the name back to the
+    operating system, which would resolve it a second time and need not agree -- has the answer it
+    checked.
+    """
+
+    if _looks_like_obfuscated_ip(hostname):
+        raise AcquisitionSecurityError(
+            "OBFUSCATED_IP",
+            "non-canonical numeric IP hostnames are denied",
+        )
+    literal = _parse_ip_literal(hostname)
+    if literal is not None:
+        addresses: tuple[str, ...] = (literal.compressed,)
+    else:
+        addresses = (SystemResolver() if resolver is None else resolver).resolve(hostname, port)
+    approved: list[str] = []
+    for raw in addresses:
+        try:
+            address = ipaddress.ip_address(raw)
+        except ValueError:
+            raise AcquisitionSecurityError(
+                "DNS_ADDRESS",
+                "resolver returned a non-IP address",
+            ) from None
+        _require_globally_routable(address)
+        approved.append(address.compressed)
+    approved_ips = tuple(sorted(dict.fromkeys(approved)))
+    if not approved_ips:
+        raise AcquisitionSecurityError("DNS_EMPTY", "target has no approved public addresses")
+    if len(approved_ips) > MAX_DNS_ANSWERS:
+        raise AcquisitionSecurityError(
+            "DNS_ANSWER_LIMIT",
+            f"target has more than {MAX_DNS_ANSWERS} approved addresses",
+        )
+    return approved_ips
+
+
 def validate_public_https_url(
     url: str,
     *,
@@ -222,34 +278,7 @@ def validate_public_https_url(
         raise AcquisitionSecurityError("METADATA_DESTINATION", "metadata/internal hosts are denied")
 
     literal = _parse_ip_literal(hostname)
-    if literal is not None:
-        addresses = (literal.compressed,)
-    else:
-        if _looks_like_obfuscated_ip(hostname):
-            raise AcquisitionSecurityError(
-                "OBFUSCATED_IP",
-                "non-canonical numeric IP hostnames are denied",
-            )
-        addresses = resolver.resolve(hostname, effective_port)
-    approved: list[str] = []
-    for raw in addresses:
-        try:
-            address = ipaddress.ip_address(raw)
-        except ValueError:
-            raise AcquisitionSecurityError(
-                "DNS_ADDRESS",
-                "resolver returned a non-IP address",
-            ) from None
-        _require_globally_routable(address)
-        approved.append(address.compressed)
-    approved_ips = tuple(sorted(dict.fromkeys(approved)))
-    if not approved_ips:
-        raise AcquisitionSecurityError("DNS_EMPTY", "target has no approved public addresses")
-    if len(approved_ips) > MAX_DNS_ANSWERS:
-        raise AcquisitionSecurityError(
-            "DNS_ANSWER_LIMIT",
-            f"target has more than {MAX_DNS_ANSWERS} approved addresses",
-        )
+    approved_ips = approved_public_addresses(hostname, effective_port, resolver=resolver)
 
     netloc = hostname
     if ":" in hostname and literal is not None:

{mostlyright_data-0.9.4 → mostlyright_data-0.9.5}/src/mostlyright/data_harness/cli.py

@@ -23,7 +23,7 @@ import sys
 import tempfile
 import time
 from collections.abc import Callable, Iterator, Mapping, Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
 from typing import Any
@@ -209,6 +209,9 @@ from mostlyright.data_harness.sources.stream_recorder import (
 from mostlyright.data_harness.sources.stream_recorder import (
     ledger_root as stream_ledger_root,
 )
+from mostlyright.data_harness.sources.stream_registry import (
+    load_stream_connector_registry,
+)
 from mostlyright.data_harness.ux.commands import discovered_commands
 from mostlyright.data_harness.ux.credentials import LoginError, resolve_cloud_credentials
 from mostlyright.data_harness.ux.headline import (
@@ -1014,6 +1017,7 @@ def _dispatch(
                     producer_principal=args.producer_principal,
                     producer_attempt=args.producer_attempt,
                     previous_run_dir=args.previous_run_dir,
+                    connector_registry=args.connector_registry,
                     visual_store=visual_store,
                 )
                 payload = result.to_dict()
@@ -1979,6 +1983,14 @@ def _parser(
         help="the run settings: which recipe fingerprint and engine versions this run is pinned to",
     )
     recipe_run.add_argument("--input-root", type=Path, required=True, help=_INPUT_ROOT)
+    recipe_run.add_argument(
+        "--connector-registry",
+        type=Path,
+        help=(
+            "the folder of stream source documents the captures under the input folder were "
+            "recorded under; without it, only the ones this build ships are held"
+        ),
+    )
     recipe_run.add_argument("--output", type=Path, required=True, help=_OUTPUT_NEW)
     recipe_run.add_argument("--previous-run-dir", type=Path, help=_PREVIOUS_BUILD)
     recipe_run.add_argument("--producer-principal", required=True, help=_BUILT_BY)
@@ -3058,6 +3070,7 @@ def _local_recipe_registry(
     sandbox: CrawlerSandbox,
     stock_sources: tuple[Any, ...],
     stream_sources: tuple[Any, ...],
+    connector_registry: Path | None = None,
 ) -> AdapterRegistry:
     """Build the exact allowlist this local run is permitted to use.
 
@@ -3069,6 +3082,14 @@ def _local_recipe_registry(
     host allowlist against the address in the query and the other hands the fetch to Studio's
     hosted clean room.  A recorded capture has no address and no hosted home yet, so an adapter
     registered in either of those would claim a capability it cannot honestly provide.
+
+    ``connector_registry`` is the folder of source documents an operator wrote, the same one they
+    recorded the capture under.  ``None`` means the versions this build ships.  It has to be
+    sayable here: a sealed capture pins the digest of the source document it was recorded under,
+    and a build that could only look in the documents it ships would refuse every capture taken
+    from a source somebody wrote -- which is every live capture, because the one source this build
+    ships may not be captured live at all.  It is not a way to reinterpret a capture: the manifest
+    still names its own digest, and this only says where that version may be looked up.
     """
 
     adapters: list[Any] = []
@@ -3082,19 +3103,37 @@ def _local_recipe_registry(
         )
     if stream_sources:
         captures_root = input_root / _CAPTURES_DIRECTORY_NAME
-        gateway = RecordedStreamGateway(
+        # Read once, here, before any Build work: a folder that is not there, or is a link to
+        # one, is refused with STREAM_CONNECTOR_REGISTRY_ROOT at the start of the run rather
+        # than found missing halfway through it. With no folder named, this is the registry of
+        # documents the build ships, which is what every run did before the folder could be said.
+        held = load_stream_connector_registry(
+            None if connector_registry is None else connector_registry.absolute()
+        )
+        reader = RecordedStreamGateway(
             captures_root=captures_root,
             # Coordinator times, not capture times. The adapter checks the batch it gets back
             # against the observation evidence recorded here, and a gateway that answered with
             # times it read out of a file would make that check a tautology.
             observed_at=execution.requested_at,
             available_at=execution.requested_at,
+            registry=held,
         )
+        governance = _recorded_stream_governance(execution, stream_sources, reader)
+        # The same gateway, stamping a batch with the availability this coordinator just wrote
+        # down rather than with the moment the run started. The two are not the same fact: a
+        # capture stops being extended when its Recorder stops, and the observation above says
+        # so deliberately, so a gateway still stamping "now" would hand the adapter a batch its
+        # own evidence contradicts and every build over a capture older than an instant would
+        # be refused with STREAM_TEMPORAL_BINDING. This is still a coordinator input -- the
+        # value is read once, into the evidence, and handed on -- and not the gateway reading a
+        # time out of the file it is serving.
+        gateway = replace(reader, available_at=governance.observation.available_at)
         adapters.append(
             StreamBatchAdapter(
                 gateway=gateway,
                 sandbox=sandbox,
-                governance=_recorded_stream_governance(execution, stream_sources, gateway),
+                governance=governance,
                 watermark_store=StreamWatermarkStore(stream_ledger_root(captures_root)),
             )
         )
@@ -3111,6 +3150,7 @@ def _execute_public_recipe_run(
     producer_principal: str,
     producer_attempt: str,
     previous_run_dir: Path | None,
+    connector_registry: Path | None = None,
     visual_store: VisualRunStore | None = None,
 ) -> Any:
     stock_sources = _stock_local_reader_sources(recipe, execution)
@@ -3147,6 +3187,7 @@ def _execute_public_recipe_run(
             sandbox=sandbox,
             stock_sources=stock_sources,
             stream_sources=stream_sources,
+            connector_registry=connector_registry,
         )
 
         def _context(source: Any, operation: str) -> AcquisitionContext:

{mostlyright_data-0.9.4 → mostlyright_data-0.9.5}/src/mostlyright/data_harness/hosted_crawler_job.py

@@ -35,6 +35,7 @@ from mostlyright.data_harness.canonical import (
 from mostlyright.data_harness.hosted_crawler import (
     CRAWLER_EGRESS_POLICY_ENV,
     WARM_UP_RESULT_SCHEMA,
+    HostedCrawlerError,
     encode_normalization_input,
 )
 from mostlyright.data_harness.hosted_crawler_fetch import fetch_crawler_source
@@ -80,6 +81,24 @@ PRE_BOOTSTRAP_FAILURE_CODES = frozenset(
         "CRAWLER_BOOTSTRAP_RESPONSE_TOO_LARGE",
     }
 )
+# Wrapper deaths after bootstrap: the fetch may have succeeded, so these must
+# never become sealed refusals (a refusal is the source's answer, and the
+# source never answered "no").  Without a control-plane report the session
+# dangles until the producer's poll deadline expires and the run dies blind
+# (harness#400).  Keep this closed in lock-step with Studio's
+# ``WRAPPER_FAILURE_CODES``; a Studio that predates that set refuses the
+# report with 400 and the job still exits non-zero, so deploy order is free.
+WRAPPER_FAILURE_CODES = frozenset(
+    {
+        "CRAWLER_CHILD_FAILED",
+        "CRAWLER_CHILD_INVALID",
+        "CRAWLER_EXECUTABLE_UNAVAILABLE",
+        "CRAWLER_JOB_FAILED",
+        "CRAWLER_RESULTS_BUCKET_REQUIRED",
+        "CRAWLER_UPLOAD_FAILED",
+        "CRAWLER_WORKLOAD_IDENTITY_UNAVAILABLE",
+    }
+)
 # Sized for an EPA-scale year: the streamed decode, the streamed verification pass, and
 # the chunked result encode are each one pure-Python walk over ~200M cells, which a live
 # run spends 12-15 minutes on.  The old 600 was calibrated to decode alone and killed a
@@ -474,7 +493,7 @@ def _report_crawler_bootstrap_failure(
     can cross back into the crawler through this path.
     """
 
-    if failure_code not in PRE_BOOTSTRAP_FAILURE_CODES:
+    if failure_code not in PRE_BOOTSTRAP_FAILURE_CODES | WRAPPER_FAILURE_CODES:
         return False
     token: bytearray | None = None
     response: bytearray | None = None
@@ -1028,19 +1047,23 @@ def _validate_public_https_result(
         from mostlyright.data_harness.readers.contracts import ReaderPin
 
         try:
+            from mostlyright.data_harness.readers.registry import TOOLBOX
+
+            family = TOOLBOX.resolve(reader["family_id"], reader["family_version"])
+            # The child digests the ADMITTED options -- ``validate_options``
+            # fills family defaults -- so the expected identity must digest the
+            # same bytes.  Digesting the sealed pin raw made every recipe that
+            # omitted a defaulted key fail ``reader_identity`` on every crawl.
             pin = ReaderPin(
                 reader["family_id"],
                 reader["family_version"],
-                reader["decode_options"],
+                dict(family.validate_options(reader["decode_options"])),
             )
         except (KeyError, TypeError, ValueError):
             _fail("CRAWLER_CHILD_INVALID", "public HTTPS Reader identity is invalid")
         reader_identity = (pin.family_id, pin.family_version, pin.options_digest)
         resource_caps = query.get("resource_caps")
         try:
-            from mostlyright.data_harness.readers.registry import TOOLBOX
-
-            family = TOOLBOX.resolve(pin.family_id, pin.family_version)
             expected_reader_budgets = asdict(
                 family.default_budgets.narrowed_by(
                     {
@@ -1324,6 +1347,10 @@ def crawler_job_main(argv: list[str] | None = None) -> int:
         return 0
     except (
         CanonicalJSONError,
+        # HostedCrawlerError is a RuntimeError, not a ValueError; leaving it out
+        # let every request/normalization refusal escape as a raw traceback with
+        # no sealed failure and no session report (harness#400).
+        HostedCrawlerError,
         HostedCrawlerJobError,
         HostedCrawlerProtocolError,
         TypeError,
@@ -1366,6 +1393,28 @@ def crawler_job_main(argv: list[str] | None = None) -> int:
                 )
                 sys.stderr.write("\n")
                 return 0
+        if code in WRAPPER_FAILURE_CODES:
+            # A post-bootstrap wrapper death has no sealed-refusal lane (the
+            # source never refused), so terminalize the session through the
+            # ``:fail`` control-plane route instead of leaving it to dangle
+            # into the producer's poll deadline.  Best-effort: the job exits
+            # non-zero either way, and an older Studio refuses the code.
+            reported = _report_crawler_bootstrap_failure(
+                crawler_session_id=crawler_session_id,
+                failure_code=code,
+            )
+            sys.stderr.write(
+                canonical_json_bytes(
+                    {
+                        "status": "failed",
+                        "code": code,
+                        "detail": detail,
+                        "session_failure_reported": reported,
+                    }
+                ).decode()
+            )
+            sys.stderr.write("\n")
+            return 1
         sys.stderr.write(
             canonical_json_bytes({"status": "failed", "code": code, "detail": detail}).decode()
         )