mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/adapters.py

@@ -0,0 +1,1214 @@
+"""Governed user-file and public-HTTPS source adapters.
+
+The steps shared with the external-provider and stream adapters live in ``_adapter_steps``. The
+private helpers below are local to this module.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import stat
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field, replace
+from pathlib import Path, PurePosixPath
+from typing import Any
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.acquisition.http import (
+    RetrievalLimits,
+    reader_retrieval_limits,
+    seal_content_addressed_snapshot,
+)
+from mostlyright.data_harness.acquisition.parsing import row_digest_for
+from mostlyright.data_harness.acquisition.sandbox import (
+    CrawlerSandbox,
+    sandbox_policy_digest,
+)
+from mostlyright.data_harness.acquisition.sandbox import (
+    FetchedMember as SandboxFetchedMember,
+)
+from mostlyright.data_harness.acquisition.url_policy import AcquisitionSecurityError
+from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
+from mostlyright.data_harness.formats import DATA_FORMATS
+from mostlyright.data_harness.readers.contracts import ReaderBudgets, ReaderPin
+from mostlyright.data_harness.readers.samples import DecodedFacts, ReaderSample, warm_up
+from mostlyright.data_harness.sources._adapter_steps import (
+    DecodeRecord,
+    _authorize_candidate,
+    _output,
+    _parse_limits,
+    _require_sandbox_binding,
+)
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionReceiptV2,
+    AcquisitionRequest,
+    AdapterDescriptor,
+    EvidenceReference,
+    RetentionPolicy,
+    RightsEvidence,
+    SnapshotReference,
+    SourceContractError,
+    SourceObservation,
+    fetched_source_revision_digest,
+)
+from mostlyright.data_harness.sources.contracts import (
+    FetchedMember as ReceiptFetchedMember,
+)
+from mostlyright.data_harness.sources.governance import (
+    classify_before_persistence,
+    require_admissible,
+)
+from mostlyright.data_harness.sources.range_reader import (
+    RangeRevisionLock,
+    acquire_range_reader,
+    default_policy_digests,
+    range_reader_policy_digest,
+    require_crawler_sandbox,
+)
+from mostlyright.data_harness.sources.registry import (
+    AcquisitionContext,
+    AcquisitionOutput,
+)
+
+# Ceilings for the composed address.  Every one is enforced before a single character is
+# joined, so a malformed composition object costs a bounded amount of work rather than an
+# unbounded one.
+URL_MAX_LENGTH = 2_048
+URL_ORIGIN_MAX_LENGTH = 255
+URL_MAX_PATH_COMPONENTS = 16
+URL_MAX_COMPONENT_PARTS = 8
+URL_PART_MAX_LENGTH = 64
+URL_COMPONENT_MAX_LENGTH = 128
+
+
+def _receipt_v2_member(member: SandboxFetchedMember) -> ReceiptFetchedMember:
+    """Project current transport evidence onto the immutable historical v2 receipt shape."""
+
+    document = member.to_dict()
+    document["peer_attempts"] = [
+        {
+            "approved_ip": attempt["approved_ip"],
+            "outcome": attempt["outcome"],
+            "failure_code": attempt["failure_code"],
+        }
+        for attempt in document["peer_attempts"]
+    ]
+    return ReceiptFetchedMember(**document)
+
+
+# What a path part may be made of.  The excluded characters are not a stylistic preference:
+# each one is how a path part stops being a path part.  ``/`` invents a component the recipe
+# did not state; ``?`` and ``#`` start a query string or a fragment the recipe never
+# declared; ``@`` and ``:`` can move the authority, so the address would reach a host the
+# allowlist was never asked about; ``%`` re-encodes any of those past a later reader; and a
+# space or a non-ASCII character is normalised differently by different readers, so what is
+# checked and what is fetched can disagree.  What remains cannot express any of them, which
+# is why the refusal is structural rather than a regular expression run over the assembled
+# result.
+_URL_PART = re.compile(rf"^[A-Za-z0-9._-]{{1,{URL_PART_MAX_LENGTH}}}$")
+
+
+@dataclass(frozen=True)
+class ObservationPolicy:
+    """Recorded, digest-bound temporal/source evidence owned by the coordinator."""
+
+    observed_at: str
+    available_at: str
+    historical_start: str
+    historical_end: str
+    live_status: str
+    publication_delay_seconds: int
+    update_frequency_seconds: int
+    response_status: int | None
+    event_time_field: str | None
+    available_at_field: str | None
+    evidence: tuple[EvidenceReference, ...]
+
+
+@dataclass(frozen=True)
+class AdapterGovernance:
+    rights: RightsEvidence
+    retention: RetentionPolicy
+    declared_classification: str
+    lawful_basis_declared: bool
+    attached_license_artifact_digests: tuple[str, ...]
+    observation: ObservationPolicy
+
+
+# Which families this process has already warmed up.  One recipe execution is one refresh, and
+# the harness already runs a fresh coordinator process per execution, so per-process memoization
+# is per-refresh in practice -- that is why "before every refresh" and "once per process" say the
+# same thing here.  The consequence, stated rather than left to be inferred: a long-lived
+# coordinator would warm up once and then never again, so if one is ever introduced this memo is
+# the thing that must change.
+_WARMED_READER_FAMILIES: set[tuple[str, str]] = set()
+
+
+def _clean_room_decode_route(
+    sandbox: CrawlerSandbox,
+    *,
+    context: AcquisitionContext,
+    sealed_format: str,
+    expected_policy_digest: str | None = None,
+) -> Callable[[ReaderSample], DecodedFacts]:
+    """Build the decode route the warm-up check runs its known file through.
+
+    The route is injected rather than called from ``readers.samples`` because ``readers/`` may
+    not import ``acquisition/``; that back-edge would close an import cycle. It is built here,
+    where the sandbox already is.
+
+    It goes through the same ``decode_and_parse`` operation the real decode uses, under the same
+    attested policy. Warming up through a shortcut would prove the decoder works on a path
+    production does not take, which is the class of green test that hides a red system.
+
+    Two consumers, two routes, one expectation. The CI coverage gate
+    (``tests/test_reader_samples.py``) takes ``readers.samples.decode_in_process``: it runs on
+    every leg including ones with no sandbox, and its job is to prove the family answers
+    correctly. A refresh takes this route: its job is to prove the family answers correctly *on
+    the path production takes*, attestation included. Both read the same packaged sample through
+    the same loader and compare against the same recorded constants, so the two cannot drift on
+    expectations -- only on transport, which is the difference they exist to cover.
+
+    The row count and column names reported here are the parser's, derived from the bytes the
+    Reader returned, where the in-process route reports the family's own declaration. That is
+    deliberate and is the one place the two routes differ in substance: a family whose declared
+    shape disagrees with the parser's reading of its own output fails this check, and that is a
+    finding worth having rather than one to reconcile away.
+
+    The family's own default budgets apply, exactly as they do for the CI gate. Narrowing the
+    warm-up by a recipe's caps would let one recipe's tight budget turn a healthy Reader into a
+    halted refresh.
+    """
+
+    def decode(sample: ReaderSample) -> DecodedFacts:
+        sandboxed = sandbox.decode_and_parse(
+            # Lower-cased and dot-joined because a sandbox request identifier admits lowercase
+            # letters, digits, and ``._-`` only, while a family identifier may carry capitals
+            # and a coordinate carries an ``@``.
+            request_id=f"warmup.{sample.family_id}.{sample.family_version}".lower(),
+            content=sample.content,
+            reader_pin={
+                "family_id": sample.family_id,
+                "family_version": sample.family_version,
+                "decode_options": dict(sample.decode_options),
+            },
+            output_format=sealed_format,
+            limits=_parse_limits(context),
+            reader_budgets=None,
+        )
+        if expected_policy_digest is None:
+            _require_sandbox_binding(sandboxed.policy_digest, context)
+        elif sandboxed.policy_digest != expected_policy_digest:
+            raise SourceContractError(
+                "SANDBOX_ATTESTATION_MISMATCH",
+                "sandbox.policy_digest",
+                "warm-up decode does not bind the composed adapter's decode policy",
+            )
+        if sandboxed.parsed is None or sandboxed.content is None or sandboxed.decode_flags is None:
+            raise SourceContractError(
+                "SANDBOX_RESULT",
+                "sandbox",
+                "warm-up decode result is incomplete",
+            )
+        return DecodedFacts(
+            sha256_bytes(sandboxed.content),
+            len(sandboxed.parsed.rows),
+            tuple(sandboxed.parsed.columns),
+            tuple(sandboxed.decode_flags),
+        )
+
+    return decode
+
+
+def _warm_up_family(
+    sandbox: CrawlerSandbox,
+    *,
+    pin: Mapping[str, Any],
+    context: AcquisitionContext,
+    sealed_format: str,
+    expected_policy_digest: str | None = None,
+) -> None:
+    """Re-open this family's one known file before it is allowed to touch source data.
+
+    Stated for the user as: before every refresh, the Reader re-opens one known file; a wrong
+    answer halts the refresh. A decoder that answers differently on a file whose answer is known
+    has changed the meaning of every dataset it has ever produced, and the datasets already
+    sealed cannot be un-sealed -- so the only useful moment to stop is before the next one is
+    built, which is why this runs before the fetch rather than beside it.
+    """
+
+    coordinate = (str(pin["family_id"]), str(pin["family_version"]))
+    if coordinate in _WARMED_READER_FAMILIES:
+        return
+    warm_up(
+        coordinate[0],
+        coordinate[1],
+        decode=_clean_room_decode_route(
+            sandbox,
+            context=context,
+            sealed_format=sealed_format,
+            expected_policy_digest=expected_policy_digest,
+        ),
+    )
+    _WARMED_READER_FAMILIES.add(coordinate)
+
+
+@dataclass(frozen=True)
+class _Decoded:
+    """What one clean-room decode handed back, in exactly the facts the adapter seals.
+
+    ``content`` is the Reader's canonical output and is what gets sealed; the fetched bytes
+    are never sealed and survive only as the digest inside ``record``.
+    """
+
+    content: bytes
+    media_type: str
+    parsed_schema_digest: str
+    record: DecodeRecord
+
+
+def _reader_budget_caps(
+    context: AcquisitionContext,
+    request: AcquisitionRequest | None = None,
+) -> dict[str, int]:
+    """Every cap that reaches the decode: the coordinator's, and the recipe's own.
+
+    ``ReaderBudgets.narrowed_by`` takes a per-field minimum, so these can only tighten the
+    family's own defaults and can never buy more room than the family allows. This is the
+    Reader-side twin of ``_parse_limits``, which does the same for the parser.
+
+    This is the only function that builds ``reader_budgets``, so recipe caps are merged here.
+
+    Where the two disagree, the tighter wins on each field independently, and that is
+    ``narrowed_by``'s own rule applied once more here rather than a second policy: a
+    coordinator that caps a source and a recipe that caps itself are both entitled to their
+    ceiling, and neither can raise the other's.
+    """
+
+    caps = {
+        "max_input_bytes": context.max_source_bytes,
+        # The decode output budget, distinct from the fetched-byte budget, mirroring the
+        # hosted Reader budget narrowing.
+        "max_output_bytes": (
+            context.max_source_bytes
+            if context.max_normalized_bytes is None
+            else context.max_normalized_bytes
+        ),
+        "max_rows": context.max_rows,
+        "max_columns": context.max_columns,
+    }
+    stated = None if request is None else request.resource_caps
+    if stated is None:
+        return caps
+    for name, value in stated.items():
+        current = caps.get(name)
+        caps[name] = int(value) if current is None else min(current, int(value))
+    return caps
+
+
+def https_reader_policy_digest() -> str:
+    """Bind the networked decode and its mandatory networkless warm-up as one policy."""
+
+    return canonical_sha256(
+        {
+            "retrieve_decode_and_parse": sandbox_policy_digest("retrieve_decode_and_parse"),
+            "decode_and_parse": sandbox_policy_digest("decode_and_parse"),
+        }
+    )
+
+
+def _decode_in_clean_room(
+    sandbox: CrawlerSandbox,
+    *,
+    request: AcquisitionRequest,
+    context: AcquisitionContext,
+    content: bytes,
+    sealed_format: str,
+) -> _Decoded:
+    """Run the request's pinned Reader on fetched bytes, inside the confinement.
+
+    Two formats meet here and are deliberately not the same thing. The **sealed** format is
+    ``sealed_format``: what the adapter query names, what the snapshot suffix carries, and what
+    the receipt's ``snapshot.data_format`` must equal. The **fetched** format is described by
+    the Reader pin, appears in no format table, and is never sealed. Conflating them would seal
+    an archive under a wire-format name.
+
+    The filename the parser reads is the family's own declaration and never the fetched file's
+    name: it is chosen inside the worker from ``ReaderResult.filename``, so a zip fetched as
+    ``cities.zip`` is parsed as the CSV the Reader produced rather than refused as
+    ``PARSE_SUFFIX_MISMATCH``.
+    """
+
+    pin = request.reader_pin
+    if pin is None:
+        raise SourceContractError(
+            "DECODE_PIN_MISMATCH",
+            "request.reader_pin",
+            "a decode was attempted for a request that entitles no Reader family",
+        )
+    try:
+        sandboxed = sandbox.decode_and_parse(
+            request_id=request.request_id,
+            content=content,
+            reader_pin=dict(pin),
+            output_format=sealed_format,
+            limits=_parse_limits(context),
+            reader_budgets=_reader_budget_caps(context, request),
+        )
+    except AcquisitionSecurityError as error:
+        # A halted refresh names the source it halted on.  The Reader cannot do this itself and
+        # should not be able to: it is handed bytes and no provenance at all, which is the
+        # property that lets its refusals be carried out of the confinement in the first place.
+        # This is the layer that knows the recipe's own name for the source, so this is where
+        # the name is attached.
+        raise AcquisitionSecurityError(
+            error.code,
+            f"source {request.source_id}: {error.detail}",
+        ) from error
+    # The decode operation carries its own attestation. A coordinator pinned to the plain
+    # ``parse`` digest does not authorize a decode and is refused here.
+    _require_sandbox_binding(sandboxed.policy_digest, context)
+    if (
+        sandboxed.parsed is None
+        or sandboxed.content is None
+        or sandboxed.media_type is None
+        or sandboxed.decode_family_id is None
+        or sandboxed.decode_family_version is None
+        or sandboxed.decode_options_digest is None
+        or sandboxed.decode_flags is None
+    ):
+        raise SourceContractError(
+            "SANDBOX_RESULT",
+            "sandbox",
+            "decode result is incomplete",
+        )
+    # The returned table must describe the returned bytes. The response boundary checks this
+    # too; both gates stand, so a worker that returned a table about other bytes has to defeat
+    # the boundary and the adapter rather than either alone.
+    if sandboxed.parsed.input_sha256 != sha256_bytes(sandboxed.content):
+        raise SourceContractError(
+            "SANDBOX_RESULT",
+            "sandbox.parsed.input_sha256",
+            "clean-room result does not bind the exact normalized bytes it returned",
+        )
+    return _Decoded(
+        content=sandboxed.content,
+        media_type=sandboxed.media_type,
+        parsed_schema_digest=sandboxed.parsed.schema_digest,
+        record=DecodeRecord(
+            fetched_content_sha256=sha256_bytes(content),
+            family_id=sandboxed.decode_family_id,
+            family_version=sandboxed.decode_family_version,
+            decode_options_digest=sandboxed.decode_options_digest,
+            flags=sandboxed.decode_flags,
+        ),
+    )
+
+
+def _local_descriptor() -> AdapterDescriptor:
+    return AdapterDescriptor(
+        adapter_id="user.file",
+        adapter_version="1.0.0",
+        source_class="user_file",
+        data_formats=tuple(sorted(DATA_FORMATS)),
+        capabilities=("snapshot",),
+    )
+
+
+@dataclass(frozen=True)
+class LocalFileAdapter:
+    """Descriptor-confined user-file acquisition with sandboxed parsing."""
+
+    input_root: Path
+    sandbox: CrawlerSandbox
+    governance: AdapterGovernance
+    descriptor: AdapterDescriptor = field(default_factory=_local_descriptor)
+
+    def __post_init__(self) -> None:
+        if (
+            not self.input_root.is_absolute()
+            or self.input_root.is_symlink()
+            or not self.input_root.is_dir()
+        ):
+            raise SourceContractError(
+                "INPUT_ROOT",
+                "adapter.input_root",
+                "must be an existing absolute non-symlink directory",
+            )
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        _bind_request(request, self.descriptor)
+        if request.credential_reference_id is not None:
+            raise SourceContractError(
+                "USER_FILE_CREDENTIAL",
+                "request.credential_reference_id",
+                "local files cannot select or receive source credentials",
+            )
+        _require_disjoint_roots(self.input_root, context.snapshot_root)
+        relative_path, data_format, media_type = _local_query(request.query)
+        _authorize_candidate(request, self.governance)
+        if request.reader_pin is not None:
+            # The Reader stage. ``data_format`` here is the **sealed** format and stays what
+            # it is; the query's ``media_type`` describes the **fetched** bytes and is not
+            # used below, because the Reader family declares the media type of what it
+            # produced. Everything after this branch is the direct-fetch path, unchanged.
+            return self._acquire_decoded(
+                request,
+                context,
+                relative_path=relative_path,
+                sealed_format=data_format,
+            )
+        content = _read_confined_file(
+            self.input_root,
+            relative_path=relative_path,
+            max_bytes=context.max_source_bytes,
+        )
+        classification = classify_before_persistence(
+            content,
+            declared_classification=self.governance.declared_classification,
+            lawful_basis_declared=self.governance.lawful_basis_declared,
+        )
+        require_admissible(classification)
+        sandboxed = self.sandbox.parse(
+            request_id=request.request_id,
+            content=content,
+            data_format=data_format,
+            media_type=media_type,
+            filename=PurePosixPath(relative_path).name,
+            limits=_parse_limits(context),
+        )
+        _require_sandbox_binding(sandboxed.policy_digest, context)
+        if (
+            sandboxed.parsed is None
+            or sandboxed.parsed.input_sha256 != classification.content_sha256
+        ):
+            raise SourceContractError(
+                "SANDBOX_RESULT",
+                "sandbox.parsed",
+                "sandbox result does not bind the classified exact source bytes",
+            )
+        snapshot_path = seal_content_addressed_snapshot(
+            context.snapshot_root,
+            content=content,
+            suffix=data_format,
+        )
+        transport_digest = canonical_sha256(
+            {
+                "kind": "descriptor-confined-local-file",
+                "content_sha256": classification.content_sha256,
+                "size_bytes": len(content),
+                "identity_checked_before_after": True,
+                "symlink_hardlink_special_file_rejected": True,
+            }
+        )
+        content_urn = f"urn:sha256:{classification.content_sha256}"
+        return _output(
+            request=request,
+            descriptor=self.descriptor,
+            governance=self.governance,
+            snapshot_path=snapshot_path,
+            content=content,
+            data_format=data_format,
+            media_type=media_type,
+            source_uri=content_urn,
+            final_uri=content_urn,
+            classification=classification.classification,
+            transport_evidence_digest=transport_digest,
+            parsed_schema_digest=sandboxed.parsed.schema_digest,
+        )
+
+    def _acquire_decoded(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+        *,
+        relative_path: str,
+        sealed_format: str,
+    ) -> AcquisitionOutput:
+        """Acquire one source whose recipe pins a Reader.
+
+        The order is the design. The family is warmed up first, so a suspect Reader never
+        touches source data. Then the fetched bytes are decoded before anything is classified
+        or sealed, so classification and the rights gate scan the normalized artifact -- the
+        data a build actually reads -- and the sealed snapshot is the Reader's canonical
+        output. The fetched bytes leave one trace, their digest, which is what gives the
+        derived snapshot a link back to what produced it.
+        """
+
+        assert request.reader_pin is not None
+        _warm_up_family(
+            self.sandbox,
+            pin=request.reader_pin,
+            context=context,
+            sealed_format=sealed_format,
+        )
+        content = _read_confined_file(
+            self.input_root,
+            relative_path=relative_path,
+            max_bytes=context.max_source_bytes,
+        )
+        decoded = _decode_in_clean_room(
+            self.sandbox,
+            request=request,
+            context=context,
+            content=content,
+            sealed_format=sealed_format,
+        )
+        classification = classify_before_persistence(
+            decoded.content,
+            declared_classification=self.governance.declared_classification,
+            lawful_basis_declared=self.governance.lawful_basis_declared,
+        )
+        require_admissible(classification)
+        snapshot_path = seal_content_addressed_snapshot(
+            context.snapshot_root,
+            content=decoded.content,
+            suffix=sealed_format,
+        )
+        transport_digest = canonical_sha256(
+            {
+                "kind": "descriptor-confined-local-file",
+                "content_sha256": decoded.record.fetched_content_sha256,
+                "size_bytes": len(content),
+                "identity_checked_before_after": True,
+                "symlink_hardlink_special_file_rejected": True,
+            }
+        )
+        # The URN names the bytes that were fetched, which is what this source is. The sealed
+        # snapshot is a derivative of them and carries its own digest on the receipt.
+        content_urn = f"urn:sha256:{decoded.record.fetched_content_sha256}"
+        return _output(
+            request=request,
+            descriptor=self.descriptor,
+            governance=self.governance,
+            snapshot_path=snapshot_path,
+            content=decoded.content,
+            data_format=sealed_format,
+            media_type=decoded.media_type,
+            source_uri=content_urn,
+            final_uri=content_urn,
+            classification=classification.classification,
+            transport_evidence_digest=transport_digest,
+            parsed_schema_digest=decoded.parsed_schema_digest,
+            decode=decoded.record,
+        )
+
+
+@dataclass(frozen=True)
+class RangeReaderHttpsAdapter:
+    """Normal registry adapter for one locked HTTPS range plus a pinned Reader."""
+
+    sandbox: CrawlerSandbox
+    allowed_hostnames: tuple[str, ...]
+    governance: AdapterGovernance
+    descriptor: AdapterDescriptor
+    retrieval_limits: RetrievalLimits = field(default_factory=RetrievalLimits)
+
+    def __post_init__(self) -> None:
+        require_crawler_sandbox(self.sandbox)
+        if self.descriptor.source_class != "external_adapter":
+            raise SourceContractError(
+                "ADAPTER_CLASS",
+                "adapter.descriptor.source_class",
+                "range Reader adapter requires external_adapter",
+            )
+        if not self.allowed_hostnames:
+            raise SourceContractError(
+                "EGRESS_ALLOWLIST",
+                "adapter.allowed_hostnames",
+                "range Reader adapter requires an exact non-empty host allowlist",
+            )
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        _bind_request(request, self.descriptor)
+        if request.credential_reference_id is not None:
+            raise SourceContractError(
+                "AUTHENTICATED_SOURCE_REQUIRES_TRUSTED_GATEWAY",
+                "request.credential_reference_id",
+                "range Courier receives no credential",
+            )
+        if request.reader_pin is None:
+            raise SourceContractError(
+                "READER_PIN_REQUIRED",
+                "request.reader_pin",
+                "range Reader acquisition requires an exact Reader pin",
+            )
+        query = _exact_query(request.query, {"revision_lock"})
+        try:
+            lock = RangeRevisionLock.from_dict(query["revision_lock"])
+            pin = ReaderPin(
+                request.reader_pin["family_id"],
+                request.reader_pin["family_version"],
+                request.reader_pin["decode_options"],
+            )
+            budgets = ReaderBudgets().narrowed_by(request.resource_caps)
+        except (AcquisitionSecurityError, KeyError) as error:
+            raise SourceContractError(
+                "RANGE_READER_REQUEST",
+                "request.query",
+                "range Reader authority is malformed",
+            ) from error
+        expected_fetch, expected_decode = default_policy_digests()
+        if context.crawler_sandbox_attestation_digest != range_reader_policy_digest():
+            raise SourceContractError(
+                "SANDBOX_ATTESTATION_MISMATCH",
+                "context.crawler_sandbox_attestation_digest",
+                "context does not pin the composed fetch-plus-decode policies",
+            )
+        _authorize_candidate(request, self.governance)
+        _warm_up_family(
+            self.sandbox,
+            pin=request.reader_pin,
+            context=context,
+            sealed_format="csv",
+            expected_policy_digest=expected_decode,
+        )
+        composed = acquire_range_reader(
+            self.sandbox,
+            request_id=request.request_id,
+            lock=lock,
+            allowed_hostnames=self.allowed_hostnames,
+            retrieval_limits=self.retrieval_limits,
+            reader_pin=pin,
+            reader_budgets=budgets,
+            parse_limits=_parse_limits(context),
+            expected_fetch_policy_digest=expected_fetch,
+            expected_decode_policy_digest=expected_decode,
+        )
+        classification = classify_before_persistence(
+            composed.content,
+            declared_classification=self.governance.declared_classification,
+            lawful_basis_declared=self.governance.lawful_basis_declared,
+        )
+        require_admissible(classification)
+        snapshot_path = seal_content_addressed_snapshot(
+            context.snapshot_root,
+            content=composed.content,
+            suffix="csv",
+        )
+        observation = SourceObservation(
+            source_id=request.source_id,
+            observed_at=self.governance.observation.observed_at,
+            event_time_field=self.governance.observation.event_time_field,
+            available_at_field=self.governance.observation.available_at_field,
+            historical_start=self.governance.observation.historical_start,
+            historical_end=self.governance.observation.historical_end,
+            live_status=self.governance.observation.live_status,
+            publication_delay_seconds=self.governance.observation.publication_delay_seconds,
+            update_frequency_seconds=self.governance.observation.update_frequency_seconds,
+            response_status=self.governance.observation.response_status,
+            evidence=self.governance.observation.evidence,
+        )
+        snapshot_digest = sha256_bytes(composed.content)
+        receipt_members = tuple(_receipt_v2_member(member) for member in composed.fetched_members)
+        receipt = AcquisitionReceiptV2(
+            receipt_id=f"receipt.{request.digest[:24]}",
+            request_digest=request.digest,
+            source_id=request.source_id,
+            adapter=self.descriptor,
+            query_digest=request.query_digest,
+            snapshot=SnapshotReference(
+                content_sha256=snapshot_digest,
+                size_bytes=len(composed.content),
+                media_type=composed.media_type,
+                data_format="csv",
+                storage_name=snapshot_path.name,
+            ),
+            fetched_content_sha256=lock.slices[0].sha256,
+            family_id=pin.family_id,
+            family_version=pin.family_version,
+            decode_options_digest=pin.options_digest,
+            decode_flags=composed.decode_flags,
+            source_uri=lock.object_url,
+            final_uri=lock.object_url,
+            event_time_field=self.governance.observation.event_time_field,
+            observed_at=self.governance.observation.observed_at,
+            available_at=self.governance.observation.available_at,
+            ingested_at=request.requested_at,
+            classification=classification.classification,
+            rights_digest=self.governance.rights.digest,
+            retention_digest=self.governance.retention.digest,
+            observation_digest=observation.digest,
+            transport_evidence_digest=canonical_sha256(
+                [member.hop_document() for member in receipt_members]
+            ),
+            fetched_members=receipt_members,
+            cycle=lock.cycle,
+            revision_lock_digest=lock.digest,
+            source_revision_digest=fetched_source_revision_digest(
+                lock.cycle,
+                lock.full_object_size_bytes,
+                receipt_members,
+            ),
+            full_object_size_bytes=lock.full_object_size_bytes,
+            efficiency_numerator=composed.efficiency_numerator,
+            efficiency_denominator=lock.full_object_size_bytes,
+            fetch_policy_digest=expected_fetch,
+            decode_policy_digest=expected_decode,
+            parsed_schema_digest=composed.parsed_schema_digest,
+        )
+        return AcquisitionOutput(
+            receipt=receipt,
+            observation=observation,
+            parsed_schema_digest=composed.parsed_schema_digest,
+            snapshot_path=snapshot_path,
+        )
+
+
+@dataclass(frozen=True)
+class PublicHttpsAdapter:
+    """Public-only URL/API adapter using the real credential-free crawler boundary."""
+
+    sandbox: CrawlerSandbox
+    allowed_hostnames: tuple[str, ...]
+    governance: AdapterGovernance
+    descriptor: AdapterDescriptor
+    retrieval_limits: RetrievalLimits = field(default_factory=RetrievalLimits)
+
+    def __post_init__(self) -> None:
+        if self.descriptor.source_class not in {"user_url", "user_api", "external_adapter"}:
+            raise SourceContractError(
+                "ADAPTER_CLASS",
+                "adapter.descriptor.source_class",
+                "public HTTPS adapter requires user_url, user_api, or external_adapter",
+            )
+        if not self.allowed_hostnames:
+            raise SourceContractError(
+                "EGRESS_ALLOWLIST",
+                "adapter.allowed_hostnames",
+                "public HTTPS adapter requires an exact non-empty host allowlist",
+            )
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        _bind_request(request, self.descriptor)
+        if request.credential_reference_id is not None:
+            raise SourceContractError(
+                "AUTHENTICATED_SOURCE_REQUIRES_TRUSTED_GATEWAY",
+                "request.credential_reference_id",
+                "generic crawler receives no credential; use a trusted scoped gateway",
+            )
+        url, data_format, filename = _https_query(request.query)
+        _authorize_candidate(request, self.governance)
+        retrieval_limits = replace(
+            self.retrieval_limits,
+            max_response_bytes=min(
+                self.retrieval_limits.max_response_bytes,
+                context.max_source_bytes,
+            ),
+        )
+        if request.reader_pin is None:
+            sandboxed = self.sandbox.retrieve_and_parse(
+                request_id=request.request_id,
+                url=url,
+                allowed_hostnames=self.allowed_hostnames,
+                data_format=data_format,
+                filename=filename,
+                parse_limits=_parse_limits(context),
+                retrieval_limits=retrieval_limits,
+            )
+            decode = None
+        else:
+            pin = request.reader_pin
+            expected_retrieval = sandbox_policy_digest("retrieve_decode_and_parse")
+            expected_decode = sandbox_policy_digest("decode_and_parse")
+            if context.crawler_sandbox_attestation_digest != https_reader_policy_digest():
+                raise SourceContractError(
+                    "SANDBOX_ATTESTATION_MISMATCH",
+                    "context.crawler_sandbox_attestation_digest",
+                    "context does not pin the composed HTTPS retrieval-plus-decode policies",
+                )
+            retrieval_limits = reader_retrieval_limits(
+                retrieval_limits,
+                family_id=str(pin["family_id"]),
+                family_version=str(pin["family_version"]),
+            )
+            _warm_up_family(
+                self.sandbox,
+                pin=pin,
+                context=context,
+                sealed_format=data_format,
+                expected_policy_digest=expected_decode,
+            )
+            sandboxed = self.sandbox.retrieve_decode_and_parse(
+                request_id=request.request_id,
+                url=url,
+                allowed_hostnames=self.allowed_hostnames,
+                reader_pin=pin,
+                output_format=data_format,
+                parse_limits=_parse_limits(context),
+                retrieval_limits=retrieval_limits,
+                reader_budgets=_reader_budget_caps(context, request),
+            )
+            if (
+                sandboxed.fetched_content_sha256 is None
+                or sandboxed.decode_family_id is None
+                or sandboxed.decode_family_version is None
+                or sandboxed.decode_options_digest is None
+                or sandboxed.decode_flags is None
+            ):
+                raise SourceContractError(
+                    "SANDBOX_RESULT",
+                    "sandbox",
+                    "network Reader acquisition result is incomplete",
+                )
+            decode = DecodeRecord(
+                fetched_content_sha256=sandboxed.fetched_content_sha256,
+                family_id=sandboxed.decode_family_id,
+                family_version=sandboxed.decode_family_version,
+                decode_options_digest=sandboxed.decode_options_digest,
+                flags=sandboxed.decode_flags,
+            )
+        if request.reader_pin is None:
+            _require_sandbox_binding(sandboxed.policy_digest, context)
+        elif sandboxed.policy_digest != expected_retrieval:
+            raise SourceContractError(
+                "SANDBOX_ATTESTATION_MISMATCH",
+                "sandbox.policy_digest",
+                "HTTPS Reader acquisition does not bind its retrieval-plus-decode policy",
+            )
+        if (
+            sandboxed.content is None
+            or sandboxed.parsed is None
+            or sandboxed.media_type is None
+            or sandboxed.final_url is None
+            or sandboxed.transport_evidence_digest is None
+        ):
+            raise SourceContractError(
+                "SANDBOX_RESULT",
+                "sandbox",
+                "network acquisition result is incomplete",
+            )
+        classification = classify_before_persistence(
+            sandboxed.content,
+            declared_classification=self.governance.declared_classification,
+            lawful_basis_declared=self.governance.lawful_basis_declared,
+        )
+        require_admissible(classification)
+        if sandboxed.parsed.input_sha256 != classification.content_sha256:
+            raise SourceContractError(
+                "SANDBOX_RESULT",
+                "sandbox.parsed.input_sha256",
+                "sandbox result does not bind the classified exact response bytes",
+            )
+        snapshot_path = seal_content_addressed_snapshot(
+            context.snapshot_root,
+            content=sandboxed.content,
+            suffix=data_format,
+        )
+        return _output(
+            request=request,
+            descriptor=self.descriptor,
+            governance=self.governance,
+            snapshot_path=snapshot_path,
+            content=sandboxed.content,
+            data_format=data_format,
+            media_type=sandboxed.media_type,
+            source_uri=url,
+            final_uri=sandboxed.final_url,
+            classification=classification.classification,
+            transport_evidence_digest=sandboxed.transport_evidence_digest,
+            parsed_schema_digest=sandboxed.parsed.schema_digest,
+            decode=decode,
+            probe_transport=sandboxed.probe_transport,
+            parsed_row_count=len(sandboxed.parsed.rows),
+            parsed_row_digest=row_digest_for(sandboxed.parsed),
+        )
+
+
+def _bind_request(request: AcquisitionRequest, descriptor: AdapterDescriptor) -> None:
+    if (request.adapter_id, request.adapter_version) != (
+        descriptor.adapter_id,
+        descriptor.adapter_version,
+    ):
+        raise SourceContractError(
+            "ADAPTER_BINDING",
+            "request",
+            "request does not bind the exact adapter identity",
+        )
+
+
+def _local_query(query: Any) -> tuple[str, str, str]:
+    data = _exact_query(query, {"relative_path", "data_format", "media_type"})
+    return (
+        _text(data["relative_path"], "query.relative_path", 1_024),
+        _format(data["data_format"]),
+        _text(data["media_type"], "query.media_type", 255),
+    )
+
+
+def _https_query(query: Any) -> tuple[str, str, str]:
+    data = _exact_query(query, {"url", "data_format", "filename"})
+    return (
+        _https_url(data["url"]),
+        _format(data["data_format"]),
+        _text(data["filename"], "query.filename", 255),
+    )
+
+
+def _https_url(value: Any) -> str:
+    """Admit the address as a literal string or as the closed composition object.
+
+    The key set of the query is unchanged; the shape change lives inside one value.  A
+    string behaves exactly as it always has.  The object exists so a model-run address --
+    ``gfs.20260806/06/atmos/gfs.t06z.pgrb2.0p25.f003`` -- can be assembled from bounded
+    literal parts and the rendered cycle coordinates, which whole-string substitution alone
+    could never express.  It is deliberately not free-form interpolation: there is no slot
+    in the shape for a scheme, an authority, a query or a fragment, so those cannot be
+    reached from a recipe at all.
+    """
+
+    if isinstance(value, dict):
+        return _compose_url(value)
+    return _text(value, "query.url", URL_MAX_LENGTH)
+
+
+def _compose_url(value: dict[str, Any]) -> str:
+    """Assemble ``origin`` and an ordered list of path components into one address."""
+
+    if set(value) != {"origin", "path"}:
+        raise SourceContractError(
+            "URL_COMPOSITION",
+            "query.url",
+            "composed address must contain exactly ['origin', 'path']",
+        )
+    origin = _composition_origin(value["origin"])
+    components = _composition_components(value["path"])
+    composed = origin + "/" + "/".join(components)
+    if len(composed) > URL_MAX_LENGTH:
+        raise SourceContractError(
+            "URL_COMPOSITION",
+            "query.url",
+            f"composed address must be no longer than {URL_MAX_LENGTH} characters",
+        )
+    return composed
+
+
+def _composition_origin(origin: Any) -> str:
+    """Return the origin, which carries a scheme, a host, an optional port, and nothing else.
+
+    The origin is a separate field that no part can be composed into, so a composed address
+    can never name a host the recipe did not state.  Everything a URL can carry after the
+    authority -- path, query, fragment, user information -- is refused here rather than
+    stripped, because stripping would silently accept an address the author wrote and the
+    adapter did not fetch.
+    """
+
+    if (
+        not isinstance(origin, str)
+        or not 1 <= len(origin) <= URL_ORIGIN_MAX_LENGTH
+        or not origin.isascii()
+    ):
+        raise SourceContractError(
+            "URL_COMPOSITION",
+            "query.url.origin",
+            f"must be ASCII text no longer than {URL_ORIGIN_MAX_LENGTH} characters",
+        )
+    try:
+        parsed = urlsplit(origin)
+        port = parsed.port
+        host = parsed.hostname
+    except ValueError:
+        raise SourceContractError(
+            "URL_COMPOSITION", "query.url.origin", "is not a parseable origin"
+        ) from None
+    if (
+        parsed.scheme != "https"
+        or not host
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.path
+        or parsed.query
+        or parsed.fragment
+        # Reconstruction equality is the structural claim: the origin is exactly a scheme
+        # and an authority, spelled the one way that round-trips.  It refuses an uppercase
+        # scheme, a stray ``?`` or ``#`` that split into empty components, and any other
+        # spelling whose parse disagrees with its text.
+        or origin != f"https://{parsed.netloc}"
+    ):
+        raise SourceContractError(
+            "URL_COMPOSITION",
+            "query.url.origin",
+            "must be an https scheme with a host and optional port and nothing else",
+        )
+    if port is not None and not 1 <= port <= 65_535:
+        raise SourceContractError("URL_COMPOSITION", "query.url.origin", "states an invalid port")
+    return origin
+
+
+def _composition_components(path: Any) -> list[str]:
+    """Return the joined path components, each built from bounded literal parts."""
+
+    if not isinstance(path, list) or not 1 <= len(path) <= URL_MAX_PATH_COMPONENTS:
+        raise SourceContractError(
+            "URL_COMPOSITION",
+            "query.url.path",
+            f"must be a list of 1 to {URL_MAX_PATH_COMPONENTS} components",
+        )
+    components: list[str] = []
+    for index, component in enumerate(path):
+        component_path = f"query.url.path[{index}]"
+        if not isinstance(component, list) or not 1 <= len(component) <= URL_MAX_COMPONENT_PARTS:
+            raise SourceContractError(
+                "URL_COMPOSITION",
+                component_path,
+                f"must be a list of 1 to {URL_MAX_COMPONENT_PARTS} parts",
+            )
+        for part_index, part in enumerate(component):
+            if not isinstance(part, str) or not _URL_PART.fullmatch(part):
+                raise SourceContractError(
+                    "URL_COMPOSITION",
+                    f"{component_path}[{part_index}]",
+                    f"must be 1 to {URL_PART_MAX_LENGTH} characters of A-Z a-z 0-9 . _ -",
+                )
+        joined = "".join(component)
+        if not 1 <= len(joined) <= URL_COMPONENT_MAX_LENGTH or joined in {".", ".."}:
+            raise SourceContractError(
+                "URL_COMPOSITION",
+                component_path,
+                f"must join to 1 to {URL_COMPONENT_MAX_LENGTH} characters and not to . or ..",
+            )
+        components.append(joined)
+    return components
+
+
+def _exact_query(query: Any, expected: set[str]) -> dict[str, Any]:
+    if not isinstance(query, dict) or set(query) != expected:
+        raise SourceContractError(
+            "QUERY_FIELDS",
+            "request.query",
+            f"must contain exactly {sorted(expected)}",
+        )
+    return query
+
+
+def _format(value: Any) -> str:
+    text = _text(value, "query.data_format", 16)
+    if text not in DATA_FORMATS:
+        raise SourceContractError(
+            "DATA_FORMAT",
+            "query.data_format",
+            "must be one of " + ", ".join(sorted(DATA_FORMATS)),
+        )
+    return text
+
+
+def _text(value: Any, path: str, maximum: int) -> str:
+    if not isinstance(value, str) or not value or len(value) > maximum:
+        raise SourceContractError("TEXT", path, f"must be a string no longer than {maximum}")
+    if any(0xD800 <= ord(char) <= 0xDFFF for char in value):
+        raise SourceContractError("UNICODE", path, "contains invalid Unicode")
+    return value
+
+
+def _require_disjoint_roots(input_root: Path, snapshot_root: Path) -> None:
+    try:
+        common = Path(os.path.commonpath((input_root, snapshot_root)))
+    except ValueError:
+        return
+    if common in {input_root, snapshot_root}:
+        raise SourceContractError(
+            "ROOT_OVERLAP",
+            "context.snapshot_root",
+            "input and snapshot roots must be disjoint",
+        )
+
+
+def _read_confined_file(root: Path, *, relative_path: str, max_bytes: int) -> bytes:
+    path = PurePosixPath(relative_path)
+    if (
+        path.is_absolute()
+        or not path.parts
+        or path.as_posix() != relative_path
+        or any(part in {"", ".", ".."} or part.startswith(".") for part in path.parts)
+    ):
+        raise SourceContractError(
+            "SOURCE_PATH",
+            "query.relative_path",
+            "must be a canonical non-hidden relative POSIX path",
+        )
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    descriptors: list[int] = [os.open(root, directory_flags)]
+    try:
+        for component in path.parts[:-1]:
+            descriptors.append(os.open(component, directory_flags, dir_fd=descriptors[-1]))
+        descriptor = os.open(
+            path.parts[-1],
+            os.O_RDONLY
+            | getattr(os, "O_NOFOLLOW", 0)
+            | getattr(os, "O_NONBLOCK", 0)
+            | getattr(os, "O_CLOEXEC", 0),
+            dir_fd=descriptors[-1],
+        )
+        try:
+            before = os.fstat(descriptor)
+            if (
+                not stat.S_ISREG(before.st_mode)
+                or before.st_nlink != 1
+                or before.st_size < 1
+                or before.st_size > max_bytes
+            ):
+                raise SourceContractError(
+                    "SOURCE_FILE",
+                    "query.relative_path",
+                    "must be one non-hardlinked, non-empty, bounded regular file",
+                )
+            chunks: list[bytes] = []
+            total = 0
+            while total <= max_bytes:
+                chunk = os.read(descriptor, min(64 * 1024, max_bytes + 1 - total))
+                if not chunk:
+                    break
+                chunks.append(chunk)
+                total += len(chunk)
+            after = os.fstat(descriptor)
+            current = os.stat(path.parts[-1], dir_fd=descriptors[-1], follow_symlinks=False)
+            if (
+                total > max_bytes
+                or total != before.st_size
+                or _source_stat_signature(before) != _source_stat_signature(after)
+                or _source_stat_signature(before) != _source_stat_signature(current)
+            ):
+                raise SourceContractError(
+                    "SOURCE_MUTATED",
+                    "query.relative_path",
+                    "source changed during its descriptor-bound read",
+                )
+            return b"".join(chunks)
+        finally:
+            os.close(descriptor)
+    except OSError:
+        raise SourceContractError(
+            "SOURCE_PATH",
+            "query.relative_path",
+            "source path could not be opened without following links",
+        ) from None
+    finally:
+        for descriptor in reversed(descriptors):
+            os.close(descriptor)
+
+
+def _source_stat_signature(info: os.stat_result) -> tuple[int, ...]:
+    """Bind the complete regular-file state used by a confined source observation."""
+
+    return (
+        info.st_dev,
+        info.st_ino,
+        info.st_mode,
+        info.st_nlink,
+        info.st_size,
+        info.st_mtime_ns,
+        info.st_ctime_ns,
+    )