mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/health.py

@@ -0,0 +1,447 @@
+"""The source health check: a bounded liveness and citation probe.
+
+The probe answers one question -- is this public source answering, and what exactly did it say --
+and it answers it without downloading the source's payload. It does that by fetching a small
+declared **citation target**: the bulk index, the CKAN ``package_show`` document, the STAC
+collection document. For FAOSTAT that is a 109,120-byte index carrying an update timestamp for
+every dataset, which is how cadence is learned without touching a 33,921,825-byte archive. If the
+citation target itself exceeds the cap, the probe refuses. There is no fallback to an unbounded
+fetch, and there is no ranged request anywhere on this path.
+
+**The probe reports facts. It never reaches a verdict.** It does not import the fitness
+adjudicator, the rights rule, or the catalog gate, and it never returns "selected", "rejected" or
+"human_escalation_required". A probe that started issuing verdicts would be a second gate, and two
+gates disagree. A test proves this structurally rather than trusting the sentence you just read --
+and note that those three words appear in this docstring and nowhere else in the module, which is
+exactly why that test strips docstrings before it looks.
+
+Two consequences of that reporting discipline are worth stating.
+
+*A blocked egress is not a dead source.* An allowlist, DNS, URL-policy or TLS failure propagates
+untouched, because recording it as "unreachable" would write a local misconfiguration into the
+catalog as a fact about the world. Only codes describing how the endpoint *answered* become an
+unreachable probe.
+
+*A refusal is still citable.* Every probe emits an ``EvidenceReference``, including the
+unreachable case, where the citation attests to the refusal rather than to a document. A liveness
+claim nobody can check later is not evidence.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from mostlyright.data_harness.acquisition.http import PinnedHttpsRetriever, RetrievalLimits
+from mostlyright.data_harness.acquisition.url_policy import AcquisitionSecurityError
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.sources.contracts import (
+    EvidenceReference,
+    SourceContractError,
+    SourceObservation,
+    _CanonicalContract,
+)
+
+HEALTH_PROBE_CONTRACT_VERSION = "harness-source-health-probe.v1"
+
+MAX_CITATION_BYTES = 1 * 1024 * 1024
+
+CITATION_PROBE_LIMITS = RetrievalLimits(
+    max_response_bytes=MAX_CITATION_BYTES,
+    max_redirects=2,
+    max_requests=3,
+    allowed_media_types=("application/json",),
+)
+
+# Codes that describe how the endpoint answered. These become an unreachable probe. Every other
+# acquisition code describes this side of the connection and propagates untouched.
+ENDPOINT_FAULT_CODES = frozenset(
+    {
+        "HTTP_STATUS",
+        "MEDIA_TYPE",
+        "RESPONSE_LIMIT",
+        "CONTENT_LENGTH",
+        "CONTENT_ENCODING",
+        "REDIRECT_LIMIT",
+        "REDIRECT_LOCATION",
+    }
+)
+
+# The parked item, named verbatim so a 401 reads as a scope boundary rather than a prompt.
+AUTHENTICATED_ADAPTER_PARKED = (
+    "this endpoint requires authentication and is out of scope; authenticated source adapters are "
+    "parked, and their trigger is the first real credentialed need plus a broker trust review"
+)
+
+# Ten years, matching the bound SourceObservation enforces on its second-valued fields.
+MAX_DECLARED_DELAY_SECONDS = 10 * 365 * 24 * 60 * 60
+
+_STATUS_TAIL = re.compile(r"status (\d{3})$")
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_TIMESTAMP = re.compile(
+    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z$"
+)
+
+
+def citation_probe_limits(*media_types: str) -> RetrievalLimits:
+    """The citation-probe caps, narrowed to one or more media types.
+
+    STAC and SDMX citation targets are not JSON in every case, and they must not have to redefine
+    the caps to say so -- a second copy of the caps is a second thing to forget to tighten.
+
+    Which non-JSON types actually reach the wire is the transport's decision, not this function's.
+    A probe is narrowed onto the coordinator's retriever, so the type named here must be one that
+    retriever's allowlist carries or one of the transport's own ``METADATA_MEDIA_TYPES`` -- the
+    closed metadata channel that exists because an SDMX structure message is not a payload format
+    and so can never be in the format table a default retriever is built from.
+    """
+
+    if not media_types:
+        raise SourceContractError(
+            "EMPTY",
+            "media_types",
+            "a citation probe must allow at least one media type",
+        )
+    return RetrievalLimits(
+        max_response_bytes=CITATION_PROBE_LIMITS.max_response_bytes,
+        max_redirects=CITATION_PROBE_LIMITS.max_redirects,
+        max_requests=CITATION_PROBE_LIMITS.max_requests,
+        allowed_media_types=tuple(media_types),
+    )
+
+
+@dataclass(frozen=True)
+class SourceHealthProbe(_CanonicalContract):
+    """What one bounded probe of one public source observed, and nothing more."""
+
+    source_id: str
+    citation_uri: str
+    probed_at: str
+    reachable: bool
+    response_status: int | None = None
+    media_type: str | None = None
+    content_sha256: str | None = None
+    content_bytes: int | None = None
+    refusal_code: str | None = None
+    refusal_detail: str | None = None
+    transport_evidence_digest: str | None = None
+    schema_version: str = HEALTH_PROBE_CONTRACT_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != HEALTH_PROBE_CONTRACT_VERSION:
+            raise SourceContractError(
+                "VERSION",
+                "probe.schema_version",
+                f"must equal {HEALTH_PROBE_CONTRACT_VERSION!r}",
+            )
+        _identifier(self.source_id, "probe.source_id")
+        _https_uri(self.citation_uri, "probe.citation_uri")
+        _timestamp(self.probed_at, "probe.probed_at")
+        if type(self.reachable) is not bool:
+            raise SourceContractError("BOOLEAN", "probe.reachable", "must be a boolean")
+        # The two states are mutually exclusive by construction: a reachable probe carries what it
+        # saw, an unreachable one carries why it saw nothing. Neither can carry both.
+        if self.reachable:
+            _integer(self.response_status, "probe.response_status", minimum=200, maximum=299)
+            _media_type(self.media_type, "probe.media_type")
+            _digest(self.content_sha256, "probe.content_sha256")
+            _integer(
+                self.content_bytes,
+                "probe.content_bytes",
+                minimum=0,
+                maximum=MAX_CITATION_BYTES,
+            )
+            _digest(self.transport_evidence_digest, "probe.transport_evidence_digest")
+            if self.refusal_code is not None or self.refusal_detail is not None:
+                raise SourceContractError(
+                    "PROBE_STATE",
+                    "probe.refusal_code",
+                    "a reachable probe carries no refusal",
+                )
+        else:
+            if self.refusal_code not in ENDPOINT_FAULT_CODES:
+                raise SourceContractError(
+                    "PROBE_REFUSAL",
+                    "probe.refusal_code",
+                    f"must be one of {sorted(ENDPOINT_FAULT_CODES)}",
+                )
+            _text(self.refusal_detail, "probe.refusal_detail", maximum=500)
+            for name in ("media_type", "content_sha256", "content_bytes"):
+                if getattr(self, name) is not None:
+                    raise SourceContractError(
+                        "PROBE_STATE",
+                        f"probe.{name}",
+                        "an unreachable probe retains no content facts",
+                    )
+            if self.response_status is not None:
+                _integer(
+                    self.response_status,
+                    "probe.response_status",
+                    minimum=100,
+                    maximum=599,
+                )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "source_id": self.source_id,
+            "citation_uri": self.citation_uri,
+            "probed_at": self.probed_at,
+            "reachable": self.reachable,
+            "response_status": self.response_status,
+            "media_type": self.media_type,
+            "content_sha256": self.content_sha256,
+            "content_bytes": self.content_bytes,
+            "refusal_code": self.refusal_code,
+            "refusal_detail": self.refusal_detail,
+            "transport_evidence_digest": self.transport_evidence_digest,
+        }
+
+
+def probe_source_health(
+    *,
+    source_id: str,
+    citation_uri: str,
+    retriever: PinnedHttpsRetriever,
+    probed_at: str,
+    limits: RetrievalLimits = CITATION_PROBE_LIMITS,
+) -> SourceHealthProbe:
+    """Fetch one bounded citation target and record exactly what came back.
+
+    ``limits`` is accepted so a caller can narrow the media types, and it governs the fetch: it is
+    threaded into the transport through ``retriever.narrowed`` rather than merely validated, so a
+    probe declaring ``application/json`` under a 1 MiB cap really does refuse an HTML interstitial
+    served by a retriever whose own allowlist would have accepted it.
+
+    Two independent ceilings apply, and both are needed. ``narrowed`` refuses anything looser than
+    what the *coordinator* configured, so a probe can never escape the transport it was handed.
+    That is not enough on its own: a coordinator's retriever is sized for harvesting, so limits
+    well inside it are still far outside what "a bounded citation probe" means. So every
+    transport-shape cap this function accepts is also checked against ``CITATION_PROBE_LIMITS``
+    before anything is fetched. Checking only the byte cap would have left the redirect and
+    request caps enforced by nothing but callers using ``citation_probe_limits`` -- a bound that
+    holds while everyone takes the sanctioned route is the pattern this package refuses everywhere
+    else.
+    """
+
+    for name in ("max_response_bytes", "max_redirects", "max_requests"):
+        asked = getattr(limits, name)
+        if asked > getattr(CITATION_PROBE_LIMITS, name):
+            raise SourceContractError(
+                "PROBE_LIMITS",
+                f"limits.{name}",
+                f"a citation probe may not exceed {getattr(CITATION_PROBE_LIMITS, name)}; "
+                f"{asked} was asked for",
+            )
+    bounded = retriever.narrowed(limits)
+    try:
+        retrieved = bounded.retrieve(citation_uri)
+    except AcquisitionSecurityError as error:
+        if error.code not in ENDPOINT_FAULT_CODES:
+            # Egress, DNS, URL policy, TLS, concurrency and rate-limit failures are facts about
+            # this side of the connection. Recording one as a dead source would put a local
+            # misconfiguration into the catalog as a claim about the world.
+            raise
+        status = _status_from_detail(error.detail)
+        return SourceHealthProbe(
+            source_id=source_id,
+            citation_uri=citation_uri,
+            probed_at=probed_at,
+            reachable=False,
+            response_status=status,
+            refusal_code=error.code,
+            refusal_detail=_refusal_detail(error, status),
+        )
+    return SourceHealthProbe(
+        source_id=source_id,
+        citation_uri=citation_uri,
+        probed_at=probed_at,
+        reachable=True,
+        response_status=retrieved.hops[-1].status,
+        media_type=retrieved.media_type,
+        content_sha256=retrieved.content_sha256,
+        content_bytes=len(retrieved.content),
+        transport_evidence_digest=retrieved.transport_evidence_digest,
+    )
+
+
+def probe_citation(probe: SourceHealthProbe) -> EvidenceReference:
+    """The citation this probe leaves behind.
+
+    For a reachable probe the citation attests to a document: its uri, when it was read, and the
+    digest of its exact bytes. For an unreachable probe there are no bytes, so the citation
+    attests to the *refusal* -- a canonical digest over the refusal code, the uri and the probe
+    time. Either way the claim is checkable later, which is the point.
+    """
+
+    if not isinstance(probe, SourceHealthProbe):
+        raise SourceContractError("TYPE", "probe", "must be a SourceHealthProbe")
+    if probe.reachable:
+        assert probe.content_sha256 is not None
+        assert probe.media_type is not None
+        return EvidenceReference(
+            uri=probe.citation_uri,
+            observed_at=probe.probed_at,
+            content_sha256=probe.content_sha256,
+            media_type=probe.media_type,
+        )
+    return EvidenceReference(
+        uri=probe.citation_uri,
+        observed_at=probe.probed_at,
+        content_sha256=canonical_sha256(
+            {
+                "refusal_code": probe.refusal_code,
+                "citation_uri": probe.citation_uri,
+                "probed_at": probe.probed_at,
+            }
+        ),
+        media_type="application/json",
+    )
+
+
+def build_probe_observation(
+    *,
+    probe: SourceHealthProbe,
+    declared_updated_at: str,
+    historical_start: str,
+    historical_end: str,
+    update_frequency_seconds: int,
+    event_time_field: str | None = None,
+    available_at_field: str | None = None,
+) -> SourceObservation:
+    """Turn one probe plus the publisher's own declared update time into a source observation.
+
+    This is the object a ``CatalogEntry`` embeds, which is what stops the probe's view of a source
+    and the entry's facts about it from drifting apart.
+    """
+
+    if not isinstance(probe, SourceHealthProbe):
+        raise SourceContractError("TYPE", "probe", "must be a SourceHealthProbe")
+    probed = _timestamp(probe.probed_at, "probe.probed_at")
+    declared = _timestamp(declared_updated_at, "declared_updated_at")
+    delay = int((probed - declared).total_seconds())
+    if delay < 0:
+        # Validate ordering only; timestamp plausibility is outside this function.
+        delay = 0
+    if delay > MAX_DECLARED_DELAY_SECONDS:
+        raise SourceContractError(
+            "PROBE_STALE_DECLARATION",
+            "declared_updated_at",
+            f"the declared update is {delay} seconds old, beyond the "
+            f"{MAX_DECLARED_DELAY_SECONDS}-second bound; a decade-stale claim is a fact worth "
+            "surfacing rather than clamping into range",
+        )
+    _integer(
+        update_frequency_seconds,
+        "update_frequency_seconds",
+        minimum=0,
+        maximum=MAX_DECLARED_DELAY_SECONDS,
+    )
+    return SourceObservation(
+        source_id=probe.source_id,
+        observed_at=probe.probed_at,
+        event_time_field=event_time_field,
+        available_at_field=available_at_field,
+        historical_start=historical_start,
+        historical_end=historical_end,
+        live_status="live" if probe.reachable else "dead",
+        publication_delay_seconds=delay,
+        update_frequency_seconds=update_frequency_seconds,
+        response_status=probe.response_status,
+        evidence=(probe_citation(probe),),
+    )
+
+
+def derive_update_frequency_seconds(earlier: str, later: str) -> int:
+    """Whole seconds between two declared update timestamps of the same source."""
+
+    first = _timestamp(earlier, "earlier")
+    second = _timestamp(later, "later")
+    seconds = int((second - first).total_seconds())
+    if seconds <= 0:
+        raise SourceContractError(
+            "PROBE_INTERVAL",
+            "later",
+            "the later declared update must follow the earlier one",
+        )
+    if seconds > MAX_DECLARED_DELAY_SECONDS:
+        raise SourceContractError(
+            "PROBE_INTERVAL",
+            "later",
+            f"an update interval of {seconds} seconds exceeds the observation bound",
+        )
+    return seconds
+
+
+def _status_from_detail(detail: str) -> int | None:
+    match = _STATUS_TAIL.search(detail)
+    if match is None:
+        return None
+    return int(match.group(1))
+
+
+def _refusal_detail(error: AcquisitionSecurityError, status: int | None) -> str:
+    if status in {401, 403}:
+        return f"HTTP {status}: {AUTHENTICATED_ADAPTER_PARKED}"
+    return error.detail[:500]
+
+
+def _text(value: Any, path: str, *, maximum: int) -> str:
+    if not isinstance(value, str) or not value or len(value) > maximum:
+        raise SourceContractError(
+            "TEXT",
+            path,
+            f"must be a non-empty string no longer than {maximum} characters",
+        )
+    return value
+
+
+def _identifier(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=128)
+    if not re.fullmatch(r"[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*", text):
+        raise SourceContractError(
+            "IDENTIFIER",
+            path,
+            "must be a lowercase dotted/dashed/underscored identifier",
+        )
+    return text
+
+
+def _https_uri(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=2_048)
+    if not text.startswith("https://") or len(text) <= len("https://"):
+        raise SourceContractError("HTTPS_URI", path, "must be an HTTPS URI")
+    return text
+
+
+def _timestamp(value: Any, path: str) -> datetime:
+    text = _text(value, path, maximum=32)
+    if not _TIMESTAMP.fullmatch(text):
+        raise SourceContractError("TIMESTAMP", path, "must be a canonical RFC 3339 UTC timestamp")
+    return datetime.fromisoformat(text[:-1] + "+00:00")
+
+
+def _media_type(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=255)
+    if not re.fullmatch(r"[a-z0-9][a-z0-9!#$&^_.+-]*/[a-z0-9][a-z0-9!#$&^_.+-]*", text):
+        raise SourceContractError("MEDIA_TYPE", path, "must be a lowercase type/subtype")
+    return text
+
+
+def _digest(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=64)
+    if not _SHA256.fullmatch(text):
+        raise SourceContractError("SHA256", path, "must be a lowercase 64-character SHA-256")
+    return text
+
+
+def _integer(value: Any, path: str, *, minimum: int, maximum: int) -> int:
+    if type(value) is not int or not minimum <= value <= maximum:
+        raise SourceContractError(
+            "INTEGER",
+            path,
+            f"must be an integer in [{minimum}, {maximum}]",
+        )
+    return value

mostlyright/data_harness/sources/catalog/hosted_catalog.py

@@ -0,0 +1,105 @@
+"""Fetch-at-start: giving one hosted worker the exact sealed catalogue its deployment pinned.
+
+A local install of the catalogue is a thing a person did once and can see in their home folder. A
+worker has neither -- it is a fresh container per attempt, and the only durable statement about
+what it should be running is its own deploy-owned configuration. So the worker does at start what
+the person did by hand: reads the pin, fetches the pinned manifest, installs the generation the
+manifest names, and verifies it with the real store loader before anything can query it.
+
+THREE OUTCOMES, ALL EXPLICIT.
+
+* No pin configured -> ``None``. This deployment does not host a catalogue. It is the state every
+  deployment is in before the channel is rolled out to it, and it must not be an error.
+* Pin configured and satisfied -> a receipt naming the generation now on disk.
+* Pin configured and not satisfiable -> a refusal, and the worker does not start.
+
+The third is the one worth being deliberate about. A worker that cannot get the catalogue it was
+told to have could plausibly carry on -- discovery would simply record the catalogue lane as
+unavailable, which is a state the product already models. It does not carry on, because the two
+situations are indistinguishable from the outside and only one of them is intended: "this
+deployment has no catalogue" is a decision, and "this deployment's catalogue fetch failed" is a
+fault, and a fault that renders as a decision is how a fleet quietly stops using its catalogue.
+
+CACHING. The generation lands under a writable path in the container -- ``/tmp`` by default, which
+on Cloud Run is an in-memory filesystem, and the install is append-only, so a warm instance that
+already has the pinned generation re-verifies it and downloads nothing. There is no shared mount:
+the hosted worker image mounts no volume for any other read-only asset either, and introducing one
+for the catalogue would make a per-instance cache into shared mutable state.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.sources.catalog.channel import pin_from_environment
+from mostlyright.data_harness.sources.catalog.channel_client import (
+    ChannelTransport,
+    install_from_channel,
+    metadata_access_token,
+)
+
+__all__ = [
+    "CATALOG_ROOT_ENV",
+    "DEFAULT_WORKER_CATALOG_ROOT",
+    "HOSTED_CATALOG_RECEIPT_SCHEMA",
+    "ensure_hosted_catalog",
+    "worker_catalog_root",
+]
+
+#: Where the worker keeps the generation it fetched. Overridable so a job image with a different
+#: writable layout, and every test, can put it somewhere else.
+CATALOG_ROOT_ENV = "MOSTLYRIGHT_WORKER_CATALOG_DIR"
+DEFAULT_WORKER_CATALOG_ROOT = "/tmp/mostlyright-catalog"
+
+HOSTED_CATALOG_RECEIPT_SCHEMA = "mostlyright-worker-catalog-fetch.v1"
+
+
+def worker_catalog_root(environment: Mapping[str, str] | None = None) -> Path:
+    """The folder this worker installs the pinned generation into."""
+
+    env = os.environ if environment is None else environment
+    return Path(env.get(CATALOG_ROOT_ENV) or DEFAULT_WORKER_CATALOG_ROOT)
+
+
+def ensure_hosted_catalog(
+    environment: Mapping[str, str] | None = None,
+    *,
+    transport: ChannelTransport | None = None,
+    authorization: str | None = None,
+) -> dict[str, Any] | None:
+    """Put the pinned generation on this worker, or answer None when none is pinned.
+
+    ``authorization`` is asked of the metadata server when the caller does not supply one, which
+    is what a real worker wants and what no test wants. Passing an empty string selects an
+    unauthenticated fetch explicitly, for a channel that is world-readable.
+    """
+
+    env = os.environ if environment is None else environment
+    pin = pin_from_environment(env)
+    if pin is None:
+        return None
+    if authorization is None:
+        authorization = metadata_access_token(transport=transport)
+    elif authorization == "":
+        authorization = None
+    root = worker_catalog_root(env)
+    root.mkdir(parents=True, exist_ok=True)
+    report = install_from_channel(
+        root, pin, transport=transport, authorization=authorization or None
+    )
+    return {
+        "schema_version": HOSTED_CATALOG_RECEIPT_SCHEMA,
+        "pin_selection_id": pin.selection_id,
+        "channel_url": pin.channel_url,
+        "manifest_sha256": pin.manifest_sha256,
+        "status": report.status,
+        "store_kind": report.state.store_kind,
+        "store_sha256": report.state.store_sha256,
+        "archive_sha256": report.state.archive_sha256,
+        "entry_count": report.state.entry_count,
+        "catalog_root": str(root),
+        "generation_root": os.path.join(str(root), report.state.generation_path),
+    }