mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/channel.py

@@ -0,0 +1,523 @@
+"""The catalogue release channel: one manifest naming the current generation, and the pin for it.
+
+WHY THIS EXISTS. Until ADR 0021 a sealed generation reached a machine exactly one way: an operator
+typed a URL and the archive's SHA-256 next to it (``mr-data catalog-update``). That works for a
+person and not at all for a fleet, and ADR 0021 puts every research lane on a backend worker. A
+worker cannot be typed at. It needs to learn, from its own deploy-owned configuration and from
+nothing else, which generation it is supposed to have -- and it must refuse anything else.
+
+THE TWO DOCUMENTS. A *channel manifest* (``channel.json``) is what the channel publishes: it names
+one release archive, its byte count, its digest, and the generation inside it. A *channel pin* is
+what a deployment configures: the channel's address and the exact manifest digest that deployment
+was rolled out with. Neither is trusted on its own. The manifest is bytes off the network and is
+believed only when its SHA-256 equals the pinned one; the archive is believed only when its
+SHA-256 equals the one the believed manifest states; and the generation inside the archive is then
+put through the same ``update.install_release`` verification a hand-typed install goes through.
+There is no new trust root here -- the pin is the root, and it is deploy-owned.
+
+WHY THE PIN CARRIES A SELECTION ID. This is deliberately the same shape ``hosted_bootstrap``'s
+``MOSTLYRIGHT_WORKER_POLICY_SELECTION_JSON`` uses: every field of the selection is folded into one
+identifier that the reader recomputes, so a deployment cannot half-change the pin -- edit the
+digest and leave the URL, say -- without the worker noticing that the selection no longer binds
+the fields it is made of. A drifted pin is refused at start, not tolerated until a query happens
+to hit a missing entry.
+
+Nothing in this module performs IO. It is the contract; ``channel_client`` is the fetch.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+    parse_json,
+)
+from mostlyright.data_harness.sources.catalog.generation_receipt import is_digest
+from mostlyright.data_harness.sources.catalog.update import (
+    SUPPORTED_STORE_KINDS,
+    ReleaseDescriptor,
+)
+from mostlyright.data_harness.sources.contracts import (
+    SourceContractError,
+    require_canonical_timestamp,
+)
+
+__all__ = [
+    "CATALOG_CHANNEL_PIN_ENV",
+    "CATALOG_CHANNEL_PIN_SCHEMA",
+    "CHANNEL_MANIFEST_OBJECT",
+    "CHANNEL_MANIFEST_SCHEMA",
+    "MAX_CHANNEL_MANIFEST_BYTES",
+    "MAX_CHANNEL_NAME",
+    "CatalogChannelPin",
+    "ChannelManifest",
+    "ChannelRelease",
+    "build_channel_manifest",
+    "channel_manifest_bytes",
+    "parse_catalog_channel_pin",
+    "parse_channel_manifest",
+    "pin_from_environment",
+    "verify_channel_manifest",
+]
+
+#: The manifest's schema token. A reader that does not know this exact string refuses the document
+#: rather than reading the fields it recognises out of it.
+CHANNEL_MANIFEST_SCHEMA = "mr-data-catalog-channel.v1"
+
+#: The one object name a channel publishes its manifest under, relative to the channel base.
+CHANNEL_MANIFEST_OBJECT = "channel.json"
+
+#: The pin's schema token, and the environment variable a deployment states the pin in. The name
+#: follows ``MOSTLYRIGHT_WORKER_POLICY_SELECTION_JSON`` because it is the same kind of thing: one
+#: deploy-owned JSON document that the worker template supplies and the worker recomputes.
+CATALOG_CHANNEL_PIN_SCHEMA = "mostlyright-worker-catalog-channel.v1"
+CATALOG_CHANNEL_PIN_ENV = "MOSTLYRIGHT_WORKER_CATALOG_CHANNEL_JSON"
+
+#: A manifest names one release and nothing else, so it is small by construction. Anything larger
+#: is refused unread rather than parsed and then found to be over-full.
+MAX_CHANNEL_MANIFEST_BYTES = 8_192
+
+#: A channel name is an operator's label for one line of successors ("public", "staging").
+MAX_CHANNEL_NAME = 64
+
+#: The largest archive a channel may name. Equal to ``update.MAX_RELEASE_ARCHIVE_BYTES``: the
+#: manifest cannot promise bytes the installer would refuse to read.
+MAX_CHANNEL_ARCHIVE_BYTES = 1_073_741_824
+
+_CHANNEL_NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,62}[a-z0-9]|[a-z0-9]")
+_ARCHIVE_OBJECT = re.compile(r"[a-z0-9][a-z0-9._-]{0,127}")
+_SELECTION_ID = re.compile(r"[0-9a-f]{64}")
+
+_RELEASE_KEYS = frozenset(
+    {
+        "archive_bytes",
+        "archive_object",
+        "archive_sha256",
+        "entry_count",
+        "predecessor_store_sha256",
+        "published_at",
+        "store_kind",
+        "store_sha256",
+    }
+)
+_MANIFEST_KEYS = frozenset({"channel", "manifest_id", "release", "schema_version"})
+_PIN_KEYS = frozenset({"channel_url", "manifest_sha256", "schema_version", "selection_id"})
+
+
+def _refuse(code: str, path: str, detail: str) -> SourceContractError:
+    return SourceContractError(code, path, detail)
+
+
+@dataclass(frozen=True)
+class ChannelRelease:
+    """The one release a channel manifest names, and everything needed to fetch and check it."""
+
+    store_kind: str
+    store_sha256: str
+    predecessor_store_sha256: str | None
+    entry_count: int
+    published_at: str
+    archive_object: str
+    archive_sha256: str
+    archive_bytes: int
+
+    def __post_init__(self) -> None:
+        if self.store_kind not in SUPPORTED_STORE_KINDS:
+            raise _refuse(
+                "CATALOG_CHANNEL_RELEASE",
+                "release.store_kind",
+                "must name a supported store kind",
+            )
+        for field in ("store_sha256", "archive_sha256"):
+            if not is_digest(getattr(self, field)):
+                raise _refuse(
+                    "CATALOG_CHANNEL_RELEASE",
+                    f"release.{field}",
+                    "must be one exact lowercase SHA-256",
+                )
+        if self.predecessor_store_sha256 is not None:
+            if not is_digest(self.predecessor_store_sha256):
+                raise _refuse(
+                    "CATALOG_CHANNEL_RELEASE",
+                    "release.predecessor_store_sha256",
+                    "must be one exact lowercase SHA-256 or null",
+                )
+            if self.predecessor_store_sha256 == self.store_sha256:
+                raise _refuse(
+                    "CATALOG_CHANNEL_RELEASE",
+                    "release.predecessor_store_sha256",
+                    "must not name the release itself",
+                )
+        if _ARCHIVE_OBJECT.fullmatch(self.archive_object) is None:
+            raise _refuse(
+                "CATALOG_CHANNEL_RELEASE",
+                "release.archive_object",
+                "must be one plain object name with no path separator",
+            )
+        for field in ("entry_count", "archive_bytes"):
+            value = getattr(self, field)
+            if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+                raise _refuse(
+                    "CATALOG_CHANNEL_RELEASE", f"release.{field}", "must be a whole number"
+                )
+        if not 1 <= self.archive_bytes <= MAX_CHANNEL_ARCHIVE_BYTES:
+            raise _refuse(
+                "CATALOG_CHANNEL_RELEASE",
+                "release.archive_bytes",
+                "must be a byte count the installer would accept",
+            )
+        require_canonical_timestamp(self.published_at, "release.published_at")
+
+    @classmethod
+    def from_dict(cls, data: Any) -> ChannelRelease:
+        if not isinstance(data, dict) or set(data) != _RELEASE_KEYS:
+            raise _refuse(
+                "CATALOG_CHANNEL_RELEASE",
+                "release",
+                "must carry exactly the declared release fields",
+            )
+        return cls(
+            store_kind=data["store_kind"],
+            store_sha256=data["store_sha256"],
+            predecessor_store_sha256=data["predecessor_store_sha256"],
+            entry_count=data["entry_count"],
+            published_at=data["published_at"],
+            archive_object=data["archive_object"],
+            archive_sha256=data["archive_sha256"],
+            archive_bytes=data["archive_bytes"],
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "archive_bytes": self.archive_bytes,
+            "archive_object": self.archive_object,
+            "archive_sha256": self.archive_sha256,
+            "entry_count": self.entry_count,
+            "predecessor_store_sha256": self.predecessor_store_sha256,
+            "published_at": self.published_at,
+            "store_kind": self.store_kind,
+            "store_sha256": self.store_sha256,
+        }
+
+    def descriptor(self) -> ReleaseDescriptor:
+        """The release statement the archive itself must carry, as this manifest states it.
+
+        The installer re-reads that statement out of the archive and never takes this one for it;
+        this is what the manifest *claims*, kept in the manifest's own shape so a mismatch between
+        the channel and the bytes is a comparison rather than a surprise.
+        """
+
+        return ReleaseDescriptor(
+            store_kind=self.store_kind,
+            store_sha256=self.store_sha256,
+            predecessor_store_sha256=self.predecessor_store_sha256,
+            entry_count=self.entry_count,
+            published_at=self.published_at,
+        )
+
+
+@dataclass(frozen=True)
+class ChannelManifest:
+    """One channel's statement of its current generation, identified by its own digest."""
+
+    channel: str
+    release: ChannelRelease
+    manifest_id: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.channel, str) or _CHANNEL_NAME.fullmatch(self.channel) is None:
+            raise _refuse(
+                "CATALOG_CHANNEL_MANIFEST",
+                "channel",
+                "must be one lowercase channel name",
+            )
+        if len(self.channel) > MAX_CHANNEL_NAME:
+            raise _refuse("CATALOG_CHANNEL_MANIFEST", "channel", "channel name is too long")
+        if not isinstance(self.manifest_id, str) or _SELECTION_ID.fullmatch(self.manifest_id) is (
+            None
+        ):
+            raise _refuse(
+                "CATALOG_CHANNEL_MANIFEST",
+                "manifest_id",
+                "must be one exact lowercase SHA-256",
+            )
+        if self.manifest_id != _manifest_identity(self.channel, self.release):
+            raise _refuse(
+                "CATALOG_CHANNEL_MANIFEST",
+                "manifest_id",
+                "does not bind the exact channel and release this manifest states",
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "channel": self.channel,
+            "manifest_id": self.manifest_id,
+            "release": self.release.to_dict(),
+            "schema_version": CHANNEL_MANIFEST_SCHEMA,
+        }
+
+
+def _manifest_identity(channel: str, release: ChannelRelease) -> str:
+    """The manifest's identity: its own digest over everything it says except the identity."""
+
+    unsigned = {
+        "channel": channel,
+        "release": release.to_dict(),
+        "schema_version": CHANNEL_MANIFEST_SCHEMA,
+    }
+    return hashlib.sha256(canonical_json_bytes(unsigned)).hexdigest()
+
+
+def parse_channel_manifest(data: Any) -> ChannelManifest:
+    """Parse one channel manifest from already-decoded JSON, recomputing its identity."""
+
+    if not isinstance(data, dict) or set(data) != _MANIFEST_KEYS:
+        raise _refuse(
+            "CATALOG_CHANNEL_MANIFEST",
+            "channel",
+            "must carry exactly the declared channel manifest fields",
+        )
+    if data["schema_version"] != CHANNEL_MANIFEST_SCHEMA:
+        raise _refuse(
+            "CATALOG_CHANNEL_MANIFEST",
+            "channel.schema_version",
+            f"must be {CHANNEL_MANIFEST_SCHEMA}",
+        )
+    return ChannelManifest(
+        channel=data["channel"],
+        release=ChannelRelease.from_dict(data["release"]),
+        manifest_id=data["manifest_id"],
+    )
+
+
+def build_channel_manifest(
+    *,
+    channel: str,
+    store_kind: str,
+    store_sha256: str,
+    predecessor_store_sha256: str | None,
+    entry_count: int,
+    published_at: str,
+    archive_object: str,
+    archive_sha256: str,
+    archive_bytes: int,
+) -> ChannelManifest:
+    """Build the manifest for one already-packaged release. Publishing side only."""
+
+    release = ChannelRelease(
+        store_kind=store_kind,
+        store_sha256=store_sha256,
+        predecessor_store_sha256=predecessor_store_sha256,
+        entry_count=entry_count,
+        published_at=published_at,
+        archive_object=archive_object,
+        archive_sha256=archive_sha256,
+        archive_bytes=archive_bytes,
+    )
+    return ChannelManifest(
+        channel=channel, release=release, manifest_id=_manifest_identity(channel, release)
+    )
+
+
+def channel_manifest_bytes(manifest: ChannelManifest) -> bytes:
+    """The exact bytes to publish as ``channel.json``. Canonical, so the digest is reproducible."""
+
+    raw = canonical_json_bytes(manifest.to_dict())
+    if len(raw) > MAX_CHANNEL_MANIFEST_BYTES:  # pragma: no cover - the shape is closed and small
+        raise _refuse(
+            "CATALOG_CHANNEL_MANIFEST", "channel", "channel manifest is larger than the bound"
+        )
+    return raw
+
+
+def verify_channel_manifest(raw: bytes, *, expected_manifest_sha256: str) -> ChannelManifest:
+    """Believe one channel manifest only when its bytes are the pinned bytes.
+
+    The digest is checked *before* the JSON is parsed, so a manifest that is not the pinned one
+    never reaches the parser at all. That ordering is the whole point: parsing is the only part of
+    this path with any surface, and unpinned bytes must not get to use it.
+    """
+
+    if not isinstance(raw, bytes):
+        raise _refuse("CATALOG_CHANNEL_MANIFEST", "channel", "manifest must be bytes")
+    if not is_digest(expected_manifest_sha256):
+        raise _refuse(
+            "CATALOG_CHANNEL_DIGEST_SPELLING",
+            "expected_manifest_sha256",
+            "must be exactly 64 lowercase hex characters",
+        )
+    if len(raw) > MAX_CHANNEL_MANIFEST_BYTES:
+        raise _refuse(
+            "CATALOG_CHANNEL_MANIFEST", "channel", "channel manifest is larger than the bound"
+        )
+    if hashlib.sha256(raw).hexdigest() != expected_manifest_sha256:
+        raise _refuse(
+            "CATALOG_CHANNEL_DIGEST",
+            "channel",
+            "channel manifest bytes do not match the pinned SHA-256",
+        )
+    try:
+        data = parse_canonical_json(raw)
+    except CanonicalJSONError as error:
+        raise _refuse(
+            "CATALOG_CHANNEL_MANIFEST", "channel", "channel manifest is not canonical JSON"
+        ) from error
+    return parse_channel_manifest(data)
+
+
+@dataclass(frozen=True)
+class CatalogChannelPin:
+    """What one deployment declares about the catalogue it is supposed to be running.
+
+    Deliberately the same shape as ``hosted_bootstrap.WorkerPolicyBinding``: every field folded
+    into one recomputed ``selection_id``, so a partially-edited pin is a refusal rather than a
+    half-applied rollout.
+    """
+
+    channel_url: str
+    manifest_sha256: str
+    selection_id: str
+
+    def __post_init__(self) -> None:
+        _require_channel_url(self.channel_url)
+        if not is_digest(self.manifest_sha256):
+            raise _refuse(
+                "CATALOG_CHANNEL_PIN",
+                "catalog_channel.manifest_sha256",
+                "must be one exact lowercase SHA-256",
+            )
+        if (
+            not isinstance(self.selection_id, str)
+            or _SELECTION_ID.fullmatch(self.selection_id) is None
+        ):
+            raise _refuse(
+                "CATALOG_CHANNEL_PIN",
+                "catalog_channel.selection_id",
+                "must be one exact lowercase SHA-256",
+            )
+        if self.selection_id != _pin_identity(self.channel_url, self.manifest_sha256):
+            raise _refuse(
+                "CATALOG_CHANNEL_PIN",
+                "catalog_channel.selection_id",
+                "does not bind the exact channel address and manifest digest pinned here",
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "channel_url": self.channel_url,
+            "manifest_sha256": self.manifest_sha256,
+            "schema_version": CATALOG_CHANNEL_PIN_SCHEMA,
+            "selection_id": self.selection_id,
+        }
+
+
+def _pin_identity(channel_url: str, manifest_sha256: str) -> str:
+    unsigned = {
+        "channel_url": channel_url,
+        "manifest_sha256": manifest_sha256,
+        "schema_version": CATALOG_CHANNEL_PIN_SCHEMA,
+    }
+    return hashlib.sha256(canonical_json_bytes(unsigned)).hexdigest()
+
+
+def _require_channel_url(value: Any) -> str:
+    """A channel address is one plain HTTPS URL ending in the manifest object, and nothing else.
+
+    No query, no fragment, no user information. A channel is a published location, not a signed
+    capability: if a deployment needs authorisation to read it, that travels in a header the
+    fetcher adds, never in the address a pin is compared by.
+    """
+
+    if not isinstance(value, str) or not 1 <= len(value) <= 2048:
+        raise _refuse("CATALOG_CHANNEL_PIN", "catalog_channel.channel_url", "is not a usable URL")
+    parsed = urlsplit(value)
+    if (
+        parsed.scheme != "https"
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.query
+        or parsed.fragment
+        or not parsed.path.endswith("/" + CHANNEL_MANIFEST_OBJECT)
+    ):
+        raise _refuse(
+            "CATALOG_CHANNEL_PIN",
+            "catalog_channel.channel_url",
+            "must be one https address, without user information, query or fragment, "
+            f"whose path ends in /{CHANNEL_MANIFEST_OBJECT}",
+        )
+    return value
+
+
+def parse_catalog_channel_pin(data: Any) -> CatalogChannelPin:
+    """Parse the deploy-owned pin, recomputing the selection identity it states."""
+
+    if not isinstance(data, dict) or set(data) != _PIN_KEYS:
+        raise _refuse(
+            "CATALOG_CHANNEL_PIN",
+            "catalog_channel",
+            "must carry exactly the declared catalogue channel pin fields",
+        )
+    if data["schema_version"] != CATALOG_CHANNEL_PIN_SCHEMA:
+        raise _refuse(
+            "CATALOG_CHANNEL_PIN",
+            "catalog_channel.schema_version",
+            f"must be {CATALOG_CHANNEL_PIN_SCHEMA}",
+        )
+    return CatalogChannelPin(
+        channel_url=data["channel_url"],
+        manifest_sha256=data["manifest_sha256"],
+        selection_id=data["selection_id"],
+    )
+
+
+def build_catalog_channel_pin(*, channel_url: str, manifest_sha256: str) -> CatalogChannelPin:
+    """Build the pin a deployment should be configured with. Publishing side only."""
+
+    _require_channel_url(channel_url)
+    if not is_digest(manifest_sha256):
+        raise _refuse(
+            "CATALOG_CHANNEL_PIN",
+            "catalog_channel.manifest_sha256",
+            "must be one exact lowercase SHA-256",
+        )
+    return CatalogChannelPin(
+        channel_url=channel_url,
+        manifest_sha256=manifest_sha256,
+        selection_id=_pin_identity(channel_url, manifest_sha256),
+    )
+
+
+def pin_from_environment(environment: Any) -> CatalogChannelPin | None:
+    """Read the pin out of a worker's environment, or answer None when none is configured.
+
+    None means "this deployment does not host a catalogue", which is a legitimate configuration
+    and the one every deployment has before the channel is rolled out to it. It is emphatically
+    not the same as a *malformed* pin, which is a refusal: a deployment that tried to say
+    something about its catalogue and said it wrongly must not start as though it had said
+    nothing.
+    """
+
+    raw = environment.get(CATALOG_CHANNEL_PIN_ENV)
+    if raw is None:
+        return None
+    if not isinstance(raw, str) or not raw.strip():
+        raise _refuse(
+            "CATALOG_CHANNEL_PIN",
+            CATALOG_CHANNEL_PIN_ENV,
+            "is present but empty; unset it to run without a hosted catalogue",
+        )
+    try:
+        data = parse_json(raw)
+    except CanonicalJSONError as error:
+        raise _refuse(
+            "CATALOG_CHANNEL_PIN", CATALOG_CHANNEL_PIN_ENV, "is not one JSON object"
+        ) from error
+    return parse_catalog_channel_pin(data)