mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/acquisition/result_download.py

@@ -0,0 +1,360 @@
+"""Pure validation for generation-pinned hosted crawler result downloads."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any, NoReturn
+from urllib.parse import parse_qsl, urlsplit
+from uuid import UUID
+
+from mostlyright.data_harness.hosted_crawler_protocol import (
+    CRAWLER_RESULT_MEDIA_TYPE,
+    MAX_CRAWLER_RESULT_BYTES,
+)
+
+RESULT_DOWNLOAD_SCHEMA = "mostlyright-public-crawl-result-download.v1"
+RESULT_MEDIA_TYPE = CRAWLER_RESULT_MEDIA_TYPE
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+_GENERATION = re.compile(r"^[1-9][0-9]{0,31}$")
+_BUCKET = re.compile(r"^[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]$")
+_LOCAL_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
+_GCS_CRC32C = re.compile(r"^[A-Za-z0-9+/]{6}==$")
+
+# The bounded unit a streaming caller transfers and verifies in.  Sixty-four kibibytes is what the
+# content-addressed snapshot verifier in `acquisition/http.py` already reads with, so the two
+# streamed paths on this side of the boundary have one number between them.
+RESULT_CHUNK_BYTES = 65_536
+
+# CRC-32C (Castagnoli), reflected, initialised and finalised with 0xFFFFFFFF -- the exact function
+# GCS reports as `crc32c`.  The table is built once at import instead of once per verification: it
+# is 256 constants derived from the polynomial and nothing about it varies per result.
+_CRC32C_POLYNOMIAL = 0x82F63B78
+
+
+def _build_crc32c_table() -> tuple[int, ...]:
+    table = []
+    for index in range(256):
+        value = index
+        for _ in range(8):
+            value = (value >> 1) ^ (_CRC32C_POLYNOMIAL if value & 1 else 0)
+        table.append(value)
+    return tuple(table)
+
+
+_CRC32C_TABLE = _build_crc32c_table()
+
+
+class ResultDownloadError(ValueError):
+    """One download authority or integrity invariant failed."""
+
+
+@dataclass(frozen=True)
+class ValidatedResultDownload:
+    url: str
+    size: int
+    media_type: str
+    integrity: Mapping[str, str]
+
+
+def _refuse(detail: str) -> NoReturn:
+    """Refuse, and say so in the signature.
+
+    ``NoReturn`` is what makes "an integrity pin this module does not implement can never reach
+    the comparison" a property a checker can see rather than a sentence in a docstring: the
+    fall-through branch in :meth:`ResultBytesVerifier.verify` needs no dead ``return`` to prove
+    it, because the refusal is known not to come back.
+    """
+
+    raise ResultDownloadError(detail)
+
+
+def _timestamp(value: Any, label: str) -> datetime:
+    if not isinstance(value, str) or not value.endswith("Z"):
+        _refuse(f"{label} is invalid")
+    try:
+        parsed = datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
+    except ValueError:
+        _refuse(f"{label} is invalid")
+    if parsed.tzinfo is None:
+        _refuse(f"{label} is invalid")
+    return parsed.astimezone(UTC)
+
+
+def _validate_transfer_url(
+    signed_url: Any,
+    *,
+    base_url: str,
+    crawler_session_id: str,
+    generation: str,
+    storage_authority: Any,
+    expected_gcs_bucket: str | None,
+) -> str:
+    if not isinstance(signed_url, str) or not isinstance(storage_authority, Mapping):
+        _refuse("result transfer authority is invalid")
+    parsed = urlsplit(signed_url)
+    base = urlsplit(base_url)
+    try:
+        port = parsed.port
+    except ValueError:
+        _refuse("result transfer authority is invalid")
+    if parsed.username is not None or parsed.password is not None or parsed.fragment:
+        _refuse("result transfer authority is invalid")
+    if storage_authority.get("kind") == "gcs_bucket":
+        if set(storage_authority) != {"kind", "bucket"}:
+            _refuse("GCS storage authority is not exact")
+        bucket = storage_authority.get("bucket")
+        if not isinstance(bucket, str) or _BUCKET.fullmatch(bucket) is None:
+            _refuse("GCS storage authority is invalid")
+        if bucket != expected_gcs_bucket:
+            _refuse("GCS storage authority differs from the trusted deployment pin")
+        if (
+            parsed.scheme != "https"
+            or parsed.hostname != "storage.googleapis.com"
+            or port is not None
+            or "%" in parsed.path
+            or "\\" in parsed.path
+        ):
+            _refuse("GCS result URL authority is invalid")
+        parts = parsed.path.split("/")
+        if (
+            len(parts) != 5
+            or parts[1] != bucket
+            or parts[2:4] != ["crawler-quarantine", "v2"]
+            or _DIGEST.fullmatch(parts[4]) is None
+        ):
+            _refuse("GCS result URL path is outside its exact quarantine authority")
+        try:
+            query = parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True)
+        except ValueError:
+            _refuse("GCS result URL query is invalid")
+        names = {
+            "X-Goog-Algorithm",
+            "X-Goog-Credential",
+            "X-Goog-Date",
+            "X-Goog-Expires",
+            "X-Goog-SignedHeaders",
+            "X-Goog-Signature",
+            "generation",
+        }
+        if len(query) != len(names) or {key for key, _ in query} != names:
+            _refuse("GCS result URL query is not exact")
+        values = dict(query)
+        if (
+            any(not item for item in values.values())
+            or values["X-Goog-Algorithm"] != "GOOG4-RSA-SHA256"
+            or values["X-Goog-SignedHeaders"] != "host"
+            or values["generation"] != generation
+            or not values["X-Goog-Expires"].isdigit()
+            or not 1 <= int(values["X-Goog-Expires"]) <= 900
+            or re.fullmatch(r"[0-9]{8}T[0-9]{6}Z", values["X-Goog-Date"]) is None
+            or re.fullmatch(r"(?:[0-9A-Fa-f]{2}){64,512}", values["X-Goog-Signature"]) is None
+            or re.fullmatch(
+                r"[^/\x00-\x20]{1,320}/[0-9]{8}/auto/storage/goog4_request",
+                values["X-Goog-Credential"],
+            )
+            is None
+            or values["X-Goog-Credential"].split("/", 2)[1] != values["X-Goog-Date"][:8]
+        ):
+            _refuse("GCS result URL signature policy mismatches")
+        return signed_url
+    if storage_authority != {"kind": "studio_origin"}:
+        _refuse("result storage authority is unsupported")
+    if expected_gcs_bucket is not None:
+        _refuse("Studio-local result authority cannot replace the trusted GCS deployment pin")
+    if generation != "1":
+        _refuse("Studio-local result generation is invalid")
+    prefix = f"/_crawler-results/{crawler_session_id}/downloads/"
+    suffix = parsed.path.removeprefix(prefix)
+    try:
+        download_id = UUID(suffix)
+    except ValueError:
+        _refuse("Studio result URL has no canonical download ID")
+    if (
+        parsed.scheme != base.scheme
+        or parsed.hostname != base.hostname
+        or port != base.port
+        or parsed.query
+        or not parsed.path.startswith(prefix)
+        or str(download_id) != suffix
+    ):
+        _refuse("Studio result URL differs from the configured origin")
+    return signed_url
+
+
+def validate_result_download(
+    download: Any,
+    *,
+    base_url: str,
+    crawler_session_id: str,
+    request_digest: str,
+    source_id: str,
+    source_authority_digest: str,
+    now: datetime,
+    expected_gcs_bucket: str | None,
+) -> ValidatedResultDownload:
+    fields = {
+        "schema_version",
+        "crawler_session_id",
+        "request_digest",
+        "source_id",
+        "source_authority_digest",
+        "media_type",
+        "method",
+        "signed_url",
+        "required_headers",
+        "storage_authority",
+        "object_generation",
+        "bundle_size_bytes",
+        "storage_integrity",
+        "max_result_bytes",
+        "issued_at",
+        "expires_at",
+        "redirect_policy",
+        "single_use",
+    }
+    if not isinstance(download, Mapping) or set(download) != fields:
+        _refuse("result download fields are not exact")
+    issued = _timestamp(download["issued_at"], "result download issued_at")
+    expires = _timestamp(download["expires_at"], "result download expires_at")
+    size = download["bundle_size_bytes"]
+    generation = download["object_generation"]
+    integrity = download["storage_integrity"]
+    if (
+        download["schema_version"] != RESULT_DOWNLOAD_SCHEMA
+        or download["crawler_session_id"] != crawler_session_id
+        or download["request_digest"] != request_digest
+        or download["source_id"] != source_id
+        or download["source_authority_digest"] != source_authority_digest
+        or download["media_type"] != RESULT_MEDIA_TYPE
+        or download["method"] != "GET"
+        or download["required_headers"] != []
+        or type(size) is not int
+        or not 1 <= size <= MAX_CRAWLER_RESULT_BYTES
+        or download["max_result_bytes"] != MAX_CRAWLER_RESULT_BYTES
+        or not isinstance(generation, str)
+        or _GENERATION.fullmatch(generation) is None
+        or download["redirect_policy"] != "deny"
+        or download["single_use"] is not False
+        or issued >= expires
+        or issued > now.astimezone(UTC) + timedelta(minutes=1)
+        or now.astimezone(UTC) >= expires
+        or expires - issued > timedelta(minutes=15)
+        or not isinstance(integrity, Mapping)
+        or set(integrity) != {"kind", "value"}
+    ):
+        _refuse("result download constants, identity, or expiry are invalid")
+    integrity_kind = integrity["kind"]
+    integrity_value = integrity["value"]
+    if not isinstance(integrity_value, str) or not (
+        (integrity_kind == "local_sha256" and _LOCAL_SHA256.fullmatch(integrity_value) is not None)
+        or (integrity_kind == "gcs_crc32c" and _GCS_CRC32C.fullmatch(integrity_value) is not None)
+    ):
+        _refuse("result storage integrity is unsupported")
+    url = _validate_transfer_url(
+        download["signed_url"],
+        base_url=base_url,
+        crawler_session_id=crawler_session_id,
+        generation=generation,
+        storage_authority=download["storage_authority"],
+        expected_gcs_bucket=expected_gcs_bucket,
+    )
+    return ValidatedResultDownload(url, size, RESULT_MEDIA_TYPE, dict(integrity))
+
+
+class ResultBytesVerifier:
+    """Accumulate a result's pinned length and integrity value without holding the result.
+
+    A hosted result is up to ``MAX_CRAWLER_RESULT_BYTES``, and the caller used to have to hand
+    the whole thing over as one ``bytes`` before either check could run.  Both checks are
+    accumulative, so neither ever needed that: the length is a counter and both pinned integrity
+    functions are streaming ones.  A transferring caller feeds this verifier bounded chunks and
+    keeps none of them; ``verify_result_bytes`` feeds it exactly one chunk and is what it always
+    was.  One implementation therefore decides both, so a streamed transfer cannot reach a
+    verdict a whole-buffer transfer would not.
+
+    The verdict order is the order the whole-buffer form used and is load-bearing: length first,
+    then the pinned value.  An unsupported integrity pin is recognised here at construction but
+    refused only in :meth:`verify`, so a download that is both the wrong length and pinned by an
+    unsupported kind still reports its length, exactly as it did before.  Nothing is accepted by
+    default -- a pin this module does not implement leaves both accumulators unset and
+    :meth:`verify` refuses rather than falling through, and ``_refuse`` is typed ``NoReturn`` so
+    that is checkable rather than merely stated here.
+    """
+
+    __slots__ = ("_crc32c", "_length", "_pinned", "_sha256", "_size")
+
+    def __init__(self, validated: ValidatedResultDownload) -> None:
+        kind = validated.integrity.get("kind")
+        value = validated.integrity.get("value")
+        self._size = validated.size
+        self._length = 0
+        self._pinned: str | None = None
+        self._sha256: Any = None
+        self._crc32c: int | None = None
+        if not isinstance(value, str):
+            return
+        if kind == "local_sha256" and value.startswith("sha256:"):
+            self._pinned = value
+            self._sha256 = hashlib.sha256()
+        elif kind == "gcs_crc32c":
+            self._pinned = value
+            self._crc32c = 0xFFFFFFFF
+
+    @property
+    def length(self) -> int:
+        """Bytes accumulated so far, which is what the length check will be decided on."""
+
+        return self._length
+
+    def update(self, chunk: bytes) -> None:
+        """Fold one transferred chunk into the length and the pinned integrity accumulator."""
+
+        self._length += len(chunk)
+        if self._sha256 is not None:
+            self._sha256.update(chunk)
+        elif self._crc32c is not None:
+            checksum = self._crc32c
+            table = _CRC32C_TABLE
+            for value in chunk:
+                checksum = table[(checksum ^ value) & 0xFF] ^ (checksum >> 8)
+            self._crc32c = checksum
+
+    def verify(self) -> None:
+        """Refuse unless the accumulated length and integrity value match the pinned ones."""
+
+        if self._length != self._size:
+            _refuse("result byte length mismatches the generation-pinned size")
+        if self._sha256 is not None:
+            observed = "sha256:" + self._sha256.hexdigest()
+        elif self._crc32c is not None:
+            finalized = ((~self._crc32c) & 0xFFFFFFFF).to_bytes(4, "big")
+            observed = base64.b64encode(finalized).decode("ascii")
+        else:
+            _refuse("result storage integrity is unsupported")
+        pinned = self._pinned or ""
+        if not hmac.compare_digest(pinned.encode("ascii"), observed.encode("ascii")):
+            _refuse("result storage integrity mismatches")
+
+
+def verify_result_bytes(validated: ValidatedResultDownload, content: bytes) -> None:
+    """Verify a result already held whole; the streamed and whole-buffer verdicts are one."""
+
+    verifier = ResultBytesVerifier(validated)
+    verifier.update(content)
+    verifier.verify()
+
+
+__all__ = [
+    "RESULT_CHUNK_BYTES",
+    "ResultBytesVerifier",
+    "ResultDownloadError",
+    "ValidatedResultDownload",
+    "validate_result_download",
+    "verify_result_bytes",
+]

mostlyright/data_harness/acquisition/retention_admission.py

@@ -0,0 +1,248 @@
+"""Fail-closed retention admission at the content-addressed sealing boundary."""
+
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import parse_json
+
+_DELETION_STATUS_VERSION = "harness-source-deletion-status.v1"
+_MAX_EVIDENCE_BYTES = 1 << 20
+
+
+def _refusal(code: str, path: str, message: str) -> Exception:
+    # Import lazily: ``sources`` re-exports adapters that import the HTTP sealer, so importing
+    # its contract module while this module is imported would make that public-package cycle.
+    from mostlyright.data_harness.sources.contracts import SourceContractError
+
+    return SourceContractError(code, path, message)
+
+
+def retention_evidence_root(snapshot_root: Path) -> Path:
+    """The one coordinator evidence directory a snapshot root admits."""
+
+    return snapshot_root.parent / f".{snapshot_root.name}.deletion-evidence"
+
+
+def configured_retention_evidence_root(snapshot_root: Path) -> Path | None:
+    """Return a configured authority root; absence means deletion is not governed here."""
+
+    candidate = retention_evidence_root(snapshot_root)
+    try:
+        os.lstat(candidate)
+    except FileNotFoundError:
+        return None
+    except OSError:
+        raise _refusal(
+            "DELETION_EVIDENCE_ROOT",
+            "evidence_root",
+            "configured deletion evidence root could not be inspected",
+        ) from None
+    return candidate
+
+
+def require_snapshot_admission(
+    *,
+    evidence_root: Path,
+    content_sha256: str,
+) -> None:
+    """Refuse to publish or reuse bytes that immutable deletion evidence withdrew.
+
+    The caller holds the snapshot name's exclusive seal lock.  Deletion writes its status
+    envelope while holding that same lock before unlinking the snapshot, so a queued
+    re-acquisition cannot pass this check between removal and the tombstone write.
+    """
+
+    if len(content_sha256) != 64 or any(
+        character not in "0123456789abcdef" for character in content_sha256
+    ):
+        raise _refusal(
+            "DELETION_ADMISSION",
+            "content_sha256",
+            "must be a lowercase SHA-256",
+        )
+    root_fd = _open_evidence_root(evidence_root)
+    try:
+        status = _read_evidence(root_fd, f"status-{content_sha256}.json")
+        tombstone = _read_evidence(root_fd, f"tombstone-{content_sha256}.json")
+    finally:
+        os.close(root_fd)
+
+    if status is not None:
+        _require_status_binding(status, content_sha256)
+        raise _refusal(
+            "SOURCE_UNAVAILABLE_BY_DELETION",
+            "source",
+            "source bytes were deleted; replay, resume, and refetch must not substitute them",
+        )
+    if tombstone is not None:
+        _require_tombstone_binding(tombstone, content_sha256)
+        raise _refusal(
+            "SOURCE_UNAVAILABLE_BY_DELETION",
+            "source",
+            "source bytes were deleted; replay, resume, and refetch must not substitute them",
+        )
+
+
+def _open_evidence_root(root: Path) -> int:
+    if not root.is_absolute():
+        raise _refusal(
+            "DELETION_EVIDENCE_ROOT",
+            "evidence_root",
+            "must be an existing absolute non-symlink directory",
+        )
+    try:
+        return os.open(
+            root,
+            os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
+        )
+    except OSError:
+        raise _refusal(
+            "DELETION_EVIDENCE_ROOT",
+            "evidence_root",
+            "must be an existing absolute non-symlink directory",
+        ) from None
+
+
+def _read_evidence(root_fd: int, name: str) -> dict[str, Any] | None:
+    try:
+        descriptor = os.open(
+            name,
+            os.O_RDONLY
+            | getattr(os, "O_NOFOLLOW", 0)
+            | getattr(os, "O_NONBLOCK", 0)
+            | getattr(os, "O_CLOEXEC", 0),
+            dir_fd=root_fd,
+        )
+    except FileNotFoundError:
+        return None
+    except OSError:
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            name,
+            "deletion evidence could not be opened safely",
+        ) from None
+    try:
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode) or info.st_size > _MAX_EVIDENCE_BYTES:
+            raise _refusal(
+                "DELETION_EVIDENCE_CONFLICT",
+                name,
+                "deletion evidence must be a bounded regular file",
+            )
+        content = _read_exact(descriptor, info.st_size)
+    finally:
+        os.close(descriptor)
+    try:
+        value = parse_json(content)
+    except Exception:
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            name,
+            "deletion evidence is not valid canonical JSON",
+        ) from None
+    if not isinstance(value, dict):
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            name,
+            "deletion evidence must be an object",
+        )
+    return value
+
+
+def _read_exact(descriptor: int, size: int) -> bytes:
+    chunks: list[bytes] = []
+    remaining = size
+    while remaining:
+        chunk = os.read(descriptor, min(64 * 1024, remaining))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    if remaining or os.read(descriptor, 1):
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            "evidence",
+            "deletion evidence changed while it was being read",
+        )
+    return b"".join(chunks)
+
+
+def _require_status_binding(value: dict[str, Any], content_sha256: str) -> None:
+    expected = {
+        "schema_version",
+        "content_sha256",
+        "status",
+        "source_access_allowed",
+        "refetch_allowed",
+        "replay",
+        "removed_at",
+        "reason_code",
+        "policy_digest",
+        "complete_reference_set_digest",
+        "references",
+        "planned_tombstone_digest",
+    }
+    if (
+        set(value) != expected
+        or value["schema_version"] != _DELETION_STATUS_VERSION
+        or value["content_sha256"] != content_sha256
+        or value["status"] != "deletion-pending"
+        or value["source_access_allowed"] is not False
+        or value["refetch_allowed"] is not False
+        or value["replay"] != "unavailable-by-deletion"
+        or not isinstance(value["references"], list)
+    ):
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            f"status-{content_sha256}.json",
+            "deletion status does not bind an unavailable source",
+        )
+
+
+def _require_tombstone_binding(value: dict[str, Any], content_sha256: str) -> None:
+    expected = {
+        "schema_version",
+        "tombstone_id",
+        "removed_content_sha256",
+        "removed_at",
+        "reason_code",
+        "policy_digest",
+        "complete_reference_set_digest",
+        "references",
+        "byte_removal_confirmed",
+    }
+    if set(value) != expected or value["removed_content_sha256"] != content_sha256:
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            f"tombstone-{content_sha256}.json",
+            "tombstone does not bind confirmed removal of this source",
+        )
+    try:
+        from mostlyright.data_harness.sources.contracts import (
+            DeletedReference,
+            DeletionTombstone,
+            SourceContractError,
+        )
+
+        references = tuple(DeletedReference(**item) for item in value["references"])
+        DeletionTombstone(
+            schema_version=value["schema_version"],
+            tombstone_id=value["tombstone_id"],
+            removed_content_sha256=value["removed_content_sha256"],
+            removed_at=value["removed_at"],
+            reason_code=value["reason_code"],
+            policy_digest=value["policy_digest"],
+            complete_reference_set_digest=value["complete_reference_set_digest"],
+            references=references,
+            byte_removal_confirmed=value["byte_removal_confirmed"],
+        )
+    except (TypeError, SourceContractError):
+        raise _refusal(
+            "DELETION_EVIDENCE_CONFLICT",
+            f"tombstone-{content_sha256}.json",
+            "tombstone has an invalid closed deletion contract",
+        ) from None