mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ingestion/gcs_store.py

@@ -0,0 +1,1162 @@
+"""Generation-fenced Google Cloud Storage implementation of :mod:`ingestion.store`.
+
+The ingestion contracts deliberately do not mention a filesystem.  This implementation keeps
+that promise: manifests use Cloud Storage's object generation as their CAS token and chunks are
+create-only objects named by their SHA-256.  The one local concern is a bounded private staging
+directory, needed to learn a stream's digest before naming its content-addressed object.  Bytes
+are never accumulated into one Python object.
+
+``GoogleCloudStorageJSONTransport`` uses only the standard library and asks a workload identity
+for a fresh access token for every request.  Callers can instead supply a transport implementing
+``GCSObjectTransport``; this is how emulators and the deterministic fake in the tests exercise
+lost acknowledgements and concurrent writers.  Neither store form accepts, persists, or logs a
+long-lived plaintext capability.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import http.client
+import os
+import re
+import stat
+from collections.abc import Callable, Iterable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+from urllib.parse import quote, urlencode
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+    parse_json,
+)
+from mostlyright.data_harness.ingestion.contracts import (
+    ChunkRef,
+    IngestionConflict,
+    IngestionCorrupt,
+    IngestionError,
+    IngestionManifestV1,
+)
+from mostlyright.data_harness.ingestion.spool import SpooledChunk
+from mostlyright.data_harness.ingestion.store import (
+    DEFAULT_MAX_SPOOL_BYTES,
+    DEFAULT_MAX_SPOOL_TOTAL_BYTES,
+    DEFAULT_STALE_SPOOL_AGE_SECONDS,
+    MAX_MANIFEST_BYTES,
+    LocalIngestionStore,
+)
+
+if TYPE_CHECKING:
+    from mostlyright.data_harness.hosted_ingestion_contract import (
+        IngestionDeliveryBindingV1,
+        IngestionManifestReferenceV1,
+    )
+
+GCS_STORAGE_HOST = "storage.googleapis.com"
+GCS_METADATA_HOST = "169.254.169.254"
+GCS_METADATA_TOKEN_PATH = "/computeMetadata/v1/instance/service-accounts/default/token"
+GCS_STORAGE_SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
+GCS_READ_CHUNK_BYTES = 1024 * 1024
+GCS_METADATA_BYTES = 64 * 1024
+GCS_TOKEN_BYTES = 16 * 1024
+
+_BUCKET = re.compile(r"[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]\Z")
+_PREFIX_SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
+_GENERATION = re.compile(r"[1-9][0-9]*\Z")
+
+
+class GCSStorageError(RuntimeError):
+    """A transport failure whose text is safe to report but deliberately nonspecific."""
+
+
+class GCSObjectNotFound(GCSStorageError):
+    """The named object does not exist."""
+
+
+class GCSPreconditionFailed(GCSStorageError):
+    """A create/CAS/read-generation precondition did not hold."""
+
+
+@dataclass(frozen=True)
+class GCSObjectVersion:
+    """A GCS object identity suitable for an exact subsequent read or CAS."""
+
+    generation: str
+    byte_length: int
+
+    def __post_init__(self) -> None:
+        if _GENERATION.fullmatch(self.generation) is None:
+            raise ValueError("generation must be one positive decimal string")
+        if (
+            isinstance(self.byte_length, bool)
+            or not isinstance(self.byte_length, int)
+            or self.byte_length < 0
+        ):
+            raise ValueError("byte_length must be one non-negative integer")
+
+
+@dataclass(frozen=True)
+class GCSObjectRead:
+    """One generation-pinned body stream; consuming it never requires object-sized memory."""
+
+    version: GCSObjectVersion
+    parts: Iterable[bytes]
+    closer: Callable[[], None] = lambda: None
+
+    def close(self) -> None:
+        """Release an unread transport response; this operation is idempotent."""
+
+        self.closer()
+
+
+@runtime_checkable
+class GCSObjectTransport(Protocol):
+    """The small create/CAS/get subset the store needs from Cloud Storage.
+
+    ``if_generation_match`` is either ``"0"`` for create-only, a generation returned by
+    :meth:`get`, or ``None`` for an unconstrained metadata lookup.  Implementations must consume
+    ``parts`` at most once and must not return success before the server has made the full object
+    durable.  This intentionally small protocol makes the production HTTP transport replaceable
+    by a GCS emulator without making a bearer token part of the store API.
+    """
+
+    def put(
+        self,
+        *,
+        bucket: str,
+        name: str,
+        parts: Iterable[bytes],
+        byte_length: int,
+        if_generation_match: str,
+    ) -> GCSObjectVersion:
+        """Create or CAS-replace one object, or raise a typed transport error."""
+
+    def get(
+        self,
+        *,
+        bucket: str,
+        name: str,
+        if_generation_match: str | None = None,
+    ) -> GCSObjectRead:
+        """Return an optionally generation-pinned stream, or raise a typed transport error."""
+
+
+def workload_identity_access_token() -> str:
+    """Read one short-lived storage token directly from the Cloud Run metadata service.
+
+    The result exists only in the call stack of a subsequent HTTP request.  It is deliberately
+    neither cached nor accepted as a constructor argument, so a persisted ingestion checkpoint
+    cannot accidentally become a bearer-capability store.
+    """
+
+    query = urlencode({"scopes": GCS_STORAGE_SCOPE})
+    connection = http.client.HTTPConnection(GCS_METADATA_HOST, timeout=10.0)
+    try:
+        connection.request(
+            "GET",
+            GCS_METADATA_TOKEN_PATH + "?" + query,
+            headers={"Metadata-Flavor": "Google"},
+        )
+        response = connection.getresponse()
+        raw = _read_bounded_response(response, GCS_TOKEN_BYTES)
+        if response.status != 200:
+            raise GCSStorageError("workload identity did not return an access token")
+    except (OSError, http.client.HTTPException) as error:
+        raise GCSStorageError("workload identity could not be reached") from error
+    finally:
+        connection.close()
+    try:
+        document = parse_json(raw)
+    except CanonicalJSONError as error:
+        raise GCSStorageError("workload identity returned invalid token metadata") from error
+    token = document.get("access_token") if isinstance(document, dict) else None
+    if (
+        not isinstance(token, str)
+        or not token
+        or len(token.encode("utf-8")) > 8192
+        or any(ord(character) < 0x21 or ord(character) > 0x7E for character in token)
+    ):
+        raise GCSStorageError("workload identity returned no usable access token")
+    return token
+
+
+@dataclass
+class GoogleCloudStorageJSONTransport:
+    """Streaming JSON-API transport with no proxy endpoint or credential configuration surface."""
+
+    access_token_provider: Callable[[], str] = workload_identity_access_token
+    timeout_seconds: float = 60.0
+
+    def __post_init__(self) -> None:
+        if not callable(self.access_token_provider):
+            raise ValueError("access_token_provider must be callable")
+        if (
+            isinstance(self.timeout_seconds, bool)
+            or not isinstance(self.timeout_seconds, (int, float))
+            or self.timeout_seconds <= 0
+            or self.timeout_seconds > 600
+        ):
+            raise ValueError("timeout_seconds must be between zero and 600")
+
+    def put(
+        self,
+        *,
+        bucket: str,
+        name: str,
+        parts: Iterable[bytes],
+        byte_length: int,
+        if_generation_match: str,
+    ) -> GCSObjectVersion:
+        _require_object_request(bucket, name, byte_length, if_generation_match)
+        query = urlencode(
+            {
+                "uploadType": "media",
+                "name": name,
+                "ifGenerationMatch": if_generation_match,
+            }
+        )
+        path = f"/upload/storage/v1/b/{quote(bucket, safe='')}/o?{query}"
+        connection = self._connection()
+        try:
+            connection.putrequest("POST", path)
+            connection.putheader("Authorization", self._authorization())
+            connection.putheader("Content-Type", "application/octet-stream")
+            connection.putheader("Content-Length", str(byte_length))
+            connection.endheaders()
+            remaining = byte_length
+            for part in parts:
+                view = _part_view(part)
+                if view.nbytes > remaining:
+                    raise GCSStorageError("object upload source exceeded its declared byte length")
+                connection.send(view)
+                remaining -= view.nbytes
+            if remaining:
+                raise GCSStorageError("object upload source ended before its declared byte length")
+            response = connection.getresponse()
+            raw = _read_bounded_response(response, GCS_METADATA_BYTES)
+            _raise_for_status(response.status, name, successful_statuses=(200, 201))
+            return _parse_object_version(raw, name, expected_size=byte_length)
+        except GCSStorageError:
+            raise
+        except (OSError, http.client.HTTPException) as error:
+            raise GCSStorageError("Cloud Storage object upload failed") from error
+        finally:
+            connection.close()
+
+    def get(
+        self,
+        *,
+        bucket: str,
+        name: str,
+        if_generation_match: str | None = None,
+    ) -> GCSObjectRead:
+        _require_get_request(bucket, name, if_generation_match)
+        metadata = self._metadata(bucket=bucket, name=name)
+        if if_generation_match is not None and metadata.generation != if_generation_match:
+            raise GCSPreconditionFailed("Cloud Storage object generation differs")
+        query = urlencode({"alt": "media", "ifGenerationMatch": metadata.generation})
+        path = f"/storage/v1/b/{quote(bucket, safe='')}/o/{quote(name, safe='')}?{query}"
+        connection = self._connection()
+        try:
+            connection.putrequest("GET", path)
+            connection.putheader("Authorization", self._authorization())
+            connection.endheaders()
+            response = connection.getresponse()
+            if response.status != 200:
+                _read_bounded_response(response, GCS_METADATA_BYTES)
+                _raise_for_status(response.status, name)
+                raise AssertionError("unreachable")
+            content_length = response.getheader("Content-Length")
+            if content_length is not None and content_length != str(metadata.byte_length):
+                response.close()
+                raise GCSStorageError("Cloud Storage media response length differs from metadata")
+            body = _HTTPResponseParts(response, connection, metadata.byte_length)
+            return GCSObjectRead(version=metadata, parts=body, closer=body.close)
+        except GCSStorageError:
+            connection.close()
+            raise
+        except (OSError, http.client.HTTPException) as error:
+            connection.close()
+            raise GCSStorageError("Cloud Storage object read failed") from error
+
+    def _metadata(self, *, bucket: str, name: str) -> GCSObjectVersion:
+        path = f"/storage/v1/b/{quote(bucket, safe='')}/o/{quote(name, safe='')}"
+        connection = self._connection()
+        try:
+            connection.request("GET", path, headers={"Authorization": self._authorization()})
+            response = connection.getresponse()
+            raw = _read_bounded_response(response, GCS_METADATA_BYTES)
+            _raise_for_status(response.status, name)
+            return _parse_object_version(raw, name)
+        except GCSStorageError:
+            raise
+        except (OSError, http.client.HTTPException) as error:
+            raise GCSStorageError("Cloud Storage object metadata read failed") from error
+        finally:
+            connection.close()
+
+    def _connection(self) -> http.client.HTTPSConnection:
+        return http.client.HTTPSConnection(GCS_STORAGE_HOST, timeout=float(self.timeout_seconds))
+
+    def _authorization(self) -> str:
+        token = self.access_token_provider()
+        if (
+            not isinstance(token, str)
+            or not token
+            or len(token.encode("utf-8")) > 8192
+            or any(ord(character) < 0x21 or ord(character) > 0x7E for character in token)
+        ):
+            raise GCSStorageError("access-token provider returned no usable token")
+        return "Bearer " + token
+
+
+@dataclass(frozen=True)
+class _VersionedManifest:
+    manifest: IngestionManifestV1
+    generation: str
+    raw_sha256: str
+    byte_length: int
+
+
+@dataclass
+class GCSIngestionStore:
+    """A bounded, generation-fenced Cloud Storage ingestion store.
+
+    ``staging_root`` must be a private local directory.  It is used only for bounded temporary
+    chunk spools: all durable manifests and immutable content-addressed chunks reside in GCS.
+    The backend's only remote authority is the configured bucket/prefix plus the workload's
+    ambient identity, not a serializable upload URL or bearer secret.
+    """
+
+    bucket: str
+    prefix: str
+    staging_root: Path
+    manifest_prefix: str | None = None
+    chunk_prefix: str | None = None
+    delivery_binding: IngestionDeliveryBindingV1 | None = None
+    transport: GCSObjectTransport | None = None
+    maximum_spool_bytes: int = DEFAULT_MAX_SPOOL_BYTES
+    maximum_spool_total_bytes: int = DEFAULT_MAX_SPOOL_TOTAL_BYTES
+    stale_spool_age_seconds: int = DEFAULT_STALE_SPOOL_AGE_SECONDS
+
+    def __post_init__(self) -> None:
+        # Import lazily: hosted delivery contracts themselves import the core ingestion contract,
+        # while this package's public initializer also exposes GCSIngestionStore.
+        from mostlyright.data_harness.hosted_ingestion_contract import IngestionDeliveryBindingV1
+
+        if not isinstance(self.bucket, str) or _BUCKET.fullmatch(self.bucket) is None:
+            raise IngestionError("INGESTION_GCS_BUCKET", "gcs.bucket", "is not a valid bucket name")
+        self.prefix = _normalize_prefix(self.prefix)
+        self.manifest_prefix = _normalize_prefix(self.manifest_prefix or self.prefix)
+        self.chunk_prefix = _normalize_prefix(self.chunk_prefix or self.prefix)
+        if self.delivery_binding is not None and not isinstance(
+            self.delivery_binding, IngestionDeliveryBindingV1
+        ):
+            raise IngestionError(
+                "INGESTION_DELIVERY_BINDING",
+                "gcs.delivery_binding",
+                "must be IngestionDeliveryBindingV1",
+            )
+        if (
+            self.delivery_binding is not None
+            and self.manifest_prefix.rsplit("/", 1)[-1] != self.delivery_binding.delivery_id
+        ):
+            raise IngestionError(
+                "INGESTION_DELIVERY_NAMESPACE",
+                "gcs.manifest_prefix",
+                "must end in delivery_binding.delivery_id",
+            )
+        self.staging_root = Path(self.staging_root)
+        if self.transport is None:
+            self.transport = GoogleCloudStorageJSONTransport()
+        if not isinstance(self.transport, GCSObjectTransport):
+            raise IngestionError(
+                "INGESTION_GCS_TRANSPORT", "gcs.transport", "does not implement GCSObjectTransport"
+            )
+        # Reuse the carefully reviewed private directory, lock, aggregate quota, and stale-spool
+        # discipline from the local store.  Its runs/chunks directories remain empty here.
+        self._staging = LocalIngestionStore(
+            self.staging_root,
+            maximum_spool_bytes=self.maximum_spool_bytes,
+            maximum_spool_total_bytes=self.maximum_spool_total_bytes,
+            maximum_store_bytes=self.maximum_spool_total_bytes,
+            stale_spool_age_seconds=self.stale_spool_age_seconds,
+        )
+
+    def create(self, manifest: IngestionManifestV1) -> IngestionManifestV1:
+        if (
+            not isinstance(manifest, IngestionManifestV1)
+            or manifest.state != "open"
+            or manifest.chunks
+            or manifest.previous_manifest_sha256 is not None
+        ):
+            raise IngestionConflict(
+                "INGESTION_CREATE_STATE",
+                "ingestion_manifest",
+                "only an initial open manifest may be created",
+            )
+        raw = self._manifest_raw(manifest)
+        try:
+            self._put(self._manifest_name(manifest.run_id), [raw], len(raw), "0")
+        except GCSPreconditionFailed:
+            existing = self.load(manifest.run_id)
+            if existing is not None and existing.digest == manifest.digest:
+                return existing
+            raise IngestionConflict(
+                "INGESTION_RUN_EXISTS",
+                "ingestion_manifest.run_id",
+                "already has a different manifest",
+            ) from None
+        except GCSStorageError as error:
+            # A connection can fail after GCS committed a create.  Resolve only the exact replay;
+            # otherwise surface an I/O failure rather than guessing the durable outcome.
+            existing = self.load(manifest.run_id)
+            if existing is not None and existing.digest == manifest.digest:
+                return existing
+            raise _storage_refusal("ingestion_manifest", error) from error
+        return manifest
+
+    def load(self, run_id: str) -> IngestionManifestV1 | None:
+        self._validate_run_id(run_id)
+        current = self._load_versioned(run_id)
+        return None if current is None else current.manifest
+
+    def current_manifest_reference(self, *, run_id: str) -> IngestionManifestReferenceV1:
+        """Return the exact raw document coordinate after validating its delivery envelope."""
+
+        self._validate_run_id(run_id)
+        current = self._require_versioned(run_id)
+        from mostlyright.data_harness.hosted_ingestion_contract import IngestionManifestReferenceV1
+
+        return IngestionManifestReferenceV1(
+            object_name=self._manifest_name(run_id),
+            generation=current.generation,
+            manifest_sha256=current.raw_sha256,
+            byte_length=current.byte_length,
+        )
+
+    def load_manifest_reference(
+        self,
+        *,
+        reference: IngestionManifestReferenceV1,
+        manifest_prefix: str,
+        run_id: str,
+    ) -> IngestionManifestV1:
+        """Load one preceding delivery manifest only when every reference coordinate holds.
+
+        ``manifest_prefix`` comes from Studio's frozen predecessor object, rather than being
+        inferred from a foreign object name.  The read is generation-pinned and verifies the raw
+        canonical envelope digest before exposing its core manifest.
+        """
+
+        from mostlyright.data_harness.hosted_ingestion_contract import IngestionManifestReferenceV1
+
+        if not isinstance(reference, IngestionManifestReferenceV1):
+            raise IngestionError(
+                "INGESTION_MANIFEST_REF", "manifest_ref", "must be IngestionManifestReferenceV1"
+            )
+        self._validate_run_id(run_id)
+        reference.require_for_prefix(prefix=manifest_prefix, run_id=run_id)
+        try:
+            stored = self.transport.get(
+                bucket=self.bucket,
+                name=reference.object_name,
+                if_generation_match=reference.generation,
+            )
+        except GCSObjectNotFound as error:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_MISSING",
+                "manifest_ref.object_name",
+                "referenced delivery manifest is absent",
+            ) from error
+        except GCSPreconditionFailed as error:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_GENERATION",
+                "manifest_ref.generation",
+                "referenced delivery manifest changed",
+            ) from error
+        except GCSStorageError as error:
+            raise _storage_refusal("manifest_ref", error) from error
+        try:
+            if stored.version.byte_length != reference.byte_length:
+                raise IngestionCorrupt(
+                    "INGESTION_MANIFEST_REF_LENGTH",
+                    "manifest_ref.byte_length",
+                    "does not match object metadata",
+                )
+            raw = _collect_bounded(stored.parts, maximum=MAX_MANIFEST_BYTES, field="manifest_ref")
+        finally:
+            stored.close()
+        if len(raw) != reference.byte_length:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_LENGTH",
+                "manifest_ref.byte_length",
+                "does not match object bytes",
+            )
+        if hashlib.sha256(raw).hexdigest() != reference.manifest_sha256:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_DIGEST",
+                "manifest_ref.manifest_sha256",
+                "does not match canonical manifest bytes",
+            )
+        try:
+            from mostlyright.data_harness.hosted_ingestion_contract import (
+                IngestionDeliveryManifestV1,
+            )
+
+            envelope = IngestionDeliveryManifestV1.from_dict(parse_canonical_json(raw))
+        except (CanonicalJSONError, IngestionError) as error:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_CORRUPT",
+                "manifest_ref",
+                "does not name a valid delivery manifest",
+            ) from error
+        if envelope.manifest.run_id != run_id:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_RUN",
+                "manifest_ref.object_name",
+                "does not bind the requested run",
+            )
+        if manifest_prefix.rsplit("/", 1)[-1] != envelope.delivery.delivery_id:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_REF_DELIVERY",
+                "manifest_ref.object_name",
+                "does not name the envelope's delivery namespace",
+            )
+        return envelope.manifest
+
+    def seed_from_reference(
+        self,
+        *,
+        reference: IngestionManifestReferenceV1,
+        manifest_prefix: str,
+        run_id: str,
+    ) -> IngestionManifestV1:
+        """Verify a predecessor then atomically copy it into this delivery's own namespace."""
+
+        if self.delivery_binding is None:
+            raise IngestionConflict(
+                "INGESTION_DELIVERY_BINDING",
+                "gcs.delivery_binding",
+                "is required to seed a predecessor into a delivery namespace",
+            )
+        manifest = self.load_manifest_reference(
+            reference=reference, manifest_prefix=manifest_prefix, run_id=run_id
+        )
+        self.delivery_binding.require_manifest(manifest)
+        raw = self._manifest_raw(manifest)
+        try:
+            self._put(self._manifest_name(run_id), [raw], len(raw), "0")
+        except (GCSPreconditionFailed, GCSStorageError) as error:
+            existing = self._load_versioned(run_id)
+            if existing is not None and existing.manifest.digest == manifest.digest:
+                return existing.manifest
+            if isinstance(error, GCSPreconditionFailed):
+                raise IngestionConflict(
+                    "INGESTION_DELIVERY_SEED_FENCE",
+                    "ingestion_manifest.run_id",
+                    "already names a different delivery manifest",
+                ) from None
+            raise _storage_refusal("ingestion_manifest", error) from error
+        return manifest
+
+    def append_stream(
+        self,
+        *,
+        run_id: str,
+        expected_manifest_sha256: str,
+        chunk: ChunkRef,
+        parts: Iterable[bytes | bytearray | memoryview],
+    ) -> IngestionManifestV1:
+        self._validate_run_id(run_id)
+        self._validate_manifest_digest(expected_manifest_sha256)
+        if not isinstance(chunk, ChunkRef):
+            raise IngestionError("INGESTION_CHUNK", "chunk", "must be a ChunkRef")
+        spooled = self._staging.spool(parts)
+        try:
+            self._verify_staged_spool(spooled, chunk)
+            current = self._require_versioned(run_id)
+            replay = _matching_replay(current.manifest, chunk)
+            if replay is not None:
+                return replay
+            if current.manifest.is_finalized:
+                raise IngestionConflict(
+                    "INGESTION_FINALIZED",
+                    "ingestion_manifest.state",
+                    "cannot append to a finalized run",
+                )
+            if current.manifest.digest != expected_manifest_sha256:
+                raise IngestionConflict(
+                    "INGESTION_MANIFEST_FENCE",
+                    "expected_manifest_sha256",
+                    "does not match the current manifest",
+                )
+            successor = current.manifest.append(chunk)
+            self._ensure_chunk(chunk, spooled)
+            return self._put_successor(
+                run_id=run_id,
+                predecessor=current,
+                successor=successor,
+                expected_manifest_sha256=expected_manifest_sha256,
+                chunk=chunk,
+            )
+        finally:
+            self._discard_staged_spool(spooled)
+
+    def finalize(self, *, run_id: str, expected_manifest_sha256: str) -> IngestionManifestV1:
+        self._validate_run_id(run_id)
+        self._validate_manifest_digest(expected_manifest_sha256)
+        current = self._require_versioned(run_id)
+        if current.manifest.is_finalized:
+            if (
+                current.manifest.digest == expected_manifest_sha256
+                or current.manifest.is_direct_successor_of(expected_manifest_sha256)
+            ):
+                return current.manifest
+            raise IngestionConflict(
+                "INGESTION_MANIFEST_FENCE",
+                "expected_manifest_sha256",
+                "does not name the finalization predecessor",
+            )
+        if current.manifest.digest != expected_manifest_sha256:
+            raise IngestionConflict(
+                "INGESTION_MANIFEST_FENCE",
+                "expected_manifest_sha256",
+                "does not match the current manifest",
+            )
+        for item in current.manifest.chunks:
+            self._verify_remote_chunk(item)
+        successor = current.manifest.finalize()
+        raw = self._manifest_raw(successor)
+        try:
+            self._put(self._manifest_name(run_id), [raw], len(raw), current.generation)
+            return successor
+        except (GCSPreconditionFailed, GCSStorageError) as error:
+            resolved = self._load_versioned(run_id)
+            if (
+                resolved is not None
+                and resolved.manifest.is_finalized
+                and resolved.manifest.is_direct_successor_of(expected_manifest_sha256)
+            ):
+                return resolved.manifest
+            if isinstance(error, GCSPreconditionFailed):
+                raise IngestionConflict(
+                    "INGESTION_MANIFEST_FENCE",
+                    "expected_manifest_sha256",
+                    "does not match the current manifest",
+                ) from None
+            raise _storage_refusal("ingestion_manifest", error) from error
+
+    def iter_chunk_parts(self, chunk: ChunkRef) -> Iterator[bytes]:
+        if not isinstance(chunk, ChunkRef):
+            raise IngestionError("INGESTION_CHUNK", "chunk", "must be a ChunkRef")
+        # Verify a complete generation before returning a readable generation.  This matches the
+        # local backend's guarantee that consumers never act on bytes later found corrupt, while
+        # retaining a bounded two-pass remote read rather than materialising the object.
+        verified = self._verify_remote_chunk(chunk)
+        try:
+            readable = self.transport.get(
+                bucket=self.bucket,
+                name=self._chunk_name(chunk.content_sha256),
+                if_generation_match=verified.generation,
+            )
+        except GCSObjectNotFound as error:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_MISSING",
+                "chunk_ref.content_sha256",
+                "referenced Cloud Storage object is absent",
+            ) from error
+        except GCSPreconditionFailed as error:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_GENERATION",
+                "chunk_ref.content_sha256",
+                "referenced Cloud Storage object changed during verified read",
+            ) from error
+        except GCSStorageError as error:
+            raise _storage_refusal("chunk_ref.content_sha256", error) from error
+        try:
+            if readable.version != verified or readable.version.byte_length != chunk.byte_length:
+                raise IngestionCorrupt(
+                    "INGESTION_BLOB_GENERATION",
+                    "chunk_ref.content_sha256",
+                    "referenced Cloud Storage generation changed during verified read",
+                )
+            yield from _bounded_parts(
+                readable.parts, expected_length=chunk.byte_length, field="chunk_ref.content_sha256"
+            )
+        finally:
+            readable.close()
+
+    def _put_successor(
+        self,
+        *,
+        run_id: str,
+        predecessor: _VersionedManifest,
+        successor: IngestionManifestV1,
+        expected_manifest_sha256: str,
+        chunk: ChunkRef,
+    ) -> IngestionManifestV1:
+        raw = self._manifest_raw(successor)
+        try:
+            self._put(self._manifest_name(run_id), [raw], len(raw), predecessor.generation)
+            return successor
+        except (GCSPreconditionFailed, GCSStorageError) as error:
+            # Both a real CAS loss and a lost response are resolved by reading the current state.
+            # Returning a matching ordinal is safe even if another writer advanced farther: it is
+            # exactly the acknowledgement-safe replay rule the local store implements.
+            resolved = self._load_versioned(run_id)
+            if resolved is not None:
+                replay = _matching_replay(resolved.manifest, chunk)
+                if replay is not None:
+                    return replay
+            if isinstance(error, GCSPreconditionFailed):
+                raise IngestionConflict(
+                    "INGESTION_MANIFEST_FENCE",
+                    "expected_manifest_sha256",
+                    "does not match the current manifest",
+                ) from None
+            raise _storage_refusal("ingestion_manifest", error) from error
+
+    def _ensure_chunk(self, chunk: ChunkRef, spooled: SpooledChunk) -> None:
+        name = self._chunk_name(chunk.content_sha256)
+        try:
+            parts = self._staged_spool_parts(spooled, chunk)
+            try:
+                self._put(name, parts, chunk.byte_length, "0")
+            finally:
+                parts.close()
+        except GCSPreconditionFailed:
+            pass
+        except GCSStorageError as error:
+            # An upload acknowledgement can disappear after GCS made the create durable.  It is
+            # safe to continue only when the immutable object now verifies byte-for-byte.
+            try:
+                self._verify_remote_chunk(chunk)
+            except IngestionCorrupt:
+                raise _storage_refusal("chunk_ref.content_sha256", error) from error
+            return
+        self._verify_remote_chunk(chunk)
+
+    def _load_versioned(self, run_id: str) -> _VersionedManifest | None:
+        try:
+            stored = self.transport.get(bucket=self.bucket, name=self._manifest_name(run_id))
+        except GCSObjectNotFound:
+            return None
+        except GCSStorageError as error:
+            raise _storage_refusal("ingestion_manifest", error) from error
+        try:
+            if stored.version.byte_length > MAX_MANIFEST_BYTES:
+                raise IngestionCorrupt(
+                    "INGESTION_FILE_LIMIT",
+                    "ingestion_manifest",
+                    "Cloud Storage manifest exceeds its limit",
+                )
+            raw = _collect_bounded(
+                stored.parts, maximum=MAX_MANIFEST_BYTES, field="ingestion_manifest"
+            )
+        finally:
+            stored.close()
+        if len(raw) != stored.version.byte_length:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_SHAPE",
+                "ingestion_manifest",
+                "Cloud Storage metadata length differs",
+            )
+        try:
+            manifest = self._parse_manifest_raw(raw)
+        except IngestionCorrupt:
+            raise
+        except (CanonicalJSONError, IngestionError) as error:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_CORRUPT",
+                "ingestion_manifest",
+                "is not a valid canonical manifest",
+            ) from error
+        if manifest.run_id != run_id:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_RUN",
+                "ingestion_manifest.run_id",
+                "does not match its object coordinate",
+            )
+        return _VersionedManifest(
+            manifest=manifest,
+            generation=stored.version.generation,
+            raw_sha256=hashlib.sha256(raw).hexdigest(),
+            byte_length=stored.version.byte_length,
+        )
+
+    def _require_versioned(self, run_id: str) -> _VersionedManifest:
+        current = self._load_versioned(run_id)
+        if current is None:
+            raise IngestionConflict(
+                "INGESTION_RUN_MISSING", "ingestion_manifest.run_id", "does not exist"
+            )
+        return current
+
+    def _manifest_raw(self, manifest: IngestionManifestV1) -> bytes:
+        if self.delivery_binding is None:
+            return canonical_json_bytes(manifest.to_dict())
+        self.delivery_binding.require_manifest(manifest)
+        from mostlyright.data_harness.hosted_ingestion_contract import IngestionDeliveryManifestV1
+
+        return canonical_json_bytes(
+            IngestionDeliveryManifestV1(delivery=self.delivery_binding, manifest=manifest).to_dict()
+        )
+
+    def _parse_manifest_raw(self, raw: bytes) -> IngestionManifestV1:
+        document = parse_canonical_json(raw)
+        if self.delivery_binding is None:
+            return IngestionManifestV1.from_dict(document)
+        from mostlyright.data_harness.hosted_ingestion_contract import IngestionDeliveryManifestV1
+
+        envelope = IngestionDeliveryManifestV1.from_dict(document)
+        if envelope.delivery != self.delivery_binding:
+            raise IngestionCorrupt(
+                "INGESTION_DELIVERY_FENCE",
+                "delivery_manifest.delivery",
+                "does not match this delivery's fenced binding",
+            )
+        return envelope.manifest
+
+    def _verify_staged_spool(self, spooled: SpooledChunk, chunk: ChunkRef) -> None:
+        parts = self._staged_spool_parts(spooled, chunk)
+        try:
+            for _part in parts:
+                pass
+        finally:
+            parts.close()
+
+    def _staged_spool_parts(self, spooled: SpooledChunk, chunk: ChunkRef) -> _StagedSpoolParts:
+        return _StagedSpoolParts(spooled=spooled, chunk=chunk, spool_directory=self._staging._spool)
+
+    def _discard_staged_spool(self, spooled: SpooledChunk) -> None:
+        try:
+            info = spooled.path.stat(follow_symlinks=False)
+            if (
+                stat.S_ISREG(info.st_mode)
+                and info.st_dev == spooled.device
+                and info.st_ino == spooled.inode
+                and spooled.path.parent == self._staging._spool
+            ):
+                spooled.path.unlink()
+        except OSError:
+            # A later bounded stale-spool sweep owns cleanup.  The durable remote transition is
+            # already known, so surfacing a local cleanup failure would make its retry ambiguous.
+            pass
+
+    def _verify_remote_chunk(self, chunk: ChunkRef) -> GCSObjectVersion:
+        try:
+            stored = self.transport.get(
+                bucket=self.bucket, name=self._chunk_name(chunk.content_sha256)
+            )
+        except GCSObjectNotFound as error:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_MISSING",
+                "chunk_ref.content_sha256",
+                "referenced Cloud Storage object is absent",
+            ) from error
+        except GCSStorageError as error:
+            raise _storage_refusal("chunk_ref.content_sha256", error) from error
+        digest = hashlib.sha256()
+        total = 0
+        try:
+            if stored.version.byte_length != chunk.byte_length:
+                raise IngestionCorrupt(
+                    "INGESTION_BLOB_SHAPE",
+                    "chunk_ref.content_sha256",
+                    "object has an unexpected length",
+                )
+            for part in _bounded_parts(
+                stored.parts, expected_length=chunk.byte_length, field="chunk_ref.content_sha256"
+            ):
+                digest.update(part)
+                total += len(part)
+        finally:
+            stored.close()
+        if total != chunk.byte_length or digest.hexdigest() != chunk.content_sha256:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_DIGEST",
+                "chunk_ref.content_sha256",
+                "object bytes differ from digest",
+            )
+        return stored.version
+
+    def _put(
+        self, name: str, parts: Iterable[bytes], byte_length: int, generation: str
+    ) -> GCSObjectVersion:
+        try:
+            result = self.transport.put(
+                bucket=self.bucket,
+                name=name,
+                parts=parts,
+                byte_length=byte_length,
+                if_generation_match=generation,
+            )
+        except (GCSStorageError, IngestionError):
+            raise
+        except Exception as error:  # pragma: no cover - protects adapter boundary only
+            raise GCSStorageError("Cloud Storage adapter failed") from error
+        if not isinstance(result, GCSObjectVersion) or result.byte_length != byte_length:
+            raise GCSStorageError("Cloud Storage returned an invalid object version")
+        return result
+
+    def _manifest_name(self, run_id: str) -> str:
+        return f"{self.manifest_prefix}/runs/{run_id}.json"
+
+    def _chunk_name(self, digest: str) -> str:
+        return f"{self.chunk_prefix}/chunks/{digest}.bin"
+
+    _validate_run_id = staticmethod(LocalIngestionStore._validate_run_id)
+    _validate_manifest_digest = staticmethod(LocalIngestionStore._validate_manifest_digest)
+
+
+class _StagedSpoolParts:
+    """An opened, checked staging spool held against the local stale reaper while uploading."""
+
+    def __init__(self, *, spooled: SpooledChunk, chunk: ChunkRef, spool_directory: Path) -> None:
+        if (
+            not isinstance(spooled, SpooledChunk)
+            or spooled.path.parent != spool_directory
+            or not spooled.path.name.startswith(".spool.")
+            or spooled.content_sha256 != chunk.content_sha256
+            or spooled.byte_length != chunk.byte_length
+        ):
+            raise IngestionError(
+                "INGESTION_SPOOL_PATH", "spooled", "does not name this store's staging spool"
+            )
+        self._descriptor = -1
+        try:
+            self._descriptor = os.open(spooled.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+            info = os.fstat(self._descriptor)
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_nlink != 1
+                or info.st_dev != spooled.device
+                or info.st_ino != spooled.inode
+                or info.st_size != spooled.byte_length
+            ):
+                raise IngestionError(
+                    "INGESTION_SPOOL_PATH", "spooled.path", "changed after staging"
+                )
+            # This prevents a concurrent LocalIngestionStore reaper from deleting the completed
+            # temporary during a slow GCS upload.  The global quota lock is intentionally not held
+            # across network I/O; the spool's own shared advisory lock is the narrow ownership bit.
+            fcntl.flock(self._descriptor, fcntl.LOCK_SH)
+            digest = hashlib.sha256()
+            total = 0
+            while raw := os.read(self._descriptor, GCS_READ_CHUNK_BYTES):
+                digest.update(raw)
+                total += len(raw)
+            if total != chunk.byte_length or digest.hexdigest() != chunk.content_sha256:
+                raise IngestionError(
+                    "INGESTION_SPOOL_DIGEST", "spooled.path", "bytes differ from chunk"
+                )
+            os.lseek(self._descriptor, 0, os.SEEK_SET)
+        except OSError as error:
+            self.close()
+            raise IngestionError(
+                "INGESTION_SPOOL_IO", "spooled.path", "cannot stream staging spool"
+            ) from error
+        except BaseException:
+            self.close()
+            raise
+
+    def __iter__(self) -> Iterator[bytes]:
+        try:
+            while raw := os.read(self._descriptor, GCS_READ_CHUNK_BYTES):
+                yield raw
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_SPOOL_IO", "spooled.path", "cannot stream staging spool"
+            ) from error
+        finally:
+            self.close()
+
+    def close(self) -> None:
+        if self._descriptor >= 0:
+            descriptor = self._descriptor
+            self._descriptor = -1
+            os.close(descriptor)
+
+
+def _matching_replay(manifest: IngestionManifestV1, chunk: ChunkRef) -> IngestionManifestV1 | None:
+    if chunk.ordinal >= len(manifest.chunks):
+        return None
+    if manifest.chunks[chunk.ordinal] != chunk:
+        raise IngestionConflict(
+            "INGESTION_REPLAY_MISMATCH",
+            "chunk_ref.ordinal",
+            "already names different committed chunk bytes or metadata",
+        )
+    return manifest
+
+
+def _normalize_prefix(value: object) -> str:
+    if not isinstance(value, str) or not value or len(value) > 512:
+        raise IngestionError(
+            "INGESTION_GCS_PREFIX", "gcs.prefix", "must be one bounded object prefix"
+        )
+    segments = value.split("/")
+    if any(_PREFIX_SEGMENT.fullmatch(segment) is None for segment in segments):
+        raise IngestionError(
+            "INGESTION_GCS_PREFIX", "gcs.prefix", "must contain only safe nonempty path segments"
+        )
+    return value
+
+
+def _require_object_request(bucket: str, name: str, byte_length: int, generation: str) -> None:
+    _require_get_request(bucket, name, generation if generation != "0" else None)
+    if generation != "0" and _GENERATION.fullmatch(generation) is None:
+        raise ValueError("if_generation_match must be zero or a positive generation")
+    if isinstance(byte_length, bool) or not isinstance(byte_length, int) or byte_length < 0:
+        raise ValueError("byte_length must be non-negative")
+
+
+def _require_get_request(bucket: str, name: str, generation: str | None) -> None:
+    if not isinstance(bucket, str) or _BUCKET.fullmatch(bucket) is None:
+        raise ValueError("bucket is invalid")
+    if (
+        not isinstance(name, str)
+        or not name
+        or name.startswith("/")
+        or "//" in name
+        or ".." in name
+    ):
+        raise ValueError("object name is invalid")
+    if generation is not None and _GENERATION.fullmatch(generation) is None:
+        raise ValueError("if_generation_match must be a positive generation")
+
+
+def _part_view(value: object) -> memoryview:
+    if not isinstance(value, (bytes, bytearray, memoryview)):
+        raise GCSStorageError("object stream yielded a non-bytes part")
+    view = memoryview(value)
+    if not view.c_contiguous:
+        raise GCSStorageError("object stream yielded a non-contiguous part")
+    if view.format != "B" or view.ndim != 1:
+        view = view.cast("B")
+    return view
+
+
+def _bounded_parts(parts: Iterable[bytes], *, expected_length: int, field: str) -> Iterator[bytes]:
+    total = 0
+    for value in parts:
+        view = _part_view(value)
+        total += view.nbytes
+        if total > expected_length:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_SHAPE", field, "object exceeds its metadata length"
+            )
+        yield bytes(view)
+    if total != expected_length:
+        raise IngestionCorrupt(
+            "INGESTION_BLOB_SHAPE", field, "object is shorter than its metadata length"
+        )
+
+
+def _collect_bounded(parts: Iterable[bytes], *, maximum: int, field: str) -> bytes:
+    result: list[bytes] = []
+    total = 0
+    for value in parts:
+        view = _part_view(value)
+        total += view.nbytes
+        if total > maximum:
+            raise IngestionCorrupt(
+                "INGESTION_FILE_LIMIT", field, "object exceeds its permitted byte limit"
+            )
+        result.append(bytes(view))
+    return b"".join(result)
+
+
+def _storage_refusal(path: str, error: GCSStorageError) -> IngestionError:
+    return IngestionError("INGESTION_GCS_IO", path, "Cloud Storage request failed")
+
+
+def _read_bounded_response(response: http.client.HTTPResponse, maximum: int) -> bytes:
+    chunks: list[bytes] = []
+    total = 0
+    while raw := response.read(min(GCS_READ_CHUNK_BYTES, maximum + 1)):
+        total += len(raw)
+        if total > maximum:
+            raise GCSStorageError("Cloud Storage metadata response exceeds its limit")
+        chunks.append(raw)
+    return b"".join(chunks)
+
+
+class _HTTPResponseParts:
+    """One response body with explicit cleanup even when callers reject it before iteration."""
+
+    def __init__(
+        self,
+        response: http.client.HTTPResponse,
+        connection: http.client.HTTPSConnection,
+        expected_length: int,
+    ) -> None:
+        self._response = response
+        self._connection = connection
+        self._expected_length = expected_length
+        self._closed = False
+
+    def __iter__(self) -> Iterator[bytes]:
+        remaining = self._expected_length
+        try:
+            while remaining:
+                raw = self._response.read(min(GCS_READ_CHUNK_BYTES, remaining))
+                if not raw:
+                    raise GCSStorageError("Cloud Storage media response ended early")
+                remaining -= len(raw)
+                yield raw
+            if self._response.read(1):
+                raise GCSStorageError("Cloud Storage media response exceeded metadata length")
+        except (OSError, http.client.HTTPException) as error:
+            raise GCSStorageError("Cloud Storage media response failed") from error
+        finally:
+            self.close()
+
+    def close(self) -> None:
+        if not self._closed:
+            self._closed = True
+            self._response.close()
+            self._connection.close()
+
+
+def _raise_for_status(
+    status: int, name: str, *, successful_statuses: tuple[int, ...] = (200,)
+) -> None:
+    if status in successful_statuses:
+        return
+    if status == 404:
+        raise GCSObjectNotFound("Cloud Storage object does not exist")
+    if status == 412:
+        raise GCSPreconditionFailed("Cloud Storage object generation precondition failed")
+    raise GCSStorageError("Cloud Storage returned an unsuccessful status")
+
+
+def _parse_object_version(
+    raw: bytes, name: str, *, expected_size: int | None = None
+) -> GCSObjectVersion:
+    try:
+        document = parse_json(raw)
+    except CanonicalJSONError as error:
+        raise GCSStorageError("Cloud Storage returned invalid object metadata") from error
+    if not isinstance(document, dict):
+        raise GCSStorageError("Cloud Storage returned invalid object metadata")
+    returned_name = document.get("name")
+    generation = document.get("generation")
+    size = document.get("size")
+    if returned_name != name or not isinstance(generation, str) or not isinstance(size, str):
+        raise GCSStorageError("Cloud Storage object metadata does not bind the requested object")
+    try:
+        parsed_size = int(size)
+    except ValueError as error:
+        raise GCSStorageError("Cloud Storage object metadata has an invalid size") from error
+    try:
+        result = GCSObjectVersion(generation=generation, byte_length=parsed_size)
+    except ValueError as error:
+        raise GCSStorageError("Cloud Storage object metadata is invalid") from error
+    if expected_size is not None and result.byte_length != expected_size:
+        raise GCSStorageError("Cloud Storage upload length differs from its response")
+    return result