mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ingestion/store.py

@@ -0,0 +1,885 @@
+"""Fenced ingestion-store protocol and a crash-safe local implementation."""
+
+from __future__ import annotations
+
+import contextlib
+import fcntl
+import hashlib
+import os
+import secrets
+import stat
+import time
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+)
+from mostlyright.data_harness.ingestion.contracts import (
+    ChunkRef,
+    IngestionConflict,
+    IngestionCorrupt,
+    IngestionError,
+    IngestionManifestV1,
+)
+from mostlyright.data_harness.ingestion.faults import DeterministicFaultInjector
+from mostlyright.data_harness.ingestion.spool import SpooledChunk, spool_hash_stream
+
+MAX_MANIFEST_BYTES = 16 * 1024 * 1024
+DEFAULT_MAX_SPOOL_BYTES = 256 * 1024 * 1024
+DEFAULT_MAX_SPOOL_TOTAL_BYTES = 512 * 1024 * 1024
+DEFAULT_MAX_STORE_BYTES = 4 * 1024 * 1024 * 1024
+DEFAULT_STALE_SPOOL_AGE_SECONDS = 60 * 60
+_READ_CHUNK_SIZE = 1024 * 1024
+
+
+@runtime_checkable
+class IngestionStore(Protocol):
+    """Durable source-pinned state transitions required by the ingestion engine.
+
+    A caller supplies the digest it observed before a transition.  Implementations must
+    either apply that exact transition or report a conflict; an immediate replay of an
+    already applied append/finalization may instead return the committed successor.
+    """
+
+    def create(self, manifest: IngestionManifestV1) -> IngestionManifestV1:
+        """Create an initial manifest, returning it on exact create replay."""
+
+    def load(self, run_id: str) -> IngestionManifestV1 | None:
+        """Return the current manifest for a run, or ``None`` if it does not exist."""
+
+    def append_stream(
+        self,
+        *,
+        run_id: str,
+        expected_manifest_sha256: str,
+        chunk: ChunkRef,
+        parts: Iterable[bytes | bytearray | memoryview],
+    ) -> IngestionManifestV1:
+        """Durably spool exact stream bytes then append their reference under a manifest fence."""
+
+    def finalize(self, *, run_id: str, expected_manifest_sha256: str) -> IngestionManifestV1:
+        """Freeze a gap-free sequence under a manifest fence."""
+
+    def iter_chunk_parts(self, chunk: ChunkRef) -> Iterator[bytes]:
+        """Yield one verified immutable chunk without materializing its full byte payload."""
+
+
+@dataclass
+class LocalIngestionStore:
+    """Private-directory reference store with durable immutable blobs and atomic manifests.
+
+    The only irreversible ordering is ``fsync(blob) -> link immutable blob -> fsync(chunk
+    directory) -> replace+fsync manifest``.  A crash before the final replacement leaves an
+    unreferenced immutable blob, which is safe to reuse; a crash after it leaves a valid
+    successor manifest, which an idempotent retry recognizes through its predecessor link.
+    """
+
+    root: Path
+    fault_injector: DeterministicFaultInjector | None = None
+    maximum_spool_bytes: int = DEFAULT_MAX_SPOOL_BYTES
+    maximum_spool_total_bytes: int = DEFAULT_MAX_SPOOL_TOTAL_BYTES
+    maximum_store_bytes: int = DEFAULT_MAX_STORE_BYTES
+    stale_spool_age_seconds: int = DEFAULT_STALE_SPOOL_AGE_SECONDS
+
+    def __post_init__(self) -> None:
+        self.root = Path(self.root)
+        if not self.root.is_absolute() or self.root.is_symlink() or not self.root.is_dir():
+            raise IngestionError(
+                "INGESTION_STORE_ROOT",
+                "ingestion_store.root",
+                "must be an existing absolute directory",
+            )
+        self._require_private_directory(self.root, "ingestion_store.root")
+        self._validate_limits()
+        for name in ("runs", "chunks", "spool"):
+            path = self.root / name
+            try:
+                path.mkdir(mode=0o700)
+            except FileExistsError:
+                pass
+            except OSError as error:
+                raise IngestionError(
+                    "INGESTION_STORE_IO", "ingestion_store.root", "could not create store directory"
+                ) from error
+            self._require_private_directory(path, f"ingestion_store.{name}")
+        self._runs = self.root / "runs"
+        self._chunks = self.root / "chunks"
+        self._spool = self.root / "spool"
+        self._lock_path = self.root / ".ingestion.lock"
+        self._checkpoint("store.before_root_fsync")
+        self._fsync_directory(self.root, "ingestion_store.root")
+        self._checkpoint("store.after_root_fsync")
+        with self._locked():
+            self._reap_stale_spools_locked(now=time.time())
+
+    def spool(
+        self,
+        parts: Iterable[bytes | bytearray | memoryview],
+        *,
+        maximum_bytes: int | None = None,
+    ) -> SpooledChunk:
+        """Spool a byte stream once; this does not mutate a run manifest."""
+
+        limit = self.maximum_spool_bytes if maximum_bytes is None else maximum_bytes
+        if (
+            isinstance(limit, bool)
+            or not isinstance(limit, int)
+            or limit < 0
+            or limit > self.maximum_spool_bytes
+        ):
+            raise IngestionError(
+                "INGESTION_SPOOL_LIMIT",
+                "spool.maximum_bytes",
+                "must not exceed the local per-spool quota",
+            )
+        return spool_hash_stream(
+            self._spool,
+            parts,
+            maximum_bytes=limit,
+            fault_injector=self.fault_injector,
+            reserve_bytes=self._reserve_spool_bytes,
+        )
+
+    def reap_stale_spools(self, *, now: float | None = None) -> int:
+        """Reclaim abandoned private temporary spools without touching a live writer."""
+
+        current = time.time() if now is None else now
+        if not isinstance(current, (int, float)) or current < 0:
+            raise IngestionError("INGESTION_SPOOL_CLOCK", "spool.now", "must be a Unix timestamp")
+        with self._locked():
+            return self._reap_stale_spools_locked(now=float(current))
+
+    def create(self, manifest: IngestionManifestV1) -> IngestionManifestV1:
+        if (
+            manifest.state != "open"
+            or manifest.chunks
+            or manifest.previous_manifest_sha256 is not None
+        ):
+            raise IngestionConflict(
+                "INGESTION_CREATE_STATE",
+                "ingestion_manifest",
+                "only an initial open manifest may be created",
+            )
+        with self._locked():
+            current = self._load_locked(manifest.run_id)
+            if current is not None:
+                if current.digest == manifest.digest:
+                    return current
+                raise IngestionConflict(
+                    "INGESTION_RUN_EXISTS",
+                    "ingestion_manifest.run_id",
+                    "already has a different manifest",
+                )
+            self._replace_manifest_locked(manifest, checkpoint_prefix="store.create")
+            return manifest
+
+    def load(self, run_id: str) -> IngestionManifestV1 | None:
+        self._validate_run_id(run_id)
+        with self._locked():
+            return self._load_locked(run_id)
+
+    def append_stream(
+        self,
+        *,
+        run_id: str,
+        expected_manifest_sha256: str,
+        chunk: ChunkRef,
+        parts: Iterable[bytes | bytearray | memoryview],
+    ) -> IngestionManifestV1:
+        """Spool one replayable stream then perform the fenced durable append."""
+
+        spooled = self.spool(parts)
+        return self.append_spooled(
+            run_id=run_id,
+            expected_manifest_sha256=expected_manifest_sha256,
+            chunk=chunk,
+            spooled=spooled,
+        )
+
+    def append_spooled(
+        self,
+        *,
+        run_id: str,
+        expected_manifest_sha256: str,
+        chunk: ChunkRef,
+        spooled: SpooledChunk,
+    ) -> IngestionManifestV1:
+        self._validate_run_id(run_id)
+        self._validate_manifest_digest(expected_manifest_sha256)
+        if not isinstance(chunk, ChunkRef):
+            raise IngestionError("INGESTION_CHUNK", "chunk", "must be a ChunkRef")
+        self._verify_spool(spooled, chunk)
+        try:
+            with self._locked():
+                current = self._require_manifest_locked(run_id)
+                if chunk.ordinal < len(current.chunks):
+                    if current.chunks[chunk.ordinal] != chunk:
+                        raise IngestionConflict(
+                            "INGESTION_REPLAY_MISMATCH",
+                            "chunk_ref.ordinal",
+                            "already names different committed chunk bytes or metadata",
+                        )
+                    result = current
+                else:
+                    if current.is_finalized:
+                        raise IngestionConflict(
+                            "INGESTION_FINALIZED",
+                            "ingestion_manifest.state",
+                            "cannot append to a finalized run",
+                        )
+                    if current.digest != expected_manifest_sha256:
+                        raise IngestionConflict(
+                            "INGESTION_MANIFEST_FENCE",
+                            "expected_manifest_sha256",
+                            "does not match the current manifest",
+                        )
+                    successor = current.append(chunk)
+                    self._publish_spooled_blob_locked(spooled, chunk)
+                    self._checkpoint("store.after_blob_publish")
+                    self._replace_manifest_locked(successor, checkpoint_prefix="store.append")
+                    result = successor
+        except IngestionConflict:
+            # No blob is published until after all logical fences (including source drift) have
+            # passed. A conflict therefore owns no durable work and can release its temporary.
+            self._discard_spool(spooled)
+            raise
+        self._discard_spool(spooled)
+        return result
+
+    def finalize(self, *, run_id: str, expected_manifest_sha256: str) -> IngestionManifestV1:
+        self._validate_run_id(run_id)
+        self._validate_manifest_digest(expected_manifest_sha256)
+        with self._locked():
+            current = self._require_manifest_locked(run_id)
+            if current.is_finalized:
+                if current.digest == expected_manifest_sha256 or current.is_direct_successor_of(
+                    expected_manifest_sha256
+                ):
+                    return current
+                raise IngestionConflict(
+                    "INGESTION_MANIFEST_FENCE",
+                    "expected_manifest_sha256",
+                    "does not name the finalization predecessor",
+                )
+            if current.digest != expected_manifest_sha256:
+                raise IngestionConflict(
+                    "INGESTION_MANIFEST_FENCE",
+                    "expected_manifest_sha256",
+                    "does not match the current manifest",
+                )
+            # A Merkle commitment protects the ordered *references*. Before making that
+            # commitment externally consumable, prove every local immutable object still has
+            # the exact bytes those references name.
+            for chunk in current.chunks:
+                self._verify_chunk_file(self._chunk_path(chunk.content_sha256), chunk)
+            successor = current.finalize()
+            self._replace_manifest_locked(successor, checkpoint_prefix="store.finalize")
+            return successor
+
+    def iter_chunk_parts(self, chunk: ChunkRef) -> Iterator[bytes]:
+        """Yield verified immutable bytes in bounded pieces without materializing a chunk.
+
+        The complete object is hashed once before yielding from the still-open descriptor.
+        Immutable-object publication means that descriptor is then stable after the store
+        lock is released, without holding writers hostage for the duration of a large read.
+        """
+
+        if not isinstance(chunk, ChunkRef):
+            raise IngestionError("INGESTION_CHUNK", "chunk", "must be a ChunkRef")
+        descriptor = -1
+        try:
+            with self._locked():
+                path = self._chunk_path(chunk.content_sha256)
+                try:
+                    descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+                except FileNotFoundError as error:
+                    raise IngestionCorrupt(
+                        "INGESTION_BLOB_MISSING",
+                        "chunk_ref.content_sha256",
+                        "referenced blob is absent",
+                    ) from error
+                except OSError as error:
+                    raise IngestionCorrupt(
+                        "INGESTION_BLOB_IO",
+                        "chunk_ref.content_sha256",
+                        "referenced blob cannot be opened",
+                    ) from error
+                info = os.fstat(descriptor)
+                if not stat.S_ISREG(info.st_mode) or info.st_size != chunk.byte_length:
+                    raise IngestionCorrupt(
+                        "INGESTION_BLOB_SHAPE",
+                        "chunk_ref.content_sha256",
+                        "referenced blob is not the expected regular-file length",
+                    )
+                if self._hash_descriptor(descriptor) != chunk.content_sha256:
+                    raise IngestionCorrupt(
+                        "INGESTION_BLOB_DIGEST",
+                        "chunk_ref.content_sha256",
+                        "referenced blob hash differs",
+                    )
+                os.lseek(descriptor, 0, os.SEEK_SET)
+            while raw := os.read(descriptor, _READ_CHUNK_SIZE):
+                yield raw
+        finally:
+            if descriptor >= 0:
+                os.close(descriptor)
+
+    def read_chunk(self, chunk: ChunkRef, *, maximum_bytes: int = 64 * 1024 * 1024) -> bytes:
+        """Read a small verified blob; streaming callers must use :meth:`iter_chunk_parts`."""
+
+        if not isinstance(chunk, ChunkRef):
+            raise IngestionError("INGESTION_CHUNK", "chunk", "must be a ChunkRef")
+        if (
+            isinstance(maximum_bytes, bool)
+            or not isinstance(maximum_bytes, int)
+            or maximum_bytes < 0
+            or chunk.byte_length > maximum_bytes
+        ):
+            raise IngestionError(
+                "INGESTION_READ_LIMIT",
+                "chunk_ref.byte_length",
+                "requires bounded streaming access",
+            )
+        return b"".join(self.iter_chunk_parts(chunk))
+
+    @contextlib.contextmanager
+    def _locked(self) -> Iterator[None]:
+        descriptor = -1
+        try:
+            # The lock coordinate itself must not be creatable or replaceable by another local
+            # user. Check its parent before opening and then check the opened inode before flock.
+            self._require_private_directory(self.root, "ingestion_store.root")
+            descriptor = os.open(
+                self._lock_path,
+                os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0),
+                0o600,
+            )
+            info = os.fstat(descriptor)
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_nlink != 1
+                or info.st_uid != os.geteuid()
+                or stat.S_IMODE(info.st_mode) != 0o600
+            ):
+                raise IngestionCorrupt(
+                    "INGESTION_LOCK_PATH",
+                    "ingestion_store.lock",
+                    "must be an effective-user-owned mode-0600 regular file",
+                )
+            fcntl.flock(descriptor, fcntl.LOCK_EX)
+            self._require_private_directory(self.root, "ingestion_store.root")
+            self._require_private_directory(self._runs, "ingestion_store.runs")
+            self._require_private_directory(self._chunks, "ingestion_store.chunks")
+            self._require_private_directory(self._spool, "ingestion_store.spool")
+            yield
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_STORE_IO", "ingestion_store.lock", "lock operation failed"
+            ) from error
+        finally:
+            if descriptor >= 0:
+                os.close(descriptor)
+
+    def _load_locked(self, run_id: str) -> IngestionManifestV1 | None:
+        path = self._manifest_path(run_id)
+        try:
+            descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+        except FileNotFoundError:
+            return None
+        except OSError as error:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_IO", "ingestion_manifest", "manifest cannot be opened"
+            ) from error
+        try:
+            info = os.fstat(descriptor)
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_nlink != 1
+                or info.st_size > MAX_MANIFEST_BYTES
+            ):
+                raise IngestionCorrupt(
+                    "INGESTION_MANIFEST_SHAPE",
+                    "ingestion_manifest",
+                    "manifest has an unsafe file shape",
+                )
+            raw = self._read_descriptor(descriptor, maximum=MAX_MANIFEST_BYTES)
+        finally:
+            os.close(descriptor)
+        try:
+            manifest = IngestionManifestV1.from_dict(parse_canonical_json(raw))
+        except (CanonicalJSONError, IngestionError) as error:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_CORRUPT",
+                "ingestion_manifest",
+                "is not a valid canonical manifest",
+            ) from error
+        if manifest.run_id != run_id:
+            raise IngestionCorrupt(
+                "INGESTION_MANIFEST_RUN",
+                "ingestion_manifest.run_id",
+                "does not match its file coordinate",
+            )
+        return manifest
+
+    def _require_manifest_locked(self, run_id: str) -> IngestionManifestV1:
+        current = self._load_locked(run_id)
+        if current is None:
+            raise IngestionConflict(
+                "INGESTION_RUN_MISSING", "ingestion_manifest.run_id", "does not exist"
+            )
+        return current
+
+    def _publish_spooled_blob_locked(self, spooled: SpooledChunk, chunk: ChunkRef) -> None:
+        self._verify_spool(spooled, chunk)
+        destination = self._chunk_path(chunk.content_sha256)
+        if destination.exists():
+            self._verify_chunk_file(destination, chunk)
+            return
+        self._checkpoint("store.before_blob_link")
+        try:
+            os.link(spooled.path, destination, follow_symlinks=False)
+        except FileExistsError:
+            self._verify_chunk_file(destination, chunk)
+            return
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_BLOB_IO", "chunk_ref.content_sha256", "could not publish immutable blob"
+            ) from error
+        try:
+            self._verify_chunk_file(destination, chunk)
+            directory_descriptor = os.open(
+                self._chunks, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+            )
+            try:
+                os.fsync(directory_descriptor)
+            finally:
+                os.close(directory_descriptor)
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_BLOB_IO", "chunk_ref.content_sha256", "could not sync blob directory"
+            ) from error
+
+    def _replace_manifest_locked(
+        self, manifest: IngestionManifestV1, *, checkpoint_prefix: str
+    ) -> None:
+        raw = canonical_json_bytes(manifest.to_dict())
+        if len(raw) > MAX_MANIFEST_BYTES:
+            raise IngestionError(
+                "INGESTION_MANIFEST_LIMIT", "ingestion_manifest", "exceeds the manifest byte limit"
+            )
+        self._ensure_store_capacity_locked(len(raw))
+        destination = self._manifest_path(manifest.run_id)
+        temporary = self._runs / f".manifest.{os.getpid()}.{secrets.token_hex(16)}.tmp"
+        descriptor = -1
+        replaced = False
+        try:
+            descriptor = os.open(
+                temporary,
+                os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
+                0o600,
+            )
+            self._write_all(descriptor, raw)
+            os.fsync(descriptor)
+            os.close(descriptor)
+            descriptor = -1
+            self._checkpoint(f"{checkpoint_prefix}.before_manifest_replace")
+            os.replace(temporary, destination)
+            replaced = True
+            directory_descriptor = os.open(self._runs, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
+            try:
+                os.fsync(directory_descriptor)
+            finally:
+                os.close(directory_descriptor)
+            self._checkpoint(f"{checkpoint_prefix}.after_manifest_replace")
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_MANIFEST_IO",
+                "ingestion_manifest",
+                "could not atomically replace manifest",
+            ) from error
+        finally:
+            if descriptor >= 0:
+                os.close(descriptor)
+            if not replaced:
+                try:
+                    temporary.unlink(missing_ok=True)
+                except OSError:
+                    pass
+
+    def _verify_spool(self, spooled: SpooledChunk, chunk: ChunkRef) -> None:
+        if not isinstance(spooled, SpooledChunk):
+            raise IngestionError("INGESTION_SPOOL", "spooled", "must be a SpooledChunk")
+        if spooled.path.parent != self._spool or spooled.path.name.startswith(".") is False:
+            raise IngestionError(
+                "INGESTION_SPOOL_PATH",
+                "spooled.path",
+                "does not belong to this store's private spool",
+            )
+        if (
+            spooled.content_sha256 != chunk.content_sha256
+            or spooled.byte_length != chunk.byte_length
+        ):
+            raise IngestionError(
+                "INGESTION_SPOOL_DIGEST", "spooled", "does not match the proposed chunk reference"
+            )
+        try:
+            descriptor = os.open(spooled.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_SPOOL_IO", "spooled.path", "cannot be opened"
+            ) from error
+        try:
+            info = os.fstat(descriptor)
+            if (
+                not stat.S_ISREG(info.st_mode)
+                or info.st_dev != spooled.device
+                or info.st_ino != spooled.inode
+                or info.st_size != spooled.byte_length
+            ):
+                raise IngestionError(
+                    "INGESTION_SPOOL_PATH", "spooled.path", "changed after it was spooled"
+                )
+            digest = self._hash_descriptor(descriptor)
+        finally:
+            os.close(descriptor)
+        if digest != spooled.content_sha256:
+            raise IngestionError(
+                "INGESTION_SPOOL_DIGEST", "spooled.path", "bytes differ from the spooled digest"
+            )
+
+    def _verify_chunk_file(self, path: Path, chunk: ChunkRef) -> None:
+        try:
+            descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+        except OSError as error:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_IO", "chunk_ref.content_sha256", "blob cannot be opened"
+            ) from error
+        try:
+            info = os.fstat(descriptor)
+            if not stat.S_ISREG(info.st_mode) or info.st_size != chunk.byte_length:
+                raise IngestionCorrupt(
+                    "INGESTION_BLOB_SHAPE",
+                    "chunk_ref.content_sha256",
+                    "blob has an unexpected shape",
+                )
+            digest = self._hash_descriptor(descriptor)
+        finally:
+            os.close(descriptor)
+        if digest != chunk.content_sha256:
+            raise IngestionCorrupt(
+                "INGESTION_BLOB_DIGEST", "chunk_ref.content_sha256", "blob digest differs"
+            )
+
+    def _discard_spool(self, spooled: SpooledChunk) -> None:
+        try:
+            info = spooled.path.stat(follow_symlinks=False)
+            if (
+                stat.S_ISREG(info.st_mode)
+                and info.st_dev == spooled.device
+                and info.st_ino == spooled.inode
+                and spooled.path.parent == self._spool
+            ):
+                spooled.path.unlink()
+        except FileNotFoundError:
+            pass
+        except OSError:
+            # The durable state is already committed.  A private temporary may be reclaimed by
+            # a later maintenance operation; surfacing this as failure would make retry unsafe.
+            pass
+
+    def _reserve_spool_bytes(self, descriptor: int, reserved_length: int) -> None:
+        """Durably reserve one spool's next logical length without holding it while it streams.
+
+        The temporary file's ``st_size`` is the durable reservation ledger.  Reserving its next
+        exact length under the global lock makes every concurrent quota check see pending bytes;
+        writing the part happens after the lock is released.  An interrupted writer therefore
+        leaves only a locked, stale-reapable temporary file, never an invisible quota promise.
+        """
+
+        if isinstance(reserved_length, bool) or not isinstance(reserved_length, int):
+            raise IngestionError(
+                "INGESTION_SPOOL_LIMIT", "spool.parts", "must have an integer byte length"
+            )
+        if reserved_length < 0:
+            raise IngestionError(
+                "INGESTION_SPOOL_LIMIT", "spool.parts", "must not have a negative byte length"
+            )
+        try:
+            before = os.fstat(descriptor)
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_SPOOL_IO", "spool.path", "cannot inspect temporary spool"
+            ) from error
+        if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
+            raise IngestionError(
+                "INGESTION_SPOOL_PATH", "spool.path", "must be a private regular file"
+            )
+        if reserved_length < before.st_size:
+            raise IngestionError(
+                "INGESTION_SPOOL_LIMIT", "spool.parts", "cannot shrink a quota reservation"
+            )
+        with self._locked():
+            spool_bytes, store_bytes = self._usage_locked()
+            additional_bytes = reserved_length - before.st_size
+            if spool_bytes + additional_bytes > self.maximum_spool_total_bytes:
+                raise IngestionError(
+                    "INGESTION_SPOOL_QUOTA",
+                    "spool.parts",
+                    "would exceed the aggregate temporary-spool quota",
+                )
+            if store_bytes + additional_bytes > self.maximum_store_bytes:
+                raise IngestionError(
+                    "INGESTION_STORE_QUOTA",
+                    "spool.parts",
+                    "would exceed the aggregate local-store quota",
+                )
+            try:
+                os.ftruncate(descriptor, reserved_length)
+                os.fsync(descriptor)
+            except OSError as error:
+                raise IngestionError(
+                    "INGESTION_SPOOL_IO", "spool.path", "could not persist quota reservation"
+                ) from error
+
+    def _reap_stale_spools_locked(self, *, now: float) -> int:
+        """Delete only stale private temporary names not locked by an active writer."""
+
+        removed = 0
+        try:
+            entries = list(os.scandir(self._spool))
+        except OSError as error:
+            raise IngestionError(
+                "INGESTION_SPOOL_IO", "spool.directory", "cannot be listed"
+            ) from error
+        for entry in entries:
+            if not entry.name.startswith(".spool.") or not entry.name.endswith(".tmp"):
+                continue
+            descriptor = -1
+            try:
+                descriptor = os.open(entry.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+                info = os.fstat(descriptor)
+                if (
+                    not stat.S_ISREG(info.st_mode)
+                    or now - info.st_mtime < self.stale_spool_age_seconds
+                ):
+                    continue
+                try:
+                    fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                except BlockingIOError:
+                    continue
+                # Re-stat after obtaining the exclusive advisory lock: a writer that was active
+                # when we inspected the directory cannot be unlinked behind its back.
+                info = os.fstat(descriptor)
+                if now - info.st_mtime < self.stale_spool_age_seconds:
+                    continue
+                os.unlink(entry.path)
+                removed += 1
+            except FileNotFoundError:
+                continue
+            except OSError as error:
+                raise IngestionError(
+                    "INGESTION_SPOOL_IO", "spool.directory", "stale spool cleanup failed"
+                ) from error
+            finally:
+                if descriptor >= 0:
+                    os.close(descriptor)
+        if removed:
+            self._fsync_directory(self._spool, "ingestion_store.spool")
+        return removed
+
+    def _usage_locked(self) -> tuple[int, int]:
+        """Return logical ``(spool_bytes, total_store_bytes)`` with hard links counted once."""
+
+        spool_seen: set[tuple[int, int]] = set()
+        store_seen: set[tuple[int, int]] = set()
+        spool_bytes = 0
+        store_bytes = 0
+        for directory, is_spool in (
+            (self._runs, False),
+            (self._chunks, False),
+            (self._spool, True),
+        ):
+            try:
+                entries = list(os.scandir(directory))
+            except OSError as error:
+                raise IngestionError(
+                    "INGESTION_STORE_IO", "ingestion_store", "cannot inspect quota usage"
+                ) from error
+            for entry in entries:
+                try:
+                    info = entry.stat(follow_symlinks=False)
+                except FileNotFoundError:
+                    continue
+                except OSError as error:
+                    raise IngestionError(
+                        "INGESTION_STORE_IO", "ingestion_store", "cannot inspect stored object"
+                    ) from error
+                if not stat.S_ISREG(info.st_mode):
+                    raise IngestionCorrupt(
+                        "INGESTION_STORE_SHAPE",
+                        "ingestion_store",
+                        "contains a non-regular stored object",
+                    )
+                identity = (info.st_dev, info.st_ino)
+                if is_spool and identity not in spool_seen:
+                    spool_seen.add(identity)
+                    spool_bytes += info.st_size
+                if identity not in store_seen:
+                    store_seen.add(identity)
+                    store_bytes += info.st_size
+        return spool_bytes, store_bytes
+
+    def _ensure_store_capacity_locked(self, additional_bytes: int) -> None:
+        """Reserve temporary manifest-write space before creating the replacement file."""
+
+        _spool_bytes, store_bytes = self._usage_locked()
+        if store_bytes + additional_bytes > self.maximum_store_bytes:
+            raise IngestionError(
+                "INGESTION_STORE_QUOTA",
+                "ingestion_manifest",
+                "would exceed the aggregate local-store quota",
+            )
+
+    def _manifest_path(self, run_id: str) -> Path:
+        return self._runs / f"{run_id}.json"
+
+    def _chunk_path(self, digest: str) -> Path:
+        return self._chunks / f"{digest}.bin"
+
+    def _checkpoint(self, name: str) -> None:
+        if self.fault_injector is not None:
+            self.fault_injector.checkpoint(name)
+
+    @staticmethod
+    def _fsync_directory(path: Path, field: str) -> None:
+        descriptor = -1
+        try:
+            descriptor = os.open(
+                path,
+                os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
+            )
+            os.fsync(descriptor)
+        except OSError as error:
+            raise IngestionError("INGESTION_STORE_IO", field, "directory sync failed") from error
+        finally:
+            if descriptor >= 0:
+                os.close(descriptor)
+
+    def _validate_limits(self) -> None:
+        for field, value in (
+            ("maximum_spool_bytes", self.maximum_spool_bytes),
+            ("maximum_spool_total_bytes", self.maximum_spool_total_bytes),
+            ("maximum_store_bytes", self.maximum_store_bytes),
+        ):
+            if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+                raise IngestionError(
+                    "INGESTION_STORE_QUOTA", field, "must be a non-negative integer"
+                )
+        if self.maximum_spool_bytes > self.maximum_spool_total_bytes:
+            raise IngestionError(
+                "INGESTION_STORE_QUOTA",
+                "maximum_spool_bytes",
+                "must not exceed the aggregate temporary-spool quota",
+            )
+        if self.maximum_spool_total_bytes > self.maximum_store_bytes:
+            raise IngestionError(
+                "INGESTION_STORE_QUOTA",
+                "maximum_spool_total_bytes",
+                "must not exceed the aggregate local-store quota",
+            )
+        if (
+            isinstance(self.stale_spool_age_seconds, bool)
+            or not isinstance(self.stale_spool_age_seconds, int)
+            or self.stale_spool_age_seconds < 0
+        ):
+            raise IngestionError(
+                "INGESTION_SPOOL_AGE",
+                "stale_spool_age_seconds",
+                "must be a non-negative integer",
+            )
+
+    @staticmethod
+    def _require_private_directory(path: Path, field: str) -> None:
+        try:
+            info = path.stat(follow_symlinks=False)
+        except OSError as error:
+            raise IngestionError("INGESTION_STORE_IO", field, "cannot be inspected") from error
+        if (
+            not stat.S_ISDIR(info.st_mode)
+            or info.st_uid != os.geteuid()
+            or stat.S_IMODE(info.st_mode) != 0o700
+        ):
+            raise IngestionError(
+                "INGESTION_STORE_PATH",
+                field,
+                "must be an effective-user-owned mode-0700 non-symlink directory",
+            )
+
+    @staticmethod
+    def _validate_run_id(run_id: str) -> None:
+        # Constructing a temporary manifest would be misleading; the manifest constructor is
+        # intentionally the sole authority for this grammar, so reject path separators here too.
+        if (
+            not isinstance(run_id, str)
+            or not run_id
+            or len(run_id) > 128
+            or any(
+                character not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-"
+                for character in run_id
+            )
+            or run_id[0] in ".-"
+        ):
+            raise IngestionError("INGESTION_RUN_ID", "run_id", "is invalid")
+
+    @staticmethod
+    def _validate_manifest_digest(value: str) -> None:
+        if (
+            not isinstance(value, str)
+            or len(value) != 64
+            or any(character not in "0123456789abcdef" for character in value)
+        ):
+            raise IngestionError(
+                "INGESTION_MANIFEST_DIGEST",
+                "expected_manifest_sha256",
+                "must be a lowercase SHA-256",
+            )
+
+    @staticmethod
+    def _write_all(descriptor: int, raw: bytes) -> None:
+        view = memoryview(raw)
+        while view:
+            written = os.write(descriptor, view)
+            if written <= 0:
+                raise OSError("short manifest write")
+            view = view[written:]
+
+    @staticmethod
+    def _hash_descriptor(descriptor: int) -> str:
+        digest = hashlib.sha256()
+        os.lseek(descriptor, 0, os.SEEK_SET)
+        while raw := os.read(descriptor, _READ_CHUNK_SIZE):
+            digest.update(raw)
+        return digest.hexdigest()
+
+    @staticmethod
+    def _read_descriptor(descriptor: int, *, maximum: int) -> bytes:
+        """Read a small control-plane document under a strict byte ceiling."""
+
+        parts: list[bytes] = []
+        total = 0
+        os.lseek(descriptor, 0, os.SEEK_SET)
+        while raw := os.read(descriptor, _READ_CHUNK_SIZE):
+            total += len(raw)
+            if total > maximum:
+                raise IngestionCorrupt(
+                    "INGESTION_FILE_LIMIT",
+                    "ingestion_store",
+                    "file exceeds its permitted byte limit",
+                )
+            parts.append(raw)
+        return b"".join(parts)