mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/authoring_shards.py

@@ -0,0 +1,1217 @@
+"""Bounded streaming authoring shards behind one atomic manifest.
+
+A converged Data.gov v4 generation is immutable staging: raw response blobs, normalized page
+manifests over bounded record segments, segmented evidence indexes, and one terminal externally
+merged pass map.  This module turns that generation into immutable authoring
+shards without ever holding it in memory, and replaces the 64 MiB monolithic
+``mr-data-catalog-fill-authoring.v1`` bundle for the provider-scale workflow.  The legacy bounded
+authoring path in :mod:`authoring` is untouched and still readable.
+
+The shape of the work is deliberate.
+
+*Two bounded passes, never one big map.*  The first pass streams the terminal pass's normalized
+pages one segment at a time into a disk-backed scratch index keyed by the exact provider identifier
+bytes.  The second pass walks that index in ascending identifier order and authors one record at a
+time into one open output shard.  Peak retained records is therefore one input segment plus one
+output shard, not one generation, and the authoring order is a property of the provider identifiers
+rather than of page arrival.
+
+*Immutable members, manifest last.*  Every shard is content-addressed, installed through a
+same-directory link with fsync and exact readback, and never rewritten.  A small journal records
+the committed shards and the exact last committed provider record so an interrupted run resumes
+without duplicating or reordering anything.  ``manifest.json`` is written only when the whole
+generation is authored, so a crash leaves either a resumable partial generation or a complete one
+-- never a half-installed head.
+
+*Exact coordinates only.*  Authoring refuses to start unless the caller names the exact live fill
+checkpoint digest, the fill reports a converged terminal pass whose pass map authenticates, and the
+indexed record population equals that pass map's unique record count.  A resume additionally
+requires the exact retained journal digest.
+
+*One stated coverage, bounded or not.*  A converged sweep can be larger than a release can carry,
+so a caller may ask for a bounded selection of it instead of all of it: an ``AuthoringSelection``
+names one registered rule and how many records to take under it.  The rule is a total order over
+the *whole* corpus, evaluated after the index is built and before one record is authored, so the
+records it selects are a property of the corpus rather than of how far this invocation got.  That
+is what makes a bounded generation ``complete``: it authored the whole of what it said it would.
+
+A bound is emphatically not the record budget.  ``limits.max_records`` is a resource guard, it is
+live on every invocation bounded or not, and a run that reaches it still stops at
+``AUTHOR_RECORD_LIMIT`` with ``status="incomplete"`` and publishes no manifest.  An operator who
+sets a budget below their own bound gets an incomplete generation, exactly as they would have
+before this existed.  Nothing here relaxes that; ``coverage`` is a claim about which records were
+asked for, ``limits`` is a claim about what this process was allowed to spend, and neither stands
+in for the other.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import re
+import secrets
+import sqlite3
+import stat
+import tempfile
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.authoring_policy import (
+    AuthoringPolicy,
+    author_catalog_record,
+    resolve_authoring_policy,
+)
+from mostlyright.data_harness.sources.catalog.bounded_io import (
+    BoundedReadFailure,
+    read_bounded_at,
+)
+from mostlyright.data_harness.sources.catalog.contracts import EMBEDDING_LAYERS
+from mostlyright.data_harness.sources.catalog.coverage import (
+    COVERAGE_EXHAUSTIVE_RULE,
+    SELECTION_RULES,
+    coverage_is_valid,
+)
+from mostlyright.data_harness.sources.catalog.fill_partitions import validate_pass_map
+from mostlyright.data_harness.sources.catalog.fill_staging import FillStaging
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+AUTHORING_SHARD_SCHEMA = "harness-catalog-authoring-shard.v1"
+AUTHORING_MANIFEST_SCHEMA = "harness-catalog-authoring-manifest.v1"
+AUTHORING_JOURNAL_SCHEMA = "harness-catalog-authoring-journal.v1"
+AUTHORING_RECEIPT_SCHEMA = "mr-data-catalog-author.v1"
+
+MANIFEST_FILENAME = "manifest.json"
+JOURNAL_FILENAME = "journal.json"
+SHARDS_DIRNAME = "shards"
+
+MAX_AUTHORING_SHARD_ENTRIES = 1_000
+MAX_AUTHORING_SHARD_BYTES = 8 * 1024 * 1024
+MAX_AUTHORING_MANIFEST_BYTES = 8 * 1024 * 1024
+MAX_AUTHORING_JOURNAL_BYTES = 8 * 1024 * 1024
+
+#: Exactly what one authored shard descriptor states, and nothing else.  The journal and the
+#: manifest both carry these descriptors, and the composite generation receipt carries them
+#: onward as authoritative evidence, so every reader closes the key set rather than reading the
+#: members it happens to know and copying whatever else the document brought with it.
+AUTHORING_SHARD_DESCRIPTOR_MEMBERS = frozenset(
+    {
+        "sha256",
+        "bytes",
+        "item_count",
+        "entry_count",
+        "first_provider_record_id",
+        "last_provider_record_id",
+    }
+)
+
+
+def shard_descriptor_is_valid(descriptor: Any) -> bool:
+    """Whether one authored shard descriptor is exactly its contract, with admissible values.
+
+    Every reader that admits a descriptor answers this one question, so that adding a member to
+    the contract is one edit rather than three that have to agree.  Each caller still raises its
+    own typed refusal, because the boundary the descriptor failed at is what an operator needs.
+    """
+
+    return (
+        isinstance(descriptor, dict)
+        and set(descriptor) == AUTHORING_SHARD_DESCRIPTOR_MEMBERS
+        and _is_digest(descriptor["sha256"])
+        and type(descriptor["bytes"]) is int
+        and 1 <= descriptor["bytes"] <= MAX_AUTHORING_SHARD_BYTES
+        and type(descriptor["item_count"]) is int
+        and 1 <= descriptor["item_count"] <= MAX_AUTHORING_SHARD_ENTRIES
+        and type(descriptor["entry_count"]) is int
+        and 0 <= descriptor["entry_count"] <= descriptor["item_count"]
+        and isinstance(descriptor["first_provider_record_id"], str)
+        and isinstance(descriptor["last_provider_record_id"], str)
+    )
+
+
+# Conservative allowance for the shard envelope around the items array: the schema label, the
+# policy digest, the two bounded provider record identifiers, and the JSON punctuation. The real
+# ceiling is still enforced exactly when the shard bytes are composed.
+_SHARD_ENVELOPE_BYTES = 4_096
+
+#: The provider observation ``last_harvested_date_desc`` orders by.  It is a *volatile* field --
+#: deliberately outside the semantic digest the pass map converges on, because a digest that moved
+#: with a nightly re-harvest would make two adjacent equal pass maps unsatisfiable.  That is
+#: exactly right for a selection key: the corpus it orders is immutable staging, so the order is
+#: reproducible from the generation even though the value is not part of any record's identity.
+_RANK_OBSERVATION_PATH = "$.last_harvested_date"
+
+#: The longest provider harvest date this rule will order by.  A well-formed instant is 20
+#: characters; 64 is room for every spelling of one.  Anything longer is not a date this rule can
+#: order by, so it ranks with the undated rather than sorting on its leading bytes.
+MAX_SELECTION_RANK_CHARS = 64
+
+#: The shape a provider value must have before this rule will treat it as a harvest date.
+#:
+#: Byte order is only chronological order over values that are actually dates.  Comparison here is
+#: bytewise, so *any* string sorts somewhere -- and because ``'N' < '2' < 'u'`` is false in the
+#: middle, a record whose harvest date the provider filled in as ``unknown`` or ``pending`` would
+#: sort above every real ISO-8601 instant and head the catalogue.  "The N most recently harvested
+#: datasets" would then be led by the N records with no harvest date at all, which is the exact
+#: opposite of what the rule's name promises.
+#:
+#: So a value is a rank only if it opens with a calendar date, ``YYYY-MM-DD``.  Every spelling of
+#: an ISO-8601 instant does, and no English word does.  The rule deliberately checks the opening
+#: rather than the whole grammar: what the bytes after the date look like varies by provider and
+#: does not change which day is later, and refusing a real date over its seconds field would drop
+#: a record from the catalogue to satisfy a parser.
+_RANK_DATE_PREFIX = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+#: What a record with no orderable harvest date ranks as.  The empty string sorts below every
+#: non-empty one, so undated records fall to the bottom of a descending order without a separate
+#: null-handling clause that a query planner could get wrong.
+_UNRANKED = ""
+
+_NORMALIZED_PAGE_SCHEMA = "harness-datagov-v4-normalized-page.v2"
+_NORMALIZED_SEGMENT_SCHEMA = "harness-datagov-v4-normalized-record-segment.v1"
+_OPEN_DIRECTORY = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+_OPEN_MEMBER = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+
+
+class CatalogAuthoringRefused(SourceContractError):
+    """A stable refusal of an authoring coordinate, input, or output boundary."""
+
+
+@dataclass(frozen=True)
+class AuthoringLimits:
+    """Explicit caller-supplied authoring bounds; none of them has a hidden default widening."""
+
+    max_records: int
+    max_input_bytes: int
+    max_shard_entries: int = MAX_AUTHORING_SHARD_ENTRIES
+    max_shard_bytes: int = MAX_AUTHORING_SHARD_BYTES
+    max_disk_bytes: int = 30 * 1024 * 1024 * 1024
+    max_wall_seconds: int = 12 * 60 * 60
+
+    def __post_init__(self) -> None:
+        for name in (
+            "max_records",
+            "max_input_bytes",
+            "max_shard_entries",
+            "max_shard_bytes",
+            "max_disk_bytes",
+            "max_wall_seconds",
+        ):
+            value = getattr(self, name)
+            if type(value) is not int or value < 1:
+                raise CatalogAuthoringRefused(
+                    "AUTHOR_LIMIT", f"limits.{name}", "must be a positive integer"
+                )
+        if (
+            self.max_shard_entries > MAX_AUTHORING_SHARD_ENTRIES
+            or self.max_shard_bytes > MAX_AUTHORING_SHARD_BYTES
+        ):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_LIMIT",
+                "limits.max_shard_entries",
+                "shard bounds may not exceed the fixed range contract",
+            )
+
+    def to_dict(self) -> dict[str, int]:
+        return {
+            "max_records": self.max_records,
+            "max_input_bytes": self.max_input_bytes,
+            "max_shard_entries": self.max_shard_entries,
+            "max_shard_bytes": self.max_shard_bytes,
+            "max_disk_bytes": self.max_disk_bytes,
+            "max_wall_seconds": self.max_wall_seconds,
+        }
+
+
+@dataclass(frozen=True)
+class AuthoringSelection:
+    """Which of a converged generation's records this authoring was asked to author.
+
+    The default is every one of them, which is what authoring did before a selection could be
+    stated and is what it still does when none is.  Defaulting to exhaustive is the safe
+    direction: a caller who forgets the flag publishes a whole catalogue, never a quiet part of
+    one.
+
+    A bounded selection names a *registered* rule rather than describing one, so that "which
+    records are in this catalogue" has an answer a reader can look up instead of a sentence
+    somebody wrote into a receipt.
+    """
+
+    rule: str = COVERAGE_EXHAUSTIVE_RULE
+    bound: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.rule == COVERAGE_EXHAUSTIVE_RULE:
+            if self.bound is not None:
+                raise CatalogAuthoringRefused(
+                    "AUTHOR_COVERAGE",
+                    "selection.bound",
+                    "an exhaustive authoring states no bound",
+                )
+            return
+        if self.rule not in SELECTION_RULES:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_COVERAGE",
+                "selection.rule",
+                f"a bounded authoring names one registered rule {sorted(SELECTION_RULES)}",
+            )
+        if type(self.bound) is not int or self.bound < 1:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_COVERAGE",
+                "selection.bound",
+                "a bounded authoring states a positive record bound",
+            )
+
+    def coverage(self, *, corpus_records: int, selected_records: int) -> dict[str, Any]:
+        """What this selection, applied to a corpus this size, says it covers."""
+
+        coverage = {
+            "rule": self.rule,
+            "bound": self.bound,
+            "corpus_records": corpus_records,
+            "selected_records": selected_records,
+        }
+        if not coverage_is_valid(coverage):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_COVERAGE",
+                "authoring.coverage",
+                "the selected population does not reproduce from the stated rule and bound",
+            )
+        return coverage
+
+
+@dataclass(frozen=True)
+class AuthoringResult:
+    """One authoring invocation's exact outcome."""
+
+    status: str
+    reason_code: str | None
+    manifest_sha256: str | None
+    journal_sha256: str | None
+    coverage: dict[str, Any]
+    receipt: dict[str, Any]
+
+
+def run_catalog_authoring(
+    *,
+    staging_root: Path,
+    expected_fill_checkpoint_sha256: str,
+    policy_id: str,
+    output_root: Path,
+    limits: AuthoringLimits,
+    selection: AuthoringSelection | None = None,
+    expected_journal_sha256: str | None = None,
+    output_descriptor: int | None = None,
+    monotonic: Callable[[], float] = time.monotonic,
+) -> AuthoringResult:
+    """Author one converged v4 generation into bounded shards behind one atomic manifest."""
+
+    selection = AuthoringSelection() if selection is None else selection
+    policy = resolve_authoring_policy(policy_id)
+    if not _is_digest(expected_fill_checkpoint_sha256):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_FILL_CHECKPOINT_MISMATCH",
+            "expected_fill_checkpoint_sha256",
+            "must be a lowercase SHA-256",
+        )
+    if expected_journal_sha256 is not None and not _is_digest(expected_journal_sha256):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_JOURNAL_MISMATCH", "expected_journal_sha256", "must be a lowercase SHA-256"
+        )
+    deadline = monotonic() + limits.max_wall_seconds
+    output = _AuthoringOutput(output_root, root_descriptor=output_descriptor)
+    with output.opened():
+        journal = _resume_point(
+            output, expected_journal_sha256, policy, expected_fill_checkpoint_sha256, selection
+        )
+        _require_bounds_admit_retained(journal, limits)
+        staging = FillStaging(staging_root)
+        with staging.locked():
+            state = _converged_state(staging, expected_fill_checkpoint_sha256)
+            pass_map = validate_pass_map(
+                staging, state["last_pass_manifest_sha256"], state["final_map_sha256"]
+            )
+            scratch_root = _scratch_root(output_root, staging_root)
+            with tempfile.TemporaryDirectory(
+                prefix="mostlyright-authoring-", dir=scratch_root
+            ) as directory:
+                index = _RecordIndex(Path(directory) / "records.sqlite3")
+                try:
+                    index.build(staging, state, limits=limits)
+                    if index.count != pass_map.unique_records:
+                        raise CatalogAuthoringRefused(
+                            "AUTHOR_RECORD_COUNT",
+                            "authoring.records",
+                            "indexed record population differs from the terminal pass map",
+                        )
+                    # The rule is applied to the whole corpus before one record is authored, so
+                    # what it selects is a property of the corpus rather than of how far this
+                    # invocation gets. `corpus_records` is the pass map's own count, never the
+                    # caller's, so a coverage statement cannot overstate what it drew from.
+                    coverage = selection.coverage(
+                        corpus_records=pass_map.unique_records,
+                        selected_records=index.select(selection),
+                    )
+                    _require_coverage_matches_retained(journal, coverage)
+                    _refuse_disk(output, index, limits)
+                    return _author_generation(
+                        index,
+                        output=output,
+                        policy=policy,
+                        staging_coordinate=_staging_coordinate(
+                            state, pass_map, expected_fill_checkpoint_sha256
+                        ),
+                        coverage=coverage,
+                        limits=limits,
+                        journal=journal,
+                        deadline=deadline,
+                        monotonic=monotonic,
+                    )
+                finally:
+                    index.close()
+
+
+# ------------------------------------------------------------------------------------------
+# Converged staging
+# ------------------------------------------------------------------------------------------
+
+
+def _converged_state(staging: FillStaging, expected_checkpoint_sha256: str) -> dict[str, Any]:
+    state = staging.load()
+    if state is None or staging.state_digest() != expected_checkpoint_sha256:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_FILL_CHECKPOINT_MISMATCH",
+            "fill.staging.state",
+            "no live checkpoint matches the caller's exact digest",
+        )
+    if "v4_checkpoint_schema_version" not in state or not isinstance(
+        state.get("evidence_indexes"), dict
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_FILL_PROTOCOL",
+            "fill.staging.state",
+            "streaming authoring reads only a segmented v4 generation",
+        )
+    if (
+        state.get("completed") is not True
+        or not _is_digest(state.get("final_map_sha256"))
+        or not _is_digest(state.get("last_pass_manifest_sha256"))
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_FILL_INCOMPLETE",
+            "fill.staging.state",
+            "authoring requires a converged terminal fill generation",
+        )
+    return state
+
+
+def _selection_rank(record: Any) -> str:
+    """The exact provider text ``last_harvested_date_desc`` orders this record by.
+
+    A record ranks by whatever the provider stated and nothing derived from it: no parsing, no
+    normalising, no reinterpretation.  Ordering the provider's own bytes is what makes the rule
+    reproducible by anyone holding the staging generation, and for the ISO-8601 instants this
+    provider publishes, byte order *is* chronological order.
+
+    A value this rule cannot order by returns ``_UNRANKED``, which sorts below every value it
+    can.  Five things are unorderable and all of them mean the same thing -- that the provider
+    published no usable harvest date for this record: no such observation at all, an observation
+    the harvester had to retain in a typed encoding rather than as canonical JSON, a value that
+    is not a string, a string too long to be any spelling of an instant, and a string that does
+    not open with a calendar date.  None of them is treated as recent, because none of them is
+    evidence of recency -- and the last of them would otherwise be treated as the *most* recent
+    of all, since bytewise ``"unknown" > "2026-08-09T00:00:00Z"``.
+    """
+
+    observations = record.get("observations") if isinstance(record, dict) else None
+    if not isinstance(observations, list):
+        return _UNRANKED
+    for field in observations:
+        if not isinstance(field, dict) or field.get("path") != _RANK_OBSERVATION_PATH:
+            continue
+        if field.get("encoding", "canonical_json") != "canonical_json":
+            return _UNRANKED
+        value = field.get("value")
+        if (
+            isinstance(value, str)
+            and 0 < len(value) <= MAX_SELECTION_RANK_CHARS
+            and _RANK_DATE_PREFIX.match(value) is not None
+        ):
+            return value
+        return _UNRANKED
+    return _UNRANKED
+
+
+class _RecordIndex:
+    """A disk-backed identifier-ordered index over one terminal pass's normalized records."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.count = 0
+        self.selected = 0
+        self.input_bytes = 0
+        self._bounded = False
+        self._connection = sqlite3.connect(path)
+        self._connection.execute("PRAGMA journal_mode=OFF")
+        self._connection.execute("PRAGMA synchronous=OFF")
+        self._connection.execute("PRAGMA temp_store=FILE")
+        self._connection.execute(
+            "CREATE TABLE record ("
+            "identifier BLOB NOT NULL PRIMARY KEY, semantic TEXT NOT NULL, "
+            "page TEXT NOT NULL, observed TEXT NOT NULL, rank TEXT NOT NULL, "
+            "payload BLOB NOT NULL"
+            ") WITHOUT ROWID"
+        )
+        self._connection.execute(
+            "CREATE TABLE response (raw TEXT NOT NULL PRIMARY KEY, observed TEXT NOT NULL) "
+            "WITHOUT ROWID"
+        )
+
+    def close(self) -> None:
+        self._connection.close()
+
+    @property
+    def disk_bytes(self) -> int:
+        try:
+            return self.path.stat().st_size
+        except OSError:
+            return 0
+
+    def build(
+        self, staging: FillStaging, state: dict[str, Any], *, limits: AuthoringLimits
+    ) -> None:
+        indexes = state["evidence_indexes"]
+        for item in staging.iter_index(indexes["response"], stream="response"):
+            raw = item.get("raw_response_sha256")
+            observed = item.get("observed_epoch_seconds")
+            if _is_digest(raw) and type(observed) is int:
+                self._connection.execute(
+                    "INSERT INTO response(raw, observed) VALUES (?, ?) ON CONFLICT(raw) DO NOTHING",
+                    (raw, _utc_second(observed)),
+                )
+        self._connection.commit()
+        for item in staging.iter_index(indexes["page"], stream="page"):
+            if item.get("pass") != state["pass_number"]:
+                continue
+            self._read_page(staging, item["normalized_sha256"], limits=limits)
+        self._connection.commit()
+        self.count = int(self._connection.execute("SELECT count(*) FROM record").fetchone()[0])
+
+    def _read_page(
+        self, staging: FillStaging, page_sha256: str, *, limits: AuthoringLimits
+    ) -> None:
+        manifest = staging.read_shard("normalized", page_sha256)
+        if manifest.get("schema_version") != _NORMALIZED_PAGE_SCHEMA or not isinstance(
+            manifest.get("record_segments"), list
+        ):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_STAGING_CORRUPT", "fill.normalized", "normalized page manifest is invalid"
+            )
+        for segment in manifest["record_segments"]:
+            payload = staging.read_shard("normalized", segment["sha256"])
+            if payload.get("schema_version") != _NORMALIZED_SEGMENT_SCHEMA or not isinstance(
+                payload.get("records"), list
+            ):
+                raise CatalogAuthoringRefused(
+                    "AUTHOR_STAGING_CORRUPT",
+                    "fill.normalized",
+                    "normalized record segment is invalid",
+                )
+            for record in payload["records"]:
+                self._insert(record, page_sha256=page_sha256, limits=limits)
+
+    def _insert(self, record: Any, *, page_sha256: str, limits: AuthoringLimits) -> None:
+        if (
+            not isinstance(record, dict)
+            or not isinstance(record.get("record_id"), str)
+            or not _is_digest(record.get("normalized_sha256"))
+            or not _is_digest(record.get("raw_response_sha256"))
+        ):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_STAGING_CORRUPT", "fill.normalized", "normalized record is invalid"
+            )
+        try:
+            raw = canonical_json_bytes(record)
+        except CanonicalJSONError as error:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_STAGING_CORRUPT", "fill.normalized", "normalized record is not canonical"
+            ) from error
+        self.input_bytes += len(raw)
+        if self.input_bytes > limits.max_input_bytes:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_INPUT_LIMIT",
+                "authoring.input",
+                "normalized input exceeds the caller's byte bound",
+            )
+        row = self._connection.execute(
+            "SELECT observed FROM response WHERE raw = ?", (record["raw_response_sha256"],)
+        ).fetchone()
+        if row is None:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_STAGING_CORRUPT",
+                "fill.normalized",
+                "normalized record cites no retained response observation",
+            )
+        identifier = record["record_id"].encode("utf-8")
+        existing = self._connection.execute(
+            "SELECT semantic FROM record WHERE identifier = ?", (identifier,)
+        ).fetchone()
+        if existing is not None:
+            if existing[0] != record["normalized_sha256"]:
+                raise CatalogAuthoringRefused(
+                    "AUTHOR_RECORD_CONFLICT",
+                    "authoring.records",
+                    "one provider identifier carries two different semantic digests",
+                )
+            return
+        self._connection.execute(
+            "INSERT INTO record(identifier, semantic, page, observed, rank, payload) "
+            "VALUES (?, ?, ?, ?, ?, ?)",
+            (
+                identifier,
+                record["normalized_sha256"],
+                page_sha256,
+                row[0],
+                _selection_rank(record),
+                raw,
+            ),
+        )
+
+    def select(self, selection: AuthoringSelection) -> int:
+        """Apply one selection rule to the whole indexed corpus and answer what it selected.
+
+        The selection is a set, computed once, before any record is authored.  Authoring still
+        walks provider identifiers in ascending order afterwards, because that order is what the
+        journal resumes on, what a shard's two range identifiers describe, and what every reader
+        of an authored generation merges on.  Ranking decides *which* records; the identifier
+        decides *when* each one is authored.  Confusing the two would make the selection depend
+        on where an interrupted run stopped, which is the truncation this whole path exists to
+        avoid.
+        """
+
+        if selection.bound is None:
+            self.selected = self.count
+            return self.selected
+        # The ordering index is what keeps the selection from re-sorting the whole corpus on
+        # every read; its pages live inside the index file, which `_refuse_disk` measures. The
+        # sort that *builds* it spills to SQLite's temp directory under `temp_store=FILE`, which
+        # is transient and outside that measure -- so the bound's disk cost is accounted where it
+        # is durable and merely bounded by the corpus where it is not. A bound is the only thing
+        # that needs an order other than the primary key, so an exhaustive authoring pays neither.
+        self._connection.execute("CREATE INDEX record_rank ON record(rank DESC, identifier ASC)")
+        self._connection.execute(
+            "CREATE TABLE selected (identifier BLOB NOT NULL PRIMARY KEY) WITHOUT ROWID"
+        )
+        self._connection.execute(
+            "INSERT INTO selected(identifier) "
+            "SELECT identifier FROM record ORDER BY rank DESC, identifier ASC LIMIT ?",
+            (selection.bound,),
+        )
+        self._connection.commit()
+        self._bounded = True
+        self.selected = int(self._connection.execute("SELECT count(*) FROM selected").fetchone()[0])
+        return self.selected
+
+    def stream(self, after: str | None) -> Iterator[tuple[str, str, str, bytes]]:
+        source = (
+            "record JOIN selected ON selected.identifier = record.identifier"
+            if self._bounded
+            else "record"
+        )
+        columns = "record.identifier, record.page, record.observed, record.payload"
+        if after is None:
+            cursor = self._connection.execute(
+                f"SELECT {columns} FROM {source} ORDER BY record.identifier"
+            )
+        else:
+            cursor = self._connection.execute(
+                f"SELECT {columns} FROM {source} "
+                "WHERE record.identifier > ? ORDER BY record.identifier",
+                (after.encode("utf-8"),),
+            )
+        while rows := cursor.fetchmany(64):
+            for identifier, page, observed, payload in rows:
+                yield identifier.decode("utf-8"), page, observed, payload
+
+
+# ------------------------------------------------------------------------------------------
+# Authoring
+# ------------------------------------------------------------------------------------------
+
+
+def _author_generation(
+    index: _RecordIndex,
+    *,
+    output: _AuthoringOutput,
+    policy: AuthoringPolicy,
+    staging_coordinate: dict[str, Any],
+    coverage: dict[str, Any],
+    limits: AuthoringLimits,
+    journal: dict[str, Any],
+    deadline: float,
+    monotonic: Callable[[], float],
+) -> AuthoringResult:
+    shards: list[dict[str, Any]] = list(journal["shards"])
+    counts = dict(journal["counts"])
+    digests = dict(journal["layer_text_digests"])
+    last_id: str | None = journal["last_provider_record_id"]
+    buffered: list[dict[str, Any]] = []
+    buffered_bytes = 0
+    stop: str | None = None
+
+    def flush() -> None:
+        nonlocal buffered, buffered_bytes
+        if not buffered:
+            return
+        payload = {
+            "schema_version": AUTHORING_SHARD_SCHEMA,
+            "policy_sha256": policy.digest,
+            "first_provider_record_id": buffered[0]["provider_record_id"],
+            "last_provider_record_id": buffered[-1]["provider_record_id"],
+            "items": buffered,
+        }
+        digest, size = output.write_shard(payload, maximum=limits.max_shard_bytes)
+        shards.append(
+            {
+                "sha256": digest,
+                "bytes": size,
+                "item_count": len(buffered),
+                "entry_count": sum(1 for item in buffered if item["entry"] is not None),
+                "first_provider_record_id": payload["first_provider_record_id"],
+                "last_provider_record_id": payload["last_provider_record_id"],
+            }
+        )
+        buffered = []
+        buffered_bytes = 0
+        _refuse_disk(output, index, limits)
+
+    for identifier, page_sha256, observed_at, payload in index.stream(last_id):
+        if counts["records"] >= limits.max_records:
+            stop = "AUTHOR_RECORD_LIMIT"
+            break
+        if monotonic() >= deadline:
+            stop = "AUTHOR_WALL_LIMIT"
+            break
+        authored = author_catalog_record(
+            parse_canonical_json(payload),
+            policy=policy,
+            observed_at=observed_at,
+            page_evidence_sha256=page_sha256,
+        )
+        item = authored.to_dict()
+        item_bytes = len(canonical_json_bytes(item))
+        if buffered and (
+            len(buffered) >= limits.max_shard_entries
+            or buffered_bytes + item_bytes + len(buffered) + _SHARD_ENVELOPE_BYTES
+            > limits.max_shard_bytes
+        ):
+            flush()
+        buffered.append(item)
+        buffered_bytes += item_bytes
+        counts["records"] += 1
+        counts[authored.disposition] += 1
+        counts["layer_text_truncated"] += len(authored.truncated_layers)
+        if authored.entry is not None:
+            for layer, text in authored.layer_texts:
+                digests[layer] = _layer_chain(digests[layer], authored.entry.entry_id, layer, text)
+        last_id = identifier
+    flush()
+
+    # The journal carries the coverage of the generation it continues, so a resume that restated
+    # the rule or the bound is refused against retained evidence rather than against an argument
+    # this process happens to hold.
+    journal_sha256 = output.publish(
+        JOURNAL_FILENAME,
+        {
+            "schema_version": AUTHORING_JOURNAL_SCHEMA,
+            "policy_sha256": policy.digest,
+            "staging_checkpoint_sha256": staging_coordinate["checkpoint_sha256"],
+            "coverage": coverage,
+            "shards": shards,
+            "counts": counts,
+            "layer_text_digests": digests,
+            "last_provider_record_id": last_id,
+            "reason_code": stop,
+        },
+        maximum=MAX_AUTHORING_JOURNAL_BYTES,
+    )
+    _refuse_disk(output, index, limits)
+    # The manifest is written only when the stream ran out, which under a selection means the
+    # selected population was authored in full: `counts["records"]` and
+    # `coverage["selected_records"]` cannot differ here without one of them having been forged
+    # after the fact. That is a reader's question rather than a writer's, and every stage that
+    # reads an authored manifest asks it -- see `admit_authored_generation` and the publisher's
+    # receipt reconciliation, both of which hold a document they did not write.
+    manifest_sha256: str | None = None
+    if stop is None:
+        manifest_sha256 = output.publish(
+            MANIFEST_FILENAME,
+            _manifest(policy, staging_coordinate, coverage, counts, digests, shards),
+            maximum=MAX_AUTHORING_MANIFEST_BYTES,
+        )
+    return AuthoringResult(
+        status="complete" if stop is None else "incomplete",
+        reason_code=stop,
+        manifest_sha256=manifest_sha256,
+        journal_sha256=journal_sha256,
+        coverage=coverage,
+        receipt={
+            "schema_version": AUTHORING_RECEIPT_SCHEMA,
+            "status": "complete" if stop is None else "incomplete",
+            "reason_code": stop,
+            "policy_id": policy.policy_id,
+            "policy_sha256": policy.digest,
+            "provider_id": policy.provider_id,
+            "harvester_coordinate": policy.harvester_coordinate,
+            "staging": staging_coordinate,
+            "coverage": coverage,
+            "counts": counts,
+            "layer_text_digests": digests,
+            "shard_count": len(shards),
+            "shards": shards,
+            "limits": limits.to_dict(),
+            "manifest_sha256": manifest_sha256,
+            "journal_sha256": journal_sha256,
+            "output_bytes": output.bytes_written,
+        },
+    )
+
+
+def _manifest(
+    policy: AuthoringPolicy,
+    staging: dict[str, Any],
+    coverage: dict[str, Any],
+    counts: dict[str, int],
+    digests: dict[str, str | None],
+    shards: list[dict[str, Any]],
+) -> dict[str, Any]:
+    body = {
+        "schema_version": AUTHORING_MANIFEST_SCHEMA,
+        "policy_id": policy.policy_id,
+        "policy_sha256": policy.digest,
+        "staging": staging,
+        "coverage": coverage,
+        "counts": counts,
+        "layer_text_digests": digests,
+        "shards": shards,
+    }
+    return {**body, "root_sha256": canonical_sha256(body)}
+
+
+def _staging_coordinate(state: dict[str, Any], pass_map, checkpoint_sha256: str) -> dict[str, Any]:
+    indexes = state["evidence_indexes"]
+    return {
+        "checkpoint_sha256": checkpoint_sha256,
+        "config_sha256": state["config_sha256"],
+        "endpoint": state["endpoint"],
+        "harvester_coordinate": state["harvester_coordinate"],
+        "pass_number": state["pass_number"],
+        "pass_map_sha256": pass_map.manifest_sha256,
+        "pass_map_root_sha256": pass_map.root_sha256,
+        "unique_records": pass_map.unique_records,
+        "raw_response_index_root_sha256": indexes["response"]["root_sha256"],
+        "raw_response_index_count": indexes["response"]["count"],
+        "normalized_page_index_root_sha256": indexes["page"]["root_sha256"],
+        "normalized_page_index_count": indexes["page"]["count"],
+        "record_index_root_sha256": indexes["record"]["root_sha256"],
+        "record_index_count": indexes["record"]["count"],
+    }
+
+
+def _layer_chain(previous: str | None, entry_id: str, layer: str, text: str) -> str:
+    return canonical_sha256(
+        {
+            "previous_sha256": previous,
+            "entry_id": entry_id,
+            "layer": layer,
+            "text_sha256": sha256_bytes(text.encode("utf-8")),
+        }
+    )
+
+
+def _refuse_disk(output: _AuthoringOutput, index: _RecordIndex, limits: AuthoringLimits) -> None:
+    if output.bytes_written + index.disk_bytes > limits.max_disk_bytes:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_DISK_LIMIT", "authoring.output", "authoring exceeds its local disk bound"
+        )
+
+
+def _require_bounds_admit_retained(journal: dict[str, Any], limits: AuthoringLimits) -> None:
+    """A resume may not narrow a bound the generation it continues has already passed.
+
+    The receipt an authoring stage writes states the bounds of the invocation that *finished* the
+    generation, and a publication reconciles those bounds against every shard in the manifest --
+    including the ones earlier invocations wrote.  Narrowing a bound mid-generation would produce
+    a complete generation that no publication can accept, and it would say so hours later, at the
+    publisher, about work that is already durable.  It is refused here instead, where the
+    narrowing happened and before any further record is authored.
+
+    A fresh authoring resumes an empty journal, so nothing here constrains a first invocation.
+    """
+
+    shards = journal["shards"]
+    if (
+        limits.max_records < journal["counts"]["records"]
+        or any(limits.max_shard_entries < descriptor["item_count"] for descriptor in shards)
+        or any(limits.max_shard_bytes < descriptor["bytes"] for descriptor in shards)
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_LIMIT",
+            "limits",
+            "a resume may not narrow a bound the retained generation has already passed",
+        )
+
+
+def _require_coverage_matches_retained(journal: dict[str, Any], coverage: dict[str, Any]) -> None:
+    """A resume continues one coverage claim; it never restates it.
+
+    ``_require_bounds_admit_retained`` lets a resume widen a *bound* because a bound is a
+    permission to spend and a wider one strands nothing.  Coverage is not that.  It is the claim
+    the generation makes about which records it holds, and shards written under one claim plus
+    shards written under another are a generation that describes neither.  So this is equality,
+    in both directions, and it is checked against the retained journal rather than against a
+    caller's argument.
+
+    A fresh authoring resumes an empty journal, so nothing here constrains a first invocation.
+    """
+
+    retained = journal["coverage"]
+    if retained is not None and retained != coverage:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_COVERAGE_MISMATCH",
+            "authoring.coverage",
+            "the retained partial generation was authored under another stated coverage",
+        )
+
+
+def _empty_journal() -> dict[str, Any]:
+    return {
+        "coverage": None,
+        "shards": [],
+        "counts": {
+            "records": 0,
+            "authored": 0,
+            "flagged": 0,
+            "skipped": 0,
+            "failed": 0,
+            "layer_text_truncated": 0,
+        },
+        "layer_text_digests": dict.fromkeys(EMBEDDING_LAYERS),
+        "last_provider_record_id": None,
+    }
+
+
+def _resume_point(
+    output: _AuthoringOutput,
+    expected_journal_sha256: str | None,
+    policy: AuthoringPolicy,
+    expected_fill_checkpoint_sha256: str,
+    selection: AuthoringSelection,
+) -> dict[str, Any]:
+    manifest = output.read(MANIFEST_FILENAME, maximum=MAX_AUTHORING_MANIFEST_BYTES)
+    if manifest is not None:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_OUTPUT_EXISTS",
+            "authoring.output",
+            "a published generation is immutable; author into a new output root",
+        )
+    raw = output.read(JOURNAL_FILENAME, maximum=MAX_AUTHORING_JOURNAL_BYTES)
+    if expected_journal_sha256 is None:
+        if raw is not None:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_EXISTS",
+                "authoring.output",
+                "a partial generation exists; resume it with its exact journal digest",
+            )
+        return _empty_journal()
+    if raw is None or sha256_bytes(raw) != expected_journal_sha256:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_JOURNAL_MISMATCH",
+            "authoring.journal",
+            "no retained journal matches the caller's exact digest",
+        )
+    try:
+        journal = parse_canonical_json(raw)
+    except CanonicalJSONError as error:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_JOURNAL_CORRUPT", "authoring.journal", "journal is not canonical"
+        ) from error
+    if (
+        not isinstance(journal, dict)
+        or journal.get("schema_version") != AUTHORING_JOURNAL_SCHEMA
+        or not isinstance(journal.get("shards"), list)
+        or not isinstance(journal.get("counts"), dict)
+        or not isinstance(journal.get("layer_text_digests"), dict)
+        or set(journal["layer_text_digests"]) != set(EMBEDDING_LAYERS)
+        or set(journal["counts"]) != set(_empty_journal()["counts"])
+        or not isinstance(journal.get("last_provider_record_id"), (str, type(None)))
+        or not _is_digest(journal.get("policy_sha256"))
+        or not _is_digest(journal.get("staging_checkpoint_sha256"))
+        or not coverage_is_valid(journal.get("coverage"))
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_JOURNAL_CORRUPT", "authoring.journal", "journal contract differs"
+        )
+    if (
+        journal["policy_sha256"] != policy.digest
+        or journal["staging_checkpoint_sha256"] != expected_fill_checkpoint_sha256
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_JOURNAL_COORDINATE",
+            "authoring.journal",
+            "the retained partial generation was authored from another policy or checkpoint",
+        )
+    # The rule and the bound are the caller's half of the claim and are answerable here, before
+    # the staging root is opened or one record is re-read. The corpus and selected populations
+    # are the generation's half; they are recomputed from immutable staging and checked against
+    # this same journal once the index exists.
+    if (journal["coverage"]["rule"], journal["coverage"]["bound"]) != (
+        selection.rule,
+        selection.bound,
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_COVERAGE_MISMATCH",
+            "authoring.journal.coverage",
+            "the retained partial generation was authored under another stated coverage",
+        )
+    for descriptor in journal["shards"]:
+        output.verify_shard(descriptor)
+    return journal
+
+
+def _scratch_root(output_root: Path, staging_root: Path) -> str | None:
+    for candidate in (Path(output_root).parent, Path(staging_root).parent):
+        if candidate.is_dir():
+            return str(candidate)
+    return None
+
+
+def _utc_second(epoch_seconds: int) -> str:
+    try:
+        return datetime.fromtimestamp(epoch_seconds, UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+    except (OverflowError, OSError, ValueError) as error:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_STAGING_CORRUPT",
+            "fill.staging.response",
+            "retained response observation is outside the representable range",
+        ) from error
+
+
+def _is_digest(value: Any) -> bool:
+    return (
+        isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+    )
+
+
+# ------------------------------------------------------------------------------------------
+# Durable, descriptor-rooted, no-follow output
+# ------------------------------------------------------------------------------------------
+
+
+class _AuthoringOutput:
+    """One locally confined output root whose members install atomically and read back exactly."""
+
+    def __init__(self, root: Path, *, root_descriptor: int | None = None) -> None:
+        self.root = Path(root)
+        self._retained_root_descriptor = root_descriptor
+        self.bytes_written = 0
+        self._root_fd = -1
+        self._shards_fd = -1
+        self._owned: list[int] = []
+
+    @contextlib.contextmanager
+    def opened(self) -> Iterator[None]:
+        try:
+            if self._retained_root_descriptor is None:
+                self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+                self._root_fd = self._open_directory(self.root)
+                (self.root / SHARDS_DIRNAME).mkdir(exist_ok=True, mode=0o700)
+                self._shards_fd = self._open_directory(self.root / SHARDS_DIRNAME)
+            else:
+                self._root_fd = os.dup(self._retained_root_descriptor)
+                self._owned.append(self._root_fd)
+                self._require_private_directory(self._root_fd)
+                try:
+                    os.mkdir(SHARDS_DIRNAME, mode=0o700, dir_fd=self._root_fd)
+                    os.fsync(self._root_fd)
+                except FileExistsError:
+                    pass
+                self._shards_fd = os.open(SHARDS_DIRNAME, _OPEN_DIRECTORY, dir_fd=self._root_fd)
+                self._owned.append(self._shards_fd)
+                self._require_private_directory(self._shards_fd)
+        except CatalogAuthoringRefused:
+            self._release()
+            raise
+        except OSError as error:
+            self._release()
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_IO", "authoring.output", "authoring output root is unusable"
+            ) from error
+        try:
+            yield
+        finally:
+            self._release()
+
+    def _release(self) -> None:
+        for descriptor in reversed(self._owned):
+            os.close(descriptor)
+        self._owned = []
+        self._root_fd = -1
+        self._shards_fd = -1
+
+    def _open_directory(self, path: Path) -> int:
+        descriptor = os.open(path, _OPEN_DIRECTORY)
+        self._owned.append(descriptor)
+        self._require_private_directory(descriptor)
+        return descriptor
+
+    @staticmethod
+    def _require_private_directory(descriptor: int) -> None:
+        info = os.fstat(descriptor)
+        if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_PATH",
+                "authoring.output",
+                "authoring output components must be private directories",
+            )
+
+    def read(self, name: str, *, maximum: int) -> bytes | None:
+        try:
+            return _read_regular_at(self._root_fd, name, maximum=maximum)
+        except FileNotFoundError:
+            return None
+
+    def write_shard(self, payload: dict[str, Any], *, maximum: int) -> tuple[str, int]:
+        raw = canonical_json_bytes(payload)
+        if len(raw) > maximum:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_SHARD_LIMIT", "authoring.shard", "shard exceeds its byte bound"
+            )
+        digest = sha256_bytes(raw)
+        self._install_immutable(f"{digest}.json", raw)
+        return digest, len(raw)
+
+    def verify_shard(self, descriptor: Any) -> None:
+        if not shard_descriptor_is_valid(descriptor):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_JOURNAL_CORRUPT", "authoring.journal", "shard descriptor is invalid"
+            )
+        try:
+            raw = _read_regular_at(
+                self._shards_fd, f"{descriptor['sha256']}.json", maximum=descriptor["bytes"]
+            )
+        except OSError as error:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_READBACK",
+                "authoring.shard",
+                "a committed shard is absent or unreadable",
+            ) from error
+        if len(raw) != descriptor["bytes"] or sha256_bytes(raw) != descriptor["sha256"]:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_READBACK", "authoring.shard", "a committed shard differs"
+            )
+
+    def _install_immutable(self, name: str, raw: bytes) -> None:
+        try:
+            existing = _read_regular_at(self._shards_fd, name, maximum=len(raw))
+        except FileNotFoundError:
+            existing = None
+        except OSError as error:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_IO", "authoring.shard", "immutable shard lookup failed"
+            ) from error
+        if existing is not None:
+            if existing != raw:
+                raise CatalogAuthoringRefused(
+                    "AUTHOR_OUTPUT_READBACK", "authoring.shard", "digest path bytes differ"
+                )
+            return
+        temporary = f".publish.{os.getpid()}.{secrets.token_hex(16)}.tmp"
+        try:
+            _write_regular_at(self._shards_fd, temporary, raw)
+            with contextlib.suppress(FileExistsError):
+                os.link(
+                    temporary,
+                    name,
+                    src_dir_fd=self._shards_fd,
+                    dst_dir_fd=self._shards_fd,
+                    follow_symlinks=False,
+                )
+        except OSError as error:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_IO", "authoring.shard", "immutable shard publication failed"
+            ) from error
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(temporary, dir_fd=self._shards_fd)
+            os.fsync(self._shards_fd)
+        if _read_regular_at(self._shards_fd, name, maximum=len(raw)) != raw:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_READBACK", "authoring.shard", "immutable shard readback differs"
+            )
+        self.bytes_written += len(raw)
+
+    def publish(self, name: str, payload: dict[str, Any], *, maximum: int) -> str:
+        raw = canonical_json_bytes(payload)
+        if len(raw) > maximum:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_SHARD_LIMIT", f"authoring.{name}", "document exceeds its byte bound"
+            )
+        temporary = f".{name}.{os.getpid()}.{secrets.token_hex(8)}.tmp"
+        try:
+            _write_regular_at(self._root_fd, temporary, raw)
+            os.replace(temporary, name, src_dir_fd=self._root_fd, dst_dir_fd=self._root_fd)
+            os.fsync(self._root_fd)
+        except OSError as error:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_IO", f"authoring.{name}", "document publication failed"
+            ) from error
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(temporary, dir_fd=self._root_fd)
+        if _read_regular_at(self._root_fd, name, maximum=maximum) != raw:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_OUTPUT_READBACK", f"authoring.{name}", "document readback differs"
+            )
+        self.bytes_written += len(raw)
+        return sha256_bytes(raw)
+
+
+def _read_regular_at(parent_fd: int, name: str, *, maximum: int) -> bytes:
+    try:
+        raw = read_bounded_at(parent_fd, name, maximum=maximum)
+    except BoundedReadFailure as error:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_OUTPUT_PATH",
+            "authoring.output",
+            "output members must be stable bounded single-link regular files",
+        ) from error
+    assert raw is not None
+    return raw
+
+
+def _write_regular_at(parent_fd: int, name: str, raw: bytes) -> None:
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(name, flags, 0o600, dir_fd=parent_fd)
+    try:
+        written = 0
+        while written < len(raw):
+            count = os.write(descriptor, raw[written:])
+            if count < 1:
+                raise CatalogAuthoringRefused(
+                    "AUTHOR_OUTPUT_IO", "authoring.output", "write made no progress"
+                )
+            written += count
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)