mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/identity_history.py

@@ -0,0 +1,1549 @@
+"""Durable provider identity and complete catalog version/absence history.
+
+A provider record and a Harness catalog entry are different things, and the binding between them
+has to survive every sweep, every rename, and every rights re-decision.  This module is that
+binding, plus the complete append-only history hanging off it.
+
+Four separations are load-bearing.
+
+*Identity versus prose.*  ``(provider_id, provider_record_id)`` binds to exactly one catalog entry
+id.  The derived spelling is ``{provider_id}.{sha256(provider_record_id)}`` -- a full
+collision-resistant digest, never a slug of a title -- and it is **persisted** with the record it
+names and verified on read rather than recomputed from mutable prose.  A declared title or landing
+page may change on every sweep without moving identity by a single byte.
+
+*Semantic versions versus evidence.*  A version link is minted only when the semantic facts digest
+changes.  ``last_harvested_date`` and ``popularity`` remain outside that digest precisely
+because Data.gov rewrites them on every re-harvest, so re-observing an unchanged record appends one
+observation and mints nothing.  Evidence-only observations never force a new semantic version.
+
+*Append-only versus rewritable.*  History only grows.  A successor that drops, reorders, or rewrites
+a retained event is a rollback; a successor that rewrites a version already published is a
+post-publication mutation.  Both are refused rather than reconciled, and so are a split identity, a
+duplicate current head, and a version link whose predecessor was never observed.
+
+*Bounded packs versus one big map.*  A 548k-member JSON object is not a data structure this harness
+will write -- ``canonical`` refuses more than 100,000 members, and nothing downstream could stream
+it anyway.  A generation is instead immutable content-addressed packs of at most
+:data:`MAX_IDENTITY_PACK_RECORDS` records each, in ascending provider-record order, behind one small
+manifest that carries only pack descriptors.  Reading keeps at most one pack resident, whether the
+caller streams the generation or looks one record up.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import re
+import secrets
+import stat
+from collections.abc import Iterable, Iterator, Sequence
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.bounded_io import (
+    BoundedReadFailure,
+    read_bounded_at,
+    read_bounded_path,
+)
+from mostlyright.data_harness.sources.contracts import SourceContractError, _CanonicalContract
+
+IDENTITY_RECORD_SCHEMA = "harness-catalog-identity-history.v1"
+IDENTITY_PACK_SCHEMA = "harness-catalog-identity-pack.v1"
+IDENTITY_MANIFEST_SCHEMA = "harness-catalog-identity-manifest.v1"
+
+IDENTITY_MANIFEST_FILENAME = "identity-manifest.json"
+IDENTITY_PACKS_DIRNAME = "packs"
+
+# One pack is the unit of residency: a reader holds at most one of these at a time.
+MAX_IDENTITY_PACK_RECORDS = 1_000
+MAX_IDENTITY_PACK_BYTES = 8 * 1024 * 1024
+MAX_IDENTITY_MANIFEST_BYTES = 8 * 1024 * 1024
+
+# One provider record's complete retained history. History is never deleted, so the bound is a
+# refusal rather than an eviction: a record that would exceed it stops the sweep visibly.
+MAX_IDENTITY_HISTORY_EVENTS = 4_096
+
+# The whole-generation ceiling. The writer's collision check is the only structure whose size grows
+# with the generation; at this bound it is tens of megabytes, well inside the plan's RSS budget.
+MAX_IDENTITY_RECORDS = 2_000_000
+
+# The preserved v1 path materializes an explicit provider-to-entry mapping in memory, so it stays
+# deliberately small. Provider-scale generations use packs, never a mapping.
+MAX_EXPLICIT_MAPPING = 10_000
+
+MAX_PROVIDER_RECORD_ID = 512
+MAX_ENTRY_ID = 256
+MAX_ENTRY_VERSION = 10_000
+MAX_VECTOR_REFERENCES = 16
+
+IDENTITY_DERIVATIONS = ("provider_record_sha256", "explicit")
+NON_ADMITTED_DISPOSITIONS = ("failed", "flagged", "skipped")
+
+_IDENTIFIER = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_UTC_SECOND = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")
+
+_OPEN_DIRECTORY = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+_OPEN_MEMBER = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+
+
+class CatalogIdentityRefused(SourceContractError):
+    """A stable refusal of an identity binding, a history transition, or a durable pack."""
+
+
+# ------------------------------------------------------------------------------------------
+# Identity
+# ------------------------------------------------------------------------------------------
+
+
+def derive_entry_id(provider_id: str, provider_record_id: str, *, prefix: str | None = None) -> str:
+    """Derive the one collision-resistant catalog entry id for a provider record."""
+
+    _identifier(provider_id, "identity.provider_id")
+    _text(provider_record_id, "identity.provider_record_id", maximum=MAX_PROVIDER_RECORD_ID)
+    stem = f"{provider_id}." if prefix is None else prefix
+    return f"{stem}{sha256_bytes(provider_record_id.encode('utf-8'))}"
+
+
+@dataclass(frozen=True)
+class ProviderIdentity(_CanonicalContract):
+    """One immutable binding of a provider record to one catalog entry id."""
+
+    provider_id: str
+    provider_record_id: str
+    entry_id: str
+    derivation: str = "provider_record_sha256"
+
+    def __post_init__(self) -> None:
+        _identifier(self.provider_id, "identity.provider_id")
+        _text(
+            self.provider_record_id, "identity.provider_record_id", maximum=MAX_PROVIDER_RECORD_ID
+        )
+        _entry_id(self.entry_id, "identity.entry_id")
+        if self.derivation not in IDENTITY_DERIVATIONS:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT",
+                "identity.derivation",
+                f"must be one of {list(IDENTITY_DERIVATIONS)}",
+            )
+        if self.derivation == "provider_record_sha256" and self.entry_id != derive_entry_id(
+            self.provider_id, self.provider_record_id
+        ):
+            raise CatalogIdentityRefused(
+                "IDENTITY_DERIVATION",
+                "identity.entry_id",
+                "a persisted derived entry id must reproduce its own derivation exactly",
+            )
+
+    @property
+    def key(self) -> tuple[str, str]:
+        return (self.provider_id, self.provider_record_id)
+
+    @property
+    def sort_key(self) -> bytes:
+        return self.provider_record_id.encode("utf-8")
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "provider_id": self.provider_id,
+            "provider_record_id": self.provider_record_id,
+            "entry_id": self.entry_id,
+            "derivation": self.derivation,
+        }
+
+
+# ------------------------------------------------------------------------------------------
+# History events
+# ------------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class SemanticVersionLink(_CanonicalContract):
+    """One semantic version of an entry, linked to the exact state it superseded."""
+
+    entry_version: int
+    semantic_facts_digest: str
+    previous_semantic_digest: str | None
+    observed_at: str
+    sweep_sha256: str
+    provider_record_sha256: str
+    vector_sha256s: tuple[str, ...] = ()
+    published: bool = False
+
+    def __post_init__(self) -> None:
+        if type(self.entry_version) is not int or not 1 <= self.entry_version <= MAX_ENTRY_VERSION:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT",
+                "version.entry_version",
+                f"must be an integer in [1, {MAX_ENTRY_VERSION}]",
+            )
+        _digest(self.semantic_facts_digest, "version.semantic_facts_digest")
+        _optional_digest(self.previous_semantic_digest, "version.previous_semantic_digest")
+        _utc_second(self.observed_at, "version.observed_at")
+        _digest(self.sweep_sha256, "version.sweep_sha256")
+        _digest(self.provider_record_sha256, "version.provider_record_sha256")
+        if type(self.published) is not bool:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT", "version.published", "must be a boolean"
+            )
+        if (
+            not isinstance(self.vector_sha256s, tuple)
+            or len(self.vector_sha256s) > MAX_VECTOR_REFERENCES
+        ):
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT",
+                "version.vector_sha256s",
+                f"must be an immutable tuple of at most {MAX_VECTOR_REFERENCES} references",
+            )
+        for index, value in enumerate(self.vector_sha256s):
+            _digest(value, f"version.vector_sha256s[{index}]")
+        if (self.entry_version == 1) != (self.previous_semantic_digest is None):
+            raise CatalogIdentityRefused(
+                "IDENTITY_PREDECESSOR_MISSING",
+                "version.previous_semantic_digest",
+                "a first version has no predecessor and a successor must name one",
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "entry_version": self.entry_version,
+            "semantic_facts_digest": self.semantic_facts_digest,
+            "previous_semantic_digest": self.previous_semantic_digest,
+            "observed_at": self.observed_at,
+            "sweep_sha256": self.sweep_sha256,
+            "provider_record_sha256": self.provider_record_sha256,
+            "vector_sha256s": list(self.vector_sha256s),
+            "published": self.published,
+        }
+
+
+@dataclass(frozen=True)
+class ObservationEvidence(_CanonicalContract):
+    """One sweep that observed a record's semantic state; never a semantic fact itself."""
+
+    observed_at: str
+    semantic_facts_digest: str
+    sweep_sha256: str
+    response_evidence_sha256: str | None = None
+    page_evidence_sha256: str | None = None
+
+    def __post_init__(self) -> None:
+        _utc_second(self.observed_at, "observation.observed_at")
+        _digest(self.semantic_facts_digest, "observation.semantic_facts_digest")
+        _digest(self.sweep_sha256, "observation.sweep_sha256")
+        _optional_digest(self.response_evidence_sha256, "observation.response_evidence_sha256")
+        _optional_digest(self.page_evidence_sha256, "observation.page_evidence_sha256")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "observed_at": self.observed_at,
+            "semantic_facts_digest": self.semantic_facts_digest,
+            "sweep_sha256": self.sweep_sha256,
+            "response_evidence_sha256": self.response_evidence_sha256,
+            "page_evidence_sha256": self.page_evidence_sha256,
+        }
+
+
+@dataclass(frozen=True)
+class AbsenceEvidence(_CanonicalContract):
+    """One complete sweep in which a previously known provider record was not present."""
+
+    observed_at: str
+    last_semantic_facts_digest: str
+    sweep_sha256: str
+
+    def __post_init__(self) -> None:
+        _utc_second(self.observed_at, "absence.observed_at")
+        _digest(self.last_semantic_facts_digest, "absence.last_semantic_facts_digest")
+        _digest(self.sweep_sha256, "absence.sweep_sha256")
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "observed_at": self.observed_at,
+            "last_semantic_facts_digest": self.last_semantic_facts_digest,
+            "sweep_sha256": self.sweep_sha256,
+        }
+
+
+@dataclass(frozen=True)
+class DispositionEvidence(_CanonicalContract):
+    """One durable non-admitted disposition, so a skipped record never simply disappears."""
+
+    observed_at: str
+    disposition: str
+    reason_code: str
+    sweep_sha256: str
+    provider_record_sha256: str | None = None
+
+    def __post_init__(self) -> None:
+        _utc_second(self.observed_at, "disposition.observed_at")
+        if self.disposition not in NON_ADMITTED_DISPOSITIONS:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT",
+                "disposition.disposition",
+                f"must be one of {list(NON_ADMITTED_DISPOSITIONS)}",
+            )
+        _text(self.reason_code, "disposition.reason_code", maximum=128)
+        _digest(self.sweep_sha256, "disposition.sweep_sha256")
+        _optional_digest(self.provider_record_sha256, "disposition.provider_record_sha256")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "observed_at": self.observed_at,
+            "disposition": self.disposition,
+            "reason_code": self.reason_code,
+            "sweep_sha256": self.sweep_sha256,
+            "provider_record_sha256": self.provider_record_sha256,
+        }
+
+
+# ------------------------------------------------------------------------------------------
+# One provider record's complete history
+# ------------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class IdentityHistory(_CanonicalContract):
+    """One provider record's immutable identity plus its complete append-only history."""
+
+    identity: ProviderIdentity
+    policy_sha256: str
+    harvester_coordinate: str
+    versions: tuple[SemanticVersionLink, ...] = ()
+    observations: tuple[ObservationEvidence, ...] = ()
+    absences: tuple[AbsenceEvidence, ...] = ()
+    dispositions: tuple[DispositionEvidence, ...] = ()
+    schema_version: str = IDENTITY_RECORD_SCHEMA
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.identity, ProviderIdentity):
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT", "history.identity", "must be a ProviderIdentity"
+            )
+        if self.schema_version != IDENTITY_RECORD_SCHEMA:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT", "history.schema_version", "unsupported record schema"
+            )
+        _digest(self.policy_sha256, "history.policy_sha256")
+        _text(self.harvester_coordinate, "history.harvester_coordinate", maximum=192)
+        for name, expected in (
+            ("versions", SemanticVersionLink),
+            ("observations", ObservationEvidence),
+            ("absences", AbsenceEvidence),
+            ("dispositions", DispositionEvidence),
+        ):
+            events = getattr(self, name)
+            if not isinstance(events, tuple) or any(
+                not isinstance(item, expected) for item in events
+            ):
+                raise CatalogIdentityRefused(
+                    "IDENTITY_CONTRACT",
+                    f"history.{name}",
+                    f"must be an immutable tuple of {expected.__name__}",
+                )
+        if self.event_count > MAX_IDENTITY_HISTORY_EVENTS:
+            raise CatalogIdentityRefused(
+                "IDENTITY_HISTORY_LIMIT",
+                "history",
+                f"one record retains at most {MAX_IDENTITY_HISTORY_EVENTS} events",
+            )
+        if not self.versions and not self.dispositions:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT",
+                "history.versions",
+                "a record with no semantic version must carry its non-admitted disposition",
+            )
+        self._validate_versions()
+        self._validate_evidence()
+
+    def _validate_versions(self) -> None:
+        seen: set[str] = set()
+        for index, link in enumerate(self.versions):
+            if link.entry_version != index + 1:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_HISTORY_ORDER",
+                    f"history.versions[{index}].entry_version",
+                    "version links must be contiguous and ascending from one",
+                )
+            expected = None if index == 0 else self.versions[index - 1].semantic_facts_digest
+            if link.previous_semantic_digest != expected:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_PREDECESSOR_MISSING",
+                    f"history.versions[{index}].previous_semantic_digest",
+                    "must name the exact semantic state this version superseded",
+                )
+            if link.semantic_facts_digest in seen:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_HISTORY_ROLLBACK",
+                    f"history.versions[{index}].semantic_facts_digest",
+                    "a superseded semantic state may not be re-presented as a new version",
+                )
+            seen.add(link.semantic_facts_digest)
+
+    def _validate_evidence(self) -> None:
+        seen: set[bytes] = set()
+        for name in ("versions", "observations", "absences", "dispositions"):
+            for item in getattr(self, name):
+                raw = canonical_json_bytes({"stream": name, "event": item.to_dict()})
+                if raw in seen:
+                    raise CatalogIdentityRefused(
+                        "IDENTITY_HISTORY_ORDER", f"history.{name}", "events must be unique"
+                    )
+                seen.add(raw)
+        for name in ("observations", "absences", "dispositions"):
+            observed = [item.observed_at for item in getattr(self, name)]
+            if observed != sorted(observed):
+                raise CatalogIdentityRefused(
+                    "IDENTITY_HISTORY_ORDER",
+                    f"history.{name}",
+                    "evidence must be appended in non-decreasing observation order",
+                )
+
+    @property
+    def key(self) -> tuple[str, str]:
+        return self.identity.key
+
+    @property
+    def sort_key(self) -> bytes:
+        return self.identity.sort_key
+
+    @property
+    def head(self) -> SemanticVersionLink | None:
+        return self.versions[-1] if self.versions else None
+
+    @property
+    def head_semantic_digest(self) -> str | None:
+        return None if self.head is None else self.head.semantic_facts_digest
+
+    @property
+    def event_count(self) -> int:
+        return (
+            len(self.versions)
+            + len(self.observations)
+            + len(self.absences)
+            + len(self.dispositions)
+        )
+
+    @property
+    def last_observed_at(self) -> str | None:
+        stamps = [
+            item.observed_at
+            for stream in (self.observations, self.absences, self.dispositions)
+            for item in stream
+        ]
+        return max(stamps) if stamps else None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "identity": self.identity.to_dict(),
+            "policy_sha256": self.policy_sha256,
+            "harvester_coordinate": self.harvester_coordinate,
+            "versions": [item.to_dict() for item in self.versions],
+            "observations": [item.to_dict() for item in self.observations],
+            "absences": [item.to_dict() for item in self.absences],
+            "dispositions": [item.to_dict() for item in self.dispositions],
+        }
+
+
+# ------------------------------------------------------------------------------------------
+# Append-only transitions
+# ------------------------------------------------------------------------------------------
+
+
+def advance_identity_history(
+    previous: IdentityHistory | None,
+    *,
+    identity: ProviderIdentity,
+    policy_sha256: str,
+    harvester_coordinate: str,
+    semantic_facts_digest: str,
+    provider_record_sha256: str,
+    observation: ObservationEvidence,
+    expected_previous_semantic_digest: str | None = None,
+    vector_sha256s: tuple[str, ...] = (),
+) -> IdentityHistory:
+    """Observe one provider record, minting a version only when the semantic facts changed."""
+
+    if not isinstance(observation, ObservationEvidence):
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", "observation", "must be an ObservationEvidence"
+        )
+    _digest(semantic_facts_digest, "history.semantic_facts_digest")
+    if observation.semantic_facts_digest != semantic_facts_digest:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT",
+            "observation.semantic_facts_digest",
+            "an observation must cite the exact semantic state it observed",
+        )
+    if previous is None:
+        if expected_previous_semantic_digest is not None:
+            raise CatalogIdentityRefused(
+                "IDENTITY_PREDECESSOR_MISSING",
+                "history.previous",
+                "no retained history carries the named predecessor state",
+            )
+        return IdentityHistory(
+            identity=identity,
+            policy_sha256=policy_sha256,
+            harvester_coordinate=harvester_coordinate,
+            versions=(
+                SemanticVersionLink(
+                    entry_version=1,
+                    semantic_facts_digest=semantic_facts_digest,
+                    previous_semantic_digest=None,
+                    observed_at=observation.observed_at,
+                    sweep_sha256=observation.sweep_sha256,
+                    provider_record_sha256=provider_record_sha256,
+                    vector_sha256s=vector_sha256s,
+                ),
+            ),
+            observations=(observation,),
+        )
+
+    _require_history(previous)
+    if previous.identity != identity:
+        raise CatalogIdentityRefused(
+            "IDENTITY_SPLIT",
+            "history.identity",
+            "one provider record keeps exactly one persisted catalog identity",
+        )
+    head = previous.head
+    if head is not None and semantic_facts_digest == head.semantic_facts_digest:
+        if (
+            expected_previous_semantic_digest is not None
+            and expected_previous_semantic_digest != head.previous_semantic_digest
+        ):
+            raise CatalogIdentityRefused(
+                "IDENTITY_PREDECESSOR_MISSING",
+                "history.previous_semantic_digest",
+                "the retained head does not carry the named predecessor state",
+            )
+        return replace(previous, observations=(*previous.observations, observation))
+
+    if expected_previous_semantic_digest is not None and expected_previous_semantic_digest != (
+        None if head is None else head.semantic_facts_digest
+    ):
+        raise CatalogIdentityRefused(
+            "IDENTITY_PREDECESSOR_MISSING",
+            "history.previous_semantic_digest",
+            "the retained head does not carry the named predecessor state",
+        )
+    if any(link.semantic_facts_digest == semantic_facts_digest for link in previous.versions):
+        raise CatalogIdentityRefused(
+            "IDENTITY_HISTORY_ROLLBACK",
+            "history.versions",
+            "a superseded semantic state may not be re-presented as a new version",
+        )
+    link = SemanticVersionLink(
+        entry_version=1 if head is None else head.entry_version + 1,
+        semantic_facts_digest=semantic_facts_digest,
+        previous_semantic_digest=None if head is None else head.semantic_facts_digest,
+        observed_at=observation.observed_at,
+        sweep_sha256=observation.sweep_sha256,
+        provider_record_sha256=provider_record_sha256,
+        vector_sha256s=vector_sha256s,
+    )
+    return replace(
+        previous,
+        versions=(*previous.versions, link),
+        observations=(*previous.observations, observation),
+    )
+
+
+def observe_identity_absence(
+    previous: IdentityHistory, *, observed_at: str, sweep_sha256: str
+) -> IdentityHistory:
+    """Append one absence observation without deleting or superseding any retained state."""
+
+    _require_history(previous)
+    head = previous.head
+    if head is None:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT",
+            "history.absences",
+            "absence is only observable for a record that once had a semantic state",
+        )
+    absence = AbsenceEvidence(
+        observed_at=observed_at,
+        last_semantic_facts_digest=head.semantic_facts_digest,
+        sweep_sha256=sweep_sha256,
+    )
+    return replace(previous, absences=(*previous.absences, absence))
+
+
+def record_identity_disposition(
+    previous: IdentityHistory | None,
+    *,
+    identity: ProviderIdentity,
+    policy_sha256: str,
+    harvester_coordinate: str,
+    disposition: str,
+    reason_code: str,
+    observed_at: str,
+    sweep_sha256: str,
+    provider_record_sha256: str | None = None,
+) -> IdentityHistory:
+    """Append one durable non-admitted disposition; it never mints or moves a version."""
+
+    evidence = DispositionEvidence(
+        observed_at=observed_at,
+        disposition=disposition,
+        reason_code=reason_code,
+        sweep_sha256=sweep_sha256,
+        provider_record_sha256=provider_record_sha256,
+    )
+    if previous is None:
+        return IdentityHistory(
+            identity=identity,
+            policy_sha256=policy_sha256,
+            harvester_coordinate=harvester_coordinate,
+            dispositions=(evidence,),
+        )
+    _require_history(previous)
+    if previous.identity != identity:
+        raise CatalogIdentityRefused(
+            "IDENTITY_SPLIT",
+            "history.identity",
+            "one provider record keeps exactly one persisted catalog identity",
+        )
+    return replace(previous, dispositions=(*previous.dispositions, evidence))
+
+
+def publish_identity_head(
+    history: IdentityHistory, *, vector_sha256s: tuple[str, ...] = ()
+) -> IdentityHistory:
+    """Mark the current head published, binding the sealed member references it was sealed with."""
+
+    _require_history(history)
+    head = history.head
+    if head is None:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", "history.versions", "no semantic head to publish"
+        )
+    if head.published:
+        raise CatalogIdentityRefused(
+            "IDENTITY_PUBLISHED_MUTATION",
+            "history.versions",
+            "a published version is immutable and may not be published again",
+        )
+    published = replace(head, published=True, vector_sha256s=vector_sha256s)
+    return replace(history, versions=(*history.versions[:-1], published))
+
+
+def verify_identity_successor(previous: IdentityHistory, successor: IdentityHistory) -> None:
+    """Refuse any successor that is not a strict append-only extension of its predecessor."""
+
+    _require_history(previous)
+    _require_history(successor)
+    if previous.key != successor.key:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT",
+            "history.identity",
+            "a successor must describe the same provider record",
+        )
+    if previous.identity.entry_id != successor.identity.entry_id:
+        raise CatalogIdentityRefused(
+            "IDENTITY_SPLIT",
+            "history.identity.entry_id",
+            "one provider record may not split across two catalog entry ids",
+        )
+    for index, link in enumerate(previous.versions):
+        if index >= len(successor.versions) or successor.versions[index] != link:
+            code = "IDENTITY_PUBLISHED_MUTATION" if link.published else "IDENTITY_HISTORY_ROLLBACK"
+            raise CatalogIdentityRefused(
+                code,
+                f"history.versions[{index}]",
+                "retained version links are immutable and may only be extended",
+            )
+    for name in ("observations", "absences", "dispositions"):
+        retained = getattr(previous, name)
+        offered = getattr(successor, name)
+        if len(offered) < len(retained) or offered[: len(retained)] != retained:
+            raise CatalogIdentityRefused(
+                "IDENTITY_HISTORY_ROLLBACK",
+                f"history.{name}",
+                "retained evidence is immutable and may only be extended",
+            )
+
+
+# ------------------------------------------------------------------------------------------
+# The preserved explicit mapping path for bounded v1 fills
+# ------------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class ExplicitIdentityBinding:
+    """One caller-chosen provider-record-to-entry binding, as v1 fills always supplied it."""
+
+    provider_record_id: str
+    entry_id: str
+    semantic_facts_digest: str
+    provider_record_sha256: str
+
+
+def identity_histories_from_explicit_mapping(
+    bindings: Sequence[ExplicitIdentityBinding],
+    *,
+    provider_id: str,
+    policy_sha256: str,
+    harvester_coordinate: str,
+    observed_at: str,
+    sweep_sha256: str,
+    maximum: int = MAX_EXPLICIT_MAPPING,
+) -> tuple[IdentityHistory, ...]:
+    """Build first-version histories from an explicit mapping, collision-checked and bounded."""
+
+    if not isinstance(bindings, Sequence) or isinstance(bindings, (str, bytes)):
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", "bindings", "must be a sequence of ExplicitIdentityBinding"
+        )
+    if len(bindings) > maximum:
+        raise CatalogIdentityRefused(
+            "IDENTITY_MAPPING_LIMIT",
+            "bindings",
+            f"the explicit mapping path carries at most {maximum} records",
+        )
+    histories: list[IdentityHistory] = []
+    by_record: set[str] = set()
+    by_entry: set[str] = set()
+    for binding in bindings:
+        if not isinstance(binding, ExplicitIdentityBinding):
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT", "bindings[]", "must be an ExplicitIdentityBinding"
+            )
+        if binding.provider_record_id in by_record:
+            raise CatalogIdentityRefused(
+                "IDENTITY_SPLIT",
+                "bindings[]",
+                "one provider record may not be bound to two catalog entry ids",
+            )
+        if binding.entry_id in by_entry:
+            raise CatalogIdentityRefused(
+                "IDENTITY_COLLISION",
+                "bindings[]",
+                "one catalog entry id may not be bound to two provider records",
+            )
+        by_record.add(binding.provider_record_id)
+        by_entry.add(binding.entry_id)
+        identity = ProviderIdentity(
+            provider_id=provider_id,
+            provider_record_id=binding.provider_record_id,
+            entry_id=binding.entry_id,
+            derivation="explicit",
+        )
+        histories.append(
+            advance_identity_history(
+                None,
+                identity=identity,
+                policy_sha256=policy_sha256,
+                harvester_coordinate=harvester_coordinate,
+                semantic_facts_digest=binding.semantic_facts_digest,
+                provider_record_sha256=binding.provider_record_sha256,
+                observation=ObservationEvidence(
+                    observed_at=observed_at,
+                    semantic_facts_digest=binding.semantic_facts_digest,
+                    sweep_sha256=sweep_sha256,
+                ),
+            )
+        )
+    return tuple(sorted(histories, key=lambda item: item.sort_key))
+
+
+# ------------------------------------------------------------------------------------------
+# Durable immutable packs behind one small manifest
+# ------------------------------------------------------------------------------------------
+
+
+class DurableDirectory:
+    """One locally confined output root whose members install atomically and read back exactly."""
+
+    def __init__(
+        self,
+        root: Path,
+        *,
+        member_dirname: str,
+        code_prefix: str,
+        refusal: type[SourceContractError] = CatalogIdentityRefused,
+        root_descriptor: int | None = None,
+    ) -> None:
+        self.root = Path(root)
+        self.bytes_written = 0
+        self._member_dirname = member_dirname
+        self._prefix = code_prefix
+        self._refusal = refusal
+        self._retained_root_descriptor = root_descriptor
+        self._root_fd = -1
+        self._members_fd = -1
+        self._owned: list[int] = []
+
+    def _refuse(self, code: str, path: str, detail: str) -> SourceContractError:
+        return self._refusal(f"{self._prefix}_{code}", path, detail)
+
+    @contextlib.contextmanager
+    def opened(self) -> Iterator[DurableDirectory]:
+        try:
+            if self._retained_root_descriptor is None:
+                self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
+                self._root_fd = self._open_directory(self.root)
+            else:
+                self._root_fd = os.dup(self._retained_root_descriptor)
+                self._owned.append(self._root_fd)
+                self._require_private_directory(self._root_fd)
+            try:
+                os.mkdir(self._member_dirname, mode=0o700, dir_fd=self._root_fd)
+            except FileExistsError:
+                pass
+            self._members_fd = self._open_directory_at(self._root_fd, self._member_dirname)
+        except OSError as error:
+            self._release()
+            raise self._refuse("OUTPUT_IO", "output", "output root is unusable") from error
+        except SourceContractError:
+            self._release()
+            raise
+        try:
+            yield self
+        finally:
+            self._release()
+
+    def _release(self) -> None:
+        for descriptor in reversed(self._owned):
+            os.close(descriptor)
+        self._owned = []
+        self._root_fd = -1
+        self._members_fd = -1
+
+    def _open_directory(self, path: Path) -> int:
+        descriptor = os.open(path, _OPEN_DIRECTORY)
+        self._owned.append(descriptor)
+        self._require_private_directory(descriptor)
+        return descriptor
+
+    def _open_directory_at(self, parent_descriptor: int, name: str) -> int:
+        descriptor = os.open(name, _OPEN_DIRECTORY, dir_fd=parent_descriptor)
+        self._owned.append(descriptor)
+        self._require_private_directory(descriptor)
+        return descriptor
+
+    def _require_private_directory(self, descriptor: int) -> None:
+        info = os.fstat(descriptor)
+        if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
+            raise self._refuse(
+                "OUTPUT_PATH", "output", "output components must be private directories"
+            )
+
+    def read(self, name: str, *, maximum: int) -> bytes | None:
+        try:
+            return _read_regular_at(
+                self._root_fd, name, maximum=maximum, prefix=self._prefix, refusal=self._refusal
+            )
+        except FileNotFoundError:
+            return None
+
+    def write_member(self, payload: dict[str, Any], *, maximum: int) -> tuple[str, int]:
+        raw = canonical_json_bytes(payload)
+        if len(raw) > maximum:
+            raise self._refuse("MEMBER_LIMIT", "output.member", "member exceeds its byte bound")
+        digest = sha256_bytes(raw)
+        name = f"{digest}.json"
+        try:
+            existing = _read_regular_at(
+                self._members_fd, name, maximum=len(raw), prefix=self._prefix, refusal=self._refusal
+            )
+        except FileNotFoundError:
+            existing = None
+        except OSError as error:
+            raise self._refuse("OUTPUT_IO", "output.member", "member lookup failed") from error
+        if existing is not None:
+            if existing != raw:
+                raise self._refuse("OUTPUT_READBACK", "output.member", "digest path bytes differ")
+            return digest, len(raw)
+        temporary = f".publish.{os.getpid()}.{secrets.token_hex(16)}.tmp"
+        try:
+            _write_regular_at(
+                self._members_fd, temporary, raw, prefix=self._prefix, refusal=self._refusal
+            )
+            with contextlib.suppress(FileExistsError):
+                os.link(
+                    temporary,
+                    name,
+                    src_dir_fd=self._members_fd,
+                    dst_dir_fd=self._members_fd,
+                    follow_symlinks=False,
+                )
+        except OSError as error:
+            raise self._refuse("OUTPUT_IO", "output.member", "member publication failed") from error
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(temporary, dir_fd=self._members_fd)
+            os.fsync(self._members_fd)
+        if (
+            _read_regular_at(
+                self._members_fd, name, maximum=len(raw), prefix=self._prefix, refusal=self._refusal
+            )
+            != raw
+        ):
+            raise self._refuse("OUTPUT_READBACK", "output.member", "member readback differs")
+        self.bytes_written += len(raw)
+        return digest, len(raw)
+
+    def publish(self, name: str, payload: dict[str, Any], *, maximum: int) -> str:
+        raw = canonical_json_bytes(payload)
+        if len(raw) > maximum:
+            raise self._refuse("MEMBER_LIMIT", f"output.{name}", "document exceeds its byte bound")
+        temporary = f".{name}.{os.getpid()}.{secrets.token_hex(8)}.tmp"
+        try:
+            _write_regular_at(
+                self._root_fd, temporary, raw, prefix=self._prefix, refusal=self._refusal
+            )
+            os.replace(temporary, name, src_dir_fd=self._root_fd, dst_dir_fd=self._root_fd)
+            os.fsync(self._root_fd)
+        except OSError as error:
+            raise self._refuse(
+                "OUTPUT_IO", f"output.{name}", "document publication failed"
+            ) from error
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                os.unlink(temporary, dir_fd=self._root_fd)
+        if (
+            _read_regular_at(
+                self._root_fd, name, maximum=maximum, prefix=self._prefix, refusal=self._refusal
+            )
+            != raw
+        ):
+            raise self._refuse("OUTPUT_READBACK", f"output.{name}", "document readback differs")
+        self.bytes_written += len(raw)
+        return sha256_bytes(raw)
+
+
+def _read_regular_at(
+    parent_fd: int,
+    name: str,
+    *,
+    maximum: int,
+    prefix: str,
+    refusal: type[SourceContractError] = CatalogIdentityRefused,
+) -> bytes:
+    try:
+        raw = read_bounded_at(parent_fd, name, maximum=maximum)
+    except BoundedReadFailure as error:
+        raise refusal(
+            f"{prefix}_OUTPUT_PATH",
+            "output",
+            "members must be stable bounded single-link regular files",
+        ) from error
+    assert raw is not None
+    return raw
+
+
+def _write_regular_at(
+    parent_fd: int,
+    name: str,
+    raw: bytes,
+    *,
+    prefix: str,
+    refusal: type[SourceContractError] = CatalogIdentityRefused,
+) -> None:
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+    descriptor = os.open(name, flags, 0o600, dir_fd=parent_fd)
+    try:
+        written = 0
+        while written < len(raw):
+            count = os.write(descriptor, raw[written:])
+            if count < 1:
+                raise refusal(f"{prefix}_OUTPUT_IO", "output", "write made no progress")
+            written += count
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+class IdentityHistoryWriter:
+    """Write one identity generation as immutable ascending packs behind one small manifest."""
+
+    def __init__(
+        self,
+        root: Path,
+        *,
+        provider_id: str,
+        sweep_sha256: str,
+        predecessor_manifest_sha256: str | None = None,
+        max_pack_records: int = MAX_IDENTITY_PACK_RECORDS,
+        max_pack_bytes: int = MAX_IDENTITY_PACK_BYTES,
+        max_records: int = MAX_IDENTITY_RECORDS,
+        output_descriptor: int | None = None,
+    ) -> None:
+        _identifier(provider_id, "identity.provider_id")
+        _digest(sweep_sha256, "identity.sweep_sha256")
+        _optional_digest(predecessor_manifest_sha256, "identity.predecessor_manifest_sha256")
+        for name, value in (
+            ("max_pack_records", max_pack_records),
+            ("max_pack_bytes", max_pack_bytes),
+            ("max_records", max_records),
+        ):
+            if type(value) is not int or value < 1:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_LIMIT", f"identity.{name}", "must be a positive integer"
+                )
+        if max_pack_records > MAX_IDENTITY_PACK_RECORDS or max_pack_bytes > MAX_IDENTITY_PACK_BYTES:
+            raise CatalogIdentityRefused(
+                "IDENTITY_LIMIT",
+                "identity.max_pack_records",
+                "pack bounds may not exceed the fixed residency contract",
+            )
+        self.provider_id = provider_id
+        self.sweep_sha256 = sweep_sha256
+        self.predecessor_manifest_sha256 = predecessor_manifest_sha256
+        self.max_pack_records = max_pack_records
+        self.max_pack_bytes = max_pack_bytes
+        self.max_records = max_records
+        self.packs: list[dict[str, Any]] = []
+        self.record_count = 0
+        self.version_count = 0
+        self.observation_count = 0
+        self.absence_count = 0
+        self.disposition_count = 0
+        self._directory = DurableDirectory(
+            root,
+            member_dirname=IDENTITY_PACKS_DIRNAME,
+            code_prefix="IDENTITY",
+            root_descriptor=output_descriptor,
+        )
+        self._buffer: list[dict[str, Any]] = []
+        self._buffered_bytes = 0
+        self._last_sort_key: bytes | None = None
+        self._entry_ids: set[str] = set()
+        self._published = False
+
+    @contextlib.contextmanager
+    def opened(self) -> Iterator[IdentityHistoryWriter]:
+        with self._directory.opened():
+            existing = self._directory.read(
+                IDENTITY_MANIFEST_FILENAME, maximum=MAX_IDENTITY_MANIFEST_BYTES
+            )
+            if existing is not None:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_OUTPUT_EXISTS",
+                    "identity.output",
+                    "a published identity generation is immutable; write a new root",
+                )
+            yield self
+
+    def append(self, history: IdentityHistory) -> None:
+        """Append one record; keys must ascend and entry ids must never collide."""
+
+        _require_history(history)
+        if history.identity.provider_id != self.provider_id:
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT",
+                "identity.provider_id",
+                "one generation carries exactly one provider",
+            )
+        if self._last_sort_key is not None:
+            if history.sort_key == self._last_sort_key:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_DUPLICATE_HEAD",
+                    "identity.records",
+                    "one provider record may appear at most once in a generation",
+                )
+            if history.sort_key < self._last_sort_key:
+                raise CatalogIdentityRefused(
+                    "IDENTITY_ORDER",
+                    "identity.records",
+                    "records must be appended in ascending provider record order",
+                )
+        if history.identity.entry_id in self._entry_ids:
+            raise CatalogIdentityRefused(
+                "IDENTITY_COLLISION",
+                "identity.entry_id",
+                "one catalog entry id may not be bound to two provider records",
+            )
+        if self.record_count >= self.max_records:
+            raise CatalogIdentityRefused(
+                "IDENTITY_RECORD_LIMIT",
+                "identity.records",
+                f"one generation carries at most {self.max_records} records",
+            )
+        payload = history.to_dict()
+        size = len(canonical_json_bytes(payload))
+        if self._buffer and (
+            len(self._buffer) >= self.max_pack_records
+            or self._buffered_bytes + size + len(self._buffer) + 4_096 > self.max_pack_bytes
+        ):
+            self._flush()
+        self._buffer.append(payload)
+        self._buffered_bytes += size
+        self._entry_ids.add(history.identity.entry_id)
+        self._last_sort_key = history.sort_key
+        self.record_count += 1
+        self.version_count += len(history.versions)
+        self.observation_count += len(history.observations)
+        self.absence_count += len(history.absences)
+        self.disposition_count += len(history.dispositions)
+
+    def _flush(self) -> None:
+        if not self._buffer:
+            return
+        payload = {
+            "schema_version": IDENTITY_PACK_SCHEMA,
+            "provider_id": self.provider_id,
+            "sweep_sha256": self.sweep_sha256,
+            "first_provider_record_id": self._buffer[0]["identity"]["provider_record_id"],
+            "last_provider_record_id": self._buffer[-1]["identity"]["provider_record_id"],
+            "records": self._buffer,
+        }
+        digest, size = self._directory.write_member(payload, maximum=self.max_pack_bytes)
+        self.packs.append(
+            {
+                "sha256": digest,
+                "bytes": size,
+                "record_count": len(self._buffer),
+                "first_provider_record_id": payload["first_provider_record_id"],
+                "last_provider_record_id": payload["last_provider_record_id"],
+            }
+        )
+        self._buffer = []
+        self._buffered_bytes = 0
+
+    def publish(self) -> str:
+        """Flush the open pack and publish the manifest last; returns its exact digest."""
+
+        if self._published:
+            raise CatalogIdentityRefused(
+                "IDENTITY_OUTPUT_EXISTS",
+                "identity.output",
+                "this generation was already published",
+            )
+        self._flush()
+        body = {
+            "schema_version": IDENTITY_MANIFEST_SCHEMA,
+            "provider_id": self.provider_id,
+            "sweep_sha256": self.sweep_sha256,
+            "predecessor_manifest_sha256": self.predecessor_manifest_sha256,
+            "record_count": self.record_count,
+            "version_count": self.version_count,
+            "observation_count": self.observation_count,
+            "absence_count": self.absence_count,
+            "disposition_count": self.disposition_count,
+            "packs": self.packs,
+        }
+        manifest = {**body, "root_sha256": canonical_sha256(body)}
+        digest = self._directory.publish(
+            IDENTITY_MANIFEST_FILENAME, manifest, maximum=MAX_IDENTITY_MANIFEST_BYTES
+        )
+        self._published = True
+        return digest
+
+
+class IdentityHistoryStore:
+    """Read one identity generation while keeping at most one pack resident."""
+
+    def __init__(
+        self,
+        root: Path,
+        manifest: dict[str, Any],
+        manifest_sha256: str,
+        *,
+        root_descriptor: int | None = None,
+    ) -> None:
+        self.root = Path(root)
+        self.manifest = manifest
+        self.manifest_sha256 = manifest_sha256
+        self._root_descriptor = root_descriptor
+        self._cached_sha256: str | None = None
+        self._cached_records: list[dict[str, Any]] = []
+
+    @classmethod
+    def open(
+        cls,
+        root: Path,
+        *,
+        expected_manifest_sha256: str | None = None,
+        root_descriptor: int | None = None,
+    ) -> IdentityHistoryStore:
+        try:
+            if root_descriptor is None:
+                raw = read_bounded_path(
+                    Path(root) / IDENTITY_MANIFEST_FILENAME,
+                    maximum=MAX_IDENTITY_MANIFEST_BYTES,
+                )
+                assert raw is not None
+            else:
+                raw = _read_regular_at(
+                    root_descriptor,
+                    IDENTITY_MANIFEST_FILENAME,
+                    maximum=MAX_IDENTITY_MANIFEST_BYTES,
+                    prefix="IDENTITY",
+                    refusal=CatalogIdentityRefused,
+                )
+        except (OSError, BoundedReadFailure) as error:
+            raise CatalogIdentityRefused(
+                "IDENTITY_MANIFEST_MISMATCH",
+                "identity.manifest",
+                "no readable identity manifest at this root",
+            ) from error
+        if len(raw) > MAX_IDENTITY_MANIFEST_BYTES:
+            raise CatalogIdentityRefused(
+                "IDENTITY_MANIFEST_CORRUPT", "identity.manifest", "manifest exceeds its byte bound"
+            )
+        digest = sha256_bytes(raw)
+        if expected_manifest_sha256 is not None and digest != expected_manifest_sha256:
+            raise CatalogIdentityRefused(
+                "IDENTITY_MANIFEST_MISMATCH",
+                "identity.manifest",
+                "the retained manifest is not the caller's exact generation",
+            )
+        try:
+            manifest = parse_canonical_json(raw)
+        except CanonicalJSONError as error:
+            raise CatalogIdentityRefused(
+                "IDENTITY_MANIFEST_CORRUPT", "identity.manifest", "manifest is not canonical"
+            ) from error
+        _validate_manifest(manifest)
+        return cls(Path(root), manifest, digest, root_descriptor=root_descriptor)
+
+    @property
+    def provider_id(self) -> str:
+        return self.manifest["provider_id"]
+
+    @property
+    def sweep_sha256(self) -> str:
+        return self.manifest["sweep_sha256"]
+
+    @property
+    def record_count(self) -> int:
+        return self.manifest["record_count"]
+
+    @property
+    def packs(self) -> list[dict[str, Any]]:
+        return self.manifest["packs"]
+
+    @property
+    def resident_packs(self) -> int:
+        return 0 if self._cached_sha256 is None else 1
+
+    @property
+    def resident_records(self) -> int:
+        return len(self._cached_records)
+
+    def stream(self) -> Iterator[IdentityHistory]:
+        """Yield every record in ascending provider record order, one pack resident."""
+
+        last: bytes | None = None
+        for descriptor in self.packs:
+            for payload in self._pack(descriptor):
+                history = identity_history_from_dict(payload)
+                if last is not None and history.sort_key <= last:
+                    raise CatalogIdentityRefused(
+                        "IDENTITY_ORDER",
+                        "identity.records",
+                        "a retained generation must ascend by provider record",
+                    )
+                last = history.sort_key
+                yield history
+
+    def lookup(self, provider_record_id: str) -> IdentityHistory | None:
+        """Return one record by reading at most the one pack whose range could carry it."""
+
+        key = provider_record_id.encode("utf-8")
+        for descriptor in self.packs:
+            first = descriptor["first_provider_record_id"].encode("utf-8")
+            last = descriptor["last_provider_record_id"].encode("utf-8")
+            if key < first:
+                return None
+            if key > last:
+                continue
+            for payload in self._pack(descriptor):
+                if payload["identity"]["provider_record_id"] == provider_record_id:
+                    return identity_history_from_dict(payload)
+            return None
+        return None
+
+    def explicit_mapping(self, *, maximum: int = MAX_EXPLICIT_MAPPING) -> dict[str, str]:
+        """Materialize the bounded v1 provider-record-to-entry mapping, or refuse to."""
+
+        if self.record_count > maximum:
+            raise CatalogIdentityRefused(
+                "IDENTITY_MAPPING_LIMIT",
+                "identity.explicit_mapping",
+                f"the explicit mapping path carries at most {maximum} records",
+            )
+        return {
+            history.identity.provider_record_id: history.identity.entry_id
+            for history in self.stream()
+        }
+
+    def _pack(self, descriptor: dict[str, Any]) -> list[dict[str, Any]]:
+        if self._cached_sha256 == descriptor["sha256"]:
+            return self._cached_records
+        self._cached_sha256 = None
+        self._cached_records = []
+        try:
+            if self._root_descriptor is None:
+                raw = read_bounded_path(
+                    self.root / IDENTITY_PACKS_DIRNAME / f"{descriptor['sha256']}.json",
+                    maximum=MAX_IDENTITY_PACK_BYTES,
+                )
+                assert raw is not None
+            else:
+                members = os.open(
+                    IDENTITY_PACKS_DIRNAME, _OPEN_DIRECTORY, dir_fd=self._root_descriptor
+                )
+                try:
+                    raw = _read_regular_at(
+                        members,
+                        f"{descriptor['sha256']}.json",
+                        maximum=MAX_IDENTITY_PACK_BYTES,
+                        prefix="IDENTITY",
+                        refusal=CatalogIdentityRefused,
+                    )
+                finally:
+                    os.close(members)
+        except (OSError, BoundedReadFailure) as error:
+            raise CatalogIdentityRefused(
+                "IDENTITY_PACK_READBACK", "identity.pack", "a committed pack is unreadable"
+            ) from error
+        if len(raw) != descriptor["bytes"] or sha256_bytes(raw) != descriptor["sha256"]:
+            raise CatalogIdentityRefused(
+                "IDENTITY_PACK_READBACK", "identity.pack", "a committed pack differs"
+            )
+        try:
+            payload = parse_canonical_json(raw)
+        except CanonicalJSONError as error:
+            raise CatalogIdentityRefused(
+                "IDENTITY_PACK_READBACK", "identity.pack", "pack is not canonical"
+            ) from error
+        if (
+            not isinstance(payload, dict)
+            or payload.get("schema_version") != IDENTITY_PACK_SCHEMA
+            or not isinstance(payload.get("records"), list)
+            or len(payload["records"]) != descriptor["record_count"]
+        ):
+            raise CatalogIdentityRefused(
+                "IDENTITY_PACK_READBACK", "identity.pack", "pack contract differs"
+            )
+        self._cached_sha256 = descriptor["sha256"]
+        self._cached_records = payload["records"]
+        return self._cached_records
+
+
+def write_identity_generation(
+    root: Path,
+    histories: Iterable[IdentityHistory],
+    *,
+    provider_id: str,
+    sweep_sha256: str,
+    predecessor_manifest_sha256: str | None = None,
+    max_pack_records: int = MAX_IDENTITY_PACK_RECORDS,
+) -> str:
+    """Write one complete generation from an ascending iterable; returns the manifest digest."""
+
+    writer = IdentityHistoryWriter(
+        root,
+        provider_id=provider_id,
+        sweep_sha256=sweep_sha256,
+        predecessor_manifest_sha256=predecessor_manifest_sha256,
+        max_pack_records=max_pack_records,
+    )
+    with writer.opened():
+        for history in histories:
+            writer.append(history)
+        return writer.publish()
+
+
+# ------------------------------------------------------------------------------------------
+# Strict rebuilding
+# ------------------------------------------------------------------------------------------
+
+
+def identity_history_from_dict(value: Any) -> IdentityHistory:
+    """Rebuild exactly one persisted record shape; never repair or infer a missing member."""
+
+    expected = {
+        "schema_version",
+        "identity",
+        "policy_sha256",
+        "harvester_coordinate",
+        "versions",
+        "observations",
+        "absences",
+        "dispositions",
+    }
+    if not isinstance(value, dict) or set(value) != expected:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", "identity.record", "persisted record fields differ"
+        )
+    identity = value["identity"]
+    if not isinstance(identity, dict) or set(identity) != {
+        "provider_id",
+        "provider_record_id",
+        "entry_id",
+        "derivation",
+    }:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", "identity.record.identity", "persisted identity fields differ"
+        )
+    try:
+        return IdentityHistory(
+            identity=ProviderIdentity(**identity),
+            policy_sha256=value["policy_sha256"],
+            harvester_coordinate=value["harvester_coordinate"],
+            versions=tuple(
+                SemanticVersionLink(
+                    entry_version=item["entry_version"],
+                    semantic_facts_digest=item["semantic_facts_digest"],
+                    previous_semantic_digest=item["previous_semantic_digest"],
+                    observed_at=item["observed_at"],
+                    sweep_sha256=item["sweep_sha256"],
+                    provider_record_sha256=item["provider_record_sha256"],
+                    vector_sha256s=tuple(item["vector_sha256s"]),
+                    published=item["published"],
+                )
+                for item in value["versions"]
+            ),
+            observations=tuple(ObservationEvidence(**item) for item in value["observations"]),
+            absences=tuple(AbsenceEvidence(**item) for item in value["absences"]),
+            dispositions=tuple(DispositionEvidence(**item) for item in value["dispositions"]),
+            schema_version=value["schema_version"],
+        )
+    except CatalogIdentityRefused:
+        raise
+    except (KeyError, TypeError, ValueError) as error:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", "identity.record", "persisted record is invalid"
+        ) from error
+
+
+def _validate_manifest(manifest: Any) -> None:
+    expected = {
+        "schema_version",
+        "provider_id",
+        "sweep_sha256",
+        "predecessor_manifest_sha256",
+        "record_count",
+        "version_count",
+        "observation_count",
+        "absence_count",
+        "disposition_count",
+        "packs",
+        "root_sha256",
+    }
+    if (
+        not isinstance(manifest, dict)
+        or set(manifest) != expected
+        or manifest["schema_version"] != IDENTITY_MANIFEST_SCHEMA
+        or not isinstance(manifest["packs"], list)
+        or type(manifest["record_count"]) is not int
+    ):
+        raise CatalogIdentityRefused(
+            "IDENTITY_MANIFEST_CORRUPT", "identity.manifest", "manifest contract differs"
+        )
+    body = {key: value for key, value in manifest.items() if key != "root_sha256"}
+    if canonical_sha256(body) != manifest["root_sha256"]:
+        raise CatalogIdentityRefused(
+            "IDENTITY_MANIFEST_CORRUPT", "identity.manifest", "manifest root digest differs"
+        )
+    total = 0
+    for descriptor in manifest["packs"]:
+        if (
+            not isinstance(descriptor, dict)
+            or set(descriptor)
+            != {
+                "sha256",
+                "bytes",
+                "record_count",
+                "first_provider_record_id",
+                "last_provider_record_id",
+            }
+            or not _is_digest(descriptor["sha256"])
+            or type(descriptor["bytes"]) is not int
+            or type(descriptor["record_count"]) is not int
+        ):
+            raise CatalogIdentityRefused(
+                "IDENTITY_MANIFEST_CORRUPT", "identity.manifest", "pack descriptor is invalid"
+            )
+        total += descriptor["record_count"]
+    if total != manifest["record_count"]:
+        raise CatalogIdentityRefused(
+            "IDENTITY_MANIFEST_CORRUPT",
+            "identity.manifest",
+            "pack descriptors do not account for the declared record count",
+        )
+
+
+# ------------------------------------------------------------------------------------------
+# Small closed validators
+# ------------------------------------------------------------------------------------------
+
+
+def _require_history(value: Any) -> None:
+    if not isinstance(value, IdentityHistory):
+        raise CatalogIdentityRefused("IDENTITY_CONTRACT", "history", "must be an IdentityHistory")
+
+
+def _text(value: Any, path: str, *, maximum: int) -> str:
+    if not isinstance(value, str) or not value or len(value) > maximum:
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", path, f"must be a non-empty string of at most {maximum} characters"
+        )
+    for character in value:
+        codepoint = ord(character)
+        if (
+            0xD800 <= codepoint <= 0xDFFF
+            or codepoint < 0x20
+            or 0x7F <= codepoint <= 0x9F
+            or (character.isspace() and character != " ")
+        ):
+            raise CatalogIdentityRefused(
+                "IDENTITY_CONTRACT", path, "must not contain unsafe or control characters"
+            )
+    return value
+
+
+def _identifier(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=MAX_ENTRY_ID)
+    if not _IDENTIFIER.fullmatch(text):
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", path, "must be a lowercase dotted/dashed/underscored identifier"
+        )
+    return text
+
+
+def _entry_id(value: Any, path: str) -> str:
+    return _identifier(value, path)
+
+
+def _digest(value: Any, path: str) -> str:
+    if not _is_digest(value):
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", path, "must be a lowercase 64-character SHA-256"
+        )
+    return value
+
+
+def _optional_digest(value: Any, path: str) -> str | None:
+    if value is None:
+        return None
+    return _digest(value, path)
+
+
+def _utc_second(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=20)
+    if not _UTC_SECOND.fullmatch(text):
+        raise CatalogIdentityRefused(
+            "IDENTITY_CONTRACT", path, "must be canonical UTC to whole seconds"
+        )
+    return text
+
+
+def _is_digest(value: Any) -> bool:
+    return isinstance(value, str) and _SHA256.fullmatch(value) is not None