vector-mtrl 0.2.0__py3-none-any.whl

mtrl/_archive_integrity.py

@@ -0,0 +1,539 @@
+"""Integrity validation and digest checks for new-format archives."""
+
+from __future__ import annotations
+
+import zipfile
+from collections.abc import Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, cast
+
+from mtrl._archive_documents import (
+    _CANONICAL_OPERATION_PARAMETER_PATHS,
+    _deserialise_metadata_field,
+    _normalise_new_object_list,
+)
+from mtrl.archive import BoundedZipReader, decode_numpy
+from mtrl.archive.validation import ArchiveDocuments, validate_decoded_numpy_payload
+from mtrl.domain import CoverageStatus, Revision
+from mtrl.exceptions import (
+    CanonicalisationError,
+    InsufficientCoverageError,
+    IntegrityLayer,
+    IntegrityMismatchError,
+    InvalidArchiveStructureError,
+)
+from mtrl.integrity.canonical import decode_schema_int
+from mtrl.integrity.digests import (
+    DigestOrder,
+    canonical_payload_digest,
+    raw_digest,
+    record_digest,
+    revision_digest,
+    source_digest,
+)
+from mtrl.integrity.digests import (
+    storage_digest as calculate_storage_digest,
+)
+from mtrl.integrity.manifest import record_body
+from mtrl.models import ArchiveEntry, CheckpointEncoding, CheckpointManifestEntry, Manifest
+
+if TYPE_CHECKING:
+    from mtrl.mtrl import Mtrl
+
+
+def _assert_integrity_mapping(
+    expected: Any,
+    candidate: Any,
+    *,
+    label: IntegrityLayer,
+) -> None:
+    """Compare one committed integrity mapping with a rebuilt mapping."""
+    if not isinstance(expected, Mapping) or not isinstance(candidate, Mapping):
+        return
+    for item_id, expected_value in expected.items():
+        if candidate.get(item_id) != expected_value:
+            raise IntegrityMismatchError(
+                f"Committed {label} '{item_id}' has an integrity mismatch.",
+                layer=label,
+            )
+
+
+def _assert_current_application_state(
+    container: Mtrl,
+    expected_integrity: Mapping[str, Any],
+    candidate_entries: Sequence[tuple[str, bytes]],
+) -> None:
+    """Compare every baseline revision identity before allowing a new revision."""
+    expected_bodies = expected_integrity.get("revision_bodies", {})
+    expected_roots = expected_integrity.get("revision_roots", {})
+    if not isinstance(expected_bodies, Mapping) or not isinstance(
+        expected_roots,
+        Mapping,
+    ):
+        return
+    current_revisions = {str(revision.revision_id): revision for revision in container._revisions}
+    for revision_id, expected_body in expected_bodies.items():
+        revision = current_revisions.get(str(revision_id))
+        if revision is None or not isinstance(expected_body, Mapping):
+            raise IntegrityMismatchError(
+                f"Committed revision '{revision_id}' has an integrity mismatch.",
+                layer="revision",
+            )
+        _validate_revision_body_identity(revision, expected_body)
+        if revision.revision_root != expected_roots.get(revision_id):
+            raise IntegrityMismatchError(
+                f"Committed revision '{revision_id}' has an integrity mismatch.",
+                layer="revision",
+            )
+    if len(current_revisions) == len(expected_bodies):
+        _assert_saved_entries_unchanged(container._archive_path, candidate_entries)
+
+
+def _assert_saved_entries_unchanged(
+    archive_path: Path | None,
+    candidate_entries: Sequence[tuple[str, bytes]],
+) -> None:
+    """Reject in-place edits when no new application revision exists."""
+    if archive_path is None:
+        return
+    with zipfile.ZipFile(archive_path) as archive:
+        saved_entries = {entry.filename: archive.read(entry) for entry in archive.infolist()}
+    if dict(candidate_entries) != saved_entries:
+        raise IntegrityMismatchError(
+            "Committed evidence changed without creating a new revision.",
+            layer="storage",
+        )
+
+
+def _assert_integrity_inventory(expected: Any, candidate: Any) -> None:
+    """Compare committed payload and source inventory descriptors."""
+    if not isinstance(expected, Mapping) or not isinstance(candidate, Mapping):
+        return
+    covered_kinds = {"payload", "source", "environment-record", "source-bundle"}
+    for entry_path, expected_descriptor in expected.items():
+        if not isinstance(expected_descriptor, Mapping):
+            continue
+        candidate_descriptor = candidate.get(entry_path)
+        if not isinstance(candidate_descriptor, Mapping):
+            raise IntegrityMismatchError(
+                f"Committed inventory entry '{entry_path}' is missing.",
+                layer="storage",
+            )
+        if expected_descriptor.get("kind") not in covered_kinds:
+            continue
+        if dict(candidate_descriptor) != dict(expected_descriptor):
+            raise IntegrityMismatchError(
+                f"Committed inventory entry '{entry_path}' has an integrity mismatch.",
+                layer="storage",
+            )
+
+
+def _validate_integrity_manifest(
+    archive: BoundedZipReader,
+    manifest: Manifest,
+    documents: ArchiveDocuments,
+    *,
+    allow_partial: bool = False,
+) -> CoverageStatus:
+    """Validate new-format inventory digests and every revision root."""
+    integrity_data = documents.integrity_data
+    if not isinstance(integrity_data, Mapping):
+        raise InvalidArchiveStructureError("Archive integrity record is incomplete.")
+    inventory = integrity_data.get("inventory")
+    if not isinstance(inventory, Mapping):
+        raise InvalidArchiveStructureError("Archive integrity inventory is incomplete.")
+    _validate_integrity_entry_storage(archive, inventory)
+    archive.preflight_numpy_headers(
+        checkpoint.entry
+        for checkpoint in documents.checkpoints
+        if checkpoint.encoding == CheckpointEncoding.NPY
+    )
+    _validate_integrity_payload_digests(archive, documents.checkpoints)
+    _validate_integrity_source_digests(archive, manifest)
+    record_coverage = _validate_integrity_record_digests(
+        archive,
+        inventory,
+        integrity_data,
+    )
+    _validate_integrity_revision_roots(documents, integrity_data)
+    inventory_coverage = _validate_integrity_inventory_keys(
+        archive,
+        manifest,
+        documents,
+        integrity_data,
+    )
+    if record_coverage and inventory_coverage:
+        return CoverageStatus.FULL
+    if not allow_partial:
+        raise InsufficientCoverageError(
+            "Archive integrity coverage is partial; full manifest coverage is required."
+        )
+    return CoverageStatus.PARTIAL
+
+
+def _validate_integrity_storage_before_restoration(
+    archive: BoundedZipReader,
+    integrity_data: Any,
+) -> None:
+    """Check covered entry bytes before structural restoration can continue."""
+    if not isinstance(integrity_data, Mapping):
+        raise InvalidArchiveStructureError("Archive integrity record is incomplete.")
+    inventory = integrity_data.get("inventory")
+    if not isinstance(inventory, Mapping):
+        raise InvalidArchiveStructureError("Archive integrity inventory is incomplete.")
+    _validate_integrity_entry_storage(
+        archive,
+        inventory,
+        entry_paths=(ArchiveEntry.REVISIONS.value,),
+    )
+
+
+def _validate_integrity_payload_digests(
+    archive: BoundedZipReader,
+    checkpoints: Sequence[CheckpointManifestEntry],
+) -> None:
+    """Verify raw and canonical payload digests before interpretation checks."""
+    for checkpoint in sorted(checkpoints, key=lambda value: value.entry):
+        entry_bytes = archive.read_entry(checkpoint.entry)
+        if checkpoint.encoding == CheckpointEncoding.BYTES:
+            if checkpoint.raw_digest is None:
+                raise InvalidArchiveStructureError(
+                    f"Payload '{checkpoint.entry}' has incomplete raw digest evidence."
+                )
+            observed_raw_digest = raw_digest(entry_bytes)
+            if observed_raw_digest != checkpoint.raw_digest:
+                raise IntegrityMismatchError(
+                    f"Payload '{checkpoint.entry}' has a raw digest mismatch: "
+                    f"expected {checkpoint.raw_digest}; observed {observed_raw_digest}.",
+                    layer="raw",
+                )
+            continue
+        if checkpoint.storage_digest is None or checkpoint.payload_digest is None:
+            raise InvalidArchiveStructureError(
+                f"Payload '{checkpoint.entry}' has incomplete dual digest evidence."
+            )
+        try:
+            archive.validate_numpy(checkpoint.entry)
+            decoded_array = decode_numpy(entry_bytes)
+        except (EOFError, ValueError, OSError) as error:
+            raise InvalidArchiveStructureError(
+                f"Payload '{checkpoint.entry}' is not valid for codec 'npy'."
+            ) from error
+        validate_decoded_numpy_payload(
+            checkpoint,
+            decoded_array,
+            checkpoint.entry,
+        )
+        if checkpoint.payload_digest is None:
+            continue
+        observed_payload_digest = canonical_payload_digest(
+            decoded_array,
+            order=cast(DigestOrder, checkpoint.order),
+        )
+        if observed_payload_digest != checkpoint.payload_digest:
+            raise IntegrityMismatchError(
+                f"Payload '{checkpoint.entry}' has a canonical payload digest mismatch: "
+                f"expected {checkpoint.payload_digest}; observed {observed_payload_digest}.",
+                layer="canonical",
+            )
+
+
+def _validate_checkpoint_storage_digests(
+    archive: BoundedZipReader,
+    checkpoints: Sequence[CheckpointManifestEntry],
+) -> None:
+    """Verify all recorded payload storage digests before NumPy preflight."""
+    for checkpoint in sorted(checkpoints, key=lambda value: value.entry):
+        if checkpoint.storage_digest is None:
+            continue
+        entry_bytes = archive.read_entry(checkpoint.entry)
+        observed_storage_digest = calculate_storage_digest(
+            checkpoint.entry,
+            entry_bytes,
+        )
+        if observed_storage_digest != checkpoint.storage_digest:
+            raise IntegrityMismatchError(
+                f"Payload '{checkpoint.entry}' has a storage digest mismatch: "
+                f"expected {checkpoint.storage_digest}; observed {observed_storage_digest}.",
+                layer="storage",
+            )
+
+
+def _validate_integrity_source_digests(
+    archive: BoundedZipReader,
+    manifest: Manifest,
+) -> None:
+    """Verify every retained source digest over its captured bytes."""
+    for source in sorted(
+        manifest.sources.values(),
+        key=lambda value: str(value.entry or value.source_id or value.filename),
+    ):
+        if source.entry is None or source.digest is None:
+            continue
+        observed_source_digest = source_digest(archive.read_entry(source.entry))
+        if observed_source_digest != source.digest:
+            raise IntegrityMismatchError(
+                f"Source '{source.entry}' has a digest mismatch: expected "
+                f"{source.digest}; observed {observed_source_digest}.",
+                layer="source",
+            )
+
+
+def _validate_integrity_entry_storage(
+    archive: BoundedZipReader,
+    inventory: Mapping[str, Any],
+    *,
+    entry_paths: Iterable[str] | None = None,
+) -> None:
+    """Validate storage digests for every physical archive entry."""
+    paths = archive.entry_paths if entry_paths is None else entry_paths
+    for entry_path in paths:
+        if entry_path == ArchiveEntry.INTEGRITY.value:
+            continue
+        descriptor = inventory.get(entry_path)
+        if not isinstance(descriptor, Mapping):
+            raise InvalidArchiveStructureError(f"Archive integrity inventory omits '{entry_path}'.")
+        expected_length = descriptor.get("stored_length")
+        if expected_length is None:
+            raise InvalidArchiveStructureError(
+                f"Archive integrity inventory entry '{entry_path}' has no stored length."
+            )
+        try:
+            decoded_length = decode_schema_int(expected_length, non_negative=True)
+        except CanonicalisationError as error:
+            raise InvalidArchiveStructureError(
+                f"Archive integrity inventory entry '{entry_path}' has an invalid stored length."
+            ) from error
+        actual_length = len(archive.read_entry(entry_path))
+        if actual_length != decoded_length:
+            raise IntegrityMismatchError(
+                f"Archive entry '{entry_path}' has a stored length mismatch: "
+                f"expected {decoded_length}; observed {actual_length}.",
+                layer="storage",
+            )
+        expected_storage = descriptor.get("storage_digest")
+        if (
+            entry_path in archive.entry_paths
+            and descriptor.get("kind")
+            in {
+                "fixed",
+                "source",
+                "payload",
+                "revision-log",
+                "environment-record",
+                "source-bundle",
+            }
+            and not isinstance(expected_storage, str)
+        ):
+            raise InvalidArchiveStructureError(
+                f"Archive integrity inventory entry '{entry_path}' has no storage digest."
+            )
+        if expected_storage is None:
+            continue
+        observed_storage = calculate_storage_digest(
+            entry_path,
+            archive.read_entry(entry_path),
+        )
+        if observed_storage != expected_storage:
+            raise IntegrityMismatchError(
+                f"Archive entry '{entry_path}' has a storage digest mismatch: "
+                f"expected {expected_storage}; observed {observed_storage}.",
+                layer="storage",
+            )
+
+
+def _validate_integrity_record_digests(
+    archive: BoundedZipReader,
+    inventory: Mapping[str, Any],
+    integrity_data: Mapping[str, Any],
+) -> bool:
+    """Validate immutable record bodies and their digest index."""
+    records_document = _normalise_new_object_list(
+        archive.read_json(
+            ArchiveEntry.RECORDS,
+            typed_metadata_paths=_CANONICAL_OPERATION_PARAMETER_PATHS,
+        ),
+        "records",
+        "records",
+    )
+    record_bodies: dict[str, dict[str, Any]] = {}
+    complete_coverage = True
+    for descriptor in records_document:
+        record_id = descriptor.get("record_id")
+        if not isinstance(record_id, str):
+            raise InvalidArchiveStructureError("Archive records require a string record_id.")
+        body = record_body(descriptor)
+        record_bodies[record_id] = body
+        expected_digest = descriptor.get("record_digest")
+        observed_digest = record_digest(body)
+        if expected_digest != observed_digest:
+            raise IntegrityMismatchError(
+                f"Record '{record_id}' has a record digest mismatch: expected "
+                f"{expected_digest}; observed {observed_digest}.",
+                layer="record",
+            )
+        inventory_descriptor = inventory.get(f"records/{record_id}")
+        if not isinstance(inventory_descriptor, Mapping):
+            complete_coverage = False
+            continue
+        if inventory_descriptor.get("record_digest") != observed_digest:
+            raise IntegrityMismatchError(
+                f"Record '{record_id}' disagrees with its inventory digest.",
+                layer="record",
+            )
+    record_digests = integrity_data.get("record_digests")
+    expected_record_digests = {
+        record_id: record_digest(body) for record_id, body in sorted(record_bodies.items())
+    }
+    if not isinstance(record_digests, Mapping) or dict(record_digests) != expected_record_digests:
+        raise IntegrityMismatchError(
+            "Archive record digest index does not match records.json.",
+            layer="record",
+        )
+    return complete_coverage
+
+
+def _validate_integrity_revision_roots(
+    documents: ArchiveDocuments,
+    integrity_data: Mapping[str, Any],
+) -> None:
+    """Validate revision snapshots, roots, head identity, and field bindings."""
+    revision_bodies = integrity_data.get("revision_bodies")
+    revision_roots = integrity_data.get("revision_roots")
+    if not isinstance(revision_bodies, Mapping) or not isinstance(
+        revision_roots,
+        Mapping,
+    ):
+        raise InvalidArchiveStructureError("Archive integrity revision roots are incomplete.")
+    revision_ids = [str(revision.revision_id) for revision in documents.revisions]
+    if set(revision_bodies) != set(revision_ids) or set(revision_roots) != set(revision_ids):
+        raise InvalidArchiveStructureError(
+            "Archive integrity revision snapshots do not cover revision history."
+        )
+    inventory = integrity_data.get("inventory")
+    revision_log = (
+        inventory.get(ArchiveEntry.REVISIONS.value) if isinstance(inventory, Mapping) else None
+    )
+    if not isinstance(revision_log, Mapping) or revision_log.get("revision_ids") != (revision_ids):
+        raise IntegrityMismatchError(
+            "Archive revision-log inventory does not match revision history.",
+            layer="revision",
+        )
+    for revision in documents.revisions:
+        revision_id = str(revision.revision_id)
+        body = revision_bodies.get(revision_id)
+        if not isinstance(body, Mapping):
+            raise InvalidArchiveStructureError(
+                f"Archive integrity omits revision snapshot '{revision_id}'."
+            )
+        expected_root = revision_digest(body)
+        stored_root = revision_roots.get(revision_id)
+        if stored_root != expected_root or revision.revision_root != expected_root:
+            raise IntegrityMismatchError(
+                f"Revision '{revision_id}' has a revision root mismatch: expected "
+                f"{expected_root}; observed {stored_root}.",
+                layer="revision",
+            )
+        _validate_revision_body_identity(revision, body)
+    head_revision_id = integrity_data.get("head_revision_id")
+    if head_revision_id != revision_ids[-1]:
+        raise InvalidArchiveStructureError(
+            "Archive integrity head revision does not match revision history."
+        )
+    manifest_body = integrity_data.get("manifest_body")
+    if manifest_body != revision_bodies[revision_ids[-1]]:
+        raise IntegrityMismatchError(
+            "Archive integrity manifest body does not match the head snapshot.",
+            layer="revision",
+        )
+    head_body = revision_bodies[revision_ids[-1]]
+    if head_body.get("inventory") != integrity_data.get("inventory"):
+        raise IntegrityMismatchError(
+            "Archive integrity inventory does not match the head snapshot.",
+            layer="revision",
+        )
+
+
+def _validate_integrity_inventory_keys(
+    archive: BoundedZipReader,
+    manifest: Manifest,
+    documents: ArchiveDocuments,
+    integrity_data: Mapping[str, Any],
+) -> bool:
+    """Require one inventory item for each physical and logical archive item."""
+    inventory = integrity_data.get("inventory")
+    if not isinstance(inventory, Mapping):
+        raise InvalidArchiveStructureError("Archive integrity inventory is incomplete.")
+    expected = {
+        entry_path
+        for entry_path in archive.entry_paths
+        if entry_path != ArchiveEntry.INTEGRITY.value
+    }
+    record_ids = {str(record.record_id) for record in documents.checkpoints}
+    expected.update({f"records/{record_id}" for record_id in record_ids})
+    expected.update({f"provenance/{record_id}" for record_id in record_ids})
+    expected.update({f"revisions/{revision.revision_id}" for revision in documents.revisions})
+    expected.add(ArchiveEntry.REVISIONS.value)
+    revision_bodies = integrity_data.get("revision_bodies", {})
+    if isinstance(revision_bodies, Mapping):
+        for revision_id, body in revision_bodies.items():
+            if not isinstance(body, Mapping):
+                continue
+            selections = body.get("active_selections", {})
+            if isinstance(selections, Mapping):
+                expected.update(
+                    f"active/{revision_id}/{checkpoint_id}" for checkpoint_id in selections
+                )
+    for source in manifest.sources.values():
+        source_id = source.source_id
+        if source_id is not None and source.entry is None:
+            expected.add(f"source-records/{source_id}")
+    expected.update(f"features/required/{feature}" for feature in manifest.required_features)
+    if set(inventory) != expected:
+        missing = sorted(expected - set(inventory))
+        extra = sorted(set(inventory) - expected)
+        physical_expected = {
+            entry_path for entry_path in expected if entry_path in archive.entry_paths
+        }
+        missing_physical = sorted(set(missing) & physical_expected)
+        if missing_physical or extra:
+            detail = f"missing={missing!r}, extra={extra!r}"
+            raise InvalidArchiveStructureError(
+                f"Archive integrity inventory is not closed: {detail}."
+            )
+        return False
+    return True
+
+
+def _validate_new_metadata_documents(documents: ArchiveDocuments) -> None:
+    """Validate typed metadata before integrity checks consume the document."""
+    for fields in documents.metadata_by_run.values():
+        if not isinstance(fields, Mapping):
+            raise InvalidArchiveStructureError("Canonical metadata by run must be an object.")
+        for field_data in fields.values():
+            _deserialise_metadata_field(field_data)
+
+
+def _validate_revision_body_identity(
+    revision: Revision,
+    body: Mapping[str, Any],
+) -> None:
+    """Validate the revision fields mirrored in its canonical snapshot."""
+    expected_fields = {
+        "revision_id": str(revision.revision_id),
+        "archive_id": str(revision.archive_id),
+        "parent_revision_id": (
+            str(revision.parent_revision_id) if revision.parent_revision_id is not None else None
+        ),
+        "parent_root": revision.parent_root,
+        "active_selections": dict(revision.active_selections),
+        "events": [dict(event) for event in revision.events],
+        "created_at": revision.created_at.isoformat(),
+    }
+    for field_name, expected in expected_fields.items():
+        if body.get(field_name) != expected:
+            raise IntegrityMismatchError(
+                f"Revision '{revision.revision_id}' body field '{field_name}' "
+                "does not match the revision log.",
+                layer="revision",
+            )