aind-metadata-reader 0.2.0__py3-none-any.whl

aind_metadata_reader/interpretation.py

@@ -0,0 +1,478 @@
+"""Explicit, replayable field changes with source-qualified derivation identities."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field, fields, replace
+from datetime import datetime
+from importlib.metadata import version
+from typing import TYPE_CHECKING, Any, Literal, TypeVar
+
+from aind_metadata_reader._wire import canonical, digest, object_fields, parse_json, string, string_array, strings
+from aind_metadata_reader.diagnostics import FieldState, MetadataError
+from aind_metadata_reader.document import Document, Snapshot, SnapshotReference, Source, _segments
+from aind_metadata_reader.formats import ReaderRecord
+
+if TYPE_CHECKING:
+    from aind_metadata_reader.applicability import RuleEvaluation
+
+_Record = TypeVar("_Record")
+
+
+@dataclass(frozen=True)
+class Bundle:
+    """Named input snapshots with attributed consistency evidence, never inferred authority."""
+
+    entries: tuple[tuple[str, Document], ...]
+    consistency: Literal["unknown", "asserted", "established"] = "unknown"
+    evidence: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        """Require unique names and explicit evidence for stronger consistency claims."""
+        object.__setattr__(self, "entries", tuple(self.entries))
+        object.__setattr__(self, "evidence", strings(self.evidence, code="invalid_bundle"))
+        if len({name for name, _ in self.entries}) != len(self.entries):
+            raise MetadataError("Bundle names must be unique", code="ambiguous")
+        if self.consistency not in ("unknown", "asserted", "established"):
+            raise MetadataError("Invalid bundle consistency", code="invalid_bundle")
+        if self.consistency != "unknown" and not self.evidence:
+            raise MetadataError("Bundle consistency needs explicit evidence", code="missing_evidence")
+
+
+@dataclass(frozen=True)
+class Change:
+    """An attributed replacement, bound to an exact input and optional supporting inputs."""
+
+    operation_id: str
+    rule_id: str
+    rule_revision: str
+    implementation: str
+    kind: Literal["correction", "inference", "processing_frame"]
+    source_digest: str
+    pointer: str
+    before_state: str
+    before_json: str
+    after_json: str
+    author: str
+    reason: str
+    evidence: tuple[str, ...]
+    scope: str
+    assumptions: tuple[str, ...] = ()
+    source: Source = field(default_factory=Source)
+    target: SnapshotReference | None = None
+    references: tuple[SnapshotReference, ...] = ()
+    readers: tuple[ReaderRecord, ...] = ()
+
+    def __post_init__(self) -> None:
+        """Detach mutable caller sequences without weakening application-time checks."""
+        object.__setattr__(self, "evidence", strings(self.evidence, code="invalid_change"))
+        object.__setattr__(self, "assumptions", strings(self.assumptions, code="invalid_change"))
+        object.__setattr__(self, "references", tuple(self.references))
+        object.__setattr__(self, "readers", tuple(self.readers))
+
+    @classmethod
+    def propose(
+        cls,
+        document: Document,
+        pointer: str,
+        value: Any,
+        *,
+        operation_id: str,
+        rule_id: str,
+        rule_revision: str,
+        implementation: str,
+        kind: Literal["correction", "inference", "processing_frame"],
+        author: str,
+        reason: str,
+        evidence: tuple[str, ...],
+        scope: str,
+        assumptions: tuple[str, ...] = (),
+        references: tuple[SnapshotReference, ...] = (),
+        readers: tuple[ReaderRecord, ...] = (),
+    ) -> Change:
+        """Capture source preconditions without modifying the recorded snapshot."""
+        before = document.field(pointer)
+        replacement = Document({"value": value}).raw["value"]
+        return cls(
+            operation_id,
+            rule_id,
+            rule_revision,
+            implementation,
+            kind,
+            document.snapshot.content_digest,
+            before.pointer,
+            before.state.value,
+            json.dumps(before.value, allow_nan=False),
+            json.dumps(replacement, allow_nan=False),
+            author,
+            reason,
+            evidence,
+            scope,
+            assumptions,
+            document.snapshot.source,
+            replace(document.reference(), pointer=before.pointer),
+            references,
+            readers,
+        )
+
+    def record(self, *, legacy: bool = False) -> dict[str, Any]:
+        """Serialize replacements as JSON text while retaining absent/null preconditions."""
+        result = {
+            f.name: getattr(self, f.name)
+            for f in fields(self)
+            if f.name not in ("source", "target", "references", "readers")
+        }
+        result["evidence"] = list(self.evidence)
+        result["assumptions"] = list(self.assumptions)
+        result["source"] = self.source.record(legacy=legacy)
+        if not legacy:
+            result.update(
+                target=None if self.target is None else self.target.record(),
+                references=[ref.record() for ref in self.references],
+                readers=[reader.record() for reader in self.readers],
+            )
+        return result
+
+    @classmethod
+    def from_record(cls, value: Any, *, legacy_document: Document | None = None, legacy: bool = False) -> Change:
+        """Restore a proposal strictly; v1 targets derive from verified manifest inputs."""
+        names = {f.name for f in fields(cls)}
+        legacy = legacy or legacy_document is not None
+        data = dict(object_fields(value, names - {"target", "references", "readers"} if legacy else names))
+        data["evidence"] = string_array(data["evidence"])
+        data["assumptions"] = string_array(data["assumptions"])
+        data["source"] = Source.from_record(data["source"], legacy=legacy)
+        string(data["pointer"], empty=True)
+        if legacy_document is not None:
+            data["target"] = replace(legacy_document.reference(), pointer=data["pointer"])
+        elif not legacy:
+            data["target"] = SnapshotReference.from_record(data["target"])
+            data["references"] = _records(data["references"], SnapshotReference.from_record)
+            data["readers"] = _records(data["readers"], ReaderRecord.from_record)
+        change = cls(**data)
+        _validate(change, allow_unbound=legacy and legacy_document is None)
+        return change
+
+
+def _records(value: Any, factory: Callable[[Any], _Record]) -> tuple[_Record, ...]:
+    if not isinstance(value, list):
+        raise MetadataError("Expected a saved-record array", code="invalid_manifest")
+    return tuple(factory(item) for item in value)
+
+
+def _application_record(
+    document: Document,
+    changes: tuple[Change, ...],
+    evaluations: tuple[RuleEvaluation, ...],
+    applied_at: str,
+    writer_version: str,
+) -> dict[str, Any]:
+    return {
+        "format": "metadata-interpretation/2",
+        "input": document.reference().record(),
+        "changes": [c.record() for c in changes],
+        "evaluations": [e.record() for e in evaluations],
+        "applied_at": applied_at,
+        "writer_version": writer_version,
+    }
+
+
+@dataclass(frozen=True)
+class Interpretation:
+    """Original and derived views, with linked decisions and applied operations."""
+
+    original: Document
+    derived: Document
+    changes: tuple[Change, ...]
+    changed: tuple[bool, ...]
+    applied_at: str
+    evaluations: tuple[RuleEvaluation, ...] = ()
+    writer_version: str = "unknown"
+
+    @property
+    def interpretation_id(self) -> str:
+        """Bind the derivation to its inputs, decisions, operations, time, and writer."""
+        return digest(
+            "metadata-interpretation-v1",
+            _application_record(self.original, self.changes, self.evaluations, self.applied_at, self.writer_version),
+        )
+
+    def manifest(self) -> dict[str, Any]:
+        """Export an independently versioned application record, not a source-schema file."""
+        record = _application_record(
+            self.original, self.changes, self.evaluations, self.applied_at, self.writer_version
+        )
+        links = {
+            change.operation_id: evaluation.evaluation_id
+            for evaluation in self.evaluations
+            for change in evaluation.changes
+            if change in self.changes
+        }
+        return {
+            **record,
+            "interpretation_id": self.interpretation_id,
+            "changed": list(self.changed),
+            "result_digest": self.derived.snapshot.content_digest,
+            "result_snapshot_id": self.derived.snapshot.snapshot_id,
+            "applied_operations": links,
+        }
+
+
+def _validate(change: Change, *, allow_unbound: bool = False) -> None:
+    for value in (
+        change.operation_id,
+        change.rule_id,
+        change.rule_revision,
+        change.implementation,
+        change.author,
+        change.reason,
+        change.scope,
+        change.source_digest,
+    ):
+        string(value, code="invalid_change")
+    if not change.evidence:
+        raise MetadataError("A change needs attributed evidence", code="missing_evidence")
+    if change.kind not in ("correction", "inference", "processing_frame"):
+        raise MetadataError("Unknown change kind", code="invalid_change")
+    if change.kind == "inference" and not change.assumptions:
+        raise MetadataError("An inference must record its assumptions", code="missing_assumptions")
+    if change.before_state not in tuple(state.value for state in FieldState):
+        raise MetadataError("Invalid source field state", code="invalid_change")
+    if not _segments(change.pointer):
+        raise MetadataError("Whole-document replacement is not a field correction", code="invalid_change")
+    if not (allow_unbound and change.target is None) and (
+        not isinstance(change.target, SnapshotReference)
+        or (
+            change.target.source != change.source
+            or change.target.content_digest != change.source_digest
+            or change.target.pointer != change.pointer
+        )
+    ):
+        raise MetadataError("Proposal must identify its exact target snapshot and field", code="invalid_change")
+    if any(not isinstance(ref, SnapshotReference) for ref in change.references):
+        raise MetadataError("Expected structured evidence references", code="invalid_change")
+    if any(
+        not isinstance(reader, ReaderRecord) or reader.document not in change.references for reader in change.readers
+    ):
+        raise MetadataError("Reader provenance must identify a referenced input", code="invalid_change")
+    parse_json(change.before_json)
+    parse_json(change.after_json)
+
+
+def _replace(data: dict[str, Any], pointer: str, value: Any) -> None:
+    segments = _segments(pointer)
+    parent: Any = data
+    for segment in segments[:-1]:
+        if isinstance(parent, dict) and segment in parent:
+            parent = parent[segment]
+        elif (
+            isinstance(parent, list)
+            and segment.isascii()
+            and segment.isdigit()
+            and str(int(segment)) == segment
+            and int(segment) < len(parent)
+        ):
+            parent = parent[int(segment)]
+        else:
+            raise MetadataError("Correction parent is absent", code="absent", pointer=pointer)
+    leaf = segments[-1]
+    if isinstance(parent, dict):
+        parent[leaf] = value
+    elif (
+        isinstance(parent, list)
+        and leaf.isascii()
+        and leaf.isdigit()
+        and str(int(leaf)) == leaf
+        and int(leaf) < len(parent)
+    ):
+        parent[int(leaf)] = value
+    else:
+        raise MetadataError(
+            "Cannot replace this field without inferring parent structure", code="invalid_change", pointer=pointer
+        )
+
+
+def _apply_operations(document: Document, operations: tuple[Change, ...]) -> tuple[dict[str, Any], tuple[bool, ...]]:
+    data = document.snapshot.raw
+    ids: set[str] = set()
+    targets: list[str] = []
+    changed = []
+    for change in operations:
+        _validate(change)
+        if change.source_digest != document.snapshot.content_digest:
+            raise MetadataError("Change targets another snapshot revision", code="stale_source", pointer=change.pointer)
+        if change.source != document.snapshot.source:
+            raise MetadataError(
+                "Change targets another source identity", code="conflicting_source", pointer=change.pointer
+            )
+        if change.target is None or change.target.snapshot_id != document.snapshot.snapshot_id:
+            raise MetadataError("Change targets another snapshot capture", code="stale_source", pointer=change.pointer)
+        if (
+            document.pointer
+            and change.pointer != document.pointer
+            and not change.pointer.startswith(document.pointer + "/")
+        ):
+            raise MetadataError("Change is outside the selected document", code="outside_focus", pointer=change.pointer)
+        if change.operation_id in ids or any(
+            change.pointer == p or change.pointer.startswith(p + "/") or p.startswith(change.pointer + "/")
+            for p in targets
+        ):
+            raise MetadataError(
+                "Conflicting changes require explicit resolution", code="conflicting_changes", pointer=change.pointer
+            )
+        before = parse_json(change.before_json)
+        after = parse_json(change.after_json)
+        original = document.snapshot.field(change.pointer)
+        if original.state != change.before_state or canonical(original.value) != canonical(before):
+            raise MetadataError(
+                "Source value precondition does not match", code="stale_precondition", pointer=change.pointer
+            )
+        _replace(data, change.pointer, after)
+        changed.append(original.state == FieldState.ABSENT or canonical(before) != canonical(after))
+        ids.add(change.operation_id)
+        targets.append(change.pointer)
+    return data, tuple(changed)
+
+
+def _check_evaluations(
+    document: Document, operations: tuple[Change, ...], evaluations: tuple[RuleEvaluation, ...]
+) -> None:
+    seen: set[str] = set()
+    proposals: dict[str, Change] = {}
+    for evaluation in evaluations:
+        if evaluation.evaluation_id in seen:
+            raise MetadataError("Repeated evaluation", code="conflicting_evaluations")
+        seen.add(evaluation.evaluation_id)
+        if not any(
+            ref.snapshot_id == document.snapshot.snapshot_id and ref.pointer == document.pointer
+            for ref in evaluation.inputs
+        ):
+            raise MetadataError("Evaluation does not identify this input view", code="conflicting_evaluations")
+        for change in evaluation.changes:
+            if change.operation_id in proposals:
+                raise MetadataError("Evaluations have ambiguous operation identities", code="conflicting_evaluations")
+            proposals[change.operation_id] = change
+    for operation in operations:
+        if operation.operation_id in proposals and operation != proposals[operation.operation_id]:
+            raise MetadataError("Applied operation differs from evaluated proposal", code="conflicting_evaluations")
+
+
+def _apply(
+    document: Document,
+    operations: tuple[Change, ...],
+    evaluations: tuple[RuleEvaluation, ...],
+    applied_at: str,
+    writer_version: str,
+) -> Interpretation:
+    try:
+        # Python 3.10 does not accept the UTC Z suffix directly. Preserve the
+        # recorded spelling in manifests while checking the equivalent offset.
+        parsed_time = (
+            applied_at[:-1] + "+00:00" if isinstance(applied_at, str) and applied_at.endswith("Z") else applied_at
+        )
+        timestamp = datetime.fromisoformat(parsed_time)
+        if timestamp.utcoffset() is None:
+            raise ValueError("Timezone required")
+    except (ValueError, TypeError) as error:
+        raise MetadataError("Record an ISO application time with timezone", code="invalid_timestamp") from error
+    string(writer_version)
+    _check_evaluations(document, operations, evaluations)
+    data, changed = _apply_operations(document, operations)
+    identity = digest(
+        "metadata-interpretation-v1", _application_record(document, operations, evaluations, applied_at, writer_version)
+    )
+    source = Source(derived_from=(document.snapshot.snapshot_id,), interpretation_id=identity)
+    derived = Document.from_snapshot(Snapshot.from_mapping(data, source), document.pointer)
+    return Interpretation(document, derived, operations, changed, applied_at, evaluations, writer_version)
+
+
+def apply_changes(
+    document: Document, changes: Iterable[Change], *, applied_at: str, evaluations: Iterable[RuleEvaluation] = ()
+) -> Interpretation:
+    """Apply explicit proposals within this view, retaining any supplied rule decisions.
+
+    Supply evaluations even for unknown and negative decisions to retain their
+    evidence. Applied operations are linked to matching evaluated proposals;
+    proposed operations may be deliberately left unapplied.
+    """
+    return _apply(document, tuple(changes), tuple(evaluations), applied_at, version("aind-metadata-reader"))
+
+
+def _check_result(result: Interpretation, manifest: dict[str, Any], *, legacy: bool = False) -> None:
+    flags = manifest["changed"]
+    if not isinstance(flags, list) or any(type(flag) is not bool for flag in flags):
+        raise MetadataError("Changed flags must be booleans", code="invalid_manifest")
+    if list(result.changed) != flags or result.derived.snapshot.content_digest != manifest["result_digest"]:
+        raise MetadataError("Manifest result does not match recorded output", code="manifest_mismatch")
+    if not legacy and result.manifest() != manifest:
+        raise MetadataError("Manifest identities or operation links do not match", code="manifest_mismatch")
+
+
+def _replay_v1(document: Document, manifest: dict[str, Any]) -> Interpretation:
+    object_fields(
+        manifest,
+        {
+            "format",
+            "source_digest",
+            "focus_pointer",
+            "source",
+            "original_digest",
+            "changes",
+            "changed",
+            "result_digest",
+            "applied_at",
+        },
+    )
+    if (
+        manifest["source_digest"] != document.snapshot.content_digest
+        or manifest["focus_pointer"] != document.pointer
+        or manifest["original_digest"] != document.snapshot.original_digest
+    ):
+        raise MetadataError("Manifest targets another source or object", code="stale_source")
+    if Source.from_record(manifest["source"], legacy=True) != document.snapshot.source:
+        raise MetadataError("Manifest source identity does not match", code="conflicting_source")
+    operations = _records(manifest["changes"], lambda item: Change.from_record(item, legacy_document=document))
+    result = apply_changes(document, operations, applied_at=manifest["applied_at"])
+    _check_result(result, manifest, legacy=True)
+    return result
+
+
+def replay(document: Document, manifest: dict[str, Any]) -> Interpretation:
+    """Apply recorded replacements to exact archived inputs; never rerun reader/rule code.
+
+    Historical v1 records are read explicitly, not relabeled as v2 inputs. Their
+    verified target is bound to the current source-qualified identity on replay;
+    newly exported results use v2 and do not invent historical reader provenance.
+    """
+    from aind_metadata_reader.applicability import RuleEvaluation
+
+    if not isinstance(manifest, dict):
+        raise MetadataError("Expected an interpretation manifest object", code="invalid_manifest")
+    if manifest.get("format") == "metadata-interpretation/1":
+        return _replay_v1(document, manifest)
+    if manifest.get("format") != "metadata-interpretation/2":
+        raise MetadataError("Unsupported interpretation manifest", code="unsupported_manifest")
+    object_fields(
+        manifest,
+        {
+            "format",
+            "input",
+            "changes",
+            "evaluations",
+            "applied_at",
+            "writer_version",
+            "interpretation_id",
+            "changed",
+            "result_digest",
+            "result_snapshot_id",
+            "applied_operations",
+        },
+    )
+    if SnapshotReference.from_record(manifest["input"]) != document.reference():
+        raise MetadataError("Manifest targets another snapshot or view", code="stale_source")
+    operations = _records(manifest["changes"], Change.from_record)
+    evaluations = _records(manifest["evaluations"], RuleEvaluation.from_record)
+    result = _apply(document, operations, evaluations, manifest["applied_at"], manifest["writer_version"])
+    _check_result(result, manifest)
+    return result

aind_metadata_reader/py.typed

@@ -0,0 +1 @@
+