aind-metadata-reader 0.2.0__py3-none-any.whl

aind_metadata_reader/__init__.py

@@ -0,0 +1,63 @@
+"""Read historical AIND metadata without rewriting source records."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from aind_metadata_reader.applicability import Applicability, RuleEvaluation
+from aind_metadata_reader.components import ArtifactReference, CodeView
+from aind_metadata_reader.diagnostics import Diagnostic, FieldState, MetadataError, UnsupportedVersionError
+from aind_metadata_reader.document import Document, Field, Snapshot, SnapshotReference, Source, read_json
+from aind_metadata_reader.formats import FORMATS, Format, ReaderRecord, ReaderSelection
+from aind_metadata_reader.inspection import ReadStatus, ReadValue
+from aind_metadata_reader.interpretation import Bundle, Change, Interpretation, apply_changes, replay
+from aind_metadata_reader.readers import (
+    Acquisition,
+    Axis,
+    CoordinateFrame,
+    Process,
+    Processing,
+    read_acquisition,
+    read_processing,
+)
+from aind_metadata_reader.validation import ValidationReport, validate
+
+__all__ = [
+    "FORMATS",
+    "Acquisition",
+    "Applicability",
+    "ArtifactReference",
+    "Axis",
+    "Bundle",
+    "Change",
+    "CodeView",
+    "CoordinateFrame",
+    "Diagnostic",
+    "Document",
+    "Field",
+    "FieldState",
+    "Format",
+    "Interpretation",
+    "MetadataError",
+    "Process",
+    "Processing",
+    "ReadStatus",
+    "ReadValue",
+    "ReaderRecord",
+    "ReaderSelection",
+    "RuleEvaluation",
+    "Snapshot",
+    "SnapshotReference",
+    "Source",
+    "UnsupportedVersionError",
+    "ValidationReport",
+    "apply_changes",
+    "read_acquisition",
+    "read_json",
+    "read_processing",
+    "replay",
+    "validate",
+]
+
+try:
+    __version__ = version("aind-metadata-reader")
+except PackageNotFoundError:
+    __version__ = "0.0.0.dev0"

aind_metadata_reader/_enum.py

@@ -0,0 +1,10 @@
+"""String-valued enums with consistent display on every supported Python version."""
+
+from enum import Enum
+
+
+class StringEnum(str, Enum):
+    """Retain string comparison/JSON encoding and display values rather than member names."""
+
+    def __str__(self) -> str:
+        return str(self.value)

aind_metadata_reader/_wire.py

@@ -0,0 +1,84 @@
+"""Strict primitives shared by independently versioned saved formats."""
+
+import hashlib
+import json
+import math
+from typing import Any
+
+from aind_metadata_reader.diagnostics import MetadataError
+
+
+def json_value(value: Any) -> None:
+    """Reject implicit conversions and nonfinite numbers at JSON boundaries."""
+    if value is None or isinstance(value, (str, bool, int)):
+        return
+    if isinstance(value, float) and math.isfinite(value):
+        return
+    if isinstance(value, list):
+        for item in value:
+            json_value(item)
+        return
+    if isinstance(value, dict) and all(isinstance(key, str) for key in value):
+        for item in value.values():
+            json_value(item)
+        return
+    raise MetadataError("Expected finite JSON values with string object keys", code="invalid_json")
+
+
+def _pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    obj: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in obj:
+            raise MetadataError(f"Duplicate JSON key {key!r}", code="duplicate_key")
+        obj[key] = value
+    return obj
+
+
+def parse_json(value: str | bytes) -> Any:
+    """Decode JSON without discarding duplicate keys or accepting nonfinite values."""
+    try:
+        result = json.loads(value, object_pairs_hook=_pairs)
+    except (ValueError, UnicodeError, TypeError) as error:
+        if isinstance(error, MetadataError):
+            raise
+        raise MetadataError(f"Invalid JSON: {error}", code="invalid_json") from error
+    json_value(result)
+    return result
+
+
+def canonical(value: Any) -> bytes:
+    """Use the documented Python JSON convention for identities, never string coercion."""
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False).encode()
+
+
+def digest(scheme: str, value: Any) -> str:
+    """Namespace identity algorithms separately from file-format versions."""
+    return scheme + ":sha256:" + hashlib.sha256(canonical(value)).hexdigest()
+
+
+def object_fields(value: Any, names: set[str], *, code: str = "invalid_manifest") -> dict[str, Any]:
+    """Require an exact field set; extensions need a new declared format version."""
+    if not isinstance(value, dict) or set(value) != names:
+        raise MetadataError("Saved record has missing or unknown fields", code=code)
+    return value
+
+
+def string(value: Any, *, code: str = "invalid_manifest", empty: bool = False) -> str:
+    """Require a string without coercing other JSON types."""
+    if not isinstance(value, str) or not empty and not value.strip():
+        raise MetadataError("Expected a nonempty string", code=code)
+    return value
+
+
+def string_array(value: Any, *, code: str = "invalid_manifest") -> tuple[str, ...]:
+    """Accept JSON arrays, never strings iterated as individual characters."""
+    if not isinstance(value, list):
+        raise MetadataError("Expected an array of strings", code=code)
+    return tuple(string(item, code=code) for item in value)
+
+
+def strings(value: Any, *, code: str) -> tuple[str, ...]:
+    """Detach caller-supplied sequences into immutable, validated string tuples."""
+    if not isinstance(value, (tuple, list)):
+        raise MetadataError("Expected a sequence of strings", code=code)
+    return tuple(string(item, code=code) for item in value)

aind_metadata_reader/applicability.py

@@ -0,0 +1,132 @@
+"""Versioned decisions link exact inputs and proposed operations without applying them."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from aind_metadata_reader._enum import StringEnum
+from aind_metadata_reader._wire import digest, object_fields, string, string_array, strings
+from aind_metadata_reader.diagnostics import MetadataError
+from aind_metadata_reader.document import SnapshotReference
+from aind_metadata_reader.formats import ReaderRecord
+from aind_metadata_reader.interpretation import Change, _records, _validate
+
+
+class Applicability(StringEnum):
+    """A negative conclusion differs from insufficient evidence."""
+
+    APPLICABLE = "applicable"
+    NOT_APPLICABLE = "not_applicable"
+    UNKNOWN = "unknown"
+
+
+@dataclass(frozen=True)
+class RuleEvaluation:
+    """A versioned decision bound to exact inputs, reader provenance, and proposals."""
+
+    rule_id: str
+    rule_revision: str
+    outcome: Applicability
+    reason: str
+    evidence: tuple[str, ...] = ()
+    changes: tuple[Change, ...] = ()
+    inputs: tuple[SnapshotReference, ...] = ()
+    readers: tuple[ReaderRecord, ...] = ()
+    implementation: str = "manual/1"
+    _legacy: bool = field(default=False, repr=False)
+
+    def __post_init__(self) -> None:
+        """Validate links as well as the distinction between proposed and applied edits."""
+        for value in (self.rule_id, self.rule_revision, self.reason, self.implementation):
+            string(value, code="invalid_evaluation")
+        object.__setattr__(self, "evidence", strings(self.evidence, code="invalid_evaluation"))
+        for name in ("changes", "inputs", "readers"):
+            object.__setattr__(self, name, tuple(getattr(self, name)))
+        if not isinstance(self.outcome, Applicability):
+            raise MetadataError("Unknown applicability outcome", code="invalid_evaluation")
+        if (self.outcome == Applicability.APPLICABLE) != bool(self.changes):
+            raise MetadataError("Only applicable corrections carry proposals", code="invalid_evaluation")
+        if not self._legacy and (not self.inputs or any(not isinstance(ref, SnapshotReference) for ref in self.inputs)):
+            raise MetadataError("An evaluation needs exact input references", code="invalid_evaluation")
+        if any(not isinstance(r, ReaderRecord) or r.document not in self.inputs for r in self.readers):
+            raise MetadataError("Reader provenance must identify an evaluation input", code="invalid_evaluation")
+        for change in self.changes:
+            _validate(change, allow_unbound=self._legacy)
+            if (change.rule_id, change.rule_revision) != (self.rule_id, self.rule_revision):
+                raise MetadataError("Proposal identifies a different rule revision", code="invalid_evaluation")
+            if not self._legacy and (
+                change.implementation != self.implementation
+                or change.references != self.inputs
+                or change.readers != self.readers
+            ):
+                raise MetadataError("Proposal provenance differs from the evaluated inputs", code="invalid_evaluation")
+        if len({c.operation_id for c in self.changes}) != len(self.changes):
+            raise MetadataError("Proposal operation identities must be unique", code="invalid_evaluation")
+
+    def _body(self) -> dict[str, Any]:
+        result = {
+            "format": "metadata-rule-evaluation/1" if self._legacy else "metadata-rule-evaluation/2",
+            "rule_id": self.rule_id,
+            "rule_revision": self.rule_revision,
+            "outcome": self.outcome.value,
+            "reason": self.reason,
+            "evidence": list(self.evidence),
+            "changes": [c.record(legacy=self._legacy) for c in self.changes],
+        }
+        if not self._legacy:
+            result.update(
+                inputs=[ref.record() for ref in self.inputs],
+                readers=[r.record() for r in self.readers],
+                implementation=self.implementation,
+            )
+        return result
+
+    @property
+    def evaluation_id(self) -> str:
+        """Identify the retained decision independently of whether proposals were applied."""
+        return digest("metadata-rule-evaluation-v1", self._body())
+
+    def record(self) -> dict[str, Any]:
+        """Retain historical v1 records verbatim; new decisions use the qualified v2 format."""
+        record = self._body()
+        if not self._legacy:
+            record["evaluation_id"] = self.evaluation_id
+        return record
+
+    @classmethod
+    def from_record(cls, value: Any) -> RuleEvaluation:
+        """Restore decisions for audit and replay without running their implementation."""
+        if not isinstance(value, dict) or value.get("format") not in (
+            "metadata-rule-evaluation/1",
+            "metadata-rule-evaluation/2",
+        ):
+            raise MetadataError("Unsupported evaluation record", code="unsupported_evaluation")
+        legacy = value["format"] == "metadata-rule-evaluation/1"
+        names = {"format", "rule_id", "rule_revision", "outcome", "reason", "evidence", "changes"}
+        data = dict(
+            object_fields(
+                value,
+                names if legacy else names | {"inputs", "readers", "implementation", "evaluation_id"},
+                code="invalid_evaluation",
+            )
+        )
+        try:
+            outcome = Applicability(data["outcome"])
+        except (TypeError, ValueError) as error:
+            raise MetadataError("Unknown applicability outcome", code="invalid_evaluation") from error
+        result = cls(
+            data["rule_id"],
+            data["rule_revision"],
+            outcome,
+            data["reason"],
+            string_array(data["evidence"], code="invalid_evaluation"),
+            _records(data["changes"], lambda item: Change.from_record(item, legacy=legacy)),
+            () if legacy else _records(data["inputs"], SnapshotReference.from_record),
+            () if legacy else _records(data["readers"], ReaderRecord.from_record),
+            "legacy-unrecorded" if legacy else data["implementation"],
+            legacy,
+        )
+        if not legacy and data["evaluation_id"] != result.evaluation_id:
+            raise MetadataError("Evaluation identity mismatch", code="evaluation_mismatch")
+        return result

aind_metadata_reader/components.py

@@ -0,0 +1,125 @@
+"""Small views preserve code and reference meanings across historical representations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from aind_metadata_reader.diagnostics import Diagnostic, FieldState
+from aind_metadata_reader.document import Document, Field
+from aind_metadata_reader.inspection import ReadStatus, ReadValue, as_object, as_string, observe, unavailable
+
+
+@dataclass(frozen=True)
+class ArtifactReference:
+    """A file, asset, group, or uninterpreted entry with its exact source preserved."""
+
+    field: Field
+    kind: Literal["file", "asset", "asset_group", "unknown"]
+
+    @property
+    def raw(self) -> Any:
+        """Return recorded content without manufacturing an asset ID or file path."""
+        return self.field.value
+
+    @property
+    def location(self) -> ReadValue[str]:
+        """Keep asset URLs distinct from file-level locations in kind and provenance."""
+        if self.kind == "file":
+            return observe(self.field, as_string)
+        if self.kind == "asset":
+            return observe(self.field.snapshot.field(self.field.pointer + "/url"), as_string)
+        return unavailable(self.field, ReadStatus.UNSUPPORTED, "Reference does not establish one location")
+
+    @property
+    def members(self) -> ReadValue[tuple[ArtifactReference, ...]]:
+        """Retain grouped assets and nesting rather than flattening their meaning."""
+        if self.kind != "asset_group":
+            return unavailable(self.field, ReadStatus.UNSUPPORTED, "Reference is not an asset group")
+        return references(self.field.snapshot.field(self.field.pointer + "/assets"), "assets")
+
+
+def references(field: Field, representation: Literal["paths", "assets"]) -> ReadValue[tuple[ArtifactReference, ...]]:
+    """Read known reference structures while retaining malformed and unfamiliar members."""
+    if field.state != FieldState.PRESENT:
+        return observe(field, lambda _: ())
+    value = field.value
+    if representation == "paths" and isinstance(value, str):
+        return ReadValue(field, ReadStatus.AVAILABLE, (ArtifactReference(field, "file"),))
+    if not isinstance(value, list):
+        return unavailable(field, ReadStatus.INVALID, "Expected recorded paths or an array of asset references")
+    items = []
+    issues: list[Diagnostic] = []
+    for i, item in enumerate(value):
+        entry = field.snapshot.field(field.pointer + "/" + str(i))
+        kind: Literal["file", "asset", "asset_group", "unknown"] = "unknown"
+        if representation == "paths" and isinstance(item, str):
+            kind = "file"
+        elif representation == "assets" and isinstance(item, dict):
+            marker = item.get("object_type")
+            if marker == "Data asset":
+                kind = "asset"
+            elif marker == "Combined data":
+                kind = "asset_group"
+        if kind == "unknown":
+            issues.extend(
+                unavailable(
+                    entry, ReadStatus.UNSUPPORTED, "Reference entry has no established interpretation"
+                ).diagnostics
+            )
+        items.append(ArtifactReference(entry, kind))
+    return ReadValue(field, ReadStatus.AVAILABLE, tuple(items), tuple(issues))
+
+
+@dataclass(frozen=True)
+class CodeView:
+    """Code interpretation can be unavailable without preventing process inspection."""
+
+    field: Field
+    representation: Literal["legacy", "nested"]
+
+    def _field(self, legacy: str, nested: str) -> Field:
+        return self.field.snapshot.field(
+            self.field.pointer + "/" + (legacy if self.representation == "legacy" else nested)
+        )
+
+    def _read(self, legacy: str, nested: str, decoder: Any) -> ReadValue[Any]:
+        if self.field.state != FieldState.PRESENT or not isinstance(self.field.value, dict):
+            status = (
+                ReadStatus.INVALID if self.field.state == FieldState.PRESENT else ReadStatus(self.field.state.value)
+            )
+            return unavailable(self.field, status, "Code object is unavailable")
+        return observe(self._field(legacy, nested), decoder)
+
+    @property
+    def version(self) -> ReadValue[str]:
+        """Read a producer code version, independent of its pipeline version."""
+        return self._read("code_version", "version", as_string)
+
+    @property
+    def url(self) -> ReadValue[str]:
+        """Retain the recorded repository location without filling missing names."""
+        return self._read("code_url", "url", as_string)
+
+    @property
+    def parameters(self) -> ReadValue[dict[str, Any]]:
+        """Inspect parameters without replacing invalid values or supplying defaults."""
+        return self._read("parameters", "parameters", as_object)
+
+    @property
+    def inputs(self) -> ReadValue[tuple[ArtifactReference, ...]]:
+        """Expose file versus asset/group semantics explicitly."""
+        if self.field.state != FieldState.PRESENT or not isinstance(self.field.value, dict):
+            status = (
+                ReadStatus.INVALID if self.field.state == FieldState.PRESENT else ReadStatus(self.field.state.value)
+            )
+            return unavailable(self.field, status, "Code object cannot establish input references")
+        field = self._field("input_location", "input_data")
+        return references(field, "paths" if self.representation == "legacy" else "assets")
+
+    @property
+    def document(self) -> Document | None:
+        """Select a valid code object while keeping invalid raw content in field."""
+        if self.field.state == FieldState.PRESENT and isinstance(self.field.value, dict):
+            return Document.from_snapshot(self.field.snapshot, self.field.pointer)
+        return None

aind_metadata_reader/diagnostics.py

@@ -0,0 +1,38 @@
+"""Machine-readable failures and field availability without fabricated values."""
+
+from dataclasses import dataclass
+
+from aind_metadata_reader._enum import StringEnum
+
+
+class FieldState(StringEnum):
+    """Distinguish recorded values from absent and explicitly null fields."""
+
+    PRESENT = "present"
+    ABSENT = "absent"
+    NULL = "null"
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    """A failure tied to a location in a preserved snapshot."""
+
+    code: str
+    message: str
+    pointer: str = ""
+    snapshot_id: str | None = None
+
+
+class MetadataError(ValueError):
+    """A requested interpretation cannot be established from the record."""
+
+    def __init__(
+        self, message: str, *, code: str = "invalid_structure", pointer: str = "", snapshot_id: str | None = None
+    ) -> None:
+        """Attach structured context while keeping a readable exception message."""
+        self.diagnostic = Diagnostic(code, message, pointer, snapshot_id)
+        super().__init__(f"{message} (at {pointer or '/'})")
+
+
+class UnsupportedVersionError(MetadataError):
+    """No supported reader matches the declared representation."""