data-annotations 2.2.0__tar.gz → 2.3.0__tar.gz

{data_annotations-2.2.0 → data_annotations-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-annotations
-Version: 2.2.0
+Version: 2.3.0
 Summary: Annotate generated data artifacts
 Keywords: annotations,data,metadata,provenance,reproducibility
 Author: Rodrigo C.  G.  Pena
@@ -102,6 +102,9 @@ Every annotation document includes provenance with:
 - Git commit, branch, dirty state, canonical repository remote, exact tags, and
   `git describe` output when available
 - The current `SLURM_JOB_ID` when available
+- Structured snapshots for recorded local inputs, including file checksums,
+  directory content digests, and upstream annotation sidecar references when
+  present
 
 You can also attach your own parameters, input file paths, and function names.
 Local filesystem paths in provenance are stored as absolute paths. URI-style inputs
@@ -380,6 +383,7 @@ File annotations store:
 - `subject.kind`
 - `subject.sha256`
 - `provenance.*`
+- `provenance.input_artifacts[]`
 - `description.title`
 - `description.summary`
 - `description.fields`
@@ -396,6 +400,7 @@ Directory annotations store:
 - `subject.child_bundles[]`
 - `subject.content_digest`
 - `provenance.*`
+- `provenance.input_artifacts[]`
 - `description.title`
 - `description.summary`
 - `description.artifact_groups[]`
@@ -529,13 +534,16 @@ per call.
 ## Recovery Helpers
 
 Use `artifact_matches_manifest(...)` to verify whether a detached artifact still
-matches an annotation document, and `checkout_manifest_source(...)` to recover the
-recorded code state from Git metadata.
+matches an annotation document. Use `analyze_provenance_chain(...)` when you also
+want to verify recorded inputs and recursively follow upstream annotation
+sidecars. Use `checkout_manifest_source(...)` to recover the recorded code state
+from Git metadata.
 
 ```python
 from pathlib import Path
 
 from data_annotations.provenance import (
+    analyze_provenance_chain,
     artifact_matches_manifest,
     checkout_manifest_source,
 )
@@ -544,6 +552,8 @@ annotation_path = Path("outputs/participants.csv.annotation.json")
 artifact_path = Path("downloads/participants.csv")
 
 if artifact_matches_manifest(artifact_path, annotation_path):
+    chain = analyze_provenance_chain(artifact_path)
+    print(chain.status)
     recovered = checkout_manifest_source(annotation_path)
     print(recovered.checkout_path)
     print(recovered.script_path)
@@ -602,12 +612,24 @@ For provenance inspection and source recovery:
 
 ```bash
 data-annotations provenance match path/to/artifact
+data-annotations provenance chain path/to/artifact
+data-annotations provenance chain path/to/artifact --full-paths
 data-annotations provenance checkout path/to/artifact
 ```
 
 Command `match` auto-discovers `*.annotation.json` for files and `data-annotations.json` for
 directories, prints a verification summary, and suggests the exact `checkout`
 command to run next when Git recovery metadata is available.
+Command `chain` uses the same sidecar discovery, then verifies the artifact,
+recorded input snapshots, and any upstream annotation sidecars reachable from
+those inputs. Its default output shows a compact relative-path tree and lists
+stale, missing, or unverifiable nodes first; use `--full-paths` when you need
+absolute paths.
+
+If `data-annotations provenance --help` does not list `chain`, your shell is
+resolving an older installed command. From a source checkout, use
+`uv run data-annotations provenance chain ...`, or reinstall the CLI from the
+updated source before using the bare `data-annotations` command.
 
 ### Run With `uvx`
 
@@ -632,6 +654,7 @@ the project environment. You can then run:
 uv run data-annotations annotate file path/to/participants.csv
 uv run data-annotations annotate directory path/to/run-001
 uv run data-annotations provenance match path/to/participants.csv
+uv run data-annotations provenance chain path/to/participants.csv
 uv run data-annotations provenance checkout path/to/participants.csv
 ```
 
@@ -684,9 +707,12 @@ uv run data-annotations provenance checkout path/to/participants.csv
 
 - `ProducedFile`
 - `ChildBundle`
+- `InputArtifact`
 - `BaseProvenance`
 - `FileManifest`
 - `DirectoryManifest`
+- `ProvenanceChainNode`
+- `ProvenanceChainReport`
 - `RecoveredSource`
 
 ### Provenance Functions
@@ -696,6 +722,8 @@ uv run data-annotations provenance checkout path/to/participants.csv
 - `write_file_manifest(...)`
 - `write_directory_manifest(...)`
 - `directory_content_digest(...)`
+- `analyze_provenance_chain(...)`
+- `provenance_chain_is_fresh(...)`
 - `artifact_matches_manifest(...)`
 - `checkout_manifest_source(...)`
 
@@ -717,6 +745,8 @@ uv run python examples/write_file_manifest.py
 uv run python examples/write_directory_manifest.py
 uv run python examples/write_file_description.py
 uv run python examples/write_directory_description.py
+uv run python examples/provenance_chain.py
+uv run python examples/provenance_chain_cli.py
 uv run python examples/recover_provenance.py
 uv run python examples/recover_provenance_cli.py
 ```

{data_annotations-2.2.0 → data_annotations-2.3.0}/README.md

@@ -73,6 +73,9 @@ Every annotation document includes provenance with:
 - Git commit, branch, dirty state, canonical repository remote, exact tags, and
   `git describe` output when available
 - The current `SLURM_JOB_ID` when available
+- Structured snapshots for recorded local inputs, including file checksums,
+  directory content digests, and upstream annotation sidecar references when
+  present
 
 You can also attach your own parameters, input file paths, and function names.
 Local filesystem paths in provenance are stored as absolute paths. URI-style inputs
@@ -351,6 +354,7 @@ File annotations store:
 - `subject.kind`
 - `subject.sha256`
 - `provenance.*`
+- `provenance.input_artifacts[]`
 - `description.title`
 - `description.summary`
 - `description.fields`
@@ -367,6 +371,7 @@ Directory annotations store:
 - `subject.child_bundles[]`
 - `subject.content_digest`
 - `provenance.*`
+- `provenance.input_artifacts[]`
 - `description.title`
 - `description.summary`
 - `description.artifact_groups[]`
@@ -500,13 +505,16 @@ per call.
 ## Recovery Helpers
 
 Use `artifact_matches_manifest(...)` to verify whether a detached artifact still
-matches an annotation document, and `checkout_manifest_source(...)` to recover the
-recorded code state from Git metadata.
+matches an annotation document. Use `analyze_provenance_chain(...)` when you also
+want to verify recorded inputs and recursively follow upstream annotation
+sidecars. Use `checkout_manifest_source(...)` to recover the recorded code state
+from Git metadata.
 
 ```python
 from pathlib import Path
 
 from data_annotations.provenance import (
+    analyze_provenance_chain,
     artifact_matches_manifest,
     checkout_manifest_source,
 )
@@ -515,6 +523,8 @@ annotation_path = Path("outputs/participants.csv.annotation.json")
 artifact_path = Path("downloads/participants.csv")
 
 if artifact_matches_manifest(artifact_path, annotation_path):
+    chain = analyze_provenance_chain(artifact_path)
+    print(chain.status)
     recovered = checkout_manifest_source(annotation_path)
     print(recovered.checkout_path)
     print(recovered.script_path)
@@ -573,12 +583,24 @@ For provenance inspection and source recovery:
 
 ```bash
 data-annotations provenance match path/to/artifact
+data-annotations provenance chain path/to/artifact
+data-annotations provenance chain path/to/artifact --full-paths
 data-annotations provenance checkout path/to/artifact
 ```
 
 Command `match` auto-discovers `*.annotation.json` for files and `data-annotations.json` for
 directories, prints a verification summary, and suggests the exact `checkout`
 command to run next when Git recovery metadata is available.
+Command `chain` uses the same sidecar discovery, then verifies the artifact,
+recorded input snapshots, and any upstream annotation sidecars reachable from
+those inputs. Its default output shows a compact relative-path tree and lists
+stale, missing, or unverifiable nodes first; use `--full-paths` when you need
+absolute paths.
+
+If `data-annotations provenance --help` does not list `chain`, your shell is
+resolving an older installed command. From a source checkout, use
+`uv run data-annotations provenance chain ...`, or reinstall the CLI from the
+updated source before using the bare `data-annotations` command.
 
 ### Run With `uvx`
 
@@ -603,6 +625,7 @@ the project environment. You can then run:
 uv run data-annotations annotate file path/to/participants.csv
 uv run data-annotations annotate directory path/to/run-001
 uv run data-annotations provenance match path/to/participants.csv
+uv run data-annotations provenance chain path/to/participants.csv
 uv run data-annotations provenance checkout path/to/participants.csv
 ```
 
@@ -655,9 +678,12 @@ uv run data-annotations provenance checkout path/to/participants.csv
 
 - `ProducedFile`
 - `ChildBundle`
+- `InputArtifact`
 - `BaseProvenance`
 - `FileManifest`
 - `DirectoryManifest`
+- `ProvenanceChainNode`
+- `ProvenanceChainReport`
 - `RecoveredSource`
 
 ### Provenance Functions
@@ -667,6 +693,8 @@ uv run data-annotations provenance checkout path/to/participants.csv
 - `write_file_manifest(...)`
 - `write_directory_manifest(...)`
 - `directory_content_digest(...)`
+- `analyze_provenance_chain(...)`
+- `provenance_chain_is_fresh(...)`
 - `artifact_matches_manifest(...)`
 - `checkout_manifest_source(...)`
 
@@ -688,6 +716,8 @@ uv run python examples/write_file_manifest.py
 uv run python examples/write_directory_manifest.py
 uv run python examples/write_file_description.py
 uv run python examples/write_directory_description.py
+uv run python examples/provenance_chain.py
+uv run python examples/provenance_chain_cli.py
 uv run python examples/recover_provenance.py
 uv run python examples/recover_provenance_cli.py
 ```

{data_annotations-2.2.0 → data_annotations-2.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data-annotations"
-version = "2.2.0"
+version = "2.3.0"
 description = "Annotate generated data artifacts"
 readme = "README.md"
 authors = [

{data_annotations-2.2.0 → data_annotations-2.3.0}/src/data_annotations/annotations/models.py

@@ -22,14 +22,14 @@ class DirectoryArtifactSubject(BaseModel):
 
 
 class FileAnnotationDocument(BaseModel):
-    annotation_version: Literal["4"] = "4"
+    annotation_version: Literal["5"] = "5"
     subject: FileArtifactSubject
     provenance: BaseProvenance
     description: FileDescription
 
 
 class DirectoryAnnotationDocument(BaseModel):
-    annotation_version: Literal["4"] = "4"
+    annotation_version: Literal["5"] = "5"
     subject: DirectoryArtifactSubject
     provenance: BaseProvenance
     description: DirectoryDescription

{data_annotations-2.2.0 → data_annotations-2.3.0}/src/data_annotations/annotations/writers.py

@@ -1,3 +1,4 @@
+from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Callable
 
@@ -154,7 +155,7 @@ def _build_file_annotation_document(
     generation_context: dict[str, Any] | None = None,
     artifact_kind: ArtifactKind = "other",
     params: dict[str, Any] | None = None,
-    inputs: list[str] | None = None,
+    inputs: Sequence[str | Path] | None = None,
     function: Callable[..., Any] | None = None,
     capture_mode: str = "runtime",
     provenance_overrides: dict[str, Any] | None = None,
@@ -205,7 +206,7 @@ def _build_directory_annotation_document(
     acquisition_context: dict[str, Any] | None = None,
     generation_context: dict[str, Any] | None = None,
     params: dict[str, Any] | None = None,
-    inputs: list[str] | None = None,
+    inputs: Sequence[str | Path] | None = None,
     function: Callable[..., Any] | None = None,
     capture_mode: str = "runtime",
     provenance_overrides: dict[str, Any] | None = None,
@@ -285,7 +286,7 @@ def write_file_annotation(
     generation_context: dict[str, Any] | None = None,
     artifact_kind: ArtifactKind = "other",
     params: dict[str, Any] | None = None,
-    inputs: list[str] | None = None,
+    inputs: Sequence[str | Path] | None = None,
     function: Callable[..., Any] | None = None,
     capture_mode: str = "runtime",
     provenance_overrides: dict[str, Any] | None = None,
@@ -324,7 +325,7 @@ def write_directory_annotation(
     acquisition_context: dict[str, Any] | None = None,
     generation_context: dict[str, Any] | None = None,
     params: dict[str, Any] | None = None,
-    inputs: list[str] | None = None,
+    inputs: Sequence[str | Path] | None = None,
     function: Callable[..., Any] | None = None,
     capture_mode: str = "runtime",
     provenance_overrides: dict[str, Any] | None = None,
@@ -363,7 +364,7 @@ def annotate_file(
     generation_context: dict[str, Any] | None = None,
     artifact_kind: ArtifactKind = "other",
     params: dict[str, Any] | None = None,
-    inputs: list[str] | None = None,
+    inputs: Sequence[str | Path] | None = None,
     function: Callable[..., Any] | None = None,
     write_readme: bool = True,
     write_schema: bool | None = None,
@@ -418,7 +419,7 @@ def annotate_directory(
     acquisition_context: dict[str, Any] | None = None,
     generation_context: dict[str, Any] | None = None,
     params: dict[str, Any] | None = None,
-    inputs: list[str] | None = None,
+    inputs: Sequence[str | Path] | None = None,
     function: Callable[..., Any] | None = None,
     write_readme: bool = True,
     write_schema: bool | None = None,

data_annotations-2.3.0/src/data_annotations/cli_app/provenance_commands.py

@@ -0,0 +1,321 @@
+import subprocess
+from dataclasses import dataclass
+from os.path import commonpath
+from pathlib import Path
+from urllib.parse import urlparse
+
+import typer
+
+from data_annotations.provenance import checkout_manifest_source
+from data_annotations.provenance import recovery as provenance_recovery
+
+from .common import (
+    _checkout_hint,
+    _echo_entries,
+    _error,
+    _match_target_path,
+    _missing_checkout_fields,
+    _resolve_manifest_path,
+    _resolved_path,
+)
+
+provenance_app = typer.Typer(
+    no_args_is_help=True,
+    help="Inspect provenance recorded in annotation documents.",
+)
+
+
+@provenance_app.command("match")
+def match_command(
+    target: Path = typer.Argument(
+        ..., help="Artifact, directory, or annotation document path."
+    ),
+    manifest: Path | None = typer.Option(
+        None,
+        "--manifest",
+        help="Explicit annotation document path to use instead of auto-discovery.",
+    ),
+) -> None:
+    manifest_path = _resolve_manifest_path(target, manifest)
+    candidate_path = _match_target_path(target, manifest)
+    loaded_manifest = provenance_recovery._load_manifest(manifest_path)
+    match = provenance_recovery._analyze_artifact_match(candidate_path, loaded_manifest)
+
+    typer.echo(f"Target: {candidate_path}")
+    typer.echo(f"Manifest: {manifest_path}")
+    typer.echo(f"Result: {match.status.replace('_', ' ').upper()}")
+
+    _echo_entries("Verified entries", match.verified_entries)
+    _echo_entries("Missing tracked entries", match.missing_tracked_entries)
+    _echo_entries("Mismatched tracked entries", match.mismatched_tracked_entries)
+    _echo_entries("Extra entries", match.extra_entries)
+    _echo_entries("Unverifiable tracked entries", match.unverifiable_tracked_entries)
+
+    if match.status in {"match", "partial_match"}:
+        missing_checkout_fields = _missing_checkout_fields(loaded_manifest)
+        if missing_checkout_fields:
+            typer.echo(
+                "Checkout unavailable: manifest is missing "
+                + ", ".join(missing_checkout_fields)
+            )
+        else:
+            typer.echo("Next step:")
+            typer.echo(
+                "  "
+                + _checkout_hint(
+                    str(_resolved_path(target)),
+                    str(_resolved_path(manifest)) if manifest is not None else None,
+                )
+            )
+        return
+
+    raise typer.Exit(code=1)
+
+
+@dataclass(frozen=True)
+class _DisplayChainNode:
+    path: str
+    status: provenance_recovery.ChainStatus
+    details: tuple[str, ...]
+    inputs: tuple["_DisplayChainNode", ...]
+
+
+def _is_uri_like(value: str) -> bool:
+    parsed = urlparse(value)
+    return bool(parsed.scheme and (parsed.netloc or "://" in value))
+
+
+def _node_paths_equal(left: str, right: str) -> bool:
+    if _is_uri_like(left) or _is_uri_like(right):
+        return left == right
+    return Path(left).expanduser().resolve() == Path(right).expanduser().resolve()
+
+
+def _should_merge_upstream_node(
+    node: provenance_recovery.ProvenanceChainNode,
+    child: provenance_recovery.ProvenanceChainNode,
+) -> bool:
+    return _node_paths_equal(node.path, child.path)
+
+
+def _display_chain_node(
+    node: provenance_recovery.ProvenanceChainNode,
+) -> _DisplayChainNode:
+    details = node.details
+    inputs = node.inputs
+    if len(inputs) == 1 and _should_merge_upstream_node(node, inputs[0]):
+        upstream = inputs[0]
+        details = (*details, *upstream.details)
+        inputs = upstream.inputs
+
+    return _DisplayChainNode(
+        path=node.path,
+        status=node.status,
+        details=details,
+        inputs=tuple(_display_chain_node(input_node) for input_node in inputs),
+    )
+
+
+def _filesystem_display_base(root: _DisplayChainNode) -> Path | None:
+    parent_paths: list[str] = []
+
+    def collect(node: _DisplayChainNode) -> None:
+        if not _is_uri_like(node.path):
+            path = Path(node.path).expanduser().resolve()
+            parent_paths.append(str(path if path.is_dir() else path.parent))
+        for input_node in node.inputs:
+            collect(input_node)
+
+    collect(root)
+    if not parent_paths:
+        return None
+    try:
+        return Path(commonpath(parent_paths))
+    except ValueError:
+        return None
+
+
+def _format_chain_path(
+    value: str,
+    *,
+    display_base: Path | None,
+    full_paths: bool,
+) -> str:
+    if full_paths or _is_uri_like(value):
+        return value
+    path = Path(value).expanduser().resolve()
+    if display_base is None:
+        return path.name
+    try:
+        return path.relative_to(display_base).as_posix()
+    except ValueError:
+        return str(path)
+
+
+def _problem_detail(details: tuple[str, ...]) -> str | None:
+    problem_terms = (
+        "changed",
+        "missing",
+        "no recorded",
+        "not locally verifiable",
+        "could not",
+        "unverifiable",
+        "stale:",
+    )
+    for detail in details:
+        lowered = detail.lower()
+        if any(term in lowered for term in problem_terms):
+            return detail
+    return None
+
+
+def _echo_chain_problems(
+    root: _DisplayChainNode,
+    *,
+    display_base: Path | None,
+    full_paths: bool,
+) -> None:
+    problems: list[tuple[_DisplayChainNode, _DisplayChainNode | None, str]] = []
+
+    def collect(
+        node: _DisplayChainNode,
+        parent: _DisplayChainNode | None = None,
+    ) -> None:
+        if node.status != "fresh":
+            detail = _problem_detail(node.details)
+            if detail is not None:
+                problems.append((node, parent, detail))
+        for input_node in node.inputs:
+            collect(input_node, node)
+
+    collect(root)
+    if not problems:
+        return
+
+    typer.echo("")
+    typer.echo("Problems:")
+    for node, parent, detail in problems:
+        typer.echo(
+            "  "
+            + f"{node.status.upper():<12}"
+            + _format_chain_path(
+                node.path,
+                display_base=display_base,
+                full_paths=full_paths,
+            )
+        )
+        typer.echo(f"  {'':<12}{detail}")
+        if parent is not None:
+            typer.echo(
+                "  "
+                + f"{'':<12}used by "
+                + _format_chain_path(
+                    parent.path,
+                    display_base=display_base,
+                    full_paths=full_paths,
+                )
+            )
+
+
+def _echo_chain_tree(
+    node: _DisplayChainNode,
+    *,
+    display_base: Path | None,
+    full_paths: bool,
+    depth: int = 0,
+) -> None:
+    indent = "  " * depth
+    path = _format_chain_path(
+        node.path,
+        display_base=display_base,
+        full_paths=full_paths,
+    )
+    typer.echo(f"{indent}[{node.status.upper()}] {path}")
+    for input_node in node.inputs:
+        _echo_chain_tree(
+            input_node,
+            display_base=display_base,
+            full_paths=full_paths,
+            depth=depth + 1,
+        )
+
+
+def _echo_chain_report(
+    report: provenance_recovery.ProvenanceChainReport,
+    *,
+    full_paths: bool,
+) -> None:
+    root = _display_chain_node(report.root)
+    display_base = _filesystem_display_base(root)
+    typer.echo(f"Result: {report.status.upper()}")
+    _echo_chain_problems(root, display_base=display_base, full_paths=full_paths)
+    typer.echo("")
+    typer.echo("Chain:")
+    _echo_chain_tree(root, display_base=display_base, full_paths=full_paths)
+
+
+@provenance_app.command("chain")
+def chain_command(
+    target: Path = typer.Argument(
+        ..., help="Artifact, directory, or annotation document path."
+    ),
+    manifest: Path | None = typer.Option(
+        None,
+        "--manifest",
+        help="Explicit annotation document path to use instead of auto-discovery.",
+    ),
+    full_paths: bool = typer.Option(
+        False,
+        "--full-paths",
+        help="Show full paths instead of compact paths relative to the chain root.",
+    ),
+) -> None:
+    manifest_path = _resolve_manifest_path(target, manifest)
+    candidate_path = _match_target_path(target, manifest)
+    report = provenance_recovery.analyze_provenance_chain(
+        candidate_path,
+        manifest_path,
+    )
+
+    _echo_chain_report(report, full_paths=full_paths)
+
+    if report.status != "fresh":
+        raise typer.Exit(code=1)
+
+
+@provenance_app.command("checkout")
+def checkout_command(
+    target: Path = typer.Argument(
+        ..., help="Artifact, directory, or annotation document path."
+    ),
+    manifest: Path | None = typer.Option(
+        None,
+        "--manifest",
+        help="Explicit annotation document path to use instead of auto-discovery.",
+    ),
+    dest: Path | None = typer.Option(
+        None,
+        "--dest",
+        help="Optional checkout destination. Defaults to a stable user cache.",
+    ),
+) -> None:
+    manifest_path = _resolve_manifest_path(target, manifest)
+
+    try:
+        recovered = checkout_manifest_source(
+            manifest_path,
+            destination_dir=dest,
+        )
+    except ValueError as exc:
+        _error(str(exc), code=1)
+    except subprocess.CalledProcessError:
+        _error("failed to clone or checkout the recorded repository state", code=1)
+
+    typer.echo(f"Manifest: {manifest_path}")
+    typer.echo(f"Checkout path: {recovered.checkout_path}")
+    if recovered.script_path is not None:
+        typer.echo(f"Recovered script: {recovered.script_path}")
+    else:
+        typer.echo(
+            "Recovered repository checkout, but the generating script could not be resolved."
+        )

{data_annotations-2.2.0 → data_annotations-2.3.0}/src/data_annotations/provenance/__init__.py

@@ -4,12 +4,20 @@ from .models import (
     ChildBundle,
     DirectoryManifest,
     FileManifest,
+    InputArtifact,
     ProducedFile,
     RecoveredSource,
 )
 from .decorators import record_directory_manifest, record_file_manifest
 from .git import capture_git_info
-from .recovery import artifact_matches_manifest, checkout_manifest_source
+from .recovery import (
+    ProvenanceChainNode,
+    ProvenanceChainReport,
+    analyze_provenance_chain,
+    artifact_matches_manifest,
+    checkout_manifest_source,
+    provenance_chain_is_fresh,
+)
 from .runtime import capture_runtime_info
 from .writers import (
     callable_name,
@@ -25,13 +33,18 @@ __all__ = [
     "ChildBundle",
     "DirectoryManifest",
     "FileManifest",
+    "InputArtifact",
     "ProducedFile",
+    "ProvenanceChainNode",
+    "ProvenanceChainReport",
     "RecoveredSource",
+    "analyze_provenance_chain",
     "artifact_matches_manifest",
     "callable_name",
     "capture_git_info",
     "capture_runtime_info",
     "checkout_manifest_source",
+    "provenance_chain_is_fresh",
     "directory_content_digest",
     "record_directory_manifest",
     "record_file_manifest",