data-annotations 2.8.1__tar.gz → 2.9.0__tar.gz

{data_annotations-2.8.1 → data_annotations-2.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-annotations
-Version: 2.8.1
+Version: 2.9.0
 Summary: Annotate data artifacts with provenance and descriptions
 Keywords: annotations,data,metadata,provenance,reproducibility
 Author: Rodrigo C.  G.  Pena
@@ -30,6 +30,11 @@ Description-Content-Type: text/markdown
 
 # data-annotations
 
+[![PyPI](https://img.shields.io/pypi/v/data-annotations?label=pypi)](https://pypi.org/project/data-annotations/)
+[![Documentation](https://img.shields.io/badge/docs-latest-blue)](https://ceda-unibas.gitlab.io/tools/data-annotations/)
+[![License](https://img.shields.io/pypi/l/data-annotations?label=license)](https://gitlab.com/ceda-unibas/tools/data-annotations/-/blob/main/LICENSE)
+[![CI](https://gitlab.com/ceda-unibas/tools/data-annotations/badges/main/pipeline.svg)](https://gitlab.com/ceda-unibas/tools/data-annotations/-/pipelines)
+
 `data-annotations` is a Python package for attaching provenance and structured
 descriptions to the files and directories your workflows produce.
 
@@ -43,12 +48,13 @@ Optional Markdown README sidecars can be generated for human-readable summaries.
 
 ## Documentation
 
-The full documentation is organized as a [Diátaxis](https://diataxis.fr/) site:
-https://ceda-unibas.gitlab.io/tools/data-annotations/
+The [full documentation](https://ceda-unibas.gitlab.io/tools/data-annotations/) is organized as a [Diátaxis](https://diataxis.fr/) site.
+
+Other links:
 
-- Source: https://gitlab.com/ceda-unibas/tools/data-annotations
-- Changelog: https://gitlab.com/ceda-unibas/tools/data-annotations/-/blob/main/CHANGELOG.md
-- Work items: https://gitlab.com/ceda-unibas/tools/data-annotations/-/work_items
+- [Source code](https://gitlab.com/ceda-unibas/tools/data-annotations)
+- [Changelog](https://gitlab.com/ceda-unibas/tools/data-annotations/-/blob/main/CHANGELOG.md)
+- [Work items](https://gitlab.com/ceda-unibas/tools/data-annotations/-/work_items)
 
 ## Installation
 

{data_annotations-2.8.1 → data_annotations-2.9.0}/README.md

@@ -1,5 +1,10 @@
 # data-annotations
 
+[![PyPI](https://img.shields.io/pypi/v/data-annotations?label=pypi)](https://pypi.org/project/data-annotations/)
+[![Documentation](https://img.shields.io/badge/docs-latest-blue)](https://ceda-unibas.gitlab.io/tools/data-annotations/)
+[![License](https://img.shields.io/pypi/l/data-annotations?label=license)](https://gitlab.com/ceda-unibas/tools/data-annotations/-/blob/main/LICENSE)
+[![CI](https://gitlab.com/ceda-unibas/tools/data-annotations/badges/main/pipeline.svg)](https://gitlab.com/ceda-unibas/tools/data-annotations/-/pipelines)
+
 `data-annotations` is a Python package for attaching provenance and structured
 descriptions to the files and directories your workflows produce.
 
@@ -13,12 +18,13 @@ Optional Markdown README sidecars can be generated for human-readable summaries.
 
 ## Documentation
 
-The full documentation is organized as a [Diátaxis](https://diataxis.fr/) site:
-https://ceda-unibas.gitlab.io/tools/data-annotations/
+The [full documentation](https://ceda-unibas.gitlab.io/tools/data-annotations/) is organized as a [Diátaxis](https://diataxis.fr/) site.
+
+Other links:
 
-- Source: https://gitlab.com/ceda-unibas/tools/data-annotations
-- Changelog: https://gitlab.com/ceda-unibas/tools/data-annotations/-/blob/main/CHANGELOG.md
-- Work items: https://gitlab.com/ceda-unibas/tools/data-annotations/-/work_items
+- [Source code](https://gitlab.com/ceda-unibas/tools/data-annotations)
+- [Changelog](https://gitlab.com/ceda-unibas/tools/data-annotations/-/blob/main/CHANGELOG.md)
+- [Work items](https://gitlab.com/ceda-unibas/tools/data-annotations/-/work_items)
 
 ## Installation
 

{data_annotations-2.8.1 → data_annotations-2.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data-annotations"
-version = "2.8.1"
+version = "2.9.0"
 description = "Annotate data artifacts with provenance and descriptions"
 readme = "README.md"
 authors = [

{data_annotations-2.8.1 → data_annotations-2.9.0}/src/data_annotations/annotations/answers.py

@@ -1,7 +1,7 @@
 import os
 import re
 import shlex
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from pathlib import Path
 from typing import Any, Literal, TypeAlias
 
@@ -124,6 +124,32 @@ _EXPLICIT_PROVENANCE_OVERRIDE_FIELDS = {
 }
 
 
+def _validate_runtime_inference_fields(values: Iterable[str]) -> list[str]:
+    normalized: list[str] = []
+    for value in values:
+        if value in _UNSUPPORTED_RUNTIME_INFERENCE_FIELDS:
+            raise ValueError(
+                "runtime inference is not supported for "
+                f"provenance.{value}; provide it explicitly"
+            )
+        if value not in _RUNTIME_INFERENCE_FIELDS:
+            allowed = sorted(_RUNTIME_INFERENCE_FIELDS)
+            raise ValueError(
+                f"unknown runtime inference field {value!r}; "
+                "expected one of: " + ", ".join(allowed)
+            )
+        if value not in normalized:
+            normalized.append(value)
+    return normalized
+
+
+def _expand_runtime_inference_fields(values: Iterable[str]) -> set[str]:
+    fields: set[str] = set()
+    for field in values:
+        fields.update(_RUNTIME_INFERENCE_GROUPS.get(field, {field}))
+    return fields
+
+
 class ProvenanceAnswers(BaseModel):
     """Optional provenance overrides supplied through an answers payload."""
 
@@ -194,22 +220,7 @@ class ProvenanceAnswers(BaseModel):
     @field_validator("infer_from_runtime")
     @classmethod
     def _validate_runtime_inference_fields(cls, values: list[str]) -> list[str]:
-        normalized: list[str] = []
-        for value in values:
-            if value in _UNSUPPORTED_RUNTIME_INFERENCE_FIELDS:
-                raise ValueError(
-                    "runtime inference is not supported for "
-                    f"provenance.{value}; provide it explicitly"
-                )
-            if value not in _RUNTIME_INFERENCE_FIELDS:
-                allowed = sorted(_RUNTIME_INFERENCE_FIELDS)
-                raise ValueError(
-                    f"unknown runtime inference field {value!r}; "
-                    "expected one of: " + ", ".join(allowed)
-                )
-            if value not in normalized:
-                normalized.append(value)
-        return normalized
+        return _validate_runtime_inference_fields(values)
 
     @model_validator(mode="after")
     def _validate_runtime_inference_conflicts(self) -> "ProvenanceAnswers":
@@ -240,10 +251,7 @@ class ProvenanceAnswers(BaseModel):
     def runtime_inference_fields(self) -> set[str]:
         """Expand runtime inference groups into concrete provenance fields."""
 
-        fields: set[str] = set()
-        for field in self.infer_from_runtime:
-            fields.update(_RUNTIME_INFERENCE_GROUPS.get(field, {field}))
-        return fields
+        return _expand_runtime_inference_fields(self.infer_from_runtime)
 
 
 class BaseAnswers(BaseModel):

{data_annotations-2.8.1 → data_annotations-2.9.0}/src/data_annotations/cli_app/annotate/__init__.py

@@ -11,6 +11,7 @@ from .helpers import (
     _child_bundle_from_annotation,
     _child_bundles_from_answers,
     _collect_post_hoc_provenance_from_sources,
+    _documented_artifacts_from_discovered_files,
     _documented_artifact_groups_from_answers,
     _documented_artifacts_from_answers,
     _load_directory_answers,
@@ -38,6 +39,7 @@ from ..common import (
     InputValuesOption,
     MaxChecksumBytesOption,
     ParamValuesOption,
+    RuntimeInferenceValuesOption,
     ScriptOption,
     ScriptRepoPathOption,
     Sha256Option,
@@ -125,6 +127,7 @@ def annotate_file_command(
     source_path: SourcePathOption = None,
     source_revision: SourceRevisionOption = None,
     source_sha256: SourceSha256Option = None,
+    infer_from_runtime: RuntimeInferenceValuesOption = None,
     checksum_policy: ChecksumPolicyOption = "auto",
     max_checksum_bytes: MaxChecksumBytesOption = 10 * 1024**3,
     sha256: Sha256Option = None,
@@ -146,8 +149,6 @@ def annotate_file_command(
         file_answers.target if file_answers is not None else None,
         expected_kind="file",
     )
-    if not is_interactive and file_answers is None:
-        _error("--no-interactive requires --answers")
     if not is_interactive and file_answers is not None:
         try:
             answer_files.require_complete_file_answers(
@@ -215,6 +216,7 @@ def annotate_file_command(
         source_path=source_path,
         source_revision=source_revision,
         source_sha256=source_sha256,
+        infer_from_runtime=infer_from_runtime,
     )
 
     fields: list[FieldDefinition] = list(file_answers.fields) if file_answers else []
@@ -295,6 +297,7 @@ def annotate_directory_command(
     source_path: SourcePathOption = None,
     source_revision: SourceRevisionOption = None,
     source_sha256: SourceSha256Option = None,
+    infer_from_runtime: RuntimeInferenceValuesOption = None,
     recursive: bool = typer.Option(
         False,
         "--recursive",
@@ -326,6 +329,14 @@ def annotate_directory_command(
         "--group-kind",
         help="Artifact kind for the corresponding --group-selector.",
     ),
+    include_discovered: bool = typer.Option(
+        False,
+        "--include-discovered",
+        help=(
+            "Include discovered files and child bundles non-interactively using "
+            "sparse descriptions."
+        ),
+    ),
     checksum_policy: ChecksumPolicyOption = "auto",
     max_checksum_bytes: MaxChecksumBytesOption = 10 * 1024**3,
     checksum_values: ChecksumValuesOption = None,
@@ -346,17 +357,6 @@ def annotate_directory_command(
         directory_answers.target if directory_answers is not None else None,
         expected_kind="directory",
     )
-    if not is_interactive and directory_answers is None:
-        _error("--no-interactive requires --answers")
-    if not is_interactive and directory_answers is not None:
-        try:
-            answer_files.require_complete_directory_answers(
-                directory_answers,
-                target=output_dir,
-            )
-        except answer_files.AnswersError as exc:
-            _error(str(exc))
-
     discovered_files, discovered_child_bundle_annotations = _discover_directory_entries(
         output_dir,
         recursive=recursive,
@@ -382,6 +382,21 @@ def annotate_directory_command(
             or directory_answers.child_bundles
         )
     )
+    has_cli_inventory = include_discovered or bool(artifact_groups)
+    if not is_interactive:
+        if directory_answers is not None and not has_cli_inventory:
+            try:
+                answer_files.require_complete_directory_answers(
+                    directory_answers,
+                    target=output_dir,
+                )
+            except answer_files.AnswersError as exc:
+                _error(str(exc))
+        elif not has_answers_inventory and not has_cli_inventory:
+            _error(
+                "--no-interactive directory annotation requires "
+                "--include-discovered, --group-selector, or --answers inventory"
+            )
     if (
         not has_answers_inventory
         and not discovered_files
@@ -442,6 +457,7 @@ def annotate_directory_command(
         source_path=source_path,
         source_revision=source_revision,
         source_sha256=source_sha256,
+        infer_from_runtime=infer_from_runtime,
     )
 
     default_kind = _validate_artifact_kind(kind) if kind is not None else None
@@ -460,6 +476,46 @@ def annotate_directory_command(
             output_dir,
             directory_answers.child_bundles,
         )
+        if include_discovered:
+            excluded_relative_paths = {artifact.path for artifact in artifacts} | {
+                relative_path
+                for group in artifact_groups
+                for relative_path in group.paths
+            }
+            artifacts.extend(
+                _documented_artifacts_from_discovered_files(
+                    output_dir,
+                    discovered_files=discovered_files,
+                    artifact_kind=default_kind or "other",
+                    excluded_relative_paths=excluded_relative_paths,
+                )
+            )
+            known_child_annotation_paths = {
+                child_bundle.annotation_path for child_bundle in child_bundles
+            }
+            child_bundles.extend(
+                _child_bundle_from_annotation(output_dir, annotation_path)
+                for annotation_path in discovered_child_bundle_annotations
+                if annotation_path.relative_to(output_dir).as_posix()
+                not in known_child_annotation_paths
+            )
+    elif include_discovered:
+        grouped_relative_paths = {
+            relative_path for group in artifact_groups for relative_path in group.paths
+        }
+        artifacts = _documented_artifacts_from_discovered_files(
+            output_dir,
+            discovered_files=discovered_files,
+            artifact_kind=default_kind or "other",
+            excluded_relative_paths=grouped_relative_paths,
+        )
+        child_bundles = [
+            _child_bundle_from_annotation(output_dir, annotation_path)
+            for annotation_path in discovered_child_bundle_annotations
+        ]
+    elif not is_interactive:
+        artifacts = []
+        child_bundles = []
     else:
         artifacts = []
         grouped_relative_paths = {

{data_annotations-2.8.1 → data_annotations-2.9.0}/src/data_annotations/cli_app/annotate/helpers.py

@@ -3,6 +3,10 @@ from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
+from data_annotations.annotations.answers import (
+    _expand_runtime_inference_fields,
+    _validate_runtime_inference_fields,
+)
 from data_annotations.annotations import (
     write_directory_annotation,
     write_file_annotation,
@@ -268,6 +272,7 @@ def _selected_git_tags(
 def _selected_source_code(
     *,
     answers: answer_files.BaseAnswers | None,
+    inferred_fields: set[str],
     source_kind: str | None,
     source_uri: str | None,
     source_download_uri: str | None,
@@ -284,10 +289,7 @@ def _selected_source_code(
         source_sha256,
     ]
     if not any(value is not None for value in source_values):
-        if (
-            answers is not None
-            and "source_code" in answers.provenance.runtime_inference_fields()
-        ):
+        if "source_code" in inferred_fields:
             return None
         return answers.provenance.source_code if answers is not None else None
 
@@ -308,19 +310,27 @@ def _selected_source_code(
     )
 
 
-def _runtime_inferred_fields(answers: answer_files.BaseAnswers | None) -> set[str]:
-    if answers is None:
-        return set()
-    return answers.provenance.runtime_inference_fields()
+def _runtime_inferred_fields(
+    answers: answer_files.BaseAnswers | None,
+    infer_from_runtime: list[str] | None,
+) -> set[str]:
+    selected_fields: list[str] = []
+    if answers is not None:
+        selected_fields.extend(answers.provenance.infer_from_runtime)
+    selected_fields.extend(infer_from_runtime or [])
+    try:
+        normalized = _validate_runtime_inference_fields(selected_fields)
+    except ValueError as exc:
+        _error(str(exc))
+    return _expand_runtime_inference_fields(normalized)
 
 
 def _filter_runtime_inferred_overrides(
     overrides: dict[str, Any],
     *,
-    answers: answer_files.BaseAnswers | None,
+    inferred_fields: set[str],
     explicit_fields: set[str],
 ) -> dict[str, Any]:
-    inferred_fields = _runtime_inferred_fields(answers)
     if not inferred_fields:
         return overrides
     if "source_code" in inferred_fields:
@@ -409,10 +419,12 @@ def _collect_post_hoc_provenance_from_sources(
     source_path: str | None,
     source_revision: str | None,
     source_sha256: str | None,
+    infer_from_runtime: list[str] | None,
 ) -> tuple[list[str], dict[str, Any], dict[str, Any]]:
     selected_inputs = _selected_inputs(input_values, answers)
     selected_params = _selected_params(param_values, answers)
     answer_command_tokens = _command_tokens_from_answers(answers)
+    inferred_fields = _runtime_inferred_fields(answers, infer_from_runtime)
     source_code_cli_values = {
         source_kind,
         source_uri,
@@ -451,6 +463,7 @@ def _collect_post_hoc_provenance_from_sources(
     selected_git_tags = _selected_git_tags(git_tags, answers)
     selected_source_code = _selected_source_code(
         answers=answers,
+        inferred_fields=inferred_fields,
         source_kind=source_kind,
         source_uri=source_uri,
         source_download_uri=source_download_uri,
@@ -505,7 +518,7 @@ def _collect_post_hoc_provenance_from_sources(
             params,
             _filter_runtime_inferred_overrides(
                 overrides,
-                answers=answers,
+                inferred_fields=inferred_fields,
                 explicit_fields=explicit_override_fields,
             ),
         )
@@ -553,12 +566,35 @@ def _collect_post_hoc_provenance_from_sources(
         selected_params or {},
         _filter_runtime_inferred_overrides(
             overrides,
-            answers=answers,
+            inferred_fields=inferred_fields,
             explicit_fields=explicit_override_fields,
         ),
     )
 
 
+def _documented_artifacts_from_discovered_files(
+    output_dir: Path,
+    *,
+    discovered_files: list[Path],
+    artifact_kind: ArtifactKind,
+    excluded_relative_paths: set[str],
+) -> list[DocumentedArtifact]:
+    artifacts: list[DocumentedArtifact] = []
+    for discovered_file in discovered_files:
+        relative_path = discovered_file.relative_to(output_dir).as_posix()
+        if relative_path in excluded_relative_paths:
+            continue
+        artifacts.append(
+            DocumentedArtifact(
+                path=relative_path,
+                kind=artifact_kind,
+                title=relative_path,
+                summary=None,
+            )
+        )
+    return artifacts
+
+
 def _documented_artifacts_from_answers(
     artifacts: list[answer_files.DirectoryArtifactAnswers],
 ) -> list[DocumentedArtifact]:

{data_annotations-2.8.1 → data_annotations-2.9.0}/src/data_annotations/cli_app/common.py

@@ -109,6 +109,12 @@ SOURCE_SHA256_OPTION = typer.Option(
     "--source-sha256",
     help="SHA-256 checksum for downloaded source archives or files.",
 )
+RUNTIME_INFERENCE_OPTION = typer.Option(
+    "--infer-from-runtime",
+    help=(
+        "Runtime provenance field or group to infer instead of overriding. Repeatable."
+    ),
+)
 CHECKSUM_POLICY_OPTION = typer.Option(
     "--checksum-policy",
     help="Checksum behavior for local files: always, auto, or never.",
@@ -150,6 +156,7 @@ SourceDownloadUriOption = Annotated[str | None, SOURCE_DOWNLOAD_URI_OPTION]
 SourcePathOption = Annotated[str | None, SOURCE_PATH_OPTION]
 SourceRevisionOption = Annotated[str | None, SOURCE_REVISION_OPTION]
 SourceSha256Option = Annotated[str | None, SOURCE_SHA256_OPTION]
+RuntimeInferenceValuesOption = Annotated[list[str] | None, RUNTIME_INFERENCE_OPTION]
 ChecksumPolicyOption = Annotated[str, CHECKSUM_POLICY_OPTION]
 MaxChecksumBytesOption = Annotated[int, MAX_CHECKSUM_BYTES_OPTION]
 Sha256Option = Annotated[str | None, SHA256_OPTION]