data-annotations 2.1.2__py3-none-any.whl

data_annotations/annotations/writers.py

@@ -0,0 +1,368 @@
+from pathlib import Path
+from typing import Any, Callable
+
+from data_annotations.description import (
+    ArtifactDescription,
+    DirectoryDescription,
+    DocumentedArtifact,
+    FieldDefinition,
+    FileDescription,
+    write_directory_readme,
+    write_file_readme,
+)
+from data_annotations.provenance import ArtifactKind, BaseProvenance, ProducedFile
+from data_annotations.provenance import writers as provenance_writers
+
+from .models import (
+    DirectoryAnnotationDocument,
+    DirectoryAnnotationResult,
+    DirectoryArtifactSubject,
+    FileAnnotationDocument,
+    FileAnnotationResult,
+    FileArtifactSubject,
+)
+
+
+def _validated_file_readme_fields(
+    *, title: str | None, summary: str | None
+) -> tuple[str, str]:
+    if title is None:
+        raise ValueError("title is required when write_readme=True")
+    if summary is None:
+        raise ValueError("summary is required when write_readme=True")
+    return title, summary
+
+
+def _validated_directory_readme_fields(
+    *, title: str | None, summary: str | None
+) -> tuple[str, str]:
+    if title is None:
+        raise ValueError("title is required when write_readme=True")
+    if summary is None:
+        raise ValueError("summary is required when write_readme=True")
+    return title, summary
+
+
+def _base_provenance_from_manifest(
+    manifest: Any,
+    *,
+    excluded_fields: set[str],
+) -> BaseProvenance:
+    return BaseProvenance.model_validate(
+        manifest.model_dump(exclude=excluded_fields),
+    )
+
+
+def _artifact_description(artifact: DocumentedArtifact) -> ArtifactDescription:
+    fields = [FieldDefinition.model_validate(field) for field in artifact.fields]
+    return ArtifactDescription(
+        path=str(Path(artifact.path)),
+        title=artifact.title,
+        summary=artifact.summary,
+        fields=fields,
+        primary_key=list(artifact.primary_key),
+        missing_value_codes=dict(artifact.missing_value_codes),
+    )
+
+
+def _to_produced_file(artifact: DocumentedArtifact) -> ProducedFile:
+    path = Path(artifact.path)
+    return ProducedFile(
+        path=str(path),
+        kind=artifact.kind,
+        sha256=provenance_writers.sha256_file(path) if path.exists() else None,
+    )
+
+
+def _coerce_fields(
+    fields: list[FieldDefinition] | None,
+) -> list[FieldDefinition]:
+    return [FieldDefinition.model_validate(field) for field in (fields or [])]
+
+
+def _build_file_annotation_document(
+    artifact_path: str | Path,
+    *,
+    title: str | None = None,
+    summary: str | None = None,
+    fields: list[FieldDefinition] | None = None,
+    primary_key: list[str] | None = None,
+    missing_value_codes: dict[str, str] | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    artifact_kind: ArtifactKind = "other",
+    params: dict[str, Any] | None = None,
+    inputs: list[str] | None = None,
+    function: Callable[..., Any] | None = None,
+    capture_mode: str = "runtime",
+    provenance_overrides: dict[str, Any] | None = None,
+    normalize_inputs: bool = True,
+) -> FileAnnotationDocument:
+    manifest = provenance_writers._build_file_manifest(
+        artifact_path,
+        artifact_kind=artifact_kind,
+        params=params,
+        inputs=inputs,
+        function=function,
+        capture_mode=capture_mode,
+        overrides=provenance_overrides,
+        normalize_inputs=normalize_inputs,
+    )
+
+    return FileAnnotationDocument(
+        subject=FileArtifactSubject(
+            path=manifest.artifact_path,
+            kind=manifest.artifact_kind,
+            sha256=manifest.artifact_sha256,
+        ),
+        provenance=_base_provenance_from_manifest(
+            manifest,
+            excluded_fields={"artifact_path", "artifact_kind", "artifact_sha256"},
+        ),
+        description=FileDescription(
+            title=title,
+            summary=summary,
+            fields=_coerce_fields(fields),
+            primary_key=primary_key or [],
+            missing_value_codes=missing_value_codes or {},
+            acquisition_context=acquisition_context or {},
+            generation_context=generation_context or {},
+            description_updated_at=manifest.created_at,
+        ),
+    )
+
+
+def _build_directory_annotation_document(
+    output_dir: str | Path,
+    *,
+    artifacts: list[DocumentedArtifact],
+    title: str | None = None,
+    summary: str | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    params: dict[str, Any] | None = None,
+    inputs: list[str] | None = None,
+    function: Callable[..., Any] | None = None,
+    capture_mode: str = "runtime",
+    provenance_overrides: dict[str, Any] | None = None,
+    normalize_inputs: bool = True,
+) -> DirectoryAnnotationDocument:
+    produced_files = [_to_produced_file(artifact) for artifact in artifacts]
+    manifest = provenance_writers._build_directory_manifest(
+        output_dir,
+        produced_files=produced_files,
+        params=params,
+        inputs=inputs,
+        function=function,
+        capture_mode=capture_mode,
+        overrides=provenance_overrides,
+        normalize_inputs=normalize_inputs,
+    )
+
+    return DirectoryAnnotationDocument(
+        subject=DirectoryArtifactSubject(
+            path=manifest.output_dir,
+            produced_files=list(manifest.produced_files),
+        ),
+        provenance=_base_provenance_from_manifest(
+            manifest,
+            excluded_fields={"output_dir", "produced_files"},
+        ),
+        description=DirectoryDescription(
+            title=title,
+            summary=summary,
+            artifacts=[_artifact_description(artifact) for artifact in artifacts],
+            acquisition_context=acquisition_context or {},
+            generation_context=generation_context or {},
+            description_updated_at=manifest.created_at,
+        ),
+    )
+
+
+def _write_annotation_document(
+    document: FileAnnotationDocument | DirectoryAnnotationDocument,
+    path: Path,
+) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(document.model_dump_json(indent=2), encoding="utf-8")
+    return path
+
+
+def write_file_annotation(
+    artifact_path: str | Path,
+    *,
+    title: str | None = None,
+    summary: str | None = None,
+    fields: list[FieldDefinition] | None = None,
+    primary_key: list[str] | None = None,
+    missing_value_codes: dict[str, str] | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    artifact_kind: ArtifactKind = "other",
+    params: dict[str, Any] | None = None,
+    inputs: list[str] | None = None,
+    function: Callable[..., Any] | None = None,
+    capture_mode: str = "runtime",
+    provenance_overrides: dict[str, Any] | None = None,
+    normalize_inputs: bool = True,
+    suffix: str = ".meta.json",
+) -> Path:
+    document = _build_file_annotation_document(
+        artifact_path,
+        title=title,
+        summary=summary,
+        fields=fields,
+        primary_key=primary_key,
+        missing_value_codes=missing_value_codes,
+        acquisition_context=acquisition_context,
+        generation_context=generation_context,
+        artifact_kind=artifact_kind,
+        params=params,
+        inputs=inputs,
+        function=function,
+        capture_mode=capture_mode,
+        provenance_overrides=provenance_overrides,
+        normalize_inputs=normalize_inputs,
+    )
+    annotation_path = Path(str(document.subject.path) + suffix)
+    return _write_annotation_document(document, annotation_path)
+
+
+def write_directory_annotation(
+    output_dir: str | Path,
+    *,
+    artifacts: list[DocumentedArtifact],
+    title: str | None = None,
+    summary: str | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    params: dict[str, Any] | None = None,
+    inputs: list[str] | None = None,
+    function: Callable[..., Any] | None = None,
+    capture_mode: str = "runtime",
+    provenance_overrides: dict[str, Any] | None = None,
+    normalize_inputs: bool = True,
+    filename: str = "manifest.json",
+) -> Path:
+    document = _build_directory_annotation_document(
+        output_dir,
+        artifacts=artifacts,
+        title=title,
+        summary=summary,
+        acquisition_context=acquisition_context,
+        generation_context=generation_context,
+        params=params,
+        inputs=inputs,
+        function=function,
+        capture_mode=capture_mode,
+        provenance_overrides=provenance_overrides,
+        normalize_inputs=normalize_inputs,
+    )
+    annotation_path = Path(document.subject.path) / filename
+    return _write_annotation_document(document, annotation_path)
+
+
+def annotate_file(
+    artifact_path: str | Path,
+    *,
+    title: str | None = None,
+    summary: str | None = None,
+    fields: list[FieldDefinition] | None = None,
+    primary_key: list[str] | None = None,
+    missing_value_codes: dict[str, str] | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    artifact_kind: ArtifactKind = "other",
+    params: dict[str, Any] | None = None,
+    inputs: list[str] | None = None,
+    function: Callable[..., Any] | None = None,
+    write_readme: bool = True,
+    write_schema: bool | None = None,
+    annotation_suffix: str = ".meta.json",
+    readme_suffix: str = ".README.md",
+) -> FileAnnotationResult:
+    document = _build_file_annotation_document(
+        artifact_path,
+        title=title,
+        summary=summary,
+        fields=fields,
+        primary_key=primary_key,
+        missing_value_codes=missing_value_codes,
+        acquisition_context=acquisition_context,
+        generation_context=generation_context,
+        artifact_kind=artifact_kind,
+        params=params,
+        inputs=inputs,
+        function=function,
+    )
+    artifact_path = Path(document.subject.path)
+    annotation_path = _write_annotation_document(
+        document,
+        Path(str(artifact_path) + annotation_suffix),
+    )
+
+    readme_path: Path | None = None
+    if write_readme:
+        _validated_file_readme_fields(title=title, summary=summary)
+        readme_path = write_file_readme(
+            Path(str(artifact_path) + readme_suffix),
+            artifact_path=document.subject.path,
+            artifact_kind=document.subject.kind,
+            description=document.description,
+        )
+
+    return FileAnnotationResult(
+        artifact_path=artifact_path,
+        annotation_path=annotation_path,
+        readme_path=readme_path,
+    )
+
+
+def annotate_directory(
+    output_dir: str | Path,
+    *,
+    artifacts: list[DocumentedArtifact],
+    title: str | None = None,
+    summary: str | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    params: dict[str, Any] | None = None,
+    inputs: list[str] | None = None,
+    function: Callable[..., Any] | None = None,
+    write_readme: bool = True,
+    write_schema: bool | None = None,
+    annotation_filename: str = "manifest.json",
+    readme_filename: str = "README.md",
+) -> DirectoryAnnotationResult:
+    document = _build_directory_annotation_document(
+        output_dir,
+        artifacts=artifacts,
+        title=title,
+        summary=summary,
+        acquisition_context=acquisition_context,
+        generation_context=generation_context,
+        params=params,
+        inputs=inputs,
+        function=function,
+    )
+    output_dir = Path(document.subject.path)
+    annotation_path = _write_annotation_document(
+        document,
+        output_dir / annotation_filename,
+    )
+
+    readme_path: Path | None = None
+    if write_readme:
+        _validated_directory_readme_fields(title=title, summary=summary)
+        readme_path = write_directory_readme(
+            output_dir / readme_filename,
+            output_dir=document.subject.path,
+            produced_files=document.subject.produced_files,
+            description=document.description,
+        )
+
+    return DirectoryAnnotationResult(
+        output_dir=output_dir,
+        annotation_path=annotation_path,
+        readme_path=readme_path,
+    )

data_annotations/cli.py

@@ -0,0 +1,37 @@
+from typing import Any
+
+_CLI_IMPORT_ERROR: ModuleNotFoundError | None = None
+_CLI_OPTIONAL_DEPENDENCIES = {"questionary", "typer"}
+app: Any = None
+
+try:
+    import typer
+
+    from data_annotations.cli_app.annotate import annotate_app
+    from data_annotations.cli_app.provenance_commands import provenance_app
+except ModuleNotFoundError as exc:
+    if exc.name not in _CLI_OPTIONAL_DEPENDENCIES:
+        raise
+    _CLI_IMPORT_ERROR = exc
+else:
+    app = typer.Typer(no_args_is_help=True)
+    app.add_typer(annotate_app, name="annotate")
+    app.add_typer(provenance_app, name="provenance")
+
+
+def main() -> None:
+    if _CLI_IMPORT_ERROR is not None:
+        missing_dependency = _CLI_IMPORT_ERROR.name or "an optional CLI dependency"
+        raise SystemExit(
+            "The data-annotations command requires the optional CLI dependencies.\n"
+            f"Missing dependency: {missing_dependency}\n\n"
+            "Install them with one of:\n"
+            '  pip install "data-annotations[cli]"\n'
+            '  uv tool install "data-annotations[cli]"\n'
+            '  uv add "data-annotations[cli]"'
+        )
+    app()
+
+
+if __name__ == "__main__":
+    main()

data_annotations/cli_app/__init__.py

@@ -0,0 +1 @@
+"""CLI command modules for data-annotations."""

data_annotations/cli_app/annotate.py

@@ -0,0 +1,307 @@
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from data_annotations.annotations import (
+    write_directory_annotation,
+    write_file_annotation,
+)
+from data_annotations.description import (
+    ArtifactKind,
+    DocumentedArtifact,
+    FieldDefinition,
+    write_directory_readme,
+    write_file_readme,
+)
+from data_annotations.provenance import recovery as provenance_recovery
+
+from . import prompts
+from .common import (
+    CommandOption,
+    ForceOption,
+    FunctionOption,
+    GitBranchOption,
+    GitDirtyOption,
+    GitRemoteNameOption,
+    GitRemoteUrlOption,
+    GitShaOption,
+    InputValuesOption,
+    ParamValuesOption,
+    ScriptOption,
+    ScriptRepoPathOption,
+    _annotation_paths_for_directory,
+    _annotation_paths_for_file,
+    _discover_directory_files,
+    _ensure_annotation_outputs_available,
+    _error,
+    _resolved_path,
+    _validate_artifact_kind,
+)
+
+annotate_app = typer.Typer(
+    no_args_is_help=True,
+    help="Create unified annotation sidecars for existing artifacts.",
+)
+
+
+def _write_post_hoc_file_bundle(
+    artifact_path: Path,
+    *,
+    title: str,
+    summary: str,
+    artifact_kind: ArtifactKind,
+    fields: list[FieldDefinition],
+    primary_key: list[str],
+    missing_value_codes: dict[str, str],
+    inputs: list[str],
+    params: dict[str, Any],
+    provenance_overrides: dict[str, Any],
+) -> tuple[Path, Path]:
+    annotation_path, readme_path = _annotation_paths_for_file(artifact_path)
+    write_file_annotation(
+        artifact_path,
+        title=title,
+        summary=summary,
+        fields=fields,
+        primary_key=primary_key,
+        missing_value_codes=missing_value_codes,
+        artifact_kind=artifact_kind,
+        params=params,
+        inputs=inputs,
+        capture_mode="post_hoc",
+        provenance_overrides=provenance_overrides,
+        normalize_inputs=False,
+    )
+    document = provenance_recovery._load_annotation_document(annotation_path)
+    write_file_readme(
+        readme_path,
+        artifact_path=document.subject.path,
+        artifact_kind=document.subject.kind,
+        description=document.description,
+    )
+    return annotation_path, readme_path
+
+
+def _write_post_hoc_directory_bundle(
+    output_dir: Path,
+    *,
+    title: str,
+    summary: str,
+    artifacts: list[DocumentedArtifact],
+    inputs: list[str],
+    params: dict[str, Any],
+    provenance_overrides: dict[str, Any],
+) -> tuple[Path, Path]:
+    annotation_path, readme_path = _annotation_paths_for_directory(output_dir)
+    write_directory_annotation(
+        output_dir,
+        artifacts=artifacts,
+        title=title,
+        summary=summary,
+        params=params,
+        inputs=inputs,
+        capture_mode="post_hoc",
+        provenance_overrides=provenance_overrides,
+        normalize_inputs=False,
+    )
+    document = provenance_recovery._load_annotation_document(annotation_path)
+    write_directory_readme(
+        readme_path,
+        output_dir=document.subject.path,
+        produced_files=document.subject.produced_files,
+        description=document.description,
+    )
+    return annotation_path, readme_path
+
+
+@annotate_app.command("file")
+def annotate_file_command(
+    target: Path = typer.Argument(..., help="Existing file to annotate."),
+    title: str | None = typer.Option(None, "--title", help="Human-readable title."),
+    summary: str | None = typer.Option(
+        None, "--summary", help="Human-readable summary."
+    ),
+    kind: str | None = typer.Option(None, "--kind", help="Artifact kind."),
+    input_values: InputValuesOption = None,
+    param_values: ParamValuesOption = None,
+    script: ScriptOption = None,
+    script_repo_path: ScriptRepoPathOption = None,
+    command: CommandOption = None,
+    function: FunctionOption = None,
+    git_sha: GitShaOption = None,
+    git_branch: GitBranchOption = None,
+    git_remote_name: GitRemoteNameOption = None,
+    git_remote_url: GitRemoteUrlOption = None,
+    git_dirty: GitDirtyOption = None,
+    force: ForceOption = False,
+) -> None:
+    artifact_path = _resolved_path(target)
+    if not artifact_path.exists():
+        _error(f"target does not exist: {artifact_path}")
+    if not artifact_path.is_file():
+        _error(f"target is not a file: {artifact_path}")
+
+    _ensure_annotation_outputs_available(
+        _annotation_paths_for_file(artifact_path),
+        force=force,
+    )
+
+    typer.echo("\nGeneral questions: optional fields may be skipped.")
+
+    artifact_kind = (
+        _validate_artifact_kind(kind)
+        if kind is not None
+        else prompts._prompt_artifact_kind(
+            "Artifact kind",
+            default="other",
+        )
+    )
+    title_value = title or prompts._prompt_required_text(
+        "Title", default=artifact_path.stem
+    )
+    summary_value = summary or prompts._prompt_required_text("Summary")
+    inputs, params, provenance_overrides = prompts._collect_post_hoc_provenance(
+        input_values=input_values,
+        param_values=param_values,
+        script=script,
+        script_repo_path=script_repo_path,
+        command=command,
+        function=function,
+        git_sha=git_sha,
+        git_branch=git_branch,
+        git_remote_name=git_remote_name,
+        git_remote_url=git_remote_url,
+        git_dirty=git_dirty,
+    )
+
+    fields: list[FieldDefinition] = []
+    primary_key: list[str] = []
+    missing_value_codes: dict[str, str] = {}
+    if artifact_kind in {"dataset", "table"}:
+        fields, primary_key, missing_value_codes = prompts._prompt_schema_details()
+
+    annotation_path, readme_path = _write_post_hoc_file_bundle(
+        artifact_path,
+        title=title_value,
+        summary=summary_value,
+        artifact_kind=artifact_kind,
+        fields=fields,
+        primary_key=primary_key,
+        missing_value_codes=missing_value_codes,
+        inputs=inputs,
+        params=params,
+        provenance_overrides=provenance_overrides,
+    )
+    typer.echo("")
+    typer.echo(f"Annotation: {annotation_path}")
+    typer.echo(f"README: {readme_path}")
+
+
+@annotate_app.command("directory")
+def annotate_directory_command(
+    target: Path = typer.Argument(..., help="Existing directory to annotate."),
+    title: str | None = typer.Option(None, "--title", help="Directory title."),
+    summary: str | None = typer.Option(None, "--summary", help="Directory summary."),
+    kind: str | None = typer.Option(
+        None,
+        "--kind",
+        help="Default artifact kind to apply to all included files.",
+    ),
+    input_values: InputValuesOption = None,
+    param_values: ParamValuesOption = None,
+    script: ScriptOption = None,
+    script_repo_path: ScriptRepoPathOption = None,
+    command: CommandOption = None,
+    function: FunctionOption = None,
+    git_sha: GitShaOption = None,
+    git_branch: GitBranchOption = None,
+    git_remote_name: GitRemoteNameOption = None,
+    git_remote_url: GitRemoteUrlOption = None,
+    git_dirty: GitDirtyOption = None,
+    force: ForceOption = False,
+) -> None:
+    output_dir = _resolved_path(target)
+    if not output_dir.exists():
+        _error(f"target does not exist: {output_dir}")
+    if not output_dir.is_dir():
+        _error(f"target is not a directory: {output_dir}")
+
+    discovered_files = _discover_directory_files(output_dir)
+    if not discovered_files:
+        _error(
+            "no immediate child files found in directory; nested files are intentionally ignored and should be annotated from their own subdirectory",
+            code=1,
+        )
+
+    _ensure_annotation_outputs_available(
+        _annotation_paths_for_directory(output_dir),
+        force=force,
+    )
+
+    title_value = title or prompts._prompt_required_text(
+        "Directory title", default=output_dir.name
+    )
+    summary_value = summary or prompts._prompt_required_text("Directory summary")
+    inputs, params, provenance_overrides = prompts._collect_post_hoc_provenance(
+        input_values=input_values,
+        param_values=param_values,
+        script=script,
+        script_repo_path=script_repo_path,
+        command=command,
+        function=function,
+        git_sha=git_sha,
+        git_branch=git_branch,
+        git_remote_name=git_remote_name,
+        git_remote_url=git_remote_url,
+        git_dirty=git_dirty,
+    )
+
+    default_kind = _validate_artifact_kind(kind) if kind is not None else None
+    artifacts: list[DocumentedArtifact] = []
+    selected_files = prompts._prompt_directory_file_selection(discovered_files)
+    for candidate in selected_files:
+        artifact_kind = (
+            default_kind
+            if default_kind is not None
+            else prompts._prompt_artifact_kind(
+                f"Artifact kind for {candidate.name}",
+                default="other",
+            )
+        )
+        artifact_title = prompts._prompt_required_text(
+            f"Title for {candidate.name}",
+            default=candidate.name,
+        )
+        artifact_summary = prompts._prompt_optional_text(
+            f"Summary for {candidate.name}"
+        )
+        fields: list[FieldDefinition] = []
+        primary_key: list[str] = []
+        missing_value_codes: dict[str, str] = {}
+        if artifact_kind in {"dataset", "table"}:
+            fields, primary_key, missing_value_codes = prompts._prompt_schema_details()
+        artifacts.append(
+            DocumentedArtifact(
+                path=str(candidate),
+                kind=artifact_kind,
+                title=artifact_title,
+                summary=artifact_summary,
+                fields=fields,
+                primary_key=primary_key,
+                missing_value_codes=missing_value_codes,
+            )
+        )
+
+    annotation_path, readme_path = _write_post_hoc_directory_bundle(
+        output_dir,
+        title=title_value,
+        summary=summary_value,
+        artifacts=artifacts,
+        inputs=inputs,
+        params=params,
+        provenance_overrides=provenance_overrides,
+    )
+
+    typer.echo(f"Annotation: {annotation_path}")
+    typer.echo(f"README: {readme_path}")