data-annotations 2.1.2__py3-none-any.whl

data_annotations/description/decorators.py

@@ -0,0 +1,145 @@
+from datetime import datetime, timezone
+from functools import wraps
+from typing import Any, Callable
+
+from data_annotations._decorators import (
+    argument_path,
+    bind_arguments,
+    coerce_documented_artifacts,
+    coerce_produced_files,
+)
+from data_annotations.provenance.models import ArtifactKind
+
+from .models import (
+    ArtifactDescription,
+    DirectoryDescription,
+    DocumentedArtifact,
+    FieldDefinition,
+    FileDescription,
+)
+from .writers import write_directory_description, write_file_description
+
+
+def _coerce_fields(fields: list[FieldDefinition] | None) -> list[FieldDefinition]:
+    return [FieldDefinition.model_validate(field) for field in (fields or [])]
+
+
+def _artifact_description(artifact: DocumentedArtifact) -> ArtifactDescription:
+    return ArtifactDescription(
+        path=artifact.path,
+        title=artifact.title,
+        summary=artifact.summary,
+        fields=_coerce_fields(artifact.fields),
+        primary_key=list(artifact.primary_key),
+        missing_value_codes=dict(artifact.missing_value_codes),
+    )
+
+
+def _timestamp() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def record_file_description(
+    *,
+    artifact_path_arg: str = "artifact_path",
+    artifact_kind: ArtifactKind = "other",
+    title: str | None = None,
+    summary: str | None = None,
+    fields: list[FieldDefinition] | None = None,
+    primary_key: list[str] | None = None,
+    missing_value_codes: dict[str, str] | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    readme_suffix: str = ".README.md",
+):
+    """
+    Decorate a function that writes one described artifact.
+
+    Wrapped function contract:
+    - Accept a local output path argument, named ``artifact_path`` by default.
+    - The decorator writes the README sidecar from ``artifact_path``.
+    - The return value is not inspected and is returned unchanged.
+    """
+
+    def deco(fn: Callable[..., Any]):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            bound = bind_arguments(fn, args, kwargs)
+            result = fn(*args, **kwargs)
+
+            artifact_path = argument_path(bound, argument_name=artifact_path_arg)
+            description_model = FileDescription(
+                title=title,
+                summary=summary,
+                fields=_coerce_fields(fields),
+                primary_key=primary_key or [],
+                missing_value_codes=missing_value_codes or {},
+                acquisition_context=acquisition_context or {},
+                generation_context=generation_context or {},
+                description_updated_at=_timestamp(),
+            )
+            write_file_description(
+                str(artifact_path) + readme_suffix,
+                artifact_path=str(artifact_path),
+                artifact_kind=artifact_kind,
+                description=description_model,
+            )
+            return result
+
+        return wrapper
+
+    return deco
+
+
+def record_directory_description(
+    *,
+    output_dir_arg: str = "output_dir",
+    title: str | None = None,
+    summary: str | None = None,
+    acquisition_context: dict[str, Any] | None = None,
+    generation_context: dict[str, Any] | None = None,
+    readme_filename: str = "README.md",
+):
+    """
+    Decorate a function that writes several described outputs in a directory.
+
+    Wrapped function contract:
+    - Accept a local output directory argument, named ``output_dir`` by default.
+    - Return a materialized iterable, usually a ``list`` or ``tuple``.
+    - Supported return items are:
+      - DocumentedArtifact
+      - ProducedFile
+      - (path, kind)
+      - path-like objects (kind defaults to ``"other"``)
+    - The original return value is passed through unchanged.
+    """
+
+    def deco(fn: Callable[..., Any]):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            bound = bind_arguments(fn, args, kwargs)
+            result = fn(*args, **kwargs)
+
+            items = list(result)
+            output_dir = argument_path(bound, argument_name=output_dir_arg)
+            artifacts = coerce_documented_artifacts(items)
+            produced_files = coerce_produced_files(items)
+            description_model = DirectoryDescription(
+                title=title,
+                summary=summary,
+                artifacts=[_artifact_description(artifact) for artifact in artifacts],
+                acquisition_context=acquisition_context or {},
+                generation_context=generation_context or {},
+                description_updated_at=_timestamp(),
+            )
+            write_directory_description(
+                output_dir / readme_filename,
+                output_dir=str(output_dir),
+                produced_files=produced_files,
+                description=description_model,
+            )
+            return result
+
+        return wrapper
+
+    return deco

data_annotations/description/models.py

@@ -0,0 +1,63 @@
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from data_annotations.provenance.models import ArtifactKind
+
+
+class AllowedValue(BaseModel):
+    value: Any
+    label: str | None = None
+    summary: str | None = None
+
+
+class FieldDefinition(BaseModel):
+    name: str
+    data_type: str | None = None
+    summary: str
+    required: bool | None = None
+    nullable: bool | None = None
+    unit: str | None = None
+    example: Any | None = None
+    notes: str | None = None
+    allowed_values: list[AllowedValue] = Field(default_factory=list)
+
+
+class ArtifactDescription(BaseModel):
+    path: str
+    title: str | None = None
+    summary: str | None = None
+    fields: list[FieldDefinition] = Field(default_factory=list)
+    primary_key: list[str] = Field(default_factory=list)
+    missing_value_codes: dict[str, str] = Field(default_factory=dict)
+
+
+class DocumentedArtifact(BaseModel):
+    path: str
+    kind: ArtifactKind = "other"
+    title: str | None = None
+    summary: str | None = None
+    fields: list[FieldDefinition] = Field(default_factory=list)
+    primary_key: list[str] = Field(default_factory=list)
+    missing_value_codes: dict[str, str] = Field(default_factory=dict)
+
+
+class FileDescription(BaseModel):
+    title: str | None = None
+    summary: str | None = None
+    fields: list[FieldDefinition] = Field(default_factory=list)
+    primary_key: list[str] = Field(default_factory=list)
+    missing_value_codes: dict[str, str] = Field(default_factory=dict)
+    acquisition_context: dict[str, Any] = Field(default_factory=dict)
+    generation_context: dict[str, Any] = Field(default_factory=dict)
+    description_updated_at: datetime
+
+
+class DirectoryDescription(BaseModel):
+    title: str | None = None
+    summary: str | None = None
+    artifacts: list[ArtifactDescription] = Field(default_factory=list)
+    acquisition_context: dict[str, Any] = Field(default_factory=dict)
+    generation_context: dict[str, Any] = Field(default_factory=dict)
+    description_updated_at: datetime

data_annotations/description/writers.py

@@ -0,0 +1,321 @@
+from pathlib import Path
+from typing import Any
+
+from data_annotations.provenance.models import ArtifactKind, ProducedFile
+
+from .models import (
+    AllowedValue,
+    ArtifactDescription,
+    DirectoryDescription,
+    FieldDefinition,
+    FileDescription,
+)
+
+
+def _stringify(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, bool):
+        return "yes" if value else "no"
+    if isinstance(value, (list, tuple, set)):
+        return ", ".join(_stringify(item) for item in value)
+    if isinstance(value, dict):
+        return ", ".join(f"{key}={_stringify(item)}" for key, item in value.items())
+    return str(value)
+
+
+def _escape_table_cell(value: Any) -> str:
+    return _stringify(value).replace("|", "\\|").replace("\n", "<br>")
+
+
+def _summarize_allowed_value(value: AllowedValue) -> str:
+    label = f" ({value.label})" if value.label else ""
+    return f"{_stringify(value.value)}{label}"
+
+
+def _allowed_values_have_descriptions(values: list[AllowedValue]) -> bool:
+    return any(value.summary for value in values)
+
+
+def _render_allowed_values(values: list[AllowedValue]) -> list[str]:
+    lines: list[str] = []
+    for value in values:
+        prefix = f"- `{_stringify(value.value)}`"
+        if value.label:
+            prefix += f" ({value.label})"
+        if value.summary:
+            prefix += f": {value.summary}"
+        lines.append(prefix)
+    return lines
+
+
+def _render_context(context: dict[str, Any]) -> list[str]:
+    return [f"- {key}: {_stringify(value)}" for key, value in context.items()]
+
+
+def _use_field_table(fields: list[FieldDefinition]) -> bool:
+    return not any(
+        field.notes or _allowed_values_have_descriptions(field.allowed_values)
+        for field in fields
+    )
+
+
+def _render_field_table(fields: list[FieldDefinition]) -> list[str]:
+    lines = [
+        "| Name | Type | Summary | Required | Nullable | Unit | Example | Allowed Values |",
+        "| ---- | ---- | ------- | -------- | -------- | ---- | ------- | -------------- |",
+    ]
+    for field in fields:
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _escape_table_cell(field.name),
+                    _escape_table_cell(field.data_type),
+                    _escape_table_cell(field.summary),
+                    _escape_table_cell(field.required),
+                    _escape_table_cell(field.nullable),
+                    _escape_table_cell(field.unit),
+                    _escape_table_cell(field.example),
+                    _escape_table_cell(
+                        ", ".join(
+                            _summarize_allowed_value(value)
+                            for value in field.allowed_values
+                        )
+                    ),
+                ]
+            )
+            + " |"
+        )
+    return lines
+
+
+def _render_field_sections(
+    fields: list[FieldDefinition],
+    *,
+    heading_level: int,
+) -> list[str]:
+    lines: list[str] = []
+    heading_prefix = "#" * heading_level
+
+    for index, field in enumerate(fields):
+        if index > 0:
+            lines.append("")
+        lines.append(f"{heading_prefix} {field.name}")
+        lines.append("")
+        lines.append(f"- Summary: {field.summary}")
+        if field.data_type:
+            lines.append(f"- Type: `{field.data_type}`")
+        if field.required is not None:
+            lines.append(f"- Required: {_stringify(field.required)}")
+        if field.nullable is not None:
+            lines.append(f"- Nullable: {_stringify(field.nullable)}")
+        if field.unit:
+            lines.append(f"- Unit: `{field.unit}`")
+        if field.example is not None:
+            lines.append(f"- Example: `{_stringify(field.example)}`")
+        if field.notes:
+            lines.append(f"- Notes: {field.notes}")
+        if field.allowed_values:
+            lines.append("- Allowed values:")
+            lines.extend(_render_allowed_values(field.allowed_values))
+
+    return lines
+
+
+def _render_fields(fields: list[FieldDefinition], *, heading_level: int) -> list[str]:
+    if not fields:
+        return []
+    if _use_field_table(fields):
+        return _render_field_table(fields)
+    return _render_field_sections(fields, heading_level=heading_level)
+
+
+def render_file_readme(
+    *,
+    artifact_path: str,
+    artifact_kind: ArtifactKind,
+    description: FileDescription,
+) -> str:
+    title = description.title or Path(artifact_path).name
+    body = description.summary or "No summary provided."
+    lines = [
+        f"# {title}",
+        "",
+        body,
+        "",
+        "## Artifact",
+        "",
+        f"- Path: `{artifact_path}`",
+        f"- Kind: `{artifact_kind}`",
+    ]
+
+    if description.fields:
+        lines.extend(["", "## Fields", ""])
+        lines.extend(_render_fields(description.fields, heading_level=3))
+
+    if description.primary_key:
+        lines.extend(
+            [
+                "",
+                "## Keys",
+                "",
+                f"- Primary key: `{', '.join(description.primary_key)}`",
+            ]
+        )
+
+    if description.missing_value_codes:
+        codes = "; ".join(
+            f"`{code}` = {meaning}"
+            for code, meaning in description.missing_value_codes.items()
+        )
+        lines.extend(["", "## Missing Value Codes", "", f"- {codes}"])
+
+    if description.acquisition_context:
+        lines.extend(["", "## Acquisition Context", ""])
+        lines.extend(_render_context(description.acquisition_context))
+
+    if description.generation_context:
+        lines.extend(["", "## Generation Context", ""])
+        lines.extend(_render_context(description.generation_context))
+
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _artifact_kind_map(
+    produced_files: list[ProducedFile],
+) -> dict[str, ArtifactKind]:
+    return {item.path: item.kind for item in produced_files}
+
+
+def _render_artifact_summary(
+    artifact: ArtifactDescription,
+    *,
+    artifact_kind: ArtifactKind | None,
+) -> list[str]:
+    lines = [f"- Path: `{artifact.path}`"]
+    if artifact_kind is not None:
+        lines.append(f"- Kind: `{artifact_kind}`")
+    if artifact.summary:
+        lines.append(f"- Summary: {artifact.summary}")
+    if artifact.primary_key:
+        lines.append(f"- Primary key: `{', '.join(artifact.primary_key)}`")
+    if artifact.missing_value_codes:
+        codes = "; ".join(
+            f"`{code}` = {meaning}"
+            for code, meaning in artifact.missing_value_codes.items()
+        )
+        lines.append(f"- Missing value codes: {codes}")
+    return lines
+
+
+def render_directory_readme(
+    *,
+    output_dir: str,
+    produced_files: list[ProducedFile],
+    description: DirectoryDescription,
+) -> str:
+    title = description.title or Path(output_dir).name
+    body = description.summary or "No summary provided."
+    lines = [
+        f"# {title}",
+        "",
+        body,
+    ]
+    artifact_kinds = _artifact_kind_map(produced_files)
+
+    if description.artifacts:
+        lines.extend(["", "## Artifacts"])
+        for artifact in description.artifacts:
+            heading = artifact.title or Path(artifact.path).name
+            lines.extend(["", f"### {heading}", ""])
+            lines.extend(
+                _render_artifact_summary(
+                    artifact,
+                    artifact_kind=artifact_kinds.get(artifact.path),
+                )
+            )
+            if artifact.fields:
+                lines.extend(["", "#### Fields", ""])
+                lines.extend(_render_fields(artifact.fields, heading_level=5))
+
+    if description.acquisition_context:
+        lines.extend(["", "## Acquisition Context", ""])
+        lines.extend(_render_context(description.acquisition_context))
+
+    if description.generation_context:
+        lines.extend(["", "## Generation Context", ""])
+        lines.extend(_render_context(description.generation_context))
+
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def write_file_readme(
+    output_path: str | Path,
+    *,
+    artifact_path: str,
+    artifact_kind: ArtifactKind,
+    description: FileDescription,
+) -> Path:
+    output_path = Path(output_path)
+    output_path.write_text(
+        render_file_readme(
+            artifact_path=artifact_path,
+            artifact_kind=artifact_kind,
+            description=description,
+        ),
+        encoding="utf-8",
+    )
+    return output_path
+
+
+def write_directory_readme(
+    output_path: str | Path,
+    *,
+    output_dir: str,
+    produced_files: list[ProducedFile],
+    description: DirectoryDescription,
+) -> Path:
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(
+        render_directory_readme(
+            output_dir=output_dir,
+            produced_files=produced_files,
+            description=description,
+        ),
+        encoding="utf-8",
+    )
+    return output_path
+
+
+def write_file_description(
+    output_path: str | Path,
+    *,
+    artifact_path: str,
+    artifact_kind: ArtifactKind,
+    description: FileDescription,
+) -> Path:
+    return write_file_readme(
+        output_path,
+        artifact_path=artifact_path,
+        artifact_kind=artifact_kind,
+        description=description,
+    )
+
+
+def write_directory_description(
+    output_path: str | Path,
+    *,
+    output_dir: str,
+    produced_files: list[ProducedFile],
+    description: DirectoryDescription,
+) -> Path:
+    return write_directory_readme(
+        output_path,
+        output_dir=output_dir,
+        produced_files=produced_files,
+        description=description,
+    )

data_annotations/provenance/__init__.py

@@ -0,0 +1,37 @@
+from .models import (
+    ArtifactKind,
+    BaseProvenance,
+    DirectoryManifest,
+    FileManifest,
+    ProducedFile,
+    RecoveredSource,
+)
+from .decorators import record_directory_manifest, record_file_manifest
+from .git import capture_git_info
+from .recovery import artifact_matches_manifest, checkout_manifest_source
+from .runtime import capture_runtime_info
+from .writers import (
+    callable_name,
+    sha256_file,
+    write_directory_manifest,
+    write_file_manifest,
+)
+
+__all__ = [
+    "ArtifactKind",
+    "BaseProvenance",
+    "DirectoryManifest",
+    "FileManifest",
+    "ProducedFile",
+    "RecoveredSource",
+    "artifact_matches_manifest",
+    "callable_name",
+    "capture_git_info",
+    "capture_runtime_info",
+    "checkout_manifest_source",
+    "record_directory_manifest",
+    "record_file_manifest",
+    "sha256_file",
+    "write_directory_manifest",
+    "write_file_manifest",
+]

data_annotations/provenance/decorators.py

@@ -0,0 +1,111 @@
+from functools import wraps
+from typing import Any, Callable
+
+from data_annotations._decorators import (
+    DEFAULT_INPUT_ARGS,
+    argument_path,
+    bind_arguments,
+    coerce_produced_files,
+    extract_inputs,
+    extract_params,
+)
+
+from . import writers
+from .models import ArtifactKind
+
+
+def record_file_manifest(
+    *,
+    artifact_path_arg: str = "artifact_path",
+    input_args: tuple[str, ...] = DEFAULT_INPUT_ARGS,
+    artifact_kind: ArtifactKind = "other",
+    suffix: str = ".meta.json",
+):
+    """
+    Decorate a function that writes one artifact to ``artifact_path_arg``.
+
+    Wrapped function contract:
+    - Accept a local output path argument, named ``artifact_path`` by default.
+    - Any bound arguments named in ``input_args`` are recorded as provenance inputs.
+    - Remaining bound arguments become provenance params.
+    - The return value is not inspected and is returned unchanged.
+    """
+
+    def deco(fn: Callable[..., Any]):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            bound = bind_arguments(fn, args, kwargs)
+            result = fn(*args, **kwargs)
+
+            artifact_path = argument_path(bound, argument_name=artifact_path_arg)
+            params = extract_params(
+                bound,
+                target_args=(artifact_path_arg,),
+                input_args=input_args,
+            )
+            inputs = extract_inputs(bound, input_args=input_args)
+
+            writers.write_file_manifest(
+                artifact_path,
+                artifact_kind=artifact_kind,
+                params=params,
+                inputs=inputs,
+                function=fn,
+                suffix=suffix,
+            )
+            return result
+
+        return wrapper
+
+    return deco
+
+
+def record_directory_manifest(
+    *,
+    output_dir_arg: str = "output_dir",
+    input_args: tuple[str, ...] = DEFAULT_INPUT_ARGS,
+    manifest_name: str = "manifest.json",
+):
+    """
+    Decorate a function that writes several outputs inside ``output_dir_arg``.
+
+    Wrapped function contract:
+    - Accept a local output directory argument, named ``output_dir`` by default.
+    - Return a materialized iterable, usually a ``list`` or ``tuple``.
+    - Supported return items are:
+      - DocumentedArtifact
+      - ProducedFile
+      - (path, kind)
+      - path-like objects (kind defaults to "other")
+    - Any bound arguments named in ``input_args`` are recorded as provenance inputs.
+    - Remaining bound arguments become provenance params.
+    - The original return value is passed through unchanged.
+    """
+
+    def deco(fn: Callable[..., Any]):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            bound = bind_arguments(fn, args, kwargs)
+            result = fn(*args, **kwargs)
+            output_dir = argument_path(bound, argument_name=output_dir_arg)
+            produced_files = coerce_produced_files(result)
+            params = extract_params(
+                bound,
+                target_args=(output_dir_arg,),
+                input_args=input_args,
+            )
+            inputs = extract_inputs(bound, input_args=input_args)
+
+            writers.write_directory_manifest(
+                output_dir,
+                produced_files=produced_files,
+                params=params,
+                inputs=inputs,
+                function=fn,
+                filename=manifest_name,
+            )
+            return result
+
+        return wrapper
+
+    return deco