eggpipe 0.3.0__py3-none-any.whl

eggpipe/core/model.py

@@ -0,0 +1,308 @@
+"""Internal immutable core models."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import date, datetime
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    field_serializer,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
+
+PRIMITIVE_TYPES: frozenset[str] = frozenset(
+    {
+        "string",
+        "text",
+        "integer",
+        "bigint",
+        "boolean",
+        "float",
+        "date",
+        "datetime",
+        "uuid",
+        "json",
+    }
+)
+
+
+class _Unset:
+    """Sentinel for "value not set"."""
+
+    def __repr__(self) -> str:
+        return "UNSET"
+
+
+UNSET = _Unset()
+
+DefaultValue = str | int | float | bool | date | datetime | None
+
+
+@dataclass(frozen=True)
+class FormatInfo:
+    """Concrete format and version."""
+
+    name: str
+    version: str
+
+
+class MergePolicy(StrEnum):
+    """Merge policy for layer collections."""
+
+    KEYED = "keyed"
+    APPEND = "append"
+    UNION = "union"
+    ERROR = "error"
+
+
+@dataclass(frozen=True)
+class FormatSchema:
+    """Pydantic schemas for one layer and the final entity."""
+
+    layer_entity: type[BaseModel]
+    full_entity: type[BaseModel]
+
+
+class EntityIdentity(BaseModel):
+    """Minimal entity identity model for a delta layer."""
+
+    name: str = Field(..., pattern=r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class FieldIdentity(BaseModel):
+    """Minimal field identity model for a delta layer."""
+
+    name: str = Field(..., pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class Constraints(BaseModel):
+    min: int | float | date | datetime | None = None
+    max: int | float | date | datetime | None = None
+    min_length: int | None = None
+    max_length: int | None = None
+    pattern: str | None = None
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class FieldDef(BaseModel):
+    name: str = Field(..., pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+    type: str
+    description: str | None = None
+    required: bool = False
+    nullable: bool = False
+    default: DefaultValue | _Unset = UNSET
+    constraints: Constraints | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    source: Path | None = Field(default=None, exclude=True, repr=False)
+
+    model_config = ConfigDict(
+        extra="forbid",
+        frozen=True,
+        arbitrary_types_allowed=True,
+    )
+
+    @field_serializer("default")
+    def _serialize_default(self, value: object, _info: object) -> object:
+        # UNSET is not serialized to JSON as a value; model_serializer removes the key.
+        return None if value is UNSET else value
+
+    @model_validator(mode="after")
+    def _coerce_temporal_default(self) -> FieldDef:
+        if self.type == "date" and isinstance(self.default, str):
+            try:
+                return self.model_copy(update={"default": date.fromisoformat(self.default)})
+            except ValueError:
+                return self
+        if self.type == "datetime" and isinstance(self.default, str):
+            try:
+                return self.model_copy(
+                    update={"default": datetime.fromisoformat(self.default.replace("Z", "+00:00"))}
+                )
+            except ValueError:
+                return self
+        return self
+
+    @model_serializer(mode="wrap")
+    def _serialize(self, handler: Callable[[Any], dict[str, Any]]) -> dict[str, Any]:
+        data = handler(self)
+        if self.default is UNSET:
+            data.pop("default", None)
+        return data
+
+
+class ExtendsRef(BaseModel):
+    import_: str = Field(alias="import")
+    entity: str
+
+    model_config = ConfigDict(
+        extra="forbid",
+        frozen=True,
+        populate_by_name=True,
+    )
+
+
+class LayerEntity(EntityIdentity):
+    """Partial entity schema for a layer file."""
+
+    description: str | None = None
+    version: str | int | None = None
+    updated_at: date | datetime | None = None
+    extends: str | ExtendsRef | None = None
+    fields: tuple[FieldDef, ...] = ()
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class Entity(BaseModel):
+    name: str = Field(..., pattern=r"^[A-Za-z_][A-Za-z0-9_]*$")
+    description: str | None = None
+    version: str | int | None = None
+    updated_at: date | datetime | None = None
+    extends: str | ExtendsRef | None = None
+    fields: tuple[FieldDef, ...] = ()
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    source: Path | None = Field(default=None, exclude=True, repr=False)
+    format: str | None = Field(default=None, exclude=True, repr=False)
+    format_version: str | None = Field(default=None, exclude=True, repr=False)
+    layer_sources: tuple[Path, ...] = Field(default=(), exclude=True, repr=False)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    @field_validator("updated_at", mode="before")
+    @classmethod
+    def _coerce_updated_at(cls, value: object) -> object:
+        # Pydantic smart union parses midnight ISO-datetime as date; preserve the type.
+        if isinstance(value, str) and ("T" in value or " " in value):
+            try:
+                return datetime.fromisoformat(value.replace("Z", "+00:00"))
+            except ValueError:
+                return value
+        return value
+
+
+class EnumValue(BaseModel):
+    name: str = Field(..., pattern=r"^[A-Za-z_][A-Za-z0-9_]*$")
+    value: str | int | None = None
+    description: str | None = None
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class EnumDef(BaseModel):
+    name: str = Field(..., pattern=r"^[A-Za-z_][A-Za-z0-9_]*$")
+    description: str | None = None
+    values: tuple[EnumValue, ...] = ()
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class CustomType(BaseModel):
+    name: str = Field(..., pattern=r"^[A-Za-z_][A-Za-z0-9_]*$")
+    description: str | None = None
+    base: str
+    constraints: Constraints | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class _IndexedModel(BaseModel):
+    """Immutable model with name indexes."""
+
+    entities: tuple[Entity, ...] = ()
+    enums: tuple[EnumDef, ...] = ()
+    types: tuple[CustomType, ...] = ()
+
+    _entity_map: dict[str, Entity] = PrivateAttr(default_factory=dict)
+    _enum_map: dict[str, EnumDef] = PrivateAttr(default_factory=dict)
+    _type_map: dict[str, CustomType] = PrivateAttr(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    @model_validator(mode="after")
+    def _build_maps(self) -> _IndexedModel:
+        object.__setattr__(self, "_entity_map", {item.name: item for item in self.entities})
+        object.__setattr__(self, "_enum_map", {item.name: item for item in self.enums})
+        object.__setattr__(self, "_type_map", {item.name: item for item in self.types})
+        return self
+
+    def entity(self, name: str) -> Entity:
+        return self._entity_map[name]
+
+    def enum(self, name: str) -> EnumDef:
+        return self._enum_map[name]
+
+    def type(self, name: str) -> CustomType:
+        return self._type_map[name]
+
+    def resolve_type(self, field: FieldDef) -> Entity | EnumDef | CustomType | str:
+        if field.type in self._entity_map:
+            return self._entity_map[field.type]
+        if field.type in self._enum_map:
+            return self._enum_map[field.type]
+        if field.type in self._type_map:
+            return self._type_map[field.type]
+        return field.type
+
+
+class ModelSlice(_IndexedModel):
+    @property
+    def formats(self) -> frozenset[FormatInfo]:
+        return frozenset(
+            FormatInfo(entity.format, entity.format_version)
+            for entity in self.entities
+            if entity.format is not None and entity.format_version is not None
+        )
+
+
+class SourceModel(ModelSlice):
+    name: str | None = None
+    description: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class InputSelection(BaseModel):
+    """Model subset selection; None/empty means all."""
+
+    entities: tuple[str, ...] | None = None
+    enums: tuple[str, ...] | None = None
+    types: tuple[str, ...] | None = None
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+__all__ = [
+    "PRIMITIVE_TYPES",
+    "UNSET",
+    "CustomType",
+    "DefaultValue",
+    "Entity",
+    "EntityIdentity",
+    "EnumDef",
+    "EnumValue",
+    "ExtendsRef",
+    "FieldDef",
+    "FieldIdentity",
+    "FormatInfo",
+    "FormatSchema",
+    "InputSelection",
+    "LayerEntity",
+    "MergePolicy",
+    "ModelSlice",
+    "SourceModel",
+]

eggpipe/core/naming.py

@@ -0,0 +1,36 @@
+"""Naming utilities; renaming the model remains a plugin decision."""
+
+from __future__ import annotations
+
+import re
+from enum import StrEnum
+
+
+class NamingStyle(StrEnum):
+    SNAKE = "snake_case"
+    CAMEL = "camelCase"
+    PASCAL = "pascal_case"
+    KEBAB = "kebab-case"
+
+
+def transform(name: str, style: NamingStyle) -> str:
+    """Deterministically convert a name to the requested style."""
+    words = _split_words(name)
+    if not words:
+        return name
+    if style is NamingStyle.SNAKE:
+        return "_".join(word.lower() for word in words)
+    if style is NamingStyle.KEBAB:
+        return "-".join(word.lower() for word in words)
+    if style is NamingStyle.PASCAL:
+        return "".join(word[:1].upper() + word[1:].lower() for word in words)
+    return words[0].lower() + "".join(word[:1].upper() + word[1:].lower() for word in words[1:])
+
+
+def _split_words(name: str) -> list[str]:
+    value = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", name)
+    value = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1 \2", value)
+    return [part for part in re.split(r"[^A-Za-z0-9]+", value) if part]
+
+
+__all__ = ["NamingStyle", "transform"]

eggpipe/core/parser.py

@@ -0,0 +1,96 @@
+"""Model document parser."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from eggpipe.core.document import ModelDocument
+from eggpipe.core.format_registry import FormatRegistry
+from eggpipe.core.model import SourceModel
+from eggpipe.utils.exceptions import (
+    DocumentError,
+    UnknownFormatError,
+)
+from eggpipe.utils.yaml import YamlLoadError, load_yaml
+
+
+def normalize_version(value: object) -> str | None:
+    """Normalize version: strings as-is, numbers to a string with one decimal place."""
+    if isinstance(value, bool) or value is None:
+        return None
+    if isinstance(value, str):
+        return value
+    if isinstance(value, int):
+        return f"{value}.0"
+    if isinstance(value, float):
+        return f"{value:.1f}"
+    return None
+
+
+class Parser:
+    def __init__(self, registry: FormatRegistry | None = None) -> None:
+        self.registry = registry or FormatRegistry()
+
+    def parse_document(self, path: Path) -> ModelDocument:
+        try:
+            raw = load_yaml(path)
+        except YamlLoadError as exc:
+            raise DocumentError(exc.code, path, exc.message) from exc
+
+        format_name = raw.get("format")
+        if not isinstance(format_name, str) or not format_name:
+            raise DocumentError("EG007", path, "`format` is required")
+
+        try:
+            plugin = self.registry.get(format_name)
+        except UnknownFormatError as exc:
+            raise DocumentError("EG007", path, str(exc)) from exc
+
+        raw_version = raw.get("version")
+        if raw_version is None:
+            version = plugin.supported_versions[-1]
+        else:
+            version = normalize_version(raw_version)
+            if version is None:
+                raise DocumentError("EG008", path, "`version` must be a string or a number")
+            if version not in plugin.supported_versions:
+                raise DocumentError(
+                    "EG008",
+                    path,
+                    f"{format_name} does not support version {version}",
+                )
+
+        raw["format"] = format_name
+        raw["version"] = version
+        try:
+            document = plugin.parse(raw, base_path=path.parent)
+        except DocumentError:
+            raise
+        except ValidationError as exc:
+            raise DocumentError("EG028", path, str(exc)) from exc
+        model = SourceModel(
+            name=document.model.name,
+            description=document.model.description,
+            metadata=document.model.metadata,
+            entities=tuple(
+                entity.model_copy(
+                    update={
+                        "source": path,
+                        "format": format_name,
+                        "format_version": version,
+                        "fields": tuple(
+                            field.model_copy(update={"source": path}) for field in entity.fields
+                        ),
+                    }
+                )
+                for entity in document.model.entities
+            ),
+            enums=document.model.enums,
+            types=document.model.types,
+        )
+        return document.model_copy(update={"path": path, "model": model})
+
+
+__all__ = ["Parser", "normalize_version"]

eggpipe/core/paths.py

@@ -0,0 +1,12 @@
+"""Shared path filtering rules."""
+
+from __future__ import annotations
+
+IGNORED_PARTS = frozenset({".git", ".eggpipe", "__pycache__", ".venv", "node_modules"})
+
+
+def is_ignored_part(part: str) -> bool:
+    return part.startswith(".") or part in IGNORED_PARTS
+
+
+__all__ = ["IGNORED_PARTS", "is_ignored_part"]

eggpipe/core/provenance.py

@@ -0,0 +1,52 @@
+"""Source provenance: the last git commit of a file."""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Provenance:
+    path: Path
+    commit: str | None
+    date: datetime | None
+
+
+def git_provenance(path: Path) -> Provenance | None:
+    """Return commit/date for a file, or None when git/history is unavailable."""
+    resolved = path.resolve()
+    try:
+        result = subprocess.run(
+            [
+                "git",
+                "-C",
+                str(resolved.parent),
+                "log",
+                "-1",
+                "--format=%H%n%cI",
+                "--",
+                resolved.name,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=5,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if result.returncode != 0:
+        return None
+    lines = [line for line in result.stdout.splitlines() if line]
+    if len(lines) < 2:
+        return None
+    try:
+        date = datetime.fromisoformat(lines[1].replace("Z", "+00:00"))
+    except ValueError:
+        date = None
+    return Provenance(path=resolved, commit=lines[0], date=date)
+
+
+__all__ = ["Provenance", "git_provenance"]

eggpipe/core/slicing.py

@@ -0,0 +1,110 @@
+"""Build a closed model slice."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from eggpipe.core.model import (
+    Entity,
+    InputSelection,
+    ModelSlice,
+    SourceModel,
+)
+from eggpipe.interfaces.diagnostics import Diagnostic, error
+
+
+def select_slice(
+    model: SourceModel,
+    selection: InputSelection | None = None,
+) -> tuple[ModelSlice, list[Diagnostic]]:
+    """Select a subset and add the transitive closure over enums/types."""
+    if selection is None:
+        return (
+            ModelSlice(
+                entities=model.entities,
+                enums=model.enums,
+                types=model.types,
+            ),
+            [],
+        )
+
+    diagnostics: list[Diagnostic] = []
+    entity_names = _requested(
+        "entities",
+        [item.name for item in model.entities],
+        selection.entities,
+        diagnostics,
+    )
+    enum_names = _requested(
+        "enums",
+        [item.name for item in model.enums],
+        selection.enums,
+        diagnostics,
+    )
+    type_names = _requested(
+        "types",
+        [item.name for item in model.types],
+        selection.types,
+        diagnostics,
+    )
+
+    entities = tuple(
+        item for item in model.entities if entity_names is None or item.name in entity_names
+    )
+    enum_map = {item.name: item for item in model.enums}
+    type_map = {item.name: item for item in model.types}
+
+    selected_enums = {item.name for item in model.enums} if enum_names is None else set(enum_names)
+    selected_types = {item.name for item in model.types} if type_names is None else set(type_names)
+
+    changed = True
+    while changed:
+        changed = False
+        for name in _referenced(entities, model, selected_types):
+            if name in enum_map and name not in selected_enums:
+                selected_enums.add(name)
+                changed = True
+            elif name in type_map and name not in selected_types:
+                selected_types.add(name)
+                changed = True
+
+    enums = tuple(item for item in model.enums if item.name in selected_enums)
+    types = tuple(item for item in model.types if item.name in selected_types)
+    return ModelSlice(entities=entities, enums=enums, types=types), diagnostics
+
+
+def _requested(
+    label: str,
+    available: list[str],
+    requested: tuple[str, ...] | None,
+    diagnostics: list[Diagnostic],
+) -> set[str] | None:
+    if not requested:
+        return None
+    available_set = set(available)
+    for name in requested:
+        if name not in available_set:
+            diagnostics.append(
+                error("EG018", f"unknown name in input.{label}: {name}", path=f"input.{label}")
+            )
+    return set(requested)
+
+
+def _referenced(
+    entities: Iterable[Entity],
+    model: SourceModel,
+    selected_types: set[str],
+) -> set[str]:
+    names: set[str] = set()
+    for entity in entities:
+        for field in entity.fields:
+            names.add(field.type)
+    type_map = {item.name: item for item in model.types}
+    for name in tuple(selected_types):
+        custom = type_map.get(name)
+        if custom is not None:
+            names.add(custom.base)
+    return names
+
+
+__all__ = ["select_slice"]