eggpipe 0.3.0__py3-none-any.whl

eggpipe/core/format_registry.py

@@ -0,0 +1,230 @@
+"""Document format registry."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+from eggpipe.core.model import FormatSchema
+from eggpipe.interfaces.formats import EntityModelFormat, FormatPlugin
+from eggpipe.utils.exceptions import (
+    FormatConflictError,
+    FormatInheritanceError,
+    UnknownFormatError,
+)
+
+MAX_FORMAT_DEPTH = 10
+
+
+@dataclass(frozen=True)
+class FormatRegistryInfo:
+    name: str
+    versions: tuple[str, ...]
+    primitive_types: frozenset[str]
+
+
+@dataclass(frozen=True)
+class _FormatContract:
+    plugin: FormatPlugin
+    names: tuple[str, ...]
+    versions: tuple[str, ...]
+    schemas: dict[str, FormatSchema]
+    primitives: frozenset[str]
+    extends: str | None
+    base_versions: dict[str, str]
+
+    @property
+    def name(self) -> str:
+        return self.names[0]
+
+
+class FormatRegistry:
+    """Built-in entity-model plus formats from plugins."""
+
+    def __init__(self, *, include_builtin: bool = True) -> None:
+        self._formats: dict[str, _FormatContract] = {}
+        self._validated = False
+        if include_builtin:
+            self.register(EntityModelFormat())
+
+    def register(self, plugin: FormatPlugin) -> None:
+        contract = _contract(plugin)
+        for name in contract.names:
+            if name == "entity-model" and not isinstance(plugin, EntityModelFormat):
+                raise FormatConflictError(
+                    "entity-model is reserved by the core and cannot be overridden"
+                )
+            if name in self._formats:
+                raise FormatConflictError(f"format is already registered: {name}")
+        for name in contract.names:
+            self._formats[name] = contract
+        self._validated = False
+
+    def get(self, name: str) -> FormatPlugin:
+        self.validate()
+        try:
+            return self._formats[name].plugin
+        except KeyError as exc:
+            raise UnknownFormatError(f"unknown format: {name}") from exc
+
+    def accepts(self, format_name: str, accepted: set[str] | None) -> bool:
+        """Check a format, following the extends chain."""
+        if accepted is None:
+            return True
+        self.validate()
+        current: str | None = format_name
+        seen: set[str] = set()
+        while current is not None and current not in seen:
+            if current in accepted:
+                return True
+            seen.add(current)
+            contract = self._formats.get(current)
+            if contract is None:
+                return False
+            current = contract.extends
+        return False
+
+    def list(self) -> list[FormatRegistryInfo]:
+        self.validate()
+        return [
+            FormatRegistryInfo(
+                name=name,
+                versions=contract.versions,
+                primitive_types=contract.primitives,
+            )
+            for name, contract in sorted(self._formats.items())
+        ]
+
+    def validate(self) -> None:
+        if self._validated:
+            return
+        contracts = {id(contract.plugin): contract for contract in self._formats.values()}
+        for contract in contracts.values():
+            if contract.extends is not None:
+                self._validate_inheritance(contract)
+        self._validated = True
+
+    def _validate_inheritance(self, contract: _FormatContract) -> None:
+        parent = self._formats.get(contract.extends or "")
+        if parent is None:
+            raise FormatInheritanceError(
+                f"{contract.name}: unknown base format {contract.extends!r}"
+            )
+
+        seen: set[int] = set()
+        current: _FormatContract | None = contract
+        depth = 0
+        while current is not None and current.extends is not None:
+            if id(current) in seen:
+                raise FormatInheritanceError(f"{contract.name}: format inheritance cycle")
+            seen.add(id(current))
+            current = self._formats.get(current.extends)
+            if current is None:
+                raise FormatInheritanceError(
+                    f"{contract.name}: unknown base format {contract.extends!r}"
+                )
+            depth += 1
+            if depth > MAX_FORMAT_DEPTH:
+                raise FormatInheritanceError(
+                    f"{contract.name}: inheritance depth exceeds {MAX_FORMAT_DEPTH}"
+                )
+
+        for version in contract.versions:
+            base_version = contract.base_versions[version]
+            if base_version not in parent.versions:
+                raise FormatInheritanceError(
+                    f"{contract.name} {version}: base version {base_version} is not "
+                    f"supported by {contract.extends}"
+                )
+            own_schema = contract.schemas[version]
+            parent_schema = parent.schemas[base_version]
+            if not issubclass(own_schema.full_entity, parent_schema.full_entity):
+                raise FormatInheritanceError(
+                    f"{contract.name} {version}: full_entity must inherit "
+                    f"full_entity {contract.extends} {base_version}"
+                )
+        missing = set(parent.primitives) - set(contract.primitives)
+        if missing:
+            raise FormatInheritanceError(
+                f"{contract.name}: primitive_types must include the primitives of "
+                f"{contract.extends}: {', '.join(sorted(missing))}"
+            )
+
+
+def _contract(plugin: FormatPlugin) -> _FormatContract:
+    names = getattr(plugin, "format_names", None)
+    if not isinstance(names, list) or not names:
+        raise FormatInheritanceError("format_names must be a non-empty list")
+    if len(set(names)) != len(names):
+        raise FormatInheritanceError("format_names contains duplicates")
+    if any(not isinstance(name, str) or not name for name in names):
+        raise FormatInheritanceError("format_names must contain strings")
+
+    versions = getattr(plugin, "supported_versions", None)
+    if not isinstance(versions, list) or not versions:
+        raise FormatInheritanceError(f"{names[0]}: supported_versions must be a non-empty list")
+    if any(not isinstance(version, str) or not version for version in versions):
+        raise FormatInheritanceError(f"{names[0]}: supported_versions must contain strings")
+    if len(set(versions)) != len(versions):
+        raise FormatInheritanceError(f"{names[0]}: supported_versions contains duplicates")
+
+    schemas = getattr(plugin, "schemas", None)
+    if not isinstance(schemas, dict):
+        raise FormatInheritanceError(f"{names[0]}: schemas must be a mapping")
+    if set(schemas) != set(versions):
+        raise FormatInheritanceError(f"{names[0]}: schemas keys must match supported_versions")
+    for version in versions:
+        schema = schemas[version]
+        if not isinstance(schema, FormatSchema):
+            raise FormatInheritanceError(f"{names[0]}: schemas[{version!r}] must be a FormatSchema")
+        if not _is_model(schema.layer_entity):
+            raise FormatInheritanceError(
+                f"{names[0]}: schemas[{version!r}].layer_entity must be a BaseModel"
+            )
+        if not _is_model(schema.full_entity):
+            raise FormatInheritanceError(
+                f"{names[0]}: schemas[{version!r}].full_entity must be a BaseModel"
+            )
+
+    primitives = getattr(plugin, "primitive_types", None)
+    if not isinstance(primitives, (set, frozenset)) or any(
+        not isinstance(item, str) for item in primitives
+    ):
+        raise FormatInheritanceError(f"{names[0]}: primitive_types must be a set of strings")
+
+    extends = getattr(plugin, "extends", None)
+    if extends is not None and (not isinstance(extends, str) or not extends):
+        raise FormatInheritanceError(f"{names[0]}: extends must be a non-empty string")
+    base_versions = getattr(plugin, "base_versions", None)
+    if not isinstance(base_versions, dict):
+        raise FormatInheritanceError(f"{names[0]}: base_versions must be a mapping")
+    if extends is None:
+        if base_versions:
+            raise FormatInheritanceError(f"{names[0]}: base_versions is only allowed with extends")
+    else:
+        if extends in names:
+            raise FormatInheritanceError(f"{names[0]}: a format cannot extend itself")
+        if set(base_versions) != set(versions):
+            raise FormatInheritanceError(
+                f"{names[0]}: base_versions must contain an entry for every version"
+            )
+        if any(not isinstance(value, str) or not value for value in base_versions.values()):
+            raise FormatInheritanceError(f"{names[0]}: versions in base_versions must be strings")
+
+    return _FormatContract(
+        plugin=plugin,
+        names=tuple(names),
+        versions=tuple(versions),
+        schemas=dict(schemas),
+        primitives=frozenset(primitives),
+        extends=extends,
+        base_versions=dict(base_versions),
+    )
+
+
+def _is_model(value: object) -> bool:
+    return isinstance(value, type) and issubclass(value, BaseModel)
+
+
+__all__ = ["FormatRegistry", "FormatRegistryInfo"]

eggpipe/core/layers.py

@@ -0,0 +1,385 @@
+"""Build a model unit from format layers (ADR-0008)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Generic, Protocol, TypeVar
+
+from pydantic import BaseModel, ValidationError
+
+from eggpipe.core.document import ModelDocument
+from eggpipe.core.model import CustomType, Entity, EnumDef, FieldDef, SourceModel
+from eggpipe.interfaces.diagnostics import Diagnostic, warning
+from eggpipe.interfaces.formats import FormatPlugin
+from eggpipe.utils.exceptions import DocumentError
+
+
+class FormatLookup(Protocol):
+    def get(self, name: str) -> FormatPlugin: ...
+
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True)
+class MergeChange(Generic[T]):
+    value: T
+    changed: bool = False
+
+
+_MISSING = object()
+
+
+@dataclass(frozen=True)
+class LayerMergeResult:
+    document: ModelDocument
+    diagnostics: list[Diagnostic]
+
+
+def merge_model_layers(
+    documents: list[ModelDocument],
+    registry: FormatLookup,
+) -> LayerMergeResult:
+    """Validate and additively merge the layers of one model."""
+    if not documents:
+        raise DocumentError("EG027", None, "empty model")
+    layers = _ordered_layers(documents, registry)
+    diagnostics: list[Diagnostic] = []
+
+    base_doc = layers[0]
+    base_plugin = registry.get(base_doc.format)
+    _validate_entity(
+        base_plugin,
+        base_doc,
+        "full_entity",
+        _entity(base_doc),
+    )
+
+    merged = _entity(base_doc)
+    enums = list(base_doc.model.enums)
+    types = list(base_doc.model.types)
+    metadata = dict(base_doc.model.metadata)
+    document_metadata = dict(base_doc.metadata)
+    layer_sources = [base_doc.path]
+
+    for index, delta_doc in enumerate(layers[1:], start=1):
+        delta_plugin = registry.get(delta_doc.format)
+        _validate_layer_chain(layers[index - 1], delta_doc, delta_plugin)
+        _validate_entity(delta_plugin, delta_doc, "layer_entity", _entity(delta_doc))
+
+        delta = _entity(delta_doc)
+        if delta.name != merged.name:
+            raise DocumentError(
+                "EG029",
+                delta_doc.path,
+                f"layer {delta_doc.format} describes {delta.name}, expected {merged.name}",
+            )
+        merged_change = _merge_entity(merged, delta, delta_doc.path)
+        enums_change = _merge_enums(enums, delta_doc.model.enums, delta_doc.path)
+        types_change = _merge_types(types, delta_doc.model.types, delta_doc.path)
+        metadata_change = _merge_metadata(
+            metadata,
+            delta_doc.model.metadata,
+            delta_doc.path,
+        )
+        document_metadata_change = _merge_metadata(
+            document_metadata,
+            delta_doc.metadata,
+            delta_doc.path,
+            path="document.metadata",
+        )
+        merged = merged_change.value
+        enums = enums_change.value
+        types = types_change.value
+        metadata = metadata_change.value
+        document_metadata = document_metadata_change.value
+        added = any(
+            change.changed
+            for change in (
+                merged_change,
+                enums_change,
+                types_change,
+                metadata_change,
+                document_metadata_change,
+            )
+        )
+        layer_sources.append(delta_doc.path)
+        if not added:
+            diagnostics.append(
+                warning(
+                    "EG035",
+                    f"delta {delta_doc.format} adds nothing",
+                    path=str(delta_doc.path),
+                )
+            )
+
+    derived_doc = layers[-1]
+    derived_plugin = registry.get(derived_doc.format)
+    final = merged.model_copy(
+        update={
+            "format": derived_doc.format,
+            "format_version": derived_doc.version,
+            "layer_sources": tuple(layer_sources),
+        }
+    )
+    _validate_entity(derived_plugin, derived_doc, "full_entity", final)
+    model = SourceModel(
+        name=base_doc.model.name,
+        description=base_doc.model.description,
+        entities=(final,),
+        enums=tuple(enums),
+        types=tuple(types),
+        metadata=metadata,
+    )
+    document = ModelDocument(
+        format=derived_doc.format,
+        version=derived_doc.version,
+        path=base_doc.path,
+        imports=(),
+        model=model,
+        metadata=document_metadata,
+    )
+    return LayerMergeResult(document=document, diagnostics=diagnostics)
+
+
+def _ordered_layers(
+    documents: list[ModelDocument],
+    registry: FormatLookup,
+) -> list[ModelDocument]:
+    by_format: dict[str, ModelDocument] = {}
+    plugins: dict[str, FormatPlugin] = {}
+    for document in documents:
+        if len(document.model.entities) != 1:
+            raise DocumentError(
+                "EG031", document.path, "a layer file must contain exactly one entity"
+            )
+        if document.imports:
+            raise DocumentError("EG029", document.path, "imports are not allowed in a layer file")
+        entity = document.model.entities[0]
+        if entity.extends is not None:
+            raise DocumentError("EG029", document.path, "extends is not allowed in a layer file")
+        if document.format in by_format:
+            raise DocumentError(
+                "EG029",
+                document.path,
+                f"multiple files of the same format: {document.format}",
+            )
+        by_format[document.format] = document
+        plugins[document.format] = registry.get(document.format)
+
+    roots = [
+        name
+        for name, plugin in plugins.items()
+        if plugin.extends is None or plugin.extends not in plugins
+    ]
+    if len(roots) != 1:
+        raise DocumentError("EG029", documents[0].path, "layer formats do not form a single chain")
+
+    ordered = [roots[0]]
+    current = roots[0]
+    while True:
+        children = [name for name, plugin in plugins.items() if plugin.extends == current]
+        if len(children) > 1:
+            raise DocumentError(
+                "EG029",
+                documents[0].path,
+                "layer formats form a branch",
+            )
+        if not children:
+            break
+        ordered.append(children[0])
+        current = children[0]
+    if set(ordered) != set(by_format):
+        raise DocumentError(
+            "EG029", documents[0].path, "layer formats are not connected by inheritance"
+        )
+    return [by_format[name] for name in ordered]
+
+
+def _validate_layer_chain(
+    parent: ModelDocument,
+    child: ModelDocument,
+    child_plugin: FormatPlugin,
+) -> None:
+    expected = child_plugin.base_versions.get(child.version)
+    if expected is None:
+        raise DocumentError(
+            "EG034",
+            child.path,
+            f"{child.format} {child.version}: base_versions is missing",
+        )
+    if expected != parent.version:
+        raise DocumentError(
+            "EG034",
+            child.path,
+            f"{child.format} {child.version} requires {parent.format} {expected}, "
+            f"but the layer has {parent.version}",
+        )
+
+
+def _validate_entity(
+    plugin: FormatPlugin,
+    document: ModelDocument,
+    schema_name: str,
+    entity: Entity,
+) -> None:
+    schema = plugin.schemas.get(document.version)
+    if schema is None:
+        raise DocumentError(
+            "EG034",
+            document.path,
+            f"{document.format} {document.version}: schema is missing",
+        )
+    model: type[BaseModel] = getattr(schema, schema_name)
+    try:
+        model.model_validate(entity, from_attributes=True)
+    except ValidationError as exc:
+        raise DocumentError(
+            "EG029",
+            document.path,
+            f"{document.format} {document.version}: {schema_name}: {exc}",
+        ) from exc
+
+
+def _entity(document: ModelDocument) -> Entity:
+    return document.model.entities[0]
+
+
+def _merge_entity(
+    base: Entity,
+    delta: Entity,
+    source: Path,
+) -> MergeChange[Entity]:
+    added = False
+    values: dict[str, Any] = {}
+    for name in ("description", "version", "updated_at"):
+        left = getattr(base, name)
+        right = getattr(delta, name)
+        if right is None:
+            continue
+        if left is None:
+            values[name] = right
+            added = True
+        elif left != right:
+            raise DocumentError(
+                "EG029",
+                source,
+                f"scalar conflict for {name}: {left!r} and {right!r}",
+            )
+    metadata_change = _merge_metadata(base.metadata, delta.metadata, source)
+    if metadata_change.changed:
+        values["metadata"] = metadata_change.value
+        added = True
+
+    fields = list(base.fields)
+    by_name = {field.name: index for index, field in enumerate(fields)}
+    for field in delta.fields:
+        index = by_name.get(field.name)
+        if index is None:
+            fields.append(field.model_copy(update={"source": source}))
+            by_name[field.name] = len(fields) - 1
+            added = True
+            continue
+        field_change = _merge_field(fields[index], field, source)
+        fields[index] = field_change.value
+        added |= field_change.changed
+    if added or len(fields) != len(base.fields):
+        values["fields"] = tuple(fields)
+    return MergeChange(base.model_copy(update=values), added)
+
+
+def _merge_field(base: FieldDef, delta: FieldDef, source: Path) -> MergeChange[FieldDef]:
+    left = base.model_dump(exclude={"metadata"})
+    right = delta.model_dump(exclude={"metadata"})
+    if left != right:
+        raise DocumentError(
+            "EG029",
+            source,
+            f"overriding field {base.name} is not allowed",
+        )
+    metadata_change = _merge_metadata(base.metadata, delta.metadata, source)
+    if not metadata_change.changed:
+        return MergeChange(base)
+    return MergeChange(
+        base.model_copy(update={"metadata": metadata_change.value}),
+        True,
+    )
+
+
+def _merge_enums(
+    base: list[EnumDef],
+    delta: tuple[EnumDef, ...],
+    source: Path,
+) -> MergeChange[list[EnumDef]]:
+    return _merge_append(base, delta, source, label="enum", key=lambda item: item.name)
+
+
+def _merge_types(
+    base: list[CustomType],
+    delta: tuple[CustomType, ...],
+    source: Path,
+) -> MergeChange[list[CustomType]]:
+    return _merge_append(base, delta, source, label="type", key=lambda item: item.name)
+
+
+def _merge_append(
+    base: list[T],
+    delta: tuple[T, ...],
+    source: Path,
+    *,
+    label: str,
+    key: Callable[[T], str],
+) -> MergeChange[list[T]]:
+    by_name = {key(item): item for item in base}
+    result = list(base)
+    added = False
+    for item in delta:
+        name = key(item)
+        existing = by_name.get(name)
+        if existing is None:
+            result.append(item)
+            by_name[name] = item
+            added = True
+        elif existing != item:
+            raise DocumentError("EG029", source, f"overriding {label} {name} is not allowed")
+    return MergeChange(result, added)
+
+
+def _merge_metadata(
+    base: dict[str, Any],
+    delta: dict[str, Any],
+    source: Path,
+    *,
+    path: str = "metadata",
+) -> MergeChange[dict[str, Any]]:
+    result = dict(base)
+    changed = False
+    for key, value in delta.items():
+        change = _merge_value(result.get(key, _MISSING), value, source, f"{path}.{key}")
+        if change.changed:
+            result[key] = change.value
+            changed = True
+    return MergeChange(result, changed)
+
+
+def _merge_value(current: Any, incoming: Any, source: Path, path: str) -> MergeChange[Any]:
+    if current is _MISSING:
+        return MergeChange(incoming, True)
+    match current, incoming:
+        case dict(), dict():
+            return _merge_metadata(current, incoming, source, path=path)
+        case list(), list():
+            return MergeChange([*current, *incoming], bool(incoming))
+        case _ if current == incoming:
+            return MergeChange(current)
+        case _:
+            raise DocumentError(
+                "EG029",
+                source,
+                f"conflict at {path}: {current!r} and {incoming!r}",
+            )
+
+
+__all__ = ["FormatLookup", "LayerMergeResult", "merge_model_layers"]