modelable 1.0.0__py3-none-any.whl

modelable/registry/index.py

@@ -0,0 +1,419 @@
+from __future__ import annotations
+
+import itertools
+import json
+import sqlite3
+from importlib.resources import files
+from pathlib import Path
+
+from modelable.compat.checker import check_model_version_compatibility
+from modelable.compiler.workspace import Workspace
+from modelable.diagnostics.model import render_diagnostic
+from modelable.governance.por import build_por_record
+from modelable.parser.ir import AccessBlock, AccessGrant, ComputedMapping, DirectMapping
+from modelable.planner.lineage import build_projection_lineage
+from modelable.registry.resolver import resolved_version_spec
+
+
+def build_registry(workspace: Workspace, output_dir: str | Path = ".modelable") -> Path:
+    """Write a rebuildable SQLite registry index for a validated workspace."""
+    if workspace.errors:
+        joined = "\n".join(render_diagnostic(diagnostic) for diagnostic in workspace.errors)
+        raise ValueError(f"Cannot build registry with validation errors:\n{joined}")
+
+    output_path = Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+    registry_path = output_path / "registry.db"
+    if registry_path.exists():
+        registry_path.unlink()
+
+    schema = files("modelable.registry").joinpath("schema.sql").read_text(encoding="utf-8")
+    with sqlite3.connect(registry_path) as conn:
+        conn.executescript(schema)
+        _insert_workspace(conn, workspace)
+        conn.commit()
+
+    return registry_path
+
+
+def _insert_workspace(conn: sqlite3.Connection, workspace: Workspace) -> None:
+    source_paths = {id(domain): source.path for source in workspace.sources for domain in source.mdl.domains}
+
+    for domain in workspace.mdl.domains:
+        conn.execute(
+            "insert into domains (name, owner, description) values (?, ?, ?)",
+            (domain.name, domain.owner, domain.description),
+        )
+
+        for model_name, versions in domain.models.items():
+            kind = versions[0].model_kind.value
+            conn.execute(
+                "insert into models (domain_name, name, kind) values (?, ?, ?)",
+                (domain.name, model_name, kind),
+            )
+            for version in versions:
+                conn.execute(
+                    """
+                    insert into model_versions
+                    (domain_name, model_name, version, change_kind, source_path)
+                    values (?, ?, ?, ?, ?)
+                    """,
+                    (
+                        domain.name,
+                        model_name,
+                        version.version,
+                        version.change_kind.value,
+                        str(source_paths[id(domain)]),
+                    ),
+                )
+                for position, field in enumerate(version.fields):
+                    classification = field.classification
+                    conn.execute(
+                        """
+                        insert into fields
+                        (
+                          domain_name, model_name, model_version, field_name,
+                          position, type_json, optional, is_key, is_pii, classification
+                        )
+                        values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            domain.name,
+                            model_name,
+                            version.version,
+                            field.name,
+                            position,
+                            _to_json(field.type),
+                            int(field.optional),
+                            int(field.is_key),
+                            int(field.is_pii),
+                            classification.value if classification else None,
+                        ),
+                    )
+                _insert_access_policies(
+                    conn,
+                    subject_ref=f"{domain.name}.{model_name}@{version.version}",
+                    access=version.access,
+                    same_domain=domain.name,
+                    owner=domain.owner,
+                )
+                _insert_por_log(
+                    conn,
+                    build_por_record(f"{domain.name}.{model_name}.v{version.version}").as_dict(),
+                )
+            _insert_compatibility_reports(conn, workspace, domain.name, model_name, versions)
+
+        for projection_name, versions in domain.projections.items():
+            conn.execute(
+                "insert into projections (domain_name, name) values (?, ?)",
+                (domain.name, projection_name),
+            )
+            for version in versions:
+                conn.execute(
+                    """
+                    insert into projection_versions
+                    (
+                      domain_name, projection_name, version, source_model,
+                      source_version_json, source_alias
+                    )
+                    values (?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        domain.name,
+                        projection_name,
+                        version.version,
+                        version.source.model,
+                        _to_json(
+                            resolved_version_spec(
+                                workspace.mdl,
+                                version.source.model,
+                                version.source.version,
+                            )
+                        ),
+                        version.source.alias,
+                    ),
+                )
+                conn.execute(
+                    """
+                    insert into projection_sources
+                    (
+                      domain_name, projection_name, projection_version,
+                      source_kind, source_model, source_version_json, source_alias
+                    )
+                    values (?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        domain.name,
+                        projection_name,
+                        version.version,
+                        "primary",
+                        version.source.model,
+                        _to_json(
+                            resolved_version_spec(
+                                workspace.mdl,
+                                version.source.model,
+                                version.source.version,
+                            )
+                        ),
+                        version.source.alias,
+                    ),
+                )
+                for join in version.joins:
+                    conn.execute(
+                        """
+                        insert into projection_sources
+                        (
+                          domain_name, projection_name, projection_version,
+                          source_kind, source_model, source_version_json,
+                          source_alias, join_on
+                        )
+                        values (?, ?, ?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            domain.name,
+                            projection_name,
+                            version.version,
+                            "join",
+                            join.model,
+                            _to_json(resolved_version_spec(workspace.mdl, join.model, join.version)),
+                            join.alias,
+                            join.on,
+                        ),
+                    )
+                for position, field in enumerate(version.fields):
+                    classification = field.classification
+                    conn.execute(
+                        """
+                        insert into projection_fields
+                        (
+                          domain_name, projection_name, projection_version,
+                          field_name, position, mapping_json, is_pii, classification
+                        )
+                        values (?, ?, ?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            domain.name,
+                            projection_name,
+                            version.version,
+                            field.name,
+                            position,
+                            _to_json(field.mapping),
+                            int(field.is_pii),
+                            classification.value if classification else None,
+                        ),
+                    )
+                    _insert_field_mapping(
+                        conn,
+                        domain.name,
+                        projection_name,
+                        version.version,
+                        field.name,
+                        field.mapping,
+                    )
+                _insert_lineage_edges(conn, workspace, domain.name, projection_name, version)
+                _insert_access_policies(
+                    conn,
+                    subject_ref=f"{domain.name}.{projection_name}@{version.version}",
+                    access=version.access,
+                    same_domain=domain.name,
+                    owner=domain.owner,
+                )
+                _insert_por_log(
+                    conn,
+                    build_por_record(f"{domain.name}.{projection_name}.v{version.version}").as_dict(),
+                )
+
+    for binding in workspace.mdl.bindings:
+        conn.execute(
+            """
+            insert into adapter_bindings (name, model_ref, adapter, table_name)
+            values (?, ?, ?, ?)
+            """,
+            (
+                binding.name,
+                f"{binding.model}@{binding.model_version}",
+                binding.adapter,
+                binding.table,
+            ),
+        )
+
+
+def _insert_field_mapping(
+    conn: sqlite3.Connection,
+    domain_name: str,
+    projection_name: str,
+    projection_version: int,
+    target_field: str,
+    mapping: DirectMapping | ComputedMapping,
+) -> None:
+    if isinstance(mapping, DirectMapping):
+        conn.execute(
+            """
+            insert into field_mappings
+            (
+              domain_name, projection_name, projection_version, target_field,
+              mapping_kind, source_alias, source_field
+            )
+            values (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                domain_name,
+                projection_name,
+                projection_version,
+                target_field,
+                mapping.kind,
+                mapping.source_alias,
+                mapping.source_field,
+            ),
+        )
+        return
+
+    conn.execute(
+        """
+        insert into field_mappings
+        (
+          domain_name, projection_name, projection_version, target_field,
+          mapping_kind, expression
+        )
+        values (?, ?, ?, ?, ?, ?)
+        """,
+        (
+            domain_name,
+            projection_name,
+            projection_version,
+            target_field,
+            mapping.kind,
+            mapping.expression,
+        ),
+    )
+
+
+def _insert_access_policies(
+    conn: sqlite3.Connection,
+    *,
+    subject_ref: str,
+    access: AccessBlock | None,
+    same_domain: str,
+    owner: str | None,
+) -> None:
+    if access is not None:
+        for grant in access.entity:
+            _insert_access_grant(
+                conn,
+                subject_ref=subject_ref,
+                grant=grant,
+            )
+        for field_name, grants in access.properties.items():
+            for grant in grants:
+                _insert_access_grant(
+                    conn,
+                    subject_ref=f"{subject_ref}.{field_name}",
+                    grant=grant,
+                )
+        return
+
+    for action in ("read", "project", "subscribe"):
+        conn.execute(
+            """
+            insert into access_policies (subject_ref, action, grantee)
+            values (?, ?, ?)
+            """,
+            (subject_ref, action, same_domain),
+        )
+
+    if owner:
+        for action in ("write", "transfer", "manage_access"):
+            conn.execute(
+                """
+                insert into access_policies (subject_ref, action, grantee)
+                values (?, ?, ?)
+                """,
+                (subject_ref, action, owner),
+            )
+
+
+def _insert_access_grant(
+    conn: sqlite3.Connection,
+    *,
+    subject_ref: str,
+    grant: AccessGrant,
+) -> None:
+    for action in grant.permissions:
+        conn.execute(
+            """
+            insert into access_policies (subject_ref, action, grantee)
+            values (?, ?, ?)
+            """,
+            (subject_ref, action, grant.principal),
+        )
+
+
+def _insert_compatibility_reports(
+    conn: sqlite3.Connection,
+    workspace: Workspace,
+    domain_name: str,
+    model_name: str,
+    versions,
+) -> None:
+    for previous, current in itertools.pairwise(versions):
+        report = check_model_version_compatibility(
+            workspace.mdl,
+            domain_name,
+            model_name,
+            previous.version,
+            current.version,
+        )
+        conn.execute(
+            """
+            insert into compatibility_reports
+            (domain_name, model_name, from_version, to_version, status)
+            values (?, ?, ?, ?, ?)
+            """,
+            (
+                domain_name,
+                model_name,
+                previous.version,
+                current.version,
+                report.status,
+            ),
+        )
+
+
+def _insert_lineage_edges(
+    conn: sqlite3.Connection,
+    workspace: Workspace,
+    domain_name: str,
+    projection_name: str,
+    version,
+) -> None:
+    lineage = build_projection_lineage(domain_name, projection_name, version, workspace.mdl)
+    target_prefix = f"{domain_name}.{projection_name}@{version.version}"
+    for field_lineage in lineage.fields:
+        target_ref = f"{target_prefix}.{field_lineage.field_name}"
+        for source_ref in field_lineage.lineage:
+            conn.execute(
+                """
+                insert into lineage_edges (source_ref, target_ref, edge_kind)
+                values (?, ?, ?)
+                """,
+                (source_ref, target_ref, field_lineage.kind),
+            )
+
+
+def _insert_por_log(conn: sqlite3.Connection, por_record: dict[str, str | None]) -> None:
+    conn.execute(
+        """
+        insert into por_log (model_ref, issuer, issued_at, signature)
+        values (?, ?, ?, ?)
+        """,
+        (
+            por_record["model"],
+            por_record["issuer"],
+            por_record["issuedAt"],
+            por_record.get("signature"),
+        ),
+    )
+
+
+def _to_json(value) -> str:
+    return json.dumps(value.model_dump(mode="json"), sort_keys=True, separators=(",", ":"))

modelable/registry/local.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from pathlib import Path
+from shutil import copy2
+
+from modelable.registry.base import Registry
+
+
+class LocalRegistry(Registry):
+    """Local file-based registry."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+
+    def push(self, registry_path: Path) -> None:
+        """Push a registry index to the registry."""
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        if registry_path.resolve() == self.path.resolve():
+            return
+        copy2(registry_path, self.path)
+
+    def pull(self, dest_path: Path) -> None:
+        """Pull a registry index from the registry."""
+        if self.path.resolve() == dest_path.resolve():
+            return
+        copy2(self.path, dest_path)

modelable/registry/oci.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from modelable.registry.base import Registry
+
+
+class OCIRegistry(Registry):
+    """OCI-compliant registry."""
+
+    def __init__(self, url: str) -> None:
+        self.url = url
+
+    def push(self, registry_path: Path) -> None:
+        """Push a registry index to the OCI registry."""
+        # TODO: Implement OCI push (e.g., skopeo copy)
+        print(f"Pushing to {self.url} (not implemented)")
+
+    def pull(self, dest_path: Path) -> None:
+        """Pull a registry index from the OCI registry."""
+        # TODO: Implement OCI pull (e.g., skopeo copy)
+        print(f"Pulling from {self.url} (not implemented)")

modelable/registry/resolver.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from modelable.parser.ir import (
+    MdlFile,
+    ModelVersion,
+    ProjectionVersion,
+    VersionExact,
+    VersionMin,
+    VersionPinned,
+    VersionRange,
+    VersionSpec,
+)
+from modelable.registry.signature import compute_version_signature
+
+
+@dataclass(frozen=True)
+class ResolvedModelRef:
+    domain_name: str
+    model_name: str
+    version: ModelVersion | ProjectionVersion
+
+
+def resolve_model_ref(
+    mdl: MdlFile,
+    model_ref: str,
+    version_spec: VersionSpec | int,
+) -> ResolvedModelRef:
+    """Resolve a model reference to a concrete published model version."""
+    domain_name, model_name = _split_model_ref(model_ref)
+    versions = _find_model_versions(mdl, domain_name, model_name)
+    if not versions:
+        raise LookupError(f"unresolved model reference {model_ref}@{_format_version_spec(version_spec)}")
+
+    matching = [version for version in versions if _matches(version, version_spec, domain_name, model_name)]
+    if not matching:
+        raise LookupError(f"unresolved model reference {model_ref}@{_format_version_spec(version_spec)}")
+
+    selected = max(matching, key=lambda version: version.version)
+
+    # If using a range or min spec, ensure no breaking change exists between
+    # the requested start and the selected version.
+    if isinstance(version_spec, (VersionRange, VersionMin)):
+        min_v = version_spec.min_inclusive
+        # Check all versions from min_v + 1 up to selected.version
+        for v in versions:
+            if min_v < v.version <= selected.version:
+                from modelable.parser.ir import ChangeKind
+
+                if v.change_kind == ChangeKind.breaking:
+                    raise LookupError(
+                        f"unresolved model reference {model_ref}@{_format_version_spec(version_spec)}: "
+                        f"breaking change at version {v.version} blocks automatic resolution"
+                    )
+
+    return ResolvedModelRef(
+        domain_name=domain_name,
+        model_name=model_name,
+        version=selected,
+    )
+
+
+def resolved_version_spec(
+    mdl: MdlFile,
+    model_ref: str,
+    version_spec: VersionSpec | int,
+) -> VersionExact:
+    """Return the concrete version selected by a model reference."""
+    resolved = resolve_model_ref(mdl, model_ref, version_spec)
+    return VersionExact(version=resolved.version.version)
+
+
+def find_dependents(
+    mdl: MdlFile,
+    domain_name: str,
+    model_name: str,
+    version: int,
+) -> list[tuple[str, str, int]]:
+    """Return list of (domain, projection, version) depending on the source model version."""
+    dependents: list[tuple[str, str, int]] = []
+    source_ref = f"{domain_name}.{model_name}"
+
+    for domain in mdl.domains:
+        for proj_name, proj_versions in domain.projections.items():
+            for pv in proj_versions:
+                # Check primary source
+                is_dependent = False
+                if pv.source.model == source_ref:
+                    try:
+                        resolved = resolve_model_ref(mdl, pv.source.model, pv.source.version)
+                        if resolved.version.version == version:
+                            is_dependent = True
+                    except LookupError:
+                        pass
+
+                # Check joins if not already found
+                if not is_dependent:
+                    for join in pv.joins:
+                        if join.model == source_ref:
+                            try:
+                                resolved = resolve_model_ref(mdl, join.model, join.version)
+                                if resolved.version.version == version:
+                                    is_dependent = True
+                                    break
+                            except LookupError:
+                                pass
+
+                if is_dependent:
+                    dependents.append((domain.name, proj_name, pv.version))
+
+    return dependents
+
+
+def validate_references(mdl: MdlFile) -> list[str]:
+    """Return unresolved reference errors for projections, joins, and bindings."""
+    errors: list[str] = []
+
+    for domain in mdl.domains:
+        for projection_name, projection_versions in domain.projections.items():
+            for projection_version in projection_versions:
+                context = f"{domain.name}.{projection_name}@{projection_version.version}"
+                _append_lookup_error(
+                    errors,
+                    context,
+                    projection_version.source.model,
+                    projection_version.source.version,
+                    mdl,
+                )
+                for join in projection_version.joins:
+                    _append_lookup_error(errors, context, join.model, join.version, mdl)
+
+    for binding in mdl.bindings:
+        if not binding.model:
+            continue
+        _append_lookup_error(
+            errors,
+            f"binding {binding.name}",
+            binding.model,
+            binding.model_version,
+            mdl,
+        )
+
+    return errors
+
+
+def _append_lookup_error(
+    errors: list[str],
+    context: str,
+    model_ref: str,
+    version_spec: VersionSpec | int,
+    mdl: MdlFile,
+) -> None:
+    try:
+        resolve_model_ref(mdl, model_ref, version_spec)
+    except LookupError as exc:
+        errors.append(f"{context}: {exc}")
+
+
+def _find_model_versions(
+    mdl: MdlFile,
+    domain_name: str,
+    model_name: str,
+) -> list[ModelVersion]:
+    for domain in mdl.domains:
+        if domain.name == domain_name:
+            versions: list[ModelVersion] = []
+            versions.extend(domain.models.get(model_name, []))
+            versions.extend(domain.projections.get(model_name, []))
+            return versions
+    return []
+
+
+def _split_model_ref(model_ref: str) -> tuple[str, str]:
+    parts = model_ref.split(".", 1)
+    if len(parts) != 2 or not parts[0] or not parts[1]:
+        raise LookupError(f"invalid model reference {model_ref}")
+    return parts[0], parts[1]
+
+
+def _matches(
+    version: ModelVersion | ProjectionVersion,
+    version_spec: VersionSpec | int,
+    domain_name: str,
+    model_name: str,
+) -> bool:
+    if isinstance(version_spec, int):
+        return version.version == version_spec
+    if isinstance(version_spec, VersionExact):
+        return version.version == version_spec.version
+    if isinstance(version_spec, VersionRange):
+        return version_spec.min_inclusive <= version.version < version_spec.max_exclusive
+    if isinstance(version_spec, VersionMin):
+        return version.version >= version_spec.min_inclusive
+    if isinstance(version_spec, VersionPinned):
+        if version.version != version_spec.version:
+            return False
+        return compute_version_signature(domain_name, model_name, version).lower() == version_spec.content_hash.lower()
+    return False
+
+
+def _format_version_spec(version_spec: VersionSpec | int) -> str:
+    if isinstance(version_spec, int):
+        return str(version_spec)
+    if isinstance(version_spec, VersionExact):
+        return str(version_spec.version)
+    if isinstance(version_spec, VersionRange):
+        return f">={version_spec.min_inclusive}<{version_spec.max_exclusive}"
+    if isinstance(version_spec, VersionMin):
+        return f">={version_spec.min_inclusive}"
+    if isinstance(version_spec, VersionPinned):
+        return f"{version_spec.version}#{version_spec.content_hash}"
+    return str(version_spec)