modelable 1.0.0__py3-none-any.whl

modelable/emitters/dbt_yaml.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from modelable.compiler.workspace import Workspace
+from modelable.emitters.base import EmittedArtifact, compute_content_hash
+from modelable.parser.ir import (
+    ArrayType,
+    DecimalType,
+    DirectMapping,
+    DomainDef,
+    EnumType,
+    MapType,
+    MdlFile,
+    ModelVersion,
+    NamedType,
+    ObjectType,
+    PrimitiveType,
+    ProjectionField,
+    ProjectionVersion,
+    RefType,
+    VersionExact,
+    VersionMin,
+    VersionPinned,
+    VersionRange,
+)
+from modelable.registry.resolver import resolve_model_ref
+
+
+def emit_dbt_yaml(workspace: Workspace, out_dir: Path) -> list[EmittedArtifact]:
+    """Emit dbt schema.yml fragments for every model and projection version."""
+    artifacts: list[EmittedArtifact] = []
+    for domain in workspace.mdl.domains:
+        for model_name, versions in domain.models.items():
+            for version in versions:
+                artifacts.append(_emit_model(domain, model_name, version, out_dir))
+        for projection_name, versions in domain.projections.items():
+            for version in versions:
+                artifacts.append(_emit_projection(domain, projection_name, version, out_dir, workspace.mdl))
+    return artifacts
+
+
+def _emit_model(domain: DomainDef, model_name: str, version: ModelVersion, out_dir: Path) -> EmittedArtifact:
+    artifact_id = f"{domain.name}.{model_name}.v{version.version}"
+
+    meta: dict[str, Any] = {
+        "modelable_domain": domain.name,
+        "modelable_name": model_name,
+        "modelable_kind": version.model_kind.value,
+        "modelable_version": version.version,
+        "modelable_change_kind": version.change_kind.value,
+    }
+    if domain.owner:
+        meta["modelable_owner"] = domain.owner
+    if domain.contact:
+        meta["modelable_contact"] = domain.contact
+
+    columns: list[dict[str, Any]] = []
+    for field in version.fields:
+        col: dict[str, Any] = {
+            "name": field.name,
+            "data_type": _type_to_dbt(field.type, wire_targets=field.wire_targets()),
+        }
+
+        # constraints
+        constraints: list[dict[str, str]] = []
+        if field.is_key:
+            constraints.append({"type": "primary_key"})
+        if not field.optional:
+            constraints.append({"type": "not_null"})
+        if constraints:
+            col["constraints"] = constraints
+
+        # meta
+        col_meta: dict[str, Any] = {}
+        if field.is_pii:
+            col_meta["modelable_pii"] = True
+        if field.classification:
+            col_meta["modelable_classification"] = field.classification.value
+        if domain.owner:
+            col_meta["modelable_owner"] = domain.owner
+
+        if col_meta:
+            col["meta"] = col_meta
+
+        columns.append(col)
+
+    model_entry: dict[str, Any] = {
+        "name": model_name,
+        "latest_version": version.version,
+        "meta": meta,
+        "versions": [
+            {
+                "v": version.version,
+                "columns": columns,
+            }
+        ],
+    }
+
+    if domain.description:
+        model_entry["description"] = domain.description
+
+    doc = {
+        "version": 2,
+        "models": [model_entry],
+    }
+
+    yaml_content = yaml.safe_dump(doc, sort_keys=False, default_flow_style=False)
+    content = f"# @generated by Modelable\n{yaml_content}"
+
+    return EmittedArtifact(
+        target="dbt-yaml",
+        ref=f"{domain.name}.{model_name}@{version.version}",
+        artifact_id=artifact_id,
+        path=out_dir / f"{artifact_id}.yml",
+        content=content,
+        content_hash=compute_content_hash(content),
+    )
+
+
+def _emit_projection(
+    domain: DomainDef,
+    projection_name: str,
+    version: ProjectionVersion,
+    out_dir: Path,
+    mdl: MdlFile,
+) -> EmittedArtifact:
+    artifact_id = f"{domain.name}.{projection_name}.v{version.version}"
+
+    meta: dict[str, Any] = {
+        "modelable_domain": domain.name,
+        "modelable_name": projection_name,
+        "modelable_kind": "projection",
+        "modelable_version": version.version,
+        "modelable_source": f"{version.source.model}@{_version_label(version.source.version)}",
+    }
+    if domain.owner:
+        meta["modelable_owner"] = domain.owner
+    if domain.contact:
+        meta["modelable_contact"] = domain.contact
+    if version.where:
+        meta["modelable_where"] = version.where
+    if version.group_by:
+        meta["modelable_group_by"] = ", ".join(version.group_by)
+
+    columns: list[dict[str, Any]] = []
+    for field in version.fields:
+        col: dict[str, Any] = {
+            "name": field.name,
+            "data_type": _resolve_projection_field_type_string(field, version, mdl),
+        }
+
+        col_meta: dict[str, Any] = {}
+        lineage = _resolve_lineage(field, version, mdl)
+        if lineage:
+            col_meta["modelable_lineage"] = [lineage]
+
+        if field.is_pii:
+            col_meta["modelable_pii"] = True
+        if field.classification:
+            col_meta["modelable_classification"] = field.classification.value
+        if domain.owner:
+            col_meta["modelable_owner"] = domain.owner
+
+        if col_meta:
+            col["meta"] = col_meta
+
+        columns.append(col)
+
+    model_entry: dict[str, Any] = {
+        "name": projection_name,
+        "latest_version": version.version,
+        "meta": meta,
+        "versions": [
+            {
+                "v": version.version,
+                "columns": columns,
+            }
+        ],
+    }
+
+    if domain.description:
+        model_entry["description"] = domain.description
+
+    doc = {
+        "version": 2,
+        "models": [model_entry],
+    }
+
+    yaml_content = yaml.safe_dump(doc, sort_keys=False, default_flow_style=False)
+    content = f"# @generated by Modelable\n{yaml_content}"
+
+    return EmittedArtifact(
+        target="dbt-yaml",
+        ref=f"{domain.name}.{projection_name}@{version.version}",
+        artifact_id=artifact_id,
+        path=out_dir / f"{artifact_id}.yml",
+        content=content,
+        content_hash=compute_content_hash(content),
+    )
+
+
+def _version_label(version_spec: Any) -> str:
+    if isinstance(version_spec, VersionExact):
+        return str(version_spec.version)
+    if isinstance(version_spec, VersionRange):
+        return f">={version_spec.min_inclusive}<{version_spec.max_exclusive}"
+    if isinstance(version_spec, VersionMin):
+        return f">={version_spec.min_inclusive}"
+    if isinstance(version_spec, VersionPinned):
+        return f"{version_spec.version}#{version_spec.content_hash}"
+    if isinstance(version_spec, int):
+        return str(version_spec)
+    return "?"
+
+
+def _type_to_dbt(field_type: Any, *, wire_targets: dict[str, Any] | None = None) -> str:
+    json_wire = wire_targets.get("json") if wire_targets is not None else None
+
+    if isinstance(field_type, PrimitiveType):
+        if (
+            json_wire is not None
+            and getattr(json_wire, "encoding", None) == "string"
+            and field_type.kind in {"int", "float"}
+        ):
+            return "varchar"
+
+        mapping = {
+            "string": "text",
+            "int": "integer",
+            "float": "float",
+            "bool": "boolean",
+            "date": "date",
+            "time": "time",
+            "timestamp": "timestamp",
+            "uuid": "uuid",
+            "duration": "varchar",
+            "binary": "binary",
+            "json": "text",
+        }
+        return mapping.get(field_type.kind, "varchar")
+
+    if isinstance(field_type, DecimalType):
+        return f"numeric({field_type.precision}, {field_type.scale})"
+    if isinstance(field_type, ArrayType):
+        return f"{_type_to_dbt(field_type.item)}[]"
+    if isinstance(field_type, MapType):
+        return "text"
+    if isinstance(field_type, RefType):
+        return "text"
+    if isinstance(field_type, EnumType):
+        return "text"
+    if isinstance(field_type, ObjectType):
+        return "text"
+    if isinstance(field_type, NamedType):
+        return "text"
+    return "text"
+
+
+def _resolve_projection_field_type_string(field: ProjectionField, projection: ProjectionVersion, mdl: MdlFile) -> str:
+    if not isinstance(field.mapping, DirectMapping):
+        return "text"
+    try:
+        source_domain, source_model = projection.source.model.rsplit(".", 1)
+        resolved = resolve_model_ref(mdl, f"{source_domain}.{source_model}", projection.source.version)
+    except ValueError, LookupError:
+        return "text"
+
+    for src_field in resolved.version.fields:
+        if src_field.name == field.mapping.source_field:
+            return _type_to_dbt(src_field.type, wire_targets=src_field.wire_targets())
+    return "text"
+
+
+def _resolve_lineage(field: ProjectionField, projection: ProjectionVersion, mdl: MdlFile) -> str | None:
+    if not isinstance(field.mapping, DirectMapping):
+        return None
+    try:
+        source_domain, source_model = projection.source.model.rsplit(".", 1)
+        resolved = resolve_model_ref(mdl, f"{source_domain}.{source_model}", projection.source.version)
+    except ValueError, LookupError:
+        return None
+
+    for src_field in resolved.version.fields:
+        if src_field.name == field.mapping.source_field:
+            return f"{resolved.domain_name}.{resolved.model_name}@{_version_label(resolved.version.version)}.{src_field.name}"
+    return None

modelable/emitters/diagnostics.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+
+def emit_warning(code: str, message: str) -> str:
+    return f"[{code}] {message}"
+
+
+def unsupported_target(target: str) -> str:
+    return emit_warning("EMIT001", f"Unsupported target: {target}")
+
+
+def type_loss(field_type: str) -> str:
+    return emit_warning("EMIT002", f"Type '{field_type}' cannot be represented without loss")
+
+
+def missing_metadata(field: str) -> str:
+    return emit_warning("EMIT003", f"Missing metadata required by target: {field}")
+
+
+def validation_failed(path: str, detail: str) -> str:
+    return emit_warning("EMIT004", f"Generated artifact failed validation: {path} ({detail})")
+
+
+def deferred_target(target: str) -> str:
+    return emit_warning("EMIT005", f"Deferred target requested in current phase: {target}")