contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/governance/sql.py

@@ -0,0 +1,31 @@
+"""Render reviewable Unity Catalog governance SQL."""
+
+from __future__ import annotations
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.annotations import render_annotations_sql
+from contractforge_databricks.governance.access import render_access_sql
+from contractforge_databricks.rendering.names import target_full_name
+
+
+def render_governance_sql(contract: SemanticContract) -> str:
+    governance = contract.governance
+    if governance is None:
+        return "-- No governance intent declared.\n"
+
+    target = target_full_name(contract)
+    lines = [
+        "-- Review before execution. Function names and privileges are contract-owned.",
+        f"-- Target: {target}",
+        "",
+    ]
+    if governance.owner:
+        lines.append(f"-- Owner intent: {governance.owner}")
+    access_sql_body = render_access_sql(contract)
+    if access_sql_body:
+        lines.append(access_sql_body)
+    access_sql = "\n".join(lines) + "\n"
+    annotations_sql = render_annotations_sql(contract)
+    if annotations_sql.startswith("-- No annotations intent declared."):
+        return access_sql
+    return annotations_sql + "\n" + access_sql

contractforge_databricks/governance/validation.py

@@ -0,0 +1,135 @@
+"""Databricks governance validation and access drift helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.coercion import mapping, mapping_list, string_list
+from contractforge_databricks.rendering.names import target_full_name
+
+
+def governance_referenced_columns(contract: SemanticContract) -> dict[str, list[str]]:
+    annotations = contract.governance.annotations if contract.governance else None
+    access = contract.governance.access if contract.governance else None
+    annotation_columns = sorted(mapping(annotations).get("columns", {}).keys())
+    row_filter_columns = sorted(
+        {
+            column
+            for row_filter in mapping_list(mapping(access).get("row_filters"))
+            for column in string_list(row_filter.get("columns"))
+        }
+    )
+    mask_columns = sorted(
+        {
+            column
+            for mask in mapping_list(mapping(access).get("column_masks"))
+            for column in [str(mask.get("column")), *string_list(mask.get("using_columns"))]
+            if column and column != "None"
+        }
+    )
+    all_columns = sorted(set(annotation_columns) | set(row_filter_columns) | set(mask_columns))
+    return {
+        "annotations": annotation_columns,
+        "row_filters": row_filter_columns,
+        "column_masks": mask_columns,
+        "all": all_columns,
+    }
+
+
+def validate_governance_contract(
+    contract: SemanticContract,
+    *,
+    existing_columns: set[str] | list[str] | tuple[str, ...],
+    target_table: str | None = None,
+) -> dict[str, Any]:
+    references = governance_referenced_columns(contract)
+    columns = set(str(column) for column in existing_columns)
+    issues = []
+    annotations = mapping(contract.governance.annotations if contract.governance else None)
+    table = mapping(annotations.get("table"))
+    annotation_columns = mapping(annotations.get("columns"))
+    contains_pii = str(mapping(table.get("tags")).get("contains_pii", "")).lower() == "true"
+    pii_columns = sorted(
+        column
+        for column, config in annotation_columns.items()
+        if bool(mapping(mapping(config).get("pii")).get("enabled", False))
+    )
+    if contains_pii and not pii_columns:
+        issues.append(_issue("fail", "annotations", "table.tags.contains_pii", "contains_pii=true requires at least one column with pii.enabled=true"))
+    for column in pii_columns:
+        if not mapping(annotation_columns.get(column)).get("description"):
+            issues.append(_issue("warn", "annotations", column, f"PII column {column!r} should declare a description"))
+
+    for scope, referenced in references.items():
+        if scope == "all":
+            continue
+        for column in sorted(set(referenced) - columns):
+            issues.append(_issue("fail", scope, column, f"Column {column!r} referenced by {scope} does not exist"))
+    return {
+        "status": "FAILED" if any(issue["severity"] == "fail" for issue in issues) else "SUCCESS",
+        "target_table": target_table or target_full_name(contract),
+        "references": references,
+        "issues": issues,
+    }
+
+
+def access_drift_report(
+    contract: SemanticContract,
+    *,
+    current_grants: set[tuple[str, str]],
+    target_table: str | None = None,
+) -> dict[str, Any]:
+    access = mapping(contract.governance.access if contract.governance else None)
+    if not access:
+        return _drift_payload("NOT_CONFIGURED", target_table or target_full_name(contract), set(), current_grants, [], [])
+    declared = {
+        (str(grant.get("principal")), str(privilege).upper())
+        for grant in mapping_list(access.get("grants"))
+        for privilege in string_list(grant.get("privileges"))
+    }
+    normalized_current = {(str(principal), str(privilege).upper()) for principal, privilege in current_grants}
+    missing = sorted(declared - normalized_current)
+    unmanaged = sorted(normalized_current - declared)
+    policy = _access_policy(access)
+    revoke_unmanaged = bool(mapping(access.get("access_policy")).get("revoke_unmanaged", access.get("revoke_unmanaged", False)))
+    issues = [
+        _issue(policy, "grant", f"{principal}:{privilege}", f"Declared grant is missing: {privilege} for {principal}")
+        for principal, privilege in missing
+    ]
+    if revoke_unmanaged:
+        issues.extend(
+            _issue(policy, "grant", f"{principal}:{privilege}", f"Current unmanaged grant was detected: {privilege} from {principal}")
+            for principal, privilege in unmanaged
+        )
+    status = "DRIFTED" if missing or (revoke_unmanaged and unmanaged) else "IN_SYNC"
+    return _drift_payload(status, target_table or target_full_name(contract), declared, normalized_current, missing, unmanaged, issues)
+
+
+def _drift_payload(
+    status: str,
+    target_table: str,
+    declared: set[tuple[str, str]],
+    current: set[tuple[str, str]],
+    missing: list[tuple[str, str]],
+    unmanaged: list[tuple[str, str]],
+    issues: list[dict[str, str]] | None = None,
+) -> dict[str, Any]:
+    return {
+        "status": status,
+        "target_table": target_table,
+        "declared_grants": sorted(declared),
+        "current_grants": sorted(current),
+        "missing_grants": missing,
+        "unmanaged_grants": unmanaged,
+        "issues": issues or [],
+    }
+
+
+def _access_policy(access: dict[str, Any]) -> str:
+    policy = mapping(access.get("access_policy"))
+    return "fail" if policy.get("on_drift", access.get("on_drift", "warn")) == "fail" else "warn"
+
+
+def _issue(severity: str, scope: str, obj: str, message: str) -> dict[str, str]:
+    return {"severity": severity, "scope": scope, "object": obj, "message": message}

contractforge_databricks/lakeflow/__init__.py

@@ -0,0 +1,21 @@
+from contractforge_databricks.lakeflow.compatibility import (
+    LakeflowCompatibility,
+    LakeflowSourceKind,
+    evaluate_lakeflow_compatibility,
+    render_lakeflow_review,
+)
+from contractforge_databricks.lakeflow.rendering import (
+    LakeflowAutoCdcArtifact,
+    render_lakeflow_auto_cdc_artifact,
+    render_lakeflow_auto_cdc_python,
+)
+
+__all__ = [
+    "LakeflowAutoCdcArtifact",
+    "LakeflowCompatibility",
+    "LakeflowSourceKind",
+    "evaluate_lakeflow_compatibility",
+    "render_lakeflow_auto_cdc_artifact",
+    "render_lakeflow_auto_cdc_python",
+    "render_lakeflow_review",
+]

contractforge_databricks/lakeflow/compatibility.py

@@ -0,0 +1,194 @@
+"""Lakeflow AUTO CDC compatibility checks for semantic contracts."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.rendering.names import target_full_name
+
+LakeflowStatus = Literal["compatible", "requires_translation", "unsupported"]
+LakeflowSourceKind = Literal["change_feed", "snapshot"]
+
+
+@dataclass(frozen=True)
+class LakeflowCompatibility:
+    status: LakeflowStatus
+    source_kind: LakeflowSourceKind
+    scd_type: int | None
+    target_table: str
+    reasons: tuple[str, ...] = ()
+    required_fields: tuple[str, ...] = ()
+    mapped_fields: dict[str, Any] | None = None
+    translation_required: tuple[str, ...] = ()
+    unsupported_fields: tuple[str, ...] = ()
+    warnings: tuple[str, ...] = ()
+
+    @property
+    def supported(self) -> bool:
+        return self.status != "unsupported"
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "status": self.status,
+            "supported": self.supported,
+            "source_kind": self.source_kind,
+            "scd_type": self.scd_type,
+            "target_table": self.target_table,
+            "reasons": list(self.reasons),
+            "required_fields": list(self.required_fields),
+            "mapped_fields": dict(self.mapped_fields or {}),
+            "translation_required": list(self.translation_required),
+            "unsupported_fields": list(self.unsupported_fields),
+            "warnings": list(self.warnings),
+        }
+
+
+def evaluate_lakeflow_compatibility(
+    contract: SemanticContract,
+    *,
+    source_kind: LakeflowSourceKind = "change_feed",
+    source_name: str | None = None,
+    keys: tuple[str, ...] = (),
+    sequence_by: str | None = None,
+    apply_as_truncates: str | None = None,
+) -> LakeflowCompatibility:
+    reasons: list[str] = []
+    required: list[str] = []
+    translation: list[str] = []
+    unsupported: list[str] = []
+    warnings: list[str] = []
+    scd_type = _scd_type(contract.write.mode)
+    target_table = target_full_name(contract)
+    effective_keys = keys or contract.write.merge_keys
+    effective_sequence_by = sequence_by or contract.write.scd2_sequence_by
+    mapped_fields: dict[str, Any] = {
+        "target": target_table,
+        "source": source_name,
+        "keys": list(effective_keys),
+        "sequence_by": effective_sequence_by,
+        "stored_as_scd_type": scd_type,
+        "apply_as_deletes": contract.write.scd2_apply_as_deletes,
+        "apply_as_truncates": apply_as_truncates,
+    }
+
+    if scd_type is None:
+        unsupported.append("mode")
+        reasons.append(f"{contract.write.mode} does not map directly to Lakeflow AUTO CDC.")
+    if not source_name:
+        required.append("source_name")
+        reasons.append("Lakeflow AUTO CDC requires a source table, view, or snapshot function.")
+    if not effective_keys:
+        required.append("keys")
+        reasons.append("Lakeflow AUTO CDC requires stable keys.")
+    if source_kind not in {"change_feed", "snapshot"}:
+        unsupported.append("source_kind")
+        reasons.append("Lakeflow source_kind must be 'change_feed' or 'snapshot'.")
+    if source_kind == "change_feed" and scd_type == 2 and not effective_sequence_by:
+        required.append("sequence_by")
+        reasons.append("Lakeflow AUTO CDC SCD2 requires sequence_by.")
+    if source_kind == "snapshot" and contract.write.scd2_apply_as_deletes:
+        unsupported.append("apply_as_deletes")
+        reasons.append("AUTO CDC FROM SNAPSHOT derives deletes from snapshots and does not use CDC delete predicates.")
+    if apply_as_truncates and scd_type == 2:
+        unsupported.append("apply_as_truncates")
+        reasons.append("Lakeflow apply_as_truncates is supported only for SCD type 1.")
+    if scd_type == 2 and contract.write.scd2_change_columns:
+        mapped_fields["track_history_column_list"] = list(contract.write.scd2_change_columns)
+    elif scd_type == 2:
+        warnings.append("SCD2 without scd2_change_columns maps to Lakeflow's default of tracking all output columns.")
+    if contract.quality:
+        translation.append("quality")
+        reasons.append("Quality rules must be materialized upstream or enforced outside AUTO CDC.")
+    metadata = contract.operations.metadata if contract.operations and contract.operations.metadata else {}
+    _translation_from_metadata(
+        metadata,
+        translation,
+        reasons,
+        "select_columns",
+        "Projection intent must be materialized upstream as the Lakeflow source table/view.",
+    )
+    _translation_from_metadata(
+        metadata,
+        translation,
+        reasons,
+        "column_mapping",
+        "Column mapping intent must be materialized upstream as the Lakeflow source table/view.",
+    )
+    _translation_from_metadata(
+        metadata,
+        translation,
+        reasons,
+        "filter_expression",
+        "Filter intent must be materialized upstream as the Lakeflow source table/view.",
+    )
+    _translation_from_metadata(
+        metadata,
+        translation,
+        reasons,
+        "watermark_columns",
+        "Watermark filtering/state remains ContractForge runtime behavior and must be resolved before AUTO CDC.",
+    )
+    if contract.shape:
+        translation.append("shape")
+        reasons.append("Shape intent must be materialized upstream as the Lakeflow source table/view.")
+    if contract.transform:
+        translation.append("transform")
+        reasons.append("Transform intent must be materialized upstream as the Lakeflow source table/view.")
+
+    if unsupported or required:
+        status: LakeflowStatus = "unsupported"
+    elif translation:
+        status = "requires_translation"
+    else:
+        status = "compatible"
+        reasons.append("Contract can map to Lakeflow AUTO CDC arguments.")
+
+    return LakeflowCompatibility(
+        status=status,
+        source_kind=source_kind,
+        scd_type=scd_type,
+        target_table=target_table,
+        reasons=tuple(reasons),
+        required_fields=tuple(required),
+        mapped_fields={key: value for key, value in mapped_fields.items() if value is not None},
+        translation_required=tuple(translation),
+        unsupported_fields=tuple(unsupported),
+        warnings=tuple(warnings),
+    )
+
+
+def render_lakeflow_review(compatibility: LakeflowCompatibility) -> str:
+    lines = [
+        "# Lakeflow AUTO CDC Compatibility",
+        "",
+        f"- Status: `{compatibility.status}`",
+        f"- SCD type: `{compatibility.scd_type}`",
+        "",
+    ]
+    for reason in compatibility.reasons:
+        lines.append(f"- {reason}")
+    for warning in compatibility.warnings:
+        lines.append(f"- Warning: {warning}")
+    return "\n".join(lines) + "\n"
+
+
+def _scd_type(mode: str) -> int | None:
+    if mode == "scd1_upsert":
+        return 1
+    if mode == "scd2_historical":
+        return 2
+    return None
+
+
+def _translation_from_metadata(
+    metadata: dict[str, Any],
+    translation: list[str],
+    reasons: list[str],
+    key: str,
+    reason: str,
+) -> None:
+    if metadata.get(key) not in (None, "", [], {}):
+        translation.append(key)
+        reasons.append(reason)

contractforge_databricks/lakeflow/rendering.py

@@ -0,0 +1,175 @@
+"""Lakeflow AUTO CDC Python artifact rendering."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.lakeflow.compatibility import (
+    LakeflowCompatibility,
+    LakeflowSourceKind,
+    evaluate_lakeflow_compatibility,
+)
+from contractforge_databricks.rendering.names import target_full_name
+
+
+@dataclass(frozen=True)
+class LakeflowAutoCdcArtifact:
+    language: Literal["python"]
+    source_kind: LakeflowSourceKind
+    code: str
+    compatibility: LakeflowCompatibility
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "language": self.language,
+            "source_kind": self.source_kind,
+            "code": self.code,
+            "compatibility": self.compatibility.as_dict(),
+        }
+
+
+def render_lakeflow_auto_cdc_python(
+    contract: SemanticContract,
+    *,
+    source_kind: LakeflowSourceKind = "change_feed",
+    source_name: str,
+    keys: tuple[str, ...] = (),
+    sequence_by: str | None = None,
+    flow_name: str | None = None,
+    apply_as_truncates: str | None = None,
+    ignore_null_updates: bool = False,
+    once: bool = False,
+) -> str:
+    return render_lakeflow_auto_cdc_artifact(
+        contract,
+        source_kind=source_kind,
+        source_name=source_name,
+        keys=keys,
+        sequence_by=sequence_by,
+        flow_name=flow_name,
+        apply_as_truncates=apply_as_truncates,
+        ignore_null_updates=ignore_null_updates,
+        once=once,
+    ).code
+
+
+def render_lakeflow_auto_cdc_artifact(
+    contract: SemanticContract,
+    *,
+    source_kind: LakeflowSourceKind = "change_feed",
+    source_name: str,
+    keys: tuple[str, ...] = (),
+    sequence_by: str | None = None,
+    flow_name: str | None = None,
+    apply_as_truncates: str | None = None,
+    ignore_null_updates: bool = False,
+    once: bool = False,
+) -> LakeflowAutoCdcArtifact:
+    compatibility = evaluate_lakeflow_compatibility(
+        contract,
+        source_kind=source_kind,
+        source_name=source_name,
+        keys=keys,
+        sequence_by=sequence_by,
+        apply_as_truncates=apply_as_truncates,
+    )
+    if compatibility.status == "unsupported":
+        raise ValueError("; ".join(compatibility.reasons))
+
+    if source_kind == "snapshot":
+        code = _render_snapshot_flow(
+            contract,
+            source_name=source_name,
+            keys=keys or contract.write.merge_keys,
+            flow_name=flow_name,
+            scd_type=compatibility.scd_type,
+        )
+    else:
+        code = _render_change_feed_flow(
+            contract,
+            source_name=source_name,
+            keys=keys or contract.write.merge_keys,
+            sequence_by=sequence_by or contract.write.scd2_sequence_by,
+            flow_name=flow_name,
+            scd_type=compatibility.scd_type,
+            apply_as_truncates=apply_as_truncates,
+            ignore_null_updates=ignore_null_updates,
+            once=once,
+        )
+    return LakeflowAutoCdcArtifact(
+        language="python",
+        source_kind=source_kind,
+        code=code,
+        compatibility=compatibility,
+    )
+
+
+def _render_change_feed_flow(
+    contract: SemanticContract,
+    *,
+    source_name: str,
+    keys: tuple[str, ...],
+    sequence_by: str | None,
+    flow_name: str | None,
+    scd_type: int | None,
+    apply_as_truncates: str | None,
+    ignore_null_updates: bool,
+    once: bool,
+) -> str:
+    target = target_full_name(contract)
+    lines = [
+        "from pyspark import pipelines as dp",
+        "",
+        f"dp.create_streaming_table(name={target!r})",
+        "",
+        "dp.create_auto_cdc_flow(",
+        f"    target={target!r},",
+        f"    source={source_name!r},",
+        f"    keys={list(keys)!r},",
+        f"    stored_as_scd_type={scd_type!r},",
+        f"    ignore_null_updates={ignore_null_updates!r},",
+    ]
+    if sequence_by:
+        lines.append(f"    sequence_by={sequence_by!r},")
+    if contract.write.scd2_apply_as_deletes:
+        lines.append(f"    apply_as_deletes={contract.write.scd2_apply_as_deletes!r},")
+    if apply_as_truncates:
+        lines.append(f"    apply_as_truncates={apply_as_truncates!r},")
+    if contract.write.scd2_change_columns:
+        lines.append(f"    track_history_column_list={list(contract.write.scd2_change_columns)!r},")
+    if flow_name:
+        lines.append(f"    name={flow_name!r},")
+    if once:
+        lines.append("    once=True,")
+    lines.append(")")
+    return "\n".join(lines) + "\n"
+
+
+def _render_snapshot_flow(
+    contract: SemanticContract,
+    *,
+    source_name: str,
+    keys: tuple[str, ...],
+    flow_name: str | None,
+    scd_type: int | None,
+) -> str:
+    target = target_full_name(contract)
+    lines = [
+        "from pyspark import pipelines as dp",
+        "",
+        f"dp.create_streaming_table(name={target!r})",
+        "",
+        "dp.create_auto_cdc_from_snapshot_flow(",
+        f"    target={target!r},",
+        f"    source={source_name!r},",
+        f"    keys={list(keys)!r},",
+        f"    stored_as_scd_type={scd_type!r},",
+    ]
+    if contract.write.scd2_change_columns:
+        lines.append(f"    track_history_column_list={list(contract.write.scd2_change_columns)!r},")
+    if flow_name:
+        lines.append(f"    name={flow_name!r},")
+    lines.append(")")
+    return "\n".join(lines) + "\n"

contractforge_databricks/lineage/__init__.py

@@ -0,0 +1,7 @@
+from contractforge_databricks.lineage.openlineage import (
+    build_openlineage_event,
+    openlineage_namespace,
+    render_openlineage_insert_sql,
+)
+
+__all__ = ["build_openlineage_event", "openlineage_namespace", "render_openlineage_insert_sql"]