contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/__init__.py

@@ -0,0 +1,172 @@
+"""Databricks adapter package for ContractForge Core."""
+
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+try:
+    __version__ = _pkg_version("contractforge-databricks")
+except PackageNotFoundError:  # pragma: no cover - editable installs without metadata
+    __version__ = "0.0.0+unknown"
+
+from contractforge_databricks.adapter import DatabricksAdapter
+from contractforge_databricks.api import plan_databricks_contract, render_databricks_contract
+from contractforge_databricks.capabilities.evaluate import evaluate_databricks_capabilities
+from contractforge_databricks.capabilities.mapping import to_core_capabilities
+from contractforge_databricks.capabilities.models import CapabilityEvidence, DatabricksCapabilities, NativeCapability
+from contractforge_databricks.capabilities.uc import uc_capability_issues
+from contractforge_databricks.cost import CostModel, build_operational_cost_report, render_operational_cost_query
+from contractforge_databricks.dashboards import render_control_dashboard_artifacts, render_control_dashboard_sql
+from contractforge_databricks.environment import DatabricksEnvironment
+from contractforge_databricks.bundles import render_databricks_project_bundle, render_databricks_project_bundle_yaml
+from contractforge_databricks.governance import (
+    access_drift_report,
+    apply_access_contract,
+    apply_governance_contract,
+    check_governance_contract,
+    governance_referenced_columns,
+    render_access_sql,
+    render_governance_sql,
+    validate_governance_contract,
+)
+from contractforge_databricks.lakeflow import (
+    LakeflowAutoCdcArtifact,
+    LakeflowCompatibility,
+    evaluate_lakeflow_compatibility,
+    render_lakeflow_auto_cdc_artifact,
+    render_lakeflow_auto_cdc_python,
+)
+from contractforge_databricks.maintenance import build_control_retention_plan, execute_control_retention_plan
+from contractforge_databricks.metrics import render_delta_history_query, resolve_write_metrics
+from contractforge_databricks.parity import (
+    ParityMetricExpectation,
+    WriteEngineParityScenario,
+    build_write_engine_parity_plan,
+    get_write_engine_parity_scenario,
+    list_write_engine_parity_scenarios,
+    scenarios_for_engine,
+    scenarios_for_mode,
+)
+from contractforge_databricks.presets import apply_preset, get_preset, list_presets, preset_details, register_preset
+from contractforge_databricks.preparation import apply_shape
+from contractforge_databricks.quality import (
+    get_quality_rule,
+    list_quality_rules,
+    register_quality_rule,
+    unregister_quality_rule,
+)
+from contractforge_databricks.runtime import (
+    DatabricksIngestOptions,
+    DatabricksIngestionHooks,
+    PreparedViewInput,
+    apply_databricks_access_bundle,
+    apply_databricks_annotations_bundle,
+    apply_databricks_governance_bundle,
+    detect_databricks_capabilities,
+    deploy_databricks_bundle,
+    deploy_databricks_project,
+    get_source_resolver,
+    ingest_databricks_bundle,
+    ingest_databricks_contract,
+    list_source_resolvers,
+    register_source_resolver,
+    render_databricks_project_bundle_file,
+    resolve_source_dataframe,
+    run_available_now_stream,
+    unregister_source_resolver,
+)
+from contractforge_databricks.templates import (
+    contract_template_details,
+    contract_template_files,
+    get_contract_template,
+    list_contract_templates,
+    recommend_contract_templates,
+)
+from contractforge_databricks.write_modes.registry import (
+    get_write_mode,
+    list_write_modes,
+    register_write_mode,
+    unregister_write_mode,
+)
+from contractforge_databricks.write_modes.strategy import WriteStrategy, choose_write_strategy
+
+__all__ = [
+    "CapabilityEvidence",
+    "CostModel",
+    "DatabricksAdapter",
+    "DatabricksCapabilities",
+    "DatabricksEnvironment",
+    "DatabricksIngestOptions",
+    "DatabricksIngestionHooks",
+    "LakeflowAutoCdcArtifact",
+    "LakeflowCompatibility",
+    "NativeCapability",
+    "ParityMetricExpectation",
+    "PreparedViewInput",
+    "WriteEngineParityScenario",
+    "WriteStrategy",
+    "access_drift_report",
+    "apply_access_contract",
+    "apply_databricks_access_bundle",
+    "apply_databricks_annotations_bundle",
+    "apply_databricks_governance_bundle",
+    "apply_governance_contract",
+    "apply_preset",
+    "apply_shape",
+    "build_control_retention_plan",
+    "build_operational_cost_report",
+    "build_write_engine_parity_plan",
+    "check_governance_contract",
+    "choose_write_strategy",
+    "contract_template_details",
+    "contract_template_files",
+    "detect_databricks_capabilities",
+    "deploy_databricks_bundle",
+    "deploy_databricks_project",
+    "evaluate_databricks_capabilities",
+    "evaluate_lakeflow_compatibility",
+    "execute_control_retention_plan",
+    "get_contract_template",
+    "get_preset",
+    "get_quality_rule",
+    "get_source_resolver",
+    "get_write_engine_parity_scenario",
+    "get_write_mode",
+    "governance_referenced_columns",
+    "ingest_databricks_bundle",
+    "ingest_databricks_contract",
+    "list_contract_templates",
+    "list_presets",
+    "list_quality_rules",
+    "list_source_resolvers",
+    "list_write_engine_parity_scenarios",
+    "list_write_modes",
+    "plan_databricks_contract",
+    "preset_details",
+    "recommend_contract_templates",
+    "register_preset",
+    "register_quality_rule",
+    "register_source_resolver",
+    "register_write_mode",
+    "render_access_sql",
+    "render_control_dashboard_artifacts",
+    "render_control_dashboard_sql",
+    "render_databricks_contract",
+    "render_databricks_project_bundle",
+    "render_databricks_project_bundle_file",
+    "render_databricks_project_bundle_yaml",
+    "render_delta_history_query",
+    "render_governance_sql",
+    "render_lakeflow_auto_cdc_artifact",
+    "render_lakeflow_auto_cdc_python",
+    "render_operational_cost_query",
+    "resolve_source_dataframe",
+    "resolve_write_metrics",
+    "run_available_now_stream",
+    "scenarios_for_engine",
+    "scenarios_for_mode",
+    "to_core_capabilities",
+    "uc_capability_issues",
+    "unregister_source_resolver",
+    "unregister_quality_rule",
+    "unregister_write_mode",
+    "validate_governance_contract",
+]

contractforge_databricks/adapter.py

@@ -0,0 +1,69 @@
+"""ContractForge Core adapter implementation for Databricks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from contractforge_core.adapters import RenderedArtifacts
+from contractforge_core.capabilities import PlatformCapabilities
+from contractforge_core.planner import ExecutionPlan, PlanningResult, plan_contract
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.capabilities import DatabricksCapabilities, evaluate_databricks_capabilities, to_core_capabilities
+from contractforge_databricks.contract_extensions import databricks_extension_warnings
+from contractforge_databricks.environment import DatabricksEnvironment
+from contractforge_databricks.rendering import render_databricks_artifacts
+
+
+@dataclass(frozen=True)
+class DatabricksAdapter:
+    """Dry-planning Databricks adapter.
+
+    Execution belongs in a later runtime module. This adapter only declares
+    capabilities, plans via the core, and renders reviewable native artifacts.
+    """
+
+    native_capabilities: DatabricksCapabilities
+    environment: DatabricksEnvironment = DatabricksEnvironment()
+    name: str = "databricks"
+
+    @classmethod
+    def from_evidence(
+        cls,
+        *,
+        target_table: str | None = None,
+        runtime_type: str | None = None,
+        spark_version: str | None = None,
+        spark_conf: dict[str, str] | None = None,
+        environment: dict[str, Any] | None = None,
+    ) -> "DatabricksAdapter":
+        env = DatabricksEnvironment.from_contract(environment)
+        return cls(
+            evaluate_databricks_capabilities(
+                target_table=target_table,
+                runtime_type=runtime_type or env.runtime_kind,
+                spark_version=spark_version,
+                spark_conf=spark_conf,
+            ),
+            environment=env,
+        )
+
+    def capabilities(self) -> PlatformCapabilities:
+        return to_core_capabilities(self.native_capabilities)
+
+    def plan(self, contract: SemanticContract) -> PlanningResult:
+        result = plan_contract(contract, self.capabilities())
+        extension_warnings = databricks_extension_warnings(contract)
+        if not extension_warnings:
+            return result
+        warnings = result.warnings + extension_warnings
+        if result.status == "SUPPORTED":
+            return PlanningResult(status="SUPPORTED_WITH_WARNINGS", plan=result.plan, blockers=result.blockers, warnings=warnings)
+        return PlanningResult(status=result.status, plan=result.plan, blockers=result.blockers, warnings=warnings)
+
+    def render(self, plan: ExecutionPlan) -> RenderedArtifacts:
+        raise NotImplementedError("Use render_contract(contract) so Databricks artifacts include contract context.")
+
+    def render_contract(self, contract: SemanticContract) -> RenderedArtifacts:
+        planning = self.plan(contract)
+        return render_databricks_artifacts(contract, planning, self.native_capabilities, environment=self.environment)

contractforge_databricks/annotations/__init__.py

@@ -0,0 +1,10 @@
+from contractforge_databricks.annotations.application import apply_annotations_contract
+from contractforge_databricks.annotations.audit import render_annotations_audit_insert_sql
+from contractforge_databricks.annotations.sql import annotation_steps, render_annotations_sql
+
+__all__ = [
+    "annotation_steps",
+    "apply_annotations_contract",
+    "render_annotations_audit_insert_sql",
+    "render_annotations_sql",
+]

contractforge_databricks/annotations/application.py

@@ -0,0 +1,52 @@
+"""Apply Databricks annotations with an injected SQL runner."""
+
+from __future__ import annotations
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_core.results import GovernanceApplyResult
+from contractforge_databricks.annotations.sql import annotation_steps
+from contractforge_databricks.execution.sql_merge import SqlRunner
+from contractforge_databricks.security import exception_message
+
+
+def apply_annotations_contract(*, runner: SqlRunner, contract: SemanticContract) -> GovernanceApplyResult:
+    steps = annotation_steps(contract)
+    if not steps:
+        return GovernanceApplyResult(status="NOT_CONFIGURED")
+    policy = _policy(contract)
+    sql_preview = tuple(str(step["sql"]) for step in steps)
+    if policy == "ignore":
+        return GovernanceApplyResult(status="IGNORED", ignored=len(steps), sql_preview=sql_preview)
+
+    applied = 0
+    errors: list[str] = []
+    for statement in sql_preview:
+        try:
+            runner.sql(statement)
+            applied += 1
+        except Exception as exc:
+            errors.append(exception_message(exc))
+            if policy == "fail":
+                return GovernanceApplyResult(
+                    status="FAILED",
+                    applied=applied,
+                    failed=len(errors),
+                    sql_preview=sql_preview,
+                    errors=tuple(errors),
+                )
+    if errors:
+        return GovernanceApplyResult(
+            status="WARNED",
+            applied=applied,
+            failed=len(errors),
+            sql_preview=sql_preview,
+            errors=tuple(errors),
+        )
+    return GovernanceApplyResult(status="SUCCESS", applied=applied, sql_preview=sql_preview)
+
+
+def _policy(contract: SemanticContract) -> str:
+    annotations = contract.governance.annotations if contract.governance else None
+    if isinstance(annotations, dict):
+        return str(annotations.get("policy", "warn"))
+    return "warn"

contractforge_databricks/annotations/audit.py

@@ -0,0 +1,49 @@
+"""Render annotation audit SQL for Databricks evidence tables."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.annotations.sql import annotation_steps
+from contractforge_databricks.evidence.tables import evidence_table_names
+from contractforge_databricks.rendering.names import target_full_name
+from contractforge_databricks.sql import quote_table_name, sql_string
+
+
+def render_annotations_audit_insert_sql(
+    contract: SemanticContract,
+    *,
+    run_id: str = "${run_id}",
+    status: str = "PLANNED",
+    captured_at_utc: datetime | None = None,
+    catalog: str = "main",
+    schema: str = "ops",
+) -> str:
+    steps = annotation_steps(contract)
+    if not steps:
+        return "-- No annotations intent declared.\n"
+    table = evidence_table_names(catalog, schema)["annotations"]
+    captured_at_utc = captured_at_utc or datetime(1970, 1, 1, 0, 0, 0)
+    statements = [
+        _audit_insert(table, run_id, target_full_name(contract), step, status, captured_at_utc)
+        for step in steps
+    ]
+    return ";\n".join(statements) + ";\n"
+
+
+def _audit_insert(table: str, run_id: str, target: str, step: dict[str, object], status: str, captured_at_utc: datetime) -> str:
+    columns = "run_id, target_table, annotation_scope, annotation_type, column_name, key, value, status, applied_sql, annotation_ts_utc"
+    values = [
+        sql_string(run_id),
+        sql_string(target),
+        sql_string(step.get("annotation_scope")),
+        sql_string(step.get("annotation_type")),
+        sql_string(step.get("column_name")),
+        sql_string(step.get("key")),
+        sql_string(step.get("value")),
+        sql_string(status),
+        sql_string(step.get("sql")),
+        f"TIMESTAMP {sql_string(captured_at_utc.strftime('%Y-%m-%d %H:%M:%S'))}",
+    ]
+    return f"INSERT INTO {quote_table_name(table)} ({columns}) VALUES ({', '.join(values)})"

contractforge_databricks/annotations/sql.py

@@ -0,0 +1,142 @@
+"""Render Unity Catalog annotations SQL."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.coercion import mapping, string_list, string_map
+from contractforge_databricks.rendering.names import target_full_name
+from contractforge_databricks.sql import quote_identifier, quote_table_name, sql_string
+
+
+def annotation_steps(contract: SemanticContract) -> list[dict[str, Any]]:
+    annotations = contract.governance.annotations if contract.governance else None
+    if not isinstance(annotations, dict):
+        return []
+
+    target = target_full_name(contract)
+    quoted_target = quote_table_name(target)
+    steps: list[dict[str, Any]] = []
+    table = mapping(annotations.get("table"))
+    if table.get("description"):
+        steps.append(
+            {
+                "annotation_scope": "table",
+                "annotation_type": "description",
+                "column_name": None,
+                "key": "description",
+                "value": str(table["description"]),
+                "sql": f"COMMENT ON TABLE {quoted_target} IS {sql_string(table['description'])}",
+            }
+        )
+    _append_tag_step(steps, quoted_target, None, _table_tags(table))
+    for column, config in mapping(annotations.get("columns")).items():
+        _append_column_steps(steps, quoted_target, str(column), mapping(config))
+    return steps
+
+
+def render_annotations_sql(contract: SemanticContract) -> str:
+    annotations = contract.governance.annotations if contract.governance else None
+    if not isinstance(annotations, dict):
+        return "-- No annotations intent declared.\n"
+
+    target = target_full_name(contract)
+    lines = [
+        "-- Review before execution. Unity Catalog tags require UC-enabled targets.",
+        f"-- Target: {target}",
+        "",
+    ]
+    lines.extend(f"{step['sql']};" for step in annotation_steps(contract))
+    return "\n".join(lines) + "\n"
+
+
+def _append_column_steps(
+    steps: list[dict[str, Any]],
+    quoted_target: str,
+    column: str,
+    config: dict[str, Any],
+) -> None:
+    quoted_column = quote_identifier(column)
+    if config.get("description"):
+        steps.append(
+            {
+                "annotation_scope": "column",
+                "annotation_type": "description",
+                "column_name": column,
+                "key": "description",
+                "value": str(config["description"]),
+                "sql": f"ALTER TABLE {quoted_target} ALTER COLUMN {quoted_column} COMMENT {sql_string(config['description'])}",
+            }
+        )
+    _append_tag_step(steps, quoted_target, column, _column_tags(config))
+
+
+def _append_tag_step(steps: list[dict[str, Any]], quoted_target: str, column: str | None, tags: dict[str, str]) -> None:
+    if tags:
+        sql = f"ALTER TABLE {quoted_target} SET TAGS ({_tag_sql(tags)})"
+        if column is not None:
+            sql = f"ALTER TABLE {quoted_target} ALTER COLUMN {quote_identifier(column)} SET TAGS ({_tag_sql(tags)})"
+        steps.append(
+            {
+                "annotation_scope": "column" if column else "table",
+                "annotation_type": "tags",
+                "column_name": column,
+                "key": "tags",
+                "value": _json(tags),
+                "sql": sql,
+            }
+        )
+
+
+def _table_tags(table: dict[str, Any]) -> dict[str, str]:
+    return {
+        **string_map(table.get("tags")),
+        **_alias_tags(table.get("aliases")),
+        **_deprecated_tags(table.get("deprecated")),
+    }
+
+
+def _column_tags(config: dict[str, Any]) -> dict[str, str]:
+    return {
+        **string_map(config.get("tags")),
+        **_alias_tags(config.get("aliases")),
+        **_pii_tags(config.get("pii")),
+        **_deprecated_tags(config.get("deprecated")),
+    }
+
+
+def _alias_tags(value: object) -> dict[str, str]:
+    return {f"alias_{idx}": alias for idx, alias in enumerate(string_list(value, sep="|"), start=1)}
+
+
+def _deprecated_tags(value: object) -> dict[str, str]:
+    deprecated = mapping(value)
+    if not deprecated:
+        return {}
+    tags = {"deprecated": "true"}
+    for key in ("since", "replacement", "removal_date"):
+        if deprecated.get(key):
+            tags[f"deprecated_{key}"] = str(deprecated[key])
+    return tags
+
+
+def _pii_tags(value: object) -> dict[str, str]:
+    pii = mapping(value)
+    if not pii:
+        return {}
+    return {
+        "pii": str(pii.get("enabled", True)).lower(),
+        "pii_type": str(pii.get("type", "unknown")),
+        "sensitivity": str(pii.get("sensitivity", "internal")),
+    }
+
+
+def _tag_sql(tags: dict[str, str]) -> str:
+    return ", ".join(f"{sql_string(key)} = {sql_string(value)}" for key, value in tags.items())
+
+
+def _json(value: dict[str, str]) -> str:
+    import json
+
+    return json.dumps(value, sort_keys=True, separators=(",", ":"))

contractforge_databricks/api.py

@@ -0,0 +1,65 @@
+"""High-level Databricks adapter API."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.adapters import RenderedArtifacts
+from contractforge_core.contracts import semantic_contract_from_mapping
+from contractforge_core.planner import PlanningResult
+from contractforge_databricks.adapter import DatabricksAdapter
+from contractforge_databricks.contract_extensions import normalize_databricks_contract
+
+
+def plan_databricks_contract(
+    contract: dict[str, Any],
+    *,
+    target_table: str | None = None,
+    runtime_type: str | None = None,
+    spark_version: str | None = None,
+    spark_conf: dict[str, str] | None = None,
+    environment: dict[str, Any] | None = None,
+) -> PlanningResult:
+    semantic = semantic_contract_from_mapping(normalize_databricks_contract(contract))
+    adapter = DatabricksAdapter.from_evidence(
+        target_table=target_table or _target_table(contract),
+        runtime_type=runtime_type,
+        spark_version=spark_version,
+        spark_conf=spark_conf,
+        environment=environment,
+    )
+    return adapter.plan(semantic)
+
+
+def render_databricks_contract(
+    contract: dict[str, Any],
+    *,
+    target_table: str | None = None,
+    runtime_type: str | None = None,
+    spark_version: str | None = None,
+    spark_conf: dict[str, str] | None = None,
+    environment: dict[str, Any] | None = None,
+) -> RenderedArtifacts:
+    semantic = semantic_contract_from_mapping(normalize_databricks_contract(contract))
+    adapter = DatabricksAdapter.from_evidence(
+        target_table=target_table or _target_table(contract),
+        runtime_type=runtime_type,
+        spark_version=spark_version,
+        spark_conf=spark_conf,
+        environment=environment,
+    )
+    return adapter.render_contract(semantic)
+
+
+def _target_table(contract: dict[str, Any]) -> str | None:
+    target = contract.get("target")
+    if isinstance(target, dict):
+        parts = [target.get("catalog"), target.get("schema"), target.get("table")]
+        if all(parts):
+            return ".".join(str(part) for part in parts)
+    table = contract.get("target_table")
+    catalog = contract.get("catalog")
+    schema = contract.get("target_schema")
+    if table and catalog and schema:
+        return f"{catalog}.{schema}.{table}"
+    return str(table) if table else None

contractforge_databricks/bundles/__init__.py

@@ -0,0 +1,9 @@
+from contractforge_databricks.bundles.assets import DatabricksJobSpec, render_databricks_asset_bundle
+from contractforge_databricks.bundles.project import render_databricks_project_bundle, render_databricks_project_bundle_yaml
+
+__all__ = [
+    "DatabricksJobSpec",
+    "render_databricks_asset_bundle",
+    "render_databricks_project_bundle",
+    "render_databricks_project_bundle_yaml",
+]

contractforge_databricks/bundles/assets.py

@@ -0,0 +1,47 @@
+"""Databricks Asset Bundle rendering."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class DatabricksJobSpec:
+    bundle_name: str
+    job_name: str
+    task_key: str
+    notebook_path: str
+    target: str = "dev"
+
+
+def render_databricks_asset_bundle(spec: DatabricksJobSpec) -> str:
+    _require_non_empty(spec.bundle_name, "bundle_name")
+    _require_non_empty(spec.job_name, "job_name")
+    _require_non_empty(spec.task_key, "task_key")
+    _require_non_empty(spec.notebook_path, "notebook_path")
+    return "\n".join(
+        [
+            "bundle:",
+            f"  name: {spec.bundle_name}",
+            "",
+            "resources:",
+            "  jobs:",
+            f"    {spec.job_name}:",
+            f"      name: {spec.job_name}",
+            "      tasks:",
+            f"        - task_key: {spec.task_key}",
+            "          notebook_task:",
+            f"            notebook_path: {spec.notebook_path}",
+            "",
+            "targets:",
+            f"  {spec.target}:",
+            "    mode: development",
+            "",
+        ]
+    )
+
+
+def _require_non_empty(value: str, field_name: str) -> None:
+    if not value or not value.strip():
+        raise ValueError(f"{field_name} must not be empty")
+