contractforge-databricks 0.1.0__tar.gz

contractforge_databricks-0.1.0/.gitignore

@@ -0,0 +1,18 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.uv-cache/
+.venv/
+.tmp/
+dist/
+build/
+*.egg
+.coverage
+.coverage.*
+htmlcov/
+.DS_Store
+.env
+.env.*
+!.env.example

contractforge_databricks-0.1.0/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+All notable changes to `contractforge-databricks` are documented in this file.
+
+The format follows Keep a Changelog, and this package follows semantic
+versioning as described in `../../docs/specs/api-stability.md`.
+
+## [0.1.0] - 2026-06-08
+
+### Added
+
+- Initial public alpha release of the Databricks adapter.
+- Databricks contract planning, SQL/artifact rendering, Asset Bundle support,
+  runtime helpers, governance, quality, evidence, lineage and operational
+  cost surfaces.

contractforge_databricks-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: contractforge-databricks
+Version: 0.1.0
+Summary: Databricks adapter for ContractForge Core.
+Project-URL: Homepage, https://github.com/marquesantero/contractforge-core/tree/main/adapters/databricks
+Project-URL: Documentation, https://marquesantero.github.io/contractforge-core/docs/adapters/databricks
+Project-URL: Repository, https://github.com/marquesantero/contractforge-core
+Project-URL: Issues, https://github.com/marquesantero/contractforge-core/issues
+Project-URL: Changelog, https://github.com/marquesantero/contractforge-core/blob/main/adapters/databricks/CHANGELOG.md
+Author: ContractForge contributors
+License: MIT
+Keywords: contractforge,data-contracts,databricks,ingestion,lakehouse
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: contractforge-core<0.2,>=0.1
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# ContractForge Databricks Adapter
+
+Databricks adapter for ContractForge.
+
+This package depends on `contractforge-core` and translates platform-neutral
+ingestion contracts into Databricks-native execution behavior and artifacts.

contractforge_databricks-0.1.0/README.md

@@ -0,0 +1,6 @@
+# ContractForge Databricks Adapter
+
+Databricks adapter for ContractForge.
+
+This package depends on `contractforge-core` and translates platform-neutral
+ingestion contracts into Databricks-native execution behavior and artifacts.

contractforge_databricks-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "contractforge-databricks"
+version = "0.1.0"
+description = "Databricks adapter for ContractForge Core."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "ContractForge contributors" }]
+keywords = ["contractforge", "databricks", "data-contracts", "ingestion", "lakehouse"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = ["contractforge-core>=0.1,<0.2"]
+
+[project.urls]
+Homepage = "https://github.com/marquesantero/contractforge-core/tree/main/adapters/databricks"
+Documentation = "https://marquesantero.github.io/contractforge-core/docs/adapters/databricks"
+Repository = "https://github.com/marquesantero/contractforge-core"
+Issues = "https://github.com/marquesantero/contractforge-core/issues"
+Changelog = "https://github.com/marquesantero/contractforge-core/blob/main/adapters/databricks/CHANGELOG.md"
+
+[project.optional-dependencies]
+dev = ["pytest>=8"]
+
+[project.scripts]
+contractforge-databricks = "contractforge_databricks.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/contractforge_databricks"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/src/contractforge_databricks", "/README.md", "/CHANGELOG.md", "/pyproject.toml"]

contractforge_databricks-0.1.0/src/contractforge_databricks/__init__.py

@@ -0,0 +1,172 @@
+"""Databricks adapter package for ContractForge Core."""
+
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+try:
+    __version__ = _pkg_version("contractforge-databricks")
+except PackageNotFoundError:  # pragma: no cover - editable installs without metadata
+    __version__ = "0.0.0+unknown"
+
+from contractforge_databricks.adapter import DatabricksAdapter
+from contractforge_databricks.api import plan_databricks_contract, render_databricks_contract
+from contractforge_databricks.capabilities.evaluate import evaluate_databricks_capabilities
+from contractforge_databricks.capabilities.mapping import to_core_capabilities
+from contractforge_databricks.capabilities.models import CapabilityEvidence, DatabricksCapabilities, NativeCapability
+from contractforge_databricks.capabilities.uc import uc_capability_issues
+from contractforge_databricks.cost import CostModel, build_operational_cost_report, render_operational_cost_query
+from contractforge_databricks.dashboards import render_control_dashboard_artifacts, render_control_dashboard_sql
+from contractforge_databricks.environment import DatabricksEnvironment
+from contractforge_databricks.bundles import render_databricks_project_bundle, render_databricks_project_bundle_yaml
+from contractforge_databricks.governance import (
+    access_drift_report,
+    apply_access_contract,
+    apply_governance_contract,
+    check_governance_contract,
+    governance_referenced_columns,
+    render_access_sql,
+    render_governance_sql,
+    validate_governance_contract,
+)
+from contractforge_databricks.lakeflow import (
+    LakeflowAutoCdcArtifact,
+    LakeflowCompatibility,
+    evaluate_lakeflow_compatibility,
+    render_lakeflow_auto_cdc_artifact,
+    render_lakeflow_auto_cdc_python,
+)
+from contractforge_databricks.maintenance import build_control_retention_plan, execute_control_retention_plan
+from contractforge_databricks.metrics import render_delta_history_query, resolve_write_metrics
+from contractforge_databricks.parity import (
+    ParityMetricExpectation,
+    WriteEngineParityScenario,
+    build_write_engine_parity_plan,
+    get_write_engine_parity_scenario,
+    list_write_engine_parity_scenarios,
+    scenarios_for_engine,
+    scenarios_for_mode,
+)
+from contractforge_databricks.presets import apply_preset, get_preset, list_presets, preset_details, register_preset
+from contractforge_databricks.preparation import apply_shape
+from contractforge_databricks.quality import (
+    get_quality_rule,
+    list_quality_rules,
+    register_quality_rule,
+    unregister_quality_rule,
+)
+from contractforge_databricks.runtime import (
+    DatabricksIngestOptions,
+    DatabricksIngestionHooks,
+    PreparedViewInput,
+    apply_databricks_access_bundle,
+    apply_databricks_annotations_bundle,
+    apply_databricks_governance_bundle,
+    detect_databricks_capabilities,
+    deploy_databricks_bundle,
+    deploy_databricks_project,
+    get_source_resolver,
+    ingest_databricks_bundle,
+    ingest_databricks_contract,
+    list_source_resolvers,
+    register_source_resolver,
+    render_databricks_project_bundle_file,
+    resolve_source_dataframe,
+    run_available_now_stream,
+    unregister_source_resolver,
+)
+from contractforge_databricks.templates import (
+    contract_template_details,
+    contract_template_files,
+    get_contract_template,
+    list_contract_templates,
+    recommend_contract_templates,
+)
+from contractforge_databricks.write_modes.registry import (
+    get_write_mode,
+    list_write_modes,
+    register_write_mode,
+    unregister_write_mode,
+)
+from contractforge_databricks.write_modes.strategy import WriteStrategy, choose_write_strategy
+
+__all__ = [
+    "CapabilityEvidence",
+    "CostModel",
+    "DatabricksAdapter",
+    "DatabricksCapabilities",
+    "DatabricksEnvironment",
+    "DatabricksIngestOptions",
+    "DatabricksIngestionHooks",
+    "LakeflowAutoCdcArtifact",
+    "LakeflowCompatibility",
+    "NativeCapability",
+    "ParityMetricExpectation",
+    "PreparedViewInput",
+    "WriteEngineParityScenario",
+    "WriteStrategy",
+    "access_drift_report",
+    "apply_access_contract",
+    "apply_databricks_access_bundle",
+    "apply_databricks_annotations_bundle",
+    "apply_databricks_governance_bundle",
+    "apply_governance_contract",
+    "apply_preset",
+    "apply_shape",
+    "build_control_retention_plan",
+    "build_operational_cost_report",
+    "build_write_engine_parity_plan",
+    "check_governance_contract",
+    "choose_write_strategy",
+    "contract_template_details",
+    "contract_template_files",
+    "detect_databricks_capabilities",
+    "deploy_databricks_bundle",
+    "deploy_databricks_project",
+    "evaluate_databricks_capabilities",
+    "evaluate_lakeflow_compatibility",
+    "execute_control_retention_plan",
+    "get_contract_template",
+    "get_preset",
+    "get_quality_rule",
+    "get_source_resolver",
+    "get_write_engine_parity_scenario",
+    "get_write_mode",
+    "governance_referenced_columns",
+    "ingest_databricks_bundle",
+    "ingest_databricks_contract",
+    "list_contract_templates",
+    "list_presets",
+    "list_quality_rules",
+    "list_source_resolvers",
+    "list_write_engine_parity_scenarios",
+    "list_write_modes",
+    "plan_databricks_contract",
+    "preset_details",
+    "recommend_contract_templates",
+    "register_preset",
+    "register_quality_rule",
+    "register_source_resolver",
+    "register_write_mode",
+    "render_access_sql",
+    "render_control_dashboard_artifacts",
+    "render_control_dashboard_sql",
+    "render_databricks_contract",
+    "render_databricks_project_bundle",
+    "render_databricks_project_bundle_file",
+    "render_databricks_project_bundle_yaml",
+    "render_delta_history_query",
+    "render_governance_sql",
+    "render_lakeflow_auto_cdc_artifact",
+    "render_lakeflow_auto_cdc_python",
+    "render_operational_cost_query",
+    "resolve_source_dataframe",
+    "resolve_write_metrics",
+    "run_available_now_stream",
+    "scenarios_for_engine",
+    "scenarios_for_mode",
+    "to_core_capabilities",
+    "uc_capability_issues",
+    "unregister_source_resolver",
+    "unregister_quality_rule",
+    "unregister_write_mode",
+    "validate_governance_contract",
+]

contractforge_databricks-0.1.0/src/contractforge_databricks/adapter.py

@@ -0,0 +1,69 @@
+"""ContractForge Core adapter implementation for Databricks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from contractforge_core.adapters import RenderedArtifacts
+from contractforge_core.capabilities import PlatformCapabilities
+from contractforge_core.planner import ExecutionPlan, PlanningResult, plan_contract
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.capabilities import DatabricksCapabilities, evaluate_databricks_capabilities, to_core_capabilities
+from contractforge_databricks.contract_extensions import databricks_extension_warnings
+from contractforge_databricks.environment import DatabricksEnvironment
+from contractforge_databricks.rendering import render_databricks_artifacts
+
+
+@dataclass(frozen=True)
+class DatabricksAdapter:
+    """Dry-planning Databricks adapter.
+
+    Execution belongs in a later runtime module. This adapter only declares
+    capabilities, plans via the core, and renders reviewable native artifacts.
+    """
+
+    native_capabilities: DatabricksCapabilities
+    environment: DatabricksEnvironment = DatabricksEnvironment()
+    name: str = "databricks"
+
+    @classmethod
+    def from_evidence(
+        cls,
+        *,
+        target_table: str | None = None,
+        runtime_type: str | None = None,
+        spark_version: str | None = None,
+        spark_conf: dict[str, str] | None = None,
+        environment: dict[str, Any] | None = None,
+    ) -> "DatabricksAdapter":
+        env = DatabricksEnvironment.from_contract(environment)
+        return cls(
+            evaluate_databricks_capabilities(
+                target_table=target_table,
+                runtime_type=runtime_type or env.runtime_kind,
+                spark_version=spark_version,
+                spark_conf=spark_conf,
+            ),
+            environment=env,
+        )
+
+    def capabilities(self) -> PlatformCapabilities:
+        return to_core_capabilities(self.native_capabilities)
+
+    def plan(self, contract: SemanticContract) -> PlanningResult:
+        result = plan_contract(contract, self.capabilities())
+        extension_warnings = databricks_extension_warnings(contract)
+        if not extension_warnings:
+            return result
+        warnings = result.warnings + extension_warnings
+        if result.status == "SUPPORTED":
+            return PlanningResult(status="SUPPORTED_WITH_WARNINGS", plan=result.plan, blockers=result.blockers, warnings=warnings)
+        return PlanningResult(status=result.status, plan=result.plan, blockers=result.blockers, warnings=warnings)
+
+    def render(self, plan: ExecutionPlan) -> RenderedArtifacts:
+        raise NotImplementedError("Use render_contract(contract) so Databricks artifacts include contract context.")
+
+    def render_contract(self, contract: SemanticContract) -> RenderedArtifacts:
+        planning = self.plan(contract)
+        return render_databricks_artifacts(contract, planning, self.native_capabilities, environment=self.environment)

contractforge_databricks-0.1.0/src/contractforge_databricks/annotations/__init__.py

@@ -0,0 +1,10 @@
+from contractforge_databricks.annotations.application import apply_annotations_contract
+from contractforge_databricks.annotations.audit import render_annotations_audit_insert_sql
+from contractforge_databricks.annotations.sql import annotation_steps, render_annotations_sql
+
+__all__ = [
+    "annotation_steps",
+    "apply_annotations_contract",
+    "render_annotations_audit_insert_sql",
+    "render_annotations_sql",
+]

contractforge_databricks-0.1.0/src/contractforge_databricks/annotations/application.py

@@ -0,0 +1,52 @@
+"""Apply Databricks annotations with an injected SQL runner."""
+
+from __future__ import annotations
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_core.results import GovernanceApplyResult
+from contractforge_databricks.annotations.sql import annotation_steps
+from contractforge_databricks.execution.sql_merge import SqlRunner
+from contractforge_databricks.security import exception_message
+
+
+def apply_annotations_contract(*, runner: SqlRunner, contract: SemanticContract) -> GovernanceApplyResult:
+    steps = annotation_steps(contract)
+    if not steps:
+        return GovernanceApplyResult(status="NOT_CONFIGURED")
+    policy = _policy(contract)
+    sql_preview = tuple(str(step["sql"]) for step in steps)
+    if policy == "ignore":
+        return GovernanceApplyResult(status="IGNORED", ignored=len(steps), sql_preview=sql_preview)
+
+    applied = 0
+    errors: list[str] = []
+    for statement in sql_preview:
+        try:
+            runner.sql(statement)
+            applied += 1
+        except Exception as exc:
+            errors.append(exception_message(exc))
+            if policy == "fail":
+                return GovernanceApplyResult(
+                    status="FAILED",
+                    applied=applied,
+                    failed=len(errors),
+                    sql_preview=sql_preview,
+                    errors=tuple(errors),
+                )
+    if errors:
+        return GovernanceApplyResult(
+            status="WARNED",
+            applied=applied,
+            failed=len(errors),
+            sql_preview=sql_preview,
+            errors=tuple(errors),
+        )
+    return GovernanceApplyResult(status="SUCCESS", applied=applied, sql_preview=sql_preview)
+
+
+def _policy(contract: SemanticContract) -> str:
+    annotations = contract.governance.annotations if contract.governance else None
+    if isinstance(annotations, dict):
+        return str(annotations.get("policy", "warn"))
+    return "warn"

contractforge_databricks-0.1.0/src/contractforge_databricks/annotations/audit.py

@@ -0,0 +1,49 @@
+"""Render annotation audit SQL for Databricks evidence tables."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.annotations.sql import annotation_steps
+from contractforge_databricks.evidence.tables import evidence_table_names
+from contractforge_databricks.rendering.names import target_full_name
+from contractforge_databricks.sql import quote_table_name, sql_string
+
+
+def render_annotations_audit_insert_sql(
+    contract: SemanticContract,
+    *,
+    run_id: str = "${run_id}",
+    status: str = "PLANNED",
+    captured_at_utc: datetime | None = None,
+    catalog: str = "main",
+    schema: str = "ops",
+) -> str:
+    steps = annotation_steps(contract)
+    if not steps:
+        return "-- No annotations intent declared.\n"
+    table = evidence_table_names(catalog, schema)["annotations"]
+    captured_at_utc = captured_at_utc or datetime(1970, 1, 1, 0, 0, 0)
+    statements = [
+        _audit_insert(table, run_id, target_full_name(contract), step, status, captured_at_utc)
+        for step in steps
+    ]
+    return ";\n".join(statements) + ";\n"
+
+
+def _audit_insert(table: str, run_id: str, target: str, step: dict[str, object], status: str, captured_at_utc: datetime) -> str:
+    columns = "run_id, target_table, annotation_scope, annotation_type, column_name, key, value, status, applied_sql, annotation_ts_utc"
+    values = [
+        sql_string(run_id),
+        sql_string(target),
+        sql_string(step.get("annotation_scope")),
+        sql_string(step.get("annotation_type")),
+        sql_string(step.get("column_name")),
+        sql_string(step.get("key")),
+        sql_string(step.get("value")),
+        sql_string(status),
+        sql_string(step.get("sql")),
+        f"TIMESTAMP {sql_string(captured_at_utc.strftime('%Y-%m-%d %H:%M:%S'))}",
+    ]
+    return f"INSERT INTO {quote_table_name(table)} ({columns}) VALUES ({', '.join(values)})"

contractforge_databricks-0.1.0/src/contractforge_databricks/annotations/sql.py

@@ -0,0 +1,142 @@
+"""Render Unity Catalog annotations SQL."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.coercion import mapping, string_list, string_map
+from contractforge_databricks.rendering.names import target_full_name
+from contractforge_databricks.sql import quote_identifier, quote_table_name, sql_string
+
+
+def annotation_steps(contract: SemanticContract) -> list[dict[str, Any]]:
+    annotations = contract.governance.annotations if contract.governance else None
+    if not isinstance(annotations, dict):
+        return []
+
+    target = target_full_name(contract)
+    quoted_target = quote_table_name(target)
+    steps: list[dict[str, Any]] = []
+    table = mapping(annotations.get("table"))
+    if table.get("description"):
+        steps.append(
+            {
+                "annotation_scope": "table",
+                "annotation_type": "description",
+                "column_name": None,
+                "key": "description",
+                "value": str(table["description"]),
+                "sql": f"COMMENT ON TABLE {quoted_target} IS {sql_string(table['description'])}",
+            }
+        )
+    _append_tag_step(steps, quoted_target, None, _table_tags(table))
+    for column, config in mapping(annotations.get("columns")).items():
+        _append_column_steps(steps, quoted_target, str(column), mapping(config))
+    return steps
+
+
+def render_annotations_sql(contract: SemanticContract) -> str:
+    annotations = contract.governance.annotations if contract.governance else None
+    if not isinstance(annotations, dict):
+        return "-- No annotations intent declared.\n"
+
+    target = target_full_name(contract)
+    lines = [
+        "-- Review before execution. Unity Catalog tags require UC-enabled targets.",
+        f"-- Target: {target}",
+        "",
+    ]
+    lines.extend(f"{step['sql']};" for step in annotation_steps(contract))
+    return "\n".join(lines) + "\n"
+
+
+def _append_column_steps(
+    steps: list[dict[str, Any]],
+    quoted_target: str,
+    column: str,
+    config: dict[str, Any],
+) -> None:
+    quoted_column = quote_identifier(column)
+    if config.get("description"):
+        steps.append(
+            {
+                "annotation_scope": "column",
+                "annotation_type": "description",
+                "column_name": column,
+                "key": "description",
+                "value": str(config["description"]),
+                "sql": f"ALTER TABLE {quoted_target} ALTER COLUMN {quoted_column} COMMENT {sql_string(config['description'])}",
+            }
+        )
+    _append_tag_step(steps, quoted_target, column, _column_tags(config))
+
+
+def _append_tag_step(steps: list[dict[str, Any]], quoted_target: str, column: str | None, tags: dict[str, str]) -> None:
+    if tags:
+        sql = f"ALTER TABLE {quoted_target} SET TAGS ({_tag_sql(tags)})"
+        if column is not None:
+            sql = f"ALTER TABLE {quoted_target} ALTER COLUMN {quote_identifier(column)} SET TAGS ({_tag_sql(tags)})"
+        steps.append(
+            {
+                "annotation_scope": "column" if column else "table",
+                "annotation_type": "tags",
+                "column_name": column,
+                "key": "tags",
+                "value": _json(tags),
+                "sql": sql,
+            }
+        )
+
+
+def _table_tags(table: dict[str, Any]) -> dict[str, str]:
+    return {
+        **string_map(table.get("tags")),
+        **_alias_tags(table.get("aliases")),
+        **_deprecated_tags(table.get("deprecated")),
+    }
+
+
+def _column_tags(config: dict[str, Any]) -> dict[str, str]:
+    return {
+        **string_map(config.get("tags")),
+        **_alias_tags(config.get("aliases")),
+        **_pii_tags(config.get("pii")),
+        **_deprecated_tags(config.get("deprecated")),
+    }
+
+
+def _alias_tags(value: object) -> dict[str, str]:
+    return {f"alias_{idx}": alias for idx, alias in enumerate(string_list(value, sep="|"), start=1)}
+
+
+def _deprecated_tags(value: object) -> dict[str, str]:
+    deprecated = mapping(value)
+    if not deprecated:
+        return {}
+    tags = {"deprecated": "true"}
+    for key in ("since", "replacement", "removal_date"):
+        if deprecated.get(key):
+            tags[f"deprecated_{key}"] = str(deprecated[key])
+    return tags
+
+
+def _pii_tags(value: object) -> dict[str, str]:
+    pii = mapping(value)
+    if not pii:
+        return {}
+    return {
+        "pii": str(pii.get("enabled", True)).lower(),
+        "pii_type": str(pii.get("type", "unknown")),
+        "sensitivity": str(pii.get("sensitivity", "internal")),
+    }
+
+
+def _tag_sql(tags: dict[str, str]) -> str:
+    return ", ".join(f"{sql_string(key)} = {sql_string(value)}" for key, value in tags.items())
+
+
+def _json(value: dict[str, str]) -> str:
+    import json
+
+    return json.dumps(value, sort_keys=True, separators=(",", ":"))