contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/templates/core.py

@@ -0,0 +1,83 @@
+"""Template helpers for Databricks adapter contract examples."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+import json
+from typing import Any
+
+from contractforge_databricks.templates.catalog import BUILTIN_CONTRACT_TEMPLATES, TEMPLATE_META_KEY, ContractTemplate
+
+
+def list_contract_templates() -> list[str]:
+    return sorted(BUILTIN_CONTRACT_TEMPLATES)
+
+
+def get_contract_template(name: str) -> ContractTemplate:
+    if name not in BUILTIN_CONTRACT_TEMPLATES:
+        raise ValueError(f"Template not found: {name}. Valid templates: {list_contract_templates()}")
+    return deepcopy(BUILTIN_CONTRACT_TEMPLATES[name])
+
+
+def contract_template_files(name: str) -> dict[str, dict[str, Any]]:
+    template = get_contract_template(name)
+    return {
+        key: deepcopy(template[key])
+        for key in ("ingestion", "annotations", "operations", "access")
+        if key in template
+    }
+
+
+def contract_template_details(name: str) -> dict[str, Any]:
+    template = get_contract_template(name)
+    meta = dict(template.get(TEMPLATE_META_KEY) or {})
+    ingestion = template.get("ingestion") or {}
+    return {
+        "name": name,
+        "description": meta.get("description", ""),
+        "category": meta.get("category", "custom"),
+        "files": [key for key in ("ingestion", "annotations", "operations", "access") if key in template],
+        "target": ingestion.get("target"),
+        "presets": ingestion.get("preset"),
+        "source": _source_kind(template),
+        "mode": ingestion.get("mode"),
+        "recommendation_priority": meta.get("recommendation_priority", 100),
+    }
+
+
+def recommend_contract_templates(
+    *,
+    layer: str | None = None,
+    source: str | None = None,
+    mode: str | None = None,
+    pattern: str | None = None,
+    limit: int | None = None,
+) -> list[dict[str, Any]]:
+    criteria = {"layer": _norm(layer), "source": _norm(source), "mode": _norm(mode), "pattern": _norm(pattern)}
+    has_criteria = any(criteria.values())
+    matches = []
+    for name in list_contract_templates():
+        details = contract_template_details(name)
+        haystack = _norm(json.dumps({"name": name, "details": details, "template": get_contract_template(name)}))
+        score = 0
+        matched: list[str] = []
+        for key, weight in (("layer", 4), ("source", 3), ("mode", 3), ("pattern", 2)):
+            if criteria[key] and criteria[key] in haystack:
+                score += weight
+                matched.append(key)
+        if has_criteria and score == 0:
+            continue
+        matches.append({**details, "score": score, "matched": matched})
+    matches.sort(key=lambda item: (-int(item["score"]), int(item["recommendation_priority"]), str(item["name"])))
+    return matches[: max(0, int(limit))] if limit is not None else matches
+
+
+def _source_kind(template: ContractTemplate) -> str:
+    source = (template.get("ingestion") or {}).get("source")
+    if isinstance(source, dict):
+        return str(source.get("type") or source.get("connector") or "connector")
+    return str(source or "unknown")
+
+
+def _norm(value: Any) -> str:
+    return str(value or "").strip().lower().replace("-", "_")

contractforge_databricks/templates/enrichment.py

@@ -0,0 +1,175 @@
+"""Parity enrichments for Databricks contract templates."""
+
+from __future__ import annotations
+
+from typing import Any
+
+ContractTemplate = dict[str, Any]
+
+
+def enrich_contractforge_parity(templates: dict[str, ContractTemplate]) -> None:
+    """Restore mature original-template parameters using core canonical names."""
+    _bronze_rest_api_incremental(templates)
+    _bronze_http_file_csv_snapshot(templates)
+    _bronze_autoloader_governed_delta(templates)
+    _silver_jdbc_scd1_upsert(templates)
+    _silver_jdbc_rds_iam_hash_diff(templates)
+    _silver_scd2_history(templates)
+    _silver_snapshot_soft_delete(templates)
+
+
+def _bronze_rest_api_incremental(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["bronze_rest_api_incremental"]["ingestion"]
+    ingestion.update(
+        {
+            "source": {
+                "type": "rest_api",
+                "name": "orders_api",
+                "request": {"url": "https://api.example.com/orders", "params": {"status": "open"}},
+                "auth": {"type": "bearer_token", "token": "{{ secret:orders_api/token }}"},
+                "pagination": {"type": "cursor", "cursor_param": "cursor", "next_cursor_path": "$.next"},
+                "response": {"records_path": "$.data"},
+                "incremental": {
+                    "watermark_param": "updated_after",
+                    "watermark_header": "X-Watermark",
+                    "initial_value": "1970-01-01T00:00:00Z",
+                },
+                "limits": {"max_pages": 100, "timeout_seconds": 60, "retry_attempts": 3},
+            },
+            "mode": "scd0_append",
+            "watermark_columns": ["updated_at"],
+            "schema_policy": "additive_only",
+            "quality_rules": {
+                "not_null": ["id"],
+                "expressions": [
+                    {
+                        "name": "valid_updated_at",
+                        "expression": "updated_at IS NOT NULL",
+                        "severity": "warn",
+                        "message": "updated_at is missing from the API payload.",
+                    }
+                ],
+            },
+        }
+    )
+
+
+def _bronze_http_file_csv_snapshot(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["bronze_http_file_csv_snapshot"]["ingestion"]
+    ingestion.update(
+        {
+            "source": {
+                "type": "http_csv",
+                "url": "https://example.com/public/orders.csv",
+                "format": "csv",
+                "options": {"header": True, "multiLine": False},
+                "read": {
+                    "source_complete": True,
+                    "schema": "order_id STRING, order_date DATE, customer_id STRING, amount DOUBLE, updated_at TIMESTAMP",
+                    "timeout_seconds": 120,
+                },
+            },
+            "schema_policy": "additive_only",
+            "quality_rules": {
+                "not_null": ["order_id"],
+                "expressions": [
+                    {
+                        "name": "valid_amount",
+                        "expression": "amount IS NULL OR amount >= 0",
+                        "severity": "warn",
+                        "message": "Negative amount in HTTP CSV.",
+                    }
+                ],
+            },
+        }
+    )
+
+
+def _bronze_autoloader_governed_delta(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["bronze_autoloader_governed_delta"]["ingestion"]
+    ingestion.update(
+        {
+            "preset": [
+                "bronze_autoloader_append",
+                "runtime_databricks_serverless",
+                "delta_cdf_enabled",
+                "delta_liquid_clustering",
+                "write_engine_native_auto_preview",
+                "governance_uc_basic",
+            ],
+            "source": {
+                "type": "incremental_files",
+                "format": "json",
+                "path": "/Volumes/main/landing/orders_json",
+                "progress_location": "/Volumes/main/ops/checkpoints/orders_json",
+                "schema_tracking_location": "/Volumes/main/ops/autoloader_schemas/orders_json",
+                "read": {"max_files_per_trigger": 50, "include_existing_files": True},
+                "schema_hints": "order_id STRING, event_time TIMESTAMP, customer_id STRING",
+                "trigger": "available_now",
+            },
+            "extensions": {"databricks": {"cluster_columns": ["order_id"]}},
+            "idempotency_key": "b_orders_events_available_now",
+        }
+    )
+    templates["bronze_autoloader_governed_delta"]["access"]["grants"] = [
+        {"principal": "data-engineering", "privileges": ["SELECT", "MODIFY"]},
+        {"principal": "sales-analytics", "privileges": ["SELECT"]},
+    ]
+
+
+def _silver_jdbc_scd1_upsert(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["silver_jdbc_scd1_upsert"]["ingestion"]
+    ingestion.update(
+        {
+            "source": {
+                "type": "postgres",
+                "url": "{{ secret:erp/postgres_url }}",
+                "table": "public.orders",
+                "auth": {"type": "basic", "username": "{{ secret:erp/user }}", "password": "{{ secret:erp/password }}"},
+                "incremental": {"watermark_column": "updated_at", "initial_value": "1970-01-01 00:00:00"},
+                "read": {"fetchsize": 10000, "partition_column": "id", "lower_bound": 1, "upper_bound": 10000000, "num_partitions": 16},
+            },
+            "transform": {"deduplicate": {"keys": ["order_id"], "order_by": "updated_at DESC NULLS LAST"}},
+            "column_mapping": {"id": "order_id"},
+            "quality_rules": {
+                "not_null": ["order_id", "updated_at"],
+                "unique_key": ["order_id"],
+                "expressions": [{"name": "positive_amount", "expression": "amount >= 0", "severity": "quarantine", "message": "Negative amount."}],
+            },
+        }
+    )
+
+
+def _silver_jdbc_rds_iam_hash_diff(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["silver_jdbc_rds_iam_hash_diff"]["ingestion"]
+    source = dict(ingestion.get("source") or {})
+    source.update(
+        {
+            "type": "postgres",
+            "url": "jdbc:postgresql://orders.cluster-abcdefghijkl.us-east-1.rds.amazonaws.com:5432/erp",
+            "table": "public.orders",
+            "auth": {"type": "rds_iam", "username": "contractforge_app", "region": "us-east-1"},
+        }
+    )
+    ingestion["source"] = source
+
+
+def _silver_scd2_history(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["silver_scd2_history"]["ingestion"]
+    ingestion.update(
+        {
+            "transform": {"deduplicate": {"keys": ["customer_id"], "order_by": "updated_at DESC NULLS LAST"}},
+            "hash_exclude_columns": ["updated_at", "ingestion_ts_utc", "__run_id"],
+            "quality_rules": {
+                "not_null": ["customer_id"],
+                "expressions": [{"name": "valid_period", "expression": "updated_at IS NOT NULL", "severity": "abort", "message": "updated_at is required for SCD2 history."}],
+            },
+        }
+    )
+
+
+def _silver_snapshot_soft_delete(templates: dict[str, ContractTemplate]) -> None:
+    ingestion = templates["silver_snapshot_soft_delete"]["ingestion"]
+    source = dict(ingestion.get("source") or {})
+    source["read"] = {"source_complete": True}
+    ingestion.update({"source": source, "quality_rules": {"not_null": ["device_id"], "unique_key": ["device_id"]}})

contractforge_databricks/transforms/__init__.py

@@ -0,0 +1,3 @@
+from contractforge_databricks.transforms.sql import render_transform_sql
+
+__all__ = ["render_transform_sql"]

contractforge_databricks/transforms/sql.py

@@ -0,0 +1,118 @@
+"""Databricks SQL review rendering for portable transform intent."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.sql import quote_identifier, quote_table_name
+
+
+def render_transform_sql(
+    contract: SemanticContract,
+    *,
+    source_view: str = "${prepared_source_view}",
+    output_view: str = "${transformed_view}",
+) -> str:
+    if not contract.transform:
+        return "-- No transform declared.\n"
+    transform = contract.transform.raw
+    select_items = _select_items(transform)
+    deduplicate = transform.get("deduplicate")
+    if isinstance(deduplicate, dict):
+        return _render_deduplicate(
+            select_items=select_items,
+            source_view=source_view,
+            output_view=output_view,
+            deduplicate=deduplicate,
+        )
+    sql = [
+        "-- Transform SQL review artifact.",
+        "-- Databricks runtime may execute equivalent PySpark preparation for complex cases.",
+        f"CREATE OR REPLACE TEMP VIEW {quote_table_name(output_view)} AS",
+        "SELECT",
+        ",\n".join(f"  {item}" for item in select_items),
+        f"FROM {quote_table_name(source_view)}",
+    ]
+    return "\n".join(sql) + ";\n"
+
+
+def _select_items(transform: dict[str, Any]) -> list[str]:
+    items: list[str] = ["*"]
+    for column, data_type in transform.get("cast", {}).items():
+        items.append(f"CAST({quote_identifier(column)} AS {data_type}) AS {quote_identifier(column)}")
+    for column, config in transform.get("standardize", {}).items():
+        items.append(f"{_standardize_expr(column, config)} AS {quote_identifier(column)}")
+    for column, expression in transform.get("derive", {}).items():
+        items.append(f"{expression} AS {quote_identifier(column)}")
+    for column, source_columns in transform.get("composite_keys", {}).items():
+        columns = [source_columns] if isinstance(source_columns, str) else list(source_columns or ())
+        parts = ", ".join(f"coalesce(CAST({quote_identifier(str(item))} AS STRING), '')" for item in columns)
+        items.append(f"concat_ws('|', {parts}) AS {quote_identifier(column)}")
+    return items
+
+
+def _standardize_expr(column: str, config: dict[str, Any]) -> str:
+    expr = quote_identifier(column)
+    if config.get("normalize_whitespace"):
+        expr = f"regexp_replace({expr}, '\\\\s+', ' ')"
+    if config.get("trim"):
+        expr = f"trim({expr})"
+    if config.get("lower"):
+        expr = f"lower({expr})"
+    if config.get("upper"):
+        expr = f"upper({expr})"
+    if config.get("empty_as_null"):
+        expr = f"nullif({expr}, '')"
+    return expr
+
+
+def _render_deduplicate(
+    *,
+    select_items: list[str],
+    source_view: str,
+    output_view: str,
+    deduplicate: dict[str, Any],
+) -> str:
+    keys = deduplicate.get("keys")
+    if isinstance(keys, str):
+        key_columns = [keys]
+    else:
+        key_columns = [str(key) for key in keys or ()]
+    order_by = _order_by(deduplicate.get("order_by"))
+    partition = ", ".join(quote_identifier(column) for column in key_columns)
+    lines = [
+        "-- Transform SQL review artifact.",
+        "-- Databricks runtime may execute equivalent PySpark preparation for complex cases.",
+        f"CREATE OR REPLACE TEMP VIEW {quote_table_name(output_view)} AS",
+        "WITH transformed AS (",
+        "  SELECT",
+        ",\n".join(f"    {item}" for item in select_items),
+        f"  FROM {quote_table_name(source_view)}",
+        "), ranked AS (",
+        "  SELECT *,",
+        f"    row_number() OVER (PARTITION BY {partition} ORDER BY {order_by}) AS __cf_row_number",
+        "  FROM transformed",
+        ")",
+        "SELECT * EXCEPT (__cf_row_number)",
+        "FROM ranked",
+        "WHERE __cf_row_number = 1",
+    ]
+    return "\n".join(lines) + ";\n"
+
+
+def _order_by(value: object) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        parts = []
+        for item in value:
+            if not isinstance(item, dict):
+                continue
+            clause = quote_identifier(str(item["column"]))
+            clause += f" {str(item.get('direction', 'desc')).upper()}"
+            if item.get("nulls"):
+                clause += f" NULLS {str(item['nulls']).upper()}"
+            parts.append(clause)
+        return ", ".join(parts)
+    raise ValueError("transform.deduplicate.order_by is required")

contractforge_databricks/watermark/__init__.py

@@ -0,0 +1,6 @@
+from contractforge_databricks.watermark.sql import (
+    render_select_watermark_candidate_sql,
+    render_watermark_filter_predicate,
+)
+
+__all__ = ["render_select_watermark_candidate_sql", "render_watermark_filter_predicate"]

contractforge_databricks/watermark/sql.py

@@ -0,0 +1,91 @@
+"""Databricks SQL helpers for ContractForge typed watermarks."""
+
+from __future__ import annotations
+
+from contractforge_core.watermark import decode_watermark_value
+from contractforge_databricks.sql import quote_identifier, quote_table_name, sql_string
+
+
+def render_watermark_filter_predicate(*, columns: tuple[str, ...], watermark_value: str | None) -> str:
+    """Render a lexicographic SQL predicate for rows after a typed watermark."""
+    if not columns or not watermark_value:
+        return "true"
+    decoded = decode_watermark_value(watermark_value, columns)
+    if not decoded:
+        return "true"
+    if len(columns) == 1:
+        column = columns[0]
+        return f"{quote_identifier(column)} > {_typed_literal(decoded[column].value, decoded[column].type)}"
+
+    branches: list[str] = []
+    for index, column in enumerate(columns):
+        comparisons = [
+            f"{quote_identifier(previous)} = {_typed_literal(decoded[previous].value, decoded[previous].type)}"
+            for previous in columns[:index]
+        ]
+        comparisons.append(
+            f"{quote_identifier(column)} > {_typed_literal(decoded[column].value, decoded[column].type)}"
+        )
+        branches.append("(" + " AND ".join(comparisons) + ")")
+    return "(" + " OR ".join(branches) + ")"
+
+
+def render_select_watermark_candidate_sql(
+    *,
+    table_name: str,
+    columns: tuple[str, ...],
+    types: dict[str, str] | None = None,
+) -> str:
+    """Render SQL that computes the next typed watermark candidate from a table."""
+    if not columns:
+        raise ValueError("watermark columns must not be empty")
+    type_map = types or {}
+    if len(columns) == 1:
+        column = columns[0]
+        return "\n".join(
+            [
+                "SELECT",
+                f"  to_json(named_struct({sql_string(column)}, named_struct(",
+                f"    'type', {sql_string(type_map.get(column, 'string'))},",
+                f"    'value', CAST(MAX({quote_identifier(column)}) AS STRING)",
+                "  ))) AS watermark_value",
+                f"FROM {quote_table_name(table_name)}",
+            ]
+        )
+
+    struct_fields = ", ".join(
+        f"{sql_string(column)}, {quote_identifier(column)}" for column in columns
+    )
+    json_fields = _candidate_json_fields(columns, type_map)
+    return "\n".join(
+        [
+            "WITH candidate AS (",
+            f"  SELECT MAX(named_struct({struct_fields})) AS wm",
+            f"  FROM {quote_table_name(table_name)}",
+            ")",
+            "SELECT",
+            f"  to_json(named_struct({json_fields})) AS watermark_value",
+            "FROM candidate",
+        ]
+    )
+
+
+def _candidate_json_fields(columns: tuple[str, ...], types: dict[str, str]) -> str:
+    fields = []
+    for column in columns:
+        fields.append(
+            ", ".join(
+                [
+                    sql_string(column),
+                    "named_struct("
+                    f"'type', {sql_string(types.get(column, 'string'))}, "
+                    f"'value', CAST(wm.{quote_identifier(column)} AS STRING)"
+                    ")",
+                ]
+            )
+        )
+    return ", ".join(fields)
+
+
+def _typed_literal(value: str | None, data_type: str) -> str:
+    return f"CAST({sql_string(value)} AS {data_type})"

contractforge_databricks/write_modes/__init__.py

@@ -0,0 +1,20 @@
+from contractforge_databricks.write_modes.registry import (
+    clear_write_mode_registry,
+    get_write_mode,
+    list_write_modes,
+    register_write_mode,
+    unregister_write_mode,
+)
+from contractforge_databricks.write_modes.sql import render_write_mode_sql_notes
+from contractforge_databricks.write_modes.strategy import WriteStrategy, choose_write_strategy
+
+__all__ = [
+    "WriteStrategy",
+    "choose_write_strategy",
+    "clear_write_mode_registry",
+    "get_write_mode",
+    "list_write_modes",
+    "register_write_mode",
+    "render_write_mode_sql_notes",
+    "unregister_write_mode",
+]

contractforge_databricks/write_modes/registry.py

@@ -0,0 +1,44 @@
+"""Databricks runtime registry for adapter-owned custom write modes."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from contractforge_core.execution import ExecutionOutcome, canonical_custom_write_mode
+
+DatabricksWriteHandler = Callable[..., ExecutionOutcome]
+WRITE_MODE_REGISTRY: dict[str, DatabricksWriteHandler] = {}
+
+
+def register_write_mode(mode: str, handler: DatabricksWriteHandler, *, overwrite: bool = False) -> str:
+    canonical = canonical_custom_write_mode(mode)
+    if not callable(handler):
+        raise ValueError("write mode handler must be callable")
+    if canonical in WRITE_MODE_REGISTRY and not overwrite:
+        raise ValueError(f"write mode already registered: {canonical}")
+    WRITE_MODE_REGISTRY[canonical] = handler
+    return canonical
+
+
+def unregister_write_mode(mode: str) -> None:
+    WRITE_MODE_REGISTRY.pop(canonical_custom_write_mode(mode), None)
+
+
+def list_write_modes() -> tuple[str, ...]:
+    return tuple(sorted(WRITE_MODE_REGISTRY))
+
+
+def get_write_mode(mode: str) -> DatabricksWriteHandler | None:
+    return WRITE_MODE_REGISTRY.get(canonical_custom_write_mode(mode))
+
+
+def execute_registered_write_mode(mode: str, **kwargs: Any) -> ExecutionOutcome:
+    handler = get_write_mode(mode)
+    if handler is None:
+        raise ValueError(f"Unsupported Databricks write mode: {mode}")
+    return handler(**kwargs)
+
+
+def clear_write_mode_registry() -> None:
+    WRITE_MODE_REGISTRY.clear()

contractforge_databricks/write_modes/sql.py

@@ -0,0 +1,33 @@
+"""Render non-executing SQL notes for Databricks write modes."""
+
+from __future__ import annotations
+
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.rendering.names import target_full_name
+
+
+def render_write_mode_sql_notes(contract: SemanticContract) -> str:
+    target = target_full_name(contract)
+    mode = contract.write.mode
+    lines = [
+        "-- Databricks write mode review notes.",
+        "-- This artifact is not an executable job script.",
+        f"-- Target: {target}",
+        f"-- Mode: {mode}",
+        "",
+    ]
+    if mode == "scd0_append":
+        lines.append("-- Expected implementation: Delta append with schema policy applied by adapter.")
+    elif mode == "scd0_overwrite":
+        lines.append("-- Expected implementation: Delta overwrite or scoped replaceWhere when declared.")
+    elif mode == "scd1_upsert":
+        lines.append("-- Expected implementation: SCD1 Delta MERGE current-state upsert by merge keys.")
+    elif mode == "scd1_hash_diff":
+        lines.append("-- Expected implementation: SCD1 hash current source rows and append changed versions.")
+    elif mode == "scd2_historical":
+        lines.append("-- Expected implementation: Delta MERGE with valid_from, valid_to, is_current, row_hash.")
+    elif mode == "snapshot_soft_delete":
+        lines.append("-- Expected implementation: Delta MERGE with NOT MATCHED BY SOURCE soft-delete update.")
+    else:
+        lines.append("-- Unsupported write mode.")
+    return "\n".join(lines) + "\n"