contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/runtime/partitioning.py

@@ -0,0 +1,93 @@
+"""Runtime partition-scope helpers for Databricks writes."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.runtime import PreparedInput, QueryOne
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.contract_extensions import databricks_extensions
+from contractforge_databricks.partitioning import render_partition_in_predicate
+from contractforge_databricks.sql import quote_identifier, quote_table_name
+
+
+def target_partition_predicate(
+    *,
+    contract: SemanticContract,
+    prepared: PreparedInput,
+    query_one: QueryOne | None,
+) -> str | None:
+    extensions = databricks_extensions(contract)
+    if contract.write.mode != "scd1_upsert" or extensions.get("merge_strategy") != "delta_by_partition":
+        return None
+    column = str(extensions.get("merge_partition_column") or extensions.get("partition_column") or "")
+    if not column:
+        raise ValueError("merge_strategy=delta_by_partition requires merge_partition_column or partition_column")
+    if column not in prepared.source_columns:
+        raise ValueError(f"partition column {column!r} is not present in prepared source columns")
+    values = _partition_values(prepared, column, query_one)
+    return f"t.{render_partition_in_predicate(column, values)}"
+
+
+def replace_partition_predicate(
+    *,
+    contract: SemanticContract,
+    prepared: PreparedInput,
+    query_one: QueryOne | None,
+) -> str | None:
+    extensions = databricks_extensions(contract)
+    if contract.write.mode != "scd1_upsert" or extensions.get("merge_strategy") != "replace_partitions":
+        return None
+    _validate_replace_partitions_contract(contract, extensions)
+    column = str(extensions.get("merge_partition_column") or "")
+    if column not in prepared.source_columns:
+        raise ValueError(f"merge_partition_column {column!r} is not present in prepared source columns")
+    values = _partition_values(prepared, column, query_one)
+    return render_partition_in_predicate(column, values)
+
+
+def _validate_replace_partitions_contract(contract: SemanticContract, extensions: dict[str, Any]) -> None:
+    column = str(extensions.get("merge_partition_column") or "")
+    if not column:
+        raise ValueError("merge_strategy=replace_partitions requires merge_partition_column")
+    partition_column = extensions.get("partition_column")
+    if partition_column and partition_column != column:
+        raise ValueError("merge_strategy=replace_partitions requires partition_column equal to merge_partition_column")
+    if extensions.get("replace_partitions_source_complete") or _source_declares_complete(contract):
+        return
+    raise ValueError(
+        "merge_strategy=replace_partitions requires replace_partitions_source_complete=true "
+        "or source.read.source_complete=true/source.read.full_snapshot=true"
+    )
+
+
+def _source_declares_complete(contract: SemanticContract) -> bool:
+    source = contract.source.raw or {}
+    read = source.get("read") if isinstance(source.get("read"), dict) else {}
+    return bool(read.get("source_complete") or read.get("full_snapshot") or source.get("source_complete"))
+
+
+def _partition_values(prepared: PreparedInput, column: str, query_one: QueryOne | None) -> tuple[Any, ...]:
+    if query_one is None:
+        metadata_values = (prepared.source_metadata or {}).get("affected_partition_values")
+        if metadata_values:
+            return tuple(metadata_values)
+        raise ValueError("delta_by_partition requires query_one or prepared.source_metadata.affected_partition_values")
+    row = query_one(
+        f"SELECT collect_set({quote_identifier(column)}) AS partition_values "
+        f"FROM {quote_table_name(prepared.source_view)}"
+    )
+    values = _row_value(row, "partition_values")
+    if not values:
+        raise ValueError("delta_by_partition could not detect affected partition values")
+    return tuple(values)
+
+
+def _row_value(row: Any, key: str) -> Any:
+    if row is None:
+        return None
+    if isinstance(row, dict):
+        return row.get(key)
+    if hasattr(row, "asDict"):
+        return row.asDict().get(key)
+    return getattr(row, key, None)

contractforge_databricks/runtime/quality_quarantine.py

@@ -0,0 +1,92 @@
+"""Persist row-level Databricks quality quarantine evidence."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from importlib import import_module
+from typing import Any
+from uuid import uuid4
+
+from contractforge_core.quality import QualityRuleResult
+from contractforge_databricks.evidence.tables import evidence_table_names
+from contractforge_databricks.quality import evaluate_quality
+from contractforge_databricks.runtime.models import DatabricksIngestOptions
+
+
+def apply_declared_quality(
+    *,
+    spark: Any,
+    contract: Any,
+    prepared: Any,
+    opts: DatabricksIngestOptions,
+    run_id: str,
+    target: str,
+    quality_results: tuple[QualityRuleResult, ...],
+) -> tuple[Any, tuple[QualityRuleResult, ...]]:
+    if quality_results or not getattr(contract, "quality", ()):
+        return prepared, quality_results
+    status, evaluated, valid_df, quarantined_df, quarantined_count = evaluate_quality(spark.table(prepared.source_view), contract)
+    if status == "NOT_CONFIGURED":
+        return prepared, evaluated
+    if quarantined_count > 0 and not opts.dry_run:
+        persist_quality_quarantine_rows(
+            quarantined_df,
+            run_id=run_id,
+            target_table=target,
+            quality_results=evaluated,
+            catalog=opts.catalog,
+            schema=opts.schema,
+        )
+    valid_df.createOrReplaceTempView(prepared.source_view)
+    return replace(prepared, rows_quarantined=quarantined_count), evaluated
+
+
+def with_run_id(opts: DatabricksIngestOptions) -> DatabricksIngestOptions:
+    return opts if opts.run_id else replace(opts, run_id=f"run-{uuid4()}")
+
+
+def persist_quality_quarantine_rows(
+    quarantined_df: Any,
+    *,
+    run_id: str,
+    target_table: str,
+    quality_results: tuple[QualityRuleResult, ...],
+    catalog: str,
+    schema: str,
+) -> None:
+    """Append quarantined row payloads to the core quarantine evidence table."""
+
+    failed = tuple(result for result in quality_results if result.failed_count and result.severity == "quarantine")
+    if not failed:
+        return
+    functions = _functions()
+    payload_columns = [functions.col(column) for column in getattr(quarantined_df, "columns", ()) or ()]
+    reason = _reason(failed)
+    quarantine_table = evidence_table_names(catalog, schema)["quarantine"]
+    (
+        quarantined_df.select(
+            functions.lit(run_id).alias("run_id"),
+            functions.lit(target_table).alias("target_table"),
+            functions.lit(_rule_name(failed)).alias("rule_name"),
+            functions.lit(reason).alias("error_reason"),
+            functions.to_json(functions.struct(*payload_columns)).alias("record_payload"),
+            functions.lit(None).cast("string").alias("record_ref"),
+            functions.lit(reason).alias("reason"),
+            functions.current_timestamp().alias("quarantined_at_utc"),
+        )
+        .write.mode("append")
+        .insertInto(quarantine_table)
+    )
+
+
+def _rule_name(results: tuple[QualityRuleResult, ...]) -> str:
+    return ", ".join(result.rule_name for result in results)
+
+
+def _reason(results: tuple[QualityRuleResult, ...]) -> str:
+    parts = [f"{result.rule_name}: {result.message or 'quality rule failed'}" for result in results]
+    return "; ".join(parts)
+
+
+def _functions() -> Any:
+    return import_module("pyspark.sql").functions

contractforge_databricks/runtime/rest_api.py

@@ -0,0 +1,46 @@
+"""Databricks runtime execution for bounded REST API connector sources.
+
+The request/pagination/auth/records logic lives in the core REST client; this
+module resolves Databricks secret placeholders, delegates the read to the core,
+and materializes the returned records into a Spark DataFrame.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import is_rest_api_connector
+from contractforge_core.connectors.api.rest import read_rest_api_records as _core_read_rest_api_records
+from contractforge_databricks.runtime.json_materialization import materialize_json_records
+from contractforge_databricks.runtime.source_schema import source_declared_schema
+from contractforge_databricks.security import resolve_databricks_secret_placeholders
+
+
+def read_rest_api_records(source: dict[str, Any]) -> list[dict[str, Any]]:
+    """Read records from a bounded REST source, resolving secrets via Databricks first."""
+
+    resolved = dict(source)
+    if resolved.get("auth") is not None:
+        resolved["auth"] = resolve_databricks_secret_placeholders(resolved.get("auth"))
+    return _core_read_rest_api_records(resolved)
+
+
+def resolve_rest_api_dataframe(spark: Any, source: dict[str, Any]) -> Any:
+    if not is_rest_api_connector(source):
+        raise ValueError(
+            "REST API runtime resolution requires source.type=rest_api or source.type=connector and connector=rest_api"
+        )
+    records = read_rest_api_records(source)
+    schema = source_declared_schema(source)
+    read = _dict(source.get("read"))
+    return materialize_json_records(
+        spark,
+        records,
+        schema=schema,
+        read_options=_dict(read.get("json_options")),
+        staging_path=read.get("staging_path"),
+    )
+
+
+def _dict(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, dict) else {}

contractforge_databricks/runtime/rest_auth.py

@@ -0,0 +1,21 @@
+"""Databricks REST auth headers: resolve secrets, then delegate to the core."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors.api.rest import rest_request_headers as _core_rest_request_headers
+from contractforge_databricks.security import resolve_databricks_secret_placeholders
+
+
+def rest_request_headers(
+    source: dict[str, Any],
+    incremental: dict[str, Any] | None = None,
+    watermark: str | None = None,
+) -> dict[str, str]:
+    """Resolve Databricks secret placeholders in auth, then build headers via the core."""
+
+    resolved = dict(source)
+    if resolved.get("auth") is not None:
+        resolved["auth"] = resolve_databricks_secret_placeholders(resolved.get("auth"))
+    return _core_rest_request_headers(resolved, incremental, watermark)

contractforge_databricks/runtime/rest_pagination.py

@@ -0,0 +1,21 @@
+"""Compatibility re-exports for the core REST pagination helpers."""
+
+from contractforge_core.connectors.api.rest.pagination import (
+    json_path,
+    link_header_next,
+    max_pages_for_source,
+    next_url,
+    page_urls,
+    pagination_type,
+    url_with_params,
+)
+
+__all__ = [
+    "json_path",
+    "link_header_next",
+    "max_pages_for_source",
+    "next_url",
+    "page_urls",
+    "pagination_type",
+    "url_with_params",
+]

contractforge_databricks/runtime/run_payload.py

@@ -0,0 +1,177 @@
+"""Databricks run evidence payload assembly."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from contractforge_core.config import CTRL_SCHEMA_VERSION, FRAMEWORK_VERSION
+from contractforge_core.runtime import PreparedInput
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.runtime.metadata import contract_metadata
+from contractforge_databricks.runtime.models import DatabricksIngestOptions
+from contractforge_databricks.runtime.write_strategy import write_strategy_evidence
+
+
+def run_payload(
+    contract: SemanticContract,
+    prepared: PreparedInput,
+    opts: DatabricksIngestOptions,
+    run_id: str,
+    target: str,
+    status: str,
+    started: str,
+    finished: str,
+    rows_written: int,
+    quality_status_value: str,
+    operation_metrics: dict[str, Any],
+    schema_changes: dict[str, Any],
+    governance_results: dict[str, Any],
+    write_started_at: str | None,
+    write_finished_at: str | None,
+    stage_durations: dict[str, float],
+    watermark_column: str | None,
+    watermark_previous: str | None,
+    watermark_current: str | None,
+    diagnostics: dict[str, bool],
+    error_message: str | None,
+    skip_reason: str | None,
+    skipped_by_run_id: str | None,
+) -> dict[str, Any]:
+    runtime = dict(opts.runtime_metadata or {})
+    source_metadata = dict(prepared.source_metadata or {})
+    operations = dict(contract.operations.metadata or {}) if contract.operations and contract.operations.metadata else {}
+    annotations_result = governance_results.get("annotations") or {}
+    operations_result = governance_results.get("operations") or {}
+    row_metrics = _row_metrics(operation_metrics, rows_written)
+    delta_version_after = _metric_int(operation_metrics, "version", default=None)
+    write_strategy = write_strategy_evidence(contract, target, runtime)
+    return {
+        "run_id": run_id,
+        "run_ts_utc": started,
+        "run_date": _date_now(),
+        "started_at_utc": started,
+        "finished_at_utc": finished,
+        "duration_seconds": _duration_seconds(started, finished),
+        "target_table": target,
+        "runtime_entrypoint": runtime.get("notebook_name"),
+        "layer": contract.target.layer,
+        "mode": contract.write.mode,
+        "write_engine_requested": write_strategy["write_engine_requested"],
+        "write_engine_selected": write_strategy["write_engine_selected"],
+        "write_engine_status": write_strategy["write_engine_status"],
+        "write_engine_reason": write_strategy["write_engine_reason"],
+        "write_engine_fallback_policy": write_strategy["write_engine_fallback_policy"],
+        "write_engine": dict(write_strategy),
+        "status": status,
+        "source_table": prepared.source_name or prepared.source_view,
+        "source_type": source_metadata.get("source_type"),
+        "source_connector": source_metadata.get("source_connector"),
+        "source_name": source_metadata.get("source_name") or prepared.source_name,
+        "source_system": source_metadata.get("source_system") or _source_system(contract),
+        "source_provider": source_metadata.get("source_provider"),
+        "source_format": source_metadata.get("source_format"),
+        "source_path": source_metadata.get("source_path"),
+        "source_options_json": _metadata_value(source_metadata, "source_options", "source_options_redacted"),
+        "source_read_json": _metadata_value(source_metadata, "source_read", "source_read_redacted"),
+        "source_request_json": _metadata_value(source_metadata, "source_request", "source_request_redacted"),
+        "source_auth_json": _metadata_value(source_metadata, "source_auth", "source_auth_redacted"),
+        "source_pagination_json": _metadata_value(source_metadata, "source_pagination", "source_pagination_redacted"),
+        "source_response_json": _metadata_value(source_metadata, "source_response", "source_response_redacted"),
+        "source_incremental_json": _metadata_value(source_metadata, "source_incremental", "source_incremental_redacted"),
+        "source_limits_json": _metadata_value(source_metadata, "source_limits", "source_limits_redacted"),
+        "source_capabilities_json": source_metadata.get("source_capabilities"),
+        "source_metrics_json": source_metadata.get("source_metrics"),
+        "source": source_metadata or None,
+        "rows_read": prepared.rows_read,
+        "rows_effective": prepared.rows_read - prepared.rows_quarantined,
+        "rows_written": rows_written,
+        "rows_inserted": row_metrics["rows_inserted"],
+        "rows_updated": row_metrics["rows_updated"],
+        "rows_deleted": row_metrics["rows_deleted"],
+        "rows_expired": row_metrics["rows_expired"],
+        "rows_quarantined": prepared.rows_quarantined,
+        "watermark_column": watermark_column,
+        "watermark_previous": watermark_previous,
+        "watermark_current": watermark_current,
+        "quality_status": quality_status_value,
+        "schema_changes": schema_changes,
+        "schema_changes_json": schema_changes,
+        "stage_durations": stage_durations,
+        "stage_durations_json": stage_durations,
+        "operation_metrics": operation_metrics,
+        "operation_metrics_json": operation_metrics,
+        "metrics_json": operation_metrics,
+        "schema_policy": contract.write.schema_policy,
+        "applied_presets": operations.get("applied_presets"),
+        "metrics_source": operation_metrics.get("metrics_source") or "adapter_runtime",
+        "table_version_after": None if delta_version_after is None else str(delta_version_after),
+        "write_delta_version": delta_version_after,
+        "write_committed": status == "SUCCESS" and rows_written >= 0,
+        "explain_captured": bool(diagnostics.get("explain_captured")),
+        "openlineage_event_emitted": bool(diagnostics.get("openlineage_event_emitted")),
+        "runtime_type": runtime.get("runtime_type"),
+        "engine_version": runtime.get("spark_version"),
+        "python_version": runtime.get("python_version"),
+        "framework_version": FRAMEWORK_VERSION,
+        "ctrl_schema_version": CTRL_SCHEMA_VERSION,
+        "contract_owner": contract.governance.owner if contract.governance else None,
+        "contract_domain": contract.target.domain,
+        "contract_tags_json": operations.get("tags"),
+        "contract_sla": operations.get("sla"),
+        "runtime_parameters_json": operations.get("runtime_parameters"),
+        "contract_metadata": contract_metadata(contract, operations),
+        "annotations_status": annotations_result.get("status"),
+        "annotations_result_json": annotations_result or None,
+        "operations_json": {"metadata": operations, "record_result": operations_result} if operations or operations_result else None,
+        "idempotency_key": opts.idempotency_key,
+        "idempotency_policy": opts.idempotency_policy,
+        "write_started_at_utc": write_started_at,
+        "write_finished_at_utc": write_finished_at,
+        "parent_run_id": operations.get("parent_run_id"),
+        "run_group_id": operations.get("run_group_id"),
+        "master_job_id": operations.get("master_job_id"),
+        "master_run_id": operations.get("master_run_id"),
+        "skip_reason": skip_reason,
+        "skipped_by_run_id": skipped_by_run_id,
+        "error_message": error_message,
+    }
+
+
+def _date_now() -> str:
+    return datetime.now(timezone.utc).date().isoformat()
+
+
+def _metadata_value(source_metadata: dict[str, Any], raw_key: str, redacted_key: str) -> Any:
+    return source_metadata.get(redacted_key) if redacted_key in source_metadata else source_metadata.get(raw_key)
+
+
+def _source_system(contract: SemanticContract) -> str | None:
+    value = (contract.source.raw or {}).get("system")
+    return str(value) if value not in (None, "") else None
+
+
+def _row_metrics(operation_metrics: dict[str, Any], rows_written: int) -> dict[str, int]:
+    return {
+        "rows_inserted": _metric_int(operation_metrics, "rows_inserted", default=rows_written),
+        "rows_updated": _metric_int(operation_metrics, "rows_updated"),
+        "rows_deleted": _metric_int(operation_metrics, "rows_deleted"),
+        "rows_expired": _metric_int(operation_metrics, "rows_expired"),
+    }
+
+
+def _metric_int(metrics: dict[str, Any], key: str, *, default: int | None = 0) -> int | None:
+    value = metrics.get(key, default)
+    try:
+        return int(value or 0)
+    except (TypeError, ValueError):
+        return default
+
+
+def _duration_seconds(started: str, finished: str) -> float | None:
+    try:
+        started_dt = datetime.strptime(started, "%Y-%m-%d %H:%M:%S")
+        finished_dt = datetime.strptime(finished, "%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        return None
+    return (finished_dt - started_dt).total_seconds()

contractforge_databricks/runtime/schema.py

@@ -0,0 +1,106 @@
+"""Runtime schema setup and schema-change evidence for Databricks."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.runtime import PreparedInput
+from contractforge_core.schema import compare_schema, validate_schema_diff
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.contract_extensions import databricks_extensions
+from contractforge_databricks.evidence import EvidenceWriter, render_schema_change_log_insert_sqls
+from contractforge_databricks.execution import SqlRunner, execute_table_setup
+from contractforge_databricks.rendering.names import target_full_name
+from contractforge_databricks.schema import render_add_columns_sql, render_type_widening_sql
+
+
+def setup_and_sync_schema(
+    *,
+    runner: SqlRunner,
+    evidence: EvidenceWriter,
+    contract: SemanticContract,
+    prepared: PreparedInput,
+    run_id: str,
+    ensure_table: bool,
+    target_schema: dict[str, str] | None,
+) -> dict[str, Any]:
+    """Apply Databricks schema policy using prepared-source schema evidence."""
+
+    if not prepared.source_schema:
+        return {}
+
+    target = target_full_name(contract)
+    if ensure_table:
+        execute_table_setup(runner=runner, contract=contract, columns=prepared.source_schema)
+
+    if target_schema is None:
+        return {"status": "new_or_unknown", "source_schema": dict(prepared.source_schema)}
+
+    allow_type_widening = bool(databricks_extensions(contract).get("allow_type_widening"))
+    diff = compare_schema(
+        prepared.source_schema,
+        target_schema,
+        allow_type_widening=allow_type_widening,
+    )
+    validate_schema_diff(diff, contract.write.schema_policy)
+    changes = diff.as_dict()
+    _apply_schema_sync_sql(runner, target, prepared.source_schema, diff)
+    _mark_applied_schema_changes(changes)
+    _write_schema_change_logs(evidence, run_id, target, changes, prepared.source_schema)
+    return changes
+
+
+def preview_schema_changes(
+    *,
+    contract: SemanticContract,
+    prepared: PreparedInput,
+    target_schema: dict[str, str] | None,
+) -> dict[str, Any]:
+    """Validate and describe schema changes without running Databricks DDL."""
+
+    if not prepared.source_schema:
+        return {}
+    if target_schema is None:
+        return {"status": "new_or_unknown", "source_schema": dict(prepared.source_schema)}
+    diff = compare_schema(
+        prepared.source_schema,
+        target_schema,
+        allow_type_widening=bool(databricks_extensions(contract).get("allow_type_widening")),
+    )
+    validate_schema_diff(diff, contract.write.schema_policy)
+    return diff.as_dict()
+
+
+def _apply_schema_sync_sql(runner: SqlRunner, target: str, source_schema: dict[str, str], diff: Any) -> None:
+    for statement in (
+        render_add_columns_sql(target_table=target, source_schema=source_schema, diff=diff),
+        render_type_widening_sql(target_table=target, diff=diff),
+    ):
+        for part in statement.split(";"):
+            sql = part.strip()
+            if sql and not sql.startswith("--"):
+                runner.sql(sql)
+
+
+def _write_schema_change_logs(
+    evidence: EvidenceWriter,
+    run_id: str,
+    target: str,
+    changes: dict[str, Any],
+    source_schema: dict[str, str],
+) -> None:
+    for statement in render_schema_change_log_insert_sqls(
+        run_id=run_id,
+        target_table=target,
+        schema_changes=changes,
+        source_schema=source_schema,
+        catalog=evidence.catalog,
+        schema=evidence.schema,
+    ):
+        evidence.runner.sql(statement)
+
+
+def _mark_applied_schema_changes(changes: dict[str, Any]) -> None:
+    for change in changes.get("type_changes") or ():
+        if change.get("allowed"):
+            change["applied"] = True

contractforge_databricks/runtime/source_metadata.py

@@ -0,0 +1,30 @@
+"""Databricks runtime source metadata helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import source_metadata_from_mapping
+
+
+def source_name(source: dict[str, Any]) -> str:
+    return str(source.get("table") or source.get("path") or source.get("url") or source.get("type") or "source")
+
+
+def source_metadata(source: dict[str, Any]) -> dict[str, Any]:
+    return source_metadata_from_mapping(source)
+
+
+def schema_types(df: Any) -> dict[str, str] | None:
+    schema = getattr(df, "schema", None)
+    fields = getattr(schema, "fields", None)
+    if not fields:
+        return None
+    return {str(field.name): str(field.dataType.simpleString()) for field in fields}
+
+
+def source_metadata_with_watermark(source: dict[str, Any], watermark_previous: str | None) -> dict[str, Any]:
+    metadata = source_metadata(source)
+    if watermark_previous is not None:
+        metadata["watermark_previous"] = watermark_previous
+    return metadata

contractforge_databricks/runtime/source_registry.py

@@ -0,0 +1,43 @@
+"""Databricks runtime source resolver registry."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Protocol
+
+_SOURCE_NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$")
+_CUSTOM_RESOLVERS: dict[str, "DatabricksSourceResolver"] = {}
+
+
+class DatabricksSourceResolver(Protocol):
+    def resolve(self, spark: Any, source: dict[str, Any]) -> Any:
+        """Resolve a source mapping into a Databricks DataFrame-like object."""
+        ...
+
+
+def register_source_resolver(source_type: str, resolver: DatabricksSourceResolver, *, overwrite: bool = False) -> None:
+    normalized = _normalize_source_name(source_type)
+    if not hasattr(resolver, "resolve"):
+        raise ValueError("resolver must implement resolve(spark, source)")
+    if normalized in _CUSTOM_RESOLVERS and not overwrite:
+        raise ValueError(f"source resolver already registered: {normalized}")
+    _CUSTOM_RESOLVERS[normalized] = resolver
+
+
+def unregister_source_resolver(source_type: str) -> None:
+    _CUSTOM_RESOLVERS.pop(_normalize_source_name(source_type), None)
+
+
+def get_source_resolver(source_type: str) -> DatabricksSourceResolver | None:
+    return _CUSTOM_RESOLVERS.get(_normalize_source_name(source_type))
+
+
+def list_source_resolvers() -> list[str]:
+    return sorted(_CUSTOM_RESOLVERS)
+
+
+def _normalize_source_name(source_type: str) -> str:
+    normalized = str(source_type or "").strip()
+    if not _SOURCE_NAME_RE.match(normalized):
+        raise ValueError("source_type must start with a letter and contain only letters, numbers, '_' or '-'")
+    return normalized

contractforge_databricks/runtime/source_schema.py

@@ -0,0 +1,24 @@
+"""Runtime source schema helpers for Databricks readers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def source_declared_schema(source: dict[str, Any]) -> str | None:
+    if source.get("schema") not in (None, ""):
+        raise ValueError("source.schema is not supported; declare source.read.schema")
+    read = source.get("read") if isinstance(source.get("read"), dict) else {}
+    options = source.get("options") if isinstance(source.get("options"), dict) else {}
+    values = [read.get("schema"), options.get("schema")]
+    declared = [str(value).strip() for value in values if value not in (None, "")]
+    if len(set(declared)) > 1:
+        raise ValueError("source.read.schema conflicts with source.options.schema")
+    if declared and not declared[0]:
+        raise ValueError("source.read.schema cannot be empty")
+    return declared[0] if declared else None
+
+
+def apply_declared_schema(reader: Any, source: dict[str, Any]) -> Any:
+    schema = source_declared_schema(source)
+    return reader.schema(schema) if schema else reader