contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/sources/artifacts.py

@@ -0,0 +1,100 @@
+"""Source artifact routing for Databricks bundles."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import JDBC_CONNECTORS
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.environment import DatabricksEnvironment
+from contractforge_databricks.rendering.names import artifact_prefix
+from contractforge_databricks.sources.autoloader import render_autoloader_python
+from contractforge_databricks.sources.bounded_streams import is_bounded_stream_source, render_bounded_stream_python
+from contractforge_databricks.sources.delta_share import is_delta_share_source, render_delta_share_python
+from contractforge_databricks.sources.files import (
+    is_catalog_source,
+    is_file_source,
+    render_catalog_source_python,
+    render_file_source_python,
+)
+from contractforge_databricks.sources.http_file import is_http_file_source, render_http_file_python
+from contractforge_databricks.sources.interpret import interpret_incremental_files_source, is_incremental_file_source
+from contractforge_databricks.sources.jdbc import render_jdbc_python
+from contractforge_databricks.sources.native_passthrough import render_native_passthrough_plan
+from contractforge_databricks.sources.rest_api import is_rest_api_connector, render_rest_api_review_plan
+from contractforge_databricks.sources.table_refs import contract_with_databricks_source_refs
+
+
+def render_source_artifacts(
+    contract: SemanticContract,
+    *,
+    environment: DatabricksEnvironment | None = None,
+) -> dict[str, str]:
+    if not contract.source.raw:
+        return {}
+    runtime_contract = contract_with_databricks_source_refs(contract)
+    source = runtime_contract.source.raw or {}
+    prefix = artifact_prefix(contract)
+    artifacts: dict[str, str] = {}
+    if is_incremental_file_source(source):
+        artifacts[f"{prefix}.source_autoloader.py"] = render_autoloader_python(
+            interpret_incremental_files_source(source, environment=environment)
+        )
+    if _is_jdbc_source(source):
+        artifacts[f"{prefix}.source_jdbc.py"] = render_jdbc_python(source)
+    if _can_render_file_source(source):
+        artifacts[f"{prefix}.source_files.py"] = render_file_source_python(source)
+    if _can_render_catalog_source(source):
+        artifacts[f"{prefix}.source_catalog.py"] = render_catalog_source_python(
+            source,
+        )
+    if _can_render_http_file_source(source):
+        artifacts[f"{prefix}.source_http_file.py"] = render_http_file_python(source)
+    if is_bounded_stream_source(source):
+        artifacts[f"{prefix}.source_bounded_stream.py"] = render_bounded_stream_python(source)
+    if _can_render_delta_share_source(source):
+        artifacts[f"{prefix}.source_delta_share.py"] = render_delta_share_python(source)
+    if source.get("type") == "native_passthrough":
+        artifacts[f"{prefix}.native_passthrough.json"] = render_native_passthrough_plan(source)
+    if is_rest_api_connector(source):
+        artifacts[f"{prefix}.source_rest_api_review.json"] = render_rest_api_review_plan(source)
+    return artifacts
+
+
+def _is_jdbc_source(source: dict[str, Any]) -> bool:
+    source_type = source.get("type")
+    connector = source.get("connector")
+    options = source.get("options") if isinstance(source.get("options"), dict) else {}
+    has_url = bool(source.get("url") or options.get("url"))
+    is_jdbc = source_type == "jdbc" or connector in JDBC_CONNECTORS
+    return is_jdbc and has_url
+
+
+def _can_render_file_source(source: dict[str, Any]) -> bool:
+    return is_file_source(source) and not is_incremental_file_source(source) and bool(source.get("path"))
+
+
+def _can_render_catalog_source(source: dict[str, Any]) -> bool:
+    if not is_catalog_source(source):
+        return False
+    options = source.get("options") if isinstance(source.get("options"), dict) else {}
+    return bool(
+        source.get("table")
+        or source.get("path")
+        or source.get("ref")
+        or source.get("table_ref")
+        or source.get("query")
+        or options.get("query")
+    )
+
+
+def _can_render_http_file_source(source: dict[str, Any]) -> bool:
+    request = source.get("request") if isinstance(source.get("request"), dict) else {}
+    return is_http_file_source(source) and bool(source.get("url") or request.get("url"))
+
+
+def _can_render_delta_share_source(source: dict[str, Any]) -> bool:
+    options = source.get("options") if isinstance(source.get("options"), dict) else {}
+    return is_delta_share_source(source) and bool(source.get("profile_file") or options.get("profileFile")) and bool(
+        source.get("table") or options.get("table")
+    )

contractforge_databricks/sources/autoloader.py

@@ -0,0 +1,48 @@
+"""Databricks Auto Loader rendering for incremental file sources."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def render_autoloader_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    source_type = source.get("type")
+    if source_type != "incremental_files":
+        raise ValueError("Auto Loader rendering requires source.type incremental_files")
+    path = source.get("path")
+    if not path:
+        raise ValueError("Auto Loader source requires path")
+    file_format = source.get("format") or "json"
+    options = {
+        "cloudFiles.format": file_format,
+        **{str(key): str(value) for key, value in source.get("options", {}).items()},
+    }
+    schema_location = source.get("schema_tracking_location")
+    if schema_location:
+        options["cloudFiles.schemaLocation"] = schema_location
+    if source.get("schema_hints"):
+        options["cloudFiles.schemaHints"] = source["schema_hints"]
+
+    lines = [
+        f"{dataframe_name} = (",
+        "    spark.readStream",
+        "    .format('cloudFiles')",
+    ]
+    for key, value in sorted(options.items()):
+        lines.append(f"    .option({key!r}, {value!r})")
+    lines.extend(
+        [
+            f"    .load({path!r})",
+            ")",
+        ]
+    )
+    checkpoint = source.get("progress_location")
+    if checkpoint:
+        lines.extend(
+            [
+                "",
+                "# Use this checkpoint when writing the available-now stream.",
+                f"checkpoint_location = {checkpoint!r}",
+            ]
+        )
+    return "\n".join(lines) + "\n"

contractforge_databricks/sources/bounded_streams.py

@@ -0,0 +1,44 @@
+"""Databricks bounded stream source rendering."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import (
+    eventhubs_bounded_options,
+    is_bounded_stream_source as is_bounded_stream_source,
+    kafka_bounded_options,
+)
+from contractforge_databricks.security import redact_value
+
+
+def render_bounded_stream_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    source_type = source.get("type")
+    if source_type == "kafka_bounded":
+        return render_kafka_bounded_python(source, dataframe_name=dataframe_name)
+    if source_type == "eventhubs_bounded":
+        return render_eventhubs_bounded_python(source, dataframe_name=dataframe_name)
+    raise ValueError("bounded stream rendering requires kafka_bounded or eventhubs_bounded")
+
+
+def render_kafka_bounded_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    return _render_reader("kafka", kafka_bounded_options(source), dataframe_name, "kafka_bounded_options_review")
+
+
+def render_eventhubs_bounded_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    return _render_reader("eventhubs", eventhubs_bounded_options(source), dataframe_name, "eventhubs_bounded_options_review")
+
+
+
+
+def _render_reader(source_format: str, options: dict[str, str], dataframe_name: str, review_name: str) -> str:
+    lines = [
+        "# Bounded replay/catch-up read. This is not a continuous streaming artifact.",
+        f"{dataframe_name} = (",
+        "    spark.read",
+        f"    .format({source_format!r})",
+    ]
+    for key, value in sorted(options.items()):
+        lines.append(f"    .option({key!r}, {value!r})")
+    lines.extend(["    .load()", ")", "", f"{review_name} = {redact_value(options)!r}"])
+    return "\n".join(lines) + "\n"

contractforge_databricks/sources/classification.py

@@ -0,0 +1,115 @@
+"""Databricks source connector classification.
+
+This module is adapter-owned: it maps portable core connector semantics to the
+Databricks rendering/runtime surface without making Databricks names portable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from contractforge_core.connectors import (
+    JDBC_CONNECTORS,
+    is_bounded_stream_source,
+    is_catalog_source,
+    is_delta_share_source,
+    is_file_source,
+    is_http_file_source,
+    is_native_passthrough_source,
+    is_rest_api_connector,
+)
+from contractforge_databricks.sources.interpret import is_incremental_file_source
+
+SUPPORTED = "SUPPORTED"
+REVIEW_REQUIRED = "REVIEW_REQUIRED"
+UNSUPPORTED = "UNSUPPORTED"
+
+
+@dataclass(frozen=True)
+class DatabricksSourceClassification:
+    source_type: str
+    status: str
+    native_mapping: str | None
+    note: str
+
+
+def classify_databricks_source(source: dict[str, Any] | str) -> DatabricksSourceClassification:
+    """Classify a source connector against Databricks support semantics."""
+
+    payload = {"type": source} if isinstance(source, str) else dict(source)
+    source_type = str(payload.get("connector") or payload.get("type") or "").strip().lower()
+    if is_incremental_file_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Auto Loader cloudFiles",
+            note="Uses core incremental_files/file_stream intent.",
+        )
+    if _is_jdbc_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Spark JDBC reader",
+            note="Core builds JDBC options; Databricks resolves secrets.",
+        )
+    if is_catalog_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="spark.table / spark.sql",
+            note="Catalog resolution is runtime-owned.",
+        )
+    if is_http_file_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Core HTTP file fetch + Spark reader",
+            note="Fetch algorithm lives in core.",
+        )
+    if is_rest_api_connector(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Core REST client + Spark JSON materialization",
+            note="Secrets resolve in Databricks.",
+        )
+    if is_bounded_stream_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Spark bounded kafka/eventhubs reader",
+            note="Not continuous streaming.",
+        )
+    if is_delta_share_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Delta Sharing Spark connector",
+            note="Runtime must provide connector support.",
+        )
+    if is_file_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=SUPPORTED,
+            native_mapping="Spark file reader",
+            note="Format and reader options are core-normalized.",
+        )
+    if is_native_passthrough_source(payload):
+        return DatabricksSourceClassification(
+            source_type=source_type,
+            status=REVIEW_REQUIRED,
+            native_mapping="Databricks native connector handoff",
+            note="Adapter-owned design review.",
+        )
+    return DatabricksSourceClassification(
+        source_type=source_type,
+        status=UNSUPPORTED,
+        native_mapping=None,
+        note="No Databricks source renderer is declared for this connector.",
+    )
+
+
+def _is_jdbc_source(source: dict[str, Any]) -> bool:
+    connector = source.get("connector") or source.get("type")
+    return connector in JDBC_CONNECTORS or source.get("type") == "jdbc"

contractforge_databricks/sources/delta_share.py

@@ -0,0 +1,21 @@
+"""Databricks Delta Sharing source rendering."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import delta_share_options, is_delta_share_source as is_delta_share_source
+from contractforge_databricks.security import redact_value
+
+
+def render_delta_share_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    options = delta_share_options(source)
+    lines = [
+        f"{dataframe_name} = (",
+        "    spark.read",
+        "    .format('deltaSharing')",
+    ]
+    for key, value in sorted(options.items()):
+        lines.append(f"    .option({key!r}, {value!r})")
+    lines.extend([")", "", f"delta_share_options_review = {redact_value(options)!r}"])
+    return "\n".join(lines) + "\n"

contractforge_databricks/sources/files.py

@@ -0,0 +1,48 @@
+"""Databricks Spark file and catalog source rendering."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import (
+    TableRefResolver,
+    catalog_source_query,
+    catalog_source_table_or_path,
+    file_reader_options,
+    file_source_format,
+    is_catalog_source as is_catalog_source,
+    is_file_source as is_file_source,
+)
+
+
+def render_file_source_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    file_format = file_source_format(source)
+    path = source.get("path")
+    if not path:
+        raise ValueError("file source requires path")
+    options = file_reader_options(source)
+    lines = [
+        f"{dataframe_name} = (",
+        "    spark.read",
+        f"    .format({file_format!r})",
+    ]
+    for key, value in sorted(options.items()):
+        lines.append(f"    .option({key!r}, {value!r})")
+    lines.extend([f"    .load({path!r})", ")"])
+    return "\n".join(lines) + "\n"
+
+
+def render_catalog_source_python(
+    source: dict[str, Any],
+    *,
+    dataframe_name: str = "df",
+    table_ref_resolver: TableRefResolver | None = None,
+) -> str:
+    source_type = source.get("type")
+    if source_type == "sql":
+        return f"{dataframe_name} = spark.sql({catalog_source_query(source, table_ref_resolver=table_ref_resolver)!r})\n"
+    table = catalog_source_table_or_path(source, table_ref_resolver=table_ref_resolver)
+    if source.get("path") and not source.get("table"):
+        file_format = "delta" if source_type == "delta_table" else source_type.replace("_table", "")
+        return f"{dataframe_name} = spark.read.format({file_format!r}).load({str(table)!r})\n"
+    return f"{dataframe_name} = spark.table({str(table)!r})\n"

contractforge_databricks/sources/http_file.py

@@ -0,0 +1,46 @@
+"""Databricks HTTP file source rendering."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import (
+    http_file_format,
+    http_file_headers,
+    http_file_reader_options,
+    is_http_file_source as is_http_file_source,
+)
+from contractforge_databricks.security import redact_value
+
+
+def render_http_file_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    if not is_http_file_source(source):
+        raise ValueError("HTTP file rendering requires source.type http_file/http_csv/http_json/http_text")
+    request = source.get("request", {})
+    url = source.get("url") or request.get("url")
+    if not url:
+        raise ValueError("HTTP file source requires url or request.url")
+    method = str(request.get("method") or "GET").upper()
+    if method != "GET":
+        raise ValueError("HTTP file source supports only GET")
+    file_format = http_file_format(source)
+    http_file_headers(source)
+    options = http_file_reader_options(source)
+    lines = [
+        "from contractforge_core.connectors import download_http_file",
+        "",
+        f"source = {source!r}",
+        "local_path = download_http_file(source)",
+        f"reader = spark.read.format({file_format!r})",
+    ]
+    for key, value in sorted(options.items()):
+        lines.append(f"reader = reader.option({key!r}, {value!r})")
+    lines.extend(
+        [
+            f"{dataframe_name} = reader.load(local_path)",
+            "",
+            "# Rendered HTTP source with sensitive values redacted for review:",
+            f"http_source_review = {redact_value(source)!r}",
+        ]
+    )
+    return "\n".join(lines) + "\n"

contractforge_databricks/sources/interpret.py

@@ -0,0 +1,76 @@
+"""Interpret core source contracts for Databricks renderers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_databricks.environment import DatabricksEnvironment
+
+_FILE_STREAM_SOURCE_TYPES = {
+    "adls",
+    "avro",
+    "azure_blob",
+    "blob",
+    "csv",
+    "delta",
+    "gcs",
+    "json",
+    "object_storage",
+    "orc",
+    "parquet",
+    "s3",
+    "text",
+    "xml",
+}
+
+
+def is_incremental_file_source(source: dict[str, Any]) -> bool:
+    source_type = str(source.get("type") or "")
+    if source_type == "incremental_files":
+        return True
+    return source.get("intent") == "file_stream" and source_type in _FILE_STREAM_SOURCE_TYPES
+
+
+def interpret_incremental_files_source(
+    source: dict[str, Any],
+    *,
+    environment: DatabricksEnvironment | None = None,
+) -> dict[str, Any]:
+    if not is_incremental_file_source(source):
+        raise ValueError("incremental file interpretation requires source.type incremental_files or source.intent file_stream")
+    rendered = dict(source)
+    rendered["type"] = "incremental_files"
+    if not rendered.get("path"):
+        raise ValueError("Databricks file_stream source requires source.path")
+    options = _options(source)
+    params = (environment.parameters if environment else {}) or {}
+    state = source.get("state") if isinstance(source.get("state"), dict) else {}
+    state_location = state.get("location") if isinstance(state.get("location"), dict) else {}
+
+    if not rendered.get("progress_location") and state_location.get("type") == "object_storage" and state_location.get("path"):
+        rendered["progress_location"] = state_location["path"]
+
+    _set_bool_option(options, "cloudFiles.inferColumnTypes", source.get("options", {}).get("infer_column_types"))
+    _set_bool_option(options, "cloudFiles.inferColumnTypes", params.get("incremental_files.infer_column_types"))
+    _set_if_present(options, "cloudFiles.maxFilesPerTrigger", source.get("max_files_per_trigger"))
+    _set_if_present(options, "cloudFiles.maxFilesPerTrigger", params.get("incremental_files.max_files_per_trigger"))
+
+    rendered["options"] = options
+    return rendered
+
+
+def _options(source: dict[str, Any]) -> dict[str, str]:
+    raw = source.get("options") if isinstance(source.get("options"), dict) else {}
+    ignored = {"infer_column_types"}
+    return {str(key): str(value) for key, value in raw.items() if key not in ignored}
+
+
+def _set_bool_option(options: dict[str, str], key: str, value: object) -> None:
+    if value is None:
+        return
+    options[key] = str(bool(value)).lower() if isinstance(value, bool) else str(value)
+
+
+def _set_if_present(options: dict[str, str], key: str, value: object) -> None:
+    if value is not None:
+        options[key] = str(value)

contractforge_databricks/sources/jdbc.py

@@ -0,0 +1,32 @@
+"""Databricks JDBC source rendering."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.connectors import JDBC_CONNECTORS, jdbc_common_options
+from contractforge_databricks.security import assert_no_inline_jdbc_secrets, redact_value
+
+
+def render_jdbc_python(source: dict[str, Any], *, dataframe_name: str = "df") -> str:
+    connector = source.get("connector") or source.get("type")
+    if connector not in JDBC_CONNECTORS and source.get("type") != "jdbc":
+        raise ValueError("JDBC rendering requires a JDBC source connector")
+    options = jdbc_options(source)
+    lines = [
+        f"{dataframe_name} = (",
+        "    spark.read",
+        "    .format('jdbc')",
+    ]
+    for key, value in sorted(options.items()):
+        lines.append(f"    .option({key!r}, {value!r})")
+    lines.extend([")", ""])
+    lines.append("# Rendered JDBC options with sensitive values redacted for review:")
+    lines.append(f"jdbc_options_review = {redact_value(options)!r}")
+    return "\n".join(lines) + "\n"
+
+
+def jdbc_options(source: dict[str, Any]) -> dict[str, str]:
+    options = jdbc_common_options(source)
+    assert_no_inline_jdbc_secrets(options)
+    return options

contractforge_databricks/sources/metadata.py

@@ -0,0 +1,18 @@
+"""Databricks source metadata evidence helpers."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from contractforge_core.connectors import source_metadata_from_contract as core_source_metadata_from_contract
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.rendering.names import target_full_name
+
+
+def source_metadata_from_contract(contract: SemanticContract) -> dict[str, Any]:
+    return core_source_metadata_from_contract(contract, target_table=target_full_name(contract))
+
+
+def render_source_metadata_json(contract: SemanticContract) -> str:
+    return json.dumps(source_metadata_from_contract(contract), indent=2, sort_keys=True) + "\n"

contractforge_databricks/sources/native_passthrough.py

@@ -0,0 +1,33 @@
+"""Databricks native passthrough planning artifacts."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from contractforge_core.connectors import native_passthrough_descriptor
+
+
+def render_native_passthrough_plan(source: dict[str, Any]) -> str:
+    descriptor = native_passthrough_descriptor(source)
+
+    payload = {
+        "kind": "databricks_native_passthrough_plan",
+        **descriptor,
+        "recommended_databricks_targets": _recommended_targets(str(descriptor["system"])),
+        "notes": [
+            "Use Databricks-native ingestion where available, such as Lakeflow Connect or Databricks Connections.",
+            "Do not implement proprietary SaaS API clients inside contractforge_databricks unless no native path exists.",
+            "Adapter execution must remain platform-owned and auditable.",
+        ],
+    }
+    return json.dumps(payload, indent=2, sort_keys=True) + "\n"
+
+
+def _recommended_targets(system: str) -> list[str]:
+    normalized = system.lower()
+    if normalized in {"salesforce", "workday", "servicenow", "google_analytics", "google_ads"}:
+        return ["lakeflow_connect"]
+    if normalized in {"sftp", "ftp"}:
+        return ["databricks_connection", "autoloader"]
+    return ["databricks_connection", "lakeflow_connect_if_available"]

contractforge_databricks/sources/rds_iam.py

@@ -0,0 +1,15 @@
+"""Compatibility re-exports for Databricks JDBC RDS IAM helpers."""
+
+from contractforge_core.connectors.databases import (
+    generate_rds_iam_auth_token,
+    infer_aws_region_from_rds_host,
+    parse_jdbc_host_port,
+    rds_iam_review_options,
+)
+
+__all__ = [
+    "generate_rds_iam_auth_token",
+    "infer_aws_region_from_rds_host",
+    "parse_jdbc_host_port",
+    "rds_iam_review_options",
+]