contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/cli_maintenance.py

@@ -0,0 +1,69 @@
+"""Maintenance CLI commands for the Databricks adapter."""
+
+from __future__ import annotations
+
+import argparse
+import json
+
+from contractforge_databricks.cost import CostModel, build_operational_cost_report
+from contractforge_databricks.maintenance import build_control_retention_plan
+
+
+def add_maintenance_parser(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
+    maintenance = subparsers.add_parser("maintenance", help="Render Databricks operational maintenance SQL")
+    maintenance_sub = maintenance.add_subparsers(dest="maintenance_command", required=True)
+    retention = maintenance_sub.add_parser("ctrl-retention")
+    retention.add_argument("--catalog", default="main")
+    retention.add_argument("--schema", default="ops")
+    retention.add_argument("--retention-days", required=True, type=int)
+    retention.add_argument("--target", dest="targets", action="append")
+    retention.add_argument("--vacuum", action="store_true")
+    retention.add_argument("--vacuum-retention-hours", type=int, default=168)
+    retention.add_argument("--indent", type=int, default=2)
+    cost = maintenance_sub.add_parser("cost-report")
+    cost.add_argument("--catalog", default="main")
+    cost.add_argument("--schema", default="ops")
+    cost.add_argument("--lookback-days", type=int, default=30)
+    cost.add_argument("--group-by", action="append")
+    cost.add_argument("--dbu-per-hour", type=float)
+    cost.add_argument("--currency-per-dbu", type=float)
+    cost.add_argument("--currency", default="USD")
+    cost.add_argument("--success-only", action="store_true")
+    cost.add_argument("--limit", type=int, default=100)
+    cost.add_argument("--indent", type=int, default=2)
+
+
+def maintenance_command(args: argparse.Namespace) -> int:
+    if args.maintenance_command == "cost-report":
+        return _cost_report(args)
+    if args.maintenance_command != "ctrl-retention":
+        raise ValueError(f"unsupported maintenance command: {args.maintenance_command}")
+    plan = build_control_retention_plan(
+        catalog=args.catalog,
+        schema=args.schema,
+        retention_days=args.retention_days,
+        vacuum=args.vacuum,
+        vacuum_retention_hours=args.vacuum_retention_hours,
+        targets=args.targets,
+    )
+    print(json.dumps({"status": "DRY_RUN", "plan": plan}, indent=args.indent, sort_keys=True, default=str))
+    return 0
+
+
+def _cost_report(args: argparse.Namespace) -> int:
+    report = build_operational_cost_report(
+        catalog=args.catalog,
+        schema=args.schema,
+        lookback_days=args.lookback_days,
+        group_by=tuple(args.group_by or ("target_table", "mode", "status")),
+        cost_model=CostModel(
+            dbu_per_hour=args.dbu_per_hour,
+            currency_per_dbu=args.currency_per_dbu,
+            currency=args.currency,
+        ),
+        include_failed=not args.success_only,
+        query_only=True,
+        limit=args.limit,
+    )
+    print(json.dumps(report, indent=args.indent, sort_keys=True, default=str))
+    return 0

contractforge_databricks/coercion.py

@@ -0,0 +1,31 @@
+"""Small adapter-local coercion helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+
+def mapping(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, Mapping) else {}
+
+
+def mapping_list(value: object) -> list[dict[str, Any]]:
+    return [dict(item) for item in value or () if isinstance(item, Mapping)]
+
+
+def string_list(value: object, *, sep: str | None = None) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        items = value.split(sep) if sep else (value,)
+        return [item.strip() for item in items if item.strip()]
+    if not isinstance(value, Iterable):
+        return [str(value)]
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def string_map(value: object) -> dict[str, str]:
+    if not isinstance(value, Mapping):
+        return {}
+    return {str(key): str(item).lower() if isinstance(item, bool) else str(item) for key, item in value.items()}

contractforge_databricks/contract_extensions.py

@@ -0,0 +1,70 @@
+"""Databricks contract extension utilities."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any
+
+from contractforge_core.planner import PlanningWarning
+
+DATABRICKS_EXTENSION_FIELDS = {
+    "allow_type_widening",
+    "autoloader",
+    "cache_source",
+    "cluster_columns",
+    "delta_properties",
+    "encoding",
+    "encoding_columns",
+    "explain_format",
+    "explain_mode",
+    "fix_encoding",
+    "hooks",
+    "lakeflow",
+    "lock_enabled",
+    "merge_partition_column",
+    "merge_strategy",
+    "openlineage_enabled",
+    "openlineage_namespace",
+    "openlineage_producer",
+    "optimize_after_write",
+    "partition_column",
+    "partition_columns",
+    "partition_value",
+    "replace_partitions_source_complete",
+    "write_engine",
+    "zorder_columns",
+}
+
+
+def normalize_databricks_contract(contract: dict[str, Any]) -> dict[str, Any]:
+    """Return a defensive copy of a Databricks contract mapping.
+
+    Databricks-owned execution fields must be declared explicitly under
+    ``extensions.databricks``. The adapter does not translate top-level aliases.
+    """
+
+    return deepcopy(contract)
+
+
+def databricks_extensions(contract: Any) -> dict[str, Any]:
+    extensions = getattr(contract, "extensions", None)
+    if not isinstance(extensions, dict):
+        return {}
+    value = extensions.get("databricks")
+    return dict(value) if isinstance(value, dict) else {}
+
+
+def databricks_extension_warnings(contract: Any) -> tuple[PlanningWarning, ...]:
+    """Return warnings for Databricks extension keys the adapter will ignore."""
+
+    unknown = sorted(set(databricks_extensions(contract)) - DATABRICKS_EXTENSION_FIELDS)
+    return tuple(
+        PlanningWarning(
+            code="DATABRICKS_UNKNOWN_EXTENSION",
+            message=(
+                f"extensions.databricks.{name} is not a recognized Databricks adapter extension "
+                "and will not be honored by planning, rendering or runtime execution."
+            ),
+        )
+        for name in unknown
+    )

contractforge_databricks/cost/__init__.py

@@ -0,0 +1,11 @@
+from contractforge_databricks.cost.model import CostModel
+from contractforge_databricks.cost.report import build_operational_cost_report
+from contractforge_databricks.cost.sql import DEFAULT_COST_GROUP_BY, VALID_COST_GROUP_FIELDS, render_operational_cost_query
+
+__all__ = [
+    "CostModel",
+    "DEFAULT_COST_GROUP_BY",
+    "VALID_COST_GROUP_FIELDS",
+    "build_operational_cost_report",
+    "render_operational_cost_query",
+]

contractforge_databricks/cost/model.py

@@ -0,0 +1,22 @@
+"""Databricks logical cost model."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class CostModel:
+    dbu_per_hour: float | None = None
+    currency_per_dbu: float | None = None
+    currency: str = "USD"
+
+    @property
+    def enabled(self) -> bool:
+        return self.dbu_per_hour is not None and self.currency_per_dbu is not None
+
+    @property
+    def hourly_rate(self) -> float | None:
+        if not self.enabled:
+            return None
+        return float(self.dbu_per_hour or 0.0) * float(self.currency_per_dbu or 0.0)

contractforge_databricks/cost/report.py

@@ -0,0 +1,65 @@
+"""JSON-friendly cost report planning."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_databricks.cost.model import CostModel
+from contractforge_databricks.cost.sql import DEFAULT_COST_GROUP_BY, render_operational_cost_query
+
+
+def build_operational_cost_report(
+    *,
+    catalog: str = "main",
+    schema: str = "ops",
+    lookback_days: int = 30,
+    group_by: tuple[str, ...] = DEFAULT_COST_GROUP_BY,
+    cost_model: CostModel | None = None,
+    include_failed: bool = True,
+    query_only: bool = True,
+    runner: Any | None = None,
+    limit: int = 100,
+) -> dict[str, Any]:
+    if limit < 1:
+        raise ValueError("limit must be greater than or equal to 1")
+    model = cost_model or CostModel()
+    query = render_operational_cost_query(
+        catalog=catalog,
+        schema=schema,
+        lookback_days=lookback_days,
+        group_by=group_by,
+        cost_model=model,
+        include_failed=include_failed,
+    )
+    rows = [] if query_only or runner is None else _collect_rows(runner.sql(f"{query}\nLIMIT {int(limit)}"))
+    return {
+        "status": "QUERY_ONLY" if query_only or runner is None else "SUCCESS",
+        "catalog": catalog,
+        "schema": schema,
+        "lookback_days": lookback_days,
+        "group_by": list(group_by),
+        "include_failed": include_failed,
+        "limit": limit,
+        "cost_model": {
+            "enabled": model.enabled,
+            "dbu_per_hour": model.dbu_per_hour,
+            "currency_per_dbu": model.currency_per_dbu,
+            "currency": model.currency,
+            "hourly_rate": model.hourly_rate,
+        },
+        "query": query,
+        "rows": rows,
+    }
+
+
+def _collect_rows(result: Any) -> list[dict[str, Any]]:
+    collected = result.collect() if hasattr(result, "collect") else result
+    rows = []
+    for row in collected or []:
+        if hasattr(row, "asDict"):
+            rows.append(row.asDict(recursive=True))
+        elif isinstance(row, dict):
+            rows.append(dict(row))
+        else:
+            rows.append(dict(row))
+    return rows

contractforge_databricks/cost/sql.py

@@ -0,0 +1,136 @@
+"""Databricks operational cost SQL rendering."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from contractforge_databricks.cost.model import CostModel
+from contractforge_databricks.evidence import evidence_table_names
+from contractforge_databricks.sql import quote_identifier, quote_table_name, sql_string
+
+VALID_COST_GROUP_FIELDS = {
+    "contract_domain",
+    "contract_owner",
+    "criticality",
+    "layer",
+    "target_table",
+    "mode",
+    "runtime_type",
+    "source_connector",
+    "source_provider",
+    "status",
+}
+DEFAULT_COST_GROUP_BY = ("target_table", "layer", "mode", "status")
+
+
+def render_operational_cost_query(
+    *,
+    catalog: str = "main",
+    schema: str = "ops",
+    lookback_days: int = 30,
+    group_by: Iterable[str] | None = None,
+    cost_model: CostModel | None = None,
+    include_failed: bool = True,
+) -> str:
+    if lookback_days < 1:
+        raise ValueError("lookback_days must be greater than or equal to 1")
+    fields = _normalize_group_by(group_by)
+    model = cost_model or CostModel()
+    _validate_float("dbu_per_hour", model.dbu_per_hour)
+    _validate_float("currency_per_dbu", model.currency_per_dbu)
+    runs_table = evidence_table_names(catalog, schema)["runs"]
+    status_filter = "" if include_failed else "AND status = 'SUCCESS'"
+    hourly_rate = "NULL" if model.hourly_rate is None else repr(float(model.hourly_rate))
+    return f"""
+WITH base AS (
+    SELECT
+        target_table,
+        layer,
+        mode,
+        status,
+        contract_domain,
+        contract_owner,
+        runtime_type,
+        source_connector,
+        source_provider,
+        COALESCE(
+            get_json_object(operations_json, '$.metadata.criticality'),
+            get_json_object(operations_json, '$.criticality'),
+            'unknown'
+        ) AS criticality,
+        CAST(COALESCE(rows_read, 0) AS BIGINT) AS rows_read,
+        CAST(COALESCE(rows_written, 0) AS BIGINT) AS rows_written,
+        CAST(COALESCE(rows_quarantined, 0) AS BIGINT) AS rows_quarantined,
+        CAST(COALESCE(duration_seconds, 0.0) AS DOUBLE) AS duration_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.read'), '0') AS DOUBLE) AS read_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.prepare'), '0') AS DOUBLE) AS prepare_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.quality'), '0') AS DOUBLE) AS quality_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.schema'), '0') AS DOUBLE) AS schema_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.preflight'), '0') AS DOUBLE) AS preflight_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.write'), '0') AS DOUBLE) AS write_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.maintenance'), '0') AS DOUBLE) AS maintenance_seconds,
+        CAST(COALESCE(get_json_object(stage_durations_json, '$.governance'), '0') AS DOUBLE) AS governance_seconds
+    FROM {quote_table_name(runs_table)}
+    WHERE run_date >= date_sub(current_date(), {int(lookback_days)})
+      {status_filter}
+),
+agg AS (
+    SELECT
+        {_group_select(fields)},
+        COUNT(*) AS runs,
+        SUM(CASE WHEN status = 'SUCCESS' THEN 1 ELSE 0 END) AS successful_runs,
+        SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) AS failed_runs,
+        SUM(rows_read) AS rows_read,
+        SUM(rows_written) AS rows_written,
+        SUM(rows_quarantined) AS rows_quarantined,
+        SUM(duration_seconds) AS duration_seconds,
+        SUM(read_seconds) AS read_seconds,
+        SUM(prepare_seconds) AS prepare_seconds,
+        SUM(quality_seconds) AS quality_seconds,
+        SUM(schema_seconds) AS schema_seconds,
+        SUM(preflight_seconds) AS preflight_seconds,
+        SUM(write_seconds) AS write_seconds,
+        SUM(maintenance_seconds) AS maintenance_seconds,
+        SUM(governance_seconds) AS governance_seconds
+    FROM base
+    GROUP BY {_group_by(fields)}
+)
+SELECT
+    *,
+    CASE WHEN duration_seconds > 0 THEN rows_written / duration_seconds ELSE NULL END AS rows_written_per_second,
+    CASE WHEN duration_seconds > 0 THEN rows_read / duration_seconds ELSE NULL END AS rows_read_per_second,
+    CASE WHEN runs > 0 THEN duration_seconds / runs ELSE NULL END AS avg_duration_seconds,
+    {hourly_rate} AS estimated_hourly_rate,
+    {sql_string(model.currency)} AS estimated_currency,
+    CASE WHEN {hourly_rate} IS NULL THEN NULL ELSE duration_seconds / 3600.0 * {hourly_rate} END AS estimated_compute_cost,
+    CASE
+        WHEN {hourly_rate} IS NULL OR rows_written <= 0 THEN NULL
+        ELSE (duration_seconds / 3600.0 * {hourly_rate}) / (rows_written / 1000000.0)
+    END AS estimated_cost_per_million_rows,
+    'estimated_from_evidence_runs' AS cost_source
+FROM agg
+ORDER BY estimated_compute_cost DESC NULLS LAST, duration_seconds DESC
+""".strip()
+
+
+def _normalize_group_by(group_by: Iterable[str] | None) -> tuple[str, ...]:
+    fields = tuple(group_by or DEFAULT_COST_GROUP_BY)
+    if not fields:
+        raise ValueError("group_by must contain at least one field")
+    unknown = sorted(set(fields) - VALID_COST_GROUP_FIELDS)
+    if unknown:
+        raise ValueError(f"unknown group_by fields: {unknown}")
+    return fields
+
+
+def _validate_float(name: str, value: float | None) -> None:
+    if value is not None and value < 0:
+        raise ValueError(f"{name} must be greater than or equal to 0")
+
+
+def _group_select(fields: tuple[str, ...]) -> str:
+    return ",\n        ".join(quote_identifier(field) for field in fields)
+
+
+def _group_by(fields: tuple[str, ...]) -> str:
+    return ", ".join(quote_identifier(field) for field in fields)

contractforge_databricks/dashboards/__init__.py

@@ -0,0 +1,15 @@
+from contractforge_databricks.dashboards.control_tables import (
+    DashboardQuery,
+    control_dashboard_blueprint,
+    control_dashboard_queries,
+    render_control_dashboard_artifacts,
+    render_control_dashboard_sql,
+)
+
+__all__ = [
+    "DashboardQuery",
+    "control_dashboard_blueprint",
+    "control_dashboard_queries",
+    "render_control_dashboard_artifacts",
+    "render_control_dashboard_sql",
+]

contractforge_databricks/dashboards/control_tables.py

@@ -0,0 +1,150 @@
+"""Databricks dashboard artifacts over ContractForge control tables."""
+
+from __future__ import annotations
+
+from contractforge_core.reporting import DashboardQuery
+from contractforge_databricks.sql import quote_table_name
+
+
+def control_dashboard_queries(*, catalog: str = "main", schema: str = "ops", lookback_days: int = 7) -> tuple[DashboardQuery, ...]:
+    t = _tables(catalog, schema)
+    days = int(lookback_days)
+    return (
+        _q("q01_executive_kpis", "Control Tower", "kpi_card_strip", f"""
+            SELECT count(*) AS total_runs,
+                   sum(CASE WHEN status = 'SUCCESS' THEN 1 ELSE 0 END) AS successful_runs,
+                   sum(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) AS failed_runs,
+                   round(100.0 * sum(CASE WHEN status = 'SUCCESS' THEN 1 ELSE 0 END) / nullif(count(*), 0), 2) AS success_rate_pct,
+                   count(DISTINCT target_table) AS active_targets,
+                   sum(coalesce(rows_read, 0)) AS rows_read,
+                   sum(coalesce(rows_written, 0)) AS rows_written,
+                   sum(coalesce(rows_quarantined, 0)) AS rows_quarantined
+            FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days})"""),
+        _q("q02_status_trend", "Run Health Trend", "stacked_area", f"""
+            SELECT run_date, layer, status, count(*) AS runs, sum(coalesce(rows_written, 0)) AS rows_written
+            FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days})
+            GROUP BY run_date, layer, status ORDER BY run_date, layer, status"""),
+        _q("q03_latest_target_health", "Target Health Radar", "table_with_conditional_formatting", f"""
+            WITH ranked AS (
+              SELECT *, row_number() OVER (PARTITION BY target_table ORDER BY run_ts_utc DESC, finished_at_utc DESC) AS rn
+              FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days})
+            )
+            SELECT target_table, layer, mode, status, quality_status, rows_read, rows_written,
+                   rows_quarantined, duration_seconds, finished_at_utc, runtime_type, error_message
+            FROM ranked WHERE rn = 1 ORDER BY status, target_table"""),
+        _q("q04_recent_failures", "Latest Incidents", "table", f"""
+            SELECT e.error_ts_utc, e.target_table, r.layer, e.mode, e.error_type, e.error_message, r.run_id
+            FROM {t['errors']} e LEFT JOIN {t['runs']} r ON e.run_id = r.run_id
+            WHERE e.error_date >= date_sub(current_date(), {days}) ORDER BY e.error_ts_utc DESC"""),
+        _q("q05_target_reliability", "Target Reliability Matrix", "heatmap_or_table", f"""
+            SELECT target_table, layer, mode, count(*) AS runs,
+                   sum(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) AS failed_runs,
+                   round(100.0 * sum(CASE WHEN status = 'SUCCESS' THEN 1 ELSE 0 END) / nullif(count(*), 0), 2) AS success_rate_pct
+            FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days})
+            GROUP BY target_table, layer, mode ORDER BY failed_runs DESC, success_rate_pct ASC"""),
+        _q("q06_sla_freshness", "Freshness SLA Board", "table_with_status_colors", f"""
+            WITH latest_success AS (
+              SELECT target_table, max(finished_at_utc) AS last_success_at_utc FROM {t['runs']} WHERE status = 'SUCCESS' GROUP BY target_table
+            ), ops AS (
+              SELECT *, row_number() OVER (PARTITION BY target_table ORDER BY recorded_at_utc DESC) AS rn FROM {t['operations']}
+            )
+            SELECT o.target_table, o.criticality, o.expected_frequency, o.freshness_sla_minutes, s.last_success_at_utc,
+                   CASE WHEN s.last_success_at_utc IS NULL THEN 'NO_SUCCESS'
+                        WHEN o.freshness_sla_minutes IS NULL THEN 'NO_SLA'
+                        WHEN (unix_timestamp(current_timestamp()) - unix_timestamp(s.last_success_at_utc)) / 60 > o.freshness_sla_minutes THEN 'BREACHED'
+                        ELSE 'OK' END AS freshness_status, o.runbook_url
+            FROM ops o LEFT JOIN latest_success s ON o.target_table = s.target_table WHERE o.rn = 1"""),
+        _q("q07_failure_taxonomy", "Failure Taxonomy", "horizontal_bar", f"""
+            SELECT coalesce(error_type, 'unknown') AS error_type, count(*) AS failures, count(DISTINCT target_table) AS affected_targets
+            FROM {t['errors']} WHERE error_date >= date_sub(current_date(), {days})
+            GROUP BY coalesce(error_type, 'unknown') ORDER BY failures DESC"""),
+        _q("q08_error_drilldown", "Error Drilldown", "table", f"SELECT * FROM {t['errors']} WHERE error_date >= date_sub(current_date(), {days}) ORDER BY error_ts_utc DESC"),
+        _q("q09_duration_percentiles", "Duration Percentiles by Mode", "grouped_bar", f"""
+            SELECT layer, mode, count(*) AS successful_runs, round(avg(duration_seconds), 2) AS avg_duration_seconds,
+                   round(percentile_approx(duration_seconds, 0.95), 2) AS p95_duration_seconds
+            FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days}) AND status = 'SUCCESS'
+            GROUP BY layer, mode ORDER BY p95_duration_seconds DESC"""),
+        _q("q10_stage_duration_breakdown", "Stage Bottlenecks", "stacked_bar", f"SELECT run_id, target_table, stage_durations_json FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days}) AND status = 'SUCCESS'"),
+        _q("q11_throughput_by_target", "Throughput by Target", "scatter_or_table", f"""
+            SELECT target_table, layer, mode, sum(rows_written) AS rows_written,
+                   round(sum(rows_written) / nullif(sum(duration_seconds), 0), 2) AS rows_written_per_second
+            FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days}) AND status = 'SUCCESS'
+            GROUP BY target_table, layer, mode ORDER BY rows_written_per_second ASC NULLS LAST"""),
+        _q("q12_slowest_runs", "Slowest Successful Runs", "table", f"SELECT run_ts_utc, target_table, mode, duration_seconds, rows_written, run_id FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days}) AND status = 'SUCCESS' ORDER BY duration_seconds DESC LIMIT 50"),
+        _q("q13_quality_summary", "Quality Outcomes", "stacked_bar", f"SELECT status, severity, count(*) AS rule_evaluations, sum(failed_count) AS failed_count FROM {t['quality']} GROUP BY status, severity"),
+        _q("q14_quality_rules_hotspots", "Rule Hotspots", "horizontal_bar", f"SELECT target_table, rule_name, status, sum(failed_count) AS failed_count FROM {t['quality']} GROUP BY target_table, rule_name, status ORDER BY failed_count DESC"),
+        _q("q15_quarantine_hotspots", "Quarantine Drilldown", "table", f"SELECT target_table, rule_name, count(*) AS quarantined_records FROM {t['quarantine']} GROUP BY target_table, rule_name ORDER BY quarantined_records DESC"),
+        _q("q16_effective_rows", "Useful Rows vs Quarantine", "stacked_bar", f"SELECT target_table, sum(rows_read) AS rows_read, sum(rows_written) AS rows_written, sum(rows_quarantined) AS rows_quarantined FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days}) GROUP BY target_table"),
+        _q("q17_stream_kpis", "Stream Control Tower", "kpi_card_strip", f"SELECT count(*) AS stream_runs, sum(batches_processed) AS batches_processed, sum(total_rows_written) AS total_rows_written FROM {t['streams']} WHERE started_at_utc >= current_timestamp() - INTERVAL {days} DAYS"),
+        _q("q18_stream_runs", "Stream Runs", "table", f"SELECT stream_run_id, target_table, source_type, trigger, status, batches_processed, total_rows_written, started_at_utc, ended_at_utc FROM {t['streams']} ORDER BY started_at_utc DESC"),
+        _q("q19_stream_child_reconciliation", "Parent/Child Reconciliation", "table_with_status_colors", f"SELECT stream_run_id, target_table, status, batches_processed, total_rows_read, total_rows_written FROM {t['streams']} ORDER BY started_at_utc DESC"),
+        _q("q20_connector_runtime_matrix", "Connector and Runtime Matrix", "grouped_bar_or_heatmap", f"SELECT source_connector, source_provider, source_format, runtime_type, count(*) AS runs FROM {t['runs']} WHERE run_date >= date_sub(current_date(), {days}) GROUP BY source_connector, source_provider, source_format, runtime_type"),
+        _q("q21_operations_coverage", "Operations Coverage", "table_with_completeness_score", f"SELECT target_table, criticality, expected_frequency, freshness_sla_minutes, runbook_url, status FROM {t['operations']}"),
+        _q("q22_governance_artifacts", "Governance Artifacts", "table", f"""
+            SELECT coalesce(s.target_table, a.target_table, x.target_table) AS target_table,
+                   s.schema_change_events, a.annotation_events, x.access_events
+            FROM (SELECT target_table, count(*) AS schema_change_events FROM {t['schema_changes']} GROUP BY target_table) s
+            FULL OUTER JOIN (SELECT target_table, count(*) AS annotation_events FROM {t['annotations']} GROUP BY target_table) a ON s.target_table = a.target_table
+            FULL OUTER JOIN (SELECT target_table, count(*) AS access_events FROM {t['access']} GROUP BY target_table) x ON coalesce(s.target_table, a.target_table) = x.target_table"""),
+    )
+
+
+def render_control_dashboard_sql(*, catalog: str = "main", schema: str = "ops", lookback_days: int = 7) -> str:
+    blocks = ["-- ContractForge Operations Command Center", "-- Databricks SQL dashboard queries"]
+    for query in control_dashboard_queries(catalog=catalog, schema=schema, lookback_days=lookback_days):
+        blocks.extend(["", f"-- {query.name}", f"-- Visualization: {query.visualization}", _clean_sql(query.sql) + ";"])
+    return "\n".join(blocks) + "\n"
+
+
+def control_dashboard_blueprint(*, catalog: str = "main", schema: str = "ops", lookback_days: int = 7) -> dict[str, object]:
+    queries = control_dashboard_queries(catalog=catalog, schema=schema, lookback_days=lookback_days)
+    return {
+        "title": "ContractForge Operations Command Center",
+        "data_source": {"catalog": catalog, "schema": schema, "lookback_days": lookback_days},
+        "pages": {
+            "overview": ["q01_executive_kpis", "q02_status_trend", "q03_latest_target_health", "q04_recent_failures"],
+            "reliability": ["q06_sla_freshness", "q05_target_reliability", "q07_failure_taxonomy", "q08_error_drilldown"],
+            "performance": ["q09_duration_percentiles", "q10_stage_duration_breakdown", "q11_throughput_by_target", "q12_slowest_runs"],
+            "quality": ["q13_quality_summary", "q14_quality_rules_hotspots", "q16_effective_rows", "q15_quarantine_hotspots"],
+            "streaming": ["q17_stream_kpis", "q19_stream_child_reconciliation", "q18_stream_runs"],
+            "connectors_governance": ["q20_connector_runtime_matrix", "q21_operations_coverage", "q22_governance_artifacts"],
+        },
+        "queries": [query.__dict__ for query in queries],
+    }
+
+
+def render_control_dashboard_artifacts(*, catalog: str = "main", schema: str = "ops", lookback_days: int = 7) -> dict[str, str]:
+    import json
+
+    return {
+        "control_tables_dashboard.sql": render_control_dashboard_sql(catalog=catalog, schema=schema, lookback_days=lookback_days),
+        "control_tables_dashboard_blueprint.json": json.dumps(
+            control_dashboard_blueprint(catalog=catalog, schema=schema, lookback_days=lookback_days),
+            indent=2,
+            sort_keys=True,
+        )
+        + "\n",
+    }
+
+
+def _tables(catalog: str, schema: str) -> dict[str, str]:
+    return {
+        "runs": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_runs"),
+        "errors": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_errors"),
+        "quality": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_quality"),
+        "quarantine": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_quarantine"),
+        "schema_changes": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_schema_changes"),
+        "streams": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_streams"),
+        "annotations": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_annotations"),
+        "access": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_access"),
+        "operations": quote_table_name(f"{catalog}.{schema}.ctrl_ingestion_operations"),
+    }
+
+
+def _q(name: str, title: str, visualization: str, sql: str) -> DashboardQuery:
+    return DashboardQuery(name=name, title=title, visualization=visualization, sql=_clean_sql(sql))
+
+
+def _clean_sql(sql: str) -> str:
+    lines = [line.strip() for line in sql.strip().splitlines()]
+    return "\n".join(line for line in lines if line)

contractforge_databricks/diagnostics/__init__.py

@@ -0,0 +1,7 @@
+from contractforge_databricks.diagnostics.explain import (
+    ExplainPlanRecord,
+    render_create_explain_table_sql,
+    render_explain_insert_sql,
+)
+
+__all__ = ["ExplainPlanRecord", "render_create_explain_table_sql", "render_explain_insert_sql"]

contractforge_databricks/diagnostics/explain.py

@@ -0,0 +1,40 @@
+"""Databricks explain-plan diagnostic SQL."""
+
+from __future__ import annotations
+
+from contractforge_core.diagnostics import ExplainPlanRecord
+from contractforge_databricks.security import redact_text
+from contractforge_databricks.sql import quote_table_name, sql_string
+
+
+def render_create_explain_table_sql(*, catalog: str = "main", schema: str = "ops") -> str:
+    table = f"{catalog}.{schema}.ctrl_ingestion_explain"
+    return "\n".join(
+        [
+            f"CREATE SCHEMA IF NOT EXISTS {quote_table_name(f'{catalog}.{schema}')};",
+            "",
+            f"CREATE TABLE IF NOT EXISTS {quote_table_name(table)} (",
+            "  run_id STRING, target_table STRING, source_table STRING, mode STRING,",
+            "  explain_format STRING, plan_text STRING, captured_at_utc TIMESTAMP",
+            ")",
+            "USING DELTA;",
+            "",
+        ]
+    )
+
+
+def render_explain_insert_sql(
+    record: ExplainPlanRecord,
+    *,
+    catalog: str = "main",
+    schema: str = "ops",
+    truncate_at: int = 100_000,
+) -> str:
+    table = f"{catalog}.{schema}.ctrl_ingestion_explain"
+    plan_text = redact_text(record.plan_text)[:truncate_at]
+    return (
+        f"INSERT INTO {quote_table_name(table)} "
+        "(run_id, target_table, source_table, mode, explain_format, plan_text, captured_at_utc) VALUES "
+        f"({sql_string(record.run_id)}, {sql_string(record.target_table)}, {sql_string(record.source_name)}, "
+        f"{sql_string(record.mode)}, {sql_string(record.explain_format)}, {sql_string(plan_text)}, current_timestamp())"
+    )