contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/quality/results.py

@@ -0,0 +1,17 @@
+"""Compatibility exports for platform-neutral quality result models."""
+
+from contractforge_core.quality import (
+    QualityRuleResult,
+    QualitySeverity,
+    QualityStatus,
+    quality_status,
+    quarantinable_results,
+)
+
+__all__ = [
+    "QualityRuleResult",
+    "QualitySeverity",
+    "QualityStatus",
+    "quality_status",
+    "quarantinable_results",
+]

contractforge_databricks/quality/sql.py

@@ -0,0 +1,113 @@
+"""Databricks SQL quality check rendering."""
+
+from __future__ import annotations
+
+from contractforge_core.config import MAX_INLINE_ACCEPTED_VALUES
+from contractforge_core.semantic import QualityIntent, SemanticContract
+from contractforge_databricks.rendering.names import target_full_name
+from contractforge_databricks.sql import quote_identifier, quote_table_name
+
+
+def render_quality_check_sql(contract: SemanticContract, *, source_view: str | None = None) -> str:
+    table = quote_table_name(source_view or target_full_name(contract))
+    statements = []
+    for quality in contract.quality:
+        statements.append(_render_quality_intent(quality, table))
+    return "\n\n".join(statements) + ("\n" if statements else "-- No quality rules declared.\n")
+
+
+def _render_quality_intent(quality: QualityIntent, table: str) -> str:
+    if quality.rule == "required_columns":
+        return _render_required_columns(quality, table)
+    if quality.rule == "not_null":
+        column = _single_column(quality)
+        return (
+            f"-- quality: {quality.name}\n"
+            f"SELECT count(*) AS failed_rows FROM {table} WHERE {quote_identifier(column)} IS NULL;"
+        )
+    if quality.rule == "unique_key":
+        keys = ", ".join(quote_identifier(column) for column in quality.columns)
+        return (
+            f"-- quality: {quality.name}\n"
+            f"SELECT count(*) AS failed_rows FROM ("
+            f"SELECT {keys}, count(*) AS row_count FROM {table} GROUP BY {keys} HAVING count(*) > 1"
+            ") duplicates;"
+        )
+    if quality.rule == "accepted_values":
+        column = _single_column(quality)
+        values = quality.value if isinstance(quality.value, (list, tuple)) else [quality.value]
+        if len(values) > MAX_INLINE_ACCEPTED_VALUES:
+            raise ValueError(
+                f"quality.accepted_values.{column} has {len(values)} values. "
+                "Use a reference table or custom quality evaluator for large value sets."
+            )
+        accepted = ", ".join(_sql_literal(value) for value in values)
+        return (
+            f"-- quality: {quality.name}\n"
+            f"SELECT count(*) AS failed_rows FROM {table} "
+            f"WHERE {quote_identifier(column)} IS NOT NULL AND {quote_identifier(column)} NOT IN ({accepted});"
+        )
+    if quality.rule == "row_count_minimum":
+        return (
+            f"-- quality: {quality.name}\n"
+            f"SELECT CASE WHEN count(*) >= {int(quality.value)} THEN 0 ELSE 1 END AS failed_rows FROM {table};"
+        )
+    if quality.rule == "max_null_ratio":
+        column = _single_column(quality)
+        ratio = float(quality.value)
+        return (
+            f"-- quality: {quality.name}\n"
+            "SELECT CASE WHEN count(*) = 0 THEN 0 "
+            f"WHEN (sum(CASE WHEN {quote_identifier(column)} IS NULL THEN 1 ELSE 0 END) / count(*)) > {ratio} "
+            "THEN 1 ELSE 0 END AS failed_rows "
+            f"FROM {table};"
+        )
+    if quality.rule == "expression":
+        return (
+            f"-- quality: {quality.name}\n"
+            f"SELECT count(*) AS failed_rows FROM {table} "
+            f"WHERE NOT ({quality.value}) OR ({quality.value}) IS NULL;"
+        )
+    return f"-- Unsupported quality rule for Databricks SQL rendering: {quality.rule}\n"
+
+
+def _single_column(quality: QualityIntent) -> str:
+    if len(quality.columns) != 1:
+        raise ValueError(f"quality rule {quality.name} requires exactly one column")
+    return quality.columns[0]
+
+
+def _sql_literal(value: object) -> str:
+    if value is None:
+        return "NULL"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    return "'" + str(value).replace("'", "''") + "'"
+
+
+def _render_required_columns(quality: QualityIntent, table: str) -> str:
+    parts = _unquote_table_parts(table)
+    expected = ", ".join(_sql_literal(column) for column in quality.columns)
+    if len(parts) != 3:
+        return (
+            f"-- quality: {quality.name}\n"
+            "-- Required column checks need runtime schema inspection for temp views or non-qualified tables.\n"
+            f"-- expected columns: {', '.join(quality.columns)}"
+        )
+    catalog, schema, table_name = parts
+    return (
+        f"-- quality: {quality.name}\n"
+        f"SELECT count(*) AS failed_rows FROM (SELECT explode(array({expected})) AS expected_column) expected "
+        "LEFT ANTI JOIN ("
+        "SELECT column_name FROM system.information_schema.columns "
+        f"WHERE table_catalog = {_sql_literal(catalog)} "
+        f"AND table_schema = {_sql_literal(schema)} "
+        f"AND table_name = {_sql_literal(table_name)}"
+        ") actual ON expected.expected_column = actual.column_name;"
+    )
+
+
+def _unquote_table_parts(table: str) -> tuple[str, ...]:
+    return tuple(part.strip("`").replace("``", "`") for part in table.split(".") if part.strip("`"))

contractforge_databricks/rendering/__init__.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def render_databricks_artifacts(*args: Any, **kwargs: Any) -> Any:
+    from contractforge_databricks.rendering.bundle import render_databricks_artifacts as _render
+
+    return _render(*args, **kwargs)
+
+__all__ = ["render_databricks_artifacts"]

contractforge_databricks/rendering/bundle.py

@@ -0,0 +1,93 @@
+"""Bundle Databricks adapter artifacts."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+
+from contractforge_core.adapters import RenderedArtifacts
+from contractforge_core.planner import PlanningResult
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.annotations import render_annotations_audit_insert_sql, render_annotations_sql
+from contractforge_databricks.bundles import DatabricksJobSpec, render_databricks_asset_bundle
+from contractforge_databricks.capabilities.models import DatabricksCapabilities
+from contractforge_databricks.cost import render_operational_cost_query
+from contractforge_databricks.diagnostics import render_create_explain_table_sql
+from contractforge_databricks.environment import DatabricksEnvironment
+from contractforge_databricks.evidence import render_create_evidence_tables_sql, render_evidence_table_notes
+from contractforge_databricks.governance import render_access_audit_insert_sql, render_governance_sql
+from contractforge_databricks.lakeflow import evaluate_lakeflow_compatibility, render_lakeflow_review
+from contractforge_databricks.lineage import render_openlineage_insert_sql
+from contractforge_databricks.operations import render_operations_insert_sql, render_operations_json
+from contractforge_databricks.quality import render_quality_check_sql
+from contractforge_databricks.rendering.markdown import render_review_markdown
+from contractforge_databricks.rendering.names import artifact_prefix, bundle_name, job_name, task_key
+from contractforge_databricks.schema import plan_schema_policy
+from contractforge_databricks.shapes import render_shape_sql
+from contractforge_databricks.sources import render_source_artifacts, render_source_metadata_json
+from contractforge_databricks.state import render_control_table_migrations_sql, render_create_state_tables_sql
+from contractforge_databricks.transforms import render_transform_sql
+from contractforge_databricks.write_modes import choose_write_strategy, render_write_mode_sql_notes
+
+
+def render_databricks_artifacts(
+    contract: SemanticContract,
+    planning: PlanningResult,
+    capabilities: DatabricksCapabilities,
+    *,
+    environment: DatabricksEnvironment | None = None,
+) -> RenderedArtifacts:
+    env = environment or DatabricksEnvironment()
+    prefix = artifact_prefix(contract)
+    artifacts = {
+        f"{prefix}.review.md": render_review_markdown(contract, planning, capabilities),
+        f"{prefix}.capabilities.json": json.dumps(capabilities.as_dict(), indent=2, sort_keys=True),
+        f"{prefix}.write_mode.sql": render_write_mode_sql_notes(contract),
+        f"{prefix}.shape.sql": render_shape_sql(contract),
+        f"{prefix}.transform.sql": render_transform_sql(contract),
+        f"{prefix}.annotations.sql": render_annotations_sql(contract),
+        f"{prefix}.annotations_audit.sql": render_annotations_audit_insert_sql(contract, catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.governance.sql": render_governance_sql(contract),
+        f"{prefix}.access_audit.sql": render_access_audit_insert_sql(contract, catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.quality.sql": render_quality_check_sql(contract),
+        f"{prefix}.schema_policy.json": json.dumps(plan_schema_policy(contract).as_dict(), indent=2, sort_keys=True),
+        f"{prefix}.source_metadata.json": render_source_metadata_json(contract),
+        f"{prefix}.evidence.sql": render_evidence_table_notes(catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.evidence_ddl.sql": render_create_evidence_tables_sql(catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.state_ddl.sql": render_create_state_tables_sql(catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.control_table_migrations.sql": render_control_table_migrations_sql(catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.openlineage.sql": _render_openlineage_template(contract, env),
+        f"{prefix}.operations.json": render_operations_json(contract),
+        f"{prefix}.operations.sql": render_operations_insert_sql(contract, catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.diagnostics_ddl.sql": render_create_explain_table_sql(catalog=env.evidence_catalog, schema=env.evidence_schema),
+        f"{prefix}.cost.sql": render_operational_cost_query(catalog=env.evidence_catalog, schema=env.evidence_schema),
+    }
+    strategy = choose_write_strategy(contract, capabilities)
+    artifacts[f"{prefix}.strategy.json"] = json.dumps(strategy.as_dict(), indent=2, sort_keys=True)
+    if capabilities.status("lakeflow_auto_cdc") != "unsupported":
+        compatibility = evaluate_lakeflow_compatibility(contract)
+        artifacts[f"{prefix}.lakeflow.md"] = render_lakeflow_review(compatibility)
+    artifacts.update(render_source_artifacts(contract, environment=env))
+    artifacts[f"{prefix}.databricks.yml"] = render_databricks_asset_bundle(
+        DatabricksJobSpec(
+            bundle_name=bundle_name(contract),
+            job_name=job_name(contract),
+            task_key=task_key(contract),
+            notebook_path=f"{env.workspace_path}/{prefix}/run",
+            target=env.bundle_target,
+        )
+    )
+    return RenderedArtifacts(artifacts=artifacts)
+
+
+def _render_openlineage_template(contract: SemanticContract, env: DatabricksEnvironment) -> str:
+    return render_openlineage_insert_sql(
+        contract,
+        run_id="${run_id}",
+        source_name=contract.source.name,
+        status="SUCCESS",
+        started_at_utc=datetime(1970, 1, 1, 0, 0, 0),
+        finished_at_utc=datetime(1970, 1, 1, 0, 0, 0),
+        catalog=env.evidence_catalog,
+        schema=env.evidence_schema,
+    )

contractforge_databricks/rendering/markdown.py

@@ -0,0 +1,50 @@
+"""Markdown review report renderer."""
+
+from __future__ import annotations
+
+from contractforge_core.planner import PlanningResult
+from contractforge_core.security import redact_value
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.capabilities.models import DatabricksCapabilities
+from contractforge_databricks.contract_extensions import databricks_extensions
+from contractforge_databricks.rendering.names import plan_title, target_full_name
+
+
+def render_review_markdown(
+    contract: SemanticContract,
+    planning: PlanningResult,
+    capabilities: DatabricksCapabilities,
+) -> str:
+    lines = [
+        f"# {plan_title(planning.plan) if planning.plan else 'ContractForge Databricks planning review'}",
+        "",
+        f"- Status: `{planning.status}`",
+        f"- Target: `{target_full_name(contract)}`",
+        f"- Write mode: `{contract.write.mode}`",
+        f"- Runtime kind: `{capabilities.runtime_kind}`",
+        "",
+        "## Plan Steps",
+        "",
+    ]
+    if planning.plan:
+        lines.extend(f"- `{step.name}`: {step.intent}" for step in planning.plan.steps)
+    else:
+        lines.append("- No executable abstract plan was produced.")
+
+    if planning.blockers:
+        lines.extend(["", "## Blockers", ""])
+        lines.extend(f"- `{blocker.code}`: {blocker.message}" for blocker in planning.blockers)
+
+    if planning.warnings:
+        lines.extend(["", "## Warnings", ""])
+        lines.extend(f"- `{warning.code}`: {warning.message}" for warning in planning.warnings)
+
+    extensions = databricks_extensions(contract)
+    if extensions:
+        lines.extend(["", "## Databricks Extensions", ""])
+        lines.extend(f"- `{name}`: `{redact_value(extensions[name])}`" for name in sorted(extensions))
+
+    lines.extend(["", "## Databricks Capability Evidence", ""])
+    for name, capability in sorted(capabilities.capabilities.items()):
+        lines.append(f"- `{name}`: `{capability.status}` - {capability.reason}")
+    return "\n".join(lines) + "\n"

contractforge_databricks/rendering/names.py

@@ -0,0 +1,56 @@
+"""Databricks artifact naming helpers."""
+
+from __future__ import annotations
+
+from contractforge_core.planner import ExecutionPlan
+from contractforge_core.semantic import SemanticContract
+from contractforge_core.naming import derive_names, naming_config_from_mapping
+
+
+def target_full_name(contract: SemanticContract) -> str:
+    if contract.target.namespace:
+        return f"{contract.target.namespace}.{contract.target.name}"
+    return contract.target.name
+
+
+def artifact_prefix(contract: SemanticContract) -> str:
+    if contract.naming:
+        return derive_names(
+            target_table=contract.target.name,
+            layer=contract.target.layer,
+            domain=contract.target.domain,
+            config=naming_config_from_mapping(contract.naming.raw),
+        ).contract_basename
+    namespace = contract.target.namespace.replace(".", "_") if contract.target.namespace else contract.target.layer
+    return f"{namespace}_{contract.target.name}"
+
+
+def bundle_name(contract: SemanticContract) -> str:
+    return derive_names(
+        target_table=contract.target.name,
+        layer=contract.target.layer,
+        domain=contract.target.domain,
+        config=naming_config_from_mapping(contract.naming.raw if contract.naming else None),
+    ).bundle_name
+
+
+def job_name(contract: SemanticContract) -> str:
+    return derive_names(
+        target_table=contract.target.name,
+        layer=contract.target.layer,
+        domain=contract.target.domain,
+        config=naming_config_from_mapping(contract.naming.raw if contract.naming else None),
+    ).job_name
+
+
+def task_key(contract: SemanticContract) -> str:
+    return derive_names(
+        target_table=contract.target.name,
+        layer=contract.target.layer,
+        domain=contract.target.domain,
+        config=naming_config_from_mapping(contract.naming.raw if contract.naming else None),
+    ).task_key
+
+
+def plan_title(plan: ExecutionPlan) -> str:
+    return f"ContractForge Databricks plan for {plan.platform}"

contractforge_databricks/results.py

@@ -0,0 +1,15 @@
+"""Compatibility exports for platform-neutral result models."""
+
+from contractforge_core.results import (
+    GovernanceApplyResult,
+    GovernanceApplyStatus,
+    OperationsRecordResult,
+    OperationsRecordStatus,
+)
+
+__all__ = [
+    "GovernanceApplyResult",
+    "GovernanceApplyStatus",
+    "OperationsRecordResult",
+    "OperationsRecordStatus",
+]

contractforge_databricks/runtime/__init__.py

@@ -0,0 +1,101 @@
+from contractforge_databricks.runtime.available_now import BatchIngestor, run_available_now_stream
+from contractforge_databricks.runtime.bundles import (
+    apply_databricks_access_bundle,
+    apply_databricks_annotations_bundle,
+    apply_databricks_governance_bundle,
+    ingest_databricks_bundle,
+)
+from contractforge_databricks.runtime.detection import detect_databricks_capabilities
+from contractforge_databricks.runtime.deploy import (
+    deploy_databricks_bundle,
+    deploy_databricks_project,
+    render_databricks_project_bundle_file,
+)
+from contractforge_databricks.runtime.hooks import DatabricksIngestionHooks
+from contractforge_databricks.runtime.http_file import download_http_file, resolve_http_file_dataframe
+from contractforge_databricks.runtime.models import DatabricksIngestOptions, PreparedViewInput
+from contractforge_databricks.runtime.orchestrator import ingest_databricks_contract
+from contractforge_databricks.runtime.rest_api import read_rest_api_records, resolve_rest_api_dataframe
+from contractforge_databricks.runtime.source_registry import (
+    DatabricksSourceResolver,
+    get_source_resolver,
+    list_source_resolvers,
+    register_source_resolver,
+    unregister_source_resolver,
+)
+from contractforge_databricks.runtime.spark import (
+    fix_encoding,
+    runtime_info,
+    safe_cache,
+    safe_cache_table,
+    safe_unpersist,
+    safe_uncache_table,
+    schema_signature,
+    sync_delta_schema,
+    table_exists,
+)
+from contractforge_databricks.runtime.sources import prepare_contract_source_view, prepare_source_view, resolve_source_dataframe
+from contractforge_databricks.runtime.streaming import (
+    prefer_child_stream_metrics,
+    stream_metrics_from_batches,
+    stream_result_payload,
+    stream_start_payload,
+)
+from contractforge_databricks.runtime.utils import (
+    as_list,
+    new_run_id,
+    safe_truncate,
+    today_str,
+    utc_now_str,
+    utc_now_ts,
+    validate_columns,
+)
+
+__all__ = [
+    "DatabricksIngestOptions",
+    "DatabricksIngestionHooks",
+    "BatchIngestor",
+    "DatabricksSourceResolver",
+    "PreparedViewInput",
+    "as_list",
+    "apply_databricks_access_bundle",
+    "apply_databricks_annotations_bundle",
+    "apply_databricks_governance_bundle",
+    "detect_databricks_capabilities",
+    "deploy_databricks_bundle",
+    "deploy_databricks_project",
+    "render_databricks_project_bundle_file",
+    "download_http_file",
+    "fix_encoding",
+    "get_source_resolver",
+    "ingest_databricks_bundle",
+    "ingest_databricks_contract",
+    "list_source_resolvers",
+    "prepare_source_view",
+    "prepare_contract_source_view",
+    "read_rest_api_records",
+    "register_source_resolver",
+    "new_run_id",
+    "resolve_http_file_dataframe",
+    "resolve_rest_api_dataframe",
+    "resolve_source_dataframe",
+    "run_available_now_stream",
+    "runtime_info",
+    "safe_cache",
+    "safe_cache_table",
+    "safe_truncate",
+    "safe_unpersist",
+    "safe_uncache_table",
+    "schema_signature",
+    "prefer_child_stream_metrics",
+    "sync_delta_schema",
+    "stream_metrics_from_batches",
+    "stream_result_payload",
+    "stream_start_payload",
+    "table_exists",
+    "today_str",
+    "utc_now_str",
+    "utc_now_ts",
+    "unregister_source_resolver",
+    "validate_columns",
+]

contractforge_databricks/runtime/available_now.py

@@ -0,0 +1,147 @@
+"""Available-now stream orchestration helpers for Databricks."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import datetime, timezone
+from typing import Any
+
+from contractforge_core.runtime import PreparedInput
+from contractforge_core.runtime import QueryOne
+from contractforge_core.semantic import SemanticContract
+from contractforge_databricks.evidence import EvidenceWriter, render_stream_child_run_metrics_sql
+from contractforge_databricks.runtime.sources import resolve_source_dataframe
+from contractforge_databricks.sources.interpret import interpret_incremental_files_source, is_incremental_file_source
+from contractforge_databricks.runtime.streaming import (
+    prefer_child_stream_metrics,
+    stream_metrics_from_batches,
+    stream_result_payload,
+    stream_start_payload,
+)
+
+BatchIngestor = Callable[[PreparedInput, int], dict[str, Any]]
+
+
+def run_available_now_stream(
+    spark: Any,
+    contract: SemanticContract,
+    *,
+    stream_run_id: str,
+    batch_ingestor: BatchIngestor,
+    source_view_prefix: str = "cf_stream_batch",
+    evidence: EvidenceWriter | None = None,
+    query_one: QueryOne | None = None,
+    runtime_metadata: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Execute an available-now stream and process micro-batches through an injected ingestor."""
+
+    source = contract.source.raw or {}
+    interpreted_source = interpret_incremental_files_source(source) if is_incremental_file_source(source) else source
+    checkpoint = interpreted_source.get("progress_location") or interpreted_source.get("checkpoint_location")
+    if not checkpoint:
+        raise ValueError("available-now stream requires source.progress_location or source.checkpoint_location")
+
+    started = datetime.now(timezone.utc)
+    batch_results: list[dict[str, Any]] = []
+    stream_df = resolve_source_dataframe(spark, source)
+    status = "SUCCESS"
+    error_message = None
+    if evidence is not None:
+        evidence.write_stream_log(
+            stream_start_payload(
+                contract,
+                stream_run_id=stream_run_id,
+                status="RUNNING",
+                started_at_utc=started,
+                runtime_metadata=runtime_metadata,
+            )
+        )
+
+    def process_batch(batch_df: Any, batch_id: int) -> None:
+        view_name = f"{source_view_prefix}_{stream_run_id}_{batch_id}".replace("-", "_")
+        batch_df.createOrReplaceTempView(view_name)
+        prepared = PreparedInput(
+            source_view=view_name,
+            source_columns=tuple(str(column) for column in getattr(batch_df, "columns", ()) or ()),
+            rows_read=int(batch_df.count()) if hasattr(batch_df, "count") else 0,
+            source_name=str(source.get("path") or source.get("table") or contract.source.name),
+            source_metadata={"stream_run_id": stream_run_id, "batch_id": batch_id},
+        )
+        result = batch_ingestor(prepared, batch_id)
+        batch_results.append(result)
+        if result.get("status") == "FAILED":
+            raise RuntimeError(f"Available-now stream batch {batch_id} failed: {result.get('error_message')}")
+
+    try:
+        query = (
+            stream_df.writeStream.foreachBatch(process_batch)
+            .option("checkpointLocation", str(checkpoint))
+            .trigger(availableNow=True)
+            .start()
+        )
+        query.awaitTermination()
+    except Exception as exc:
+        status = "FAILED"
+        error_message = str(exc)
+    local_metrics = stream_metrics_from_batches(batch_results)
+    child_metrics = _child_stream_metrics(evidence, query_one, stream_run_id)
+    metrics = child_metrics if child_metrics and prefer_child_stream_metrics(local_metrics, child_metrics) else local_metrics
+    result = stream_result_payload(
+        contract,
+        stream_run_id=stream_run_id,
+        status=status,
+        started_at_utc=started,
+        batch_results=batch_results,
+        error_message=error_message,
+        stream_metrics=metrics,
+        runtime_metadata=runtime_metadata,
+    )
+    if evidence is not None:
+        evidence.finish_stream_log(stream_run_id=stream_run_id, payload=result)
+        if status == "FAILED":
+            evidence.write_error_log(_stream_error_payload(contract, stream_run_id, result, error_message))
+    return result
+
+
+def _child_stream_metrics(
+    evidence: EvidenceWriter | None,
+    query_one: QueryOne | None,
+    stream_run_id: str,
+) -> dict[str, int] | None:
+    if evidence is None or query_one is None:
+        return None
+    row = query_one(
+        render_stream_child_run_metrics_sql(
+            stream_run_id=stream_run_id,
+            runs_table=f"{evidence.catalog}.{evidence.schema}.ctrl_ingestion_runs",
+        )
+    )
+    if not row:
+        return None
+    return {
+        "batches_processed": int(row.get("batches_processed") or 0),
+        "total_rows_read": int(row.get("total_rows_read") or 0),
+        "total_rows_written": int(row.get("total_rows_written") or 0),
+        "total_rows_quarantined": int(row.get("total_rows_quarantined") or 0),
+    }
+
+
+def _stream_error_payload(
+    contract: SemanticContract,
+    stream_run_id: str,
+    result: dict[str, Any],
+    error_message: str | None,
+) -> dict[str, Any]:
+    source = contract.source.raw or {}
+    return {
+        "run_id": stream_run_id,
+        "error_ts_utc": result.get("ended_at_utc"),
+        "error_date": str(result.get("ended_at_utc") or "")[:10],
+        "target_table": result.get("target_table"),
+        "source_table": source.get("path") or source.get("table") or contract.source.name,
+        "mode": contract.write.mode,
+        "status": result.get("status"),
+        "error_type": "AvailableNowStreamError",
+        "error_message": error_message,
+        "stack_trace": error_message,
+    }