contractforge-databricks 0.1.0__py3-none-any.whl

contractforge_databricks/state/queries.py

@@ -0,0 +1,149 @@
+"""Databricks SQL lookup queries for control state."""
+
+from __future__ import annotations
+
+from contractforge_databricks.sql import quote_table_name, sql_int, sql_string
+
+
+def render_lock_status_sql(*, target_table: str, locks_table: str = "main.ops.ctrl_ingestion_locks") -> str:
+    return "\n".join(
+        [
+            "SELECT run_id, owner, status, acquired_at_utc, expires_at_utc, ttl_minutes",
+            f"FROM {quote_table_name(locks_table)}",
+            f"WHERE target_table = {sql_string(target_table)}",
+            "LIMIT 1",
+        ]
+    )
+
+
+def render_find_idempotent_run_sql(
+    *,
+    target_table: str,
+    idempotency_key: str,
+    status: str | None = None,
+    runs_table: str = "main.ops.ctrl_ingestion_runs",
+) -> str:
+    filters = [
+        f"target_table = {sql_string(target_table)}",
+        f"idempotency_key = {sql_string(idempotency_key)}",
+    ]
+    if status:
+        filters.append(f"status = {sql_string(status)}")
+    return "\n".join(
+        [
+            "SELECT run_id, status",
+            f"FROM {quote_table_name(runs_table)}",
+            f"WHERE {' AND '.join(filters)}",
+            "ORDER BY run_ts_utc DESC NULLS LAST",
+            "LIMIT 1",
+        ]
+    )
+
+
+def render_find_idempotent_stream_sql(
+    *,
+    target_table: str,
+    idempotency_key: str,
+    status: str | None = None,
+    streams_table: str = "main.ops.ctrl_ingestion_streams",
+) -> str:
+    filters = [
+        f"target_table = {sql_string(target_table)}",
+        f"idempotency_key = {sql_string(idempotency_key)}",
+    ]
+    if status:
+        filters.append(f"status = {sql_string(status)}")
+    return "\n".join(
+        [
+            "SELECT stream_run_id, status",
+            f"FROM {quote_table_name(streams_table)}",
+            f"WHERE {' AND '.join(filters)}",
+            "ORDER BY started_at_utc DESC NULLS LAST",
+            "LIMIT 1",
+        ]
+    )
+
+
+def render_has_successful_run_sql(
+    *,
+    target_table: str,
+    idempotency_key: str,
+    runs_table: str = "main.ops.ctrl_ingestion_runs",
+) -> str:
+    return "\n".join(
+        [
+            "SELECT count(1) > 0 AS has_successful_run",
+            f"FROM {quote_table_name(runs_table)}",
+            "WHERE "
+            f"target_table = {sql_string(target_table)} "
+            f"AND idempotency_key = {sql_string(idempotency_key)} "
+            "AND status = 'SUCCESS'",
+        ]
+    )
+
+
+def render_select_previous_watermark_sql(
+    *,
+    target_table: str,
+    state_table: str = "main.ops.ctrl_ingestion_state",
+) -> str:
+    return "\n".join(
+        [
+            "SELECT watermark_value",
+            f"FROM {quote_table_name(state_table)}",
+            f"WHERE target_table = {sql_string(target_table)}",
+            "LIMIT 1",
+        ]
+    )
+
+
+def render_control_metadata_current_sql(
+    *,
+    framework_version: str,
+    ctrl_schema_version: int,
+    metadata_table: str = "main.ops.ctrl_ingestion_metadata",
+) -> str:
+    return "\n".join(
+        [
+            "SELECT 1",
+            f"FROM {quote_table_name(metadata_table)}",
+            "WHERE component = 'contractforge'",
+            f"  AND framework_version = {sql_string(framework_version)}",
+            f"  AND ctrl_schema_version = {sql_int(ctrl_schema_version)}",
+            "LIMIT 1",
+        ]
+    )
+
+
+def render_record_control_metadata_sql(
+    *,
+    framework_version: str,
+    ctrl_schema_version: int,
+    metadata_table: str = "main.ops.ctrl_ingestion_metadata",
+) -> str:
+    return f"""
+MERGE INTO {quote_table_name(metadata_table)} t
+USING (
+  SELECT
+    'contractforge' AS component,
+    {sql_string(framework_version)} AS framework_version,
+    {sql_int(ctrl_schema_version)} AS ctrl_schema_version,
+    current_timestamp() AS updated_at_utc
+) s
+ON t.component = s.component
+WHEN MATCHED THEN UPDATE SET
+  t.framework_version = s.framework_version,
+  t.ctrl_schema_version = s.ctrl_schema_version,
+  t.updated_at_utc = s.updated_at_utc
+WHEN NOT MATCHED THEN INSERT (
+  component,
+  framework_version,
+  ctrl_schema_version,
+  updated_at_utc
+) VALUES (
+  s.component,
+  s.framework_version,
+  s.ctrl_schema_version,
+  s.updated_at_utc
+)
+""".strip()

contractforge_databricks/state/sql.py

@@ -0,0 +1,116 @@
+"""Databricks SQL for locks, state and idempotency lookups."""
+
+from __future__ import annotations
+
+from contractforge_databricks.sql import quote_table_name, sql_int, sql_string
+from contractforge_databricks.state.tables import state_table_names
+
+
+def render_acquire_lock_sql(
+    *,
+    target_table: str,
+    run_id: str,
+    owner: str | None = None,
+    ttl_minutes: int = 60,
+    catalog: str = "main",
+    schema: str = "ops",
+) -> str:
+    table = state_table_names(catalog, schema)["locks"]
+    return f"""
+MERGE INTO {quote_table_name(table)} t
+USING (
+  SELECT
+    {sql_string(target_table)} AS target_table,
+    {sql_string(run_id)} AS run_id,
+    {sql_string(owner)} AS owner,
+    current_timestamp() AS acquired_at_utc,
+    current_timestamp() + INTERVAL {int(ttl_minutes)} MINUTES AS expires_at_utc,
+    {sql_int(ttl_minutes)} AS ttl_minutes,
+    CAST(NULL AS TIMESTAMP) AS released_at_utc,
+    'ACTIVE' AS status
+) s
+ON t.target_table = s.target_table
+WHEN MATCHED AND (t.status <> 'ACTIVE' OR t.expires_at_utc < current_timestamp()) THEN UPDATE SET *
+WHEN NOT MATCHED THEN INSERT *
+""".strip()
+
+
+def render_release_lock_sql(
+    *, target_table: str, run_id: str, catalog: str = "main", schema: str = "ops"
+) -> str:
+    table = state_table_names(catalog, schema)["locks"]
+    return f"""
+UPDATE {quote_table_name(table)}
+SET status = 'RELEASED',
+    released_at_utc = current_timestamp()
+WHERE target_table = {sql_string(target_table)} AND run_id = {sql_string(run_id)}
+""".strip()
+
+
+def render_upsert_state_sql(
+    *,
+    target_table: str,
+    run_id: str,
+    status: str,
+    rows_written: int,
+    watermark_column: str | None = None,
+    watermark_value: str | None = None,
+    success_at_utc: str | None = None,
+    error_message: str | None = None,
+    table_version: str | None = None,
+    write_completed_at_utc: str | None = None,
+    watermark_candidate: str | None = None,
+    parent_run_id: str | None = None,
+    run_group_id: str | None = None,
+    master_job_id: str | None = None,
+    master_run_id: str | None = None,
+    catalog: str = "main",
+    schema: str = "ops",
+) -> str:
+    table = state_table_names(catalog, schema)["state"]
+    return f"""
+MERGE INTO {quote_table_name(table)} t
+USING (
+  SELECT
+    {sql_string(target_table)} AS target_table,
+    {sql_string(watermark_column)} AS watermark_column,
+    {sql_string(watermark_value)} AS watermark_value,
+    CAST({sql_string(success_at_utc)} AS TIMESTAMP) AS last_success_at_utc,
+    {sql_string(run_id)} AS last_run_id,
+    {sql_string(status)} AS last_status,
+    {sql_int(rows_written)} AS last_rows_written,
+    {sql_string(_truncate(error_message))} AS last_error_message,
+    {sql_string(parent_run_id)} AS parent_run_id,
+    {sql_string(run_group_id)} AS run_group_id,
+    {sql_string(master_job_id)} AS master_job_id,
+    {sql_string(master_run_id)} AS master_run_id,
+    {sql_string(table_version)} AS last_table_version,
+    CAST({sql_string(write_completed_at_utc)} AS TIMESTAMP) AS last_write_completed_at_utc,
+    {sql_string(watermark_candidate)} AS last_watermark_candidate,
+    current_timestamp() AS last_updated_at_utc
+) s
+ON t.target_table = s.target_table
+WHEN MATCHED THEN UPDATE SET
+  t.watermark_column = s.watermark_column,
+  t.watermark_value = s.watermark_value,
+  t.last_success_at_utc = s.last_success_at_utc,
+  t.last_run_id = s.last_run_id,
+  t.last_status = s.last_status,
+  t.last_rows_written = s.last_rows_written,
+  t.last_error_message = s.last_error_message,
+  t.parent_run_id = s.parent_run_id,
+  t.run_group_id = s.run_group_id,
+  t.master_job_id = s.master_job_id,
+  t.master_run_id = s.master_run_id,
+  t.last_table_version = s.last_table_version,
+  t.last_write_completed_at_utc = s.last_write_completed_at_utc,
+  t.last_watermark_candidate = s.last_watermark_candidate,
+  t.last_updated_at_utc = s.last_updated_at_utc
+WHEN NOT MATCHED THEN INSERT *
+""".strip()
+
+
+def _truncate(value: str | None, limit: int = 4000) -> str | None:
+    if value is None or len(value) <= limit:
+        return value
+    return value[:limit]

contractforge_databricks/state/tables.py

@@ -0,0 +1,9 @@
+"""Databricks operational state table names."""
+
+from __future__ import annotations
+
+from contractforge_core.evidence import STATE_TABLES
+
+
+def state_table_names(catalog: str, schema: str) -> dict[str, str]:
+    return {name: f"{catalog}.{schema}.{table}" for name, table in STATE_TABLES.items()}

contractforge_databricks/state/writer.py

@@ -0,0 +1,83 @@
+"""State writer using an injected SQL runner."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from contractforge_databricks.execution.sql_merge import SqlRunner
+from contractforge_databricks.state.queries import render_lock_status_sql, render_record_control_metadata_sql
+from contractforge_databricks.state.sql import (
+    render_acquire_lock_sql,
+    render_release_lock_sql,
+    render_upsert_state_sql,
+)
+
+
+logger = logging.getLogger("contractforge_databricks")
+
+
+class StateWriter:
+    def __init__(
+        self,
+        runner: SqlRunner,
+        *,
+        catalog: str = "main",
+        schema: str = "ops",
+        query_one: Callable[[str], dict[str, Any] | None] | None = None,
+    ) -> None:
+        self.runner = runner
+        self.catalog = catalog
+        self.schema = schema
+        self.query_one = query_one
+
+    def acquire_lock(self, *, target_table: str, run_id: str, owner: str | None = None, ttl_minutes: int = 60) -> None:
+        self.runner.sql(
+            render_acquire_lock_sql(
+                target_table=target_table,
+                run_id=run_id,
+                owner=owner,
+                ttl_minutes=ttl_minutes,
+                catalog=self.catalog,
+                schema=self.schema,
+            )
+        )
+        if self.query_one is None:
+            return
+        row = self.query_one(
+            render_lock_status_sql(
+                target_table=target_table,
+                locks_table=f"{self.catalog}.{self.schema}.ctrl_ingestion_locks",
+            )
+        )
+        if not row or row.get("run_id") != run_id or row.get("status") != "ACTIVE":
+            raise RuntimeError(
+                f"Lock is busy for {target_table}. This run_id={run_id} did not acquire the lock. "
+                f"Current lock: {row}"
+            )
+
+    def release_lock(self, *, target_table: str, run_id: str) -> None:
+        try:
+            self.runner.sql(
+                render_release_lock_sql(
+                    target_table=target_table,
+                    run_id=run_id,
+                    catalog=self.catalog,
+                    schema=self.schema,
+                )
+            )
+        except Exception as exc:
+            logger.warning("Failed to release lock for %s: %s", target_table, exc)
+
+    def upsert_state(self, **kwargs: object) -> None:
+        self.runner.sql(render_upsert_state_sql(catalog=self.catalog, schema=self.schema, **kwargs))
+
+    def record_control_metadata(self, *, framework_version: str, ctrl_schema_version: int) -> None:
+        self.runner.sql(
+            render_record_control_metadata_sql(
+                framework_version=framework_version,
+                ctrl_schema_version=ctrl_schema_version,
+                metadata_table=f"{self.catalog}.{self.schema}.ctrl_ingestion_metadata",
+            )
+        )

contractforge_databricks/templates/__init__.py

@@ -0,0 +1,15 @@
+from contractforge_databricks.templates.core import (
+    contract_template_details,
+    contract_template_files,
+    get_contract_template,
+    list_contract_templates,
+    recommend_contract_templates,
+)
+
+__all__ = [
+    "contract_template_details",
+    "contract_template_files",
+    "get_contract_template",
+    "list_contract_templates",
+    "recommend_contract_templates",
+]

contractforge_databricks/templates/catalog.py

@@ -0,0 +1,205 @@
+"""Databricks template catalog for split ContractForge contracts."""
+
+from __future__ import annotations
+
+from typing import Any
+
+ContractTemplate = dict[str, Any]
+TEMPLATE_META_KEY = "_template"
+
+
+def _template(
+    name: str,
+    category: str,
+    description: str,
+    ingestion: dict[str, Any],
+    *,
+    annotations: dict[str, Any] | None = None,
+    operations: dict[str, Any] | None = None,
+    access: dict[str, Any] | None = None,
+    priority: int = 50,
+) -> ContractTemplate:
+    result: ContractTemplate = {
+        TEMPLATE_META_KEY: {
+            "name": name,
+            "category": category,
+            "description": description,
+            "recommendation_priority": priority,
+        },
+        "ingestion": ingestion,
+    }
+    if annotations:
+        result["annotations"] = annotations
+    if operations:
+        result["operations"] = operations
+    if access:
+        result["access"] = access
+    return result
+
+
+def _target(schema: str, table: str) -> dict[str, str]:
+    return {"catalog": "main", "schema": schema, "table": table}
+
+
+def _ops(domain: str) -> dict[str, Any]:
+    return {
+        "owner": "data-platform",
+        "domain": domain,
+        "criticality": "medium",
+        "expected_frequency": "daily",
+        "runbook_url": f"https://wiki.example.com/runbooks/{domain}",
+    }
+
+
+def _ann(description: str) -> dict[str, Any]:
+    return {"policy": "warn", "table": {"comment": description, "tags": {"contractforge": "databricks"}}}
+
+
+def _access(group: str) -> dict[str, Any]:
+    return {"access_policy": {"mode": "validate_only", "on_drift": "warn"}, "grants": [{"principal": group, "privileges": ["SELECT"]}]}
+
+
+from contractforge_databricks.templates.catalog_parity import PARITY_CONTRACT_TEMPLATES  # noqa: E402
+from contractforge_databricks.templates.enrichment import enrich_contractforge_parity  # noqa: E402
+
+
+BUILTIN_CONTRACT_TEMPLATES: dict[str, ContractTemplate] = {
+    **PARITY_CONTRACT_TEMPLATES,
+    "bronze_rest_api_incremental": _template(
+        "bronze_rest_api_incremental",
+        "bronze",
+        "REST API landing through reviewed native passthrough or bounded file fetch.",
+        {
+            "preset": "bronze_file_append",
+            "source": {"type": "native_passthrough", "system": "rest_api", "object": "orders"},
+            "target": _target("raw", "b_orders_api"),
+        },
+        annotations=_ann("Raw REST API order events."),
+        operations=_ops("b_orders_api"),
+    ),
+    "bronze_http_file_csv_snapshot": _template(
+        "bronze_http_file_csv_snapshot",
+        "bronze",
+        "HTTP CSV snapshot landing.",
+        {"preset": "bronze_full_overwrite", "source": {"type": "http_csv", "url": "https://example.com/orders.csv"}, "target": _target("raw", "b_orders_http")},
+        annotations=_ann("Raw HTTP CSV orders."),
+        operations=_ops("b_orders_http"),
+    ),
+    "bronze_autoloader_json": _template(
+        "bronze_autoloader_json",
+        "bronze",
+        "Portable incremental files rendered as Databricks Auto Loader.",
+        {"preset": "bronze_autoloader_append", "source": {"type": "incremental_files", "format": "json", "path": "s3://bucket/landing/orders/"}, "target": _target("raw", "b_orders_json")},
+        annotations=_ann("Raw incremental JSON orders."),
+        operations=_ops("b_orders_json"),
+    ),
+    "bronze_autoloader_available_now_json": _template(
+        "bronze_autoloader_available_now_json",
+        "bronze",
+        "Available-now incremental JSON ingestion.",
+        {"preset": "bronze_autoloader_append", "source": {"type": "incremental_files", "format": "json", "trigger": "available_now", "path": "s3://bucket/landing/orders/"}, "target": _target("raw", "b_orders_available_now")},
+        annotations=_ann("Available-now incremental JSON orders."),
+        operations=_ops("b_orders_available_now"),
+    ),
+    "bronze_autoloader_governed_delta": _template(
+        "bronze_autoloader_governed_delta",
+        "bronze",
+        "Governed Auto Loader landing with Delta optimization preview.",
+        {"preset": ["bronze_autoloader_append", "delta_optimized_writes", "governance_uc_basic"], "source": {"type": "incremental_files", "format": "json", "path": "s3://bucket/landing/governed/"}, "target": _target("raw", "b_governed_delta")},
+        annotations=_ann("Governed raw landing table."),
+        operations=_ops("b_governed_delta"),
+        access=_access("data-engineers"),
+    ),
+    "bronze_object_storage_nested_json_shape": _template(
+        "bronze_object_storage_nested_json_shape",
+        "bronze",
+        "Object-storage nested JSON with shape intent.",
+        {"preset": "bronze_file_append", "source": {"type": "json", "path": "s3://bucket/events/"}, "shape": {"parse_json": [{"column": "payload", "schema": "STRUCT<id: STRING>", "alias": "payload_obj"}]}, "target": _target("raw", "b_nested_events")},
+        annotations=_ann("Nested JSON event landing."),
+        operations=_ops("b_nested_events"),
+    ),
+    "bronze_object_storage_small_files": _template(
+        "bronze_object_storage_small_files",
+        "bronze",
+        "Object-storage small files batch append.",
+        {"preset": "bronze_file_append", "source": {"type": "parquet", "path": "s3://bucket/small-files/"}, "target": _target("raw", "b_small_files")},
+        annotations=_ann("Small-file batch landing."),
+        operations=_ops("b_small_files"),
+    ),
+    "silver_jdbc_scd1_upsert": _template(
+        "silver_jdbc_scd1_upsert",
+        "silver",
+        "JDBC SCD1 current-state upsert.",
+        {"preset": ["silver_incremental_watermark_upsert", "quality_quarantine", "delta_optimized_writes"], "source": {"type": "jdbc", "table": "public.orders"}, "target": _target("curated", "s_orders"), "merge_keys": ["order_id"], "watermark_columns": ["updated_at"]},
+        annotations=_ann("Current-state orders from JDBC."),
+        operations=_ops("s_orders"),
+        access=_access("sales-analytics"),
+    ),
+    "silver_jdbc_rds_iam_hash_diff": _template(
+        "silver_jdbc_rds_iam_hash_diff",
+        "silver",
+        "JDBC RDS IAM hash-diff append.",
+        {"preset": ["silver_hash_diff_append", "quality_quarantine"], "source": {"type": "postgres", "table": "public.orders", "auth": {"type": "rds_iam"}}, "target": _target("curated", "s_orders_hash_diff"), "hash_keys": ["order_id"]},
+        annotations=_ann("Hash-diff order changes from JDBC."),
+        operations=_ops("s_orders_hash_diff"),
+    ),
+    "silver_lakeflow_auto_cdc_scd1_preview": _template(
+        "silver_lakeflow_auto_cdc_scd1_preview",
+        "silver",
+        "Lakeflow AUTO CDC SCD1 review artifact with Delta fallback semantics.",
+        {"preset": ["silver_scd1_upsert", "delta_liquid_clustering"], "source": {"type": "table", "table": "main.raw.customer_cdc"}, "target": _target("curated", "s_customers_current"), "merge_keys": ["customer_id"], "extensions": {"databricks": {"cluster_columns": ["customer_id"], "write_engine": {"requested": "lakeflow_auto_cdc", "fallback_policy": "preview_only"}}}},
+        annotations=_ann("Current customers with Lakeflow review evidence."),
+        operations=_ops("s_customers_current"),
+    ),
+    "silver_lakeflow_auto_cdc_scd2_preview": _template(
+        "silver_lakeflow_auto_cdc_scd2_preview",
+        "silver",
+        "Lakeflow AUTO CDC SCD2 review artifact with Delta baseline.",
+        {"preset": ["silver_scd2_historical", "delta_liquid_clustering"], "source": {"type": "table", "table": "main.raw.product_cdc"}, "target": _target("curated", "s_products_history"), "merge_keys": ["product_id"], "extensions": {"databricks": {"cluster_columns": ["product_id"], "write_engine": {"requested": "lakeflow_auto_cdc", "fallback_policy": "preview_only"}}}},
+        annotations=_ann("Product SCD2 history with Lakeflow review evidence."),
+        operations=_ops("s_products_history"),
+    ),
+    "silver_raw_json_payload_shape": _template(
+        "silver_raw_json_payload_shape",
+        "silver",
+        "Raw JSON payload parsing into a curated table.",
+        {"preset": "silver_scd1_upsert", "source": {"type": "table", "table": "main.raw.b_events"}, "shape": {"parse_json": [{"column": "payload", "schema": "STRUCT<event_id: STRING>", "alias": "payload_obj"}], "columns": {"payload_obj.event_id": {"alias": "event_id", "cast": "STRING"}}}, "target": _target("curated", "s_events"), "merge_keys": ["event_id"]},
+        annotations=_ann("Curated event payloads."),
+        operations=_ops("s_events"),
+    ),
+    "silver_parallel_arrays_shape": _template(
+        "silver_parallel_arrays_shape",
+        "silver",
+        "Parallel array normalization review template.",
+        {"preset": "silver_scd1_upsert", "source": {"type": "table", "table": "main.raw.b_forecast"}, "shape": {"zip_arrays": [{"alias": "hour", "columns": {"times": "time", "values": "value"}}], "arrays": [{"path": "hour", "mode": "explode_outer", "alias": "hour"}]}, "target": _target("curated", "s_hourly_forecast"), "merge_keys": ["forecast_id"]},
+        annotations=_ann("Forecast rows derived from parallel arrays."),
+        operations=_ops("s_hourly_forecast"),
+    ),
+    "silver_snapshot_soft_delete": _template(
+        "silver_snapshot_soft_delete",
+        "silver",
+        "Current-state snapshot with soft delete for missing rows.",
+        {"preset": "silver_snapshot_soft_delete", "source": {"type": "table", "table": "main.raw.b_devices_snapshot"}, "target": _target("curated", "s_devices"), "merge_keys": ["device_id"]},
+        annotations=_ann("Device snapshot with soft delete semantics."),
+        operations=_ops("s_devices"),
+    ),
+    "silver_scd2_history": _template(
+        "silver_scd2_history",
+        "silver",
+        "SCD2 historical table.",
+        {"preset": "silver_scd2_historical", "source": {"type": "table", "table": "main.raw.b_customers"}, "target": _target("curated", "s_customers_history"), "merge_keys": ["customer_id"]},
+        annotations=_ann("Customer SCD2 history."),
+        operations=_ops("s_customers_history"),
+    ),
+    "gold_full_refresh_kpi": _template(
+        "gold_full_refresh_kpi",
+        "gold",
+        "Gold KPI table recalculated by full refresh.",
+        {"preset": "gold_full_refresh", "source": {"type": "sql", "query": "SELECT order_date, count(*) AS orders FROM main.curated.s_orders GROUP BY order_date"}, "target": _target("analytics", "g_daily_orders")},
+        annotations=_ann("Daily order KPI table."),
+        operations=_ops("g_daily_orders"),
+        access=_access("executive-dashboards"),
+    ),
+}
+
+enrich_contractforge_parity(BUILTIN_CONTRACT_TEMPLATES)

contractforge_databricks/templates/catalog_parity.py

@@ -0,0 +1,85 @@
+"""Additional templates ported from the original ContractForge catalog."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_databricks.templates.catalog import _access, _ann, _ops, _target, _template
+
+ContractTemplate = dict[str, Any]
+
+PARITY_CONTRACT_TEMPLATES: dict[str, ContractTemplate] = {
+    "bronze_blob_partitioned_files": _template(
+        "bronze_blob_partitioned_files",
+        "bronze",
+        "Bronze batch ingestion for partitioned files in object storage.",
+        {
+            "preset": "bronze_file_append",
+            "source": {
+                "type": "s3",
+                "format": "parquet",
+                "path": "s3://company-landing/orders/",
+                "options": {"recursiveFileLookup": True, "pathGlobFilter": "*.parquet"},
+                "read": {
+                    "source_complete": True,
+                    "schema": "order_id STRING, order_date DATE, customer_id STRING, amount DOUBLE",
+                    "file_regex": r"^year=2026/month=05/.*/orders_\d+\.parquet$",
+                    "file_regex_scope": "relative_path",
+                    "file_regex_max_listed": 50000,
+                },
+            },
+            "target": _target("raw", "b_orders_files"),
+            "layer": "bronze",
+            "mode": "scd0_append",
+            "schema_policy": "additive_only",
+            "quality_rules": {
+                "not_null": ["order_id"],
+                "expressions": [
+                    {
+                        "name": "valid_amount",
+                        "expression": "amount IS NULL OR amount >= 0",
+                        "severity": "warn",
+                        "message": "Negative amount in raw file.",
+                    }
+                ],
+            },
+        },
+        annotations=_ann("Partitioned order files in object storage."),
+        operations=_ops("b_orders_files"),
+    ),
+    "silver_scd1_hash_diff": _template(
+        "silver_scd1_hash_diff",
+        "silver",
+        "Silver append-only hash diff retaining changed versions.",
+        {
+            "preset": "silver_hash_diff_append",
+            "source": {"type": "table", "table": "main.raw.b_products"},
+            "target": _target("curated", "s_products_hash_diff"),
+            "layer": "silver",
+            "mode": "scd1_hash_diff",
+            "hash_keys": ["product_id"],
+            "hash_exclude_columns": ["updated_at"],
+            "transform": {
+                "deduplicate": {
+                    "keys": ["product_id"],
+                    "order_by": "updated_at DESC NULLS LAST",
+                }
+            },
+            "quality_rules": {
+                "not_null": ["product_id"],
+                "expressions": [
+                    {
+                        "name": "valid_product_status",
+                        "expression": "status IS NULL OR status IN ('active', 'inactive', 'discontinued')",
+                        "severity": "quarantine",
+                        "message": "Invalid product status.",
+                    }
+                ],
+            },
+        },
+        annotations=_ann("Changed product versions detected by hash diff."),
+        operations=_ops("s_products_hash_diff"),
+        access=_access("catalog-analytics"),
+        priority=10,
+    ),
+}