aetherdialect 0.1.0__py3-none-any.whl

text2sql/schema_profiling.py

@@ -0,0 +1,2075 @@
+"""Column and table profiling, role inference, schema parsing, and partition filter injection.
+
+Profiles columns for statistical properties (distinct count, null ratio, min/max,
+top-K values), supporting both a local SQLAlchemy engine and a Databricks Spark
+session. Infers column roles (CATEGORICAL, NUMERIC_MEASURE, TEMPORAL, IDENTIFIER,
+BOOLEAN, FREE_TEXT, NUMERIC_CATEGORICAL, AUDIT) and table roles (FACT, DIMENSION,
+BRIDGE) via a single schema-wide LLM call with a heuristic fallback. Also provides
+SQL DDL parsing (sqlglot + LLM fallback), Unity Catalog constraint extraction,
+and partition filter injection for Databricks Spark SQL.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from .config import (
+    BOOLEAN_TRUE_FALSE_MAP,
+    BOOLEAN_VALUE_PATTERNS,
+    EngineConfig,
+    PolicyConfig,
+    QSimConfig,
+)
+from .contracts_base import (
+    ColumnMetadata,
+    ColumnRole,
+    SchemaGraph,
+    TableMetadata,
+    TableRole,
+)
+from .contracts_core import FilterParam, NormalizedExpr, RuntimeIntent
+from .core_utils import debug, llm_chat, safe_json_loads, stable_json
+from .utils import flatten_param_values
+
+
+def _has_boolean_like_values(col: ColumnMetadata) -> bool:
+    """Check if a column's top-K values match a known boolean-like pattern.
+
+    Handles both string and numeric values (e.g. integer 0/1).
+
+    Args:
+
+        col: The ``ColumnMetadata`` to inspect.
+
+    Returns:
+
+        ``True`` if ``distinct_count`` is exactly 2 and the two top-K values form a recognised boolean pair.
+    """
+    if col.distinct_count != 2:
+        return False
+    if not col.top_k_values or len(col.top_k_values) != 2:
+        return False
+    values_lower = frozenset(str(v).lower().strip() for v in col.top_k_values)
+    return values_lower in BOOLEAN_VALUE_PATTERNS
+
+
+def _populate_boolean_mapping(col: ColumnMetadata) -> None:
+    """Populate ``boolean_true_value`` and ``boolean_false_value`` on a boolean-like column.
+
+    For native boolean data types the canonical ``"true"``/``"false"`` pair is assigned directly. For columns whose profiled ``top_k_values`` match a known pattern in ``_BOOLEAN_TRUE_FALSE_MAP`` the actual database-cased values are stored so downstream normalisation preserves the original representation.
+
+    Args:
+
+        col: The ``ColumnMetadata`` to update in-place.
+    """
+    dtype_lower = (col.data_type or "").lower()
+    if "bool" in dtype_lower:
+        col.boolean_true_value = "true"
+        col.boolean_false_value = "false"
+        return
+    if not col.top_k_values or len(col.top_k_values) != 2:
+        return
+    values_lower = frozenset(str(v).lower().strip() for v in col.top_k_values)
+    true_false = BOOLEAN_TRUE_FALSE_MAP.get(values_lower)
+    if not true_false:
+        return
+    true_lower, false_lower = true_false
+    for v in col.top_k_values:
+        v_lower = str(v).lower().strip()
+        if v_lower == true_lower:
+            col.boolean_true_value = str(v)
+        elif v_lower == false_lower:
+            col.boolean_false_value = str(v)
+
+
+def _profile_column(
+    engine: Any,
+    col: ColumnMetadata,
+    table_name: str,
+    row_count: int,
+    sample_threshold: int = None,
+    sample_size: int = None,
+) -> None:
+    """Profile a single column and update its metadata in-place.
+
+    Computes distinct count, distinct ratio, null ratio, min/max (for numeric and date columns), and top-K values. Uses sampling for large tables.
+
+    Args:
+
+        engine: SQLAlchemy engine connected to the target database.
+
+        col: The ``ColumnMetadata`` to update.
+
+        table_name: The name of the table containing the column.
+
+        row_count: Total row count for the table (used for sample size calculation).
+
+        sample_threshold: Row count above which sampling is used; defaults to ``QSimConfig.PROFILING_SAMPLE_THRESHOLD``.
+
+        sample_size: Number of rows to sample; defaults to ``QSimConfig.PROFILING_SAMPLE_SIZE``.
+    """
+    debug(f"[schema_profiling.profile_column] profiling {table_name}.{col.name}")
+    if sample_threshold is None:
+        sample_threshold = QSimConfig.PROFILING_SAMPLE_THRESHOLD
+    if sample_size is None:
+        sample_size = QSimConfig.PROFILING_SAMPLE_SIZE
+
+    col.row_count = row_count
+    use_sample = row_count > sample_threshold
+
+    from sqlalchemy import text
+
+    try:
+        with engine.connect() as conn:
+            if use_sample:
+                if EngineConfig.TYPE == "postgresql":
+                    sample_clause = f"TABLESAMPLE BERNOULLI ({100 * sample_size / row_count:.2f}) REPEATABLE ({QSimConfig.RANDOM_SEED})"
+                else:
+                    sample_clause = f"LIMIT {sample_size}"
+            else:
+                sample_clause = ""
+
+            if use_sample and EngineConfig.TYPE == "postgresql":
+                stats_sql = f"""
+                    SELECT 
+                        COUNT(*) as cnt,
+                        COUNT(DISTINCT "{col.name}") as dist,
+                        COUNT(*) - COUNT("{col.name}") as nulls
+                    FROM "{table_name}" {sample_clause}
+                """
+            elif use_sample:
+                stats_sql = f"""
+                    SELECT 
+                        COUNT(*) as cnt,
+                        COUNT(DISTINCT "{col.name}") as dist,
+                        COUNT(*) - COUNT("{col.name}") as nulls
+                    FROM (SELECT "{col.name}" FROM "{table_name}" {sample_clause}) t
+                """
+            else:
+                stats_sql = f"""
+                    SELECT 
+                        COUNT(*) as cnt,
+                        COUNT(DISTINCT "{col.name}") as dist,
+                        COUNT(*) - COUNT("{col.name}") as nulls
+                    FROM "{table_name}"
+                """
+
+            result = conn.execute(text(stats_sql)).fetchone()
+            cnt = result[0] or 1
+            dist = result[1] or 0
+            nulls = result[2] or 0
+
+            col.distinct_count = dist
+            col.distinct_ratio = dist / cnt if cnt > 0 else 0.0
+            col.null_ratio = nulls / cnt if cnt > 0 else 0.0
+
+            if col.value_type in ("integer", "number") or col.value_type == "date":
+                minmax_sql = f'SELECT MIN("{col.name}"), MAX("{col.name}") FROM "{table_name}"'
+                minmax_result = conn.execute(text(minmax_sql)).fetchone()
+                if minmax_result:
+                    col.min_val = str(minmax_result[0]) if minmax_result[0] is not None else None
+                    col.max_val = str(minmax_result[1]) if minmax_result[1] is not None else None
+
+            topk_sql = f"""
+                SELECT "{col.name}", COUNT(*) as freq 
+                FROM "{table_name}" 
+                WHERE "{col.name}" IS NOT NULL
+                GROUP BY "{col.name}" 
+                ORDER BY freq DESC 
+                LIMIT {PolicyConfig.CATEGORICAL_SAMPLE_SIZE}
+            """
+            topk_result = conn.execute(text(topk_sql)).fetchall()
+            col.top_k_values = [str(row[0]) for row in topk_result if row[0] is not None]
+    except Exception as e:
+        debug(f"[schema_profiling.profile_column] failed: {table_name}.{col.name}: {e}")
+
+
+_NAME_COLUMN_PATTERN = re.compile(r"(first.?name|last.?name|given.?name|family.?name)", re.IGNORECASE)
+
+
+def _profile_composite_descriptive(
+    engine: Any,
+    table: TableMetadata,
+) -> None:
+    """Compute composite distinct ratios for name-like column pairs.
+
+    Detects pairs of string columns whose names match common
+    name patterns (first_name / last_name) and measures the
+    distinct ratio of their concatenation.  Results are stored in
+    ``table.composite_descriptive_ratios``.
+    """
+    name_cols = [
+        col_name
+        for col_name, col_meta in table.columns.items()
+        if (col_meta.value_type or "").lower() == "string"
+        and _NAME_COLUMN_PATTERN.search(col_name)
+        and not col_meta.is_primary_key
+        and not col_meta.is_foreign_key
+    ]
+    if len(name_cols) < 2:
+        return
+    row_count = table.row_count or 0
+    if row_count == 0:
+        return
+
+    from sqlalchemy import text
+
+    try:
+        with engine.connect() as conn:
+            for i in range(len(name_cols)):
+                for j in range(i + 1, len(name_cols)):
+                    c1, c2 = name_cols[i], name_cols[j]
+                    sql = (
+                        f'SELECT COUNT(DISTINCT CONCAT("{c1}", \' \', "{c2}")) '
+                        f'FROM "{table.name}"'
+                    )
+                    composite_distinct = conn.execute(text(sql)).scalar() or 0
+                    ratio = composite_distinct / row_count
+                    table.composite_descriptive_ratios[(c1, c2)] = ratio
+                    debug(
+                        f"[schema_profiling._profile_composite_descriptive] "
+                        f"{table.name}.({c1}, {c2}) composite_ratio={ratio:.4f}"
+                    )
+    except Exception as exc:
+        debug(
+            f"[schema_profiling._profile_composite_descriptive] "
+            f"failed for {table.name}: {exc}"
+        )
+
+
+def _profile_table(engine: Any, table: TableMetadata) -> None:
+    """Profile all columns in a table and update metadata in-place.
+
+    Args:
+
+        engine: SQLAlchemy engine connected to the target database.
+
+        table: The ``TableMetadata`` to update (row_count and all column stats).
+    """
+    from sqlalchemy import text
+
+    debug(f"[schema_profiling.profile_table] profiling {table.name} ({len(table.columns)} columns)")
+    try:
+        with engine.connect() as conn:
+            count_sql = f'SELECT COUNT(*) FROM "{table.name}"'
+            row_count = conn.execute(text(count_sql)).scalar() or 0
+            table.row_count = row_count
+    except Exception as e:
+        debug(f"[schema_profiling.profile_table] row count failed: {table.name}: {e}")
+        row_count = 0
+        table.row_count = 0
+
+    for col in table.columns.values():
+        _profile_column(engine, col, table.name, row_count)
+
+    _profile_composite_descriptive(engine, table)
+
+    debug(f"[schema_profiling.profile_table] completed: {table.name}")
+
+
+def profile_schema(engine: Any, schema: SchemaGraph) -> None:
+    """Profile all tables in a schema and update metadata in-place.
+
+    Args:
+
+        engine: SQLAlchemy engine connected to the target database.
+
+        schema: The ``SchemaGraph`` whose tables will be profiled.
+    """
+    debug(f"[schema_profiling.profile_schema] profiling {len(schema.tables)} tables")
+    for table in schema.tables.values():
+        _profile_table(engine, table)
+    debug("[schema_profiling.profile_schema] completed")
+
+
+def _profile_column_spark(
+    spark,
+    catalog: str,
+    schema_name: str,
+    col: ColumnMetadata,
+    table_name: str,
+    row_count: int,
+    sample_threshold: int = None,
+    sample_size: int = None,
+) -> None:
+    """Profile a single column from a Databricks table via Spark SQL and update metadata in-place.
+
+    Args:
+
+        spark: Active ``SparkSession``.
+
+        catalog: The Unity Catalog name.
+
+        schema_name: The schema (database) name within the catalog.
+
+        col: The ``ColumnMetadata`` to update.
+
+        table_name: The table name within the schema.
+
+        row_count: Total row count used for sampling decisions.
+
+        sample_threshold: Row count above which sampling is used; defaults to ``QSimConfig.PROFILING_SAMPLE_THRESHOLD``.
+
+        sample_size: Number of rows to sample; defaults to ``QSimConfig.PROFILING_SAMPLE_SIZE``.
+    """
+    debug(f"[schema_profiling.profile_column_spark] profiling {table_name}.{col.name}")
+    if sample_threshold is None:
+        sample_threshold = QSimConfig.PROFILING_SAMPLE_THRESHOLD
+    if sample_size is None:
+        sample_size = QSimConfig.PROFILING_SAMPLE_SIZE
+
+    col.row_count = row_count
+    use_sample = row_count > sample_threshold
+
+    try:
+        full_table = f"`{catalog}`.`{schema_name}`.`{table_name}`"
+
+        if use_sample:
+            sample_clause = f"TABLESAMPLE ({sample_size} ROWS)"
+        else:
+            sample_clause = ""
+
+        if use_sample:
+            stats_sql = f"""
+                SELECT 
+                    COUNT(*) as cnt,
+                    COUNT(DISTINCT `{col.name}`) as dist,
+                    COUNT(*) - COUNT(`{col.name}`) as nulls
+                FROM {full_table} {sample_clause}
+            """
+        else:
+            stats_sql = f"""
+                SELECT 
+                    COUNT(*) as cnt,
+                    COUNT(DISTINCT `{col.name}`) as dist,
+                    COUNT(*) - COUNT(`{col.name}`) as nulls
+                FROM {full_table}
+            """
+
+        result = spark.sql(stats_sql).collect()[0]
+        cnt = result["cnt"] or 1
+        dist = result["dist"] or 0
+        nulls = result["nulls"] or 0
+
+        col.distinct_count = dist
+        col.distinct_ratio = dist / cnt if cnt > 0 else 0.0
+        col.null_ratio = nulls / cnt if cnt > 0 else 0.0
+
+        if col.value_type in ("integer", "number") or col.value_type == "date":
+            minmax_sql = f"SELECT MIN(`{col.name}`), MAX(`{col.name}`) FROM {full_table}"
+            minmax_result = spark.sql(minmax_sql).collect()[0]
+            if minmax_result:
+                col.min_val = str(minmax_result[0]) if minmax_result[0] is not None else None
+                col.max_val = str(minmax_result[1]) if minmax_result[1] is not None else None
+
+        topk_sql = f"""
+            SELECT `{col.name}`, COUNT(*) as freq 
+            FROM {full_table} 
+            WHERE `{col.name}` IS NOT NULL
+            GROUP BY `{col.name}` 
+            ORDER BY freq DESC 
+            LIMIT {PolicyConfig.CATEGORICAL_SAMPLE_SIZE}
+        """
+        topk_result = spark.sql(topk_sql).collect()
+        col.top_k_values = [str(row[col.name]) for row in topk_result if row[col.name] is not None]
+    except Exception as e:
+        debug(f"[schema_profiling.profile_column_spark] failed: {table_name}.{col.name}: {e}")
+
+
+def _profile_composite_descriptive_spark(
+    spark,
+    catalog: str,
+    schema_name: str,
+    table: TableMetadata,
+) -> None:
+    """Compute composite distinct ratios for name-like column pairs via Spark.
+
+    Spark equivalent of ``_profile_composite_descriptive``.
+    """
+    name_cols = [
+        col_name
+        for col_name, col_meta in table.columns.items()
+        if (col_meta.value_type or "").lower() == "string"
+        and _NAME_COLUMN_PATTERN.search(col_name)
+        and not col_meta.is_primary_key
+        and not col_meta.is_foreign_key
+    ]
+    if len(name_cols) < 2:
+        return
+    row_count = table.row_count or 0
+    if row_count == 0:
+        return
+    full_table = f"`{catalog}`.`{schema_name}`.`{table.name}`"
+    try:
+        for i in range(len(name_cols)):
+            for j in range(i + 1, len(name_cols)):
+                c1, c2 = name_cols[i], name_cols[j]
+                sql = (
+                    f"SELECT COUNT(DISTINCT CONCAT(`{c1}`, ' ', `{c2}`)) "
+                    f"FROM {full_table}"
+                )
+                composite_distinct = spark.sql(sql).collect()[0][0] or 0
+                ratio = composite_distinct / row_count
+                table.composite_descriptive_ratios[(c1, c2)] = ratio
+                debug(
+                    f"[schema_profiling._profile_composite_descriptive_spark] "
+                    f"{table.name}.({c1}, {c2}) composite_ratio={ratio:.4f}"
+                )
+    except Exception as exc:
+        debug(
+            f"[schema_profiling._profile_composite_descriptive_spark] "
+            f"failed for {table.name}: {exc}"
+        )
+
+
+def _profile_table_spark(spark, catalog: str, schema_name: str, table: TableMetadata) -> None:
+    """Profile all columns in a Databricks table via Spark queries.
+
+    Args:
+
+        spark: Active ``SparkSession``.
+
+        catalog: The Unity Catalog name.
+
+        schema_name: The schema (database) name.
+
+        table: The ``TableMetadata`` to update.
+    """
+    debug(f"[schema_profiling.profile_table_spark] profiling {table.name} ({len(table.columns)} columns)")
+    try:
+        full_table = f"`{catalog}`.`{schema_name}`.`{table.name}`"
+        count_sql = f"SELECT COUNT(*) FROM {full_table}"
+        row_count = spark.sql(count_sql).collect()[0][0] or 0
+        table.row_count = row_count
+    except Exception as e:
+        debug(f"[schema_profiling.profile_table_spark] row count failed: {table.name}: {e}")
+        row_count = 0
+        table.row_count = 0
+
+    for col in table.columns.values():
+        _profile_column_spark(spark, catalog, schema_name, col, table.name, row_count)
+
+    _profile_composite_descriptive_spark(spark, catalog, schema_name, table)
+
+    debug(f"[schema_profiling.profile_table_spark] completed: {table.name}")
+
+
+def profile_schema_spark(spark, catalog: str, schema_name: str, schema: SchemaGraph) -> None:
+    """Profile all tables in a Databricks schema via Spark queries.
+
+    Args:
+
+        spark: Active ``SparkSession``.
+
+        catalog: The Unity Catalog name.
+
+        schema_name: The schema (database) name.
+
+        schema: The ``SchemaGraph`` whose tables will be profiled.
+    """
+    debug(f"[schema_profiling.profile_schema_spark] profiling {len(schema.tables)} tables")
+    for table in schema.tables.values():
+        _profile_table_spark(spark, catalog, schema_name, table)
+    debug("[schema_profiling.profile_schema_spark] completed")
+
+
+def _cursor_rows_as_dicts(cursor) -> list[dict]:
+    """Convert cursor result rows to list of dicts keyed by column name."""
+    if not cursor.description:
+        return []
+    col_names = [d[0] for d in cursor.description]
+    return [dict(zip(col_names, row)) for row in cursor.fetchall()]
+
+
+def _profile_column_sql_connector(
+    connection,
+    catalog: str,
+    schema_name: str,
+    col: ColumnMetadata,
+    table_name: str,
+    row_count: int,
+    sample_threshold: int = None,
+    sample_size: int = None,
+) -> None:
+    """Profile a single column via databricks-sql-connector and update metadata in-place."""
+    if sample_threshold is None:
+        sample_threshold = QSimConfig.PROFILING_SAMPLE_THRESHOLD
+    if sample_size is None:
+        sample_size = QSimConfig.PROFILING_SAMPLE_SIZE
+    col.row_count = row_count
+    use_sample = row_count > sample_threshold
+    full_table = f"`{catalog}`.`{schema_name}`.`{table_name}`"
+    sample_clause = f"TABLESAMPLE ({sample_size} ROWS)" if use_sample else ""
+    try:
+        with connection.cursor() as cursor:
+            if use_sample:
+                stats_sql = f"""
+                    SELECT
+                        COUNT(*) as cnt,
+                        COUNT(DISTINCT `{col.name}`) as dist,
+                        COUNT(*) - COUNT(`{col.name}`) as nulls
+                    FROM {full_table} {sample_clause}
+                """
+            else:
+                stats_sql = f"""
+                    SELECT
+                        COUNT(*) as cnt,
+                        COUNT(DISTINCT `{col.name}`) as dist,
+                        COUNT(*) - COUNT(`{col.name}`) as nulls
+                    FROM {full_table}
+                """
+            cursor.execute(stats_sql)
+            rows = _cursor_rows_as_dicts(cursor)
+            if rows:
+                r = rows[0]
+                cnt = r.get("cnt") or 1
+                dist = r.get("dist") or 0
+                nulls = r.get("nulls") or 0
+                col.distinct_count = dist
+                col.distinct_ratio = dist / cnt if cnt > 0 else 0.0
+                col.null_ratio = nulls / cnt if cnt > 0 else 0.0
+            if col.value_type in ("integer", "number") or col.value_type == "date":
+                minmax_sql = f"SELECT MIN(`{col.name}`) as mn, MAX(`{col.name}`) as mx FROM {full_table}"
+                cursor.execute(minmax_sql)
+                minmax_rows = _cursor_rows_as_dicts(cursor)
+                if minmax_rows:
+                    r = minmax_rows[0]
+                    col.min_val = str(r["mn"]) if r.get("mn") is not None else None
+                    col.max_val = str(r["mx"]) if r.get("mx") is not None else None
+            topk_sql = f"""
+                SELECT `{col.name}` as topval, COUNT(*) as freq
+                FROM {full_table}
+                WHERE `{col.name}` IS NOT NULL
+                GROUP BY `{col.name}`
+                ORDER BY freq DESC
+                LIMIT {PolicyConfig.CATEGORICAL_SAMPLE_SIZE}
+            """
+            cursor.execute(topk_sql)
+            topk_rows = _cursor_rows_as_dicts(cursor)
+            col.top_k_values = [
+                str(r["topval"]) for r in topk_rows if r and r.get("topval") is not None
+            ]
+    except Exception as e:
+        debug(f"[schema_profiling._profile_column_sql_connector] failed: {table_name}.{col.name}: {e}")
+
+
+def _profile_composite_descriptive_sql_connector(
+    connection,
+    catalog: str,
+    schema_name: str,
+    table: TableMetadata,
+) -> None:
+    """Compute composite distinct ratios for name-like column pairs via SQL connector."""
+    name_cols = [
+        col_name
+        for col_name, col_meta in table.columns.items()
+        if (col_meta.value_type or "").lower() == "string"
+        and _NAME_COLUMN_PATTERN.search(col_name)
+        and not col_meta.is_primary_key
+        and not col_meta.is_foreign_key
+    ]
+    if len(name_cols) < 2:
+        return
+    row_count = table.row_count or 0
+    if row_count == 0:
+        return
+    full_table = f"`{catalog}`.`{schema_name}`.`{table.name}`"
+    try:
+        with connection.cursor() as cursor:
+            for i in range(len(name_cols)):
+                for j in range(i + 1, len(name_cols)):
+                    c1, c2 = name_cols[i], name_cols[j]
+                    sql = f"SELECT COUNT(DISTINCT CONCAT(`{c1}`, ' ', `{c2}`)) FROM {full_table}"
+                    cursor.execute(sql)
+                    rows = _cursor_rows_as_dicts(cursor)
+                    composite_distinct = 0
+                    if rows and rows[0]:
+                        composite_distinct = list(rows[0].values())[0] or 0
+                    ratio = composite_distinct / row_count
+                    table.composite_descriptive_ratios[(c1, c2)] = ratio
+                    debug(
+                        f"[schema_profiling._profile_composite_descriptive_sql_connector] "
+                        f"{table.name}.({c1}, {c2}) composite_ratio={ratio:.4f}"
+                    )
+    except Exception as exc:
+        debug(
+            f"[schema_profiling._profile_composite_descriptive_sql_connector] "
+            f"failed for {table.name}: {exc}"
+        )
+
+
+def _profile_table_sql_connector(
+    connection,
+    catalog: str,
+    schema_name: str,
+    table: TableMetadata,
+) -> None:
+    """Profile all columns in a Databricks table via databricks-sql-connector."""
+    debug(f"[schema_profiling._profile_table_sql_connector] profiling {table.name}")
+    full_table = f"`{catalog}`.`{schema_name}`.`{table.name}`"
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(f"SELECT COUNT(*) FROM {full_table}")
+            rows = _cursor_rows_as_dicts(cursor)
+            row_count = rows[0].get(list(rows[0].keys())[0], 0) or 0 if rows else 0
+    except Exception as e:
+        debug(f"[schema_profiling._profile_table_sql_connector] row count failed: {table.name}: {e}")
+        row_count = 0
+    table.row_count = row_count
+    for col in table.columns.values():
+        _profile_column_sql_connector(
+            connection, catalog, schema_name, col, table.name, row_count
+        )
+    _profile_composite_descriptive_sql_connector(connection, catalog, schema_name, table)
+    debug(f"[schema_profiling._profile_table_sql_connector] completed: {table.name}")
+
+
+def profile_schema_sql_connector(
+    connection,
+    catalog: str,
+    schema_name: str,
+    schema: SchemaGraph,
+) -> None:
+    """Profile all tables in a Databricks schema via databricks-sql-connector.
+
+    Args:
+
+        connection: Active ``databricks.sql`` connection.
+
+        catalog: The Unity Catalog name.
+
+        schema_name: The schema (database) name.
+
+        schema: The ``SchemaGraph`` whose tables will be profiled.
+    """
+    debug(f"[schema_profiling.profile_schema_sql_connector] profiling {len(schema.tables)} tables")
+    for table in schema.tables.values():
+        _profile_table_sql_connector(connection, catalog, schema_name, table)
+    debug("[schema_profiling.profile_schema_sql_connector] completed")
+
+
+def extract_tables_from_catalog_sql_connector(
+    connection,
+    catalog: str,
+    schema: str,
+) -> dict[str, dict]:
+    """Extract full table metadata from a Databricks Unity Catalog schema via SQL connector.
+
+    Args:
+
+        connection: Active ``databricks.sql`` connection.
+
+        catalog: The catalog name.
+
+        schema: The schema (database) name.
+
+    Returns:
+
+        Dict mapping table name to a metadata dict with keys ``table_name_original``,
+        ``column_names_original``, ``column_types``, ``primary_keys``, ``foreign_keys``,
+        ``table_comment``, and ``properties``.
+    """
+    tables = {}
+    with connection.cursor() as cursor:
+        cursor.execute(f"SHOW TABLES IN {catalog}.{schema}")
+        table_rows = _cursor_rows_as_dicts(cursor)
+    table_col = "tableName"
+    if table_rows and table_rows[0]:
+        row0 = table_rows[0]
+        if "tableName" in row0:
+            table_col = "tableName"
+        elif "tablename" in row0:
+            table_col = "tablename"
+        else:
+            table_col = list(row0.keys())[0]
+    for row in table_rows or []:
+        table_name = row.get(table_col) if row else None
+        if not table_name:
+            continue
+        full_table = f"{catalog}.{schema}.{table_name}"
+        debug(f"[schema_profiling.extract_tables_from_catalog_sql_connector] extracting: {full_table}")
+        with connection.cursor() as cursor:
+            cursor.execute(f"DESCRIBE TABLE {full_table}")
+            cols = _cursor_rows_as_dicts(cursor)
+        column_names = []
+        column_types = []
+        for col in cols:
+            cname = col.get("col_name") or col.get("colname")
+            if not cname or str(cname).startswith("#"):
+                break
+            column_names.append(cname)
+            column_types.append(col.get("data_type") or "STRING")
+        primary_keys = []
+        foreign_keys = []
+        partition_columns: list[str] = []
+        try:
+            with connection.cursor() as cursor:
+                cursor.execute(f"SHOW CREATE TABLE {full_table}")
+                create_rows = _cursor_rows_as_dicts(cursor)
+            if create_rows:
+                row0 = create_rows[0] or {}
+                stmt = row0.get("createtab_stmt") or (
+                    list(row0.values())[0] if row0 else None
+                )
+                if stmt:
+                    primary_keys, foreign_keys = _parse_unity_catalog_constraints(stmt)
+                    partition_columns = _parse_partition_columns_from_create_stmt(stmt)
+                    debug(f"[schema_profiling.extract_tables_from_catalog_sql_connector] ddl_found: {full_table}")
+        except Exception as e:
+            debug(f"[schema_profiling.extract_tables_from_catalog_sql_connector] ddl_error: {full_table} {e}")
+
+        if not partition_columns:
+            partition_columns = _extract_partition_columns_from_describe_detail_sql_connector(
+                connection, full_table
+            )
+
+        properties = {}
+        try:
+            with connection.cursor() as cursor:
+                cursor.execute(f"SHOW TBLPROPERTIES {full_table}")
+                prop_rows = _cursor_rows_as_dicts(cursor)
+            for r in prop_rows or []:
+                k = r.get("key")
+                v = r.get("value")
+                if k is not None:
+                    properties[k] = v
+        except Exception:
+            pass
+        table_comment = None
+        try:
+            with connection.cursor() as cursor:
+                cursor.execute(f"DESCRIBE TABLE EXTENDED {full_table}")
+                ext_rows = _cursor_rows_as_dicts(cursor)
+            for r in ext_rows or []:
+                cname = r.get("col_name") or r.get("colname")
+                if cname == "Comment":
+                    table_comment = r.get("data_type")
+                    break
+        except Exception:
+            pass
+        tables[table_name] = {
+            "table_name_original": table_name,
+            "column_names_original": column_names,
+            "column_types": column_types,
+            "primary_keys": primary_keys,
+            "foreign_keys": foreign_keys,
+            "partition_columns": partition_columns,
+            "table_comment": table_comment,
+            "properties": properties,
+        }
+    debug(f"[schema_profiling.extract_tables_from_catalog_sql_connector] complete: {len(tables)} tables")
+    return tables
+
+
+def _enrich_fk_column_descriptions(schema: SchemaGraph) -> None:
+    """Append navigational hints to FK column descriptions.
+
+    For each foreign-key column whose description does not already mention the target table, appends a short suffix listing the target table and its notable descriptive (non-PK, non-FK) columns. This helps the intent LLM discover join relationships from the schema text alone.
+    """
+    for table in schema.tables.values():
+        for col in table.columns.values():
+            if not col.fk_target:
+                continue
+            dst_table_name, _dst_col = col.fk_target
+            dst_table = schema.tables.get(dst_table_name)
+            if not dst_table:
+                continue
+            if dst_table_name.lower() in (col.description or "").lower():
+                continue
+            descriptive_cols = [
+                c.name
+                for c in dst_table.columns.values()
+                if not c.is_primary_key and not c.is_foreign_key and c.role not in ("identifier", "")
+            ][:3]
+            if not descriptive_cols:
+                descriptive_cols = [
+                    c.name for c in dst_table.columns.values() if not c.is_primary_key and not c.is_foreign_key
+                ][:3]
+            if descriptive_cols:
+                suffix = f"join {dst_table_name} for {', '.join(descriptive_cols)}"
+            else:
+                suffix = f"join {dst_table_name}"
+            existing = (col.description or "").rstrip(". ")
+            col.description = f"{existing} — {suffix}" if existing else suffix
+
+
+def _infer_column_role(col: ColumnMetadata) -> ColumnRole:
+    """Infer a column's role from its metadata using heuristic rules (fallback).
+
+    Evaluation priority: BOOLEAN (type) → IDENTIFIER (PK/FK) → BOOLEAN (value pattern) → TEMPORAL → FREE_TEXT (high uniqueness) → CATEGORICAL / NUMERIC_CATEGORICAL → NUMERIC_MEASURE → FREE_TEXT.
+
+    Args:
+
+        col: The ``ColumnMetadata`` to classify.
+
+    Returns:
+
+        The inferred ``ColumnRole`` enum value.
+    """
+    if col.value_type == "boolean":
+        return ColumnRole.BOOLEAN
+
+    if col.is_primary_key:
+        return ColumnRole.IDENTIFIER
+
+    if col.is_foreign_key:
+        return ColumnRole.IDENTIFIER
+
+    if _has_boolean_like_values(col):
+        return ColumnRole.BOOLEAN
+
+    if col.value_type == "date":
+        return ColumnRole.TEMPORAL
+
+    if col.distinct_ratio >= PolicyConfig.IDENTIFIER_MIN_UNIQUENESS:
+        return ColumnRole.FREE_TEXT
+
+    is_categorical = (
+        col.distinct_count <= PolicyConfig.CATEGORICAL_MAX_CARDINALITY
+        or col.distinct_ratio <= PolicyConfig.CATEGORICAL_MAX_RATIO
+    )
+    if is_categorical:
+        if col.value_type in ("integer", "number"):
+            return ColumnRole.NUMERIC_CATEGORICAL
+        return ColumnRole.CATEGORICAL
+
+    if col.value_type in ("integer", "number"):
+        return ColumnRole.NUMERIC_MEASURE
+
+    return ColumnRole.FREE_TEXT
+
+
+def _validate_column_classification(col: ColumnMetadata, role: str) -> tuple[list[str], list[str]]:
+    """Validate an LLM-assigned column role against profiling data.
+
+    Args:
+
+        col: The ``ColumnMetadata`` with profiling statistics.
+
+        role: The role string assigned by the LLM.
+
+    Returns:
+
+        Tuple of ``(hard_errors, soft_warnings)`` where hard errors block acceptance and trigger a retry, and soft warnings are logged only.
+    """
+    hard_errors = []
+    soft_warnings = []
+
+    is_numeric = col.value_type in ("integer", "number")
+    is_temporal = col.value_type == "date"
+    col_name_lower = col.name.lower()
+    dtype = (col.data_type or "").upper()
+
+    if role == ColumnRole.NUMERIC_MEASURE.value and not is_numeric:
+        hard_errors.append(f"{col.name}: NUMERIC_MEASURE requires numeric type, got '{col.data_type}'")
+
+    if role == ColumnRole.TEMPORAL.value and not is_temporal:
+        if "year" in col_name_lower or "DOMAIN" in dtype or "YEAR" in dtype:
+            soft_warnings.append(f"{col.name}: TEMPORAL on year/domain column, recommend NUMERIC_CATEGORICAL")
+        else:
+            hard_errors.append(f"{col.name}: TEMPORAL requires date/time type, got '{col.data_type}'")
+
+    if role == ColumnRole.BOOLEAN.value and col.distinct_count and col.distinct_count > 2:
+        hard_errors.append(f"{col.name}: BOOLEAN requires distinct_count <= 2, got {col.distinct_count}")
+
+    if role == ColumnRole.CATEGORICAL.value and col.distinct_count and col.distinct_count > 1000:
+        soft_warnings.append(f"{col.name}: CATEGORICAL with high cardinality ({col.distinct_count})")
+
+    if role == ColumnRole.NUMERIC_MEASURE.value and col.distinct_count and col.distinct_count <= 5:
+        soft_warnings.append(f"{col.name}: NUMERIC_MEASURE with low cardinality ({col.distinct_count})")
+
+    if role == ColumnRole.IDENTIFIER.value and not col.is_primary_key and not col.is_foreign_key:
+        soft_warnings.append(f"{col.name}: IDENTIFIER on non-PK/FK column")
+
+    return hard_errors, soft_warnings
+
+
+def _build_column_profile_for_llm(col: ColumnMetadata) -> dict:
+    """Build a column profile dict for inclusion in the LLM classification prompt.
+
+    Args:
+
+        col: The ``ColumnMetadata`` to summarise.
+
+    Returns:
+
+        Dict with ``name``, ``data_type``, ``is_primary_key``, ``is_foreign_key``, and an optional ``profile_hints`` sub-dict.
+    """
+    profile = {
+        "name": col.name,
+        "data_type": col.data_type,
+        "is_primary_key": col.is_primary_key,
+        "is_foreign_key": col.is_foreign_key,
+    }
+    hints: dict = {}
+    if col.distinct_count is not None:
+        hints["distinct_count"] = col.distinct_count
+    if col.distinct_ratio is not None:
+        hints["distinct_ratio"] = round(col.distinct_ratio, 3)
+    if col.null_ratio is not None:
+        hints["null_ratio"] = round(col.null_ratio, 3)
+    if hints:
+        profile["profile_hints"] = hints
+    return profile
+
+
+def _llm_classify_schema(
+    schema: SchemaGraph,
+) -> dict[str, tuple[str, str, dict[str, tuple[str, str]]]]:
+    """Use a single LLM call to classify all table roles, column roles, column semantic hints, and table descriptions.
+
+    Args:
+
+        schema: The ``SchemaGraph`` containing all tables and columns to classify.
+
+    Returns:
+
+        Dict mapping table name to a tuple of ``(table_role, description, {col_name: (role, hint), ...})``.
+
+    Raises:
+
+        ValueError: If the LLM returns invalid or non-dict JSON.
+    """
+    tables_data = []
+    for table in schema.tables.values():
+        fks = [",".join(fk.src_cols) + "->" + fk.dst_table + "." + ",".join(fk.dst_cols) for fk in table.foreign_keys]
+        column_profiles = [_build_column_profile_for_llm(col) for col in table.columns.values()]
+        tables_data.append(
+            {
+                "table": table.name,
+                "fks": fks,
+                "columns": column_profiles,
+            }
+        )
+    system = (
+        "Classify every table's role and every column's role in this schema.\n\n"
+        "TABLE ROLES:\n"
+        "- dimension: reference/lookup table referenced by others, descriptive attributes\n"
+        "- fact: transactional/event table with FKs to dimensions, contains measures\n"
+        "- bridge: junction table for many-to-many, mostly FKs, few own columns\n"
+        "- unknown: cannot confidently classify\n"
+        "Use FK topology: tables referenced by many others are dimension; tables with many outbound FKs are fact; tables with only 2+ FKs and minimal columns are bridge.\n\n"
+        "COLUMN ROLE DECISION PRIORITY (evaluate in order, first match wins):\n"
+        "1. is_primary_key or is_foreign_key → identifier\n"
+        "2. data_type is date/time/timestamp → temporal\n"
+        "3. name suggests binary state (is_*, has_*, active) and distinct_count = 2 → boolean\n"
+        "4. numeric and name suggests quantity/amount/duration/size/distance/price/count → numeric_measure (integer or decimal — data type does not restrict this)\n"
+        "5. numeric and name suggests code/rating/level/rank/status/tier/type → numeric_categorical\n"
+        "6. numeric with no clear name signal → numeric_measure (default for numeric)\n"
+        "7. text and very high distinct_ratio → free_text\n"
+        "8. text → categorical (default for text)\n\n"
+        "PROFILE HINTS (supporting evidence only — never override name/type signals):\n"
+        "Each column may include a profile_hints object with distinct_count, distinct_ratio, and null_ratio. Use these to confirm or disambiguate when name and type are ambiguous.\n"
+        "Do NOT use profile_hints as the primary reason to choose a role.\n\n"
+        "CROSS-TABLE CONSISTENCY:\n"
+        "- Columns with the same name and data type across tables MUST receive the same role.\n\n"
+        "COLUMN HINTS:\n"
+        "For each column, provide a short semantic hint (max 8 words) describing what the column represents in business terms. Include common synonyms a user might use when asking questions about this data. The hint should help map natural language to the correct column.\n"
+        "Role-based guidance for hints:\n"
+        "- identifier columns: describe what entity the ID refers to.\n"
+        "- numeric_measure columns: state the unit or what is measured.\n"
+        "- categorical columns: mention common category values or groupings.\n"
+        "- temporal columns: state what event the date/time marks.\n"
+        "- boolean columns: describe the yes/no condition.\n"
+        "- FK columns: MUST state what business data the target table provides when joined. Name the key descriptive columns on the target table (e.g. 'links to target_table for name, title, description').\n\n"
+        "TABLE DESCRIPTIONS:\n"
+        "For each table provide a one-line business purpose that includes: (a) what entity or event the table represents, (b) which related tables it connects to via foreign keys, and (c) the notable descriptive or measure columns it provides that users commonly ask about.\n\n"
+        "Reason internally, output only JSON:\n"
+        '{"table1": {"table_role": "...", "description": "one-line business purpose including related tables and key columns", "columns": {"col1": {"role": "...", "hint": "..."}, ...}}, ...}'
+    )
+    user = stable_json({"tables": tables_data})
+    raw = llm_chat(system, user, timeout=120.0, task="schema")
+    result = safe_json_loads(raw)
+    if not result or not isinstance(result, dict):
+        raise ValueError(f"LLM returned invalid JSON for schema classification: {raw[:200]}")
+    valid_table_roles = {"dimension", "fact", "bridge", "unknown"}
+    valid_col_roles = {r.value for r in ColumnRole}
+    classifications = {}
+    for table_name, table_data in result.items():
+        if not isinstance(table_data, dict):
+            continue
+        table_role = table_data.get("table_role", "unknown").lower()
+        if table_role not in valid_table_roles:
+            table_role = "unknown"
+        description = str(table_data.get("description", "")).strip()
+        columns_data = table_data.get("columns", {})
+        column_classifications: dict[str, tuple[str, str]] = {}
+        for col_name, classification in columns_data.items():
+            if isinstance(classification, dict):
+                role = classification.get("role", "").lower()
+                hint = str(classification.get("hint", "")).strip()
+            else:
+                role = str(classification).lower()
+                hint = ""
+            if role not in valid_col_roles:
+                role = ColumnRole.FREE_TEXT.value
+            column_classifications[col_name] = (role, hint)
+        classifications[table_name] = (table_role, description, column_classifications)
+    return classifications
+
+
+def apply_column_roles_llm(schema: SchemaGraph) -> None:
+    """Apply LLM-inferred roles and table descriptions to the schema in-place.
+
+    Retries up to ``QSimConfig.MAX_ROLE_CLASSIFICATION_RETRIES`` times on hard validation errors. Falls back to ``infer_column_role`` for all tables if all LLM attempts fail. Also applies hard overrides for PK/FK columns and boolean-value-pattern columns after LLM assignment.
+
+    Args:
+
+        schema: The ``SchemaGraph`` to update in-place.
+    """
+    debug(f"[schema_profiling.apply_column_roles_llm] classifying {len(schema.tables)} tables via LLM (single-call)")
+    total_columns = sum(len(table.columns) for table in schema.tables.values())
+    debug(f"[schema_profiling.apply_column_roles_llm] total columns: {total_columns}")
+    role_counts: dict[str, int] = {}
+    table_role_counts: dict[str, int] = {}
+    llm_success = 0
+    llm_fallback = 0
+    success = False
+    for attempt in range(QSimConfig.MAX_ROLE_CLASSIFICATION_RETRIES + 1):
+        try:
+            classifications = _llm_classify_schema(schema)
+            all_hard_errors = []
+            all_soft_warnings = []
+            for table in schema.tables.values():
+                if table.name not in classifications:
+                    all_hard_errors.append(f"{table.name}: missing from LLM response")
+                    continue
+                table_role, _desc, column_classifications = classifications[table.name]
+                for col in table.columns.values():
+                    if col.name not in column_classifications:
+                        all_hard_errors.append(f"{table.name}.{col.name}: missing from LLM response")
+                        continue
+                    role, _hint = column_classifications[col.name]
+                    hard_errors, soft_warnings = _validate_column_classification(col, role)
+                    all_hard_errors.extend([f"{table.name}.{e}" for e in hard_errors])
+                    all_soft_warnings.extend([f"{table.name}.{w}" for w in soft_warnings])
+            for warning in all_soft_warnings:
+                debug(f"[apply_column_roles_llm] WARNING: {warning}")
+            if all_hard_errors:
+                debug(
+                    f"[apply_column_roles_llm] {len(all_hard_errors)} hard errors (attempt {attempt + 1}): {all_hard_errors[:5]}"
+                )
+                continue
+            for table in schema.tables.values():
+                if table.name in classifications:
+                    table_role, description, column_classifications = classifications[table.name]
+                    table.role = table_role
+                    table.description = description
+                    table_role_counts[table_role] = table_role_counts.get(table_role, 0) + 1
+                    for col in table.columns.values():
+                        if col.name in column_classifications:
+                            role, hint = column_classifications[col.name]
+                            col.role = role
+                            col.description = hint
+                            if col.is_primary_key or col.is_foreign_key:
+                                if col.role != ColumnRole.IDENTIFIER.value:
+                                    debug(
+                                        f"[apply_column_roles_llm] override {table.name}.{col.name}: {col.role} → identifier (pk/fk)"
+                                    )
+                                    col.role = ColumnRole.IDENTIFIER.value
+                            elif _has_boolean_like_values(col):
+                                if col.role != ColumnRole.BOOLEAN.value:
+                                    debug(
+                                        f"[apply_column_roles_llm] override {table.name}.{col.name}: {col.role} → boolean (value pattern)"
+                                    )
+                                    col.role = ColumnRole.BOOLEAN.value
+                            elif (
+                                col.role == ColumnRole.FREE_TEXT.value
+                                and col.distinct_count is not None
+                                and col.distinct_count <= PolicyConfig.FREE_TEXT_CATEGORICAL_MAX_CARDINALITY
+                            ):
+                                debug(
+                                    f"[apply_column_roles_llm] override {table.name}.{col.name}: free_text → categorical (distinct={col.distinct_count})"
+                                )
+                                col.role = ColumnRole.CATEGORICAL.value
+                            role_counts[col.role] = role_counts.get(col.role, 0) + 1
+            success = True
+            llm_success = len(schema.tables)
+            debug("[apply_column_roles_llm] single-call classification successful")
+            break
+        except Exception as e:
+            debug(f"[apply_column_roles_llm] attempt {attempt + 1} failed: {e}")
+            continue
+    if not success:
+        debug("[apply_column_roles_llm] LLM failed, using heuristic fallback for all tables")
+        for table in schema.tables.values():
+            llm_fallback += 1
+            fk_out = len(table.foreign_keys)
+            fk_in = sum(1 for t in schema.tables.values() for fk in t.foreign_keys if fk.dst_table == table.name)
+            if fk_out >= 2:
+                table_role = TableRole.FACT.value
+            elif fk_out == 0 and fk_in >= 1:
+                table_role = TableRole.DIMENSION.value
+            elif fk_out == 2 and fk_in == 0 and len(table.columns) <= 4:
+                table_role = TableRole.BRIDGE.value
+            else:
+                table_role = TableRole.UNKNOWN.value
+            table.role = table_role
+            table_role_counts[table_role] = table_role_counts.get(table_role, 0) + 1
+            for col in table.columns.values():
+                role = _infer_column_role(col)
+                col.role = role.value
+                role_counts[role.value] = role_counts.get(role.value, 0) + 1
+    debug(f"[apply_column_roles_llm] completed: {llm_success} LLM, {llm_fallback} fallback")
+    debug(f"[apply_column_roles_llm] table distribution: {table_role_counts}")
+    debug(f"[apply_column_roles_llm] column distribution: {role_counts}")
+    _enrich_fk_column_descriptions(schema)
+    boolean_mapped = 0
+    for table in schema.tables.values():
+        for col in table.columns.values():
+            if col.is_boolean_like or "bool" in (col.data_type or "").lower():
+                _populate_boolean_mapping(col)
+                if col.boolean_true_value is not None:
+                    boolean_mapped += 1
+    if boolean_mapped:
+        debug(f"[apply_column_roles_llm] boolean mappings populated: {boolean_mapped}")
+
+
+def assign_column_ops(schema: SchemaGraph) -> None:
+    """Assign valid filter, aggregation, and HAVING ops to each column based on its final role.
+
+    Deterministic; run after role assignment. Removes string-only operators from non-string columns and numeric-only aggregations from non-numeric columns. Columns marked ``is_filterable=False`` receive an empty filter-ops list.
+
+    Args:
+
+        schema: The ``SchemaGraph`` to update in-place.
+    """
+    debug("[schema_profiling.assign_column_ops] assigning ops to columns")
+
+    null_ops = ["is null", "is not null"]
+    string_only_ops = {"like", "ilike", "not like", "not ilike"}
+    numeric_only_aggs = {"sum", "avg"}
+
+    for table in schema.tables.values():
+        for col in table.columns.values():
+            role = col.role
+            vt = col.value_type
+            string = vt == "string"
+            numeric = vt in ("integer", "number")
+
+            if role == ColumnRole.AUDIT.value:
+                col.valid_filter_ops = []
+                col.valid_aggregations = []
+                col.valid_having_ops = []
+            elif col.is_primary_key or col.is_foreign_key or role == ColumnRole.IDENTIFIER.value:
+                col.valid_filter_ops = [
+                    "=",
+                    "!=",
+                    "<",
+                    "<=",
+                    ">",
+                    ">=",
+                    "between",
+                    "in",
+                    "not in",
+                ] + null_ops
+                col.valid_aggregations = ["count"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+            elif role == ColumnRole.CATEGORICAL.value:
+                col.valid_filter_ops = [
+                    "=",
+                    "!=",
+                    "in",
+                    "not in",
+                    "like",
+                    "ilike",
+                    "not like",
+                    "not ilike",
+                ] + null_ops
+                col.valid_aggregations = ["count", "min", "max"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+            elif role == ColumnRole.NUMERIC_CATEGORICAL.value:
+                col.valid_filter_ops = [
+                    "=",
+                    "!=",
+                    "in",
+                    "not in",
+                    "<",
+                    "<=",
+                    ">",
+                    ">=",
+                    "between",
+                ] + null_ops
+                col.valid_aggregations = ["count", "min", "max"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+            elif role == ColumnRole.NUMERIC_MEASURE.value:
+                col.valid_filter_ops = [
+                    "=",
+                    "!=",
+                    "<",
+                    "<=",
+                    ">",
+                    ">=",
+                    "between",
+                    "in",
+                    "not in",
+                ] + null_ops
+                col.valid_aggregations = ["sum", "avg", "min", "max", "count"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+            elif role == ColumnRole.TEMPORAL.value:
+                col.valid_filter_ops = [
+                    "=",
+                    "!=",
+                    "<",
+                    "<=",
+                    ">",
+                    ">=",
+                    "between",
+                    "in",
+                    "not in",
+                ] + null_ops
+                col.valid_aggregations = ["min", "max", "count"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+            elif role == ColumnRole.BOOLEAN.value:
+                col.valid_filter_ops = ["=", "!=", "in", "not in"] + null_ops
+                col.valid_aggregations = ["count"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+            elif role == ColumnRole.FREE_TEXT.value:
+                col.valid_filter_ops = [
+                    "like",
+                    "ilike",
+                    "not like",
+                    "not ilike",
+                ] + null_ops
+                col.valid_aggregations = ["count"]
+                col.valid_having_ops = []
+            else:
+                col.valid_filter_ops = ["=", "!="] + null_ops
+                col.valid_aggregations = ["count"]
+                col.valid_having_ops = ["=", "!=", "<", "<=", ">", ">="]
+
+            if not string:
+                col.valid_filter_ops = [op for op in col.valid_filter_ops if op not in string_only_ops]
+            if not numeric:
+                col.valid_aggregations = [agg for agg in col.valid_aggregations if agg not in numeric_only_aggs]
+
+            if not col.is_filterable:
+                if role == ColumnRole.FREE_TEXT.value:
+                    pattern_ops = {"like", "ilike", "not like", "not ilike",
+                                   "is null", "is not null"}
+                    col.valid_filter_ops = [
+                        op for op in col.valid_filter_ops if op in pattern_ops
+                    ]
+                else:
+                    col.valid_filter_ops = []
+
+    debug("[schema_profiling.assign_column_ops] completed")
+
+
+def extract_tables_from_catalog(spark, catalog: str, schema: str) -> dict[str, dict]:
+    """Extract full table metadata from a Databricks Unity Catalog schema.
+
+    Args:
+
+        spark: Active ``SparkSession``.
+
+        catalog: The catalog name.
+
+        schema: The schema (database) name.
+
+    Returns:
+
+        Dict mapping table name to a metadata dict with keys ``table_name_original``, ``column_names_original``, ``column_types``, ``primary_keys``, ``foreign_keys``, ``table_comment``, and ``properties``.
+    """
+    tables = {}
+
+    table_list = spark.sql(f"SHOW TABLES IN {catalog}.{schema}").collect()
+
+    for row in table_list:
+        table_name = row["tableName"]
+        full_table = f"{catalog}.{schema}.{table_name}"
+
+        debug(f"[schema_profiling.extract_tables_from_catalog] extracting: {full_table}")
+
+        cols = spark.sql(f"DESCRIBE TABLE {full_table}").collect()
+
+        column_names = []
+        column_types = []
+        primary_keys = []
+
+        for col in cols:
+            col_name = col["col_name"]
+
+            if col_name.startswith("#"):
+                break
+
+            column_names.append(col_name)
+            column_types.append(col["data_type"])
+
+        partition_columns: list[str] = []
+        try:
+            create_result = spark.sql(f"SHOW CREATE TABLE {full_table}").collect()
+            if create_result:
+                create_stmt = create_result[0]["createtab_stmt"]
+                primary_keys, foreign_keys = _parse_unity_catalog_constraints(create_stmt)
+                partition_columns = _parse_partition_columns_from_create_stmt(create_stmt)
+                debug(f"[schema_profiling.extract_tables_from_catalog] ddl_found: {full_table}")
+        except Exception as e:
+            debug(f"[schema_profiling.extract_tables_from_catalog] ddl_error: {full_table} {e}")
+            primary_keys = []
+            foreign_keys = []
+
+        if not partition_columns:
+            partition_columns = _extract_partition_columns_from_describe_detail_spark(
+                spark, full_table
+            )
+
+        try:
+            props = spark.sql(f"SHOW TBLPROPERTIES {full_table}").collect()
+            properties = {p["key"]: p["value"] for p in props}
+        except Exception:
+            properties = {}
+
+        try:
+            extended = spark.sql(f"DESCRIBE TABLE EXTENDED {full_table}").collect()
+            table_comment = None
+            for row in extended:
+                if row["col_name"] == "Comment":
+                    table_comment = row["data_type"]
+                    break
+        except Exception:
+            table_comment = None
+
+        tables[table_name] = {
+            "table_name_original": table_name,
+            "column_names_original": column_names,
+            "column_types": column_types,
+            "primary_keys": primary_keys,
+            "foreign_keys": foreign_keys,
+            "partition_columns": partition_columns,
+            "table_comment": table_comment,
+            "properties": properties,
+        }
+
+    debug(f"[schema_profiling.extract_tables_from_catalog] complete: {len(tables)} tables")
+    return tables
+
+
+def _extract_partition_columns_from_describe_detail_spark(
+    spark, full_table: str
+) -> list[str]:
+    """Extract partition column names via DESCRIBE DETAIL, fallback INFORMATION_SCHEMA.
+
+    Args:
+
+        spark: Active SparkSession.
+
+        full_table: Fully qualified table name (catalog.schema.table).
+
+    Returns:
+
+        List of partition column name strings.
+    """
+    try:
+        detail_df = spark.sql(f"DESCRIBE DETAIL {full_table}")
+        row = detail_df.collect()
+        if row:
+            r = row[0]
+            cols = r.get("partitionColumns") or r.get("partition_columns")
+            if isinstance(cols, list) and cols:
+                return [str(c) for c in cols]
+    except Exception as e:
+        debug(f"[schema_profiling._extract_partition_from_detail] DESCRIBE DETAIL failed: {e}")
+
+    try:
+        parts = full_table.split(".")
+        if len(parts) >= 3:
+            catalog_name, schema_name, table_name = parts[0], parts[1], parts[2]
+            info_schema = f"{catalog_name}.information_schema.columns"
+            q = f"""
+                SELECT column_name FROM {info_schema}
+                WHERE table_catalog = '{catalog_name}'
+                  AND table_schema = '{schema_name}'
+                  AND table_name = '{table_name}'
+                  AND partition_ordinal_position IS NOT NULL
+                ORDER BY partition_ordinal_position
+            """
+            info_rows = spark.sql(q).collect()
+            return [str(r["column_name"]) for r in info_rows if r.get("column_name")]
+    except Exception as e:
+        debug(f"[schema_profiling._extract_partition_from_detail] INFORMATION_SCHEMA failed: {e}")
+
+    return []
+
+
+def _extract_partition_columns_from_describe_detail_sql_connector(
+    connection, full_table: str
+) -> list[str]:
+    """Extract partition column names via DESCRIBE DETAIL, fallback INFORMATION_SCHEMA.
+
+    Args:
+
+        connection: Active databricks.sql connection.
+
+        full_table: Fully qualified table name (catalog.schema.table).
+
+    Returns:
+
+        List of partition column name strings.
+    """
+    try:
+        with connection.cursor() as cursor:
+            cursor.execute(f"DESCRIBE DETAIL {full_table}")
+            rows = _cursor_rows_as_dicts(cursor)
+        if rows:
+            r = rows[0]
+            cols = r.get("partitionColumns") or r.get("partition_columns")
+            if isinstance(cols, list) and cols:
+                return [str(c) for c in cols]
+    except Exception as e:
+        debug(
+            f"[schema_profiling._extract_partition_sql_connector] DESCRIBE DETAIL failed: {e}"
+        )
+
+    try:
+        parts = full_table.split(".")
+        if len(parts) >= 3:
+            catalog_name, schema_name, table_name = parts[0], parts[1], parts[2]
+            info_schema = f"{catalog_name}.information_schema.columns"
+            q = f"""
+                SELECT column_name FROM {info_schema}
+                WHERE table_catalog = '{catalog_name}'
+                  AND table_schema = '{schema_name}'
+                  AND table_name = '{table_name}'
+                  AND partition_ordinal_position IS NOT NULL
+                ORDER BY partition_ordinal_position
+            """
+            with connection.cursor() as cursor:
+                cursor.execute(q)
+                info_rows = _cursor_rows_as_dicts(cursor)
+            return [str(r["column_name"]) for r in info_rows if r.get("column_name")]
+    except Exception as e:
+        debug(
+            f"[schema_profiling._extract_partition_sql_connector] INFORMATION_SCHEMA failed: {e}"
+        )
+
+    return []
+
+
+def _parse_partition_columns_from_create_stmt(create_stmt: str) -> list[str]:
+    """Extract partition column names from a CREATE TABLE DDL string.
+
+    Matches ``PARTITIONED BY (col1, col2, ...)`` (case-insensitive).
+
+    Args:
+
+        create_stmt: The raw CREATE TABLE DDL string.
+
+    Returns:
+
+        List of partition column name strings, or empty list if not found.
+    """
+    match = re.search(r"PARTITIONED\s+BY\s*\(([^)]+)\)", create_stmt, re.IGNORECASE)
+    if not match:
+        return []
+    return [c.strip().strip("`").strip('"') for c in match.group(1).split(",")]
+
+
+def _parse_unity_catalog_constraints(create_stmt: str) -> tuple[list[str], list[dict]]:
+    """Parse PRIMARY KEY and FOREIGN KEY constraints from a CREATE TABLE DDL string.
+
+    Args:
+
+        create_stmt: The raw CREATE TABLE DDL string from ``SHOW CREATE TABLE``.
+
+    Returns:
+
+        Tuple of ``(primary_keys, foreign_keys)`` where ``primary_keys`` is a list of column name strings and ``foreign_keys`` is a list of dicts with keys ``src_cols``, ``dst_table``, and ``dst_cols``.
+    """
+    primary_keys = []
+    foreign_keys = []
+
+    pk_pattern = r"CONSTRAINT\s+\w+\s+PRIMARY\s+KEY\s*\(([^)]+)\)"
+    pk_matches = re.findall(pk_pattern, create_stmt, re.IGNORECASE)
+    for match in pk_matches:
+        cols = [c.strip().strip("`").strip('"') for c in match.split(",")]
+        primary_keys.extend(cols)
+
+    fk_pattern = r"CONSTRAINT\s+\w+\s+FOREIGN\s+KEY\s*\(([^)]+)\)\s+REFERENCES\s+(\w+)\s*\(([^)]+)\)"
+    fk_matches = re.findall(fk_pattern, create_stmt, re.IGNORECASE)
+    for match in fk_matches:
+        src_cols = [c.strip().strip("`").strip('"') for c in match[0].split(",")]
+        ref_table = match[1].strip("`").strip('"')
+        ref_cols = [c.strip().strip("`").strip('"') for c in match[2].split(",")]
+
+        foreign_keys.append({"src_cols": src_cols, "dst_table": ref_table, "dst_cols": ref_cols})
+
+    return primary_keys, foreign_keys
+
+
+def parse_sql_file(sql_path: Path) -> dict[str, dict]:
+    """Parse CREATE TABLE statements from a SQL file using sqlglot with LLM fallback.
+
+    Attempts sqlglot first; if it returns 0 tables, falls back to an LLM call to extract metadata.
+
+    Args:
+
+        sql_path: Path to the SQL file to parse.
+
+    Returns:
+
+        Dict mapping table name to a metadata dict (same structure as ``extract_tables_from_catalog``).
+    """
+    with open(sql_path, encoding="utf-8-sig") as f:
+        sql_content = f.read()
+
+    debug(f"[schema_profiling.parse_sql_file] reading: {len(sql_content)} chars")
+
+    tables = _parse_sql_file_fallback(sql_content)
+    if tables:
+        debug(f"[schema_profiling.parse_sql_file] sqlglot parsed: {len(tables)} tables")
+        return tables
+
+    debug("[schema_profiling.parse_sql_file] sqlglot returned 0 tables, falling back to LLM")
+
+    system = """You are a deterministic SQL parser. Extract CREATE TABLE statements and output ONLY valid JSON. Be precise and consistent. Follow the output format exactly."""
+
+    user = stable_json(
+        {
+            "task": "Parse all CREATE TABLE statements from the SQL and extract complete metadata",
+            "sql_content": sql_content,
+            "output_format": {
+                "tables": {
+                    "<table_name>": {
+                        "table_name_original": "exact table name from SQL (without schema prefix)",
+                        "column_names_original": ["column1", "column2"],
+                        "column_types": ["TYPE1", "TYPE2"],
+                        "primary_keys": ["column1"],
+                        "foreign_keys": [
+                            {
+                                "src_cols": ["column1"],
+                                "dst_table": "other_table",
+                                "dst_cols": ["column1"],
+                            }
+                        ],
+                    }
+                }
+            },
+            "rules": [
+                "Extract ALL CREATE TABLE statements, ignore other SQL commands",
+                "Preserve exact table and column names (case-sensitive)",
+                "Strip schema prefixes from table names (e.g., 'public.users' → 'users')",
+                "Remove quotes from identifiers (e.g., '\"user_id\"' → 'user_id')",
+                "Capture PRIMARY KEY constraints (inline and separate CONSTRAINT clauses)",
+                "Capture FOREIGN KEY constraints with all source/destination columns",
+                "Normalize data types to uppercase (e.g., 'varchar(50)' → 'VARCHAR(50)')",
+                "Convert SERIAL → INTEGER, BIGSERIAL → BIGINT",
+                "Use empty arrays [] for tables with no PKs or FKs",
+                "Ignore CHECK constraints, DEFAULT values, and UNIQUE constraints",
+                "Handle multi-line statements and SQL comments (-- and /* */)",
+                "Output ONLY the JSON object, no markdown code blocks, no explanations",
+            ],
+            "examples": [
+                {
+                    "input": "CREATE TABLE public.table1 (column1 SERIAL PRIMARY KEY, column2 VARCHAR(100));",
+                    "output": {
+                        "tables": {
+                            "table1": {
+                                "table_name_original": "table1",
+                                "column_names_original": ["column1", "column2"],
+                                "column_types": ["INTEGER", "VARCHAR(100)"],
+                                "primary_keys": ["column1"],
+                                "foreign_keys": [],
+                            }
+                        }
+                    },
+                },
+                {
+                    "input": "CREATE TABLE table2 (column1 INT, column2 INT, FOREIGN KEY (column2) REFERENCES table1(column1));",
+                    "output": {
+                        "tables": {
+                            "table2": {
+                                "table_name_original": "table2",
+                                "column_names_original": ["column1", "column2"],
+                                "column_types": ["INT", "INT"],
+                                "primary_keys": [],
+                                "foreign_keys": [
+                                    {
+                                        "src_cols": ["column2"],
+                                        "dst_table": "table1",
+                                        "dst_cols": ["column1"],
+                                    }
+                                ],
+                            }
+                        }
+                    },
+                },
+            ],
+        }
+    )
+
+    response = llm_chat(system, user, task="schema")
+    parsed = safe_json_loads(response)
+
+    if not isinstance(parsed, dict) or "tables" not in parsed:
+        debug("[schema_profiling.parse_sql_file] llm also failed, returning empty")
+        return {}
+
+    tables = parsed["tables"]
+    debug(f"[schema_profiling.parse_sql_file] llm parsed: {len(tables)} tables")
+
+    if PolicyConfig.DEBUG:
+        for tname, tinfo in tables.items():
+            debug(
+                f"[schema_profiling.parse_sql_file] table: {tname} cols={len(tinfo.get('column_names_original', []))}"
+            )
+
+    return tables
+
+
+def _parse_sql_file_fallback(sql_content: str) -> dict[str, dict]:
+    """Parse CREATE TABLE statements from SQL content using sqlglot.
+
+    Uses dialect from EngineConfig.TYPE (spark for Databricks, postgres for PostgreSQL).
+
+    Args:
+
+        sql_content: The full SQL file content as a string.
+
+    Returns:
+
+        Dict mapping table name to a metadata dict, or an empty dict if no tables are found.
+    """
+    import sqlglot
+    from sqlglot import exp
+
+    dialect_name = "spark" if EngineConfig.TYPE == "databricks" else "postgres"
+    try:
+        statements = sqlglot.parse(sql_content, dialect=dialect_name)
+    except Exception as e:
+        debug(f"[schema_profiling._parse_sql_file_fallback] parse failed: {e}")
+        return {}
+
+    debug(f"[schema_profiling._parse_sql_file_fallback] statements: {len(statements)}")
+    tables: dict[str, dict] = {}
+
+    for stmt in statements:
+        if not isinstance(stmt, exp.Create) or not stmt.this:
+            continue
+
+        table_ref = stmt.this
+        if hasattr(table_ref, "this") and table_ref.this is not None:
+            table_name = getattr(table_ref.this, "name", None) or str(table_ref.this)
+        else:
+            table_name = getattr(table_ref, "name", None) or str(table_ref)
+        if not table_name or "." in str(table_name):
+            table_name = str(table_ref).split(".")[-1].strip("`\"")
+        table_name = str(table_name).strip("`\"")
+
+        if not table_name:
+            continue
+
+        debug(f"[schema_profiling._parse_sql_file_fallback] parsing: {table_name}")
+
+        col_block = _extract_column_block_from_create(stmt)
+        columns, types, pks, fks = _parse_columns_and_constraints(col_block)
+        full_stmt = stmt.sql(dialect=dialect_name)
+        partition_cols = _parse_partition_columns_from_create_stmt(full_stmt)
+
+        debug(f"[schema_profiling._parse_sql_file_fallback] ddl_generated: {table_name}")
+
+        tables[table_name] = {
+            "table_name_original": table_name,
+            "column_names_original": columns,
+            "column_types": types,
+            "primary_keys": pks,
+            "foreign_keys": fks,
+            "partition_columns": partition_cols,
+        }
+
+    debug(f"[schema_profiling._parse_sql_file_fallback] complete: {len(tables)} tables")
+    return tables
+
+
+def _extract_column_block_from_create(create_expr: Any) -> str:
+    """Extract the inner content of the column definition block from a sqlglot Create.
+
+    In sqlglot, Create.this is a Schema (table + columns); Create.expression is often None.
+    Schema.expressions holds ColumnDef nodes. We join their SQL to form the column block.
+
+    Args:
+
+        create_expr: A parsed sqlglot ``Create`` expression.
+
+    Returns:
+
+        The column definition string with surrounding parentheses stripped.
+    """
+    schema = create_expr.this if create_expr.this else create_expr.expression
+    if schema is None:
+        return ""
+    expressions = getattr(schema, "expressions", None)
+    if expressions:
+        return ", ".join(e.sql() for e in expressions if hasattr(e, "sql"))
+    schema_sql = schema.sql()
+    if schema_sql.startswith("(") and schema_sql.endswith(")"):
+        return schema_sql[1:-1].strip()
+    full_sql = create_expr.sql()
+    match = re.search(r"\(([\s\S]*)\)\s*(?:PARTITIONED|STORED|LOCATION|AS|$)", full_sql)
+    if match:
+        return match.group(1).strip()
+    return schema_sql.strip()
+
+
+def _split_by_top_level_comma(s: str) -> list[str]:
+    """Split string by commas that are outside parentheses.
+
+    Preserves commas inside type definitions such as NUMERIC(4,2) or VARCHAR(10,2).
+
+    Args:
+
+        s: The string to split (e.g. a CREATE TABLE column block).
+
+    Returns:
+
+        List of non-empty segments between top-level commas.
+    """
+    result: list[str] = []
+    current: list[str] = []
+    depth = 0
+    for char in s:
+        if char == "(":
+            depth += 1
+            current.append(char)
+        elif char == ")":
+            depth -= 1
+            current.append(char)
+        elif char == "," and depth == 0:
+            segment = "".join(current).strip()
+            if segment:
+                result.append(segment)
+            current = []
+        else:
+            current.append(char)
+    segment = "".join(current).strip()
+    if segment:
+        result.append(segment)
+    return result
+
+
+def _parse_columns_and_constraints(col_block: str) -> tuple[list, list, list, list]:
+    """Parse column definitions and constraints from a column block string.
+
+    Args:
+
+        col_block: The inner column definition block (contents between ``(`` and ``)``) from a CREATE TABLE statement.
+
+    Returns:
+
+        Tuple of ``(columns, types, pks, fks)``.
+    """
+    lines = [line.strip() for line in _split_by_top_level_comma(col_block)]
+
+    columns = []
+    types = []
+    pks = []
+    fks = []
+
+    for line in lines:
+        line_upper = line.upper()
+
+        if line_upper.startswith("PRIMARY KEY"):
+            pk_cols = _extract_pk_columns(line)
+            pks.extend(pk_cols)
+            continue
+
+        if line_upper.startswith("FOREIGN KEY"):
+            fk_def = _extract_fk_definition(line)
+            if fk_def:
+                fks.append(fk_def)
+            continue
+
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+
+        col_name = parts[0].strip("`").strip('"')
+        col_type = parts[1]
+
+        columns.append(col_name)
+        types.append(col_type)
+
+        if "PRIMARY KEY" in line_upper:
+            pks.append(col_name)
+
+    return columns, types, pks, fks
+
+
+def _extract_pk_columns(line: str) -> list[str]:
+    """Extract column names from a PRIMARY KEY (col1, col2) definition line.
+
+    Args:
+
+        line: A single DDL line starting with ``PRIMARY KEY``.
+
+    Returns:
+
+        List of unquoted column name strings.
+    """
+    match = re.search(r"PRIMARY\s+KEY\s*\(([^)]+)\)", line, re.IGNORECASE)
+    if match:
+        return [c.strip().strip("`").strip('"') for c in match.group(1).split(",")]
+    return []
+
+
+def _extract_fk_definition(line: str) -> dict:
+    """Extract a FOREIGN KEY definition from a DDL constraint line.
+
+    Args:
+
+        line: A single DDL line starting with ``FOREIGN KEY``.
+
+    Returns:
+
+        Dict with keys ``src_cols``, ``dst_table``, and ``dst_cols``, or ``None`` if the pattern does not match.
+    """
+    match = re.search(
+        r"FOREIGN\s+KEY\s*\(([^)]+)\)\s+REFERENCES\s+(\w+)\s*\(([^)]+)\)",
+        line,
+        re.IGNORECASE,
+    )
+    if match:
+        return {
+            "src_cols": [c.strip().strip("`").strip('"') for c in match.group(1).split(",")],
+            "dst_table": match.group(2).strip("`").strip('"'),
+            "dst_cols": [c.strip().strip("`").strip('"') for c in match.group(3).split(",")],
+        }
+    return None
+
+
+def inject_partition_filters(
+    spark_sql: str,
+    schema: SchemaGraph,
+    intent: RuntimeIntent,
+) -> str:
+    """Ensure partition column predicates from intent filters are in the WHERE clause.
+
+    Builds predicates from filters_param for partition columns, formats them for
+    Spark (backticks), and appends any missing predicates to the WHERE clause.
+
+    Args:
+
+        spark_sql: Spark SQL string (with params already substituted).
+
+        schema: SchemaGraph with partition_columns per table.
+
+        intent: RuntimeIntent with filters_param and param_values.
+
+    Returns:
+
+        Spark SQL with partition predicates ensured in WHERE.
+    """
+    params = dict(flatten_param_values(intent))
+    predicates = _build_partition_predicates(schema, intent, params)
+    if not predicates:
+        return spark_sql
+    combined = " AND ".join(predicates)
+    if _predicate_already_in_sql(spark_sql, combined, predicates):
+        return spark_sql
+    return _append_to_where(spark_sql, combined)
+
+
+def _build_partition_predicates(
+    schema: SchemaGraph,
+    intent: RuntimeIntent,
+    params: dict[str, Any],
+) -> list[str]:
+    """Build Spark-formatted partition predicates from filters_param."""
+    tables = intent.tables or []
+    filters = intent.filters_param or []
+    if not tables or not filters:
+        return []
+
+    grouped: dict[tuple[str, str], list[FilterParam]] = {}
+
+    for table_name in tables:
+        table_meta = schema.tables.get(table_name)
+        if not table_meta or not table_meta.partition_columns:
+            continue
+        part_cols_lower = {c.lower(): c for c in table_meta.partition_columns}
+
+        for fp in filters:
+            col_ref = _get_column_ref(fp.left_expr)
+            if not col_ref:
+                continue
+            table_part, col_part = col_ref
+            col_lower = col_part.lower() if col_part else ""
+            if col_lower not in part_cols_lower:
+                continue
+            actual_col = part_cols_lower[col_lower]
+            table_for_pred = table_part or (tables[0] if tables else "")
+            if table_part and table_part.lower() not in {t.lower() for t in tables}:
+                continue
+            key = (table_for_pred.lower(), actual_col.lower())
+            grouped.setdefault(key, []).append(fp)
+
+    result: list[str] = []
+    for (table_key, col_key), fps in grouped.items():
+        table_name = next(
+            (t for t in tables if t.lower() == table_key), tables[0] if tables else ""
+        )
+        table_meta = schema.tables.get(table_name)
+        col_name = (
+            next(
+                (c for c in table_meta.partition_columns if c.lower() == col_key),
+                col_key,
+            )
+            if table_meta
+            else col_key
+        )
+        pred = _format_grouped_predicate(table_name, col_name, fps, params)
+        if pred:
+            result.append(pred)
+
+    return result
+
+
+def _format_grouped_predicate(
+    table: str,
+    col: str,
+    fps: list[FilterParam],
+    params: dict[str, Any],
+) -> str | None:
+    """Format a grouped predicate for IN (= with OR) or BETWEEN (>= and <=)."""
+    qual = f"`{table}`.`{col}`"
+    ops = {fp.op for fp in fps}
+    if ops <= {"="} and len(fps) > 1:
+        parts = []
+        for fp in fps:
+            val = fp.param_key and params.get(fp.param_key) or fp.raw_value
+            if val is not None:
+                parts.append(_format_partition_literal(val))
+        if parts:
+            return f"{qual} IN ({', '.join(parts)})"
+        return None
+    if ops <= {">=", "<="} and len(fps) == 2:
+        ge = next((f for f in fps if f.op == ">="), None)
+        le = next((f for f in fps if f.op == "<="), None)
+        if ge and le:
+            v1 = ge.param_key and params.get(ge.param_key) or ge.raw_value
+            v2 = le.param_key and params.get(le.param_key) or le.raw_value
+            if v1 is not None and v2 is not None:
+                return f"({qual} >= {_format_partition_literal(v1)} AND {qual} <= {_format_partition_literal(v2)})"
+        return None
+    if len(fps) == 1:
+        return _format_partition_predicate(table, col, fps[0], params)
+    return None
+
+
+def _get_column_ref(expr: NormalizedExpr) -> tuple[str | None, str | None]:
+    """Extract (table, column) from primary_term like 'table.col' or 'col'."""
+    term = (expr.primary_term or "").strip()
+    if not term:
+        return None, None
+    if "." in term:
+        parts = term.rsplit(".", 1)
+        return parts[0].strip() or None, parts[1].strip() or None
+    return None, term
+
+
+def _format_partition_predicate(
+    table: str,
+    col: str,
+    fp: FilterParam,
+    params: dict[str, Any],
+) -> str | None:
+    """Format a single partition predicate for Spark SQL."""
+    qual = f"`{table}`.`{col}`"
+    val = fp.param_key and params.get(fp.param_key)
+    if val is None and fp.raw_value is not None:
+        val = fp.raw_value
+
+    if fp.op == "=":
+        if val is None:
+            return None
+        lit = _format_partition_literal(val)
+        return f"{qual} = {lit}"
+    if fp.op in (">=", "<=", ">", "<"):
+        if val is None:
+            return None
+        lit = _format_partition_literal(val)
+        return f"{qual} {fp.op} {lit}"
+    if fp.op == "in":
+        if val is None:
+            return None
+        if isinstance(val, list):
+            parts = [_format_partition_literal(v) for v in val]
+            return f"{qual} IN ({', '.join(parts)})"
+        lit = _format_partition_literal(val)
+        return f"{qual} IN ({lit})"
+    return None
+
+
+def _format_partition_literal(val: Any) -> str:
+    """Format a value as a Spark SQL literal."""
+    if isinstance(val, str):
+        escaped = val.replace("\\", "\\\\").replace("'", "\\'")
+        return f"'{escaped}'"
+    if isinstance(val, bool):
+        return "TRUE" if val else "FALSE"
+    if isinstance(val, (int, float)):
+        return str(val)
+    return f"'{str(val)}'"
+
+
+def _predicate_already_in_sql(
+    sql: str,
+    combined: str,
+    predicates: list[str],
+) -> bool:
+    """Check if partition predicates are already present in the SQL."""
+    sql_norm = sql.replace(" ", "").replace("\n", " ").lower()
+    for pred in predicates:
+        pred_norm = pred.replace(" ", "").lower()
+        if pred_norm not in sql_norm:
+            return False
+    return True
+
+
+def _append_to_where(sql: str, predicate: str) -> str:
+    """Append predicate to the WHERE clause, creating WHERE if absent."""
+    where_match = re.search(r"\bWHERE\b", sql, re.IGNORECASE)
+    if where_match:
+        insert_pos = where_match.end()
+        next_clause = re.search(
+            r"\b(GROUP\s+BY|ORDER\s+BY|LIMIT|HAVING)\b",
+            sql[insert_pos:],
+            re.IGNORECASE,
+        )
+        if next_clause:
+            end_pos = insert_pos + next_clause.start()
+            clause = sql[insert_pos:end_pos].rstrip()
+            new_clause = clause + " AND " + predicate
+            return sql[:insert_pos] + new_clause + sql[end_pos:]
+        return sql.rstrip() + " AND " + predicate
+
+    from_match = re.search(r"\bFROM\b", sql, re.IGNORECASE)
+    if not from_match:
+        return sql
+    group_match = re.search(
+        r"\bGROUP\s+BY\b",
+        sql[from_match.end() :],
+        re.IGNORECASE,
+    )
+    order_match = re.search(
+        r"\bORDER\s+BY\b",
+        sql[from_match.end() :],
+        re.IGNORECASE,
+    )
+    limit_match = re.search(r"\bLIMIT\b", sql[from_match.end() :], re.IGNORECASE)
+    having_match = re.search(r"\bHAVING\b", sql[from_match.end() :], re.IGNORECASE)
+
+    insert_pos = len(sql)
+    for m in [group_match, order_match, limit_match, having_match]:
+        if m:
+            pos = from_match.end() + m.start()
+            if pos < insert_pos:
+                insert_pos = pos
+
+    before = sql[:insert_pos].rstrip()
+    after = sql[insert_pos:].lstrip()
+    if after:
+        return before + " WHERE " + predicate + " " + after
+    return before + " WHERE " + predicate