aetherdialect 0.1.0__py3-none-any.whl

text2sql/validation_execute.py

@@ -0,0 +1,1092 @@
+"""SQL execution, validation, confidence scoring, and CTE chain validation.
+
+Executes SQL via the configured dialect and validates safety
+(SELECT-only enforcement) and structural integrity of CTE chains
+(dependency order, join paths, table necessity, output column
+types, cardinality).  Also provides the confidence scoring formula
+and the LLM-based rejection classifier used during user feedback
+collection.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .config import EngineConfig, PolicyConfig
+from .schema_profiling import inject_partition_filters
+from .contracts_base import (
+    CteOutputColumnMeta,
+    IntentIssue,
+    IntentValidationResult,
+    SchemaGraph,
+)
+from .contracts_core import (
+    RuntimeCteStep,
+    RuntimeIntent,
+    SelectCol,
+)
+from .core_utils import canonicalize_sql, debug, llm_json, stable_json, substitute_params
+from .dialect import Dialect, get_dialect
+from .validation_agg import (
+    validate_column_types,
+    validate_having_agg_per_role,
+    validate_order_by_agg_per_role,
+    validate_order_by_agg_semantics,
+    validate_pk_fk_aggregation,
+    validate_scalar_expression_semantics,
+    validate_scalar_func_type_semantics,
+    validate_select_agg_per_role,
+    validate_select_agg_semantics,
+    validate_temporal_columns,
+)
+from .validation_schema import (
+    validate_date_diff_units,
+    validate_date_window_units,
+    validate_filter_ops_per_column,
+    validate_filter_value_type_alignment,
+    validate_filters_schema,
+    validate_group_by_cols_schema,
+    validate_having_ops_per_column,
+    validate_having_schema,
+    validate_no_between_ops,
+    validate_null_filters,
+    validate_order_by_cols_schema,
+    validate_select_cols_schema,
+)
+from .validation_semantic import (
+    validate_agg_vs_agg_having,
+    validate_arith_expression_semantics,
+    validate_count_threshold_missing_having,
+    validate_cte_dependency_grains,
+    validate_cte_grain_consistency,
+    validate_expr_vs_expr_filters,
+    validate_filter_expr_types,
+    validate_filter_no_aggregation,
+    validate_for_each_grouping,
+    validate_grain_consistency,
+    validate_grouped_requires_aggregation,
+    validate_having_expr_types,
+    validate_having_requires_aggregation,
+    validate_mixed_aggregation_in_mulgroup,
+    validate_no_nested_aggregation,
+    validate_no_pk_fk_filters,
+    validate_order_by_aggregation_context,
+    validate_order_by_expr_types,
+    validate_question_agg_keyword_coverage,
+    validate_question_aggregation_hint,
+    validate_question_distinct_hint,
+    validate_question_numeric_coverage,
+    validate_question_table_mentions,
+    validate_select_expr_types,
+    validate_select_group_by_membership,
+    validate_semantic_contradictions,
+    validate_threshold_missing_having,
+)
+
+
+def _enforce_select_only(sql: str) -> tuple[bool, str]:
+    """Check if SQL is a safe SELECT-only statement.
+
+    Args:
+
+        sql: The SQL string to check.
+
+    Returns:
+
+        Tuple of ``(ok, reason)`` where ``ok`` is True for safe SELECT statements, and ``reason`` is ``'ok'``, ``'not_select'``, or ``'forbidden_sql'``.
+    """
+    s = sql.lower().strip()
+    if not s.startswith("select"):
+        return False, "not_select"
+    for p in PolicyConfig.FORBIDDEN_SQL:
+        if re.search(p, s, re.IGNORECASE):
+            return False, "forbidden_sql"
+    return True, "ok"
+
+
+def validate_sql(
+    dialect: Dialect,
+    sql: str,
+    engine: Any | None = None,
+    params: dict[str, Any] | None = None,
+) -> tuple[bool, str | None]:
+    """Validate SQL is a safe SELECT and syntactically valid via the dialect AST, optionally checking executability via ``dialect.explain_sql``.
+
+    Args:
+
+        dialect: The database dialect instance for AST validation.
+        sql: The SQL string to validate.
+        engine: Optional SQLAlchemy ``Engine`` for explain-based executability check; when ``None``, only AST validation runs.
+        params: Optional bind-parameter values keyed by placeholder name (for example ``{"p1": "value"}``), forwarded to ``explain_sql`` so the database can resolve ``:p1`` style placeholders.
+
+    Returns:
+
+        Tuple of ``(ok, error_message)`` where ``error_message`` is ``None`` on success.
+    """
+    debug(f"[validation_execute.validate_sql] checking SQL length={len(sql)}")
+    ok, reason = _enforce_select_only(sql)
+    if not ok:
+        debug(f"[validation_execute.validate_sql] enforce_select_only FAILED: {reason}")
+        return False, reason
+    ok, ast_err = dialect.ast_validate(sql)
+    if not ok:
+        debug(f"[validation_execute.validate_sql] AST validation failed: {ast_err}")
+        return False, f"SQL structure error: {ast_err}"
+    debug("[validation_execute.validate_sql] structural validation succeeded")
+    if engine is not None:
+        ok, explain_err = dialect.explain_sql(engine, sql, params)
+        if not ok:
+            debug(f"[validation_execute.validate_sql] explain_sql failed: {explain_err}")
+            return False, explain_err
+        debug("[validation_execute.validate_sql] explain_sql passed")
+    return True, None
+
+
+def get_spark_sql_for_execution(
+    sql_param: str,
+    params: dict[str, Any],
+    schema: SchemaGraph,
+    intent: RuntimeIntent,
+    dialect: Dialect,
+    spark_sql_param_override: str | None = None,
+) -> str:
+    """Compute Spark SQL for execution: qualify, substitute, then partition inject.
+
+    Returns empty string when not Databricks.
+
+    Args:
+
+        sql_param: Parameterized Spark SQL.
+        params: Resolved parameter values.
+        schema: Schema graph for partition metadata.
+        intent: Runtime intent for partition filter injection.
+        dialect: Dialect instance (must have prepare_for_execution).
+        spark_sql_param_override: Optional pre-qualified Spark SQL; when
+            provided, used instead of qualifying sql_param.
+
+    Returns:
+
+        Spark SQL string ready for execution, or empty when not Databricks.
+    """
+    if EngineConfig.TYPE != "databricks":
+        return ""
+    spark_sql = spark_sql_param_override or dialect.prepare_for_execution(sql_param)
+    substituted = substitute_params(spark_sql, params)
+    return inject_partition_filters(substituted, schema, intent)
+
+
+def execute_sql(
+    dialect: Dialect,
+    sql: str,
+    spark_sql_for_execution: str | None = None,
+) -> list[tuple]:
+    """Execute SQL and return result rows via the configured dialect.
+
+    Uses Spark execution for Databricks engine type and SQLAlchemy otherwise.
+    When Databricks and spark_sql_for_execution is provided, uses it directly;
+    otherwise qualifies table references and executes.
+
+    Args:
+
+        dialect: The database dialect instance.
+        sql: The Spark SQL string to execute for Databricks.
+        spark_sql_for_execution: Pre-qualified Spark SQL for Databricks;
+            when provided, used directly without qualification.
+
+    Returns:
+
+        List of result row tuples.
+    """
+    debug("[validation_execute.execute_sql] executing query via dialect")
+    if EngineConfig.TYPE == "databricks":
+        to_run = spark_sql_for_execution if spark_sql_for_execution else sql
+        rows = dialect.execute_sql_spark(
+            to_run, sql_already_spark=bool(spark_sql_for_execution)
+        )
+    else:
+        from sqlalchemy import text
+
+        with dialect.engine.connect() as conn:
+            rows = conn.execute(text(sql)).fetchall()
+    debug(f"[validation_execute.execute_sql] returned {len(rows)} rows")
+    return rows
+
+
+def compute_confidence(
+    best_score: float,
+    score_gap: float,
+    used_new_tables: bool,
+    shape_penalty: float,
+    negative_pen: float,
+    colmap_pen: float,
+    num_cte_pen: float = 0.0,
+) -> float:
+    """Compute overall confidence score for a query result.
+
+    Combines template similarity score, gap to next-best, new-table penalty, shape distance penalty, negative memory penalty, column-map penalty, and CTE count penalty into a single float.
+
+    Args:
+
+        best_score: Similarity score of the best matching template in [0, 1].
+        score_gap: Gap between best and second-best template scores.
+        used_new_tables: Whether the query uses tables not seen in any template.
+        shape_penalty: Shape distance penalty in [0, 1].
+        negative_pen: Accumulated negative memory penalty in [0, 1].
+        colmap_pen: Column-map rejection penalty in [0, 1].
+        num_cte_pen: Optional CTE-count penalty in [0, 1].
+
+    Returns:
+
+        Confidence float clamped to [0, 1].
+    """
+    debug("[validation_execute.compute_confidence] inputs:")
+    debug(
+        f"[validation_execute.compute_confidence] used_new_tables={used_new_tables}, shape_penalty={shape_penalty:.3f}, num_cte_pen={num_cte_pen:.3f}"
+    )
+    c = 0.0
+    c += 0.62 * best_score
+    debug(f"[validation_execute.compute_confidence] +{0.62 * best_score:.3f} from best_score")
+    gap_contrib = 0.18 * max(0.0, min(1.0, score_gap * 2.0))
+    c += gap_contrib
+    debug(f"[validation_execute.compute_confidence] +{gap_contrib:.3f} from score_gap")
+    if used_new_tables:
+        c -= 0.12
+        debug("[validation_execute.compute_confidence] -0.120 from used_new_tables")
+    shape_deduct = 0.18 * shape_penalty
+    c -= shape_deduct
+    debug(f"[validation_execute.compute_confidence] -{shape_deduct:.3f} from shape_penalty")
+    neg_deduct = 0.35 * min(1.0, max(0.0, negative_pen))
+    c -= neg_deduct
+    debug(f"[validation_execute.compute_confidence] -{neg_deduct:.3f} from negative_pen")
+    colmap_deduct = 0.20 * min(1.0, max(0.0, colmap_pen))
+    c -= colmap_deduct
+    debug(f"[validation_execute.compute_confidence] -{colmap_deduct:.3f} from colmap_pen")
+    cte_deduct = 0.10 * min(1.0, max(0.0, num_cte_pen))
+    c -= cte_deduct
+    debug(f"[validation_execute.compute_confidence] -{cte_deduct:.3f} from num_cte_pen")
+    result = max(0.0, min(1.0, c))
+    if best_score == 0.0:
+        cold_start_floor = max(0.0, 0.50 - neg_deduct - colmap_deduct - cte_deduct)
+        if result < cold_start_floor:
+            debug(
+                f"[validation_execute.compute_confidence] cold-start floor applied: {result:.3f} → {cold_start_floor:.3f}"
+            )
+            result = cold_start_floor
+    debug(f"[validation_execute.compute_confidence] FINAL confidence={result:.3f}")
+    return result
+
+
+def _keyword_classify_rejection(reason_lower: str) -> str | None:
+    """Fast deterministic pre-classifier for common rejection phrases.
+
+    Scans the lowered rejection reason for unambiguous keyword patterns and returns a canonical category string if matched, or ``None`` to fall through to the LLM classifier.
+    """
+    _KEYWORD_MAP: list[tuple[list[str], str]] = [
+        (
+            ["wrong table", "missing table", "incorrect table", "should use table"],
+            "wrong_tables",
+        ),
+        (
+            [
+                "wrong join",
+                "relationship wrong",
+                "join issue",
+                "self-join",
+                "self join",
+                "self_join",
+            ],
+            "wrong_join",
+        ),
+        (
+            [
+                "wrong aggregation",
+                "wrong grouping",
+                "aggregation or grouping",
+                "group by",
+            ],
+            "wrong_aggregation_or_grouping",
+        ),
+        (
+            ["wrong column", "missing column", "columns selected", "wrong columns"],
+            "wrong_columns_selected",
+        ),
+        (
+            ["wrong filter", "wrong condition", "wrong having", "wrong threshold"],
+            "wrong_filters_or_having",
+        ),
+        (["wrong intent", "misunderstood", "question wrong"], "wrong_intent"),
+        (["too many rows", "too many results"], "too_many_rows"),
+        (["too few rows", "too few results", "missing rows"], "too_few_rows"),
+        (
+            [
+                "invalid structure",
+                "query format",
+                "sql structure",
+                "wrong query structure",
+                "wrong structure",
+            ],
+            "invalid_structure",
+        ),
+    ]
+    for phrases, category in _KEYWORD_MAP:
+        if any(p in reason_lower for p in phrases):
+            return category
+    return None
+
+
+def llm_classify_rejection(q_norm: str, intent: RuntimeIntent, sql: str, ux: str, user_reason: str) -> dict[str, str]:
+    """Use an LLM to classify a user's rejection reason into a canonical category.
+
+    A fast keyword pre-classifier is tried first and the LLM is only invoked when no deterministic match is found.
+
+    Args:
+
+        q_norm: The normalised user question.
+        intent: The ``RuntimeIntent`` for the rejected query.
+        sql: The rejected SQL.
+        ux: The UX explanation shown to the user (unused in the prompt; reserved).
+        user_reason: The free-text reason provided by the user.
+
+    Returns:
+
+        Dict with ``"category"`` (one of ``PolicyConfig.REJECT_CATEGORIES``) and ``"normalized_reason"`` (concise 5–10 word summary).
+    """
+    debug("[validation_execute.llm_classify_rejection] classifying user rejection")
+    debug(f"[validation_execute.llm_classify_rejection] user_reason='{user_reason or ''}'")
+
+    reason_lower = (user_reason or "").lower()
+    keyword_cat = _keyword_classify_rejection(reason_lower)
+    if keyword_cat is not None:
+        debug(f"[validation_execute.llm_classify_rejection] keyword hit -> {keyword_cat}")
+        return {"category": keyword_cat, "normalized_reason": user_reason.strip()}
+
+    system = "You are a deterministic rejection classifier. Output ONLY valid JSON. Identical inputs must produce identical outputs."
+    user = stable_json(
+        {
+            "task": "Classify the user's rejection reason into exactly one category.",
+            "categories": PolicyConfig.REJECT_CATEGORIES,
+            "category_rules": {
+                "wrong_intent": "User says the question was misunderstood, intent is wrong.",
+                "wrong_tables": "User says wrong table, missing table, incorrect table, or should use different table.",
+                "wrong_join": "User says tables connected wrong, relationship wrong, or join issue.",
+                "wrong_filters_or_having": "User says wrong condition, filter, WHERE clause, HAVING clause, or threshold issue.",
+                "wrong_aggregation_or_grouping": "User says wrong count/sum/avg, grouping wrong, or should group differently.",
+                "wrong_columns_selected": "User says wrong column, missing column, or column from wrong table.",
+                "too_many_rows": "User says too many results, needs to be filtered more.",
+                "too_few_rows": "User says results are incomplete, missing rows, or too few results.",
+                "invalid_structure": "User says SQL structure is wrong, query format is bad.",
+                "other": "Cannot classify into above categories.",
+            },
+            "classification_rules": [
+                "Read the user_reason carefully.",
+                "Match keywords: 'table'/'tables' -> wrong_tables, 'column' -> wrong_columns_selected, 'join' -> wrong_join, 'filter'/'condition'/'value'/'having'/'threshold' -> wrong_filters_or_having.",
+                "If multiple categories apply, choose the most specific one.",
+                "Only use 'other' if no category matches at all.",
+            ],
+            "output_format": {
+                "category": "one of the category keys",
+                "normalized_reason": "concise 5-10 word summary of the issue",
+            },
+            "context": {
+                "question": q_norm,
+                "tables_used": intent.tables or [],
+                "user_reason": user_reason,
+            },
+        }
+    )
+    parsed = llm_json(system, user, retries=1, task="intent")
+    cat = parsed.get("category") if isinstance(parsed.get("category"), str) else "other"
+    if cat not in PolicyConfig.REJECT_CATEGORIES:
+        cat = "other"
+    norm = parsed.get("normalized_reason") if isinstance(parsed.get("normalized_reason"), str) else user_reason
+    debug(f"[validation_execute.llm_classify_rejection] classified: category={cat}")
+    return {"category": cat, "normalized_reason": norm.strip()}
+
+
+def _validate_main_query_cte_usage(intent: RuntimeIntent, cte_outputs: dict[str, list[str]]) -> list[IntentIssue]:
+    """Validate that the main query references CTE outputs correctly.
+
+    Checks for unreferenced CTEs, main query select columns not present
+    in their referenced CTE outputs, column_map references missing from
+    CTE outputs, and filter column references missing from CTE outputs.
+
+    Args:     intent: The main RuntimeIntent.     cte_outputs: Dict of
+    CTE name -> output column list from ``validate_cte_chain``.
+
+    Returns:     List of IntentIssue objects.
+    """
+    issues = []
+    debug(
+        f"[validation_execute.validate_main_query_cte_usage] checking main query uses CTEs: {list(cte_outputs.keys())}"
+    )
+    if not cte_outputs:
+        return issues
+    intent_tables = set(t.lower() for t in (intent.tables or []))
+    cte_names = set(c.lower() for c in cte_outputs.keys())
+    unreferenced_ctes = cte_names - intent_tables
+    if unreferenced_ctes:
+        issues.append(
+            IntentIssue(
+                issue_id=f"cte_unreferenced_{','.join(sorted(unreferenced_ctes))}",
+                category="cte_usage",
+                severity="warning",
+                message=f"CTEs defined but not used in main query: {sorted(unreferenced_ctes)}",
+                context={"unreferenced": list(unreferenced_ctes)},
+            )
+        )
+        debug(f"[validation_execute.validate_main_query_cte_usage] unreferenced CTEs: {unreferenced_ctes}")
+    for sc in intent.select_cols or []:
+        col_expr = sc.expr.primary_column
+        if not col_expr or "." not in col_expr:
+            continue
+        table_ref, col_name = col_expr.rsplit(".", 1)
+        if table_ref.lower() in cte_names:
+            cte_cols = cte_outputs.get(table_ref, [])
+            if col_name.lower() not in {c.lower() for c in cte_cols}:
+                issues.append(
+                    IntentIssue(
+                        issue_id=f"main_col_not_in_cte_{table_ref}_{col_name}",
+                        category="cte_column_reference",
+                        severity="error",
+                        message=f"Main query references column '{col_name}' not in CTE '{table_ref}'",
+                        context={
+                            "cte": table_ref,
+                            "column": col_name,
+                            "available": cte_cols,
+                        },
+                    )
+                )
+                debug(f"[validation_execute.validate_main_query_cte_usage] column {col_name} not in CTE {table_ref}")
+    column_map = intent.column_map or {}
+    for col_name, source in column_map.items():
+        if source.lower() in cte_names:
+            cte_cols = cte_outputs.get(source, [])
+            if col_name.lower() not in {c.lower() for c in cte_cols}:
+                issues.append(
+                    IntentIssue(
+                        issue_id=f"main_colmap_not_in_cte_{source}_{col_name}",
+                        category="cte_column_reference",
+                        severity="error",
+                        message=f"column_map references '{col_name}' from CTE '{source}' but column not in CTE outputs",
+                        context={
+                            "cte": source,
+                            "column": col_name,
+                            "available": cte_cols,
+                        },
+                    )
+                )
+                debug(f"[validation_execute.validate_main_query_cte_usage] column_map {col_name} not in CTE {source}")
+    for fp in intent.filters_param or []:
+        col_expr = fp.left_expr.primary_column
+        if not col_expr or "." not in col_expr:
+            continue
+        table_ref, col_name = col_expr.rsplit(".", 1)
+        if table_ref.lower() in cte_names:
+            cte_cols = cte_outputs.get(table_ref, [])
+            if col_name.lower() not in {c.lower() for c in cte_cols}:
+                issues.append(
+                    IntentIssue(
+                        issue_id=f"main_filter_not_in_cte_{table_ref}_{col_name}",
+                        category="cte_column_reference",
+                        severity="error",
+                        message=f"Main query filter references column '{col_name}' not in CTE '{table_ref}'",
+                        context={
+                            "cte": table_ref,
+                            "column": col_name,
+                            "available": cte_cols,
+                        },
+                    )
+                )
+                debug(f"[validation_execute.validate_main_query_cte_usage] filter {col_name} not in CTE {table_ref}")
+    if issues:
+        debug(f"[validation_execute.validate_main_query_cte_usage] found {len(issues)} issues")
+    else:
+        debug("[validation_execute.validate_main_query_cte_usage] all CTE references valid")
+    return issues
+
+
+def _validate_cte_output_types(cte_steps: list[RuntimeCteStep], schema: SchemaGraph) -> list[IntentIssue]:
+    """Validate that CTE output column types are consistent with
+    aggregation usage.
+
+    Warns when SUM or AVG is applied to a column whose inferred type is
+    not numeric.
+
+    Args:     cte_steps: Ordered list of RuntimeCteStep objects.
+    schema: The SchemaGraph for column type lookup.
+
+    Returns:     List of IntentIssue objects (severity ``'warning'``)
+    for type mismatches.
+    """
+    issues = []
+    cte_output_types: dict[str, dict[str, str]] = {}
+    debug(f"[validation_execute.validate_cte_output_types] validating {len(cte_steps)} CTE output types")
+    for cte in cte_steps:
+        cte_name = cte.cte_name
+        output_cols = cte.output_columns or []
+        select_cols = cte.select_cols or []
+        col_types: dict[str, str] = {}
+        for col in output_cols:
+            if "." in col:
+                table, col_name = col.rsplit(".", 1)
+            else:
+                col_name = col
+                table = cte.tables[0] if cte.tables else None
+            for sc in select_cols:
+                if sc.is_aggregated:
+                    term = sc.expr.primary_term
+                    agg_name = term.split("(")[0].lower() if "(" in term else ""
+                    alias = f"{agg_name}_{sc.expr.primary_column.replace('.', '_')}"
+                    if col_name.lower() == alias.lower() or col_name.lower().startswith(f"{agg_name}_"):
+                        col_types[col_name] = "numeric"
+                        break
+            else:
+                if table and table in schema.tables:
+                    schema_col = schema.tables[table].columns.get(col_name)
+                    if schema_col and schema_col.value_type:
+                        col_types[col_name] = schema_col.value_type
+                elif table and table in cte_output_types:
+                    dep_types = cte_output_types[table]
+                    if col_name in dep_types:
+                        col_types[col_name] = dep_types[col_name]
+        cte_output_types[cte_name] = col_types
+    for cte in cte_steps:
+        cte_name = cte.cte_name
+        select_cols = cte.select_cols or []
+        numeric_aggs = {"sum", "avg"}
+        for sc in select_cols:
+            if not sc.is_aggregated:
+                continue
+            term = sc.expr.primary_term
+            agg_func = term.split("(")[0].lower() if "(" in term else ""
+            if agg_func not in numeric_aggs:
+                continue
+            col_expr = sc.expr.primary_column
+            if not col_expr:
+                continue
+            if "." in col_expr:
+                table, col_name = col_expr.rsplit(".", 1)
+            else:
+                col_name = col_expr
+                table = cte.tables[0] if cte.tables else None
+            if table in cte_output_types:
+                col_type = cte_output_types[table].get(col_name, "")
+                if col_type and col_type not in ("integer", "number"):
+                    issues.append(
+                        IntentIssue(
+                            issue_id=f"cte_agg_type_mismatch_{cte_name}_{agg_func}_{col_name}",
+                            category="cte_type_consistency",
+                            severity="warning",
+                            message=f"CTE '{cte_name}' applies {agg_func.upper()} to non-numeric column '{col_name}' (type: {col_type})",
+                            context={
+                                "cte_name": cte_name,
+                                "agg": agg_func,
+                                "column": col_name,
+                                "type": col_type,
+                            },
+                        )
+                    )
+                    debug(
+                        f"[validation_execute.validate_cte_output_types] type mismatch: {agg_func} on {col_name}({col_type})"
+                    )
+    if issues:
+        debug(f"[validation_execute.validate_cte_output_types] found {len(issues)} type issues")
+    else:
+        debug("[validation_execute.validate_cte_output_types] all CTE output types valid")
+    return issues
+
+
+def _validate_cte_cardinality(cte_steps: list[RuntimeCteStep]) -> list[IntentIssue]:
+    """Validate CTE cardinality expectations are internally consistent.
+
+    Warns when scalar-grain CTEs have ``expected_rows != 'one'``, LIMIT
+    1 CTEs have ``expected_rows != 'one'``, and notes when a many-row
+    CTE depends on a single-row CTE.
+
+    Args:     cte_steps: Ordered list of RuntimeCteStep objects.
+
+    Returns:     List of IntentIssue objects.
+    """
+    issues = []
+    cte_expected_rows: dict[str, str] = {}
+    debug(f"[validation_execute.validate_cte_cardinality] validating {len(cte_steps)} CTE cardinalities")
+    for cte in cte_steps:
+        cte_expected_rows[cte.cte_name] = cte.expected_rows or "many"
+    for cte in cte_steps:
+        cte_name = cte.cte_name
+        expected = cte.expected_rows or "many"
+        grain = cte.grain
+        if grain == "scalar" and expected != "one":
+            issues.append(
+                IntentIssue(
+                    issue_id=f"cte_scalar_cardinality_{cte_name}",
+                    category="cte_cardinality",
+                    severity="warning",
+                    message=f"CTE '{cte_name}' has scalar grain but expected_rows='{expected}'",
+                    context={
+                        "cte_name": cte_name,
+                        "grain": grain,
+                        "expected_rows": expected,
+                    },
+                )
+            )
+            debug(f"[validation_execute.validate_cte_cardinality] scalar CTE with expected_rows={expected}")
+        if cte.limit == 1 and expected != "one":
+            issues.append(
+                IntentIssue(
+                    issue_id=f"cte_limit1_cardinality_{cte_name}",
+                    category="cte_cardinality",
+                    severity="warning",
+                    message=f"CTE '{cte_name}' has LIMIT 1 but expected_rows='{expected}'",
+                    context={
+                        "cte_name": cte_name,
+                        "limit": 1,
+                        "expected_rows": expected,
+                    },
+                )
+            )
+            debug(f"[validation_execute.validate_cte_cardinality] LIMIT 1 CTE with expected_rows={expected}")
+        for table in cte.tables or []:
+            if table in cte_expected_rows:
+                dep_expected = cte_expected_rows[table]
+                if expected in {"few", "many"} and dep_expected == "one":
+                    issues.append(
+                        IntentIssue(
+                            issue_id=f"cte_cardinality_expansion_{cte_name}_{table}",
+                            category="cte_cardinality",
+                            severity="info",
+                            message=f"CTE '{cte_name}' (expected_rows={expected}) depends on single-row CTE '{table}'",
+                            context={
+                                "cte_name": cte_name,
+                                "expected": expected,
+                                "dep_cte": table,
+                                "dep_expected": dep_expected,
+                            },
+                        )
+                    )
+                    debug(
+                        f"[validation_execute.validate_cte_cardinality] cardinality note: {cte_name}({expected}) <- {table}({dep_expected})"
+                    )
+    if issues:
+        debug(f"[validation_execute.validate_cte_cardinality] found {len(issues)} cardinality issues")
+    else:
+        debug("[validation_execute.validate_cte_cardinality] all cardinalities valid")
+    return issues
+
+
+def validate_semantics(intent: RuntimeIntent, schema: SchemaGraph) -> IntentValidationResult:
+    """Run the full semantic validation suite against a
+    ``RuntimeIntent``.
+
+    Applies every schema-level and semantic-level validation function to
+    the main query and each CTE step, then aggregates CTE grain
+    compatibility checks across the full CTE chain.
+
+    Args:     intent: The resolved runtime intent to validate.
+    schema: Schema graph providing table, column, and relationship
+    metadata.
+
+    Returns:     An ``IntentValidationResult`` containing all
+    ``IntentIssue`` entries     found across the main query and all CTE
+    steps.
+    """
+    all_issues: list[IntentIssue] = []
+    debug("[validation_semantic.validate_semantics] running semantic validation suite")
+    if not intent.tables:
+        all_issues.append(
+            IntentIssue(
+                issue_id="tables_empty",
+                category="structural",
+                severity="error",
+                message="Intent has no tables specified",
+                context={},
+            )
+        )
+        debug("[validation_semantic.validate_semantics] tables empty")
+    cte_steps = intent.cte_steps or []
+    cte_outputs: dict[str, dict[str, CteOutputColumnMeta]] = {}
+    for cte in cte_steps:
+        if not cte.cte_name:
+            all_issues.append(
+                IntentIssue(
+                    issue_id="cte_name_empty",
+                    category="cte_structure",
+                    severity="error",
+                    message="CTE step has no name specified",
+                    context={},
+                )
+            )
+        if not cte.output_columns:
+            all_issues.append(
+                IntentIssue(
+                    issue_id=f"cte_output_columns_empty_{cte.cte_name}",
+                    category="cte_structure",
+                    severity="error",
+                    message=f"CTE '{cte.cte_name}' has no output columns specified",
+                    context={"cte_name": cte.cte_name},
+                )
+            )
+        else:
+            cte_outputs[cte.cte_name] = dict(cte.output_column_metadata or {})
+    if cte_steps:
+        cte_names = [c.cte_name for c in cte_steps]
+        if len(cte_names) != len(set(cte_names)):
+            duplicates = [n for n in cte_names if cte_names.count(n) > 1]
+            all_issues.append(
+                IntentIssue(
+                    issue_id=f"cte_duplicate_names_{','.join(sorted(set(duplicates)))}",
+                    category="cte_structure",
+                    severity="error",
+                    message=f"Duplicate CTE names: {sorted(set(duplicates))}",
+                    context={"duplicates": list(set(duplicates))},
+                )
+            )
+        for i, cte in enumerate(cte_steps):
+            for table in cte.tables or []:
+                if table.lower() in {n.lower() for n in cte_names[i + 1 :]}:
+                    all_issues.append(
+                        IntentIssue(
+                            issue_id=f"cte_forward_ref_{cte.cte_name}_{table}",
+                            category="cte_structure",
+                            severity="error",
+                            message=f"CTE '{cte.cte_name}' forward-references CTE '{table}' defined later",
+                            context={"cte_name": cte.cte_name, "forward_ref": table},
+                        )
+                    )
+        known_tables = set(schema.tables.keys())
+        for i, cte in enumerate(cte_steps):
+            available = known_tables | {c.cte_name for c in cte_steps[:i]}
+            for table in cte.tables or []:
+                if table.lower() not in {t.lower() for t in available}:
+                    all_issues.append(
+                        IntentIssue(
+                            issue_id=f"cte_unknown_table_{cte.cte_name}_{table}",
+                            category="cte_table_reference",
+                            severity="error",
+                            message=f"CTE '{cte.cte_name}' references unknown table '{table}'",
+                            context={"cte_name": cte.cte_name, "table": table},
+                        )
+                    )
+        cte_outputs_list = {c.cte_name: (c.output_columns or []) for c in cte_steps}
+        all_issues.extend(_validate_main_query_cte_usage(intent, cte_outputs_list))
+        all_issues.extend(_validate_cte_output_types(cte_steps, schema))
+        all_issues.extend(_validate_cte_cardinality(cte_steps))
+    allowed_tables = set(intent.tables or [])
+    all_issues.extend(
+        validate_select_cols_schema(intent.select_cols or [], schema, allowed_tables, cte_outputs, "main query")
+    )
+    all_issues.extend(
+        validate_order_by_cols_schema(
+            intent.order_by_cols or [],
+            schema,
+            allowed_tables,
+            cte_outputs,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_group_by_cols_schema(
+            intent.group_by_cols or [],
+            schema,
+            allowed_tables,
+            cte_outputs,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_filters_schema(
+            intent.filters_param or [],
+            schema,
+            allowed_tables,
+            cte_outputs,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_having_schema(intent.having_param or [], schema, allowed_tables, cte_outputs, "main query")
+    )
+    all_issues.extend(validate_filter_ops_per_column(intent.filters_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_having_agg_per_role(intent.having_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_having_ops_per_column(intent.having_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_select_agg_per_role(intent.select_cols or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_select_agg_semantics(intent.select_cols or [], schema, "main query"))
+    all_issues.extend(validate_order_by_agg_per_role(intent.order_by_cols or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_order_by_agg_semantics(intent.order_by_cols or [], schema, "main query"))
+    all_issues.extend(
+        validate_scalar_func_type_semantics(intent.select_cols or [], intent.order_by_cols or [], schema, "main query")
+    )
+    all_issues.extend(validate_null_filters(intent.filters_param or [], cte_steps, "main query"))
+    all_issues.extend(validate_date_window_units(intent.filters_param or [], cte_steps, "main query"))
+    all_issues.extend(validate_date_diff_units(intent.filters_param or [], cte_steps, "main query"))
+    all_issues.extend(validate_column_types(intent.select_cols or [], schema, "main query"))
+    all_issues.extend(
+        validate_filter_value_type_alignment(intent.filters_param or [], schema, "main query", cte_outputs)
+    )
+    all_issues.extend(validate_no_between_ops(intent.filters_param or [], intent.having_param or [], "main query"))
+    all_issues.extend(
+        validate_grain_consistency(
+            intent.grain,
+            intent.select_cols or [],
+            intent.group_by_cols or [],
+            intent.having_param or [],
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_grouped_requires_aggregation(
+            intent.grain,
+            intent.select_cols or [],
+            intent.group_by_cols or [],
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_semantic_contradictions(
+            intent.select_cols or [],
+            intent.natural_language,
+            intent.grain,
+            intent.expected_rows,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_question_aggregation_hint(
+            intent.natural_language,
+            intent.select_cols or [],
+            intent.having_param or [],
+            intent.grain,
+            "main query",
+            filters_param=intent.filters_param or [],
+            schema=schema,
+        )
+    )
+    all_issues.extend(
+        validate_threshold_missing_having(
+            intent.natural_language,
+            intent.select_cols or [],
+            intent.having_param or [],
+            intent.grain,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_count_threshold_missing_having(
+            intent.natural_language,
+            intent.tables or [],
+            intent.having_param or [],
+            schema,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_question_numeric_coverage(
+            intent.natural_language,
+            intent.filters_param or [],
+            intent.having_param or [],
+            intent.limit,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_question_distinct_hint(
+            intent.natural_language,
+            intent.select_cols or [],
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_no_pk_fk_filters(
+            intent.filters_param or [],
+            schema,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_question_table_mentions(
+            intent.natural_language,
+            intent.tables or [],
+            schema,
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_question_agg_keyword_coverage(
+            intent.natural_language,
+            intent.select_cols or [],
+            intent.having_param or [],
+            "main query",
+        )
+    )
+    _has_agg = any(sc.is_aggregated for sc in (intent.select_cols or [])) or bool(intent.having_param)
+    all_issues.extend(
+        validate_for_each_grouping(
+            intent.natural_language,
+            intent.group_by_cols or [],
+            schema,
+            _has_agg,
+            "main query",
+        )
+    )
+    all_issues.extend(validate_expr_vs_expr_filters(intent.filters_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_agg_vs_agg_having(intent.having_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_scalar_expression_semantics(intent.select_cols or [], schema, "main query"))
+    all_issues.extend(
+        validate_arith_expression_semantics(
+            intent.filters_param or [],
+            intent.having_param or [],
+            schema,
+            cte_outputs,
+            "main query",
+        )
+    )
+    all_issues.extend(validate_temporal_columns(intent.select_cols or [], schema, "main query"))
+    all_issues.extend(validate_pk_fk_aggregation(intent.select_cols or [], schema, "main query"))
+    all_issues.extend(validate_select_expr_types(intent.select_cols or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_order_by_expr_types(intent.order_by_cols or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_filter_expr_types(intent.filters_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_having_expr_types(intent.having_param or [], schema, cte_outputs, "main query"))
+    all_issues.extend(validate_filter_no_aggregation(intent.filters_param or [], "main query"))
+    all_issues.extend(validate_having_requires_aggregation(intent.having_param or [], "main query"))
+    all_issues.extend(
+        validate_no_nested_aggregation(
+            intent.select_cols or [],
+            intent.order_by_cols or [],
+            intent.filters_param or [],
+            intent.having_param or [],
+            "main query",
+        )
+    )
+    all_issues.extend(
+        validate_mixed_aggregation_in_mulgroup(
+            intent.select_cols or [],
+            intent.order_by_cols or [],
+            intent.filters_param or [],
+            intent.having_param or [],
+            "main query",
+        )
+    )
+    all_issues.extend(validate_order_by_aggregation_context(intent.order_by_cols or [], intent.grain, "main query"))
+    all_issues.extend(
+        validate_select_group_by_membership(
+            intent.select_cols or [],
+            intent.group_by_cols or [],
+            intent.grain,
+            "main query",
+        )
+    )
+    for cte in cte_steps:
+        cte_context = f"CTE '{cte.cte_name}'"
+        cte_allowed = set(cte.tables or [])
+        all_issues.extend(
+            validate_select_cols_schema(cte.select_cols or [], schema, cte_allowed, cte_outputs, cte_context)
+        )
+        all_issues.extend(
+            validate_order_by_cols_schema(cte.order_by_cols or [], schema, cte_allowed, cte_outputs, cte_context)
+        )
+        all_issues.extend(
+            validate_group_by_cols_schema(cte.group_by_cols or [], schema, cte_allowed, cte_outputs, cte_context)
+        )
+        all_issues.extend(
+            validate_filters_schema(cte.filters_param or [], schema, cte_allowed, cte_outputs, cte_context)
+        )
+        all_issues.extend(validate_having_schema(cte.having_param or [], schema, cte_allowed, cte_outputs, cte_context))
+        all_issues.extend(validate_filter_ops_per_column(cte.filters_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_date_window_units(cte.filters_param or [], [], cte_context))
+        all_issues.extend(validate_date_diff_units(cte.filters_param or [], [], cte_context))
+        all_issues.extend(validate_having_agg_per_role(cte.having_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_having_ops_per_column(cte.having_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_select_agg_per_role(cte.select_cols or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_select_agg_semantics(cte.select_cols or [], schema, cte_context))
+        all_issues.extend(validate_order_by_agg_per_role(cte.order_by_cols or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_order_by_agg_semantics(cte.order_by_cols or [], schema, cte_context))
+        all_issues.extend(
+            validate_scalar_func_type_semantics(cte.select_cols or [], cte.order_by_cols or [], schema, cte_context)
+        )
+        all_issues.extend(validate_column_types(cte.select_cols or [], schema, cte_context))
+        all_issues.extend(
+            validate_filter_value_type_alignment(cte.filters_param or [], schema, cte_context, cte_outputs)
+        )
+        all_issues.extend(validate_no_between_ops(cte.filters_param or [], cte.having_param or [], cte_context))
+        all_issues.extend(validate_cte_grain_consistency(cte, cte_context))
+        all_issues.extend(
+            validate_grouped_requires_aggregation(
+                cte.grain, cte.select_cols or [], cte.group_by_cols or [], cte_context
+            )
+        )
+        all_issues.extend(
+            validate_semantic_contradictions(
+                cte.select_cols or [],
+                cte.description or "",
+                cte.grain,
+                cte.expected_rows,
+                cte_context,
+            )
+        )
+        all_issues.extend(
+            validate_question_aggregation_hint(
+                cte.description or "",
+                cte.select_cols or [],
+                cte.having_param or [],
+                cte.grain,
+                cte_context,
+                filters_param=cte.filters_param or [],
+                schema=schema,
+            )
+        )
+        all_issues.extend(validate_expr_vs_expr_filters(cte.filters_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_agg_vs_agg_having(cte.having_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_scalar_expression_semantics(cte.select_cols or [], schema, cte_context))
+        all_issues.extend(
+            validate_arith_expression_semantics(
+                cte.filters_param or [],
+                cte.having_param or [],
+                schema,
+                cte_outputs,
+                cte_context,
+            )
+        )
+        all_issues.extend(validate_temporal_columns(cte.select_cols or [], schema, cte_context))
+        all_issues.extend(validate_pk_fk_aggregation(cte.select_cols or [], schema, cte_context))
+        all_issues.extend(validate_select_expr_types(cte.select_cols or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_order_by_expr_types(cte.order_by_cols or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_filter_expr_types(cte.filters_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_having_expr_types(cte.having_param or [], schema, cte_outputs, cte_context))
+        all_issues.extend(validate_filter_no_aggregation(cte.filters_param or [], cte_context))
+        all_issues.extend(validate_having_requires_aggregation(cte.having_param or [], cte_context))
+        all_issues.extend(
+            validate_no_nested_aggregation(
+                cte.select_cols or [],
+                cte.order_by_cols or [],
+                cte.filters_param or [],
+                cte.having_param or [],
+                cte_context,
+            )
+        )
+        all_issues.extend(
+            validate_mixed_aggregation_in_mulgroup(
+                cte.select_cols or [],
+                cte.order_by_cols or [],
+                cte.filters_param or [],
+                cte.having_param or [],
+                cte_context,
+            )
+        )
+        all_issues.extend(validate_order_by_aggregation_context(cte.order_by_cols or [], cte.grain, cte_context))
+        all_issues.extend(
+            validate_select_group_by_membership(cte.select_cols or [], cte.group_by_cols or [], cte.grain, cte_context)
+        )
+        if cte.output_columns:
+            cte_outputs[cte.cte_name] = dict(cte.output_column_metadata or {})
+    if cte_steps:
+        all_issues.extend(validate_cte_dependency_grains(cte_steps, intent.grain))
+    if all_issues:
+        debug(f"[validation_semantic.validate_semantics] found {len(all_issues)} total issues")
+    else:
+        debug("[validation_execute.validate_semantics] all validations passed")
+    for idx, iss in enumerate(all_issues):
+        debug(
+            f"[validation_execute.validate_semantics] issue[{idx}]: "
+            f"{iss.issue_id} | {iss.category} | {iss.severity} | {iss.message} | context={iss.context}"
+        )
+    return IntentValidationResult(issues=all_issues)