aetherdialect 0.1.0__py3-none-any.whl

text2sql/utils.py

@@ -0,0 +1,973 @@
+"""Shared utility functions for intent normalisation, question validation, fingerprint generation, and natural language question generation.
+
+Provides SHA-256 intent fingerprinting over canonical stable JSON, fuzzy question matching with Levenshtein distance, and filter/HAVING/CTE normalisation helpers used by both the main pipeline and the query simulator. Also exposes the ``generate_question`` LLM helper used by QSim and Simulator to produce human-readable questions from structured query intent.
+"""
+
+from __future__ import annotations
+
+import random
+import re
+from typing import Any
+
+from .config import (
+    AGG_PATTERN,
+    QUESTION_STARTS_AGG,
+    QUESTION_STARTS_GROUP,
+    QUESTION_STARTS_LIST,
+    VALID_AGGREGATION_FUNCTIONS,
+    VALID_EXPECTED_ROWS,
+    VALID_FILTER_OPS,
+    VALID_GRAINS,
+    VALID_HAVING_OPS,
+    PolicyConfig,
+)
+from .contracts_base import CteOutputColumnMeta, SchemaGraph, SQLShape
+from .contracts_core import (
+    FilterParam,
+    HavingParam,
+    NormalizedExpr,
+    OrderByCol,
+    RuntimeCteStep,
+    RuntimeIntent,
+    SelectCol,
+)
+from .core_utils import debug, llm_json, normalize_question, sha256, stable_json
+from .intent_resolve import sort_filters, sort_having
+
+_QUESTION_VALIDATION_SYSTEM = (
+    "You decide if user input is a database query request or not.\n\n"
+    "A database query request asks to retrieve, count, sum, filter, sort, or analyze data.\n"
+    "When in doubt, treat it as a valid query request.\n\n"
+    "Only mark as invalid if it is clearly:\n"
+    '- Chitchat (e.g. "hello", "thanks", "who are you")\n'
+    '- A help or instruction request (e.g. "how do I query", "explain SQL")\n'
+    '- General knowledge unrelated to data (e.g. "what is the capital of France")\n\n'
+    "Respond with JSON containing exactly three fields:\n"
+    '- "valid_database_question": "yes" or "no"\n'
+    '- "query_type": "allowed" if read/SELECT operation, "restricted" if write/destructive operation (UPDATE/DELETE/INSERT/DROP/ALTER/CREATE/TRUNCATE), "unspecified" if unclear\n'
+    '- "corrected": the input with spelling typos fixed only. Do NOT remove, reorder, or rephrase any words. If no corrections needed, return the original text unchanged.\n\n'
+    "Respond ONLY with valid JSON, no explanation."
+)
+
+
+def validate_question(question: str) -> tuple[bool, str, str]:
+    """Validate a question and return typo-corrected text via LLM.
+
+    Args:
+
+        question: The raw user input string.
+
+    Returns:
+
+        Tuple of ``(is_valid, query_type, corrected_text)`` where ``is_valid`` is True for valid database questions, ``query_type`` is one of ``'allowed'``, ``'restricted'``, or ``'unspecified'``, and ``corrected_text`` is the typo-corrected version of the original input.
+    """
+    result = llm_json(_QUESTION_VALIDATION_SYSTEM, question, task="default")
+    if not result:
+        return False, "invalid", question
+    valid = result.get("valid_database_question", "").lower() == "yes"
+    query_type = result.get("query_type", "unspecified").lower()
+    corrected = result.get("corrected", question) or question
+    if not valid:
+        return False, "invalid", corrected
+    if query_type == "restricted":
+        return False, "restricted", corrected
+    return True, "allowed", corrected
+
+
+def sql_shape(sql: str, intent: RuntimeIntent) -> SQLShape:
+    """Extract structural features from SQL and intent for shape comparison.
+
+    Args:
+
+        sql: The generated SQL string.
+        intent: The RuntimeIntent providing filter and HAVING counts (including CTE steps).
+
+    Returns:
+
+        SQLShape with join count, GROUP BY flag, aggregation flag, CTE count, filter count, HAVING count, and DISTINCT flag.
+    """
+    s = sql.lower()
+    num_filters = len(intent.filters_param or [])
+    num_having = len(intent.having_param or [])
+    for cte in intent.cte_steps or []:
+        num_filters += len(cte.filters_param or [])
+        num_having += len(cte.having_param or [])
+    return SQLShape(
+        num_joins=len(re.findall(r"\bjoin\b", s)),
+        has_group_by=bool(re.search(r"\bgroup\s+by\b", s)),
+        has_agg=bool(re.search(r"\b(count|sum|avg|min|max)\s*\(", s)),
+        num_cte=len(intent.cte_steps or []),
+        num_filters=num_filters,
+        num_having=num_having,
+        has_distinct=bool(re.search(r"\bselect\s+distinct\b", s)),
+    )
+
+
+def _levenshtein_distance(s1: str, s2: str) -> int:
+    """Compute Levenshtein edit distance between two strings.
+
+    Args:
+
+        s1: The first string.
+        s2: The second string.
+
+    Returns:
+
+        Integer edit distance (number of single-character insertions, deletions, or substitutions required to transform s1 into s2).
+    """
+    if len(s1) < len(s2):
+        return _levenshtein_distance(s2, s1)
+    if len(s2) == 0:
+        return len(s1)
+    previous_row = range(len(s2) + 1)
+    for i, c1 in enumerate(s1):
+        current_row = [i + 1]
+        for j, c2 in enumerate(s2):
+            insertions = previous_row[j + 1] + 1
+            deletions = current_row[j] + 1
+            substitutions = previous_row[j] + (c1 != c2)
+            current_row.append(min(insertions, deletions, substitutions))
+        previous_row = current_row
+    return previous_row[-1]
+
+
+def exact_question_match(
+    q1: str,
+    q2: str,
+    max_distance: int = PolicyConfig.FUZZY_MATCH_MAX_DISTANCE,
+    label: str = "",
+) -> bool:
+    """Check if two questions are identical after normalisation, within a fuzzy tolerance.
+
+    Args:
+
+        q1: The first question string.
+        q2: The second question string.
+        max_distance: Maximum total Levenshtein distance across all token pairs.
+        label: Optional debug label appended to log messages.
+
+    Returns:
+
+        True if the questions match within the specified fuzzy tolerance.
+    """
+    tag = f"[utils.exact_question_match] {label}" if label else "[utils.exact_question_match]"
+    q1_norm = normalize_question(q1)
+    q2_norm = normalize_question(q2)
+    tokens1 = _tokenize(q1_norm)
+    tokens2 = _tokenize(q2_norm)
+    if len(tokens1) == 0 or len(tokens2) == 0:
+        debug(f"{tag} FAIL empty_tokens t1={len(tokens1)} t2={len(tokens2)}")
+        return False
+    if len(tokens1) != len(tokens2):
+        debug(f"{tag} FAIL token_count t1={len(tokens1)} t2={len(tokens2)}")
+        return False
+    total_dist = 0
+    worst_pair = ("", "")
+    for t1, t2 in zip(tokens1, tokens2, strict=False):
+        dist = _levenshtein_distance(t1, t2)
+        total_dist += dist
+        if dist > 0:
+            worst_pair = (t1, t2)
+    if total_dist > max_distance:
+        debug(f"{tag} FAIL total_dist={total_dist} worst_pair='{worst_pair[0]}'→'{worst_pair[1]}'")
+        return False
+    debug(f"{tag} MATCH tokens={len(tokens1)} total_dist={total_dist}")
+    return True
+
+
+def _normalize_filters(filters: list[Any]) -> list[FilterParam]:
+    """Normalise a filter list into sorted FilterParam objects for intent-key hashing.
+
+    Accepts both FilterParam instances and raw dicts (as returned by the LLM). Operators are lowercased; invalid values are defaulted to ``'='``.
+
+    Args:
+
+        filters: List of FilterParam instances or raw dicts.
+
+    Returns:
+
+        Sorted list of normalised FilterParam instances.
+    """
+    if not filters:
+        return []
+    out = []
+    for f in filters:
+        if isinstance(f, FilterParam):
+            left_expr = f.left_expr
+            op = f.op.strip().lower() if f.op else "="
+            vtype = f.value_type.strip().lower() if isinstance(f.value_type, str) else "unknown"
+            right_expr = f.right_expr
+            bool_op = f.bool_op
+            filter_group = f.filter_group
+        elif isinstance(f, dict):
+            left_raw = f.get("left_expr")
+            if isinstance(left_raw, dict):
+                left_expr = NormalizedExpr.from_dict(left_raw)
+            else:
+                col = f.get("column", "")
+                left_expr = NormalizedExpr.from_column(col.strip().lower() if isinstance(col, str) else "")
+            op = f.get("op", "=").strip().lower()
+            vtype = f.get("value_type", "unknown").strip().lower()
+            right_raw = f.get("right_expr")
+            right_expr = NormalizedExpr.from_dict(right_raw) if isinstance(right_raw, dict) and right_raw else None
+            bool_op = f.get("bool_op", "AND")
+            fg_raw = f.get("filter_group")
+            filter_group = int(fg_raw) if fg_raw is not None else None
+        else:
+            continue
+        if not left_expr or not isinstance(op, str):
+            continue
+        fp = FilterParam(
+            left_expr=left_expr,
+            op=op,
+            value_type=vtype,
+            param_key="",
+            right_expr=right_expr,
+            bool_op=bool_op,
+            filter_group=filter_group,
+        )
+        out.append(fp)
+    return sort_filters(out)
+
+
+def _normalize_having_conditions(conditions: list[Any]) -> list[HavingParam]:
+    """Normalise a HAVING conditions list into sorted HavingParam objects.
+
+    Accepts both HavingParam instances and raw dicts. Operators are validated against the allowed set and defaulted to ``'='`` when invalid.
+
+    Args:
+
+        conditions: List of HavingParam instances or raw dicts.
+
+    Returns:
+
+        Sorted list of normalised HavingParam instances.
+    """
+    if not conditions:
+        return []
+    out = []
+    for c in conditions:
+        if isinstance(c, HavingParam):
+            left_expr = c.left_expr
+            op = c.op.strip().lower() if c.op else "="
+            value_type = c.value_type.strip().lower() if c.value_type else "number"
+            right_expr = c.right_expr
+            bool_op = c.bool_op
+            filter_group = c.filter_group
+        elif isinstance(c, dict):
+            left_raw = c.get("left_expr")
+            if isinstance(left_raw, dict):
+                left_expr = NormalizedExpr.from_dict(left_raw)
+            else:
+                agg = c.get("aggregation", "")
+                left_expr = NormalizedExpr.from_column(str(agg)) if agg else NormalizedExpr()
+            op = c.get("op", "=")
+            value_type = c.get("value_type", "number")
+            right_raw = c.get("right_expr")
+            right_expr = NormalizedExpr.from_dict(right_raw) if isinstance(right_raw, dict) and right_raw else None
+            bool_op = c.get("bool_op", "AND")
+            fg_raw = c.get("filter_group")
+            filter_group = int(fg_raw) if fg_raw is not None else None
+        else:
+            continue
+        if not left_expr or not left_expr.primary_term:
+            continue
+        op_norm = op.strip().lower() if isinstance(op, str) else "="
+        if op_norm not in VALID_HAVING_OPS:
+            op_norm = "="
+        hp = HavingParam(
+            left_expr=left_expr,
+            op=op_norm,
+            value_type=str(value_type),
+            param_key="",
+            right_expr=right_expr,
+            bool_op=bool_op,
+            filter_group=filter_group,
+        )
+        out.append(hp)
+    return sort_having(out)
+
+
+def _normalize_cte_steps(steps: Any, available_ctes: dict[str, list[str]] | None = None) -> list[RuntimeCteStep]:
+    """Normalise a CTE steps list into RuntimeCteStep objects with resolved column maps.
+
+    Handles both RuntimeCteStep instances and raw dicts. Normalises filters, HAVING conditions, grain, select columns, order-by columns, and infers column-map and output column metadata from available CTE output names.
+
+    Args:
+
+        steps: List of RuntimeCteStep instances or raw dicts.
+        available_ctes: Dict of already-processed CTE name -> output columns (mutated in place as each step is processed).
+
+    Returns:
+
+        List of normalised RuntimeCteStep instances in input order.
+    """
+    if not isinstance(steps, list):
+        return []
+    if available_ctes is None:
+        available_ctes = {}
+    out = []
+    for s in steps:
+        if isinstance(s, RuntimeCteStep):
+            cte_name = s.cte_name
+            tables = s.tables or []
+            grain = s.grain or "row_level"
+            select_cols = s.select_cols or []
+            group_by_cols = s.group_by_cols or []
+            output_columns = s.output_columns or []
+            filters_param = s.filters_param or []
+            having_param = s.having_param or []
+            param_values = s.param_values or {}
+            order_by_cols = s.order_by_cols or []
+            limit = s.limit
+            column_map = s.column_map or {}
+            output_column_metadata = s.output_column_metadata or {}
+            chosen_join_candidate_id = s.chosen_join_candidate_id or ""
+            chosen_join_path_signature = s.chosen_join_path_signature or []
+        elif isinstance(s, dict):
+            cte_name = s.get("cte_name", "")
+            tables = s.get("tables", [])
+            grain = s.get("grain", "row_level")
+            sc_raw = s.get("select_cols", [])
+            select_cols = [
+                (
+                    SelectCol.from_dict(sc)
+                    if isinstance(sc, dict)
+                    else (SelectCol(expr=NormalizedExpr.from_column(sc)) if isinstance(sc, str) else sc)
+                )
+                for sc in sc_raw
+            ]
+            group_by_cols = s.get("group_by_cols", [])
+            group_by_cols = [
+                (
+                    NormalizedExpr.from_dict(g)
+                    if isinstance(g, dict)
+                    else (NormalizedExpr.from_column(g) if isinstance(g, str) else g)
+                )
+                for g in group_by_cols
+            ]
+            output_columns = s.get("output_columns", [])
+            fp_raw = s.get("filters_param", [])
+            filters_param = [FilterParam.from_dict(f) if isinstance(f, dict) else f for f in fp_raw]
+            hp_raw = s.get("having_param", [])
+            having_param = [HavingParam.from_dict(h) if isinstance(h, dict) else h for h in hp_raw]
+            param_values = s.get("param_values", {})
+            obc_raw = s.get("order_by_cols", [])
+            order_by_cols = [
+                (
+                    OrderByCol.from_dict(o)
+                    if isinstance(o, dict)
+                    else (OrderByCol(expr=NormalizedExpr.from_column(o)) if isinstance(o, str) else o)
+                )
+                for o in obc_raw
+            ]
+            limit = s.get("limit")
+            column_map = s.get("column_map", {})
+            ocm_raw = s.get("output_column_metadata", {})
+            output_column_metadata = {
+                k: CteOutputColumnMeta.from_dict(v) if isinstance(v, dict) else v for k, v in ocm_raw.items()
+            }
+            chosen_join_candidate_id = s.get("chosen_join_candidate_id", "")
+            chosen_join_path_signature = s.get("chosen_join_path_signature", [])
+        else:
+            continue
+        if not cte_name:
+            continue
+        if grain not in VALID_GRAINS:
+            grain = "row_level"
+        normalized_fp = []
+        for f in filters_param:
+            if isinstance(f, FilterParam):
+                op = f.op.strip().lower() if f.op else "="
+                if op not in VALID_FILTER_OPS:
+                    op = "="
+                vtype = f.value_type.strip().lower() if f.value_type else "unknown"
+                fp = FilterParam(
+                    left_expr=f.left_expr,
+                    op=op,
+                    value_type=vtype,
+                    param_key=f.param_key or "",
+                    right_expr=f.right_expr,
+                )
+                normalized_fp.append(fp)
+        normalized_hp = []
+        for h in having_param:
+            if isinstance(h, HavingParam):
+                op = h.op.strip().lower() if h.op else "="
+                if op not in VALID_HAVING_OPS:
+                    op = "="
+                vtype = h.value_type.strip().lower() if h.value_type else "number"
+                hp = HavingParam(
+                    left_expr=h.left_expr,
+                    op=op,
+                    value_type=vtype,
+                    param_key=h.param_key or "",
+                    right_expr=h.right_expr,
+                )
+                normalized_hp.append(hp)
+        cte_column_map = {}
+        all_cols_raw = [g.primary_column for g in group_by_cols] + [f.left_expr.primary_column for f in normalized_fp]
+        for sc in select_cols:
+            if isinstance(sc, SelectCol):
+                all_cols_raw.append(sc.expr.primary_column)
+            elif isinstance(sc, str):
+                all_cols_raw.append(sc)
+        for col in all_cols_raw:
+            if "." in col:
+                parts = col.split(".", 1)
+                table_ref = parts[0].lower()
+                col_name = parts[1].lower()
+                if table_ref in {t.lower() for t in tables} or table_ref in {c.lower() for c in available_ctes.keys()}:
+                    cte_column_map[col_name] = table_ref
+        if column_map:
+            cte_column_map.update(column_map)
+        ocm = {}
+        for out_col in output_columns:
+            out_col_lower = out_col.lower()
+            if out_col_lower in output_column_metadata:
+                ocm[out_col_lower] = output_column_metadata[out_col_lower]
+            else:
+                is_agg = any(out_col_lower.startswith(f"{agg}_") for agg in VALID_AGGREGATION_FUNCTIONS)
+                base_col = ""
+                agg_func = ""
+                if is_agg:
+                    for agg in VALID_AGGREGATION_FUNCTIONS:
+                        if out_col_lower.startswith(f"{agg}_"):
+                            agg_func = agg
+                            base_col = out_col_lower[len(agg) + 1 :].replace("_", ".")
+                            break
+                ocm[out_col_lower] = CteOutputColumnMeta(
+                    source="aggregation" if is_agg else "passthrough",
+                    base_column=base_col,
+                    agg_func=agg_func,
+                    filterable=True,
+                    aggregatable=True,
+                    data_type=("integer" if is_agg and agg_func == "count" else "unknown"),
+                )
+        normalized_select_cols = (
+            select_cols
+            if select_cols and isinstance(select_cols[0], SelectCol)
+            else ([SelectCol(expr=NormalizedExpr.from_column(c)) for c in select_cols] if select_cols else [])
+        )
+        normalized_order_by = (
+            order_by_cols
+            if order_by_cols and isinstance(order_by_cols[0], OrderByCol)
+            else (
+                [(OrderByCol(expr=NormalizedExpr.from_column(c)) if isinstance(c, str) else c) for c in order_by_cols]
+                if order_by_cols
+                else []
+            )
+        )
+        cte = RuntimeCteStep(
+            cte_name=str(cte_name),
+            tables=sorted(set(str(t) for t in tables)),
+            grain=grain,
+            select_cols=normalized_select_cols,
+            group_by_cols=group_by_cols,
+            output_columns=list(str(c) for c in output_columns),
+            filters_param=sorted(
+                normalized_fp,
+                key=lambda x: (
+                    x.left_expr.signature_key,
+                    x.op,
+                    x.right_expr.signature_key if x.right_expr else "",
+                    x.value_type,
+                ),
+            ),
+            having_param=sorted(
+                normalized_hp,
+                key=lambda x: (
+                    x.left_expr.signature_key,
+                    x.op,
+                    x.right_expr.signature_key if x.right_expr else "",
+                    x.value_type,
+                ),
+            ),
+            param_values=param_values,
+            order_by_cols=normalized_order_by,
+            limit=limit,
+            column_map=cte_column_map,
+            output_column_metadata=ocm,
+            chosen_join_candidate_id=chosen_join_candidate_id,
+            chosen_join_path_signature=chosen_join_path_signature,
+        )
+        out.append(cte)
+        available_ctes[cte_name] = output_columns
+    return out
+
+
+def _normalize_cte_steps_for_key(steps: list[RuntimeCteStep]) -> list[dict[str, Any]]:
+    """Convert RuntimeCteStep list to stable structural dicts for intent-key hashing.
+
+    Args:
+
+        steps: List of normalised RuntimeCteStep instances.
+
+    Returns:
+
+        List of dicts with sorted keys suitable for stable JSON serialisation.
+    """
+    result = []
+    for cte in steps:
+        select_sigs = []
+        for sc in cte.select_cols or []:
+            if isinstance(sc, SelectCol):
+                select_sigs.append(sc.signature_key)
+            elif isinstance(sc, str):
+                select_sigs.append(sc)
+        order_sigs = []
+        for obc in cte.order_by_cols or []:
+            if isinstance(obc, OrderByCol):
+                order_sigs.append(obc.signature_key)
+            elif isinstance(obc, str):
+                order_sigs.append(obc)
+        cte_dict = {
+            "cte_name": cte.cte_name,
+            "tables": sorted(cte.tables or []),
+            "select_cols": sorted(select_sigs),
+            "group_by_cols": sorted([g.signature_key for g in (cte.group_by_cols or [])]),
+            "output_columns": sorted(cte.output_columns or []),
+            "filters_param": [
+                f"{f.signature_key}|{f.bool_op}|{f.filter_group}"
+                for f in sorted(
+                    cte.filters_param or [],
+                    key=lambda x: (
+                        x.filter_group if x.filter_group is not None else -1,
+                        x.left_expr.signature_key,
+                        x.op,
+                        x.right_expr.signature_key if x.right_expr else "",
+                        x.value_type,
+                    ),
+                )
+            ],
+            "having_param": [
+                f"{h.signature_key}|{h.bool_op}|{h.filter_group}"
+                for h in sorted(
+                    cte.having_param or [],
+                    key=lambda x: (
+                        x.filter_group if x.filter_group is not None else -1,
+                        x.left_expr.signature_key,
+                        x.op,
+                        x.right_expr.signature_key if x.right_expr else "",
+                        x.value_type,
+                    ),
+                )
+            ],
+            "order_by_cols": sorted(order_sigs),
+        }
+        result.append(cte_dict)
+    return result
+
+
+def flatten_param_values(intent: RuntimeIntent) -> dict[str, Any]:
+    """Flatten all param_values from CTE steps and the main query into a single dict.
+
+    CTE values are merged first; main query values take precedence on key collision.
+
+    Args:
+
+        intent: The RuntimeIntent whose CTE steps and main param_values are merged.
+
+    Returns:
+
+        Merged dict of all parameter values.
+    """
+    merged = {}
+    for cte in intent.cte_steps or []:
+        merged.update(cte.param_values or {})
+    merged.update(intent.param_values or {})
+    return merged
+
+
+def intent_key(intent: RuntimeIntent) -> str:
+    """Generate a stable SHA-256 hash key for a RuntimeIntent.
+
+    Normalises tables, select columns, filters, GROUP BY, ORDER BY, HAVING, and CTE steps before hashing so that semantically equivalent intents produce the same key regardless of input order.
+
+    Args:
+
+        intent: The RuntimeIntent to fingerprint.
+
+    Returns:
+
+        Hex-encoded SHA-256 hash string.
+    """
+    select_cols = intent.select_cols or []
+
+    grain = intent.grain or "row_level"
+    if grain not in VALID_GRAINS:
+        debug(f"[utils.intent_key] invalid_grain: '{grain}' defaulting to 'row_level'")
+        grain = "row_level"
+
+    expected_rows = intent.expected_rows or "many"
+    if expected_rows not in VALID_EXPECTED_ROWS:
+        if grain == "scalar":
+            expected_rows = "one"
+        elif grain == "grouped":
+            expected_rows = "few"
+        else:
+            expected_rows = "many"
+        debug(f"[utils.intent_key] inferred_expected_rows: grain={grain} -> expected_rows={expected_rows}")
+
+    filters_normalized = _normalize_filters(intent.filters_param or [])
+    having_conditions_normalized = _normalize_having_conditions(intent.having_param or [])
+    cte_steps_normalized = _normalize_cte_steps(intent.cte_steps or [])
+    cte_steps_for_key = _normalize_cte_steps_for_key(cte_steps_normalized)
+
+    select_cols_sorted = sorted([s.signature_key for s in select_cols])
+    order_by_sorted = sorted([o.signature_key for o in (intent.order_by_cols or [])])
+
+    normalized = {
+        "tables": sorted(intent.tables or []),
+        "select_cols": select_cols_sorted,
+        "filters": [f.to_dict() for f in filters_normalized],
+        "group_by_cols": sorted([g.signature_key for g in (intent.group_by_cols or [])]),
+        "order_by_cols": order_by_sorted,
+        "having_conditions": [hc.to_dict() for hc in having_conditions_normalized],
+        "cte_steps": cte_steps_for_key,
+    }
+    key = sha256(stable_json(normalized))
+    debug(f"[utils.intent_key] computed: tables={normalized['tables']} key={key[:16]}...")
+    return key
+
+
+def _tokenize(q: str) -> list[str]:
+    """Extract sorted, unique tokens from a normalised question, excluding stopwords.
+
+    Args:
+
+        q: The normalised question string.
+
+    Returns:
+
+        Sorted list of alphanumeric/underscore tokens with stopwords removed.
+    """
+    return sorted(t for t in re.findall(r"[a-z0-9_]+", q) if t and t not in PolicyConfig.STOPWORDS)
+
+
+def extract_tables_from_sql(sql: str, known_tables: list[str]) -> list[str]:
+    """Extract real table names referenced in SQL from a known list, excluding CTE names.
+
+    Args:
+
+        sql: The SQL string to scan.
+        known_tables: List of base table names to match against.
+
+    Returns:
+
+        Sorted list of matched table names that are not CTE aliases.
+    """
+    s = sql.lower()
+
+    cte_names = set()
+    if re.search(r"\bwith\b", s):
+        for m in re.finditer(r"\b(\w+)\s+AS\s*\(", s, re.IGNORECASE):
+            cte_names.add(m.group(1).lower())
+
+    hits = []
+    for t in known_tables:
+        tn = t.lower()
+        if tn not in cte_names and re.search(rf"\b{re.escape(tn)}\b", s):
+            hits.append(t)
+    return sorted(set(hits))
+
+
+def _describe_operation(select_terms: list[str]) -> str:
+    """Derive the primary operation type from a list of SELECT expression strings.
+
+    Args:
+
+        select_terms: List of SELECT column/expression strings (e.g. ``["COUNT(t.id)"]``).
+
+    Returns:
+
+        Lowercase aggregation function name (e.g. ``'count'``, ``'sum'``) or ``'list'`` if no aggregation is detected.
+    """
+    for sc in select_terms:
+        m = AGG_PATTERN.match(sc)
+        if m:
+            return m.group(1).lower()
+    return "list"
+
+
+def _pick_question_style(select_terms: list[str], has_grouping: bool) -> str:
+    """Pick a random question-opening phrase appropriate for the query structure.
+
+    Args:
+
+        select_terms: List of SELECT expression strings used to detect aggregation.
+        has_grouping: True if the query has GROUP BY columns.
+
+    Returns:
+
+        A randomly selected opening phrase string.
+    """
+    agg_funcs = []
+    for sc in select_terms:
+        m = AGG_PATTERN.match(sc)
+        if m:
+            agg_funcs.append(m.group(1).lower())
+
+    if has_grouping:
+        return random.choice(QUESTION_STARTS_GROUP)
+    elif agg_funcs:
+        agg = agg_funcs[0]
+        if agg == "count":
+            return random.choice(["How many", "Count", "What is the number of"])
+        elif agg == "sum":
+            return random.choice(["What is the total", "Find the sum of", "Calculate the total"])
+        elif agg == "avg":
+            return random.choice(["What is the average", "Calculate the average", "Find the mean"])
+        elif agg in ("min", "max"):
+            return random.choice([f"What is the {agg}imum", f"Find the {agg}imum", f"Get the {agg}"])
+        return random.choice(QUESTION_STARTS_AGG)
+    else:
+        return random.choice(QUESTION_STARTS_LIST)
+
+
+def generate_question(
+    tables: list[str],
+    select_terms: list[str],
+    filter_descriptions: list[dict[str, str]],
+    group_by_terms: list[str],
+    having_descriptions: list[dict[str, str]],
+    schema: SchemaGraph,
+) -> str | None:
+    """Generate a natural language question from a plain-data intent description via LLM.
+
+    Builds a structured prompt including semantic context from the schema graph and enforces a randomly selected question-opening phrase for variety. Callers are responsible for resolving all param placeholder values into the condition strings of ``filter_descriptions`` and ``having_descriptions`` before calling.
+
+    Args:
+
+        tables: List of table names involved in the query.
+        select_terms: List of SELECT expression strings.
+        filter_descriptions: List of dicts with ``'column'`` and ``'condition'`` keys, with concrete values already substituted.
+        group_by_terms: List of GROUP BY column expression strings.
+        having_descriptions: List of dicts with HAVING condition descriptions, with concrete values already substituted.
+        schema: The SchemaGraph used to look up table and column descriptions.
+
+    Returns:
+
+        The generated question string, or None if the LLM call fails or returns a response that violates the required opening phrase.
+    """
+    semantics = {}
+    for table in tables:
+        table_ir = schema.tables.get(table)
+        if table_ir:
+            semantics[table] = {
+                "description": table_ir.description or f"{table} records",
+                "columns": {col: (getattr(meta, "description", None) or col) for col, meta in table_ir.columns.items()},
+            }
+
+    roles = {}
+    all_columns = set()
+    for fd in filter_descriptions:
+        all_columns.add(fd.get("column", ""))
+    for col in group_by_terms:
+        all_columns.add(col)
+    for col in all_columns:
+        parts = col.split(".")
+        if len(parts) == 2 and parts[0] in schema.tables:
+            col_meta = schema.tables[parts[0]].columns.get(parts[1])
+            if col_meta and col_meta.role:
+                roles[col] = col_meta.role
+
+    operation = _describe_operation(select_terms)
+
+    intent_structure = {
+        "tables": tables,
+        "operation": operation,
+        "columns": select_terms,
+        "filters": filter_descriptions if filter_descriptions else None,
+        "grouping": group_by_terms if group_by_terms else None,
+        "having": having_descriptions if having_descriptions else None,
+    }
+
+    selected_style = _pick_question_style(select_terms, bool(group_by_terms))
+
+    system = (
+        "You are a natural language question generator for database queries. Convert structured query intent to conversational questions.\n\n"
+        "Output Requirements:\n"
+        '- Output ONLY valid JSON with single "question" field\n'
+        "- Do NOT include markdown, explanations, or commentary\n"
+        "- Identical inputs must produce identical outputs\n\n"
+        "Generation Rules:\n"
+        "- Question MUST reflect EXACTLY the intent structure (same tables, columns, filters, aggregation)\n"
+        "- Do NOT add columns, tables, or filters not in the intent\n"
+        "- Do NOT change aggregation type or omit filter values\n"
+        "- Use natural, conversational language - avoid SQL jargon and raw table names\n"
+        "- Use semantic context to refer to business concepts naturally\n"
+        "- For multi-table queries, describe relationships naturally\n"
+        "- Inject filter values naturally into the question\n"
+        "- ONE sentence only\n"
+        "- Vary phrasing naturally within the required start constraint\n"
+    )
+
+    user_prompt = {
+        "task": "Generate a natural language question for this query intent",
+        "intent": intent_structure,
+        "semantic_context": semantics,
+        "column_roles": roles,
+        "phrasing_constraint": {
+            "required_start": selected_style,
+            "description": f"Question MUST start with exactly: '{selected_style}'",
+            "strict_mode": not filter_descriptions and not group_by_terms,
+            "phrasing_flexibility": "Vary word choice and sentence structure while keeping the required start",
+        },
+        "examples": [
+            {
+                "intent": {
+                    "tables": ["table1"],
+                    "operation": "count",
+                    "columns": ["COUNT(table1.column1)"],
+                    "filters": [{"column": "table1.column2", "condition": "= 1"}],
+                },
+                "required_start": "How many",
+                "question": "How many active records do we have?",
+            },
+            {
+                "intent": {
+                    "tables": ["table1"],
+                    "operation": "list",
+                    "columns": ["table1.column1", "table1.column2"],
+                    "filters": [{"column": "table1.column2", "condition": "= 'value1'"}],
+                },
+                "required_start": "What are",
+                "question": "What are the records with column2 equal to value1?",
+            },
+            {
+                "intent": {
+                    "tables": ["table1", "table2"],
+                    "operation": "sum",
+                    "columns": ["SUM(table2.column1)", "table1.column1"],
+                    "grouping": ["table2.column2"],
+                },
+                "required_start": "Show me",
+                "question": "Show me the total column1 for each group.",
+            },
+        ],
+        "output_format": {"question": "Your natural language question here"},
+    }
+
+    try:
+        response = llm_json(system, stable_json(user_prompt), retries=1, task="default")
+        question = response.get("question")
+        if question and isinstance(question, str):
+            question = question.strip()
+            template_start = selected_style.split("{")[0].strip()
+            if template_start and not question.startswith(template_start):
+                debug(
+                    f"[utils.generate_question] phrasing_violation: expected_start={template_start}, got={question[:30]}"
+                )
+                return None
+            debug(f"[utils.generate_question] generated: {question[:50]}")
+            return question
+        debug("[utils.generate_question] missing_question_field")
+        return None
+    except Exception as e:
+        debug(f"[utils.generate_question] failed: {e}")
+        return None
+
+
+_QUESTION_FROM_SQL_SYSTEM = (
+    "You are given a SQL query and a schema description. "
+    "Your job is to decide whether the query represents a realistic, "
+    "meaningful business question and, if so, generate a natural language "
+    "question that a human analyst would ask to get this query's result.\n\n"
+    "Rules:\n"
+    "- If the query is unrealistic, nonsensical, or produces meaningless "
+    "results, set is_realistic to false and explain why in drop_reason.\n"
+    "- If realistic, write ONE clear, conversational question that a "
+    "non-technical user would ask. Do NOT use SQL jargon or raw column "
+    "names — use natural business language.\n"
+    "- Output ONLY valid JSON with exactly three fields: "
+    '"question", "is_realistic" (boolean), "drop_reason" (string or null).\n'
+)
+
+
+def generate_question_from_sql(
+    sql: str,
+    schema: SchemaGraph,
+    tables: list[str],
+) -> dict[str, Any] | None:
+    """Generate an NL question from a substituted SQL query with a realism gate.
+
+    Sends the SQL and table/column descriptions to the LLM (default
+    task profile = gpt-4o-mini).  Returns a dict with ``question``,
+    ``is_realistic``, and ``drop_reason`` keys, or ``None`` on failure.
+
+    Args:
+
+        sql: Fully substituted SQL string with literal values.
+        schema: Schema graph for table/column descriptions.
+        tables: Tables referenced by the query.
+
+    Returns:
+
+        Dict with question and realism judgment, or None on LLM failure.
+    """
+    col_descriptions: list[str] = []
+    for table in tables:
+        table_meta = schema.tables.get(table)
+        if not table_meta:
+            continue
+        cols = []
+        for col_name, col_meta in table_meta.columns.items():
+            desc = f"{col_name} ({col_meta.data_type or 'unknown'})"
+            if col_meta.role:
+                desc += f" [{col_meta.role}]"
+            cols.append(desc)
+        col_descriptions.append(
+            f"TABLE {table}: {', '.join(cols)}"
+        )
+    schema_context = "\n".join(col_descriptions)
+
+    user_prompt = stable_json({
+        "sql": sql,
+        "schema": schema_context,
+        "output_format": {
+            "question": "natural language question or empty string",
+            "is_realistic": True,
+            "drop_reason": None,
+        },
+    })
+
+    try:
+        response = llm_json(
+            _QUESTION_FROM_SQL_SYSTEM, user_prompt,
+            retries=1, task="default",
+        )
+        is_realistic = response.get("is_realistic", False)
+        question = response.get("question", "")
+        drop_reason = response.get("drop_reason")
+
+        if not isinstance(is_realistic, bool):
+            is_realistic = str(is_realistic).lower() in ("true", "1", "yes")
+
+        if not is_realistic:
+            debug(
+                f"[utils.generate_question_from_sql] dropped: "
+                f"reason={drop_reason}"
+            )
+            return {
+                "question": "",
+                "is_realistic": False,
+                "drop_reason": drop_reason or "unrealistic",
+            }
+
+        if not question or not isinstance(question, str):
+            debug("[utils.generate_question_from_sql] empty question")
+            return None
+
+        debug(
+            f"[utils.generate_question_from_sql] generated: "
+            f"{question[:60]}"
+        )
+        return {
+            "question": question.strip(),
+            "is_realistic": True,
+            "drop_reason": None,
+        }
+    except Exception as e:
+        debug(f"[utils.generate_question_from_sql] failed: {e}")
+        return None