aetherdialect 0.1.0__py3-none-any.whl

text2sql/simulator.py

@@ -0,0 +1,970 @@
+"""Coverage simulator for synthetic template generation.
+
+Orchestrates the deterministic simulation pipeline: gold intent generation
+from seed questions, fully deterministic multi-depth expansion, join
+resolution (cached per table-set, LLM only for ambiguous multi-table),
+SQL building and validation, value instantiation, and LLM-based NL
+question generation with a realism quality gate.
+
+Also provides helpers for persisting simulator artefacts.
+"""
+
+from __future__ import annotations
+
+import csv
+import glob
+import json
+import os
+import re
+from dataclasses import replace
+from typing import Any
+
+from .config import EngineConfig, SimulatorConfig
+from .contracts_base import SchemaGraph, TemplateStats, ValueDomain
+from .contracts_core import (
+    ConcreteIntent,
+    FilterParam,
+    HavingParam,
+    RuntimeIntent,
+    SimulatorIntent,
+    SimulatorResult,
+    Template,
+    ValueHistory,
+)
+from .core_utils import (
+    ask_user_choice,
+    canonicalize_sql,
+    colmap_signature,
+    debug,
+    log,
+    normalize_question,
+    parameter_abstract,
+    print_info,
+    sql_fp,
+    substitute_params,
+)
+from .intent_expr import (
+    assign_param_keys,
+    collect_raw_param_values,
+    extract_structural_params,
+)
+from .intent_process import full_intent_parse
+from .qsim_sample import (
+    deterministic_having_value,
+    sample_coordinated_range,
+    sample_value_from_domain,
+)
+from .sql_gen import (
+    build_deterministic_sql,
+    get_join_choice_from_llm,
+    inject_join_into_deterministic_sql,
+    join_candidate_map,
+    join_hints_multi,
+    physical_tables_for_join_hints,
+)
+from .utils import (
+    extract_tables_from_sql,
+    flatten_param_values,
+    generate_question_from_sql,
+    intent_key,
+    sql_shape,
+)
+from .validation_execute import execute_sql, get_spark_sql_for_execution, validate_sql
+
+
+# ---------------------------------------------------------------------------
+# Versioning / file helpers
+# ---------------------------------------------------------------------------
+
+def get_next_simulator_version(output_dir: str) -> int:
+    """Return the next auto-incrementing version number for simulator output files.
+
+    Scans *output_dir* for files matching ``gold_intents_v*.json`` and
+    returns the highest existing version number plus one.
+
+    Args:
+
+        output_dir: Directory to scan for versioned output files.
+
+    Returns:
+
+        The next available integer version number (starting at 1).
+    """
+    pattern = os.path.join(output_dir, "gold_intents_v*.json")
+    existing = glob.glob(pattern)
+    if not existing:
+        return 1
+    versions = []
+    for f in existing:
+        try:
+            v = int(os.path.basename(f).split("_v")[1].split(".")[0])
+            versions.append(v)
+        except (IndexError, ValueError):
+            continue
+    return max(versions) + 1 if versions else 1
+
+
+# ---------------------------------------------------------------------------
+# Seed / gold intent generation
+# ---------------------------------------------------------------------------
+
+def _load_seed_questions(filepath: str) -> list[dict[str, Any]]:
+    """Load seed questions from a text file.
+
+    Supports numbered formats (``1. question``, ``1) question``),
+    quoted strings, and plain lines. Phase sections are parsed from
+    ``# Phase N`` comment headers.
+
+    Args:
+
+        filepath: Path to the seed questions text file.
+
+    Returns:
+
+        List of dicts with ``number``, ``question``, and ``phase`` keys.
+
+    Raises:
+
+        FileNotFoundError: If *filepath* does not exist.
+    """
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"Seed questions file not found: {filepath}")
+
+    questions: list[dict[str, Any]] = []
+    current_phase = "unknown"
+    auto_number = 1
+
+    with open(filepath, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            if line.startswith("#"):
+                phase_match = re.search(
+                    r"Phase\s+(\d+)", line, re.IGNORECASE,
+                )
+                if phase_match:
+                    current_phase = f"phase_{phase_match.group(1)}"
+                continue
+
+            question_text = None
+            number = None
+
+            match = re.match(r"^(\d+)\.\s*(.+)$", line)
+            if match:
+                number = int(match.group(1))
+                question_text = match.group(2).strip()
+
+            if not question_text:
+                match = re.match(r"^(\d+)\)\s*(.+)$", line)
+                if match:
+                    number = int(match.group(1))
+                    question_text = match.group(2).strip()
+
+            if not question_text:
+                match = re.match(r'^["\'](.+)["\']$', line)
+                if match:
+                    question_text = match.group(1).strip()
+                    number = auto_number
+                    auto_number += 1
+
+            if not question_text:
+                question_text = line
+                number = auto_number
+                auto_number += 1
+
+            question_text = question_text.strip("\"'")
+            if question_text:
+                questions.append({
+                    "number": number,
+                    "question": question_text,
+                    "phase": current_phase,
+                })
+
+    debug(
+        f"[simulator.load_seed_questions] loaded: {len(questions)} questions"
+    )
+    return questions
+
+
+def _parse_gold_intent(
+    question: str, schema: SchemaGraph,
+) -> RuntimeIntent | None:
+    """Parse a seed question into a gold RuntimeIntent via full intent pipeline."""
+    q_norm = normalize_question(question)
+    intent, semantic_issues, _llm_calls = full_intent_parse(q_norm, schema)
+    if intent is None:
+        debug(f"[simulator.parse_gold_intent] parse_failed: {q_norm}")
+        return None
+    if semantic_issues:
+        debug(
+            f"[simulator.parse_gold_intent] semantic_issues: "
+            f"{len(semantic_issues)} issues (ignored)"
+        )
+    return intent
+
+
+def _confirm_gold_intent(
+    question: str, intent: RuntimeIntent,
+) -> tuple[bool, RuntimeIntent | None]:
+    """Interactively confirm a parsed gold intent with the user."""
+    nl_summary = intent.natural_language or ""
+    if not nl_summary:
+        nl_summary = (
+            f"Query {', '.join(intent.tables or [])} "
+            f"for {intent.grain or 'data'}"
+        )
+    grain = intent.grain or "row_level"
+    if grain == "scalar":
+        expected_display = "scalar"
+    else:
+        expected_display = f"{intent.expected_rows or 'many'} row(s)"
+
+    agg_ops = []
+    for sc in intent.select_cols or []:
+        if sc.is_aggregated:
+            e = sc.expr
+            agg = e.agg_func or (
+                e.add_groups[0].agg_func
+                if e.add_groups and e.add_groups[0].agg_func
+                else None
+            )
+            term = e.primary_term
+            if agg:
+                agg_ops.append(f"{agg.upper()}({term})")
+            else:
+                agg_ops.append(term)
+
+    print_info(
+        f"Question: {question}\n\nI understood: {nl_summary}",
+        items={
+            "Tables": intent.tables or [],
+            "Aggregations": agg_ops or ["none"],
+            "Expected": expected_display,
+        },
+    )
+    choice = ask_user_choice("\nIs this correct?", ["y", "n"])
+    if choice is None:
+        return False, None
+    elif choice == "y":
+        print("\nIntent accepted.")
+        return True, intent
+    else:
+        print("\nIntent rejected.")
+        return False, None
+
+
+def _abstract_values(intent: RuntimeIntent) -> RuntimeIntent:
+    """Return *intent* with ``param_values`` cleared."""
+    return RuntimeIntent(
+        tables=intent.tables,
+        grain=intent.grain,
+        select_cols=intent.select_cols,
+        group_by_cols=intent.group_by_cols,
+        order_by_cols=intent.order_by_cols,
+        filters_param=intent.filters_param,
+        having_param=intent.having_param,
+        param_values={},
+        cte_steps=intent.cte_steps,
+        column_map=intent.column_map,
+        natural_language=intent.natural_language,
+        limit=intent.limit,
+    )
+
+
+def _save_gold_questions(
+    gold_intents: list[dict[str, Any]], filepath: str,
+) -> None:
+    """Persist confirmed gold intents to a JSON file."""
+    output = {"count": len(gold_intents), "intents": gold_intents}
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(output, f, indent=2, ensure_ascii=False)
+    log(
+        f"save_gold_questions: saved {len(gold_intents)} gold intents "
+        f"to {filepath}"
+    )
+
+
+def _load_gold_questions(filepath: str) -> list[dict[str, Any]]:
+    """Load previously saved gold intents from a JSON file."""
+    if not os.path.exists(filepath):
+        return []
+    with open(filepath, encoding="utf-8") as f:
+        data = json.load(f)
+    intents = data.get("intents", [])
+    debug(f"[simulator.load_gold_questions] loaded: {len(intents)} intents")
+    return intents
+
+
+def run_gold_intent_generation(
+    schema: SchemaGraph,
+    seed_filepath: str,
+    output_filepath: str,
+    interactive: bool = True,
+) -> list[dict[str, Any]]:
+    """Run the full gold intent generation pipeline from seed questions.
+
+    Args:
+
+        schema: Schema graph used for intent parsing.
+        seed_filepath: Path to the seed questions text file.
+        output_filepath: JSON file path for saving gold intents.
+        interactive: When True, interactively confirm each parsed intent.
+
+    Returns:
+
+        List of all gold intent dicts.
+    """
+    log("gold_intent_generation: starting")
+    seeds = _load_seed_questions(seed_filepath)
+    log(f"gold_intent_generation: {len(seeds)} seed questions loaded")
+
+    existing = _load_gold_questions(output_filepath)
+    existing_questions = {
+        i.get("normalized_question")
+        for i in existing if i.get("normalized_question")
+    }
+    log(f"gold_intent_generation: {len(existing)} existing gold intents")
+
+    gold_intents = list(existing)
+    new_count = 0
+    skip_count = 0
+    fail_count = 0
+
+    for seed in seeds:
+        q = seed["question"]
+        q_norm = normalize_question(q)
+        if q_norm in existing_questions:
+            skip_count += 1
+            continue
+
+        log(f"gold_intent_generation: processing [{seed['number']}] {q}")
+        intent = _parse_gold_intent(q, schema)
+        if intent is None:
+            log(
+                f"gold_intent_generation: FAILED to parse [{seed['number']}]"
+            )
+            fail_count += 1
+            continue
+
+        if interactive:
+            confirmed, final_intent = _confirm_gold_intent(q, intent)
+            if not confirmed or final_intent is None:
+                log(
+                    f"gold_intent_generation: user rejected "
+                    f"[{seed['number']}]"
+                )
+                continue
+            intent = final_intent
+
+        intent = _abstract_values(intent)
+        gold_dict = intent.to_dict()
+        gold_dict["normalized_question"] = q_norm
+        gold_intents.append(gold_dict)
+        existing_questions.add(q_norm)
+        new_count += 1
+
+        if new_count % 10 == 0:
+            _save_gold_questions(gold_intents, output_filepath)
+
+    _save_gold_questions(gold_intents, output_filepath)
+    log(
+        f"gold_intent_generation: complete. "
+        f"new={new_count}, skipped={skip_count}, failed={fail_count}"
+    )
+    print("\nGold intent generation complete.")
+    print(f"  New: {new_count}")
+    print(f"  Skipped: {skip_count}")
+    print(f"  Failed: {fail_count}")
+    print(f"  Total: {len(gold_intents)}")
+    return gold_intents
+
+
+# ---------------------------------------------------------------------------
+# Join resolution cache
+# ---------------------------------------------------------------------------
+
+JoinCacheEntry = tuple[str, str, list[Any]]
+
+
+def resolve_joins_for_table_set(
+    tables: list[str],
+    schema: SchemaGraph,
+    question_hint: str,
+    join_cache: dict[frozenset[str], JoinCacheEntry],
+) -> JoinCacheEntry:
+    """Resolve join path for a table set, using cache when available.
+
+    Returns ``(chosen_join_id, join_path_signature, join_candidates)``
+    and populates the cache.  LLM is only called when there are
+    multiple ambiguous candidates.
+
+    Args:
+
+        tables: Sorted table list for the intent.
+        schema: Schema graph for join hint generation.
+        question_hint: Text hint passed to the LLM for disambiguation.
+        join_cache: Shared cache keyed by frozenset of table names.
+
+    Returns:
+
+        Tuple of (join_id, signature_string, candidates_list).
+    """
+    key = frozenset(tables)
+    if key in join_cache:
+        return join_cache[key]
+
+    if len(tables) <= 1:
+        entry: JoinCacheEntry = ("J00", "", [])
+        join_cache[key] = entry
+        return entry
+
+    join_tables = physical_tables_for_join_hints(tables, schema)
+    candidates = join_hints_multi(schema, join_tables)
+    cmap = join_candidate_map(candidates)
+
+    if not candidates:
+        entry = ("J00", "", [])
+        join_cache[key] = entry
+        return entry
+
+    non_trivial = [c for c in cmap if c != "J00"]
+    if len(non_trivial) <= 1:
+        chosen = non_trivial[0] if non_trivial else "J00"
+        sig = str(cmap.get(chosen, []))
+        entry = (chosen, sig, candidates)
+        join_cache[key] = entry
+        return entry
+
+    chosen = get_join_choice_from_llm(
+        question_hint, "", candidates, task="sql",
+    )
+    sig = str(cmap.get(chosen, []))
+    entry = (chosen, sig, candidates)
+    join_cache[key] = entry
+    return entry
+
+
+# ---------------------------------------------------------------------------
+# Value instantiation
+# ---------------------------------------------------------------------------
+
+def _decompose_between_filter_param(f: FilterParam) -> list[FilterParam]:
+    """Decompose a BETWEEN filter into >= and <= pair."""
+    if f.op != "between":
+        return [f]
+    return [
+        replace(
+            f, op=">=",
+            param_key=f"{f.param_key}_lower" if f.param_key else None,
+        ),
+        replace(
+            f, op="<=",
+            param_key=f"{f.param_key}_upper" if f.param_key else None,
+        ),
+    ]
+
+
+def _identify_range_pairs(
+    filters: list[FilterParam],
+) -> dict[str, dict[str, int]]:
+    """Identify columns with paired lower/upper bound filters."""
+    column_ops: dict[str, dict[str, int]] = {}
+    for idx, f in enumerate(filters):
+        if f.right_expr:
+            continue
+        if f.op in (">", ">="):
+            column_ops.setdefault(f.left_expr.primary_column, {})[
+                "lower_idx"
+            ] = idx
+        elif f.op in ("<", "<="):
+            column_ops.setdefault(f.left_expr.primary_column, {})[
+                "upper_idx"
+            ] = idx
+    return {
+        col: ops for col, ops in column_ops.items()
+        if "lower_idx" in ops and "upper_idx" in ops
+    }
+
+
+def instantiate_intent(
+    intent: SimulatorIntent,
+    value_domains: dict[str, ValueDomain],
+) -> SimulatorIntent | None:
+    """Populate filter and HAVING values from profiling data.
+
+    Samples one concrete value set per intent.
+
+    Args:
+
+        intent: Abstract simulator intent to instantiate.
+        value_domains: ``table.column`` -> ValueDomain map.
+
+    Returns:
+
+        A new SimulatorIntent with param_values populated, or None.
+    """
+    decomposed: list[FilterParam] = []
+    for f in intent.filters_param:
+        decomposed.extend(_decompose_between_filter_param(f))
+
+    range_pairs = _identify_range_pairs(decomposed)
+    range_values: dict[str, tuple[str, str]] = {}
+    for col_key, pair_indices in range_pairs.items():
+        domain = value_domains.get(col_key)
+        if domain is None:
+            continue
+        lower_idx = pair_indices["lower_idx"]
+        vtype = decomposed[lower_idx].value_type
+        lower_val, upper_val = sample_coordinated_range(domain, vtype, 0)
+        if lower_val is not None and upper_val is not None:
+            range_values[col_key] = (lower_val, upper_val)
+
+    new_filters: list[FilterParam] = []
+    new_param_values: dict[str, Any] = {}
+
+    for filter_idx, f in enumerate(decomposed):
+        col_key = f.left_expr.primary_column
+        op = f.op
+        param_key = f.param_key or f"filter_{filter_idx}"
+
+        if f.right_expr:
+            new_filters.append(f)
+            continue
+
+        if f.value_type == "null" or op in ("is null", "is not null"):
+            new_filters.append(
+                replace(f, op=op, value_type="null", param_key=param_key)
+            )
+            continue
+
+        if f.value_type in ("date_window", "date_diff"):
+            new_filters.append(replace(f, param_key=param_key))
+            continue
+
+        domain = value_domains.get(col_key)
+        if domain is None:
+            new_filters.append(replace(f, param_key=param_key))
+            continue
+
+        if col_key in range_values:
+            lower_val, upper_val = range_values[col_key]
+            if f.op in (">", ">="):
+                value = lower_val
+            elif f.op in ("<", "<="):
+                value = upper_val
+            else:
+                value = sample_value_from_domain(
+                    domain, f.value_type, f.op, 0,
+                )
+        else:
+            value = sample_value_from_domain(
+                domain, f.value_type, f.op, 0,
+            )
+
+        if value is not None:
+            new_param_values[param_key] = value
+        new_filters.append(replace(f, param_key=param_key))
+
+    new_having: list[HavingParam] = []
+    for having_idx, h in enumerate(intent.having_param):
+        param_key = h.param_key or f"having_{having_idx}"
+        if h.right_expr is not None:
+            new_having.append(replace(h, param_key=param_key))
+            continue
+        value = deterministic_having_value(
+            h.left_expr.primary_term, 0, having_idx,
+        )
+        new_param_values[param_key] = value
+        new_having.append(replace(h, param_key=param_key))
+
+    return SimulatorIntent(
+        intent_id=intent.intent_id,
+        tables=intent.tables,
+        grain=intent.grain,
+        select_cols=intent.select_cols,
+        group_by_cols=intent.group_by_cols,
+        order_by_cols=intent.order_by_cols,
+        filters_param=new_filters,
+        having_param=new_having,
+        param_values=new_param_values,
+        cte_steps=intent.cte_steps,
+        question="",
+        expansion_metadata=intent.expansion_metadata,
+        limit=intent.limit,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Template creation
+# ---------------------------------------------------------------------------
+
+def _create_template_from_result(
+    result: SimulatorResult,
+    schema: SchemaGraph,
+    next_id: int,
+    dialect: Any | None = None,
+    source: str = "synthetic",
+    trust_level: int = 0,
+) -> Template | None:
+    """Create a Template from a successful simulation result."""
+    if not result.success or not result.sql:
+        return None
+
+    sql_canon = canonicalize_sql(result.sql)
+    sql_param_str, _ = parameter_abstract(sql_canon)
+    sql_fp_val = sql_fp(sql_param_str)
+
+    intent = result.intent
+    tables_used = extract_tables_from_sql(
+        sql_canon, list(schema.tables.keys()),
+    )
+
+    param_values = (
+        intent.param_values if hasattr(intent, "param_values") else {}
+    )
+    natural_language = (
+        intent.natural_language
+        if hasattr(intent, "natural_language") else ""
+    )
+    chosen_join_id = (
+        intent.chosen_join_candidate_id
+        if hasattr(intent, "chosen_join_candidate_id") else ""
+    )
+
+    ikey = intent_key(intent)
+    tid = f"T{next_id:04d}"
+
+    intent_sig = ConcreteIntent(
+        intent_id=ikey,
+        tables=intent.tables or [],
+        grain=intent.grain or "row_level",
+        select_cols=intent.select_cols or [],
+        group_by_cols=intent.group_by_cols or [],
+        order_by_cols=intent.order_by_cols or [],
+        filters_param=intent.filters_param or [],
+        having_param=intent.having_param or [],
+        cte_steps=(
+            intent.cte_steps if hasattr(intent, "cte_steps") else []
+        ),
+        limit=intent.limit if hasattr(intent, "limit") else None,
+        chosen_join_candidate_id=chosen_join_id,
+        chosen_join_path_signature=(
+            intent.chosen_join_path_signature
+            if hasattr(intent, "chosen_join_path_signature")
+            else []
+        ),
+    )
+
+    spark_sql_param = ""
+    if EngineConfig.TYPE == "databricks" and dialect:
+        spark_sql_param = dialect.prepare_for_execution(sql_param_str)
+
+    tmpl = Template(
+        id=tid,
+        schema_hash=schema.schema_hash,
+        intent_signature=intent_sig,
+        intent_key=ikey,
+        tables_used=sorted(tables_used),
+        sql_param=sql_param_str,
+        spark_sql_param=spark_sql_param,
+        sql_display_param=sql_param_str,
+        sql_fp=sql_fp_val,
+        shape=sql_shape(sql_canon, intent),
+        colmap_sig=colmap_signature({}),
+        value_history=ValueHistory(
+            param_values=[param_values],
+            questions=[result.question],
+            natural_language=[natural_language or ""],
+        ),
+        stats=TemplateStats(accept=0, reject=0),
+        source=source,
+        trust_level=trust_level,
+        aliased_sql="",
+    )
+    debug(f"[simulator.create_template] created: id={tmpl.id}")
+    return tmpl
+
+
+# ---------------------------------------------------------------------------
+# Deterministic simulation pipeline
+# ---------------------------------------------------------------------------
+
+def _build_value_domains(
+    schema: SchemaGraph,
+) -> dict[str, ValueDomain]:
+    """Build value domains from schema column metadata."""
+    domains: dict[str, ValueDomain] = {}
+    for table_name, table_meta in schema.tables.items():
+        for col_name, col_meta in table_meta.columns.items():
+            col_key = f"{table_name}.{col_name}"
+            domains[col_key] = ValueDomain(
+                values=col_meta.top_k_values or [],
+                min_val=col_meta.min_val,
+                max_val=col_meta.max_val,
+            )
+    return domains
+
+
+def run_deterministic_simulation(
+    intents: list[SimulatorIntent],
+    schema: SchemaGraph,
+    dialect: Any,
+    next_id: int,
+    join_cache: dict[frozenset[str], JoinCacheEntry] | None = None,
+    csv_output_path: str | None = None,
+) -> tuple[list[SimulatorResult], list[Template], int]:
+    """Run the deterministic simulation pipeline for expanded intents.
+
+    For each intent: resolve joins (cached), build SQL, instantiate
+    values, substitute, validate + execute, then call LLM for question
+    generation with realism gate.  One question per intent.
+
+    Args:
+
+        intents: Expanded and deduplicated SimulatorIntents.
+        schema: Schema graph.
+        dialect: SQL dialect executor.
+        next_id: Starting template ID counter.
+        join_cache: Shared join cache; created if None.
+        csv_output_path: Optional CSV summary output path.
+
+    Returns:
+
+        Tuple of (results, templates, updated_next_id).
+    """
+    if join_cache is None:
+        join_cache = {}
+
+    value_domains = _build_value_domains(schema)
+    results: list[SimulatorResult] = []
+    templates: list[Template] = []
+    current_id = next_id
+
+    success_count = 0
+    fail_count = 0
+    validation_drop = 0
+    realism_drop = 0
+
+    log(
+        f"run_deterministic_simulation: processing {len(intents)} intents"
+    )
+
+    for intent in intents:
+        runtime = intent.to_runtime_intent()
+        result = SimulatorResult(runtime, "")
+
+        try:
+            join_id, join_sig, candidates = resolve_joins_for_table_set(
+                intent.tables or [], schema,
+                intent.intent_id, join_cache,
+            )
+        except Exception as e:
+            result.error = f"join_resolution_failed: {e}"
+            results.append(result)
+            fail_count += 1
+            continue
+
+        try:
+            fp, hp, cte, _ = assign_param_keys(
+                runtime.filters_param or [],
+                runtime.having_param or [],
+                runtime.cte_steps or [],
+            )
+            runtime.filters_param = fp
+            runtime.having_param = hp
+            runtime.cte_steps = cte
+            extract_structural_params(runtime)
+            collect_raw_param_values(runtime)
+
+            det_sql = build_deterministic_sql(runtime)
+            if candidates and join_id != "J00":
+                det_sql = inject_join_into_deterministic_sql(
+                    det_sql, join_id, candidates,
+                )
+            runtime.sql_param = det_sql
+            runtime.deterministic_sql = det_sql
+            runtime.chosen_join_candidate_id = join_id
+            runtime.chosen_join_path_signature = join_sig
+        except Exception as e:
+            result.error = f"sql_build_failed: {e}"
+            results.append(result)
+            fail_count += 1
+            continue
+
+        instantiated = instantiate_intent(intent, value_domains)
+        if instantiated is None:
+            result.error = "instantiation_failed"
+            results.append(result)
+            fail_count += 1
+            continue
+
+        all_params = dict(instantiated.param_values)
+        all_params.update(runtime.param_values or {})
+        try:
+            final_sql = substitute_params(det_sql, all_params)
+        except Exception as e:
+            result.error = f"substitution_failed: {e}"
+            results.append(result)
+            fail_count += 1
+            continue
+
+        if not final_sql or not final_sql.strip():
+            result.error = "empty_sql_after_substitution"
+            results.append(result)
+            fail_count += 1
+            continue
+
+        try:
+            ok, err = validate_sql(dialect, final_sql)
+        except Exception as e:
+            result.error = f"validation_exception: {e}"
+            results.append(result)
+            validation_drop += 1
+            fail_count += 1
+            continue
+
+        if not ok:
+            result.error = f"validation_failed: {err}"
+            results.append(result)
+            validation_drop += 1
+            fail_count += 1
+            continue
+
+        try:
+            spark_sql = get_spark_sql_for_execution(
+                runtime.sql_param or "",
+                all_params,
+                schema,
+                runtime,
+                dialect,
+            )
+            rows = execute_sql(
+                dialect, final_sql,
+                spark_sql_for_execution=spark_sql if spark_sql else None,
+            )
+        except Exception as e:
+            result.error = f"execution_failed: {e}"
+            results.append(result)
+            validation_drop += 1
+            fail_count += 1
+            continue
+
+        qresult = generate_question_from_sql(
+            final_sql, schema, intent.tables or [],
+        )
+        if qresult is None:
+            result.error = "question_generation_failed"
+            results.append(result)
+            fail_count += 1
+            continue
+
+        if not qresult.get("is_realistic", False):
+            result.error = (
+                f"realism_dropped: {qresult.get('drop_reason', '')}"
+            )
+            results.append(result)
+            realism_drop += 1
+            fail_count += 1
+            continue
+
+        question = qresult["question"]
+        result.sql = final_sql
+        result.question = question
+        result.rows = rows
+        result.success = True
+        result.confidence = 1.0
+
+        runtime.param_values = all_params
+
+        tmpl = _create_template_from_result(
+            result, schema, current_id, dialect,
+        )
+        if tmpl:
+            templates.append(tmpl)
+            current_id += 1
+        success_count += 1
+        results.append(result)
+
+    log(
+        f"run_deterministic_simulation: "
+        f"{success_count} success, {fail_count} failed "
+        f"(validation_drop={validation_drop}, "
+        f"realism_drop={realism_drop}), "
+        f"{len(templates)} templates"
+    )
+
+    if csv_output_path:
+        with open(csv_output_path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.writer(f)
+            writer.writerow(["question", "sql", "status", "error"])
+            for r in results:
+                writer.writerow([
+                    r.question,
+                    r.sql or "",
+                    "success" if r.success else "failed",
+                    r.error or "",
+                ])
+
+    return results, templates, current_id
+
+
+# ---------------------------------------------------------------------------
+# Report persistence
+# ---------------------------------------------------------------------------
+
+def save_simulation_report(
+    results: list[SimulatorResult], filepath: str,
+) -> None:
+    """Save simulation results to a JSON report file."""
+
+    def result_to_dict(r: SimulatorResult) -> dict[str, Any]:
+        return {
+            "question": r.question,
+            "intent": r.intent,
+            "sql": r.sql,
+            "success": r.success,
+            "error": r.error,
+            "confidence": r.confidence,
+            "validation_issues": r.validation_issues,
+        }
+
+    report = {
+        "total": len(results),
+        "success": sum(1 for r in results if r.success),
+        "failed": sum(1 for r in results if not r.success),
+        "results": [result_to_dict(r) for r in results],
+    }
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2, ensure_ascii=False)
+    log(f"save_simulation_report: saved {len(results)} results to {filepath}")
+
+
+def save_simulation_failures(
+    results: list[SimulatorResult], filepath: str,
+) -> None:
+    """Save detailed failure information to a JSON file."""
+    failures = []
+    for r in results:
+        if not r.success and r.error:
+            error_category = (
+                r.error.split(":")[0] if ":" in r.error else "unknown_error"
+            )
+            failures.append({
+                "question": r.question,
+                "intent": r.intent,
+                "sql": r.sql or "",
+                "llm_response": r.llm_response or "",
+                "error_message": r.error,
+                "error_category": error_category,
+                "validation_issues": r.validation_issues,
+            })
+    if failures:
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(failures, f, indent=2, ensure_ascii=False)
+        log(
+            f"save_simulation_failures: saved {len(failures)} "
+            f"failures to {filepath}"
+        )
+    else:
+        log("save_simulation_failures: no failures to save")