aetherdialect 0.1.0__py3-none-any.whl

text2sql/templates.py

@@ -0,0 +1,1037 @@
+"""Template and rejected template persistence with trust management and promotion workflows.
+
+Manages the on-disk template store for accepted and rejected query patterns. Accepted templates carry trust levels (0/1/2) that are promoted or demoted based on user feedback ratios. Rejected templates accumulate category-specific rejection counters used to decide hard blocks. Provides lookup, similarity-based search, promotion and demotion workflows between the two stores, and serialisation helpers for the JSON store format.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+from .config import EngineConfig, PolicyConfig
+from .contracts_base import SchemaGraph, TemplateStats
+from .contracts_core import (
+    RejectedTemplate,
+    RejectedValueHistory,
+    RuntimeIntent,
+    Template,
+    ValueHistory,
+    runtime_intent_to_concrete,
+)
+from .core_utils import (
+    canonicalize_sql,
+    colmap_signature,
+    debug,
+    normalize_sql,
+    parameter_abstract,
+    sql_fp,
+)
+from .intent_process import compute_intent_union, intent_similarity, llm_ux_explain
+from .dialect import get_dialect
+from .utils import extract_tables_from_sql, flatten_param_values, intent_key, sql_shape
+
+
+def increment_rejected_template(
+    rt: RejectedTemplate,
+    category: str,
+    reason: str,
+    q_norm: str,
+    intent: RuntimeIntent,
+    sql: str,
+) -> None:
+    """Increment rejection count and update categories and reasons for a rejected template.
+
+    Args:
+
+        rt: The ``RejectedTemplate`` to update in-place.
+
+        category: Rejection category string (for example, ``"schema_mismatch"``).
+
+        reason: Human-readable rejection reason.
+
+        q_norm: The normalised question that triggered the rejection.
+
+        intent: The ``RuntimeIntent`` associated with the rejection.
+
+        sql: The SQL that was rejected (currently unused but kept for signature consistency).
+    """
+    all_pv = flatten_param_values(intent)
+    rt.value_history.add(all_pv, q_norm, intent.natural_language, reason, category)
+    debug(f"[templates.increment_rejected_template] updated: id={rt.id} entries={len(rt.value_history.questions)}")
+
+
+def get_hard_block_info(rt: RejectedTemplate) -> dict[str, Any]:
+    """Get hard block information for a rejected template based on category-specific thresholds.
+
+    Args:
+
+        rt: The ``RejectedTemplate`` to evaluate.
+
+    Returns:
+
+        Dict with keys ``should_block`` (bool), ``total_rejects`` (int), ``categories`` (dict of category to count), ``blocking_category`` (str or ``None``), and ``last_reason`` (str).
+    """
+    category_counts = {}
+    vh = rt.value_history
+    total_rejections = len(vh.rejection_categories)
+
+    for cat in vh.rejection_categories:
+        category_counts[cat] = category_counts.get(cat, 0) + 1
+
+    last_reason = vh.rejection_reasons[-1] if vh.rejection_reasons else ""
+
+    blocking_category = None
+    for cat in PolicyConfig.STRUCTURAL_REJECT_CATEGORIES:
+        if category_counts.get(cat, 0) >= 2:
+            blocking_category = cat
+            break
+
+    return {
+        "should_block": blocking_category is not None,
+        "total_rejects": total_rejections,
+        "categories": category_counts,
+        "blocking_category": blocking_category,
+        "last_reason": last_reason,
+    }
+
+
+def get_aggregated_hard_block_info(
+    rejected: dict[str, RejectedTemplate],
+    intent: RuntimeIntent,
+    similarity_threshold: float = 0.70,
+) -> dict[str, Any]:
+    """Aggregate hard block info across all similar rejected templates.
+
+    Args:
+
+        rejected: The full rejected template dict.
+
+        intent: The ``RuntimeIntent`` to match against.
+
+        similarity_threshold: Minimum similarity score to include a rejected template.
+
+    Returns:
+
+        Dict with keys ``should_block`` (bool), ``total_rejects``, ``total_structural_rejects``, ``categories``, ``blocking_category``, and ``matched_templates`` (list of ``(RejectedTemplate, score)`` tuples).
+    """
+
+    category_counts = {}
+    total_structural_rejections = 0
+    total_rejections = 0
+    matched_templates = []
+
+    for rt in rejected.values():
+        sim = intent_similarity(intent, rt.intent_signature) if rt.intent_signature else 0.0
+        if sim >= similarity_threshold:
+            matched_templates.append((rt, sim))
+            for cat in rt.value_history.rejection_categories:
+                category_counts[cat] = category_counts.get(cat, 0) + 1
+                total_rejections += 1
+                if cat in PolicyConfig.STRUCTURAL_REJECT_CATEGORIES:
+                    total_structural_rejections += 1
+
+    debug(f"[templates.get_aggregated_hard_block_info] found {len(matched_templates)} similar rejected templates")
+    debug(f"[templates.get_aggregated_hard_block_info] category_counts: {category_counts}")
+    debug(f"[templates.get_aggregated_hard_block_info] total_structural_rejections: {total_structural_rejections}")
+
+    should_block = total_structural_rejections >= 2
+
+    blocking_category = None
+    if should_block:
+        for cat in PolicyConfig.STRUCTURAL_REJECT_CATEGORIES:
+            if cat in category_counts:
+                if blocking_category is None or category_counts[cat] > category_counts.get(blocking_category, 0):
+                    blocking_category = cat
+
+        debug(
+            f"[templates.get_aggregated_hard_block_info] BLOCK: total_structural={total_structural_rejections} primary_category={blocking_category}"
+        )
+
+    return {
+        "should_block": should_block,
+        "total_rejects": total_rejections,
+        "total_structural_rejects": total_structural_rejections,
+        "categories": category_counts,
+        "blocking_category": blocking_category,
+        "matched_templates": matched_templates,
+    }
+
+
+def find_rejected_by_intent_key(rejected: dict[str, RejectedTemplate], ikey: str) -> RejectedTemplate | None:
+    """Find a rejected template by intent key.
+
+    Args:
+
+        rejected: The full rejected template dict.
+
+        ikey: The intent key string to search for.
+
+    Returns:
+
+        The matching ``RejectedTemplate``, or ``None`` if not found.
+    """
+    for rt in rejected.values():
+        if rt.intent_key == ikey:
+            return rt
+    return None
+
+
+def find_rejected_by_similarity(
+    rejected: dict[str, RejectedTemplate],
+    intent: RuntimeIntent,
+    threshold: float = 0.85,
+) -> RejectedTemplate | None:
+    """Find best matching rejected template by similarity.
+
+    Args:
+
+        rejected: The full rejected template dict.
+
+        intent: The ``RuntimeIntent`` to match against.
+
+        threshold: Minimum similarity score to consider a match.
+
+    Returns:
+
+        The best-matching ``RejectedTemplate`` above the threshold, or ``None``.
+    """
+    best_rt = None
+    best_sim = 0.0
+
+    for rt in rejected.values():
+        sim = intent_similarity(intent, rt.intent_signature) if rt.intent_signature else 0.0
+        if sim >= threshold and sim > best_sim:
+            best_sim = sim
+            best_rt = rt
+
+    if best_rt:
+        debug(f"[templates.find_rejected_by_similarity] found: id={best_rt.id} sim={best_sim:.3f}")
+    return best_rt
+
+
+def promote_rejected_to_template(
+    store: dict[str, Any],
+    templates: dict[str, Template],
+    rejected: dict[str, RejectedTemplate],
+    rt: RejectedTemplate,
+    q_norm: str,
+    intent: RuntimeIntent,
+    sql: str,
+    schema_hash: str,
+) -> Template:
+    """Promote a rejected template to an accepted template after user override and acceptance.
+
+    Creates a new ``Template`` from the rejected template's data, removes the rejected entry from the store, and cleans up associated negative memory entries.
+
+    Args:
+
+        store: The mutable template store dict.
+
+        templates: The accepted templates dict to insert into.
+
+        rejected: The rejected templates dict to remove from.
+
+        rt: The ``RejectedTemplate`` being promoted.
+
+        q_norm: The normalised question for the initial value history entry.
+
+        intent: The ``RuntimeIntent`` at the time of promotion.
+
+        sql: The SQL that was accepted.
+
+        schema_hash: The schema hash to stamp on the new template.
+
+    Returns:
+
+        The newly created ``Template``.
+    """
+    tid = f"T{int(store['next_id']):04d}"
+    store["next_id"] = int(store["next_id"]) + 1
+
+    sql_canon = canonicalize_sql(sql)
+    if intent.sql_param:
+        sql_param = intent.sql_param
+    else:
+        sql_norm = normalize_sql(sql_canon)
+        sql_param, _ = parameter_abstract(sql_norm)
+    sql_fp_val = sql_fp(sql_param)
+
+    colmap_sig_val = colmap_signature(intent.column_map)
+
+    intent_signature = runtime_intent_to_concrete(intent, "")
+
+    all_pv = flatten_param_values(intent)
+    ux_summary = llm_ux_explain(intent, q_norm)
+
+    tmpl = Template(
+        id=tid,
+        schema_hash=schema_hash,
+        intent_signature=intent_signature,
+        intent_key=rt.intent_key,
+        tables_used=sorted(intent.tables),
+        sql_param=sql_param,
+        spark_sql_param=getattr(rt, "spark_sql_param", "") or "",
+        sql_display_param=rt.sql_display_param or intent.sql_display_param or sql_param,
+        sql_fp=sql_fp_val,
+        shape=sql_shape(sql_canon, intent),
+        colmap_sig=colmap_sig_val,
+        value_history=ValueHistory(
+            param_values=[all_pv],
+            questions=[q_norm],
+            natural_language=[intent.natural_language or ""],
+        ),
+        stats=TemplateStats(accept=1, reject=0),
+        ux_summary=ux_summary,
+        source="human",
+        trust_level=1,
+        structural_defaults={k: v for k, v in all_pv.items() if k.startswith("s")},
+        aliased_sql=rt.aliased_sql or intent.sql_display_param or "",
+    )
+
+    templates[tid] = tmpl
+
+    _cleanup_negative_memory_for_rejected(store, rt)
+
+    if rt.id in rejected:
+        del rejected[rt.id]
+        debug(f"[templates.promote_rejected_to_template] removed_rejected: id={rt.id}")
+
+    debug(f"[templates.promote_rejected_to_template] created_template: id={tid} from_rejected={rt.id}")
+    return tmpl
+
+
+def _cleanup_negative_memory_for_rejected(store: dict[str, Any], rt: RejectedTemplate) -> None:
+    """Remove negative memory entries associated with a promoted rejected template.
+
+    Cleans up ``by_intent_key``, ``by_sql_fp``, ``by_join_sig``, and ``by_colmap_sig`` entries in the store's ``negative_memory`` section that correspond to this rejected template.
+
+    Args:
+
+        store: The mutable template store dict containing ``negative_memory``.
+
+        rt: The ``RejectedTemplate`` whose negative memory entries should be removed.
+    """
+    mem = store.get("negative_memory", {})
+    vh = rt.value_history
+    reasons_to_remove = list(vh.rejection_reasons)
+    categories_to_remove = list(vh.rejection_categories)
+
+    if rt.intent_key and rt.intent_key in mem.get("by_intent_key", {}):
+        del mem["by_intent_key"][rt.intent_key]
+        debug(f"[templates.cleanup_negative_memory_for_rejected] removed by_intent_key: {rt.intent_key[:16]}...")
+
+    if rt.sql_fp and rt.sql_fp in mem.get("by_sql_fp", {}):
+        del mem["by_sql_fp"][rt.sql_fp]
+        debug(f"[templates.cleanup_negative_memory_for_rejected] removed by_sql_fp: {rt.sql_fp[:16]}...")
+
+    join_sig_key = str(rt.chosen_join_path_signature) if rt.chosen_join_path_signature else None
+    if join_sig_key and join_sig_key in mem.get("by_join_sig", {}):
+        bucket = mem["by_join_sig"][join_sig_key]
+        for reason, cat in zip(reasons_to_remove, categories_to_remove, strict=False):
+            if reason in bucket.get("reasons", []):
+                bucket["reasons"].remove(reason)
+                bucket["rejects"] = max(0, bucket.get("rejects", 0) - 1)
+                if cat in bucket.get("categories", {}):
+                    bucket["categories"][cat] = max(0, bucket["categories"].get(cat, 0) - 1)
+                    if bucket["categories"][cat] == 0:
+                        del bucket["categories"][cat]
+        if bucket.get("rejects", 0) == 0:
+            del mem["by_join_sig"][join_sig_key]
+            debug(f"[templates.cleanup_negative_memory_for_rejected] removed by_join_sig: {join_sig_key[:32]}...")
+        else:
+            debug(
+                f"[templates.cleanup_negative_memory_for_rejected] decremented by_join_sig: {join_sig_key[:32]}... rejects={bucket['rejects']}"
+            )
+
+    if rt.colmap_sig and rt.colmap_sig in mem.get("by_colmap_sig", {}):
+        bucket = mem["by_colmap_sig"][rt.colmap_sig]
+        for reason, cat in zip(reasons_to_remove, categories_to_remove, strict=False):
+            if reason in bucket.get("reasons", []):
+                bucket["reasons"].remove(reason)
+                bucket["rejects"] = max(0, bucket.get("rejects", 0) - 1)
+                if cat in bucket.get("categories", {}):
+                    bucket["categories"][cat] = max(0, bucket["categories"].get(cat, 0) - 1)
+                    if bucket["categories"][cat] == 0:
+                        del bucket["categories"][cat]
+        if bucket.get("rejects", 0) == 0:
+            del mem["by_colmap_sig"][rt.colmap_sig]
+            debug(f"[templates.cleanup_negative_memory_for_rejected] removed by_colmap_sig: {rt.colmap_sig[:16]}...")
+        else:
+            debug(
+                f"[templates.cleanup_negative_memory_for_rejected] decremented by_colmap_sig: {rt.colmap_sig[:16]}... rejects={bucket['rejects']}"
+            )
+
+
+def demote_template_to_rejected(
+    store: dict[str, Any],
+    templates: dict[str, Template],
+    rejected: dict[str, RejectedTemplate],
+    tmpl: Template,
+    schema: SchemaGraph,
+    intent: RuntimeIntent,
+    sql: str,
+    q_norm: str,
+    category: str,
+    reason: str,
+) -> RejectedTemplate:
+    """Demote an accepted template to a rejected template.
+
+    Removes the template from the accepted store, creates a new ``RejectedTemplate`` with the rejection metadata, and inserts it into the rejected store.
+
+    Args:
+
+        store: The mutable template store dict.
+
+        templates: The accepted templates dict to remove from.
+
+        rejected: The rejected templates dict to insert into.
+
+        tmpl: The ``Template`` being demoted.
+
+        schema: The schema graph (for ``schema_hash`` and table enumeration).
+
+        intent: The ``RuntimeIntent`` at the time of demotion.
+
+        sql: The SQL that was rejected.
+
+        q_norm: The normalised question.
+
+        category: The rejection category string.
+
+        reason: The human-readable rejection reason.
+
+    Returns:
+
+        The newly created ``RejectedTemplate``.
+    """
+    rid = f"R{int(store.get('next_reject_id', 1)):04d}"
+    store["next_reject_id"] = int(store.get("next_reject_id", 1)) + 1
+
+    sql_canon = canonicalize_sql(sql)
+    if intent.sql_param:
+        sql_param = intent.sql_param
+    else:
+        sql_norm = normalize_sql(sql_canon)
+        sql_param, _ = parameter_abstract(sql_norm)
+    sql_fp_val = sql_fp(sql_param)
+    tables_used = extract_tables_from_sql(sql_canon, list(schema.tables.keys()))
+    colmap_sig_val = colmap_signature(intent.column_map)
+
+    intent_sig = runtime_intent_to_concrete(intent, "")
+
+    all_pv = flatten_param_values(intent)
+
+    rt = RejectedTemplate(
+        id=rid,
+        schema_hash=schema.schema_hash,
+        intent_signature=intent_sig,
+        intent_key=tmpl.intent_key,
+        tables_used=sorted(tables_used),
+        sql_param=sql_param,
+        spark_sql_param=getattr(tmpl, "spark_sql_param", "") or "",
+        sql_display_param=tmpl.sql_display_param or sql_param,
+        sql_fp=sql_fp_val,
+        shape=sql_shape(sql_canon, intent),
+        colmap_sig=colmap_sig_val,
+        value_history=RejectedValueHistory(
+            param_values=[all_pv],
+            questions=[q_norm],
+            natural_language=[intent.natural_language or ""],
+            rejection_reasons=[reason],
+            rejection_categories=[category],
+        ),
+        aliased_sql=tmpl.aliased_sql or "",
+    )
+
+    rejected[rid] = rt
+
+    if tmpl.id in templates:
+        del templates[tmpl.id]
+
+    debug(f"[templates.demote_template_to_rejected] demoted: template={tmpl.id} to_rejected={rid}")
+    return rt
+
+
+def promote_trust(template: Template) -> bool:
+    """Promote template trust level based on accept count and ratio.
+
+    Trust 0 → 1 is immediate on first call. Trust 1 → 2 requires total count greater than or equal to ``PolicyConfig.TRUST_PROMOTE_MIN_TOTAL`` and reject ratio less than or equal to ``PolicyConfig.TRUST_PROMOTE_MAX_REJECT_RATIO``.
+
+    Args:
+
+        template: The ``Template`` to potentially promote.
+
+    Returns:
+
+        ``True`` if the trust level was incremented, ``False`` otherwise.
+    """
+    accept_count = template.stats.accept
+    reject_count = template.stats.reject
+    total_count = accept_count + reject_count
+    current_trust = template.trust_level
+
+    if current_trust >= 2:
+        return False
+
+    if current_trust == 0:
+        template.trust_level = 1
+        debug(f"[templates.promote_trust] promoted: id={template.id} level=1")
+        return True
+
+    if current_trust == 1 and total_count >= PolicyConfig.TRUST_PROMOTE_MIN_TOTAL:
+        reject_ratio = reject_count / total_count
+        if reject_ratio <= PolicyConfig.TRUST_PROMOTE_MAX_REJECT_RATIO:
+            template.trust_level = 2
+            debug(f"[templates.promote_trust] promoted: id={template.id} level=2 ratio={reject_ratio:.2f}")
+            return True
+
+    return False
+
+
+def record_template_feedback(template: Template, accept: bool) -> None:
+    """Record accept or reject feedback on a template.
+
+    Args:
+
+        template: The ``Template`` to update in-place.
+
+        accept: ``True`` to increment the accept counter, ``False`` for the reject counter.
+    """
+    if accept:
+        template.stats.accept += 1
+    else:
+        template.stats.reject += 1
+    debug(f"[templates.record_template_feedback] recorded: id={template.id} accept={accept}")
+
+
+def maybe_demote_trust(template: Template) -> bool:
+    """Demote template trust level based on reject ratio.
+
+    Trust 2 → 1 when reject ratio exceeds ``PolicyConfig.TRUST_DEMOTE_REJECT_RATIO_T2``. Trust 1 → 0 when ratio exceeds ``PolicyConfig.TRUST_DEMOTE_REJECT_RATIO_T1``.
+
+    Args:
+
+        template: The ``Template`` to potentially demote.
+
+    Returns:
+
+        ``True`` if the trust level was decremented, ``False`` otherwise.
+    """
+    accept_count = template.stats.accept
+    reject_count = template.stats.reject
+    total_count = accept_count + reject_count
+    current_trust = template.trust_level
+
+    if total_count == 0:
+        return False
+
+    reject_ratio = reject_count / total_count
+
+    if current_trust == 2 and reject_ratio > PolicyConfig.TRUST_DEMOTE_REJECT_RATIO_T2:
+        template.trust_level = 1
+        debug(f"[templates.maybe_demote_trust] demoted: id={template.id} level=1 ratio={reject_ratio:.2f}")
+        return True
+    elif current_trust == 1 and reject_ratio > PolicyConfig.TRUST_DEMOTE_REJECT_RATIO_T1:
+        template.trust_level = 0
+        debug(f"[templates.maybe_demote_trust] demoted: id={template.id} level=0 ratio={reject_ratio:.2f}")
+        return True
+
+    return False
+
+
+def load_template_store(schema_hash: str) -> dict[str, Any]:
+    """Load template store from disk or create empty one.
+
+    Returns a fresh empty store if the file does not exist or if the stored schema hash does not match the provided hash.
+
+    Args:
+
+        schema_hash: The current schema hash used to invalidate stale stores.
+
+    Returns:
+
+        The template store dict with ``templates``, ``rejected_templates``, ``negative_memory``, and counter fields populated.
+    """
+    if PolicyConfig.REGENERATE_TEMPLATE_STORE:
+        debug("[templates.load_template_store] REGENERATE_TEMPLATE_STORE: returning empty store")
+        return {
+            "schema_hash": schema_hash,
+            "next_id": 1,
+            "next_reject_id": 1,
+            "templates": {},
+            "rejected_templates": {},
+            "negative_memory": {
+                "by_intent_key": {},
+                "by_join_sig": {},
+                "by_sql_fp": {},
+                "by_colmap_sig": {},
+            },
+        }
+
+    template_json_path = EngineConfig.TEMPLATE_JSON_PATH
+
+    if not os.path.exists(template_json_path):
+        debug(f"[templates.load_template_store] no_file: path={template_json_path}")
+        return {
+            "schema_hash": schema_hash,
+            "next_id": 1,
+            "next_reject_id": 1,
+            "templates": {},
+            "rejected_templates": {},
+            "negative_memory": {
+                "by_intent_key": {},
+                "by_join_sig": {},
+                "by_sql_fp": {},
+                "by_colmap_sig": {},
+            },
+        }
+    with open(template_json_path, encoding="utf-8") as f:
+        d = json.load(f)
+    if d.get("schema_hash") != schema_hash:
+        debug("[templates.load_template_store] schema_hash_mismatch: resetting")
+        return {
+            "schema_hash": schema_hash,
+            "next_id": 1,
+            "next_reject_id": 1,
+            "templates": {},
+            "rejected_templates": {},
+            "negative_memory": {
+                "by_intent_key": {},
+                "by_join_sig": {},
+                "by_sql_fp": {},
+                "by_colmap_sig": {},
+            },
+        }
+    d.setdefault("next_id", 1)
+    d.setdefault("next_reject_id", 1)
+    d.setdefault("templates", {})
+    d.setdefault("rejected_templates", {})
+    d.setdefault("negative_memory", {})
+    d["negative_memory"].setdefault("by_intent_key", {})
+    d["negative_memory"].setdefault("by_join_sig", {})
+    d["negative_memory"].setdefault("by_sql_fp", {})
+    d["negative_memory"].setdefault("by_colmap_sig", {})
+    debug(
+        f"[templates.load_template_store] loaded: templates={len(d.get('templates', {}))} rejected={len(d.get('rejected_templates', {}))}"
+    )
+    return d
+
+
+def save_template_store(store: dict[str, Any]) -> None:
+    """Save template store to disk.
+
+    Converts all non-serialisable objects (for example, sets) before writing JSON.
+
+    Args:
+
+        store: The template store dict to serialise and save.
+    """
+    template_json_path = EngineConfig.TEMPLATE_JSON_PATH
+
+    debug(
+        f"[templates.save_template_store] saving: templates={len(store.get('templates', {}))} rejected={len(store.get('rejected_templates', {}))}"
+    )
+    _debug_check_types(store, "store")
+    store = _convert_to_json_serializable(store)
+    with open(template_json_path, "w", encoding="utf-8") as f:
+        json.dump(store, f, indent=2, sort_keys=True, ensure_ascii=False)
+    debug(f"[templates.save_template_store] complete: path={template_json_path}")
+
+
+def store_to_templates(store: dict[str, Any]) -> dict[str, Template]:
+    """Convert store dict to ``Template`` objects with nested dataclass reconstruction.
+
+    Args:
+
+        store: The raw template store dict loaded from disk.
+
+    Returns:
+
+        Dict mapping template ID to ``Template`` dataclass instance.
+    """
+    out = {}
+    for tid, v in store.get("templates", {}).items():
+        t = Template.from_dict(v)
+        t.sql_fp = sql_fp(parameter_abstract(canonicalize_sql(t.sql_param))[0])
+        out[tid] = t
+    return out
+
+
+def store_to_rejected_templates(store: dict[str, Any]) -> dict[str, RejectedTemplate]:
+    """Convert store dict to ``RejectedTemplate`` objects with nested dataclass reconstruction.
+
+    Args:
+
+        store: The raw template store dict loaded from disk.
+
+    Returns:
+
+        Dict mapping rejected template ID to ``RejectedTemplate`` dataclass instance.
+    """
+    out: dict[str, RejectedTemplate] = {}
+    for rid, v in store.get("rejected_templates", {}).items():
+        rt = RejectedTemplate.from_dict(v)
+        rt.sql_fp = sql_fp(parameter_abstract(canonicalize_sql(rt.sql_param))[0])
+        out[rid] = rt
+    return out
+
+
+def _convert_to_json_serializable(obj: Any) -> Any:
+    if isinstance(obj, (set, frozenset)):
+        return list(obj)
+    elif isinstance(obj, dict):
+        return {k: _convert_to_json_serializable(v) for k, v in obj.items()}
+    elif isinstance(obj, list | tuple):
+        return [_convert_to_json_serializable(item) for item in obj]
+    return obj
+
+
+def _debug_check_types(obj: Any, path: str = "root") -> None:
+    if isinstance(obj, set):
+        debug(f"[templates.debug_check_types] found_set: path={path}")
+    elif isinstance(obj, dict):
+        for k, v in obj.items():
+            _debug_check_types(v, f"{path}.{k}")
+    elif isinstance(obj, list | tuple):
+        for i, item in enumerate(obj):
+            _debug_check_types(item, f"{path}[{i}]")
+
+
+def templates_to_store(store: dict[str, Any], templates: dict[str, Template]) -> dict[str, Any]:
+    """Convert Template objects to store dict format.
+
+    Args:
+
+        store: The mutable template store dict to update in-place.
+
+        templates: Dict of template ID to ``Template`` to serialise.
+
+    Returns:
+
+        The updated store dict with ``templates`` key populated.
+    """
+    debug(f"[templates.templates_to_store] converting: count={len(templates)}")
+    for tid, t in templates.items():
+        template_dict = t.to_dict()
+        _debug_check_types(template_dict, f"template[{tid}]")
+    store["templates"] = {k: _convert_to_json_serializable(v.to_dict()) for k, v in templates.items()}
+    return store
+
+
+def rejected_templates_to_store(store: dict[str, Any], rejected: dict[str, RejectedTemplate]) -> dict[str, Any]:
+    """Convert ``RejectedTemplate`` objects back to store dict with nested dataclass conversion.
+
+    Args:
+
+        store: The mutable template store dict to update in-place.
+
+        rejected: Dict of rejected template ID to ``RejectedTemplate`` to serialise.
+
+    Returns:
+
+        The updated store dict with ``rejected_templates`` key populated.
+    """
+    store["rejected_templates"] = {k: _convert_to_json_serializable(v.to_dict()) for k, v in rejected.items()}
+    return store
+
+
+def top_rejected_examples(
+    rejected: dict[str, RejectedTemplate],
+    intent: RuntimeIntent,
+    k: int = 3,
+    exclude_id: str = None,
+) -> list[dict[str, Any]]:
+    """Get top ``k`` rejected examples for negative learning based on similarity.
+
+    Only considers rejected templates with at least two rejections and at least one structural rejection category. Ranks candidates by similarity to the intent.
+
+    Args:
+
+        rejected: The full rejected template dict.
+
+        intent: The ``RuntimeIntent`` to match against.
+
+        k: Maximum number of examples to return.
+
+        exclude_id: Optional rejected template ID to exclude from results.
+
+    Returns:
+
+        List of dicts each describing a rejected example with ``sql_param``, join path, intent fields, category, and reason.
+    """
+    debug(f"[templates.top_rejected_examples] searching: k={k} rejected_count={len(rejected)}")
+
+    scored: list[tuple] = []
+    for r in rejected.values():
+        if exclude_id and r.id == exclude_id:
+            debug(f"[templates.top_rejected_examples] skipping: id={r.id} excluded")
+            continue
+        vh = r.value_history
+        reject_count = len(vh.rejection_categories)
+        if reject_count <= 1:
+            continue
+
+        category_counts: dict[str, int] = {}
+        for cat in vh.rejection_categories:
+            category_counts[cat] = category_counts.get(cat, 0) + 1
+
+        has_structural = any(category_counts.get(cat, 0) > 0 for cat in PolicyConfig.STRUCTURAL_REJECT_CATEGORIES)
+        if not has_structural:
+            continue
+
+        sim = intent_similarity(intent, r.intent_signature) if r.intent_signature else 0.0
+        if sim >= PolicyConfig.AUTO_PROCEED_THRESHOLD:
+            scored.append((r, sim))
+
+    scored.sort(key=lambda x: (-x[1], x[0].id))
+    debug(f"[templates.top_rejected_examples] candidates: {len(scored)}")
+
+    out: list[dict[str, Any]] = []
+    for r, sim in scored[:k]:
+        vh = r.value_history
+        last_category = vh.rejection_categories[-1] if vh.rejection_categories else ""
+        last_reason = vh.rejection_reasons[-1] if vh.rejection_reasons else ""
+        intent_sig = r.intent_signature
+
+        out.append(
+            {
+                "sql_param": r.sql_param,
+                "join_path_signature": r.chosen_join_path_signature or [],
+                "tables": intent_sig.tables if intent_sig else [],
+                "grain": intent_sig.grain if intent_sig else "row_level",
+                "select_cols": ([s.to_dict() for s in (intent_sig.select_cols or [])] if intent_sig else []),
+                "group_by_cols": intent_sig.group_by_cols if intent_sig else [],
+                "order_by_cols": ([o.to_dict() for o in (intent_sig.order_by_cols or [])] if intent_sig else []),
+                "filters_param": ([f.to_dict() for f in (intent_sig.filters_param or [])] if intent_sig else []),
+                "having_param": ([h.to_dict() for h in (intent_sig.having_param or [])] if intent_sig else []),
+                "category": last_category,
+                "reason": last_reason,
+            }
+        )
+        debug(f"[templates.top_rejected_examples] selected: id={r.id} sim={sim:.3f}")
+
+    debug(f"[templates.top_rejected_examples] returning: {len(out)} examples")
+    return out
+
+
+def insert_template(
+    store: dict,
+    templates: dict[str, Template],
+    schema,
+    q_norm: str,
+    intent: RuntimeIntent,
+    sql: str,
+    ux_summary: str = "",
+    dialect: Any | None = None,
+) -> Template:
+    """Create and insert a new accepted template, or merge question into an existing one.
+
+    If a template with the same ``(intent_key, join_path, sql_fp)`` triple already exists, the question is merged into its value history and its accept count is incremented. Otherwise a new ``Template`` is created.
+
+    Args:
+
+        store: The mutable template store dict.
+
+        templates: The accepted templates dict.
+
+        schema: The schema graph (for ``schema_hash`` and table enumeration).
+
+        q_norm: The normalised question.
+
+        intent: The ``RuntimeIntent``.
+
+        sql: The accepted SQL.
+
+        ux_summary: Optional pre-computed UX summary; generated via LLM if empty.
+
+    Returns:
+
+        The existing or newly created ``Template``.
+    """
+    sql_canon = canonicalize_sql(sql)
+    if intent.sql_param:
+        sql_param = intent.sql_param
+    else:
+        sql_norm = normalize_sql(sql_canon)
+        sql_param, _ = parameter_abstract(sql_norm)
+    sql_fp_val = sql_fp(sql_param)
+    tables_used = extract_tables_from_sql(sql_canon, list(schema.tables.keys()))
+    colmap_sig_val = colmap_signature(intent.column_map)
+    ikey = intent_key(intent)
+
+    intent_sig = runtime_intent_to_concrete(intent, "")
+
+    debug(f"[templates.insert_template] checking_duplicate: ikey={ikey[:32]} sql_fp={sql_fp_val[:16]})")
+
+    for t in templates.values():
+        if t.intent_key != ikey:
+            continue
+        _, cc = compute_intent_union(intent, t.intent_signature)
+        if not cc:
+            debug(f"[templates.insert_template] duplicate_found: id={t.id}")
+            t.stats.accept += 1
+            promote_trust(t)
+
+            all_pv = flatten_param_values(intent)
+            t.value_history.add(all_pv, q_norm, intent.natural_language)
+            debug(f"[templates.insert_template] value_history_added: entries={len(t.value_history.questions)}")
+            return t
+
+    debug("[templates.insert_template] no_duplicate: creating_new")
+
+    tid = f"T{int(store['next_id']):04d}"
+    store["next_id"] = int(store["next_id"]) + 1
+
+    all_pv = flatten_param_values(intent)
+    if not ux_summary:
+        ux_summary = llm_ux_explain(intent, q_norm)
+
+    spark_sql_param = ""
+    if EngineConfig.TYPE == "databricks":
+        d = dialect or get_dialect()
+        if d:
+            spark_sql_param = d.prepare_for_execution(sql_param)
+
+    tmpl = Template(
+        id=tid,
+        schema_hash=schema.schema_hash,
+        intent_signature=intent_sig,
+        intent_key=ikey,
+        tables_used=sorted(tables_used),
+        sql_param=sql_param,
+        spark_sql_param=spark_sql_param,
+        sql_display_param=intent.sql_display_param or sql_param,
+        sql_fp=sql_fp_val,
+        shape=intent.sql_shape if intent.sql_shape else sql_shape(sql_canon, intent),
+        colmap_sig=colmap_sig_val,
+        value_history=ValueHistory(
+            param_values=[all_pv],
+            questions=[q_norm],
+            natural_language=[intent.natural_language or ""],
+        ),
+        stats=TemplateStats(accept=1, reject=0),
+        ux_summary=ux_summary,
+        source="human",
+        trust_level=1,
+        structural_defaults={k: v for k, v in all_pv.items() if k.startswith("s")},
+        deterministic_sql=intent.deterministic_sql,
+        aliased_sql=intent.sql_display_param or "",
+    )
+
+    debug(
+        f"[templates.insert_template] CREATED template natural_language={tmpl.value_history.natural_language} chosen_join_candidate_id='{tmpl.chosen_join_candidate_id}' chosen_join_path_signature={tmpl.chosen_join_path_signature}"
+    )
+
+    templates[tid] = tmpl
+    debug(f"[templates.insert_template] created: id={tid}")
+    return tmpl
+
+
+def insert_rejected_template(
+    store: dict[str, Any],
+    rejected: dict[str, RejectedTemplate],
+    schema: SchemaGraph,
+    q_norm: str,
+    intent: RuntimeIntent,
+    sql: str,
+    category: str,
+    reason: str,
+    dialect: Any | None = None,
+) -> RejectedTemplate:
+    """Create and insert a new rejected template.
+
+    Args:
+
+        store: The mutable template store dict.
+
+        rejected: The rejected templates dict to insert into.
+
+        schema: The schema graph.
+
+        q_norm: The normalised question.
+
+        intent: The ``RuntimeIntent``.
+
+        sql: The rejected SQL.
+
+        category: The rejection category string.
+
+        reason: The human-readable rejection reason.
+
+    Returns:
+
+        The newly created ``RejectedTemplate``.
+    """
+    rid = f"R{int(store.get('next_reject_id', 1)):04d}"
+    store["next_reject_id"] = int(store.get("next_reject_id", 1)) + 1
+
+    sql_canon = canonicalize_sql(sql)
+    if intent.sql_param:
+        sql_param = intent.sql_param
+    else:
+        sql_norm = normalize_sql(sql_canon)
+        sql_param, _ = parameter_abstract(sql_norm)
+    sql_fp_val = sql_fp(sql_param)
+    tables_used = extract_tables_from_sql(sql_canon, list(schema.tables.keys()))
+    colmap_sig_val = colmap_signature(intent.column_map)
+    ikey = intent_key(intent)
+
+    intent_sig = runtime_intent_to_concrete(intent, "")
+
+    all_pv = flatten_param_values(intent)
+
+    debug(f"[templates.insert_rejected_template] creating: id={rid} category={category}")
+
+    spark_sql_param = ""
+    if EngineConfig.TYPE == "databricks":
+        d = dialect or get_dialect()
+        if d:
+            spark_sql_param = d.prepare_for_execution(sql_param)
+
+    rt = RejectedTemplate(
+        id=rid,
+        schema_hash=schema.schema_hash,
+        intent_signature=intent_sig,
+        intent_key=ikey,
+        tables_used=sorted(tables_used),
+        sql_param=sql_param,
+        spark_sql_param=spark_sql_param,
+        sql_display_param=intent.sql_display_param or sql_param,
+        sql_fp=sql_fp_val,
+        shape=sql_shape(sql_canon, intent),
+        colmap_sig=colmap_sig_val,
+        value_history=RejectedValueHistory(
+            param_values=[all_pv],
+            questions=[q_norm],
+            natural_language=[intent.natural_language or ""],
+            rejection_reasons=[reason],
+            rejection_categories=[category],
+        ),
+        aliased_sql=intent.sql_display_param or "",
+    )
+    rejected[rid] = rt
+    debug(f"[templates.insert_rejected_template] total_rejected: {len(rejected)}")
+    return rt
+
+
+def best_template_by_intent_key(templates: dict[str, Template], ikey: str) -> Template | None:
+    """Find best template matching an intent key.
+
+    Args:
+
+        templates: The accepted templates dict.
+
+        ikey: The intent key string to search for.
+
+    Returns:
+
+        The matching ``Template``, or ``None`` if not found.
+    """
+    debug(f"[templates.best_template_by_intent_key] searching: ikey={ikey[:32]}")
+    for tid, t in templates.items():
+        if t.intent_key == ikey:
+            debug(f"[templates.best_template_by_intent_key] found: id={tid}")
+            return t
+    debug("[templates.best_template_by_intent_key] no_match")
+    return None