aetherdialect 0.1.0__py3-none-any.whl

text2sql/contracts_core.py

@@ -0,0 +1,1890 @@
+"""Intent and template contracts for the text-to-SQL pipeline.
+
+Defines the expression model (ExprValue, MulGroup, NormalizedExpr) used throughout the pipeline for canonical sum-of-products representations, along with all intent containers (RuntimeIntent, ConcreteIntent, SimulatorIntent, QSimIntent), filter and having conditions, select and order-by columns, CTE step representations, and the Template/RejectedTemplate structures used for query reuse.
+
+Also provides conversion helpers between runtime and concrete forms used at different pipeline stages.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .config import AGG_PREFIXES, OP_FLIP
+from .contracts_base import (
+    CteOutputColumnMeta,
+    ExpansionMetadata,
+    IntentIssue,
+    SQLShape,
+    TemplateStats,
+)
+
+ScalarArg = str | int | float
+ParamValue = str | int | float | bool | list[str | int | float]
+RawValue = str | int | float | bool | list[str | int | float] | dict[str, str | int] | None
+
+
+@dataclass
+class ExprValue:
+    """Parameterized literal value for expression arithmetic with param_key for template reuse."""
+
+    value: float = 0.0
+    param_key: str = ""
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> ExprValue:
+        """Create ExprValue from dictionary.
+
+        Args:
+
+            d: Dictionary with 'value' and 'param_key' keys, or a bare numeric value.
+
+        Returns:
+
+            Populated ExprValue instance.
+        """
+        if isinstance(d, int | float):
+            return ExprValue(value=float(d))
+        return ExprValue(value=d.get("value", 0.0), param_key=d.get("param_key", ""))
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with 'value' and 'param_key' keys.
+        """
+        return {"value": self.value, "param_key": self.param_key}
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching (value-agnostic)."""
+        return "val"
+
+
+@dataclass
+class MulGroup:
+    """Single multiplicative term: scalar_func(agg_func(inner_scalar_func(coefficient * multiply[0] * ... / divide[0] / ...))) with scalar_func_args and inner_scalar_func_args."""
+
+    coefficient: float = 1.0
+    multiply: list[str] = field(default_factory=list)
+    divide: list[str] = field(default_factory=list)
+    agg_func: str | None = None
+    scalar_func: str | None = None
+    inner_scalar_func: str | None = None
+    scalar_func_args: list[ScalarArg] = field(default_factory=list)
+    inner_scalar_func_args: list[ScalarArg] = field(default_factory=list)
+    coeff_param_key: str = ""
+    sarg_param_keys: list[str] = field(default_factory=list)
+    isarg_param_keys: list[str] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        """Sort multiply and divide lists and lowercase function names."""
+        self.multiply = sorted(self.multiply)
+        self.divide = sorted(self.divide)
+        if self.agg_func:
+            self.agg_func = self.agg_func.lower()
+        if self.scalar_func:
+            self.scalar_func = self.scalar_func.lower()
+        if self.inner_scalar_func:
+            self.inner_scalar_func = self.inner_scalar_func.lower()
+        if self.scalar_func and self.inner_scalar_func:
+            if self.scalar_func == "extract":
+                pass
+            elif self.inner_scalar_func == "extract":
+                self.scalar_func, self.inner_scalar_func = (
+                    self.inner_scalar_func,
+                    self.scalar_func,
+                )
+                self.scalar_func_args, self.inner_scalar_func_args = (
+                    self.inner_scalar_func_args,
+                    self.scalar_func_args,
+                )
+            elif self.scalar_func > self.inner_scalar_func:
+                self.scalar_func, self.inner_scalar_func = (
+                    self.inner_scalar_func,
+                    self.scalar_func,
+                )
+                self.scalar_func_args, self.inner_scalar_func_args = (
+                    self.inner_scalar_func_args,
+                    self.scalar_func_args,
+                )
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> MulGroup:
+        """Create MulGroup from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching MulGroup fields.
+
+        Returns:
+
+            Populated MulGroup instance.
+        """
+        return MulGroup(
+            coefficient=d.get("coefficient", 1.0),
+            multiply=d.get("multiply", []),
+            divide=d.get("divide", []),
+            agg_func=d.get("agg_func"),
+            scalar_func=d.get("scalar_func"),
+            inner_scalar_func=d.get("inner_scalar_func"),
+            scalar_func_args=d.get("scalar_func_args", []),
+            inner_scalar_func_args=d.get("inner_scalar_func_args", []),
+            coeff_param_key=d.get("coeff_param_key", ""),
+            sarg_param_keys=d.get("sarg_param_keys", []),
+            isarg_param_keys=d.get("isarg_param_keys", []),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "coefficient": self.coefficient,
+            "multiply": self.multiply,
+            "divide": self.divide,
+            "agg_func": self.agg_func,
+            "scalar_func": self.scalar_func,
+            "inner_scalar_func": self.inner_scalar_func,
+            "scalar_func_args": self.scalar_func_args,
+            "inner_scalar_func_args": self.inner_scalar_func_args,
+            "coeff_param_key": self.coeff_param_key,
+            "sarg_param_keys": self.sarg_param_keys,
+            "isarg_param_keys": self.isarg_param_keys,
+        }
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching."""
+        parts = ["coeff"]
+        if self.agg_func:
+            parts.append(f"agg={self.agg_func}")
+        if self.scalar_func:
+            parts.append(f"scalar={self.scalar_func}")
+        if self.scalar_func_args:
+            parts.append(f"sargs={len(self.scalar_func_args)}")
+        if self.inner_scalar_func:
+            parts.append(f"inner={self.inner_scalar_func}")
+        if self.inner_scalar_func_args:
+            parts.append(f"iargs={len(self.inner_scalar_func_args)}")
+        parts.extend(f"*{m}" for m in self.multiply)
+        parts.extend(f"/{d}" for d in self.divide)
+        return "|".join(parts)
+
+    @property
+    def structural_key(self) -> str:
+        """Return coefficient-agnostic structural key for like-term combining."""
+        parts: list[str] = []
+        if self.agg_func:
+            parts.append(f"agg={self.agg_func}")
+        if self.scalar_func:
+            parts.append(f"scalar={self.scalar_func}")
+        if self.scalar_func_args:
+            parts.append(f"sargs={len(self.scalar_func_args)}")
+        if self.inner_scalar_func:
+            parts.append(f"inner={self.inner_scalar_func}")
+        if self.inner_scalar_func_args:
+            parts.append(f"iargs={len(self.inner_scalar_func_args)}")
+        parts.extend(f"*{m}" for m in self.multiply)
+        parts.extend(f"/{d}" for d in self.divide)
+        return "|".join(parts)
+
+
+@dataclass
+class NormalizedExpr:
+    """Canonical sum-of-products expression: scalar_func(agg_func(inner_scalar_func(sum of add_groups minus sub_groups plus add_values minus sub_values))) with scalar_func_args and inner_scalar_func_args."""
+
+    add_groups: list[MulGroup] = field(default_factory=list)
+    sub_groups: list[MulGroup] = field(default_factory=list)
+    add_values: list[ExprValue] = field(default_factory=list)
+    sub_values: list[ExprValue] = field(default_factory=list)
+    agg_func: str | None = None
+    scalar_func: str | None = None
+    inner_scalar_func: str | None = None
+    scalar_func_args: list[ScalarArg] = field(default_factory=list)
+    inner_scalar_func_args: list[ScalarArg] = field(default_factory=list)
+    sarg_param_keys: list[str] = field(default_factory=list)
+    isarg_param_keys: list[str] = field(default_factory=list)
+    is_numeric: bool = True
+
+    def __post_init__(self) -> None:
+        """Sort groups by signature_key, sort values by value, and lowercase function names."""
+        self.add_groups = sorted(self.add_groups, key=lambda g: g.signature_key)
+        self.sub_groups = sorted(self.sub_groups, key=lambda g: g.signature_key)
+        self.add_values = sorted(self.add_values, key=lambda v: v.value)
+        self.sub_values = sorted(self.sub_values, key=lambda v: v.value)
+        if self.agg_func:
+            self.agg_func = self.agg_func.lower()
+        if self.scalar_func:
+            self.scalar_func = self.scalar_func.lower()
+        if self.inner_scalar_func:
+            self.inner_scalar_func = self.inner_scalar_func.lower()
+        if self.scalar_func and self.inner_scalar_func:
+            if self.scalar_func == "extract":
+                pass
+            elif self.inner_scalar_func == "extract":
+                self.scalar_func, self.inner_scalar_func = (
+                    self.inner_scalar_func,
+                    self.scalar_func,
+                )
+                self.scalar_func_args, self.inner_scalar_func_args = (
+                    self.inner_scalar_func_args,
+                    self.scalar_func_args,
+                )
+            elif self.scalar_func > self.inner_scalar_func:
+                self.scalar_func, self.inner_scalar_func = (
+                    self.inner_scalar_func,
+                    self.scalar_func,
+                )
+                self.scalar_func_args, self.inner_scalar_func_args = (
+                    self.inner_scalar_func_args,
+                    self.scalar_func_args,
+                )
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> NormalizedExpr:
+        """Create NormalizedExpr from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching NormalizedExpr fields.
+
+        Returns:
+
+            Populated NormalizedExpr instance with nested MulGroup and ExprValue objects.
+        """
+        return NormalizedExpr(
+            add_groups=[MulGroup.from_dict(g) for g in d.get("add_groups", [])],
+            sub_groups=[MulGroup.from_dict(g) for g in d.get("sub_groups", [])],
+            add_values=[ExprValue.from_dict(v) for v in d.get("add_values", [])],
+            sub_values=[ExprValue.from_dict(v) for v in d.get("sub_values", [])],
+            agg_func=d.get("agg_func"),
+            scalar_func=d.get("scalar_func"),
+            inner_scalar_func=d.get("inner_scalar_func"),
+            scalar_func_args=d.get("scalar_func_args", []),
+            inner_scalar_func_args=d.get("inner_scalar_func_args", []),
+            sarg_param_keys=d.get("sarg_param_keys", []),
+            isarg_param_keys=d.get("isarg_param_keys", []),
+            is_numeric=d.get("is_numeric", True),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all NormalizedExpr fields, with nested groups and values serialized.
+        """
+        return {
+            "add_groups": [g.to_dict() for g in self.add_groups],
+            "sub_groups": [g.to_dict() for g in self.sub_groups],
+            "add_values": [v.to_dict() for v in self.add_values],
+            "sub_values": [v.to_dict() for v in self.sub_values],
+            "agg_func": self.agg_func,
+            "scalar_func": self.scalar_func,
+            "inner_scalar_func": self.inner_scalar_func,
+            "scalar_func_args": self.scalar_func_args,
+            "inner_scalar_func_args": self.inner_scalar_func_args,
+            "sarg_param_keys": self.sarg_param_keys,
+            "isarg_param_keys": self.isarg_param_keys,
+            "is_numeric": self.is_numeric,
+        }
+
+    @staticmethod
+    def from_column(col: str) -> NormalizedExpr:
+        """Create expression for a bare column reference."""
+        return NormalizedExpr(add_groups=[MulGroup(multiply=[col])])
+
+    @staticmethod
+    def from_agg(agg_func: str, col: str) -> NormalizedExpr:
+        """Create expression for a single aggregation call."""
+        return NormalizedExpr(add_groups=[MulGroup(multiply=[col], agg_func=agg_func.lower())])
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching."""
+        parts = []
+        if self.agg_func:
+            parts.append(f"expr_agg={self.agg_func}")
+        if self.scalar_func:
+            parts.append(f"expr_scalar={self.scalar_func}")
+        if self.scalar_func_args:
+            parts.append(f"expr_sargs={len(self.scalar_func_args)}")
+        if self.inner_scalar_func:
+            parts.append(f"expr_inner={self.inner_scalar_func}")
+        if self.inner_scalar_func_args:
+            parts.append(f"expr_iargs={len(self.inner_scalar_func_args)}")
+        parts.extend(f"+{g.signature_key}" for g in self.add_groups)
+        parts.extend(f"-{g.signature_key}" for g in self.sub_groups)
+        parts.extend(f"+{v.signature_key}" for v in self.add_values)
+        parts.extend(f"-{v.signature_key}" for v in self.sub_values)
+        return "|".join(parts)
+
+    @property
+    def has_aggregation(self) -> bool:
+        """Return True if any term contains a SQL aggregation function, any group has agg_func, or the expression has agg_func."""
+        if self.agg_func:
+            return True
+        for group in self.add_groups + self.sub_groups:
+            if group.agg_func:
+                return True
+            for term in group.multiply + group.divide:
+                upper = term.upper()
+                if any(p in upper for p in AGG_PREFIXES):
+                    return True
+        return False
+
+    @property
+    def primary_column(self) -> str:
+        """Return innermost column reference from the first term, stripping function wrappers and DISTINCT prefix."""
+        if not self.add_groups or not self.add_groups[0].multiply:
+            return ""
+        term = self.add_groups[0].multiply[0]
+        while "(" in term:
+            start = term.index("(")
+            end = term.rindex(")")
+            term = term[start + 1 : end]
+        if term.upper().startswith("DISTINCT "):
+            term = term[9:].strip()
+        return term
+
+    @property
+    def primary_term(self) -> str:
+        """Return the first atomic term as-is."""
+        if self.add_groups and self.add_groups[0].multiply:
+            return self.add_groups[0].multiply[0]
+        return ""
+
+
+@dataclass
+class FilterParam:
+    """Filter condition with left expression, operator, and optional right expression for expr-vs-expr comparisons."""
+
+    left_expr: NormalizedExpr = field(default_factory=NormalizedExpr)
+    op: str = "="
+    right_expr: NormalizedExpr | None = None
+    value_type: str = "string"
+    param_key: str = ""
+    raw_value: RawValue = None
+    bool_op: str = "AND"
+    filter_group: int | None = None
+
+    def __post_init__(self) -> None:
+        """Normalize op, value_type, and bool_op, canonicalize ordering, and consolidate ExprValues to the value side."""
+        self.op = self.op.strip().lower()
+        self.value_type = self.value_type.strip().lower()
+        self.bool_op = self.bool_op.strip().upper() if self.bool_op else "AND"
+        if self.bool_op not in ("AND", "OR"):
+            self.bool_op = "AND"
+        if self.right_expr is not None:
+            if self.left_expr.signature_key > self.right_expr.signature_key:
+                left_has_col = "." in self.left_expr.signature_key
+                right_has_col = "." in self.right_expr.signature_key
+                if not (left_has_col and not right_has_col):
+                    self.left_expr, self.right_expr = self.right_expr, self.left_expr
+                    self.op = OP_FLIP.get(self.op, self.op)
+            for ev in self.left_expr.add_values:
+                self.right_expr.sub_values.append(ExprValue(value=ev.value, param_key=ev.param_key))
+            for ev in self.left_expr.sub_values:
+                self.right_expr.add_values.append(ExprValue(value=ev.value, param_key=ev.param_key))
+            self.left_expr.add_values = []
+            self.left_expr.sub_values = []
+        elif (
+            self.raw_value is not None
+            and isinstance(self.raw_value, int | float)
+            and not isinstance(self.raw_value, bool)
+        ):
+            offset = sum(ev.value for ev in self.left_expr.add_values) - sum(
+                ev.value for ev in self.left_expr.sub_values
+            )
+            self.raw_value = self.raw_value - offset
+            self.left_expr.add_values = []
+            self.left_expr.sub_values = []
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> FilterParam:
+        """Create FilterParam from dictionary.
+
+        Args:
+
+            d: Dictionary with 'left_expr', 'op', optional 'right_expr', 'value_type', and 'param_key'.
+
+        Returns:
+
+            Populated FilterParam instance.
+        """
+        left_raw = d.get("left_expr", {})
+        right_raw = d.get("right_expr")
+        fg_raw = d.get("filter_group")
+        return FilterParam(
+            left_expr=(NormalizedExpr.from_dict(left_raw) if isinstance(left_raw, dict) else left_raw),
+            op=d.get("op", "="),
+            right_expr=(NormalizedExpr.from_dict(right_raw) if isinstance(right_raw, dict) and right_raw else None),
+            value_type=d.get("value_type", "string"),
+            param_key=d.get("param_key", ""),
+            raw_value=d.get("value") or d.get("raw_value"),
+            bool_op=d.get("bool_op", "AND"),
+            filter_group=int(fg_raw) if fg_raw is not None else None,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with filter fields; raw_value is intentionally excluded.
+        """
+        d: dict[str, Any] = {
+            "left_expr": self.left_expr.to_dict(),
+            "op": self.op,
+            "right_expr": self.right_expr.to_dict() if self.right_expr else None,
+            "value_type": self.value_type,
+            "param_key": self.param_key,
+        }
+        if self.bool_op != "AND":
+            d["bool_op"] = self.bool_op
+        if self.filter_group is not None:
+            d["filter_group"] = self.filter_group
+        return d
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching."""
+        parts = [self.left_expr.signature_key, self.op, self.value_type]
+        if self.right_expr:
+            parts.append(f"r:{self.right_expr.signature_key}")
+        return "|".join(parts)
+
+
+@dataclass
+class HavingParam:
+    """Having condition with left expression, operator, and optional right expression for expr-vs-expr comparisons."""
+
+    left_expr: NormalizedExpr = field(default_factory=NormalizedExpr)
+    op: str = "="
+    right_expr: NormalizedExpr | None = None
+    value_type: str = "number"
+    param_key: str = ""
+    raw_value: RawValue = None
+    bool_op: str = "AND"
+    filter_group: int | None = None
+
+    def __post_init__(self) -> None:
+        """Normalize op, value_type, and bool_op, canonicalize ordering, and consolidate ExprValues to the value side."""
+        self.op = self.op.strip().lower()
+        self.value_type = self.value_type.strip().lower()
+        self.bool_op = self.bool_op.strip().upper() if self.bool_op else "AND"
+        if self.bool_op not in ("AND", "OR"):
+            self.bool_op = "AND"
+        if self.right_expr is not None:
+            if self.left_expr.signature_key > self.right_expr.signature_key:
+                left_has_col = "." in self.left_expr.signature_key
+                right_has_col = "." in self.right_expr.signature_key
+                if not (left_has_col and not right_has_col):
+                    self.left_expr, self.right_expr = self.right_expr, self.left_expr
+                    self.op = OP_FLIP.get(self.op, self.op)
+            for ev in self.left_expr.add_values:
+                self.right_expr.sub_values.append(ExprValue(value=ev.value, param_key=ev.param_key))
+            for ev in self.left_expr.sub_values:
+                self.right_expr.add_values.append(ExprValue(value=ev.value, param_key=ev.param_key))
+            self.left_expr.add_values = []
+            self.left_expr.sub_values = []
+        elif (
+            self.raw_value is not None
+            and isinstance(self.raw_value, int | float)
+            and not isinstance(self.raw_value, bool)
+        ):
+            offset = sum(ev.value for ev in self.left_expr.add_values) - sum(
+                ev.value for ev in self.left_expr.sub_values
+            )
+            self.raw_value = self.raw_value - offset
+            self.left_expr.add_values = []
+            self.left_expr.sub_values = []
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> HavingParam:
+        """Create HavingParam from dictionary.
+
+        Args:
+
+            d: Dictionary with 'left_expr', 'op', optional 'right_expr', 'value_type', and 'param_key'.
+
+        Returns:
+
+            Populated HavingParam instance.
+        """
+        left_raw = d.get("left_expr", {})
+        right_raw = d.get("right_expr")
+        fg_raw = d.get("filter_group")
+        return HavingParam(
+            left_expr=(NormalizedExpr.from_dict(left_raw) if isinstance(left_raw, dict) else left_raw),
+            op=d.get("op", "="),
+            right_expr=(NormalizedExpr.from_dict(right_raw) if isinstance(right_raw, dict) and right_raw else None),
+            value_type=d.get("value_type", "number"),
+            param_key=d.get("param_key", ""),
+            raw_value=d.get("value") or d.get("raw_value"),
+            bool_op=d.get("bool_op", "AND"),
+            filter_group=int(fg_raw) if fg_raw is not None else None,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        d: dict[str, Any] = {
+            "left_expr": self.left_expr.to_dict(),
+            "op": self.op,
+            "right_expr": self.right_expr.to_dict() if self.right_expr else None,
+            "value_type": self.value_type,
+            "param_key": self.param_key,
+        }
+        if self.bool_op != "AND":
+            d["bool_op"] = self.bool_op
+        if self.filter_group is not None:
+            d["filter_group"] = self.filter_group
+        return d
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching."""
+        parts = [self.left_expr.signature_key, self.op, self.value_type]
+        if self.right_expr:
+            parts.append(f"r:{self.right_expr.signature_key}")
+        return "|".join(parts)
+
+
+@dataclass
+class SelectCol:
+    """Select column with normalized expression."""
+
+    expr: NormalizedExpr = field(default_factory=NormalizedExpr)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> SelectCol:
+        """Create SelectCol from dictionary.
+
+        Args:
+
+            d: Dictionary with an 'expr' key containing a NormalizedExpr dict.
+
+        Returns:
+
+            Populated SelectCol instance.
+        """
+        expr_raw = d.get("expr", {})
+        return SelectCol(
+            expr=(NormalizedExpr.from_dict(expr_raw) if isinstance(expr_raw, dict) else expr_raw),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with the serialized expr.
+        """
+        return {"expr": self.expr.to_dict()}
+
+    @property
+    def is_aggregated(self) -> bool:
+        """Return True if expression contains an aggregation function."""
+        return self.expr.has_aggregation
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching."""
+        return self.expr.signature_key
+
+
+@dataclass
+class OrderByCol:
+    """Order by column with expression and sort direction."""
+
+    expr: NormalizedExpr = field(default_factory=NormalizedExpr)
+    direction: str = "ASC"
+
+    def __post_init__(self) -> None:
+        """Uppercase direction for consistent comparison."""
+        self.direction = self.direction.strip().upper()
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> OrderByCol:
+        """Create OrderByCol from dictionary.
+
+        Args:
+
+            d: Dictionary with 'expr' and 'direction' keys.
+
+        Returns:
+
+            Populated OrderByCol instance.
+        """
+        expr_raw = d.get("expr", {})
+        return OrderByCol(
+            expr=(NormalizedExpr.from_dict(expr_raw) if isinstance(expr_raw, dict) else expr_raw),
+            direction=d.get("direction", "ASC"),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with the serialized expr and direction string.
+        """
+        return {"expr": self.expr.to_dict(), "direction": self.direction}
+
+    @property
+    def is_aggregated(self) -> bool:
+        """Return True if expression contains an aggregation function."""
+        return self.expr.has_aggregation
+
+    @property
+    def signature_key(self) -> str:
+        """Return structural signature for template matching."""
+        return "|".join([self.expr.signature_key, self.direction])
+
+
+@dataclass
+class ConcreteCteStep:
+    """CTE step structural signature for template storage."""
+
+    cte_name: str
+    tables: list[str] = field(default_factory=list)
+    select_cols: list[SelectCol] = field(default_factory=list)
+    group_by_cols: list[NormalizedExpr] = field(default_factory=list)
+    order_by_cols: list[OrderByCol] = field(default_factory=list)
+    filters_param: list[FilterParam] = field(default_factory=list)
+    having_param: list[HavingParam] = field(default_factory=list)
+    output_columns: list[str] = field(default_factory=list)
+    grain: str = "row_level"
+    limit: int | None = None
+    column_map: dict[str, str] = field(default_factory=dict)
+    output_column_metadata: dict[str, CteOutputColumnMeta] = field(default_factory=dict)
+    chosen_join_candidate_id: str = ""
+    chosen_join_path_signature: list[str] = field(default_factory=list)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> ConcreteCteStep:
+        """Create ConcreteCteStep from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching ConcreteCteStep fields.
+
+        Returns:
+
+            Populated ConcreteCteStep with nested expression objects.
+        """
+        sc_raw = d.get("select_cols", [])
+        gbc_raw = d.get("group_by_cols", [])
+        obc_raw = d.get("order_by_cols", [])
+        fp_raw = d.get("filters_param", [])
+        hp_raw = d.get("having_param", [])
+        ocm_raw = d.get("output_column_metadata", {})
+        select_cols = [
+            (
+                SelectCol.from_dict(s)
+                if isinstance(s, dict)
+                else (SelectCol(expr=NormalizedExpr.from_column(s)) if isinstance(s, str) else s)
+            )
+            for s in sc_raw
+        ]
+        group_by_cols = [
+            (
+                NormalizedExpr.from_dict(g)
+                if isinstance(g, dict)
+                else (NormalizedExpr.from_column(g) if isinstance(g, str) else g)
+            )
+            for g in gbc_raw
+        ]
+        order_by_cols = [
+            (
+                OrderByCol.from_dict(o)
+                if isinstance(o, dict)
+                else (OrderByCol(expr=NormalizedExpr.from_column(o)) if isinstance(o, str) else o)
+            )
+            for o in obc_raw
+        ]
+        return ConcreteCteStep(
+            cte_name=d.get("cte_name", ""),
+            tables=d.get("tables", []),
+            select_cols=select_cols,
+            group_by_cols=group_by_cols,
+            order_by_cols=order_by_cols,
+            filters_param=[FilterParam.from_dict(f) if isinstance(f, dict) else f for f in fp_raw],
+            having_param=[HavingParam.from_dict(h) if isinstance(h, dict) else h for h in hp_raw],
+            output_columns=d.get("output_columns", []),
+            grain=d.get("grain", "row_level"),
+            limit=d.get("limit"),
+            column_map=d.get("column_map", {}),
+            output_column_metadata={
+                k: CteOutputColumnMeta.from_dict(v) if isinstance(v, dict) else v for k, v in ocm_raw.items()
+            },
+            chosen_join_candidate_id=d.get("chosen_join_candidate_id", ""),
+            chosen_join_path_signature=d.get("chosen_join_path_signature", []),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all ConcreteCteStep fields, with nested expressions serialized.
+        """
+        return {
+            "cte_name": self.cte_name,
+            "tables": self.tables,
+            "select_cols": [s.to_dict() for s in self.select_cols],
+            "group_by_cols": [g.to_dict() for g in self.group_by_cols],
+            "order_by_cols": [o.to_dict() for o in self.order_by_cols],
+            "filters_param": [f.to_dict() for f in self.filters_param],
+            "having_param": [h.to_dict() for h in self.having_param],
+            "output_columns": self.output_columns,
+            "grain": self.grain,
+            "limit": self.limit,
+            "column_map": self.column_map,
+            "output_column_metadata": {k: v.to_dict() for k, v in self.output_column_metadata.items()},
+            "chosen_join_candidate_id": self.chosen_join_candidate_id,
+            "chosen_join_path_signature": self.chosen_join_path_signature,
+        }
+
+
+@dataclass
+class RuntimeCteStep:
+    """CTE step specification for WITH clause queries with runtime values."""
+
+    cte_name: str
+    description: str = ""
+    tables: list[str] = field(default_factory=list)
+    select_cols: list[SelectCol] = field(default_factory=list)
+    group_by_cols: list[NormalizedExpr] = field(default_factory=list)
+    order_by_cols: list[OrderByCol] = field(default_factory=list)
+    filters_param: list[FilterParam] = field(default_factory=list)
+    having_param: list[HavingParam] = field(default_factory=list)
+    param_values: dict[str, ParamValue] = field(default_factory=dict)
+    output_columns: list[str] = field(default_factory=list)
+    grain: str = "row_level"
+    limit: int | None = None
+    limit_param_key: str = ""
+    column_map: dict[str, str] = field(default_factory=dict)
+    output_column_metadata: dict[str, CteOutputColumnMeta] = field(default_factory=dict)
+    chosen_join_candidate_id: str = ""
+    chosen_join_path_signature: list[str] = field(default_factory=list)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> RuntimeCteStep:
+        """Create RuntimeCteStep from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching RuntimeCteStep fields.
+
+        Returns:
+
+            Populated RuntimeCteStep with nested expression objects and runtime param_values.
+        """
+        sc_raw = d.get("select_cols", [])
+        gbc_raw = d.get("group_by_cols", [])
+        obc_raw = d.get("order_by_cols", [])
+        fp_raw = d.get("filters_param", [])
+        hp_raw = d.get("having_param", [])
+        ocm_raw = d.get("output_column_metadata", {})
+        select_cols = [
+            (
+                SelectCol.from_dict(s)
+                if isinstance(s, dict)
+                else (SelectCol(expr=NormalizedExpr.from_column(s)) if isinstance(s, str) else s)
+            )
+            for s in sc_raw
+        ]
+        group_by_cols = [
+            (
+                NormalizedExpr.from_dict(g)
+                if isinstance(g, dict)
+                else (NormalizedExpr.from_column(g) if isinstance(g, str) else g)
+            )
+            for g in gbc_raw
+        ]
+        order_by_cols = [
+            (
+                OrderByCol.from_dict(o)
+                if isinstance(o, dict)
+                else (OrderByCol(expr=NormalizedExpr.from_column(o)) if isinstance(o, str) else o)
+            )
+            for o in obc_raw
+        ]
+        return RuntimeCteStep(
+            cte_name=d.get("cte_name", ""),
+            description=d.get("description", ""),
+            tables=d.get("tables", []),
+            select_cols=select_cols,
+            group_by_cols=group_by_cols,
+            order_by_cols=order_by_cols,
+            filters_param=[FilterParam.from_dict(f) if isinstance(f, dict) else f for f in fp_raw],
+            having_param=[HavingParam.from_dict(h) if isinstance(h, dict) else h for h in hp_raw],
+            param_values=d.get("param_values", {}),
+            output_columns=d.get("output_columns", []),
+            grain=d.get("grain", "row_level"),
+            limit=d.get("limit"),
+            limit_param_key=d.get("limit_param_key", ""),
+            column_map=d.get("column_map", {}),
+            output_column_metadata={
+                k: CteOutputColumnMeta.from_dict(v) if isinstance(v, dict) else v for k, v in ocm_raw.items()
+            },
+            chosen_join_candidate_id=d.get("chosen_join_candidate_id", ""),
+            chosen_join_path_signature=d.get("chosen_join_path_signature", []),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all RuntimeCteStep fields including param_values and limit_param_key.
+        """
+        return {
+            "cte_name": self.cte_name,
+            "description": self.description,
+            "tables": self.tables,
+            "select_cols": [s.to_dict() for s in self.select_cols],
+            "group_by_cols": [g.to_dict() for g in self.group_by_cols],
+            "order_by_cols": [o.to_dict() for o in self.order_by_cols],
+            "filters_param": [f.to_dict() for f in self.filters_param],
+            "having_param": [h.to_dict() for h in self.having_param],
+            "param_values": self.param_values,
+            "output_columns": self.output_columns,
+            "grain": self.grain,
+            "limit": self.limit,
+            "limit_param_key": self.limit_param_key,
+            "column_map": self.column_map,
+            "output_column_metadata": {k: v.to_dict() for k, v in self.output_column_metadata.items()},
+            "chosen_join_candidate_id": self.chosen_join_candidate_id,
+            "chosen_join_path_signature": self.chosen_join_path_signature,
+        }
+
+    @property
+    def expected_rows(self) -> str:
+        """Derive expected_rows from grain and limit."""
+        if self.grain == "scalar":
+            return "one"
+        return "few" if self.limit else "many"
+
+
+@dataclass
+class CteValidationResult:
+    """Result container for CTE chain validation."""
+
+    is_valid: bool
+    cte_outputs: dict[str, list[str]]
+    issues: list[IntentIssue] = field(default_factory=list)
+
+
+def _runtime_cte_to_concrete(runtime: RuntimeCteStep) -> ConcreteCteStep:
+    """Convert RuntimeCteStep to ConcreteCteStep for template storage.
+
+    Args:
+
+        runtime: The runtime CTE step containing values and expressions.
+
+    Returns:
+
+        ConcreteCteStep with runtime-only fields (param_values and limit_param_key) stripped.
+    """
+    return ConcreteCteStep(
+        cte_name=runtime.cte_name,
+        tables=runtime.tables,
+        select_cols=runtime.select_cols,
+        group_by_cols=runtime.group_by_cols,
+        order_by_cols=runtime.order_by_cols,
+        filters_param=runtime.filters_param,
+        having_param=runtime.having_param,
+        output_columns=runtime.output_columns,
+        grain=runtime.grain,
+        limit=runtime.limit,
+        column_map=runtime.column_map,
+        output_column_metadata=runtime.output_column_metadata,
+        chosen_join_candidate_id=runtime.chosen_join_candidate_id,
+        chosen_join_path_signature=runtime.chosen_join_path_signature,
+    )
+
+
+def concrete_cte_to_runtime(concrete: ConcreteCteStep) -> RuntimeCteStep:
+    """Convert ConcreteCteStep to RuntimeCteStep for pipeline execution.
+
+    Args:
+
+        concrete: The stored concrete CTE step from a template.
+
+    Returns:
+
+        RuntimeCteStep with blank description and empty param_values ready for execution.
+    """
+    return RuntimeCteStep(
+        cte_name=concrete.cte_name,
+        description="",
+        tables=concrete.tables,
+        select_cols=concrete.select_cols,
+        group_by_cols=concrete.group_by_cols,
+        order_by_cols=concrete.order_by_cols,
+        filters_param=concrete.filters_param,
+        having_param=concrete.having_param,
+        param_values={},
+        output_columns=concrete.output_columns,
+        grain=concrete.grain,
+        limit=concrete.limit,
+        column_map=concrete.column_map,
+        output_column_metadata=concrete.output_column_metadata,
+        chosen_join_candidate_id=concrete.chosen_join_candidate_id,
+        chosen_join_path_signature=concrete.chosen_join_path_signature,
+    )
+
+
+@dataclass
+class RuntimeIntent:
+    """Runtime intent container for pipeline execution with structural fields and values."""
+
+    tables: list[str]
+    grain: str
+    select_cols: list[SelectCol]
+    group_by_cols: list[NormalizedExpr]
+    order_by_cols: list[OrderByCol]
+    filters_param: list[FilterParam]
+    having_param: list[HavingParam] = field(default_factory=list)
+    param_values: dict[str, ParamValue] = field(default_factory=dict)
+    cte_steps: list[RuntimeCteStep] = field(default_factory=list)
+    natural_language: str = ""
+    limit: int | None = None
+    limit_param_key: str = ""
+    column_map: dict[str, str] = field(default_factory=dict)
+    chosen_join_candidate_id: str = ""
+    chosen_join_path_signature: list[str] = field(default_factory=list)
+    extra_tables: set[str] = field(default_factory=set)
+    sql_param: str = ""
+    sql_display_param: str = ""
+    sql_substituted: str = ""
+    deterministic_sql: str = ""
+    sql_shape: SQLShape | None = None
+    schema_invalid: bool = False
+
+    @property
+    def expected_rows(self) -> str:
+        """Derive expected_rows from grain and limit."""
+        if self.grain == "scalar":
+            return "one"
+        return "few" if self.limit else "many"
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> RuntimeIntent:
+        """Create RuntimeIntent from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching RuntimeIntent fields, typically from JSON storage.
+
+        Returns:
+
+            Populated RuntimeIntent with all nested objects deserialized.
+        """
+        sc_raw = d.get("select_cols", [])
+        gbc_raw = d.get("group_by_cols", [])
+        obc_raw = d.get("order_by_cols", [])
+        fp_raw = d.get("filters_param", [])
+        hp_raw = d.get("having_param", [])
+        cte_raw = d.get("cte_steps", [])
+        join_sig_raw = d.get("chosen_join_path_signature", [])
+        if isinstance(join_sig_raw, str):
+            join_sig_raw = [join_sig_raw] if join_sig_raw else []
+        select_cols = [
+            (
+                SelectCol.from_dict(s)
+                if isinstance(s, dict)
+                else (SelectCol(expr=NormalizedExpr.from_column(s)) if isinstance(s, str) else s)
+            )
+            for s in sc_raw
+        ]
+        group_by_cols = [
+            (
+                NormalizedExpr.from_dict(g)
+                if isinstance(g, dict)
+                else (NormalizedExpr.from_column(g) if isinstance(g, str) else g)
+            )
+            for g in gbc_raw
+        ]
+        order_by_cols = [
+            (
+                OrderByCol.from_dict(o)
+                if isinstance(o, dict)
+                else (OrderByCol(expr=NormalizedExpr.from_column(o)) if isinstance(o, str) else o)
+            )
+            for o in obc_raw
+        ]
+        return RuntimeIntent(
+            tables=d.get("tables", []),
+            grain=d.get("grain", "row_level"),
+            select_cols=select_cols,
+            group_by_cols=group_by_cols,
+            order_by_cols=order_by_cols,
+            filters_param=[FilterParam.from_dict(fp) if isinstance(fp, dict) else fp for fp in fp_raw],
+            having_param=[HavingParam.from_dict(hp) if isinstance(hp, dict) else hp for hp in hp_raw],
+            param_values=d.get("param_values", {}),
+            cte_steps=[RuntimeCteStep.from_dict(cte) if isinstance(cte, dict) else cte for cte in cte_raw],
+            natural_language=d.get("natural_language", ""),
+            limit=d.get("limit"),
+            limit_param_key=d.get("limit_param_key", ""),
+            column_map=d.get("column_map", {}),
+            chosen_join_candidate_id=d.get("chosen_join_candidate_id", ""),
+            chosen_join_path_signature=join_sig_raw,
+            extra_tables=set(d.get("extra_tables", [])),
+            sql_param=d.get("sql_param", ""),
+            sql_display_param=d.get("sql_display_param", ""),
+            sql_substituted=d.get("sql_substituted", ""),
+            deterministic_sql=d.get("deterministic_sql", ""),
+            sql_shape=(SQLShape.from_dict(d["sql_shape"]) if d.get("sql_shape") else None),
+            schema_invalid=d.get("schema_invalid", False),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all RuntimeIntent fields, with nested objects serialized recursively.
+        """
+        return {
+            "tables": self.tables,
+            "grain": self.grain,
+            "expected_rows": self.expected_rows,
+            "select_cols": [s.to_dict() for s in self.select_cols],
+            "group_by_cols": [g.to_dict() for g in self.group_by_cols],
+            "order_by_cols": [o.to_dict() for o in self.order_by_cols],
+            "filters_param": [fp.to_dict() for fp in self.filters_param],
+            "having_param": [hp.to_dict() for hp in self.having_param],
+            "param_values": self.param_values,
+            "cte_steps": [cte.to_dict() for cte in self.cte_steps],
+            "natural_language": self.natural_language,
+            "limit": self.limit,
+            "limit_param_key": self.limit_param_key,
+            "column_map": self.column_map,
+            "chosen_join_candidate_id": self.chosen_join_candidate_id,
+            "chosen_join_path_signature": self.chosen_join_path_signature,
+            "extra_tables": sorted(self.extra_tables),
+            "sql_param": self.sql_param,
+            "sql_display_param": self.sql_display_param,
+            "sql_substituted": self.sql_substituted,
+            "deterministic_sql": self.deterministic_sql,
+            "sql_shape": self.sql_shape.to_dict() if self.sql_shape else None,
+            "schema_invalid": self.schema_invalid,
+        }
+
+    @property
+    def has_aggregation(self) -> bool:
+        """Return True if any select column uses aggregation."""
+        return any(s.is_aggregated for s in self.select_cols)
+
+
+@dataclass
+class ConcreteIntent:
+    """Structural intent for template storage without values or natural language."""
+
+    intent_id: str
+    tables: list[str]
+    grain: str
+    select_cols: list[SelectCol]
+    group_by_cols: list[NormalizedExpr]
+    order_by_cols: list[OrderByCol]
+    filters_param: list[FilterParam]
+    having_param: list[HavingParam] = field(default_factory=list)
+    cte_steps: list[ConcreteCteStep] = field(default_factory=list)
+    limit: int | None = None
+    column_map: dict[str, str] = field(default_factory=dict)
+    chosen_join_candidate_id: str = ""
+    chosen_join_path_signature: list[str] = field(default_factory=list)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> ConcreteIntent:
+        """Create ConcreteIntent from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching ConcreteIntent fields.
+
+        Returns:
+
+            Populated ConcreteIntent with nested expression objects.
+        """
+        sc_raw = d.get("select_cols", [])
+        gbc_raw = d.get("group_by_cols", [])
+        obc_raw = d.get("order_by_cols", [])
+        fp_raw = d.get("filters_param", [])
+        hp_raw = d.get("having_param", [])
+        cte_raw = d.get("cte_steps", [])
+        select_cols = [
+            (
+                SelectCol.from_dict(s)
+                if isinstance(s, dict)
+                else (SelectCol(expr=NormalizedExpr.from_column(s)) if isinstance(s, str) else s)
+            )
+            for s in sc_raw
+        ]
+        group_by_cols = [
+            (
+                NormalizedExpr.from_dict(g)
+                if isinstance(g, dict)
+                else (NormalizedExpr.from_column(g) if isinstance(g, str) else g)
+            )
+            for g in gbc_raw
+        ]
+        order_by_cols = [
+            (
+                OrderByCol.from_dict(o)
+                if isinstance(o, dict)
+                else (OrderByCol(expr=NormalizedExpr.from_column(o)) if isinstance(o, str) else o)
+            )
+            for o in obc_raw
+        ]
+        return ConcreteIntent(
+            intent_id=d.get("intent_id", ""),
+            tables=d.get("tables", []),
+            grain=d.get("grain", "row_level"),
+            select_cols=select_cols,
+            group_by_cols=group_by_cols,
+            order_by_cols=order_by_cols,
+            filters_param=[FilterParam.from_dict(fp) if isinstance(fp, dict) else fp for fp in fp_raw],
+            having_param=[HavingParam.from_dict(hp) if isinstance(hp, dict) else hp for hp in hp_raw],
+            cte_steps=[ConcreteCteStep.from_dict(cte) if isinstance(cte, dict) else cte for cte in cte_raw],
+            limit=d.get("limit"),
+            column_map=d.get("column_map", {}),
+            chosen_join_candidate_id=d.get("chosen_join_candidate_id", ""),
+            chosen_join_path_signature=d.get("chosen_join_path_signature", []),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all ConcreteIntent fields, with nested expressions serialized.
+        """
+        return {
+            "intent_id": self.intent_id,
+            "tables": self.tables,
+            "grain": self.grain,
+            "select_cols": [s.to_dict() for s in self.select_cols],
+            "group_by_cols": [g.to_dict() for g in self.group_by_cols],
+            "order_by_cols": [o.to_dict() for o in self.order_by_cols],
+            "filters_param": [fp.to_dict() for fp in self.filters_param],
+            "having_param": [hp.to_dict() for hp in self.having_param],
+            "cte_steps": [cte.to_dict() for cte in self.cte_steps],
+            "limit": self.limit,
+            "column_map": self.column_map,
+            "chosen_join_candidate_id": self.chosen_join_candidate_id,
+            "chosen_join_path_signature": self.chosen_join_path_signature,
+        }
+
+
+@dataclass
+class SimulatorIntent:
+    """Unified intent for Simulator with inline values and expansion metadata."""
+
+    intent_id: str
+    tables: list[str]
+    grain: str
+    select_cols: list[SelectCol]
+    group_by_cols: list[NormalizedExpr]
+    order_by_cols: list[OrderByCol]
+    filters_param: list[FilterParam]
+    having_param: list[HavingParam]
+    param_values: dict[str, ParamValue] = field(default_factory=dict)
+    cte_steps: list[RuntimeCteStep] = field(default_factory=list)
+    question: str = ""
+    expansion_metadata: ExpansionMetadata | None = None
+    limit: int | None = None
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> SimulatorIntent:
+        """Create SimulatorIntent from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching SimulatorIntent fields.
+
+        Returns:
+
+            Populated SimulatorIntent with nested expression and metadata objects.
+        """
+        sc_raw = d.get("select_cols", [])
+        gbc_raw = d.get("group_by_cols", [])
+        obc_raw = d.get("order_by_cols", [])
+        fp_raw = d.get("filters_param", d.get("filters", []))
+        hp_raw = d.get("having_param", d.get("having", []))
+        cte_raw = d.get("cte_steps", [])
+        em_raw = d.get("expansion_metadata")
+        select_cols = [
+            (
+                SelectCol.from_dict(s)
+                if isinstance(s, dict)
+                else (SelectCol(expr=NormalizedExpr.from_column(s)) if isinstance(s, str) else s)
+            )
+            for s in sc_raw
+        ]
+        group_by_cols = [
+            (
+                NormalizedExpr.from_dict(g)
+                if isinstance(g, dict)
+                else (NormalizedExpr.from_column(g) if isinstance(g, str) else g)
+            )
+            for g in gbc_raw
+        ]
+        order_by_cols = [
+            (
+                OrderByCol.from_dict(o)
+                if isinstance(o, dict)
+                else (OrderByCol(expr=NormalizedExpr.from_column(o)) if isinstance(o, str) else o)
+            )
+            for o in obc_raw
+        ]
+        return SimulatorIntent(
+            intent_id=d.get("intent_id", ""),
+            tables=d.get("tables", []),
+            grain=d.get("grain", "row_level"),
+            select_cols=select_cols,
+            group_by_cols=group_by_cols,
+            order_by_cols=order_by_cols,
+            filters_param=[FilterParam.from_dict(fp) if isinstance(fp, dict) else fp for fp in fp_raw],
+            having_param=[HavingParam.from_dict(hp) if isinstance(hp, dict) else hp for hp in hp_raw],
+            param_values=d.get("param_values", {}),
+            cte_steps=[RuntimeCteStep.from_dict(cte) if isinstance(cte, dict) else cte for cte in cte_raw],
+            question=d.get("question", ""),
+            expansion_metadata=ExpansionMetadata.from_dict(em_raw) if em_raw else None,
+            limit=d.get("limit"),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        result = {
+            "intent_id": self.intent_id,
+            "tables": self.tables,
+            "grain": self.grain,
+            "select_cols": [s.to_dict() for s in self.select_cols],
+            "group_by_cols": [g.to_dict() for g in self.group_by_cols],
+            "order_by_cols": [o.to_dict() for o in self.order_by_cols],
+            "filters_param": [fp.to_dict() for fp in self.filters_param],
+            "having_param": [hp.to_dict() for hp in self.having_param],
+            "param_values": self.param_values,
+            "cte_steps": [cte.to_dict() for cte in self.cte_steps],
+            "question": self.question,
+            "limit": self.limit,
+        }
+        if self.expansion_metadata:
+            result["expansion_metadata"] = self.expansion_metadata.to_dict()
+        return result
+
+    def to_runtime_intent(self) -> RuntimeIntent:
+        """Convert to RuntimeIntent for pipeline execution.
+
+        Returns:
+
+            RuntimeIntent built from this SimulatorIntent with an inferred column_map.
+        """
+        column_map = {}
+        for sc in self.select_cols:
+            col = sc.expr.primary_column
+            if "." in col:
+                table, bare = col.split(".", 1)
+                if table in self.tables:
+                    column_map[bare] = table
+        for fp in self.filters_param:
+            col = fp.left_expr.primary_column
+            if "." in col:
+                table, bare = col.split(".", 1)
+                if table in self.tables:
+                    column_map[bare] = table
+
+        return RuntimeIntent(
+            tables=self.tables,
+            grain=self.grain,
+            select_cols=self.select_cols,
+            group_by_cols=self.group_by_cols,
+            order_by_cols=self.order_by_cols,
+            filters_param=self.filters_param,
+            having_param=self.having_param,
+            param_values=self.param_values,
+            cte_steps=self.cte_steps,
+            natural_language="",
+            limit=self.limit,
+            column_map=column_map,
+        )
+
+
+@dataclass
+class QSimFilter:
+    """Lightweight filter for QSim intent with column reference and operator."""
+
+    column: str
+    op: str
+    value_type: str
+    right_column: str = ""
+
+    @property
+    def is_expr_comparison(self) -> bool:
+        """True if filter is expr-vs-expr (right_column references another expression)."""
+        return bool(self.right_column)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> QSimFilter:
+        """Create QSimFilter from dictionary.
+
+        Args:
+
+            d: Dictionary with 'column', 'op', 'value_type', and optional 'right_column' keys.
+
+        Returns:
+
+            Populated QSimFilter instance.
+        """
+        return QSimFilter(
+            column=d.get("column", ""),
+            op=d.get("op", "="),
+            value_type=d.get("value_type", "categorical"),
+            right_column=d.get("right_column", ""),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with filter fields; right_column only included when set.
+        """
+        result = {"column": self.column, "op": self.op, "value_type": self.value_type}
+        if self.right_column:
+            result["right_column"] = self.right_column
+        return result
+
+
+@dataclass
+class QSimHaving:
+    """Lightweight having condition for QSim intent with aggregate expression."""
+
+    expression: str
+    op: str
+    value_type: str
+    right_expression: str = ""
+
+    @property
+    def is_expression_comparison(self) -> bool:
+        """True if this is an expression-to-expression comparison."""
+        return bool(self.right_expression)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> QSimHaving:
+        """Create QSimHaving from dictionary.
+
+        Args:
+
+            d: Dictionary with 'expression', 'op', 'value_type', and optional 'right_expression' keys.
+
+        Returns:
+
+            Populated QSimHaving instance.
+        """
+        return QSimHaving(
+            expression=d.get("expression", ""),
+            op=d.get("op", ">"),
+            value_type=d.get("value_type", "number"),
+            right_expression=d.get("right_expression", ""),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with having fields; right_expression only included when set.
+        """
+        result = {
+            "expression": self.expression,
+            "op": self.op,
+            "value_type": self.value_type,
+        }
+        if self.right_expression:
+            result["right_expression"] = self.right_expression
+        return result
+
+
+@dataclass
+class QSimIntent:
+    """Unified intent for QSim question generation with optional values."""
+
+    intent_id: str
+    tables: list[str]
+    grain: str
+    select_cols: list[str]
+    group_by_cols: list[str]
+    order_by_cols: list[str]
+    filters_param: list[QSimFilter]
+    having_param: list[QSimHaving]
+    param_values: dict[str, ParamValue] = field(default_factory=dict)
+    question: str = ""
+    variant_idx: int = 0
+    limit: int | None = None
+    distinct: bool = False
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> QSimIntent:
+        """Create QSimIntent from dictionary.
+
+        Args:
+
+            d: Dictionary with keys matching QSimIntent fields.
+
+        Returns:
+
+            Populated QSimIntent instance.
+        """
+        fp_raw = d.get("filters_param", d.get("filters", []))
+        hp_raw = d.get("having_param", d.get("having", []))
+        return QSimIntent(
+            intent_id=d.get("intent_id", ""),
+            tables=d.get("tables", []),
+            grain=d.get("grain", "row_level"),
+            select_cols=d.get("select_cols", []),
+            group_by_cols=d.get("group_by_cols", []),
+            order_by_cols=d.get("order_by_cols", []),
+            filters_param=[QSimFilter.from_dict(fp) if isinstance(fp, dict) else fp for fp in fp_raw],
+            having_param=[QSimHaving.from_dict(hp) if isinstance(hp, dict) else hp for hp in hp_raw],
+            param_values=d.get("param_values", {}),
+            question=d.get("question", ""),
+            variant_idx=d.get("variant_idx", 0),
+            limit=d.get("limit"),
+            distinct=d.get("distinct", False),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all QSimIntent fields.
+        """
+        return {
+            "intent_id": self.intent_id,
+            "tables": self.tables,
+            "grain": self.grain,
+            "select_cols": self.select_cols,
+            "group_by_cols": self.group_by_cols,
+            "order_by_cols": self.order_by_cols,
+            "filters_param": [fp.to_dict() for fp in self.filters_param],
+            "having_param": [hp.to_dict() for hp in self.having_param],
+            "param_values": self.param_values,
+            "question": self.question,
+            "variant_idx": self.variant_idx,
+            "limit": self.limit,
+            "distinct": self.distinct,
+        }
+
+
+def runtime_intent_to_concrete(runtime: RuntimeIntent, intent_id: str) -> ConcreteIntent:
+    """Extract ConcreteIntent from RuntimeIntent for template storage.
+
+    Args:
+
+        runtime: The RuntimeIntent to convert.
+        intent_id: Unique identifier to assign to the resulting ConcreteIntent.
+
+    Returns:
+
+        ConcreteIntent with runtime-only fields (param_values and natural_language) stripped.
+    """
+    return ConcreteIntent(
+        intent_id=intent_id,
+        tables=runtime.tables,
+        grain=runtime.grain,
+        select_cols=runtime.select_cols,
+        group_by_cols=runtime.group_by_cols,
+        order_by_cols=runtime.order_by_cols,
+        filters_param=runtime.filters_param,
+        having_param=runtime.having_param,
+        cte_steps=[_runtime_cte_to_concrete(cte) for cte in runtime.cte_steps],
+        limit=runtime.limit,
+        column_map=runtime.column_map,
+        chosen_join_candidate_id=runtime.chosen_join_candidate_id,
+        chosen_join_path_signature=runtime.chosen_join_path_signature,
+    )
+
+
+@dataclass
+class ValueHistory:
+    """Value history for Template tracking historical query values as a flat dict."""
+
+    param_values: list[dict[str, ParamValue]]
+    questions: list[str]
+    natural_language: list[str]
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> ValueHistory:
+        """Create ValueHistory from dictionary.
+
+        Args:
+
+            d: Dictionary with 'param_values', 'questions', and 'natural_language' list keys.
+
+        Returns:
+
+            Populated ValueHistory instance.
+        """
+        return ValueHistory(
+            param_values=d.get("param_values", []),
+            questions=d.get("questions", []),
+            natural_language=d.get("natural_language", []),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with the three parallel history lists.
+        """
+        return {
+            "param_values": self.param_values,
+            "questions": self.questions,
+            "natural_language": self.natural_language,
+        }
+
+    def add(self, param_values: dict[str, ParamValue], question: str, natural_language: str) -> None:
+        """Append a new entry to all parallel history lists.
+
+        Args:
+
+            param_values: Flat dict of parameter keys to resolved values.
+            question: The natural language question associated with this execution.
+            natural_language: The intent's natural language description.
+        """
+        self.param_values.append(param_values)
+        self.questions.append(question)
+        self.natural_language.append(natural_language or "")
+
+    def __len__(self) -> int:
+        """Return number of entries in history."""
+        return len(self.questions)
+
+
+@dataclass
+class RejectedValueHistory:
+    """Value history for RejectedTemplate tracking historical rejected values."""
+
+    param_values: list[dict[str, ParamValue]]
+    questions: list[str]
+    natural_language: list[str]
+    rejection_reasons: list[str]
+    rejection_categories: list[str]
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> RejectedValueHistory:
+        """Create RejectedValueHistory from dictionary.
+
+        Args:
+
+            d: Dictionary with all RejectedValueHistory list keys.
+
+        Returns:
+
+            Populated RejectedValueHistory instance.
+        """
+        return RejectedValueHistory(
+            param_values=d.get("param_values", []),
+            questions=d.get("questions", []),
+            natural_language=d.get("natural_language", []),
+            rejection_reasons=d.get("rejection_reasons", []),
+            rejection_categories=d.get("rejection_categories", []),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all five parallel history lists.
+        """
+        return {
+            "param_values": self.param_values,
+            "questions": self.questions,
+            "natural_language": self.natural_language,
+            "rejection_reasons": self.rejection_reasons,
+            "rejection_categories": self.rejection_categories,
+        }
+
+    def add(
+        self,
+        param_values: dict[str, ParamValue],
+        question: str,
+        natural_language: str,
+        reason: str,
+        category: str,
+    ) -> None:
+        """Append a new entry to all parallel history lists.
+
+        Args:
+
+            param_values: Flat dict of parameter keys to resolved values.
+            question: The natural language question associated with this rejection.
+            natural_language: The intent's natural language description.
+            reason: Full rejection reason message.
+            category: Coarse rejection category string.
+        """
+        self.param_values.append(param_values)
+        self.questions.append(question)
+        self.natural_language.append(natural_language or "")
+        self.rejection_reasons.append(reason)
+        self.rejection_categories.append(category)
+
+    def __len__(self) -> int:
+        """Return number of entries in history."""
+        return len(self.questions)
+
+
+@dataclass
+class Template:
+    """Validated and accepted query template."""
+
+    id: str
+    schema_hash: str
+    intent_signature: ConcreteIntent
+    intent_key: str
+    tables_used: list[str]
+    sql_param: str
+    sql_display_param: str
+    sql_fp: str
+    shape: SQLShape
+    colmap_sig: str
+    value_history: ValueHistory
+    stats: TemplateStats
+    ux_summary: str = ""
+    source: str = "human"
+    trust_level: int = 1
+    structural_defaults: dict[str, str | int | float] = field(default_factory=dict)
+    deterministic_sql: str = ""
+    aliased_sql: str = ""
+    spark_sql_param: str = ""
+
+    @property
+    def chosen_join_candidate_id(self) -> str:
+        """Delegate to intent_signature."""
+        return self.intent_signature.chosen_join_candidate_id
+
+    @property
+    def chosen_join_path_signature(self) -> list[str]:
+        """Delegate to intent_signature."""
+        return self.intent_signature.chosen_join_path_signature
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> Template:
+        """Create Template from dictionary with nested dataclass reconstruction.
+
+        Args:
+
+            d: Dictionary with all Template fields, including nested intent_signature, value_history, stats, and shape dicts.
+
+        Returns:
+
+            Fully populated Template instance.
+        """
+        intent_sig = d.get("intent_signature", {})
+        if isinstance(intent_sig, dict):
+            intent_sig = ConcreteIntent.from_dict(intent_sig)
+        vh_data = d.get("value_history", {})
+        value_history = ValueHistory.from_dict(vh_data) if isinstance(vh_data, dict) else vh_data
+        stats_data = d.get("stats", {})
+        stats = TemplateStats.from_dict(stats_data) if isinstance(stats_data, dict) else stats_data
+        shape_data = d.get("shape", {})
+        shape = SQLShape.from_dict(shape_data) if isinstance(shape_data, dict) else shape_data
+        return Template(
+            id=d.get("id", ""),
+            schema_hash=d.get("schema_hash", ""),
+            intent_signature=intent_sig,
+            intent_key=d.get("intent_key", ""),
+            tables_used=d.get("tables_used", []),
+            sql_param=d.get("sql_param", ""),
+            sql_display_param=d.get("sql_display_param", ""),
+            sql_fp=d.get("sql_fp", ""),
+            shape=shape,
+            colmap_sig=d.get("colmap_sig", ""),
+            value_history=value_history,
+            stats=stats,
+            ux_summary=d.get("ux_summary", ""),
+            source=d.get("source", "human"),
+            trust_level=d.get("trust_level", 1),
+            structural_defaults=d.get("structural_defaults", {}),
+            deterministic_sql=d.get("deterministic_sql", ""),
+            aliased_sql=d.get("aliased_sql", ""),
+            spark_sql_param=d.get("spark_sql_param", ""),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary with nested dataclass conversion.
+
+        Returns:
+
+            Dictionary with all Template fields, with intent_signature, value_history, stats, and shape serialized recursively.
+        """
+        intent_sig = (
+            self.intent_signature.to_dict() if hasattr(self.intent_signature, "to_dict") else self.intent_signature
+        )
+        vh_dict = self.value_history.to_dict() if hasattr(self.value_history, "to_dict") else self.value_history
+        stats_dict = self.stats.to_dict() if hasattr(self.stats, "to_dict") else self.stats
+        return {
+            "id": self.id,
+            "schema_hash": self.schema_hash,
+            "intent_signature": intent_sig,
+            "intent_key": self.intent_key,
+            "tables_used": self.tables_used,
+            "sql_param": self.sql_param,
+            "sql_display_param": self.sql_display_param,
+            "sql_fp": self.sql_fp,
+            "shape": self.shape.to_dict(),
+            "colmap_sig": self.colmap_sig,
+            "value_history": vh_dict,
+            "stats": stats_dict,
+            "ux_summary": self.ux_summary,
+            "source": self.source,
+            "trust_level": self.trust_level,
+            "structural_defaults": self.structural_defaults,
+            "deterministic_sql": self.deterministic_sql,
+            "aliased_sql": self.aliased_sql,
+            "spark_sql_param": self.spark_sql_param,
+        }
+
+
+@dataclass
+class RejectedTemplate:
+    """Rejected query template for negative learning."""
+
+    id: str
+    schema_hash: str
+    intent_signature: ConcreteIntent
+    intent_key: str
+    tables_used: list[str]
+    sql_param: str
+    sql_display_param: str
+    sql_fp: str
+    shape: SQLShape
+    colmap_sig: str
+    value_history: RejectedValueHistory
+    aliased_sql: str = ""
+    spark_sql_param: str = ""
+
+    @property
+    def chosen_join_candidate_id(self) -> str:
+        """Delegate to intent_signature."""
+        return self.intent_signature.chosen_join_candidate_id
+
+    @property
+    def chosen_join_path_signature(self) -> list[str]:
+        """Delegate to intent_signature."""
+        return self.intent_signature.chosen_join_path_signature
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> RejectedTemplate:
+        """Create RejectedTemplate from dictionary with nested dataclass reconstruction.
+
+        Args:
+
+            d: Dictionary with all RejectedTemplate fields, including nested intent_signature, value_history, and shape dicts.
+
+        Returns:
+
+            Fully populated RejectedTemplate instance.
+        """
+        intent_sig = d.get("intent_signature", {})
+        if isinstance(intent_sig, dict):
+            intent_sig = ConcreteIntent.from_dict(intent_sig)
+        vh_data = d.get("value_history", {})
+        value_history = RejectedValueHistory.from_dict(vh_data) if isinstance(vh_data, dict) else vh_data
+        shape_data = d.get("shape", {})
+        shape = SQLShape.from_dict(shape_data) if isinstance(shape_data, dict) else shape_data
+        return RejectedTemplate(
+            id=d.get("id", ""),
+            schema_hash=d.get("schema_hash", ""),
+            intent_signature=intent_sig,
+            intent_key=d.get("intent_key", ""),
+            tables_used=d.get("tables_used", []),
+            sql_param=d.get("sql_param", ""),
+            sql_display_param=d.get("sql_display_param", d.get("sql_param", "")),
+            sql_fp=d.get("sql_fp", ""),
+            shape=shape,
+            colmap_sig=d.get("colmap_sig", ""),
+            value_history=value_history,
+            aliased_sql=d.get("aliased_sql", ""),
+            spark_sql_param=d.get("spark_sql_param", ""),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary with nested dataclass conversion."""
+        intent_sig = (
+            self.intent_signature.to_dict() if hasattr(self.intent_signature, "to_dict") else self.intent_signature
+        )
+        vh_dict = self.value_history.to_dict() if hasattr(self.value_history, "to_dict") else self.value_history
+        return {
+            "id": self.id,
+            "schema_hash": self.schema_hash,
+            "intent_signature": intent_sig,
+            "intent_key": self.intent_key,
+            "tables_used": self.tables_used,
+            "sql_param": self.sql_param,
+            "sql_display_param": self.sql_display_param,
+            "sql_fp": self.sql_fp,
+            "shape": self.shape.to_dict(),
+            "colmap_sig": self.colmap_sig,
+            "value_history": vh_dict,
+            "aliased_sql": self.aliased_sql,
+            "spark_sql_param": self.spark_sql_param,
+        }
+
+
+@dataclass
+class SimulatorResult:
+    """Result of a single simulator run."""
+
+    intent: RuntimeIntent
+    question: str
+    sql: str | None = None
+    rows: list | None = None
+    success: bool = False
+    error: str | None = None
+    validation_issues: list[str] = field(default_factory=list)
+    confidence: float = 0.0
+    llm_response: str | None = None
+    sql_generation_attempts: int = 0
+    repair_loop_count: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a plain dictionary.
+
+        Returns:
+
+            Dictionary with all SimulatorResult fields including the serialized intent.
+        """
+        return {
+            "intent": self.intent.to_dict() if self.intent else None,
+            "question": self.question,
+            "sql": self.sql,
+            "rows": self.rows,
+            "success": self.success,
+            "error": self.error,
+            "validation_issues": self.validation_issues,
+            "confidence": self.confidence,
+            "llm_response": self.llm_response,
+            "sql_generation_attempts": self.sql_generation_attempts,
+            "repair_loop_count": self.repair_loop_count,
+        }
+
+
+@dataclass
+class TemplateMatch:
+    """Result of template matching against intent."""
+
+    intent: RuntimeIntent | None = None
+    best_template: Template | None = None
+    similarity_score: float = 0.0
+    reuse_type: str = "none"
+    semantic_warnings: list[str] = field(default_factory=list)
+    llm_calls: int = 0