dataface 0.1.2__py3-none-any.whl

dataface/core/execute/executor.py

@@ -0,0 +1,1249 @@
+"""Query execution engine.
+
+Stage: EXECUTE (Service)
+Purpose: Execute queries for compiled datafaces.
+
+Entry Points:
+    - Executor.execute_query(query_name, variables) -> List[Dict]
+    - Executor.execute_chart(chart, variables) -> List[Dict]
+
+The executor:
+- Resolves which adapter to use for each query
+- Executes queries and returns data
+- Caches results for efficiency
+
+Does NOT:
+- Compile datafaces
+- Render charts
+
+Dependencies:
+    - dataface.compile (for types)
+    - .adapters (for data source adapters)
+
+See also:
+    - render/renderer.py: Uses executor for data
+"""
+
+import hashlib
+import json
+import logging
+import re
+from functools import lru_cache
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+logger = logging.getLogger(__name__)
+
+from dataface.core.compile.jinja import resolve_jinja_template
+from dataface.core.compile.models.chart.authored import FilterDef
+from dataface.core.compile.models.chart.compiled import (
+    Chart,
+)
+from dataface.core.compile.models.face.compiled import Face, VariableValues
+from dataface.core.compile.models.query.compiled import (
+    AnyQuery,
+    SqlQuery,
+    is_sql_query,
+)
+from dataface.core.compile.variables import parse_variable_json_strings
+from dataface.core.execute.errors import ExecutionError, QueryError
+
+if TYPE_CHECKING:
+    from dataface.core.execute.adapters.adapter_registry import AdapterRegistry
+    from dataface.core.execute.adapters.base import ResolvedRelation
+    from dataface.core.execute.batch import BatchStatement
+    from dataface.core.execute.cache_backend import QueryResultCache
+
+
+@lru_cache(maxsize=256)
+def _sql_like_to_regex(pattern: str) -> re.Pattern[str]:
+    """Compile a SQL LIKE pattern to a regex (cached — same pattern reused across rows).
+
+    SQL specials: ``%`` → any sequence (``.*``), ``_`` → any one char (``.``).
+    All other characters — including fnmatch specials ``*``, ``?``, ``[``, ``]``
+    which are *literal* in SQL LIKE — are escaped via ``re.escape``.
+    Backslash escapes: ``\\%`` → literal ``%``, ``\\_`` → literal ``_``.
+    """
+    out: list[str] = []
+    i = 0
+    while i < len(pattern):
+        ch = pattern[i]
+        if ch == "\\" and i + 1 < len(pattern):
+            out.append(re.escape(pattern[i + 1]))
+            i += 2
+        elif ch == "%":
+            out.append(".*")
+            i += 1
+        elif ch == "_":
+            out.append(".")
+            i += 1
+        else:
+            out.append(re.escape(ch))
+            i += 1
+    return re.compile(r"\A(?:" + "".join(out) + r")\Z", re.DOTALL)
+
+
+# Matches a Jinja template that is a pure variable reference: "{{ var_name }}".
+# Used in _apply_filters to short-circuit through a type-preserving dict lookup
+# instead of Jinja rendering (which always produces strings and would break
+# numeric/boolean column comparisons).
+_SIMPLE_JINJA_VAR_RE: re.Pattern[str] = re.compile(r"^\{\{\s*(\w+)\s*\}\}$")
+
+# Comparison functions for ordered operators — hoisted to module level to
+# avoid rebuilding the dict on every _row_matches call.
+_CMP_FUNCS: dict[str, Any] = {
+    "gt": lambda a, b: a > b,
+    "gte": lambda a, b: a >= b,
+    "lt": lambda a, b: a < b,
+    "lte": lambda a, b: a <= b,
+}
+
+
+def _row_matches(op: str, row_val: Any, filter_val: Any) -> bool:
+    """Return True if ``row_val`` satisfies the filter predicate.
+
+    Used by ``Executor._apply_filters`` for post-execution row filtering.
+    Mirrors the operator semantics of ``filter_injection._build_condition``
+    but applied to Python values instead of a SQL AST.
+
+    Args:
+        op: Operator key — one of eq, neq, gt, gte, lt, lte, like, ilike,
+            in, not_in, between.
+        row_val: Value from the result row.
+        filter_val: Resolved variable value.
+
+    Returns:
+        True if the row passes the predicate; False to exclude it.
+
+    Raises:
+        ValueError: If ``filter_val`` is the wrong type for the operator
+            (e.g. non-list for ``in``/``not_in``, non-2-element list for
+            ``between``) or if the comparison raises TypeError due to
+            incompatible types.
+        ValueError: If ``op`` is not a recognized operator key.
+    """
+    if op == "in" or (op == "eq" and isinstance(filter_val, list)):
+        if not isinstance(filter_val, (list, tuple)):
+            raise ValueError(
+                f"Filter operator 'in' requires a list value, got {type(filter_val).__name__}"
+            )
+        return row_val in filter_val
+    if op == "not_in":
+        if not isinstance(filter_val, (list, tuple)):
+            raise ValueError(
+                f"Filter operator 'not_in' requires a list value, got {type(filter_val).__name__}"
+            )
+        return row_val not in filter_val
+    if op == "between":
+        if not isinstance(filter_val, (list, tuple)) or len(filter_val) != 2:
+            raise ValueError(
+                f"Filter operator 'between' requires a 2-element [start, end] list, "
+                f"got {filter_val!r}"
+            )
+        start, end = filter_val
+        if start is None or end is None:
+            raise ValueError(
+                f"Filter operator 'between' bounds must be non-None, got {filter_val!r}"
+            )
+        try:
+            return start <= row_val <= end
+        except TypeError as e:
+            raise ValueError(
+                f"Filter 'between' cannot compare {type(row_val).__name__} with "
+                f"bounds of type {type(start).__name__}: {e}"
+            ) from e
+    if op == "neq":
+        return row_val != filter_val
+    if op in _CMP_FUNCS:
+        try:
+            return _CMP_FUNCS[op](row_val, filter_val)
+        except TypeError as e:
+            raise ValueError(
+                f"Filter '{op}' cannot compare {type(row_val).__name__} "
+                f"with {type(filter_val).__name__}: {e}"
+            ) from e
+    if op == "like":
+        return bool(_sql_like_to_regex(str(filter_val)).match(str(row_val)))
+    if op == "ilike":
+        return bool(
+            _sql_like_to_regex(str(filter_val).lower()).match(str(row_val).lower())
+        )
+    if op == "eq":
+        return row_val == filter_val
+    raise ValueError(f"Unknown filter operator {op!r}")
+
+
+def _get_relevant_variables(
+    query: AnyQuery, variables: VariableValues | None
+) -> dict[str, Any]:
+    """Filter variables to only those this query depends on."""
+    if variables and query.variable_dependencies:
+        return {k: v for k, v in variables.items() if k in query.variable_dependencies}
+    return {}
+
+
+def _query_name_for(query: AnyQuery, registry: dict[str, AnyQuery]) -> str | None:
+    """Find the registry name for a query object, or None if not found."""
+    for name, q in registry.items():
+        if q is query:
+            return name
+    return None
+
+
+class Executor:
+    """Executes queries for datafaces.
+
+    Stage: EXECUTE (Service Module)
+
+    The executor manages query execution for a compiled dataface.
+    It handles:
+    - Query lookup from the face
+    - Adapter selection based on query type
+    - Result caching for efficiency
+    - Variable substitution
+
+    Does NOT:
+    - Compile datafaces (use compile module)
+    - Render charts (use render module)
+
+    Attributes:
+        face: The compiled dataface
+        adapter_registry: Registry of adapters
+        query_registry: Complete query registry (for cross-references)
+
+    Example:
+        >>> from pathlib import Path
+        >>> from dataface.core.compile import compile
+        >>> from dataface.core.execute import Executor
+        >>> from dataface.core.execute.adapters import build_adapter_registry
+        >>>
+        >>> result = compile(yaml_content)
+        >>> registry = build_adapter_registry(Path.cwd())
+        >>> executor = Executor(result.face, registry, query_registry=result.query_registry)
+        >>>
+        >>> # Execute a query
+        >>> data = executor.execute_query("sales", {"year": 2024})
+        >>> print(data)  # [{"date": "2024-01", "amount": 1000}, ...]
+    """
+
+    def __init__(
+        self,
+        face: Face,
+        adapter_registry: "AdapterRegistry",
+        query_registry: dict[str, AnyQuery] | None = None,
+        project_root: Path | None = None,
+        use_cache: bool = True,
+        duckdb_cache: "QueryResultCache | None" = None,
+    ):
+        """Initialize executor.
+
+        Args:
+            face: Compiled dataface
+            adapter_registry: Adapter registry. Build at the entry point with
+                ``build_adapter_registry(project_root, ...)``; the executor
+                does not invent one from cwd.
+            query_registry: Optional query registry for cross-file references
+            project_root: Root directory for resolving relative file paths
+            use_cache: Default cache behavior for query execution
+            duckdb_cache: Optional DuckDBCache for persistent caching (Suite context)
+        """
+        self.face = face
+        self.query_registry = query_registry or {}
+        self._cache: dict[str, list[dict[str, Any]]] = {}
+        self._descriptions_cache: dict[str, dict[str, tuple]] = {}
+        self._provenance_cache: dict[str, list[ResolvedRelation]] = {}
+        self._query_errors: dict[str, Exception] = {}
+        self._use_cache = use_cache
+        self._duckdb_cache = duckdb_cache
+        self.project_root = project_root
+        self.adapter_registry = adapter_registry
+
+    def execute_query(
+        self,
+        query_name: str,
+        variables: VariableValues | None = None,
+        use_cache: bool | None = None,
+        force_refresh: bool = False,
+    ) -> list[dict[str, Any]]:
+        """Execute a query and return results.
+
+        Stage: EXECUTE (Main Entry Point)
+
+        Looks up the query by name, resolves the appropriate adapter,
+        executes the query, and returns the data.
+
+        Args:
+            query_name: Name of query to execute (may include "queries." prefix)
+            variables: Variable values for query resolution
+            use_cache: Whether to use cached results if available
+            force_refresh: If True, clear both success and failure caches
+                for this query and re-run it from scratch.
+
+        Returns:
+            List of dictionaries with query results (each dict is a row)
+
+        Raises:
+            ExecutionError: If query not found or execution fails
+            CachedQueryFailure: If a cached failure exists within TTL
+
+        Example:
+            >>> data = executor.execute_query("sales", {"year": 2024})
+            >>> for row in data:
+            ...     print(f"{row['date']}: {row['amount']}")
+        """
+        # ────────────────────────────────────────────────────────────────
+        # Step 1: Normalize query name (strip "queries." prefix)
+        # ────────────────────────────────────────────────────────────────
+        if query_name.startswith("queries."):
+            query_name = query_name[8:]
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 1b: Merge variables with face defaults
+        # ────────────────────────────────────────────────────────────────
+        # Trust the normalizer - use pre-computed variable_defaults
+        variable_registry = self.face.variable_registry or {}
+
+        # Merge variables: start with None for all vars, then defaults, then user values
+        all_variables: dict[str, Any] = dict.fromkeys(variable_registry)
+        all_variables.update(self.face.variable_defaults)  # Pre-computed by normalizer
+        # Parse JSON strings in variables (from URL parameters) and merge
+        parsed_variables = parse_variable_json_strings(variables or {})
+        variables = {**all_variables, **parsed_variables}
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 2: Look up query
+        # ────────────────────────────────────────────────────────────────
+        query = self._get_query(query_name)
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 3: Check cache (use instance default if not explicitly set)
+        # ────────────────────────────────────────────────────────────────
+        should_use_cache = use_cache if use_cache is not None else self._use_cache
+        cache_key = self._cache_key(query, variables)
+
+        # force_refresh: clear all caches for this key before running
+        if force_refresh:
+            self._cache.pop(cache_key, None)
+            self._query_errors.pop(query_name, None)
+            if self._duckdb_cache:
+                from dataface.core.execute.duckdb_cache import (
+                    compute_query_hash,
+                    compute_source_hash,
+                    compute_variables_hash,
+                )
+
+                relevant_vars = _get_relevant_variables(query, variables)
+                variables_hash = compute_variables_hash(relevant_vars)
+                if is_sql_query(query):
+                    pre_sql = query.sql
+                    if query.setup_sql:
+                        pre_sql += "\n" + query.setup_sql
+                    q_hash = compute_query_hash(pre_sql)
+                    source_hash = compute_source_hash(
+                        query.source, face_sources=self.face.sources
+                    )
+                else:
+                    q_hash = compute_query_hash(query.source_description)
+                    source_hash = compute_source_hash(None)
+                self._duckdb_cache.clear(source_hash, q_hash, variables_hash)
+        else:
+            # Check in-memory cache first (CLI context)
+            if should_use_cache and cache_key in self._cache:
+                return self._cache[cache_key]
+
+            # ────────────────────────────────────────────────────────────
+            # Step 3a: Check for stored pre-execution error
+            # ────────────────────────────────────────────────────────────
+            if query_name in self._query_errors:
+                stored = self._query_errors[query_name]
+                raise QueryError(str(stored), query_name) from stored
+
+            # ────────────────────────────────────────────────────────────
+            # Step 3b: Check DuckDB failure then success cache
+            # ────────────────────────────────────────────────────────────
+            if should_use_cache and self._duckdb_cache:
+                from dataface.core.execute.cache_backend import CachedQueryFailure
+                from dataface.core.execute.duckdb_cache import (
+                    compute_query_hash,
+                    compute_source_hash,
+                    compute_variables_hash,
+                )
+
+                relevant_vars = _get_relevant_variables(query, variables)
+                variables_hash = compute_variables_hash(relevant_vars)
+                if is_sql_query(query):
+                    pre_sql = query.sql
+                    if query.setup_sql:
+                        pre_sql += "\n" + query.setup_sql
+                    q_hash = compute_query_hash(pre_sql)
+                    source_hash = compute_source_hash(
+                        query.source, face_sources=self.face.sources
+                    )
+                else:
+                    q_hash = compute_query_hash(query.source_description)
+                    source_hash = compute_source_hash(None)
+
+                # Step 3b: failure cache check
+
+                cached_failure = self._duckdb_cache.get_failure(
+                    source_hash, q_hash, variables_hash
+                )
+                if isinstance(cached_failure, CachedQueryFailure):
+                    raise cached_failure
+
+                # Step 3c: Check DuckDB success cache.
+                # Key: (source_hash, query_hash, variables_hash) — no face/query noise.
+                # A different query_hash (changed SQL) is a miss by definition since the
+                # key doesn't match.
+                cached_rows = self._duckdb_cache.get(
+                    source_hash, q_hash, variables_hash
+                )
+                if cached_rows is not None:
+                    self._cache[cache_key] = cached_rows  # warm in-memory cache
+                    return cached_rows
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 3d: Check for {{ results.X }} references (Suite context)
+        # ────────────────────────────────────────────────────────────────
+        if self._has_results_refs(query):
+            # {{ results.X }} queries bypass _resolve_query_references,
+            # so setup_sql is not supported on this path.
+            if is_sql_query(query) and query.setup_sql:
+                raise QueryError(
+                    "setup_sql is not supported on queries using {{ results.X }} references",
+                    query_name,
+                )
+            result_data = self._execute_results_query(query, variables)
+            if should_use_cache:
+                self._cache[cache_key] = result_data
+            return result_data
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 4: Resolve query references in SQL
+        # ────────────────────────────────────────────────────────────────
+        query = self._resolve_query_references(query, variables, query_name)
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 5: Execute via adapter
+        # ────────────────────────────────────────────────────────────────
+        try:
+            result = self.adapter_registry.execute(query, variables, face=self.face)
+        except Exception as e:
+            self._cache_failure_to_duckdb(query_name, query, variables, e)
+            raise QueryError(str(e), query_name) from e
+
+        if not result.is_success:
+            err = QueryError(result.error or "Unknown error", query_name)
+            self._cache_failure_to_duckdb(query_name, query, variables, err)
+            raise err
+
+        # ────────────────────────────────────────────────────────────────
+        # Step 6: Cache and return
+        # ────────────────────────────────────────────────────────────────
+        if should_use_cache:
+            self._cache[cache_key] = result.data
+
+        # Cache column descriptions from cursor.description for chart decisions
+        if result.column_descriptions:
+            self._descriptions_cache[query_name] = result.column_descriptions
+
+        # Cache dbt relation provenance for chart schema-status badges
+        if result.resolved_relations:
+            self._provenance_cache[query_name] = result.resolved_relations
+
+        # Also cache to DuckDB if configured (Suite context)
+        if self._duckdb_cache:
+            self._cache_to_duckdb(query_name, query, result.data, variables)
+
+        return result.data
+
+    def execute_chart(
+        self,
+        chart: Chart | str,
+        variables: VariableValues | None = None,
+        use_cache: bool | None = None,
+    ) -> list[dict[str, Any]]:
+        """Execute the query for a chart.
+
+        Convenience method that handles chart → query lookup.
+
+        Args:
+            chart: Chart or chart name string
+            variables: Variable values
+            use_cache: Whether to use cache (defaults to instance setting)
+
+        Returns:
+            Query results for the chart
+
+        Example:
+            >>> chart = face.charts["revenue"]
+            >>> data = executor.execute_chart(chart, {"year": 2024})
+        """
+        if isinstance(chart, str):
+            if chart not in self.face.charts:
+                raise ExecutionError(f"Chart '{chart}' not found")
+            chart = self.face.charts[chart]
+
+        # Handle blank charts (no query) - return empty data
+        if chart.query_name is None:
+            return []
+
+        # Get data (use_cache will fall back to instance default in execute_query)
+        data = self.execute_query(chart.query_name, variables, use_cache)
+
+        # Apply chart-level filters if present
+        if chart.filters:
+            data = self._apply_filters(data, chart.filters, variables)
+
+        return data
+
+    def get_column_descriptions(self, query_name: str) -> dict[str, tuple] | None:
+        """Get cached PEP 249 cursor.description tuples for a query.
+
+        Returns the full description tuples captured from cursor.description
+        during the last execution. Each value is a 7-tuple:
+        (name, type_code, display_size, internal_size, precision, scale, null_ok).
+
+        Returns None if the query hasn't been executed or the adapter
+        didn't provide descriptions (e.g. CSV, HTTP adapters).
+        """
+        if query_name and query_name.startswith("queries."):
+            query_name = query_name[8:]
+        return self._descriptions_cache.get(query_name)
+
+    def get_query_provenance(self, query_name: str) -> list["ResolvedRelation"] | None:
+        """Get cached dbt relation provenance for a query.
+
+        Returns a list of ResolvedRelation objects captured during execution,
+        or None if no provenance is available (non-dbt queries, not yet executed).
+        """
+        if query_name and query_name.startswith("queries."):
+            query_name = query_name[8:]
+        return self._provenance_cache.get(query_name)
+
+    def execute_face_batch(
+        self,
+        variables: VariableValues | None = None,
+    ) -> dict[str, list[dict[str, Any]]]:
+        """Execute all dataface queries using batch temp table optimization.
+
+        This method executes all queries needed for the dataface using temp tables
+        for queries that are referenced by other queries. This is more efficient than
+        executing queries individually when there are shared dependencies.
+
+        The batch execution:
+        1. Collects all queries used by charts (and their dependencies)
+        2. Groups queries by database source
+        3. For each source, generates batch SQL with temp tables
+        4. Executes the batch and caches results
+        5. Cleans up temp tables after execution
+
+        IMPORTANT: Batching + Caching Interaction
+        ------------------------------------------
+        While queries are EXECUTED together as a batch, they are CACHED individually.
+        This means:
+        - If you re-run the dataface with different variables, only queries that
+          depend on those variables will be re-executed (cache miss)
+        - Queries that don't use the changed variables will return cached results
+        - Each query's cache key is based on its SQL content + relevant variables,
+          NOT on whether it was part of a batch
+
+        Example: Dataface with 10 queries, 2 use variable 'region':
+          Initial load (region=west): Executes all 10 queries, caches all 10
+          Second load (region=east): Re-executes only 2 queries, uses cache for 8
+
+        IMPORTANT: Connection/Session Requirements
+        ------------------------------------------
+        Temp tables are session-scoped in most databases. For batch execution
+        to work correctly, all statements in a batch MUST execute on the same
+        database connection/session. If the adapter creates a new connection
+        per query, temp tables created in earlier statements won't be visible
+        to later statements.
+
+        Note:
+            Variable names should not include "queries" as a key, as this is
+            a reserved namespace for query references.
+
+        Args:
+            variables: Variable values for query resolution
+
+        Returns:
+            Dict mapping query names to their results
+
+        Raises:
+            CrossSourceReferenceError: If a query references another
+                query on a different database source
+            ValueError: If variables contain reserved namespace "queries"
+
+        Example:
+            >>> executor = Executor(compiled_face, adapter_registry)
+            >>> results = executor.execute_face_batch({"region": "west"})
+            >>> orders_data = results["orders"]
+            >>> high_value_data = results["high_value"]
+        """
+        from dataface.core.execute.batch import (
+            TEMP_TABLE_PREFIX,
+            create_batch_execution_plan,
+        )
+
+        # Validate variables don't conflict with reserved namespaces
+        if variables and "queries" in variables:
+            raise ValueError(
+                "Variable name 'queries' is reserved for query references. "
+                "Please use a different variable name."
+            )
+
+        # Generate batch execution plan
+        plan = create_batch_execution_plan(self.face)
+
+        results: dict[str, list[dict[str, Any]]] = {}
+        created_temp_tables: dict[str, list[str]] = {}  # source -> temp table names
+
+        try:
+            for source, statements in plan.items():
+                # Track temp tables created for cleanup
+                created_temp_tables[source] = [
+                    f"{TEMP_TABLE_PREFIX}{stmt.query_name}"
+                    for stmt in statements
+                    if stmt.is_temp_table_create
+                ]
+
+                # Execute each statement in the batch
+                source_results = self._execute_batch_statements(
+                    statements, source, variables
+                )
+                results.update(source_results)
+        finally:
+            # Clean up temp tables
+            self._cleanup_temp_tables(created_temp_tables)
+
+        return results
+
+    def _cleanup_temp_tables(
+        self,
+        temp_tables: dict[str, list[str]],
+    ) -> None:
+        """Clean up temporary tables created during batch execution.
+
+        This method attempts to drop all temp tables created during batch
+        execution. Failures are logged but don't raise exceptions, since
+        temp tables typically auto-drop at session end anyway.
+
+        Args:
+            temp_tables: Dict mapping source to list of temp table names
+        """
+        from dataface.core.execute.dialects import get_dialect
+
+        for source, table_names in temp_tables.items():
+            dialect = get_dialect(source)
+
+            for table_name in table_names:
+                try:
+                    drop_sql = dialect.drop_temp_table(table_name)
+                    temp_query = SqlQuery(sql=drop_sql, source=source)
+                    self.adapter_registry.execute(temp_query, None, face=self.face)
+                except Exception as e:  # noqa: BLE001
+                    # Log but don't fail - temp tables auto-drop at session end
+                    logger.warning("Failed to drop temp table '%s': %s", table_name, e)
+
+    def _execute_batch_statements(
+        self,
+        statements: list["BatchStatement"],
+        source: str,
+        variables: VariableValues | None,
+    ) -> dict[str, list[dict[str, Any]]]:
+        """Execute a list of batch statements for a single source.
+
+        IMPORTANT: Connection/Session Requirements
+        ------------------------------------------
+        Temp tables are session-scoped in most databases. For batch execution
+        to work correctly, all statements in a batch MUST execute on the same
+        database connection/session. The adapter_registry should maintain a
+        persistent connection for the duration of the batch.
+
+        If the adapter creates a new connection per query, temp tables created
+        in earlier statements won't be visible to later statements.
+
+        Security:
+        ---------
+        Uses parameterized queries via render_parameterized() for SQL injection
+        prevention. Variable values are passed as parameters to the database
+        driver, not interpolated into the SQL string.
+
+        Args:
+            statements: List of BatchStatement objects to execute
+            source: Database source name
+            variables: Variable values for query resolution
+
+        Returns:
+            Dict mapping query names to their results
+        """
+        from dataface.core.compile.parameterized import render_parameterized
+        from dataface.core.execute.dialects import get_dialect
+
+        results: dict[str, list[dict[str, Any]]] = {}
+
+        # Get dialect for parameterization
+        dialect = get_dialect(source)
+
+        for stmt in statements:
+            # TWO-PHASE JINJA RESOLUTION:
+            # Phase 1 (already done in generate_batch_sql): {{ queries.X }} refs
+            #         are replaced with temp table references (_df_temp_X)
+            # Phase 2 (here): Variable templates like {{ region }} are resolved
+            #         using parameterized queries for SQL injection prevention
+            parameterized = render_parameterized(
+                stmt.sql,
+                variables=variables or {},
+                dialect=dialect,
+            )
+
+            # Create a temporary SqlQuery for execution with parameterized SQL
+            temp_query = SqlQuery(sql=parameterized.sql, source=source)
+
+            try:
+                # Pass params directly to the adapter to skip re-parameterization
+                result = self.adapter_registry.execute(
+                    temp_query, variables, params=parameterized.params, face=self.face
+                )
+
+                if not result.is_success:
+                    # Add context about whether this was a temp table creation
+                    error_context = (
+                        f"Failed to create temp table for '{stmt.query_name}'"
+                        if stmt.is_temp_table_create
+                        else f"Query execution failed for '{stmt.query_name}'"
+                    )
+                    raise QueryError(
+                        f"{error_context}: {result.error or 'Unknown error'}",
+                        stmt.name,
+                    )
+
+                # For temp table creation, we don't need to store results
+                # (the temp table itself is the "result" - accessible by later queries)
+                if not stmt.is_temp_table_create:
+                    # Use query_name for caching/results (not statement name)
+                    query_name = stmt.query_name or stmt.name
+                    results[query_name] = result.data
+
+                    # Cache each query result INDIVIDUALLY (not the whole batch)
+                    # This is critical: even though we execute as a batch, we cache
+                    # per-query so that future requests can benefit from granular caching.
+                    #
+                    # Example: If 2 out of 10 queries change due to variable changes,
+                    # only those 2 will cache-miss and re-execute. The other 8 will
+                    # return from cache without hitting the database.
+                    if self._use_cache:
+                        cache_key = self._cache_key(temp_query, variables)
+                        self._cache[cache_key] = result.data
+                    if result.column_descriptions:
+                        self._descriptions_cache[query_name] = (
+                            result.column_descriptions
+                        )
+                    if result.resolved_relations:
+                        self._provenance_cache[query_name] = result.resolved_relations
+
+            except Exception as e:
+                if isinstance(e, QueryError):
+                    raise
+                # Add context about whether this was a temp table creation
+                error_context = (
+                    f"Failed to create temp table for '{stmt.query_name}'"
+                    if stmt.is_temp_table_create
+                    else f"Query execution failed for '{stmt.query_name}'"
+                )
+                raise QueryError(f"{error_context}: {e}", stmt.name) from e
+
+        return results
+
+    def clear_cache(self) -> None:
+        """Clear all caches (query results, column descriptions, provenance, errors)."""
+        self._cache.clear()
+        self._descriptions_cache.clear()
+        self._provenance_cache.clear()
+        self._query_errors.clear()
+
+    def _get_query(self, query_name: str) -> AnyQuery:
+        """Look up a query by name.
+
+        Checks face.queries first, then query_registry.
+
+        Args:
+            query_name: Query name
+
+        Returns:
+            AnyQuery
+
+        Raises:
+            ExecutionError: If query not found
+        """
+        if query_name in self.face.queries:
+            return self.face.queries[query_name]
+
+        if query_name in self.query_registry:
+            return self.query_registry[query_name]
+
+        raise ExecutionError(f"Query '{query_name}' not found")
+
+    def _cache_key(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None,
+    ) -> str:
+        """Generate cache key from query content hash + relevant variables.
+
+        The cache key is designed to:
+        1. Use query content (SQL template, file path, etc.) - not query name
+           so that changing SQL invalidates the cache even if name stays same
+        2. Only include variables the query actually uses (from variable_dependencies)
+           so changing unrelated variables doesn't cause cache misses
+        3. Use deterministic hashing (SHA-256) that works across processes
+
+        This granular cache key design enables efficient partial re-execution:
+        When variables change, only queries that depend on those specific variables
+        will have different cache keys (cache miss). Other queries return from cache.
+
+        Example: Dataface with queries A (uses var 'region'), B (uses var 'date'),
+                 C (uses no variables). When 'region' changes:
+                 - Query A: cache miss (key changes) → re-executes
+                 - Query B: cache hit (key unchanged) → returns cached data
+                 - Query C: cache hit (key unchanged) → returns cached data
+
+        Args:
+            query: The compiled query object
+            variables: All available variable values
+
+        Returns:
+            Cache key in format "{query_hash}:{variables_hash}"
+        """
+        # Hash the query content - use SQL + setup_sql for SQL queries
+        if is_sql_query(query):
+            query_content = query.sql
+            if query.setup_sql:
+                query_content += "\n" + query.setup_sql
+        else:
+            query_content = query.source_description
+
+        query_hash = hashlib.sha256(query_content.encode()).hexdigest()[:16]
+
+        relevant_vars = _get_relevant_variables(query, variables)
+
+        # Deterministic hash using json.dumps with sorted keys
+        # sorted(items()) ensures consistent ordering
+        if relevant_vars:
+            variables_hash = hashlib.sha256(
+                json.dumps(sorted(relevant_vars.items()), default=str).encode()
+            ).hexdigest()[:16]
+        else:
+            variables_hash = "0" * 16
+
+        return f"{query_hash}:{variables_hash}"
+
+    def _resolve_query_references(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None,
+        query_name: str | None = None,
+    ) -> AnyQuery:
+        """Resolve {{ queries.* }} references in SQL.
+
+        Resolves nested references in topological dependency order so that
+        multi-level query composition (e.g. style -> calc -> base) works.
+        Also propagates setup_sql from referenced queries so that their
+        preambles run before the dependent query.
+
+        Args:
+            query: Query to resolve
+            variables: Variable values
+            query_name: Name of this query (for dependency-ordered lookup)
+
+        Returns:
+            Query with fully resolved SQL and accumulated setup_sql
+        """
+        # Only SQL queries can have query references
+        if not is_sql_query(query):
+            return query
+
+        # Merge face queries and registry for resolution
+        all_queries = {**self.face.queries, **self.query_registry}
+
+        # Collect setup_sql from the full dependency chain
+        from dataface.core.execute.setup_sql import collect_setup_sql
+
+        setup_stmts = collect_setup_sql(
+            # We need a name to look up, but we may be called with an unnamed query.
+            # Pass the query's own name by searching all_queries; fallback to just
+            # using the query's own setup_sql.
+            _query_name_for(query, all_queries) or "__anonymous__",
+            {**all_queries, "__anonymous__": query},
+        )
+        merged_setup = "\n".join(setup_stmts) if setup_stmts else None
+
+        if "{{ queries." not in query.sql:
+            # No Jinja refs to resolve, but may still need propagated setup_sql
+            if merged_setup != query.setup_sql:
+                return query.model_copy(update={"setup_sql": merged_setup})
+            return query
+
+        from dataface.core.compile.jinja import resolve_query_sql_with_dependencies
+
+        resolved_sql = resolve_query_sql_with_dependencies(
+            query_name or "",
+            all_queries,
+            variables=variables,
+        )
+
+        # Create new SqlQuery with resolved SQL and propagated setup_sql
+        updates: dict = {"sql": resolved_sql}
+        if merged_setup:
+            updates["setup_sql"] = merged_setup
+        return query.model_copy(update=updates)
+
+    def _apply_filters(
+        self,
+        data: list[dict[str, Any]],
+        filters: dict[str, FilterDef],
+        variables: VariableValues | None,
+    ) -> list[dict[str, Any]]:
+        """Apply chart-level FilterDef bindings to data rows (AND logic).
+
+        Each binding specifies an operator and either a plain variable name
+        (``var``) or a Jinja template (``template``). The value is resolved once
+        before the row loop; if it is None, "", or the string "none" (case-insensitive)
+        the predicate is skipped entirely (null-value means "no filter applied").
+
+        Args:
+            data: Query result rows.
+            filters: Mapping of column name → FilterDef instance.
+            variables: Runtime variable values.
+
+        Returns:
+            Filtered rows (all bindings must match — AND semantics).
+        """
+        if not filters or not data:
+            return data
+
+        vars_: dict[str, Any] = variables or {}
+
+        # Resolve all filter values once before the row loop.
+        # active_filters holds only predicates that are not skipped.
+        active_filters: list[tuple[str, str, Any]] = []  # (col, op, value)
+        for col, binding in filters.items():
+            if binding.template is not None:
+                # Simple "{{ var_name }}" → type-preserving direct lookup so
+                # numeric/boolean column values compare correctly (Jinja always
+                # renders to strings, which would break eq on non-string columns).
+                m = _SIMPLE_JINJA_VAR_RE.match(binding.template)
+                if m:
+                    value: Any = vars_.get(m.group(1))
+                    if value is None or value == "":
+                        continue
+                else:
+                    # Complex expression (e.g. "{{ x if x else '' }}") — render
+                    # through Jinja; result is always a string.
+                    value = resolve_jinja_template(
+                        binding.template, vars_, strict=False
+                    )
+                    # "" / "none" (Jinja renders Python None as "None") → skip.
+                    if not value or str(value).lower() == "none":
+                        continue
+            else:
+                assert binding.var is not None
+                value = vars_.get(binding.var)
+                if value is None or value == "":
+                    continue
+
+            # Empty collection → skip (mirrors SQL: no IN()/NOT IN() clause emitted).
+            # Also applies to eq when value is a list (eq delegates to in).
+            if (
+                binding.op in ("in", "not_in", "eq")
+                and isinstance(value, (list, tuple))
+                and not value
+            ):
+                continue
+            # between with None bound → skip (mirrors SQL: no BETWEEN clause emitted).
+            if (
+                binding.op == "between"
+                and isinstance(value, (list, tuple))
+                and len(value) == 2
+                and (value[0] is None or value[1] is None)
+            ):
+                continue
+            active_filters.append((col, binding.op, value))
+
+        if not active_filters:
+            return data
+
+        filtered = []
+        for row in data:
+            keep = True
+            for col, op, value in active_filters:
+                if not _row_matches(op, row.get(col), value):
+                    keep = False
+                    break
+            if keep:
+                filtered.append(row)
+
+        return filtered
+
+    # ─────────────────────────────────────────────────────────────────────
+    # DuckDB Cache Integration (Suite context)
+    # ─────────────────────────────────────────────────────────────────────
+
+    def _has_results_refs(self, query: AnyQuery) -> bool:
+        """Check if query has {{ results.X }} references.
+
+        Args:
+            query: Query to check
+
+        Returns:
+            True if query references cached results
+        """
+        if not is_sql_query(query):
+            return False
+        return "{{ results." in query.sql or "{{results." in query.sql
+
+    def _extract_results_refs(self, sql: str) -> list[str]:
+        """Extract {{ results.X }} reference names from SQL.
+
+        Args:
+            sql: SQL with potential results references
+
+        Returns:
+            List of query names referenced
+        """
+        import re
+
+        pattern = r"\{\{\s*results\.(\w+)\s*\}\}"
+        return re.findall(pattern, sql)
+
+    def _cache_to_duckdb(
+        self,
+        query_name: str,
+        query: AnyQuery,
+        data: list[dict[str, Any]],
+        variables: VariableValues | None,
+    ) -> None:
+        """Cache query result to DuckDB using the new (source, query, vars) key."""
+        if not self._duckdb_cache or not data:
+            return
+
+        from dataface.core.execute.duckdb_cache import (
+            compute_query_hash,
+            compute_source_hash,
+            compute_variables_hash,
+        )
+
+        if is_sql_query(query):
+            query_content = query.sql
+            if query.setup_sql:
+                query_content += "\n" + query.setup_sql
+            query_hash = compute_query_hash(query_content)
+            source_hash = compute_source_hash(
+                query.source, face_sources=self.face.sources
+            )
+        else:
+            query_hash = compute_query_hash(query.source_description)
+            source_hash = compute_source_hash(None)
+
+        relevant_vars = _get_relevant_variables(query, variables)
+        variables_hash = compute_variables_hash(relevant_vars)
+        face_slug = self._get_face_slug()
+
+        self._duckdb_cache.put(
+            source_hash,
+            query_hash,
+            variables_hash,
+            data,
+            face_slug=face_slug,
+            query_name=query_name,
+            variables=relevant_vars,
+        )
+
+    def _cache_failure_to_duckdb(
+        self,
+        query_name: str,
+        query: AnyQuery,
+        variables: VariableValues | None,
+        exception: Exception,
+    ) -> None:
+        """Cache a query failure to DuckDB (if configured). Never raises."""
+        if not self._duckdb_cache:
+            return
+        try:
+            from dataface.core.execute.duckdb_cache import (
+                compute_query_hash,
+                compute_source_hash,
+                compute_variables_hash,
+            )
+
+            if is_sql_query(query):
+                query_content = query.sql
+                if query.setup_sql:
+                    query_content += "\n" + query.setup_sql
+                query_hash = compute_query_hash(query_content)
+                source_hash = compute_source_hash(
+                    query.source, face_sources=self.face.sources
+                )
+            else:
+                query_hash = compute_query_hash(query.source_description)
+                source_hash = compute_source_hash(None)
+
+            relevant_vars = _get_relevant_variables(query, variables)
+            variables_hash = compute_variables_hash(relevant_vars)
+            face_slug = self._get_face_slug()
+
+            self._duckdb_cache.put_failure(
+                source_hash,
+                query_hash,
+                variables_hash,
+                exception,
+                face_slug=face_slug,
+                query_name=query_name,
+            )
+        except Exception:  # noqa: BLE001
+            logger.debug(
+                "Failed to cache query failure for '%s'",
+                query_name,
+                exc_info=True,
+            )
+
+    def _execute_results_query(
+        self,
+        query: AnyQuery,
+        variables: VariableValues | None,
+    ) -> list[dict[str, Any]]:
+        """Execute a query that uses {{ results.X }} references.
+
+        First ensures all referenced queries are cached, then executes
+        the query against DuckDB.
+
+        Args:
+            query: Query with results references
+            variables: Variable values
+
+        Returns:
+            Query result data
+
+        Raises:
+            ExecutionError: If DuckDB cache not configured
+        """
+        if not self._duckdb_cache:
+            raise ExecutionError(
+                "{{ results.X }} queries require a query result cache. "
+                "Use Suite context or configure duckdb_cache."
+            )
+
+        if not self._duckdb_cache.supports_results_refs():
+            raise ExecutionError(
+                "{{ results.X }} requires a SQL-engine cache backend. "
+                "The configured cache does not support result references."
+            )
+
+        if not is_sql_query(query):
+            raise ExecutionError("{{ results.X }} can only be used in SQL queries")
+
+        # Extract referenced queries
+        refs = self._extract_results_refs(query.sql)
+
+        # Ensure each referenced query is cached
+        for ref_name in refs:
+            ref_query = self._get_query(ref_name)
+            ref_data = self._execute_and_cache_for_results(
+                ref_name, ref_query, variables
+            )
+            if ref_data is None:
+                raise ExecutionError(
+                    f"Failed to cache query '{ref_name}' for results reference"
+                )
+
+        # Rewrite SQL to use DuckDB view names
+        face_slug = self._get_face_slug()
+        rewritten_sql = self._duckdb_cache.rewrite_results_refs(face_slug, query.sql)
+
+        # Execute against DuckDB
+        return self._duckdb_cache.execute_results_query(face_slug, rewritten_sql)
+
+    def _execute_and_cache_for_results(
+        self,
+        query_name: str,
+        query: AnyQuery,
+        variables: VariableValues | None,
+    ) -> list[dict[str, Any]] | None:
+        """Execute a query and cache it for {{ results.X }} access.
+
+        Args:
+            query_name: Query name
+            query: Query to execute
+            variables: Variable values
+
+        Returns:
+            Query result data
+        """
+        if not self._duckdb_cache:
+            return None
+
+        from dataface.core.execute.duckdb_cache import (
+            compute_query_hash,
+            compute_source_hash,
+            compute_variables_hash,
+        )
+
+        relevant_vars = _get_relevant_variables(query, variables)
+        variables_hash = compute_variables_hash(relevant_vars)
+
+        if is_sql_query(query):
+            pre_sql = query.sql
+            if query.setup_sql:
+                pre_sql += "\n" + query.setup_sql
+            q_hash = compute_query_hash(pre_sql)
+            source_hash = compute_source_hash(
+                query.source, face_sources=self.face.sources
+            )
+        else:
+            q_hash = compute_query_hash(query.source_description)
+            source_hash = compute_source_hash(None)
+
+        cached = self._duckdb_cache.get(source_hash, q_hash, variables_hash)
+        if cached:
+            return cached
+
+        # Execute the query
+        try:
+            if is_sql_query(query):
+                resolved_query = self._resolve_query_references(
+                    query, variables, query_name
+                )
+                result = self.adapter_registry.execute(
+                    resolved_query, variables, face=self.face
+                )
+            else:
+                result = self.adapter_registry.execute(query, variables, face=self.face)
+
+            if not result.is_success:
+                return None
+
+            # Cache to DuckDB
+            self._cache_to_duckdb(query_name, query, result.data, variables)
+
+            return result.data
+
+        except Exception:  # noqa: BLE001
+            # Log but don't fail - returns None to indicate caching failed
+            # This is intentional: the query may still succeed via direct execution
+            logging.getLogger(__name__).debug(
+                "Failed to execute and cache query '%s' for results reference",
+                query_name,
+                exc_info=True,
+            )
+            return None
+
+    def _get_face_slug(self) -> str:
+        """Get face slug for table naming.
+
+        Returns:
+            Sanitized face slug
+        """
+        # Trust normalizer guarantees - use direct attribute access
+        # Face always has title (may be empty string) and id (always present)
+        return self.face.title or self.face.id or "default"