dataface 0.1.2__py3-none-any.whl

dataface/core/compile/jinja.py

@@ -0,0 +1,511 @@
+"""Jinja template resolution module.
+
+Stage: COMPILE (Part of Step 3: Normalization)
+Purpose: Resolve Jinja templates in queries and SQL strings.
+
+Entry Points:
+    - resolve_jinja_template(template: str, variables: Dict) -> str
+    - resolve_query_filters(filters: Dict, variables: Dict) -> Dict
+    - extract_variable_dependencies(template: str) -> Set[str]
+
+Inputs:
+    - Template strings with {{ variable }} syntax
+    - Variable values for substitution
+
+Outputs:
+    - Resolved strings with variables substituted
+    - Set of variable names referenced in templates
+
+Dependencies:
+    - jinja2 (Environment, Template, meta)
+
+Errors:
+    - JinjaError: Template syntax or resolution errors
+
+See also:
+    - compile/normalizer.py: Uses this module
+
+Security Note:
+    When using strict=False (lenient mode), undefined variables are silently
+    converted to empty strings. This mode should ONLY be used in interactive
+    contexts (chart editor) where the user is actively editing and expects
+    partial results. Production rendering should always use strict=True.
+"""
+
+import logging
+import re
+from typing import Any
+
+from jinja2 import (
+    Environment,
+    StrictUndefined,
+    TemplateError,
+    TemplateSyntaxError,
+    Undefined,
+    UndefinedError,
+    meta,
+)
+
+from dataface.core.compile._jinja_helpers import _LenientUndefined, _QueryNamespace
+from dataface.core.compile.errors import JinjaError
+
+logger = logging.getLogger(__name__)
+
+
+# Jinja environment with strict undefined (errors on missing variables)
+_jinja_env = Environment(undefined=StrictUndefined)
+
+# Lenient Jinja environment (undefined variables become empty/None)
+_jinja_env_lenient = Environment(undefined=_LenientUndefined)
+
+# Known context items that are NOT user variables (helper functions, namespaces)
+_KNOWN_CONTEXT = {"filter", "filter_date_range", "queries"}
+
+
+def extract_variable_dependencies(template_str: str) -> set[str]:
+    """Extract variable names referenced in a Jinja template.
+
+    Uses Jinja's AST parser to find all undeclared variables in the template.
+    Filters out known helper functions and namespaces.
+
+    This is used during normalization to track which variables a query,
+    chart title, or other template depends on.
+
+    Args:
+        template_str: String potentially containing Jinja expressions
+
+    Returns:
+        Set of variable names referenced in the template
+
+    Example:
+        >>> extract_variable_dependencies(
+        ...     "SELECT * FROM orders WHERE region = '{{ region }}'"
+        ... )
+        {'region'}
+
+        >>> extract_variable_dependencies(
+        ...     "SELECT * WHERE {{ filter('region', region) }}"
+        ... )
+        {'region'}
+    """
+    if not template_str or not isinstance(template_str, str):
+        return set()
+
+    # Quick check - no Jinja syntax
+    if "{{" not in template_str and "{%" not in template_str:
+        return set()
+
+    try:
+        env = Environment()
+        ast = env.parse(template_str)
+        undeclared = meta.find_undeclared_variables(ast)
+        # Filter out known context items (helpers, not user variables)
+        return undeclared - _KNOWN_CONTEXT
+    except TemplateSyntaxError:
+        # Jinja syntax errors are expected here for invalid templates.
+        # Return empty set - the actual render will catch and report the syntax error
+        # with proper context. This is intentionally silent because this function
+        # is only used for dependency detection, not validation.
+        return set()
+
+
+def resolve_jinja_template(
+    template: str,
+    variables: dict[str, Any] | None = None,
+    queries: dict[str, Any] | None = None,
+    strict: bool = True,
+) -> str:
+    """Resolve a Jinja template string.
+
+    Stage: COMPILE (Step 3: Normalization - Jinja Resolution)
+
+    Resolves {{ variable }} expressions in template strings.
+    Also handles {{ queries.query_name }} references for SQL composition.
+
+    Args:
+        template: String potentially containing Jinja expressions
+        variables: Variable values for substitution
+        queries: Query registry for {{ queries.* }} resolution
+        strict: If True (default), raises error on undefined variables.
+                If False, undefined variables become None/empty (useful for
+                chart editor where variables may not all be set).
+
+    Returns:
+        Resolved string with variables substituted
+
+    Raises:
+        JinjaError: If template syntax is invalid or (if strict) variable not found
+
+    Example:
+        >>> resolve_jinja_template(
+        ...     "SELECT * FROM users WHERE status = '{{ status }}'",
+        ...     variables={"status": "active"}
+        ... )
+        "SELECT * FROM users WHERE status = 'active'"
+    """
+    if not template or not isinstance(template, str):
+        return template
+
+    # Quick check for Jinja syntax
+    if "{{" not in template and "{%" not in template:
+        return template
+
+    variables = variables or {}
+    queries = queries or {}
+
+    # Build context with variables and query helper
+    context = {**variables}
+
+    # Add queries namespace for {{ queries.query_name }} resolution
+    if queries:
+        context["queries"] = _QueryNamespace(queries)
+
+    # Add helper functions
+    context["filter"] = _filter_helper
+    context["filter_date_range"] = _filter_date_range_helper
+
+    # Use strict or lenient environment
+    jinja_env = _jinja_env if strict else _jinja_env_lenient
+
+    try:
+        jinja_template = jinja_env.from_string(template)
+        return jinja_template.render(context)
+    except UndefinedError as e:
+        raise JinjaError(f"Undefined variable: {e}", template) from e
+    except TemplateSyntaxError as e:
+        raise JinjaError(f"Template syntax error: {e}", template) from e
+    except TemplateError as e:
+        raise JinjaError(f"Template error: {e}", template) from e
+
+
+def resolve_query_filters(
+    filters: dict[str, Any],
+    variables: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Resolve Jinja expressions in filter values.
+
+    Args:
+        filters: Dictionary of filter conditions
+        variables: Variable values for substitution
+
+    Returns:
+        Filters with resolved values
+
+    Example:
+        >>> resolve_query_filters(
+        ...     {"status": "{{ selected_status }}"},
+        ...     variables={"selected_status": "active"}
+        ... )
+        {"status": "active"}
+    """
+    if not filters:
+        return filters
+
+    resolved: dict[str, Any] = {}
+    for key, value in filters.items():
+        if isinstance(value, str):
+            resolved[key] = resolve_jinja_template(value, variables)
+        elif isinstance(value, dict):
+            resolved[key] = resolve_query_filters(value, variables)
+        elif isinstance(value, list):
+            resolved[key] = [
+                resolve_jinja_template(v, variables) if isinstance(v, str) else v
+                for v in value
+            ]
+        else:
+            resolved[key] = value
+
+    return resolved
+
+
+def detect_query_dependencies(queries: dict[str, Any]) -> dict[str, list[str]]:
+    """Detect dependencies between queries for circular reference detection.
+
+    Scans query SQL for {{ queries.* }} references and builds a dependency graph.
+    Raises an error if circular dependencies are detected.
+
+    Args:
+        queries: Dictionary of query definitions
+
+    Returns:
+        Dictionary mapping query name to list of dependencies
+
+    Raises:
+        JinjaError: If circular dependencies detected
+    """
+    dependencies: dict[str, list[str]] = {}
+
+    for name, query in queries.items():
+        deps: list[str] = []
+
+        # Get SQL content
+        sql = None
+        if hasattr(query, "sql"):
+            sql = query.sql
+        elif isinstance(query, dict):
+            sql = query.get("sql")
+
+        if sql:
+            # Find {{ queries.* }} references
+            pattern = r"\{\{\s*queries\.(\w+)\s*\}\}"
+            matches = re.findall(pattern, sql)
+            deps.extend(matches)
+
+        dependencies[name] = deps
+
+    # Check for circular dependencies
+    _detect_circular(dependencies)
+
+    return dependencies
+
+
+def _detect_circular(dependencies: dict[str, list[str]]) -> None:
+    """Detect circular dependencies using DFS.
+
+    Args:
+        dependencies: Dependency graph
+
+    Raises:
+        JinjaError: If circular dependency found
+    """
+    visited: set = set()
+    rec_stack: set = set()
+
+    def dfs(node: str, path: list[str]) -> None:
+        if node in rec_stack:
+            cycle = path[path.index(node) :] + [node]
+            raise JinjaError(
+                f"Circular query dependency detected: {' -> '.join(cycle)}"
+            )
+
+        if node in visited:
+            return
+
+        visited.add(node)
+        rec_stack.add(node)
+
+        for dep in dependencies.get(node, []):
+            if dep in dependencies:  # Only follow known queries
+                dfs(dep, path + [node])
+
+        rec_stack.remove(node)
+
+    for query_name in dependencies:
+        dfs(query_name, [])
+
+
+def _topological_sort(dependencies: dict[str, list[str]]) -> list[str]:
+    """Return query names in topological order (dependencies first).
+
+    Args:
+        dependencies: Dependency graph from detect_query_dependencies()
+
+    Returns:
+        List of query names with leaf nodes first
+    """
+    visited: set[str] = set()
+    order: list[str] = []
+
+    def visit(name: str) -> None:
+        if name in visited:
+            return
+        visited.add(name)
+        for dep in dependencies.get(name, []):
+            if dep in dependencies:
+                visit(dep)
+        order.append(name)
+
+    for name in dependencies:
+        visit(name)
+
+    return order
+
+
+def _get_query_sql(query: Any) -> str:
+    """Extract the SQL string a _QueryNamespace would return for a query."""
+    if hasattr(query, "sql"):
+        sql = query.sql
+        return sql if isinstance(sql, str) else ""
+    elif isinstance(query, dict):
+        sql = query.get("sql", "")
+        return sql if isinstance(sql, str) else ""
+    return str(query)
+
+
+def _substitute_query_refs(sql: str, resolved: dict[str, str]) -> str:
+    """Replace {{ queries.X }} tokens with already-resolved SQL.
+
+    Uses regex substitution so that only query references are expanded;
+    variable expressions ({{ var }}) and other Jinja constructs are
+    left untouched for a later resolve_jinja_template pass.
+    """
+    _QUERY_REF = re.compile(r"\{\{\s*queries\.(\w+)\s*\}\}")
+
+    def _replacer(m: re.Match) -> str:
+        name = m.group(1)
+        if name in resolved:
+            return resolved[name]
+        return m.group(0)  # leave unresolved if not in map
+
+    return _QUERY_REF.sub(_replacer, sql)
+
+
+def resolve_query_sql_with_dependencies(
+    query_name: str,
+    queries: dict[str, Any],
+    variables: dict[str, Any] | None = None,
+) -> str:
+    """Resolve a query's SQL, fully expanding nested {{ queries.* }} refs.
+
+    Two-phase approach:
+      Phase 1 — Expand {{ queries.* }} in topological order using regex so
+                each query sees fully expanded dependency SQL.  Variable
+                expressions are left untouched.
+      Phase 2 — Run the target query's expanded SQL through Jinja to resolve
+                variables, filters, and other template constructs.
+
+    Args:
+        query_name: The target query to resolve
+        queries: Full query registry (name -> query object or dict)
+        variables: Variable values for Jinja substitution
+
+    Returns:
+        Fully resolved SQL string for the target query
+
+    Raises:
+        JinjaError: If circular dependencies detected or resolution fails
+    """
+    # Build dependency graph (validates no cycles)
+    deps = detect_query_dependencies(queries)
+
+    # Topological sort — leaves (no deps) come first
+    order = _topological_sort(deps)
+
+    # Phase 1: expand {{ queries.* }} refs in dependency order
+    expanded: dict[str, str] = {}
+    for name in order:
+        if name not in queries:
+            continue
+        raw_sql = _get_query_sql(queries[name])
+        if raw_sql and ("{{ queries." in raw_sql or "{{queries." in raw_sql):
+            expanded[name] = _substitute_query_refs(raw_sql, expanded)
+        else:
+            expanded[name] = raw_sql
+
+    target_sql = expanded.get(query_name, _get_query_sql(queries.get(query_name, "")))
+
+    # Phase 2: resolve variables / filters via Jinja
+    if target_sql and ("{{" in target_sql or "{%" in target_sql):
+        target_sql = resolve_jinja_template(target_sql, variables=variables)
+
+    return target_sql
+
+
+def _filter_helper(
+    column: str,
+    value: Any,
+    operator: str = "=",
+    none: str = "allow",
+) -> str:
+    """Generate SQL filter clause (DEPRECATED - use parameterized version).
+
+    .. deprecated::
+        This helper uses string interpolation which is vulnerable to SQL injection.
+        Use the parameterized filter helper from dataface.core.compile.parameterized instead,
+        which is automatically used when executing queries via the SqlAdapter.
+
+    Security Warning:
+        This function directly interpolates values into SQL strings. If values
+        come from user input, this creates SQL injection vulnerabilities.
+        The parameterized version in parameterized.py should be used instead.
+
+    Helper function available in templates as {{ filter(...) }}.
+
+    Args:
+        column: Column name
+        value: Filter value
+        operator: SQL operator (=, !=, >, <, etc.)
+        none: Fallback when value is null/empty/undefined. 'allow' (default) returns
+            '1=1' (no constraint, show all rows). 'deny' returns '1=0' (zero rows).
+
+    Returns:
+        SQL clause like "column = 'value'", "1=1" (allow all) if value is None/undefined,
+        or "1=0" (deny all) if value is None/undefined and none='deny'.
+
+    Raises:
+        ValueError: If none is not 'allow' or 'deny'.
+    """
+    if none not in ("allow", "deny"):
+        raise ValueError(f"none= must be 'allow' or 'deny', got {none!r}")
+
+    # Handle undefined Jinja variables (from lenient mode)
+    if isinstance(value, Undefined):
+        return "1=0" if none == "deny" else "1=1"
+    if value is None or value == "":
+        return "1=0" if none == "deny" else "1=1"
+
+    if isinstance(value, str):
+        return f"{column} {operator} '{value}'"
+    elif isinstance(value, (int, float)):
+        return f"{column} {operator} {value}"
+    elif isinstance(value, list):
+        if not value:
+            return "1=0" if none == "deny" else "1=1"
+        quoted = [f"'{v}'" if isinstance(v, str) else str(v) for v in value]
+        return f"{column} IN ({', '.join(quoted)})"
+
+    return f"{column} {operator} '{value}'"
+
+
+def _filter_date_range_helper(
+    column: str,
+    date_range: Any,
+) -> str:
+    """Generate SQL date range filter (DEPRECATED - use parameterized version).
+
+    .. deprecated::
+        This helper uses string interpolation which is vulnerable to SQL injection.
+        Use the parameterized filter_date_range helper from dataface.core.compile.parameterized
+        instead, which is automatically used when executing queries via the SqlAdapter.
+
+    Security Warning:
+        This function directly interpolates values into SQL strings. If values
+        come from user input, this creates SQL injection vulnerabilities.
+        The parameterized version in parameterized.py should be used instead.
+
+    Helper function for {{ filter_date_range(column, date_range) }}.
+
+    Args:
+        column: Date column name
+        date_range: Tuple/list of [start, end] dates, or JSON string
+
+    Returns:
+        SQL BETWEEN clause or "1=1" (always true) if invalid/undefined
+    """
+    if not date_range:
+        return "1=1"
+
+    # Handle JSON string (from URL parameters)
+    if isinstance(date_range, str):
+        import json
+
+        try:
+            date_range = json.loads(date_range)
+        except (json.JSONDecodeError, ValueError) as e:
+            raise ValueError(f"Invalid date_range JSON: {date_range!r}") from e
+
+    if not isinstance(date_range, (list, tuple)):
+        raise ValueError(
+            f"date_range must be a list/tuple [start, end], got {type(date_range).__name__}"
+        )
+
+    if len(date_range) != 2:
+        raise ValueError(
+            f"date_range must have exactly 2 elements [start, end], got {len(date_range)}"
+        )
+
+    start, end = date_range
+    if not start or not end:
+        return "1=1"  # Empty dates = no filter (intentional)
+
+    return f"{column} BETWEEN '{start}' AND '{end}'"

dataface/core/compile/labels_env.py

@@ -0,0 +1,52 @@
+"""Jinja env for chart-label templates and ``where:`` expressions.
+
+Lives in compile/ so ChartLabels' pydantic validators can parse without
+reaching into the render layer (the dependency direction stays
+compile → shared, render → shared, never render → compile or compile →
+render).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from jinja2 import Environment, StrictUndefined
+
+from dataface.core.render.format_utils import format_d3
+
+
+def label_jinja_env() -> Environment:
+    """Return the Jinja env used to compile and render label templates.
+
+    Strict-undefined: a typo like ``{{ pct }}`` (instead of ``percent``)
+    raises immediately rather than silently rendering empty.
+    """
+    env = Environment(undefined=StrictUndefined, autoescape=False)
+    env.filters["format"] = _format_filter
+    return env
+
+
+def _format_filter(value: Any, spec: str) -> str:
+    """Jinja filter: ``{{ value | format('$,.0f') }}`` → d3-formatted."""
+    return format_d3(value, spec)
+
+
+def strip_jinja_braces(expr: str) -> str:
+    """Accept ``{{ x }}`` or bare ``x`` for ``where:`` — return the bare expression.
+
+    Authors naturally write ``where: "{{ value > 5 }}"`` (matching the
+    ``template:`` syntax), but Jinja's ``compile_expression`` wants the
+    bare expression (``value > 5``). Tolerate the single-outer-pair case;
+    reject mixed forms (``"{{ a }} and {{ b }}"`` etc.) loudly.
+    """
+    s = expr.strip()
+    if not (s.startswith("{{") and s.endswith("}}")):
+        return s
+    inner = s[2:-2]
+    if "{{" in inner or "}}" in inner:
+        raise ValueError(
+            "where: expression contains nested {{ ... }}; use a bare Python "
+            "expression instead (e.g. ``value > 5`` not "
+            "``{{ value > 5 }} and {{ other }}``)."
+        )
+    return inner.strip()