ripple-sql 0.1.0__py3-none-any.whl

ripple/engine/jinja.py

@@ -0,0 +1,282 @@
+"""Convert Jinja-templated dbt SQL into parseable SQL.
+
+This module handles the transformation of dbt's Jinja-templated SQL into
+standard SQL that can be parsed by SQLGlot for column lineage extraction.
+
+Key transformations:
+- {{ ref('model') }} → model
+- {{ source('schema', 'table') }} → schema__table
+- {{ config(...) }} → removed
+- {% if/for/... %} → removed
+- Custom macros → NULL placeholder
+
+Security:
+- ReDoS protection via size limits on regex processing
+- Bounded regex patterns to prevent catastrophic backtracking
+"""
+
+import logging
+import re
+from typing import Final
+
+logger = logging.getLogger(__name__)
+
+# Maximum SQL size for regex processing (500KB) - prevents ReDoS on huge files
+MAX_SQL_SIZE_FOR_REGEX: Final[int] = 500 * 1024
+
+
+def convert_jinja_to_sql(raw_sql: str) -> str:
+    """Convert Jinja-templated SQL to parseable SQL.
+
+    This is the full conversion that handles:
+
+    Core dbt functions:
+    - {{ ref('model') }} → model
+    - {{ source('src', 'table') }} → src__table
+    - {{ config(...) }} → removed
+    - {{ var('x') }} → 'x' (variable reference)
+    - {{ this }} → this_model
+
+    dbt-utils macros:
+    - {{ group_by(n) }} → GROUP BY n
+    - {{ dbt_utils.star(...) }} → *
+    - {{ dbt_utils.surrogate_key(...) }} → md5(...)
+
+    SQL normalization:
+    - Escaped quotes: ''value'' → 'value'
+    - Dollar-quoted strings: $$text$$ → 'text'
+    - Prepared statement placeholders: $1, ?, :name → 'param'
+
+    Args:
+        raw_sql: SQL with Jinja templates
+
+    Returns:
+        Clean SQL parseable by SQLGlot
+
+    Security:
+        Returns raw SQL unchanged if size exceeds MAX_SQL_SIZE_FOR_REGEX
+        to prevent ReDoS attacks.
+    """
+    if len(raw_sql) > MAX_SQL_SIZE_FOR_REGEX:
+        logger.warning(
+            f"SQL too large for Jinja cleaning ({len(raw_sql)} bytes > {MAX_SQL_SIZE_FOR_REGEX}). "
+            "Returning raw SQL to avoid ReDoS risk."
+        )
+        return raw_sql
+
+    cleaned = raw_sql
+
+    # ==========================================================================
+    # SQL NORMALIZATION FOR CROSS-DIALECT PARSING
+    # These patterns handle syntax variations across Snowflake, BigQuery,
+    # Databricks, Trino, Redshift, Postgres that sqlglot may struggle with.
+    # ==========================================================================
+
+    # 1. Escaped single quotes: ''value'' → 'value'
+    #    Common in Snowflake/Trino compiled SQL for Jinja variables
+    cleaned = re.sub(r"''([^']+)''", r"'\1'", cleaned)
+
+    # 2. Dollar-quoted strings (PostgreSQL/Redshift): $$text$$ → 'text'
+    cleaned = re.sub(r"\$\$([^$]*)\$\$", r"'\1'", cleaned)
+    cleaned = re.sub(r"\$(\w+)\$([^$]*)\$\1\$", r"'\2'", cleaned)  # Tagged: $tag$text$tag$
+
+    # 3. Unicode escape sequences: U&'text' → 'text'
+    cleaned = re.sub(r"U&'([^']*)'", r"'\1'", cleaned, flags=re.IGNORECASE)
+
+    # 4. National character strings: N'text' → 'text'
+    cleaned = re.sub(r"(?<!\w)N'([^']*)'", r"'\1'", cleaned)
+
+    # 5. Raw string literals (BigQuery): r'pattern' → 'pattern'
+    cleaned = re.sub(r"(?<!\w)[rR]'([^']*)'", r"'\1'", cleaned)
+    cleaned = re.sub(r'(?<!\w)[rR]"([^"]*)"', r"'\1'", cleaned)
+
+    # 6. Byte literals (BigQuery): b'bytes' → 'bytes'
+    cleaned = re.sub(r"(?<!\w)[bB]'([^']*)'", r"'\1'", cleaned)
+    cleaned = re.sub(r'(?<!\w)[bB]"([^"]*)"', r"'\1'", cleaned)
+
+    # 7. Prepared statement placeholders → literal placeholder
+    cleaned = re.sub(r"\$(\d+)", r"'param\1'", cleaned)  # $1, $2 (Postgres)
+    cleaned = re.sub(r"(?<!\?)\?(?!\?)", "'param'", cleaned)  # ? (JDBC)
+    cleaned = re.sub(
+        r"(?<!:):(\w+)(?=\s|,|\)|$|AND|OR|=|>|<|\+|-|\*|/)",
+        r"'\1'",
+        cleaned,
+        flags=re.IGNORECASE,
+    )  # :name (Oracle)
+
+    # 8. Interval literals with quoted units
+    cleaned = re.sub(r"INTERVAL\s+'(\d+)'\s+(\w+)", r"INTERVAL \1 \2", cleaned, flags=re.IGNORECASE)
+
+    # ==========================================================================
+    # CORE DBT FUNCTIONS
+    # ==========================================================================
+
+    # ref('model') → model
+    cleaned = re.sub(r"\{\{\s*ref\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}", r"\1", cleaned)
+
+    # source('schema', 'table') → schema__table
+    cleaned = re.sub(
+        r"\{\{\s*source\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}",
+        r"\1__\2",
+        cleaned,
+    )
+
+    # config() blocks (can be multiline, nested)
+    cleaned = re.sub(r"\{\{\s*config\s*\(.*?\)\s*\}\}", "", cleaned, flags=re.DOTALL)
+    cleaned = re.sub(r"\{\{\s*config\([^}]*\}\}\s*\}\}", "", cleaned)
+
+    # ==========================================================================
+    # DBT-UTILS MACROS
+    # ==========================================================================
+
+    cleaned = re.sub(r"\{\{\s*group_by\s*\(\s*(\d+)\s*\)\s*\}\}", r"GROUP BY \1", cleaned)
+    cleaned = re.sub(r"\{\{\s*order_by\s*\(\s*(\d+)\s*\)\s*\}\}", r"ORDER BY \1", cleaned)
+    cleaned = re.sub(r"\{\{\s*limit\s*\(\s*(\d+)\s*\)\s*\}\}", r"LIMIT \1", cleaned)
+
+    # dbt_utils.star() → *
+    cleaned = re.sub(r"\{\{\s*dbt_utils\.star\s*\([^}]*\)\s*\}\}", "*", cleaned)
+
+    # dbt_utils.surrogate_key() → md5('key')
+    cleaned = re.sub(r"\{\{\s*dbt_utils\.surrogate_key\s*\([^)]*\)\s*\}\}", "md5('key')", cleaned)
+    cleaned = re.sub(
+        r"\{\{\s*dbt_utils\.generate_surrogate_key\s*\([^)]*\)\s*\}\}",
+        "md5('key')",
+        cleaned,
+    )
+
+    # dbt_utils.pivot/unpivot → comment placeholder
+    cleaned = re.sub(r"\{\{\s*dbt_utils\.pivot\s*\([^)]*\)\s*\}\}", "/* pivot */", cleaned)
+    cleaned = re.sub(r"\{\{\s*dbt_utils\.unpivot\s*\([^)]*\)\s*\}\}", "/* unpivot */", cleaned)
+
+    # dbt_utils.generate_series() → subquery
+    cleaned = re.sub(
+        r"\{\{\s*dbt_utils\.generate_series\s*\([^)]*\)\s*\}\}",
+        "(SELECT 1 AS n)",
+        cleaned,
+    )
+
+    # dbt_utils.get_column_values() → tuple
+    cleaned = re.sub(
+        r"\{\{\s*dbt_utils\.get_column_values\s*\([^)]*\)\s*\}\}",
+        "('value1', 'value2')",
+        cleaned,
+    )
+
+    # dbt_utils.safe_cast() → TRY_CAST
+    cleaned = re.sub(
+        r"\{\{\s*dbt_utils\.safe_cast\s*\(\s*['\"]?([^'\"]+)['\"]?\s*,\s*['\"]?([^'\"]+)['\"]?\s*\)\s*\}\}",
+        r"TRY_CAST(\1 AS \2)",
+        cleaned,
+    )
+
+    # dbt_utils.date_trunc() → DATE_TRUNC
+    cleaned = re.sub(
+        r"\{\{\s*dbt_utils\.date_trunc\s*\(\s*['\"](\w+)['\"]\s*,\s*([^)]+)\s*\)\s*\}\}",
+        r"DATE_TRUNC('\1', \2)",
+        cleaned,
+    )
+
+    # Generic dbt_utils.* → NULL
+    cleaned = re.sub(r"\{\{\s*dbt_utils\.\w+\s*\([^)]*\)\s*\}\}", "NULL", cleaned)
+
+    # ==========================================================================
+    # OTHER DBT MACROS
+    # ==========================================================================
+
+    # dbt_date macros
+    cleaned = re.sub(
+        r"\{\{\s*dbt_date\.date_spine\s*\([^)]*\)\s*\}\}",
+        "(SELECT CURRENT_DATE AS date_day)",
+        cleaned,
+    )
+    cleaned = re.sub(r"\{\{\s*dbt_date\.\w+\s*\([^)]*\)\s*\}\}", "CURRENT_DATE", cleaned)
+
+    # fivetran_utils macros
+    cleaned = re.sub(
+        r"\{\{\s*fivetran_utils\.\w+\s*\([^)]*\)\s*\}\}", "/* fivetran_macro */", cleaned
+    )
+
+    # dbt_expectations macros
+    cleaned = re.sub(r"\{\{\s*dbt_expectations\.\w+\s*\([^)]*\)\s*\}\}", "TRUE", cleaned)
+
+    # ==========================================================================
+    # VARIABLE REFERENCES
+    # ==========================================================================
+
+    cleaned = re.sub(r"\{\{\s*var\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}", r"'\1'", cleaned)
+    cleaned = re.sub(r"\{\{\s*env_var\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}", r"'\1'", cleaned)
+    cleaned = re.sub(r"\{\{\s*this\s*\}\}", "this_model", cleaned)
+    cleaned = re.sub(r"\{\{\s*target\.\w+\s*\}\}", "'target_value'", cleaned)
+    cleaned = re.sub(r"\{\{\s*run_started_at\s*\}\}", "CURRENT_TIMESTAMP", cleaned)
+    cleaned = re.sub(r"\{\{\s*invocation_id\s*\}\}", "'invocation_id'", cleaned)
+
+    # ==========================================================================
+    # CONTROL FLOW AND CLEANUP
+    # ==========================================================================
+
+    # {% ... %} control blocks carry no lineage
+    cleaned = re.sub(r"\{%[\s\S]*?%\}", "", cleaned)
+
+    # Any remaining {{ ... }} → NULL
+    cleaned = re.sub(r"\{\{[^}]*\}\}", "NULL", cleaned)
+
+    # Clean up leftover Jinja fragments
+    cleaned = re.sub(r'["\')]*\s*\}\}', "", cleaned)
+    cleaned = re.sub(r'\{\{\s*["\(\']*', "", cleaned)
+    cleaned = re.sub(r'^\s*["\']+["\')]*\s*$', "", cleaned, flags=re.MULTILINE)
+
+    # a stripped config block leaves its indentation behind
+    cleaned = cleaned.lstrip()
+
+    return cleaned
+
+
+def convert_jinja_to_sql_minimal(raw_sql: str) -> str:
+    """Minimal Jinja conversion - only guaranteed-safe transformations.
+
+    Use this as a fallback when full conversion breaks valid SQL.
+    Only handles patterns that cannot produce invalid SQL.
+
+    Safe transformations:
+    - {{ ref('model') }} → model
+    - {{ source('src', 'table') }} → src__table
+    - {{ config(...) }} → removed
+    - {% ... %} → removed
+    - Other {{ ... }} → NULL
+
+    Does NOT apply:
+    - SQL string literal normalization
+    - Prepared statement placeholder handling
+    - Dialect-specific pattern fixes
+
+    Args:
+        raw_sql: SQL with Jinja templates
+
+    Returns:
+        Minimally cleaned SQL
+    """
+    if len(raw_sql) > MAX_SQL_SIZE_FOR_REGEX:
+        return raw_sql
+
+    cleaned = raw_sql
+
+    # Only the safest Jinja transformations
+    cleaned = re.sub(r"\{\{\s*ref\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}", r"\1", cleaned)
+    cleaned = re.sub(
+        r"\{\{\s*source\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}",
+        r"\1__\2",
+        cleaned,
+    )
+    cleaned = re.sub(r"\{\{\s*config\s*\(.*?\)\s*\}\}", "", cleaned, flags=re.DOTALL)
+
+    # Control flow blocks
+    cleaned = re.sub(r"\{%[\s\S]*?%\}", "", cleaned)
+
+    # Remaining Jinja macros → NULL
+    cleaned = re.sub(r"\{\{[^}]*\}\}", "NULL", cleaned)
+
+    # Minimal cleanup
+    cleaned = re.sub(r'["\')]*\s*\}\}', "", cleaned)
+    cleaned = re.sub(r'\{\{\s*["\(\']*', "", cleaned)
+
+    return cleaned

ripple/engine/json_sources.py

@@ -0,0 +1,241 @@
+"""JSON-access source extraction for select items.
+
+Handles column references reached through JSON path expressions
+(JSON_EXTRACT, ->, data:field) and bracket syntax (data['key']), preserving
+the JSON path on the emitted source entry.
+"""
+
+from typing import Any
+
+from sqlglot import exp
+
+from ripple.engine.column_ref import (
+    resolve_column_ref,
+    under_subquery_where,
+)
+
+# built once: per-column construction showed up in profiles
+# Covers all dialects:
+# - Snowflake: data:field::string (parses to JSONExtract)
+# - BigQuery: JSON_EXTRACT, JSON_EXTRACT_SCALAR, JSON_VALUE
+# - Trino: json_extract, json_extract_scalar
+# - Redshift: json_extract_path_text
+# - Postgres: ->, ->> (JSONExtract), #>, #>> (JSONBExtract)
+# - Databricks: get_json_object
+JSON_EXPR_TYPES: tuple[type, ...] = (
+    exp.JSONExtract,  # Most dialects: JSON_EXTRACT, ->, data:field
+    exp.JSONExtractScalar,  # JSON_EXTRACT_SCALAR, ->>, JSON_VALUE
+)
+# the JSONB variants exist only in newer sqlglot
+if hasattr(exp, "JSONBExtract"):
+    JSON_EXPR_TYPES = JSON_EXPR_TYPES + (exp.JSONBExtract,)
+if hasattr(exp, "JSONBExtractScalar"):
+    JSON_EXPR_TYPES = JSON_EXPR_TYPES + (exp.JSONBExtractScalar,)
+
+
+def _extract_json_path(json_path_expr: "exp.Expression | None") -> str:
+    """Extract JSON path string from various JSON path expression types.
+
+    Handles:
+    - JSONPath expressions: $.field.subfield -> "field.subfield"
+    - String literals: '$.path' -> "path"
+    - Trino array arguments: 'key1', 'key2' -> "key1.key2"
+    - Redshift json_extract_path_text: col, 'key1', 'key2' -> "key1.key2"
+
+    Returns:
+        Extracted path string (without leading $.) or empty string if not extractable.
+    """
+    if json_path_expr is None:
+        return ""
+
+    if hasattr(json_path_expr, "expressions"):
+        path_parts = []
+        for path_part in json_path_expr.expressions:
+            # Skip JSONPathRoot ($)
+            if type(path_part).__name__ == "JSONPathRoot":
+                continue
+            if hasattr(path_part, "this"):
+                part_name = str(path_part.this)
+                path_parts.append(part_name)
+            elif hasattr(path_part, "name"):
+                path_parts.append(path_part.name)
+        if path_parts:
+            return ".".join(path_parts)
+
+    if hasattr(json_path_expr, "this"):
+        path_str = str(json_path_expr.this)
+        # Strip quotes and leading $.
+        return path_str.strip("'\"").lstrip("$.")
+
+    # Fallback: try string representation
+    path_str = str(json_path_expr)
+    return path_str.strip("'\"").lstrip("$.")
+
+
+def collect_json_sources(
+    source_expr: "exp.Expression",
+    known_relations: set[str],
+    alias_map: dict[str, str],
+    array_expansion_sources: dict[str, dict[str, str]],
+    default_table: str,
+    qualified_via_schema: bool,
+    sources: list[dict[str, Any]],
+    seen: set,
+) -> set[int]:
+    """Append JSON-path and bracket-access sources for one select item.
+
+    Mutates sources/seen in place and returns the ids of Column nodes
+    already handled, so the plain column sweep can skip them.
+    """
+    json_columns_handled = set()
+    for json_expr in source_expr.find_all(*JSON_EXPR_TYPES):
+        base_col = json_expr.this
+        json_path_expr = json_expr.expression
+
+        # JSON_EXTRACT(JSON_EXTRACT(...)): the inner call owns the column
+        while isinstance(base_col, JSON_EXPR_TYPES):
+            base_col = base_col.this
+
+        if isinstance(base_col, exp.Column):
+            if under_subquery_where(base_col, source_expr):
+                json_columns_handled.add(id(base_col))
+                continue
+            alias, column, _struct_path, _certain = resolve_column_ref(base_col, known_relations)
+
+            json_path = _extract_json_path(json_path_expr)
+
+            # Mark this column as handled via JSON path
+            json_columns_handled.add(id(base_col))
+
+            source_entry: dict[str, Any] = {
+                "column": column,
+                "inferred": False,
+                "json_path": json_path if json_path else None,
+                "is_json_access": True,
+            }
+
+            if not alias and column in array_expansion_sources:
+                # the expansion alias used bare: elem->>'k' reads the value
+                # jsonb_array_elements produced, so the true source is the
+                # expanded array column (FAC findings_text, holdout round 4)
+                expansion_info = array_expansion_sources[column]
+                table = expansion_info.get("source_table", "")
+                confidence = 0.5
+                trust_level = "moderate"
+                source_entry["from_array_expansion"] = True
+                source_entry["expansion_type"] = expansion_info.get("expansion_type")
+                src_col = expansion_info.get("source_column", column)
+                source_entry["source_column_in_array"] = src_col
+                if src_col:
+                    source_entry["expanded_as"] = column
+                    source_entry["column"] = src_col
+            elif alias:
+                if alias in array_expansion_sources:
+                    expansion_info = array_expansion_sources[alias]
+                    table = expansion_info.get("source_table", alias)
+                    confidence = 0.5
+                    trust_level = "moderate"
+                    source_entry["from_array_expansion"] = True
+                    source_entry["expansion_type"] = expansion_info.get("expansion_type")
+                    source_entry["source_column_in_array"] = expansion_info.get(
+                        "source_column", column
+                    )
+                else:
+                    table = alias_map.get(alias, alias)
+                    confidence = 1.0
+                    trust_level = "verified" if qualified_via_schema else "high_confidence"
+            elif default_table:
+                table = default_table
+                confidence = 0.8
+                trust_level = "high_confidence"
+                source_entry["inferred"] = True
+            else:
+                table = ""
+                confidence = 0.5
+                trust_level = "moderate"
+
+            source_entry["table"] = table
+            source_entry["confidence"] = confidence
+            source_entry["trust_level"] = trust_level
+
+            key = (table, column, json_path)
+            if key not in seen:
+                seen.add(key)
+                sources.append(source_entry)
+
+    # This pattern is used in Trino, Redshift (SUPER), and some other dialects
+    for bracket_expr in source_expr.find_all(exp.Bracket):
+        base = bracket_expr.this
+        # Skip if already handled via JSON expression
+        if any(id(base) in json_columns_handled for base in [bracket_expr.this]):
+            continue
+
+        # Walk up to find the base column
+        json_path_parts = []
+        while isinstance(base, exp.Bracket):
+            if base.expressions:
+                key = base.expressions[0]
+                if hasattr(key, "this"):
+                    json_path_parts.insert(0, str(key.this))
+                elif hasattr(key, "name"):
+                    json_path_parts.insert(0, key.name)
+                else:
+                    json_path_parts.insert(0, str(key))
+            base = base.this
+
+        if isinstance(base, exp.Column):
+            json_columns_handled.add(id(base))
+            if under_subquery_where(base, source_expr):
+                continue
+            alias, column, _struct_path, _certain = resolve_column_ref(base, known_relations)
+            json_path = ".".join(json_path_parts) if json_path_parts else None
+
+            source_entry: dict[str, Any] = {
+                "column": column,
+                "inferred": False,
+                "json_path": json_path,
+                "is_json_access": True,
+            }
+
+            if not alias and column in array_expansion_sources:
+                # bare expansion alias (see the JSONExtract branch above)
+                expansion_info = array_expansion_sources[column]
+                table = expansion_info.get("source_table", "")
+                confidence = 0.5
+                trust_level = "moderate"
+                source_entry["from_array_expansion"] = True
+                src_col = expansion_info.get("source_column", column)
+                if src_col:
+                    source_entry["expanded_as"] = column
+                    source_entry["column"] = src_col
+            elif alias:
+                if alias in array_expansion_sources:
+                    expansion_info = array_expansion_sources[alias]
+                    table = expansion_info.get("source_table", alias)
+                    confidence = 0.5
+                    trust_level = "moderate"
+                    source_entry["from_array_expansion"] = True
+                else:
+                    table = alias_map.get(alias, alias)
+                    confidence = 1.0
+                    trust_level = "verified" if qualified_via_schema else "high_confidence"
+            elif default_table:
+                table = default_table
+                confidence = 0.8
+                trust_level = "high_confidence"
+                source_entry["inferred"] = True
+            else:
+                table = ""
+                confidence = 0.5
+                trust_level = "moderate"
+
+            source_entry["table"] = table
+            source_entry["confidence"] = confidence
+            source_entry["trust_level"] = trust_level
+
+            key = (table, column, json_path)
+            if key not in seen:
+                seen.add(key)
+                sources.append(source_entry)
+
+    return json_columns_handled

ripple/engine/macro_source.py

@@ -0,0 +1,127 @@
+"""Primary-source detection from dbt macro patterns.
+
+Regex-only: finds the source() or ref() call that names a model's primary
+data source, before any SQL parsing happens.
+"""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def extract_primary_source_from_macro(raw_sql: str) -> dict | None:
+    """Extract primary source from staging macro patterns.
+
+    Handles:
+    1. ANY custom macro wrapping source() or ref() - not just known names
+    2. Direct source()/ref() calls as first statement
+    3. Various whitespace and formatting patterns
+    4. Multi-line macro calls
+
+    Strategy:
+    - First, strip config blocks to find the "main body"
+    - Look for source() or ref() as the FIRST reference in the main body
+    - This is almost always the primary data source (not WHERE clause refs)
+
+    Returns:
+        {
+            "type": "source"|"ref",
+            "source": "source_name",  # for source type
+            "table": "table_name",    # for source type
+            "ref": "model_name",      # for ref type
+            "detection_method": "macro_wrapper"|"direct_call"|"first_ref"
+        }
+        or None if no pattern found
+    """
+    import re
+
+    if not raw_sql or not raw_sql.strip():
+        return None
+
+    # Step 1: Strip config block(s) to get to the main SQL body
+    config_pattern = re.compile(
+        r"\{\{\s*config\s*\([^)]*(?:\([^)]*\)[^)]*)*\)\s*\}\}", re.IGNORECASE | re.DOTALL
+    )
+    main_body = config_pattern.sub("", raw_sql).strip()
+
+    # Step 2: Try to find ANY macro that wraps source() or ref()
+    # Generic macro wrapping source()
+    generic_macro_source = re.compile(
+        r"\{\{\s*(\w+)\s*\(\s*source\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)",
+        re.IGNORECASE | re.DOTALL,
+    )
+
+    match = generic_macro_source.search(main_body)
+    if match:
+        macro_name = match.group(1).lower()
+        skip_macros = {"config", "if", "elif", "else", "endif", "for", "endfor", "set", "do"}
+        if macro_name not in skip_macros:
+            return {
+                "type": "source",
+                "source": match.group(2),
+                "table": match.group(3),
+                "detection_method": "macro_wrapper",
+                "macro_name": macro_name,
+            }
+
+    # Generic macro wrapping ref()
+    generic_macro_ref = re.compile(
+        r"\{\{\s*(\w+)\s*\(\s*ref\s*\(\s*['\"]([^'\"]+)['\"]\s*\)", re.IGNORECASE | re.DOTALL
+    )
+
+    match = generic_macro_ref.search(main_body)
+    if match:
+        macro_name = match.group(1).lower()
+        skip_macros = {"config", "if", "elif", "else", "endif", "for", "endfor", "set", "do"}
+        if macro_name not in skip_macros:
+            return {
+                "type": "ref",
+                "ref": match.group(2),
+                "detection_method": "macro_wrapper",
+                "macro_name": macro_name,
+            }
+
+    # Step 3: Look for direct source() or ref() as first Jinja call
+    first_source = re.search(
+        r"\{\{\s*source\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}",
+        main_body,
+        re.IGNORECASE,
+    )
+    first_ref = re.search(
+        r"\{\{\s*ref\s*\(\s*['\"]([^'\"]+)['\"]\s*\)\s*\}\}", main_body, re.IGNORECASE
+    )
+
+    where_pos = main_body.upper().find("WHERE")
+    if where_pos == -1:
+        where_pos = len(main_body)
+
+    candidates = []
+    if first_source and first_source.start() < where_pos:
+        candidates.append(
+            (
+                first_source.start(),
+                {
+                    "type": "source",
+                    "source": first_source.group(1),
+                    "table": first_source.group(2),
+                    "detection_method": "direct_call",
+                },
+            )
+        )
+    if first_ref and first_ref.start() < where_pos:
+        candidates.append(
+            (
+                first_ref.start(),
+                {
+                    "type": "ref",
+                    "ref": first_ref.group(1),
+                    "detection_method": "direct_call",
+                },
+            )
+        )
+
+    if candidates:
+        candidates.sort(key=lambda x: x[0])
+        return candidates[0][1]
+
+    return None