ripple-sql 0.1.0__py3-none-any.whl

ripple/engine/cte_tracing.py

@@ -0,0 +1,1309 @@
+"""CTE column lineage extraction and tracing.
+
+This module handles Common Table Expressions (CTEs) for column lineage:
+- Extract column mappings from CTEs
+- Detect CTE name collisions with real tables
+- Recursively trace columns through nested CTEs
+
+CTE Collision Detection:
+If a CTE name shadows a real table name (e.g., WITH orders AS ...),
+lineage would incorrectly trace to the CTE instead of the real table.
+This module detects and warns about such collisions.
+
+Key Functions:
+- extract_cte_column_lineage: Per-CTE column sources
+- extract_cte_column_mappings: CTE name -> column mappings
+- trace_through_ctes: Recursive column tracing
+"""
+
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from ripple.engine.column_ref import (
+    alias_is_foldable,
+    collect_unnest_expansions,
+    column_parts,
+    expansion_field_reads,
+    in_selector_argument,
+    in_window_ordering,
+    left_join_anchor,
+    nested_scope_sole_table,
+    owning_select,
+    qualified_table_name,
+    qualifier_is_quoted,
+    register_subquery_from_aliases,
+    resolve_column_ref,
+    scoped_unnest_entry,
+    table_function_kind,
+    under_subquery_where,
+    unwrap_select,
+)
+from ripple.engine.preprocess import restore_jinja_dots
+from ripple.engine.safe_gen import safe_sql
+from ripple.engine.scope import register_table
+from ripple.engine.tsql_catalog import system_catalog_owner
+from ripple.engine.types import (
+    LATERAL_ALIAS_DIALECTS,
+    META_CTE_COLLISIONS,
+    META_CTE_RECURSIVE,
+    META_CTE_RELATIONS,
+    META_CTE_SHADOWED,
+    META_PREFIX,
+)
+
+if TYPE_CHECKING:
+    from sqlglot import exp
+
+logger = logging.getLogger(__name__)
+
+
+def extract_cte_column_lineage(
+    raw_sql: str,
+    dialect: str = "snowflake",
+    clean_jinja_func: Callable[[str], str] | None = None,
+    real_tables: set[str] | None = None,
+    self_names: set[str] | None = None,
+) -> dict[str, dict[str, list[dict]] | list[dict]]:
+    """Extract column lineage for each CTE in the SQL.
+
+    For SQL like:
+        WITH enriched AS (
+            SELECT c.name AS customer_name
+            FROM orders o
+            LEFT JOIN {{ ref('customers') }} AS c ON o.customer_id = c.id
+        )
+        SELECT * FROM enriched
+
+    Returns:
+        {
+            "enriched": {
+                "customer_name": [{"table": "c", "column": "name"}]
+            },
+            META_CTE_COLLISIONS: [...]  # Only if collisions detected
+        }
+
+    CTE Collision Detection:
+        If `real_tables` is provided, this function checks whether any CTE name
+        shadows a real table name. This is critical for lineage correctness:
+        a CTE named "orders" would shadow the real "orders" table, causing
+        lineage to trace to the wrong source.
+
+    This enables tracing THROUGH CTEs to find the actual column sources.
+
+    Args:
+        raw_sql: Raw SQL with Jinja templates
+        dialect: SQL dialect (snowflake, bigquery, databricks, etc.)
+        clean_jinja_func: Function to clean Jinja from SQL (injected dependency)
+        real_tables: Set of real table names (for collision detection)
+    """
+    import sqlglot
+    from sqlglot import exp
+
+    # Use injected cleaner or identity function
+    if clean_jinja_func is None:
+
+        def clean_jinja_func(sql):
+            return sql
+
+    cte_lineage: dict[str, dict[str, list[dict]]] = {}
+    cte_collisions: list[dict] = []
+    real_tables_lower = {t.lower() for t in (real_tables or set())}
+    # a CTE named after the model being analyzed shadows only itself: the CTE
+    # always wins within its statement and a genuine self-read would resolve
+    # elsewhere, so it is not a collision (jaffle-shop's house style flagged
+    # 17 correct pass-through edges this way)
+    real_tables_lower -= {t.lower() for t in (self_names or set())}
+
+    try:
+        from ripple.engine.preprocess import prepare_sql_for_parse
+
+        cleaned_sql, _ = prepare_sql_for_parse(raw_sql, dialect, clean_jinja_func)
+        parsed = sqlglot.parse_one(cleaned_sql, dialect=dialect)
+        if (dialect or "").lower() == "tsql":
+            from ripple.engine.tsql_xml import rewrite_xml_method_calls
+
+            parsed = rewrite_xml_method_calls(parsed)
+
+        def extract_select_columns(select_node: exp.Select) -> dict[str, list[dict]]:
+            """Extract column lineage from a SELECT node."""
+            alias_map: dict[str, str] = {}
+            all_tables: list[str] = []
+            subquery_columns_by_alias: dict[str, dict[str, list[dict]]] = {}
+
+            from_clause = select_node.args.get("from_") or select_node.args.get("from")
+            if from_clause:
+                for subquery in from_clause.find_all(exp.Subquery):
+                    sub_inner = unwrap_select(subquery.this)
+                    if subquery.alias and isinstance(sub_inner, exp.Select):
+                        subquery_columns_by_alias[subquery.alias] = extract_select_columns(
+                            sub_inner
+                        )
+                for table in from_clause.find_all(exp.Table):
+                    register_table(table, alias_map, all_tables)
+
+            for join in select_node.find_all(exp.Join):
+                join_table = join.this
+                if isinstance(join_table, exp.Subquery) and join_table.alias:
+                    join_inner = unwrap_select(join_table.this)
+                    if isinstance(join_inner, exp.Select):
+                        subquery_columns_by_alias[join_table.alias] = extract_select_columns(
+                            join_inner
+                        )
+                if isinstance(join_table, exp.Table):
+                    register_table(join_table, alias_map, all_tables)
+
+            default_table = all_tables[0] if len(all_tables) == 1 else ""
+
+            columns: dict[str, list[dict]] = {}
+            for select_expr in select_node.selects:
+                if isinstance(select_expr, exp.Column) and isinstance(select_expr.this, exp.Star):
+                    # vv.* over an inline subquery: absorb the subquery's own
+                    # column map, star chain included, instead of letting the
+                    # alias surface as a phantom relation (pensjon's dedup
+                    # wrapper, holdout round 5: the round's only
+                    # wrong-confident predictions)
+                    qualifier = select_expr.table or ""
+                    sub_map = subquery_columns_by_alias.get(qualifier)
+                    if sub_map is not None:
+                        for name, srcs in sub_map.items():
+                            columns.setdefault(name, list(srcs))
+                        continue
+                    rel = alias_map.get(qualifier, qualifier)
+                    if rel:
+                        columns.setdefault("*", []).append(
+                            {"table": rel, "column": "*", "confidence": 0.5}
+                        )
+                    continue
+                if isinstance(select_expr, exp.Alias):
+                    col_name = select_expr.alias
+                    source_expr = select_expr.this
+                elif isinstance(select_expr, exp.Column):
+                    col_name = select_expr.name
+                    source_expr = select_expr
+                elif isinstance(select_expr, exp.Star):
+                    col_name = "*"
+                    source_expr = select_expr
+                else:
+                    continue
+
+                if not col_name:
+                    continue
+
+                sources: list[dict] = []
+                seen: set[tuple[str, str]] = set()
+                _known = (
+                    {str(k).lower() for k in alias_map}
+                    | {str(v).lower() for v in alias_map.values()}
+                    | {str(k).lower() for k in subquery_columns_by_alias}
+                )
+                for col in source_expr.find_all(exp.Column):
+                    if in_window_ordering(col, source_expr):
+                        continue
+                    if in_selector_argument(col, source_expr):
+                        continue
+                    alias, column, _field_path, _certain = resolve_column_ref(col, _known)
+                    if "{" in column or (not _certain and any("{" in p for p in column_parts(col))):
+                        # a {placeholder} or jinja chain in column/qualifier
+                        # position has no citable identity (cycle-12, F11)
+                        continue
+                    if not _certain:
+                        # unknown leading identifier reads as a struct root;
+                        # anchor to the sole relation instead of fabricating
+                        alias = "" if default_table else alias
+
+                    if alias and alias in subquery_columns_by_alias:
+                        sub_sources = subquery_columns_by_alias[alias].get(column, [])
+                        for src in sub_sources:
+                            key = (src.get("table"), src.get("column"))
+                            if key not in seen and (src.get("table") or src.get("column")):
+                                seen.add(key)
+                                sources.append(src)
+                        if sub_sources:
+                            continue
+
+                    if not alias and not default_table and len(subquery_columns_by_alias) == 1:
+                        sub_alias = next(iter(subquery_columns_by_alias))
+                        sub_sources = subquery_columns_by_alias[sub_alias].get(column, [])
+                        for src in sub_sources:
+                            key = (src.get("table"), src.get("column"))
+                            if key not in seen and (src.get("table") or src.get("column")):
+                                seen.add(key)
+                                sources.append(src)
+                        if sub_sources:
+                            continue
+
+                    if alias:
+                        table = alias_map.get(alias, alias)
+                    elif default_table:
+                        table = default_table
+                    else:
+                        table = ""
+
+                    key = (table, column)
+                    if key not in seen and (table or column):
+                        seen.add(key)
+                        sources.append({"table": table, "column": column})
+
+                if sources:
+                    columns[col_name] = sources
+
+            return columns
+
+        # Find all CTEs
+        for cte in parsed.find_all(exp.CTE):
+            cte_name = restore_jinja_dots(cte.alias.lower()) if cte.alias else None
+            if not cte_name:
+                continue
+
+            # CTE collision detection: check if CTE name shadows a real table
+            if real_tables_lower and cte_name in real_tables_lower:
+                cte_collisions.append(
+                    {
+                        "cte_name": cte_name,
+                        "collision_type": "shadows_real_table",
+                        "warning": f"CTE '{cte_name}' shadows real table with same name. "
+                        "Lineage may be incorrect.",
+                        "trust_level": "review_required",
+                    }
+                )
+
+            cte_select = unwrap_select(cte.this)
+            columns: dict[str, list[dict]] = {}
+
+            if isinstance(cte_select, exp.Select):
+                columns = extract_select_columns(cte_select)
+            elif isinstance(cte_select, exp.SetOperation):
+                # value branches only (EXCEPT/INTERSECT right sides filter,
+                # never surface), mapped positionally onto the left branch's
+                # output names: 'a AS x UNION ALL b AS y' exposes one column
+                # x fed by both branches, not a phantom y. UNION BY NAME is
+                # the exception and merges by name.
+                branches = set_op_value_branches(cte_select)
+                by_name = any(
+                    n.args.get("by_name")
+                    for n in cte_select.find_all(exp.Union, exp.Except, exp.Intersect)
+                )
+                if branches:
+                    columns = extract_select_columns(branches[0])
+                    out_names = [n for n in columns if n != "*"]
+                    for branch in branches[1:]:
+                        branch_columns = extract_select_columns(branch)
+                        branch_names = [n for n in branch_columns if n != "*"]
+                        for i, bname in enumerate(branch_names):
+                            if by_name:
+                                target = bname
+                            else:
+                                target = out_names[i] if i < len(out_names) else bname
+                            existing = columns.setdefault(target, [])
+                            seen = {(s.get("table"), s.get("column")) for s in existing}
+                            for src in branch_columns[bname]:
+                                key = (src.get("table"), src.get("column"))
+                                if key not in seen:
+                                    seen.add(key)
+                                    existing.append(src)
+                        for src in branch_columns.get("*", []):
+                            existing = columns.setdefault("*", [])
+                            key = (src.get("table"), src.get("column"))
+                            if key not in {(s.get("table"), s.get("column")) for s in existing}:
+                                existing.append(src)
+
+            if columns:
+                cte_lineage[cte_name] = columns
+
+    except Exception as e:
+        logger.debug(f"Failed to extract CTE column lineage: {e}")
+
+    # Include CTE collisions in result if any were detected
+    result: dict[str, Any] = dict(cte_lineage)
+    if cte_collisions:
+        result[META_CTE_COLLISIONS] = cte_collisions
+
+    return result
+
+
+def set_op_value_branches(node: "exp.Expression") -> list["exp.Select"]:
+    """Selects whose rows can appear in a set operation's output. EXCEPT and
+    INTERSECT right sides only filter; their values never surface."""
+    from sqlglot import exp
+
+    if isinstance(node, (exp.Subquery, exp.Paren)):
+        return set_op_value_branches(node.this)
+    if isinstance(node, exp.Union):
+        return set_op_value_branches(node.left) + set_op_value_branches(node.right)
+    if isinstance(node, (exp.Except, exp.Intersect)):
+        return set_op_value_branches(node.left)
+    if isinstance(node, exp.Select):
+        return [node]
+    return []
+
+
+def extract_set_op_column_sources(
+    set_op: "exp.Expression",
+    dialect: str,
+    correlated_aliases: dict[str, str] | None = None,
+    correlated_maps: dict[str, dict[str, list[dict]]] | None = None,
+    correlated_tables: list[str] | None = None,
+) -> dict[str, list[dict]]:
+    """Column map for a set-operation body: every value branch contributes
+    to its positionally or by-name matched output column.
+
+    Output names come from the leftmost branch. A star branch adds its
+    relations under '*' and pass-through entries for output columns its own
+    select list does not name; trace_through_ctes resolves both later. A
+    set operation inside an APPLY is as correlated as a plain SELECT, so
+    the enclosing scope's names forward to every branch (cycle-9
+    review, F9).
+    """
+    from sqlglot import exp
+
+    branches = set_op_value_branches(set_op)
+    if not branches:
+        return {}
+
+    def _branch_columns(branch: "exp.Select") -> dict[str, list[dict]]:
+        return extract_select_column_sources(
+            branch,
+            dialect,
+            correlated_aliases=correlated_aliases,
+            correlated_maps=correlated_maps,
+            correlated_tables=correlated_tables,
+        )
+
+    columns = _branch_columns(branches[0])
+    by_name = any(
+        n.args.get("by_name") for n in set_op.find_all(exp.Union, exp.Except, exp.Intersect)
+    )
+    output_names = [k for k in columns if k != "*"]
+    left_has_star = "*" in columns
+    lower_names = {n.lower(): n for n in output_names}
+
+    def append(target: str, entries: list[dict]) -> None:
+        existing = columns.setdefault(target, [])
+        seen = {(s.get("table"), s.get("column")) for s in existing}
+        for src in entries:
+            key = (src.get("table"), src.get("column"))
+            if key not in seen and (key[0] or key[1]):
+                seen.add(key)
+                existing.append(src)
+
+    for branch in branches[1:]:
+        branch_cols = _branch_columns(branch)
+        named_items = [(n, s) for n, s in branch_cols.items() if n != "*"]
+        star_sources = branch_cols.get("*") or []
+        use_positional = (
+            not by_name
+            and not left_has_star
+            and not star_sources
+            and len(named_items) == len(output_names)
+        )
+        covered: set[str] = set()
+        for idx, (name, srcs) in enumerate(named_items):
+            target = output_names[idx] if use_positional else lower_names.get(name.lower())
+            if target is None:
+                continue
+            covered.add(target)
+            append(target, srcs)
+        star_relations = [s.get("table") for s in star_sources if s.get("table")]
+        for rel in star_relations:
+            for target in output_names:
+                if target in covered:
+                    continue
+                append(target, [{"table": rel, "column": target, "confidence": 0.8}])
+        if star_sources:
+            append("*", star_sources)
+    return columns
+
+
+def subquery_column_entries(sub_map: dict[str, list[dict]], column: str) -> list[dict]:
+    """An inline subquery's own sources for one of its output columns: the
+    named entry, or the column carried through the subquery's star
+    passthrough. Used wherever an alias-qualified column's qualifier turns
+    out to be a subquery alias, so the alias never surfaces as a relation
+    (pensjon under qualify(), the gap named in PR #28)."""
+    entries = sub_map.get(column)
+    if entries is None:
+        entries = next((v for k, v in sub_map.items() if k.lower() == column.lower()), None)
+    if entries is None:
+        entries = [{**src, "column": column} for src in sub_map.get("*", []) if src.get("table")]
+    return entries or []
+
+
+def rename_derived_columns(
+    columns: dict[str, list[dict]], names: list[str]
+) -> dict[str, list[dict]]:
+    """Apply a derived table's column-list alias (AS x (a, b)) positionally.
+
+    The list renames the subquery's outputs in order, so an unaliased
+    select item (a bare CAST) is read by its declared name, never by its
+    SQL spelling (DarkQueries' event_file_value_xml ([xml]), holdout
+    round 8)."""
+    if not names:
+        return columns
+    renamed: dict[str, list[dict]] = {}
+    idx = 0
+    for key, value in columns.items():
+        if key == "*" or str(key).startswith(META_PREFIX):
+            renamed[key] = value
+            continue
+        renamed[names[idx] if idx < len(names) else key] = value
+        idx += 1
+    return renamed
+
+
+def alias_select_items(inner: "exp.Select", names: list[str]) -> "exp.Select":
+    """A copy with the column-list alias applied to the select items in
+    place, so two items spelling the same output (t.x, u.x) extract under
+    their distinct declared names instead of collapsing onto one dict key
+    (cycle-9 review, F3). A star item makes the positions
+    unknowable offline; the caller falls back to renaming afterwards."""
+    from sqlglot import exp
+
+    renamed = inner.copy()
+    for idx, item in enumerate(renamed.selects):
+        if idx >= len(names):
+            break
+        if isinstance(item, exp.Alias):
+            item.set("alias", exp.to_identifier(names[idx]))
+        else:
+            item.replace(exp.alias_(item, names[idx]))
+    return renamed
+
+
+def lateral_subquery_map(
+    lateral: "exp.Expression",
+    dialect: str,
+    correlated_aliases: dict[str, str] | None = None,
+    correlated_maps: dict[str, dict[str, list[dict]]] | None = None,
+    correlated_tables: list[str] | None = None,
+) -> dict[str, list[dict]] | None:
+    """Column map of a lateral derived table (CROSS/OUTER APPLY (SELECT ...)).
+
+    APPLY subqueries are correlated: their reads resolve against the outer
+    scope's aliases, tables and the maps of earlier applies, so all three
+    are passed through (DarkQueries' stacked CROSS APPLY chain, holdout
+    round 8). Returns None when the lateral wraps anything but a derived
+    table (Explode laterals are expansions, handled by the scope
+    machinery)."""
+    from sqlglot import exp
+
+    if not isinstance(lateral.this, (exp.Subquery, exp.Paren)):
+        return None
+    inner = unwrap_select(lateral.this)
+    alias_node = lateral.args.get("alias")
+    names = [
+        c.name for c in ((alias_node.columns if alias_node is not None else None) or []) if c.name
+    ]
+    if isinstance(inner, exp.Select):
+        bases = [item.this if isinstance(item, exp.Alias) else item for item in inner.selects]
+        renamed_on_ast = bool(names) and not any(
+            isinstance(b, exp.Star) or (isinstance(b, exp.Column) and isinstance(b.this, exp.Star))
+            for b in bases
+        )
+        if renamed_on_ast:
+            inner = alias_select_items(inner, names)
+        columns = extract_select_column_sources(
+            inner,
+            dialect,
+            correlated_aliases=correlated_aliases,
+            correlated_maps=correlated_maps,
+            correlated_tables=correlated_tables,
+        )
+        if renamed_on_ast:
+            return columns
+    elif isinstance(inner, (exp.Union, exp.Except, exp.Intersect)):
+        columns = extract_set_op_column_sources(
+            inner,
+            dialect,
+            correlated_aliases=correlated_aliases,
+            correlated_maps=correlated_maps,
+            correlated_tables=correlated_tables,
+        )
+    else:
+        return None
+    return rename_derived_columns(columns, names)
+
+
+def extract_select_column_sources(
+    select_node: "exp.Select",
+    dialect: str,
+    correlated_aliases: dict[str, str] | None = None,
+    correlated_maps: dict[str, dict[str, list[dict]]] | None = None,
+    correlated_tables: list[str] | None = None,
+) -> dict[str, list[dict]]:
+    """Column name -> sources for a single SELECT, in select-list order.
+
+    Shared by CTE mapping extraction and set-operation branch tracing.
+    Qualified stars (rel.*) and bare stars land under the '*' key with one
+    entry per spanned relation. correlated_aliases, correlated_maps and
+    correlated_tables are the enclosing scope's names, passed only for
+    correlated lateral derived tables (APPLY); local definitions win on
+    collision.
+    """
+    from sqlglot import exp
+
+    alias_map: dict[str, str] = dict(correlated_aliases or {})
+    all_tables: list[str] = []
+
+    def _subquery_map(inner: "exp.Expression | None") -> dict[str, list[dict]] | None:
+        # a set-operation derived table is as absorbable as a plain SELECT;
+        # skipping it left the alias as a phantom upstream (review of
+        # PR #28)
+        if isinstance(inner, exp.Select):
+            return extract_select_column_sources(inner, dialect)
+        if isinstance(inner, (exp.Union, exp.Except, exp.Intersect)):
+            return extract_set_op_column_sources(inner, dialect)
+        return None
+
+    subquery_columns_by_alias: dict[str, dict[str, list[dict]]] = {}
+    subquery_columns_folded: dict[str, dict[str, list[dict]]] = {}
+    for name, sub_map in (correlated_maps or {}).items():
+        if name:
+            subquery_columns_by_alias[name] = sub_map
+            subquery_columns_folded.setdefault(name.lower(), sub_map)
+
+    def _register_subquery(subquery: "exp.Subquery") -> None:
+        sub_map = _subquery_map(unwrap_select(subquery.this))
+        if sub_map is None:
+            return
+        if not subquery.alias:
+            # an UNALIASED from-subquery: its projection IS this scope's
+            # source. Dropping it let the sole-source fallthrough chain to
+            # the external relation and invent columns on it
+            # (transfermarkt's datetime_str, holdout round 7)
+            subquery_columns_by_alias.setdefault("", sub_map)
+            return
+        subquery_columns_by_alias[subquery.alias] = sub_map
+        if alias_is_foldable(subquery):
+            subquery_columns_folded.setdefault(subquery.alias.lower(), sub_map)
+
+    def _sub_map_for(alias: str, quoted: bool) -> dict[str, list[dict]] | None:
+        # a quoted reference matches only exactly; an unquoted one folds
+        # case against foldable definitions (the cycle-6 review)
+        exact = subquery_columns_by_alias.get(alias)
+        if exact is not None or quoted:
+            return exact
+        return subquery_columns_folded.get(alias.lower())
+
+    from_clause = select_node.args.get("from_") or select_node.args.get("from")
+    local_real_table = False
+    joined_relation = False
+    anon_candidates = 0
+    if from_clause:
+        for subquery in from_clause.find_all(exp.Subquery):
+            # nested subqueries belong to their own select; registering
+            # them here leaked inner projections into the outer scope
+            # (the cycle-8 review)
+            if owning_select(subquery) is not select_node:
+                continue
+            if not subquery.alias:
+                anon_candidates += 1
+            _register_subquery(subquery)
+        for table in from_clause.find_all(exp.Table):
+            register_table(table, alias_map, all_tables)
+            if owning_select(table) is select_node and table_function_kind(table) != "srf":
+                local_real_table = True
+
+    for join in select_node.find_all(exp.Join):
+        # joins inside CTE bodies or nested subqueries belong to their own
+        # select; letting them in shadows this scope's aliases
+        if owning_select(join) is not select_node:
+            continue
+        join_table = join.this
+        if isinstance(join_table, exp.Subquery):
+            _register_subquery(join_table)
+            joined_relation = True
+        if isinstance(join_table, exp.Lateral) and join_table.alias:
+            lat_map = lateral_subquery_map(
+                join_table,
+                dialect,
+                correlated_aliases=alias_map,
+                correlated_maps=subquery_columns_by_alias,
+                correlated_tables=all_tables,
+            )
+            if lat_map is not None:
+                subquery_columns_by_alias[join_table.alias] = lat_map
+                if alias_is_foldable(join_table):
+                    subquery_columns_folded[join_table.alias.lower()] = lat_map
+                joined_relation = True
+        if isinstance(join_table, exp.Table):
+            register_table(join_table, alias_map, all_tables)
+            if table_function_kind(join_table) != "srf":
+                local_real_table = True
+
+    # the anonymous map speaks for this scope only when the unaliased
+    # subquery is the SOLE relation: a second subquery or any other
+    # relation makes its outputs a coin flip (the cycle-8 review)
+    if "" in subquery_columns_by_alias and (
+        anon_candidates != 1 or local_real_table or joined_relation
+    ):
+        subquery_columns_by_alias.pop("", None)
+
+    # a correlated body with no FROM of its own resolves against the outer
+    # scope; without the outer relations an unqualified read had no
+    # ownership candidate at all (cycle-9 review, F8)
+    if not all_tables and correlated_tables:
+        all_tables.extend(dict.fromkeys(correlated_tables))
+
+    default_table = all_tables[0] if len(all_tables) == 1 else ""
+    join_anchor = "" if default_table else left_join_anchor(select_node)
+    register_subquery_from_aliases(list(select_node.selects), alias_map)
+    columns: dict[str, list[dict]] = {}
+
+    # UNNEST element aliases in this select's FROM scope, including the
+    # synthetic _0-style names sqlglot's qualify() invents. An element
+    # read resolves to the array root column, never to a phantom.
+    expansions = collect_unnest_expansions(select_node, alias_map, all_tables)
+    lateral_aliases: dict[str, list[dict]] = {}
+    allow_lateral = (dialect or "").lower() in LATERAL_ALIAS_DIALECTS
+
+    _known = (
+        {str(k).lower() for k in alias_map}
+        | {str(v).lower() for v in alias_map.values()}
+        | {str(k).lower() for k in expansions}
+        | {str(k).lower() for k in subquery_columns_by_alias}
+    )
+
+    for select_expr in select_node.selects:
+        is_computed = False
+
+        if isinstance(select_expr, exp.Alias):
+            col_name = select_expr.alias
+            source_expr = select_expr.this
+            is_computed = not isinstance(source_expr, exp.Column)
+        elif isinstance(select_expr, exp.Column) and isinstance(select_expr.this, exp.Star):
+            # qualified star: rel.* spans one relation. Over an inline
+            # subquery alias, absorb the subquery's own map instead of
+            # letting the alias surface as a phantom relation (pensjon's
+            # dedup wrapper, holdout round 5)
+            qualifier = select_expr.table or ""
+            sub_map = _sub_map_for(qualifier, qualifier_is_quoted(select_expr))
+            if sub_map is not None:
+                for name, srcs in sub_map.items():
+                    if name == "*":
+                        existing = columns.setdefault("*", [])
+                        for src in srcs:
+                            if all(s.get("table") != src.get("table") for s in existing):
+                                existing.append(dict(src))
+                    else:
+                        columns.setdefault(name, [dict(s) for s in srcs])
+                continue
+            rel = alias_map.get(qualifier, qualifier)
+            if rel:
+                existing = columns.setdefault("*", [])
+                if all(s.get("table") != rel for s in existing):
+                    existing.append({"table": rel, "column": "*", "confidence": 0.5})
+            continue
+        elif isinstance(select_expr, exp.Column):
+            col_name = select_expr.name
+            source_expr = select_expr
+        elif isinstance(select_expr, exp.Star):
+            if not local_real_table and subquery_columns_by_alias:
+                # bare star over a scope of only derived tables: the
+                # columns are the subqueries' own projections, all of them
+                for sub_map_ in subquery_columns_by_alias.values():
+                    for name, srcs in sub_map_.items():
+                        if str(name).startswith(META_PREFIX):
+                            continue
+                        if name == "*":
+                            existing = columns.setdefault("*", [])
+                            for src in srcs:
+                                if all(s.get("table") != src.get("table") for s in existing):
+                                    existing.append(dict(src))
+                        else:
+                            columns.setdefault(name, [dict(s) for s in srcs])
+                continue
+            existing = columns.setdefault("*", [])
+            for t in all_tables:
+                if all(s.get("table") != t for s in existing):
+                    existing.append({"table": t, "column": "*", "confidence": 0.5})
+            continue
+        else:
+            col_sql = safe_sql(select_expr, dialect)
+            col_name = col_sql if col_sql is not None and len(col_sql) < 50 else None
+            source_expr = select_expr
+            is_computed = True
+
+        if not col_name:
+            continue
+
+        sources: list[dict] = []
+        seen: set[tuple[str, str]] = set()
+
+        # qualify() rewrites a read of a table-valued alias (CROSS JOIN
+        # jsonb_array_elements_text(roles) AS role ... SELECT role) into a
+        # TableColumn node, which the Column sweep never sees (kingfisher,
+        # holdout round 7)
+        tc_cls = getattr(exp, "TableColumn", None)
+        if tc_cls is not None:
+            for tc in source_expr.find_all(tc_cls):
+                entry = expansions.get(tc.name) or expansions.get(tc.name.lower())
+                if not entry:
+                    continue
+                reads = [(entry.get("source_table", ""), entry.get("source_column", ""))] + [
+                    (x.get("source_table", ""), x.get("source_column", ""))
+                    for x in entry.get("extra_columns", [])
+                ]
+                for key in reads:
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(
+                            {
+                                "table": key[0],
+                                "column": key[1],
+                                "confidence": 0.5,
+                                "inferred": False,
+                                "from_array_expansion": True,
+                            }
+                        )
+
+        for col in source_expr.find_all(exp.Column):
+            if under_subquery_where(col, source_expr):
+                continue
+            if in_window_ordering(col, source_expr):
+                continue
+            if in_selector_argument(col, source_expr):
+                continue
+            alias, column, _field_path, _certain = resolve_column_ref(col, _known)
+            if "{" in column or (not _certain and any("{" in p for p in column_parts(col))):
+                # a {placeholder} or jinja chain in column/qualifier position
+                # has no citable identity; anchoring it fabricated a struct
+                # root on the FROM table (cycle-12, F11/F19)
+                continue
+
+            expansion = None
+            if _certain and alias:
+                expansion = expansions.get(alias)
+            elif _certain and not alias:
+                expansion = expansions.get(column)
+            elif not _certain:
+                expansion = scoped_unnest_entry(
+                    col, source_expr, alias_map, all_tables
+                ) or expansions.get("__anonymous_unnest__")
+
+            if expansion is not None:
+                table = expansion.get("source_table") or default_table
+                array_column = expansion["source_column"]
+                selective = expansion_field_reads(expansion, col, column, _field_path)
+                if selective is not None:
+                    expanded = [
+                        (e.get("source_table") or default_table, e.get("source_column", ""))
+                        for e in selective
+                    ]
+                else:
+                    expanded = [(table, array_column)] + [
+                        (e.get("source_table") or default_table, e.get("source_column", ""))
+                        for e in expansion.get("extra_columns", [])
+                    ]
+                for exp_table, exp_column in expanded:
+                    key = (exp_table, exp_column)
+                    if key not in seen and (exp_table or exp_column):
+                        seen.add(key)
+                        sources.append(
+                            {
+                                "table": exp_table,
+                                "column": exp_column,
+                                "confidence": 0.5,
+                                "inferred": False,
+                                "from_array_expansion": True,
+                            }
+                        )
+                continue
+
+            if _certain and not alias and allow_lateral and column.lower() in lateral_aliases:
+                # lateral column alias defined earlier in this list
+                for src in lateral_aliases[column.lower()]:
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append({**src, "via_lateral_alias": True})
+                continue
+
+            if not _certain:
+                # struct-root read: anchor to the sole relation at reduced
+                # confidence rather than fabricating the leading part
+                table = default_table or join_anchor or ""
+                confidence = 0.5
+            elif alias and (sub_map := _sub_map_for(alias, qualifier_is_quoted(col))) is not None:
+                for src in subquery_column_entries(sub_map, column):
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(dict(src))
+                continue
+            elif alias:
+                table = alias_map.get(alias, alias)
+                confidence = 1.0
+            elif (anon := subquery_columns_by_alias.get("")) is not None and any(
+                k != "*" and k.lower() == column.lower() for k in anon
+            ):
+                # unqualified read of a column the unaliased from-subquery
+                # projects (transfermarkt's `where n = 1` wrapper)
+                for src in subquery_column_entries(anon, column):
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(dict(src))
+                continue
+            elif (
+                not local_real_table
+                and all("*" not in m for m in subquery_columns_by_alias.values())
+                and len(
+                    owner_maps := [
+                        m
+                        for m in subquery_columns_by_alias.values()
+                        if any(
+                            k != "*"
+                            and not str(k).startswith(META_PREFIX)
+                            and str(k).lower() == column.lower()
+                            for k in m
+                        )
+                    ]
+                )
+                == 1
+            ):
+                # this select reads only derived tables, and exactly one
+                # projects the column: anchoring to a table found INSIDE
+                # the subquery skips the derivation (kingfisher's role
+                # expansion read through id_role, holdout round 7)
+                for src in subquery_column_entries(owner_maps[0], column):
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(dict(src))
+                continue
+            elif (nested := nested_scope_sole_table(col, select_node)) is not None:
+                # the column lives in a subselect with its own FROM; the
+                # outer scope is the wrong place to anchor it (review
+                # of PR #28: correlated scalar subqueries)
+                table = nested
+                confidence = 0.8 if nested else 0.5
+            elif default_table:
+                table = default_table
+                confidence = 0.8
+            elif len(all_tables) > 1 and (
+                owner := system_catalog_owner(column, all_tables, dialect)
+            ):
+                # every relation in scope is a documented sys object and
+                # exactly one owns the column (InvestigateWaits, round 8);
+                # documented ownership outranks the join-anchor guess
+                # (cycle-9 review, F5)
+                table = owner
+                confidence = 0.8
+            elif join_anchor:
+                table = join_anchor
+                confidence = 0.6
+            else:
+                table = ""
+                confidence = 0.5
+
+            key = (table, column)
+            if key not in seen and (table or column):
+                seen.add(key)
+                sources.append(
+                    {
+                        "table": table,
+                        "column": column,
+                        "confidence": confidence,
+                        "inferred": alias == "" and table != "",
+                    }
+                )
+
+        if not sources and is_computed:
+            sources = [{"table": "", "column": col_name, "computed": True, "confidence": 0.3}]
+
+        if isinstance(select_expr, exp.Alias) and sources:
+            lateral_aliases.setdefault(col_name.lower(), sources)
+
+        columns[col_name] = sources
+
+    return columns
+
+
+def extract_cte_column_mappings(
+    parsed: "exp.Expression",
+    dialect: str,
+) -> dict[str, dict[str, list[dict]]]:
+    """Extract column lineage from each CTE in the query.
+
+    Returns a mapping: CTE name -> {column_name -> [sources]}
+
+    This enables recursive tracing through CTEs.
+    """
+    from sqlglot import exp
+
+    cte_columns: dict[str, dict[str, list[dict]]] = {}
+
+    # nested WITHs flatten into one map, so a name defined twice (an inner
+    # CTE shadowing an outer one) resolves to neither; recursive CTEs read
+    # their own name legitimately and it must not pass for a real relation
+    # (the cycle-7 review)
+    counts: dict[str, int] = {}
+    recursive_names: set[str] = set()
+    for with_node in parsed.find_all(exp.With):
+        if with_node.args.get("recursive"):
+            for cte in with_node.expressions:
+                if cte.alias:
+                    recursive_names.add(restore_jinja_dots(cte.alias.lower()))
+    for cte in parsed.find_all(exp.CTE):
+        if cte.alias:
+            key = restore_jinja_dots(cte.alias.lower())
+            counts[key] = counts.get(key, 0) + 1
+
+    per_name: dict[str, list[dict]] = {}
+    for cte in parsed.find_all(exp.CTE):
+        # keys restored so a templated CTE name agrees with the restored
+        # table reference that reads it (cycle-12, F10)
+        cte_name = restore_jinja_dots(cte.alias.lower()) if cte.alias else ""
+        if not cte_name:
+            continue
+
+        cte_select = unwrap_select(cte.this)
+        if isinstance(cte_select, exp.Select):
+            entry = extract_select_column_sources(cte_select, dialect)
+        elif isinstance(cte_select, (exp.Union, exp.Except, exp.Intersect)):
+            entry = extract_set_op_column_sources(cte_select, dialect)
+        else:
+            continue
+        # relations the CTE reads, for the sole-source fallthrough: a joined
+        # relation whose loop-generated projection vanished offline is
+        # invisible in the projected sources but not in the FROM scope
+        entry[META_CTE_RELATIONS] = sorted(
+            {
+                qualified_table_name(t)
+                for t in cte_select.find_all(exp.Table)
+                if qualified_table_name(t)
+            }
+        )
+        if cte_name in recursive_names:
+            entry[META_CTE_RECURSIVE] = True
+        per_name.setdefault(cte_name, []).append(entry)
+
+    for cte_name, entries in per_name.items():
+        keeper = entries[-1]  # find_all is pre-order: last = innermost
+        if len(entries) > 1 and not all(_references_only(e, cte_name) for e in entries[:-1]):
+            # genuinely different definitions share the name; the flat map
+            # cannot know which one a reference means
+            keeper = dict(keeper)
+            keeper[META_CTE_SHADOWED] = True
+        cte_columns[cte_name] = keeper
+
+    return cte_columns
+
+
+def _references_only(entry: dict, name: str) -> bool:
+    """True when a CTE definition's column sources reference nothing but
+    `name` itself: a pure self-wrapper. Macro-emitted models nest WITH X
+    inside WITH X (... SELECT * FROM X), where the innermost definition is
+    the meaning everywhere; only materially different definitions make the
+    name ambiguous."""
+    return {
+        (src.get("table") or "").lower()
+        for key, sources in entry.items()
+        if not key.startswith(META_PREFIX)
+        for src in sources
+        if src.get("table") and not src.get("computed")
+    } <= {name}
+
+
+def _unknown_source(column: str) -> dict:
+    return {"table": "", "column": column, "computed": True, "confidence": 0.3}
+
+
+def _cte_sole_relation(cte_cols: dict) -> str | None:
+    """The single relation a CTE reads from, or None if it reads several.
+
+    Prefers the FROM-scope relations recorded at extraction: a joined
+    relation whose loop-generated projection vanished offline is invisible
+    in the projected sources but was still read, and guessing the other
+    relation would be wrong-confident (the cycle-7 review). Falls back
+    to the tables the surviving column sources name."""
+    relations = cte_cols.get(META_CTE_RELATIONS)
+    if relations is not None:
+        return relations[0] if len(relations) == 1 else None
+    inferred = {
+        src["table"]
+        for key, sources in cte_cols.items()
+        if not key.startswith(META_PREFIX)
+        for src in sources
+        if src.get("table") and not src.get("computed")
+    }
+    return next(iter(inferred)) if len(inferred) == 1 else None
+
+
+def _dedupe(results: list[dict]) -> list[dict]:
+    """Collapse duplicate (table, column) entries, keeping the most
+    confident. Branching CTE DAGs (two CTEs per level, each reading both
+    parents) otherwise multiply identical paths exponentially — 2^18
+    entries at depth 18 (the cycle-7 review)."""
+    best: dict[tuple, dict] = {}
+    order: list[tuple] = []
+    for entry in results:
+        key = (
+            (entry.get("table") or "").lower(),
+            (entry.get("column") or "").lower(),
+            bool(entry.get("computed")),
+        )
+        held = best.get(key)
+        if held is None:
+            best[key] = entry
+            order.append(key)
+        elif entry.get("confidence", 1.0) > held.get("confidence", 1.0):
+            best[key] = entry
+    return [best[k] for k in order]
+
+
+def _seal(
+    entries: list[dict],
+    cte_columns: dict[str, dict[str, list[dict]]],
+    shadow: str = "",
+) -> list[dict]:
+    """A CTE is a statement-scoped name, never a relation; reporting one as a
+    source is always wrong (holdout round 6 scored issue_multiselect_history
+    and set_values as tables). Anything still CTE-named degrades to
+    computed/unknown.
+
+    `shadow` is the CTE whose own body the entries came from: SQL forbids
+    non-recursive self-reference, so that one spelling denotes the real
+    relation the CTE shadows (the dbt import idiom, `with orders as
+    (select * from orders)`) and must survive the seal."""
+    return [
+        e
+        if (t := (e.get("table") or "").lower()) not in cte_columns or t == shadow
+        else _unknown_source(e.get("column", ""))
+        for e in entries
+    ]
+
+
+def trace_through_ctes(
+    column: str,
+    table: str,
+    cte_columns: dict[str, dict[str, list[dict]]],
+    visited: set[str] | None = None,
+    max_depth: int = 32,
+    _budget: list[int] | None = None,
+) -> list[dict]:
+    """Recursively trace a column through CTEs to find actual sources.
+
+    Args:
+        column: Column name to trace
+        table: Table/CTE name the column comes from
+        cte_columns: CTE name -> column mappings
+        visited: Set of visited (table, column) pairs to prevent cycles
+        max_depth: Maximum recursion depth (dbt_jira's pivot chains twelve
+            CTEs, holdout round 6; the cap only guards pathological fanout)
+
+    Returns:
+        List of source columns with full tracing through CTEs
+    """
+    if visited is None:
+        visited = set()
+    if _budget is None:
+        # visited is copied per branch, so a branching DAG (two CTEs per
+        # level, each reading both parents) re-explores 2^n paths inside
+        # the depth cap. The shared budget bounds total work; exhaustion
+        # degrades to unknown, never to a wrong answer (review of
+        # cycle 7). Real chains are linear and use a few hundred calls.
+        _budget = [50_000]
+    _budget[0] -= 1
+    if _budget[0] <= 0:
+        return [_unknown_source(column)]
+
+    key = f"{table.lower()}.{column.lower()}"
+    if key in visited or max_depth <= 0:
+        return []
+
+    visited.add(key)
+
+    table_lower = table.lower()
+    if table_lower not in cte_columns:
+        # Not a CTE - this is an actual table reference
+        return [{"table": table, "column": column, "confidence": 1.0}]
+
+    # It's a CTE - look up the column's sources
+    cte_cols = cte_columns[table_lower]
+
+    if cte_cols.get(META_CTE_SHADOWED):
+        # the name is defined more than once (nested WITH shadowing); the
+        # flat map cannot know which definition a reference means
+        return [_unknown_source(column)]
+    self_is_real = not cte_cols.get(META_CTE_RECURSIVE)
+
+    if column == "*" and "*" in cte_cols:
+        return _seal(cte_cols["*"], cte_columns, shadow=table_lower if self_is_real else "")
+
+    if column not in cte_cols and column.lower() not in {k.lower() for k in cte_cols}:
+        # Column not found in CTE - might be SELECT * situation
+        if "*" in cte_cols:
+            star_sources = cte_cols["*"]
+            results = []
+            for src in star_sources:
+                if (src.get("table") or "").lower() == table_lower:
+                    # self-named import CTE: the spelling is the real
+                    # relation it shadows, not a recursion target (a
+                    # RECURSIVE cte's self-read is the cte and adds nothing)
+                    if self_is_real:
+                        results.append(
+                            {
+                                "table": src["table"],
+                                "column": column,
+                                "confidence": src.get("confidence", 1.0),
+                            }
+                        )
+                    continue
+                traced = trace_through_ctes(
+                    column,
+                    src["table"],
+                    cte_columns,
+                    visited.copy(),
+                    max_depth - 1,
+                    _budget,
+                )
+                results.extend(traced)
+            return _dedupe(results) if results else [_unknown_source(column)]
+        # Explicit projection without the column: offline rendering loses
+        # loop-generated select items (dbt_jira's adapter.get_columns_in_
+        # relation loops), so a sole-source CTE passes the read through at
+        # guess confidence; several sources would be a coin flip, refuse.
+        sole = _cte_sole_relation(cte_cols)
+        if sole is not None and sole.lower() == table_lower:
+            if self_is_real:
+                return [{"table": sole, "column": column, "confidence": 0.4}]
+            return [_unknown_source(column)]
+        if sole is not None:
+            traced = trace_through_ctes(
+                column, sole, cte_columns, visited.copy(), max_depth - 1, _budget
+            )
+            if traced:
+                return [
+                    {**t, "confidence": min(t.get("confidence", 1.0), 0.4)}
+                    for t in _seal(traced, cte_columns)
+                ]
+        return [_unknown_source(column)]
+
+    # Find the column (case-insensitive)
+    col_key = column
+    for k in cte_cols:
+        if k.lower() == column.lower():
+            col_key = k
+            break
+
+    sources = cte_cols.get(col_key, [])
+    if not sources:
+        return [_unknown_source(column)]
+
+    # Recursively trace each source
+    results = []
+    for src in sources:
+        if src.get("computed"):
+            results.append(src)
+        elif src.get("table"):
+            if src["table"].lower() == table_lower:
+                # self-named import CTE (with orders as (select * from
+                # orders)): the inner spelling denotes the real relation
+                # the CTE shadows, never a recursive lookup. A RECURSIVE
+                # cte's self-read IS the cte and contributes nothing.
+                if self_is_real:
+                    results.append(dict(src))
+                continue
+            traced = trace_through_ctes(
+                src["column"],
+                src["table"],
+                cte_columns,
+                visited.copy(),
+                max_depth - 1,
+                _budget,
+            )
+            if traced:
+                # a hop through an uncertain mapping (array expansion,
+                # struct guess) never comes out more confident than it went in
+                src_confidence = src.get("confidence", 1.0)
+                for t in traced:
+                    if t.get("confidence", 1.0) > src_confidence:
+                        t = {**t, "confidence": src_confidence}
+                    results.append(t)
+            else:
+                # budget or cycle exhaustion mid-chain: the raw src may still
+                # be CTE-qualified and must not surface as a relation
+                results.extend(_seal([src], cte_columns))
+        else:
+            results.append(src)
+
+    return _dedupe(results)
+
+
+def star_except_spellings(idents) -> frozenset[str]:
+    """The spellings a star's EXCEPT/EXCLUDE list names.
+
+    Unquoted (or already-lowercase) identifiers fold and are stored
+    lowercase. A quoted mixed-case identifier (Snowflake EXCLUDE
+    ("MiXeD")) names exactly that column and keeps its case, so it can
+    never collapse onto the unquoted column of the same letters
+    (cycle-12, F20)."""
+    out = set()
+    for c in idents:
+        name = getattr(c, "name", "") or ""
+        if not name:
+            continue
+        ident = getattr(c, "this", None)
+        quoted = bool(getattr(ident, "quoted", False) or getattr(c, "args", {}).get("quoted"))
+        out.add(name if quoted and name != name.lower() else name.lower())
+    return frozenset(out)
+
+
+def star_excepted(name: str, excepts) -> bool:
+    """True when a star exclusion covers this column spelling: exact match
+    for quoted mixed-case exclusions, case-folded for the rest."""
+    return bool(name) and (name in excepts or name.lower() in excepts)
+
+
+def expand_star_columns(
+    relations: list[str],
+    cte_columns: dict[str, dict[str, list[dict]]],
+    except_cols: set[str] | frozenset[str] = frozenset(),
+) -> tuple[dict[str, list[dict]], list[dict]]:
+    """Expand a star over known CTEs into their named output columns.
+
+    Walks star-passthrough chains (a CTE whose own select is `*` or `rel.*`
+    over another relation) so named columns anywhere down the chain surface
+    as outputs, each traced to its real sources. Relations the chain cannot
+    resolve (real tables) come back as residual wildcard sources for
+    schema-level expansion. except_cols (lowercase) are excluded at every
+    level. Returns (named_columns, residual_star_sources).
+    """
+    named: dict[str, list[dict]] = {}
+    residual: list[dict] = []
+    seen_relations: set[str] = set()
+
+    def walk(rel: str) -> None:
+        rel_lower = rel.lower()
+        if rel_lower in seen_relations:
+            return
+        seen_relations.add(rel_lower)
+        cols = cte_columns.get(rel_lower)
+        if cols is None:
+            entry: dict = {"table": rel, "column": "*", "confidence": 0.5}
+            if except_cols:
+                # the graph retains these so a named column can later
+                # resolve through the star unless excepted (class D)
+                entry["except_columns"] = sorted(except_cols)
+            residual.append(entry)
+            return
+        for name in cols:
+            if name == "*" or name.startswith(META_PREFIX):
+                continue
+            if star_excepted(name, except_cols) or name in named:
+                continue
+            traced = trace_through_ctes(name, rel, cte_columns)
+            # rel is a CTE here (cols is not None); its name must never
+            # stand in as a relation when tracing comes back empty
+            named[name] = traced or [_unknown_source(name)]
+        for src in cols.get("*", []):
+            src_table = src.get("table") or ""
+            if src_table:
+                walk(src_table)
+
+    for rel in relations:
+        if rel:
+            walk(rel)
+    return named, residual