ripple-sql 0.1.0__py3-none-any.whl

ripple/engine/select_sources.py

@@ -0,0 +1,1038 @@
+"""Select-item source extraction.
+
+Walks a statement's select list and fills the result map: for every output
+column, which upstream (table, column) pairs feed it, with confidence and
+trust metadata. Handles aliases, stars, qualified stars, script TRANSFORM,
+JSON access, array-expansion aliases, lateral column aliases, ambiguity
+resolution against the warehouse schema, and CTE tracing of each source.
+"""
+
+import logging
+from typing import Any
+
+from sqlglot import exp
+
+from ripple.engine.column_ref import (
+    column_parts,
+    expansion_field_reads,
+    in_selector_argument,
+    in_window_ordering,
+    register_subquery_from_aliases,
+    resolve_column_ref,
+    scoped_unnest_entry,
+    under_subquery_where,
+)
+from ripple.engine.cte_tracing import (
+    expand_star_columns,
+    star_except_spellings,
+    trace_through_ctes,
+)
+from ripple.engine.json_sources import collect_json_sources
+from ripple.engine.safe_gen import safe_sql
+from ripple.engine.schema_qualification import resolve_column_table
+from ripple.engine.tsql_catalog import system_catalog_owner
+from ripple.engine.types import (
+    LATERAL_ALIAS_DIALECTS,
+    META_CTE_SHADOWED,
+    META_NEEDS_EXPANSION,
+    META_PREFIX,
+    WarehouseColumns,
+)
+
+logger = logging.getLogger(__name__)
+
+try:
+    from sqlglot import exp as _sqlglot_exp
+
+    _QUERY_TRANSFORM_CLS = getattr(_sqlglot_exp, "QueryTransform", None)
+except ImportError:  # pragma: no cover
+    _QUERY_TRANSFORM_CLS = None
+
+
+def _append_column_sources(
+    result: dict[str, Any], column: str, entries: list[dict[str, Any]]
+) -> None:
+    """Append entries to result[column], deduped by (table, column)."""
+    existing = result.setdefault(column, [])
+    seen = {(e.get("table", ""), e.get("column", "")) for e in existing}
+    for entry in entries:
+        key = (entry.get("table", ""), entry.get("column", ""))
+        if key in seen or not (key[0] or key[1]):
+            continue
+        seen.add(key)
+        existing.append(entry)
+
+
+def _expansion_extra_entries(
+    expansion_info: dict[str, Any], expanded_as: str
+) -> list[dict[str, Any]]:
+    """Source entries for the extra columns a computed array expression reads.
+
+    UNNEST(GENERATE_DATE_ARRAY(start_date, LEAST(end_date, ...))) AS dt: the
+    element derives from every argument column, not just the first."""
+    entries = []
+    for extra in expansion_info.get("extra_columns", []):
+        column = extra.get("source_column", "")
+        if not column:
+            continue
+        entries.append(
+            {
+                "table": extra.get("source_table", ""),
+                "column": column,
+                "confidence": 0.5,
+                "trust_level": "moderate",
+                "inferred": False,
+                "from_array_expansion": True,
+                "expansion_type": expansion_info.get("expansion_type"),
+                "source_column_in_array": column,
+                "expanded_as": expanded_as,
+            }
+        )
+    return entries
+
+
+def _field_selective_entries(
+    expansion_info: dict[str, Any],
+    column: str,
+    col: "exp.Column",
+    field_path: str | None = None,
+) -> list[dict[str, Any]] | None:
+    """Source entries for a read of one named STRUCT field through an
+    array-of-structs expansion; None when the expansion has no field map
+    for the read (callers keep the every-column behavior)."""
+    selective = expansion_field_reads(expansion_info, col, column, field_path)
+    if selective is None:
+        return None
+    return [
+        {
+            "table": fs.get("source_table", ""),
+            "column": fs.get("source_column", ""),
+            "confidence": 0.5,
+            "trust_level": "moderate",
+            "inferred": False,
+            "from_array_expansion": True,
+            "expansion_type": expansion_info.get("expansion_type"),
+            "source_column_in_array": fs.get("source_column", ""),
+            "expanded_as": column,
+        }
+        for fs in selective
+        if fs.get("source_column")
+    ]
+
+
+def process_select_items(
+    selects_list: list["exp.Expression"],
+    select_for_from: "exp.Expression",
+    result: dict[str, Any],
+    warnings: list[dict],
+    alias_map: dict[str, str],
+    all_tables: list[str],
+    array_expansion_sources: dict[str, dict[str, str]],
+    default_table: str,
+    join_anchor: str,
+    cte_columns: dict[str, dict[str, list[dict]]],
+    cte_collision_names: set[str],
+    dialect: str,
+    warehouse_columns: WarehouseColumns | None,
+    qualified_via_schema: bool,
+) -> None:
+    """Fill result with per-output-column sources; mutates result/warnings."""
+
+    def _sole_enumerated_cte_owner(column: str) -> str | None:
+        """The one relation in scope proven to own the column, or None.
+
+        Provable only when EVERY relation is a CTE whose outputs are fully
+        enumerated (present in cte_columns, no star, not shadowed); a real
+        table or star-veiled CTE makes ownership unprovable and keeps the
+        ambiguous refusal."""
+        owners: list[str] = []
+        for t in all_tables:
+            entry = cte_columns.get(str(t).lower())
+            if entry is None or "*" in entry or entry.get(META_CTE_SHADOWED):
+                return None
+            if any(
+                not str(k).startswith(META_PREFIX) and str(k).lower() == column.lower()
+                for k in entry
+            ):
+                owners.append(t)
+        return owners[0] if len(owners) == 1 else None
+
+    def _lookup_anchor(column: str) -> str:
+        """Lookup-shape fallback for an unqualified read; never a CTE
+        whose known outputs prove the column absent."""
+        if not join_anchor:
+            return ""
+        anchor_cols = cte_columns.get(join_anchor.lower())
+        if anchor_cols is not None and not any(k.lower() == column.lower() for k in anchor_cols):
+            return ""
+        return join_anchor
+
+    register_subquery_from_aliases(selects_list, alias_map)
+    # Output maps of THIS select's aliased FROM/JOIN subqueries, plus the
+    # relations registered at this level (find_all descends into subqueries,
+    # so all_tables alone cannot say what sits at this level). Together they
+    # support elimination: an unqualified column absent from every derived
+    # item's outputs must come from the one real table (patentsview's
+    # webtool joins, holdout round 5).
+    from ripple.engine.column_ref import (
+        alias_is_foldable,
+        nested_scope_sole_table,
+        owning_select,
+        qualified_table_name,
+        qualifier_is_quoted,
+        unwrap_select,
+    )
+    from ripple.engine.cte_tracing import (
+        extract_select_column_sources as _sub_map,
+    )
+    from ripple.engine.cte_tracing import (
+        extract_set_op_column_sources as _sub_set_op_map,
+    )
+    from ripple.engine.cte_tracing import (
+        lateral_subquery_map as _lateral_subquery_map,
+    )
+    from ripple.engine.cte_tracing import subquery_column_entries as _subquery_column_entries
+
+    subquery_output_maps: list[dict] = []
+    subquery_maps_by_alias: dict[str, dict] = {}
+    subquery_maps_folded: dict[str, dict] = {}
+    local_real_tables: list[str] = []
+    from ripple.engine.column_ref import table_function_kind as _tf_kind
+
+    _from = select_for_from.args.get("from_") or select_for_from.args.get("from")
+    # nested subqueries belong to their own select; registering them here
+    # leaked inner projections into the outer scope (the cycle-8 review)
+    _sub_nodes = (
+        [sq for sq in _from.find_all(exp.Subquery) if owning_select(sq) is select_for_from]
+        if _from is not None
+        else []
+    )
+    if _from is not None:
+        for _t in _from.find_all(exp.Table):
+            if owning_select(_t) is select_for_from and _tf_kind(_t) != "srf":
+                local_real_tables.append(qualified_table_name(_t))
+    for _join in select_for_from.find_all(exp.Join):
+        if owning_select(_join) is not select_for_from:
+            continue
+        if isinstance(_join.this, exp.Subquery):
+            _sub_nodes.append(_join.this)
+        elif isinstance(_join.this, exp.Table) and _tf_kind(_join.this) != "srf":
+            local_real_tables.append(qualified_table_name(_join.this))
+    for _sq in _sub_nodes:
+        _inner = unwrap_select(_sq.this)
+        try:
+            if isinstance(_inner, exp.Select):
+                _map = _sub_map(_inner, dialect)
+            elif isinstance(_inner, (exp.Union, exp.Except, exp.Intersect)):
+                _map = _sub_set_op_map(_inner, dialect)
+            else:
+                continue
+        except Exception:
+            continue
+        subquery_output_maps.append(_map)
+        if _sq.alias:
+            subquery_maps_by_alias[_sq.alias] = _map
+            if alias_is_foldable(_sq):
+                subquery_maps_folded.setdefault(_sq.alias.lower(), _map)
+        else:
+            # an UNALIASED from-subquery: its projection IS this scope's
+            # source (transfermarkt's dedup wrapper, holdout round 7) but
+            # only when it is the SOLE relation; a peer subquery or any
+            # other relation makes its outputs a coin flip (review
+            # of cycle 8)
+            if (
+                len([q for q in _sub_nodes if not q.alias]) == 1
+                and len(_sub_nodes) == 1
+                and not local_real_tables
+            ):
+                subquery_maps_by_alias.setdefault("", _map)
+
+    # lateral derived tables (CROSS/OUTER APPLY (SELECT ...) AS x (cols)),
+    # in document order so each apply sees the maps of earlier ones
+    # (DarkQueries' stacked CROSS APPLY chain, holdout round 8)
+    for _join in select_for_from.find_all(exp.Join):
+        if owning_select(_join) is not select_for_from:
+            continue
+        _lat = _join.this
+        if not isinstance(_lat, exp.Lateral) or not _lat.alias:
+            continue
+        try:
+            _lat_map = _lateral_subquery_map(
+                _lat,
+                dialect,
+                correlated_aliases=alias_map,
+                correlated_maps=subquery_maps_by_alias,
+                correlated_tables=all_tables,
+            )
+        except Exception:
+            continue
+        if _lat_map is None:
+            continue
+        subquery_maps_by_alias[_lat.alias] = _lat_map
+        if alias_is_foldable(_lat):
+            subquery_maps_folded.setdefault(_lat.alias.lower(), _lat_map)
+
+    def _subquery_map_for(alias: str, quoted: bool) -> dict | None:
+        # a quoted reference matches only exactly; an unquoted one folds
+        # case against foldable definitions (the cycle-6 review:
+        # "Q" and q are different postgres aliases)
+        exact = subquery_maps_by_alias.get(str(alias))
+        if exact is not None or quoted:
+            return exact
+        return subquery_maps_folded.get(str(alias).lower())
+
+    def _derived_expansion_entries(expansion_info: dict, expanded_as: str) -> list[dict] | None:
+        """An expansion whose root column is a derived table's output reads
+        through that derivation: data.nodes(...) over FROM (SELECT
+        CONVERT(XML, target_data) AS data ...) t traces to target_data,
+        never to a phantom relation (InvestigateWaits, holdout round 8)."""
+        root_table = expansion_info.get("source_table") or ""
+        root_column = expansion_info.get("source_column") or ""
+        if not root_column:
+            return None
+        sub_map = _subquery_map_for(root_table, False) if root_table else None
+        if sub_map is None:
+            if root_table or local_real_tables:
+                return None
+            owners = [
+                m
+                for m in subquery_output_maps
+                if any(
+                    k != "*"
+                    and not str(k).startswith(META_PREFIX)
+                    and str(k).lower() == root_column.lower()
+                    for k in m
+                )
+            ]
+            if len(owners) != 1 or any("*" in m for m in subquery_output_maps):
+                return None
+            sub_map = owners[0]
+
+        def _chained(chain_map: dict, chain_column: str) -> list[dict]:
+            entries = []
+            for src in _subquery_column_entries(chain_map, chain_column):
+                if not (src.get("table") or src.get("column")):
+                    continue
+                entry = dict(src)
+                entry["confidence"] = min(entry.get("confidence", 1.0), 0.5)
+                entry["trust_level"] = "moderate"
+                entry["from_array_expansion"] = True
+                entry["expansion_type"] = expansion_info.get("expansion_type")
+                entry["expanded_as"] = expanded_as
+                entries.append(entry)
+            return entries
+
+        entries = _chained(sub_map, root_column)
+        if not entries:
+            return None
+        # every argument counts: the extras chain through their own derived
+        # tables exactly like the primary (cycle-9 review, F10)
+        for extra in expansion_info.get("extra_columns", []):
+            extra_column = extra.get("source_column") or ""
+            if not extra_column:
+                continue
+            extra_table = extra.get("source_table") or ""
+            extra_map = _subquery_map_for(extra_table, False) if extra_table else None
+            if extra_map is not None:
+                entries.extend(_chained(extra_map, extra_column))
+            else:
+                entries.extend(
+                    _expansion_extra_entries(
+                        {**expansion_info, "extra_columns": [extra]}, expanded_as
+                    )
+                )
+        return entries
+
+    def _selective_expansion_entries(
+        expansion_info: dict, column: str, col: "exp.Column", field_path: str | None
+    ) -> list[dict] | None:
+        """Field-selective entries with each source chained through its own
+        derived table's map. Selectivity runs BEFORE the derived-expansion
+        fallback: a named-struct melt over subquery outputs must keep each
+        field's own sources, not every input column (cycle-12, F16)."""
+        selective = _field_selective_entries(expansion_info, column, col, field_path)
+        if selective is None:
+            return None
+        entries: list[dict] = []
+        for entry in selective:
+            table = entry.get("table") or ""
+            sub_map = _subquery_map_for(table, False) if table else None
+            if sub_map is None:
+                entries.append(entry)
+                continue
+            for src in _subquery_column_entries(sub_map, entry.get("column", "")):
+                if not (src.get("table") or src.get("column")):
+                    continue
+                chained_entry = dict(src)
+                chained_entry["confidence"] = min(chained_entry.get("confidence", 1.0), 0.5)
+                chained_entry["trust_level"] = "moderate"
+                chained_entry["from_array_expansion"] = True
+                chained_entry["expansion_type"] = expansion_info.get("expansion_type")
+                chained_entry["expanded_as"] = column
+                entries.append(chained_entry)
+        return entries
+
+    # Every name a multi-part column reference could legitimately anchor
+    # to: aliases, table names (bare and as written), expansion aliases,
+    # inline-subquery aliases. Used to tell struct access (o.customer.name)
+    # from qualified table.column (ds.orders.customer) without fabricating
+    # tables.
+    _known_relations = (
+        {str(k).lower() for k in alias_map}
+        | {str(v).lower() for v in alias_map.values()}
+        | {str(k).lower() for k in array_expansion_sources}
+        | {str(k).lower() for k in subquery_maps_by_alias}
+    )
+
+    # Select-item aliases already resolved, in list order, for dialects
+    # where a later item may reference an earlier alias (lateral column
+    # aliases). The alias is a name of the expression, not a column of
+    # the upstream relation.
+    lateral_aliases: dict[str, list[dict[str, Any]]] = {}
+    allow_lateral = (dialect or "").lower() in LATERAL_ALIAS_DIALECTS
+
+    for item_idx, select_expr in enumerate(selects_list):
+        # Qualified star over a known CTE: rel.* (except ...). Expand
+        # through the CTE column map (following star-passthrough chains)
+        # and trace each named column; collapsing to '*' would silently
+        # drop most of the model's columns.
+        if isinstance(select_expr, exp.Column) and isinstance(select_expr.this, exp.Star):
+            star = select_expr.this
+            qualifier = select_expr.table or ""
+            rel = str(alias_map.get(qualifier, qualifier))
+            except_cols = star_except_spellings(
+                star.args.get("except_") or star.args.get("except") or []
+            )
+            # rel.* over an inline subquery alias: absorb the subquery's own
+            # map. The round-5 fix landed only in the CTE mappers; at the
+            # model's own top level the alias still surfaced as a struct
+            # column of the inner table (the review of PR #28).
+            sub_absorbed = _subquery_map_for(qualifier, qualifier_is_quoted(select_expr))
+            if sub_absorbed is not None:
+                for name, srcs in sub_absorbed.items():
+                    if name == "*":
+                        _append_column_sources(result, "*", srcs)
+                        result[META_NEEDS_EXPANSION] = True
+                    elif name.lower() not in {e.lower() for e in except_cols}:
+                        result.setdefault(name, [dict(s) for s in srcs])
+                continue
+            expanded, residual = expand_star_columns([rel], cte_columns, except_cols)
+            if expanded:
+                for name, srcs in expanded.items():
+                    result.setdefault(name, srcs)
+                if residual:
+                    _append_column_sources(result, "*", residual)
+                    result[META_NEEDS_EXPANSION] = True
+                continue
+            if residual:
+                # rel.* over an unexpandable relation keeps its wildcard
+                # link WITH its exclusions, appended so a second qualified
+                # star never overwrites the first (cycle-12, F4/F5)
+                _append_column_sources(result, "*", residual)
+                result[META_NEEDS_EXPANSION] = True
+                continue
+            # a nameless qualifier falls through to the ordinary paths
+
+        is_computed = False
+
+        if isinstance(select_expr, exp.Alias):
+            col_name = select_expr.alias
+            source_expr = select_expr.this
+            is_computed = not isinstance(source_expr, exp.Column)
+        elif isinstance(select_expr, exp.Column):
+            col_name = select_expr.name
+            source_expr = select_expr
+        elif isinstance(select_expr, exp.Star):
+            if not local_real_tables and subquery_output_maps:
+                # bare star over a scope of only derived tables: the
+                # columns are the subqueries' own projections, all of
+                # them (transfermarkt's dedup wrapper, holdout round 7;
+                # generalized to peers by the cycle-8 review)
+                for m in subquery_output_maps:
+                    for name, srcs in m.items():
+                        if str(name).startswith("\x00"):
+                            continue
+                        if name == "*":
+                            _append_column_sources(result, "*", [dict(s) for s in srcs])
+                            result[META_NEEDS_EXPANSION] = True
+                        else:
+                            result.setdefault(name, [dict(s) for s in srcs])
+                continue
+            # A bare star over known CTEs expands through the CTE column
+            # map, so named CTE columns become outputs instead of
+            # vanishing behind '*' with the CTE name as a phantom source.
+            except_cols = star_except_spellings(
+                select_expr.args.get("except_") or select_expr.args.get("except") or []
+            )
+            expanded, residual = expand_star_columns(all_tables, cte_columns, except_cols)
+            if expanded:
+                for name, srcs in expanded.items():
+                    result.setdefault(name, srcs)
+                if residual:
+                    _append_column_sources(result, "*", residual)
+                    result[META_NEEDS_EXPANSION] = True
+            elif residual:
+                # a pure star chain (b -> a -> real) yields no named columns
+                # but the walk still found the real relations; rebuilding
+                # from all_tables here resurfaced the CTE itself as a
+                # phantom source (the cycle-7 review)
+                _append_column_sources(result, "*", residual)
+                result[META_NEEDS_EXPANSION] = True
+            else:
+                result["*"] = [
+                    {"table": table_name, "column": "*", "confidence": 0.5}
+                    for table_name in all_tables
+                ]
+                result[META_NEEDS_EXPANSION] = True
+            continue
+        elif _QUERY_TRANSFORM_CLS is not None and isinstance(select_expr, _QUERY_TRANSFORM_CLS):
+            # SELECT TRANSFORM(...) USING 'script': the mapping through
+            # the script is unknowable, but inputs and declared outputs
+            # are static. Honest contract: every output depends on every
+            # input, all at review_required. Never assume identity, even
+            # for 'cat' (ROW FORMAT can reshape fields).
+            schema_arg = select_expr.args.get("schema")
+            outputs = (
+                [s.name for s in schema_arg.expressions]
+                if schema_arg is not None and schema_arg.expressions
+                else ["key", "value"]  # Spark's documented default
+            )
+            script_node = select_expr.args.get("command_script")
+            script = script_node.name if script_node is not None else "?"
+            input_sources = []
+            input_names = []
+            for item in select_expr.expressions or []:
+                if isinstance(item, exp.Star):
+                    for table_name in all_tables:
+                        input_sources.append(
+                            {
+                                "table": table_name,
+                                "column": "*",
+                                "confidence": 0.3,
+                                "trust_level": "review_required",
+                                "via_script_transform": True,
+                                "script": script,
+                            }
+                        )
+                    result[META_NEEDS_EXPANSION] = True
+                    input_names.append("*")
+                    continue
+                for col in item.find_all(exp.Column):
+                    col_alias = col.table or ""
+                    table = (
+                        alias_map.get(col_alias, col_alias) if col_alias else (default_table or "")
+                    )
+                    input_sources.append(
+                        {
+                            "table": table,
+                            "column": col.name,
+                            "confidence": 0.3,
+                            "trust_level": "review_required",
+                            "via_script_transform": True,
+                            "script": script,
+                        }
+                    )
+                    input_names.append(col.name)
+            for out in outputs:
+                result[out] = [dict(s) for s in input_sources]
+            warnings.append(
+                {
+                    "warning_type": "script_transform",
+                    "message": (
+                        f"SELECT TRANSFORM pipes rows through external script '{script}'; "
+                        f"the column mapping through the script is opaque. Each of the "
+                        f"{len(outputs)} output columns is conservatively linked to all "
+                        f"{len(input_names)} input columns at review_required."
+                    ),
+                    "script": script,
+                    "output_columns": outputs,
+                    "input_columns": input_names,
+                }
+            )
+            continue
+        else:
+            col_sql = safe_sql(select_expr, dialect)
+            col_name = col_sql if col_sql is not None and len(col_sql) < 50 else None
+            source_expr = select_expr
+            is_computed = True
+
+        if not col_name:
+            continue
+        if "{" in col_name:
+            # a template placeholder is not a real output name; exposing it
+            # (or its internal dot encoding) leaked into reports and
+            # exported nodes (cycle-12, F19). The output stays, opaque.
+            col_name = f"_col_{item_idx}"
+
+        sources = []
+        seen = set()
+
+        json_columns_handled = collect_json_sources(
+            source_expr,
+            _known_relations,
+            alias_map,
+            array_expansion_sources,
+            default_table,
+            qualified_via_schema,
+            sources,
+            seen,
+        )
+
+        # qualify() rewrites a read of a table-valued alias (CROSS JOIN
+        # jsonb_array_elements_text(roles) AS role ... SELECT role) into a
+        # TableColumn node the Column sweep never sees (kingfisher, round 7)
+        _tc_cls = getattr(exp, "TableColumn", None)
+        if _tc_cls is not None:
+            for tc in source_expr.find_all(_tc_cls):
+                entry = array_expansion_sources.get(tc.name) or array_expansion_sources.get(
+                    tc.name.lower()
+                )
+                if not entry:
+                    continue
+                reads = [(entry.get("source_table", ""), entry.get("source_column", ""))] + [
+                    (x.get("source_table", ""), x.get("source_column", ""))
+                    for x in entry.get("extra_columns", [])
+                ]
+                for key in reads:
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(
+                            {
+                                "table": key[0],
+                                "column": key[1],
+                                "confidence": 0.5,
+                                "trust_level": "moderate",
+                                "inferred": False,
+                                "from_array_expansion": True,
+                                "expansion_type": entry.get("expansion_type"),
+                            }
+                        )
+
+        # TODO: this classification block near-duplicates cte_tracing.extract_select_column_sources; merging needs judgment on trust/warning metadata.
+        for col in source_expr.find_all(exp.Column):
+            if id(col) in json_columns_handled:
+                continue  # Already handled as part of JSONExtract
+            if under_subquery_where(col, source_expr):
+                # a nested subquery's WHERE selects rows there; it never
+                # feeds the produced value
+                continue
+            if in_window_ordering(col, source_expr):
+                # frames the window, does not flow into its value
+                continue
+            if in_selector_argument(col, source_expr):
+                # picks the arg_max/arg_min row, does not flow into its value
+                continue
+            alias, column, field_path, certain = resolve_column_ref(col, _known_relations)
+
+            if "{" in column or (not certain and any("{" in p for p in column_parts(col))):
+                # a {placeholder} or jinja chain in column OR qualifier
+                # position may format to anything; only relation-position
+                # braces carry a citable identity. Anchoring the qualifier
+                # as a struct root fabricated a source (cycle-12, F11).
+                spelled = ".".join(column_parts(col))
+                warnings.append(
+                    {
+                        "column": spelled,
+                        "warning_type": "template_placeholder_expression",
+                        "trust_level": "review_required",
+                        "message": f"template placeholder {spelled} in an expression; "
+                        "its lineage cannot be resolved offline",
+                    }
+                )
+                continue
+
+            source_entry: dict[str, Any] = {
+                "column": column,
+                "inferred": False,
+            }
+            if field_path:
+                source_entry["field_path"] = field_path
+
+            if not certain:
+                # multi-part reference whose leading identifier matches no
+                # known relation: read as a struct root. Anchor it to the
+                # UNNEST of its own subquery scope, the anonymous UNNEST
+                # of this select, or the sole relation when possible;
+                # never emit a confident phantom table.
+                anon = scoped_unnest_entry(
+                    col, select_for_from, alias_map, all_tables
+                ) or array_expansion_sources.get("__anonymous_unnest__")
+                if anon is not None:
+                    table = anon.get("source_table", "")
+                    confidence = 0.5
+                    trust_level = "moderate"
+                    source_entry["from_array_expansion"] = True
+                    source_entry["expansion_type"] = anon.get("expansion_type")
+                    source_entry["source_column_in_array"] = anon.get("source_column", "")
+                    if anon.get("source_column"):
+                        source_entry["expanded_as"] = column
+                        source_entry["column"] = anon["source_column"]
+                elif default_table:
+                    table = default_table
+                    confidence = 0.5
+                    trust_level = "moderate"
+                    source_entry["inferred"] = True
+                    source_entry["struct_guess"] = True
+                else:
+                    table = ""
+                    confidence = 0.4
+                    trust_level = "review_required"
+                    source_entry["inferred"] = True
+                    source_entry["unresolved_reference"] = True
+            elif alias:
+                if alias in array_expansion_sources:
+                    expansion_info = array_expansion_sources[alias]
+                    selective = _selective_expansion_entries(
+                        expansion_info, column, col, field_path
+                    )
+                    if selective is not None:
+                        for entry in selective:
+                            key = (entry.get("table", ""), entry.get("column", ""))
+                            if key not in seen:
+                                seen.add(key)
+                                sources.append(entry)
+                        continue
+                    chained = _derived_expansion_entries(expansion_info, column)
+                    if chained:
+                        for entry in chained:
+                            chain_key = (entry.get("table", ""), entry.get("column", ""))
+                            if chain_key not in seen:
+                                seen.add(chain_key)
+                                sources.append(entry)
+                        continue
+                    # Trace to the actual source table/column
+                    table = expansion_info.get("source_table", alias)
+                    source_column = expansion_info.get("source_column", column)
+                    # Lower confidence: we know the source but can't track row expansion
+                    confidence = 0.5
+                    trust_level = "moderate"
+                    source_entry["from_array_expansion"] = True
+                    source_entry["expansion_type"] = expansion_info.get("expansion_type")
+                    source_entry["source_column_in_array"] = source_column
+                    if source_column:
+                        # the upstream column IS the array; the alias's
+                        # output name does not exist on the source table
+                        source_entry["expanded_as"] = column
+                        source_entry["column"] = source_column
+                    if expansion_info.get("json_path"):
+                        source_entry["json_path"] = expansion_info["json_path"]
+                    for extra_entry in _expansion_extra_entries(expansion_info, column):
+                        extra_key = (extra_entry["table"], extra_entry["column"])
+                        if extra_key not in seen:
+                            seen.add(extra_key)
+                            sources.append(extra_entry)
+                    warnings.append(
+                        {
+                            "column": column,
+                            "warning_type": "array_expansion",
+                            "message": f"Column '{column}' comes from array expansion ({expansion_info.get('expansion_type')}). "
+                            f"Source array: {table}.{source_column}",
+                            "source_table": table,
+                            "source_column": source_column,
+                            "expansion_type": expansion_info.get("expansion_type"),
+                        }
+                    )
+                elif (
+                    sub_absorbed := _subquery_map_for(alias, qualifier_is_quoted(col))
+                ) is not None:
+                    # the qualifier is an inline-subquery alias: the
+                    # column's sources are the subquery's own, never a
+                    # relation named after the alias (pensjon under
+                    # qualify(), the gap named in PR #28)
+                    for src in _subquery_column_entries(sub_absorbed, column):
+                        key = (src.get("table", ""), src.get("column", ""))
+                        if key not in seen and (key[0] or key[1]):
+                            seen.add(key)
+                            sources.append(dict(src))
+                    continue
+                else:
+                    # Column is explicitly qualified - high confidence
+                    table = alias_map.get(alias, alias)
+                    confidence = 1.0
+                    trust_level = "verified" if qualified_via_schema else "high_confidence"
+            elif column in array_expansion_sources:
+                # Unqualified column that matches an UNNEST output column
+                expansion_info = array_expansion_sources[column]
+                selective = _selective_expansion_entries(expansion_info, column, col, field_path)
+                if selective is not None:
+                    for entry in selective:
+                        key = (entry.get("table", ""), entry.get("column", ""))
+                        if key not in seen:
+                            seen.add(key)
+                            sources.append(entry)
+                    continue
+                chained = _derived_expansion_entries(expansion_info, column)
+                if chained:
+                    for entry in chained:
+                        chain_key = (entry.get("table", ""), entry.get("column", ""))
+                        if chain_key not in seen:
+                            seen.add(chain_key)
+                            sources.append(entry)
+                    continue
+                table = expansion_info.get("source_table", "")
+                source_column = expansion_info.get("source_column", column)
+                confidence = 0.5
+                trust_level = "moderate"
+                source_entry["from_array_expansion"] = True
+                source_entry["expansion_type"] = expansion_info.get("expansion_type")
+                source_entry["source_column_in_array"] = source_column
+                source_entry["is_unnest_column"] = expansion_info.get("is_unnest_column", False)
+                if source_column:
+                    source_entry["expanded_as"] = column
+                    source_entry["column"] = source_column
+                for extra_entry in _expansion_extra_entries(expansion_info, column):
+                    extra_key = (extra_entry["table"], extra_entry["column"])
+                    if extra_key not in seen:
+                        seen.add(extra_key)
+                        sources.append(extra_entry)
+                warnings.append(
+                    {
+                        "column": column,
+                        "warning_type": "array_expansion",
+                        "message": f"Column '{column}' comes from array expansion ({expansion_info.get('expansion_type')}). "
+                        f"Source array: {table}.{source_column}",
+                        "source_table": table,
+                        "source_column": source_column,
+                        "expansion_type": expansion_info.get("expansion_type"),
+                    }
+                )
+            elif (
+                allow_lateral
+                and column.lower() in lateral_aliases
+                and not (
+                    warehouse_columns
+                    and resolve_column_table(column, all_tables, warehouse_columns)[1]
+                )
+            ):
+                # lateral column alias: the name belongs to an earlier
+                # select item, not to the upstream relation. When the
+                # schema proves a real column of that name exists, the
+                # table wins; otherwise resolve to the definition.
+                for src in lateral_aliases[column.lower()]:
+                    entry = dict(src)
+                    entry["via_lateral_alias"] = True
+                    key = (entry.get("table", ""), entry.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(entry)
+                continue
+            elif (anon_map := subquery_maps_by_alias.get("")) is not None and any(
+                not str(k).startswith("\x00") and k != "*" and str(k).lower() == column.lower()
+                for k in anon_map
+            ):
+                # unqualified read of a column the unaliased from-subquery
+                # projects (transfermarkt's `where n = 1` wrapper,
+                # holdout round 7)
+                for src in _subquery_column_entries(anon_map, column):
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(dict(src))
+                continue
+            elif (
+                not local_real_tables
+                and all("*" not in m for m in subquery_output_maps)
+                and len(
+                    owner_maps := [
+                        m
+                        for m in subquery_output_maps
+                        if any(
+                            not str(k).startswith("\x00")
+                            and k != "*"
+                            and str(k).lower() == column.lower()
+                            for k in m
+                        )
+                    ]
+                )
+                == 1
+            ):
+                # this select reads only derived tables, and exactly one of
+                # them projects the column: anchoring to a table found
+                # INSIDE the subquery skips the derivation (kingfisher's
+                # role expansion read through id_role, holdout round 7)
+                for src in _subquery_column_entries(owner_maps[0], column):
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen and (key[0] or key[1]):
+                        seen.add(key)
+                        sources.append(dict(src))
+                continue
+            elif (nested := nested_scope_sole_table(col, select_for_from)) is not None:
+                # the column lives in a subselect with its own FROM; the
+                # outer sole table or elimination would anchor it a scope
+                # too high (the review of PR #28: scalar subqueries in
+                # UPDATE SET expressions)
+                table = nested
+                if nested:
+                    confidence = 0.8
+                    trust_level = "high_confidence"
+                else:
+                    confidence = 0.4
+                    trust_level = "review_required"
+                    source_entry["unresolved_reference"] = True
+                source_entry["inferred"] = True
+            elif default_table:
+                # Single table in FROM - inferred but reliable
+                table = default_table
+                confidence = 0.8
+                trust_level = "high_confidence"
+                source_entry["inferred"] = True
+            elif (
+                subquery_output_maps
+                and len(local_real_tables) == 1
+                and all(
+                    column.lower() not in {str(k).lower() for k in m} for m in subquery_output_maps
+                )
+                and all("*" not in m for m in subquery_output_maps)
+            ):
+                # every derived FROM item enumerates its outputs and none
+                # carries this name, so the one real table must. A '*' entry
+                # means a subquery's outputs are NOT enumerated: the column
+                # may well live behind the star, and eliminating on it
+                # fabricated an anchor (the review of PR #28)
+                table = local_real_tables[0]
+                confidence = 0.6
+                trust_level = "moderate"
+                source_entry["inferred"] = True
+            elif (
+                len(all_tables) > 1
+                and not (
+                    warehouse_columns
+                    and resolve_column_table(column, all_tables, warehouse_columns)[1]
+                )
+                and (_sys_owner := system_catalog_owner(column, all_tables, dialect))
+            ):
+                # every relation in scope is a documented sys object and
+                # exactly one owns the column (InvestigateWaits, round 8);
+                # a supplied schema that resolves the column outranks the
+                # built-in catalog (cycle-9 review, F1)
+                table = _sys_owner
+                confidence = 0.8
+                trust_level = "high_confidence"
+                source_entry["inferred"] = True
+            elif len(all_tables) > 1 and (_owner := _sole_enumerated_cte_owner(column)):
+                # SQL resolves an unqualified name to the one relation that
+                # can own it. Provable only when every relation in scope is
+                # a CTE with fully enumerated outputs and exactly one
+                # projects the name (dbt_salesforce's manager_id coalesce
+                # across three joined aggregate CTEs, holdout round 7).
+                table = _owner
+                confidence = 0.8
+                trust_level = "high_confidence"
+                source_entry["inferred"] = True
+            elif len(all_tables) > 1 and warehouse_columns:
+                # Multiple tables and warehouse schema available - try to resolve
+                resolved_table, was_resolved = resolve_column_table(
+                    column, all_tables, warehouse_columns
+                )
+                if was_resolved and resolved_table:
+                    # Successfully resolved ambiguity via warehouse schema
+                    table = resolved_table
+                    confidence = 1.0
+                    trust_level = "verified"
+                    source_entry["qualified_via_schema"] = True
+                    source_entry["inferred"] = True
+                elif _lookup_anchor(column):
+                    table = _lookup_anchor(column)
+                    confidence = 0.6
+                    trust_level = "moderate"
+                    source_entry["inferred"] = True
+                else:
+                    # Could not resolve - mark as ambiguous
+                    table = ""
+                    confidence = 0.5
+                    trust_level = "review_required"
+                    source_entry["ambiguous_tables"] = all_tables
+                    source_entry["ambiguity_unresolved"] = True
+                    warnings.append(
+                        {
+                            "column": column,
+                            "warning_type": "ambiguous_column",
+                            "message": f"Column '{column}' could come from any of: {', '.join(all_tables)}",
+                            "candidate_tables": all_tables,
+                        }
+                    )
+            elif len(all_tables) > 1 and _lookup_anchor(column):
+                table = _lookup_anchor(column)
+                confidence = 0.6
+                trust_level = "moderate"
+                source_entry["inferred"] = True
+            elif len(all_tables) > 1:
+                # Multiple tables but no warehouse schema - ambiguous, needs resolution
+                table = ""
+                confidence = 0.5
+                trust_level = "moderate"
+                source_entry["ambiguous_tables"] = all_tables
+                source_entry["needs_warehouse_connection"] = True
+                warnings.append(
+                    {
+                        "column": column,
+                        "warning_type": "ambiguous_column_no_schema",
+                        "message": f"Column '{column}' is ambiguous. Connect a warehouse to enable automatic resolution.",
+                        "candidate_tables": all_tables,
+                        "hint": "Connect a warehouse to enable automatic column resolution.",
+                    }
+                )
+            else:
+                # No FROM clause or empty tables list
+                table = ""
+                confidence = 0.5
+                trust_level = "moderate"
+
+            source_entry["table"] = table
+            source_entry["confidence"] = confidence
+            source_entry["trust_level"] = trust_level
+
+            key = (table, column)
+            if key not in seen and (table or column):
+                seen.add(key)
+                sources.append(source_entry)
+
+        if not sources and is_computed:
+            sources = [
+                {
+                    "table": "",
+                    "column": col_name,
+                    "computed": True,
+                    "confidence": 0.3,
+                    "trust_level": "moderate",
+                }
+            ]
+
+        # Trace sources through CTEs
+        if sources and cte_columns:
+            traced_sources = []
+            seen_traced = set()
+            for src in sources:
+                if src.get("computed"):
+                    traced_sources.append(src)
+                elif src.get("table") and src["table"].lower() in cte_columns:
+                    traced = trace_through_ctes(
+                        src["column"],
+                        src["table"],
+                        cte_columns,
+                    )
+                    for t in traced:
+                        key = (t.get("table", ""), t.get("column", ""))
+                        if key not in seen_traced:
+                            seen_traced.add(key)
+                            # Degrade confidence if source CTE collides with real table
+                            if cte_collision_names and src["table"].lower() in cte_collision_names:
+                                t["confidence"] = min(t.get("confidence", 1.0), 0.5)
+                                t["trust_level"] = "review_required"
+                                t["cte_collision"] = True
+                            # Preserve trust metadata from traced sources
+                            elif src.get("qualified_via_schema"):
+                                t["qualified_via_schema"] = True
+                                t["trust_level"] = "verified"
+                            traced_sources.append(t)
+                else:
+                    key = (src.get("table", ""), src.get("column", ""))
+                    if key not in seen_traced:
+                        seen_traced.add(key)
+                        traced_sources.append(src)
+            sources = traced_sources if traced_sources else sources
+
+        if isinstance(select_expr, exp.Alias) and sources:
+            lateral_aliases.setdefault(col_name.lower(), sources)
+
+        result[col_name] = sources