ripple-sql 0.1.0__py3-none-any.whl

ripple/ci_signature.py

@@ -0,0 +1,374 @@
+"""What a changed model's SQL means, as a comparable signature.
+
+`ripple ci` diffs the base and head versions of every changed model. The
+signature per output column is (upstream columns, top-level expression, the
+inner expressions its lineage passes through), plus one entry for the
+statement's row-deciding closure and one for the inner expressions no output
+column reaches. Pure functions over SQL text; nothing here touches git or the
+graph.
+
+The row-deciding closure is the one rule behind the predicates entry: every
+clause that decides which rows survive or how they group (WHERE, HAVING,
+QUALIFY, JOIN, GROUP BY, DISTINCT, a row cap with the ORDER BY it ranks by)
+plus the whole chain of expressions behind every column those clauses read,
+through CTEs, derived tables and set operations, down to the physical tables.
+A change anywhere in it widens to the whole model. When sqlglot cannot resolve
+the statement the raw clause text is the floor: the failure mode is a spare
+rebuild, never a missed one.
+"""
+
+from __future__ import annotations
+
+import sqlglot
+from sqlglot import exp
+from sqlglot.lineage import lineage
+from sqlglot.optimizer.qualify import qualify
+from sqlglot.optimizer.scope import Scope, ScopeType, build_scope
+
+from ripple.engine.safe_gen import safe_sql
+
+PREDICATES_KEY = "(predicates)"
+CTE_EXPRESSIONS_KEY = "(cte expressions)"
+
+_KEY_HOLDERS = (exp.Group, exp.Tuple, exp.GroupingSets, exp.Cube, exp.Rollup, exp.Distinct)
+_CORRELATING = (ScopeType.SUBQUERY, ScopeType.UDTF, ScopeType.UNION)
+# the spellings of a row cap: TOP and FETCH FIRST land in `limit` on most
+# dialects, but not all versions of every one
+_CAPS = ("limit", "offset", "fetch", "top")
+# clauses that reshape the row set without filtering a column
+_ROW_SHAPERS = tuple(
+    node
+    for name in ("TableSample", "Connect", "Pivot", "Unpivot", "MatchRecognize")
+    if (node := getattr(exp, name, None)) is not None
+)
+
+
+def _lineage_signature(sql: str, dialect: str) -> dict[str, tuple]:
+    """column -> (upstream columns, defining expression, inner chain), plus
+    the row-deciding closure entry and the leftover inner expressions entry.
+    The expression matters: amount/100.0 to amount changes every downstream
+    number. The closure matters more: WHERE status='pending' to 'paid'
+    changes every downstream row while touching no column definition."""
+    from ripple.engine import extract_lineage_complete
+    from ripple.engine.jinja import convert_jinja_to_sql
+    from ripple.engine.preprocess import prepare_sql_for_parse
+
+    result = extract_lineage_complete(sql, dialect=dialect, clean_jinja_func=convert_jinja_to_sql)
+    # signature reparses share the engine's pre-parse pipeline, or a braced
+    # model diffs as an empty signature (cycle-10 review, F8 class)
+    cleaned, _ = prepare_sql_for_parse(sql, dialect, convert_jinja_to_sql)
+    expressions = _column_expressions(cleaned, dialect)
+    traced = _trace_statement(cleaned, dialect)
+    chains, leftover, closure = traced or ({}, _cte_expression_signature(cleaned, dialect), [])
+    signature = {
+        column: (
+            frozenset(f"{s.get('table', '?')}.{s.get('column', '?')}" for s in sources),
+            expressions.get(column.lower(), ""),
+            chains.get(column.lower(), ""),
+        )
+        for column, sources in result.contributing.items()
+    }
+    signature[PREDICATES_KEY] = (frozenset(), _predicate_signature(cleaned, dialect, closure))
+    signature[CTE_EXPRESSIONS_KEY] = (frozenset(), leftover)
+    return signature
+
+
+def _trace_statement(cleaned: str, dialect: str) -> tuple[dict[str, str], str, list[str]] | None:
+    """Per output column, the inner expressions its lineage passes through;
+    the leftover signature of the inner expressions no output column passes
+    through; the row-deciding closure texts. Per column because one hash over
+    every CTE item widened any one-column edit in `with source, renamed
+    select *` to the whole model (jaffle-shop stg_orders). None when sqlglot
+    cannot resolve the statement; the caller falls back to the single hash."""
+    try:
+        parsed = sqlglot.parse_one(cleaned, dialect=dialect)
+        qualified = qualify(parsed, dialect=dialect, validate_qualify_columns=False)
+        root = build_scope(qualified)
+        if root is None:
+            return None
+        closure = _Closure(root, dialect)
+        closure.run(qualified)
+        traced = (
+            _lineage_chains(qualified, root, dialect) if isinstance(qualified, exp.Select) else None
+        )
+    except Exception:
+        return None
+    if traced is None:
+        return {}, _cte_expression_signature(cleaned, dialect), closure.texts
+    chains, reached, items = traced
+    # items on the closure stay in the leftover so the edit also reports as
+    # reworked; two tests pin that label for a CTE flag read by a WHERE
+    leftover = sorted(
+        item.sql(dialect=dialect).lower() for key, item in items.items() if key not in reached
+    )
+    return chains, "|".join(leftover), closure.texts
+
+
+def _lineage_chains(
+    qualified: exp.Select, root: Scope, dialect: str
+) -> tuple[dict[str, str], set[int], dict[int, exp.Expression]] | None:
+    """Per output column, the inner items its lineage passes through; the ids
+    of every expression any output column reaches; the inner items by id.
+    None for an output sqlglot cannot name."""
+    items = {
+        id(item): item
+        for select in qualified.find_all(exp.Select)
+        if select is not qualified
+        for item in select.expressions
+    }
+    chains: dict[str, str] = {}
+    reached: set[int] = set()
+    for item in qualified.expressions:
+        name = item.alias_or_name
+        if not name or name == "*":
+            return None
+        stack = list(lineage(name, qualified, scope=root, dialect=dialect).downstream)
+        parts = []
+        while stack:
+            node = stack.pop()
+            key = id(node.expression)
+            reached.add(key)
+            if key in items:
+                parts.append(node.expression.sql(dialect=dialect).lower())
+            stack.extend(node.downstream)
+        chains[name.lower()] = "|".join(sorted(parts))
+    return chains, reached, items
+
+
+class _Closure:
+    """One walk over the row-deciding clauses of a qualified statement:
+    `texts` is the clause texts plus every expression behind the columns they
+    read, `absorbed` the ids of everything already in `texts`."""
+
+    def __init__(self, root: Scope, dialect: str) -> None:
+        self.root = root
+        self.dialect = dialect
+        self.owners = {id(scope.expression): scope for scope in root.traverse()}
+        self.texts: list[str] = []
+        self.absorbed: set[int] = set()
+        self.asked: set[tuple[int, str, str]] = set()
+
+    def run(self, qualified: exp.Expression) -> None:
+        for clause in _row_deciding_clauses(qualified):
+            self.absorb(clause)
+            for item in _ordinal_items(clause):
+                self.absorb(item)
+
+    def scope_of(self, node: exp.Expression) -> Scope:
+        return self.owners.get(id(node.find_ancestor(exp.Select)), self.root)
+
+    def ask(self, scope: Scope, table: str, name: str) -> None:
+        """Absorb what defines `table.name` seen from `scope`; a star passes
+        the name through to its own sources."""
+        key = (id(scope), table.lower(), name.lower())
+        if key in self.asked:
+            return
+        self.asked.add(key)
+        for item, item_scope in _defining_items(scope, table.lower(), name.lower()):
+            if item.is_star:
+                self.ask(item_scope, item.table if isinstance(item, exp.Column) else "", name)
+            else:
+                self.absorb(item)
+
+    def absorb(self, item: exp.Expression) -> None:
+        if id(item) in self.absorbed:
+            return
+        self.absorbed.add(id(item))
+        self.texts.append(item.sql(dialect=self.dialect).lower())
+        for column in item.find_all(exp.Column):
+            self.ask(self.scope_of(column), column.table, column.name)
+
+
+def _row_deciding_clauses(statement: exp.Expression):
+    """Every clause in the statement that decides which rows survive or how
+    they group."""
+    yield from statement.find_all(exp.Where, exp.Having, exp.Qualify, exp.Group)
+    yield from statement.find_all(*_ROW_SHAPERS)
+    for join in statement.find_all(exp.Join):
+        yield from _join_clauses(join)
+    for query in statement.find_all(exp.Select, exp.SetOperation):
+        yield from _row_set_clauses(query)
+
+
+def _join_clauses(join: exp.Join):
+    """A join decides rows by its kind, its source and its condition. What an
+    inline subquery source projects decides none of the three."""
+    kinds = " ".join(join.text(part) for part in ("method", "side", "kind"))
+    pivots = " ".join(_rendered(pivot) for pivot in join.args.get("pivots") or [])
+    shape = f"join {kinds} {_source_shape(join.this)} {pivots}"
+    yield exp.Literal.string(" ".join(shape.lower().split()))
+    for part in ("on", "match_condition"):
+        if (condition := join.args.get(part)) is not None:
+            yield condition
+    yield from join.args.get("using") or []
+
+
+def _source_shape(source: exp.Expression | None) -> str:
+    """Where a join source draws its rows from: a table by name, a derived
+    table by its own FROM, WHERE and GROUP BY. Anything else renders whole."""
+    if source is None:
+        return ""
+    inner = source.this if isinstance(source, exp.Subquery) else None
+    if not isinstance(inner, exp.Select):
+        return _rendered(source)
+    # "from_" is the sqlglot 30 arg name, "from" the older one
+    args = (inner.args.get(name) for name in ("from", "from_", "where", "group"))
+    return " ".join(_rendered(arg) for arg in args if arg is not None)
+
+
+def _row_set_clauses(query: exp.Expression):
+    """A row cap or a DISTINCT ON makes the ORDER BY decide which rows
+    survive, not just the order they come in. A plain DISTINCT node carries
+    no reference to the select list, so every select item decides which rows
+    are duplicates."""
+    caps = [cap for name in _CAPS if (cap := query.args.get(name)) is not None]
+    yield from caps
+    if isinstance(query, exp.SetOperation):
+        # union keeps one row per duplicate, union all keeps both, and except
+        # is not intersect: the operator itself decides the rows
+        mode = "all" if query.args.get("distinct") is False else "distinct"
+        yield exp.Literal.string(f"{query.key} {mode}".lower())
+        return
+    distinct = query.args.get("distinct") if isinstance(query, exp.Select) else None
+    picks_a_row = bool(caps) or (distinct is not None and distinct.args.get("on") is not None)
+    if picks_a_row and (order := query.args.get("order")) is not None:
+        yield order
+    if distinct is None:
+        return
+    yield distinct
+    if distinct.args.get("on") is None:
+        # duplicates are compared by value; the output's name is not part of it
+        yield from (item.unalias() for item in query.expressions)
+
+
+def _rendered(node: exp.Expression) -> str:
+    return safe_sql(node, None) or ""
+
+
+def _ordinal_items(clause: exp.Expression) -> list[exp.Expression]:
+    """Select items behind `group by 1` style keys, for the dialects where
+    qualify leaves the ordinal in place."""
+    select = clause.find_ancestor(exp.Select)
+    if select is None or not isinstance(clause, (exp.Group, exp.Distinct)):
+        return []
+    found = []
+    for key in clause.find_all(exp.Literal):
+        if key.is_int and isinstance(key.parent, _KEY_HOLDERS):
+            index = int(key.this) - 1
+            if 0 <= index < len(select.expressions):
+                found.append(select.expressions[index])
+    return found
+
+
+def _defining_items(scope: Scope, table: str, name: str) -> list[tuple[exp.Expression, Scope]]:
+    """Items defining `table.name` seen from `scope`: the matching source's
+    item, every same-named item when unqualified, the select's own alias
+    (HAVING, QUALIFY, GROUP BY). Subqueries walk outward for correlated
+    references. Physical tables end the chain."""
+    while scope is not None:
+        found = [
+            pair
+            for alias, source in scope.sources.items()
+            if isinstance(source, Scope) and (not table or alias.lower() == table)
+            for pair in _named_items(source, name)
+        ]
+        if not table and isinstance(scope.expression, exp.Select):
+            found += [
+                (item, scope)
+                for item in scope.expression.expressions
+                if item.alias_or_name.lower() == name
+            ]
+        aliases = {alias.lower() for alias in scope.sources}
+        if found or table in aliases or scope.scope_type not in _CORRELATING:
+            return found
+        scope = scope.parent
+    return []
+
+
+def _named_items(source: Scope, name: str) -> list[tuple[exp.Expression, Scope]]:
+    """Items putting `name` on a source's output, stars included; both
+    branches at that position for a set operation; a table function whole."""
+    expression = source.expression
+    if isinstance(expression, exp.SetOperation):
+        names = _output_names(source)
+        return _items_at(source, names.index(name)) if name in names else []
+    if not isinstance(expression, exp.Select):
+        return [(expression, source)]
+    return [
+        (item, source)
+        for item in expression.expressions
+        if item.is_star or item.alias_or_name.lower() == name
+    ]
+
+
+def _output_names(scope: Scope) -> list[str]:
+    if isinstance(scope.expression, exp.SetOperation):
+        return _output_names(scope.union_scopes[0]) if scope.union_scopes else []
+    if isinstance(scope.expression, exp.Select):
+        return [item.alias_or_name.lower() for item in scope.expression.expressions]
+    return []
+
+
+def _items_at(scope: Scope, position: int) -> list[tuple[exp.Expression, Scope]]:
+    if isinstance(scope.expression, exp.SetOperation):
+        return [pair for branch in scope.union_scopes for pair in _items_at(branch, position)]
+    items = scope.expression.expressions if isinstance(scope.expression, exp.Select) else []
+    return [(items[position], scope)] if position < len(items) else []
+
+
+def _cte_expression_signature(sql: str, dialect: str) -> str:
+    """Normalized SELECT expressions from every CTE and subquery: the
+    whole-model fallback when per-column tracing fails. A formula change
+    inside a CTE changes every downstream number while leaving the top-level
+    expressions untouched, so it needs its own signature."""
+    try:
+        parsed = sqlglot.parse_one(sql, dialect=dialect)
+    except Exception:
+        return ""
+    # the top-level select has its own per-column signature
+    top = parsed if isinstance(parsed, exp.Select) else parsed.find(exp.Select)
+    parts = [
+        rendered.lower()
+        for select in parsed.find_all(exp.Select)
+        if select is not top
+        for item in select.expressions
+        if (rendered := safe_sql(item, dialect)) is not None
+    ]
+    return "|".join(sorted(parts))
+
+
+def _predicate_signature(sql: str, dialect: str, closure: list[str]) -> str:
+    """The row-deciding closure plus the raw text of every row-deciding
+    clause. The raw text is the floor that survives a failed qualification."""
+    parts = list(closure)
+    try:
+        parsed = sqlglot.parse_one(sql, dialect=dialect)
+    except Exception:
+        return "|".join(sorted(parts))
+    for node in _row_deciding_clauses(parsed):
+        rendered = safe_sql(node, dialect)
+        if rendered is not None:
+            parts.append(rendered.lower())
+    return "|".join(sorted(parts))
+
+
+def _column_expressions(sql: str, dialect: str) -> dict[str, str]:
+    """Normalized top-level SELECT expression per output column."""
+    try:
+        parsed = sqlglot.parse_one(sql, dialect=dialect)
+    except Exception:
+        return {}
+    select = parsed
+    while select is not None and not isinstance(select, exp.Select):
+        select = select.this if hasattr(select, "this") else None
+        if not isinstance(select, exp.Expression):
+            return {}
+    if select is None:
+        return {}
+    out = {}
+    for item in select.expressions:
+        name = item.alias_or_name
+        rendered = safe_sql(item, dialect)
+        if name and name != "*" and rendered is not None:
+            out[name.lower()] = rendered.lower()
+    return out