ripple-sql 0.1.0__py3-none-any.whl

ripple/engine/tsql_xml.py

@@ -0,0 +1,97 @@
+"""T-SQL XML method calls, normalized for lineage.
+
+col.value('xpath', 'type'), col.query(...), col.exist(...) and
+col.nodes('xpath') parse in sqlglot's tsql as a Dot chain ending in an
+Anonymous call whose base is bare Identifiers, so the column sweep sees
+no Column at all and a .nodes() rowset never reaches the expansion
+machinery (sqlserver_kit DarkQueries and InvestigateWaits, holdout
+round 8). Each such Dot is rewritten into an ordinary call with the base
+column as the first argument: value(col, 'xpath', 'type'). The produced
+value derives from the shredded column exactly like any function
+argument, and .nodes() in a lateral becomes a set-returning function the
+scope machinery already expands.
+"""
+
+from __future__ import annotations
+
+import re
+
+from sqlglot import exp
+
+XML_METHODS = frozenset({"value", "query", "exist", "modify", "nodes"})
+
+# sql:column("t.id") exposes a relational column inside an XQuery literal.
+# Conservative on purpose: double-quoted, plain dotted identifiers only
+# (brackets stripped); anything fancier stays an opaque literal.
+_SQL_COLUMN_REF = re.compile(r'sql:column\s*\(\s*"([^"]+)"\s*\)', re.IGNORECASE)
+
+
+def rewrite_xml_method_calls(parsed: exp.Expression) -> exp.Expression:
+    """Rewrite every XML-method Dot in the tree, outermost first."""
+    for dot in list(parsed.find_all(exp.Dot)):
+        method = dot.expression
+        if not isinstance(method, exp.Anonymous):
+            continue
+        if method.name.lower() not in XML_METHODS:
+            continue
+        base = _base_column(dot.this)
+        if base is None:
+            continue
+        args = [e.copy() for e in method.expressions or []]
+        dot.replace(
+            exp.Anonymous(
+                this=method.name.lower(),
+                expressions=[base, *args, *_embedded_column_reads(args)],
+            )
+        )
+    return parsed
+
+
+def _embedded_column_reads(args: list[exp.Expression]) -> list[exp.Column]:
+    """Columns the XQuery string arguments read through sql:column(), each
+    emitted as an ordinary argument of the rewritten call (cycle-9
+    review, F7)."""
+    columns: list[exp.Column] = []
+    for arg in args:
+        if not isinstance(arg, exp.Literal) or not arg.is_string:
+            continue
+        for ref in _SQL_COLUMN_REF.findall(arg.this):
+            parts = [p.strip('[]" ') for p in ref.split(".")]
+            parts = [p for p in parts if p]
+            if not parts or len(parts) > 4:
+                continue
+            slots = ("this", "table", "db", "catalog")
+            columns.append(
+                exp.Column(
+                    **{
+                        slot: exp.Identifier(this=part, quoted=False)
+                        for slot, part in zip(slots, reversed(parts), strict=False)
+                    }
+                )
+            )
+    return columns
+
+
+def _base_column(node: exp.Expression) -> exp.Column | None:
+    """The method's receiver as a Column, or None when it is not a plain
+    (optionally qualified) column reference."""
+    if isinstance(node, exp.Column):
+        return node.copy()
+    parts = _identifier_parts(node)
+    if not parts or len(parts) > 4:
+        return None
+    slots = ("this", "table", "db", "catalog")
+    return exp.Column(
+        **{slot: ident.copy() for slot, ident in zip(slots, reversed(parts), strict=False)}
+    )
+
+
+def _identifier_parts(node: exp.Expression) -> list[exp.Identifier] | None:
+    if isinstance(node, exp.Identifier):
+        return [node]
+    if isinstance(node, exp.Dot):
+        left = _identifier_parts(node.this)
+        right = _identifier_parts(node.expression)
+        if left is not None and right is not None:
+            return left + right
+    return None

ripple/engine/types.py

@@ -0,0 +1,167 @@
+"""Column lineage type definitions.
+
+This module defines the core types used across the column lineage system.
+Centralizing types here enables clean imports and avoids circular dependencies.
+
+Type Categories:
+- Trust levels: Confidence indicators for lineage correctness
+- Data structures: Dataclasses for column dependencies and lineage results
+- Type aliases: Common type patterns for warehouse schemas
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+# Type aliases for clarity
+TrustLevel = Literal["verified", "high_confidence", "moderate", "review_required"]
+"""Trust level indicating lineage confidence.
+
+- verified: 1.0 confidence - warehouse-qualified or exact match
+- high_confidence: 0.8-0.99 - strong evidence
+- moderate: 0.5-0.79 - heuristic match
+- review_required: <0.5 - ambiguity or collision detected
+"""
+
+WarehouseColumns = dict[str, dict[str, str]]
+"""Warehouse schema format: {table_name: {column_name: data_type}}"""
+
+
+# Size limits for DoS protection
+MAX_WAREHOUSE_TABLES = 10000  # arrives from outside via ingest_schema
+# real_tables is derived from the caller's own repo, after Ripple has already
+# read every one of those files off disk, so a low cap guards nothing and
+# turns a big monorepo into silence: Dune's spellbook needs 11,962 and used to
+# fail all 7,439 models. Kept as a runaway guard, set far above any real repo.
+MAX_REAL_TABLES = 1_000_000
+MAX_SQL_LENGTH = 5_000_000  # 5MB max SQL size
+
+# Internal metadata keys carried inside extraction result dicts. The NUL
+# prefix cannot appear in a SQL identifier, so a real output column named
+# _name (or even _warnings) can never collide with engine metadata.
+META_PREFIX = "\x00ripple:"
+META_WARNINGS = META_PREFIX + "warnings"
+META_EXTRACTION_ERROR = META_PREFIX + "extraction_error"
+META_EXTRACTION_ERROR_TYPE = META_PREFIX + "extraction_error_type"
+META_QUALIFIED_VIA_SCHEMA = META_PREFIX + "qualified_via_schema"
+META_FALLBACK_DIALECT = META_PREFIX + "fallback_dialect"
+META_NEEDS_EXPANSION = META_PREFIX + "needs_expansion"
+META_UNION_SOURCES = META_PREFIX + "union_sources"
+META_UNION_VALIDATION = META_PREFIX + "union_validation"
+META_CTE_COLLISIONS = META_PREFIX + "cte_collisions"
+# Per-CTE metadata carried inside a CTE's own column map (cte_columns[name]):
+# the relations its FROM scope reads, whether the name is defined more than
+# once in the statement (nested WITH shadowing), and whether it is a
+# recursive CTE. Consumers iterating column names must skip META_PREFIX keys.
+META_CTE_RELATIONS = META_PREFIX + "cte_relations"
+META_CTE_SHADOWED = META_PREFIX + "cte_shadowed"
+META_CTE_RECURSIVE = META_PREFIX + "cte_recursive"
+
+# Dialects where a select item may reference an alias defined earlier in the
+# same select list (lateral column aliases). In these dialects a bare
+# reference matching an earlier alias resolves to that alias's expression,
+# not to a column of the upstream relation.
+LATERAL_ALIAS_DIALECTS = frozenset(
+    {"snowflake", "redshift", "databricks", "spark", "spark2", "duckdb", "clickhouse", "teradata"}
+)
+
+
+@dataclass
+class FilterDependency:
+    """A column used in WHERE, HAVING, or QUALIFY clauses.
+
+    These are C_ref columns (referenced but not contributing to output)
+    following the LineageX paper terminology.
+
+    Attributes:
+        column: Column name used in the filter
+        table: Source table (None if unqualified)
+        predicate_type: Type of filter clause (where, having, qualify, on,
+            window for a PARTITION BY / ORDER BY key)
+        operator: Comparison operator if detectable (=, IN, BETWEEN, etc.)
+    """
+
+    column: str
+    table: str | None
+    predicate_type: Literal["where", "having", "qualify", "on", "window"]
+    operator: str | None = None
+
+
+@dataclass
+class JoinKeyPair:
+    """A pair of columns in a JOIN condition.
+
+    Tracks the relationship between columns used in JOIN ON clauses,
+    which is critical for understanding data cardinality and integrity.
+
+    Attributes:
+        left_column: Column from the left side of the join
+        left_table: Table/alias for the left column
+        right_column: Column from the right side of the join
+        right_table: Table/alias for the right column
+        join_type: Type of join (inner, left, right, full, cross)
+    """
+
+    left_column: str
+    left_table: str
+    right_column: str
+    right_table: str
+    join_type: Literal["inner", "left", "right", "full", "cross"]
+
+
+@dataclass
+class ColumnDependencies:
+    """C_ref: Columns that affect row selection but not output values.
+
+    This complements C_con (contributing columns) to provide complete
+    lineage following the LineageX model.
+
+    Attributes:
+        filter_columns: Columns used in WHERE/HAVING/QUALIFY
+        join_keys: Column pairs used in JOIN ON conditions
+        window_keys: Columns in a window's PARTITION BY / ORDER BY
+        warnings: Any issues encountered during extraction
+    """
+
+    filter_columns: list[FilterDependency] = field(default_factory=list)
+    join_keys: list[JoinKeyPair] = field(default_factory=list)
+    window_keys: list[FilterDependency] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+@dataclass
+class UnifiedLineageResult:
+    """Complete lineage following the LineageX C_con + C_ref model.
+
+    Carries both:
+    - C_con: Columns that contribute to output values (SELECT sources)
+    - C_ref: Columns referenced in WHERE/JOIN (affect row selection)
+
+    Attributes:
+        contributing: Dict mapping output columns to their sources
+        filter_columns: Columns used in WHERE/HAVING/QUALIFY
+        join_keys: Column pairs used in JOIN conditions
+        window_keys: Columns in a window's PARTITION BY / ORDER BY
+        warnings: Any issues encountered during extraction
+        qualified_via_schema: True if warehouse schema was used
+    """
+
+    contributing: dict[str, list[dict[str, Any]]]
+    filter_columns: list[FilterDependency] = field(default_factory=list)
+    join_keys: list[JoinKeyPair] = field(default_factory=list)
+    window_keys: list[FilterDependency] = field(default_factory=list)
+    warnings: list[dict[str, Any]] = field(default_factory=list)
+    qualified_via_schema: bool = False
+
+
+@dataclass
+class CTEColumnMapping:
+    """Column mapping for a single CTE.
+
+    Tracks which columns a CTE produces and their sources,
+    enabling recursive tracing through nested CTEs.
+    """
+
+    cte_name: str
+    columns: dict[str, list[dict[str, Any]]]
+    has_star: bool = False
+    is_union: bool = False