ripple-sql 0.1.0__py3-none-any.whl

ripple/__init__.py

@@ -0,0 +1,31 @@
+"""Ripple: offline column-level SQL lineage. See what breaks before you merge.
+
+Library use starts here:
+
+    from ripple import LineageGraph, load_project
+
+    graph = LineageGraph.build(load_project("path/to/repo"))
+    graph.breaks("orders", "revenue")
+
+Everything else is internal and may move between releases.
+"""
+
+from ripple.graph import Edge, LineageGraph, UnknownTarget
+from ripple.project import Model, Project, load_project
+
+try:
+    from importlib.metadata import version
+
+    __version__ = version("ripple-sql")
+except Exception:  # not installed (e.g. running from a checkout)
+    __version__ = "0.0.0.dev"
+
+__all__ = [
+    "Edge",
+    "LineageGraph",
+    "Model",
+    "Project",
+    "UnknownTarget",
+    "__version__",
+    "load_project",
+]

ripple/answer.py

@@ -0,0 +1,473 @@
+"""One answer, one shape, every door.
+
+A `breaks` or `trace` result becomes nodes and links with a hop depth and a
+trust label, plus the summary counts every surface prints. The CLI text,
+the MCP payload, the pull-request comment, and the answer page all read
+this shape, so they cannot disagree about what a change reaches.
+
+Pure functions over the dicts the graph returns. Nothing here touches the
+graph, files, or output formats.
+"""
+
+from __future__ import annotations
+
+DASHBOARD_PREFIXES = ("metric:", "semantic:", "exposure:")
+ASKED = "asked"  # the column the question was about; depth 0 in either direction
+PREVIEW_MODELS = 8
+PREVIEW_COLUMNS = 5
+
+
+KIND_WORDS = {"semantic": "semantic model", "metric": "metric", "exposure": "exposure"}
+
+
+def node_kind(model: str) -> str:
+    return "dashboard" if model.startswith(DASHBOARD_PREFIXES) else "model"
+
+
+def noun_for(mode: str) -> str:
+    """'model' is dbt's word; a plain SQL folder deserves a plain one."""
+    return "table" if mode == "sql-dir" else "model"
+
+
+def count_hits(by_model: dict) -> tuple[int, int, int]:
+    """(model columns, models, dashboard numbers) from a graph's by_model.
+
+    The graph counts every hit as a column. A dashboard number is not one:
+    "19 columns in 4 models and 7 dashboard numbers" read as 26 things when
+    it was 12 columns plus the 7 numbers themselves."""
+    columns = models = dashboards = 0
+    for model, hits in by_model.items():
+        if node_kind(model) == "dashboard":
+            dashboards += 1
+        else:
+            models += 1
+            columns += len(hits)
+    return columns, models, dashboards
+
+
+def where_phrase(models: int, dashboards: int, noun: str = "model") -> str:
+    parts = []
+    if models:
+        parts.append(_plural(models, noun))
+    if dashboards:
+        parts.append(_plural(dashboards, "dashboard number"))
+    return " and ".join(parts) if parts else f"0 {noun}s"
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}" + ("" if n == 1 else "s")
+
+
+def has_hits(answer: dict) -> bool:
+    """True when the change reaches anything: a column, a dashboard number, or a row filter."""
+    s = answer["summary"]
+    return bool(s["columns"] or s["dashboard_numbers"] or answer["row_level"])
+
+
+def _node(model: str, column: str, depth: int, trust: str) -> dict:
+    return {
+        "id": f"{model}.{column}",
+        "model": model,
+        "column": column,
+        "kind": node_kind(model),
+        "depth": depth,
+        "trust": trust,
+    }
+
+
+def _worse(a: str, b: str) -> str:
+    # graph.TRUST_ORDER reversed; a heuristic match is rated "moderate"
+    order = ("verified", "high_confidence", "moderate", "review_required")
+    return a if order.index(a) >= order.index(b) else b
+
+
+def _merge(nodes: list[dict]) -> list[dict]:
+    """One node per column: the shortest path's depth, the worst path's trust."""
+    merged: dict[str, dict] = {}
+    for node in nodes:
+        seen = merged.get(node["id"])
+        if seen is None:
+            merged[node["id"]] = dict(node)
+        else:
+            seen["depth"] = min(seen["depth"], node["depth"])
+            seen["trust"] = _worse(seen["trust"], node["trust"])
+    return list(merged.values())
+
+
+def _close_edges(nodes: list[dict], edges: list[dict], kind: str) -> list[dict]:
+    """A link may name a column the walk never listed: a wildcard, or a
+    column reached through one. Give it a node so no link dangles.
+
+    Links point away from the asked column on breaks and toward it on trace,
+    so which end sits one step nearer depends on the kind."""
+    ids = {n["id"] for n in nodes}
+    by_id = {n["id"]: n for n in nodes}
+    for edge in edges:
+        for end, other in (("src", "dst"), ("dst", "src")):
+            if edge[end] in ids:
+                continue
+            model, _, column = edge[end].rpartition(".")
+            neighbour = by_id.get(edge[other])
+            nearer = (end == "src") == (kind == "breaks")
+            depth = max(0, (neighbour["depth"] if neighbour else 1) + (-1 if nearer else 1))
+            node = _node(model, column, depth, "review_required")
+            node["kind"] = "unknown"
+            nodes.append(node)
+            ids.add(node["id"])
+            by_id[node["id"]] = node
+    return nodes
+
+
+def breaks_answer(result: dict, noun: str = "model") -> dict:
+    """Everything downstream of one column, as nodes and links."""
+    src = result["source"]
+    asked = _node(src["model"], src["column"], 0, "verified")
+    asked["kind"] = ASKED
+    nodes = [asked]
+    edges: list[dict] = []
+    for hits in result["by_model"].values():
+        for hit in hits:
+            nodes.append(_node(hit["model"], hit["column"], hit["depth"], hit["trust"]))
+            edge = {
+                "src": hit["via"],
+                "dst": nodes[-1]["id"],
+                "trust": hit["edge_trust"],
+                "path_trust": hit["trust"],
+            }
+            if hit.get("reason"):
+                edge["reason"] = hit["reason"]
+            edges.append(edge)
+    nodes = _close_edges(_merge(nodes), edges, "breaks")
+    rows = [
+        {"model": r["model"], "uses": list(r.get("uses", [])), "trust": r.get("trust")}
+        for r in result.get("row_level_impact", [])
+    ]
+    columns, models, dashboards = count_hits(result["by_model"])
+    answer = {
+        "kind": "breaks",
+        "target": asked["id"],
+        "noun": noun,
+        "summary": {
+            "columns": columns,
+            "models": models,
+            "dashboard_numbers": dashboards,
+            "review": result["review_required"],
+            "row_level_models": len(rows),
+            "deepest_hops": max(n["depth"] for n in nodes),
+            "complete": result.get("complete", True),
+        },
+        "nodes": nodes,
+        "edges": edges,
+        "row_level": rows,
+    }
+    for key in ("truncated_at_depth", "also_matches"):
+        if result.get(key):
+            answer[key] = result[key]
+    return answer
+
+
+def trace_answer(result: dict, noun: str = "model") -> dict:
+    """Where one column comes from, as nodes and links pointing downstream."""
+    tgt = result["target"]
+    asked = _node(tgt["model"], tgt["column"], 0, "verified")
+    asked["kind"] = ASKED
+    nodes = [asked]
+    edges: list[dict] = []
+    for hop in result["upstream"]:
+        nodes.append(_node(hop["model"], hop["column"], hop["depth"], hop["trust"]))
+        edge = {
+            "src": nodes[-1]["id"],
+            "dst": hop["feeds"],
+            "trust": hop["edge_trust"],
+            "path_trust": hop["trust"],
+        }
+        if hop.get("reason"):
+            edge["reason"] = hop["reason"]
+        edges.append(edge)
+    nodes = _close_edges(_merge(nodes), edges, "trace")
+    upstream = [n for n in nodes if n["depth"] > 0]
+    models = {n["model"] for n in upstream}
+    answer = {
+        "kind": "trace",
+        "target": asked["id"],
+        "noun": noun,
+        "summary": {
+            "columns": len(upstream),
+            "models": sum(1 for m in models if node_kind(m) == "model"),
+            "dashboard_numbers": sum(1 for m in models if node_kind(m) == "dashboard"),
+            "review": sum(1 for n in upstream if n["trust"] == "review_required"),
+            "row_level_models": 0,
+            "deepest_hops": max(n["depth"] for n in nodes),
+            "complete": result.get("complete", True),
+        },
+        "nodes": nodes,
+        "edges": edges,
+        "row_level": [],
+    }
+    if result.get("truncated_at_depth"):
+        answer["truncated_at_depth"] = result["truncated_at_depth"]
+    return answer
+
+
+def ci_answers(report: dict, noun: str = "model") -> list[dict]:
+    """One breaks answer per changed column that reaches anything."""
+    answers = []
+    for finding in report["findings"]:
+        for col in finding.get("columns", []):
+            if not col.get("by_model") and not col.get("row_level_models"):
+                continue
+            rows = col.get("row_level_impact") or [
+                {"model": m} for m in col.get("row_level_models", [])
+            ]
+            result = {
+                "source": {"model": finding["model"], "column": col["column"]},
+                "impacted_columns": col["impacted_columns"],
+                "impacted_models": len(col["by_model"]),
+                "review_required": col["review_required"],
+                "by_model": col["by_model"],
+                "row_level_impact": rows,
+                "complete": not col.get("truncated"),
+            }
+            if col.get("truncated_at_depth"):
+                result["truncated_at_depth"] = col["truncated_at_depth"]
+            answer = breaks_answer(result, noun)
+            answer["change"] = col["state"]
+            answers.append(answer)
+    return answers
+
+
+def headline_parts(answer: dict) -> tuple[str, str, str]:
+    """(count, where, target) so a surface can style them apart.
+
+    A metric is not a table; lumping dashboard numbers into the model count
+    made "6 models" both wrong for plain-SQL users and vague for dbt users."""
+    s = answer["summary"]
+    where = where_phrase(s["models"], s["dashboard_numbers"], answer["noun"])
+    return str(s["columns"]), where, answer["target"]
+
+
+def headline(answer: dict) -> str:
+    count, where, target = headline_parts(answer)
+    if answer["kind"] == "trace":
+        return f"{target} comes from {count} columns in {where}"
+    return f"{count} columns in {where} affected by {target}"
+
+
+def hit_hardest(answer: dict, limit: int = 3) -> list[tuple[str, int]]:
+    """The models with the most touched columns, worst first."""
+    counts: dict[str, int] = {}
+    for node in answer["nodes"]:
+        if node["depth"] > 0 and node["kind"] == "model":
+            counts[node["model"]] = counts.get(node["model"], 0) + 1
+    return sorted(counts.items(), key=lambda kv: -kv[1])[:limit]
+
+
+def _groups(answer: dict) -> list[tuple[str, list[dict]]]:
+    """(model, nodes) in first-seen order; the asked column and stand-ins skipped."""
+    groups: dict[str, list[dict]] = {}
+    for node in answer["nodes"]:
+        if node["depth"] == 0 or node["kind"] == "unknown":
+            continue
+        groups.setdefault(node["model"], []).append(node)
+    return list(groups.items())
+
+
+def row_impact_kind(row: dict) -> str:
+    """How a reader uses the changed column on rows: filter, join, or window.
+    A window key frames the window; saying it filters rows would be wrong."""
+    kinds = {u.rsplit(" (", 1)[-1].rstrip(")") for u in row.get("uses") or [] if " (" in u}
+    if kinds == {"window"}:
+        return "window"
+    if kinds == {"join"}:
+        return "join"
+    return "filter"
+
+
+def _row_line(row: dict) -> list[tuple[str, str]]:
+    uses = row.get("uses") or []
+    used = uses[0].rsplit(" (", 1) if uses else []
+    how = used[1].rstrip(")") if len(used) > 1 else "filter"
+    note = (
+        "partitions or orders a window on it"
+        if row_impact_kind(row) == "window"
+        else f"filters rows on it ({how})"
+    )
+    segments = [(f"  {row['model']} ", "plain"), (f"· {note}", "dim")]
+    if row.get("trust") == "review_required":
+        segments.append((" review", "warn"))
+    return segments
+
+
+def _source_label(node: dict, via: dict[str, str]) -> str:
+    """The arrow names the source model when the column keeps its name, and
+    the source column only when the name changes. Fifteen of nineteen lines
+    on jaffle-shop said the same name twice."""
+    src = via.get(node["id"], "?")
+    src_model, _, src_column = src.rpartition(".")
+    return src_model if src_model and src_column == node["column"] else src
+
+
+def _model_block(model: str, nodes: list[dict], via: dict[str, str], full: bool) -> list:
+    """One line per (source, trust) group. A wide model repeats the same
+    source for every column; without full, each line names the first few
+    columns and one dim line counts the rest."""
+    depth = min(n["depth"] for n in nodes)
+    lines = [[(f"  {model} ", "plain"), (f"· {_plural(depth, 'step')}", "dim")]]
+    groups: dict[tuple[str, bool], list[str]] = {}
+    for node in nodes:
+        key = (_source_label(node, via), node["trust"] == "review_required")
+        groups.setdefault(key, []).append(node["column"])
+    items = list(groups.items())
+    shown = items if full else items[:PREVIEW_COLUMNS]
+    hidden = sum(len(columns) for _, columns in items[len(shown) :])
+    for (label, review), columns in shown:
+        named = columns if full else columns[:PREVIEW_COLUMNS]
+        hidden += len(columns) - len(named)
+        segments = [(f"    {', '.join(named)} ", "plain"), (f"← {label}", "dim")]
+        if review:
+            segments.append((" needs review", "warn"))
+        lines.append(segments)
+    if hidden:
+        lines.append([(f"    and {hidden} more (--full)", "dim")])
+    return lines
+
+
+def _dashboard_lines(groups: list[tuple[str, list[dict]]], full: bool) -> list:
+    """Dashboard numbers in their own section, the kind as a word after the
+    name, so an analyst can skip it and an engineer knows which yml to open."""
+    entries: dict[tuple[str, str], dict] = {}
+    for model, nodes in groups:
+        kind, _, rest = model.partition(":")
+        if kind == "semantic":
+            entity, _, _ = rest.rpartition(".")
+            key = (entity or rest, kind)
+            columns = [n["column"] for n in nodes]
+        else:
+            key = (kind, kind)
+            columns = [rest]
+        entry = entries.setdefault(key, {"columns": [], "review": False})
+        entry["columns"].extend(columns)
+        entry["review"] |= any(n["trust"] == "review_required" for n in nodes)
+    lines: list = [[], [("Dashboard numbers", "plain")]]
+    items = list(entries.items())
+    shown = items if full else items[:PREVIEW_MODELS]
+    for (entity, kind), entry in shown:
+        names = ", ".join(entry["columns"])
+        text = (
+            f"  {kind}s: {names}" if entity == kind else f"  {entity} ({KIND_WORDS[kind]}): {names}"
+        )
+        segments = [(text, "plain")]
+        if entry["review"]:
+            segments.append((" needs review", "warn"))
+        lines.append(segments)
+    if len(items) > len(shown):
+        lines.append([(f"  ...and {len(items) - len(shown)} more (--full)", "dim")])
+    return lines
+
+
+def breaks_lines(answer: dict, full: bool = False) -> list[list[tuple[str, str]]]:
+    """The text every door prints, as lines of (text, role) segments.
+
+    Roles: plain, bold, dim, warn. A terminal styles them; a page or an
+    agent joins the text. A hub column can feed hundreds of models, so the
+    headline is the answer and the blocks are detail: counts and names
+    first, everything on full.
+    """
+    s, noun, target = answer["summary"], answer["noun"], answer["target"]
+    if not has_hits(answer):
+        return [[(f"Nothing downstream reads {target}.", "plain")]]
+    count, where, _ = headline_parts(answer)
+    lines = [
+        [
+            (count, "bold"),
+            (" columns in ", "plain"),
+            (where, "bold"),
+            (f" affected by {target}", "plain"),
+        ]
+    ]
+    if s["review"]:
+        lines.append([(f"{s['review']} of them need review (lineage uncertain)", "warn")])
+    groups = _groups(answer)
+    models = [(m, nodes) for m, nodes in groups if node_kind(m) == "model"]
+    dashboards = [(m, nodes) for m, nodes in groups if node_kind(m) == "dashboard"]
+    if len(models) > 1:
+        named = ", ".join(f"{m} ({n})" for m, n in hit_hardest(answer))
+        lines.append([(f"Hit hardest: {named}", "dim")])
+    lines.append([])
+    via: dict[str, str] = {}
+    for edge in answer["edges"]:
+        via.setdefault(edge["dst"], edge["src"])
+    shown = models if full else models[:PREVIEW_MODELS]
+    for model, nodes in shown:
+        lines.extend(_model_block(model, nodes, via, full))
+    if len(models) > len(shown):
+        lines.append([(f"  ...and {len(models) - len(shown)} more {noun}s (--full)", "dim")])
+    if dashboards:
+        lines.extend(_dashboard_lines(dashboards, full))
+    rows = answer["row_level"]
+    shown_rows = rows if full else rows[:PREVIEW_MODELS]
+    lines.extend(_row_line(row) for row in shown_rows)
+    if len(rows) > len(shown_rows):
+        lines.append([(f"  ...and {len(rows) - len(shown_rows)} more (--full)", "dim")])
+    if not s["complete"]:
+        # breaks_all reports an incomplete walk without saying at which depth
+        depth = answer.get("truncated_at_depth")
+        stopped = f"at depth {depth}" if depth else "at the depth cap"
+        lines.append([(f"  stopped {stopped}; the full radius may be larger", "warn")])
+    lines.append([])
+    return lines
+
+
+def trace_lines(answer: dict) -> list[list[tuple[str, str]]]:
+    """Where the asked column comes from, one line per hop, indented by depth."""
+    target = answer["target"]
+    by_id = {n["id"]: n for n in answer["nodes"]}
+    hops = [e for e in answer["edges"] if by_id.get(e["src"], {}).get("kind") != "unknown"]
+    if not hops:
+        return [[(f"No upstream lineage for {target}.", "plain")]]
+    lines = [[(f"{target} comes from:", "plain")]]
+    for edge in hops:
+        node = by_id[edge["src"]]
+        segments = [("  " * node["depth"] + edge["src"], "plain")]
+        if edge.get("path_trust", node["trust"]) == "review_required":
+            segments.append((" review", "warn"))
+        segments.append((" ", "plain"))
+        segments.append(("→ " + edge["dst"], "dim"))
+        lines.append(segments)
+    if answer.get("truncated_at_depth"):
+        lines.append(
+            [
+                (
+                    f"  stopped at depth {answer['truncated_at_depth']}; "
+                    "the trail may go further back (--depth to follow it)",
+                    "warn",
+                )
+            ]
+        )
+    return lines
+
+
+def plain_text(lines: list[list[tuple[str, str]]]) -> str:
+    """The lines with no styling, one string, for a page or an agent."""
+    return "\n".join("".join(text for text, _ in segments) for segments in lines)
+
+
+def attach(result: dict, kind: str, noun: str) -> dict:
+    """A graph result plus its contract (`answer`) and preview text (`text`),
+    the top-level counts aligned to the contract. Every other key stays.
+
+    The graph counts a dashboard number as a column; the contract does not,
+    so a payload carrying both used to disagree with its own text."""
+    if kind == "breaks":
+        answer = breaks_answer(result, noun)
+        text = plain_text(breaks_lines(answer))
+    else:
+        answer = trace_answer(result, noun)
+        text = plain_text(trace_lines(answer))
+    s = answer["summary"]
+    out = {**result, "answer": answer, "text": text}
+    out["impacted_columns"] = s["columns"]
+    out["dashboard_numbers"] = s["dashboard_numbers"]
+    if "impacted_models" in result:
+        out["impacted_models"] = s["models"]
+    return out