ripple-sql 0.1.0__py3-none-any.whl

ripple/ci.py

@@ -0,0 +1,422 @@
+"""CI mode: what does this change break?
+
+`ripple ci --base origin/main` compares every changed model against the base
+ref, works out which output columns changed lineage (added, removed, or
+re-derived), and prints the union blast radius as a PR-ready markdown comment.
+
+Runs entirely in the repo's own CI with the repo's own checkout. No server,
+no token, nothing leaves the runner.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from ripple.answer import (
+    DASHBOARD_PREFIXES,
+    breaks_lines,
+    ci_answers,
+    count_hits,
+    noun_for,
+    plain_text,
+    row_impact_kind,
+    where_phrase,
+)
+from ripple.ci_signature import CTE_EXPRESSIONS_KEY, PREDICATES_KEY, _lineage_signature
+from ripple.graph import LineageGraph, UnknownTarget
+from ripple.project import load_project
+
+
+def _git(args: list[str], cwd: Path) -> str:
+    result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()}")
+    return result.stdout
+
+
+def changed_sql_files(base: str, cwd: Path) -> tuple[list[str], Path]:
+    """Changed .sql paths (relative to the git toplevel) and the toplevel itself.
+
+    Union of committed changes since the merge-base (the PR diff) and any
+    uncommitted worktree changes, so local runs work too.
+    """
+    toplevel = Path(_git(["rev-parse", "--show-toplevel"], cwd).strip())
+    committed = _git(["diff", "--name-only", f"{base}...HEAD"], cwd).splitlines()
+    worktree = _git(["diff", "--name-only", base], cwd).splitlines()
+    seen: list[str] = []
+    for f in [*committed, *worktree]:
+        if f.endswith(".sql") and f not in seen:
+            seen.append(f)
+    return seen, toplevel
+
+
+def analyze(base: str, path: str = ".", dialect: str | None = None) -> dict:
+    project = load_project(path, dialect=dialect)
+    return analyze_graph(base, project, LineageGraph.build(project))
+
+
+def analyze_graph(base: str, project, graph: LineageGraph) -> dict:
+    """The report for one loaded project and its graph, so a caller that
+    also writes the answer page builds the graph once."""
+    root = project.root
+    changed, toplevel = changed_sql_files(base, root)
+    models_by_abspath = {(root / m.path).resolve(): m for m in project.models if m.path}
+    findings = []
+    base_sql: dict[str, str] = {}
+    pending_removed: list[tuple[dict, str, str]] = []
+    for file_path in changed:
+        model = models_by_abspath.get((toplevel / file_path).resolve())
+        try:
+            old_sql = _git(["show", f"{base}:{file_path}"], root)
+        except RuntimeError:
+            if model is None:
+                continue  # not a model we can say anything about
+            old_sql = ""  # new model
+        if model is None:
+            findings.append(_missing_model_finding(file_path, toplevel, graph))
+            continue
+        if old_sql:
+            base_sql[model.name] = old_sql
+        finding = _model_finding(graph, project.dialect, model, old_sql)
+        findings.append(finding)
+        pending_removed.extend(
+            (entry, model.name, entry["column"])
+            for entry in finding["columns"]
+            if entry["state"] == "removed"
+        )
+
+    if pending_removed:
+        _fill_removed_blasts(project, base_sql, pending_removed)
+    return _report(base, findings)
+
+
+def _missing_model_finding(file_path: str, toplevel: Path, graph: LineageGraph) -> dict:
+    """The changed file is absent from the current project: deleted, or it
+    still exists but no longer reads as a model (garbage SQL)."""
+    name = Path(file_path).stem
+    still_referenced = sorted({e.dst_model for e in graph.edges if e.src_model == name})
+    gone = not (toplevel / file_path).exists()
+    finding = {
+        "model": name,
+        "status": "deleted" if gone else "unparseable",
+        "columns": [],
+        "still_referenced_by": still_referenced,
+    }
+    if not gone:
+        finding["error"] = "changed file no longer reads as a model"
+    return finding
+
+
+def _model_finding(graph: LineageGraph, dialect: str, model, old_sql: str) -> dict:
+    try:
+        old_sig = _lineage_signature(old_sql, dialect) if old_sql else {}
+        new_sig = _lineage_signature(model.sql, dialect)
+    except Exception as e:
+        return {"model": model.name, "status": "unparseable", "error": str(e), "columns": []}
+    touched = sorted(
+        column for column in {*old_sig, *new_sig} if old_sig.get(column) != new_sig.get(column)
+    )
+    columns = []
+    for column in touched:
+        entry = _column_finding(graph, model.name, column, old_sig, new_sig, bool(old_sql))
+        if entry is not None:
+            columns.append(entry)
+    return {
+        "model": model.name,
+        "status": "new" if not old_sql else "changed",
+        "columns": columns,
+    }
+
+
+def _column_finding(
+    graph: LineageGraph,
+    model_name: str,
+    column: str,
+    old_sig: dict,
+    new_sig: dict,
+    has_base: bool,
+) -> dict | None:
+    state = (
+        "added" if column not in old_sig else "removed" if column not in new_sig else "rederived"
+    )
+    blast = None
+    if column == PREDICATES_KEY:
+        if not has_base:
+            return None  # a new model has no predicate diff to report
+        # a row-filter change affects every output of this model
+        state = "rows_refiltered"
+        blast = graph.breaks_all(model_name)
+    elif column == CTE_EXPRESSIONS_KEY:
+        if not has_base:
+            return None
+        # a formula changed inside a CTE: outputs derive differently
+        state = "reworked"
+        blast = graph.breaks_all(model_name)
+    elif state == "removed":
+        # the column and its edges are gone from the current graph, so asking
+        # it says "nothing downstream" about the one change most likely to
+        # break consumers; the real blast lives in the base state and is
+        # filled in by _fill_removed_blasts
+        blast = None
+    elif state != "added":
+        try:
+            blast = graph.breaks(model_name, column)
+        except UnknownTarget:
+            blast = None
+    return {"column": column, "state": state, **_blast_fields(blast)}
+
+
+def _blast_fields(blast: dict | None) -> dict:
+    if not blast:
+        return {
+            "impacted_columns": 0,
+            "impacted_models": 0,
+            "dashboard_numbers": 0,
+            "review_required": 0,
+            "by_model": {},
+            "row_level_models": [],
+            "row_level_impact": [],
+            "truncated": False,
+        }
+    columns, models, dashboards = count_hits(blast["by_model"])
+    rows = blast.get("row_level_impact", [])
+    fields = {
+        "impacted_columns": columns,
+        "impacted_models": models,
+        "dashboard_numbers": dashboards,
+        "review_required": blast["review_required"],
+        "by_model": blast["by_model"],
+        "row_level_models": sorted({r["model"] for r in rows}),
+        "row_level_impact": rows,
+        "truncated": not blast.get("complete", True),
+    }
+    if blast.get("truncated_at_depth"):
+        fields["truncated_at_depth"] = blast["truncated_at_depth"]
+    return fields
+
+
+def _fill_removed_blasts(project, base_sql: dict[str, str], pending: list) -> None:
+    """One extra graph build, only when a change removes columns."""
+    base_models = [
+        dataclasses.replace(m, sql=base_sql[m.name]) if m.name in base_sql else m
+        for m in project.models
+    ]
+    base_graph = LineageGraph.build(dataclasses.replace(project, models=base_models))
+    for entry, model_name, column in pending:
+        try:
+            blast = base_graph.breaks(model_name, column)
+        except UnknownTarget:
+            continue  # nothing referenced it at base either
+        entry.update(_blast_fields(blast))
+
+
+def _report(base: str, findings: list[dict]) -> dict:
+    report = {
+        "base": base,
+        "changed_models": len(findings),
+        "impacted_columns": sum(c["impacted_columns"] for f in findings for c in f["columns"]),
+        "dashboard_numbers": sum(c["dashboard_numbers"] for f in findings for c in f["columns"]),
+        "review_required": sum(c["review_required"] for f in findings for c in f["columns"]),
+        "row_level_models": len(
+            {m for f in findings for c in f["columns"] for m in c.get("row_level_models", [])}
+        ),
+        "deleted_still_referenced": [
+            f["model"]
+            for f in findings
+            if f["status"] == "deleted" and f.get("still_referenced_by")
+        ],
+        "unparseable": [f["model"] for f in findings if f["status"] == "unparseable"],
+        "findings": findings,
+    }
+    report["rebuild"] = rebuild_tokens(report)
+    return report
+
+
+def write_change_page(report: dict, project, graph: LineageGraph, path: Path) -> Path | None:
+    """One self-contained page for the whole change: a view per changed
+    column that reaches anything, the rebuild list, and the graph embedded
+    so a reader can ask the next question. None when nothing changed."""
+    from ripple import answer_page
+
+    answers = ci_answers(report, noun_for(project.mode))
+    if not answers:
+        return None
+    views = [{"answer": a, "text": plain_text(breaks_lines(a, full=True))} for a in answers]
+    info = answer_page.project_info(project, Path(project.root))
+    change = {
+        "base": report["base"],
+        "changed_models": report["changed_models"],
+        "impacted_columns": report["impacted_columns"],
+        "dashboard_numbers": report["dashboard_numbers"],
+        "review_required": report["review_required"],
+        "rebuild": report["rebuild"],
+    }
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        answer_page.render(
+            views[0]["answer"],
+            views[0]["text"],
+            info,
+            answer_page.graph_export(graph),
+            views=views,
+            change=change,
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def rebuild_tokens(report: dict) -> list[str]:
+    """The models to rebuild after this change, as dbt --select tokens.
+
+    dbt's own state:modified+ rebuilds every descendant of any changed file;
+    column lineage knows which descendants actually derive differently, so
+    the enumerated set is usually much smaller. The one rule: over-select,
+    never under-select. Enumeration is only allowed when the analysis saw
+    everything; an unparseable diff, a depth-capped blast, a changed row
+    filter, or a deleted model's readers get dbt's descendant closure
+    (`model+`) instead. A missed rebuild ships stale data; a spare rebuild
+    only costs compute.
+    """
+    exact: set[str] = set()
+    widened: set[str] = set()
+    for finding in report["findings"]:
+        model = finding["model"]
+        if finding["status"] == "deleted":
+            # its readers will fail or change; their descendants follow
+            widened.update(finding.get("still_referenced_by", []))
+            continue
+        if finding["status"] == "unparseable":
+            widened.add(model)
+            widened.update(finding.get("still_referenced_by", []))
+            continue
+        exact.add(model)
+        for column in finding["columns"]:
+            if column.get("truncated"):
+                widened.add(model)
+            # metric:/semantic:/exposure: readers are dashboard numbers, not
+            # models dbt can build
+            exact.update(
+                name
+                for name in column.get("by_model", {})
+                if not name.startswith(DASHBOARD_PREFIXES)
+            )
+            # a changed row set changes every output of the filtering model
+            widened.update(column.get("row_level_models", []))
+    exact -= widened
+    return sorted(exact) + sorted(f"{name}+" for name in widened)
+
+
+_ROW_VERBS = {"filter": "filters rows in", "join": "joins rows in", "window": "frames a window in"}
+
+
+def _row_level_phrase(col: dict) -> str:
+    """The readers whose every row this column decides, as "filters rows in `paid`"."""
+    rows = col.get("row_level_impact") or [{"model": m} for m in col.get("row_level_models", [])]
+    by_verb: dict[str, list[str]] = {}
+    for row in rows:
+        by_verb.setdefault(_ROW_VERBS[row_impact_kind(row)], []).append(f"`{row['model']}`")
+    return "; ".join(f"{verb} {', '.join(names)}" for verb, names in by_verb.items())
+
+
+def to_markdown(report: dict) -> str:
+    lines = ["### Ripple"]
+    if not report["findings"]:
+        lines.append("No model changes detected.")
+        return "\n".join(lines)
+    hit = f"**{report['impacted_columns']}** downstream columns"
+    if report.get("dashboard_numbers"):
+        hit += f" and **{report['dashboard_numbers']}** dashboard numbers"
+    if report.get("row_level_models"):
+        hit += f" and every row of **{report['row_level_models']}** model(s)"
+    lines.append(f"{hit} affected across {report['changed_models']} changed model(s).")
+    if report["review_required"]:
+        lines.append(
+            f"{report['review_required']} could not be fully verified. Check those by hand."
+        )
+    lines.append("")
+    for finding in report["findings"]:
+        if finding["status"] == "unparseable":
+            lines.append(f"- `{finding['model']}`: could not parse ({finding['error'][:120]})")
+            continue
+        if finding["status"] == "deleted":
+            refs = finding.get("still_referenced_by", [])
+            note = f", still referenced by {', '.join(f'`{r}`' for r in refs[:5])}" if refs else ""
+            lines.append(f"- `{finding['model']}` deleted{note}")
+            continue
+        for col in finding["columns"]:
+            if col["state"] in ("rows_refiltered", "reworked"):
+                what = (
+                    "row filter changed"
+                    if col["state"] == "rows_refiltered"
+                    else "internal logic changed"
+                )
+                head = f"- `{finding['model']}` {what}"
+                rows = _row_level_phrase(col)
+                if col["impacted_columns"] or col.get("dashboard_numbers"):
+                    where = where_phrase(col["impacted_models"], col.get("dashboard_numbers", 0))
+                    head += f": {col['impacted_columns']} columns in {where} derive differently"
+                    head += f"; {rows}" if rows else ""
+                elif rows:
+                    head += f": {rows}"
+                lines.append(head)
+                continue
+            head = f"- `{finding['model']}.{col['column']}` ({col['state']})"
+            rows = _row_level_phrase(col)
+            if col["impacted_columns"] or col.get("dashboard_numbers"):
+                top = sorted(col["by_model"].items(), key=lambda kv: -len(kv[1]))[:3]
+                names = ", ".join(f"`{name}`" for name, _ in top)
+                more = len(col["by_model"]) - len(top)
+                suffix = f" and {more} more" if more > 0 else ""
+                where = where_phrase(col["impacted_models"], col.get("dashboard_numbers", 0))
+                head += f": {col['impacted_columns']} columns in {where} ({names}{suffix})"
+                if col["review_required"]:
+                    head += f", {col['review_required']} need review"
+                head += f"; {rows}" if rows else ""
+            elif rows:
+                head += f": {rows}"
+            elif col["state"] != "added":
+                head += ": nothing downstream"
+            lines.append(head)
+    if report.get("rebuild"):
+        lines.append("")
+        lines.append(f"Rebuild: `{' '.join(report['rebuild'])}`")
+    return "\n".join(lines)
+
+
+def run(
+    base: str,
+    path: str,
+    dialect: str | None,
+    as_json: bool,
+    fail_on: str,
+    select: str | None = None,
+    html: str | None = None,
+) -> int:
+    project = load_project(path, dialect=dialect)
+    graph = LineageGraph.build(project)
+    report = analyze_graph(base, project, graph)
+    if html:
+        written = write_change_page(report, project, graph, Path(html))
+        if written:
+            print(f"answer page: {written}", file=sys.stderr)
+    if select == "dbt":
+        # tokens only, pipeable: dbt build --select "$(ripple ci --select dbt)"
+        if report["rebuild"]:
+            print(" ".join(report["rebuild"]))
+        return 0
+    print(json.dumps(report, indent=2) if as_json else to_markdown(report))
+    if fail_on == "breaks" and (
+        report["impacted_columns"]
+        or report["dashboard_numbers"]
+        or report["row_level_models"]
+        or report["deleted_still_referenced"]
+    ):
+        return 1
+    if fail_on == "review" and (report["review_required"] or report["unparseable"]):
+        return 1
+    return 0