redcliff 0.1.0__py3-none-any.whl

redcliff/__init__.py

@@ -0,0 +1,43 @@
+"""Redcliff: open-source visual pipelines with lineage built in."""
+
+from redcliff.columns import infer_columns
+from redcliff.compile import CompileError, compile_pyspark, compile_query, compile_script
+from redcliff.importer import ImportResult, SQLImportError, import_sql
+from redcliff.lineage import Ref, blast_radius, build_graph, column_issues, lineage_report, openlineage_facet
+from redcliff.powercenter import PowerCenterImportError, import_powercenter
+from redcliff.scorecard import Scorecard, ScoreItem
+from redcliff.ssis import SSISImportError, import_ssis
+from redcliff.spec import Issue, Node, Pipeline, SpecError, json_schema, load_spec, parse_spec, validate
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CompileError",
+    "compile_pyspark",
+    "compile_query",
+    "compile_script",
+    "infer_columns",
+    "ImportResult",
+    "SQLImportError",
+    "import_sql",
+    "PowerCenterImportError",
+    "import_powercenter",
+    "SSISImportError",
+    "import_ssis",
+    "Scorecard",
+    "ScoreItem",
+    "Ref",
+    "blast_radius",
+    "build_graph",
+    "column_issues",
+    "lineage_report",
+    "openlineage_facet",
+    "Issue",
+    "Node",
+    "Pipeline",
+    "SpecError",
+    "json_schema",
+    "load_spec",
+    "parse_spec",
+    "validate",
+]

redcliff/cli.py

@@ -0,0 +1,503 @@
+"""The ``redcliff`` command line: import, validate, compile, inspect lineage and run pipelines.
+
+    redcliff import legacy.sql --dialect teradata --schema schema.json -o pipeline.yaml
+    redcliff import mapping.xml -o pipeline.yaml --scorecard scorecard.md   (Informatica PowerCenter)
+    redcliff import package.dtsx -o pipeline.yaml --scorecard scorecard.md  (SSIS)
+    redcliff validate pipeline.yaml
+    redcliff compile pipeline.yaml --to spark
+    redcliff compile pipeline.yaml --to dbt -o my_dbt_project       (or --to lakeflow)
+    redcliff lineage pipeline.yaml --impact orders.status
+    redcliff run pipeline.yaml --db warehouse.duckdb
+    redcliff test pipeline.yaml                       (data quality expectations)
+    redcliff trace pipeline.yaml out --where "region = 'South'"
+    redcliff diff old.yaml new.yaml --data            (or: redcliff diff --git origin/main)
+    redcliff mcp --root .                             (an MCP server for AI agents, over stdio)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from redcliff.compile import CompileError, compile_pyspark, compile_query, compile_script
+from redcliff.importer import SQLImportError, import_sql
+from redcliff.lineage import Ref, blast_radius, build_graph, openlineage_facet, trace_to_sources
+from redcliff.powercenter import import_powercenter
+from redcliff.spec import SpecError, dump_spec, load_spec
+
+
+def _read_mapping(path: str) -> dict:
+    text = Path(path).read_text(encoding="utf-8")
+    if text.lstrip().startswith("{"):
+        return json.loads(text)
+    import yaml
+
+    return yaml.safe_load(text)
+
+
+def _write_spec(data: dict, path: str | None) -> str:
+    text = dump_spec(data, "yaml" if path is None or path.endswith((".yaml", ".yml")) else "json")
+    if path:
+        Path(path).write_text(text, encoding="utf-8")
+    return text
+
+
+def _spec(args):
+    """Load the spec, with --var overrides applied to its variables."""
+    pipeline = load_spec(args.spec)
+    for item in getattr(args, "var", None) or []:
+        name, sep, value = item.partition("=")
+        if not sep or not name.strip():
+            raise SystemExit(f"--var expects NAME=VALUE, got {item!r}")
+        if name.strip() not in pipeline.variables:
+            _err(f"note: {name.strip()} isn't declared in the spec's variables; adding it")
+        pipeline.variables[name.strip()] = value
+    return pipeline
+
+
+def _ref(text: str) -> Ref:
+    node, _, column = text.partition(".")
+    if not node or not column:
+        raise SystemExit(f"expected NODE.COLUMN, got {text!r}")
+    return Ref(node, column)
+
+
+def _err(message: str) -> None:
+    print(message, file=sys.stderr)
+
+
+# --------------------------------------------------------------------------- commands
+
+
+def cmd_import(args) -> int:
+    text_in = Path(args.sql).read_text(encoding="utf-8")
+    from redcliff.ssis import import_ssis, is_ssis
+
+    if is_ssis(text_in, args.sql):
+        result = import_ssis(text_in, name=args.name)
+    elif args.sql.lower().endswith(".xml") or text_in.lstrip().startswith("<"):
+        params = Path(args.params).read_text(encoding="utf-8") if args.params else None
+        result = import_powercenter(text_in, mapping=args.mapping or None, name=args.name, params=params)
+    else:
+        schema = _read_mapping(args.schema) if args.schema else None
+        name = args.name or Path(args.sql).stem
+        result = import_sql(text_in, dialect=args.dialect, name=name, schema=schema)
+    text = _write_spec(result.pipeline.to_dict(), args.output)
+    if not args.output:
+        print(text, end="")
+    nodes = result.pipeline.nodes
+    _err(f"imported {len(nodes)} nodes ({sum(n.type == 'target' for n in nodes)} targets)"
+         + (f" to {args.output}" if args.output else ""))
+    for w in result.warnings:
+        _err(f"warning: {w}")
+    card = result.scorecard
+    if card is not None:
+        c = card.counts()
+        _err(f"scorecard: {c['converted']} of {len(card.items)} converted ({card.automated():.0%}), "
+             f"{c['review']} to review, {c['manual']} to rewrite by hand")
+        if args.scorecard:
+            out = Path(args.scorecard)
+            out.write_text(json.dumps(card.to_dict(), indent=2) + "\n" if out.suffix == ".json"
+                           else card.to_markdown(), encoding="utf-8")
+            _err(f"scorecard written to {args.scorecard}")
+    return 0
+
+
+def cmd_validate(args) -> int:
+    pipeline = load_spec(args.spec)
+    issues = build_graph(pipeline).issues
+    for i in issues:
+        _err(f"column issue: {i}")
+    if issues:
+        return 1
+    print(f"{args.spec}: valid ({len(pipeline.nodes)} nodes)")
+    return 0
+
+
+def _write_project(files: dict[str, str], out_dir: str | None) -> None:
+    if not out_dir:  # one stream, a header per file
+        for rel, content in files.items():
+            print(f"# ==== {rel}\n{content}", end="" if content.endswith("\n") else "\n")
+        return
+    for rel, content in files.items():
+        path = Path(out_dir) / rel
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content, encoding="utf-8")
+    _err(f"wrote {len(files)} files to {out_dir}")
+
+
+def cmd_compile(args) -> int:
+    pipeline = _spec(args)
+    base = Path(args.spec).parent if args.resolve_paths else None
+    if args.to == "dbt":
+        from redcliff.dbt import compile_dbt
+
+        _write_project(compile_dbt(pipeline, args.adapter, base_dir=Path(args.spec).parent), args.output)
+        return 0
+    if args.to == "lakeflow":
+        from redcliff.lakeflow import compile_lakeflow
+
+        _write_project(compile_lakeflow(pipeline, base_dir=base, storage=args.storage), args.output)
+        return 0
+    if args.to == "pyspark":
+        out = compile_pyspark(pipeline, base_dir=base)
+    elif args.node:
+        out = compile_query(pipeline, args.node, args.to, base_dir=base) + "\n"
+    else:
+        out = compile_script(pipeline, args.to, base_dir=base)
+    if args.output:
+        Path(args.output).write_text(out, encoding="utf-8")
+        _err(f"wrote {args.output}")
+    else:
+        print(out, end="")
+    return 0
+
+
+def cmd_lineage(args) -> int:
+    graph = build_graph(_spec(args))
+    for i in graph.issues:
+        _err(f"column issue: {i}")
+    if args.openlineage:
+        print(json.dumps(openlineage_facet(graph, args.openlineage), indent=2))
+    elif args.impact:
+        impact = blast_radius(graph, _ref(args.impact)).to_dict()
+        print(f"If {impact['column']} changes:")
+        print(f"  values change in: {', '.join(impact['values']) or 'nothing'}")
+        print(f"  rows change in:   {', '.join(impact['rows']) or 'nothing'}")
+        print(f"  a rename breaks:  {', '.join(impact['breaks']) or 'nothing'}")
+    elif args.column:
+        ref = _ref(args.column)
+        traced = trace_to_sources(graph, ref)
+        print(f"{ref} comes from:")
+        for src, (kind, sub) in sorted(traced.items()):
+            print(f"  {src}  ({kind.lower()}, {sub.lower()})")
+        if not traced:
+            print("  no source columns (a literal or COUNT(*))")
+    else:
+        for node, cols in graph.columns.items():
+            for col in cols:
+                srcs = ", ".join(str(s) for s in sorted(trace_to_sources(graph, Ref(node, col))))
+                print(f"{node}.{col} <- {srcs or '-'}")
+    return 0
+
+
+def cmd_run(args) -> int:
+    try:
+        import duckdb
+    except ImportError:
+        _err("run needs DuckDB: pip install duckdb")
+        return 2
+    pipeline = _spec(args)
+    base = Path(args.spec).parent
+    con = duckdb.connect(args.db) if args.db else duckdb.connect()
+    if args.node:
+        rel = con.sql(compile_query(pipeline, args.node, "duckdb", base_dir=base))
+        print(" | ".join(rel.columns))
+        for row in rel.fetchmany(args.limit):
+            print(" | ".join("" if v is None else str(v) for v in row))
+        return 0
+    if not args.skip_checks:
+        from redcliff.expect import run_checks
+
+        results = run_checks(con, pipeline, base_dir=base)
+        _report_checks(results, quiet=True)
+        if any(not r.passed and r.expectation.severity == "error" for r in results):
+            _err("not writing: an expectation failed (see above); fix the data or pass --skip-checks")
+            return 1
+    for target in (n for n in pipeline.nodes if n.type == "target" and n.params["format"] != "table"):
+        (base / target.params["path"]).parent.mkdir(parents=True, exist_ok=True)
+    con.execute(compile_script(pipeline, "duckdb", base_dir=base))
+    targets = [n.params.get("table") or n.params.get("path") for n in pipeline.nodes if n.type == "target"]
+    print(f"ran {pipeline.name}: wrote {', '.join(targets)}")
+    return 0
+
+
+def _report_checks(results, quiet: bool = False) -> None:
+    """Print expectation results; with quiet, only the failures."""
+    for r in results:
+        e = r.expectation
+        if r.passed:
+            if not quiet:
+                print(f"pass  {e.node}.{e.name}  ({e.describe()})")
+            continue
+        label = "warn " if e.severity == "warn" else "FAIL "
+        what = r.error or f"{r.failures} failing row{'s' if r.failures != 1 else ''}"
+        print(f"{label} {e.node}.{e.name}  ({e.describe()}): {what}")
+        if r.sample:
+            print("        " + " | ".join(r.columns))
+            for row in r.sample:
+                print("        " + " | ".join("" if v is None else str(v) for v in row))
+
+
+def cmd_test(args) -> int:
+    try:
+        import duckdb
+    except ImportError:
+        _err("test needs DuckDB: pip install duckdb")
+        return 2
+    from redcliff.expect import run_checks
+
+    pipeline = _spec(args)
+    con = duckdb.connect(args.db, read_only=True) if args.db else duckdb.connect()
+    results = run_checks(con, pipeline, args.node, Path(args.spec).parent)
+    if not results:
+        print("no expectations to check; add 'expect' to a step (see docs/spec.md)")
+        return 0
+    _report_checks(results)
+    failed = [r for r in results if not r.passed and r.expectation.severity == "error"]
+    warned = [r for r in results if not r.passed and r.expectation.severity == "warn"]
+    print(f"\n{len(results) - len(failed) - len(warned)} passed, {len(failed)} failed, {len(warned)} warned")
+    return 1 if failed else 0
+
+
+def _cell(v) -> str:
+    return "" if v is None else str(v)
+
+
+def cmd_trace(args) -> int:
+    try:
+        import duckdb
+    except ImportError:
+        _err("trace needs DuckDB: pip install duckdb")
+        return 2
+    from redcliff.trace import trace_query, trace_rows
+
+    pipeline = _spec(args)
+    base = Path(args.spec).parent
+    con = duckdb.connect(args.db, read_only=True) if args.db else duckdb.connect()
+    tq = trace_query(pipeline, args.node, base)
+    for step in tq.opaque:
+        _err(f"note: rows pass through the sql step {step} untraced")
+    rows = trace_rows(con, pipeline, args.node, args.where, base)
+    if not rows:
+        print("no rows" + (f" where {args.where}" if args.where else ""))
+    for i, traced in enumerate(rows[: args.limit]):
+        if i:
+            print()
+        print(f"{tq.node}: " + ", ".join(f"{k}={_cell(v)}" for k, v in traced["row"].items()))
+        for source, src_rows in traced["sources"].items():
+            print(f"  from {source} ({len(src_rows)} row{'s' if len(src_rows) != 1 else ''}):")
+            for r in src_rows:
+                print(f"    #{r['row']}: " + ", ".join(f"{k}={_cell(v)}" for k, v in r.items() if k != "row"))
+    if len(rows) > args.limit:
+        print(f"\n... {len(rows) - args.limit} more rows; narrow them with --where")
+    return 0
+
+
+MARKER = "<!-- redcliff-diff -->"
+
+
+def _git(*args: str, cwd: str | None = None) -> str:
+    import subprocess
+
+    done = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True)
+    if done.returncode:
+        raise FileNotFoundError(f"git {' '.join(args)}: {done.stderr.strip()}")
+    return done.stdout
+
+
+def _as_pipeline(text: str | None):
+    """A spec's text as a Pipeline, or None when it isn't a pipeline spec."""
+    from redcliff.spec import Pipeline
+
+    if text is None:
+        return None
+    try:
+        if text.lstrip().startswith("{"):
+            data = json.loads(text)
+        else:
+            import yaml
+
+            data = yaml.safe_load(text)
+        if not isinstance(data, dict) or not isinstance(data.get("nodes"), list):
+            return None
+        return Pipeline.from_dict(data)
+    except Exception:  # noqa: BLE001 - not a spec, or not parseable: skip it
+        return None
+
+
+def _diff_one(base, head, args, base_dir, head_dir):
+    from redcliff.diff import data_diff, diff_pipelines
+    from redcliff.spec import Pipeline
+
+    base = base or Pipeline(name=head.name, nodes=[])
+    diff = diff_pipelines(base, head)
+    if args.data and not diff.issues:
+        try:
+            import duckdb
+        except ImportError:
+            _err("--data needs DuckDB: pip install duckdb")
+        else:
+            con = duckdb.connect(args.db, read_only=True) if args.db else duckdb.connect()
+            diff.data = data_diff(base, head, con, base_dir, head_dir)
+    return diff
+
+
+def cmd_diff(args) -> int:
+
+    results = []  # (label, diff)
+    if args.git:
+        root = _git("rev-parse", "--show-toplevel").strip()
+        names = _git("diff", "--name-only", "--diff-filter=ACMRD", args.git, "--", *(args.paths or ["."]), cwd=None)
+        prefix = _git("rev-parse", "--show-prefix").strip()
+        for rel in names.split():
+            if not rel.endswith((".yaml", ".yml", ".json")):
+                continue
+            path = Path(root) / rel
+            head_text = path.read_text(encoding="utf-8") if path.exists() else None
+            try:
+                base_text = _git("show", f"{args.git}:{rel}", cwd=root)
+            except FileNotFoundError:
+                base_text = None
+            base, head = _as_pipeline(base_text), _as_pipeline(head_text)
+            if head is None and base is None:
+                continue
+            label = rel[len(prefix):] if prefix and rel.startswith(prefix) else rel
+            if head is None:
+                results.append((label, None))
+                continue
+            results.append((label, _diff_one(base, head, args, path.parent, path.parent)))
+    else:
+        if not args.paths or len(args.paths) != 2:
+            _err("give two specs (BASE HEAD), or --git REF")
+            return 2
+        a, b = (Path(p) for p in args.paths)
+        base = _as_pipeline(a.read_text(encoding="utf-8"))
+        head = _as_pipeline(b.read_text(encoding="utf-8"))
+        if base is None or head is None:
+            _err("both files must be pipeline specs")
+            return 1
+        results.append((str(b), _diff_one(base, head, args, a.parent, b.parent)))
+
+    if args.format == "json":
+        text = json.dumps({label: (d.to_dict() if d else {"removed": True}) for label, d in results}, indent=2) + "\n"
+    else:
+        lines = [MARKER, "## Redcliff: what this change does to the pipelines", ""]
+        if not results:
+            lines.append("No pipeline specs changed.")
+        for label, d in results:
+            if d is None:
+                lines += [f"### `{label}`", "", "**pipeline removed**", ""]
+            else:
+                lines.append(d.to_markdown(title=f"`{label}`", level=3))
+        lines += ["<sub>Generated by [Redcliff](https://github.com/Fullcountr/redcliff) from the pipeline specs; "
+                  "data compares each target on the sample data in the repository.</sub>" if args.data else
+                  "<sub>Generated by [Redcliff](https://github.com/Fullcountr/redcliff) from the pipeline specs.</sub>"]
+        text = "\n".join(lines).rstrip() + "\n"
+    if args.output:
+        Path(args.output).write_text(text, encoding="utf-8")
+        _err(f"wrote {args.output}")
+    else:
+        print(text, end="")
+    has_issues = any(d is not None and d.issues for _l, d in results)
+    return 1 if has_issues and args.fail_on_issues else 0
+
+
+def cmd_mcp(args) -> int:
+    from redcliff.mcp import serve
+
+    serve(args.root, args.db)
+    return 0
+
+
+# --------------------------------------------------------------------------- entry point
+
+
+def build_parser() -> argparse.ArgumentParser:
+    from redcliff import __version__
+
+    parser = argparse.ArgumentParser(prog="redcliff", description="Legacy ETL in, clean Spark out.")
+    parser.add_argument("--version", action="version", version=f"redcliff {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("import", help="turn legacy SQL, a PowerCenter export or an SSIS package into a pipeline spec")
+    p.add_argument("sql", help="SQL file (one or more statements), a PowerCenter XML export or an SSIS .dtsx")
+    p.add_argument("--dialect", help="sqlglot dialect, e.g. teradata, oracle, tsql, snowflake")
+    p.add_argument("--schema", help="JSON/YAML mapping of table -> {column: type}")
+    p.add_argument("--name", help="pipeline name (default: the file name)")
+    p.add_argument("--mapping", action="append", help="PowerCenter: import only this mapping (repeatable)")
+    p.add_argument("--scorecard", help="write the migration scorecard here (.md or .json)")
+    p.add_argument("--params", help="PowerCenter: a parameter file, for $$PARAMS in mappings and SQL")
+    p.add_argument("-o", "--output", help="write the spec here (.yaml or .json); default: stdout")
+    p.set_defaults(func=cmd_import)
+
+    p = sub.add_parser("validate", help="check a spec, including column-level checks")
+    p.add_argument("spec")
+    p.set_defaults(func=cmd_validate)
+
+    p = sub.add_parser("compile", help="generate DuckDB SQL, Spark SQL, PySpark, a dbt project or a Lakeflow pipeline")
+    p.add_argument("spec")
+    p.add_argument("--var", action="append", metavar="NAME=VALUE", help="set a pipeline variable (repeatable)")
+    p.add_argument("--to", choices=["duckdb", "spark", "pyspark", "dbt", "lakeflow"], default="duckdb")
+    p.add_argument("--adapter", default="duckdb", help="dbt: the adapter to write SQL for (duckdb, databricks, "
+                   "spark, snowflake, bigquery, postgres, redshift)")
+    p.add_argument("--storage", help="lakeflow: checkpoint root, an absolute URI (file://..., s3a://...)")
+    p.add_argument("--node", help="compile a SELECT for one node instead of the full script")
+    p.add_argument("--resolve-paths", action="store_true", help="make file paths absolute")
+    p.add_argument("-o", "--output", help="a file; for dbt and lakeflow, a directory")
+    p.set_defaults(func=cmd_compile)
+
+    p = sub.add_parser("lineage", help="column lineage, blast radius or OpenLineage export")
+    p.add_argument("spec")
+    p.add_argument("--var", action="append", metavar="NAME=VALUE", help="set a pipeline variable (repeatable)")
+    g = p.add_mutually_exclusive_group()
+    g.add_argument("--column", help="NODE.COLUMN: where does it come from?")
+    g.add_argument("--impact", help="NODE.COLUMN: what does a change to it touch?")
+    g.add_argument("--openlineage", metavar="NODE", help="print the OpenLineage column facet for NODE")
+    p.set_defaults(func=cmd_lineage)
+
+    p = sub.add_parser("run", help="run the pipeline locally with DuckDB")
+    p.add_argument("spec")
+    p.add_argument("--var", action="append", metavar="NAME=VALUE", help="set a pipeline variable (repeatable)")
+    p.add_argument("--db", help="DuckDB database file holding table sources and targets")
+    p.add_argument("--node", help="preview one node's rows instead of writing targets")
+    p.add_argument("--limit", type=int, default=20)
+    p.add_argument("--skip-checks", action="store_true", help="write even if an expectation fails")
+    p.set_defaults(func=cmd_run)
+
+    p = sub.add_parser("test", help="run the pipeline's expectations (data quality checks) in DuckDB")
+    p.add_argument("spec")
+    p.add_argument("--var", action="append", metavar="NAME=VALUE", help="set a pipeline variable (repeatable)")
+    p.add_argument("--node", help="check only this step")
+    p.add_argument("--db", help="DuckDB database file holding table sources")
+    p.set_defaults(func=cmd_test)
+
+    p = sub.add_parser("trace", help="row-level trace: the source rows behind a step's rows")
+    p.add_argument("spec")
+    p.add_argument("--var", action="append", metavar="NAME=VALUE", help="set a pipeline variable (repeatable)")
+    p.add_argument("node", help="the step (or target) whose rows to trace")
+    p.add_argument("--where", help="SQL condition picking the rows, e.g. \"region = 'South'\"")
+    p.add_argument("--db", help="DuckDB database file holding table sources")
+    p.add_argument("--limit", type=int, default=10)
+    p.set_defaults(func=cmd_trace)
+
+    p = sub.add_parser("diff", help="what a change does: steps, columns, lineage and data")
+    p.add_argument("paths", nargs="*", help="BASE HEAD specs; with --git, paths to limit the search to")
+    p.add_argument("--git", metavar="REF", help="compare every changed spec with its version at REF")
+    p.add_argument("--data", action="store_true", help="also run each target on sample data, both ways")
+    p.add_argument("--db", help="DuckDB database file holding table sources")
+    p.add_argument("--format", choices=["md", "json"], default="md")
+    p.add_argument("--fail-on-issues", action="store_true", help="exit 1 if a new spec has problems")
+    p.add_argument("-o", "--output")
+    p.set_defaults(func=cmd_diff)
+
+    p = sub.add_parser("mcp", help="run an MCP server over stdio, so AI agents can read and edit pipelines")
+    p.add_argument("--root", default=".", help="the folder whose specs agents may read and edit (default: here)")
+    p.add_argument("--db", help="DuckDB database file holding table sources, for previews and traces")
+    p.set_defaults(func=cmd_mcp)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except (SpecError, SQLImportError, CompileError, FileNotFoundError, KeyError, ValueError) as e:
+        _err(f"error: {e}")
+        return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

redcliff/columns.py

@@ -0,0 +1,70 @@
+"""Work out each node's output columns from the columns declared on its sources.
+
+Knowing columns lets the compiler emit explicit, portable SQL (a derive that replaces a column,
+a join that drops duplicate names) and is the base the lineage engine builds on. When a source
+does not declare its columns, everything downstream of it is ``None`` (unknown).
+"""
+
+from __future__ import annotations
+
+import sqlglot
+from sqlglot import exp
+
+from redcliff.spec import Node, Pipeline, bind
+
+Columns = list[str] | None
+
+
+def output_columns(node: Node, inputs: list[Columns]) -> Columns:
+    """Output columns of one node, given its inputs' columns (in input order)."""
+    t, p = node.type, node.params
+    if t == "source":
+        declared = p.get("columns")
+        return list(declared) if declared else None
+    if t == "target":
+        return []
+    if t == "select":
+        return list(p["columns"])
+    if t == "aggregate":
+        return list(p.get("group_by", [])) + list(p.get("measures") or {})
+    if t == "sql":
+        return _sql_columns(p["query"], inputs)
+
+    first = inputs[0] if inputs else None
+    if t in ("filter", "union"):
+        return list(first) if first is not None else None
+    if t == "derive":
+        if first is None:
+            return None
+        new = [c for c in p["columns"] if c not in first]
+        return list(first) + new
+    if t == "join":
+        left, right = inputs
+        if left is None or right is None:
+            return None
+        return list(left) + [c for c in right if c not in left]
+    raise ValueError(f"unknown node type {t!r}")
+
+
+def _sql_columns(query: str, inputs: list[Columns]) -> Columns:
+    """Output columns of a raw SQL step, when they can be read off its SELECT list."""
+    tree = sqlglot.parse_one(query)
+    select = tree if isinstance(tree, exp.Select) else tree.find(exp.Select)
+    if select is None:
+        return None
+    projections = select.expressions
+    if len(projections) == 1 and isinstance(projections[0], exp.Star) and len(inputs) == 1:
+        return list(inputs[0]) if inputs[0] is not None else None
+    if any(isinstance(e, exp.Star) or isinstance(e.unalias(), exp.Star) for e in projections):
+        return None
+    names = select.named_selects
+    return names if len(names) == len(projections) and all(names) else None
+
+
+def infer_columns(pipeline: Pipeline) -> dict[str, Columns]:
+    """Map every node id to its output column names, or None where they can't be known."""
+    pipeline = bind(pipeline)
+    cols: dict[str, Columns] = {}
+    for node in pipeline.topo_order():
+        cols[node.id] = output_columns(node, [cols[i] for i in node.inputs])
+    return cols