dbt-graphify 0.1.0__py3-none-any.whl

dbt_graphify/__init__.py

@@ -0,0 +1,3 @@
+"""dbt-graphify: Parse a dbt manifest.json into a graphify-queryable knowledge graph."""
+
+__version__ = "0.1.0"

dbt_graphify/__main__.py

@@ -0,0 +1,3 @@
+from dbt_graphify.cli import main
+
+main()

dbt_graphify/builder.py

@@ -0,0 +1,420 @@
+"""Build graphify-out/ artifacts from parsed dbt graph data."""
+
+import json
+import shutil
+import subprocess
+from collections import defaultdict, deque
+from pathlib import Path
+
+
+# ── Community detection ───────────────────────────────────────────────────────
+
+_LAYER_COMMUNITY = {
+    "source": 0,
+    "staging": 1,
+    "intermediate": 2,
+    "mart": 3,
+    "seed": 4,
+    "macro": 5,
+    "test": 6,
+}
+
+
+def build_community_map(nodes: list[dict]) -> tuple[dict[str, int], dict[int, str]]:
+    """
+    Assign community integers via priority chain:
+      1. dbt group field
+      2. first tag
+      3. Louvain topology clustering (if networkx available)
+      4. layer-based fallback
+
+    Returns (uid -> community_int, community_int -> label).
+    """
+    # Check if any node has groups or tags
+    group_index: dict[str, int] = {}
+    tag_index: dict[str, int] = {}
+    counter = [0]
+
+    def _get_or_create(index: dict, key: str) -> int:
+        if key not in index:
+            index[key] = counter[0]
+            counter[0] += 1
+        return index[key]
+
+    has_groups = any(n.get("group") for n in nodes)
+    has_tags = any(n.get("tags") for n in nodes) and not has_groups
+
+    if has_groups:
+        uid_to_community = {}
+        for n in nodes:
+            group = n.get("group") or ""
+            cid = _get_or_create(group_index, group or "__ungrouped__")
+            uid_to_community[n["uid"]] = cid
+        labels = {v: k for k, v in group_index.items()}
+        return uid_to_community, labels
+
+    if has_tags:
+        uid_to_community = {}
+        for n in nodes:
+            tags = n.get("tags") or []
+            key = tags[0] if tags else "__untagged__"
+            cid = _get_or_create(tag_index, key)
+            uid_to_community[n["uid"]] = cid
+        labels = {v: k for k, v in tag_index.items()}
+        return uid_to_community, labels
+
+    # Try topology clustering with networkx
+    try:
+        import networkx as nx
+        G = nx.DiGraph()
+        for n in nodes:
+            G.add_node(n["uid"])
+        for n in nodes:
+            for tgt in n.get("downstream", []):
+                G.add_edge(n["uid"], tgt)
+
+        undirected = G.to_undirected()
+        try:
+            from networkx.algorithms.community import louvain_communities
+            communities = louvain_communities(undirected, seed=42)
+        except (ImportError, Exception):
+            # greedy modularity as fallback
+            from networkx.algorithms.community import greedy_modularity_communities
+            communities = list(greedy_modularity_communities(undirected))
+
+        uid_to_community = {}
+        labels = {}
+        for cid, members in enumerate(communities):
+            # Name the community after the most-connected node in the cluster
+            best = max(members, key=lambda n: G.degree(n), default=next(iter(members)))
+            labels[cid] = best.split(".")[-1]
+            for uid in members:
+                uid_to_community[uid] = cid
+        # Fill any missing (isolated nodes)
+        for n in nodes:
+            if n["uid"] not in uid_to_community:
+                uid_to_community[n["uid"]] = _LAYER_COMMUNITY.get(n["layer"], 0)
+        return uid_to_community, labels
+
+    except ImportError:
+        pass
+
+    # Layer fallback
+    uid_to_community = {n["uid"]: _LAYER_COMMUNITY.get(n["layer"], 0) for n in nodes}
+    labels = {v: k.capitalize() for k, v in _LAYER_COMMUNITY.items()}
+    return uid_to_community, labels
+
+
+# ── BFS ───────────────────────────────────────────────────────────────────────
+
+def bfs_reachable(start: str, adjacency: dict[str, list]) -> list[str]:
+    visited, queue = set(), deque(adjacency.get(start, []))
+    while queue:
+        node = queue.popleft()
+        if node not in visited:
+            visited.add(node)
+            queue.extend(n for n in adjacency.get(node, []) if n not in visited)
+    return sorted(visited)
+
+
+# ── Descriptions ──────────────────────────────────────────────────────────────
+
+_AUTO_DESCS: dict[str, str] = {
+    "stg_students": "Cleans raw students. Derives: full_name, age, years_enrolled, academic_standing, current_status.",
+    "stg_courses": "Cleans raw courses. Derives: difficulty_description, credit_category.",
+    "stg_departments": "Cleans raw departments. Derives: department_size, budget_millions.",
+    "stg_faculty": "Cleans raw faculty. Derives: full_name, years_of_service, rank_level, salary_band.",
+    "stg_enrollments": "Cleans raw enrollments. Derives: grade_category, enrollment_status, attendance_level.",
+    "stg_semesters": "Cleans raw semesters. Derives: semester_type, semester_duration_days, semester_status.",
+    "stg_class_sessions": "Cleans raw class sessions. Derives: day_of_week, session_hour, time_block, day_name.",
+    "stg_assignments": "Cleans raw assignments. Derives: assignment_category, due_status, days_until_due, weight_category.",
+    "stg_assignment_submissions": "Cleans raw submissions. Derives: grading_status, submission_timeliness, feedback_status.",
+    "stg_financial_aid": "Cleans raw financial aid. Derives: aid_category, support_level, disbursement_period.",
+    "stg_tuition_payments": "Cleans raw tuition payments. Derives: total_payment, payment_timeliness, payment_method_category.",
+    "int_student_enrollment_history": "Per-enrollment grain with window aggregates per student: total_enrollments, credits_attempted/earned, failed_courses_count, avg_grade_points.",
+    "int_course_performance_metrics": "Per-course aggregates: total_enrollments, pass_rate, withdrawal_rate, avg_grade_points, avg_attendance.",
+    "int_department_analytics": "Per-department aggregates: faculty_count, course_count, student_count, avg_faculty_salary, student_faculty_ratio.",
+    "int_faculty_teaching_load": "Per-faculty workload: unique_courses_taught, total_students_taught, salary_per_course, teaching_load_category.",
+    "int_student_at_risk_indicators": "Per-student risk scoring: 8 binary flags summed into risk_level (Low/Moderate/High/Critical) + recommended_intervention.",
+    "student_academic_summary": "Final mart: student academic profile with class_standing, completion_rate, progress_indicator.",
+    "student_financial_profile": "Final mart: student financial summary with aid_recipient_category, payment_reliability, primary_aid_type.",
+    "faculty_performance_dashboard": "Final mart: faculty dashboard with teaching_impact_category, engagement_effectiveness, career_stage.",
+    "department_overview": "Final mart: department overview with scale, efficiency ratios, faculty/student metrics.",
+    "course_performance_summary": "Final mart: course performance category (High/Good/Average/Needs Improvement) with grade distribution.",
+    "student_at_risk_report": "Final mart: at-risk students only (risk_level != Low Risk), ordered by total_risk_score for intervention prioritization.",
+}
+
+
+def _auto_desc(name: str, layer: str) -> str:
+    return _AUTO_DESCS.get(name, f"{layer.capitalize()} model: {name}")
+
+
+def _infer_materialized(layer: str, config: dict) -> str:
+    if config and "materialized" in config:
+        return config["materialized"]
+    return {
+        "source": "external",
+        "staging": "view",
+        "intermediate": "view",
+        "mart": "table",
+        "seed": "seed",
+        "macro": "macro",
+    }.get(layer, "view")
+
+
+# ── graph.json ────────────────────────────────────────────────────────────────
+
+def build_graph_json(
+    parsed: dict,
+    project_name: str,
+    uid_to_community: dict[str, int],
+    community_labels: dict[int, str],
+) -> dict:
+    """Build graphify-compatible graph.json in NetworkX node-link format.
+
+    All node properties are FLAT at the top level — no nested 'properties' key.
+    Required by graphify's node_link_graph() loader.
+    """
+    nodes_out = []
+    edges_out = []
+
+    for n in parsed["nodes"]:
+        uid = n["uid"]
+        name = n["name"]
+        layer = n["layer"]
+        description = n.get("description") or _auto_desc(name, layer)
+        materialized = _infer_materialized(layer, n.get("config", {}))
+
+        # FLAT node — all fields at top level
+        node = {
+            "id": uid,
+            "label": name,
+            "community": uid_to_community.get(uid, 0),
+            "type": n["resource_type"],
+            "layer": layer,
+            "source_file": n.get("file_path", ""),
+            "description": description,
+            "materialized": materialized,
+            "upstream": n.get("upstream", []),
+            "downstream": n.get("downstream", []),
+            "columns": n.get("columns", []),
+            "refs": n.get("refs", []),
+            "sources_used": n.get("sources", []),
+            "tags": n.get("tags", []),
+        }
+        if n.get("schema"):
+            node["schema"] = n["schema"]
+
+        nodes_out.append(node)
+
+        # Edges: upstream → this node
+        for parent_uid in n.get("upstream", []):
+            edges_out.append({
+                "source": parent_uid,
+                "target": uid,
+                "type": "LINEAGE",
+                "provenance": "EXTRACTED",
+                "key": 0,
+            })
+
+    # God nodes: one per layer (not added as graph nodes to avoid polluting BFS)
+    layer_groups: dict[str, list[str]] = defaultdict(list)
+    for n in parsed["nodes"]:
+        if n["layer"] not in ("test", "macro"):
+            layer_groups[n["layer"]].append(n["name"])
+
+    god_nodes = [
+        {
+            "layer": layer,
+            "label": f"LAYER:{layer.upper()}",
+            "count": len(names),
+            "models": sorted(names),
+        }
+        for layer, names in sorted(layer_groups.items())
+    ]
+
+    return {
+        "directed": True,
+        "multigraph": False,
+        "graph": {
+            "project": project_name,
+            "generated_by": "dbt-graphify",
+            "communities": {
+                str(cid): label for cid, label in community_labels.items()
+            },
+        },
+        "god_nodes": god_nodes,
+        "nodes": nodes_out,
+        "edges": edges_out,
+    }
+
+
+# ── lineage.json ──────────────────────────────────────────────────────────────
+
+def build_lineage_json(nodes: list[dict], parsed: dict) -> dict:
+    adj_up = parsed.get("upstream", {})
+    adj_down = parsed.get("downstream", {})
+
+    ancestors = {n["uid"]: bfs_reachable(n["uid"], adj_up) for n in nodes}
+    descendants = {n["uid"]: bfs_reachable(n["uid"], adj_down) for n in nodes}
+
+    return {
+        "description": (
+            "Blast-radius maps. "
+            "'ancestors' = full upstream chain (what a node reads from). "
+            "'descendants' = full downstream chain (what breaks if this node changes)."
+        ),
+        "ancestors": ancestors,
+        "descendants": descendants,
+    }
+
+
+# ── GRAPH_REPORT.md ───────────────────────────────────────────────────────────
+
+def write_report(
+    parsed: dict,
+    out_dir: Path,
+    project_name: str,
+    lineage: dict,
+    uid_to_community: dict[str, int],
+    community_labels: dict[int, str],
+):
+    nodes = parsed["nodes"]
+
+    layer_order = ["source", "staging", "intermediate", "mart", "seed", "macro"]
+    layer_labels = {
+        "source": "Sources",
+        "staging": "Staging — views, clean & standardize",
+        "intermediate": "Intermediate — views, aggregate & join",
+        "mart": "Marts — tables, BI-ready",
+        "seed": "Seeds — static reference data",
+        "macro": "Macros — reusable Jinja2 logic",
+    }
+
+    lines = [
+        f"# dbt Knowledge Graph — {project_name}",
+        "",
+        "_Generated by dbt-graphify. Query: `graphify query \"<question>\"`_",
+        "",
+        "## Architecture",
+        "",
+        "```",
+        "Source tables ──► staging/ (views, clean + standardize)",
+        "                      │",
+        "                      ▼",
+        "               intermediate/ (views, aggregate + join)",
+        "                      │",
+        "                      ▼",
+        "                  marts/ (tables, BI-ready outputs)",
+        "```",
+        "",
+        "## Node Inventory",
+        "",
+    ]
+
+    for layer in layer_order:
+        layer_nodes = [n for n in nodes if n["layer"] == layer]
+        if not layer_nodes:
+            continue
+        lines.append(f"### {layer_labels.get(layer, layer)} ({len(layer_nodes)})")
+        lines.append("")
+        for n in sorted(layer_nodes, key=lambda x: x["name"]):
+            desc = n.get("description") or _auto_desc(n["name"], layer)
+            mat = _infer_materialized(layer, n.get("config", {}))
+            up = [u.split(".")[-1] for u in n.get("upstream", [])]
+            down = [d.split(".")[-1] for d in n.get("downstream", [])]
+            cols = n.get("columns", [])
+            community_label = community_labels.get(uid_to_community.get(n["uid"], 0), "")
+            lines.append(f"**{n['name']}** `[{mat}]` · community: _{community_label}_")
+            lines.append(f"> {desc}")
+            if up:
+                lines.append(f"- Reads from: `{'`, `'.join(sorted(up))}`")
+            if down:
+                lines.append(f"- Feeds into: `{'`, `'.join(sorted(down))}`")
+            if cols:
+                shown = cols[:15]
+                suffix = "..." if len(cols) > 15 else ""
+                lines.append(f"- Columns: `{'`, `'.join(shown)}`{suffix}")
+            lines.append("")
+
+    # Blast-radius table
+    lines += [
+        "## Blast Radius",
+        "",
+        "Downstream models affected when a model changes:",
+        "",
+        "| Model | Affected downstream |",
+        "|---|---|",
+    ]
+
+    for n in sorted(nodes, key=lambda x: -len(lineage["descendants"].get(x["uid"], []))):
+        if n["layer"] in ("test", "macro", "seed"):
+            continue
+        desc = [d.split(".")[-1] for d in lineage["descendants"].get(n["uid"], [])]
+        if desc:
+            lines.append(f"| `{n['name']}` | {', '.join(sorted(desc))} |")
+
+    # Community legend
+    lines += [
+        "",
+        "## Communities",
+        "",
+        "| ID | Label |",
+        "|---|---|",
+    ]
+    for cid in sorted(community_labels):
+        lines.append(f"| {cid} | {community_labels[cid]} |")
+
+    lines += [
+        "",
+        "---",
+        "_Regenerate: `dbt-graphify` or `python -m dbt_graphify`_",
+    ]
+
+    (out_dir / "GRAPH_REPORT.md").write_text("\n".join(lines), encoding="utf-8")
+
+
+# ── Sidecar files ─────────────────────────────────────────────────────────────
+
+def write_graphify_root(out_dir: Path, project_root: Path):
+    (out_dir / ".graphify_root").write_text(str(project_root.resolve()), encoding="utf-8")
+
+
+def write_graphify_labels(out_dir: Path, community_labels: dict[int, str]):
+    data = {str(cid): label for cid, label in community_labels.items()}
+    (out_dir / ".graphify_labels.json").write_text(
+        json.dumps(data, indent=2), encoding="utf-8"
+    )
+
+
+# ── HTML via graphify cluster-only ────────────────────────────────────────────
+
+def generate_html(out_dir: Path, project_root: Path):
+    graphify_bin = shutil.which("graphify")
+    if not graphify_bin:
+        print(
+            "  [dbt-graphify] Note: install graphifyy (`pip install graphifyy`) "
+            "to also generate graph.html"
+        )
+        return
+
+    try:
+        result = subprocess.run(
+            [graphify_bin, "cluster-only", str(project_root), "--no-label"],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        if result.returncode == 0:
+            print("  [dbt-graphify] graph.html generated via graphify cluster-only")
+        else:
+            print(
+                f"  [dbt-graphify] Warning: graphify cluster-only exited {result.returncode} "
+                f"— graph.html not generated. graph.json is still valid."
+            )
+            if result.stderr:
+                print(f"  {result.stderr.strip()[:200]}")
+    except subprocess.TimeoutExpired:
+        print("  [dbt-graphify] Warning: graphify cluster-only timed out — skipping graph.html")
+    except Exception as exc:
+        print(f"  [dbt-graphify] Warning: could not run graphify cluster-only ({exc})")

dbt_graphify/cli.py

@@ -0,0 +1,158 @@
+"""CLI entry point for dbt-graphify."""
+
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+from dbt_graphify.parser import (
+    find_manifest,
+    find_graph_summary,
+    parse_manifest,
+    parse_graph_summary,
+)
+from dbt_graphify.builder import (
+    build_community_map,
+    build_graph_json,
+    build_lineage_json,
+    write_report,
+    write_graphify_root,
+    write_graphify_labels,
+    generate_html,
+)
+
+
+def main():
+    args = sys.argv[1:]
+    manifest_path: Path | None = None
+    out_dir = Path("graphify-out")
+    skip_html = False
+
+    i = 0
+    while i < len(args):
+        if args[i] in ("--out", "-o") and i + 1 < len(args):
+            out_dir = Path(args[i + 1])
+            i += 2
+        elif args[i].startswith("--out="):
+            out_dir = Path(args[i].split("=", 1)[1])
+            i += 1
+        elif args[i] == "--no-html":
+            skip_html = True
+            i += 1
+        elif args[i] in ("-h", "--help"):
+            _print_help()
+            sys.exit(0)
+        elif not args[i].startswith("-"):
+            manifest_path = Path(args[i])
+            i += 1
+        else:
+            print(f"Unknown argument: {args[i]}", file=sys.stderr)
+            i += 1
+
+    cwd = Path.cwd()
+
+    # ── Source selection ──────────────────────────────────────────────────────
+    parsed = None
+    source_label = ""
+    project_root = cwd
+
+    if manifest_path is None:
+        manifest_path = find_manifest(cwd)
+
+    if manifest_path and manifest_path.exists() and manifest_path.stat().st_size > 10:
+        print(f"Reading manifest: {manifest_path}")
+        with open(manifest_path, encoding="utf-8") as f:
+            manifest = json.load(f)
+        project_root = manifest_path.parent.parent
+        parsed = parse_manifest(manifest, project_root)
+        source_label = "manifest.json"
+    else:
+        summary_path = find_graph_summary(cwd)
+        if summary_path is None:
+            print(
+                "ERROR: No manifest.json or graph_summary.json found with content.\n"
+                "Run `dbt compile` or `dbt parse` first, then retry.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        print(f"manifest.json empty — falling back to: {summary_path}")
+        with open(summary_path, encoding="utf-8") as f:
+            summary = json.load(f)
+        project_root = summary_path.parent.parent
+        parsed = parse_graph_summary(summary, project_root)
+        source_label = "graph_summary.json"
+
+    project_name = parsed.get("project_name", "dbt_project")
+    nodes = parsed["nodes"]
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # ── Community detection ───────────────────────────────────────────────────
+    uid_to_community, community_labels = build_community_map(nodes)
+
+    # ── graph.json ────────────────────────────────────────────────────────────
+    graph = build_graph_json(parsed, project_name, uid_to_community, community_labels)
+    with open(out_dir / "graph.json", "w", encoding="utf-8") as f:
+        json.dump(graph, f, indent=2)
+
+    # ── lineage.json ──────────────────────────────────────────────────────────
+    lineage = build_lineage_json(nodes, parsed)
+    with open(out_dir / "lineage.json", "w", encoding="utf-8") as f:
+        json.dump(lineage, f, indent=2)
+
+    # ── GRAPH_REPORT.md ───────────────────────────────────────────────────────
+    write_report(parsed, out_dir, project_name, lineage, uid_to_community, community_labels)
+
+    # ── Sidecar files ─────────────────────────────────────────────────────────
+    write_graphify_root(out_dir, project_root)
+    write_graphify_labels(out_dir, community_labels)
+
+    # ── graph.html via graphify ───────────────────────────────────────────────
+    # cluster-only needs the directory that *contains* graphify-out/, not the dbt project root
+    if not skip_html:
+        generate_html(out_dir, out_dir.parent.resolve())
+
+    # ── Summary ───────────────────────────────────────────────────────────────
+    layer_counts: dict[str, int] = defaultdict(int)
+    for n in nodes:
+        layer_counts[n["layer"]] += 1
+
+    n_nodes = len(graph["nodes"])
+    n_edges = len(graph["edges"])
+
+    print(f"\n✓ {out_dir}/graph.json          ({n_nodes} nodes, {n_edges} edges) [source: {source_label}]")
+    print(f"✓ {out_dir}/lineage.json")
+    print(f"✓ {out_dir}/GRAPH_REPORT.md")
+    print(f"✓ {out_dir}/.graphify_root")
+    print(f"✓ {out_dir}/.graphify_labels.json")
+    print(f"\nNode breakdown:")
+    for layer in ["source", "staging", "intermediate", "mart", "seed", "macro"]:
+        if layer_counts[layer]:
+            print(f"  {layer:14s} {layer_counts[layer]}")
+    print(f"\nQuery the graph:")
+    print(f'  graphify query "which models use stg_students?"')
+    print(f'  graphify query "trace the lineage of student_at_risk_report"')
+    print(f'  graphify query "what breaks if I change stg_departments?"')
+
+
+def _print_help():
+    print(
+        "dbt-graphify — parse dbt manifest.json into a graphify knowledge graph\n"
+        "\n"
+        "Usage:\n"
+        "  dbt-graphify [manifest.json] [--out DIR] [--no-html]\n"
+        "  python -m dbt_graphify [manifest.json] [--out DIR] [--no-html]\n"
+        "\n"
+        "Arguments:\n"
+        "  manifest.json   Path to dbt manifest.json (auto-detected if omitted)\n"
+        "  --out DIR       Output directory (default: graphify-out/)\n"
+        "  --no-html       Skip graph.html generation via graphify cluster-only\n"
+        "\n"
+        "Outputs:\n"
+        "  graphify-out/graph.json          NetworkX node-link graph\n"
+        "  graphify-out/lineage.json        Ancestor/descendant blast-radius maps\n"
+        "  graphify-out/GRAPH_REPORT.md     Human-readable architecture summary\n"
+        "  graphify-out/.graphify_root      Project root path\n"
+        "  graphify-out/.graphify_labels.json Community labels\n"
+        "  graphify-out/graph.html          Interactive visualization (via graphify)\n"
+    )

dbt_graphify/parser.py

@@ -0,0 +1,340 @@
+"""Parse dbt manifest.json or graph_summary.json into a unified graph dict."""
+
+import json
+import re
+from collections import defaultdict
+from pathlib import Path
+
+
+# ── Layer inference ───────────────────────────────────────────────────────────
+
+def infer_layer(resource_type: str, name: str, fqn: list[str]) -> str:
+    if resource_type == "source":
+        return "source"
+    if resource_type == "seed":
+        return "seed"
+    if resource_type in ("test", "analysis"):
+        return resource_type
+    if resource_type == "macro":
+        return "macro"
+    # model — name prefix first
+    if name.startswith("stg_"):
+        return "staging"
+    if name.startswith("int_"):
+        return "intermediate"
+    # fqn segments
+    mart_segments = {"marts", "mart", "core", "academic", "finance", "financial", "reporting"}
+    staging_segments = {"staging", "stg"}
+    intermediate_segments = {"intermediate", "int"}
+    for seg in fqn:
+        seg_lower = seg.lower()
+        if seg_lower in staging_segments:
+            return "staging"
+        if seg_lower in intermediate_segments:
+            return "intermediate"
+        if seg_lower in mart_segments:
+            return "mart"
+    return "mart"
+
+
+# ── SQL extraction ────────────────────────────────────────────────────────────
+
+def extract_refs_from_sql(sql: str) -> list[str]:
+    return re.findall(r"\{\{\s*ref\s*\(\s*['\"](\w+)['\"]\s*\)\s*\}\}", sql)
+
+
+def extract_sources_from_sql(sql: str) -> list[tuple[str, str]]:
+    return re.findall(
+        r"\{\{\s*source\s*\(\s*['\"](\w+)['\"]\s*,\s*['\"](\w+)['\"]\s*\)\s*\}\}", sql
+    )
+
+
+def _extract_columns_sql(sql: str) -> list[str]:
+    """Best-effort: extract column alias names from the final SELECT clause."""
+    clean = re.sub(r"\{\{.*?\}\}", "__jinja__", sql, flags=re.DOTALL)
+    clean = re.sub(r"\{%-?.*?-?%\}", "", clean, flags=re.DOTALL)
+
+    selects = list(re.finditer(r"\bselect\b(.*?)\bfrom\b", clean, re.IGNORECASE | re.DOTALL))
+    if not selects:
+        return []
+
+    clause = selects[-1].group(1).strip()
+    if clause.strip() == "*" and len(selects) > 1:
+        clause = selects[-2].group(1).strip()
+
+    depth, current, parts = 0, [], []
+    for ch in clause:
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            depth -= 1
+        elif ch == "," and depth == 0:
+            parts.append("".join(current).strip())
+            current = []
+            continue
+        current.append(ch)
+    if current:
+        parts.append("".join(current).strip())
+
+    result = []
+    for part in parts:
+        part = part.strip()
+        if not part:
+            continue
+        alias = re.search(r"\bas\s+(\w+)\s*$", part, re.IGNORECASE)
+        if alias:
+            result.append(alias.group(1))
+        else:
+            word = re.search(r"(\w+)\s*$", part)
+            if word:
+                result.append(word.group(1))
+    return result
+
+
+# ── File discovery ────────────────────────────────────────────────────────────
+
+def find_manifest(start: Path) -> Path | None:
+    candidates = [
+        start / "target" / "manifest.json",
+        start / "dbt_project" / "target" / "manifest.json",
+        start / "manifest.json",
+    ]
+    for c in candidates:
+        if c.exists() and c.stat().st_size > 10:
+            return c
+    return None
+
+
+def find_graph_summary(start: Path) -> Path | None:
+    candidates = [
+        start / "target" / "graph_summary.json",
+        start / "dbt_project" / "target" / "graph_summary.json",
+        start / "graph_summary.json",
+    ]
+    for c in candidates:
+        if c.exists() and c.stat().st_size > 10:
+            return c
+    return None
+
+
+def _find_sql_file(name: str, project_root: Path) -> Path | None:
+    for p in project_root.rglob(f"{name}.sql"):
+        # Skip compiled/run output directories
+        if "compiled" not in p.parts and "run" not in p.parts:
+            return p
+    return None
+
+
+# ── Manifest parser ───────────────────────────────────────────────────────────
+
+def parse_manifest(manifest: dict, project_root: Path) -> dict:
+    """Parse dbt manifest.json into a unified graph dict."""
+    nodes_raw = manifest.get("nodes", {})
+    sources_raw = manifest.get("sources", {})
+    macros_raw = manifest.get("macros", {})
+    parent_map = manifest.get("parent_map", {})
+    child_map = manifest.get("child_map", {})
+
+    # Build upstream/downstream from parent_map
+    upstream: dict[str, set] = defaultdict(set)
+    downstream: dict[str, set] = defaultdict(set)
+    for node_id, parents in parent_map.items():
+        for parent_id in parents:
+            upstream[node_id].add(parent_id)
+            downstream[parent_id].add(node_id)
+
+    nodes = []
+
+    def _process_model(uid: str, raw: dict, resource_type: str):
+        name = raw.get("name", uid.split(".")[-1])
+        fqn = raw.get("fqn", [])
+        layer = infer_layer(resource_type, name, fqn)
+        config = raw.get("config", {})
+        description = raw.get("description", "").strip()
+        columns = list(raw.get("columns", {}).keys())
+        file_path = raw.get("original_file_path", raw.get("path", ""))
+        raw_code = raw.get("raw_code", raw.get("raw_sql", ""))
+        tags = raw.get("tags", [])
+        group = raw.get("group", None)
+
+        refs_in_code = extract_refs_from_sql(raw_code) if raw_code else []
+        sources_in_code = extract_sources_from_sql(raw_code) if raw_code else []
+
+        # Enrich from SQL file if code or columns missing
+        if (not raw_code or not columns) and project_root and file_path:
+            sql_path = project_root / file_path
+            if not sql_path.exists():
+                found = _find_sql_file(name, project_root)
+                if found:
+                    sql_path = found
+            if sql_path.exists():
+                raw_code = sql_path.read_text(encoding="utf-8")
+                refs_in_code = extract_refs_from_sql(raw_code)
+                sources_in_code = extract_sources_from_sql(raw_code)
+                if not columns:
+                    columns = _extract_columns_sql(raw_code)
+                if not file_path:
+                    try:
+                        file_path = str(sql_path.relative_to(project_root.parent))
+                    except ValueError:
+                        file_path = str(sql_path)
+
+        up_ids = sorted(upstream.get(uid, set()))
+        down_ids = sorted(downstream.get(uid, set()))
+
+        nodes.append({
+            "uid": uid,
+            "name": name,
+            "resource_type": resource_type,
+            "layer": layer,
+            "description": description,
+            "file_path": file_path,
+            "config": config,
+            "columns": columns[:40],
+            "refs": refs_in_code,
+            "sources": [f"{s[0]}.{s[1]}" for s in sources_in_code],
+            "tags": tags,
+            "group": group,
+            "upstream": up_ids,
+            "downstream": down_ids,
+        })
+
+    for uid, raw in nodes_raw.items():
+        rt = raw.get("resource_type", "model")
+        if rt in ("test", "analysis"):
+            continue
+        _process_model(uid, raw, rt)
+
+    for uid, raw in sources_raw.items():
+        name = raw.get("name", "")
+        source_name = raw.get("source_name", "")
+        description = raw.get("description", "").strip()
+        columns = list(raw.get("columns", {}).keys())
+        down_ids = sorted(downstream.get(uid, set()))
+
+        nodes.append({
+            "uid": uid,
+            "name": name,
+            "resource_type": "source",
+            "layer": "source",
+            "description": description or f"Source table: {source_name}.{name}",
+            "file_path": raw.get("original_file_path", ""),
+            "config": {"materialized": "external"},
+            "columns": columns[:40],
+            "refs": [],
+            "sources": [],
+            "tags": raw.get("tags", []),
+            "group": raw.get("group", None),
+            "schema": source_name,
+            "upstream": [],
+            "downstream": down_ids,
+        })
+
+    for uid, raw in macros_raw.items():
+        name = raw.get("name", uid.split(".")[-1])
+        nodes.append({
+            "uid": uid,
+            "name": name,
+            "resource_type": "macro",
+            "layer": "macro",
+            "description": raw.get("description", "").strip() or f"Jinja2 macro: {name}",
+            "file_path": raw.get("original_file_path", ""),
+            "config": {"materialized": "macro"},
+            "columns": [],
+            "refs": [],
+            "sources": [],
+            "tags": [],
+            "group": None,
+            "upstream": [],
+            "downstream": [],
+        })
+
+    meta = manifest.get("metadata", {})
+    return {
+        "meta": meta,
+        "project_name": meta.get("project_name", "dbt_project"),
+        "nodes": nodes,
+        "upstream": {k: list(v) for k, v in upstream.items()},
+        "downstream": {k: list(v) for k, v in downstream.items()},
+    }
+
+
+# ── graph_summary.json fallback ───────────────────────────────────────────────
+
+def parse_graph_summary(summary: dict, project_root: Path) -> dict:
+    """Fallback: parse dbt graph_summary.json + raw SQL files."""
+    linked = summary.get("linked", {})
+
+    id_to_name: dict[int, str] = {}
+    id_to_type: dict[int, str] = {}
+    for nid, nd in linked.items():
+        id_to_name[int(nid)] = nd["name"]
+        id_to_type[int(nid)] = nd["type"]
+
+    upstream: dict[str, set] = defaultdict(set)
+    downstream: dict[str, set] = defaultdict(set)
+
+    for nid_str, nd in linked.items():
+        nid = int(nid_str)
+        if nd["type"] == "test":
+            continue
+        src_name = nd["name"]
+        for succ_id in nd.get("succ", []):
+            tgt_name = id_to_name[succ_id]
+            if id_to_type[succ_id] == "test":
+                continue
+            upstream[tgt_name].add(src_name)
+            downstream[src_name].add(tgt_name)
+
+    nodes = []
+    for nid_str, nd in linked.items():
+        node_type = nd["type"]
+        if node_type == "test":
+            continue
+
+        full_name = nd["name"]
+        short_name = full_name.split(".")[-1]
+        fqn = full_name.split(".")
+        layer = infer_layer(node_type, short_name, fqn)
+
+        columns, refs_in_code, sources_in_code, file_path = [], [], [], ""
+
+        if node_type == "model" and project_root:
+            sql_file = _find_sql_file(short_name, project_root)
+            if sql_file:
+                try:
+                    file_path = str(sql_file.relative_to(project_root.parent))
+                except ValueError:
+                    file_path = str(sql_file)
+                sql = sql_file.read_text(encoding="utf-8")
+                refs_in_code = extract_refs_from_sql(sql)
+                sources_in_code = extract_sources_from_sql(sql)
+                columns = _extract_columns_sql(sql)
+
+        up_ids = sorted(upstream.get(full_name, set()))
+        down_ids = sorted(downstream.get(full_name, set()))
+
+        nodes.append({
+            "uid": full_name,
+            "name": short_name,
+            "resource_type": node_type,
+            "layer": layer,
+            "description": "",
+            "file_path": file_path,
+            "config": {},
+            "columns": columns[:40],
+            "refs": refs_in_code,
+            "sources": [f"{s[0]}.{s[1]}" for s in sources_in_code],
+            "tags": [],
+            "group": None,
+            "upstream": up_ids,
+            "downstream": down_ids,
+        })
+
+    return {
+        "meta": {"source": "graph_summary.json"},
+        "project_name": "dbt_project",
+        "nodes": nodes,
+        "upstream": {k: list(v) for k, v in upstream.items()},
+        "downstream": {k: list(v) for k, v in downstream.items()},
+    }

dbt_graphify-0.1.0.dist-info/METADATA

@@ -0,0 +1,181 @@
+Metadata-Version: 2.4
+Name: dbt-graphify
+Version: 0.1.0
+Summary: Parse a dbt manifest.json into a graphify-queryable knowledge graph
+License-Expression: MIT
+Keywords: dbt,graphify,lineage,knowledge-graph,data-catalog
+Author: Ponder Labs
+Requires-Python: >=3.9
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Database
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: all
+Provides-Extra: clustering
+Provides-Extra: yaml
+Requires-Dist: PyYAML (>=6.0) ; extra == "all"
+Requires-Dist: PyYAML (>=6.0) ; extra == "yaml"
+Requires-Dist: networkx (>=3.0) ; extra == "all"
+Requires-Dist: networkx (>=3.0) ; extra == "clustering"
+Project-URL: Homepage, https://github.com/ponder-co/dbt-graphify
+Project-URL: Repository, https://github.com/ponder-co/dbt-graphify
+Description-Content-Type: text/markdown
+
+# dbt-graphify
+
+Parse a dbt `manifest.json` into a [graphify](https://github.com/graphifyy/graphifyy)-queryable knowledge graph. Ask AI questions about your dbt lineage without reading hundreds of SQL files.
+
+```
+dbt manifest.json  ──►  dbt-graphify  ──►  graphify-out/graph.json
+                                                    │
+                              ┌─────────────────────┤
+                              ▼                     ▼
+                    graphify query              graph.html
+                  "what breaks if I            (interactive
+                  change stg_orders?"           D3 viewer)
+```
+
+## Why
+
+Graphify can't read dbt projects directly — Jinja2 SQL (`{{ ref('model') }}`) breaks its extractor, and it doesn't understand dbt's layer semantics (staging → intermediate → mart). The result is a noisy, incomplete graph that costs more tokens to query than just grepping the files.
+
+dbt already computes the perfect graph in `manifest.json`. This tool reshapes it into the format graphify expects, with zero LLM calls.
+
+## Installation
+
+```bash
+pip install dbt-graphify
+
+# With topology-based community detection (recommended):
+pip install "dbt-graphify[clustering]"
+
+# With YAML source description parsing:
+pip install "dbt-graphify[yaml]"
+
+# Everything:
+pip install "dbt-graphify[all]"
+```
+
+No runtime dependencies without extras — stdlib only.
+
+## Usage
+
+```bash
+# Auto-detect manifest.json in standard dbt locations
+dbt-graphify
+
+# Explicit manifest path
+dbt-graphify path/to/target/manifest.json
+
+# Custom output directory
+dbt-graphify --out /tmp/my-graph
+
+# Skip graph.html generation
+dbt-graphify --no-html
+
+# Module form (no install required)
+python -m dbt_graphify
+```
+
+### Fallback: no compiled manifest
+
+If `manifest.json` is empty (dbt not compiled), the tool falls back to `graph_summary.json`, which dbt writes even during `dbt parse`:
+
+```bash
+cd your-dbt-project/
+dbt parse               # fast, no DB connection needed
+dbt-graphify            # auto-finds target/graph_summary.json
+```
+
+## Output
+
+All files written to `graphify-out/` (or `--out` dir):
+
+| File | Description |
+|---|---|
+| `graph.json` | NetworkX node-link graph — loaded by `graphify query` |
+| `lineage.json` | Full ancestor/descendant maps for blast-radius analysis |
+| `GRAPH_REPORT.md` | Human-readable architecture summary with blast-radius table |
+| `.graphify_root` | Project root path (read by graphify for incremental updates) |
+| `.graphify_labels.json` | Community integer → label name mapping |
+| `graph.html` | Interactive D3 visualization (generated via `graphify cluster-only`) |
+
+## Querying with graphify
+
+After running `dbt-graphify`, use [graphify](https://github.com/graphifyy/graphifyy) to query the graph:
+
+```bash
+graphify query "which models depend on stg_customers?"
+graphify query "trace the full lineage of orders_mart"
+graphify query "what breaks if I change stg_payments?"
+graphify path "raw_orders" "revenue_report"
+graphify explain "int_order_metrics"
+```
+
+### Claude Code integration
+
+Install the graphify skill and run `/dbt-graphify` to regenerate the graph, then use `/graphify query` for any lineage question:
+
+```bash
+graphify install claude   # installs the graphify skill + hook
+```
+
+After that, any `graphify query` or `graphify path` command in Claude Code will use the knowledge graph instead of reading raw SQL files — typically 70–90% fewer tokens per question.
+
+## Node structure
+
+Each node in `graph.json` carries these flat fields (readable by `graphify query`):
+
+| Field | Example |
+|---|---|
+| `id` | `model.my_project.stg_orders` |
+| `label` | `stg_orders` |
+| `layer` | `staging` |
+| `type` | `model` |
+| `materialized` | `view` |
+| `description` | `Cleans raw orders. Derives: status_label, days_to_ship.` |
+| `source_file` | `models/staging/stg_orders.sql` |
+| `community` | `1` |
+| `upstream` | `["source.my_project.raw.orders"]` |
+| `downstream` | `["model.my_project.int_order_metrics", ...]` |
+| `columns` | `["order_id", "customer_id", "status_label", ...]` |
+
+## Community detection
+
+Communities are assigned via this priority chain so the tool works on any dbt project, not just a specific domain:
+
+1. **dbt `group`** — if your models use [dbt groups](https://docs.getdbt.com/docs/build/groups), each group becomes a community
+2. **dbt `tags`** — first tag on each model becomes the community
+3. **Topology clustering** — Louvain modularity on the DAG (requires `pip install "dbt-graphify[clustering]"`)
+4. **Layer fallback** — source / staging / intermediate / mart / seed (always available)
+
+## Blast-radius analysis
+
+`lineage.json` contains full ancestor and descendant maps computed via BFS:
+
+```python
+import json
+
+with open("graphify-out/lineage.json") as f:
+    lineage = json.load(f)
+
+# Everything downstream of stg_customers
+affected = lineage["descendants"]["model.my_project.stg_customers"]
+print(f"Changing stg_customers breaks: {affected}")
+```
+
+## Requirements
+
+- Python ≥ 3.9
+- A dbt project with `dbt compile` or `dbt parse` run (produces `target/manifest.json` or `target/graph_summary.json`)
+- [graphify](https://github.com/graphifyy/graphifyy) (`pip install graphifyy`) for `graph.html` and query CLI — optional but recommended
+
+## License
+
+MIT
+

dbt_graphify-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+dbt_graphify/__init__.py,sha256=UJ-5DtBgzGJeV64yM2JkKB-YWwQ0Fv2K9NmivOREB4k,112
+dbt_graphify/__main__.py,sha256=Xxg8lpMbaSrz_E_p1ixA9ZzmWZTvltVqSzZovvo2aKE,42
+dbt_graphify/builder.py,sha256=7JKbZWWI67gBsI1NABXTydFOW7JH6KnVNJLocSV8wgU,17110
+dbt_graphify/cli.py,sha256=wsaR_8zysjZQLaoyLa4kYvqG_asOT9glvRuCYWUpBkk,6910
+dbt_graphify/parser.py,sha256=-KNNsZ5Cql2BoRoQuN8GKH8EZZJnSah9e6UmW_VnygQ,12328
+dbt_graphify-0.1.0.dist-info/METADATA,sha256=MJWFxk0DQDUf8rlG-rUW0UKqnmivfBGZyCoWhETETR4,6460
+dbt_graphify-0.1.0.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
+dbt_graphify-0.1.0.dist-info/entry_points.txt,sha256=O4eNv-wgpxqgaQz0KdXPLygY9aksEUe7GWGmm_aBahE,54
+dbt_graphify-0.1.0.dist-info/RECORD,,

dbt_graphify-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.4.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dbt_graphify-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dbt-graphify=dbt_graphify.cli:main
+