codegraph-nav 0.1.0__py3-none-any.whl

codegraph_nav/graph/communities.py

@@ -0,0 +1,402 @@
+"""Community detection — file-based clustering with optional Leiden algorithm.
+
+Groups graph nodes into communities based on directory structure (always available)
+or Leiden algorithm (requires igraph). Computes cohesion and coupling metrics.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import cast
+
+from .store import GraphStore
+
+# Edge weights for Leiden (if igraph available)
+EDGE_WEIGHTS = {
+    "CALLS": 1.0,
+    "IMPORTS_FROM": 0.5,
+    "INHERITS": 0.8,
+    "CONTAINS": 0.3,
+    "TESTED_BY": 0.4,
+}
+
+# Check igraph availability
+try:
+    import igraph
+
+    HAS_IGRAPH = True
+except ImportError:
+    HAS_IGRAPH = False
+
+
+# ==============================================================================
+# Community Detection
+# ==============================================================================
+
+
+def detect_communities(store: GraphStore, min_size: int = 2) -> list[dict]:
+    """Detect communities. Uses Leiden if igraph available, file-based otherwise."""
+    if HAS_IGRAPH:
+        try:
+            return detect_communities_leiden(store, min_size)
+        except Exception:
+            pass  # Fall back to file-based
+    return detect_communities_file_based(store, min_size)
+
+
+def detect_communities_file_based(store: GraphStore, min_size: int = 2) -> list[dict]:
+    """Group non-File nodes by directory prefix (first 2 components)."""
+    nodes = store.get_all_nodes()
+    groups: dict[str, list] = defaultdict(list)
+
+    for node in nodes:
+        if node["kind"] == "File":
+            continue
+        fp = node["file_path"]
+        parts = Path(fp).parts
+        # Use first 2 directory components as group key
+        if len(parts) >= 2:
+            prefix = str(Path(parts[0]) / parts[1])
+        elif len(parts) == 1:
+            prefix = parts[0]
+        else:
+            prefix = "root"
+        groups[prefix].append(node)
+
+    communities = []
+    for prefix, members in groups.items():
+        if len(members) < min_size:
+            continue
+
+        member_ids = [m["id"] for m in members]
+        member_qns = {m["qualified_name"] for m in members}
+
+        # Compute cohesion
+        cohesion = _compute_cohesion(store, member_qns)
+
+        # Generate name
+        name = _generate_community_name(members, prefix)
+
+        # Extract keywords
+        keywords = _extract_keywords(members)
+
+        communities.append(
+            {
+                "name": name,
+                "node_count": len(members),
+                "cohesion": cohesion,
+                "file_prefix": prefix,
+                "keywords": keywords[:5],
+                "member_ids": member_ids,
+            }
+        )
+
+    communities.sort(key=lambda c: cast(int, c["node_count"]), reverse=True)
+    return communities
+
+
+def detect_communities_leiden(store: GraphStore, min_size: int = 2) -> list[dict]:
+    """Leiden community detection using igraph."""
+    if not HAS_IGRAPH:
+        return detect_communities_file_based(store, min_size)
+
+    nodes = [n for n in store.get_all_nodes() if n["kind"] != "File"]
+    if not nodes:
+        return []
+
+    # Build igraph
+    qn_to_idx = {n["qualified_name"]: i for i, n in enumerate(nodes)}
+    edges_data = []
+    weights = []
+
+    conn = store.conn
+    all_edges = conn.execute("SELECT * FROM edges").fetchall()
+
+    for edge in all_edges:
+        src_idx = qn_to_idx.get(edge["source_qualified"])
+        tgt_idx = qn_to_idx.get(edge["target_qualified"])
+        if src_idx is not None and tgt_idx is not None and src_idx != tgt_idx:
+            edges_data.append((src_idx, tgt_idx))
+            weights.append(EDGE_WEIGHTS.get(edge["kind"], 0.5))
+
+    if not edges_data:
+        return detect_communities_file_based(store, min_size)
+
+    g = igraph.Graph(n=len(nodes), edges=edges_data, directed=True)
+    g.es["weight"] = weights
+
+    # Run Leiden
+    partition = g.community_leiden(
+        objective_function="modularity",
+        weights="weight",
+        resolution=1.0,
+    )
+
+    # Build communities from partition
+    community_nodes: dict[int, list] = defaultdict(list)
+    for node_idx, comm_id in enumerate(partition.membership):
+        community_nodes[comm_id].append(nodes[node_idx])
+
+    communities = []
+    for comm_id, members in community_nodes.items():
+        if len(members) < min_size:
+            continue
+
+        member_ids = [m["id"] for m in members]
+        member_qns = {m["qualified_name"] for m in members}
+
+        # Compute cohesion
+        cohesion = _compute_cohesion(store, member_qns)
+
+        # File prefix from members
+        file_paths = [m["file_path"] for m in members]
+        prefix = _extract_file_prefix(file_paths)
+
+        name = _generate_community_name(members, prefix)
+        keywords = _extract_keywords(members)
+
+        communities.append(
+            {
+                "name": name,
+                "node_count": len(members),
+                "cohesion": cohesion,
+                "file_prefix": prefix,
+                "keywords": keywords[:5],
+                "member_ids": member_ids,
+            }
+        )
+
+    communities.sort(key=lambda c: cast(int, c["node_count"]), reverse=True)
+    return communities
+
+
+# ==============================================================================
+# Helpers
+# ==============================================================================
+
+
+def _compute_cohesion(store: GraphStore, member_qns: set[str]) -> float:
+    """Cohesion = internal_edges / (internal_edges + external_edges)."""
+    internal = 0
+    external = 0
+
+    for qn in member_qns:
+        for edge in store.get_edges_from(qn):
+            if edge["target_qualified"] in member_qns:
+                internal += 1
+            else:
+                external += 1
+
+    total = internal + external
+    if total == 0:
+        return 0.0
+    return round(internal / total, 4)
+
+
+def _generate_community_name(members: list, prefix: str) -> str:
+    """Generate community name from prefix + dominant class/keyword."""
+    # Find dominant class (>40% of members)
+    classes = [m["name"] for m in members if m["kind"] == "Class"]
+    if classes:
+        class_counts = Counter(classes)
+        top_class, top_count = class_counts.most_common(1)[0]
+        if top_count / len(members) > 0.4:
+            return f"{Path(prefix).name}-{top_class}"
+
+    # Use directory name + top keyword
+    keywords = _extract_keywords(members)
+    if keywords:
+        return f"{Path(prefix).name}-{keywords[0]}"
+
+    return Path(prefix).name
+
+
+def _extract_file_prefix(file_paths: list[str]) -> str:
+    """Find common directory prefix from file paths."""
+    if not file_paths:
+        return "root"
+    parts_list = [Path(fp).parts for fp in file_paths]
+    if not parts_list:
+        return "root"
+
+    prefix_parts = []
+    for i in range(min(len(p) for p in parts_list)):
+        vals = {p[i] for p in parts_list}
+        if len(vals) == 1:
+            prefix_parts.append(vals.pop())
+        else:
+            break
+
+    return str(Path(*prefix_parts)) if prefix_parts else "root"
+
+
+def _extract_keywords(members: list) -> list[str]:
+    """Extract frequent keywords from member names via camelCase/snake_case splitting."""
+    words: Counter = Counter()
+    stop_words = {
+        "get",
+        "set",
+        "is",
+        "has",
+        "do",
+        "to",
+        "from",
+        "on",
+        "the",
+        "a",
+        "an",
+        "test",
+        "self",
+        "init",
+        "new",
+        "create",
+        "make",
+        "build",
+    }
+
+    for m in members:
+        name = m["name"]
+        # Split camelCase and snake_case
+        parts = _split_name(name)
+        for part in parts:
+            part_lower = part.lower()
+            if len(part_lower) > 2 and part_lower not in stop_words:
+                words[part_lower] += 1
+
+    return [w for w, _ in words.most_common(5)]
+
+
+def _split_name(name: str) -> list[str]:
+    """Split camelCase and snake_case into words."""
+    # Snake case
+    if "_" in name:
+        return [p for p in name.split("_") if p]
+    # CamelCase
+    parts = re.sub(r"([A-Z])", r" \1", name).split()
+    return [p for p in parts if p]
+
+
+# ==============================================================================
+# Coupling Warnings
+# ==============================================================================
+
+
+def get_coupling_warnings(store: GraphStore, communities: list[dict]) -> list[str]:
+    """Detect cross-community coupling and oversized communities."""
+    warnings = []
+
+    # Build node → community mapping
+    node_to_community: dict[int, str] = {}
+    for comm in communities:
+        for nid in comm["member_ids"]:
+            node_to_community[nid] = comm["name"]
+
+    # Count cross-community edges
+    cross_edges: Counter = Counter()
+    conn = store.conn
+    all_edges = conn.execute(
+        "SELECT source_qualified, target_qualified FROM edges WHERE kind = 'CALLS'"
+    ).fetchall()
+
+    for edge in all_edges:
+        src_node = store.get_node(edge[0])
+        tgt_node = store.get_node(edge[1])
+        if src_node and tgt_node:
+            src_comm = node_to_community.get(src_node["id"])
+            tgt_comm = node_to_community.get(tgt_node["id"])
+            if src_comm and tgt_comm and src_comm != tgt_comm:
+                key = f"{src_comm}→{tgt_comm}"
+                cross_edges[key] += 1
+
+    # Top coupling warnings
+    for pair, count in cross_edges.most_common(5):
+        if count >= 3:
+            warnings.append(f"coupling: {pair} ({count} edges)")
+
+    # Oversized community warning
+    for comm in communities:
+        if comm["node_count"] > 50:
+            warnings.append(f"WARN: {comm['name']} too large ({comm['node_count']} nodes)")
+
+    # Low cohesion warning
+    for comm in communities:
+        if comm["cohesion"] < 0.3 and comm["node_count"] > 5:
+            warnings.append(f"WARN: {comm['name']} low cohesion ({comm['cohesion']:.2f})")
+
+    return warnings
+
+
+# ==============================================================================
+# Persistence
+# ==============================================================================
+
+
+def store_communities(store: GraphStore, communities: list[dict]):
+    """Save communities to DB."""
+    store.clear_communities()
+    for comm in communities:
+        cid = store.insert_community(
+            name=comm["name"],
+            cohesion=comm["cohesion"],
+            node_count=comm["node_count"],
+            file_prefix=comm.get("file_prefix"),
+            keywords=comm.get("keywords"),
+        )
+        store.add_community_members(cid, comm["member_ids"])
+
+
+# ==============================================================================
+# Formatters
+# ==============================================================================
+
+
+def format_communities_minimal(communities: list[dict], limit: int = 10) -> str:
+    """One line per community."""
+    if not communities:
+        return "No communities detected."
+
+    lines = [f"{len(communities)} communities:"]
+    for comm in communities[:limit]:
+        kw = ",".join(comm.get("keywords", [])[:3])
+        kw_str = f" [{kw}]" if kw else ""
+        lines.append(
+            f"  {comm['name']} size:{comm['node_count']} "
+            f"cohesion:{comm['cohesion']:.2f}{kw_str}"
+        )
+    if len(communities) > limit:
+        lines.append(f"  ... +{len(communities) - limit} more")
+    return "\n".join(lines)
+
+
+def format_architecture_overview(
+    communities: list[dict],
+    coupling_warnings: list[str],
+    hubs: list[dict],
+    flows_count: int,
+) -> str:
+    """Compact architecture summary (<150 tokens)."""
+    lines = []
+
+    # Communities summary
+    if communities:
+        top = communities[:5]
+        comm_strs = [f"{c['name']}({c['node_count']})" for c in top]
+        lines.append(f"communities({len(communities)}): {', '.join(comm_strs)}")
+
+    # Coupling warnings
+    if coupling_warnings:
+        for w in coupling_warnings[:3]:
+            lines.append(f"  {w}")
+
+    # Hubs
+    if hubs:
+        hub_strs = [f"{h['file']}({h['imports']}←)" for h in hubs[:3]]
+        lines.append(f"hubs: {', '.join(hub_strs)}")
+
+    # Flows
+    if flows_count:
+        lines.append(f"flows: {flows_count}")
+
+    return "\n".join(lines) if lines else "No architecture data available."

codegraph_nav/graph/flows.py

@@ -0,0 +1,311 @@
+"""Execution flow detection, BFS tracing, and criticality scoring.
+
+Detects entry points via decorator patterns, conventional names, and
+graph topology (no incoming CALLS). Traces flows via forward BFS.
+Scores criticality using a 5-factor weighted formula.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import deque
+
+from .schema import MAX_BFS_DEPTH, SECURITY_KEYWORDS
+from .store import GraphStore
+
+# ==============================================================================
+# Entry Point Detection
+# ==============================================================================
+
+ENTRY_DECORATOR_PATTERN = re.compile(
+    r"@(app|router|blueprint)\.(get|post|put|delete|patch|route|websocket)"
+    r"|@click\.(command|group)"
+    r"|@celery\.task"
+    r"|@(api_view|action)"
+    r"|@(Get|Post|Put|Delete|Patch|RequestMapping)",
+    re.IGNORECASE,
+)
+
+ENTRY_NAME_PATTERN = re.compile(
+    r"^(main|__main__|cli|run|start|setup|app|create_app|entrypoint|handler)$"
+    r"|^test_"
+    r"|^(on_|handle_|process_|dispatch_)",
+    re.IGNORECASE,
+)
+
+
+def detect_entry_points(store: GraphStore) -> list:
+    """Find execution entry points in the graph.
+
+    An entry point is a Function/Method node that satisfies ANY of:
+    1. No incoming CALLS edges (true root)
+    2. Has a framework decorator (e.g., @app.get, @click.command)
+    3. Has a conventional entry point name (main, test_*, handle_*)
+
+    Returns list of node Rows.
+    """
+    conn = store.conn
+
+    # Get all Function/Method nodes
+    candidates = conn.execute("SELECT * FROM nodes WHERE kind IN ('Function', 'Method')").fetchall()
+
+    entry_points = []
+    seen = set()
+
+    for node in candidates:
+        qn = node["qualified_name"]
+        if qn in seen:
+            continue
+
+        is_entry = False
+
+        # Strategy 1: No incoming CALLS
+        incoming_calls = store.get_edges_to(qn, kind="CALLS")
+        if not incoming_calls:
+            is_entry = True
+
+        # Strategy 2: Decorator pattern
+        if not is_entry:
+            extra = node["extra"] or "{}"
+            try:
+                extra_data = json.loads(extra)
+            except (json.JSONDecodeError, TypeError):
+                extra_data = {}
+            decorators = extra_data.get("decorators", [])
+            for dec in decorators:
+                if ENTRY_DECORATOR_PATTERN.search(dec):
+                    is_entry = True
+                    break
+
+        # Strategy 3: Name pattern
+        if not is_entry and ENTRY_NAME_PATTERN.search(node["name"]):
+            is_entry = True
+
+        if is_entry:
+            seen.add(qn)
+            entry_points.append(node)
+
+    return entry_points
+
+
+# ==============================================================================
+# BFS Flow Tracing
+# ==============================================================================
+
+
+def trace_flows(
+    store: GraphStore,
+    max_depth: int = MAX_BFS_DEPTH,
+    limit: int = 100,
+) -> list[dict]:
+    """Trace execution flows from entry points via forward BFS.
+
+    Returns list of flow dicts, each with: name, entry_point, path, depth,
+    node_count, file_count, criticality.
+    """
+    entry_points = detect_entry_points(store)
+
+    # Skip test entry points for flow tracing (too many)
+    entry_points = [ep for ep in entry_points if not ep["is_test"]]
+
+    flows = []
+    for ep in entry_points[:limit]:
+        flow = _trace_single_flow(store, ep, max_depth)
+        if flow:
+            flows.append(flow)
+
+    # Sort by criticality
+    flows.sort(key=lambda f: f["criticality"], reverse=True)
+    return flows
+
+
+def _trace_single_flow(
+    store: GraphStore,
+    entry_point,
+    max_depth: int = MAX_BFS_DEPTH,
+) -> dict | None:
+    """BFS from a single entry point, following forward CALLS edges."""
+    queue = deque([(entry_point["qualified_name"], 0)])
+    visited = {entry_point["qualified_name"]}
+    path_ids = [entry_point["id"]]
+    path_names = [entry_point["name"]]
+    files = {entry_point["file_path"]}
+    max_reached_depth = 0
+
+    while queue:
+        current_qn, depth = queue.popleft()
+        if depth >= max_depth:
+            continue
+
+        # Follow forward CALLS edges
+        edges = store.get_edges_from(current_qn, kind="CALLS")
+        for edge in edges:
+            target_qn = edge["target_qualified"]
+            if target_qn in visited or target_qn.startswith("__unresolved__"):
+                continue
+            visited.add(target_qn)
+
+            target_node = store.get_node(target_qn)
+            if target_node:
+                path_ids.append(target_node["id"])
+                path_names.append(target_node["name"])
+                files.add(target_node["file_path"])
+                max_reached_depth = max(max_reached_depth, depth + 1)
+                queue.append((target_qn, depth + 1))
+
+    # Skip trivial flows
+    if len(path_ids) < 2:
+        return None
+
+    criticality = compute_criticality(path_ids, path_names, files, max_reached_depth, store)
+
+    return {
+        "name": entry_point["name"],
+        "entry_point": entry_point["qualified_name"],
+        "entry_point_id": entry_point["id"],
+        "path_ids": path_ids,
+        "path_names": path_names,
+        "depth": max_reached_depth,
+        "node_count": len(path_ids),
+        "file_count": len(files),
+        "files": sorted(files),
+        "criticality": criticality,
+    }
+
+
+# ==============================================================================
+# Criticality Scoring
+# ==============================================================================
+
+
+def compute_criticality(
+    path_ids: list[int],
+    path_names: list[str],
+    files: set[str],
+    depth: int,
+    store: GraphStore,
+) -> float:
+    """Compute flow criticality score (0.0–1.0).
+
+    Factors:
+    - file_spread (0.30):     How many files the flow touches
+    - external_calls (0.20):  Unresolved/external dependencies
+    - security_hits (0.25):   Symbols with security-related names
+    - test_gap (0.15):        Fraction of flow nodes without tests
+    - depth (0.10):           How deep the call chain goes
+    """
+    node_count = len(path_ids)
+    if node_count == 0:
+        return 0.0
+
+    # 1. File spread (0–1), weight 0.30
+    file_count = len(files)
+    file_spread = min((file_count - 1) / 4.0, 1.0) if file_count > 1 else 0.0
+
+    # 2. External calls (0–1), weight 0.20
+    # Count edges from flow nodes to unresolved targets
+    external_count = 0
+    for nid in path_ids:
+        node = store.get_node_by_id(nid)
+        if node:
+            edges = store.get_edges_from(node["qualified_name"], kind="CALLS")
+            external_count += sum(
+                1 for e in edges if e["target_qualified"].startswith("__unresolved__")
+            )
+    external_score = min(external_count / 5.0, 1.0)
+
+    # 3. Security sensitivity (0–1), weight 0.25
+    security_hits = 0
+    all_names = [n.lower() for n in path_names]
+    for name in all_names:
+        if any(kw in name for kw in SECURITY_KEYWORDS):
+            security_hits += 1
+    security_score = min(security_hits / max(node_count, 1), 1.0)
+
+    # 4. Test coverage gap (0–1), weight 0.15
+    tested_count = 0
+    for nid in path_ids:
+        node = store.get_node_by_id(nid)
+        if node:
+            tested_by = store.get_edges_to(node["qualified_name"], kind="TESTED_BY")
+            if tested_by:
+                tested_count += 1
+    coverage = tested_count / max(node_count, 1)
+    test_gap = 1.0 - coverage
+
+    # 5. Depth (0–1), weight 0.10
+    depth_score = min(depth / 10.0, 1.0)
+
+    criticality = (
+        file_spread * 0.30
+        + external_score * 0.20
+        + security_score * 0.25
+        + test_gap * 0.15
+        + depth_score * 0.10
+    )
+
+    return round(min(max(criticality, 0.0), 1.0), 4)
+
+
+# ==============================================================================
+# Flow Persistence
+# ==============================================================================
+
+
+def store_flows(store: GraphStore, flows: list[dict]):
+    """Persist traced flows to the database."""
+    store.clear_flows()
+    for flow in flows:
+        store.insert_flow(
+            name=flow["name"],
+            entry_point_id=flow["entry_point_id"],
+            depth=flow["depth"],
+            node_count=flow["node_count"],
+            file_count=flow["file_count"],
+            criticality=flow["criticality"],
+            path_ids=flow["path_ids"],
+        )
+
+
+# ==============================================================================
+# Formatters
+# ==============================================================================
+
+
+def format_flow_minimal(flow: dict) -> str:
+    """Format a single flow as compact string (<30 tokens).
+
+    Example: flow login → authenticate → verify_password [auth] crit:0.82 gaps:1
+    """
+    names = flow["path_names"][:5]
+    path_str = " → ".join(names)
+    if len(flow["path_names"]) > 5:
+        path_str += f" +{len(flow['path_names']) - 5}"
+
+    # Detect domain tags from names
+    tags = set()
+    for name in flow["path_names"]:
+        name_lower = name.lower()
+        if any(kw in name_lower for kw in ("auth", "login", "password", "token", "session")):
+            tags.add("auth")
+        if any(kw in name_lower for kw in ("db", "query", "sql", "model")):
+            tags.add("db")
+        if any(kw in name_lower for kw in ("http", "request", "api", "route")):
+            tags.add("api")
+
+    tag_str = f" [{','.join(sorted(tags))}]" if tags else ""
+    return f"flow {path_str}{tag_str} crit:{flow['criticality']:.2f}"
+
+
+def format_flows_minimal(flows: list[dict], limit: int = 10) -> str:
+    """Format flow list as compact string."""
+    if not flows:
+        return "No execution flows detected."
+
+    lines = [f"{len(flows)} flows detected:"]
+    for flow in flows[:limit]:
+        lines.append(f"  {format_flow_minimal(flow)}")
+    if len(flows) > limit:
+        lines.append(f"  ... +{len(flows) - limit} more")
+    return "\n".join(lines)