codebeacon 0.1.2__py3-none-any.whl

codebeacon/graph/cluster.py

@@ -0,0 +1,160 @@
+"""Community detection for the codebeacon knowledge graph.
+
+Attempts clustering in order of quality:
+  1. graspologic Leiden   — best quality, requires: pip install graspologic
+  2. leidenalg            — good quality, requires: pip install leidenalg igraph
+  3. NetworkX Louvain     — built into networkx >= 3.0 (seed-stable)
+  4. Weakly connected components — always available fallback
+
+Public API:
+    cluster(G)              → dict[node_id, community_id]
+    apply_communities(G, communities)  → writes community attr to G nodes
+    score_all(G, communities)          → dict[community_id, cohesion_score]
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Optional
+
+import networkx as nx
+
+
+def cluster(G: nx.DiGraph) -> dict[str, int]:
+    """Detect communities in the graph.
+
+    Returns:
+        node_id → community_id mapping (community IDs are consecutive integers
+        starting from 0).
+    """
+    if G.number_of_nodes() == 0:
+        return {}
+
+    result = _try_graspologic(G)
+    if result is not None:
+        return result
+
+    result = _try_leidenalg(G)
+    if result is not None:
+        return result
+
+    result = _try_louvain(G)
+    if result is not None:
+        return result
+
+    return _connected_components(G)
+
+
+def apply_communities(G: nx.DiGraph, communities: dict[str, int]) -> None:
+    """Write community labels back as node attributes (in-place)."""
+    for node_id, community_id in communities.items():
+        if node_id in G:
+            G.nodes[node_id]["community"] = community_id
+
+
+def score_all(G: nx.DiGraph, communities: dict[str, int]) -> dict[int, float]:
+    """Compute a simple edge-based cohesion score for each community.
+
+    Cohesion = internal_edges / (internal_edges + boundary_edges).
+    A score of 1.0 means all edges stay within the community.
+
+    Returns:
+        community_id → cohesion score (0.0–1.0).
+    """
+    if not communities:
+        return {}
+
+    # Build community → member set
+    community_nodes: dict[int, set[str]] = {}
+    for node, cid in communities.items():
+        community_nodes.setdefault(cid, set()).add(node)
+
+    scores: dict[int, float] = {}
+    for cid, members in community_nodes.items():
+        internal = sum(
+            1 for u, v in G.edges()
+            if u in members and v in members
+        )
+        boundary = sum(
+            1 for u, v in G.edges()
+            if (u in members) != (v in members)
+        )
+        total = internal + boundary
+        scores[cid] = internal / total if total > 0 else 1.0
+
+    return scores
+
+
+# ── Algorithm implementations ─────────────────────────────────────────────────
+
+def _try_graspologic(G: nx.DiGraph) -> Optional[dict[str, int]]:
+    """Leiden via graspologic."""
+    try:
+        from graspologic.partition import leiden
+
+        UG = G.to_undirected()
+        if UG.number_of_edges() == 0:
+            return None
+
+        communities, _ = leiden(UG)
+        return {str(k): int(v) for k, v in communities.items()}
+    except ImportError:
+        return None
+    except Exception as exc:
+        warnings.warn(f"graspologic leiden failed: {exc}", stacklevel=2)
+        return None
+
+
+def _try_leidenalg(G: nx.DiGraph) -> Optional[dict[str, int]]:
+    """Leiden via leidenalg + python-igraph."""
+    try:
+        import leidenalg
+        import igraph as ig
+
+        UG = G.to_undirected()
+        if UG.number_of_edges() == 0:
+            return None
+
+        nodes = list(UG.nodes())
+        node_idx = {n: i for i, n in enumerate(nodes)}
+        edges = [(node_idx[u], node_idx[v]) for u, v in UG.edges()]
+
+        g = ig.Graph(n=len(nodes), edges=edges, directed=False)
+        partition = leidenalg.find_partition(g, leidenalg.ModularityVertexPartition)
+
+        result: dict[str, int] = {}
+        for community_id, members in enumerate(partition):
+            for member_idx in members:
+                result[nodes[member_idx]] = community_id
+        return result
+    except ImportError:
+        return None
+    except Exception as exc:
+        warnings.warn(f"leidenalg failed: {exc}", stacklevel=2)
+        return None
+
+
+def _try_louvain(G: nx.DiGraph) -> Optional[dict[str, int]]:
+    """Louvain via networkx built-in (nx >= 3.0)."""
+    try:
+        UG = G.to_undirected()
+        if UG.number_of_edges() == 0:
+            return None
+
+        communities = nx.community.louvain_communities(UG, seed=42)
+        result: dict[str, int] = {}
+        for community_id, members in enumerate(communities):
+            for node in members:
+                result[node] = community_id
+        return result
+    except (AttributeError, Exception):
+        return None
+
+
+def _connected_components(G: nx.DiGraph) -> dict[str, int]:
+    """Fallback: weakly connected components as pseudo-communities."""
+    result: dict[str, int] = {}
+    for community_id, component in enumerate(nx.weakly_connected_components(G)):
+        for node in component:
+            result[node] = community_id
+    return result

codebeacon/graph/enrich.py

@@ -0,0 +1,206 @@
+"""Graph enrichment: HTTP API cross-service edges + shared DB entity edges.
+
+Two enrichment passes run AFTER the base graph is built by build.py:
+  1. enrich_http_api()  — frontend URL calls → backend controller routes (calls_api edges)
+  2. enrich_shared_db() — same DAO/Entity used by multiple services (shares_db_entity edges)
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import networkx as nx
+
+
+# Regexes to extract API URLs from frontend source files
+_API_URL_RES = [
+    re.compile(r'''(?:fetch|invoke|\$fetch)\s*\(\s*[`"']([^`"'$]+)[`"']'''),
+    re.compile(r'''axios\.\w+\s*\(\s*[`"']([^`"'$]+)[`"']'''),
+    re.compile(r'''(?:api|http|client)\.\w+\s*\(\s*[`"']([^`"'$]+)[`"']'''),
+    re.compile(r'''url\s*[:=]\s*[`"']([^`"'$]+)[`"']'''),
+    re.compile(r'''["'](/api/[^"'`\s]+)["'`]'''),
+]
+_URL_LIKE = re.compile(r'^/[a-zA-Z]')
+
+
+def _extract_api_urls(source_file: str) -> list[str]:
+    """Scan a source file for HTTP API URL patterns."""
+    try:
+        content = Path(source_file).read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return []
+    urls: set[str] = set()
+    for pat in _API_URL_RES:
+        for m in pat.finditer(content):
+            url = m.group(1).split("?")[0].split("#")[0].strip()
+            if _URL_LIKE.match(url):
+                urls.add(url)
+    return list(urls)
+
+
+def enrich_http_api(G: nx.DiGraph) -> int:
+    """Add calls_api edges where frontend URL patterns match backend route paths.
+
+    Strategy:
+    - Collect all 'route' nodes with their path attribute
+    - For each frontend component, scan its source file for API URL patterns
+    - Match URLs to routes: exact first, then parameterized
+
+    Returns:
+        Number of new calls_api edges added.
+    """
+    added = 0
+
+    # Build route lookup: normalized_path → (node_id, project)
+    route_map: dict[str, tuple[str, str]] = {}
+    for node_id, data in G.nodes(data=True):
+        if data.get("type") != "route":
+            continue
+        path = data.get("path", "")
+        if path:
+            proj = data.get("project", "")
+            route_map[_normalize_path(path)] = (node_id, proj)
+
+    if not route_map:
+        return 0
+
+    # Find component/class nodes and scan their source for API calls
+    for node_id, data in G.nodes(data=True):
+        if data.get("type") not in ("component", "class"):
+            continue
+        src_proj = data.get("project", "")
+
+        src_file = data.get("source_file", "")
+        if not src_file:
+            continue
+
+        # Use metadata api_calls if set, otherwise scan file
+        api_calls = data.get("api_calls", [])
+        if not api_calls:
+            api_calls = _extract_api_urls(src_file)
+
+        for url in api_calls:
+            normalized = _normalize_path(url)
+
+            # Exact match
+            if normalized in route_map:
+                target_id, target_proj = route_map[normalized]
+                # Only create cross-project edges (skip same-project)
+                if target_proj == src_proj:
+                    continue
+                if not G.has_edge(node_id, target_id):
+                    G.add_edge(
+                        node_id, target_id,
+                        relation="calls_api",
+                        confidence="EXTRACTED",
+                        confidence_score=1.0,
+                        source_file=src_file,
+                    )
+                    added += 1
+                continue
+
+            # Parameterized match: /api/users/123 → /api/users/:id
+            for route_path, (route_node_id, route_proj) in route_map.items():
+                if route_proj == src_proj:
+                    continue
+                if _paths_match(normalized, route_path):
+                    if not G.has_edge(node_id, route_node_id):
+                        G.add_edge(
+                            node_id, route_node_id,
+                            relation="calls_api",
+                            confidence="INFERRED",
+                            confidence_score=0.8,
+                            source_file=src_file,
+                        )
+                        added += 1
+                    break
+
+    return added
+
+
+def enrich_shared_db(G: nx.DiGraph) -> int:
+    """Add shares_db_entity edges when the same entity is accessed by multiple services.
+
+    Detection strategy:
+    - Find all entity nodes
+    - For each entity, collect which service/class nodes reference it (via any edge)
+    - If references span more than one project → add shares_db_entity between those projects
+
+    Returns:
+        Number of new shares_db_entity edges added.
+    """
+    added = 0
+
+    # Collect entity nodes
+    entity_ids: set[str] = {
+        node_id
+        for node_id, data in G.nodes(data=True)
+        if data.get("type") == "entity"
+    }
+
+    if not entity_ids:
+        return 0
+
+    for entity_id in entity_ids:
+        entity_data = G.nodes[entity_id]
+        entity_src = entity_data.get("source_file", "")
+
+        # Find all nodes that reference this entity (in-edges or out-edges)
+        referencing: list[str] = [
+            n for n in list(G.predecessors(entity_id)) + list(G.successors(entity_id))
+            if G.nodes[n].get("type") in ("class", "route", "component")
+        ]
+
+        # Group referencing nodes by project
+        project_reps: dict[str, str] = {}  # project → first node_id as representative
+        for ref_id in referencing:
+            proj = G.nodes[ref_id].get("project", "")
+            if proj and proj not in project_reps:
+                project_reps[proj] = ref_id
+
+        if len(project_reps) < 2:
+            continue  # only interesting when multiple projects share an entity
+
+        projects = list(project_reps.keys())
+        for i in range(len(projects)):
+            for j in range(i + 1, len(projects)):
+                src_rep = project_reps[projects[i]]
+                tgt_rep = project_reps[projects[j]]
+                if not G.has_edge(src_rep, tgt_rep):
+                    G.add_edge(
+                        src_rep, tgt_rep,
+                        relation="shares_db_entity",
+                        confidence="INFERRED",
+                        confidence_score=0.9,
+                        source_file=entity_src,
+                        shared_entity=entity_id,
+                    )
+                    added += 1
+
+    return added
+
+
+# ── URL / path utilities ──────────────────────────────────────────────────────
+
+def _normalize_path(path: str) -> str:
+    """Normalize a URL or route path for comparison."""
+    path = path.split("?")[0].split("#")[0]  # strip query + fragment
+    path = path.rstrip("/") or "/"
+    return path.lower()
+
+
+def _paths_match(url: str, route_pattern: str) -> bool:
+    """Check if a concrete URL matches a parameterized route pattern.
+
+    Handles :param, {param}, [param], and <param> styles.
+    """
+    # Convert all parameter styles to a regex segment
+    pattern = re.sub(r":[^/]+", r"[^/]+", route_pattern)
+    pattern = re.sub(r"\{[^}]+\}", r"[^/]+", pattern)
+    pattern = re.sub(r"\[[^\]]+\]", r"[^/]+", pattern)
+    pattern = re.sub(r"<[^>]+>", r"[^/]+", pattern)
+    try:
+        return bool(re.fullmatch(pattern, url))
+    except re.error:
+        return False

codebeacon/skill/SKILL.md

@@ -0,0 +1,127 @@
+---
+name: codebeacon
+description: Scan a codebase → AST extraction → knowledge graph → wiki + CLAUDE.md context map. Supports 17 frameworks (Spring Boot, NestJS, Django, FastAPI, Rails, Express, React, Vue, Angular, and more).
+trigger: /codebeacon
+---
+
+# /codebeacon
+
+Scan source code with AST analysis → build a knowledge graph → generate a navigable wiki + `CLAUDE.md` context map ready for AI agents.
+
+## Usage
+
+```
+/codebeacon                         # scan current directory
+/codebeacon <path>                  # scan specific path or workspace root
+/codebeacon <path> --update         # incremental: only reprocess changed files
+/codebeacon <path> --wiki-only      # regenerate wiki without re-extracting
+/codebeacon sync                    # sync from codebeacon.yaml (multi-project)
+/codebeacon serve <path>            # start MCP server pointing at .codebeacon/
+```
+
+## What You Must Do When Invoked
+
+If no path was given, use `.` (current directory). Do not ask the user for a path.
+
+Follow these steps in order.
+
+### Step 1 — Ensure codebeacon is installed
+
+```bash
+python3 -c "import codebeacon" 2>/dev/null || pip install codebeacon -q --break-system-packages 2>&1 | tail -5
+python3 -c "import sys; open('.codebeacon_python', 'w').write(sys.executable)"
+```
+
+In every subsequent bash block, replace `python3` with `$(cat .codebeacon_python)`.
+
+If import succeeds, print nothing and move to Step 2.
+
+### Step 2 — Detect mode and run
+
+Check if `codebeacon.yaml` exists in the target directory:
+
+```bash
+TARGET="${1:-.}"
+
+if [ -f "$TARGET/codebeacon.yaml" ]; then
+    echo "Found codebeacon.yaml — running sync mode"
+    $(cat .codebeacon_python) -m codebeacon sync --config "$TARGET/codebeacon.yaml"
+else
+    echo "Scanning $TARGET ..."
+    $(cat .codebeacon_python) -m codebeacon scan "$TARGET"
+fi
+```
+
+The command prints wave progress as it goes:
+- Framework detection per project
+- `[pct%] done/total files processed` (wave progress per project)
+- Route / Service / Entity / Component counts after each project
+- Final: `Nodes: N, Edges: E, Communities: K`
+
+Let it run to completion. Do not interrupt.
+
+### Step 3 — Report results
+
+After the command exits, read the REPORT.md:
+
+```bash
+TARGET="${1:-.}"
+OUTPUT_DIR="$TARGET/.codebeacon"
+[ -f "$OUTPUT_DIR/REPORT.md" ] && head -40 "$OUTPUT_DIR/REPORT.md"
+```
+
+Then summarise for the user:
+- Which projects/frameworks were detected
+- Total nodes, edges, communities
+- Output location (`.codebeacon/wiki/`, `.codebeacon/CLAUDE.md`, etc.)
+- Any god nodes or surprising connections worth mentioning
+
+### Step 4 — (Optional) MCP serve
+
+If the user asked for `serve`:
+
+```bash
+TARGET="${1:-.}"
+$(cat .codebeacon_python) -m codebeacon serve --dir "$TARGET/.codebeacon"
+```
+
+This blocks — run it only when the user explicitly wants an MCP server.
+
+## Output structure
+
+```
+.codebeacon/
+  beacon.json          ← full knowledge graph (node-link JSON)
+  REPORT.md            ← god nodes, surprising connections, hub files
+  CLAUDE.md            ← AI context map (also written to project root)
+  .cursorrules         ← Cursor IDE context
+  AGENTS.md            ← OpenAI Agents context
+  wiki/
+    index.md           ← global index (~200 tokens)
+    overview.md        ← platform stats + cross-project connections
+    routes.md          ← all routes table
+    cross-project/
+      connections.md   ← cross-service edges
+    <project>/
+      index.md
+      routes.md
+      controllers/<Name>.md
+      services/<Name>.md
+      entities/<Name>.md
+      components/<Name>.md
+  obsidian/            ← Obsidian vault (one note per node)
+```
+
+## Supported frameworks
+
+| Language | Frameworks |
+|----------|-----------|
+| Java/Kotlin | Spring Boot, Ktor |
+| Python | Django, FastAPI, Flask |
+| JavaScript/TypeScript | Express, NestJS, React, Vue, Angular, Svelte |
+| Go | Gin |
+| Ruby | Rails |
+| PHP | Laravel |
+| Rust | Actix-Web |
+| C# | ASP.NET Core |
+| Swift | Vapor |