codebeacon 0.1.2__py3-none-any.whl

codebeacon/wave.py

@@ -0,0 +1,292 @@
+"""Automatic wave / segment processing (Pass 1).
+
+auto_wave() splits source files into chunks and processes them in parallel
+using a ThreadPoolExecutor. Each file is run through all extractors:
+  routes, services, entities, components, dependencies.
+
+Results are merged into a WaveResult.
+Pass 2 (symbol resolution + graph wiring) happens in graph/build.py after
+all waves complete.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import warnings
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+from codebeacon.common.types import (
+    ComponentInfo,
+    Edge,
+    EntityInfo,
+    ProjectInfo,
+    RouteInfo,
+    ServiceInfo,
+    UnresolvedRef,
+)
+
+
+@dataclass
+class WaveResult:
+    """Aggregated Pass-1 extraction results across all chunks for one project."""
+    project: ProjectInfo
+    routes: list[RouteInfo] = field(default_factory=list)
+    services: list[ServiceInfo] = field(default_factory=list)
+    entities: list[EntityInfo] = field(default_factory=list)
+    components: list[ComponentInfo] = field(default_factory=list)
+    import_edges: list[Edge] = field(default_factory=list)
+    unresolved: list[UnresolvedRef] = field(default_factory=list)
+    file_count: int = 0
+    skipped_count: int = 0   # cache hits
+
+
+# ── Single-file extraction ────────────────────────────────────────────────────
+
+def _extract_file(
+    file_path: str,
+    framework: str,
+    project_path: str,
+    cache=None,
+    semantic: bool = False,
+) -> Optional[dict]:
+    """Run all extractors on a single file.
+
+    Returns a plain dict (JSON-serializable) or None on hard failure.
+    The dict has a '_cache_hit' key if the result came from cache.
+    """
+    # Check cache before parsing
+    if cache is not None:
+        cached = cache.get(file_path)
+        if cached is not None:
+            return {"_cache_hit": True, **cached}
+
+    try:
+        from codebeacon.extract.routes import extract_routes
+        from codebeacon.extract.services import extract_services
+        from codebeacon.extract.entities import extract_entities
+        from codebeacon.extract.components import extract_components
+        from codebeacon.extract.dependencies import extract_dependencies
+
+        routes = extract_routes(file_path, framework, project_path)
+        services, unresolved = extract_services(file_path, framework)
+        entities = extract_entities(file_path, framework)
+        components = extract_components(file_path, framework, project_path)
+        import_edges = extract_dependencies(file_path, framework)
+
+        # Optional semantic extraction (structured comment parsing)
+        semantic_edges: list[Edge] = []
+        if semantic:
+            from codebeacon.extract.semantic import extract_semantic_refs
+            semantic_edges = extract_semantic_refs(file_path, framework)
+
+        result: dict[str, Any] = {
+            "routes": [_route_to_dict(r) for r in routes],
+            "services": [_service_to_dict(s) for s in services],
+            "entities": [_entity_to_dict(e) for e in entities],
+            "components": [_component_to_dict(c) for c in components],
+            "import_edges": [_edge_to_dict(e) for e in import_edges + semantic_edges],
+            "unresolved": [_unresolved_to_dict(u) for u in unresolved],
+        }
+
+        if cache is not None:
+            fh = cache.file_hash(file_path)
+            cache.put(file_path, result, fh)
+
+        return result
+
+    except Exception as exc:
+        warnings.warn(f"Extraction failed [{framework}] {file_path}: {exc}", stacklevel=2)
+        return None
+
+
+# ── Main public function ──────────────────────────────────────────────────────
+
+def auto_wave(
+    project: ProjectInfo,
+    files: list[str],
+    chunk_size: int = 300,
+    max_parallel: int = 5,
+    cache=None,
+    progress_callback: Optional[Callable[[int, int], None]] = None,
+    semantic: bool = False,
+) -> WaveResult:
+    """Process all files in parallel chunks and merge results (Pass 1).
+
+    Args:
+        project: the ProjectInfo for the project being scanned
+        files: absolute file paths to process
+        chunk_size: files per wave chunk (controls peak memory)
+        max_parallel: max ThreadPoolExecutor workers per chunk
+        cache: optional Cache instance for incremental processing
+        progress_callback: optional callable(processed_count, total_count)
+
+    Returns:
+        WaveResult with all extraction data merged.
+        Pass 2 (symbol resolve + graph wiring) is NOT done here.
+    """
+    wave_result = WaveResult(project=project, file_count=len(files))
+
+    if not files:
+        return wave_result
+
+    processed = 0
+    chunks = [files[i: i + chunk_size] for i in range(0, len(files), chunk_size)]
+
+    for chunk in chunks:
+        chunk_results = _process_chunk(chunk, project.framework, project.path, cache, max_parallel, semantic)
+        for file_result in chunk_results:
+            if file_result is None:
+                continue
+            if file_result.get("_cache_hit"):
+                wave_result.skipped_count += 1
+            _merge_file_result(file_result, wave_result)
+
+        processed += len(chunk)
+        if progress_callback:
+            progress_callback(processed, len(files))
+
+    return wave_result
+
+
+def _process_chunk(
+    chunk: list[str],
+    framework: str,
+    project_path: str,
+    cache,
+    max_workers: int,
+    semantic: bool = False,
+) -> list[Optional[dict]]:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {
+            pool.submit(_extract_file, fp, framework, project_path, cache, semantic): fp
+            for fp in chunk
+        }
+        results: list[Optional[dict]] = []
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                results.append(future.result())
+            except Exception as exc:
+                fp = futures[future]
+                warnings.warn(f"Chunk worker failed for {fp}: {exc}", stacklevel=2)
+                results.append(None)
+    return results
+
+
+def _merge_file_result(result: dict, wave: WaveResult) -> None:
+    """Merge one file's extraction dict into the WaveResult."""
+    for r in result.get("routes", []):
+        wave.routes.append(_dict_to_route(r))
+    for s in result.get("services", []):
+        wave.services.append(_dict_to_service(s))
+    for e in result.get("entities", []):
+        wave.entities.append(_dict_to_entity(e))
+    for c in result.get("components", []):
+        wave.components.append(_dict_to_component(c))
+    for e in result.get("import_edges", []):
+        wave.import_edges.append(_dict_to_edge(e))
+    for u in result.get("unresolved", []):
+        wave.unresolved.append(_dict_to_unresolved(u))
+
+
+# ── Serialisation helpers (dataclass ↔ JSON-safe dict) ───────────────────────
+
+def _route_to_dict(r: RouteInfo) -> dict:
+    return {
+        "method": r.method, "path": r.path, "handler": r.handler,
+        "source_file": r.source_file, "line": r.line,
+        "framework": r.framework, "prefix": r.prefix,
+        "tags": list(r.tags),
+    }
+
+def _service_to_dict(s: ServiceInfo) -> dict:
+    return {
+        "name": s.name, "class_name": s.class_name,
+        "source_file": s.source_file, "line": s.line,
+        "framework": s.framework,
+        "methods": list(s.methods),
+        "dependencies": list(s.dependencies),
+        "annotations": list(s.annotations),
+    }
+
+def _entity_to_dict(e: EntityInfo) -> dict:
+    return {
+        "name": e.name, "table_name": e.table_name,
+        "source_file": e.source_file, "line": e.line,
+        "framework": e.framework,
+        "fields": list(e.fields),
+        "relations": list(e.relations),
+    }
+
+def _component_to_dict(c: ComponentInfo) -> dict:
+    return {
+        "name": c.name, "source_file": c.source_file, "line": c.line,
+        "framework": c.framework,
+        "props": list(c.props), "hooks": list(c.hooks), "imports": list(c.imports),
+        "is_page": c.is_page, "route_path": c.route_path,
+    }
+
+def _edge_to_dict(e: Edge) -> dict:
+    return {
+        "source": e.source, "target": e.target,
+        "relation": e.relation, "confidence": e.confidence,
+        "confidence_score": e.confidence_score,
+        "source_file": e.source_file,
+    }
+
+def _unresolved_to_dict(u: UnresolvedRef) -> dict:
+    return {
+        "source_node_id": u.source_node_id, "ref_type": u.ref_type,
+        "ref_name": u.ref_name, "framework": u.framework,
+    }
+
+
+def _dict_to_route(d: dict) -> RouteInfo:
+    return RouteInfo(
+        method=d["method"], path=d["path"], handler=d["handler"],
+        source_file=d["source_file"], line=d["line"],
+        framework=d["framework"], prefix=d.get("prefix", ""),
+        tags=d.get("tags", []),
+    )
+
+def _dict_to_service(d: dict) -> ServiceInfo:
+    return ServiceInfo(
+        name=d["name"], class_name=d["class_name"],
+        source_file=d["source_file"], line=d["line"],
+        framework=d["framework"],
+        methods=d.get("methods", []),
+        dependencies=d.get("dependencies", []),
+        annotations=d.get("annotations", []),
+    )
+
+def _dict_to_entity(d: dict) -> EntityInfo:
+    return EntityInfo(
+        name=d["name"], table_name=d["table_name"],
+        source_file=d["source_file"], line=d["line"],
+        framework=d["framework"],
+        fields=d.get("fields", []),
+        relations=d.get("relations", []),
+    )
+
+def _dict_to_component(d: dict) -> ComponentInfo:
+    return ComponentInfo(
+        name=d["name"], source_file=d["source_file"], line=d["line"],
+        framework=d["framework"],
+        props=d.get("props", []), hooks=d.get("hooks", []),
+        imports=d.get("imports", []),
+        is_page=d.get("is_page", False), route_path=d.get("route_path", ""),
+    )
+
+def _dict_to_edge(d: dict) -> Edge:
+    return Edge(
+        source=d["source"], target=d["target"],
+        relation=d["relation"], confidence=d["confidence"],
+        confidence_score=d["confidence_score"],
+        source_file=d["source_file"],
+    )
+
+def _dict_to_unresolved(d: dict) -> UnresolvedRef:
+    return UnresolvedRef(
+        source_node_id=d["source_node_id"], ref_type=d["ref_type"],
+        ref_name=d["ref_name"], framework=d["framework"],
+    )

codebeacon/wiki/generator.py

@@ -0,0 +1,376 @@
+"""Wiki generator: read the NetworkX graph, write per-project markdown articles.
+
+Output structure:
+    <output_dir>/wiki/
+        index.md                        ← global index (short)
+        overview.md                     ← platform stats + cross-project
+        routes.md                       ← all routes table
+        cross-project/
+            connections.md              ← cross-service edges
+        <project>/
+            index.md                    ← project index
+            routes.md                   ← project routes
+            controllers/<Name>.md
+            services/<Name>.md
+            entities/<Name>.md
+            components/<Name>.md
+
+Public API:
+    generate_wiki(G, communities, output_dir)  → None
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import networkx as nx
+
+from codebeacon.wiki import templates
+
+
+# ── Classification helpers ────────────────────────────────────────────────────
+
+_CONTROLLER_ANNOTATIONS = frozenset({
+    # Spring
+    "@Controller", "@RestController",
+    # NestJS
+    "@Controller",
+    # ASP.NET
+    "[ApiController]", "[Controller]",
+    # Generic
+    "Controller", "RestController",
+})
+
+_CONTROLLER_NAME_SUFFIXES = ("Controller", "Router", "Handler", "Resource")
+
+
+def _is_controller(label: str, annotations: list[str]) -> bool:
+    """Heuristic: is this class node a controller rather than a service?"""
+    if any(a in _CONTROLLER_ANNOTATIONS for a in annotations):
+        return True
+    return label.endswith(_CONTROLLER_NAME_SUFFIXES)
+
+
+def _safe_filename(label: str) -> str:
+    """Strip characters that are unsafe in filenames."""
+    return "".join(c if c.isalnum() or c in "-_." else "_" for c in label)
+
+
+# ── Node neighbour helpers ────────────────────────────────────────────────────
+
+_CALL_RELATIONS = frozenset({"calls", "injects", "depends"})
+_ENTITY_TYPES = frozenset({"entity"})
+
+
+def _predecessors_labels(G: nx.DiGraph, node_id: str, relations: frozenset[str]) -> list[str]:
+    """Labels of predecessors connected via the given relation types."""
+    result = []
+    for pred in G.predecessors(node_id):
+        edge_data = G.edges[pred, node_id]
+        if edge_data.get("relation") in relations:
+            result.append(G.nodes[pred].get("label", pred))
+    return result
+
+
+def _successors_labels(G: nx.DiGraph, node_id: str, relations: frozenset[str]) -> list[str]:
+    """Labels of successors connected via the given relation types."""
+    result = []
+    for succ in G.successors(node_id):
+        edge_data = G.edges[node_id, succ]
+        if edge_data.get("relation") in relations:
+            result.append(G.nodes[succ].get("label", succ))
+    return result
+
+
+def _related_entities(G: nx.DiGraph, node_id: str) -> list[str]:
+    """Entity node labels reachable via imports/calls edges."""
+    result = []
+    for succ in G.successors(node_id):
+        if G.nodes[succ].get("type") in _ENTITY_TYPES:
+            result.append(G.nodes[succ].get("label", succ))
+    return result
+
+
+# ── Cross-project connections ─────────────────────────────────────────────────
+
+def _cross_project_edges(G: nx.DiGraph) -> list[dict[str, Any]]:
+    """Edges that cross project boundaries."""
+    result = []
+    for src, tgt, data in G.edges(data=True):
+        src_proj = G.nodes[src].get("project", "")
+        tgt_proj = G.nodes[tgt].get("project", "")
+        if src_proj and tgt_proj and src_proj != tgt_proj:
+            result.append({
+                "source": G.nodes[src].get("label", src),
+                "target": G.nodes[tgt].get("label", tgt),
+                "relation": data.get("relation", ""),
+                "source_project": src_proj,
+                "target_project": tgt_proj,
+            })
+    return result
+
+
+# ── Route collector ───────────────────────────────────────────────────────────
+
+def _collect_routes(G: nx.DiGraph) -> dict[str, list[dict[str, Any]]]:
+    """Collect route nodes grouped by project."""
+    routes_by_project: dict[str, list[dict[str, Any]]] = {}
+    for node_id, data in G.nodes(data=True):
+        if data.get("type") != "route":
+            continue
+        project = data.get("project", "_unknown")
+        routes_by_project.setdefault(project, []).append({
+            "method": data.get("method", ""),
+            "path": data.get("path", ""),
+            "handler": data.get("label", ""),
+            "source_file": data.get("source_file", ""),
+            "framework": data.get("framework", ""),
+            "tags": data.get("tags", []),
+        })
+    return routes_by_project
+
+
+# ── Main generator ────────────────────────────────────────────────────────────
+
+def generate_wiki(
+    G: nx.DiGraph,
+    communities: dict[str, int],
+    output_dir: str | Path,
+) -> None:
+    """Generate full wiki from the knowledge graph.
+
+    Args:
+        G:            built NetworkX DiGraph (output of graph/build.py + enrich.py)
+        communities:  node_id → community_id (output of graph/cluster.py)
+        output_dir:   root output directory (e.g. /path/to/project/.codebeacon)
+    """
+    wiki_dir = Path(output_dir) / "wiki"
+    wiki_dir.mkdir(parents=True, exist_ok=True)
+
+    # Group nodes by project and type
+    projects: dict[str, dict[str, list[tuple[str, dict]]]] = {}
+    # project → type → [(node_id, data)]
+
+    for node_id, data in G.nodes(data=True):
+        project = data.get("project", "_unknown")
+        ntype = data.get("type", "unknown")
+        projects.setdefault(project, {}).setdefault(ntype, []).append((node_id, data))
+
+    # Collect routes for routes.md (all projects)
+    routes_by_project = _collect_routes(G)
+
+    # Per-project stats accumulator for overview
+    project_summary: list[dict[str, Any]] = []
+
+    for project_name, type_map in sorted(projects.items()):
+        proj_dir = wiki_dir / project_name
+        _write_project(G, project_name, type_map, routes_by_project, proj_dir)
+
+        # Collect summary stats
+        route_count = len(routes_by_project.get(project_name, []))
+        service_count = 0
+        entity_count = 0
+        component_count = 0
+        framework = ""
+
+        for node_id, data in type_map.get("class", []):
+            annotations = data.get("annotations", [])
+            if not _is_controller(data.get("label", ""), annotations):
+                service_count += 1
+            fw = data.get("framework", "")
+            if fw:
+                framework = fw
+
+        entity_count = len(type_map.get("entity", []))
+        component_count = len(type_map.get("component", []))
+
+        project_summary.append({
+            "name": project_name,
+            "framework": framework,
+            "route_count": route_count,
+            "service_count": service_count,
+            "entity_count": entity_count,
+            "component_count": component_count,
+        })
+
+    # Global stats
+    total_routes = sum(len(rs) for rs in routes_by_project.values())
+    total_services = sum(p["service_count"] for p in project_summary)
+    total_entities = sum(p["entity_count"] for p in project_summary)
+    total_components = sum(p["component_count"] for p in project_summary)
+    total_stats = {
+        "nodes": G.number_of_nodes(),
+        "edges": G.number_of_edges(),
+        "communities": len(set(communities.values())) if communities else 0,
+        "routes": total_routes,
+        "services": total_services,
+        "entities": total_entities,
+        "components": total_components,
+    }
+
+    # Cross-project connections
+    cross_edges = _cross_project_edges(G)
+
+    # Write global files (delegated to index.py's generate_index)
+    from codebeacon.wiki.index import generate_index
+    generate_index(
+        wiki_dir=wiki_dir,
+        project_summary=project_summary,
+        routes_by_project=routes_by_project,
+        cross_edges=cross_edges,
+        total_stats=total_stats,
+    )
+
+
+# ── Per-project writer ────────────────────────────────────────────────────────
+
+def _write_project(
+    G: nx.DiGraph,
+    project_name: str,
+    type_map: dict[str, list[tuple[str, dict]]],
+    routes_by_project: dict[str, list[dict[str, Any]]],
+    proj_dir: Path,
+) -> None:
+    """Write all wiki files for one project."""
+    proj_dir.mkdir(parents=True, exist_ok=True)
+
+    controllers: list[str] = []
+    services: list[str] = []
+
+    # Class nodes → controller or service
+    for node_id, data in type_map.get("class", []):
+        label = data.get("label", node_id)
+        annotations = data.get("annotations", [])
+        methods = data.get("methods", [])
+        dependencies = data.get("dependencies", [])
+        source_file = data.get("source_file", "")
+        framework = data.get("framework", "")
+
+        called_by = _predecessors_labels(G, node_id, _CALL_RELATIONS)
+        calls = _successors_labels(G, node_id, _CALL_RELATIONS)
+
+        if _is_controller(label, annotations):
+            controllers.append(label)
+            # Gather routes for this controller
+            ctrl_routes = [
+                r for r in routes_by_project.get(project_name, [])
+                if label in r.get("handler", "")
+            ]
+            content = templates.controller_article(
+                label=label,
+                routes=ctrl_routes,
+                source_file=source_file,
+                called_by=called_by,
+                calls=calls,
+                project_name=project_name,
+            )
+            _write_file(proj_dir / "controllers" / f"{_safe_filename(label)}.md", content)
+        else:
+            services.append(label)
+            entities = _related_entities(G, node_id)
+            content = templates.service_article(
+                label=label,
+                methods=methods,
+                dependencies=dependencies,
+                source_file=source_file,
+                called_by=called_by,
+                calls=calls,
+                related_entities=entities,
+                annotations=annotations,
+                project_name=project_name,
+            )
+            _write_file(proj_dir / "services" / f"{_safe_filename(label)}.md", content)
+
+    # Entity nodes
+    entity_names: list[str] = []
+    for node_id, data in type_map.get("entity", []):
+        label = data.get("label", node_id)
+        entity_names.append(label)
+        table_name = data.get("table_name", "")
+        fields = data.get("fields", [])
+        relations = data.get("relations", [])
+        source_file = data.get("source_file", "")
+        framework = data.get("framework", "")
+        used_by = _predecessors_labels(G, node_id, frozenset({"imports", "imports_from", "calls"}))
+
+        content = templates.entity_article(
+            label=label,
+            table_name=table_name,
+            fields=fields,
+            relations=relations,
+            source_file=source_file,
+            used_by=used_by,
+            framework=framework,
+            project_name=project_name,
+        )
+        _write_file(proj_dir / "entities" / f"{_safe_filename(label)}.md", content)
+
+    # Component nodes
+    component_names: list[str] = []
+    for node_id, data in type_map.get("component", []):
+        label = data.get("label", node_id)
+        component_names.append(label)
+        props = data.get("props", [])
+        hooks = data.get("hooks", [])
+        imports_list = data.get("imports", [])
+        is_page = data.get("is_page", False)
+        route_path = data.get("route_path", "")
+        source_file = data.get("source_file", "")
+        framework = data.get("framework", "")
+
+        content = templates.component_article(
+            label=label,
+            props=props,
+            hooks=hooks,
+            imports=imports_list,
+            is_page=is_page,
+            route_path=route_path,
+            source_file=source_file,
+            framework=framework,
+            project_name=project_name,
+        )
+        _write_file(proj_dir / "components" / f"{_safe_filename(label)}.md", content)
+
+    # Detect framework from any node in this project
+    framework = ""
+    for type_nodes in type_map.values():
+        for _, data in type_nodes:
+            fw = data.get("framework", "")
+            if fw:
+                framework = fw
+                break
+        if framework:
+            break
+
+    # Per-project routes.md
+    proj_routes = routes_by_project.get(project_name, [])
+    if proj_routes:
+        content = templates.routes_summary({project_name: proj_routes})
+        _write_file(proj_dir / "routes.md", content)
+
+    # Per-project index.md
+    stats = {
+        "routes": len(proj_routes),
+        "services": len(services),
+        "entities": len(entity_names),
+        "components": len(component_names),
+    }
+    content = templates.project_index(
+        project_name=project_name,
+        framework=framework,
+        stats=stats,
+        controllers=controllers,
+        services=services,
+        entities=entity_names,
+        components=component_names,
+    )
+    _write_file(proj_dir / "index.md", content)
+
+
+# ── File writer ───────────────────────────────────────────────────────────────
+
+def _write_file(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")