graph-dependency-analyzer 0.2.0__py3-none-any.whl

src/infrastructure/persistence/graph_export_repository.py

@@ -0,0 +1,443 @@
+"""Graph Export Repository — reads Neo4j and converts to Graphify-compatible schema."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+# Map Neo4j node labels to Graphify file_type values
+_NODE_TYPE_MAP: dict[str, str] = {
+    "CLASS": "code",
+    "METHOD": "code",
+    "REPOSITORY": "code",
+    "LIBRARY": "module",
+    "API": "concept",
+    "DATABASE": "concept",
+    "Component": "code",
+    "NPM_PACKAGE": "module",
+    "Microfrontend": "code",
+    "WORKSPACE": "concept",
+    "PROJECT": "code",
+    # DB schema objects
+    "DB_TABLE":      "database",
+    "DB_VIEW":       "database",
+    "DB_PROCEDURE":  "database",
+    "DB_PACKAGE":    "database",
+    "DB_TRIGGER":    "database",
+    "DB_SEQUENCE":   "database",
+    "DB_FUNCTION":   "database",
+    "DB_INDEX":      "database",
+    "DB_TYPE":       "database",
+    "DB_COLLECTION": "database",
+    "ENDPOINT":      "concept",
+}
+
+# Map Neo4j relationship types to Graphify relation names
+_EDGE_RELATION_MAP: dict[str, str] = {
+    "CONTAINS": "contains",
+    "HAS_METHOD": "method",
+    "CALLS": "calls",
+    "IMPORTS": "imports",
+    "IMPORTS_COMPONENT": "imports",
+    "USES_LIBRARY": "references",
+    "USES_PACKAGE": "references",
+    "CALLS_API": "calls",
+    "CONNECTS_TO": "references",
+    "DEPENDS_ON": "references",
+    "EXPOSES_MODULE": "references",
+    "CONSUMES_MODULE": "references",
+    # DB schema edges
+    "CONTAINS_DB_OBJECT": "contains",
+    "READS_TABLE":        "reads",
+    "WRITES_TABLE":       "writes",
+    "CALLS_PROCEDURE":    "calls",
+    "CALLS_BACKEND":      "calls",
+    "EXPOSES":            "references",
+}
+
+
+@dataclass
+class GraphifyNode:
+    id: str
+    label: str
+    file_type: str
+    file: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class GraphifyEdge:
+    source: str
+    target: str
+    relation: str
+    confidence: str = "EXTRACTED"
+    weight: float = 1.0
+    source_file: str | None = None
+
+
+@dataclass
+class GraphifyGraph:
+    nodes: list[dict[str, Any]]
+    edges: list[dict[str, Any]]
+    meta: dict[str, Any] = field(default_factory=dict)
+
+
+class GraphExportRepository:
+    """Reads Neo4j graph and exports it in Graphify-compatible JSON schema."""
+
+    def __init__(self, driver) -> None:
+        self.driver = driver
+
+    async def export_graph(
+        self,
+        repository: str | None = None,
+        group_packages: bool = True,
+        node_types: list[str] | None = None,
+        max_nodes: int = 500,
+    ) -> GraphifyGraph:
+        """
+        Export dependency graph with optional package grouping and type filtering.
+
+        Args:
+            repository: Filter by repository name.
+            group_packages: If True, collapse Java/Python class nodes into
+                            their top-level package groups (default True).
+            node_types: Which node types to include. None = all.
+                        Examples: ["CLASS","DB_TABLE","API","DATABASE"]
+            max_nodes: Hard cap on nodes returned (default 500 for performance).
+        """
+        async with self.driver.session() as session:
+            nodes = await self._fetch_nodes(session, repository)
+            edges = await self._fetch_edges(session, repository)
+
+        graphify_nodes = [self._convert_node(n) for n in nodes]
+        graphify_edges = [self._convert_edge(e) for e in edges if e]
+
+        # ── Filter by node type ──────────────────────────────────────────
+        if node_types:
+            allowed = {t.upper() for t in node_types}
+            graphify_nodes = [
+                n for n in graphify_nodes
+                if (n.get("metadata", {}).get("neo4j_label", "").upper() in allowed
+                    or n.get("type", "").upper() in allowed)
+            ]
+
+        # ── Deduplicate nodes ────────────────────────────────────────────
+        seen_ids: set[str] = set()
+        unique_nodes: list[dict] = []
+        for node in graphify_nodes:
+            if node["id"] not in seen_ids:
+                seen_ids.add(node["id"])
+                unique_nodes.append(node)
+
+        # ── Package grouping ─────────────────────────────────────────────
+        if group_packages:
+            unique_nodes, edges_remap = self._group_by_package(unique_nodes)
+        else:
+            edges_remap = {}
+
+        # ── Apply edge remapping + filter to existing nodes ───────────────
+        node_id_set = {n["id"] for n in unique_nodes}
+        valid_edges = []
+        for e in graphify_edges:
+            if e is None:
+                continue
+            src = edges_remap.get(e["source"], e["source"])
+            tgt = edges_remap.get(e["target"], e["target"])
+            if src == tgt:
+                continue
+            if src in node_id_set and tgt in node_id_set:
+                valid_edges.append({**e, "source": src, "target": tgt})
+
+        # Deduplicate edges after remapping
+        seen_edges: set[tuple] = set()
+        deduped_edges = []
+        for e in valid_edges:
+            key = (e["source"], e["target"], e["relation"])
+            if key not in seen_edges:
+                seen_edges.add(key)
+                deduped_edges.append(e)
+
+        # ── Cap nodes for performance ─────────────────────────────────────
+        # Prioritise: DB objects > API/ENDPOINT > packages with most edges > rest
+        # Also cap edges per node to avoid vis.js freezing on hub nodes.
+        edge_degree: dict[str, int] = {}
+        for e in deduped_edges:
+            edge_degree[e["source"]] = edge_degree.get(e["source"], 0) + 1
+            edge_degree[e["target"]] = edge_degree.get(e["target"], 0) + 1
+
+        if len(unique_nodes) > max_nodes:
+            def priority(n: dict) -> tuple:
+                lbl = n.get("metadata", {}).get("neo4j_label", "")
+                deg = edge_degree.get(n["id"], 0)
+                is_db  = 1 if lbl.startswith("DB_") else 0
+                is_api = 1 if lbl in ("API", "ENDPOINT", "DATABASE") else 0
+                return (-is_db, -is_api, -deg)
+
+            unique_nodes = sorted(unique_nodes, key=priority)[:max_nodes]
+            node_id_set = {n["id"] for n in unique_nodes}
+            deduped_edges = [
+                e for e in deduped_edges
+                if e["source"] in node_id_set and e["target"] in node_id_set
+            ]
+            # Recalculate degree after node cap
+            edge_degree = {}
+            for e in deduped_edges:
+                edge_degree[e["source"]] = edge_degree.get(e["source"], 0) + 1
+                edge_degree[e["target"]] = edge_degree.get(e["target"], 0) + 1
+
+        # Cap edges per node to MAX_DEGREE to prevent vis.js hub freeze.
+        # Nodes with >50 connections get their edges trimmed to 50 (keep highest-degree neighbours).
+        MAX_DEGREE = 50
+        hub_ids = {nid for nid, deg in edge_degree.items() if deg > MAX_DEGREE}
+        if hub_ids:
+            # For each hub, keep only its MAX_DEGREE edges (by neighbour degree, descending)
+            kept_edges = []
+            hub_edge_count: dict[str, int] = {}
+            # Sort edges so high-degree neighbours come first
+            def edge_priority(e: dict) -> int:
+                return -(edge_degree.get(e["source"], 0) + edge_degree.get(e["target"], 0))
+            for e in sorted(deduped_edges, key=edge_priority):
+                src_hub = e["source"] in hub_ids
+                tgt_hub = e["target"] in hub_ids
+                if src_hub and hub_edge_count.get(e["source"], 0) >= MAX_DEGREE:
+                    continue
+                if tgt_hub and hub_edge_count.get(e["target"], 0) >= MAX_DEGREE:
+                    continue
+                kept_edges.append(e)
+                if src_hub:
+                    hub_edge_count[e["source"]] = hub_edge_count.get(e["source"], 0) + 1
+                if tgt_hub:
+                    hub_edge_count[e["target"]] = hub_edge_count.get(e["target"], 0) + 1
+            deduped_edges = kept_edges
+
+        # ── Sanitise node IDs for vis.js compatibility ──────────────────────
+        # vis.js treats '.' as namespace separator and chokes on './', '../', '.'
+        # Replace relative path fragments used as TypeScript import IDs.
+        import re as _re
+        def _safe_id(nid: str) -> str:
+            if not nid:
+                return "__empty__"
+            s = str(nid)
+            s = _re.sub(r'^\.\.\/', '__parent__/', s)
+            s = _re.sub(r'^\.\/', '__rel__/', s)
+            if s in (".", ".."):
+                s = "__dot__" if s == "." else "__dotdot__"
+            return s
+
+        id_remap: dict[str, str] = {}
+        for n in unique_nodes:
+            orig = n["id"]
+            safe = _safe_id(orig)
+            if safe != orig:
+                id_remap[orig] = safe
+                n["id"] = safe
+
+        if id_remap:
+            for e in deduped_edges:
+                if e["source"] in id_remap:
+                    e["source"] = id_remap[e["source"]]
+                if e["target"] in id_remap:
+                    e["target"] = id_remap[e["target"]]
+
+        meta = {
+            "source": "neo4j",
+            "repository_filter": repository or "all",
+            "node_count": len(unique_nodes),
+            "edge_count": len(deduped_edges),
+            "grouped": group_packages,
+        }
+        logger.info("graph_exported", repository=repository or "all",
+                    nodes=len(unique_nodes), edges=len(deduped_edges))
+        return GraphifyGraph(nodes=unique_nodes, edges=deduped_edges, meta=meta)
+
+    # ── Package grouping ──────────────────────────────────────────────────
+
+    @staticmethod
+    def _pkg_group(node_id: str, neo4j_label: str) -> str | None:
+        """
+        Return a package group id for CLASS nodes.
+        Groups com.medline.wms.WebServiceEJB.Foo → com.medline.wms.WebServiceEJB
+        Non-CLASS nodes return None (keep as-is).
+        """
+        if neo4j_label not in ("CLASS", ""):
+            return None
+        # Only group internal classes (not external/java.*)
+        if not node_id or node_id.startswith("java.") or node_id.startswith("javax."):
+            return None
+        parts = node_id.split(".")
+        if len(parts) <= 2:
+            return None
+        # Use up to 4 segments as group key
+        return ".".join(parts[:min(4, len(parts) - 1)])
+
+    def _group_by_package(
+        self,
+        nodes: list[dict],
+    ) -> tuple[list[dict], dict[str, str]]:
+        """
+        Collapse CLASS nodes into package groups.
+        Returns (new_nodes, remap_dict) where remap_dict maps
+        old_node_id → group_node_id for edge rewriting.
+        """
+        remap: dict[str, str] = {}      # original_id → group_id
+        groups: dict[str, dict] = {}    # group_id → group_node
+        kept: list[dict] = []           # non-CLASS nodes kept as-is
+
+        # Pre-build set of IDs that are kept as-is (non-CLASS nodes)
+        # so we can detect collisions before creating group nodes.
+        kept_ids: set[str] = set()
+        groupable: list[dict] = []
+        for n in nodes:
+            lbl = n.get("metadata", {}).get("neo4j_label", "")
+            grp = GraphExportRepository._pkg_group(n["id"], lbl)
+            if grp is None:
+                kept.append(n)
+                kept_ids.add(n["id"])
+            else:
+                groupable.append(n)
+
+        for n in groupable:
+            lbl = n.get("metadata", {}).get("neo4j_label", "")
+            grp = GraphExportRepository._pkg_group(n["id"], lbl)
+
+            # If the group ID collides with an existing kept node,
+            # append __pkg suffix to avoid duplicate IDs in vis.js.
+            safe_grp = grp
+            if grp in kept_ids:
+                safe_grp = grp + "__pkg"
+
+            remap[n["id"]] = safe_grp
+
+            if safe_grp not in groups:
+                # Derive short label from last segment of original group
+                short = grp.split(".")[-1]
+                repo = n.get("metadata", {}).get("repository", "")
+                groups[safe_grp] = {
+                    "id":    safe_grp,
+                    "label": short,
+                    "type":  "code",
+                    "file":  None,
+                    "metadata": {
+                        "neo4j_label": "PACKAGE",
+                        "repository":  repo,
+                        "qualified_name": grp,
+                        "_class_count": 0,
+                    },
+                }
+            groups[safe_grp]["metadata"]["_class_count"] = \
+                groups[safe_grp]["metadata"].get("_class_count", 0) + 1
+
+        # Add class count to group label
+        for grp_node in groups.values():
+            cnt = grp_node["metadata"].get("_class_count", 0)
+            grp_node["label"] = f"{grp_node['label']} ({cnt})"
+
+        return kept + list(groups.values()), remap
+
+    async def _fetch_nodes(self, session, repository: str | None) -> list[dict]:
+        if repository:
+            # For DB schema repos (many stub nodes), only fetch real DDL objects
+            # (is_stub IS NULL = real DDL-backed node) plus their connected neighbours.
+            # This prevents 16k+ stub tables from flooding the graph.
+            query = """
+            MATCH (n)
+            WHERE n.repository = $repository
+              AND coalesce(n.is_stub, 'false') <> 'true'
+            RETURN labels(n) AS labels, properties(n) AS props
+            LIMIT 8000
+            """
+            params = {"repository": repository}
+        else:
+            query = """
+            MATCH (n)
+            WHERE coalesce(n.is_stub, 'false') <> 'true'
+            RETURN labels(n) AS labels, properties(n) AS props
+            LIMIT 8000
+            """
+            params = {}
+        result = await session.run(query, **params)
+        return await result.data()
+
+    async def _fetch_edges(self, session, repository: str | None) -> list[dict]:
+        if repository:
+            query = """
+            MATCH (source)-[r]->(target)
+            WHERE (source.repository = $repository OR target.repository = $repository)
+              AND coalesce(source.is_stub, 'false') <> 'true'
+              AND coalesce(target.is_stub, 'false') <> 'true'
+            RETURN
+                source.id        AS source_id,
+                target.id        AS target_id,
+                type(r)          AS relation_type,
+                source.file_path AS source_file
+            LIMIT 20000
+            """
+            params = {"repository": repository}
+        else:
+            query = """
+            MATCH (source)-[r]->(target)
+            WHERE coalesce(source.is_stub, 'false') <> 'true'
+              AND coalesce(target.is_stub, 'false') <> 'true'
+            RETURN
+                source.id        AS source_id,
+                target.id        AS target_id,
+                type(r)          AS relation_type,
+                source.file_path AS source_file
+            LIMIT 20000
+            """
+            params = {}
+        result = await session.run(query, **params)
+        return await result.data()
+
+    def _convert_node(self, record: dict) -> dict[str, Any]:
+        node_labels: list[str] = record.get("labels") or []
+        primary_label = node_labels[0] if node_labels else "UNKNOWN"
+        file_type = _NODE_TYPE_MAP.get(primary_label, "concept")
+
+        props: dict = record.get("props") or {}
+        qname: str = props.get("qualified_name") or ""
+        node_id: str = str(props.get("id") or qname or props.get("url") or "unknown")
+
+        label = (
+            props.get("name")
+            or (qname.split(".")[-1] if qname else None)
+            or props.get("url")
+            or node_id
+        )
+
+        metadata: dict[str, Any] = {"neo4j_label": primary_label}
+        for key in ("repository", "version", "framework", "language", "qualified_name",
+                    "url", "db_type", "host", "database", "type"):
+            val = props.get(key)
+            if val is not None:
+                metadata[key] = str(val)
+
+        return {
+            "id": node_id,
+            "label": str(label),
+            "type": file_type,
+            "file": props.get("file_path"),
+            "metadata": metadata,
+        }
+
+    def _convert_edge(self, record: dict) -> dict[str, Any] | None:
+        source_id = record.get("source_id")
+        target_id = record.get("target_id")
+        if not source_id or not target_id:
+            return None
+
+        relation_type: str = record.get("relation_type") or "references"
+        relation = _EDGE_RELATION_MAP.get(relation_type, "references")
+
+        return {
+            "source": str(source_id),
+            "target": str(target_id),
+            "relation": relation,
+            "confidence": "EXTRACTED",
+            "weight": 1.0,
+            "source_file": record.get("source_file"),
+        }