graph-dependency-analyzer 0.2.0__py3-none-any.whl

src/infrastructure/persistence/neo4j_dependency_repository.py

@@ -0,0 +1,657 @@
+"""
+Neo4j Dependency Graph Repository Implementation (Task 5.1)
+
+Implements persistence layer for dependency graphs in Neo4j.
+Supports 6 node types and 7 relationship types for impact analysis.
+"""
+from dataclasses import dataclass
+from typing import Any
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class GraphStorageResult:
+    """Result of storing a dependency graph"""
+    nodes_created: int
+    relationships_created: int
+    batch_count: int
+
+
+@dataclass
+class ImpactAnalysisResult:
+    """Result of impact analysis query"""
+    affected_components: list[str | dict]
+    warning: str | None = None  # Task 10.6: Warning for non-existent entities
+    suggestions: list[str] | None = None  # Task 10.6: Suggestions for similar entities
+
+
+class Neo4jDependencyGraphRepository:
+    """
+    Neo4j-based repository for storing dependency graphs.
+
+    Implements:
+    - 6 node types: Repository, Class, Method, Library, API, Database
+    - 7 relationship types: CONTAINS, HAS_METHOD, CALLS, IMPORTS,
+      USES_LIBRARY, CALLS_API, CONNECTS_TO, DEPENDS_ON
+    - Batch processing with UNWIND (batch_size=100)
+    - MERGE for idempotency
+    - Impact analysis queries
+
+    Requirements: 6.1-6.14, 16.1-16.5
+    """
+
+    def __init__(self, driver, batch_size: int = 100):
+        """
+        Initialize repository with Neo4j driver.
+
+        Args:
+            driver: Neo4j AsyncGraphDatabase driver
+            batch_size: Batch size for UNWIND operations (default 100)
+        """
+        self.driver = driver
+        self.batch_size = batch_size
+
+    async def store_dependency_graph(
+        self,
+        graph,
+        repository_name: str
+    ) -> GraphStorageResult:
+        """
+        Store complete dependency graph in Neo4j.
+
+        Uses MERGE for idempotency and UNWIND for batch processing.
+
+        Args:
+            graph: DependencyGraph with nodes and edges
+            repository_name: Name of repository being indexed
+
+        Returns:
+            GraphStorageResult with metrics
+
+        Raises:
+            Exception: If Neo4j write fails
+        """
+        nodes_created = 0
+        relationships_created = 0
+        batch_count = 0
+
+        async with self.driver.session() as session:
+            # Store nodes in batches using UNWIND and MERGE
+            batch_count = await self._store_nodes_in_batches(session, graph.nodes)
+            nodes_created = len(graph.nodes)
+
+            # Store relationships in batches using UNWIND and MERGE
+            await self._store_relationships_in_batches(session, graph.edges)
+            relationships_created = len(graph.edges)
+
+        logger.info(
+            "dependency_graph_stored",
+            repository=repository_name,
+            nodes_created=nodes_created,
+            relationships_created=relationships_created,
+            batch_count=batch_count
+        )
+
+        return GraphStorageResult(
+            nodes_created=nodes_created,
+            relationships_created=relationships_created,
+            batch_count=batch_count
+        )
+
+    async def _store_nodes_in_batches(self, session, nodes: list) -> int:
+        """
+        Store nodes in batches of batch_size using UNWIND.
+
+        Uses MERGE to ensure idempotency (no duplicates).
+
+        Args:
+            session: Neo4j session
+            nodes: List of DependencyNode
+
+        Returns:
+            Number of batches processed
+        """
+        batch_count = 0
+
+        for i in range(0, len(nodes), self.batch_size):
+            batch = nodes[i:i + self.batch_size]
+            batch_count += 1
+
+            # Convert nodes to dicts for UNWIND
+            node_dicts = []
+            for node in batch:
+                node_dict = {
+                    'id': node.id,
+                    'node_type': node.node_type,
+                    'properties': node.properties
+                }
+                node_dicts.append(node_dict)
+
+            # Use UNWIND + MERGE for batch insert
+            query = """
+            UNWIND $nodes AS node
+            CALL apoc.merge.node(
+                [node.node_type],
+                {id: node.id},
+                node.properties,
+                {}
+            ) YIELD node AS n
+            RETURN count(n) AS created
+            """
+
+            # Fallback if APOC not available - use standard MERGE per node type
+            if batch:
+                await self._merge_nodes_by_type(session, batch)
+
+        return batch_count
+
+    async def _merge_nodes_by_type(self, session, nodes: list) -> None:
+        """
+        MERGE nodes grouped by type (fallback without APOC).
+
+        Args:
+            session: Neo4j session
+            nodes: List of DependencyNode
+        """
+        # Group nodes by type
+        nodes_by_type = {}
+        for node in nodes:
+            if node.node_type not in nodes_by_type:
+                nodes_by_type[node.node_type] = []
+            nodes_by_type[node.node_type].append({
+                'id': node.id,
+                'properties': node.properties
+            })
+
+        # MERGE each type separately
+        for node_type, node_list in nodes_by_type.items():
+            query = f"""
+            UNWIND $nodes AS node
+            MERGE (n:{node_type} {{id: node.id}})
+            SET n += node.properties
+            RETURN count(n) AS created
+            """
+            await session.run(query, nodes=node_list)
+
+    async def _store_relationships_in_batches(self, session, edges: list) -> None:
+        """
+        Store relationships in batches using UNWIND and MERGE.
+
+        Args:
+            session: Neo4j session
+            edges: List of DependencyEdge
+        """
+        for i in range(0, len(edges), self.batch_size):
+            batch = edges[i:i + self.batch_size]
+
+            # Convert edges to dicts
+            edge_dicts = []
+            for edge in batch:
+                edge_dict = {
+                    'source': edge.source,
+                    'target': edge.target,
+                    'edge_type': edge.edge_type,
+                    'properties': edge.properties
+                }
+                edge_dicts.append(edge_dict)
+
+            # Use UNWIND + MERGE for relationships
+            query = """
+            UNWIND $edges AS edge
+            MATCH (source {id: edge.source})
+            MATCH (target {id: edge.target})
+            CALL apoc.merge.relationship(
+                source,
+                edge.edge_type,
+                {},
+                edge.properties,
+                target,
+                {}
+            ) YIELD rel
+            RETURN count(rel) AS created
+            """
+
+            # Fallback without APOC
+            await self._merge_relationships_by_type(session, batch)
+
+    async def _merge_relationships_by_type(self, session, edges: list) -> None:
+        """
+        MERGE relationships grouped by type (fallback without APOC).
+
+        Args:
+            session: Neo4j session
+            edges: List of DependencyEdge
+        """
+        # Group edges by type
+        edges_by_type = {}
+        for edge in edges:
+            if edge.edge_type not in edges_by_type:
+                edges_by_type[edge.edge_type] = []
+            edges_by_type[edge.edge_type].append({
+                'source': edge.source,
+                'target': edge.target,
+                'properties': edge.properties
+            })
+
+        # MERGE each type separately
+        for edge_type, edge_list in edges_by_type.items():
+            query = f"""
+            UNWIND $edges AS edge
+            MATCH (source {{id: edge.source}})
+            MATCH (target {{id: edge.target}})
+            MERGE (source)-[r:{edge_type}]->(target)
+            SET r += edge.properties
+            RETURN count(r) AS created
+            """
+            await session.run(query, edges=edge_list)
+
+    async def query_impact_analysis(
+        self,
+        query_type: str,
+        params: dict[str, Any]
+    ) -> ImpactAnalysisResult:
+        """
+        Execute impact analysis query.
+
+        Supports query types (Requirements 16.1-16.4, Tasks 10.1-10.4):
+        - REPO_DEPENDENTS: Find repositories that depend on target
+        - CLASS_IMPACT: Find methods affected by class change
+        - API_CONSUMERS: Find repositories calling an API
+        - DEPENDENCY_PATH: Find shortest path between repos
+        - COMPONENT_IMPACT: Find components importing target component (Task 10.1)
+        - API_CONSUMER_IMPACT: Find all consumers calling target API (Task 10.2)
+        - MICROFRONTEND_IMPACT: Find microfrontends consuming target modules (Task 10.3)
+        - PACKAGE_IMPACT: Find components depending on target npm package (Task 10.4)
+
+        Args:
+            query_type: Type of analysis
+            params: Query parameters
+
+        Returns:
+            ImpactAnalysisResult with affected components
+        """
+        async with self.driver.session() as session:
+            if query_type == "REPO_DEPENDENTS":
+                return await self._query_repo_dependents(session, params)
+            elif query_type == "CLASS_IMPACT":
+                return await self._query_class_impact(session, params)
+            elif query_type == "API_CONSUMERS":
+                return await self._query_api_consumers(session, params)
+            elif query_type == "DEPENDENCY_PATH":
+                return await self._query_dependency_path(session, params)
+            elif query_type == "COMPONENT_IMPACT":
+                return await self._query_component_impact(session, params)
+            elif query_type == "API_CONSUMER_IMPACT":
+                return await self._query_api_consumer_impact(session, params)
+            elif query_type == "MICROFRONTEND_IMPACT":
+                return await self._query_microfrontend_impact(session, params)
+            elif query_type == "PACKAGE_IMPACT":
+                return await self._query_package_impact(session, params)
+            else:
+                raise ValueError(f"Unknown query type: {query_type}")
+
+    async def _query_repo_dependents(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which repositories depend on target repository? (Requirement 16.1)
+
+        Args:
+            session: Neo4j session
+            params: {"repo_name": str}
+
+        Returns:
+            ImpactAnalysisResult with list of dependent repository names
+        """
+        query = """
+        MATCH (target:REPOSITORY {name: $repo_name})<-[:DEPENDS_ON*1..3]-(dependent:REPOSITORY)
+        RETURN DISTINCT dependent.name AS dependent_repo
+        ORDER BY dependent_repo
+        """
+
+        result = await session.run(query, repo_name=params["repo_name"])
+
+        affected = []
+        async for record in result:
+            affected.append(record["dependent_repo"])
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_class_impact(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which methods are affected if class changes? (Requirement 16.2)
+
+        Args:
+            session: Neo4j session
+            params: {"class_name": str}
+
+        Returns:
+            ImpactAnalysisResult with affected methods
+        """
+        query = """
+        MATCH (target:CLASS {qualified_name: $class_name})<-[:IMPORTS]-(dependent:CLASS)-[:HAS_METHOD]->(method:METHOD)
+        RETURN DISTINCT
+            method.qualified_name AS affected_method,
+            method.file_path AS file_path,
+            dependent.name AS importing_class
+        ORDER BY file_path
+        """
+
+        result = await session.run(query, class_name=params["class_name"])
+
+        affected = []
+        async for record in result:
+            affected.append(record["affected_method"])
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_api_consumers(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which repositories call this API? (Requirement 16.3)
+
+        Args:
+            session: Neo4j session
+            params: {"api_url": str}
+
+        Returns:
+            ImpactAnalysisResult with repository names
+        """
+        query = """
+        MATCH (api:API {url: $api_url})<-[:CALLS_API]-(method:METHOD)<-[:HAS_METHOD]-(class:CLASS)<-[:CONTAINS]-(repo:REPOSITORY)
+        RETURN DISTINCT
+            repo.name AS repository,
+            method.qualified_name AS calling_method,
+            class.file_path AS file_path
+        ORDER BY repo.name, file_path
+        """
+
+        result = await session.run(query, api_url=params["api_url"])
+
+        affected = []
+        async for record in result:
+            affected.append(record["repository"])
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_dependency_path(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: What's the dependency path between two repos? (Requirement 16.4)
+
+        Args:
+            session: Neo4j session
+            params: {"repo_a": str, "repo_b": str}
+
+        Returns:
+            ImpactAnalysisResult with path chain
+        """
+        query = """
+        MATCH path = shortestPath(
+            (a:REPOSITORY {name: $repo_a})-[:DEPENDS_ON*..10]->(b:REPOSITORY {name: $repo_b})
+        )
+        RETURN [node IN nodes(path) | node.name] AS dependency_chain
+        """
+
+        result = await session.run(query, repo_a=params["repo_a"], repo_b=params["repo_b"])
+
+        affected = []
+        async for record in result:
+            affected.append(record["dependency_chain"])
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_component_impact(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which components import target component (transitive)? (Task 10.1)
+
+        Args:
+            session: Neo4j session
+            params: {"component_name": str, "depth": int (optional, default 5)}
+
+        Returns:
+            ImpactAnalysisResult with list of dependent components
+        """
+        depth = params.get("depth", 5)
+        component_name = params["component_name"]
+
+        # Check if component exists (Task 10.6)
+        check_query = """
+        MATCH (c:Component {qualified_name: $component_name})
+        RETURN c.qualified_name AS name
+        """
+        check_result = await session.run(check_query, component_name=component_name)
+        exists = False
+        async for _ in check_result:
+            exists = True
+            break
+
+        if not exists:
+            # Find similar component names for suggestions (Task 10.6)
+            suggestions_query = """
+            MATCH (c:Component)
+            WHERE c.qualified_name CONTAINS $partial_name
+            RETURN c.qualified_name AS name
+            LIMIT 5
+            """
+            suggestions_result = await session.run(
+                suggestions_query,
+                partial_name=component_name[:min(len(component_name), 5)]
+            )
+            suggestions = []
+            async for record in suggestions_result:
+                suggestions.append(record["name"])
+
+            logger.warning(
+                "component_not_found",
+                component_name=component_name,
+                suggestions=suggestions
+            )
+
+            result = ImpactAnalysisResult(affected_components=[])
+            result.warning = f"Component '{component_name}' not found"
+            result.suggestions = suggestions
+            return result
+
+        # Find transitive dependencies with circular detection
+        query = f"""
+        MATCH path = (target:Component {{qualified_name: $component_name}})
+                     <-[:IMPORTS_COMPONENT*1..{depth}]-(dependent:Component)
+        WITH dependent, target, path,
+             size([n IN nodes(path) WHERE n = dependent | 1]) > 1 AS is_circular
+        RETURN DISTINCT
+            dependent.qualified_name AS qualified_name,
+            dependent.file_path AS file_path,
+            dependent.repository AS repository,
+            is_circular AS circular
+        ORDER BY dependent.repository, dependent.file_path
+        """
+
+        result = await session.run(query, component_name=component_name)
+
+        affected = []
+        async for record in result:
+            affected.append({
+                "qualified_name": record["qualified_name"],
+                "file_path": record["file_path"],
+                "repository": record["repository"],
+                "circular": record.get("circular", False)
+            })
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_api_consumer_impact(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which components/services call target API endpoint? (Task 10.2)
+
+        Args:
+            session: Neo4j session
+            params: {"api_url": str, "method": str (optional)}
+
+        Returns:
+            ImpactAnalysisResult with list of consumers (cross-language)
+        """
+        api_url = params["api_url"]
+        http_method = params.get("method")
+
+        # Build query with optional HTTP method filter
+        method_filter = ""
+        if http_method:
+            method_filter = "AND r.http_method = $http_method"
+
+        query = f"""
+        MATCH (api:API {{url: $api_url}})<-[r:CALLS_API]-(consumer)
+        WHERE (consumer:Class OR consumer:Component)
+        {method_filter}
+        RETURN DISTINCT
+            labels(consumer) AS type,
+            consumer.qualified_name AS name,
+            consumer.repository AS repository,
+            CASE
+                WHEN consumer:Class THEN 'java'
+                WHEN consumer:Component THEN coalesce(consumer.language, 'typescript')
+                ELSE 'unknown'
+            END AS language,
+            r.http_method AS http_method
+        ORDER BY repository, language, name
+        """
+
+        query_params = {"api_url": api_url}
+        if http_method:
+            query_params["http_method"] = http_method
+
+        result = await session.run(query, **query_params)
+
+        affected = []
+        async for record in result:
+            affected.append({
+                "type": record["type"],
+                "name": record["name"],
+                "repository": record["repository"],
+                "language": record["language"],
+                "http_method": record.get("http_method")
+            })
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_microfrontend_impact(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which microfrontends consume modules from target? (Task 10.3)
+
+        Args:
+            session: Neo4j session
+            params: {"microfrontend_name": str}
+
+        Returns:
+            ImpactAnalysisResult with list of consuming microfrontends
+        """
+        microfrontend_name = params["microfrontend_name"]
+
+        query = """
+        MATCH (target:Microfrontend {name: $microfrontend_name})
+              -[:EXPOSES_MODULE]->()
+              <-[:CONSUMES_MODULE]-(consumer:Microfrontend)
+        OPTIONAL MATCH (consumer)-[:USES_PACKAGE]->(pkg:NPM_PACKAGE)
+        WHERE pkg.name IN ['react', 'vue', 'angular']
+        WITH consumer, collect({name: pkg.name, version: pkg.version}) AS shared_deps
+        RETURN DISTINCT
+            consumer.name AS name,
+            consumer.filename AS url,
+            shared_deps
+        ORDER BY consumer.name
+        """
+
+        result = await session.run(query, microfrontend_name=microfrontend_name)
+
+        affected = []
+        async for record in result:
+            # Convert shared_deps list to dict for easier conflict detection
+            shared_dict = {}
+            for dep in record["shared_deps"]:
+                if dep["name"]:
+                    shared_dict[dep["name"]] = dep["version"]
+
+            affected.append({
+                "name": record["name"],
+                "url": record["url"],
+                "shared_deps": shared_dict if shared_dict else None
+            })
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def _query_package_impact(self, session, params: dict) -> ImpactAnalysisResult:
+        """
+        Query: Which components depend on target npm package? (Task 10.4)
+
+        Args:
+            session: Neo4j session
+            params: {"package_name": str, "max_depth": int (optional, default 3), "scope": str (optional)}
+
+        Returns:
+            ImpactAnalysisResult with list of dependent components
+        """
+        package_name = params["package_name"]
+        max_depth = params.get("max_depth", 3)
+        scope = params.get("scope")
+
+        # Build scope filter
+        scope_filter = ""
+        if scope:
+            scope_filter = "AND pkg.scope = $scope"
+
+        query = f"""
+        MATCH path = (pkg:NPM_PACKAGE {{name: $package_name}})
+                     <-[:USES_PACKAGE*1..{max_depth}]-(component)
+        WHERE (component:Component OR component:Repository)
+        {scope_filter}
+        WITH component, pkg, length(path) AS depth
+        RETURN DISTINCT
+            component.qualified_name AS component,
+            component.repository AS repository,
+            depth,
+            pkg.version AS version,
+            pkg.scope AS scope
+        ORDER BY repository, depth
+        """
+
+        query_params = {"package_name": package_name}
+        if scope:
+            query_params["scope"] = scope
+
+        result = await session.run(query, **query_params)
+
+        affected = []
+        async for record in result:
+            affected.append({
+                "component": record["component"],
+                "repository": record["repository"],
+                "depth": record["depth"],
+                "version": record.get("version"),
+                "scope": record.get("scope", "production")
+            })
+
+        return ImpactAnalysisResult(affected_components=affected)
+
+    async def create_indexes(self) -> None:
+        """
+        Create indexes for query performance (Requirement 6.6).
+
+        Creates indexes on:
+        - Class.qualified_name
+        - Method.qualified_name
+        - Repository.name
+        - Library.group_id, artifact_id
+        - API.url
+        """
+        async with self.driver.session() as session:
+            indexes = [
+                "CREATE INDEX IF NOT EXISTS FOR (n:CLASS) ON (n.qualified_name)",
+                "CREATE INDEX IF NOT EXISTS FOR (n:METHOD) ON (n.qualified_name)",
+                "CREATE INDEX IF NOT EXISTS FOR (n:REPOSITORY) ON (n.name)",
+                "CREATE INDEX IF NOT EXISTS FOR (n:LIBRARY) ON (n.group_id)",
+                "CREATE INDEX IF NOT EXISTS FOR (n:LIBRARY) ON (n.artifact_id)",
+                "CREATE INDEX IF NOT EXISTS FOR (n:API) ON (n.url)",
+            ]
+
+            for index_query in indexes:
+                await session.run(index_query)
+
+        logger.info("neo4j_indexes_created", count=len(indexes))