ai-coding-assistant 0.5.0__py3-none-any.whl

coding_assistant/documentation/graph/dependency_builder.py

@@ -0,0 +1,468 @@
+"""Build comprehensive dependency graphs for code repositories."""
+
+from typing import Dict, List, Set, Tuple, Optional
+from pathlib import Path
+from dataclasses import dataclass, field
+import networkx as nx
+from collections import defaultdict
+
+from coding_assistant.codebase.parser import CodeParser
+from coding_assistant.codebase.crawler import CodebaseCrawler
+from coding_assistant.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class CodeEntity:
+    """Represents a code entity (file, module, class, function)."""
+    type: str  # 'file', 'module', 'class', 'function'
+    name: str
+    path: str
+    imports: List[str] = field(default_factory=list)
+    exports: List[str] = field(default_factory=list)
+    calls: List[str] = field(default_factory=list)  # Function/method calls
+    line_count: int = 0
+    language: str = 'python'
+    metadata: Dict = field(default_factory=dict)
+
+    def __hash__(self):
+        """Make entity hashable for use in sets and as dict keys."""
+        return hash((self.type, self.name, self.path))
+
+
+class DependencyGraphBuilder:
+    """
+    Build multi-level dependency graphs for code analysis.
+
+    Creates three types of graphs:
+    1. File-level: Dependencies between files based on imports
+    2. Module-level: Dependencies between logical modules/packages
+    3. Call-level: Function/method call relationships
+    """
+
+    def __init__(self, codebase_path: Optional[Path] = None):
+        """
+        Initialize the dependency graph builder.
+
+        Args:
+            codebase_path: Path to the codebase root (defaults to current directory)
+        """
+        self.codebase_path = codebase_path or Path.cwd()
+        self.file_graph = nx.DiGraph()
+        self.module_graph = nx.DiGraph()
+        self.call_graph = nx.DiGraph()
+        self.parser = CodeParser()
+        self.crawler = CodebaseCrawler(self.codebase_path)
+
+        # Cache for parsed files
+        self._parsed_cache: Dict[str, Dict] = {}
+        self._entities: Dict[str, CodeEntity] = {}
+
+    def build_file_graph(self, codebase_path: Optional[Path] = None) -> nx.DiGraph:
+        """
+        Build file-level dependency graph based on imports.
+
+        Args:
+            codebase_path: Path to analyze (overrides instance path if provided)
+
+        Returns:
+            NetworkX DiGraph with files as nodes and imports as edges
+        """
+        path = codebase_path or self.codebase_path
+        logger.info(f"Building file dependency graph for {path}")
+
+        # Update crawler path if different
+        if path != self.codebase_path:
+            self.crawler = CodebaseCrawler(path)
+            self.codebase_path = path
+
+        # Crawl for all code files
+        file_info_list = self.crawler.scan(max_files=1000)
+
+        # Filter for Python files (we'll add multi-language support later)
+        python_files = [
+            f['absolute_path'] for f in file_info_list
+            if f['extension'] == '.py'
+        ]
+
+        logger.info(f"Found {len(python_files)} Python files to analyze")
+
+        # Parse each file and extract imports
+        for file_path in python_files:
+            try:
+                self._process_file_for_graph(file_path)
+            except Exception as e:
+                logger.warning(f"Failed to process {file_path}: {e}")
+                continue
+
+        # Build edges based on imports
+        self._build_file_edges()
+
+        logger.info(
+            f"File graph built: {self.file_graph.number_of_nodes()} nodes, "
+            f"{self.file_graph.number_of_edges()} edges"
+        )
+
+        return self.file_graph
+
+    def _process_file_for_graph(self, file_path: str):
+        """Process a single file and add to graph."""
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+        except Exception as e:
+            logger.warning(f"Could not read {file_path}: {e}")
+            return
+
+        # Parse file
+        parsed = self.parser.parse_file(file_path, content)
+        self._parsed_cache[file_path] = parsed
+
+        # Create entity for this file
+        entity = CodeEntity(
+            type='file',
+            name=Path(file_path).name,
+            path=file_path,
+            imports=parsed.get('imports', []),
+            line_count=len(content.split('\n')),
+            language='python',
+            metadata={
+                'functions': len(parsed.get('functions', [])),
+                'classes': len(parsed.get('classes', []))
+            }
+        )
+
+        self._entities[file_path] = entity
+
+        # Add node to graph
+        self.file_graph.add_node(
+            file_path,
+            entity=entity,
+            label=entity.name,
+            imports=entity.imports,
+            line_count=entity.line_count
+        )
+
+    def _build_file_edges(self):
+        """Build edges between files based on import relationships."""
+        # Build mapping of module names to file paths
+        module_to_file = self._build_module_mapping()
+
+        for file_path, entity in self._entities.items():
+            for import_stmt in entity.imports:
+                # Parse import statement to extract module name
+                imported_module = self._extract_module_from_import(import_stmt)
+
+                if imported_module in module_to_file:
+                    target_file = module_to_file[imported_module]
+                    if target_file != file_path:  # Avoid self-loops
+                        self.file_graph.add_edge(
+                            file_path,
+                            target_file,
+                            import_type='direct'
+                        )
+
+    def _build_module_mapping(self) -> Dict[str, str]:
+        """
+        Build mapping from Python module names to file paths.
+
+        Example: 'coding_assistant.llm.client' -> 'src/coding_assistant/llm/client.py'
+        """
+        module_to_file = {}
+
+        for file_path in self._entities.keys():
+            # Convert file path to module name
+            path_obj = Path(file_path)
+
+            # Find the root of the Python package
+            parts = path_obj.parts
+
+            # Look for src/ or the project root
+            try:
+                if 'src' in parts:
+                    src_idx = parts.index('src')
+                    module_parts = parts[src_idx + 1:]
+                else:
+                    # Use relative to codebase path
+                    rel_path = path_obj.relative_to(self.codebase_path)
+                    module_parts = rel_path.parts
+
+                # Remove .py extension
+                if module_parts[-1].endswith('.py'):
+                    module_parts = list(module_parts[:-1]) + [module_parts[-1][:-3]]
+
+                # Remove __init__ from module name
+                if module_parts[-1] == '__init__':
+                    module_parts = module_parts[:-1]
+
+                module_name = '.'.join(module_parts)
+                module_to_file[module_name] = file_path
+
+            except ValueError:
+                # Path is not relative to codebase_path, skip
+                continue
+
+        return module_to_file
+
+    def _extract_module_from_import(self, import_stmt: str) -> str:
+        """
+        Extract module name from import statement.
+
+        Examples:
+            'import foo.bar' -> 'foo.bar'
+            'from foo.bar import baz' -> 'foo.bar'
+            'from . import something' -> '' (relative import, skip)
+        """
+        import_stmt = import_stmt.strip()
+
+        if import_stmt.startswith('from '):
+            # from X import Y
+            parts = import_stmt.split()
+            if len(parts) >= 2:
+                module = parts[1]
+                # Skip relative imports for now
+                if module.startswith('.'):
+                    return ''
+                return module
+        elif import_stmt.startswith('import '):
+            # import X
+            parts = import_stmt.split()
+            if len(parts) >= 2:
+                module = parts[1]
+                # Handle 'import X as Y'
+                if 'as' in parts:
+                    as_idx = parts.index('as')
+                    module = parts[as_idx - 1]
+                return module
+
+        return ''
+
+    def build_module_graph(self, file_graph: Optional[nx.DiGraph] = None) -> nx.DiGraph:
+        """
+        Build module-level graph by clustering related files.
+
+        Groups files into logical modules (packages/directories) and
+        creates module-level dependencies.
+
+        Args:
+            file_graph: File graph to use (uses instance graph if not provided)
+
+        Returns:
+            NetworkX DiGraph with modules as nodes
+        """
+        graph = file_graph or self.file_graph
+
+        if graph.number_of_nodes() == 0:
+            logger.warning("File graph is empty, building it first")
+            self.build_file_graph()
+            graph = self.file_graph
+
+        logger.info("Building module-level graph")
+
+        # Group files by directory (module)
+        module_files: Dict[str, List[str]] = defaultdict(list)
+
+        for file_path in graph.nodes():
+            module_name = self._get_module_name(file_path)
+            module_files[module_name].append(file_path)
+
+        # Create module nodes
+        for module_name, files in module_files.items():
+            total_lines = sum(
+                graph.nodes[f].get('line_count', 0) for f in files
+            )
+
+            self.module_graph.add_node(
+                module_name,
+                files=files,
+                file_count=len(files),
+                line_count=total_lines
+            )
+
+        # Create module edges based on file dependencies
+        for source_file, target_file in graph.edges():
+            source_module = self._get_module_name(source_file)
+            target_module = self._get_module_name(target_file)
+
+            if source_module != target_module:
+                # Add or strengthen edge
+                if self.module_graph.has_edge(source_module, target_module):
+                    self.module_graph[source_module][target_module]['weight'] += 1
+                else:
+                    self.module_graph.add_edge(
+                        source_module,
+                        target_module,
+                        weight=1
+                    )
+
+        logger.info(
+            f"Module graph built: {self.module_graph.number_of_nodes()} modules, "
+            f"{self.module_graph.number_of_edges()} dependencies"
+        )
+
+        return self.module_graph
+
+    def _get_module_name(self, file_path: str) -> str:
+        """
+        Get module name from file path.
+
+        Example: 'src/coding_assistant/llm/client.py' -> 'coding_assistant.llm'
+        """
+        path_obj = Path(file_path)
+
+        try:
+            parts = path_obj.parts
+
+            # Find src or project root
+            if 'src' in parts:
+                src_idx = parts.index('src')
+                module_parts = parts[src_idx + 1:-1]  # Exclude filename
+            else:
+                rel_path = path_obj.relative_to(self.codebase_path)
+                module_parts = rel_path.parts[:-1]  # Exclude filename
+
+            if not module_parts:
+                return 'root'
+
+            return '.'.join(module_parts)
+
+        except ValueError:
+            return 'external'
+
+    def detect_cycles(self, graph: Optional[nx.DiGraph] = None) -> List[List[str]]:
+        """
+        Detect circular dependencies in the graph.
+
+        Args:
+            graph: Graph to analyze (defaults to file_graph)
+
+        Returns:
+            List of cycles, where each cycle is a list of node names
+        """
+        graph = graph or self.file_graph
+
+        try:
+            cycles = list(nx.simple_cycles(graph))
+
+            if cycles:
+                logger.warning(f"Found {len(cycles)} circular dependencies")
+                for i, cycle in enumerate(cycles[:5]):  # Log first 5
+                    logger.warning(f"  Cycle {i+1}: {' -> '.join(cycle)}")
+            else:
+                logger.info("No circular dependencies found")
+
+            return cycles
+
+        except Exception as e:
+            logger.error(f"Error detecting cycles: {e}")
+            return []
+
+    def compute_metrics(self, graph: Optional[nx.DiGraph] = None) -> Dict:
+        """
+        Compute graph metrics (centrality, coupling, complexity, etc.).
+
+        Args:
+            graph: Graph to analyze (defaults to file_graph)
+
+        Returns:
+            Dictionary of metrics
+        """
+        graph = graph or self.file_graph
+
+        if graph.number_of_nodes() == 0:
+            return {
+                'nodes': 0,
+                'edges': 0,
+                'density': 0.0,
+                'avg_degree': 0.0,
+            }
+
+        metrics = {
+            'nodes': graph.number_of_nodes(),
+            'edges': graph.number_of_edges(),
+            'density': nx.density(graph),
+            'avg_degree': sum(dict(graph.degree()).values()) / graph.number_of_nodes(),
+        }
+
+        # Weakly connected components
+        if graph.is_directed():
+            metrics['connected_components'] = nx.number_weakly_connected_components(graph)
+        else:
+            metrics['connected_components'] = nx.number_connected_components(graph)
+
+        # Compute centrality for important nodes
+        try:
+            centrality = nx.betweenness_centrality(graph)
+            metrics['max_centrality'] = max(centrality.values()) if centrality else 0
+            metrics['avg_centrality'] = sum(centrality.values()) / len(centrality) if centrality else 0
+        except:
+            metrics['max_centrality'] = 0
+            metrics['avg_centrality'] = 0
+
+        logger.info(f"Graph metrics: {metrics}")
+
+        return metrics
+
+    def get_important_files(self, top_n: int = 10, graph: Optional[nx.DiGraph] = None) -> List[Tuple[str, float]]:
+        """
+        Get the most important/central files in the codebase.
+
+        Uses betweenness centrality to identify files that are
+        crucial connectors in the dependency graph.
+
+        Args:
+            top_n: Number of files to return
+            graph: Graph to analyze (defaults to file_graph)
+
+        Returns:
+            List of (file_path, centrality_score) tuples
+        """
+        graph = graph or self.file_graph
+
+        if graph.number_of_nodes() == 0:
+            return []
+
+        try:
+            centrality = nx.betweenness_centrality(graph)
+            sorted_files = sorted(
+                centrality.items(),
+                key=lambda x: x[1],
+                reverse=True
+            )
+
+            return sorted_files[:top_n]
+
+        except Exception as e:
+            logger.error(f"Error computing important files: {e}")
+            return []
+
+    def export_graph(self, output_path: str, graph: Optional[nx.DiGraph] = None, format: str = 'gml'):
+        """
+        Export graph to file for visualization or external analysis.
+
+        Args:
+            output_path: Path to save the graph
+            graph: Graph to export (defaults to file_graph)
+            format: Output format ('gml', 'graphml', 'json')
+        """
+        graph = graph or self.file_graph
+
+        try:
+            if format == 'gml':
+                nx.write_gml(graph, output_path)
+            elif format == 'graphml':
+                nx.write_graphml(graph, output_path)
+            elif format == 'json':
+                from networkx.readwrite import json_graph
+                import json
+                data = json_graph.node_link_data(graph)
+                with open(output_path, 'w') as f:
+                    json.dump(data, f, indent=2)
+            else:
+                raise ValueError(f"Unsupported format: {format}")
+
+            logger.info(f"Graph exported to {output_path}")
+
+        except Exception as e:
+            logger.error(f"Failed to export graph: {e}")