ai-coding-assistant 0.5.0__py3-none-any.whl

coding_assistant/documentation/decomposition/partitioner.py

@@ -0,0 +1,621 @@
+"""Dynamic programming-based repository partitioning for scalable processing.
+
+This module implements hierarchical decomposition inspired by CodeWiki's approach,
+using dynamic programming to optimally partition large repositories while preserving
+architectural context and maximizing module cohesion.
+"""
+
+from typing import List, Dict, Set, Tuple, Optional
+from dataclasses import dataclass, field
+from pathlib import Path
+import networkx as nx
+from collections import defaultdict
+
+from coding_assistant.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class Partition:
+    """
+    Represents a partition of the repository.
+
+    A partition is a coherent group of files that can be processed together,
+    with defined dependencies on other partitions.
+    """
+    name: str
+    files: List[str]
+    size_loc: int  # Total lines of code
+    dependencies: List[str] = field(default_factory=list)  # Other partition names
+    cohesion_score: float = 0.0
+    level: int = 0  # Hierarchy level (0=top, 1=module, 2=component)
+    metadata: Dict = field(default_factory=dict)
+
+    def __repr__(self):
+        return f"Partition({self.name}, files={len(self.files)}, LOC={self.size_loc})"
+
+
+class HierarchicalPartitioner:
+    """
+    Partition repository hierarchically using dynamic programming.
+
+    Inspired by CodeWiki's approach:
+    - Break large repos into coherent modules
+    - Maintain architectural relationships
+    - Optimize for processing efficiency
+    - Handle repos from 10K to 1M+ LOC
+
+    The algorithm uses dynamic programming to find optimal partition boundaries
+    that maximize internal cohesion while minimizing external coupling.
+    """
+
+    def __init__(self,
+                 max_partition_size: int = 10000,  # Max LOC per partition
+                 min_cohesion: float = 0.3,
+                 min_partition_size: int = 500):  # Min LOC per partition
+        """
+        Initialize the hierarchical partitioner.
+
+        Args:
+            max_partition_size: Maximum lines of code per partition
+            min_cohesion: Minimum cohesion score to accept a partition
+            min_partition_size: Minimum lines of code per partition
+        """
+        self.max_partition_size = max_partition_size
+        self.min_cohesion = min_cohesion
+        self.min_partition_size = min_partition_size
+
+    def partition(self,
+                  dependency_graph: nx.DiGraph,
+                  file_sizes: Dict[str, int],
+                  modules: Optional[Dict[str, List[str]]] = None) -> List[Partition]:
+        """
+        Partition repository into hierarchical modules.
+
+        Algorithm:
+        1. Start with detected modules (from Phase 1) or files
+        2. Split large modules recursively using DP
+        3. Merge small, highly coupled modules
+        4. Optimize partition boundaries
+
+        Args:
+            dependency_graph: File dependency graph from Phase 1
+            file_sizes: Dictionary mapping file paths to line counts
+            modules: Pre-detected modules (optional, will detect if not provided)
+
+        Returns:
+            List of Partition objects
+        """
+        logger.info("Starting hierarchical partitioning")
+
+        # If no modules provided, detect them first
+        if modules is None:
+            from coding_assistant.documentation.graph.module_analyzer import ModuleAnalyzer
+            analyzer = ModuleAnalyzer(dependency_graph)
+            modules = analyzer.detect_modules()
+            logger.info(f"Detected {len(modules)} initial modules")
+
+        partitions = []
+
+        # Process each module
+        for module_name, files in modules.items():
+            total_size = sum(file_sizes.get(f, 0) for f in files)
+
+            logger.debug(f"Processing module '{module_name}': {len(files)} files, {total_size} LOC")
+
+            if total_size > self.max_partition_size:
+                # Module too large, split recursively
+                logger.info(f"Module '{module_name}' exceeds max size ({total_size} > {self.max_partition_size}), splitting...")
+                sub_partitions = self._partition_large_module(
+                    module_name, files, dependency_graph, file_sizes
+                )
+                partitions.extend(sub_partitions)
+            elif total_size < self.min_partition_size:
+                # Module too small, mark for potential merging
+                logger.debug(f"Module '{module_name}' below min size ({total_size} < {self.min_partition_size})")
+                partitions.append(self._create_partition(
+                    module_name, files, dependency_graph, file_sizes, level=1
+                ))
+            else:
+                # Module is appropriately sized
+                partitions.append(self._create_partition(
+                    module_name, files, dependency_graph, file_sizes, level=1
+                ))
+
+        # Optimize partitions (merge small ones, adjust boundaries)
+        partitions = self._optimize_partitions(partitions, dependency_graph, file_sizes)
+
+        logger.info(f"Partitioning complete: {len(partitions)} partitions created")
+
+        return partitions
+
+    def _partition_large_module(self,
+                                 module_name: str,
+                                 files: List[str],
+                                 graph: nx.DiGraph,
+                                 file_sizes: Dict[str, int],
+                                 depth: int = 0) -> List[Partition]:
+        """
+        Recursively partition a large module using dynamic programming.
+
+        Uses graph partitioning to find optimal split points that:
+        - Maximize internal cohesion within each partition
+        - Minimize coupling between partitions
+        - Respect size constraints
+
+        Args:
+            module_name: Name of the module to partition
+            files: List of file paths in the module
+            graph: Dependency graph
+            file_sizes: File size dictionary
+            depth: Recursion depth (for naming)
+
+        Returns:
+            List of sub-partitions
+        """
+        if len(files) <= 1:
+            # Can't split a single file
+            return [self._create_partition(
+                f"{module_name}_part{depth}",
+                files,
+                graph,
+                file_sizes,
+                level=2
+            )]
+
+        # Create subgraph for this module
+        try:
+            subgraph = graph.subgraph(files).copy()
+        except:
+            # Files not in graph, create partition as-is
+            return [self._create_partition(
+                f"{module_name}_part{depth}",
+                files,
+                graph,
+                file_sizes,
+                level=2
+            )]
+
+        # Find optimal split using dynamic programming
+        split = self._find_optimal_split(subgraph, file_sizes, files)
+
+        if split is None or len(split[0]) == 0 or len(split[1]) == 0:
+            # Couldn't find good split, keep as one partition
+            return [self._create_partition(
+                f"{module_name}_part{depth}",
+                files,
+                graph,
+                file_sizes,
+                level=2
+            )]
+
+        partition1_files, partition2_files = split
+
+        # Create partitions for both splits
+        partitions = []
+
+        # Recursively partition if still too large
+        size1 = sum(file_sizes.get(f, 0) for f in partition1_files)
+        if size1 > self.max_partition_size and len(partition1_files) > 1:
+            partitions.extend(self._partition_large_module(
+                f"{module_name}_a", partition1_files, graph, file_sizes, depth + 1
+            ))
+        else:
+            partitions.append(self._create_partition(
+                f"{module_name}_a{depth}",
+                partition1_files,
+                graph,
+                file_sizes,
+                level=2
+            ))
+
+        size2 = sum(file_sizes.get(f, 0) for f in partition2_files)
+        if size2 > self.max_partition_size and len(partition2_files) > 1:
+            partitions.extend(self._partition_large_module(
+                f"{module_name}_b", partition2_files, graph, file_sizes, depth + 1
+            ))
+        else:
+            partitions.append(self._create_partition(
+                f"{module_name}_b{depth}",
+                partition2_files,
+                graph,
+                file_sizes,
+                level=2
+            ))
+
+        return partitions
+
+    def _find_optimal_split(self,
+                             graph: nx.DiGraph,
+                             file_sizes: Dict[str, int],
+                             files: List[str]) -> Optional[Tuple[List[str], List[str]]]:
+        """
+        Find optimal split point using dynamic programming approach.
+
+        Objective: Maximize (internal_cohesion - external_coupling)
+        Subject to: Size constraints
+
+        This uses a modified version of the balanced graph partitioning problem.
+        We use spectral bisection for efficiency.
+
+        Args:
+            graph: Subgraph of files to split
+            file_sizes: File size dictionary
+            files: List of files to split
+
+        Returns:
+            Tuple of (partition1_files, partition2_files) or None if no good split
+        """
+        if graph.number_of_nodes() < 2:
+            return None
+
+        try:
+            # Use spectral bisection for graph partitioning
+            # This is an approximation but much faster than exact DP
+
+            # Convert to undirected for spectral analysis
+            undirected = graph.to_undirected()
+
+            # Compute Fiedler vector (second eigenvector of Laplacian)
+            try:
+                import numpy as np
+                from scipy.sparse.linalg import eigsh
+
+                # Build Laplacian matrix
+                L = nx.laplacian_matrix(undirected)
+
+                # Compute second smallest eigenvalue and eigenvector
+                eigenvalues, eigenvectors = eigsh(L.asfptype(), k=2, which='SM')
+                fiedler_vector = eigenvectors[:, 1]
+
+                # Split based on sign of Fiedler vector
+                nodes = list(undirected.nodes())
+                partition1 = [nodes[i] for i in range(len(nodes)) if fiedler_vector[i] >= 0]
+                partition2 = [nodes[i] for i in range(len(nodes)) if fiedler_vector[i] < 0]
+
+            except ImportError:
+                # Scipy not available, use simple heuristic
+                logger.warning("scipy not available, using simple bisection")
+                nodes = list(graph.nodes())
+                mid = len(nodes) // 2
+                partition1 = nodes[:mid]
+                partition2 = nodes[mid:]
+
+        except Exception as e:
+            logger.warning(f"Spectral bisection failed: {e}, using simple split")
+            # Fallback to simple bisection
+            nodes = list(graph.nodes())
+            mid = len(nodes) // 2
+            partition1 = nodes[:mid]
+            partition2 = nodes[mid:]
+
+        # Validate split meets size constraints
+        size1 = sum(file_sizes.get(f, 0) for f in partition1)
+        size2 = sum(file_sizes.get(f, 0) for f in partition2)
+
+        # Check if split is balanced enough and meets constraints
+        total_size = size1 + size2
+        if total_size == 0:
+            return None
+
+        balance = min(size1, size2) / total_size
+
+        # Require at least 20% balance
+        if balance < 0.2:
+            logger.debug(f"Split too unbalanced: {balance:.2f}")
+            # Try to rebalance
+            partition1, partition2 = self._rebalance_split(
+                partition1, partition2, file_sizes
+            )
+
+        return (partition1, partition2)
+
+    def _rebalance_split(self,
+                          partition1: List[str],
+                          partition2: List[str],
+                          file_sizes: Dict[str, int]) -> Tuple[List[str], List[str]]:
+        """
+        Rebalance an unbalanced partition split.
+
+        Moves files from larger partition to smaller one until balanced.
+        """
+        p1 = list(partition1)
+        p2 = list(partition2)
+
+        size1 = sum(file_sizes.get(f, 0) for f in p1)
+        size2 = sum(file_sizes.get(f, 0) for f in p2)
+
+        # Move files from larger to smaller
+        if size1 > size2:
+            larger, smaller = p1, p2
+            larger_size, smaller_size = size1, size2
+        else:
+            larger, smaller = p2, p1
+            larger_size, smaller_size = size2, size1
+
+        # Sort larger partition by file size
+        larger_sorted = sorted(larger, key=lambda f: file_sizes.get(f, 0))
+
+        # Move smallest files until balanced
+        while larger_size > smaller_size * 1.5 and len(larger_sorted) > 1:
+            file_to_move = larger_sorted.pop(0)
+            file_size = file_sizes.get(file_to_move, 0)
+
+            larger.remove(file_to_move)
+            smaller.append(file_to_move)
+
+            larger_size -= file_size
+            smaller_size += file_size
+
+        if size1 > size2:
+            return (larger, smaller)
+        else:
+            return (smaller, larger)
+
+    def _create_partition(self,
+                           name: str,
+                           files: List[str],
+                           graph: nx.DiGraph,
+                           file_sizes: Dict[str, int],
+                           level: int = 1) -> Partition:
+        """
+        Create a Partition object with computed metrics.
+
+        Args:
+            name: Partition name
+            files: List of files in partition
+            graph: Full dependency graph
+            file_sizes: File size dictionary
+            level: Hierarchy level
+
+        Returns:
+            Partition object with computed metrics
+        """
+        total_size = sum(file_sizes.get(f, 0) for f in files)
+        dependencies = self._compute_dependencies(files, graph)
+        cohesion = self._compute_cohesion(files, graph)
+
+        return Partition(
+            name=name,
+            files=files,
+            size_loc=total_size,
+            dependencies=dependencies,
+            cohesion_score=cohesion,
+            level=level,
+            metadata={
+                'file_count': len(files),
+                'avg_file_size': total_size / len(files) if files else 0
+            }
+        )
+
+    def _compute_dependencies(self, files: List[str], graph: nx.DiGraph) -> List[str]:
+        """
+        Compute which other partitions this partition depends on.
+
+        Returns list of external files that this partition depends on.
+        """
+        dependencies = set()
+
+        for file in files:
+            if not graph.has_node(file):
+                continue
+
+            # Check all outgoing edges
+            for _, target in graph.out_edges(file):
+                if target not in files:
+                    dependencies.add(target)
+
+        return list(dependencies)
+
+    def _compute_cohesion(self, files: List[str], graph: nx.DiGraph) -> float:
+        """
+        Compute cohesion score for a group of files.
+
+        Cohesion = (internal edges) / (possible internal edges)
+        """
+        if len(files) < 2:
+            return 1.0
+
+        try:
+            subgraph = graph.subgraph(files)
+            internal_edges = subgraph.number_of_edges()
+
+            # Possible edges in directed graph: n * (n-1)
+            n = len(files)
+            possible_edges = n * (n - 1)
+
+            if possible_edges == 0:
+                return 1.0
+
+            cohesion = internal_edges / possible_edges
+            return min(cohesion, 1.0)
+
+        except:
+            return 0.0
+
+    def _optimize_partitions(self,
+                              partitions: List[Partition],
+                              graph: nx.DiGraph,
+                              file_sizes: Dict[str, int]) -> List[Partition]:
+        """
+        Optimize partition boundaries.
+
+        - Merge small, highly coupled partitions
+        - Adjust boundaries to reduce coupling
+        - Ensure all partitions meet quality thresholds
+
+        Args:
+            partitions: Initial list of partitions
+            graph: Dependency graph
+            file_sizes: File size dictionary
+
+        Returns:
+            Optimized list of partitions
+        """
+        logger.info("Optimizing partitions...")
+
+        optimized = list(partitions)
+
+        # Merge small, highly coupled partitions
+        optimized = self._merge_small_partitions(optimized, graph, file_sizes)
+
+        # Update dependencies after merging
+        optimized = self._update_partition_dependencies(optimized, graph)
+
+        logger.info(f"Optimization complete: {len(optimized)} final partitions")
+
+        return optimized
+
+    def _merge_small_partitions(self,
+                                  partitions: List[Partition],
+                                  graph: nx.DiGraph,
+                                  file_sizes: Dict[str, int]) -> List[Partition]:
+        """
+        Merge small partitions that are highly coupled.
+        """
+        merged = []
+        to_merge = []
+
+        for partition in partitions:
+            if partition.size_loc < self.min_partition_size:
+                to_merge.append(partition)
+            else:
+                merged.append(partition)
+
+        if not to_merge:
+            return partitions
+
+        logger.info(f"Merging {len(to_merge)} small partitions")
+
+        # Group small partitions by coupling
+        while to_merge:
+            current = to_merge.pop(0)
+
+            # Find most coupled partition
+            best_match = None
+            best_coupling = 0
+
+            for other in merged + to_merge:
+                if other.name == current.name:
+                    continue
+
+                coupling = self._compute_partition_coupling(
+                    current.files, other.files, graph
+                )
+
+                if coupling > best_coupling:
+                    best_coupling = coupling
+                    best_match = other
+
+            if best_match and best_coupling > 0.1:
+                # Merge with best match
+                if best_match in merged:
+                    merged.remove(best_match)
+                else:
+                    to_merge.remove(best_match)
+
+                # Create merged partition
+                merged_files = current.files + best_match.files
+                merged_name = f"{current.name}+{best_match.name}"
+
+                merged_partition = self._create_partition(
+                    merged_name,
+                    merged_files,
+                    graph,
+                    file_sizes,
+                    level=min(current.level, best_match.level)
+                )
+
+                merged.append(merged_partition)
+            else:
+                # No good match, keep as is
+                merged.append(current)
+
+        return merged
+
+    def _compute_partition_coupling(self,
+                                      files1: List[str],
+                                      files2: List[str],
+                                      graph: nx.DiGraph) -> float:
+        """
+        Compute coupling between two groups of files.
+
+        Coupling = (edges between groups) / (possible edges)
+        """
+        if not files1 or not files2:
+            return 0.0
+
+        edges_between = 0
+
+        for file1 in files1:
+            for file2 in files2:
+                if graph.has_edge(file1, file2):
+                    edges_between += 1
+                if graph.has_edge(file2, file1):
+                    edges_between += 1
+
+        possible_edges = len(files1) * len(files2) * 2
+
+        if possible_edges == 0:
+            return 0.0
+
+        return edges_between / possible_edges
+
+    def _update_partition_dependencies(self,
+                                         partitions: List[Partition],
+                                         graph: nx.DiGraph) -> List[Partition]:
+        """
+        Update partition dependencies to reference partition names instead of files.
+        """
+        # Build file -> partition mapping
+        file_to_partition = {}
+        for partition in partitions:
+            for file in partition.files:
+                file_to_partition[file] = partition.name
+
+        # Update dependencies
+        for partition in partitions:
+            external_files = partition.dependencies
+            dependent_partitions = set()
+
+            for file in external_files:
+                if file in file_to_partition:
+                    dep_partition = file_to_partition[file]
+                    if dep_partition != partition.name:
+                        dependent_partitions.add(dep_partition)
+
+            partition.dependencies = sorted(list(dependent_partitions))
+
+        return partitions
+
+    def analyze_partitioning_quality(self, partitions: List[Partition]) -> Dict:
+        """
+        Analyze the quality of the partitioning.
+
+        Returns metrics about partition quality:
+        - Average cohesion
+        - Average size
+        - Size distribution
+        - Dependency complexity
+        """
+        if not partitions:
+            return {}
+
+        cohesions = [p.cohesion_score for p in partitions]
+        sizes = [p.size_loc for p in partitions]
+        dep_counts = [len(p.dependencies) for p in partitions]
+
+        return {
+            'partition_count': len(partitions),
+            'avg_cohesion': sum(cohesions) / len(cohesions),
+            'min_cohesion': min(cohesions),
+            'max_cohesion': max(cohesions),
+            'avg_size': sum(sizes) / len(sizes),
+            'min_size': min(sizes),
+            'max_size': max(sizes),
+            'avg_dependencies': sum(dep_counts) / len(dep_counts),
+            'max_dependencies': max(dep_counts),
+            'total_files': sum(len(p.files) for p in partitions),
+            'total_loc': sum(p.size_loc for p in partitions),
+        }

coding_assistant/documentation/generators/__init__.py

@@ -0,0 +1,14 @@
+"""Visual diagram generators for documentation.
+
+This module provides generators for various diagram types including
+architecture diagrams, class diagrams, sequence diagrams, and data flow
+visualizations using Mermaid syntax.
+"""
+
+from .diagram_generator import MermaidDiagramGenerator
+from .dataflow_generator import DataFlowGenerator
+
+__all__ = [
+    'MermaidDiagramGenerator',
+    'DataFlowGenerator',
+]