smart-commit-cli 0.1.0__py3-none-any.whl

smart_commit/clustering/semantic.py

@@ -0,0 +1,201 @@
+import numpy as np
+from typing import List, Optional
+from pathlib import Path
+
+from ..models import FileDiff, CommitCluster
+from ..embeddings import EmbeddingsGenerator, SimilarityGraph
+from ..analysis.dependency_graph import DependencyGraph
+from ..logger import get_logger
+from ..constants import DEFAULT_SIMILARITY_THRESHOLD, MAX_CLUSTER_SIZE
+from .heuristic import HeuristicClustering
+from .graph_clustering import GraphClustering
+
+logger = get_logger("clustering.semantic")
+
+
+class SemanticClustering:
+    """Clusters files using semantic similarity and graph clustering."""
+
+    def __init__(self, threshold: float = DEFAULT_SIMILARITY_THRESHOLD):
+        """Initialize semantic clustering."""
+        self.threshold = threshold
+        self.embeddings_gen = None
+        self.similarity_graph = None
+        self.graph_clustering = GraphClustering()
+        self.dep_graph = None
+        try:
+            self.embeddings_gen = EmbeddingsGenerator()
+        except Exception as exc:
+            logger.warning("Embeddings unavailable: %s", exc)
+
+    def cluster_files(self, file_diffs: list[FileDiff], repo_path: Optional[Path] = None) -> list[CommitCluster]:
+        """Cluster files using semantic and graph-based clustering."""
+        if len(file_diffs) <= 1:
+            return [
+                CommitCluster(
+                    id="cluster_0",
+                    files=file_diffs,
+                    confidence=0.99,
+                    reason="Single file commit.",
+                )
+            ]
+
+        # Init dep graph if possible
+        if repo_path and not self.dep_graph:
+            self.dep_graph = DependencyGraph(repo_path)
+            
+        if not self.similarity_graph and self.embeddings_gen:
+            self.similarity_graph = SimilarityGraph(self.embeddings_gen, self.dep_graph)
+
+        if not self.embeddings_gen or not self.similarity_graph:
+            return HeuristicClustering().cluster_files(file_diffs, MAX_CLUSTER_SIZE)
+        
+        # Build similarity matrix
+        logger.info("Building similarity graph...")
+        similarity_matrix = self.similarity_graph.build_graph(file_diffs)
+        
+        # Generate embeddings
+        logger.info("Generating embeddings...")
+        embeddings = []
+        for file_diff in file_diffs:
+            embedding = self.embeddings_gen.generate_file_embedding(file_diff)
+            embeddings.append(embedding)
+        
+        # Perform clustering
+        logger.info("Clustering files...")
+        
+        # Select strategy: graph for large, agglomerative for small
+        if self.graph_clustering.is_available() and len(file_diffs) > 15:
+            logger.info("Using Graph Clustering (Louvain)")
+            cluster_labels = self.graph_clustering.cluster_files(similarity_matrix, file_diffs)
+        else:
+            try:
+                cluster_labels = self._agglomerative_clustering(similarity_matrix, file_diffs)
+            except Exception as e:
+                logger.warning(f"Agglomerative clustering failed: {e}, falling back")
+                cluster_labels = list(range(len(file_diffs)))
+        
+        # Group by cluster
+        clusters_dict = {}
+        for idx, label in enumerate(cluster_labels):
+            if label not in clusters_dict:
+                clusters_dict[label] = []
+            clusters_dict[label].append(file_diffs[idx])
+        
+        # Create clusters with embeddings
+        clusters = []
+        for cluster_idx, (label, files) in enumerate(sorted(clusters_dict.items())):
+            valid_embeddings = [e for e in [embeddings[file_diffs.index(f)] for f in files] if e]
+            cluster_embedding = np.mean(valid_embeddings, axis=0).tolist() if valid_embeddings else None
+            
+            reason = self._generate_reason(files, similarity_matrix, file_diffs)
+            cluster = CommitCluster(
+                id=f"cluster_{cluster_idx}",
+                files=files,
+                embedding=cluster_embedding,
+                confidence=self._cluster_confidence(files, similarity_matrix, file_diffs),
+                reason=reason,
+            )
+            clusters.append(cluster)
+        
+        logger.info(f"Created {len(clusters)} clusters from {len(file_diffs)} files")
+        return clusters
+
+    def _agglomerative_clustering(self, similarity_matrix: np.ndarray,
+                                  file_diffs: list[FileDiff]) -> list[int]:
+        """Perform agglomerative clustering."""
+        try:
+            from sklearn.cluster import AgglomerativeClustering
+            distance_matrix = 1.0 - similarity_matrix
+            n_clusters = max(1, len(file_diffs) // 3)
+            
+            clustering = AgglomerativeClustering(
+                n_clusters=n_clusters,
+                linkage='average',
+                metric='precomputed'
+            )
+            
+            labels = clustering.fit_predict(distance_matrix)
+            return labels.tolist()
+            
+        except ImportError:
+            logger.warning("scikit-learn not available, using greedy clustering")
+            return self._greedy_clustering(similarity_matrix)
+
+    @staticmethod
+    def _greedy_clustering(similarity_matrix: np.ndarray) -> list[int]:
+        """Greedy clustering fallback."""
+        n = len(similarity_matrix)
+        labels = list(range(n))
+        
+        for i in range(n):
+            for j in range(i + 1, n):
+                if similarity_matrix[i][j] > 0.6:
+                    if labels[j] != labels[i]:
+                        old_label = labels[j]
+                        new_label = labels[i]
+                        labels = [new_label if x == old_label else x for x in labels]
+        
+        unique_labels = list(set(labels))
+        label_map = {old: new for new, old in enumerate(unique_labels)}
+        return [label_map[x] for x in labels]
+
+    @staticmethod
+    def _cluster_confidence(
+        files: list[FileDiff],
+        similarity_matrix: np.ndarray,
+        all_files: list[FileDiff],
+    ) -> float:
+        if len(files) <= 1:
+            return 0.99
+
+        indexes = [all_files.index(file_diff) for file_diff in files]
+        scores = []
+        for left_index, left in enumerate(indexes):
+            for right in indexes[left_index + 1 :]:
+                scores.append(float(similarity_matrix[left][right]))
+
+        if not scores:
+            return 0.74
+        return max(0.55, min(0.98, sum(scores) / len(scores)))
+
+    @staticmethod
+    def _generate_reason(
+        files: list[FileDiff],
+        similarity_matrix: np.ndarray,
+        all_files: list[FileDiff],
+    ) -> str:
+        """Generate human-readable subsystem-level reason for grouping."""
+        if len(files) <= 1:
+            if files and files[0].is_renamed:
+                return "File rename operation."
+            return "Single file commit."
+            
+        dirs = set(f.file_path.rsplit('/', 1)[0] if '/' in f.file_path else 'root' for f in files)
+        primary_dir = list(dirs)[0] if dirs else "core"
+        
+        # Subsystem-level explanation
+        subsystem = f"'{primary_dir}' subsystem" if len(dirs) == 1 else "multiple subsystems"
+        
+        # Aggregate significant symbols
+        all_added = []
+        all_modified = []
+        for f in files:
+            all_added.extend(f.extracted_symbols.added + f.extracted_symbols.classes)
+            all_modified.extend(f.extracted_symbols.modified + f.extracted_symbols.functions)
+            
+        all_added = list(set(all_added))
+        all_modified = list(set(all_modified))
+        
+        actions = []
+        if all_added:
+            actions.append(f"introducing {', '.join(all_added[:2])}")
+        if all_modified:
+            actions.append(f"updating {', '.join(all_modified[:2])}")
+            
+        if actions:
+            reason = f"These files modify the {subsystem} by {' and '.join(actions)}."
+        else:
+            reason = f"These files heavily relate to the {subsystem} based on shared dependencies and structural similarity."
+            
+        return reason

smart_commit/clustering.py

@@ -0,0 +1,186 @@
+"""Semantic clustering of file changes."""
+
+from typing import List, Optional
+import numpy as np
+from .models import CommitCluster, FileDiff
+from .embeddings import EmbeddingsGenerator
+
+
+class Clustering:
+    """Groups related file changes using embeddings and ML."""
+
+    def __init__(self, method: str = "agglomerative", threshold: float = 0.5):
+        """Initialize clustering.
+        
+        Args:
+            method: Clustering algorithm ('agglomerative' or 'hdbscan').
+            threshold: Similarity threshold for grouping.
+        """
+        self.method = method
+        self.threshold = threshold
+        self.embeddings_gen = EmbeddingsGenerator(use_local=True)
+
+    def cluster_files(self, file_diffs: List[FileDiff]) -> List[CommitCluster]:
+        """Cluster files into logical groups.
+        
+        Args:
+            file_diffs: List of FileDiff objects.
+        
+        Returns:
+            List of CommitCluster objects.
+        """
+        if not file_diffs:
+            return []
+        
+        # Generate embeddings for all files
+        embeddings = []
+        for file_diff in file_diffs:
+            embedding = self.embeddings_gen.generate_file_embedding(file_diff)
+            embeddings.append(embedding)
+        
+        # Perform clustering
+        if self.method == "agglomerative":
+            cluster_labels = self._agglomerative_clustering(embeddings)
+        else:
+            cluster_labels = self._rule_based_clustering(file_diffs)
+        
+        # Group files by cluster label
+        clusters_dict = {}
+        for idx, label in enumerate(cluster_labels):
+            if label not in clusters_dict:
+                clusters_dict[label] = []
+            clusters_dict[label].append(file_diffs[idx])
+        
+        # Create CommitCluster objects
+        clusters = []
+        for idx, (label, files) in enumerate(sorted(clusters_dict.items())):
+            cluster = CommitCluster(
+                id=f"cluster_{idx}",
+                files=files,
+                embedding=embeddings[files[0] == file_diffs[0] and next(i for i, f in enumerate(file_diffs) if f == files[0])]
+                if files else None
+            )
+            clusters.append(cluster)
+        
+        return clusters
+
+    def _agglomerative_clustering(self, embeddings: List[Optional[List[float]]]) -> List[int]:
+        """Perform agglomerative clustering.
+        
+        Args:
+            embeddings: List of embedding vectors.
+        
+        Returns:
+            Cluster labels for each embedding.
+        """
+        try:
+            from sklearn.cluster import AgglomerativeClustering
+            
+            # Filter out None embeddings
+            valid_embeddings = []
+            valid_indices = []
+            
+            for idx, emb in enumerate(embeddings):
+                if emb is not None:
+                    valid_embeddings.append(emb)
+                    valid_indices.append(idx)
+            
+            if len(valid_embeddings) < 2:
+                return list(range(len(embeddings)))
+            
+            # Compute distance threshold
+            n_clusters = max(1, len(valid_embeddings) // 3)
+            
+            clustering = AgglomerativeClustering(
+                n_clusters=n_clusters,
+                linkage='average'
+            )
+            
+            labels = clustering.fit_predict(valid_embeddings)
+            
+            # Map back to original indices
+            result = [0] * len(embeddings)
+            for i, idx in enumerate(valid_indices):
+                result[idx] = labels[i]
+            
+            return result
+            
+        except ImportError:
+            return self._rule_based_clustering_simple(embeddings)
+
+    def _rule_based_clustering(self, file_diffs: List[FileDiff]) -> List[int]:
+        """Perform rule-based clustering by directory and extension.
+        
+        Args:
+            file_diffs: List of FileDiff objects.
+        
+        Returns:
+            Cluster labels for each file.
+        """
+        labels = []
+        
+        for file_diff in file_diffs:
+            # Extract directory and extension
+            path_parts = file_diff.file_path.split('/')
+            directory = path_parts[0] if path_parts else "root"
+            
+            # Convert to a simple hash label
+            label = hash((directory, file_diff.file_path.split('.')[-1])) % 100
+            labels.append(label)
+        
+        return labels
+
+    def _rule_based_clustering_simple(self, embeddings: List[Optional[List[float]]]) -> List[int]:
+        """Simple rule-based clustering as fallback.
+        
+        Args:
+            embeddings: List of embedding vectors (unused, kept for compatibility).
+        
+        Returns:
+            Cluster labels.
+        """
+        # Each file gets its own cluster if no embeddings
+        return list(range(len(embeddings)))
+
+    def merge_similar_clusters(self, clusters: List[CommitCluster]) -> List[CommitCluster]:
+        """Merge clusters that are very similar.
+        
+        Args:
+            clusters: List of CommitCluster objects.
+        
+        Returns:
+            Merged list of clusters.
+        """
+        if len(clusters) < 2:
+            return clusters
+        
+        merged = []
+        used = set()
+        
+        for i, cluster1 in enumerate(clusters):
+            if i in used:
+                continue
+            
+            merged_files = cluster1.files.copy()
+            
+            for j, cluster2 in enumerate(clusters[i+1:], i+1):
+                if j in used:
+                    continue
+                
+                similarity = self.embeddings_gen.compute_cluster_similarity(cluster1, cluster2)
+                
+                if similarity > self.threshold:
+                    merged_files.extend(cluster2.files)
+                    used.add(j)
+            
+            # Create new merged cluster
+            merged_cluster = CommitCluster(
+                id=cluster1.id,
+                files=merged_files,
+                embedding=cluster1.embedding
+            )
+            merged.append(merged_cluster)
+            used.add(i)
+        
+        return merged
+

smart_commit/commit_messages.py

@@ -0,0 +1,232 @@
+"""Commit message generation."""
+
+from typing import Optional
+from .models import CommitCluster, CommitType
+from .config import Config
+
+
+class CommitMessageGenerator:
+    """Generates Conventional Commit messages."""
+
+    def __init__(self, config: Optional[Config] = None):
+        """Initialize commit message generator.
+        
+        Args:
+            config: Configuration object.
+        """
+        self.config = config or Config.load()
+
+    def generate_message(self, cluster: CommitCluster) -> str:
+        """Generate a full commit message for a cluster.
+        
+        Args:
+            cluster: CommitCluster to generate message for.
+        
+        Returns:
+            Commit message string.
+        """
+        title = self.generate_title(cluster)
+        description = self.generate_description(cluster)
+        
+        if description:
+            return f"{title}\n\n{description}"
+        
+        return title
+
+    def generate_title(self, cluster: CommitCluster) -> str:
+        """Generate the commit title (first line).
+        
+        Args:
+            cluster: CommitCluster to analyze.
+        
+        Returns:
+            Commit title.
+        """
+        commit_type = cluster.commit_type or self._infer_commit_type(cluster)
+        scope = cluster.scope or self._infer_scope(cluster)
+        message = cluster.message or self._generate_message_text(cluster)
+        message = self._normalize_subject(message)
+        
+        # Format as conventional commit
+        title = f"{commit_type.value}({scope}): {message}" if scope else f"{commit_type.value}: {message}"
+        return self._truncate_title(title)
+
+    def _infer_commit_type(self, cluster: CommitCluster) -> CommitType:
+        paths = [file_diff.file_path.lower() for file_diff in cluster.files]
+        if paths and all(self._get_file_type(path) == "docs" for path in paths):
+            return CommitType.DOCS
+        if any("test" in path or "spec" in path for path in paths):
+            return CommitType.TEST
+        if any(
+            path.endswith((".toml", ".yaml", ".yml", ".json", ".ini", ".cfg", ".env"))
+            or path in {".gitignore", ".env.example"}
+            for path in paths
+        ):
+            return CommitType.CHORE
+        return self._infer_commit_type_from_summary(cluster.summary)
+
+    @staticmethod
+    def _infer_commit_type_from_summary(summary: Optional[str]) -> CommitType:
+        """Infer commit type from summary text.
+        
+        Args:
+            summary: Summary text.
+        
+        Returns:
+            Inferred CommitType.
+        """
+        if not summary:
+            return CommitType.CHORE
+        
+        summary_lower = summary.lower()
+        
+        if any(x in summary_lower for x in ['fix', 'bug', 'issue', 'broken']):
+            return CommitType.FIX
+        elif any(x in summary_lower for x in ['feat', 'feature', 'add', 'implement', 'new']):
+            return CommitType.FEAT
+        elif any(x in summary_lower for x in ['refactor', 'clean', 'improve', 'simplify']):
+            return CommitType.REFACTOR
+        elif any(x in summary_lower for x in ['test', 'spec', 'unit']):
+            return CommitType.TEST
+        elif any(x in summary_lower for x in ['doc', 'readme', 'comment', 'document']):
+            return CommitType.DOCS
+        elif any(x in summary_lower for x in ['style', 'format', 'lint']):
+            return CommitType.STYLE
+        elif any(x in summary_lower for x in ['perf', 'performance', 'optimize', 'speed']):
+            return CommitType.PERF
+        
+        return CommitType.CHORE
+
+    def generate_description(self, cluster: CommitCluster) -> str:
+        """Generate commit description (body).
+        
+        Args:
+            cluster: CommitCluster to analyze.
+        
+        Returns:
+            Commit description.
+        """
+        if not cluster.files:
+            return ""
+        
+        files_by_type = {}
+        for file_diff in cluster.files:
+            file_type = self._get_file_type(file_diff.file_path)
+            if file_type not in files_by_type:
+                files_by_type[file_type] = []
+            files_by_type[file_type].append(file_diff.file_path)
+        
+        description_lines = ["Changed files:"]
+        
+        for file_type, files in sorted(files_by_type.items()):
+            description_lines.append(f"\n{file_type}:")
+            for file_path in files[:5]:  # Limit to 5 per type
+                description_lines.append(f"  - {file_path}")
+            
+            if len(files) > 5:
+                description_lines.append(f"  - and {len(files) - 5} more...")
+        
+        return "\n".join(description_lines)
+
+    def _infer_scope(self, cluster: CommitCluster) -> Optional[str]:
+        """Infer scope from files in cluster.
+        
+        Args:
+            cluster: CommitCluster to analyze.
+        
+        Returns:
+            Scope string or None.
+        """
+        if not cluster.files:
+            return None
+        
+        # Extract common directory or feature name
+        file_path = cluster.files[0].file_path
+        parts = file_path.split('/')
+        
+        ignored = {"src", "lib", "app", "apps", "packages", "tests", "test", "smart_commit"}
+        for part in parts[:-1]:
+            clean = part.replace("_", "-").lower()
+            if clean and clean not in ignored:
+                return clean
+        
+        # Try to infer from file names
+        if any('auth' in f.file_path.lower() for f in cluster.files):
+            return "auth"
+        elif any('api' in f.file_path.lower() for f in cluster.files):
+            return "api"
+        elif any('ui' in f.file_path.lower() or 'component' in f.file_path.lower() for f in cluster.files):
+            return "ui"
+        
+        return None
+
+    def _generate_message_text(self, cluster: CommitCluster) -> str:
+        """Generate the message text from summary.
+        
+        Args:
+            cluster: CommitCluster to analyze.
+        
+        Returns:
+            Message text.
+        """
+        if cluster.summary:
+            summary = cluster.summary.strip().split(";")[0]
+            normalized = self._normalize_subject(summary)
+            if normalized in {"add content", "update content", "remove content"}:
+                return self._fallback_subject(cluster, normalized.split()[0])
+            return normalized
+        
+        return self._fallback_subject(cluster, "update")
+
+    def _fallback_subject(self, cluster: CommitCluster, verb: str) -> str:
+        if len(cluster.files) == 1:
+            return f"{verb} {cluster.files[0].file_path.split('/')[-1]}"
+
+        scope = cluster.scope or self._infer_scope(cluster)
+        if scope:
+            return f"{verb} {scope} changes"
+        return f"{verb} related files"
+
+    @staticmethod
+    def _normalize_subject(text: str) -> str:
+        text = text.strip().rstrip(".")
+        if not text:
+            return "update files"
+        text = text[0].lower() + text[1:]
+        replacements = {
+            "added new content": "add content",
+            "modified content": "update content",
+            "removed content": "remove content",
+        }
+        return replacements.get(text, text)
+
+    @staticmethod
+    def _truncate_title(title: str, limit: int = 72) -> str:
+        if len(title) <= limit:
+            return title
+        truncated = title[:limit].rsplit(" ", 1)[0]
+        return truncated if len(truncated) >= 30 else title[:limit].rstrip()
+
+    @staticmethod
+    def _get_file_type(file_path: str) -> str:
+        """Get file type category.
+        
+        Args:
+            file_path: Path to file.
+        
+        Returns:
+            File type category.
+        """
+        file_lower = file_path.lower()
+        
+        if any(x in file_lower for x in ['.jsx', '.tsx', '.vue', '.css']):
+            return "frontend"
+        elif any(x in file_lower for x in ['.py', '.java', '.go', '.rs', '.rb']):
+            return "backend"
+        elif any(x in file_lower for x in ['.test.', '.spec.']):
+            return "tests"
+        elif any(x in file_lower for x in ['.md', '.txt']):
+            return "docs"
+        
+        return "other"
+