smart-commit-cli 0.1.0__py3-none-any.whl

smart_commit/analysis/diff_parser.py

@@ -0,0 +1,172 @@
+import re
+from ..models import FileDiff
+from ..logger import get_logger
+
+logger = get_logger("analysis.diff_parser")
+
+
+class DiffParser:
+    """Parses git diffs with enhanced language awareness."""
+
+    # Language-specific keywords
+    LANGUAGE_PATTERNS = {
+        ".py": {
+            "keywords": ["def ", "class ", "async def ", "import ", "from "],
+            "symbols": r"(?:def|class|async def)\s+(\w+)",
+        },
+        ".js": {
+            "keywords": ["function ", "class ", "const ", "import ", "export "],
+            "symbols": r"(?:function|class)\s+(\w+)|(?:const|let|var)\s+(\w+)",
+        },
+        ".ts": {
+            "keywords": ["interface ", "type ", "class ", "function ", "export "],
+            "symbols": r"(?:interface|type|class|function)\s+(\w+)",
+        },
+        ".tsx": {
+            "keywords": ["interface ", "function ", "const ", "export "],
+            "symbols": r"(?:interface|function)\s+(\w+)|(?:const)\s+(\w+)",
+        },
+        ".jsx": {
+            "keywords": ["function ", "export ", "const ", "import "],
+            "symbols": r"(?:function|const)\s+(\w+)",
+        },
+        ".java": {
+            "keywords": ["class ", "public ", "private ", "interface "],
+            "symbols": r"(?:class|interface)\s+(\w+)",
+        },
+        ".go": {
+            "keywords": ["func ", "type ", "package ", "import "],
+            "symbols": r"func\s+(?:\(\w+\s+\*?\w+\))?\s*(\w+)",
+        },
+        ".rs": {
+            "keywords": ["fn ", "struct ", "impl ", "pub ", "use "],
+            "symbols": r"(?:fn|struct|impl)\s+(\w+)",
+        },
+    }
+
+    def parse_diff(self, raw_diff: str, file_path: str) -> list[str]:
+        """Extract meaningful changes from a raw diff.
+        
+        Args:
+            raw_diff: Raw diff content.
+            file_path: Path to file.
+        
+        Returns:
+            List of change descriptions.
+        """
+        if not raw_diff.strip():
+            return []
+        
+        changes = []
+        file_ext = self._get_file_extension(file_path)
+        
+        # Parse diff lines
+        lines = raw_diff.split('\n')
+        added_lines = [l[1:] for l in lines if l.startswith('+') and not l.startswith('+++')]
+        removed_lines = [l[1:] for l in lines if l.startswith('-') and not l.startswith('---')]
+        
+        # High-level change type
+        if added_lines and not removed_lines:
+            changes.append("Added new content")
+        elif removed_lines and not added_lines:
+            changes.append("Removed content")
+        elif added_lines and removed_lines:
+            changes.append("Modified content")
+        
+        patterns = self.LANGUAGE_PATTERNS.get(file_ext, {})
+        if "symbols" in patterns:
+            symbols = self._extract_symbols(added_lines, patterns["symbols"])
+            for symbol in symbols:
+                changes.append(f"changed {symbol}")
+        
+        # Stats
+        num_added = len(added_lines)
+        num_removed = len(removed_lines)
+        
+        if num_added > 0 or num_removed > 0:
+            changes.append(f"+{num_added}/-{num_removed}")
+        
+        return changes
+
+    def extract_summary(self, raw_diff: str, file_path: str) -> str:
+        """Extract one-line summary of changes.
+        
+        Args:
+            raw_diff: Raw diff content.
+            file_path: Path to file.
+        
+        Returns:
+            Summary string.
+        """
+        changes = self.parse_diff(raw_diff, file_path)
+        
+        if not changes:
+            return "Modified file"
+        
+        significant = [
+            c for c in changes
+            if c not in ["Added new content", "Modified content", "Removed content"]
+            and not c.startswith("+")
+        ]
+        
+        if significant:
+            return significant[0]
+        
+        return changes[0]
+
+    def enrich_file_diff(self, file_diff: FileDiff) -> FileDiff:
+        """Attach deterministic symbols and imports to a file diff."""
+        file_ext = self._get_file_extension(file_diff.file_path)
+        patterns = self.LANGUAGE_PATTERNS.get(file_ext, {})
+        changed_lines = self._changed_lines(file_diff.raw_diff)
+        if "symbols" in patterns:
+            file_diff.symbols = self._extract_symbols(changed_lines, patterns["symbols"])
+        file_diff.imports = self._extract_imports(changed_lines)
+        return file_diff
+
+    @staticmethod
+    def _get_file_extension(file_path: str) -> str:
+        """Get file extension."""
+        if '.' not in file_path:
+            return ""
+        return '.' + file_path.split('.')[-1]
+
+    @staticmethod
+    def _extract_symbols(lines: list[str], pattern: str) -> list[str]:
+        """Extract symbols using regex pattern."""
+        symbols = []
+        
+        for line in lines:
+            matches = re.findall(pattern, line)
+            for match in matches:
+                if isinstance(match, tuple):
+                    symbol = next((m for m in match if m), None)
+                else:
+                    symbol = match
+                
+                if symbol:
+                    symbols.append(symbol)
+        
+        return list(set(symbols))  # Deduplicate
+
+    @staticmethod
+    def _changed_lines(raw_diff: str) -> list[str]:
+        lines = []
+        for line in raw_diff.splitlines():
+            if line.startswith(("+++", "---")):
+                continue
+            if line.startswith(("+", "-")):
+                lines.append(line[1:])
+        return lines
+
+    @staticmethod
+    def _extract_imports(lines: list[str]) -> list[str]:
+        imports = []
+        for line in lines:
+            stripped = line.strip()
+            if stripped.startswith(("import ", "from ")):
+                imports.append(stripped)
+            elif " require(" in stripped:
+                imports.append(stripped)
+        return sorted(set(imports))
+

smart_commit/analysis/summarizer.py

@@ -0,0 +1,101 @@
+"""Summary generation from diffs and rich semantics."""
+
+from ..models import FileDiff
+from ..logger import get_logger
+from .diff_parser import DiffParser
+from ..cache import cache_manager
+
+logger = get_logger("analysis.summarizer")
+
+
+class Summarizer:
+    """Converts raw diffs and structured symbols into semantic summaries."""
+
+    def __init__(self):
+        """Initialize summarizer."""
+        self.diff_parser = DiffParser()
+
+    def summarize_file_diff(self, file_diff: FileDiff) -> str:
+        """Generate summary for a file diff using AST and diff analysis.
+        
+        Args:
+            file_diff: FileDiff object.
+        
+        Returns:
+            Summary string (target: 20-80 words).
+        """
+        if file_diff.summary:
+            return file_diff.summary
+
+        # Check cache
+        cache_key = file_diff.raw_diff + str(file_diff.extracted_symbols)
+        cached = cache_manager.get("summaries", cache_key)
+        if cached:
+            return cached
+            
+        sym = file_diff.extracted_symbols
+        sentences = []
+        
+        if sym.added:
+            sentences.append(f"Added {', '.join(sym.added[:3])}{' and more' if len(sym.added)>3 else ''}.")
+        if sym.modified:
+            sentences.append(f"Refactored {', '.join(sym.modified[:3])}.")
+        if sym.removed:
+            sentences.append(f"Removed {', '.join(sym.removed[:3])}.")
+        if sym.imported:
+            sentences.append(f"Introduced dependencies on {', '.join(sym.imported[:3])}.")
+            
+        if not sentences:
+            # Fallback
+            summary = self.diff_parser.extract_summary(
+                file_diff.raw_diff,
+                file_diff.file_path
+            )
+            if len(summary) > 80:
+                summary = summary[:77] + "..."
+            sentences.append(summary)
+
+        summary = " ".join(sentences)
+        if len(summary) > 120:
+            summary = summary[:117] + "..."
+            
+        cache_manager.set("summaries", cache_key, summary)
+        return summary
+
+    def summarize_cluster(self, file_diffs: list[FileDiff]) -> str:
+        """Generate summary for a cluster of files."""
+        if not file_diffs:
+            return "No changes"
+        
+        summaries = [self.summarize_file_diff(diff) for diff in file_diffs]
+        unique_summaries = list(dict.fromkeys(summaries))
+        
+        if len(unique_summaries) == 1:
+            return unique_summaries[0]
+        
+        combined = " ".join(unique_summaries[:3])
+        if len(combined) > 120:
+            combined = combined[:117] + "..."
+        
+        return combined
+
+    def extract_keywords(self, file_diff: FileDiff) -> list[str]:
+        """Extract keywords from a file diff."""
+        keywords = []
+        sym = file_diff.extracted_symbols
+        keywords.extend(sym.added)
+        keywords.extend(sym.modified)
+        keywords.extend(sym.removed)
+        keywords.extend(sym.imported)
+        keywords.extend(sym.classes)
+        keywords.extend(sym.functions)
+        
+        if not keywords:
+            # Fallback
+            path_parts = file_diff.file_path.lower().split('/')
+            for part in path_parts:
+                clean = part.replace('.', '').replace('-', '_')
+                if clean and len(clean) > 2 and clean not in ['src', 'lib', 'utils', 'test']:
+                    keywords.append(clean)
+        
+        return list(set(keywords))

smart_commit/analysis/symbol_extractor.py

@@ -0,0 +1,88 @@
+"""Symbol extraction bridging diffs and AST."""
+
+from typing import List, Dict, Set
+from ..models import FileDiff, SymbolExtraction
+from .ast_parser import ASTParser
+from .diff_parser import DiffParser
+from ..logger import get_logger
+
+logger = get_logger("analysis.symbol_extractor")
+
+
+class SymbolExtractor:
+    """Extracts symbols from changed files."""
+
+    def __init__(self):
+        self.diff_parser = DiffParser()
+
+    def extract_symbols(self, file_diff: FileDiff) -> SymbolExtraction:
+        """Extract rich symbol information from a file diff.
+        
+        Args:
+            file_diff: FileDiff object containing raw diff.
+            
+        Returns:
+            SymbolExtraction with categorized symbols.
+        """
+        extraction = SymbolExtraction()
+        
+        if not file_diff.raw_diff.strip():
+            return extraction
+            
+        file_path = file_diff.file_path
+        ext = self.diff_parser._get_file_extension(file_path).lstrip(".")
+        if ext in ["py"]:
+            lang = "python"
+        elif ext in ["js", "jsx"]:
+            lang = "javascript"
+        elif ext in ["ts", "tsx"]:
+            lang = "typescript"
+        else:
+            lang = ext
+
+        # Try to parse full added content for AST if available
+        # But we only have diff. So we can extract added/removed lines
+        added_lines = [l[1:] for l in file_diff.raw_diff.split('\n') if l.startswith('+') and not l.startswith('+++')]
+        removed_lines = [l[1:] for l in file_diff.raw_diff.split('\n') if l.startswith('-') and not l.startswith('---')]
+        
+        added_content = "\n".join(added_lines)
+        removed_content = "\n".join(removed_lines)
+
+        ast_parser = ASTParser(lang)
+        
+        # Parse added content
+        if added_content:
+            extraction.classes.extend(ast_parser.extract_classes(added_content))
+            extraction.functions.extend(ast_parser.extract_functions(added_content))
+            extraction.imported.extend(ast_parser.extract_imports(added_content))
+            
+            # Use diff parser for standard symbols as a fallback
+            patterns = self.diff_parser.LANGUAGE_PATTERNS.get("." + ext, {})
+            if "symbols" in patterns:
+                symbols = self.diff_parser._extract_symbols(added_lines, patterns["symbols"])
+                extraction.added.extend(symbols)
+                
+            imports = self.diff_parser._extract_imports(added_lines)
+            extraction.imported.extend(imports)
+
+        # Parse removed content
+        if removed_content:
+            patterns = self.diff_parser.LANGUAGE_PATTERNS.get("." + ext, {})
+            if "symbols" in patterns:
+                symbols = self.diff_parser._extract_symbols(removed_lines, patterns["symbols"])
+                extraction.removed.extend(symbols)
+
+        # Deduplicate
+        extraction.classes = list(set(extraction.classes))
+        extraction.functions = list(set(extraction.functions))
+        extraction.imported = list(set(extraction.imported))
+        extraction.added = list(set(extraction.added))
+        extraction.removed = list(set(extraction.removed))
+
+        # Determine modified (present in both added and removed)
+        modified = set(extraction.added).intersection(set(extraction.removed))
+        extraction.modified = list(modified)
+        extraction.added = list(set(extraction.added) - modified)
+        extraction.removed = list(set(extraction.removed) - modified)
+
+        return extraction

smart_commit/benchmark.py

@@ -0,0 +1,88 @@
+"""Benchmarking for Smart Commit pipeline."""
+
+import time
+from pathlib import Path
+
+from .git import RepositoryScanner
+from .analysis import DiffParser, Summarizer
+from .analysis.symbol_extractor import SymbolExtractor
+from .analysis.dependency_graph import DependencyGraph
+from .clustering import SemanticClustering
+
+try:
+    from rich.console import Console
+    from rich.table import Table
+except ImportError:
+    Console = None
+    Table = None
+
+class BenchmarkHarness:
+    """Measures performance of the semantic pipeline."""
+
+    def __init__(self, repo_path: Path):
+        self.repo_path = repo_path
+        self.console = Console() if Console else None
+
+    def run(self):
+        """Execute the benchmark."""
+        print(f"Starting benchmark on {self.repo_path}...")
+        results = []
+        
+        # 1. Scanning
+        start = time.time()
+        snapshot = RepositoryScanner(self.repo_path).scan()
+        results.append(("Scanning", time.time() - start))
+        
+        num_files = snapshot.total_files_changed
+        if num_files == 0:
+            print("No files changed in repo. Cannot benchmark.")
+            return
+
+        # 2. Parsing & Symbols
+        start = time.time()
+        parser = DiffParser()
+        extractor = SymbolExtractor()
+        for file_diff in snapshot.all_files:
+            parser.enrich_file_diff(file_diff)
+            file_diff.extracted_symbols = extractor.extract_symbols(file_diff)
+        results.append(("AST & Symbols", time.time() - start))
+
+        # 3. Summaries
+        start = time.time()
+        summarizer = Summarizer()
+        for file_diff in snapshot.all_files:
+            file_diff.summary = summarizer.summarize_file_diff(file_diff)
+        results.append(("Summarization", time.time() - start))
+
+        # 4. Dependency Graph
+        start = time.time()
+        dep_graph = DependencyGraph(self.repo_path)
+        results.append(("Dependency Graph", time.time() - start))
+
+        # 5. Clustering (includes Embeddings)
+        start = time.time()
+        clustering = SemanticClustering()
+        clustering.dep_graph = dep_graph
+        clusters = clustering.cluster_files(snapshot.all_files, repo_path=self.repo_path)
+        results.append(("Clustering & Embeddings", time.time() - start))
+        
+        total_time = sum(t for _, t in results)
+        
+        if self.console and Table:
+            table = Table(title=f"Benchmark Results ({num_files} files)")
+            table.add_column("Stage", style="cyan")
+            table.add_column("Time (s)", justify="right", style="green")
+            
+            for stage, t in results:
+                table.add_row(stage, f"{t:.3f}")
+            table.add_row("Total", f"{total_time:.3f}", style="bold yellow")
+            
+            self.console.print()
+            self.console.print(table)
+        else:
+            print(f"\nBenchmark Results ({num_files} files)")
+            for stage, t in results:
+                print(f"{stage}: {t:.3f}s")
+            print(f"Total: {total_time:.3f}s")
+            
+        print(f"\nGenerated {len(clusters)} clusters.")

smart_commit/cache.py

@@ -0,0 +1,65 @@
+"""Disk-based caching for AST, summaries, and embeddings."""
+
+import json
+import hashlib
+from pathlib import Path
+from typing import Any, Optional, Dict
+
+from .logger import get_logger
+
+logger = get_logger("cache")
+
+
+class CacheManager:
+    """Manages disk-based caching for expensive operations."""
+
+    def __init__(self, cache_dir: str = ".smart_commit_cache"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_file = self.cache_dir / "cache.json"
+        self._cache: Dict[str, Any] = {}
+        self._load_cache()
+
+    def _load_cache(self):
+        """Load cache from disk."""
+        if not self.cache_file.exists():
+            return
+        try:
+            with open(self.cache_file, "r", encoding="utf-8") as f:
+                self._cache = json.load(f)
+        except Exception as e:
+            logger.debug(f"Failed to load cache: {e}")
+            self._cache = {}
+
+    def _save_cache(self):
+        """Save cache to disk."""
+        try:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            with open(self.cache_file, "w", encoding="utf-8") as f:
+                json.dump(self._cache, f)
+        except Exception as e:
+            logger.debug(f"Failed to save cache: {e}")
+
+    def _generate_key(self, namespace: str, text: str) -> str:
+        """Generate a stable cache key."""
+        text_hash = hashlib.md5(text.encode("utf-8")).hexdigest()
+        return f"{namespace}_{text_hash}"
+
+    def get(self, namespace: str, text: str) -> Optional[Any]:
+        """Retrieve a value from the cache."""
+        key = self._generate_key(namespace, text)
+        return self._cache.get(key)
+
+    def set(self, namespace: str, text: str, value: Any):
+        """Set a value in the cache."""
+        key = self._generate_key(namespace, text)
+        self._cache[key] = value
+        self._save_cache()
+
+    def clear(self):
+        """Clear the cache."""
+        self._cache = {}
+        if self.cache_file.exists():
+            self.cache_file.unlink()
+
+# Global cache instance
+cache_manager = CacheManager()