velune-cli 0.9.0__py3-none-any.whl

velune/repository/import_graph.py

@@ -0,0 +1,263 @@
+"""Import graph builder for Python, JavaScript, and TypeScript files.
+
+Constructs a directed graph where an edge A → B means "A imports from B".
+Computes structural metrics: fan_in (who imports this?), fan_out (what does this import?).
+
+Phase 2a uses simple regex for JS/TS; full tree-sitter parsing in Phase 2b-1.
+"""
+
+from __future__ import annotations
+
+import ast
+import logging
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger("velune.repository.import_graph")
+
+
+@dataclass
+class ImportMetrics:
+    """Metrics for a module/symbol in the import graph."""
+
+    module_path: str
+    fan_in: int = 0  # Count of modules that import this module
+    fan_out: int = 0  # Count of modules this module imports
+    is_imported_by_tests: bool = False
+    importers: set[str] = field(default_factory=set)  # Modules that import this
+    imports: set[str] = field(default_factory=set)  # Modules this imports
+
+
+class ImportGraphBuilder:
+    """Builds an import dependency graph for a codebase.
+
+    Extracts import statements from Python, JavaScript, and TypeScript files,
+    then computes fan_in/fan_out metrics for each module.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the import graph builder."""
+        self._graph: dict[str, ImportMetrics] = {}
+        self._file_extensions = {".py", ".js", ".ts", ".jsx", ".tsx"}
+
+    def build_from_directory(self, root_path: Path) -> dict[str, ImportMetrics]:
+        """Scan directory recursively and build the import graph.
+
+        Parameters
+        ----------
+        root_path:
+            Root directory to scan for Python/JS/TS files.
+
+        Returns
+        -------
+        dict[str, ImportMetrics]:
+            Graph mapping module paths to their metrics.
+        """
+        self._graph.clear()
+
+        # First pass: discover all files
+        file_paths = self._discover_files(root_path)
+        logger.info("Discovered %d source files", len(file_paths))
+
+        # Second pass: extract imports from each file
+        for file_path in file_paths:
+            self._extract_imports(file_path, root_path)
+
+        # Third pass: compute metrics
+        self._compute_metrics()
+
+        logger.info("Built import graph with %d modules", len(self._graph))
+        return self._graph
+
+    def _discover_files(self, root_path: Path) -> list[Path]:
+        """Recursively discover all Python/JS/TS files."""
+        files = []
+        try:
+            for path in root_path.rglob("*"):
+                if path.is_file() and path.suffix in self._file_extensions:
+                    # Skip common exclusions
+                    if any(
+                        part in ("node_modules", ".git", "__pycache__", ".venv", "venv")
+                        for part in path.parts
+                    ):
+                        continue
+                    files.append(path)
+        except Exception as exc:
+            logger.warning("Error discovering files in %s: %s", root_path, exc)
+        return sorted(files)
+
+    def _extract_imports(self, file_path: Path, root_path: Path) -> None:
+        """Extract import statements from a single file."""
+        try:
+            relative_path = file_path.relative_to(root_path).as_posix()
+
+            # Ensure module is in graph
+            if relative_path not in self._graph:
+                self._graph[relative_path] = ImportMetrics(module_path=relative_path)
+
+            if file_path.suffix == ".py":
+                self._extract_python_imports(file_path, relative_path)
+            elif file_path.suffix in {".js", ".ts", ".jsx", ".tsx"}:
+                self._extract_js_imports(file_path, relative_path)
+
+        except Exception as exc:
+            logger.debug("Failed to extract imports from %s: %s", file_path, exc)
+
+    def _extract_python_imports(self, file_path: Path, relative_path: str) -> None:
+        """Extract imports from a Python file using ast.parse()."""
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="ignore")
+            tree = ast.parse(content)
+
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        module_name = alias.name.split(".")[0]
+                        self._add_import_edge(relative_path, module_name)
+
+                elif isinstance(node, ast.ImportFrom):
+                    if node.module:
+                        module_name = node.module.split(".")[0]
+                        self._add_import_edge(relative_path, module_name)
+
+        except SyntaxError:
+            logger.debug("Syntax error parsing %s", file_path)
+        except Exception as exc:
+            logger.debug("Error extracting imports from %s: %s", file_path, exc)
+
+    def _extract_js_imports(self, file_path: Path, relative_path: str) -> None:
+        """Extract imports from JS/TS files using regex (Phase 2a).
+
+        Phase 2b will use full tree-sitter parsing for better accuracy.
+        """
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="ignore")
+
+            # ES6 import: import ... from 'module' or "module"
+            es6_pattern = r"import\s+(?:{[^}]*}|[^'\"]*)\s+from\s+['\"]([^'\"]+)['\"]"
+            for match in re.finditer(es6_pattern, content):
+                module = match.group(1)
+                # Normalize: ./foo → foo, ../foo → foo
+                module = module.lstrip("./").split("/")[0]
+                if module and not module.startswith("."):
+                    self._add_import_edge(relative_path, module)
+
+            # CommonJS require: require('module') or require("module")
+            require_pattern = r"require\s*\(\s*['\"]([^'\"]+)['\"]\s*\)"
+            for match in re.finditer(require_pattern, content):
+                module = match.group(1)
+                module = module.lstrip("./").split("/")[0]
+                if module and not module.startswith("."):
+                    self._add_import_edge(relative_path, module)
+
+        except Exception as exc:
+            logger.debug("Error extracting imports from %s: %s", file_path, exc)
+
+    def _add_import_edge(self, from_module: str, to_module: str) -> None:
+        """Add an import edge A → B (A imports from B)."""
+        if from_module == to_module:
+            return  # Skip self-imports
+
+        # Ensure both modules are in graph
+        if from_module not in self._graph:
+            self._graph[from_module] = ImportMetrics(module_path=from_module)
+        if to_module not in self._graph:
+            self._graph[to_module] = ImportMetrics(module_path=to_module)
+
+        # Add edge
+        self._graph[from_module].imports.add(to_module)
+        self._graph[to_module].importers.add(from_module)
+
+    def _compute_metrics(self) -> None:
+        """Compute fan_in, fan_out, and test coverage metrics."""
+        for metrics in self._graph.values():
+            metrics.fan_in = len(metrics.importers)
+            metrics.fan_out = len(metrics.imports)
+
+            # Check if imported by any test file
+            metrics.is_imported_by_tests = any(
+                "test_" in importer or ".test." in importer for importer in metrics.importers
+            )
+
+    def get_metrics(self, module_path: str) -> ImportMetrics | None:
+        """Get metrics for a specific module.
+
+        Parameters
+        ----------
+        module_path:
+            The relative module path.
+
+        Returns
+        -------
+        ImportMetrics | None:
+            Metrics if module exists, None otherwise.
+        """
+        return self._graph.get(module_path)
+
+    def get_all_metrics(self) -> dict[str, ImportMetrics]:
+        """Return the complete import graph."""
+        return self._graph.copy()
+
+    def get_importers(self, module_path: str) -> set[str]:
+        """Get all modules that import the given module.
+
+        Parameters
+        ----------
+        module_path:
+            The module to query.
+
+        Returns
+        -------
+        set[str]:
+            Modules that import the given module.
+        """
+        metrics = self._graph.get(module_path)
+        return metrics.importers if metrics else set()
+
+    def get_imports(self, module_path: str) -> set[str]:
+        """Get all modules imported by the given module.
+
+        Parameters
+        ----------
+        module_path:
+            The module to query.
+
+        Returns
+        -------
+        set[str]:
+            Modules imported by the given module.
+        """
+        metrics = self._graph.get(module_path)
+        return metrics.imports if metrics else set()
+
+    def get_transitive_dependents(self, module_path: str, max_depth: int = 5) -> set[str]:
+        """Get all modules that depend on this module (transitively).
+
+        Parameters
+        ----------
+        module_path:
+            The module to query.
+        max_depth:
+            Maximum traversal depth to avoid cycles.
+
+        Returns
+        -------
+        set[str]:
+            All modules that transitively depend on this module.
+        """
+        dependents = set()
+        queue = [(module_path, 0)]
+
+        while queue:
+            current, depth = queue.pop(0)
+            if depth > max_depth:
+                continue
+
+            importers = self.get_importers(current)
+            for importer in importers:
+                if importer not in dependents:
+                    dependents.add(importer)
+                    queue.append((importer, depth + 1))
+
+        return dependents

velune/repository/incremental_indexer.py

@@ -0,0 +1,275 @@
+"""Incremental repository indexer that skips unchanged files between sessions.
+
+Algorithm
+---------
+1. Load the stored ``IndexState`` (or treat as empty on first run).
+2. Fetch the current git HEAD SHA.
+3. **Fast path**: if HEAD SHA equals the stored SHA *and* the working tree is
+   clean (no uncommitted changes), return an empty ``IndexDelta`` immediately —
+   no file I/O required.
+4. **Slow path**: walk workspace source files, compute SHA-256 for each, compare
+   to the stored hashes, and build the delta (added / updated / removed).
+5. ``apply_delta`` parses only the files in the delta, updates ``IndexState``,
+   and saves it to disk.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import subprocess
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from velune.repository.index_state import IndexedFile, IndexState
+
+logger = logging.getLogger("velune.repository.incremental_indexer")
+
+# Directories always excluded from the walk (on top of .veluneignore)
+_ALWAYS_SKIP = frozenset(
+    {
+        ".git",
+        "__pycache__",
+        "node_modules",
+        ".venv",
+        "venv",
+        "dist",
+        "build",
+        ".velune",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".pytest_cache",
+        "*.egg-info",
+    }
+)
+
+_CODE_EXTENSIONS = frozenset(
+    {
+        ".py",
+        ".js",
+        ".ts",
+        ".jsx",
+        ".tsx",
+        ".go",
+        ".rs",
+        ".java",
+        ".c",
+        ".cpp",
+        ".h",
+        ".cs",
+        ".php",
+        ".rb",
+        ".swift",
+        ".kt",
+    }
+)
+
+
+@dataclass
+class IndexDelta:
+    """Describes which files need to be added, re-parsed, or removed."""
+
+    to_add: list[str] = field(default_factory=list)  # new files not in stored state
+    to_update: list[str] = field(default_factory=list)  # files whose hash changed
+    to_remove: list[str] = field(default_factory=list)  # files deleted from disk
+
+    @property
+    def is_empty(self) -> bool:
+        return not self.to_add and not self.to_update and not self.to_remove
+
+    @property
+    def total(self) -> int:
+        return len(self.to_add) + len(self.to_update) + len(self.to_remove)
+
+
+class IncrementalIndexer:
+    """Computes and applies file-level deltas to keep ``IndexState`` current."""
+
+    def __init__(self, workspace_root: Path, state_path: Path) -> None:
+        self.workspace_root = workspace_root.resolve()
+        self.state_path = state_path
+
+    # ------------------------------------------------------------------
+    # Public async API
+    # ------------------------------------------------------------------
+
+    async def compute_delta(self) -> IndexDelta:
+        """Compare disk state to the stored ``IndexState`` and return the diff.
+
+        The git HEAD SHA is checked first.  If it matches the stored SHA *and*
+        the working tree has no uncommitted changes, an empty delta is returned
+        immediately — no file reads required.
+        """
+        state = IndexState.load(self.state_path)
+        git_sha = await asyncio.to_thread(self._get_git_sha)
+
+        # --- Fast path ---
+        if (
+            state is not None
+            and state.last_commit_sha is not None
+            and state.last_commit_sha == git_sha
+        ):
+            clean = await asyncio.to_thread(self._working_tree_is_clean)
+            if clean:
+                logger.debug("Fast path: git SHA matches and working tree is clean.")
+                return IndexDelta()
+
+        # --- Slow path: walk files and compare hashes ---
+        return await asyncio.to_thread(self._compute_file_delta, state)
+
+    async def apply_delta(self, delta: IndexDelta) -> IndexState:
+        """Parse only the files in *delta*, update stored ``IndexState``, save to disk.
+
+        Files in ``delta.to_remove`` are dropped from the state.
+        Files in ``delta.to_add`` and ``delta.to_update`` are hashed and parsed.
+        """
+        state = IndexState.load(self.state_path) or IndexState.empty(str(self.workspace_root))
+
+        # Remove deleted files
+        for rel_path in delta.to_remove:
+            state.remove_file(rel_path)
+            logger.debug("Removed from index: %s", rel_path)
+
+        # Parse added and modified files
+        now = time.time()
+        for rel_path in delta.to_add + delta.to_update:
+            full_path = self.workspace_root / rel_path
+            if not full_path.exists():
+                continue
+            try:
+                content = full_path.read_text(encoding="utf-8", errors="ignore")
+                sha = self._hash_file(full_path)
+                symbols, language = await asyncio.to_thread(self._parse_file, full_path, content)
+                state.update_file(
+                    IndexedFile(
+                        path=rel_path,
+                        content_hash=sha,
+                        language=language,
+                        symbol_count=len(symbols),
+                        indexed_at=now,
+                    )
+                )
+                logger.debug("Indexed: %s (%d symbols)", rel_path, len(symbols))
+            except Exception as exc:
+                logger.debug("Skipped %s during apply_delta: %s", rel_path, exc)
+
+        # Update metadata and persist
+        git_sha = await asyncio.to_thread(self._get_git_sha)
+        state.touch(git_sha)
+        state.workspace_root = str(self.workspace_root)
+        state.save(self.state_path)
+        return state
+
+    # ------------------------------------------------------------------
+    # Synchronous helpers (run in thread pool)
+    # ------------------------------------------------------------------
+
+    def _get_git_sha(self) -> str | None:
+        """Return the current git HEAD SHA, or None if git is unavailable."""
+        try:
+            result = subprocess.run(
+                ["git", "rev-parse", "HEAD"],
+                capture_output=True,
+                text=True,
+                cwd=str(self.workspace_root),
+                timeout=5,
+            )
+            if result.returncode == 0:
+                return result.stdout.strip()
+        except Exception:
+            pass
+        return None
+
+    def _working_tree_is_clean(self) -> bool:
+        """Return True when there are no staged or unstaged modifications."""
+        try:
+            result = subprocess.run(
+                ["git", "diff", "HEAD", "--name-only"],
+                capture_output=True,
+                text=True,
+                cwd=str(self.workspace_root),
+                timeout=5,
+            )
+            if result.returncode == 0:
+                return result.stdout.strip() == ""
+        except Exception:
+            pass
+        # If git is unavailable, assume dirty so we do the file scan
+        return False
+
+    def _compute_file_delta(self, state: IndexState | None) -> IndexDelta:
+        """Walk workspace files, compare hashes, and build the delta."""
+        stored = state.file_index if state else {}
+
+        # Discover current source files (reuse scanner for .veluneignore support)
+        try:
+            from velune.repository.scanner import FilesystemScanner
+
+            scanner = FilesystemScanner(self.workspace_root)
+            current_paths = scanner.scan_code_files()
+        except Exception:
+            current_paths = list(self._fallback_scan())
+
+        current: dict[str, Path] = {}
+        for p in current_paths:
+            try:
+                rel = str(p.relative_to(self.workspace_root)).replace("\\", "/")
+                current[rel] = p
+            except ValueError:
+                continue
+
+        to_add: list[str] = []
+        to_update: list[str] = []
+
+        for rel_path, abs_path in current.items():
+            try:
+                sha = self._hash_file(abs_path)
+            except Exception:
+                continue
+
+            stored_entry = stored.get(rel_path)
+            if stored_entry is None:
+                to_add.append(rel_path)
+            elif stored_entry.content_hash != sha:
+                to_update.append(rel_path)
+
+        to_remove = [p for p in stored if p not in current]
+
+        return IndexDelta(to_add=to_add, to_update=to_update, to_remove=to_remove)
+
+    def _fallback_scan(self) -> list[Path]:
+        """Minimal walk used when FilesystemScanner is unavailable."""
+        results: list[Path] = []
+        for path in self.workspace_root.rglob("*"):
+            if any(part in _ALWAYS_SKIP for part in path.parts):
+                continue
+            if path.is_file() and path.suffix.lower() in _CODE_EXTENSIONS:
+                results.append(path)
+        return results
+
+    def _parse_file(self, full_path: Path, content: str) -> tuple[list, str]:
+        """Parse *full_path* and return (symbols, language_str)."""
+        try:
+            from velune.repository.parser import ASTParser
+
+            parser = ASTParser()
+            symbols, _ = parser.parse(full_path, content)
+            lang = parser._detect_language(full_path)
+            return symbols, lang.value
+        except Exception:
+            return [], "unknown"
+
+    @staticmethod
+    def _hash_file(path: Path) -> str:
+        """SHA-256 of a file's raw bytes, read in 8-KiB chunks."""
+        sha = hashlib.sha256()
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                sha.update(chunk)
+        return sha.hexdigest()
+
+    @staticmethod
+    def _hash_content(data: bytes) -> str:
+        return hashlib.sha256(data).hexdigest()

velune/repository/index_state.py

@@ -0,0 +1,96 @@
+"""Persistent index state for incremental repository indexing."""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger("velune.repository.index_state")
+
+
+@dataclass
+class IndexedFile:
+    """Per-file metadata stored between indexing sessions."""
+
+    path: str  # Relative to workspace root (forward-slash)
+    content_hash: str  # SHA-256 of file contents at index time
+    language: str  # Detected language (e.g. "python")
+    symbol_count: int  # Number of parsed symbols (0 if parsing failed)
+    indexed_at: float  # Unix timestamp when this entry was written
+
+
+@dataclass
+class IndexState:
+    """Full workspace index state persisted between Velune sessions.
+
+    Saved as JSON at ``.velune/index_state.json``.  The ``last_commit_sha``
+    is used for the primary fast-path check: if git HEAD has not moved since
+    the last index run, file scanning can be skipped entirely.
+    """
+
+    workspace_root: str
+    last_commit_sha: str | None  # git HEAD SHA at last full index; None if no git
+    last_indexed_at: float  # Unix timestamp of the last completed index
+    file_index: dict[str, IndexedFile] = field(default_factory=dict)  # rel_path → IndexedFile
+
+    # ------------------------------------------------------------------
+    # Persistence helpers
+    # ------------------------------------------------------------------
+
+    def save(self, path: Path) -> None:
+        """Serialize this state to *path* as JSON (creates parent dirs as needed)."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        data = {
+            "workspace_root": self.workspace_root,
+            "last_commit_sha": self.last_commit_sha,
+            "last_indexed_at": self.last_indexed_at,
+            "file_index": {k: asdict(v) for k, v in self.file_index.items()},
+        }
+        try:
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(data, f, indent=2)
+        except Exception as exc:
+            logger.warning("Could not save index state to %s: %s", path, exc)
+
+    @classmethod
+    def load(cls, path: Path) -> IndexState | None:
+        """Deserialize from *path*.  Returns ``None`` if the file is missing or corrupt."""
+        if not path.exists():
+            return None
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+            file_index = {k: IndexedFile(**v) for k, v in data.get("file_index", {}).items()}
+            return cls(
+                workspace_root=data["workspace_root"],
+                last_commit_sha=data.get("last_commit_sha"),
+                last_indexed_at=float(data.get("last_indexed_at", 0.0)),
+                file_index=file_index,
+            )
+        except Exception as exc:
+            logger.warning("Could not load index state from %s: %s", path, exc)
+            return None
+
+    @classmethod
+    def empty(cls, workspace_root: str) -> IndexState:
+        """Return a blank state (first-run placeholder)."""
+        return cls(
+            workspace_root=workspace_root,
+            last_commit_sha=None,
+            last_indexed_at=0.0,
+            file_index={},
+        )
+
+    def update_file(self, indexed_file: IndexedFile) -> None:
+        self.file_index[indexed_file.path] = indexed_file
+
+    def remove_file(self, path: str) -> None:
+        self.file_index.pop(path, None)
+
+    def touch(self, commit_sha: str | None) -> None:
+        """Mark the state as freshly indexed at the current time."""
+        self.last_commit_sha = commit_sha
+        self.last_indexed_at = time.time()