codevira 1.6.0__py3-none-any.whl

indexer/chunker.py

@@ -0,0 +1,428 @@
+"""
+Multi-language source chunker for codebase indexing.
+Splits source files into function/class/module chunks for semantic search.
+
+Language support:
+  - Python: stdlib ast module (full support)
+  - TypeScript, Go, Rust: tree-sitter grammars via treesitter_parser
+"""
+from __future__ import annotations
+
+import ast
+import functools
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator
+
+from indexer.treesitter_parser import (
+    parse_file as ts_parse_file,
+    get_language as ts_get_language,
+    EXTENSION_MAP as TS_EXTENSION_MAP,
+)
+
+
+def _load_config() -> dict:
+    from mcp_server.paths import get_data_dir
+    config_path = get_data_dir() / "config.yaml"
+    if config_path.exists():
+        try:
+            import yaml
+            with open(config_path) as f:
+                return yaml.safe_load(f) or {}
+        except Exception:
+            pass
+    return {}
+
+
+SKIP_DIRS = {"__pycache__", ".venv", "venv", ".git", "node_modules", "migrations"}
+SKIP_FILES = {"__init__.py"}
+
+# All tree-sitter supported extensions for dispatch
+_TS_SUPPORTED_EXTENSIONS = set(TS_EXTENSION_MAP.keys())
+
+
+@functools.lru_cache(maxsize=None)
+def _get_project_config() -> tuple[frozenset[str], tuple[str, ...]]:
+    """Lazily load config.yaml and return (TARGET_DIRS, FILE_EXTENSIONS).
+
+    Cached so subsequent calls are free. lru_cache is used so that the
+    config is only loaded once per process after the data directory is known.
+    """
+    cfg = _load_config()
+    project_cfg = cfg.get("project", cfg)
+    target_dirs: frozenset[str] = frozenset(project_cfg.get("watched_dirs", ["src"]))
+    file_extensions: tuple[str, ...] = tuple(project_cfg.get("file_extensions", [".py"]))
+    return target_dirs, file_extensions
+
+
+@dataclass
+class CodeChunk:
+    file_path: str        # relative to project root
+    chunk_type: str       # "function" | "class" | "module"
+    name: str             # function/class name or filename for module chunks
+    source_text: str      # the actual source code
+    start_line: int
+    end_line: int
+    docstring: str        # first docstring if present, else ""
+    layer: str            # inferred from file path
+
+
+def _infer_layer(file_path: str) -> str:
+    parts = Path(file_path).parts
+    for i, part in enumerate(parts):
+        if part in {"generator", "assembler", "indexer", "scanner", "drift", "graph", "context"}:
+            return part
+        if part in {"api", "routes"}:
+            return "api"
+        if part in {"core", "datastore", "schemas"}:
+            return part
+        if part in {"contexts", "application", "providers", "control", "services", "handlers"}:
+            return part
+    return "unknown"
+
+
+def _get_docstring(node: ast.AST) -> str:
+    try:
+        return ast.get_docstring(node) or ""
+    except Exception:
+        return ""
+
+
+def _extract_source_lines(source_lines: list[str], start: int, end: int) -> str:
+    return "".join(source_lines[start - 1:end])
+
+
+def extract_imports(file_path: str, project_root: str) -> list[str]:
+    """
+    Parse a source file's import statements and return relative paths of
+    project-local imports only (skips stdlib and third-party packages).
+
+    Dispatches to Python ast or tree-sitter based on file extension.
+    Returns list of relative file paths (e.g. 'src/services/provider.py').
+    Paths that cannot be resolved to an existing file are omitted.
+    """
+    ext = Path(file_path).suffix.lower()
+
+    # Non-Python files: use tree-sitter import extraction
+    if ext in _TS_SUPPORTED_EXTENSIONS:
+        return _extract_imports_treesitter(file_path, project_root)
+
+    # Python files: existing ast-based extraction
+    return _extract_imports_python(file_path, project_root)
+
+
+def _extract_imports_treesitter(file_path: str, project_root: str) -> list[str]:
+    """
+    Extract import paths from a non-Python file using tree-sitter.
+    Resolves relative/local imports to actual project file paths where possible.
+    Falls back to raw module strings for unresolvable imports.
+    """
+    try:
+        parsed = ts_parse_file(file_path)
+    except (FileNotFoundError, ValueError):
+        return []
+
+    project_root_path = Path(project_root)
+    file_dir = Path(file_path).parent
+    results: list[str] = []
+
+    for imp in parsed.imports:
+        raw = imp.module
+        resolved = _resolve_ts_import(raw, file_dir, project_root_path)
+        if resolved and resolved not in results:
+            results.append(resolved)
+
+    return results
+
+
+def _resolve_ts_import(raw_module: str, file_dir: Path, project_root: Path) -> str | None:
+    """
+    Try to resolve a tree-sitter import string to a relative file path.
+    Handles TypeScript/JS relative imports, Go package imports, and Rust use paths.
+    """
+    # TypeScript/JS: relative imports like './foo' or '../bar'
+    if raw_module.startswith('.'):
+        # Resolve relative to the importing file's directory
+        candidates = [
+            file_dir / f"{raw_module}.ts",
+            file_dir / f"{raw_module}.tsx",
+            file_dir / f"{raw_module}.js",
+            file_dir / f"{raw_module}.jsx",
+            file_dir / raw_module / "index.ts",
+            file_dir / raw_module / "index.tsx",
+            file_dir / raw_module / "index.js",
+        ]
+        for c in candidates:
+            resolved = c.resolve()
+            if resolved.exists():
+                try:
+                    return str(resolved.relative_to(project_root))
+                except ValueError:
+                    continue
+        return None
+
+    # Non-relative: try as a project-local path (e.g. 'src/utils/foo')
+    # Check common extensions
+    for ext in ['.ts', '.tsx', '.js', '.go', '.rs']:
+        candidate = project_root / f"{raw_module}{ext}"
+        if candidate.exists():
+            return str(candidate.relative_to(project_root))
+
+    # Try as directory with index file
+    for index in ['index.ts', 'index.tsx', 'index.js', 'mod.rs']:
+        candidate = project_root / raw_module / index
+        if candidate.exists():
+            return str(candidate.relative_to(project_root))
+
+    # Go: package paths like 'project/internal/services'
+    # Try mapping to directory with .go files
+    candidate_dir = project_root / raw_module
+    if candidate_dir.is_dir():
+        go_files = list(candidate_dir.glob('*.go'))
+        if go_files:
+            return str(go_files[0].relative_to(project_root))
+
+    return None
+
+
+def _extract_imports_python(file_path: str, project_root: str) -> list[str]:
+    """
+    Parse a Python file's import statements and return relative paths of
+    project-local imports only (skips stdlib and third-party packages).
+    """
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            source = f.read()
+    except (OSError, UnicodeDecodeError):
+        return []
+
+    try:
+        tree = ast.parse(source, filename=file_path)
+    except SyntaxError:
+        return []
+
+    project_root_path = Path(project_root)
+    target_dirs, _ = _get_project_config()
+    project_packages = set(target_dirs)
+
+    results: list[str] = []
+
+    def _module_to_path(module: str) -> str | None:
+        """Convert a dotted module name to a relative file path if project-local."""
+        parts = module.split(".")
+        if not parts or parts[0] not in project_packages:
+            return None
+        candidates = [
+            project_root_path / Path(*parts) / "__init__.py",
+            project_root_path / Path(*parts[:-1]) / f"{parts[-1]}.py",
+            project_root_path / Path(*parts).with_suffix(".py"),
+        ]
+        for candidate in candidates:
+            if candidate.exists():
+                return str(candidate.relative_to(project_root_path))
+        direct = project_root_path / Path(*parts[:-1]) / f"{parts[-1]}.py"
+        rel = str(direct.relative_to(project_root_path))
+        if (project_root_path / Path(*parts[:-1])).exists():
+            return rel
+        return None
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                path = _module_to_path(alias.name)
+                if path and path not in results:
+                    results.append(path)
+        elif isinstance(node, ast.ImportFrom):
+            if node.level and node.level > 0:
+                file_rel = os.path.relpath(file_path, project_root)
+                file_parts = Path(file_rel).parts
+                base_parts = list(file_parts[:-node.level]) if node.level < len(file_parts) else []
+                if node.module:
+                    module_parts = base_parts + str(node.module).split(".")
+                else:
+                    module_parts = base_parts
+                abs_module = ".".join(module_parts)
+            elif node.module:
+                abs_module = str(node.module)
+            else:
+                continue
+            path = _module_to_path(abs_module)
+            if path and path not in results:
+                results.append(path)
+
+    return results
+
+
+def chunk_file(file_path: str, project_root: str) -> list[CodeChunk]:
+    """
+    Parse a source file and return all meaningful code chunks.
+    Dispatches to Python ast or tree-sitter based on file extension.
+    """
+    ext = Path(file_path).suffix.lower()
+
+    # Non-Python files: dispatch to tree-sitter chunker
+    if ext in _TS_SUPPORTED_EXTENSIONS:
+        return _chunk_file_treesitter(file_path, project_root)
+
+    # Python files: existing ast-based chunking
+    return _chunk_file_python(file_path, project_root)
+
+
+def _chunk_file_treesitter(file_path: str, project_root: str) -> list[CodeChunk]:
+    """Chunk a non-Python file using tree-sitter symbol extraction."""
+    rel_path = os.path.relpath(file_path, project_root)
+    layer = _infer_layer(rel_path)
+
+    try:
+        parsed = ts_parse_file(file_path)
+    except (FileNotFoundError, ValueError):
+        return []
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            source_lines = f.read().splitlines(keepends=True)
+    except (OSError, UnicodeDecodeError):
+        return []
+
+    chunks: list[CodeChunk] = []
+
+    # Module-level docstring chunk
+    if parsed.module_docstring:
+        chunks.append(CodeChunk(
+            file_path=rel_path,
+            chunk_type="module",
+            name=Path(file_path).stem,
+            source_text=parsed.module_docstring,
+            start_line=1,
+            end_line=1,
+            docstring=parsed.module_docstring,
+            layer=layer,
+        ))
+
+    for sym in parsed.symbols:
+        # Skip very short symbols (< 3 lines) like Python chunker does
+        if sym.end_line - sym.start_line < 3:
+            continue
+
+        source_text = _extract_source_lines(source_lines, sym.start_line, sym.end_line)
+
+        # For classes/structs/impl, limit source to first 15 lines (like Python chunker)
+        chunk_type = sym.kind
+        if chunk_type in ("class", "struct", "impl", "interface", "trait", "enum"):
+            sig_end = min(sym.start_line + 15, sym.end_line)
+            source_text = _extract_source_lines(source_lines, sym.start_line, sig_end)
+
+        chunks.append(CodeChunk(
+            file_path=rel_path,
+            chunk_type=chunk_type,
+            name=sym.name,
+            source_text=source_text,
+            start_line=sym.start_line,
+            end_line=sym.end_line,
+            docstring=sym.docstring or "",
+            layer=layer,
+        ))
+
+    return chunks
+
+
+def _chunk_file_python(file_path: str, project_root: str) -> list[CodeChunk]:
+    """Parse a Python file and return all meaningful code chunks."""
+    rel_path = os.path.relpath(file_path, project_root)
+    layer = _infer_layer(rel_path)
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            source = f.read()
+            source_lines = source.splitlines(keepends=True)
+    except (OSError, UnicodeDecodeError):
+        return []
+
+    try:
+        tree = ast.parse(source, filename=file_path)
+    except SyntaxError:
+        return []
+
+    chunks: list[CodeChunk] = []
+
+    # Module-level docstring chunk
+    module_doc = _get_docstring(tree)
+    if module_doc:
+        chunks.append(CodeChunk(
+            file_path=rel_path,
+            chunk_type="module",
+            name=Path(file_path).stem,
+            source_text=module_doc,
+            start_line=1,
+            end_line=1,
+            docstring=module_doc,
+            layer=layer,
+        ))
+
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            if node.name.startswith("__") and node.name.endswith("__"):
+                continue
+            end_line = getattr(node, "end_lineno", node.lineno)
+            source_text = _extract_source_lines(source_lines, node.lineno, end_line)
+            if end_line - node.lineno < 3:
+                continue
+            chunks.append(CodeChunk(
+                file_path=rel_path,
+                chunk_type="function",
+                name=node.name,
+                source_text=source_text,
+                start_line=node.lineno,
+                end_line=end_line,
+                docstring=_get_docstring(node),
+                layer=layer,
+            ))
+
+        elif isinstance(node, ast.ClassDef):
+            end_line = getattr(node, "end_lineno", node.lineno)
+            sig_end = min(node.lineno + 15, end_line)
+            source_text = _extract_source_lines(source_lines, node.lineno, sig_end)
+            chunks.append(CodeChunk(
+                file_path=rel_path,
+                chunk_type="class",
+                name=node.name,
+                source_text=source_text,
+                start_line=node.lineno,
+                end_line=end_line,
+                docstring=_get_docstring(node),
+                layer=layer,
+            ))
+
+    return chunks
+
+
+def iter_source_files(project_root: str) -> Iterator[str]:
+    """Yield source files in TARGET_DIRS matching configured file_extensions."""
+    target_dirs, file_extensions = _get_project_config()
+    extensions = file_extensions
+    seen_files = set()
+
+    for target_dir in target_dirs:
+        target_path = os.path.join(project_root, target_dir)
+        if not os.path.exists(target_path):
+            continue
+            
+        for root, dirs, files in os.walk(target_path):
+            # Prune skipped dirs
+            dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+            
+            for fname in files:
+                if fname.endswith(extensions) and fname not in SKIP_FILES:
+                    full_path = os.path.abspath(os.path.join(root, fname))
+                    if full_path not in seen_files:
+                        seen_files.add(full_path)
+                        yield full_path
+
+
+def chunk_project(project_root: str) -> list[CodeChunk]:
+    """Chunk all source files in the project. Returns flat list of all chunks."""
+    all_chunks: list[CodeChunk] = []
+    for file_path in iter_source_files(project_root):
+        all_chunks.extend(chunk_file(file_path, project_root))
+    return all_chunks

indexer/global_db.py

@@ -0,0 +1,197 @@
+"""
+global_db.py — Global SQLite database for cross-project intelligence.
+
+Stores aggregated preferences, learned rules, and project registry in
+~/.codevira/global.db. Enables new projects to inherit intelligence from
+all past projects on day 1.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class GlobalDB:
+    """Lightweight SQLite wrapper for the global cross-project database."""
+
+    def __init__(self, db_path: str | Path):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.conn = sqlite3.connect(str(self.db_path), timeout=5)
+        self.conn.row_factory = sqlite3.Row
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.execute("PRAGMA foreign_keys=ON")
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        self.conn.executescript("""
+            CREATE TABLE IF NOT EXISTS projects (
+                path TEXT PRIMARY KEY,
+                name TEXT NOT NULL,
+                language TEXT,
+                git_remote TEXT,
+                last_synced_at DATETIME DEFAULT CURRENT_TIMESTAMP
+            );
+
+            CREATE TABLE IF NOT EXISTS global_preferences (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                category TEXT NOT NULL,
+                signal TEXT NOT NULL,
+                example TEXT,
+                frequency INTEGER DEFAULT 1,
+                source_projects TEXT DEFAULT '[]',
+                updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+                UNIQUE(category, signal)
+            );
+
+            CREATE TABLE IF NOT EXISTS global_rules (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                rule_text TEXT NOT NULL UNIQUE,
+                confidence REAL DEFAULT 0.5,
+                source_projects TEXT DEFAULT '[]',
+                category TEXT,
+                language TEXT,
+                updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+            );
+        """)
+        self.conn.commit()
+
+    def close(self) -> None:
+        self.conn.close()
+
+    # ------------------------------------------------------------------
+    # Project registry
+    # ------------------------------------------------------------------
+
+    def register_project(self, path: str, name: str, language: str,
+                         git_remote: str | None = None) -> None:
+        # Ensure git_remote column exists (handles DBs created before v1.6)
+        try:
+            cols = [row[1] for row in self.conn.execute("PRAGMA table_info(projects)").fetchall()]
+            if "git_remote" not in cols:
+                self.conn.execute("ALTER TABLE projects ADD COLUMN git_remote TEXT")
+                self.conn.commit()
+        except Exception:
+            pass
+        self.conn.execute(
+            "INSERT OR REPLACE INTO projects (path, name, language, git_remote, last_synced_at) "
+            "VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)",
+            (path, name, language, git_remote),
+        )
+        self.conn.commit()
+
+    def find_project_by_remote(self, remote_url: str) -> str | None:
+        """Return the registered path for a project matching the given git remote URL, or None."""
+        try:
+            row = self.conn.execute(
+                "SELECT path FROM projects WHERE git_remote = ? LIMIT 1",
+                (remote_url,),
+            ).fetchone()
+            return row["path"] if row else None
+        except Exception:
+            return None
+
+    def get_project_count(self) -> int:
+        row = self.conn.execute("SELECT COUNT(*) FROM projects").fetchone()
+        return row[0] if row else 0
+
+    # ------------------------------------------------------------------
+    # Preferences
+    # ------------------------------------------------------------------
+
+    def upsert_preference(self, category: str, signal: str, example: str | None,
+                          source_project: str, frequency: int = 1) -> None:
+        """Insert or update a global preference. Aggregates frequency across projects."""
+        existing = self.conn.execute(
+            "SELECT id, frequency, source_projects FROM global_preferences WHERE category = ? AND signal = ?",
+            (category, signal),
+        ).fetchone()
+
+        if existing:
+            projects = json.loads(existing["source_projects"] or "[]")
+            if source_project not in projects:
+                projects.append(source_project)
+            new_freq = existing["frequency"] + frequency
+            self.conn.execute(
+                "UPDATE global_preferences SET frequency = ?, source_projects = ?, example = COALESCE(?, example), "
+                "updated_at = CURRENT_TIMESTAMP WHERE id = ?",
+                (new_freq, json.dumps(projects), example, existing["id"]),
+            )
+        else:
+            self.conn.execute(
+                "INSERT INTO global_preferences (category, signal, example, frequency, source_projects) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (category, signal, example, frequency, json.dumps([source_project])),
+            )
+        self.conn.commit()
+
+    def get_preferences(self, min_frequency: int = 3, language: str | None = None) -> list[dict]:
+        """Get global preferences above the frequency threshold."""
+        rows = self.conn.execute(
+            "SELECT category, signal, example, frequency, source_projects FROM global_preferences "
+            "WHERE frequency >= ? ORDER BY frequency DESC",
+            (min_frequency,),
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    # ------------------------------------------------------------------
+    # Rules
+    # ------------------------------------------------------------------
+
+    def upsert_rule(self, rule_text: str, confidence: float, source_project: str,
+                    category: str | None = None, language: str | None = None) -> None:
+        """Insert or update a global rule. Merges confidence via weighted average."""
+        existing = self.conn.execute(
+            "SELECT id, confidence, source_projects FROM global_rules WHERE rule_text = ?",
+            (rule_text,),
+        ).fetchone()
+
+        if existing:
+            projects = json.loads(existing["source_projects"] or "[]")
+            if source_project not in projects:
+                projects.append(source_project)
+            new_conf = existing["confidence"] * 0.6 + confidence * 0.4
+            self.conn.execute(
+                "UPDATE global_rules SET confidence = ?, source_projects = ?, "
+                "updated_at = CURRENT_TIMESTAMP WHERE id = ?",
+                (new_conf, json.dumps(projects), existing["id"]),
+            )
+        else:
+            self.conn.execute(
+                "INSERT INTO global_rules (rule_text, confidence, source_projects, category, language) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (rule_text, confidence, json.dumps([source_project]), category, language),
+            )
+        self.conn.commit()
+
+    def get_rules(self, min_confidence: float = 0.6, language: str | None = None) -> list[dict]:
+        """Get global rules above confidence threshold, optionally filtered by language."""
+        if language:
+            rows = self.conn.execute(
+                "SELECT rule_text, confidence, source_projects, category, language FROM global_rules "
+                "WHERE confidence >= ? AND (language = ? OR language IS NULL) ORDER BY confidence DESC",
+                (min_confidence, language),
+            ).fetchall()
+        else:
+            rows = self.conn.execute(
+                "SELECT rule_text, confidence, source_projects, category, language FROM global_rules "
+                "WHERE confidence >= ? ORDER BY confidence DESC",
+                (min_confidence,),
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    # ------------------------------------------------------------------
+    # Stats
+    # ------------------------------------------------------------------
+
+    def get_stats(self) -> dict:
+        """Return summary stats for the global database."""
+        return {
+            "project_count": self.get_project_count(),
+            "total_preferences": self.conn.execute("SELECT COUNT(*) FROM global_preferences").fetchone()[0],
+            "total_rules": self.conn.execute("SELECT COUNT(*) FROM global_rules").fetchone()[0],
+        }