codegraph-engine 3.0.0__2-py3-none-any.whl

codegraph/indexing/pool.py

@@ -0,0 +1,136 @@
+"""Thread-safe SQLite connection pool for concurrent MCP and CLI execution."""
+from __future__ import annotations
+
+import queue
+import sqlite3
+import threading
+from collections.abc import Callable
+from pathlib import Path
+
+from .classifier import make_compat_row
+
+
+class SQLiteConnectionPool:
+    """Thread-safe, re-entrant connection pool for SQLite databases.
+
+    Features:
+    - Thread-safe LIFO pool to maximize SQLite cache reuse.
+    - WAL mode, synchronous=NORMAL, foreign_keys=ON, busy_timeout=30000ms.
+    - check_same_thread=False for cross-thread pool handoff.
+    - Re-entrant per-thread acquisition (nesting depth tracking via thread-local state).
+    - Optional initializer hook for idempotent schema / setup execution.
+    """
+
+    def __init__(
+        self,
+        db_path: Path | str,
+        max_size: int = 16,
+        timeout: float = 30.0,
+        initializer: Callable[[sqlite3.Connection], None] | None = None,
+    ) -> None:
+        self.db_path = Path(db_path) if isinstance(db_path, str) else db_path
+        self.max_size = max(1, max_size)
+        self.timeout = timeout
+        self.initializer = initializer
+        self._pool: queue.LifoQueue[sqlite3.Connection] = queue.LifoQueue()
+        self._created = 0
+        self._lock = threading.Lock()
+        self._local = threading.local()
+
+    def _create_connection(self) -> sqlite3.Connection:
+        con = sqlite3.connect(
+            self.db_path,
+            check_same_thread=False,
+            timeout=self.timeout,
+        )
+        con.row_factory = make_compat_row
+        con.execute("PRAGMA journal_mode = WAL")
+        con.execute("PRAGMA synchronous = NORMAL")
+        con.execute("PRAGMA foreign_keys = ON")
+        con.execute("PRAGMA busy_timeout = 30000")
+        con.execute("PRAGMA cache_size = -64000")
+        con.execute("PRAGMA temp_store = MEMORY")
+        if self.initializer is not None:
+            self.initializer(con)
+        return con
+
+    def acquire(self) -> tuple[sqlite3.Connection, bool]:
+        """Acquire a connection. Returns (connection, is_owner).
+
+        If the current thread already holds a connection from this pool,
+        it increments the re-entrancy depth and returns (connection, False).
+        """
+        active_con = getattr(self._local, "con", None)
+        if active_con is not None:
+            self._local.nesting = getattr(self._local, "nesting", 0) + 1
+            return active_con, False
+
+        con: sqlite3.Connection | None = None
+        try:
+            con = self._pool.get_nowait()
+        except queue.Empty:
+            with self._lock:
+                if self._created < self.max_size:
+                    con = self._create_connection()
+                    self._created += 1
+
+            if con is None:
+                con = self._pool.get(timeout=self.timeout)
+
+        self._local.con = con
+        self._local.nesting = 1
+        return con, True
+
+    def release(
+        self,
+        con: sqlite3.Connection,
+        is_owner: bool,
+        rollback: bool = False,
+    ) -> None:
+        """Release a connection back to the pool."""
+        if not is_owner:
+            cur_nesting = getattr(self._local, "nesting", 1) - 1
+            self._local.nesting = max(0, cur_nesting)
+            if rollback:
+                try:
+                    con.rollback()
+                except Exception:
+                    pass
+            return
+
+        self._local.con = None
+        self._local.nesting = 0
+        if rollback:
+            try:
+                con.rollback()
+            except Exception:
+                pass
+        else:
+            try:
+                con.commit()
+            except Exception:
+                pass
+
+        try:
+            self._pool.put_nowait(con)
+        except Exception:
+            try:
+                con.close()
+            except Exception:
+                pass
+
+    def close_all(self) -> None:
+        """Close and dispose all connections in the pool."""
+        with self._lock:
+            while True:
+                try:
+                    con = self._pool.get_nowait()
+                    try:
+                        con.close()
+                    except Exception:
+                        pass
+                except queue.Empty:
+                    break
+            self._created = 0
+            self._local.con = None
+            self._local.nesting = 0

codegraph/indexing/scanner.py

@@ -0,0 +1,211 @@
+"""Secure, deterministic repository scanner.
+
+Architectural Invariants:
+1. Low-level primitive `safe_path()` is strict and fail-closed (raises SecurityError on escape).
+2. Repository scanner catches SecurityError on individual unsafe symlink/traversal entries,
+   records the security skip, and continues scanning without aborting the entire repository.
+3. Unsafe/escaped paths are NEVER indexed or exposed.
+4. Scan statistics are tracked explicitly.
+"""
+from __future__ import annotations
+
+import fnmatch
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from codegraph.errors import SecurityError
+from codegraph.indexing.classifier import (
+    FileCategory,
+    classify_file,
+    is_ignored_by_codegraphignore,
+    load_codegraphignore,
+)
+from codegraph.security import is_sensitive, safe_path
+
+IGNORED_DIRS = {
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "__pycache__",
+    "dist",
+    "build",
+    ".mypy_cache",
+    ".pytest_cache",
+}
+
+LANGUAGES = {
+    ".py": "python",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".sql": "sql",
+    ".prisma": "prisma",
+    ".html": "html",
+    ".htm": "html",
+    ".jinja": "html",
+    ".jinja2": "html",
+    ".njk": "html",
+    ".ejs": "html",
+    ".lua": "luau",
+    ".luau": "luau",
+}
+
+
+@dataclass(frozen=True)
+class ScanResult:
+    path: Path
+    relative_path: Path
+    language: str
+    category: FileCategory = FileCategory.SOURCE
+
+
+@dataclass
+class ScanStats:
+    scanned: int = 0
+    accepted: int = 0
+    skipped: int = 0
+    skipped_security: int = 0
+    skipped_binary: int = 0
+    skipped_secret: int = 0
+    skipped_large: int = 0
+    skipped_ignored: int = 0
+    skipped_details: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "scanned": self.scanned,
+            "accepted": self.accepted,
+            "skipped": self.skipped,
+            "skipped_security": self.skipped_security,
+            "skipped_binary": self.skipped_binary,
+            "skipped_secret": self.skipped_secret,
+            "skipped_large": self.skipped_large,
+            "skipped_ignored": self.skipped_ignored,
+        }
+
+
+def is_binary(path: Path) -> bool:
+    try:
+        with path.open("rb") as fh:
+            return b"\x00" in fh.read(8192)
+    except OSError:
+        return True
+
+
+def scan_with_stats(
+    repository: Path,
+    max_file_size: int,
+    exclusions: tuple[str, ...] = (),
+) -> tuple[list[ScanResult], ScanStats]:
+    """Scan repository for supported source files, catching unsafe symlinks and collecting stats."""
+    root = repository.resolve(strict=True)
+    gitignore = _gitignore_patterns(root)
+    codegraphignore = load_codegraphignore(root)
+    results: list[ScanResult] = []
+    stats = ScanStats()
+
+    for directory, dirs, files in os.walk(root, followlinks=False):
+        dirs[:] = [d for d in dirs if d not in IGNORED_DIRS]
+        for filename in files:
+            stats.scanned += 1
+            absolute = Path(directory) / filename
+            try:
+                rel_unresolved = absolute.relative_to(root)
+            except ValueError:
+                stats.skipped += 1
+                stats.skipped_security += 1
+                continue
+
+            try:
+                # safe_path raises SecurityError if symlink resolves outside root or uses traversal escapes
+                resolved = safe_path(root, rel_unresolved)
+            except SecurityError:
+                stats.skipped += 1
+                stats.skipped_security += 1
+                stats.skipped_details.append(f"Security blocked: {rel_unresolved}")
+                continue
+
+            relative = resolved.relative_to(root)
+
+            try:
+                if not resolved.is_file():
+                    stats.skipped += 1
+                    continue
+                if resolved.stat().st_size > max_file_size:
+                    stats.skipped += 1
+                    stats.skipped_large += 1
+                    continue
+                if is_sensitive(relative):
+                    stats.skipped += 1
+                    stats.skipped_secret += 1
+                    continue
+                if (
+                    _ignored_by_gitignore(relative, gitignore)
+                    or is_ignored_by_codegraphignore(relative, codegraphignore)
+                    or any(relative.match(p) for p in exclusions)
+                ):
+                    stats.skipped += 1
+                    stats.skipped_ignored += 1
+                    continue
+                if is_binary(resolved):
+                    stats.skipped += 1
+                    stats.skipped_binary += 1
+                    continue
+
+                category = classify_file(relative)
+                if category in (FileCategory.BUILD_ARTIFACT, FileCategory.VENDOR, FileCategory.BINARY):
+                    stats.skipped += 1
+                    if category == FileCategory.BINARY:
+                        stats.skipped_binary += 1
+                    else:
+                        stats.skipped_ignored += 1
+                    continue
+            except OSError:
+                stats.skipped += 1
+                continue
+
+            language = LANGUAGES.get(relative.suffix.lower())
+            if language:
+                stats.accepted += 1
+                results.append(ScanResult(resolved, relative, language, category=category))
+            else:
+                stats.skipped += 1
+
+    results.sort(key=lambda item: item.relative_path.as_posix())
+    return results, stats
+
+
+def scan(
+    repository: Path,
+    max_file_size: int,
+    exclusions: tuple[str, ...] = (),
+) -> list[ScanResult]:
+    """Securely scan repository for supported source files."""
+    results, _ = scan_with_stats(repository, max_file_size, exclusions)
+    return results
+
+
+def _gitignore_patterns(root: Path) -> list[str]:
+    path = root / ".gitignore"
+    try:
+        return [
+            line.strip()
+            for line in path.read_text().splitlines()
+            if line.strip() and not line.lstrip().startswith("#") and not line.startswith("!")
+        ]
+    except OSError:
+        return []
+
+
+def _ignored_by_gitignore(relative: Path, patterns: list[str]) -> bool:
+    """Small, deterministic subset: ordinary glob and directory patterns; negation is unsupported."""
+    value = relative.as_posix()
+    return any(
+        fnmatch.fnmatch(value, pattern.rstrip("/"))
+        or (pattern.endswith("/") and value.startswith(pattern))
+        or fnmatch.fnmatch(relative.name, pattern)
+        for pattern in patterns
+    )

codegraph/indexing/telemetry.py

@@ -0,0 +1,326 @@
+"""Deterministic 16-phase indexing telemetry, memory tracking, and progress formatting (v2.1.7)."""
+from __future__ import annotations
+
+import sys
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+INDEXING_PHASES: tuple[tuple[str, str], ...] = (
+    ("repository_scan", "Repository Scan"),
+    ("file_classification", "File Classification"),
+    ("file_reading", "File Reading"),
+    ("ast_parsing", "AST Parsing"),
+    ("symbol_extraction", "Symbol Extraction"),
+    ("import_extraction", "Import Extraction"),
+    ("framework_analysis", "Framework Analysis"),
+    ("database_analysis", "Database Analysis"),
+    ("graph_edge_creation", "Graph Edge Creation"),
+    ("symbol_resolution", "Symbol Resolution"),
+    ("relationship_resolution", "Relationship Resolution"),
+    ("ranking_index_building", "Ranking/Index Building"),
+    ("sqlite_writes", "SQLite Writes"),
+    ("fts_updates", "FTS Updates"),
+    ("post_processing", "Post-Processing"),
+    ("final_commit_checkpoint", "Final Commit/Checkpoint"),
+)
+
+PHASE_LABELS: dict[str, str] = dict(INDEXING_PHASES)
+
+
+def get_process_rss_mb() -> float:
+    """Return current/peak RSS in megabytes safely across macOS, Linux, and Windows."""
+    try:
+        import resource as res_mod
+
+        if res_mod is None:
+            return 0.0
+        usage = res_mod.getrusage(res_mod.RUSAGE_SELF).ru_maxrss
+        if sys.platform == "darwin":
+            return round(usage / (1024.0 * 1024.0), 2)
+        return round(usage / 1024.0, 2)
+    except Exception:
+        return 0.0
+
+
+def format_duration_concise(seconds: float) -> str:
+    """Format elapsed seconds as e.g. '4m 12s' or '1.42s'."""
+    if seconds >= 60.0:
+        mins = int(seconds // 60)
+        secs = int(round(seconds % 60))
+        return f"{mins}m {secs:02d}s"
+    return f"{seconds:.2f}s"
+
+
+def format_progress_block(
+    phase: str,
+    completed_files: int,
+    total_files: int,
+    elapsed_seconds: float,
+    *,
+    memory_mb: float | None = None,
+) -> str:
+    """Render a concise multi-line progress block suitable for CLI display."""
+    label = PHASE_LABELS.get(phase, phase)
+    total = max(1, total_files)
+    pct = min(100, int(round((completed_files / total) * 100)))
+    rate_per_min = int(round((completed_files / max(0.001, elapsed_seconds)) * 60.0))
+    mem_val = int(round(memory_mb if memory_mb is not None else get_process_rss_mb()))
+    return (
+        f"Phase: {label}\n"
+        f"{completed_files:,} / {total_files:,} files\n"
+        f"{pct}%\n"
+        f"elapsed: {format_duration_concise(elapsed_seconds)}\n"
+        f"rate: {rate_per_min:,} files/min\n"
+        f"memory: {mem_val} MB"
+    )
+
+
+@dataclass
+class PhaseMetrics:
+    """Telemetry counters and timing for a single indexing pipeline phase."""
+
+    phase: str
+    label: str
+    elapsed_seconds: float = 0.0
+    files_processed: int = 0
+    items_processed: int = 0
+    memory_mb: float = 0.0
+    errors: int = 0
+    skips: int = 0
+    db_writes: int = 0
+
+    @property
+    def errors_skips(self) -> int:
+        return self.errors + self.skips
+
+    @errors_skips.setter
+    def errors_skips(self, value: int) -> None:
+        self.skips = max(0, value - self.errors)
+
+    @property
+    def throughput_per_sec(self) -> float:
+        if self.elapsed_seconds <= 0.0:
+            return 0.0
+        basis = self.items_processed if self.items_processed > 0 else self.files_processed
+        return round(basis / self.elapsed_seconds, 2)
+
+    @property
+    def files_per_min(self) -> float:
+        if self.elapsed_seconds <= 0.0:
+            return 0.0
+        return round((self.files_processed / self.elapsed_seconds) * 60.0, 1)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "phase": self.phase,
+            "label": self.label,
+            "elapsed_seconds": round(self.elapsed_seconds, 4),
+            "files_processed": self.files_processed,
+            "items_processed": self.items_processed,
+            "throughput_per_sec": self.throughput_per_sec,
+            "memory_mb": round(self.memory_mb, 2),
+            "errors": self.errors,
+            "skips": self.skips,
+            "errors_skips": self.errors + self.skips,
+            "db_writes": self.db_writes,
+        }
+
+
+@dataclass
+class IndexingTelemetry:
+    """Complete 16-phase deterministic telemetry for an indexing pass."""
+
+    phases: dict[str, PhaseMetrics] = field(default_factory=dict)
+    started_at: float = field(default_factory=time.perf_counter)
+    total_elapsed_seconds: float = 0.0
+    scanned_files: int = 0
+    indexed_files: int = 0
+    unchanged_files: int = 0
+    removed_files: int = 0
+    parse_failed_files: int = 0
+    symbols_count: int = 0
+    edges_count: int = 0
+    rss_before_mb: float = 0.0
+    post_parse_rss_mb: float = 0.0
+    peak_rss_mb: float = 0.0
+    steady_state_rss_mb: float = 0.0
+    peak_wal_bytes: int = 0
+    final_wal_bytes: int = 0
+    final_db_bytes: int = 0
+    commit_batches: int = 0
+    checkpoints: int = 0
+    interrupted: bool = False
+    interrupted_phase: str | None = None
+    warnings: list[str] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        rss = get_process_rss_mb()
+        if self.rss_before_mb == 0.0:
+            self.rss_before_mb = rss
+        if self.peak_rss_mb == 0.0:
+            self.peak_rss_mb = rss
+        for key, label in INDEXING_PHASES:
+            if key not in self.phases:
+                self.phases[key] = PhaseMetrics(phase=key, label=label)
+
+    @property
+    def batch_commits(self) -> int:
+        return self.commit_batches
+
+    @batch_commits.setter
+    def batch_commits(self, value: int) -> None:
+        self.commit_batches = value
+
+    @property
+    def wal_checkpoints(self) -> int:
+        return self.checkpoints
+
+    @wal_checkpoints.setter
+    def wal_checkpoints(self, value: int) -> None:
+        self.checkpoints = value
+
+    @property
+    def max_wal_size_mb(self) -> float:
+        return round(self.peak_wal_bytes / (1024.0 * 1024.0), 3)
+
+    @property
+    def final_wal_size_mb(self) -> float:
+        return round(self.final_wal_bytes / (1024.0 * 1024.0), 3)
+
+    @contextmanager
+    def phase(self, name: str) -> Iterator[PhaseMetrics]:
+        """Time a phase and update its memory snapshot upon exit."""
+        if name not in self.phases:
+            self.phases[name] = PhaseMetrics(phase=name, label=PHASE_LABELS.get(name, name))
+        metrics = self.phases[name]
+        t0 = time.perf_counter()
+        try:
+            yield metrics
+        except BaseException:
+            if self.interrupted_phase is None:
+                self.interrupted_phase = name
+            raise
+        finally:
+            metrics.elapsed_seconds += time.perf_counter() - t0
+            rss = get_process_rss_mb()
+            metrics.memory_mb = max(metrics.memory_mb, rss)
+            if rss > self.peak_rss_mb:
+                self.peak_rss_mb = rss
+
+    def record(
+        self,
+        phase: str,
+        elapsed: float = 0.0,
+        files: int = 0,
+        items: int = 0,
+        errors: int = 0,
+        skips: int = 0,
+        db_writes: int = 0,
+        update_memory: bool = False,
+        *,
+        files_processed: int = 0,
+        items_processed: int = 0,
+        errors_skips: int = 0,
+    ) -> None:
+        """Accumulate metrics for a given phase."""
+        if phase not in self.phases:
+            self.phases[phase] = PhaseMetrics(phase=phase, label=PHASE_LABELS.get(phase, phase))
+        m = self.phases[phase]
+        m.elapsed_seconds += elapsed
+        m.files_processed += files + files_processed
+        m.items_processed += items + items_processed
+        m.errors += errors
+        m.skips += skips + errors_skips
+        m.db_writes += db_writes
+        if update_memory or m.memory_mb == 0.0:
+            rss = get_process_rss_mb()
+            m.memory_mb = max(m.memory_mb, rss)
+            if rss > self.peak_rss_mb:
+                self.peak_rss_mb = rss
+
+    def observe_wal(self, db_path: Path) -> None:
+        """Sample the SQLite WAL file size and update peak_wal_bytes."""
+        wal_path = Path(f"{db_path}-wal")
+        try:
+            if wal_path.exists():
+                sz = wal_path.stat().st_size
+                if sz > self.peak_wal_bytes:
+                    self.peak_wal_bytes = sz
+                self.final_wal_bytes = sz
+            else:
+                self.final_wal_bytes = 0
+        except OSError:
+            pass
+
+    def finish(self, db_path: Path | None = None) -> None:
+        """Finalize total elapsed time, steady-state RSS, and final DB/WAL sizes."""
+        self.total_elapsed_seconds = time.perf_counter() - self.started_at
+        rss = get_process_rss_mb()
+        self.steady_state_rss_mb = rss
+        if rss > self.peak_rss_mb:
+            self.peak_rss_mb = rss
+        if db_path is not None:
+            self.observe_wal(db_path)
+            try:
+                if db_path.exists():
+                    self.final_db_bytes = db_path.stat().st_size
+            except OSError:
+                pass
+
+    @property
+    def slowest_phase(self) -> str:
+        candidates = [
+            k for k in self.phases if k != "post_processing" and self.phases[k].elapsed_seconds > 0
+        ]
+        if not candidates:
+            return "repository_scan"
+        return max(candidates, key=lambda k: self.phases[k].elapsed_seconds)
+
+    def format_progress_block(
+        self,
+        phase: str,
+        completed_files: int,
+        total_files: int,
+        elapsed_seconds: float,
+    ) -> str:
+        """Render a concise multi-line progress block suitable for CLI display."""
+        return format_progress_block(
+            phase,
+            completed_files,
+            total_files,
+            elapsed_seconds,
+            memory_mb=max(self.peak_rss_mb, get_process_rss_mb()),
+        )
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "total_elapsed_seconds": round(self.total_elapsed_seconds, 4),
+            "scanned_files": self.scanned_files,
+            "indexed_files": self.indexed_files,
+            "unchanged_files": self.unchanged_files,
+            "removed_files": self.removed_files,
+            "parse_failed_files": self.parse_failed_files,
+            "symbols_count": self.symbols_count,
+            "edges_count": self.edges_count,
+            "files_per_sec": round(self.scanned_files / max(0.0001, self.total_elapsed_seconds), 2),
+            "symbols_per_sec": round(self.symbols_count / max(0.0001, self.total_elapsed_seconds), 2),
+            "edges_per_sec": round(self.edges_count / max(0.0001, self.total_elapsed_seconds), 2),
+            "rss_before_mb": round(self.rss_before_mb, 2),
+            "post_parse_rss_mb": round(self.post_parse_rss_mb, 2),
+            "peak_rss_mb": round(self.peak_rss_mb, 2),
+            "steady_state_rss_mb": round(self.steady_state_rss_mb, 2),
+            "peak_wal_mb": round(self.peak_wal_bytes / (1024.0 * 1024.0), 3),
+            "final_wal_mb": round(self.final_wal_bytes / (1024.0 * 1024.0), 3),
+            "final_db_mb": round(self.final_db_bytes / (1024.0 * 1024.0), 3),
+            "commit_batches": self.commit_batches,
+            "checkpoints": self.checkpoints,
+            "slowest_phase": self.slowest_phase,
+            "interrupted": self.interrupted,
+            "interrupted_phase": self.interrupted_phase,
+            "warnings": list(self.warnings),
+            "phases": {k: self.phases[k].as_dict() for k, _ in INDEXING_PHASES},
+        }