codexa 0.4.0__py3-none-any.whl

semantic_code_intelligence/services/search_service.py

@@ -0,0 +1,269 @@
+"""Search service — handles semantic, keyword, regex, and hybrid search.
+
+Supports four modes:
+- **semantic** (default): FAISS cosine-similarity search
+- **keyword**: BM25-ranked keyword search
+- **regex**: grep-compatible regex pattern matching
+- **hybrid**: Reciprocal Rank Fusion of semantic + BM25
+
+Also supports ``--full-section`` expansion and auto-indexing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from semantic_code_intelligence.config.settings import AppConfig, load_config
+from semantic_code_intelligence.embeddings.generator import generate_embeddings
+from semantic_code_intelligence.storage.query_history import QueryHistory
+from semantic_code_intelligence.storage.vector_store import ChunkMetadata, VectorStore
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("services.search")
+
+SearchMode = Literal["semantic", "keyword", "regex", "hybrid"]
+
+
+@dataclass
+class SearchResult:
+    """A single search result with metadata and similarity score."""
+
+    file_path: str
+    start_line: int
+    end_line: int
+    language: str
+    content: str
+    score: float
+    chunk_index: int
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to a JSON-serializable dictionary."""
+        return {
+            "file_path": self.file_path,
+            "start_line": self.start_line,
+            "end_line": self.end_line,
+            "language": self.language,
+            "content": self.content,
+            "score": round(self.score, 4),
+            "chunk_index": self.chunk_index,
+        }
+
+
+def _auto_index_if_needed(project_root: Path, index_dir: Path) -> None:
+    """Run indexing transparently if no vector store exists yet."""
+    vectors_path = index_dir / "vectors.faiss"
+    if vectors_path.exists():
+        return
+    logger.info("No index found — auto-indexing %s", project_root)
+    from semantic_code_intelligence.services.indexing_service import run_indexing
+    run_indexing(project_root, force=False)
+
+
+def _load_store(index_dir: Path) -> VectorStore:
+    """Load the vector store (raises FileNotFoundError if missing)."""
+    store = VectorStore.load(index_dir)
+    logger.info("Loaded vector store with %d vectors.", store.size)
+    return store
+
+
+def _record_history(
+    index_dir: Path,
+    query: str,
+    results: list[SearchResult],
+) -> None:
+    """Record a query in the persistent history (best-effort)."""
+    try:
+        history = QueryHistory.load(index_dir)
+        languages = sorted(set(r.language for r in results if r.language))
+        top_files = list(dict.fromkeys(r.file_path for r in results))[:5]
+        history.record(
+            query=query,
+            result_count=len(results),
+            top_score=results[0].score if results else 0.0,
+            languages=languages,
+            top_files=top_files,
+        )
+        history.save(index_dir)
+    except Exception:
+        logger.debug("Failed to record query history.")
+
+
+# ------------------------------------------------------------------
+# Semantic search (original behaviour)
+# ------------------------------------------------------------------
+
+def _semantic_search(
+    query: str,
+    store: VectorStore,
+    config: Any,
+    top_k: int,
+    threshold: float,
+) -> list[SearchResult]:
+    query_embedding = generate_embeddings(
+        [query], model_name=config.embedding.model_name,
+    )[0]
+    raw_results = store.search(query_embedding, top_k=top_k)
+
+    results: list[SearchResult] = []
+    for meta, score in raw_results:
+        if score < threshold:
+            continue
+        results.append(
+            SearchResult(
+                file_path=meta.file_path,
+                start_line=meta.start_line,
+                end_line=meta.end_line,
+                language=meta.language,
+                content=meta.content,
+                score=score,
+                chunk_index=meta.chunk_index,
+            )
+        )
+    return results
+
+
+# ------------------------------------------------------------------
+# Keyword / regex / hybrid helpers
+# ------------------------------------------------------------------
+
+def _keyword_search(
+    query: str,
+    store: VectorStore,
+    index_dir: Path,
+    top_k: int,
+) -> list[SearchResult]:
+    from semantic_code_intelligence.search.keyword_search import keyword_search
+    hits = keyword_search(query, store, index_dir, top_k=top_k)
+    return [
+        SearchResult(
+            file_path=h.file_path,
+            start_line=h.start_line,
+            end_line=h.end_line,
+            language=h.language,
+            content=h.content,
+            score=h.score,
+            chunk_index=h.chunk_index,
+        )
+        for h in hits
+    ]
+
+
+def _regex_search(
+    pattern: str,
+    store: VectorStore,
+    top_k: int,
+    case_insensitive: bool = True,
+) -> list[SearchResult]:
+    from semantic_code_intelligence.search.keyword_search import regex_search
+    hits = regex_search(pattern, store, top_k=top_k, case_insensitive=case_insensitive)
+    return [
+        SearchResult(
+            file_path=h.file_path,
+            start_line=h.start_line,
+            end_line=h.end_line,
+            language=h.language,
+            content=h.content,
+            score=h.score,
+            chunk_index=h.chunk_index,
+        )
+        for h in hits
+    ]
+
+
+def _hybrid_search(
+    query: str,
+    store: VectorStore,
+    index_dir: Path,
+    config: Any,
+    top_k: int,
+) -> list[SearchResult]:
+    from semantic_code_intelligence.search.hybrid_search import hybrid_search
+    hits = hybrid_search(
+        query, store, index_dir,
+        model_name=config.embedding.model_name,
+        top_k=top_k,
+    )
+    return [
+        SearchResult(
+            file_path=h.file_path,
+            start_line=h.start_line,
+            end_line=h.end_line,
+            language=h.language,
+            content=h.content,
+            score=h.score,
+            chunk_index=h.chunk_index,
+        )
+        for h in hits
+    ]
+
+
+# ------------------------------------------------------------------
+# Public API
+# ------------------------------------------------------------------
+
+def search_codebase(
+    query: str,
+    project_root: Path,
+    top_k: int | None = None,
+    threshold: float | None = None,
+    mode: SearchMode = "semantic",
+    full_section: bool = False,
+    auto_index: bool = True,
+    case_insensitive: bool = True,
+) -> list[SearchResult]:
+    """Search the indexed codebase.
+
+    Args:
+        query: Natural language query, keywords, or regex pattern.
+        project_root: Root directory of the project.
+        top_k: Number of top results to return. Uses config default if None.
+        threshold: Minimum similarity score. Uses config default if None.
+        mode: One of ``"semantic"``, ``"keyword"``, ``"regex"``, ``"hybrid"``.
+        full_section: If True, expand results to full enclosing function/class.
+        auto_index: If True, index automatically when no index exists.
+        case_insensitive: For regex mode, whether to ignore case.
+
+    Returns:
+        List of SearchResult objects sorted by descending score.
+
+    Raises:
+        FileNotFoundError: If no vector index exists and auto_index is False.
+    """
+    project_root = project_root.resolve()
+    config = load_config(project_root)
+    index_dir = AppConfig.index_dir(project_root)
+
+    top_k = top_k or config.search.top_k
+    threshold = threshold if threshold is not None else config.search.similarity_threshold
+
+    # Auto-index if needed
+    if auto_index:
+        _auto_index_if_needed(project_root, index_dir)
+
+    store = _load_store(index_dir)
+    if store.size == 0:
+        return []
+
+    # Dispatch to the appropriate search backend
+    if mode == "keyword":
+        results = _keyword_search(query, store, index_dir, top_k)
+    elif mode == "regex":
+        results = _regex_search(query, store, top_k, case_insensitive)
+    elif mode == "hybrid":
+        results = _hybrid_search(query, store, index_dir, config, top_k)
+    else:
+        results = _semantic_search(query, store, config, top_k, threshold)
+
+    logger.info("Found %d results (mode=%s).", len(results), mode)
+
+    # Full-section expansion
+    if full_section and results:
+        from semantic_code_intelligence.search.section_expander import expand_to_full_section
+        results = expand_to_full_section(results, project_root, index_dir)
+
+    # Record query history
+    _record_history(index_dir, query, results)
+
+    return results

semantic_code_intelligence/storage/chunk_hash_store.py

@@ -0,0 +1,86 @@
+"""Chunk hash store — tracks content hashes at the individual chunk level.
+
+Enables chunk-level incremental indexing: when a file changes, only the
+chunks whose content actually differs are re-embedded, while unchanged
+chunks keep their existing vectors.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("storage.chunk_hashes")
+
+CHUNK_HASH_FILE = "chunk_hashes.json"
+
+
+def compute_chunk_hash(content: str) -> str:
+    """Compute a fast SHA-256 hash for a chunk's content."""
+    return hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest()
+
+
+class ChunkHashStore:
+    """Persists a mapping of chunk keys to content hashes.
+
+    Chunk key format: ``"file_path:start_line:end_line"``
+
+    Used for chunk-level incremental indexing: only re-embed chunks
+    whose content has actually changed, even if the file was modified.
+    """
+
+    def __init__(self) -> None:
+        self._hashes: dict[str, str] = {}
+
+    @staticmethod
+    def chunk_key(file_path: str, start_line: int, end_line: int) -> str:
+        return f"{file_path}:{start_line}:{end_line}"
+
+    @property
+    def count(self) -> int:
+        return len(self._hashes)
+
+    def get(self, key: str) -> str | None:
+        return self._hashes.get(key)
+
+    def set(self, key: str, content_hash: str) -> None:
+        self._hashes[key] = content_hash
+
+    def has_changed(self, key: str, content_hash: str) -> bool:
+        stored = self._hashes.get(key)
+        return stored != content_hash
+
+    def remove(self, key: str) -> None:
+        self._hashes.pop(key, None)
+
+    def remove_by_file(self, file_path: str) -> int:
+        """Remove all chunk entries whose key starts with file_path."""
+        prefix = file_path + ":"
+        keys_to_remove = [k for k in self._hashes if k.startswith(prefix)]
+        for k in keys_to_remove:
+            del self._hashes[k]
+        return len(keys_to_remove)
+
+    def keys_for_file(self, file_path: str) -> list[str]:
+        prefix = file_path + ":"
+        return [k for k in self._hashes if k.startswith(prefix)]
+
+    def save(self, directory: Path) -> None:
+        directory = Path(directory)
+        directory.mkdir(parents=True, exist_ok=True)
+        path = directory / CHUNK_HASH_FILE
+        path.write_text(
+            json.dumps(self._hashes, ensure_ascii=False),
+            encoding="utf-8",
+        )
+
+    @classmethod
+    def load(cls, directory: Path) -> "ChunkHashStore":
+        store = cls()
+        path = Path(directory) / CHUNK_HASH_FILE
+        if path.exists():
+            store._hashes = json.loads(path.read_text(encoding="utf-8"))
+        return store

semantic_code_intelligence/storage/hash_store.py

@@ -0,0 +1,66 @@
+"""Hash store — tracks file content hashes for incremental indexing."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("storage.hashes")
+
+HASH_FILE_NAME = "file_hashes.json"
+
+
+class HashStore:
+    """Persists a mapping of file paths to content hashes.
+
+    Used for incremental indexing: only re-index files whose hash has changed.
+    """
+
+    def __init__(self) -> None:
+        self._hashes: dict[str, str] = {}
+
+    @property
+    def count(self) -> int:
+        """Number of tracked files."""
+        return len(self._hashes)
+
+    def get(self, file_path: str) -> str | None:
+        """Get the stored hash for a file path."""
+        return self._hashes.get(file_path)
+
+    def set(self, file_path: str, content_hash: str) -> None:
+        """Store or update the hash for a file path."""
+        self._hashes[file_path] = content_hash
+
+    def has_changed(self, file_path: str, content_hash: str) -> bool:
+        """Check if a file's content has changed since last indexed.
+
+        Returns True if the file is new or its hash differs.
+        """
+        stored = self._hashes.get(file_path)
+        return stored != content_hash
+
+    def remove(self, file_path: str) -> None:
+        """Remove a file from the hash store."""
+        self._hashes.pop(file_path, None)
+
+    def save(self, directory: Path) -> None:
+        """Save hashes to disk."""
+        directory = Path(directory)
+        directory.mkdir(parents=True, exist_ok=True)
+        path = directory / HASH_FILE_NAME
+        path.write_text(
+            json.dumps(self._hashes, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+
+    @classmethod
+    def load(cls, directory: Path) -> "HashStore":
+        """Load hashes from disk. Returns empty store if file doesn't exist."""
+        store = cls()
+        path = Path(directory) / HASH_FILE_NAME
+        if path.exists():
+            store._hashes = json.loads(path.read_text(encoding="utf-8"))
+        return store

semantic_code_intelligence/storage/index_manifest.py

@@ -0,0 +1,85 @@
+"""Index manifest — versioned metadata for the persistent intelligence index.
+
+Tracks index schema version, embedding model, creation/update timestamps,
+and file counts to enable integrity checks and safe index upgrades.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+MANIFEST_FILE = "index_manifest.json"
+SCHEMA_VERSION = 1
+
+
+@dataclass
+class IndexManifest:
+    """Metadata describing a persisted intelligence index."""
+
+    schema_version: int = SCHEMA_VERSION
+    embedding_model: str = "all-MiniLM-L6-v2"
+    embedding_dimension: int = 384
+    created_at: float = 0.0
+    updated_at: float = 0.0
+    total_files: int = 0
+    total_chunks: int = 0
+    total_symbols: int = 0
+    languages: list[str] = field(default_factory=list)
+    project_root: str = ""
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> IndexManifest:
+        known = {f.name for f in cls.__dataclass_fields__.values()}  # type: ignore[attr-defined]
+        filtered = {k: v for k, v in data.items() if k in known}
+        return cls(**filtered)
+
+    # ------------------------------------------------------------------
+    # Persistence
+    # ------------------------------------------------------------------
+
+    def save(self, directory: str | Path) -> None:
+        """Write the manifest to *directory*/index_manifest.json."""
+        path = Path(directory)
+        path.mkdir(parents=True, exist_ok=True)
+        (path / MANIFEST_FILE).write_text(
+            json.dumps(self.to_dict(), indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+
+    @classmethod
+    def load(cls, directory: str | Path) -> IndexManifest | None:
+        """Load an existing manifest, or return ``None`` if absent."""
+        path = Path(directory) / MANIFEST_FILE
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return cls.from_dict(data)
+        except (json.JSONDecodeError, OSError):
+            return None
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def touch(self) -> None:
+        """Update ``updated_at`` to now; set ``created_at`` if zero."""
+        now = time.time()
+        if self.created_at == 0.0:
+            self.created_at = now
+        self.updated_at = now
+
+    def is_compatible(self, model: str, dimension: int) -> bool:
+        """Check whether the index was built with the given model/dimension."""
+        return self.embedding_model == model and self.embedding_dimension == dimension

semantic_code_intelligence/storage/index_stats.py

@@ -0,0 +1,138 @@
+"""Index statistics — health metrics, coverage, and staleness tracking.
+
+Provides detailed statistics about the intelligence index including
+per-language coverage, chunk distribution, and staleness metrics
+for monitoring index quality.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+STATS_FILE = "index_stats.json"
+
+
+@dataclass
+class LanguageCoverage:
+    """Per-language indexing statistics."""
+
+    language: str = ""
+    files: int = 0
+    chunks: int = 0
+    symbols: int = 0
+    total_lines: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> LanguageCoverage:
+        known = {f.name for f in cls.__dataclass_fields__.values()}  # type: ignore[attr-defined]
+        return cls(**{k: v for k, v in data.items() if k in known})
+
+
+@dataclass
+class IndexStats:
+    """Comprehensive index health and coverage statistics."""
+
+    # Counts
+    total_files: int = 0
+    total_chunks: int = 0
+    total_symbols: int = 0
+    total_vectors: int = 0
+
+    # Timing
+    last_indexed_at: float = 0.0
+    indexing_duration_seconds: float = 0.0
+
+    # Per-language breakdown
+    language_coverage: list[LanguageCoverage] = field(default_factory=list)
+
+    # Staleness
+    stale_files: int = 0  # files changed since last index
+
+    # Quality
+    avg_chunk_size: float = 0.0
+    embedding_model: str = ""
+    embedding_dimension: int = 0
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        d["language_coverage"] = [lc.to_dict() for lc in self.language_coverage]
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> IndexStats:
+        lang_data = data.pop("language_coverage", [])
+        known = {f.name for f in cls.__dataclass_fields__.values()}  # type: ignore[attr-defined]
+        filtered = {k: v for k, v in data.items() if k in known and k != "language_coverage"}
+        stats = cls(**filtered)
+        stats.language_coverage = [
+            LanguageCoverage.from_dict(lc) if isinstance(lc, dict) else lc
+            for lc in lang_data
+        ]
+        return stats
+
+    # ------------------------------------------------------------------
+    # Persistence
+    # ------------------------------------------------------------------
+
+    def save(self, directory: str | Path) -> None:
+        """Write stats to disk."""
+        path = Path(directory)
+        path.mkdir(parents=True, exist_ok=True)
+        (path / STATS_FILE).write_text(
+            json.dumps(self.to_dict(), indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+
+    @classmethod
+    def load(cls, directory: str | Path) -> IndexStats | None:
+        """Load stats from disk, or return ``None`` if absent."""
+        path = Path(directory) / STATS_FILE
+        if not path.exists():
+            return None
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return cls.from_dict(data)
+        except (json.JSONDecodeError, OSError):
+            return None
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @property
+    def staleness_seconds(self) -> float:
+        """Seconds since the last indexing run."""
+        if self.last_indexed_at == 0.0:
+            return 0.0
+        return time.time() - self.last_indexed_at
+
+    @property
+    def languages(self) -> list[str]:
+        """Return all indexed languages."""
+        return [lc.language for lc in self.language_coverage]
+
+    def get_language(self, language: str) -> LanguageCoverage | None:
+        """Return coverage for a specific language."""
+        for lc in self.language_coverage:
+            if lc.language == language:
+                return lc
+        return None
+
+    def set_language(self, coverage: LanguageCoverage) -> None:
+        """Add or replace per-language coverage entry."""
+        for i, lc in enumerate(self.language_coverage):
+            if lc.language == coverage.language:
+                self.language_coverage[i] = coverage
+                return
+        self.language_coverage.append(coverage)