embgrep 0.1.0__py3-none-any.whl

embgrep/__init__.py

@@ -0,0 +1,79 @@
+"""embgrep — Local semantic search, embedding-powered grep for files."""
+
+from __future__ import annotations
+
+from embgrep.indexer import EmbGrep, IndexStatus, SearchResult
+
+__all__ = ["EmbGrep", "IndexStatus", "SearchResult", "index", "search", "status", "update"]
+__version__ = "0.1.0"
+
+
+def index(directory: str, patterns: list[str] | None = None, db_path: str | None = None) -> dict:
+    """Index files in a directory.
+
+    Args:
+        directory: Path to the directory to index.
+        patterns: Optional list of glob patterns to filter files.
+        db_path: Optional path to the SQLite database.
+
+    Returns:
+        Dictionary with files_indexed, chunks_created, index_size_mb.
+    """
+    eg = EmbGrep(db_path=db_path) if db_path else EmbGrep()
+    try:
+        return eg.index(directory, patterns=patterns)
+    finally:
+        eg.close()
+
+
+def search(
+    query: str, top_k: int = 5, path_filter: str | None = None, db_path: str | None = None
+) -> list[SearchResult]:
+    """Semantic search across indexed files.
+
+    Args:
+        query: Natural language search query.
+        top_k: Number of results to return.
+        path_filter: Optional LIKE pattern to filter by file path.
+        db_path: Optional path to the SQLite database.
+
+    Returns:
+        List of SearchResult sorted by similarity score.
+    """
+    eg = EmbGrep(db_path=db_path) if db_path else EmbGrep()
+    try:
+        return eg.search(query, top_k=top_k, path_filter=path_filter)
+    finally:
+        eg.close()
+
+
+def status(db_path: str | None = None) -> IndexStatus:
+    """Get index statistics.
+
+    Args:
+        db_path: Optional path to the SQLite database.
+
+    Returns:
+        IndexStatus with total_files, total_chunks, last_updated, index_size_mb.
+    """
+    eg = EmbGrep(db_path=db_path) if db_path else EmbGrep()
+    try:
+        return eg.status()
+    finally:
+        eg.close()
+
+
+def update(db_path: str | None = None) -> dict:
+    """Incremental update — re-index changed files only.
+
+    Args:
+        db_path: Optional path to the SQLite database.
+
+    Returns:
+        Dictionary with updated_files, new_chunks, removed_files.
+    """
+    eg = EmbGrep(db_path=db_path) if db_path else EmbGrep()
+    try:
+        return eg.update()
+    finally:
+        eg.close()

embgrep/__main__.py

@@ -0,0 +1,141 @@
+"""CLI entry point for embgrep — embedding-powered grep for files."""
+
+from __future__ import annotations
+
+import sys
+
+
+def main() -> None:
+    """Main CLI entry point."""
+    try:
+        import click
+        from rich.console import Console
+        from rich.table import Table
+    except ImportError:
+        print("CLI requires extra dependencies: pip install embgrep[cli]")
+        sys.exit(1)
+
+    console = Console()
+
+    @click.group()
+    @click.version_option(package_name="embgrep")
+    def cli() -> None:
+        """embgrep — Local semantic search, embedding-powered grep for files."""
+
+    @cli.command()
+    @click.argument("path", type=click.Path(exists=True))
+    @click.option("--patterns", "-p", default=None, help="Comma-separated glob patterns (e.g., '*.md,*.py').")
+    @click.option("--db-path", default=None, help="Path to SQLite database.")
+    @click.option("--model", default="BAAI/bge-small-en-v1.5", help="Embedding model name.")
+    def index(path: str, patterns: str | None, db_path: str | None, model: str) -> None:
+        """Index files in PATH for semantic search."""
+        from embgrep.indexer import EmbGrep
+
+        pattern_list = [p.strip() for p in patterns.split(",")] if patterns else None
+
+        kwargs: dict = {"model": model}
+        if db_path:
+            kwargs["db_path"] = db_path
+
+        eg = EmbGrep(**kwargs)
+        try:
+            with console.status("[bold green]Indexing files..."):
+                result = eg.index(path, patterns=pattern_list)
+            console.print(f"[green]Indexed {result['files_indexed']} files, {result['chunks_created']} chunks[/green]")
+            console.print(f"Index size: {result['index_size_mb']} MB")
+        finally:
+            eg.close()
+
+    @cli.command()
+    @click.argument("query")
+    @click.option("--top-k", "-k", default=5, help="Number of results to return.")
+    @click.option("--path-filter", "-f", default=None, help="SQL LIKE pattern for file path filter.")
+    @click.option("--db-path", default=None, help="Path to SQLite database.")
+    @click.option("--model", default="BAAI/bge-small-en-v1.5", help="Embedding model name.")
+    def search(query: str, top_k: int, path_filter: str | None, db_path: str | None, model: str) -> None:
+        """Semantic search across indexed files."""
+        from embgrep.indexer import EmbGrep
+
+        kwargs: dict = {"model": model}
+        if db_path:
+            kwargs["db_path"] = db_path
+
+        eg = EmbGrep(**kwargs)
+        try:
+            with console.status("[bold green]Searching..."):
+                results = eg.search(query, top_k=top_k, path_filter=path_filter)
+
+            if not results:
+                console.print("[yellow]No results found.[/yellow]")
+                return
+
+            table = Table(title=f"Search: {query!r}", show_lines=True)
+            table.add_column("#", style="dim", width=3)
+            table.add_column("Score", style="cyan", width=7)
+            table.add_column("File", style="green")
+            table.add_column("Lines", style="yellow", width=10)
+            table.add_column("Preview", max_width=60)
+
+            for i, r in enumerate(results, 1):
+                preview = r.chunk_text[:120].replace("\n", " ").strip()
+                if len(r.chunk_text) > 120:
+                    preview += "..."
+                table.add_row(
+                    str(i),
+                    f"{r.score:.4f}",
+                    r.file_path,
+                    f"{r.line_start}-{r.line_end}",
+                    preview,
+                )
+
+            console.print(table)
+        finally:
+            eg.close()
+
+    @cli.command()
+    @click.option("--db-path", default=None, help="Path to SQLite database.")
+    def status(db_path: str | None) -> None:
+        """Show index statistics."""
+        from embgrep.indexer import EmbGrep
+
+        kwargs: dict = {}
+        if db_path:
+            kwargs["db_path"] = db_path
+
+        eg = EmbGrep(**kwargs)
+        try:
+            st = eg.status()
+            console.print("[bold]embgrep Index Status[/bold]")
+            console.print(f"  Files:        {st.total_files}")
+            console.print(f"  Chunks:       {st.total_chunks}")
+            console.print(f"  Last updated: {st.last_updated}")
+            console.print(f"  Index size:   {st.index_size_mb} MB")
+        finally:
+            eg.close()
+
+    @cli.command()
+    @click.option("--db-path", default=None, help="Path to SQLite database.")
+    @click.option("--model", default="BAAI/bge-small-en-v1.5", help="Embedding model name.")
+    def update(db_path: str | None, model: str) -> None:
+        """Incremental update — re-index changed files only."""
+        from embgrep.indexer import EmbGrep
+
+        kwargs: dict = {"model": model}
+        if db_path:
+            kwargs["db_path"] = db_path
+
+        eg = EmbGrep(**kwargs)
+        try:
+            with console.status("[bold green]Updating index..."):
+                result = eg.update()
+            console.print(f"[green]Updated {result['updated_files']} files, {result['new_chunks']} new chunks[/green]")
+            if result["removed_files"]:
+                console.print(f"[yellow]Removed {result['removed_files']} deleted files[/yellow]")
+        finally:
+            eg.close()
+
+    cli()
+
+
+if __name__ == "__main__":
+    main()

embgrep/chunker.py

@@ -0,0 +1,205 @@
+"""File chunking for embgrep — split files into semantically meaningful chunks."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+# File extensions grouped by chunking strategy
+_CODE_EXTENSIONS = {".py", ".js", ".ts", ".java", ".go", ".rs"}
+_DOC_EXTENSIONS = {".md", ".txt"}
+_CONFIG_EXTENSIONS = {".yaml", ".yml", ".json", ".toml", ".cfg", ".ini"}
+_SHELL_EXTENSIONS = {".sh", ".bash"}
+
+SUPPORTED_EXTENSIONS = _CODE_EXTENSIONS | _DOC_EXTENSIONS | _CONFIG_EXTENSIONS | _SHELL_EXTENSIONS
+
+# Regex patterns for detecting function/class boundaries per language
+_CODE_BOUNDARY_PATTERNS: dict[str, re.Pattern[str]] = {
+    ".py": re.compile(r"^(def |class |async def )", re.MULTILINE),
+    ".js": re.compile(r"^(function |class |export |const \w+ = )", re.MULTILINE),
+    ".ts": re.compile(r"^(function |class |export |const \w+ = )", re.MULTILINE),
+    ".java": re.compile(r"^(public |private |class )", re.MULTILINE),
+    ".go": re.compile(r"^(func |type )", re.MULTILINE),
+    ".rs": re.compile(r"^(fn |pub fn |pub struct |struct |impl |pub impl )", re.MULTILINE),
+}
+
+# Markdown heading pattern
+_MD_HEADING = re.compile(r"^#{1,6}\s", re.MULTILINE)
+
+
+def chunk_file(file_path: str, max_chunk_size: int = 1000) -> list[dict]:
+    """Split a file into chunks for embedding.
+
+    Args:
+        file_path: Path to the file to chunk.
+        max_chunk_size: Maximum number of characters per chunk for fixed-size fallback.
+
+    Returns:
+        List of dicts with keys: text, line_start, line_end.
+
+    Strategy:
+        - .py/.js/.ts/.java/.go/.rs: split by function/class definitions (regex)
+        - .md/.txt: split by headings (## or blank line groups)
+        - Others: split by max_chunk_size characters
+    """
+    path = Path(file_path)
+    suffix = path.suffix.lower()
+
+    try:
+        content = path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        return []
+    except UnicodeDecodeError:
+        try:
+            content = path.read_text(encoding="latin-1")
+        except Exception:
+            return []
+
+    if not content.strip():
+        return []
+
+    lines = content.splitlines(keepends=True)
+
+    if suffix in _CODE_EXTENSIONS:
+        chunks = _chunk_code(lines, suffix, max_chunk_size)
+    elif suffix in _DOC_EXTENSIONS:
+        chunks = _chunk_docs(lines, suffix, max_chunk_size)
+    else:
+        chunks = _chunk_fixed(lines, max_chunk_size)
+
+    # Filter out empty chunks
+    return [c for c in chunks if c["text"].strip()]
+
+
+def _chunk_code(lines: list[str], suffix: str, max_chunk_size: int) -> list[dict]:
+    """Split code files by function/class boundaries."""
+    pattern = _CODE_BOUNDARY_PATTERNS.get(suffix)
+    if pattern is None:
+        return _chunk_fixed(lines, max_chunk_size)
+
+    # Find boundary line numbers
+    boundaries: list[int] = []
+    for i, line in enumerate(lines):
+        if pattern.match(line):
+            boundaries.append(i)
+
+    if not boundaries:
+        return _chunk_fixed(lines, max_chunk_size)
+
+    chunks: list[dict] = []
+
+    # Lines before the first boundary (imports, module docstring, etc.)
+    if boundaries[0] > 0:
+        text = "".join(lines[: boundaries[0]])
+        if text.strip():
+            chunks.append({"text": text, "line_start": 1, "line_end": boundaries[0]})
+
+    # Each boundary to the next
+    for i, start in enumerate(boundaries):
+        end = boundaries[i + 1] if i + 1 < len(boundaries) else len(lines)
+        text = "".join(lines[start:end])
+        if text.strip():
+            chunks.append({"text": text, "line_start": start + 1, "line_end": end})
+
+    return chunks
+
+
+def _chunk_docs(lines: list[str], suffix: str, max_chunk_size: int) -> list[dict]:
+    """Split document files by headings or blank-line groups."""
+    if suffix == ".md":
+        return _chunk_markdown(lines, max_chunk_size)
+    return _chunk_by_blank_lines(lines, max_chunk_size)
+
+
+def _chunk_markdown(lines: list[str], max_chunk_size: int) -> list[dict]:
+    """Split markdown by heading boundaries."""
+    boundaries: list[int] = []
+    for i, line in enumerate(lines):
+        if _MD_HEADING.match(line):
+            boundaries.append(i)
+
+    if not boundaries:
+        return _chunk_by_blank_lines(lines, max_chunk_size)
+
+    chunks: list[dict] = []
+
+    # Lines before the first heading
+    if boundaries[0] > 0:
+        text = "".join(lines[: boundaries[0]])
+        if text.strip():
+            chunks.append({"text": text, "line_start": 1, "line_end": boundaries[0]})
+
+    for i, start in enumerate(boundaries):
+        end = boundaries[i + 1] if i + 1 < len(boundaries) else len(lines)
+        text = "".join(lines[start:end])
+        if text.strip():
+            chunks.append({"text": text, "line_start": start + 1, "line_end": end})
+
+    return chunks
+
+
+def _chunk_by_blank_lines(lines: list[str], max_chunk_size: int) -> list[dict]:
+    """Split text by groups of blank lines."""
+    chunks: list[dict] = []
+    current_lines: list[str] = []
+    start_line = 0
+
+    for i, line in enumerate(lines):
+        if not line.strip() and current_lines:
+            # End of a paragraph
+            text = "".join(current_lines)
+            if len(text) > max_chunk_size:
+                # Split large paragraphs
+                sub_chunks = _split_text_fixed(current_lines, start_line, max_chunk_size)
+                chunks.extend(sub_chunks)
+            else:
+                chunks.append({"text": text, "line_start": start_line + 1, "line_end": i})
+            current_lines = []
+            start_line = i + 1
+        else:
+            if not current_lines:
+                start_line = i
+            current_lines.append(line)
+
+    # Remaining lines
+    if current_lines:
+        text = "".join(current_lines)
+        if text.strip():
+            chunks.append({"text": text, "line_start": start_line + 1, "line_end": len(lines)})
+
+    return chunks
+
+
+def _chunk_fixed(lines: list[str], max_chunk_size: int) -> list[dict]:
+    """Split by fixed character size."""
+    return _split_text_fixed(lines, 0, max_chunk_size)
+
+
+def _split_text_fixed(lines: list[str], offset: int, max_chunk_size: int) -> list[dict]:
+    """Split a list of lines into chunks of approximately max_chunk_size characters."""
+    chunks: list[dict] = []
+    current_lines: list[str] = []
+    current_size = 0
+    start_line = offset
+
+    for i, line in enumerate(lines):
+        if current_size + len(line) > max_chunk_size and current_lines:
+            text = "".join(current_lines)
+            chunks.append({"text": text, "line_start": start_line + 1, "line_end": offset + i})
+            current_lines = [line]
+            current_size = len(line)
+            start_line = offset + i
+        else:
+            current_lines.append(line)
+            current_size += len(line)
+
+    if current_lines:
+        text = "".join(current_lines)
+        if text.strip():
+            chunks.append({
+                "text": text,
+                "line_start": start_line + 1,
+                "line_end": offset + len(lines),
+            })
+
+    return chunks

embgrep/db.py

@@ -0,0 +1,159 @@
+"""SQLite storage for embgrep — files and chunks tables."""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+from pathlib import Path
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS indexed_files (
+    id INTEGER PRIMARY KEY,
+    file_path TEXT UNIQUE NOT NULL,
+    file_hash TEXT NOT NULL,
+    indexed_at TEXT DEFAULT CURRENT_TIMESTAMP,
+    chunk_count INTEGER DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    id INTEGER PRIMARY KEY,
+    file_id INTEGER NOT NULL,
+    chunk_text TEXT NOT NULL,
+    line_start INTEGER NOT NULL,
+    line_end INTEGER NOT NULL,
+    embedding BLOB NOT NULL,
+    FOREIGN KEY (file_id) REFERENCES indexed_files(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_file_id ON chunks(file_id);
+CREATE INDEX IF NOT EXISTS idx_files_path ON indexed_files(file_path);
+"""
+
+DEFAULT_DB_PATH = "~/.local/share/embgrep/embgrep.db"
+
+
+class Database:
+    """SQLite database wrapper for embgrep index storage."""
+
+    def __init__(self, db_path: str = DEFAULT_DB_PATH) -> None:
+        self.db_path = os.path.expanduser(db_path)
+        Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(self.db_path)
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA foreign_keys=ON")
+        self._conn.executescript(_SCHEMA)
+        self._conn.commit()
+
+    def close(self) -> None:
+        """Close the database connection."""
+        self._conn.close()
+
+    # --- indexed_files operations ---
+
+    def insert_file(self, file_path: str, file_hash: str) -> int:
+        """Insert a file record and return its id."""
+        cur = self._conn.execute(
+            "INSERT INTO indexed_files (file_path, file_hash, chunk_count) VALUES (?, ?, 0)",
+            (file_path, file_hash),
+        )
+        self._conn.commit()
+        return cur.lastrowid  # type: ignore[return-value]
+
+    def get_file(self, file_path: str) -> tuple[int, str, str, str, int] | None:
+        """Get file record by path. Returns (id, file_path, file_hash, indexed_at, chunk_count) or None."""
+        cur = self._conn.execute("SELECT id, file_path, file_hash, indexed_at, chunk_count FROM indexed_files WHERE file_path = ?", (file_path,))
+        return cur.fetchone()
+
+    def get_all_files(self) -> list[tuple[int, str, str, str, int]]:
+        """Get all file records."""
+        cur = self._conn.execute("SELECT id, file_path, file_hash, indexed_at, chunk_count FROM indexed_files")
+        return cur.fetchall()
+
+    def update_file_hash(self, file_id: int, file_hash: str) -> None:
+        """Update file hash and reset indexed_at."""
+        self._conn.execute(
+            "UPDATE indexed_files SET file_hash = ?, indexed_at = CURRENT_TIMESTAMP WHERE id = ?",
+            (file_hash, file_id),
+        )
+        self._conn.commit()
+
+    def update_chunk_count(self, file_id: int, count: int) -> None:
+        """Update the chunk count for a file."""
+        self._conn.execute("UPDATE indexed_files SET chunk_count = ? WHERE id = ?", (count, file_id))
+        self._conn.commit()
+
+    def delete_file(self, file_id: int) -> None:
+        """Delete a file and its chunks (cascading)."""
+        self._conn.execute("DELETE FROM indexed_files WHERE id = ?", (file_id,))
+        self._conn.commit()
+
+    def file_count(self) -> int:
+        """Get total number of indexed files."""
+        cur = self._conn.execute("SELECT COUNT(*) FROM indexed_files")
+        return cur.fetchone()[0]
+
+    # --- chunks operations ---
+
+    def insert_chunks(self, file_id: int, chunks: list[tuple[str, int, int, bytes]]) -> int:
+        """Insert multiple chunks for a file. Each chunk is (text, line_start, line_end, embedding_blob).
+
+        Returns the number of chunks inserted.
+        """
+        self._conn.executemany(
+            "INSERT INTO chunks (file_id, chunk_text, line_start, line_end, embedding) VALUES (?, ?, ?, ?, ?)",
+            [(file_id, text, ls, le, emb) for text, ls, le, emb in chunks],
+        )
+        self._conn.commit()
+        return len(chunks)
+
+    def delete_chunks_for_file(self, file_id: int) -> None:
+        """Delete all chunks for a file."""
+        self._conn.execute("DELETE FROM chunks WHERE file_id = ?", (file_id,))
+        self._conn.commit()
+
+    def get_chunks(
+        self, path_filter: str | None = None
+    ) -> list[tuple[int, str, str, int, int, bytes]]:
+        """Get chunks with optional path filter.
+
+        Returns list of (chunk_id, file_path, chunk_text, line_start, line_end, embedding_blob).
+        """
+        if path_filter:
+            cur = self._conn.execute(
+                """
+                SELECT c.id, f.file_path, c.chunk_text, c.line_start, c.line_end, c.embedding
+                FROM chunks c
+                JOIN indexed_files f ON c.file_id = f.id
+                WHERE f.file_path LIKE ?
+                """,
+                (path_filter,),
+            )
+        else:
+            cur = self._conn.execute(
+                """
+                SELECT c.id, f.file_path, c.chunk_text, c.line_start, c.line_end, c.embedding
+                FROM chunks c
+                JOIN indexed_files f ON c.file_id = f.id
+                """
+            )
+        return cur.fetchall()
+
+    def chunk_count(self) -> int:
+        """Get total number of chunks."""
+        cur = self._conn.execute("SELECT COUNT(*) FROM chunks")
+        return cur.fetchone()[0]
+
+    # --- statistics ---
+
+    def last_updated(self) -> str | None:
+        """Get the most recent indexed_at timestamp."""
+        cur = self._conn.execute("SELECT MAX(indexed_at) FROM indexed_files")
+        row = cur.fetchone()
+        return row[0] if row else None
+
+    def db_size_mb(self) -> float:
+        """Get database file size in MB."""
+        try:
+            return os.path.getsize(self.db_path) / (1024 * 1024)
+        except OSError:
+            return 0.0

embgrep/embedder.py

@@ -0,0 +1,60 @@
+"""fastembed wrapper for embgrep — lazy-loading embedding model."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from fastembed import TextEmbedding
+
+DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
+EMBEDDING_DIM = 384
+
+
+class Embedder:
+    """Wrapper around fastembed TextEmbedding with lazy initialization.
+
+    The ONNX model is downloaded and loaded only on first use.
+    """
+
+    def __init__(self, model: str = DEFAULT_MODEL) -> None:
+        self._model_name = model
+        self._model: TextEmbedding | None = None
+
+    def _ensure_model(self) -> TextEmbedding:
+        """Lazily initialize the fastembed model."""
+        if self._model is None:
+            from fastembed import TextEmbedding
+
+            self._model = TextEmbedding(self._model_name)
+        return self._model
+
+    def embed(self, texts: list[str]) -> list[np.ndarray]:
+        """Embed a list of texts.
+
+        Args:
+            texts: List of text strings to embed.
+
+        Returns:
+            List of numpy arrays, each of shape (EMBEDDING_DIM,).
+        """
+        if not texts:
+            return []
+        model = self._ensure_model()
+        embeddings = list(model.embed(texts))
+        return [e.astype(np.float32) for e in embeddings]
+
+    def embed_query(self, query: str) -> np.ndarray:
+        """Embed a single query string.
+
+        Args:
+            query: The query text to embed.
+
+        Returns:
+            Numpy array of shape (EMBEDDING_DIM,).
+        """
+        model = self._ensure_model()
+        embeddings = list(model.embed([query]))
+        return embeddings[0].astype(np.float32)

embgrep/indexer.py

@@ -0,0 +1,237 @@
+"""EmbGrep — main orchestrator for indexing and semantic search."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+
+from embgrep.chunker import SUPPORTED_EXTENSIONS, chunk_file
+from embgrep.db import DEFAULT_DB_PATH, Database
+from embgrep.embedder import EMBEDDING_DIM, Embedder
+
+
+@dataclass
+class SearchResult:
+    """A single search result with file location and similarity score."""
+
+    file_path: str
+    chunk_text: str
+    score: float
+    line_start: int
+    line_end: int
+
+
+@dataclass
+class IndexStatus:
+    """Statistics about the current embgrep index."""
+
+    total_files: int
+    total_chunks: int
+    last_updated: str
+    index_size_mb: float
+
+
+def _file_hash(file_path: str) -> str:
+    """Compute SHA-256 hash of file contents."""
+    h = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(8192), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _collect_files(directory: str, patterns: list[str] | None = None) -> list[str]:
+    """Collect files from directory matching patterns or supported extensions."""
+    root = Path(directory).resolve()
+    if not root.is_dir():
+        msg = f"Directory not found: {directory}"
+        raise FileNotFoundError(msg)
+
+    if patterns:
+        files: list[str] = []
+        for pattern in patterns:
+            files.extend(str(p) for p in root.rglob(pattern) if p.is_file())
+        return sorted(set(files))
+
+    # Default: collect files with supported extensions
+    files = []
+    for p in root.rglob("*"):
+        if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS:
+            files.append(str(p))
+    return sorted(files)
+
+
+class EmbGrep:
+    """Main orchestrator for embedding-powered file search.
+
+    Args:
+        db_path: Path to the SQLite database file.
+        model: Name of the fastembed model to use.
+    """
+
+    def __init__(
+        self,
+        db_path: str = DEFAULT_DB_PATH,
+        model: str = "BAAI/bge-small-en-v1.5",
+    ) -> None:
+        self._db = Database(db_path)
+        self._embedder = Embedder(model)
+
+    def index(self, directory: str, patterns: list[str] | None = None) -> dict:
+        """Index files in a directory.
+
+        Args:
+            directory: Path to the directory to index.
+            patterns: Optional list of glob patterns to filter files (e.g., ["*.py", "*.md"]).
+
+        Returns:
+            Dict with keys: files_indexed, chunks_created, index_size_mb.
+        """
+        files = _collect_files(directory, patterns)
+
+        files_indexed = 0
+        chunks_created = 0
+
+        for file_path in files:
+            fhash = _file_hash(file_path)
+            existing = self._db.get_file(file_path)
+
+            if existing and existing[2] == fhash:
+                # File unchanged, skip
+                continue
+
+            if existing:
+                # File changed, remove old data
+                self._db.delete_chunks_for_file(existing[0])
+                self._db.delete_file(existing[0])
+
+            # Index the file
+            n_chunks = self._index_single_file(file_path, fhash)
+            if n_chunks > 0:
+                files_indexed += 1
+                chunks_created += n_chunks
+
+        return {
+            "files_indexed": files_indexed,
+            "chunks_created": chunks_created,
+            "index_size_mb": round(self._db.db_size_mb(), 2),
+        }
+
+    def _index_single_file(self, file_path: str, fhash: str) -> int:
+        """Index a single file: chunk, embed, store. Returns number of chunks created."""
+        chunks = chunk_file(file_path)
+        if not chunks:
+            return 0
+
+        texts = [c["text"] for c in chunks]
+        embeddings = self._embedder.embed(texts)
+
+        file_id = self._db.insert_file(file_path, fhash)
+
+        chunk_records = []
+        for chunk_data, emb in zip(chunks, embeddings, strict=True):
+            blob = emb.astype(np.float32).tobytes()
+            chunk_records.append((chunk_data["text"], chunk_data["line_start"], chunk_data["line_end"], blob))
+
+        self._db.insert_chunks(file_id, chunk_records)
+        self._db.update_chunk_count(file_id, len(chunk_records))
+
+        return len(chunk_records)
+
+    def search(self, query: str, top_k: int = 5, path_filter: str | None = None) -> list[SearchResult]:
+        """Semantic search across indexed chunks.
+
+        Args:
+            query: Natural language search query.
+            top_k: Number of top results to return.
+            path_filter: Optional SQL LIKE pattern to filter by file path (e.g., "%.py").
+
+        Returns:
+            List of SearchResult sorted by descending similarity score.
+        """
+        query_emb = self._embedder.embed_query(query)
+        db_chunks = self._db.get_chunks(path_filter=path_filter)
+
+        if not db_chunks:
+            return []
+
+        results: list[SearchResult] = []
+
+        for _chunk_id, file_path, chunk_text, line_start, line_end, emb_blob in db_chunks:
+            emb = np.frombuffer(emb_blob, dtype=np.float32)
+            if emb.shape[0] != EMBEDDING_DIM:
+                continue
+            score = _cosine_similarity(query_emb, emb)
+            results.append(SearchResult(
+                file_path=file_path,
+                chunk_text=chunk_text,
+                score=float(score),
+                line_start=line_start,
+                line_end=line_end,
+            ))
+
+        results.sort(key=lambda r: r.score, reverse=True)
+        return results[:top_k]
+
+    def update(self) -> dict:
+        """Incremental update — re-index changed files, remove deleted files.
+
+        Returns:
+            Dict with keys: updated_files, new_chunks, removed_files.
+        """
+        all_files = self._db.get_all_files()
+        updated_files = 0
+        new_chunks = 0
+        removed_files = 0
+
+        for file_id, file_path, stored_hash, _indexed_at, _chunk_count in all_files:
+            if not os.path.isfile(file_path):
+                # File was deleted
+                self._db.delete_file(file_id)
+                removed_files += 1
+                continue
+
+            current_hash = _file_hash(file_path)
+            if current_hash != stored_hash:
+                # File changed, re-index
+                self._db.delete_chunks_for_file(file_id)
+                self._db.delete_file(file_id)
+                n_chunks = self._index_single_file(file_path, current_hash)
+                updated_files += 1
+                new_chunks += n_chunks
+
+        return {
+            "updated_files": updated_files,
+            "new_chunks": new_chunks,
+            "removed_files": removed_files,
+        }
+
+    def status(self) -> IndexStatus:
+        """Get current index statistics.
+
+        Returns:
+            IndexStatus dataclass with summary information.
+        """
+        return IndexStatus(
+            total_files=self._db.file_count(),
+            total_chunks=self._db.chunk_count(),
+            last_updated=self._db.last_updated() or "never",
+            index_size_mb=round(self._db.db_size_mb(), 2),
+        )
+
+    def close(self) -> None:
+        """Close the database connection."""
+        self._db.close()
+
+
+def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
+    """Compute cosine similarity between two vectors."""
+    norm_a = np.linalg.norm(a)
+    norm_b = np.linalg.norm(b)
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return float(np.dot(a, b) / (norm_a * norm_b))

embgrep/mcp_server.py

@@ -0,0 +1,119 @@
+"""FastMCP server for embgrep — 4 semantic search tools."""
+
+from __future__ import annotations
+
+import json
+
+
+def _create_server():
+    """Create and configure the FastMCP server."""
+    try:
+        from fastmcp import FastMCP
+    except ImportError:
+        msg = "MCP server requires extra dependencies: pip install embgrep[mcp]"
+        raise ImportError(msg)  # noqa: B904
+
+    from embgrep.indexer import EmbGrep
+
+    mcp = FastMCP("embgrep", instructions="Local semantic search — embedding-powered grep for files.")
+
+    def _get_embgrep() -> EmbGrep:
+        return EmbGrep()
+
+    @mcp.tool()
+    def index_directory(path: str, patterns: str = "*.md,*.py,*.txt") -> str:
+        """Index files in a directory for semantic search.
+
+        Args:
+            path: Directory path to index.
+            patterns: Comma-separated glob patterns (default: "*.md,*.py,*.txt").
+
+        Returns:
+            JSON string with indexing results.
+        """
+        pattern_list = [p.strip() for p in patterns.split(",")]
+        eg = _get_embgrep()
+        try:
+            result = eg.index(path, patterns=pattern_list)
+            return json.dumps(result, indent=2)
+        finally:
+            eg.close()
+
+    @mcp.tool()
+    def semantic_search(query: str, top_k: int = 5, path_filter: str | None = None) -> str:
+        """Search indexed files using natural language.
+
+        Args:
+            query: Natural language search query.
+            top_k: Number of results to return (default: 5).
+            path_filter: Optional SQL LIKE pattern to filter by file path.
+
+        Returns:
+            JSON string with search results.
+        """
+        eg = _get_embgrep()
+        try:
+            results = eg.search(query, top_k=top_k, path_filter=path_filter)
+            return json.dumps(
+                [
+                    {
+                        "file_path": r.file_path,
+                        "score": round(r.score, 4),
+                        "line_start": r.line_start,
+                        "line_end": r.line_end,
+                        "chunk_text": r.chunk_text[:500],
+                    }
+                    for r in results
+                ],
+                indent=2,
+            )
+        finally:
+            eg.close()
+
+    @mcp.tool()
+    def index_status() -> str:
+        """Get current index statistics.
+
+        Returns:
+            JSON string with index status information.
+        """
+        eg = _get_embgrep()
+        try:
+            st = eg.status()
+            return json.dumps(
+                {
+                    "total_files": st.total_files,
+                    "total_chunks": st.total_chunks,
+                    "last_updated": st.last_updated,
+                    "index_size_mb": st.index_size_mb,
+                },
+                indent=2,
+            )
+        finally:
+            eg.close()
+
+    @mcp.tool()
+    def update_index() -> str:
+        """Incremental update — re-index changed files only (hash comparison).
+
+        Returns:
+            JSON string with update results.
+        """
+        eg = _get_embgrep()
+        try:
+            result = eg.update()
+            return json.dumps(result, indent=2)
+        finally:
+            eg.close()
+
+    return mcp
+
+
+def main() -> None:
+    """Run the MCP server."""
+    server = _create_server()
+    server.run()
+
+
+if __name__ == "__main__":
+    main()

embgrep-0.1.0.dist-info/METADATA

@@ -0,0 +1,194 @@
+Metadata-Version: 2.4
+Name: embgrep
+Version: 0.1.0
+Summary: Local semantic search — embedding-powered grep for files, zero external services.
+Project-URL: Homepage, https://github.com/QuartzUnit/embgrep
+Project-URL: Repository, https://github.com/QuartzUnit/embgrep
+Author: QuartzUnit
+License-Expression: MIT
+License-File: LICENSE
+Keywords: embeddings,grep,local,mcp,rag,semantic-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: fastembed>=0.4
+Requires-Dist: numpy>=1.24
+Provides-Extra: all
+Requires-Dist: click>=8.0; extra == 'all'
+Requires-Dist: fastmcp>=2.0; extra == 'all'
+Requires-Dist: rich>=13.0; extra == 'all'
+Provides-Extra: cli
+Requires-Dist: click>=8.0; extra == 'cli'
+Requires-Dist: rich>=13.0; extra == 'cli'
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Provides-Extra: mcp
+Requires-Dist: fastmcp>=2.0; extra == 'mcp'
+Description-Content-Type: text/markdown
+
+# embgrep
+
+**Local semantic search — embedding-powered grep for files, zero external services.**
+
+[![PyPI](https://img.shields.io/pypi/v/embgrep)](https://pypi.org/project/embgrep/)
+[![Python](https://img.shields.io/pypi/pyversions/embgrep)](https://pypi.org/project/embgrep/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Search your codebase and documentation by *meaning*, not just keywords. embgrep indexes files into local embeddings and lets you run semantic queries — no API keys, no cloud services, no vector database servers.
+
+## Features
+
+- **Local embeddings** — Uses [fastembed](https://github.com/qdrant/fastembed) (ONNX Runtime), no API keys needed
+- **SQLite storage** — Single-file index, no external vector DB
+- **Incremental indexing** — Only re-indexes changed files (SHA-256 hash comparison)
+- **Smart chunking** — Function-level splitting for code, heading-level for docs
+- **MCP native** — 4-tool FastMCP server for LLM agent integration
+- **15+ file types** — `.py`, `.js`, `.ts`, `.java`, `.go`, `.rs`, `.md`, `.txt`, `.yaml`, `.json`, `.toml`, and more
+
+## Install
+
+```bash
+pip install embgrep              # core (fastembed + numpy)
+pip install embgrep[cli]         # + click/rich CLI
+pip install embgrep[mcp]         # + FastMCP server
+pip install embgrep[all]         # everything
+```
+
+## Quick Start
+
+### Python API
+
+```python
+from embgrep import EmbGrep
+
+eg = EmbGrep()
+
+# Index a directory
+eg.index("./my-project", patterns=["*.py", "*.md"])
+
+# Semantic search
+results = eg.search("database connection pooling", top_k=5)
+for r in results:
+    print(f"{r.file_path}:{r.line_start}-{r.line_end} (score: {r.score:.4f})")
+    print(f"  {r.chunk_text[:80]}...")
+
+# Incremental update (only changed files)
+eg.update()
+
+# Index statistics
+status = eg.status()
+print(f"{status.total_files} files, {status.total_chunks} chunks, {status.index_size_mb} MB")
+
+eg.close()
+```
+
+### CLI
+
+```bash
+# Index a project
+embgrep index ./my-project --patterns "*.py,*.md"
+
+# Search
+embgrep search "error handling patterns"
+
+# Filter by file type
+embgrep search "async database query" --path-filter "%.py"
+
+# Check status
+embgrep status
+
+# Update changed files
+embgrep update
+```
+
+### Convenience functions
+
+```python
+import embgrep
+
+embgrep.index("./src")
+results = embgrep.search("authentication middleware")
+status = embgrep.status()
+embgrep.update()
+```
+
+## MCP Server
+
+Add to your Claude Desktop / MCP client configuration:
+
+```json
+{
+  "mcpServers": {
+    "embgrep": {
+      "command": "embgrep-mcp"
+    }
+  }
+}
+```
+
+Or with uvx:
+
+```json
+{
+  "mcpServers": {
+    "embgrep": {
+      "command": "uvx",
+      "args": ["--from", "embgrep[mcp]", "embgrep-mcp"]
+    }
+  }
+}
+```
+
+### MCP Tools
+
+| Tool | Description |
+|------|-------------|
+| `index_directory` | Index files in a directory for semantic search |
+| `semantic_search` | Search indexed files using natural language |
+| `index_status` | Get current index statistics |
+| `update_index` | Incremental update — re-index changed files only |
+
+## How It Works
+
+1. **Chunking** — Files are split into semantically meaningful chunks:
+   - Code files (`.py`, `.js`, `.ts`, etc.): split by function/class boundaries
+   - Documents (`.md`, `.txt`): split by headings or paragraph breaks
+   - Config files: fixed-size chunking
+
+2. **Embedding** — Each chunk is converted to a 384-dimensional vector using [BGE-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) via ONNX Runtime (no PyTorch needed)
+
+3. **Storage** — Embeddings are stored as BLOBs in a local SQLite database
+
+4. **Search** — Query text is embedded and compared against all chunks using cosine similarity
+
+## Configuration
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `db_path` | `~/.local/share/embgrep/embgrep.db` | SQLite database location |
+| `model` | `BAAI/bge-small-en-v1.5` | fastembed model name |
+| `max_chunk_size` | 1000 chars | Maximum chunk size for fixed-size splitting |
+| `top_k` | 5 | Number of search results |
+
+## QuartzUnit Ecosystem
+
+| Package | Description |
+|---------|-------------|
+| [markgrab](https://github.com/QuartzUnit/markgrab) | HTML/YouTube/PDF/DOCX to LLM-ready markdown |
+| [snapgrab](https://github.com/QuartzUnit/snapgrab) | URL to screenshot + metadata |
+| [docpick](https://github.com/QuartzUnit/docpick) | OCR + LLM document structure extraction |
+| [browsegrab](https://github.com/QuartzUnit/browsegrab) | Local LLM browser agent |
+| [feedkit](https://github.com/QuartzUnit/feedkit) | RSS feed collection + MCP |
+| **embgrep** | **Local semantic search for files** |
+
+## License
+
+MIT
+
+<!-- mcp-name: io.github.ArkNill/embgrep -->

embgrep-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+embgrep/__init__.py,sha256=U4l_kzuD2ISXldt79TwK7pAQk3VFLROcQiSMTCmRJM4,2277
+embgrep/__main__.py,sha256=9fKlLfoa2vAib2tdwdrtcYfgI9WhDlozES8GhKq7XRA,5325
+embgrep/chunker.py,sha256=f4iTL3oCs1nK0eMH2GrqfrzLpPanDVU8zAd5XBi5HWI,7042
+embgrep/db.py,sha256=zX4AAjZ-3wm7QRKzuAkOVt4J3D-UPsRaSFjtaqVuYy4,5885
+embgrep/embedder.py,sha256=NXIN7ZwodBO-gAGBD9nGvIfMMbcl0iu3r-t1WTk25lI,1659
+embgrep/indexer.py,sha256=PIAMe8qidD-e8mjnjT7HsutF99kOwuRy5PRk5BQbJjY,7519
+embgrep/mcp_server.py,sha256=fqYF-UDWqWugGkzMAGOG_2o3JaJZY8F3SzDklRNrtRA,3385
+embgrep-0.1.0.dist-info/METADATA,sha256=41KnmBPsRdoG1rh0MsipEnAQqUspimm8s_WrDpXHHOs,5984
+embgrep-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+embgrep-0.1.0.dist-info/entry_points.txt,sha256=zIgMaq7SJsf_tCQilxas66N9OnPuhVhJOZ02-OTMPuI,88
+embgrep-0.1.0.dist-info/licenses/LICENSE,sha256=fFlwqojwD_YW8kxc0bAa5n0luMCtwf04FZ-Ad0aQ7RI,1067
+embgrep-0.1.0.dist-info/RECORD,,

embgrep-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

embgrep-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+embgrep = embgrep.__main__:main
+embgrep-mcp = embgrep.mcp_server:main

embgrep-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 QuartzUnit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.