infogrep 0.0.1__py3-none-any.whl

infogrep/__init__.py

@@ -0,0 +1,3 @@
+"""InfoGrep: local-first content search for coding agents."""
+
+__version__ = "0.0.1"

infogrep/cli.py

@@ -0,0 +1,217 @@
+"""InfoGrep command-line interface.
+
+Thin wrapper over the core engine; also the entry point used by the daily
+scheduled re-index. Subcommands are stubbed until their milestones land.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+import typer
+
+from . import __version__
+from .config import Config
+
+app = typer.Typer(
+    add_completion=False,
+    help="Local-first content search (sparse + dense + knowledge base) for coding agents.",
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"infogrep {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def _root(
+    version: bool = typer.Option(
+        False, "--version", callback=_version_callback, is_eager=True, help="Show version."
+    ),
+) -> None:
+    """InfoGrep: index and search the content of local files."""
+
+
+@app.command()
+def index(
+    directory: Path = typer.Argument(..., help="Directory to index."),
+    full: bool = typer.Option(False, "--full", help="Force a full re-index."),
+) -> None:
+    """Build or incrementally update the side-car index for a directory."""
+    from .indexer import Indexer
+
+    cfg = Config.load(directory)
+    if not cfg.target_dir.is_dir():
+        typer.echo(f"[infogrep] not a directory: {cfg.target_dir}", err=True)
+        raise typer.Exit(code=2)
+
+    typer.echo(f"[infogrep] indexing {cfg.target_dir}")
+    typer.echo(f"[infogrep] index location: {cfg.index_dir}")
+
+    def _progress(done: int, total: int) -> None:
+        typer.echo(f"[infogrep] extracted {done}/{total} files…", err=True)
+
+    report = Indexer(cfg).reindex(full=full, on_progress=_progress)
+    typer.echo(
+        "[infogrep] "
+        f"added={report.added} modified={report.modified} deleted={report.deleted} "
+        f"unchanged={report.unchanged} name_only={report.name_only}"
+    )
+    typer.echo(f"[infogrep] index now holds {report.n_files} files, {report.n_passages} passages")
+    for err in report.errors:
+        typer.echo(f"[infogrep] error: {err}", err=True)
+
+
+@app.command()
+def search(
+    query: str = typer.Argument(..., help="Search query."),
+    directory: Path = typer.Option(Path.cwd(), "--dir", "-d", help="Indexed directory."),
+    k: int = typer.Option(10, "--k", help="Number of results."),
+    mode: str = typer.Option("hybrid", "--mode", "-m", help="hybrid | sparse | dense | kb | graph."),
+    prf: bool = typer.Option(False, "--prf", help="RM3 pseudo-relevance feedback (sparse)."),
+) -> None:
+    """Query indexed content."""
+    from .engine import SearchEngine
+
+    engine = SearchEngine(Config.load(directory))
+
+    try:
+        if mode == "sparse":
+            results = engine.search_sparse(query, k=k, prf=prf)
+        elif mode == "dense":
+            results = engine.search_dense(query, k=k)
+        elif mode == "hybrid":
+            out = engine.search_hybrid(query, k=k, prf=prf)
+            results = out.results
+            if out.used:
+                typer.echo(f"[infogrep] fused: {', '.join(out.used)}")
+            for name, reason in out.skipped.items():
+                typer.echo(f"[infogrep] skipped {name}: {reason}")
+        elif mode == "kb":
+            results = engine.search_kb(query, k=k)
+        elif mode == "graph":
+            results = engine.search_graph(query, k=k)
+        else:
+            typer.echo(f"[infogrep] unknown mode: {mode}", err=True)
+            raise typer.Exit(code=2)
+    except FileNotFoundError as exc:
+        typer.echo(f"[infogrep] {exc}", err=True)
+        raise typer.Exit(code=2)
+
+    if not results:
+        typer.echo("[infogrep] no results.")
+        return
+    for i, r in enumerate(results, start=1):
+        file_ref = r.abs_path or r.path  # original file path when known
+        loc = file_ref + (f" p.{r.page}" if r.page is not None else "")
+        typer.echo(f"{i:2}. [{r.score:.3f}] {loc}  ({r.retriever})")
+        typer.echo(f"    {r.snippet.strip()[:160]}")
+
+
+@app.command()
+def status(
+    directory: Path = typer.Argument(Path.cwd(), help="Indexed directory."),
+) -> None:
+    """Show index status and staleness for a directory."""
+    from .indexer import Indexer
+
+    cfg = Config.load(directory)
+    info = Indexer(cfg).status()
+    typer.echo(f"[infogrep] target: {cfg.target_dir}")
+    typer.echo(f"[infogrep] index location: {cfg.index_dir}")
+    if not info.get("indexed"):
+        typer.echo("[infogrep] indexed: no")
+        typer.echo("[infogrep] run `infogrep index <dir>` to build the index.")
+        return
+    typer.echo("[infogrep] indexed: yes")
+    typer.echo(f"[infogrep] files: {info['n_files']}  passages: {info['n_passages']}")
+    typer.echo(f"[infogrep] index version: {info['index_version']}")
+    last = info.get("last_indexed_at")
+    if last:
+        when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(float(last)))
+        typer.echo(f"[infogrep] last indexed: {when}")
+    if info.get("stale"):
+        typer.echo(
+            f"[infogrep] STALE: {info['pending']} pending "
+            f"(+{info['pending_added']} ~{info['pending_modified']} -{info['pending_deleted']}) "
+            "— run `infogrep index`"
+        )
+    elif "stale" in info:
+        typer.echo("[infogrep] up to date")
+
+
+@app.command()
+def mcp(
+    directory: Path = typer.Option(Path.cwd(), "--dir", "-d", help="Default indexed directory."),
+) -> None:
+    """Run the MCP server (stdio) so coding agents can call InfoGrep's search tools."""
+    from .mcp_server import main as serve
+
+    serve(directory=str(Path(directory).expanduser().resolve()))
+
+
+@app.command()
+def serve(
+    directory: Path = typer.Option(Path.cwd(), "--dir", "-d", help="Indexed directory to search."),
+    port: int = typer.Option(7421, "--port", "-p", help="Port (uncommon by default)."),
+    host: str = typer.Option("127.0.0.1", "--host", help="Bind host (localhost by default)."),
+) -> None:
+    """Run a local web UI to test search in a browser."""
+    from .web import serve as run_web
+
+    run_web(directory=directory, host=host, port=port)
+
+
+schedule_app = typer.Typer(help="Manage daily auto-reindex (macOS launchd).")
+app.add_typer(schedule_app, name="schedule")
+
+
+@schedule_app.command("install")
+def schedule_install(
+    directory: Path = typer.Argument(..., help="Directory to reindex daily."),
+    at: str = typer.Option("03:00", "--at", help="Daily run time, HH:MM (24h)."),
+) -> None:
+    """Install a daily reindex agent for a directory."""
+    from . import scheduler
+
+    try:
+        hour, minute = (int(x) for x in at.split(":", 1))
+    except ValueError:
+        typer.echo(f"[infogrep] invalid --at time: {at!r} (use HH:MM)", err=True)
+        raise typer.Exit(code=2)
+    path = scheduler.install(directory, hour=hour, minute=minute)
+    typer.echo(f"[infogrep] scheduled daily reindex of {Path(directory).resolve()} at {at}")
+    typer.echo(f"[infogrep] launchd agent: {path}")
+
+
+@schedule_app.command("uninstall")
+def schedule_uninstall(
+    directory: Path = typer.Argument(..., help="Directory whose schedule to remove."),
+) -> None:
+    """Remove the daily reindex agent for a directory."""
+    from . import scheduler
+
+    if scheduler.uninstall(directory):
+        typer.echo(f"[infogrep] removed reindex schedule for {Path(directory).resolve()}")
+    else:
+        typer.echo("[infogrep] no schedule found for that directory.")
+
+
+@schedule_app.command("list")
+def schedule_list() -> None:
+    """List installed daily reindex agents."""
+    from . import scheduler
+
+    agents = scheduler.list_agents()
+    if not agents:
+        typer.echo("[infogrep] no reindex schedules installed.")
+        return
+    for a in agents:
+        typer.echo(f"[infogrep] {a['hour']:02d}:{a['minute']:02d} daily  {a['directory']}")
+
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

infogrep/config.py

@@ -0,0 +1,217 @@
+"""Configuration model and per-directory config loading.
+
+Indexing never writes into the indexed folder. Each directory's index lives in a
+separate location under ``$INFOGREP_HOME`` (default ``~/.infogrep``):
+``$INFOGREP_HOME/indexes/<name>-<hash-of-abs-path>/``. Per-directory config is read from
+that index dir's ``config.toml`` (with an optional global ``$INFOGREP_HOME/config.toml``
+as a base).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import sys
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - exercised only on 3.10
+    import tomli as tomllib
+
+# Legacy in-folder side-car name — still pruned during the walk so an old one (or a
+# stray) inside a target never gets indexed. InfoGrep no longer creates it.
+SIDECAR_DIRNAME = ".infogrep"
+
+
+def index_home() -> Path:
+    """Root for all InfoGrep indexes (override with the INFOGREP_HOME env var)."""
+    return Path(os.environ.get("INFOGREP_HOME", "~/.infogrep")).expanduser()
+
+
+def index_dir_for(target_dir: Path) -> Path:
+    """Stable, separate index location for a target directory (outside the target)."""
+    target = Path(target_dir).expanduser().resolve()
+    digest = hashlib.sha256(str(target).encode("utf-8")).hexdigest()[:12]
+    name = re.sub(r"[^A-Za-z0-9._-]", "_", target.name) or "root"
+    return index_home() / "indexes" / f"{name}-{digest}"
+
+
+@dataclass
+class ChunkConfig:
+    """How long documents are split into passages."""
+
+    size: int = 512  # target chunk size in tokens/words
+    overlap: int = 64  # overlap between adjacent chunks
+
+
+@dataclass
+class IngestConfig:
+    """Ingestion-side options."""
+
+    ocr: bool = False  # OCR PDF pages that have little/no extractable text (needs tesseract)
+    ocr_min_chars: int = 16  # below this many chars on a page, try OCR
+    workers: int = 0  # parallel extraction processes; 0 = auto (min(8, cpu count))
+
+
+@dataclass
+class DenseConfig:
+    """Dense retrieval settings.
+
+    Off by default: embedding a large corpus needs a model download and significant
+    RAM/GPU. Enable per directory with ``[dense] enabled = true`` once you want semantics.
+    """
+
+    enabled: bool = False
+    embedder: str = "qwen"  # registry key; see infogrep.retrieval.embedders
+    model_name: str = "Qwen/Qwen3-Embedding-0.6B"
+    device: str = "auto"  # "auto" -> mps/cuda/cpu
+
+
+@dataclass
+class SparseConfig:
+    """Sparse (Pyserini/BM25) settings."""
+
+    enabled: bool = True
+    prf: bool = False  # pseudo-relevance feedback (query expansion), off by default
+    prf_fb_docs: int = 10  # feedback documents (top multi-field results) to expand from
+    prf_fb_terms: int = 10  # expansion terms to add to the query
+    # Analyzer language. Default "en+zh" handles English (Porter stemming) AND CJK
+    # (bigrams) together. Also: "en" (English only), "zh"/"ja"/"ko" (single CJK).
+    # Changing it triggers a full re-index.
+    language: str = "en+zh"
+    # Multi-field BM25 weights: passage text + file name + path.
+    field_boosts: dict = field(
+        default_factory=lambda: {"contents": 1.0, "filename": 2.0, "pathtext": 1.0}
+    )
+
+
+@dataclass
+class KnowledgeBaseConfig:
+    """Obsidian knowledge-base settings (backed by the Obsidian CLI)."""
+
+    enabled: bool = False
+    vault: str | None = None  # Obsidian vault name; None -> the CLI's active vault
+    cli: str = "obsidian"  # path to the Obsidian CLI binary
+    hops: int = 1  # graph link hops to expand (follows links + backlinks)
+    search_limit: int = 10  # how many search hits to seed graph expansion from
+
+
+@dataclass
+class GraphConfig:
+    """Metadata knowledge-graph over folder/file structure.
+
+    Built from file *paths and names only* (never content) on every reindex, and
+    materialized as an Obsidian-compatible vault of folder notes under the index's
+    ``graph_vault/`` side-car dir, browsable in Obsidian if you open it there. Hybrid
+    search uses it (no Obsidian app required — InfoGrep reads its own graph directly) to
+    pull in sibling files from the folder(s) whose name/contents best match the query,
+    not only files whose own content matched. Cheap (pure path manipulation, no model),
+    so on by default.
+    """
+
+    enabled: bool = True
+    hops: int = 1  # folder hops to expand from a matched folder (parent/children/siblings)
+    max_folders: int = 5  # top-scoring folders to expand into file candidates per query
+
+
+# Documents indexed by content (and, where supported, OCR). Code/config files are not
+# included by default — set include = ["**/*"] to index everything.
+DEFAULT_DOC_TYPES = [
+    "pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "rtf",
+    "odt", "ods", "odp", "txt", "md", "markdown", "rst", "tex", "csv", "tsv",
+    "json", "jsonl",
+]
+# Images: indexed by file name / path (content only if OCR is enabled).
+DEFAULT_IMAGE_TYPES = [
+    "png", "jpg", "jpeg", "gif", "bmp", "tif", "tiff", "webp", "svg", "heic", "heif",
+]
+DEFAULT_INCLUDE = [f"**/*.{ext}" for ext in DEFAULT_DOC_TYPES + DEFAULT_IMAGE_TYPES]
+
+# Skip dependency / VCS / cache trees and editor/OS junk during the walk.
+DEFAULT_EXCLUDE = [
+    ".infogrep/**", "**/.git/**", "**/node_modules/**",
+    "**/.venv/**", "**/venv/**", "**/site-packages/**", "**/__pycache__/**",
+    "**/.cache/**", "**/.tox/**", "**/.mypy_cache/**", "**/.pytest_cache/**",
+    "**/.Trash/**", "**/~$*", "**/.dropbox.cache/**",
+]
+
+
+@dataclass
+class Config:
+    """Top-level InfoGrep configuration for one indexed directory."""
+
+    target_dir: Path
+    # Documents + images by default; set include = ["**/*"] to index every file.
+    include: list[str] = field(default_factory=lambda: list(DEFAULT_INCLUDE))
+    exclude: list[str] = field(default_factory=lambda: list(DEFAULT_EXCLUDE))
+    chunk: ChunkConfig = field(default_factory=ChunkConfig)
+    ingest: IngestConfig = field(default_factory=IngestConfig)
+    sparse: SparseConfig = field(default_factory=SparseConfig)
+    dense: DenseConfig = field(default_factory=DenseConfig)
+    kb: KnowledgeBaseConfig = field(default_factory=KnowledgeBaseConfig)
+    graph: GraphConfig = field(default_factory=GraphConfig)
+
+    @property
+    def index_dir(self) -> Path:
+        """Where this directory's index lives — a separate location, not in the target."""
+        return index_dir_for(self.target_dir)
+
+    @property
+    def manifest_path(self) -> Path:
+        return self.index_dir / "manifest.sqlite"
+
+    @property
+    def sparse_dir(self) -> Path:
+        return self.index_dir / "sparse"
+
+    @property
+    def dense_dir(self) -> Path:
+        return self.index_dir / "dense"
+
+    @property
+    def cache_dir(self) -> Path:
+        return self.index_dir / "cache"
+
+    @property
+    def graph_vault_dir(self) -> Path:
+        """Obsidian-compatible vault of folder notes (metadata graph), for browsing."""
+        return self.index_dir / "graph_vault"
+
+    @property
+    def graph_json_path(self) -> Path:
+        """Compact JSON form of the same graph, read directly by the graph retriever."""
+        return self.index_dir / "graph.json"
+
+    @classmethod
+    def load(cls, target_dir: str | Path) -> "Config":
+        """Load config for ``target_dir`` (global config.toml, then per-index override)."""
+        target = Path(target_dir).expanduser().resolve()
+        cfg = cls(target_dir=target)
+        for config_file in (index_home() / "config.toml", cfg.index_dir / "config.toml"):
+            if config_file.is_file():
+                with config_file.open("rb") as fh:
+                    cfg = cls._merge(cfg, tomllib.load(fh))
+        return cfg
+
+    @staticmethod
+    def _merge(base: "Config", data: dict) -> "Config":
+        """Shallow-merge a parsed TOML dict onto a default Config."""
+        for key in ("include", "exclude"):
+            if key in data:
+                setattr(base, key, list(data[key]))
+        if "chunk" in data:
+            base.chunk = ChunkConfig(**{**asdict(base.chunk), **data["chunk"]})
+        if "ingest" in data:
+            base.ingest = IngestConfig(**{**asdict(base.ingest), **data["ingest"]})
+        if "sparse" in data:
+            base.sparse = SparseConfig(**{**asdict(base.sparse), **data["sparse"]})
+        if "dense" in data:
+            base.dense = DenseConfig(**{**asdict(base.dense), **data["dense"]})
+        if "kb" in data:
+            base.kb = KnowledgeBaseConfig(**{**asdict(base.kb), **data["kb"]})
+        if "graph" in data:
+            base.graph = GraphConfig(**{**asdict(base.graph), **data["graph"]})
+        return base

infogrep/engine.py

@@ -0,0 +1,166 @@
+"""Search engine: the shared core behind both the CLI and the MCP server.
+
+Owns the retrievers for one indexed directory, runs them individually or fused (RRF),
+and degrades gracefully when a backend's index is missing or a backend errors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .config import Config
+from .retrieval.base import Result, with_file_metadata
+from .retrieval.fusion import reciprocal_rank_fusion
+
+# Per-retriever candidate pool size for fusion (>= k so RRF has material to work with).
+_POOL_MIN = 20
+
+ALL_RETRIEVERS = ("sparse", "dense", "kb", "graph")
+
+
+@dataclass
+class HybridResults:
+    """Fused results plus which retrievers actually contributed / were skipped."""
+
+    results: list[Result]
+    used: list[str] = field(default_factory=list)
+    skipped: dict[str, str] = field(default_factory=dict)  # retriever -> reason
+
+
+class SearchEngine:
+    def __init__(self, config: Config):
+        self.config = config
+        self._sparse = None
+        self._dense = None
+        self._kb = None
+        self._graph = None
+
+    # -- lazy backends -----------------------------------------------------
+
+    @property
+    def sparse(self):
+        if self._sparse is None:
+            from .retrieval.sparse import SparseIndex
+
+            self._sparse = SparseIndex(
+                self.config.sparse_dir,
+                self.config.cache_dir,
+                field_boosts=self.config.sparse.field_boosts,
+                language=self.config.sparse.language,
+                prf_fb_docs=self.config.sparse.prf_fb_docs,
+                prf_fb_terms=self.config.sparse.prf_fb_terms,
+            )
+        return self._sparse
+
+    @property
+    def dense(self):
+        if self._dense is None:
+            from .retrieval.dense import DenseIndex
+
+            self._dense = DenseIndex(self.config)
+        return self._dense
+
+    @property
+    def kb(self):
+        if self._kb is None:
+            from .retrieval.kb import KnowledgeBaseIndex
+
+            self._kb = KnowledgeBaseIndex(self.config)
+        return self._kb
+
+    @property
+    def graph(self):
+        if self._graph is None:
+            from .retrieval.graph import FolderGraphIndex
+
+            self._graph = FolderGraphIndex(
+                self.config.index_dir,
+                hops=self.config.graph.hops,
+                max_folders=self.config.graph.max_folders,
+            )
+        return self._graph
+
+    # -- individual retrievers --------------------------------------------
+
+    def _enrich(self, results: list[Result], root) -> list[Result]:
+        """Attach the original file path + metadata to each result."""
+        return [with_file_metadata(r, root) for r in results]
+
+    def search_sparse(self, query: str, k: int = 10, prf: bool = False) -> list[Result]:
+        # Content-file retrievers: paths are relative to the indexed directory.
+        return self._enrich(self.sparse.search(query, k=k, prf=prf), self.config.target_dir)
+
+    def search_dense(self, query: str, k: int = 10) -> list[Result]:
+        return self._enrich(self.dense.search(query, k=k), self.config.target_dir)
+
+    def search_kb(self, query: str, k: int = 10) -> list[Result]:
+        # KB paths are vault-relative; we have the vault name (CLI target), not its
+        # filesystem root, so set filename/ext only (root=None leaves abs_path unset).
+        return self._enrich(self.kb.search(query, k=k), None)
+
+    def search_graph(self, query: str, k: int = 10) -> list[Result]:
+        # Graph paths reference real files in the indexed directory, just like sparse/dense.
+        return self._enrich(self.graph.search(query, k=k), self.config.target_dir)
+
+    def _run(self, name: str, query: str, k: int, prf: bool) -> list[Result]:
+        if name == "sparse":
+            return self.search_sparse(query, k=k, prf=prf)
+        if name == "dense":
+            return self.search_dense(query, k=k)
+        if name == "kb":
+            return self.search_kb(query, k=k)
+        if name == "graph":
+            return self.search_graph(query, k=k)
+        raise ValueError(f"unknown retriever: {name}")
+
+    def _enabled(self, name: str) -> bool:
+        return {
+            "sparse": self.config.sparse.enabled,
+            "dense": self.config.dense.enabled,
+            "kb": self.config.kb.enabled,
+            "graph": self.config.graph.enabled,
+        }.get(name, False)
+
+    # -- fused -------------------------------------------------------------
+
+    def search_hybrid(
+        self,
+        query: str,
+        k: int = 10,
+        retrievers: list[str] | None = None,
+        prf: bool = False,
+    ) -> HybridResults:
+        names = retrievers or [r for r in ALL_RETRIEVERS if self._enabled(r)]
+        pool = max(k, _POOL_MIN)
+
+        lists: list[list[Result]] = []
+        out = HybridResults(results=[])
+        for name in names:
+            if not self._enabled(name):
+                out.skipped[name] = "disabled in config"
+                continue
+            try:
+                hits = self._run(name, query, pool, prf)
+            except FileNotFoundError as exc:
+                out.skipped[name] = str(exc)
+                continue
+            except Exception as exc:  # one backend failing shouldn't sink the query
+                out.skipped[name] = f"error: {exc}"
+                continue
+            lists.append(hits)
+            out.used.append(name)
+
+        out.results = reciprocal_rank_fusion(lists, top_n=k) if lists else []
+        return out
+
+    # -- maintenance -------------------------------------------------------
+
+    def status(self) -> dict:
+        from .indexer import Indexer
+
+        return Indexer(self.config).status()
+
+    def reindex(self, full: bool = False) -> dict:
+        from .indexer import Indexer
+
+        return Indexer(self.config).reindex(full=full).as_dict()