ragrep 0.2.1__py3-none-any.whl

ragrep/__init__.py

@@ -0,0 +1,41 @@
+"""RAGrep public package API."""
+
+from __future__ import annotations
+
+__version__ = "0.2.1"
+__author__ = "RAGrep Team"
+
+
+def __getattr__(name: str):
+    if name in {"RAGrep", "RAGSystem"}:
+        from .core.rag_system import RAGrep, RAGSystem
+
+        return RAGrep if name == "RAGrep" else RAGSystem
+
+    if name == "DocumentProcessor":
+        from .core.document_processor import DocumentProcessor
+
+        return DocumentProcessor
+
+    if name == "VectorStore":
+        from .retrieval.vector_store import VectorStore
+
+        return VectorStore
+
+    if name in {"LocalEmbedder", "OllamaEmbedder"}:
+        # Keep OllamaEmbedder alias for backward compatibility.
+        from .retrieval.embeddings import LocalEmbedder
+
+        return LocalEmbedder
+
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+    "RAGrep",
+    "RAGSystem",
+    "DocumentProcessor",
+    "VectorStore",
+    "LocalEmbedder",
+    "OllamaEmbedder",
+]

ragrep/cli.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""Command-line interface for RAGrep."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+from typing import List
+
+from .core.rag_system import RAGrep
+from .retrieval.embeddings import get_runtime_device_info
+
+
+def setup_logging(verbose: bool) -> None:
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.WARNING,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+
+def _build_common_parser(description: str) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument(
+        "--db-path",
+        default=os.getenv("RAGREP_DB_PATH", "./.ragrep.db"),
+        help="Path to local SQLite database",
+    )
+    parser.add_argument(
+        "--chunk-size",
+        type=int,
+        default=int(os.getenv("CHUNK_SIZE", "1000")),
+        help="Chunk size",
+    )
+    parser.add_argument(
+        "--chunk-overlap",
+        type=int,
+        default=int(os.getenv("CHUNK_OVERLAP", "200")),
+        help="Chunk overlap",
+    )
+    parser.add_argument(
+        "--model",
+        default=os.getenv("EMBEDDING_MODEL", "mxbai-embed-large"),
+        help="Embedding model name",
+    )
+    parser.add_argument(
+        "--model-dir",
+        default=os.getenv("RAGREP_MODEL_DIR"),
+        help="Optional directory for downloaded embedding models",
+    )
+    parser.add_argument(
+        "--device",
+        default=os.getenv("RAGREP_DEVICE", "auto"),
+        help="Embedding device: auto, cpu, cuda, mps, or explicit device (e.g. cuda:0)",
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Enable debug logging")
+    return parser
+
+
+def _build_recall_parser(prog: str = "ragrep") -> argparse.ArgumentParser:
+    parser = _build_common_parser("Recall relevant chunks (auto-indexes when files changed)")
+    parser.prog = prog
+    parser.add_argument("query", nargs="+", help="Semantic query")
+    parser.add_argument(
+        "--path",
+        default=None,
+        help="Directory or file to index when needed (defaults to existing indexed root, else current dir)",
+    )
+    parser.add_argument("--limit", type=int, default=20, help="Maximum number of results")
+    parser.add_argument("--no-auto-index", action="store_true", help="Disable automatic index updates")
+    parser.add_argument("--json", action="store_true", help="Output JSON")
+    return parser
+
+
+def _build_index_parser() -> argparse.ArgumentParser:
+    parser = _build_common_parser("Index a directory or file")
+    parser.add_argument("path", nargs="?", default=".", help="Path to index")
+    parser.add_argument("--force", action="store_true", help="Force a full re-index")
+    parser.add_argument("--json", action="store_true", help="Output JSON")
+    return parser
+
+
+def _build_stats_parser() -> argparse.ArgumentParser:
+    parser = _build_common_parser("Show index statistics")
+    parser.add_argument("--json", action="store_true", help="Output JSON")
+    return parser
+
+
+def _build_gpu_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Show GPU/device support for embeddings")
+    parser.add_argument(
+        "--device",
+        default=os.getenv("RAGREP_DEVICE", "auto"),
+        help="Requested embedding device: auto, cpu, cuda, mps, cuda:0, etc.",
+    )
+    parser.add_argument("--json", action="store_true", help="Output JSON")
+    return parser
+
+
+def _run_gpu_info(args: argparse.Namespace) -> int:
+    info = get_runtime_device_info(args.device)
+    if args.json:
+        print(json.dumps(info, indent=2))
+    else:
+        print(f"Requested: {info['requested_device']}")
+        print(f"Resolved: {info['resolved_device']}")
+        print(f"PyTorch available: {info['torch_available']}")
+        print(f"CUDA available: {info['cuda_available']}")
+        print(f"CUDA device count: {info['cuda_device_count']}")
+        if info["cuda_devices"]:
+            print("CUDA devices:")
+            for index, name in enumerate(info["cuda_devices"]):
+                print(f"  {index}: {name}")
+        print(f"MPS available: {info['mps_available']}")
+    return 0
+
+
+def _print_new_file_paths(index_result: dict) -> None:
+    new_files = index_result.get("new_files") or []
+    if not new_files:
+        return
+    print("New files indexed:")
+    for path in new_files:
+        print(path)
+
+
+def _run_recall(args: argparse.Namespace) -> int:
+    setup_logging(args.verbose)
+    query = " ".join(args.query).strip()
+
+    with RAGrep(
+        db_path=args.db_path,
+        chunk_size=args.chunk_size,
+        chunk_overlap=args.chunk_overlap,
+        embedding_model=args.model,
+        model_dir=args.model_dir,
+        embedding_device=args.device,
+    ) as rag:
+        result = rag.recall(
+            query,
+            limit=args.limit,
+            path=args.path,
+            auto_index=not args.no_auto_index,
+        )
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+        return 0
+
+    index_info = result.get("auto_index")
+    if index_info and index_info.get("indexed"):
+        _print_new_file_paths(index_info)
+        print(
+            f"Indexed {index_info['indexed_files']} changed files "
+            f"({index_info['chunks_indexed']} chunks updated, {index_info['chunks']} total): "
+            f"{index_info['reason']}"
+        )
+
+    matches = result.get("matches", [])
+    print(f"Results: {len(matches)}")
+    for position, match in enumerate(matches, start=1):
+        source = match.get("metadata", {}).get("source", "unknown")
+        print(f"{position}. score={match['score']:.4f} source={source}")
+        print(match.get("text", "").rstrip())
+
+    return 0
+
+
+def _run_index(args: argparse.Namespace) -> int:
+    setup_logging(args.verbose)
+
+    with RAGrep(
+        db_path=args.db_path,
+        chunk_size=args.chunk_size,
+        chunk_overlap=args.chunk_overlap,
+        embedding_model=args.model,
+        model_dir=args.model_dir,
+        embedding_device=args.device,
+    ) as rag:
+        result = rag.index(path=args.path, force=args.force)
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+        return 0
+
+    if result["indexed"]:
+        _print_new_file_paths(result)
+        print(
+            f"Indexed {result['indexed_files']} changed files "
+            f"({result['chunks_indexed']} chunks updated, {result['chunks']} total)"
+        )
+    else:
+        print(f"Index unchanged: {result['reason']}")
+
+    return 0
+
+
+def _run_stats(args: argparse.Namespace) -> int:
+    setup_logging(args.verbose)
+
+    with RAGrep(
+        db_path=args.db_path,
+        chunk_size=args.chunk_size,
+        chunk_overlap=args.chunk_overlap,
+        embedding_model=args.model,
+        model_dir=args.model_dir,
+        embedding_device=args.device,
+    ) as rag:
+        result = rag.stats()
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(f"Database: {result['persist_path']}")
+        print(f"Indexed root: {result.get('indexed_root')}")
+        print(f"Embedding model: {result.get('embedding_model')}")
+        print(f"Files: {result['total_files']}")
+        print(f"Chunks: {result['total_chunks']}")
+        print(f"Indexed at: {result.get('indexed_at')}")
+
+    return 0
+
+
+def main(argv: List[str] | None = None) -> int:
+    args_list = list(argv) if argv is not None else sys.argv[1:]
+
+    if not args_list:
+        parser = _build_recall_parser()
+        parser.print_help()
+        return 0
+
+    try:
+        first = args_list[0]
+        if first in {"--check-gpu", "--gpu-info"}:
+            parser = _build_gpu_parser()
+            args = parser.parse_args(args_list[1:])
+            return _run_gpu_info(args)
+
+        if first in {"--stats", "-s"}:
+            parser = _build_stats_parser()
+            args = parser.parse_args(args_list[1:])
+            return _run_stats(args)
+
+        if first == "index":
+            parser = _build_index_parser()
+            args = parser.parse_args(args_list[1:])
+            return _run_index(args)
+
+        if first == "stats":
+            parser = _build_stats_parser()
+            args = parser.parse_args(args_list[1:])
+            return _run_stats(args)
+
+        if first == "recall":
+            parser = _build_recall_parser("ragrep recall")
+            args = parser.parse_args(args_list[1:])
+            return _run_recall(args)
+
+        parser = _build_recall_parser()
+        args = parser.parse_args(args_list)
+        return _run_recall(args)
+    except Exception as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ragrep/core/__init__.py

@@ -0,0 +1,8 @@
+"""Core RAGrep functionality."""
+
+from __future__ import annotations
+
+from .document_processor import DocumentProcessor
+from .rag_system import RAGrep, RAGSystem
+
+__all__ = ["RAGrep", "RAGSystem", "DocumentProcessor"]

ragrep/core/document_processor.py

@@ -0,0 +1,242 @@
+"""Document loading and chunking utilities."""
+
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+from typing import Any, Dict, Iterable, List
+
+
+_DEFAULT_EXTENSIONS = {
+    ".c",
+    ".cc",
+    ".cpp",
+    ".css",
+    ".go",
+    ".h",
+    ".hpp",
+    ".html",
+    ".java",
+    ".js",
+    ".json",
+    ".md",
+    ".py",
+    ".rb",
+    ".rs",
+    ".sql",
+    ".toml",
+    ".ts",
+    ".txt",
+    ".xml",
+    ".yaml",
+    ".yml",
+}
+
+_DEFAULT_IGNORE_PATTERNS = {
+    ".git/",
+    "__pycache__/",
+    "*.pyc",
+    "*.sqlite",
+    "*.sqlite3",
+    "*.db",
+    ".ragrep.db",
+    ".ragrep.db.legacy/",
+    "venv/",
+    ".venv/",
+    "node_modules/",
+    "dist/",
+    "build/",
+}
+
+
+class DocumentProcessor:
+    """Read text files from disk and split them into overlapping chunks."""
+
+    def __init__(self, chunk_size: int = 1000, chunk_overlap: int = 200) -> None:
+        if chunk_size <= 0:
+            raise ValueError("chunk_size must be > 0")
+        if chunk_overlap < 0:
+            raise ValueError("chunk_overlap must be >= 0")
+        if chunk_overlap >= chunk_size:
+            raise ValueError("chunk_overlap must be smaller than chunk_size")
+
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+
+    def process_path(
+        self,
+        path: str,
+        *,
+        extra_ignore_paths: Iterable[Path] | None = None,
+    ) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]], Path]:
+        files, file_records, scan_root = self.discover_path(
+            path,
+            extra_ignore_paths=extra_ignore_paths,
+        )
+        chunks = self.process_files(files, scan_root)
+        return chunks, file_records, scan_root
+
+    def discover_path(
+        self,
+        path: str,
+        *,
+        extra_ignore_paths: Iterable[Path] | None = None,
+    ) -> tuple[List[Path], List[Dict[str, Any]], Path]:
+        root = Path(path).resolve()
+        if not root.exists():
+            raise ValueError(f"Path does not exist: {path}")
+
+        if root.is_file():
+            files = [root]
+            scan_root = root.parent
+        else:
+            scan_root = root
+            files = self.scan_files(scan_root, extra_ignore_paths=extra_ignore_paths)
+
+        file_records = self.collect_file_records(files, scan_root)
+        return files, file_records, scan_root
+
+    def process_files(self, files: Iterable[Path], scan_root: Path) -> List[Dict[str, Any]]:
+        chunks: List[Dict[str, Any]] = []
+        for file_path in files:
+            relative_path = file_path.relative_to(scan_root).as_posix()
+            text = self._load_text(file_path)
+            chunks.extend(self._chunk_text(text, relative_path))
+        return chunks
+
+    def scan_files(
+        self,
+        root: Path,
+        *,
+        extra_ignore_paths: Iterable[Path] | None = None,
+    ) -> List[Path]:
+        ignore_patterns = self._load_ignore_patterns(root)
+        resolved_ignores = [path.expanduser().resolve() for path in (extra_ignore_paths or [])]
+        files: List[Path] = []
+
+        for file_path in root.rglob("*"):
+            if not file_path.is_file():
+                continue
+            if self._matches_extra_ignore(file_path.resolve(), resolved_ignores):
+                continue
+            if file_path.suffix.lower() not in _DEFAULT_EXTENSIONS:
+                continue
+
+            relative = file_path.relative_to(root).as_posix()
+            if self._should_ignore(relative, ignore_patterns):
+                continue
+
+            files.append(file_path)
+
+        files.sort()
+        return files
+
+    def collect_file_records(self, files: Iterable[Path], root: Path) -> List[Dict[str, Any]]:
+        records: List[Dict[str, Any]] = []
+
+        for file_path in files:
+            stat = file_path.stat()
+            records.append(
+                {
+                    "path": file_path.relative_to(root).as_posix(),
+                    "size": int(stat.st_size),
+                    "mtime_ns": int(stat.st_mtime_ns),
+                }
+            )
+
+        records.sort(key=lambda item: item["path"])
+        return records
+
+    def _chunk_text(self, text: str, relative_path: str) -> List[Dict[str, Any]]:
+        chunks: List[Dict[str, Any]] = []
+        start = 0
+
+        while start < len(text):
+            end = min(start + self.chunk_size, len(text))
+            chunk_text = text[start:end]
+            chunk_index = len(chunks)
+            chunk_id = f"{relative_path}:{chunk_index}:{start}:{end}"
+
+            metadata = {
+                "source": relative_path,
+                "chunk_index": chunk_index,
+                "start_char": start,
+                "end_char": end,
+            }
+            chunks.append(
+                {
+                    "id": chunk_id,
+                    "file_path": relative_path,
+                    "chunk_index": chunk_index,
+                    "start_char": start,
+                    "end_char": end,
+                    "text": chunk_text,
+                    "metadata": metadata,
+                }
+            )
+
+            if end >= len(text):
+                break
+            start = end - self.chunk_overlap
+
+        return chunks
+
+    @staticmethod
+    def _load_text(file_path: Path) -> str:
+        return file_path.read_text(encoding="utf-8", errors="ignore")
+
+    @staticmethod
+    def _load_ignore_patterns(root: Path) -> List[str]:
+        patterns = set(_DEFAULT_IGNORE_PATTERNS)
+
+        current = root
+        while True:
+            gitignore = current / ".gitignore"
+            if gitignore.exists():
+                for line in gitignore.read_text(encoding="utf-8", errors="ignore").splitlines():
+                    stripped = line.strip()
+                    if not stripped or stripped.startswith("#"):
+                        continue
+                    patterns.add(stripped)
+            if current.parent == current:
+                break
+            current = current.parent
+
+        return sorted(patterns)
+
+    @staticmethod
+    def _should_ignore(relative_path: str, patterns: Iterable[str]) -> bool:
+        path = relative_path
+
+        for pattern in patterns:
+            if pattern.startswith("!"):
+                continue
+
+            normalized = pattern.strip()
+            if not normalized:
+                continue
+
+            if normalized.endswith("/"):
+                directory = normalized.rstrip("/")
+                if path == directory or path.startswith(directory + "/"):
+                    return True
+                continue
+
+            if fnmatch.fnmatch(path, normalized):
+                return True
+            if fnmatch.fnmatch(Path(path).name, normalized):
+                return True
+
+        return False
+
+    @staticmethod
+    def _matches_extra_ignore(file_path: Path, ignored_paths: Iterable[Path]) -> bool:
+        for ignored in ignored_paths:
+            if file_path == ignored:
+                return True
+            try:
+                file_path.relative_to(ignored)
+                return True
+            except ValueError:
+                continue
+        return False