rapid-rag 0.1.0__tar.gz

rapid_rag-0.1.0/.gitignore

@@ -0,0 +1,74 @@
+# Secrets & env
+.env
+*.env
+*.secret
+
+# Keys & certs
+*.key
+*.pem
+certs/
+secrets/
+
+# Databases & dumps
+*.db
+*.sqlite
+*.sql
+dump_*/
+
+# EXCEPT: Allow database schemas (needed for server rebuild)
+!database-schemas/*.sql
+
+# Logs & runtime data
+logs/
+*.log
+__pycache__/
+*.pyc
+venv/
+.venv/
+**/venv/
+**/.venv/
+
+# Configs met secrets (we gebruiken straks templates)
+config/
+brain_api/provisioning.local.json
+brain_api/provisioning.json
+
+# Landing pages (privé - niet open source)
+landing-pages/
+humotica.com/
+jtel.nl/
+
+# Social media posts (strategie - niet open source)
+SOCIAL-MEDIA-POSTS.md
+HN-POST-UNDER-4000.md
+STRATO-DEPLOY-HUMOTICA.md
+
+# Endorsement outreach (privaat contact)
+ARXIV-ENDORSEMENT-OUTREACH.md
+
+# Deployment secrets
+DEPLOYMENT-GUIDE.md
+
+# R Project files (Dirty Data Challenge)
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+*.zip
+.mural_tokens.json
+auth.json
+gen-lang-client*.json
+*.credentials.json
+
+# Rust build artifacts
+**/target/
+*.whl
+
+# Compiled binaries (build locally)
+jis-router/jis-router
+sentinel-rs/sentinel-rs
+
+# Build distribution
+sandbox/ai/codex/dist/
+sandbox_backup/
+did-jis-core

rapid_rag-0.1.0/PKG-INFO

@@ -0,0 +1,119 @@
+Metadata-Version: 2.4
+Name: rapid-rag
+Version: 0.1.0
+Summary: Fast local RAG - search your documents with AI, no cloud needed
+Project-URL: Homepage, https://humotica.com
+Project-URL: Repository, https://github.com/humotica/rapid-rag
+Project-URL: Documentation, https://humotica.com/docs/rapid-rag
+Author-email: "J. van de Meent" <jasper@humotica.com>, "R. AI" <info@humotica.com>
+License: MIT
+Keywords: ai,augmented,chromadb,documents,embeddings,generation,llm,local,offline,rag,retrieval,search,semantic-search,vector-search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.10
+Requires-Dist: chromadb>=0.4.0
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: sentence-transformers>=2.2.0
+Provides-Extra: all
+Requires-Dist: fastapi>=0.100.0; extra == 'all'
+Requires-Dist: ollama>=0.1.0; extra == 'all'
+Requires-Dist: pdfplumber>=0.9.0; extra == 'all'
+Requires-Dist: pypdf>=3.0.0; extra == 'all'
+Requires-Dist: uvicorn>=0.22.0; extra == 'all'
+Provides-Extra: api
+Requires-Dist: fastapi>=0.100.0; extra == 'api'
+Requires-Dist: uvicorn>=0.22.0; extra == 'api'
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: ollama
+Requires-Dist: ollama>=0.1.0; extra == 'ollama'
+Provides-Extra: pdf
+Requires-Dist: pdfplumber>=0.9.0; extra == 'pdf'
+Requires-Dist: pypdf>=3.0.0; extra == 'pdf'
+Description-Content-Type: text/markdown
+
+# rapid-rag
+
+Fast local RAG - search your documents with AI, no cloud needed.
+
+## Installation
+
+```bash
+pip install rapid-rag
+```
+
+For PDF support:
+```bash
+pip install rapid-rag[pdf]
+```
+
+## Quick Start
+
+```python
+from rapid_rag import RapidRAG
+
+# Create a RAG instance
+rag = RapidRAG("my_documents")
+
+# Add documents
+rag.add("doc1", "The quick brown fox jumps over the lazy dog.")
+rag.add_file("report.pdf")
+rag.add_directory("./docs/")
+
+# Semantic search
+results = rag.search("fox jumping")
+for r in results:
+    print(f"{r['score']:.3f}: {r['content'][:100]}")
+
+# RAG query with LLM (requires Ollama)
+answer = rag.query("What does the fox do?", model="qwen2.5:7b")
+print(answer["answer"])
+```
+
+## CLI Usage
+
+```bash
+# Initialize a collection
+rapid-rag init my_docs
+
+# Add documents
+rapid-rag add ./documents/ -c my_docs -r
+
+# Search
+rapid-rag search "query here" -c my_docs
+
+# RAG query (requires Ollama)
+rapid-rag query "What is X?" -c my_docs -m qwen2.5:7b
+
+# Info
+rapid-rag info -c my_docs
+```
+
+## Features
+
+- **Local-first**: Everything runs on your machine
+- **Fast**: ChromaDB + sentence-transformers
+- **Simple API**: Add, search, query in 3 lines
+- **File support**: .txt, .md, .pdf
+- **Chunking**: Automatic with overlap
+- **LLM integration**: Works with Ollama
+
+## Requirements
+
+- Python 3.10+
+- For LLM queries: [Ollama](https://ollama.ai) running locally
+
+## License
+
+MIT - Humotica

rapid_rag-0.1.0/README.md

@@ -0,0 +1,74 @@
+# rapid-rag
+
+Fast local RAG - search your documents with AI, no cloud needed.
+
+## Installation
+
+```bash
+pip install rapid-rag
+```
+
+For PDF support:
+```bash
+pip install rapid-rag[pdf]
+```
+
+## Quick Start
+
+```python
+from rapid_rag import RapidRAG
+
+# Create a RAG instance
+rag = RapidRAG("my_documents")
+
+# Add documents
+rag.add("doc1", "The quick brown fox jumps over the lazy dog.")
+rag.add_file("report.pdf")
+rag.add_directory("./docs/")
+
+# Semantic search
+results = rag.search("fox jumping")
+for r in results:
+    print(f"{r['score']:.3f}: {r['content'][:100]}")
+
+# RAG query with LLM (requires Ollama)
+answer = rag.query("What does the fox do?", model="qwen2.5:7b")
+print(answer["answer"])
+```
+
+## CLI Usage
+
+```bash
+# Initialize a collection
+rapid-rag init my_docs
+
+# Add documents
+rapid-rag add ./documents/ -c my_docs -r
+
+# Search
+rapid-rag search "query here" -c my_docs
+
+# RAG query (requires Ollama)
+rapid-rag query "What is X?" -c my_docs -m qwen2.5:7b
+
+# Info
+rapid-rag info -c my_docs
+```
+
+## Features
+
+- **Local-first**: Everything runs on your machine
+- **Fast**: ChromaDB + sentence-transformers
+- **Simple API**: Add, search, query in 3 lines
+- **File support**: .txt, .md, .pdf
+- **Chunking**: Automatic with overlap
+- **LLM integration**: Works with Ollama
+
+## Requirements
+
+- Python 3.10+
+- For LLM queries: [Ollama](https://ollama.ai) running locally
+
+## License
+
+MIT - Humotica

rapid_rag-0.1.0/pyproject.toml

@@ -0,0 +1,60 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "rapid-rag"
+version = "0.1.0"
+description = "Fast local RAG - search your documents with AI, no cloud needed"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.10"
+authors = [
+    {name = "J. van de Meent", email = "jasper@humotica.com"},
+    {name = "R. AI", email = "info@humotica.com"},
+]
+keywords = [
+    "rag", "retrieval", "augmented", "generation", "local",
+    "chromadb", "embeddings", "search", "documents", "ai",
+    "vector-search", "semantic-search", "llm", "offline"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing :: Indexing",
+]
+dependencies = [
+    "chromadb>=0.4.0",
+    "sentence-transformers>=2.2.0",
+    "httpx>=0.24.0",
+]
+
+[project.optional-dependencies]
+api = ["fastapi>=0.100.0", "uvicorn>=0.22.0"]
+pdf = ["pypdf>=3.0.0", "pdfplumber>=0.9.0"]
+ollama = ["ollama>=0.1.0"]
+all = ["rapid-rag[api,pdf,ollama]"]
+dev = ["pytest>=7.0", "ruff>=0.1.0"]
+
+[project.urls]
+Homepage = "https://humotica.com"
+Repository = "https://github.com/humotica/rapid-rag"
+Documentation = "https://humotica.com/docs/rapid-rag"
+
+[project.scripts]
+rapid-rag = "rapid_rag.cli:main"
+
+[tool.hatch.build.targets.sdist]
+include = ["/src"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/rapid_rag"]

rapid_rag-0.1.0/src/rapid_rag/__init__.py

@@ -0,0 +1,27 @@
+"""
+rapid-rag: Fast local RAG - search your documents with AI, no cloud needed.
+
+Usage:
+    from rapid_rag import RapidRAG
+
+    # Create a RAG instance
+    rag = RapidRAG("my_documents")
+
+    # Add documents
+    rag.add("doc1", "The quick brown fox jumps over the lazy dog.")
+    rag.add_file("report.pdf")
+    rag.add_directory("./docs/")
+
+    # Search
+    results = rag.search("fox jumping")
+
+    # RAG query (with LLM)
+    answer = rag.query("What does the fox do?", model="qwen2.5:7b")
+"""
+
+from .core import RapidRAG
+from .ingest import DocumentIngester
+from .search import SemanticSearch
+
+__version__ = "0.1.0"
+__all__ = ["RapidRAG", "DocumentIngester", "SemanticSearch"]

rapid_rag-0.1.0/src/rapid_rag/cli.py

@@ -0,0 +1,150 @@
+"""
+rapid-rag CLI - Command line interface for local RAG.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        prog="rapid-rag",
+        description="Fast local RAG - search your documents with AI"
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Commands")
+
+    # Init command
+    init_parser = subparsers.add_parser("init", help="Initialize a new RAG collection")
+    init_parser.add_argument("name", help="Collection name")
+    init_parser.add_argument("--dir", help="Persist directory", default=None)
+
+    # Add command
+    add_parser = subparsers.add_parser("add", help="Add documents")
+    add_parser.add_argument("path", help="File or directory to add")
+    add_parser.add_argument("-c", "--collection", default="default", help="Collection name")
+    add_parser.add_argument("-r", "--recursive", action="store_true", help="Recursive directory scan")
+    add_parser.add_argument("--chunk-size", type=int, default=1000, help="Chunk size")
+    add_parser.add_argument("--chunk-overlap", type=int, default=200, help="Chunk overlap")
+
+    # Search command
+    search_parser = subparsers.add_parser("search", help="Semantic search")
+    search_parser.add_argument("query", help="Search query")
+    search_parser.add_argument("-c", "--collection", default="default", help="Collection name")
+    search_parser.add_argument("-n", "--num", type=int, default=5, help="Number of results")
+
+    # Query command (RAG with LLM)
+    query_parser = subparsers.add_parser("query", help="RAG query with LLM")
+    query_parser.add_argument("question", help="Question to answer")
+    query_parser.add_argument("-c", "--collection", default="default", help="Collection name")
+    query_parser.add_argument("-m", "--model", default="qwen2.5:7b", help="Ollama model")
+    query_parser.add_argument("-n", "--num", type=int, default=5, help="Context documents")
+
+    # Info command
+    info_parser = subparsers.add_parser("info", help="Collection info")
+    info_parser.add_argument("-c", "--collection", default="default", help="Collection name")
+
+    # Clear command
+    clear_parser = subparsers.add_parser("clear", help="Clear collection")
+    clear_parser.add_argument("-c", "--collection", default="default", help="Collection name")
+    clear_parser.add_argument("-y", "--yes", action="store_true", help="Skip confirmation")
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        return 0
+
+    # Import here to avoid slow startup
+    from .core import RapidRAG
+
+    if args.command == "init":
+        rag = RapidRAG(args.name, persist_dir=args.dir)
+        print(f"Initialized collection '{args.name}'")
+        print(f"Persist dir: {rag.persist_dir}")
+        return 0
+
+    if args.command == "add":
+        rag = RapidRAG(args.collection)
+        path = Path(args.path)
+
+        if path.is_file():
+            ids = rag.add_file(
+                path,
+                chunk_size=args.chunk_size,
+                chunk_overlap=args.chunk_overlap
+            )
+            print(f"Added {len(ids)} chunks from {path.name}")
+        elif path.is_dir():
+            ids = rag.add_directory(
+                path,
+                recursive=args.recursive,
+                chunk_size=args.chunk_size,
+                chunk_overlap=args.chunk_overlap
+            )
+            print(f"Added {len(ids)} chunks total")
+        else:
+            print(f"Path not found: {path}", file=sys.stderr)
+            return 1
+
+        return 0
+
+    if args.command == "search":
+        rag = RapidRAG(args.collection)
+        results = rag.search(args.query, n_results=args.num)
+
+        if not results:
+            print("No results found.")
+            return 0
+
+        for i, r in enumerate(results, 1):
+            score = r["score"]
+            source = r["metadata"].get("source", r["id"])
+            content = r["content"][:200].replace("\n", " ")
+            print(f"\n[{i}] {source} (score: {score:.3f})")
+            print(f"    {content}...")
+
+        return 0
+
+    if args.command == "query":
+        rag = RapidRAG(args.collection)
+        result = rag.query(args.question, n_results=args.num, model=args.model)
+
+        print("\n" + "=" * 60)
+        print("ANSWER:")
+        print("=" * 60)
+        print(result["answer"])
+        print("\n" + "-" * 60)
+        print("SOURCES:")
+        for s in result["sources"]:
+            source = s["metadata"].get("source", s["id"])
+            print(f"  - {source} (score: {s['score']:.3f})")
+
+        return 0
+
+    if args.command == "info":
+        rag = RapidRAG(args.collection)
+        print(f"Collection: {args.collection}")
+        print(f"Documents: {rag.count()}")
+        print(f"Persist dir: {rag.persist_dir}")
+        print(f"Embedding model: {rag.embedding_model}")
+        return 0
+
+    if args.command == "clear":
+        if not args.yes:
+            confirm = input(f"Clear collection '{args.collection}'? [y/N] ")
+            if confirm.lower() != "y":
+                print("Cancelled.")
+                return 0
+
+        rag = RapidRAG(args.collection)
+        rag.clear()
+        print(f"Cleared collection '{args.collection}'")
+        return 0
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rapid_rag-0.1.0/src/rapid_rag/core.py

@@ -0,0 +1,417 @@
+"""
+Core RapidRAG class - the main interface for local RAG.
+"""
+
+import os
+import hashlib
+from pathlib import Path
+from typing import Optional, List, Dict, Any, Union
+from datetime import datetime
+
+import chromadb
+from chromadb.config import Settings
+
+# Optional imports
+try:
+    import httpx
+    HTTPX_AVAILABLE = True
+except ImportError:
+    HTTPX_AVAILABLE = False
+
+
+class RapidRAG:
+    """
+    Fast local RAG - search your documents with AI, no cloud needed.
+
+    Example:
+        rag = RapidRAG("my_project")
+        rag.add("doc1", "Some text content")
+        results = rag.search("query")
+        answer = rag.query("What is...?", model="qwen2.5:7b")
+    """
+
+    def __init__(
+        self,
+        collection_name: str = "default",
+        persist_dir: Optional[str] = None,
+        embedding_model: str = "all-MiniLM-L6-v2",
+        ollama_url: str = "http://localhost:11434",
+    ):
+        """
+        Initialize RapidRAG.
+
+        Args:
+            collection_name: Name for the document collection
+            persist_dir: Directory to persist the database (default: ./rapid_rag_data/)
+            embedding_model: Sentence-transformers model for embeddings
+            ollama_url: URL for Ollama API (for LLM queries)
+        """
+        self.collection_name = collection_name
+        self.persist_dir = persist_dir or f"./rapid_rag_data/{collection_name}"
+        self.embedding_model = embedding_model
+        self.ollama_url = ollama_url
+
+        # Create persist directory
+        os.makedirs(self.persist_dir, exist_ok=True)
+
+        # Initialize ChromaDB with persistence
+        self.client = chromadb.PersistentClient(
+            path=self.persist_dir,
+            settings=Settings(anonymized_telemetry=False)
+        )
+
+        # Get or create collection
+        self.collection = self.client.get_or_create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"}
+        )
+
+        # Lazy load embedding model
+        self._embedder = None
+
+    @property
+    def embedder(self):
+        """Lazy load sentence-transformers model."""
+        if self._embedder is None:
+            from sentence_transformers import SentenceTransformer
+            self._embedder = SentenceTransformer(self.embedding_model)
+        return self._embedder
+
+    def _generate_id(self, content: str) -> str:
+        """Generate a unique ID for content."""
+        return hashlib.sha256(content.encode()).hexdigest()[:16]
+
+    def add(
+        self,
+        doc_id: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Add a document to the collection.
+
+        Args:
+            doc_id: Unique document identifier
+            content: Text content to index
+            metadata: Optional metadata dict
+
+        Returns:
+            Document ID
+        """
+        metadata = metadata or {}
+        metadata["added_at"] = datetime.now().isoformat()
+        metadata["content_hash"] = self._generate_id(content)
+
+        # Generate embedding
+        embedding = self.embedder.encode(content).tolist()
+
+        # Add to collection
+        self.collection.add(
+            ids=[doc_id],
+            embeddings=[embedding],
+            documents=[content],
+            metadatas=[metadata]
+        )
+
+        return doc_id
+
+    def add_texts(
+        self,
+        texts: List[str],
+        ids: Optional[List[str]] = None,
+        metadatas: Optional[List[Dict]] = None
+    ) -> List[str]:
+        """
+        Add multiple documents at once (faster).
+
+        Args:
+            texts: List of text contents
+            ids: Optional list of IDs (auto-generated if not provided)
+            metadatas: Optional list of metadata dicts
+
+        Returns:
+            List of document IDs
+        """
+        if ids is None:
+            ids = [self._generate_id(t) for t in texts]
+
+        if metadatas is None:
+            metadatas = [{} for _ in texts]
+
+        # Add timestamps
+        now = datetime.now().isoformat()
+        for meta in metadatas:
+            meta["added_at"] = now
+
+        # Generate embeddings (batch)
+        embeddings = self.embedder.encode(texts).tolist()
+
+        # Add to collection
+        self.collection.add(
+            ids=ids,
+            embeddings=embeddings,
+            documents=texts,
+            metadatas=metadatas
+        )
+
+        return ids
+
+    def add_file(
+        self,
+        file_path: Union[str, Path],
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200
+    ) -> List[str]:
+        """
+        Add a file to the collection.
+
+        Supports: .txt, .md, .pdf (with pdf extra)
+
+        Args:
+            file_path: Path to file
+            chunk_size: Characters per chunk
+            chunk_overlap: Overlap between chunks
+
+        Returns:
+            List of chunk IDs
+        """
+        file_path = Path(file_path)
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Read content based on file type
+        suffix = file_path.suffix.lower()
+
+        if suffix in [".txt", ".md"]:
+            content = file_path.read_text(encoding="utf-8", errors="ignore")
+        elif suffix == ".pdf":
+            content = self._read_pdf(file_path)
+        else:
+            # Try reading as text
+            content = file_path.read_text(encoding="utf-8", errors="ignore")
+
+        # Chunk the content
+        chunks = self._chunk_text(content, chunk_size, chunk_overlap)
+
+        # Create IDs and metadata
+        ids = [f"{file_path.stem}_{i}" for i in range(len(chunks))]
+        metadatas = [
+            {
+                "source": str(file_path),
+                "chunk_index": i,
+                "total_chunks": len(chunks)
+            }
+            for i in range(len(chunks))
+        ]
+
+        return self.add_texts(chunks, ids, metadatas)
+
+    def add_directory(
+        self,
+        dir_path: Union[str, Path],
+        extensions: Optional[List[str]] = None,
+        recursive: bool = True,
+        **kwargs
+    ) -> List[str]:
+        """
+        Add all files in a directory.
+
+        Args:
+            dir_path: Path to directory
+            extensions: File extensions to include (default: [".txt", ".md", ".pdf"])
+            recursive: Search subdirectories
+            **kwargs: Passed to add_file()
+
+        Returns:
+            List of all chunk IDs
+        """
+        dir_path = Path(dir_path)
+        extensions = extensions or [".txt", ".md", ".pdf"]
+
+        all_ids = []
+        pattern = "**/*" if recursive else "*"
+
+        for ext in extensions:
+            for file_path in dir_path.glob(f"{pattern}{ext}"):
+                if file_path.is_file():
+                    try:
+                        ids = self.add_file(file_path, **kwargs)
+                        all_ids.extend(ids)
+                        print(f"✓ {file_path.name}: {len(ids)} chunks")
+                    except Exception as e:
+                        print(f"✗ {file_path.name}: {e}")
+
+        return all_ids
+
+    def search(
+        self,
+        query: str,
+        n_results: int = 5,
+        where: Optional[Dict] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Semantic search in the collection.
+
+        Args:
+            query: Search query
+            n_results: Number of results to return
+            where: Optional filter dict
+
+        Returns:
+            List of results with content, metadata, and score
+        """
+        # Generate query embedding
+        query_embedding = self.embedder.encode(query).tolist()
+
+        # Search
+        results = self.collection.query(
+            query_embeddings=[query_embedding],
+            n_results=n_results,
+            where=where,
+            include=["documents", "metadatas", "distances"]
+        )
+
+        # Format results
+        output = []
+        for i in range(len(results["ids"][0])):
+            output.append({
+                "id": results["ids"][0][i],
+                "content": results["documents"][0][i],
+                "metadata": results["metadatas"][0][i],
+                "score": 1 - results["distances"][0][i]  # Convert distance to similarity
+            })
+
+        return output
+
+    def query(
+        self,
+        question: str,
+        n_results: int = 5,
+        model: str = "qwen2.5:7b",
+        system_prompt: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        RAG query - search + LLM analysis.
+
+        Args:
+            question: Question to answer
+            n_results: Number of documents to retrieve
+            model: Ollama model to use
+            system_prompt: Optional system prompt
+
+        Returns:
+            Dict with answer and sources
+        """
+        if not HTTPX_AVAILABLE:
+            raise ImportError("httpx required for LLM queries: pip install httpx")
+
+        # Search for relevant documents
+        results = self.search(question, n_results=n_results)
+
+        if not results:
+            return {
+                "answer": "No relevant documents found.",
+                "sources": [],
+                "query": question
+            }
+
+        # Build context from results
+        context = "\n\n---\n\n".join([
+            f"[Source: {r['metadata'].get('source', r['id'])}]\n{r['content']}"
+            for r in results
+        ])
+
+        # Default system prompt
+        if system_prompt is None:
+            system_prompt = (
+                "You are a helpful assistant. Answer questions based on the provided context. "
+                "If the answer is not in the context, say so. Cite your sources."
+            )
+
+        # Build prompt
+        prompt = f"""Context:
+{context}
+
+Question: {question}
+
+Answer based on the context above:"""
+
+        # Call Ollama
+        try:
+            with httpx.Client(timeout=60.0) as client:
+                response = client.post(
+                    f"{self.ollama_url}/api/generate",
+                    json={
+                        "model": model,
+                        "prompt": prompt,
+                        "system": system_prompt,
+                        "stream": False
+                    }
+                )
+                response.raise_for_status()
+                answer = response.json().get("response", "")
+        except Exception as e:
+            answer = f"LLM error: {e}"
+
+        return {
+            "answer": answer,
+            "sources": results,
+            "query": question,
+            "model": model
+        }
+
+    def count(self) -> int:
+        """Return number of documents in collection."""
+        return self.collection.count()
+
+    def delete(self, ids: List[str]) -> None:
+        """Delete documents by ID."""
+        self.collection.delete(ids=ids)
+
+    def clear(self) -> None:
+        """Clear all documents from collection."""
+        self.client.delete_collection(self.collection_name)
+        self.collection = self.client.create_collection(
+            name=self.collection_name,
+            metadata={"hnsw:space": "cosine"}
+        )
+
+    def _chunk_text(
+        self,
+        text: str,
+        chunk_size: int,
+        chunk_overlap: int
+    ) -> List[str]:
+        """Split text into overlapping chunks."""
+        chunks = []
+        start = 0
+
+        while start < len(text):
+            end = start + chunk_size
+            chunk = text[start:end]
+
+            # Try to break at sentence boundary
+            if end < len(text):
+                last_period = chunk.rfind(". ")
+                if last_period > chunk_size // 2:
+                    chunk = chunk[:last_period + 1]
+                    end = start + last_period + 1
+
+            chunks.append(chunk.strip())
+            start = end - chunk_overlap
+
+        return [c for c in chunks if c]  # Remove empty chunks
+
+    def _read_pdf(self, file_path: Path) -> str:
+        """Read text from PDF file."""
+        try:
+            from pypdf import PdfReader
+            reader = PdfReader(str(file_path))
+            text = "\n".join(page.extract_text() or "" for page in reader.pages)
+            return text
+        except ImportError:
+            raise ImportError("PDF support requires: pip install rapid-rag[pdf]")
+
+    def __repr__(self) -> str:
+        return f"RapidRAG(collection='{self.collection_name}', docs={self.count()})"

rapid_rag-0.1.0/src/rapid_rag/ingest.py

@@ -0,0 +1,134 @@
+"""
+Document ingestion utilities for RapidRAG.
+"""
+
+from pathlib import Path
+from typing import Optional, List, Dict, Any, Generator
+import hashlib
+
+
+class DocumentIngester:
+    """
+    Ingest documents from various sources.
+
+    Example:
+        ingester = DocumentIngester(rag)
+        ingester.from_directory("./docs/")
+        ingester.from_url("https://example.com/doc.txt")
+    """
+
+    def __init__(self, rag: "RapidRAG"):
+        """
+        Initialize ingester.
+
+        Args:
+            rag: RapidRAG instance to add documents to
+        """
+        self.rag = rag
+        self._stats = {"files": 0, "chunks": 0, "errors": 0}
+
+    def from_directory(
+        self,
+        path: str,
+        extensions: Optional[List[str]] = None,
+        recursive: bool = True,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200
+    ) -> Dict[str, int]:
+        """
+        Ingest all documents from a directory.
+
+        Args:
+            path: Directory path
+            extensions: File extensions to process
+            recursive: Include subdirectories
+            chunk_size: Characters per chunk
+            chunk_overlap: Overlap between chunks
+
+        Returns:
+            Stats dict with files, chunks, errors
+        """
+        self._stats = {"files": 0, "chunks": 0, "errors": 0}
+
+        ids = self.rag.add_directory(
+            path,
+            extensions=extensions,
+            recursive=recursive,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+
+        self._stats["chunks"] = len(ids)
+        return self._stats
+
+    def from_texts(
+        self,
+        texts: List[str],
+        ids: Optional[List[str]] = None,
+        metadatas: Optional[List[Dict]] = None
+    ) -> int:
+        """
+        Ingest a list of texts.
+
+        Args:
+            texts: List of text strings
+            ids: Optional document IDs
+            metadatas: Optional metadata dicts
+
+        Returns:
+            Number of documents added
+        """
+        added_ids = self.rag.add_texts(texts, ids, metadatas)
+        return len(added_ids)
+
+    def from_jsonl(
+        self,
+        path: str,
+        content_field: str = "text",
+        id_field: Optional[str] = None,
+        metadata_fields: Optional[List[str]] = None
+    ) -> int:
+        """
+        Ingest documents from a JSONL file.
+
+        Args:
+            path: Path to JSONL file
+            content_field: Field containing text content
+            id_field: Field containing document ID
+            metadata_fields: Fields to include as metadata
+
+        Returns:
+            Number of documents added
+        """
+        import json
+
+        texts = []
+        ids = []
+        metadatas = []
+
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                doc = json.loads(line)
+                content = doc.get(content_field, "")
+                if not content:
+                    continue
+
+                texts.append(content)
+
+                if id_field and id_field in doc:
+                    ids.append(str(doc[id_field]))
+                else:
+                    ids.append(hashlib.sha256(content.encode()).hexdigest()[:16])
+
+                if metadata_fields:
+                    meta = {k: doc.get(k) for k in metadata_fields if k in doc}
+                else:
+                    meta = {k: v for k, v in doc.items() if k != content_field}
+                metadatas.append(meta)
+
+        return self.from_texts(texts, ids, metadatas)
+
+    @property
+    def stats(self) -> Dict[str, int]:
+        """Get ingestion stats."""
+        return self._stats

rapid_rag-0.1.0/src/rapid_rag/search.py

@@ -0,0 +1,115 @@
+"""
+Semantic search utilities for RapidRAG.
+"""
+
+from typing import Optional, List, Dict, Any
+
+
+class SemanticSearch:
+    """
+    Semantic search utilities.
+
+    Wraps RapidRAG.search() with additional functionality.
+    """
+
+    def __init__(self, rag: "RapidRAG"):
+        """Initialize with a RapidRAG instance."""
+        self.rag = rag
+
+    def search(
+        self,
+        query: str,
+        n_results: int = 5,
+        min_score: float = 0.0,
+        source_filter: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Search with additional filters.
+
+        Args:
+            query: Search query
+            n_results: Max results
+            min_score: Minimum similarity score (0-1)
+            source_filter: Filter by source path (contains)
+
+        Returns:
+            Filtered search results
+        """
+        results = self.rag.search(query, n_results=n_results * 2)
+
+        # Filter by score
+        results = [r for r in results if r["score"] >= min_score]
+
+        # Filter by source
+        if source_filter:
+            results = [
+                r for r in results
+                if source_filter in r["metadata"].get("source", "")
+            ]
+
+        return results[:n_results]
+
+    def find_similar(
+        self,
+        doc_id: str,
+        n_results: int = 5
+    ) -> List[Dict[str, Any]]:
+        """
+        Find documents similar to a given document.
+
+        Args:
+            doc_id: ID of the reference document
+            n_results: Number of similar documents to find
+
+        Returns:
+            List of similar documents
+        """
+        # Get the document
+        result = self.rag.collection.get(ids=[doc_id], include=["documents"])
+
+        if not result["documents"]:
+            return []
+
+        content = result["documents"][0]
+
+        # Search for similar (excluding the original)
+        results = self.rag.search(content, n_results=n_results + 1)
+
+        # Remove the original document
+        return [r for r in results if r["id"] != doc_id][:n_results]
+
+    def hybrid_search(
+        self,
+        query: str,
+        keywords: List[str],
+        n_results: int = 5,
+        keyword_boost: float = 0.3
+    ) -> List[Dict[str, Any]]:
+        """
+        Hybrid search combining semantic + keyword matching.
+
+        Args:
+            query: Semantic search query
+            keywords: Keywords to boost
+            n_results: Number of results
+            keyword_boost: Score boost for keyword matches (0-1)
+
+        Returns:
+            Reranked results
+        """
+        # Semantic search
+        results = self.rag.search(query, n_results=n_results * 2)
+
+        # Boost scores for keyword matches
+        for r in results:
+            content_lower = r["content"].lower()
+            matches = sum(1 for kw in keywords if kw.lower() in content_lower)
+            if matches > 0:
+                boost = min(keyword_boost * matches, keyword_boost * 3)
+                r["score"] = min(1.0, r["score"] + boost)
+                r["keyword_matches"] = matches
+
+        # Sort by score
+        results.sort(key=lambda x: x["score"], reverse=True)
+
+        return results[:n_results]