ragleap-rag 0.1.0__tar.gz

ragleap_rag-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+# Environment
+.env
+*.env.local
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+venv/
+.venv/
+
+# Node
+node_modules/
+
+# Docker
+docker-compose.override.yml
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+.env
+
+# Package build artifacts
+dist/
+build/

ragleap_rag-0.1.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: ragleap-rag
+Version: 0.1.0
+Summary: A fast, honest, self-hosted RAG engine — hybrid dense+sparse retrieval, streaming, provider fallback, and real token usage reporting. BYOK, no vendor lock-in.
+Project-URL: Homepage, https://github.com/antonyrag/ragleap-core
+Project-URL: Repository, https://github.com/antonyrag/ragleap-core
+Project-URL: Documentation, https://docs.ragleap.com
+Project-URL: Issues, https://github.com/antonyrag/ragleap-core/issues
+Author-email: Antony <antony@ragleap.com>
+License-Expression: MIT
+Keywords: hybrid-search,llm,pgvector,rag,retrieval-augmented-generation,vector-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: pypdf>=4.0.0
+Requires-Dist: python-docx>=1.1.0
+Requires-Dist: requests>=2.31.0
+Provides-Extra: all
+Requires-Dist: anthropic>=0.34.0; extra == 'all'
+Requires-Dist: google-genai>=0.3.0; extra == 'all'
+Requires-Dist: openai>=1.40.0; extra == 'all'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.34.0; extra == 'anthropic'
+Provides-Extra: gemini
+Requires-Dist: google-genai>=0.3.0; extra == 'gemini'
+Provides-Extra: openai
+Requires-Dist: openai>=1.40.0; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# ragleap-rag
+
+A fast, honest, self-hosted RAG engine. Hybrid dense+sparse retrieval,
+streaming, provider fallback, and real token usage reporting — no
+vendor lock-in, bring your own API keys.
+
+```bash
+pip install ragleap-rag[gemini]
+```
+
+```python
+from ragleap import RagLeap, ProviderConfig
+
+rag = RagLeap(
+    database_url="postgresql://user:pass@localhost/mydb",
+    embedder_api_key="your-gemini-key",
+    primary=ProviderConfig(provider="gemini", api_key="your-gemini-key"),
+)
+rag.init_schema()  # one-time, idempotent
+
+with open("document.pdf", "rb") as f:
+    result = rag.ingest("document.pdf", f.read())
+print(f"Ingested {result.chunks_stored} chunks")
+
+answer = rag.ask("What does this document say?")
+print(answer["answer"])
+print("Sources:", answer["sources"])
+print("Tokens used:", answer["usage"])
+```
+
+## Why ragleap-rag
+
+- **Hybrid search by default** — combines pgvector dense similarity with
+  Postgres full-text sparse search via Reciprocal Rank Fusion, catching
+  both semantic matches and exact keyword/identifier matches.
+- **Real provider fallback** — configure backup LLM providers; if your
+  primary fails (rate limit, outage, bad key), it retries automatically.
+- **Streaming** — real per-provider streaming for Gemini, Anthropic, and
+  any OpenAI-compatible endpoint.
+- **Real token usage** — every answer reports actual `prompt_tokens`,
+  `completion_tokens`, `total_tokens` from the provider's own response,
+  not an estimate.
+- **Context budget trimming** — automatically drops lowest-ranked chunks
+  to stay within a character budget, so you're not paying for more
+  context than necessary.
+- **BYOK, always** — no system-provided keys, ever. You own your data,
+  your database, and your API costs.
+
+## Requirements
+
+A PostgreSQL database with the [pgvector](https://github.com/pgvector/pgvector)
+extension available (`CREATE EXTENSION vector` — `rag.init_schema()` handles
+the rest). Embeddings currently use Google Gemini (`gemini-embedding-001`);
+generation supports Gemini, Anthropic, and any OpenAI-compatible provider
+(OpenAI, Groq, Mistral, Together, OpenRouter, Ollama, DeepSeek, xAI, Cohere,
+Perplexity, or a custom endpoint).
+
+## Status
+
+This is a young, actively-developed extraction from
+[ragleap-core](https://github.com/antonyrag/ragleap-core), the open-source
+self-hosted RAG platform. Full documentation, more embedding providers, and
+companion packages (`ragleap-graph` for knowledge-graph-boosted retrieval,
+`ragleap-integrations` for CRM/database connectors) are in progress.
+
+## License
+
+MIT

ragleap_rag-0.1.0/README.md

@@ -0,0 +1,68 @@
+# ragleap-rag
+
+A fast, honest, self-hosted RAG engine. Hybrid dense+sparse retrieval,
+streaming, provider fallback, and real token usage reporting — no
+vendor lock-in, bring your own API keys.
+
+```bash
+pip install ragleap-rag[gemini]
+```
+
+```python
+from ragleap import RagLeap, ProviderConfig
+
+rag = RagLeap(
+    database_url="postgresql://user:pass@localhost/mydb",
+    embedder_api_key="your-gemini-key",
+    primary=ProviderConfig(provider="gemini", api_key="your-gemini-key"),
+)
+rag.init_schema()  # one-time, idempotent
+
+with open("document.pdf", "rb") as f:
+    result = rag.ingest("document.pdf", f.read())
+print(f"Ingested {result.chunks_stored} chunks")
+
+answer = rag.ask("What does this document say?")
+print(answer["answer"])
+print("Sources:", answer["sources"])
+print("Tokens used:", answer["usage"])
+```
+
+## Why ragleap-rag
+
+- **Hybrid search by default** — combines pgvector dense similarity with
+  Postgres full-text sparse search via Reciprocal Rank Fusion, catching
+  both semantic matches and exact keyword/identifier matches.
+- **Real provider fallback** — configure backup LLM providers; if your
+  primary fails (rate limit, outage, bad key), it retries automatically.
+- **Streaming** — real per-provider streaming for Gemini, Anthropic, and
+  any OpenAI-compatible endpoint.
+- **Real token usage** — every answer reports actual `prompt_tokens`,
+  `completion_tokens`, `total_tokens` from the provider's own response,
+  not an estimate.
+- **Context budget trimming** — automatically drops lowest-ranked chunks
+  to stay within a character budget, so you're not paying for more
+  context than necessary.
+- **BYOK, always** — no system-provided keys, ever. You own your data,
+  your database, and your API costs.
+
+## Requirements
+
+A PostgreSQL database with the [pgvector](https://github.com/pgvector/pgvector)
+extension available (`CREATE EXTENSION vector` — `rag.init_schema()` handles
+the rest). Embeddings currently use Google Gemini (`gemini-embedding-001`);
+generation supports Gemini, Anthropic, and any OpenAI-compatible provider
+(OpenAI, Groq, Mistral, Together, OpenRouter, Ollama, DeepSeek, xAI, Cohere,
+Perplexity, or a custom endpoint).
+
+## Status
+
+This is a young, actively-developed extraction from
+[ragleap-core](https://github.com/antonyrag/ragleap-core), the open-source
+self-hosted RAG platform. Full documentation, more embedding providers, and
+companion packages (`ragleap-graph` for knowledge-graph-boosted retrieval,
+`ragleap-integrations` for CRM/database connectors) are in progress.
+
+## License
+
+MIT

ragleap_rag-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ragleap-rag"
+version = "0.1.0"
+description = "A fast, honest, self-hosted RAG engine — hybrid dense+sparse retrieval, streaming, provider fallback, and real token usage reporting. BYOK, no vendor lock-in."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "Antony", email = "antony@ragleap.com" }
+]
+keywords = ["rag", "retrieval-augmented-generation", "llm", "vector-search", "pgvector", "hybrid-search"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing :: Linguistic",
+]
+
+dependencies = [
+    "psycopg2-binary>=2.9.0",
+    "requests>=2.31.0",
+    "pypdf>=4.0.0",
+    "python-docx>=1.1.0",
+]
+
+[project.optional-dependencies]
+gemini = ["google-genai>=0.3.0"]
+anthropic = ["anthropic>=0.34.0"]
+openai = ["openai>=1.40.0"]
+all = ["google-genai>=0.3.0", "anthropic>=0.34.0", "openai>=1.40.0"]
+
+[project.urls]
+Homepage = "https://github.com/antonyrag/ragleap-core"
+Repository = "https://github.com/antonyrag/ragleap-core"
+Documentation = "https://docs.ragleap.com"
+Issues = "https://github.com/antonyrag/ragleap-core/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ragleap"]

ragleap_rag-0.1.0/src/ragleap/__init__.py

@@ -0,0 +1,188 @@
+"""
+ragleap-rag: a fast, honest, self-hosted RAG engine.
+
+    from ragleap import RagLeap, ProviderConfig
+
+    rag = RagLeap(
+        database_url="postgresql://user:pass@localhost/mydb",
+        embedder_api_key="...",       # Gemini API key (embeddings)
+        primary=ProviderConfig(provider="gemini", api_key="..."),
+    )
+    rag.init_schema()                  # one-time, idempotent
+
+    result = rag.ingest("doc.pdf", raw_bytes)
+    print(result.document_id, result.chunks_stored)
+
+    answer = rag.ask("What does this document say about X?")
+    print(answer["answer"], answer["sources"], answer["usage"])
+"""
+import uuid
+import logging
+from dataclasses import dataclass
+from typing import Dict, Iterator, List, Optional
+
+from ragleap.chunker import TextChunker
+from ragleap.embedding import EmbeddingService
+from ragleap.retrieval import VectorRetrievalService
+from ragleap.generation import GenerationService, ProviderConfig
+from ragleap.parsers import extract_text
+from ragleap import schema as _schema
+
+logger = logging.getLogger(__name__)
+
+__version__ = "0.1.0"
+__all__ = ["RagLeap", "ProviderConfig", "IngestResult"]
+
+
+@dataclass
+class IngestResult:
+    document_id: str
+    chunks_stored: int
+
+
+class RagLeap:
+    """
+    The main entry point for ragleap-rag. Wires together chunking,
+    embedding, hybrid retrieval, and generation (with fallback,
+    streaming, and token usage reporting) over a PostgreSQL + pgvector
+    database you control.
+    """
+
+    def __init__(
+        self,
+        database_url: str,
+        primary: ProviderConfig,
+        embedder_api_key: Optional[str] = None,
+        fallbacks: Optional[List[ProviderConfig]] = None,
+        embedding_dimensions: int = 3072,
+        default_temperature: float = 0.3,
+        default_max_tokens: int = 1024,
+        max_context_chars: int = 12000,
+        chunk_size: Optional[int] = None,
+        chunk_overlap: Optional[int] = None,
+    ):
+        self.database_url = database_url
+        self.embedding_dimensions = embedding_dimensions
+
+        self._chunker = TextChunker(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        self._embedder = EmbeddingService(api_key=embedder_api_key, dimensions=embedding_dimensions)
+        self._retriever = VectorRetrievalService(database_url=database_url, embedding_dimensions=embedding_dimensions)
+        self._generator = GenerationService(
+            primary=primary,
+            fallbacks=fallbacks,
+            default_temperature=default_temperature,
+            default_max_tokens=default_max_tokens,
+            max_context_chars=max_context_chars,
+        )
+
+    def init_schema(self) -> None:
+        """Create the required tables/indexes if they don't already exist. Idempotent."""
+        _schema.init_schema(self.database_url, dimensions=self.embedding_dimensions)
+
+    def _get_connection(self):
+        import psycopg2
+        return psycopg2.connect(self.database_url)
+
+    def ingest(self, filename: str, raw_bytes: bytes) -> IngestResult:
+        """
+        Extract text (from .txt/.pdf/.docx bytes), chunk, embed, and
+        store it. Returns an IngestResult with the new document_id and
+        chunk count.
+        """
+        text = extract_text(filename, raw_bytes)
+        return self.ingest_text(filename, text)
+
+    def ingest_text(self, filename: str, text: str) -> IngestResult:
+        """Same as ingest(), but for text you've already extracted yourself."""
+        chunks = self._chunker.chunk_text(text)
+        if not chunks:
+            raise ValueError("No chunks produced from input text — is it empty?")
+
+        document_id = str(uuid.uuid4())
+        conn = self._get_connection()
+        try:
+            cur = conn.cursor()
+            cur.execute("INSERT INTO documents (id, filename) VALUES (%s, %s)", (document_id, filename))
+
+            stored = 0
+            for chunk in chunks:
+                embedding = self._embedder.embed_text(chunk["text"])
+                if embedding is None:
+                    logger.warning(f"Skipping chunk {chunk['chunk_index']} — embedding failed")
+                    continue
+
+                embedding_literal = "[" + ",".join(str(float(x)) for x in embedding) + "]"
+                cur.execute(
+                    """
+                    INSERT INTO chunks (document_id, document_name, chunk_index, text, token_count, embedding)
+                    VALUES (%s, %s, %s, %s, %s, %s::vector)
+                    """,
+                    (document_id, filename, chunk["chunk_index"], chunk["text"], chunk["token_count"], embedding_literal),
+                )
+                stored += 1
+
+            if stored == 0:
+                conn.rollback()
+                raise ValueError(f"All {len(chunks)} chunk(s) failed to embed — nothing was stored.")
+
+            conn.commit()
+            cur.close()
+            logger.info(f"Ingested '{filename}': {stored}/{len(chunks)} chunks stored")
+            return IngestResult(document_id=document_id, chunks_stored=stored)
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    def ask(
+        self,
+        query: str,
+        top_k: int = 5,
+        temperature: Optional[float] = None,
+        system_prompt: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+        hybrid: bool = True,
+    ) -> Dict:
+        """
+        Answer a question grounded in previously ingested documents.
+        Returns: {"answer": str, "sources": List[str], "provider_used": str,
+                  "usage": dict|None, "chunks_sent": int}
+        """
+        query_embedding = self._embedder.embed_text(query)
+        if query_embedding is None:
+            return {"answer": "Sorry, I couldn't process your question (embedding failed).",
+                    "sources": [], "provider_used": None, "usage": None, "chunks_sent": 0}
+
+        if hybrid:
+            chunks = self._retriever.search_hybrid_chunks(query, query_embedding, top_k=top_k)
+        else:
+            chunks = self._retriever.search_similar_chunks(query_embedding, top_k=top_k)
+
+        return self._generator.generate_answer(
+            query, chunks, temperature=temperature, system_prompt=system_prompt, max_tokens=max_tokens
+        )
+
+    def ask_stream(
+        self,
+        query: str,
+        top_k: int = 5,
+        temperature: Optional[float] = None,
+        system_prompt: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+        hybrid: bool = True,
+    ) -> Iterator[str]:
+        """Same as ask(), but yields the answer incrementally as it's generated."""
+        query_embedding = self._embedder.embed_text(query)
+        if query_embedding is None:
+            yield "Sorry, I couldn't process your question (embedding failed)."
+            return
+
+        if hybrid:
+            chunks = self._retriever.search_hybrid_chunks(query, query_embedding, top_k=top_k)
+        else:
+            chunks = self._retriever.search_similar_chunks(query_embedding, top_k=top_k)
+
+        yield from self._generator.generate_answer_stream(
+            query, chunks, temperature=temperature, system_prompt=system_prompt, max_tokens=max_tokens
+        )

ragleap_rag-0.1.0/src/ragleap/chunker.py

@@ -0,0 +1,67 @@
+"""
+Text Chunking for ragleap-rag.
+"""
+import logging
+from typing import List
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CHUNK_SIZE = 512
+DEFAULT_CHUNK_OVERLAP = 50
+
+
+class TextChunker:
+    """Splits documents into overlapping chunks for embedding and retrieval."""
+
+    def __init__(self, chunk_size: int = None, chunk_overlap: int = None):
+        self.chunk_size = chunk_size or DEFAULT_CHUNK_SIZE
+        self.chunk_overlap = chunk_overlap or DEFAULT_CHUNK_OVERLAP
+
+        if self.chunk_overlap >= self.chunk_size:
+            raise ValueError("Chunk overlap must be less than chunk size")
+
+        logger.info(f"Initialized TextChunker: size={self.chunk_size}, overlap={self.chunk_overlap}")
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Simple whitespace tokenization."""
+        return text.split()
+
+    def chunk_text(self, text: str) -> List[dict]:
+        """
+        Split text into overlapping chunks with metadata.
+        Returns a list of dicts with: text, chunk_index, token_count
+        """
+        if not text or not text.strip():
+            logger.warning("Empty text provided for chunking")
+            return []
+
+        tokens = self._tokenize(text)
+        if len(tokens) == 0:
+            logger.warning("No tokens extracted from text")
+            return []
+
+        chunks = []
+        step = self.chunk_size - self.chunk_overlap
+        chunk_index = 0
+
+        for start in range(0, len(tokens), step):
+            end = min(start + self.chunk_size, len(tokens))
+            chunk_tokens = tokens[start:end]
+            chunk_text_content = " ".join(chunk_tokens)
+
+            chunks.append({
+                "text": chunk_text_content,
+                "chunk_index": chunk_index,
+                "token_count": len(chunk_tokens),
+            })
+            chunk_index += 1
+
+            if end == len(tokens):
+                break
+
+        logger.info(f"Chunked text into {len(chunks)} chunks")
+        return chunks
+
+
+def create_chunker(chunk_size: int = None, chunk_overlap: int = None) -> TextChunker:
+    return TextChunker(chunk_size=chunk_size, chunk_overlap=chunk_overlap)

ragleap_rag-0.1.0/src/ragleap/embedding.py

@@ -0,0 +1,60 @@
+"""
+Embedding for ragleap-rag.
+Uses Google Gemini embeddings (gemini-embedding-001, 3072 dimensions).
+Bring-your-own-key: pass api_key explicitly, or set GEMINI_API_KEY in
+the environment as a convenience fallback (useful for scripts/notebooks,
+but explicit is preferred for library usage inside a larger app).
+"""
+import os
+import logging
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MODEL = "models/gemini-embedding-001"
+DEFAULT_DIMENSIONS = 3072
+
+
+class EmbeddingService:
+    """Generates vector embeddings using Google Gemini."""
+
+    def __init__(self, api_key: Optional[str] = None, model: str = None, dimensions: int = None):
+        self.api_key = api_key or os.environ.get("GEMINI_API_KEY")
+        self.model = model or os.environ.get("GEMINI_EMBEDDING_MODEL", DEFAULT_MODEL)
+        self.dimensions = dimensions or int(os.environ.get("EMBEDDING_DIMENSIONS", str(DEFAULT_DIMENSIONS)))
+
+        if not self.api_key:
+            raise ValueError(
+                "No Gemini API key provided. Pass api_key= to EmbeddingService(), "
+                "or set GEMINI_API_KEY in your environment. Get a free key at "
+                "https://aistudio.google.com/apikey — there is no system-provided key."
+            )
+
+    def embed_text(self, text: str) -> Optional[List[float]]:
+        """Generate an embedding vector for a single piece of text."""
+        if not text or not text.strip():
+            logger.warning("Empty text provided for embedding")
+            return None
+
+        try:
+            import google.genai as genai
+            client = genai.Client(api_key=self.api_key)
+            response = client.models.embed_content(model=self.model, contents=text)
+            return response.embeddings[0].values
+        except Exception as e:
+            logger.error(f"Embedding generation failed: {e}")
+            return None
+
+    def embed_batch(self, texts: List[str]) -> List[Optional[List[float]]]:
+        """Generate embeddings for multiple texts."""
+        if not texts:
+            return []
+
+        try:
+            import google.genai as genai
+            client = genai.Client(api_key=self.api_key)
+            response = client.models.embed_content(model=self.model, contents=texts)
+            return [e.values for e in response.embeddings]
+        except Exception as e:
+            logger.error(f"Batch embedding generation failed: {e}")
+            return [None] * len(texts)

ragleap_rag-0.1.0/src/ragleap/generation.py

@@ -0,0 +1,276 @@
+"""
+Answer generation for ragleap-rag.
+Bring-your-own-key: supports Gemini, Anthropic, and any OpenAI-compatible
+provider. Configure explicitly via ProviderConfig, or let it fall back to
+environment variables for convenience (useful for scripts/notebooks).
+"""
+import os
+import logging
+from dataclasses import dataclass, field
+from typing import List, Dict, Iterator, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+PROVIDER_BASE_URLS = {
+    "openai":     "https://api.openai.com/v1",
+    "mistral":    "https://api.mistral.ai/v1",
+    "groq":       "https://api.groq.com/openai/v1",
+    "together":   "https://api.together.xyz/v1",
+    "openrouter": "https://openrouter.ai/api/v1",
+    "ollama":     "http://localhost:11434/v1",
+    "deepseek":   "https://api.deepseek.com/v1",
+    "xai":        "https://api.x.ai/v1",
+    "cohere":     "https://api.cohere.ai/v1",
+    "perplexity": "https://api.perplexity.ai",
+}
+
+DEFAULT_SYSTEM_PROMPT = """You are a helpful assistant that answers questions using ONLY the provided context.
+If the answer is not in the context, say clearly that you don't have that information — do not make things up.
+Always be concise and cite which document your answer came from when possible."""
+
+
+@dataclass
+class ProviderConfig:
+    """Explicit provider configuration. Use this for library integration
+    rather than relying on environment variables."""
+    provider: str
+    api_key: Optional[str] = None
+    model: Optional[str] = None
+    base_url: Optional[str] = None
+
+    def __post_init__(self):
+        self.provider = self.provider.lower()
+
+        # Convenience env-var fallback (e.g. for quick scripts) — explicit
+        # values passed to the constructor always take precedence.
+        if self.provider == "gemini":
+            self.api_key = self.api_key or os.environ.get("GEMINI_API_KEY")
+            self.model = self.model or os.environ.get("GEMINI_CHAT_MODEL", "gemini-2.5-flash")
+        elif self.provider == "anthropic":
+            self.api_key = self.api_key or os.environ.get("ANTHROPIC_API_KEY")
+            self.model = self.model or os.environ.get("ANTHROPIC_MODEL", "claude-sonnet-4-5")
+        elif self.provider in PROVIDER_BASE_URLS:
+            self.api_key = self.api_key or os.environ.get(f"{self.provider.upper()}_API_KEY")
+            self.model = self.model or os.environ.get(f"{self.provider.upper()}_MODEL")
+            self.base_url = self.base_url or PROVIDER_BASE_URLS[self.provider]
+        elif self.provider == "custom":
+            self.base_url = self.base_url or os.environ.get("CUSTOM_BASE_URL")
+        else:
+            raise ValueError(
+                f"Unknown provider '{self.provider}'. Supported: gemini, anthropic, custom, "
+                f"{', '.join(PROVIDER_BASE_URLS.keys())}."
+            )
+
+        if not self.api_key and self.provider != "ollama":
+            raise ValueError(f"No API key for provider '{self.provider}'. Pass api_key= explicitly.")
+        if self.provider not in ("gemini", "anthropic") and not self.base_url:
+            raise ValueError(f"No base_url for provider '{self.provider}'. Pass base_url= explicitly.")
+        if self.provider not in ("gemini", "anthropic") and not self.model:
+            raise ValueError(f"No model for provider '{self.provider}'. Pass model= explicitly.")
+
+
+class GenerationService:
+    """
+    Generates a grounded answer using the configured provider, with an
+    optional fallback chain, streaming, and real token usage reporting.
+    """
+
+    def __init__(
+        self,
+        primary: ProviderConfig,
+        fallbacks: Optional[List[ProviderConfig]] = None,
+        default_temperature: float = 0.3,
+        default_max_tokens: int = 1024,
+        max_context_chars: int = 12000,
+        system_prompt: str = DEFAULT_SYSTEM_PROMPT,
+    ):
+        self.primary = primary
+        self.fallbacks = fallbacks or []
+        self.default_temperature = default_temperature
+        self.default_max_tokens = default_max_tokens
+        self.max_context_chars = max_context_chars
+        self.system_prompt = system_prompt
+
+    def _chain(self) -> List[ProviderConfig]:
+        return [self.primary] + [f for f in self.fallbacks if f.provider != self.primary.provider]
+
+    def _trim_chunks_to_budget(self, chunks: List[Dict]) -> List[Dict]:
+        if self.max_context_chars <= 0 or not chunks:
+            return chunks
+        kept, running_total = [], 0
+        for chunk in chunks:
+            chunk_len = len(chunk.get("text", ""))
+            if running_total + chunk_len > self.max_context_chars and kept:
+                break
+            kept.append(chunk)
+            running_total += chunk_len
+        if len(kept) < len(chunks):
+            logger.info(f"Trimmed context: {len(chunks)} -> {len(kept)} chunks ({running_total} chars)")
+        return kept
+
+    def _build_context(self, chunks: List[Dict]) -> str:
+        if not chunks:
+            return "No relevant context was found."
+        parts = []
+        for i, chunk in enumerate(chunks, start=1):
+            doc_name = chunk.get("document_name", "unknown document")
+            parts.append(f"[Source {i}: {doc_name}]\n{chunk.get('text', '')}")
+        return "\n\n".join(parts)
+
+    def _build_prompt(self, query: str, chunks: List[Dict], system_prompt: Optional[str]) -> str:
+        context = self._build_context(chunks)
+        instructions = system_prompt or self.system_prompt
+        return f"{instructions}\n\nContext:\n{context}\n\nQuestion: {query}\nAnswer:"
+
+    def generate_answer(
+        self,
+        query: str,
+        chunks: List[Dict],
+        temperature: Optional[float] = None,
+        system_prompt: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+    ) -> Dict:
+        """
+        Returns: {"answer": str, "sources": List[str], "provider_used": str,
+                  "usage": dict|None, "chunks_sent": int}
+        """
+        temp = self.default_temperature if temperature is None else temperature
+        max_tok = self.default_max_tokens if max_tokens is None else max_tokens
+
+        trimmed = self._trim_chunks_to_budget(chunks)
+        sources = list({c.get("document_name", "unknown") for c in trimmed})
+        prompt = self._build_prompt(query, trimmed, system_prompt)
+
+        last_error = None
+        for i, config in enumerate(self._chain()):
+            try:
+                answer_text, usage = self._call_provider(config, prompt, temp, max_tok)
+                if i > 0:
+                    logger.info(f"Answer generated via fallback provider '{config.provider}'")
+                return {
+                    "answer": answer_text, "sources": sources,
+                    "provider_used": config.provider, "usage": usage,
+                    "chunks_sent": len(trimmed),
+                }
+            except Exception as e:
+                last_error = e
+                logger.warning(f"Provider '{config.provider}' failed: {e}")
+                continue
+
+        return {
+            "answer": f"Sorry, all configured providers failed. Last error: {last_error}",
+            "sources": [], "provider_used": None, "usage": None, "chunks_sent": 0,
+        }
+
+    def generate_answer_stream(
+        self,
+        query: str,
+        chunks: List[Dict],
+        temperature: Optional[float] = None,
+        system_prompt: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+    ) -> Iterator[str]:
+        """Streams the answer incrementally. Usage reporting isn't
+        available for streaming (each provider handles it differently)."""
+        temp = self.default_temperature if temperature is None else temperature
+        max_tok = self.default_max_tokens if max_tokens is None else max_tokens
+
+        trimmed = self._trim_chunks_to_budget(chunks)
+        prompt = self._build_prompt(query, trimmed, system_prompt)
+
+        last_error = None
+        for i, config in enumerate(self._chain()):
+            yielded = False
+            try:
+                for piece in self._stream_provider(config, prompt, temp, max_tok):
+                    yielded = True
+                    yield piece
+                return
+            except Exception as e:
+                last_error = e
+                logger.warning(f"Provider '{config.provider}' failed during streaming: {e}")
+                if yielded:
+                    yield f"\n[Error: generation interrupted — {e}]"
+                    return
+                continue
+
+        yield f"Sorry, all configured providers failed. Last error: {last_error}"
+
+    def _call_provider(self, config: ProviderConfig, prompt: str, temperature: float, max_tokens: int) -> Tuple[str, Optional[Dict]]:
+        if config.provider == "gemini":
+            return self._call_gemini(prompt, temperature, max_tokens, config.api_key, config.model)
+        elif config.provider == "anthropic":
+            return self._call_anthropic(prompt, temperature, max_tokens, config.api_key, config.model)
+        else:
+            return self._call_openai_compatible(prompt, temperature, max_tokens, config.api_key, config.model, config.base_url)
+
+    def _stream_provider(self, config: ProviderConfig, prompt: str, temperature: float, max_tokens: int) -> Iterator[str]:
+        if config.provider == "gemini":
+            yield from self._stream_gemini(prompt, temperature, max_tokens, config.api_key, config.model)
+        elif config.provider == "anthropic":
+            yield from self._stream_anthropic(prompt, temperature, max_tokens, config.api_key, config.model)
+        else:
+            yield from self._stream_openai_compatible(prompt, temperature, max_tokens, config.api_key, config.model, config.base_url)
+
+    def _call_gemini(self, prompt, temperature, max_tokens, api_key, model) -> Tuple[str, Optional[Dict]]:
+        import google.genai as genai
+        from google.genai import types
+        client = genai.Client(api_key=api_key)
+        response = client.models.generate_content(
+            model=model, contents=prompt,
+            config=types.GenerateContentConfig(temperature=temperature, max_output_tokens=max_tokens),
+        )
+        text = response.text.strip() if response.text else "No answer generated."
+        usage = None
+        if getattr(response, "usage_metadata", None):
+            um = response.usage_metadata
+            usage = {"prompt_tokens": um.prompt_token_count, "completion_tokens": um.candidates_token_count, "total_tokens": um.total_token_count}
+        return text, usage
+
+    def _stream_gemini(self, prompt, temperature, max_tokens, api_key, model) -> Iterator[str]:
+        import google.genai as genai
+        from google.genai import types
+        client = genai.Client(api_key=api_key)
+        stream = client.models.generate_content_stream(
+            model=model, contents=prompt,
+            config=types.GenerateContentConfig(temperature=temperature, max_output_tokens=max_tokens),
+        )
+        for chunk in stream:
+            if chunk.text:
+                yield chunk.text
+
+    def _call_anthropic(self, prompt, temperature, max_tokens, api_key, model) -> Tuple[str, Optional[Dict]]:
+        import anthropic
+        client = anthropic.Anthropic(api_key=api_key)
+        response = client.messages.create(model=model, max_tokens=max_tokens, temperature=temperature, messages=[{"role": "user", "content": prompt}])
+        text = response.content[0].text.strip() if response.content else "No answer generated."
+        usage = None
+        if getattr(response, "usage", None):
+            usage = {"prompt_tokens": response.usage.input_tokens, "completion_tokens": response.usage.output_tokens, "total_tokens": response.usage.input_tokens + response.usage.output_tokens}
+        return text, usage
+
+    def _stream_anthropic(self, prompt, temperature, max_tokens, api_key, model) -> Iterator[str]:
+        import anthropic
+        client = anthropic.Anthropic(api_key=api_key)
+        with client.messages.stream(model=model, max_tokens=max_tokens, temperature=temperature, messages=[{"role": "user", "content": prompt}]) as stream:
+            for text in stream.text_stream:
+                yield text
+
+    def _call_openai_compatible(self, prompt, temperature, max_tokens, api_key, model, base_url) -> Tuple[str, Optional[Dict]]:
+        import openai
+        client = openai.OpenAI(api_key=api_key or "not-needed", base_url=base_url)
+        response = client.chat.completions.create(model=model, messages=[{"role": "user", "content": prompt}], temperature=temperature, max_tokens=max_tokens)
+        text = response.choices[0].message.content.strip() if response.choices else "No answer generated."
+        usage = None
+        if getattr(response, "usage", None):
+            usage = {"prompt_tokens": response.usage.prompt_tokens, "completion_tokens": response.usage.completion_tokens, "total_tokens": response.usage.total_tokens}
+        return text, usage
+
+    def _stream_openai_compatible(self, prompt, temperature, max_tokens, api_key, model, base_url) -> Iterator[str]:
+        import openai
+        client = openai.OpenAI(api_key=api_key or "not-needed", base_url=base_url)
+        stream = client.chat.completions.create(model=model, messages=[{"role": "user", "content": prompt}], temperature=temperature, max_tokens=max_tokens, stream=True)
+        for chunk in stream:
+            delta = chunk.choices[0].delta.content if chunk.choices else None
+            if delta:
+                yield delta

ragleap_rag-0.1.0/src/ragleap/parsers.py

@@ -0,0 +1,69 @@
+"""
+Document text extraction for ragleap-rag.
+Supports .txt, .pdf, .docx.
+"""
+import io
+import logging
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_EXTENSIONS = {".txt", ".pdf", ".docx"}
+
+
+def extract_text(filename: str, raw_bytes: bytes) -> str:
+    """Extract plain text from raw file bytes, dispatching on file extension."""
+    ext = filename.lower().rsplit(".", 1)[-1] if "." in filename else ""
+    ext = f".{ext}"
+    if ext == ".txt":
+        return _extract_txt(raw_bytes)
+    elif ext == ".pdf":
+        return _extract_pdf(raw_bytes)
+    elif ext == ".docx":
+        return _extract_docx(raw_bytes)
+    else:
+        raise ValueError(f"Unsupported file type '{ext}'. Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}.")
+
+
+def _extract_txt(raw_bytes: bytes) -> str:
+    try:
+        return raw_bytes.decode("utf-8")
+    except UnicodeDecodeError:
+        logger.warning("UTF-8 decode failed, retrying with latin-1")
+        return raw_bytes.decode("latin-1")
+
+
+def _extract_pdf(raw_bytes: bytes) -> str:
+    try:
+        from pypdf import PdfReader
+    except ImportError as e:
+        raise ValueError("pypdf is required for PDF support — pip install ragleap-rag installs it by default") from e
+
+    reader = PdfReader(io.BytesIO(raw_bytes))
+    pages_text = []
+    for i, page in enumerate(reader.pages):
+        text = page.extract_text() or ""
+        if text.strip():
+            pages_text.append(text)
+        else:
+            logger.warning(f"No extractable text on PDF page {i + 1} (likely scanned/image-only)")
+    full_text = "\n\n".join(pages_text)
+    if not full_text.strip():
+        raise ValueError(
+            "No text could be extracted from this PDF. It may be a scanned "
+            "image-only document, which requires OCR (not currently supported)."
+        )
+    return full_text
+
+
+def _extract_docx(raw_bytes: bytes) -> str:
+    try:
+        import docx
+    except ImportError as e:
+        raise ValueError("python-docx is required for DOCX support — pip install ragleap-rag installs it by default") from e
+
+    document = docx.Document(io.BytesIO(raw_bytes))
+    paragraphs = [p.text for p in document.paragraphs if p.text.strip()]
+    full_text = "\n\n".join(paragraphs)
+    if not full_text.strip():
+        raise ValueError("No text could be extracted from this DOCX file — it may be empty.")
+    return full_text

ragleap_rag-0.1.0/src/ragleap/retrieval.py

@@ -0,0 +1,195 @@
+"""
+Retrieval for ragleap-rag: dense (pgvector), sparse (Postgres full-text),
+and hybrid (Reciprocal Rank Fusion) search.
+
+Requires a PostgreSQL database with the pgvector extension and the
+schema created by ragleap.schema.init_schema() (or your own compatible
+schema — see that module for the exact DDL).
+
+No knowledge-graph coupling here by design — ragleap-graph (a separate
+package) extends retrieval with graph-boosted ranking on top of this.
+"""
+import logging
+from typing import List, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+RRF_K = 60  # Reciprocal Rank Fusion constant — 60 is the standard default.
+
+
+class VectorRetrievalService:
+    """
+    Performs dense, sparse, and hybrid search against a PostgreSQL
+    'chunks' table (see ragleap.schema for the required DDL).
+    """
+
+    def __init__(self, database_url: str, embedding_dimensions: int = 3072, min_similarity: float = 0.05):
+        self.database_url = database_url
+        self.embedding_dimensions = embedding_dimensions
+        self.min_similarity = min_similarity
+
+    def _get_connection(self):
+        import psycopg2
+        return psycopg2.connect(self.database_url)
+
+    def search_similar_chunks(
+        self,
+        query_embedding: List[float],
+        top_k: int = 5,
+        document_id: Optional[str] = None,
+    ) -> List[Dict]:
+        """Dense retrieval via pgvector cosine distance."""
+        if not query_embedding:
+            return []
+
+        if len(query_embedding) != self.embedding_dimensions:
+            logger.warning(
+                f"Query embedding dim={len(query_embedding)} != expected {self.embedding_dimensions}; skipping search"
+            )
+            return []
+
+        literal = "[" + ",".join(str(float(x)) for x in query_embedding) + "]"
+
+        sql = """
+            SELECT
+                id, text, document_id, document_name, chunk_index,
+                1 - (embedding::halfvec(%s) <=> %s::halfvec(%s)) / 2 AS similarity_score
+            FROM chunks
+        """
+        params = [self.embedding_dimensions, literal, self.embedding_dimensions]
+
+        if document_id:
+            sql += " WHERE document_id = %s"
+            params.append(document_id)
+
+        sql += " ORDER BY embedding::halfvec(%s) <=> %s::halfvec(%s) LIMIT %s"
+        params.extend([self.embedding_dimensions, literal, self.embedding_dimensions, top_k])
+
+        try:
+            conn = self._get_connection()
+            cur = conn.cursor()
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+            cur.close()
+            conn.close()
+
+            results = []
+            for row in rows:
+                chunk_id, text, doc_id, doc_name, chunk_index, similarity_score = row
+                if similarity_score < self.min_similarity:
+                    continue
+                results.append({
+                    "chunk_id": str(chunk_id),
+                    "text": text,
+                    "similarity_score": round(float(similarity_score), 4),
+                    "document_id": str(doc_id),
+                    "document_name": doc_name,
+                    "chunk_index": chunk_index,
+                })
+
+            logger.info(f"Vector search returned {len(results)} chunks (top_k={top_k})")
+            return results
+
+        except Exception as e:
+            logger.error(f"Vector search error: {e}", exc_info=True)
+            raise
+
+    def search_sparse_chunks(
+        self,
+        query_text: str,
+        top_k: int = 5,
+        document_id: Optional[str] = None,
+    ) -> List[Dict]:
+        """Sparse (keyword/full-text) retrieval via Postgres text search."""
+        if not query_text or not query_text.strip():
+            return []
+
+        sql = """
+            SELECT id, text, document_id, document_name, chunk_index,
+                   ts_rank(text_search_vector, websearch_to_tsquery('english', %s)) AS rank_score
+            FROM chunks
+            WHERE text_search_vector @@ websearch_to_tsquery('english', %s)
+        """
+        params = [query_text, query_text]
+
+        if document_id:
+            sql += " AND document_id = %s"
+            params.append(document_id)
+
+        sql += " ORDER BY rank_score DESC LIMIT %s"
+        params.append(top_k)
+
+        try:
+            conn = self._get_connection()
+            cur = conn.cursor()
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+            cur.close()
+            conn.close()
+
+            results = []
+            for row in rows:
+                chunk_id, text, doc_id, doc_name, chunk_index, rank_score = row
+                results.append({
+                    "chunk_id": str(chunk_id),
+                    "text": text,
+                    "similarity_score": round(float(rank_score), 4),
+                    "document_id": str(doc_id),
+                    "document_name": doc_name,
+                    "chunk_index": chunk_index,
+                })
+
+            logger.info(f"Sparse search returned {len(results)} chunks (top_k={top_k})")
+            return results
+
+        except Exception as e:
+            logger.error(f"Sparse search error: {e}", exc_info=True)
+            raise
+
+    def search_hybrid_chunks(
+        self,
+        query_text: str,
+        query_embedding: List[float],
+        top_k: int = 5,
+        document_id: Optional[str] = None,
+    ) -> List[Dict]:
+        """
+        Combines dense + sparse via Reciprocal Rank Fusion. similarity_score
+        in the returned dicts is the RRF-fused score, meaningful only for
+        ranking within this result set (not a 0-1 cosine similarity).
+        """
+        dense = self.search_similar_chunks(query_embedding, top_k=top_k * 3, document_id=document_id)
+        sparse = self.search_sparse_chunks(query_text, top_k=top_k * 3, document_id=document_id)
+
+        dense_ranks = {c["chunk_id"]: i for i, c in enumerate(dense)}
+        sparse_ranks = {c["chunk_id"]: i for i, c in enumerate(sparse)}
+
+        chunk_lookup: Dict[str, Dict] = {c["chunk_id"]: c for c in dense}
+        for c in sparse:
+            chunk_lookup.setdefault(c["chunk_id"], c)
+
+        all_ids = set(dense_ranks) | set(sparse_ranks)
+        if not all_ids:
+            return []
+
+        rrf_scores = {}
+        for cid in all_ids:
+            score = 0.0
+            if cid in dense_ranks:
+                score += 1.0 / (RRF_K + dense_ranks[cid] + 1)
+            if cid in sparse_ranks:
+                score += 1.0 / (RRF_K + sparse_ranks[cid] + 1)
+            rrf_scores[cid] = score
+
+        results = [dict(chunk_lookup[cid]) for cid in all_ids]
+        for c in results:
+            c["similarity_score"] = round(rrf_scores[c["chunk_id"]], 6)
+            c["retrieval_method"] = "hybrid_rrf"
+
+        results.sort(key=lambda c: c["similarity_score"], reverse=True)
+
+        logger.info(
+            f"Hybrid search: {len(dense)} dense + {len(sparse)} sparse -> "
+            f"{len(results)} fused, returning top {min(top_k, len(results))}"
+        )
+        return results[:top_k]

ragleap_rag-0.1.0/src/ragleap/schema.py

@@ -0,0 +1,58 @@
+"""
+Database schema management for ragleap-rag.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+SCHEMA_SQL_TEMPLATE = """
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE IF NOT EXISTS documents (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    filename TEXT NOT NULL,
+    uploaded_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+    document_name TEXT NOT NULL,
+    chunk_index INTEGER NOT NULL,
+    text TEXT NOT NULL,
+    token_count INTEGER,
+    embedding vector({dimensions}),
+    text_search_vector tsvector GENERATED ALWAYS AS (to_tsvector('english', text)) STORED,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS chunks_embedding_idx
+    ON chunks USING hnsw ((embedding::halfvec({dimensions})) halfvec_cosine_ops);
+
+CREATE INDEX IF NOT EXISTS chunks_text_search_idx
+    ON chunks USING GIN (text_search_vector);
+"""
+
+
+def get_schema_sql(dimensions: int = 3072) -> str:
+    """Return the DDL for the given embedding dimensionality."""
+    return SCHEMA_SQL_TEMPLATE.format(dimensions=dimensions)
+
+
+def init_schema(database_url: str, dimensions: int = 3072) -> None:
+    """
+    Create the required tables/indexes in the given database if they
+    don't already exist. Safe to call repeatedly (idempotent —
+    everything uses IF NOT EXISTS).
+    """
+    import psycopg2
+
+    conn = psycopg2.connect(database_url)
+    try:
+        cur = conn.cursor()
+        cur.execute(get_schema_sql(dimensions))
+        conn.commit()
+        cur.close()
+        logger.info(f"Schema initialized (embedding dimensions={dimensions})")
+    finally:
+        conn.close()