raglib-py 0.1.3__tar.gz

raglib_py-0.1.3/.gitignore

@@ -0,0 +1,38 @@
+.venv/
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+
+# Build artifacts
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Test and tooling caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+
+# Local runtime/state files
+.raglib_chroma/
+*.sqlite3
+*.db
+
+# Editors/OS
+.vscode/
+.idea/
+Thumbs.db
+Desktop.ini
+
+# Secrets/env files
+.env
+.env.*
+*.pem
+*.key
+*.token

raglib_py-0.1.3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 raglib maintainers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

raglib_py-0.1.3/PKG-INFO

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: raglib-py
+Version: 0.1.3
+Summary: A modular, production-grade Retrieval-Augmented Generation library
+Project-URL: Homepage, https://github.com/himanshau/raglib
+Project-URL: Repository, https://github.com/himanshau/raglib
+Project-URL: Bug Tracker, https://github.com/himanshau/raglib/issues
+Author: raglib maintainers
+License: MIT License
+        
+        Copyright (c) 2026 raglib maintainers
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: ai,llm,nlp,rag,retrieval,vector-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: duckduckgo-search>=3.9
+Provides-Extra: all
+Requires-Dist: chromadb>=0.5.0; extra == 'all'
+Requires-Dist: langchain-anthropic>=0.1.0; extra == 'all'
+Requires-Dist: langchain-core>=0.2.0; extra == 'all'
+Requires-Dist: langchain-google-genai>=1.0.0; extra == 'all'
+Requires-Dist: langchain-groq>=0.1.0; extra == 'all'
+Requires-Dist: langchain-huggingface>=0.0.3; extra == 'all'
+Requires-Dist: langchain-ollama>=0.1.0; extra == 'all'
+Requires-Dist: langchain-openai>=0.1.0; extra == 'all'
+Requires-Dist: pillow>=10.0.0; extra == 'all'
+Requires-Dist: pymupdf>=1.23.0; extra == 'all'
+Requires-Dist: pytesseract>=0.3.10; extra == 'all'
+Requires-Dist: python-docx>=1.0.0; extra == 'all'
+Requires-Dist: python-pptx>=0.6.21; extra == 'all'
+Requires-Dist: sentence-transformers>=2.6.0; extra == 'all'
+Provides-Extra: all-llm
+Requires-Dist: langchain-anthropic>=0.1.0; extra == 'all-llm'
+Requires-Dist: langchain-core>=0.2.0; extra == 'all-llm'
+Requires-Dist: langchain-google-genai>=1.0.0; extra == 'all-llm'
+Requires-Dist: langchain-groq>=0.1.0; extra == 'all-llm'
+Requires-Dist: langchain-ollama>=0.1.0; extra == 'all-llm'
+Requires-Dist: langchain-openai>=0.1.0; extra == 'all-llm'
+Provides-Extra: anthropic
+Requires-Dist: langchain-anthropic>=0.1.0; extra == 'anthropic'
+Provides-Extra: chroma
+Requires-Dist: chromadb>=0.5.0; extra == 'chroma'
+Provides-Extra: dev
+Requires-Dist: build>=1.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest-mock>=3.11.0; extra == 'dev'
+Requires-Dist: pytest>=7.4.0; extra == 'dev'
+Requires-Dist: twine>=4.0.0; extra == 'dev'
+Provides-Extra: docx
+Requires-Dist: python-docx>=1.0.0; extra == 'docx'
+Provides-Extra: faiss
+Requires-Dist: faiss-cpu>=1.7.0; extra == 'faiss'
+Provides-Extra: free-embed
+Requires-Dist: langchain-huggingface>=0.0.3; extra == 'free-embed'
+Requires-Dist: sentence-transformers>=2.6.0; extra == 'free-embed'
+Provides-Extra: google
+Requires-Dist: langchain-google-genai>=1.0.0; extra == 'google'
+Provides-Extra: groq
+Requires-Dist: langchain-groq>=0.1.0; extra == 'groq'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2.0; extra == 'langchain'
+Provides-Extra: ocr
+Requires-Dist: pillow>=10.0.0; extra == 'ocr'
+Requires-Dist: pytesseract>=0.3.10; extra == 'ocr'
+Provides-Extra: ollama
+Requires-Dist: langchain-ollama>=0.1.0; extra == 'ollama'
+Provides-Extra: openai
+Requires-Dist: langchain-openai>=0.1.0; extra == 'openai'
+Provides-Extra: pdf
+Requires-Dist: pymupdf>=1.23.0; extra == 'pdf'
+Provides-Extra: pptx
+Requires-Dist: python-pptx>=0.6.21; extra == 'pptx'
+Provides-Extra: serpapi
+Requires-Dist: google-search-results>=2.4.0; extra == 'serpapi'
+Provides-Extra: tavily
+Requires-Dist: tavily-python>=0.2.0; extra == 'tavily'
+Provides-Extra: vision
+Requires-Dist: langchain-anthropic>=0.1.0; extra == 'vision'
+Requires-Dist: langchain-google-genai>=1.0.0; extra == 'vision'
+Requires-Dist: langchain-openai>=0.1.0; extra == 'vision'
+Description-Content-Type: text/markdown
+
+# raglib
+
+raglib is a modular, production-grade Python library for Retrieval-Augmented Generation (RAG).
+
+## Installation
+
+Install from PyPI:
+
+pip install raglib-py
+
+Install with optional extras:
+
+pip install raglib-py[tavily,serpapi,faiss]
+
+## Quick import check
+
+python -c "from raglib.rag_types.naive_rag import NaiveRAG; print('Import successful')"
+
+## One-line API quickstart
+
+```python
+from raglib import RAG
+
+rag = RAG("my_documents/")
+result = rag.query("What is this report about?")
+
+print(result.answer)
+print(result.sources)
+```
+
+You can also add more sources later:
+
+```python
+rag.add("new_data.docx")
+rag.add("https://example.com/article")
+```
+
+## Features
+
+- Modular retrieval components (vector, web, hybrid, routing, multi-query, multi-hop)
+- RAG strategy orchestrators (naive, advanced, corrective, self, agentic, and more)
+- Mock LLM client for offline testing
+- Provider abstraction with fallback chaining
+- Full test suite with unit, integration, e2e, and edge coverage
+
+## Detailed usage guide
+
+See docs/USAGE.md for a full Python usage guide with quickstart, component wiring, and advanced orchestrator examples.
+
+## License
+
+MIT

raglib_py-0.1.3/README.md

@@ -0,0 +1,52 @@
+# raglib
+
+raglib is a modular, production-grade Python library for Retrieval-Augmented Generation (RAG).
+
+## Installation
+
+Install from PyPI:
+
+pip install raglib-py
+
+Install with optional extras:
+
+pip install raglib-py[tavily,serpapi,faiss]
+
+## Quick import check
+
+python -c "from raglib.rag_types.naive_rag import NaiveRAG; print('Import successful')"
+
+## One-line API quickstart
+
+```python
+from raglib import RAG
+
+rag = RAG("my_documents/")
+result = rag.query("What is this report about?")
+
+print(result.answer)
+print(result.sources)
+```
+
+You can also add more sources later:
+
+```python
+rag.add("new_data.docx")
+rag.add("https://example.com/article")
+```
+
+## Features
+
+- Modular retrieval components (vector, web, hybrid, routing, multi-query, multi-hop)
+- RAG strategy orchestrators (naive, advanced, corrective, self, agentic, and more)
+- Mock LLM client for offline testing
+- Provider abstraction with fallback chaining
+- Full test suite with unit, integration, e2e, and edge coverage
+
+## Detailed usage guide
+
+See docs/USAGE.md for a full Python usage guide with quickstart, component wiring, and advanced orchestrator examples.
+
+## License
+
+MIT

raglib_py-0.1.3/pyproject.toml

@@ -0,0 +1,99 @@
+[build-system]
+requires = ["hatchling>=1.25.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "raglib-py"
+version = "0.1.3"
+description = "A modular, production-grade Retrieval-Augmented Generation library"
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+  { name = "raglib maintainers" }
+]
+requires-python = ">=3.9"
+dependencies = [
+  "duckduckgo-search>=3.9"
+]
+keywords = ["rag", "llm", "retrieval", "ai", "nlp", "vector-search"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://github.com/himanshau/raglib"
+Repository = "https://github.com/himanshau/raglib"
+"Bug Tracker" = "https://github.com/himanshau/raglib/issues"
+
+[project.optional-dependencies]
+tavily = ["tavily-python>=0.2.0"]
+serpapi = ["google-search-results>=2.4.0"]
+faiss = ["faiss-cpu>=1.7.0"]
+chroma = ["chromadb>=0.5.0"]
+openai = ["langchain-openai>=0.1.0"]
+anthropic = ["langchain-anthropic>=0.1.0"]
+groq = ["langchain-groq>=0.1.0"]
+google = ["langchain-google-genai>=1.0.0"]
+ollama = ["langchain-ollama>=0.1.0"]
+langchain = ["langchain-core>=0.2.0"]
+pdf = ["PyMuPDF>=1.23.0"]
+docx = ["python-docx>=1.0.0"]
+pptx = ["python-pptx>=0.6.21"]
+free-embed = ["langchain-huggingface>=0.0.3", "sentence-transformers>=2.6.0"]
+ocr = ["pytesseract>=0.3.10", "Pillow>=10.0.0"]
+vision = [
+  "langchain-openai>=0.1.0",
+  "langchain-anthropic>=0.1.0",
+  "langchain-google-genai>=1.0.0",
+]
+all-llm = [
+  "langchain-core>=0.2.0",
+  "langchain-openai>=0.1.0",
+  "langchain-anthropic>=0.1.0",
+  "langchain-groq>=0.1.0",
+  "langchain-google-genai>=1.0.0",
+  "langchain-ollama>=0.1.0",
+]
+all = [
+  "chromadb>=0.5.0",
+  "langchain-core>=0.2.0",
+  "langchain-openai>=0.1.0",
+  "langchain-anthropic>=0.1.0",
+  "langchain-groq>=0.1.0",
+  "langchain-google-genai>=1.0.0",
+  "langchain-ollama>=0.1.0",
+  "langchain-huggingface>=0.0.3",
+  "sentence-transformers>=2.6.0",
+  "PyMuPDF>=1.23.0",
+  "python-docx>=1.0.0",
+  "python-pptx>=0.6.21",
+  "pytesseract>=0.3.10",
+  "Pillow>=10.0.0",
+]
+dev = [
+  "pytest>=7.4.0",
+  "pytest-cov>=4.1.0",
+  "pytest-mock>=3.11.0",
+  "build>=1.0.0",
+  "twine>=4.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["raglib"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "raglib/**/*.py",
+  "README.md",
+  "LICENSE",
+  "pyproject.toml",
+]

raglib_py-0.1.3/raglib/__init__.py

@@ -0,0 +1,17 @@
+"""Public package exports for raglib."""
+
+from __future__ import annotations
+
+from raglib.facade import RAG
+from raglib.llm.mock_client import MockLLMClient
+from raglib.schemas import Document, GenerationResult, QueryResult
+
+__all__ = [
+    "RAG",
+    "Document",
+    "QueryResult",
+    "GenerationResult",
+    "MockLLMClient",
+]
+
+__version__ = "0.1.3"

raglib_py-0.1.3/raglib/components/__init__.py

@@ -0,0 +1,41 @@
+"""Component exports for raglib retrieval and generation building blocks."""
+
+from raglib.components.context_reducer import ContextReducer
+from raglib.components.decision import DecisionEngine
+from raglib.components.deduplicator import Deduplicator
+from raglib.components.evaluator import Evaluator
+from raglib.components.generator import Generator
+from raglib.components.hybrid_retriever import HybridRetriever
+from raglib.components.memory import MemoryModule
+from raglib.components.multi_hop_retriever import MultiHopRetriever
+from raglib.components.multi_query_retriever import MultiQueryRetriever
+from raglib.components.planner import Planner
+from raglib.components.query_expander import QueryExpander
+from raglib.components.query_rewriter import QueryRewriter
+from raglib.components.refiner import Refiner
+from raglib.components.reflection import ReflectionModule
+from raglib.components.reranker import Reranker
+from raglib.components.retriever import Retriever
+from raglib.components.router_retriever import RouterRetriever
+from raglib.components.web_retriever import WebRetriever
+
+__all__ = [
+    "Retriever",
+    "WebRetriever",
+    "HybridRetriever",
+    "MultiQueryRetriever",
+    "RouterRetriever",
+    "MultiHopRetriever",
+    "Reranker",
+    "Evaluator",
+    "Refiner",
+    "QueryRewriter",
+    "QueryExpander",
+    "Deduplicator",
+    "ContextReducer",
+    "MemoryModule",
+    "Planner",
+    "ReflectionModule",
+    "DecisionEngine",
+    "Generator",
+]

raglib_py-0.1.3/raglib/components/context_reducer.py

@@ -0,0 +1,65 @@
+"""Context reduction utilities for prompt token budgeting."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import replace
+from typing import List, Optional
+
+from raglib.schemas import Document
+
+logger = logging.getLogger(__name__)
+
+
+class ContextReducer:
+    """Trims retrieved context to fit a target token budget."""
+
+    def __init__(self, max_context_tokens: int = 3000):
+        """Initialize context token budget."""
+
+        self.max_context_tokens = max_context_tokens
+
+    def reduce(self, documents: List[Document], max_tokens: Optional[int] = None) -> List[Document]:
+        """Reduce documents until the token budget is satisfied."""
+
+        budget = max_tokens if max_tokens is not None else self.max_context_tokens
+        reduced: List[Document] = []
+        used_tokens = 0
+
+        for doc in documents:
+            doc_tokens = self._estimate_tokens(doc.content)
+            if used_tokens + doc_tokens <= budget:
+                reduced.append(doc)
+                used_tokens += doc_tokens
+                continue
+
+            remaining = budget - used_tokens
+            if remaining <= 0:
+                break
+
+            trimmed_content = self._truncate_to_tokens(doc.content, remaining)
+            if trimmed_content.strip():
+                reduced.append(
+                    replace(
+                        doc,
+                        content=trimmed_content,
+                        metadata={**doc.metadata, "trimmed": True},
+                    )
+                )
+                used_tokens = budget
+                break
+
+        logger.info("ContextReducer used %d/%d tokens", used_tokens, budget)
+        return reduced
+
+    def _estimate_tokens(self, text: str) -> int:
+        """Estimate token count from whitespace-separated words."""
+
+        return max(1, int(len(text.split()) * 1.2))
+
+    def _truncate_to_tokens(self, text: str, max_tokens: int) -> str:
+        """Truncate text to an approximate token count."""
+
+        words = text.split()
+        max_words = max(1, int(max_tokens / 1.2))
+        return " ".join(words[:max_words])

raglib_py-0.1.3/raglib/components/decision.py

@@ -0,0 +1,63 @@
+"""Decision engine for retrieval gating and mode selection."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from raglib.llm.base_client import BaseLLMClient
+
+logger = logging.getLogger(__name__)
+
+RETRIEVAL_VECTOR: str = "vector"
+RETRIEVAL_WEB: str = "web"
+RETRIEVAL_HYBRID: str = "hybrid"
+RETRIEVAL_NONE: str = "none"
+ALLOWED_RETRIEVAL_TYPES = {
+    RETRIEVAL_VECTOR,
+    RETRIEVAL_WEB,
+    RETRIEVAL_HYBRID,
+    RETRIEVAL_NONE,
+}
+
+
+class DecisionEngine:
+    """Decides whether retrieval is needed and what retrieval mode to use."""
+
+    def __init__(self, llm_client: Optional[BaseLLMClient] = None):
+        """Initialize decision policy with an optional LLM client."""
+
+        self.llm_client = llm_client
+
+    def should_retrieve(self, query: str) -> bool:
+        """Return whether retrieval should run for this query."""
+
+        route = self.retrieval_type(query)
+        decision = route != RETRIEVAL_NONE
+        logger.info("DecisionEngine should_retrieve=%s route=%s", decision, route)
+        return decision
+
+    def retrieval_type(self, query: str) -> str:
+        """Return one of: vector, web, hybrid, none."""
+
+        if self.llm_client is not None:
+            prompt = (
+                "Decide retrieval_type for this query. Allowed: vector, web, hybrid, none.\n"
+                f"Query: {query}\n"
+                "Return one token only."
+            )
+            response = self.llm_client.complete(prompt=prompt).strip().lower()
+            if response in ALLOWED_RETRIEVAL_TYPES:
+                logger.info("DecisionEngine selected via LLM: %s", response)
+                return response
+
+        lower = query.lower().strip()
+        if not lower:
+            return RETRIEVAL_NONE
+        if any(token in lower for token in ("hello", "thanks", "hi", "good morning")):
+            return RETRIEVAL_NONE
+        if any(token in lower for token in ("latest", "today", "news", "current", "web")):
+            return RETRIEVAL_WEB
+        if any(token in lower for token in ("compare", "versus", "vs", "trend")):
+            return RETRIEVAL_HYBRID
+        return RETRIEVAL_VECTOR

raglib_py-0.1.3/raglib/components/deduplicator.py

@@ -0,0 +1,58 @@
+"""Near-duplicate removal for retrieved document sets."""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import List, Set
+
+from raglib.schemas import Document
+
+logger = logging.getLogger(__name__)
+
+
+class Deduplicator:
+    """Removes semantically similar duplicate document chunks."""
+
+    def __init__(self, similarity_threshold: float = 0.85):
+        """Initialize deduplication threshold."""
+
+        self.similarity_threshold = similarity_threshold
+
+    def deduplicate(self, documents: List[Document]) -> List[Document]:
+        """Return documents after near-duplicate filtering."""
+
+        unique_docs: List[Document] = []
+        signatures: List[Set[str]] = []
+
+        for doc in documents:
+            signature = self._signature(doc.content)
+            if not signatures:
+                unique_docs.append(doc)
+                signatures.append(signature)
+                continue
+
+            is_duplicate = any(
+                self._jaccard(signature, existing) >= self.similarity_threshold
+                for existing in signatures
+            )
+            if not is_duplicate:
+                unique_docs.append(doc)
+                signatures.append(signature)
+
+        logger.info("Deduplicator kept %d/%d documents", len(unique_docs), len(documents))
+        return unique_docs
+
+    def _signature(self, text: str) -> Set[str]:
+        """Build a token-set signature for similarity checks."""
+
+        return set(re.findall(r"[a-zA-Z0-9]+", text.lower()))
+
+    def _jaccard(self, a: Set[str], b: Set[str]) -> float:
+        """Compute Jaccard similarity between token signatures."""
+
+        if not a and not b:
+            return 1.0
+        if not a or not b:
+            return 0.0
+        return len(a & b) / len(a | b)

raglib_py-0.1.3/raglib/components/evaluator.py

@@ -0,0 +1,57 @@
+"""Retrieval quality evaluator used for corrective feedback loops."""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import replace
+from typing import List, Set
+
+from raglib.schemas import Document
+
+logger = logging.getLogger(__name__)
+
+
+class Evaluator:
+    """Scores relevance and filters low-quality retrieval results."""
+
+    def __init__(self, relevance_threshold: float = 0.3):
+        """Initialize evaluator with a minimum relevance threshold."""
+
+        self.relevance_threshold = relevance_threshold
+
+    def evaluate(self, query: str, documents: List[Document]) -> List[Document]:
+        """Filter documents by relevance score threshold."""
+
+        passed: List[Document] = []
+        for doc in documents:
+            score = self.score_document(query, doc)
+            if score >= self.relevance_threshold:
+                passed.append(replace(doc, score=score))
+
+        logger.info(
+            "Evaluator kept %d/%d documents with threshold %.2f",
+            len(passed),
+            len(documents),
+            self.relevance_threshold,
+        )
+        return passed
+
+    def score_document(self, query: str, doc: Document) -> float:
+        """Compute a relevance score for a single document."""
+
+        query_terms = self._tokenize(query)
+        doc_terms = self._tokenize(doc.content)
+        if not query_terms or not doc_terms:
+            return 0.0
+
+        overlap = len(query_terms & doc_terms) / len(query_terms)
+        density = len(query_terms & doc_terms) / max(len(doc_terms), 1)
+        phrase_bonus = 0.2 if query.lower() in doc.content.lower() else 0.0
+        score = min(1.0, overlap * 0.7 + density * 0.3 + phrase_bonus)
+        return score
+
+    def _tokenize(self, text: str) -> Set[str]:
+        """Tokenize text into a normalized set of terms."""
+
+        return set(re.findall(r"[a-zA-Z0-9]+", text.lower()))