haiku.rag 0.13.2__py3-none-any.whl → 0.13.3__py3-none-any.whl

haiku/rag/app.py

@@ -81,13 +81,10 @@ class HaikuRAGApp:
                 raw = rows[0].get("settings") or "{}"
                 data = json.loads(raw) if isinstance(raw, str) else (raw or {})
                 stored_version = str(data.get("version", stored_version))
-                embed_provider = data.get("EMBEDDINGS_PROVIDER")
-                embed_model = data.get("EMBEDDINGS_MODEL")
-                vector_dim = (
-                    int(data.get("EMBEDDINGS_VECTOR_DIM"))  # pyright: ignore[reportArgumentType]
-                    if data.get("EMBEDDINGS_VECTOR_DIM") is not None
-                    else None
-                )
+                embeddings = data.get("embeddings", {})
+                embed_provider = embeddings.get("provider")
+                embed_model = embeddings.get("model")
+                vector_dim = embeddings.get("vector_dim")
 
         num_docs = 0
         if "documents" in table_names:
@@ -195,9 +192,9 @@ class HaikuRAGApp:
                     f"[yellow]Document with id {doc_id} not found.[/yellow]"
                 )
 
-    async def search(self, query: str, limit: int = 5):
+    async def search(self, query: str, limit: int = 5, filter: str | None = None):
         async with HaikuRAG(db_path=self.db_path) as self.client:
-            results = await self.client.search(query, limit=limit)
+            results = await self.client.search(query, limit=limit, filter=filter)
             if not results:
                 self.console.print("[yellow]No results found.[/yellow]")
                 return

haiku/rag/cli.py

@@ -221,6 +221,12 @@ def search(
         "-l",
         help="Maximum number of results to return",
     ),
+    filter: str | None = typer.Option(
+        None,
+        "--filter",
+        "-f",
+        help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
+    ),
     db: Path = typer.Option(
         Config.storage.data_dir / "haiku.rag.lancedb",
         "--db",
@@ -230,7 +236,7 @@ def search(
     from haiku.rag.app import HaikuRAGApp
 
     app = HaikuRAGApp(db_path=db)
-    asyncio.run(app.search(query=query, limit=limit))
+    asyncio.run(app.search(query=query, limit=limit, filter=filter))
 
 
 @cli.command("ask", help="Ask a question using the QA agent")

haiku/rag/client.py

@@ -135,9 +135,6 @@ class HaikuRAG:
             ValueError: If the file/URL cannot be parsed or doesn't exist
             httpx.RequestError: If URL request fails
         """
-        # Lazy import to avoid loading docling
-        from haiku.rag.reader import FileReader
-
         # Normalize metadata
         metadata = metadata or {}
 
@@ -157,15 +154,17 @@ class HaikuRAG:
 
         # Handle directories
         if source_path.is_dir():
+            from haiku.rag.monitor import FileFilter
+
             documents = []
-            supported_extensions = set(FileReader.extensions)
-            for file_path in source_path.rglob("*"):
-                if (
-                    file_path.is_file()
-                    and file_path.suffix.lower() in supported_extensions
-                ):
+            filter = FileFilter(
+                ignore_patterns=self._config.monitor.ignore_patterns or None,
+                include_patterns=self._config.monitor.include_patterns or None,
+            )
+            for path in source_path.rglob("*"):
+                if path.is_file() and filter.include_file(str(path)):
                     doc = await self._create_document_from_file(
-                        file_path, title=None, metadata=metadata
+                        path, title=None, metadata=metadata
                     )
                     documents.append(doc)
             return documents
@@ -424,7 +423,11 @@ class HaikuRAG:
         return await self.document_repository.list_all(limit=limit, offset=offset)
 
     async def search(
-        self, query: str, limit: int = 5, search_type: str = "hybrid"
+        self,
+        query: str,
+        limit: int = 5,
+        search_type: str = "hybrid",
+        filter: str | None = None,
     ) -> list[tuple[Chunk, float]]:
         """Search for relevant chunks using the specified search method with optional reranking.
 
@@ -432,6 +435,7 @@ class HaikuRAG:
             query: The search query string.
             limit: Maximum number of results to return.
             search_type: Type of search - "vector", "fts", or "hybrid" (default).
+            filter: Optional SQL WHERE clause to filter documents before searching chunks.
 
         Returns:
             List of (chunk, score) tuples ordered by relevance.
@@ -441,12 +445,12 @@ class HaikuRAG:
 
         if reranker is None:
             # No reranking - return direct search results
-            return await self.chunk_repository.search(query, limit, search_type)
+            return await self.chunk_repository.search(query, limit, search_type, filter)
 
         # Get more initial results (3X) for reranking
         search_limit = limit * 3
         search_results = await self.chunk_repository.search(
-            query, search_limit, search_type
+            query, search_limit, search_type, filter
         )
 
         # Apply reranking

haiku/rag/monitor.py

@@ -40,6 +40,14 @@ class FileFilter(DefaultFilter):
         super().__init__()
 
     def __call__(self, change: Change, path: str) -> bool:
+        if not self.include_file(path):
+            return False
+
+        # Apply default watchfiles filter
+        return super().__call__(change, path)
+
+    def include_file(self, path: str) -> bool:
+        """Check if a file should be included based on filters."""
         # Check extension filter
         if not path.endswith(self.extensions):
             return False
@@ -54,8 +62,7 @@ class FileFilter(DefaultFilter):
             if self.ignore_spec.match_file(path):
                 return False
 
-        # Apply default watchfiles filter
-        return super().__call__(change, path)
+        return True
 
 
 class FileWatcher:

haiku/rag/reranking/__init__.py

@@ -41,5 +41,23 @@ def get_reranker(config: AppConfig = Config) -> RerankerBase | None:
         except ImportError:
             reranker = None
 
+    elif config.reranking.provider == "vllm":
+        try:
+            from haiku.rag.reranking.vllm import VLLMReranker
+
+            reranker = VLLMReranker(config.reranking.model)
+        except ImportError:
+            reranker = None
+
+    elif config.reranking.provider == "zeroentropy":
+        try:
+            from haiku.rag.reranking.zeroentropy import ZeroEntropyReranker
+
+            # Use configured model or default to zerank-1
+            model = config.reranking.model or "zerank-1"
+            reranker = ZeroEntropyReranker(model)
+        except ImportError:
+            reranker = None
+
     _reranker_cache[config_id] = reranker
     return reranker

haiku/rag/reranking/zeroentropy.py

@@ -0,0 +1,59 @@
+from zeroentropy import ZeroEntropy
+
+from haiku.rag.reranking.base import RerankerBase
+from haiku.rag.store.models.chunk import Chunk
+
+
+class ZeroEntropyReranker(RerankerBase):
+    """Zero Entropy reranker implementation using the zerank-1 model."""
+
+    def __init__(self, model: str = "zerank-1"):
+        """Initialize the Zero Entropy reranker.
+
+        Args:
+            model: The Zero Entropy model to use (default: "zerank-1")
+        """
+        self._model = model
+        # Zero Entropy SDK reads ZEROENTROPY_API_KEY from environment by default
+        self._client = ZeroEntropy()
+
+    async def rerank(
+        self, query: str, chunks: list[Chunk], top_n: int = 10
+    ) -> list[tuple[Chunk, float]]:
+        """Rerank the given chunks based on relevance to the query.
+
+        Args:
+            query: The query to rank against
+            chunks: The chunks to rerank
+            top_n: The number of top results to return
+
+        Returns:
+            A list of (chunk, score) tuples, sorted by relevance
+        """
+        if not chunks:
+            return []
+
+        # Prepare documents for Zero Entropy API
+        documents = [chunk.content for chunk in chunks]
+
+        # Call Zero Entropy reranking API
+        response = self._client.models.rerank(
+            model=self._model,
+            query=query,
+            documents=documents,
+        )
+
+        # Extract results and map back to chunks
+        # Zero Entropy returns results sorted by relevance with scores
+        reranked_results = []
+
+        # Get top_n results
+        for i, result in enumerate(response.results[:top_n]):
+            # Zero Entropy returns index and score for each document
+            chunk_index = result.index
+            score = result.relevance_score
+
+            if chunk_index < len(chunks):
+                reranked_results.append((chunks[chunk_index], score))
+
+        return reranked_results

haiku/rag/store/repositories/chunk.py

@@ -230,7 +230,11 @@ class ChunkRepository:
         return True
 
     async def search(
-        self, query: str, limit: int = 5, search_type: str = "hybrid"
+        self,
+        query: str,
+        limit: int = 5,
+        search_type: str = "hybrid",
+        filter: str | None = None,
     ) -> list[tuple[Chunk, float]]:
         """Search for relevant chunks using the specified search method.
 
@@ -238,6 +242,7 @@ class ChunkRepository:
             query: The search query string.
             limit: Maximum number of results to return.
             search_type: Type of search - "vector", "fts", or "hybrid" (default).
+            filter: Optional SQL WHERE clause to filter documents before searching chunks.
 
         Returns:
             List of (chunk, score) tuples ordered by relevance.
@@ -245,19 +250,42 @@ class ChunkRepository:
         if not query.strip():
             return []
 
+        chunk_where_clause = None
+        if filter:
+            # We perform filtering as a two-step process, first filtering documents, then
+            # filtering chunks based on those document IDs.
+            # This is because LanceDB does not support joins directly in search queries.
+            matching_doc_ids = self._get_filtered_document_ids(filter)
+
+            if not matching_doc_ids:
+                return []
+
+            # Build WHERE clause for chunks table
+            # Use IN clause with document IDs
+            id_list = "', '".join(matching_doc_ids)
+            chunk_where_clause = f"document_id IN ('{id_list}')"
+
         if search_type == "vector":
             query_embedding = await self.embedder.embed(query)
 
             results = self.store.chunks_table.search(
                 query_embedding, query_type="vector", vector_column_name="vector"
-            ).limit(limit)
+            )
+
+            if chunk_where_clause:
+                results = results.where(chunk_where_clause)
+
+            results = results.limit(limit)
 
             return await self._process_search_results(results)
 
         elif search_type == "fts":
-            results = self.store.chunks_table.search(query, query_type="fts").limit(
-                limit
-            )
+            results = self.store.chunks_table.search(query, query_type="fts")
+
+            if chunk_where_clause:
+                results = results.where(chunk_where_clause)
+
+            results = results.limit(limit)
             return await self._process_search_results(results)
 
         else:  # hybrid (default)
@@ -267,9 +295,13 @@ class ChunkRepository:
             reranker = RRFReranker()
 
             # Perform native hybrid search with RRF reranking
+            results = self.store.chunks_table.search(query_type="hybrid")
+
+            if chunk_where_clause:
+                results = results.where(chunk_where_clause)
+
             results = (
-                self.store.chunks_table.search(query_type="hybrid")
-                .vector(query_embedding)
+                results.vector(query_embedding)
                 .text(query)
                 .rerank(reranker)
                 .limit(limit)
@@ -332,6 +364,15 @@ class ChunkRepository:
 
         return adjacent_chunks
 
+    def _get_filtered_document_ids(self, filter: str) -> list[str]:
+        """Query documents table with filter and return matching document IDs."""
+        filtered_docs = (
+            self.store.documents_table.search()
+            .where(filter)
+            .to_pydantic(DocumentRecord)
+        )
+        return [doc.id for doc in filtered_docs]
+
     async def _process_search_results(self, query_result) -> list[tuple[Chunk, float]]:
         """Process search results into chunks with document info and scores."""
         chunks_with_scores = []

{haiku_rag-0.13.2.dist-info → haiku_rag-0.13.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.13.2
+Version: 0.13.3
 Summary: Agentic Retrieval Augmented Generation (RAG) with LanceDB
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -33,10 +33,14 @@ Requires-Dist: typer<0.20.0,>=0.19.2
 Requires-Dist: watchfiles>=1.1.1
 Provides-Extra: a2a
 Requires-Dist: fasta2a>=0.1.0; extra == 'a2a'
+Provides-Extra: cohere
+Requires-Dist: cohere>=5.0.0; extra == 'cohere'
 Provides-Extra: mxbai
 Requires-Dist: mxbai-rerank>=0.1.6; extra == 'mxbai'
 Provides-Extra: voyageai
 Requires-Dist: voyageai>=0.3.5; extra == 'voyageai'
+Provides-Extra: zeroentropy
+Requires-Dist: zeroentropy>=0.1.0a6; extra == 'zeroentropy'
 Description-Content-Type: text/markdown
 
 # Haiku RAG
@@ -56,7 +60,7 @@ Retrieval-Augmented Generation (RAG) library built on LanceDB.
 - **Multiple QA providers**: Any provider/model supported by Pydantic AI
 - **Research graph (multi‑agent)**: Plan → Search → Evaluate → Synthesize with agentic AI
 - **Native hybrid search**: Vector + full-text search with native LanceDB RRF reranking
-- **Reranking**: Default search result reranking with MixedBread AI, Cohere, or vLLM
+- **Reranking**: Default search result reranking with MixedBread AI, Cohere, Zero Entropy, or vLLM
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
 - **40+ file formats**: PDF, DOCX, HTML, Markdown, code files, URLs
@@ -79,6 +83,9 @@ haiku-rag add-src document.pdf --meta source=manual
 # Search
 haiku-rag search "query"
 
+# Search with filters
+haiku-rag search "query" --filter "uri LIKE '%.pdf' AND title LIKE '%paper%'"
+
 # Ask questions
 haiku-rag ask "Who is the author of haiku.rag?"
 

{haiku_rag-0.13.2.dist-info → haiku_rag-0.13.3.dist-info}/RECORD

@@ -1,11 +1,11 @@
 haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=hjHZoXfMvb2Pvd4sSdzQVq759zO7UGR9YuyXXA6ZbDI,21514
+haiku/rag/app.py,sha256=8035RY74-RHNwK69D2PKc-NY9KiHdIc0euLhxuno-UU,21414
 haiku/rag/chunker.py,sha256=pA0S0fFKAuvzGm2dGyp7FAkeFZA0YTCm_ata83Pnflw,1566
-haiku/rag/cli.py,sha256=Y42tnlVFGvCZVjBcLWrIVgM0A7KjNYX9MAuk9-zQvvE,14523
-haiku/rag/client.py,sha256=cG6DAhzJJ4vdo8QFn9p8iA6YTa0arMrTtIswoZc7sY0,26816
+haiku/rag/cli.py,sha256=k5T59NsV9uVugYNU5HwhVRhQSsDCYSkN3HZvxxKYEDc,14716
+haiku/rag/client.py,sha256=50yJIocbKJt7bSLUQL_aDvWG-pc38PVkEHfUvBtKUkw,26991
 haiku/rag/logging.py,sha256=dm65AwADpcQsH5OAPtRA-4hsw0w5DK-sGOvzYkj6jzw,1720
 haiku/rag/mcp.py,sha256=txuEnrUMWvs_shQBk15gEkJD7xNdSYzp3z75UUWaHFM,9328
-haiku/rag/monitor.py,sha256=EGQ-sKH1uKBSHtglehuW1COytx-4zKCKAT198tTr0Fo,4889
+haiku/rag/monitor.py,sha256=6Vgty7UxyvFkGsxC5q8DVsHQZdlr1SW-CkieIfn8dq4,5090
 haiku/rag/reader.py,sha256=aW8LG0X31kVWS7kU2tKVpe8RqP3Ne_oIidd_X3UDLH0,3307
 haiku/rag/utils.py,sha256=47ehVYJlLz6Of_Ua89qj94JclO5ZPBFU9eyonifvnVg,6131
 haiku/rag/a2a/__init__.py,sha256=tY_jLSUM0zKzyBctMkjpqmDWpxWc9QVEK1qAsb-plGs,5933
@@ -45,11 +45,12 @@ haiku/rag/qa/deep/models.py,sha256=siZMQXD21_3nk8kaLCv0BCuD9TydLYo-yC4-9CxQy3E,6
 haiku/rag/qa/deep/nodes.py,sha256=XbDujD_xg-NsJS2oYm3LqkfxeHZCzT2f9wBNhVh0wns,10442
 haiku/rag/qa/deep/prompts.py,sha256=t1fEvoD5Rdab92eAOvefv2wBVmkPFuR0BQ8Kh4X0-mY,2565
 haiku/rag/qa/deep/state.py,sha256=ICFIX0oRtBs6Sdq9YnmnP04BkGiQYwucfS3Mf8XLcCU,570
-haiku/rag/reranking/__init__.py,sha256=34xH2YZ7OCC3H8Yb-zyPuehTRQdijMSY9TC8AmZDOGQ,1296
+haiku/rag/reranking/__init__.py,sha256=cwkydVEJr7Tgs4uAWB057y9j5N3F1BDO-71YJNVkL-s,1900
 haiku/rag/reranking/base.py,sha256=Yji15nAR8LyIJGqZvEZifTWmortNQ4k_7ZHst_5mRYk,408
 haiku/rag/reranking/cohere.py,sha256=BhBPPnaSnDoVlkL_MHF74kegXQBrsZGKnWqC40ztiAk,1050
 haiku/rag/reranking/mxbai.py,sha256=qR55dmpaBz15lSN_wXD3-Z6Kqr_bmNKU9q4Pwef_wB8,911
 haiku/rag/reranking/vllm.py,sha256=Ip83qzV2RM7qXTj0mE2St66hvXykovoNW8Hu3AUebDc,1489
+haiku/rag/reranking/zeroentropy.py,sha256=bVW5gcdSEz8A97xVSD0jhWGN1l4lUZ10I-5vufINGKE,1913
 haiku/rag/research/__init__.py,sha256=xfVzYkt8QETjZaP8v4LdK8MC2R_JmxKDD34LefvrJbo,201
 haiku/rag/research/common.py,sha256=mrKXolTnDxcONjodmSPVAtXYWqbU0Bie2W-4lOTclGY,2988
 haiku/rag/research/dependencies.py,sha256=hiGImk7HyJF4LRYnTmsLFGzY1QrGjAyPW5vb_2JQRDo,8148
@@ -64,14 +65,14 @@ haiku/rag/store/models/__init__.py,sha256=kc7Ctf53Jr483tk4QTIrcgqBbXDz4ZoeYSkFXf
 haiku/rag/store/models/chunk.py,sha256=3EuZav4QekJIeHBCub48EM8SjNX8HEJ6wVDXGot4PEQ,421
 haiku/rag/store/models/document.py,sha256=cZXy_jEti-hnhq7FKhuhCfd99ccY9fIHMLovB_Thbb8,425
 haiku/rag/store/repositories/__init__.py,sha256=Olv5dLfBQINRV3HrsfUpjzkZ7Qm7goEYyMNykgo_DaY,291
-haiku/rag/store/repositories/chunk.py,sha256=bXa-NBfLdLKJuFLGEKQhFlsLi-XNbojhQYVyBjwUxz8,14205
+haiku/rag/store/repositories/chunk.py,sha256=W3aVKZWQfKULT9T3sNOR6rjIjZv-ONLu05Zm9DjjUuY,15684
 haiku/rag/store/repositories/document.py,sha256=UOC_5QEUl-3RnGPJzn92KjrCnhmh5TmWln4yd5cJ4Ss,8133
 haiku/rag/store/repositories/settings.py,sha256=15gS7Xj7cG4qetv_ioxZO_r31by7GuSqtpowOsMkHmc,6129
 haiku/rag/store/upgrades/__init__.py,sha256=RQ8A6rEXBASLb5PD9vdDnEas_m_GgRzzdVu4B88Snqc,1975
 haiku/rag/store/upgrades/v0_10_1.py,sha256=qNGnxj6hoHaHJ1rKTiALfw0c9NQOi0KAK-VZCD_073A,1959
 haiku/rag/store/upgrades/v0_9_3.py,sha256=NrjNilQSgDtFWRbL3ZUtzQzJ8tf9u0dDRJtnDFwwbdw,3322
-haiku_rag-0.13.2.dist-info/METADATA,sha256=YyvjWsHy7Y8DX8-spZcc7wzUsb7yn2okaMS3SiWw2Nk,8179
-haiku_rag-0.13.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.13.2.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.13.2.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.13.2.dist-info/RECORD,,
+haiku_rag-0.13.3.dist-info/METADATA,sha256=eSnLerdKwrwbhBX5UXjOfXTBfCT0gNT4b1cjI7E0ZqQ,8453
+haiku_rag-0.13.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.13.3.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.13.3.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.13.3.dist-info/RECORD,,