haiku.rag 0.11.1__py3-none-any.whl → 0.11.2__py3-none-any.whl

haiku/rag/client.py

@@ -1,4 +1,5 @@
 import hashlib
+import logging
 import mimetypes
 import tempfile
 from collections.abc import AsyncGenerator
@@ -18,6 +19,8 @@ from haiku.rag.store.repositories.document import DocumentRepository
 from haiku.rag.store.repositories.settings import SettingsRepository
 from haiku.rag.utils import text_to_docling_document
 
+logger = logging.getLogger(__name__)
+
 
 class HaikuRAG:
     """High-level haiku-rag client."""
@@ -538,8 +541,8 @@ class HaikuRAG:
         """Rebuild the database by deleting all chunks and re-indexing all documents.
 
         For documents with URIs:
-        - Deletes the document and re-adds it from source if source exists
-        - Skips documents where source no longer exists
+        - Re-adds from source if source exists
+        - Re-embeds from existing content if source is missing
 
         For documents without URIs:
         - Re-creates chunks from existing content
@@ -559,29 +562,51 @@ class HaikuRAG:
         for doc in documents:
             assert doc.id is not None, "Document ID should not be None"
             if doc.uri:
-                # Document has a URI - delete and try to re-add from source
-                try:
-                    # Delete the old document first
-                    await self.delete_document(doc.id)
+                # Document has a URI - check if source is accessible
+                source_accessible = False
+                parsed_url = urlparse(doc.uri)
 
-                    # Try to re-create from source (this creates the document with chunks)
-                    new_doc = await self.create_document_from_source(
-                        source=doc.uri, metadata=doc.metadata or {}
+                try:
+                    if parsed_url.scheme == "file":
+                        # Check if file exists
+                        source_path = Path(parsed_url.path)
+                        source_accessible = source_path.exists()
+                    elif parsed_url.scheme in ("http", "https"):
+                        # For URLs, we'll try to create and catch errors
+                        source_accessible = True
+                    else:
+                        source_accessible = False
+                except Exception:
+                    source_accessible = False
+
+                if source_accessible:
+                    # Source exists - delete and recreate from source
+                    try:
+                        await self.delete_document(doc.id)
+                        new_doc = await self.create_document_from_source(
+                            source=doc.uri, metadata=doc.metadata or {}
+                        )
+                        assert new_doc.id is not None, (
+                            "New document ID should not be None"
+                        )
+                        yield new_doc.id
+                    except Exception as e:
+                        logger.error(
+                            "Error recreating document from source %s: %s",
+                            doc.uri,
+                            e,
+                        )
+                        continue
+                else:
+                    # Source missing - re-embed from existing content
+                    logger.warning(
+                        "Source missing for %s, re-embedding from content", doc.uri
                     )
-
-                    assert new_doc.id is not None, "New document ID should not be None"
-                    yield new_doc.id
-
-                except (FileNotFoundError, ValueError, OSError) as e:
-                    # Source doesn't exist or can't be accessed - document already deleted, skip
-                    print(f"Skipping document with URI {doc.uri}: {e}")
-                    continue
-                except Exception as e:
-                    # Unexpected error - log it and skip
-                    print(
-                        f"Unexpected error processing document with URI {doc.uri}: {e}"
+                    docling_document = text_to_docling_document(doc.content)
+                    await self.chunk_repository.create_chunks_for_document(
+                        doc.id, docling_document
                     )
-                    continue
+                    yield doc.id
             else:
                 # Document without URI - re-create chunks from existing content
                 docling_document = text_to_docling_document(doc.content)

haiku/rag/config.py

@@ -20,8 +20,8 @@ class AppConfig(BaseModel):
     MONITOR_DIRECTORIES: list[Path] = []
 
     EMBEDDINGS_PROVIDER: str = "ollama"
-    EMBEDDINGS_MODEL: str = "mxbai-embed-large"
-    EMBEDDINGS_VECTOR_DIM: int = 1024
+    EMBEDDINGS_MODEL: str = "qwen3-embedding"
+    EMBEDDINGS_VECTOR_DIM: int = 4096
 
     RERANK_PROVIDER: str = ""
     RERANK_MODEL: str = ""

haiku/rag/embeddings/ollama.py

@@ -7,6 +7,8 @@ from haiku.rag.embeddings.base import EmbedderBase
 class Embedder(EmbedderBase):
     async def embed(self, text: str | list[str]) -> list[float] | list[list[float]]:
         client = AsyncOpenAI(base_url=f"{Config.OLLAMA_BASE_URL}/v1", api_key="dummy")
+        if not text:
+            return []
         response = await client.embeddings.create(
             model=self._model,
             input=text,

haiku/rag/embeddings/openai.py

@@ -6,6 +6,8 @@ from haiku.rag.embeddings.base import EmbedderBase
 class Embedder(EmbedderBase):
     async def embed(self, text: str | list[str]) -> list[float] | list[list[float]]:
         client = AsyncOpenAI()
+        if not text:
+            return []
         response = await client.embeddings.create(
             model=self._model,
             input=text,

haiku/rag/embeddings/vllm.py

@@ -9,6 +9,8 @@ class Embedder(EmbedderBase):
         client = AsyncOpenAI(
             base_url=f"{Config.VLLM_EMBEDDINGS_BASE_URL}/v1", api_key="dummy"
         )
+        if not text:
+            return []
         response = await client.embeddings.create(
             model=self._model,
             input=text,

haiku/rag/embeddings/voyageai.py

@@ -6,6 +6,8 @@ try:
     class Embedder(EmbedderBase):
         async def embed(self, text: str | list[str]) -> list[float] | list[list[float]]:
             client = Client()
+            if not text:
+                return []
             if isinstance(text, str):
                 res = client.embed([text], model=self._model, output_dtype="float")
                 return res.embeddings[0]  # type: ignore[return-value]

haiku/rag/monitor.py

@@ -1,13 +1,13 @@
+import logging
 from pathlib import Path
 
 from watchfiles import Change, DefaultFilter, awatch
 
 from haiku.rag.client import HaikuRAG
-from haiku.rag.logging import get_logger
 from haiku.rag.reader import FileReader
 from haiku.rag.store.models.document import Document
 
-logger = get_logger()
+logger = logging.getLogger(__name__)
 
 
 class FileFilter(DefaultFilter):

haiku/rag/reranking/__init__.py

@@ -1,3 +1,5 @@
+import os
+
 from haiku.rag.config import Config
 from haiku.rag.reranking.base import RerankerBase
 
@@ -17,6 +19,7 @@ def get_reranker() -> RerankerBase | None:
         try:
             from haiku.rag.reranking.mxbai import MxBAIReranker
 
+            os.environ["TOKENIZERS_PARALLELISM"] = "true"
             _reranker = MxBAIReranker()
             return _reranker
         except ImportError:

haiku/rag/store/repositories/settings.py

@@ -133,17 +133,17 @@ class SettingsRepository:
 
         if stored_provider and stored_provider != current_provider:
             incompatible_changes.append(
-                f"Embedding provider changed from '{stored_provider}' to '{current_provider}'"
+                f"Stored (db) embedding provider: '{stored_provider}' -> Environment (current) embedding provider: '{current_provider}'"
             )
 
         if stored_model and stored_model != current_model:
             incompatible_changes.append(
-                f"Embedding model changed from '{stored_model}' to '{current_model}'"
+                f"Stored (db) embedding model '{stored_model}' -> Environment (current) embedding model '{current_model}'"
             )
 
         if stored_vector_dim and stored_vector_dim != current_vector_dim:
             incompatible_changes.append(
-                f"Vector dimension changed from {stored_vector_dim} to {current_vector_dim}"
+                f"Stored (db) embedding vector dimension {stored_vector_dim} -> Environment (current) embedding vector dimension {current_vector_dim}"
             )
 
         if incompatible_changes:

{haiku_rag-0.11.1.dist-info → haiku_rag-0.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.11.1
+Version: 0.11.2
 Summary: Agentic Retrieval Augmented Generation (RAG) with LanceDB
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT

{haiku_rag-0.11.1.dist-info → haiku_rag-0.11.2.dist-info}/RECORD

@@ -2,24 +2,24 @@ haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 haiku/rag/app.py,sha256=TRFwMP9mzLaM7EPc7dhsPODKZxCDkSSgPCnGAdj65VU,17929
 haiku/rag/chunker.py,sha256=PVe6ysv8UlacUd4Zb3_8RFWIaWDXnzBAy2VDJ4TaUsE,1555
 haiku/rag/cli.py,sha256=wreAxyXSRnn7f09t9SGe4uAXQjlieUQIpNpOapJT7y8,12910
-haiku/rag/client.py,sha256=iUaa6YUac3CXFniIm8DsaaNsiyHsi4cp8-fPhF5XuVU,22925
-haiku/rag/config.py,sha256=SEV2OzaKavYwHZ0LmRzBj-0dbI6YFIRuNiTw9el7SO0,2307
+haiku/rag/client.py,sha256=fz5bZP1KNWbc9cvpEC8puMBHEvt-vVtFjRMItt_WD0M,23920
+haiku/rag/config.py,sha256=c2WoaieI3-HAWb6lCmVnJHY22NXl2SGLsndRbiqCzeA,2305
 haiku/rag/logging.py,sha256=dm65AwADpcQsH5OAPtRA-4hsw0w5DK-sGOvzYkj6jzw,1720
 haiku/rag/mcp.py,sha256=H7XibtSNUviFeaJVsXzHiRqUm0nJCpA7A1QHuBv6SKQ,5057
 haiku/rag/migration.py,sha256=zm0-60PiS1hIQnZz65B7qfsgM7GwZVXFqMFowjpVBs8,11058
-haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
+haiku/rag/monitor.py,sha256=VP3bqY0mEodOP60eN4RMldgrL1ti5gMjuDuQ-_vBvFc,2759
 haiku/rag/reader.py,sha256=aW8LG0X31kVWS7kU2tKVpe8RqP3Ne_oIidd_X3UDLH0,3307
 haiku/rag/utils.py,sha256=dBzhKaOHI9KRiJqHErcXUnqtnXY2AgOK8PCLA3rhO0A,6115
 haiku/rag/embeddings/__init__.py,sha256=44IfDITGIFTflGT6UEmiYOwpWFVbYv5smLY59D0YeCs,1419
 haiku/rag/embeddings/base.py,sha256=BnSviKrlzjv3L0sZJs_T-pxfawd-bcTak-rsX-D2f3A,497
-haiku/rag/embeddings/ollama.py,sha256=LuLlHH6RGoO9_gFCIlbmesuXOj017gTw6z-p8Ez0CfE,595
-haiku/rag/embeddings/openai.py,sha256=fIFCk-jpUtaW0xsnrQnJ824O0UCjaGG2sgvBzREhilc,503
-haiku/rag/embeddings/vllm.py,sha256=vhaUnCn6VMkfSluLhWKtSV-sekFaPsp4pKo2N7-SBCY,626
-haiku/rag/embeddings/voyageai.py,sha256=UW-MW4tJKnPB6Fs2P7A3yt-ZeRm46H9npckchSriPX8,661
+haiku/rag/embeddings/ollama.py,sha256=c1BeKTgpymniZw1sm4iAIdK5vA0MYoRzHLcd2_pFA44,638
+haiku/rag/embeddings/openai.py,sha256=bwoUVlzu9UtbDpN7CtG6OPt0d5tfJNeje4lR81Btpl0,546
+haiku/rag/embeddings/vllm.py,sha256=7ocp9D9bD1R5rqRIC4-Vih9VlKQNuD429k8-9wu234E,669
+haiku/rag/embeddings/voyageai.py,sha256=I4kVdT2KPtwcbjxD22GWJmgcIQIEEHpkOY2_QbFh7mQ,712
 haiku/rag/qa/__init__.py,sha256=Sl7Kzrg9CuBOcMF01wc1NtQhUNWjJI0MhIHfCWrb8V4,434
 haiku/rag/qa/agent.py,sha256=rtUkEmnD8lMHIxpPPVY6TdmF4aSlZnLjad5eDefrlBw,3145
 haiku/rag/qa/prompts.py,sha256=Lqwn3m4zCsu_CJiC4s9cLsuPNbb9nq6j2PqEF3lw1eA,3380
-haiku/rag/reranking/__init__.py,sha256=IRXHs4qPu6VbGJQpzSwhgtVWWumURH_vEoVFE-extlo,894
+haiku/rag/reranking/__init__.py,sha256=95ApqN51rcog9MLkTh_uNE69qOVozO1Z6KMbZZj8nH0,963
 haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,405
 haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
 haiku/rag/reranking/mxbai.py,sha256=uveGFIdmNmepd2EQsvYr64wv0ra2_wB845hdSZXy5Cw,908
@@ -44,12 +44,12 @@ haiku/rag/store/models/document.py,sha256=cZXy_jEti-hnhq7FKhuhCfd99ccY9fIHMLovB_
 haiku/rag/store/repositories/__init__.py,sha256=Olv5dLfBQINRV3HrsfUpjzkZ7Qm7goEYyMNykgo_DaY,291
 haiku/rag/store/repositories/chunk.py,sha256=UfajEWf5VmMuSozGRDlWBjJNR0ngvOVFDrp6_augzBg,15217
 haiku/rag/store/repositories/document.py,sha256=C9GbIl8sa2-Djaml4hlaPTtjV2HwHaz_Wzs35sdbdhg,7876
-haiku/rag/store/repositories/settings.py,sha256=7XMBMavU8zRgdBoQzQg0Obfa7UKjuVnBugidTC6sEW0,5548
+haiku/rag/store/repositories/settings.py,sha256=ObrDrzxHn-yA1WcbgIoJoVmAbVvQHAFvEdRyJFt5Opc,5685
 haiku/rag/store/upgrades/__init__.py,sha256=RQ8A6rEXBASLb5PD9vdDnEas_m_GgRzzdVu4B88Snqc,1975
 haiku/rag/store/upgrades/v0_10_1.py,sha256=qNGnxj6hoHaHJ1rKTiALfw0c9NQOi0KAK-VZCD_073A,1959
 haiku/rag/store/upgrades/v0_9_3.py,sha256=NrjNilQSgDtFWRbL3ZUtzQzJ8tf9u0dDRJtnDFwwbdw,3322
-haiku_rag-0.11.1.dist-info/METADATA,sha256=hNE2XuctQAN0UeUYW1l0eVYDioV0iHNy3ZfMmDjFI_M,6542
-haiku_rag-0.11.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.11.1.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.11.1.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.11.1.dist-info/RECORD,,
+haiku_rag-0.11.2.dist-info/METADATA,sha256=Eij1eM8K5MOYM1QZccm96n6v97r9x-lS3RG8tdPcXPw,6542
+haiku_rag-0.11.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.11.2.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.11.2.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.11.2.dist-info/RECORD,,