raglite-toolkit 1.2.0__tar.gz → 1.2.1__tar.gz

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.2.1] - 2026-10-02
+
+### Fixed
+- **URL sources:** `Document.build()` no longer fails with `LoaderError: File does not exist` for web URLs, so URLs work on their own and inside a `DocumentCollection`.
+- **Custom `VectorStore` instances:** Passing a `VectorStore` subclass instance as `{"vectorStore": store}` no longer fails config validation, so the documented custom store usage works.
+- **Qdrant shared collections:** When `indexName` is set, documents sharing one Qdrant collection no longer wipe each other. `reset()`, search and index metadata are now scoped to each document's namespace instead of the whole collection. Without `indexName`, behaviour is unchanged.
+- **Shared `VectorStore` instances in collections:** `DocumentCollection.build()` now raises `ConfigError` when a single `VectorStore` instance would be shared by more than one document, instead of each document silently resetting the previous one's index. Pass a vector store provider config instead.
+- **Web URL re-indexing:** URL sources are now fingerprinted by their fetched content rather than the URL string, so a changed page is re-indexed on the next `build()`.
+- **Query embedder after reload:** Searching an existing index in a new process now embeds queries with the provider and model the index was built with, rather than the constructor default. Configured credentials are reused when the provider matches.
+- **Collection search errors:** `DocumentCollection.search()` (and `ask`/`ask_stream`) now logs per-document search failures instead of silently dropping them, and raises if every document fails.
+- **Web loader User-Agent:** Now reports the actual package version.
+
+### Upgrade notes
+- As with every release, cached indexes are rebuilt once on first `build()` because the package version is part of the cache key.
+- Existing Qdrant indexes created with `indexName` are rebuilt once. Their old untagged points stay in the collection but are no longer returned by search; drop and recreate the collection to remove them.
+- Each `build()` on a URL source now fetches the page to check for changes, even when the cached index is reused.
+- When every document in a collection fails to search, the REST server now returns an error response (400 for RAGLite errors, 500 otherwise) instead of empty results.
+
 ## [1.2.0] - 2026-08-02
 
 ### Added

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: raglite-toolkit
-Version: 1.2.0
+Version: 1.2.1
 Summary: Build semantic search, multi-provider question answering, and REST APIs over your documents in a few lines of Python.
 Project-URL: Homepage, https://github.com/creatorpiyush/raglite-py
 Project-URL: Repository, https://github.com/creatorpiyush/raglite-py
@@ -182,6 +182,7 @@ doc = Document("./policy.pdf", {
 ```
 
 ### Qdrant Store
+By default each document gets its own collection (`raglite_<namespace>`). Set `indexName` to keep several documents in one shared collection; each document's points are tagged with its namespace, so rebuilding one document never affects the others.
 ```python
 doc = Document("./policy.pdf", {
     "vectorStore": {

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/README.md

@@ -136,6 +136,7 @@ doc = Document("./policy.pdf", {
 ```
 
 ### Qdrant Store
+By default each document gets its own collection (`raglite_<namespace>`). Set `indexName` to keep several documents in one shared collection; each document's points are tagged with its namespace, so rebuilding one document never affects the others.
 ```python
 doc = Document("./policy.pdf", {
     "vectorStore": {

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/pyproject.toml

@@ -8,7 +8,7 @@ packages = ["src/raglite"]
 
 [project]
 name = "raglite-toolkit"
-version = "1.2.0"
+version = "1.2.1"
 description = "Build semantic search, multi-provider question answering, and REST APIs over your documents in a few lines of Python."
 readme = "README.md"
 requires-python = ">=3.10"

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/src/raglite/config.py

@@ -10,6 +10,7 @@ from .constants import (
     DEFAULT_TOP_K,
 )
 from .types import EmbeddingProviderConfig, LLMProviderConfig, VectorStoreProviderConfig
+from .vectordb.base import VectorStore
 
 
 class DocumentOptions(BaseModel):
@@ -44,6 +45,10 @@ def resolve_config(options: Optional[Union[DocumentOptions, Dict[str, Any]]] = N
     if options is None:
         opts = DocumentOptions()
     elif isinstance(options, dict):
+        # A VectorStore instance is read straight from the raw options by
+        # Document, so keep it out of the provider-config validation.
+        if isinstance(options.get("vectorStore"), VectorStore):
+            options = {k: v for k, v in options.items() if k != "vectorStore"}
         opts = DocumentOptions.model_validate(options)
     else:
         opts = options

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/src/raglite/constants.py

@@ -5,7 +5,7 @@ try:
     PACKAGE_VERSION = importlib.metadata.version("raglite-toolkit")
 except importlib.metadata.PackageNotFoundError:
     PACKAGE_NAME = "raglite-toolkit"
-    PACKAGE_VERSION = "1.2.0"  # local development fallback
+    PACKAGE_VERSION = "1.2.1"  # local development fallback
 
 
 SUPPORTED_EXTENSIONS = {".pdf", ".txt", ".json", ".md", ".markdown", ".docx"}

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/src/raglite/core/collection.py

@@ -4,11 +4,12 @@ from pathlib import Path
 from typing import Any, Dict, Generator, List, Optional, Union
 
 from ..config import DocumentOptions, ResolvedConfig, resolve_config
-from ..errors import RagLiteError
+from ..errors import ConfigError, RagLiteError
 from ..llm import generate_answer, stream_answer
 from ..loaders import DirectoryLoader, is_supported_file, is_url
 from ..types import AnswerResult, SearchResult
 from ..utils.logger import Logger, create_logger
+from ..vectordb import VectorStore
 from .document import Document
 
 
@@ -91,6 +92,16 @@ class DocumentCollection:
                     self.logger.warning(msg)
                     errors.append({"source": src, "error": msg})
 
+        # A VectorStore instance has a single namespace, so every document would
+        # share it and each build() would reset the previous document's index.
+        raw_store = self.options.get("vectorStore") if isinstance(self.options, dict) else None
+        if isinstance(raw_store, VectorStore) and len(set(file_list)) > 1:
+            raise ConfigError(
+                "A VectorStore instance cannot be shared by multiple documents in a "
+                "DocumentCollection. Pass a vector store provider config "
+                '(e.g. {"provider": "qdrant", ...}) instead.'
+            )
+
         total_chunks = 0
         cached_docs = 0
         new_docs = 0
@@ -146,12 +157,19 @@ class DocumentCollection:
         st = score_threshold if score_threshold is not None else opts.get("scoreThreshold", opts.get("score_threshold", self.config.scoreThreshold))
 
         all_hits: List[SearchResult] = []
+        failures: List[Exception] = []
         for doc in self.documents.values():
             try:
                 hits = doc.search(query, top_k=tk * 2, score_threshold=st)
                 all_hits.extend(hits)
-            except Exception:
-                continue
+            except Exception as err:
+                failures.append(err)
+                self.logger.warning(f'Search failed for document "{doc.file_path}": {err}')
+
+        # One broken document should not hide results from the others, but if
+        # every document failed the caller needs the error, not an empty list.
+        if len(failures) == len(self.documents):
+            raise failures[0]
 
         all_hits.sort(key=lambda h: h.score, reverse=True)
         return all_hits[:tk]

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/src/raglite/core/document.py

@@ -10,7 +10,13 @@ from ..errors import FileNotIndexedError, LoaderError, RagLiteError
 from ..llm import generate_answer, stream_answer
 from ..loaders import get_loader, is_url
 from ..retrieval import Retriever
-from ..types import AnswerResult, ChunkMetadata, IndexMetadata, StoredChunk
+from ..types import (
+    AnswerResult,
+    ChunkMetadata,
+    EmbeddingProviderConfig,
+    IndexMetadata,
+    StoredChunk,
+)
 from ..utils.hash import hash_file, hash_string, namespace_from_path
 from ..utils.logger import create_logger
 from ..vectordb import MemoryVectorStore, VectorStore, create_vector_store
@@ -74,16 +80,17 @@ class Document:
         if should_rebuild is None:
             should_rebuild = opts.get("rebuild", False)
 
-        if not os.path.exists(self.file_path):
+        if not is_url(self.file_path) and not os.path.exists(self.file_path):
             raise LoaderError(f"File does not exist: {self.file_path}")
 
         self.store.load()
         existing = self.store.read_index_metadata()
-        source_hash = (
-            hash_string(self.file_path)
-            if is_url(self.file_path)
-            else hash_file(self.file_path)
-        )
+        # Web pages change without their URL changing, so fingerprint the
+        # fetched content rather than the URL.
+        text: Optional[str] = None
+        if is_url(self.file_path):
+            text = get_loader(self.file_path).load()
+        source_hash = hash_string(text) if text is not None else hash_file(self.file_path)
 
         if (
             not should_rebuild
@@ -110,8 +117,8 @@ class Document:
         self.store.reset()
         self.store.load()
 
-        loader = get_loader(self.file_path)
-        text = loader.load()
+        if text is None:
+            text = get_loader(self.file_path).load()
         if not text:
             raise LoaderError(
                 f"Loader returned empty text for {self.file_path}"
@@ -340,19 +347,10 @@ class Document:
                 f'No RagLite index found for "{self.file_path}". Call build() first.'
             )
 
-        embed_config = self.config.embeddings
-        if embed_config.model is None:
-            from ..types import EmbeddingProviderConfig
-
-            embed_config = EmbeddingProviderConfig(
-                provider=embed_config.provider,
-                model=existing.embeddingModel,
-                apiKey=embed_config.apiKey,
-                baseURL=embed_config.baseURL,
-            )
-
         if self.embedder is None:
-            self.embedder = create_embedder(embed_config)
+            self.embedder = create_embedder(
+                _query_embeddings_config(existing, self.config.embeddings)
+            )
         self.ready = True
 
     def _cache_still_valid(
@@ -378,3 +376,19 @@ class Document:
         if req_model is not None and req_model != existing.embeddingModel:
             return False
         return True
+
+
+def _query_embeddings_config(
+    existing: IndexMetadata, configured: EmbeddingProviderConfig
+) -> EmbeddingProviderConfig:
+    """Queries must be embedded with the same provider and model as the index,
+    which may differ from the constructor default when ``build(embeddings=...)``
+    overrode it. Credentials from the configured provider are reused when it
+    matches; otherwise the provider falls back to its environment variables."""
+    same_provider = configured.provider == existing.embeddingProvider
+    return EmbeddingProviderConfig(
+        provider=existing.embeddingProvider,
+        model=existing.embeddingModel,
+        apiKey=configured.apiKey if same_provider else None,
+        baseURL=configured.baseURL if same_provider else None,
+    )

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/src/raglite/loaders/web.py

@@ -1,6 +1,7 @@
 import html
 import re
 
+from ..constants import PACKAGE_VERSION
 from ..errors import LoaderError
 from .base import BaseLoader
 
@@ -18,7 +19,7 @@ class WebLoader(BaseLoader):
             import httpx
 
             headers = {
-                "User-Agent": "RAGLite/1.2.0 (Python/3.10+)",
+                "User-Agent": f"RAGLite/{PACKAGE_VERSION} (Python/3.10+)",
                 "Accept": "text/html,text/plain,application/xhtml+xml;q=0.9,*/*;q=0.8",
             }
             response = httpx.get(self.url, headers=headers, follow_redirects=True, timeout=15.0)

{raglite_toolkit-1.2.0 → raglite_toolkit-1.2.1}/src/raglite/vectordb/qdrant.py

@@ -31,12 +31,22 @@ class QdrantVectorStore(VectorStore):
         self._url = url.rstrip("/")
         self._api_key = api_key
         self._collection = collection_name or f"raglite_{namespace}"
+        # With an explicit collection name, several namespaces may share one
+        # collection, so every operation is scoped to this namespace instead
+        # of touching the whole collection.
+        self._shared = collection_name is not None
+        self._metadata_id = (
+            _uuid_from_str(f"{namespace}__metadata__") if self._shared else _METADATA_UUID
+        )
         self._count_cache: int = 0
 
     @property
     def namespace(self) -> str:
         return self._namespace
 
+    def _namespace_filter(self) -> dict:
+        return {"must": [{"key": "namespace", "match": {"value": self._namespace}}]}
+
     def _headers(self) -> dict:
         h: dict = {"Content-Type": "application/json"}
         if self._api_key:
@@ -70,7 +80,14 @@ class QdrantVectorStore(VectorStore):
 
     def reset(self) -> None:
         try:
-            self._request("DELETE", f"/collections/{self._collection}")
+            if self._shared:
+                self._request(
+                    "POST",
+                    f"/collections/{self._collection}/points/delete?wait=true",
+                    {"filter": self._namespace_filter()},
+                )
+            else:
+                self._request("DELETE", f"/collections/{self._collection}")
         except VectorDBError:
             pass
         self._count_cache = 0
@@ -85,6 +102,7 @@ class QdrantVectorStore(VectorStore):
                 "vector": c.embedding,
                 "payload": {
                     "id": c.id,
+                    "namespace": self._namespace,
                     "text": c.text,
                     "metadata": c.metadata.model_dump(by_alias=True),
                     "isMetadata": False,
@@ -96,15 +114,18 @@ class QdrantVectorStore(VectorStore):
         self._count_cache += len(chunks)
 
     def search(self, embedding: List[float], top_k: int) -> List[VectorSearchHit]:
+        body: dict = {
+            "vector": embedding,
+            "limit": top_k + 1,
+            "with_payload": True,
+        }
+        if self._shared:
+            body["filter"] = self._namespace_filter()
         try:
             data = self._request(
                 "POST",
                 f"/collections/{self._collection}/points/search",
-                {
-                    "vector": embedding,
-                    "limit": top_k + 1,
-                    "with_payload": True,
-                },
+                body,
             )
         except VectorDBError:
             return []
@@ -112,7 +133,7 @@ class QdrantVectorStore(VectorStore):
         hits: List[VectorSearchHit] = []
         for r in data.get("result", []):
             payload = r.get("payload", {}) or {}
-            if r.get("id") == _METADATA_UUID or payload.get("isMetadata"):
+            if r.get("id") == self._metadata_id or payload.get("isMetadata"):
                 continue
             raw_meta = payload.get("metadata") or {}
             metadata = ChunkMetadata(
@@ -141,13 +162,14 @@ class QdrantVectorStore(VectorStore):
         zero_vec = [0.0] * dim
         payload_data = metadata.model_dump(by_alias=True)
         payload_data["isMetadata"] = True
+        payload_data["namespace"] = self._namespace
         self._request(
             "PUT",
             f"/collections/{self._collection}/points",
             {
                 "points": [
                     {
-                        "id": _METADATA_UUID,
+                        "id": self._metadata_id,
                         "vector": zero_vec,
                         "payload": payload_data,
                     }
@@ -161,7 +183,7 @@ class QdrantVectorStore(VectorStore):
             data = self._request(
                 "POST",
                 f"/collections/{self._collection}/points",
-                {"ids": [_METADATA_UUID], "with_payload": True},
+                {"ids": [self._metadata_id], "with_payload": True},
             )
         except VectorDBError:
             return None

raglite_toolkit-1.2.1/tests/unit/test_regressions.py

@@ -0,0 +1,210 @@
+"""
+Regression tests for URL indexing and shared vector stores across documents.
+"""
+import shutil
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from raglite.core.collection import DocumentCollection
+from raglite.core.document import Document
+from raglite.embeddings.local import LocalEmbedder
+from raglite.errors import ConfigError
+from raglite.types import ChunkMetadata, EmbeddingProviderConfig, StoredChunk
+from raglite.vectordb import MemoryVectorStore
+from raglite.vectordb.qdrant import QdrantVectorStore
+
+
+def unit_vec_384():
+    return [1.0] + [0.0] * 383
+
+
+@pytest.fixture
+def tmp_dir():
+    d = tempfile.mkdtemp(prefix="raglite_regress_")
+    yield d
+    shutil.rmtree(d, ignore_errors=True)
+
+
+def test_document_builds_from_url(tmp_dir):
+    with patch(
+        "raglite.loaders.web.WebLoader.load", return_value="Refunds are issued within 30 days."
+    ), patch.object(
+        LocalEmbedder,
+        "embed_documents",
+        side_effect=lambda texts: [unit_vec_384() for _ in texts],
+    ):
+        doc = Document("https://example.com/policy", {"storeDir": tmp_dir, "logLevel": "silent"})
+        result = doc.build()
+
+    assert result["chunkCount"] == 1
+    assert result["cached"] is False
+
+
+def test_collection_rejects_shared_store_instance(tmp_dir):
+    (Path(tmp_dir) / "a.txt").write_text("first document", encoding="utf-8")
+    (Path(tmp_dir) / "b.txt").write_text("second document", encoding="utf-8")
+    store = MemoryVectorStore(str(Path(tmp_dir) / ".raglite"), "shared")
+
+    collection = DocumentCollection(tmp_dir, {"vectorStore": store, "logLevel": "silent"})
+    with pytest.raises(ConfigError):
+        collection.build()
+
+
+def test_qdrant_shared_collection_is_scoped_to_namespace():
+    calls = []
+
+    def fake_request(self, method, path, body=None):
+        calls.append((self.namespace, method, path, body))
+        return {"result": []}
+
+    with patch.object(QdrantVectorStore, "_request", fake_request):
+        a = QdrantVectorStore("http://qdrant:6333", "ns-a", collection_name="shared")
+        b = QdrantVectorStore("http://qdrant:6333", "ns-b", collection_name="shared")
+
+        a.reset()
+        a.add(
+            [
+                StoredChunk(
+                    id="ns-a_000001",
+                    text="hello",
+                    embedding=[1.0, 0.0, 0.0],
+                    metadata=ChunkMetadata(source="a.txt", chunk=1, totalChunks=1),
+                )
+            ]
+        )
+        a.search([1.0, 0.0, 0.0], 3)
+        a.read_index_metadata()
+        b.read_index_metadata()
+
+    assert not any(method == "DELETE" for _, method, _, _ in calls)
+
+    delete = next(c for c in calls if c[2].endswith("/points/delete?wait=true"))
+    assert delete[3]["filter"]["must"][0]["match"]["value"] == "ns-a"
+
+    upsert = next(c for c in calls if c[1] == "PUT" and c[2].endswith("/points"))
+    assert upsert[3]["points"][0]["payload"]["namespace"] == "ns-a"
+
+    search = next(c for c in calls if c[2].endswith("/points/search"))
+    assert search[3]["filter"]["must"][0]["match"]["value"] == "ns-a"
+
+    meta_ids = [c[3]["ids"][0] for c in calls if c[1] == "POST" and c[2] == "/collections/shared/points"]
+    assert len(meta_ids) == 2
+    assert meta_ids[0] != meta_ids[1]
+
+
+def test_qdrant_dedicated_collection_keeps_existing_behaviour():
+    calls = []
+
+    def fake_request(self, method, path, body=None):
+        calls.append((method, path, body))
+        return {"result": []}
+
+    with patch.object(QdrantVectorStore, "_request", fake_request):
+        store = QdrantVectorStore("http://qdrant:6333", "ns-a")
+        store.reset()
+        store.search([1.0, 0.0, 0.0], 3)
+
+    assert ("DELETE", "/collections/raglite_ns-a", None) in calls
+    search = next(c for c in calls if c[1].endswith("/points/search"))
+    assert "filter" not in search[2]
+
+
+class FakeEmbedder:
+    def __init__(self, config):
+        self.provider = config.provider
+        self.model = config.model or "fake-model"
+        self.dimensions = 384
+
+    def embed_documents(self, texts):
+        return [unit_vec_384() for _ in texts]
+
+    def embed_query(self, text):
+        return unit_vec_384()
+
+
+@pytest.fixture
+def embedder_configs():
+    configs = []
+
+    def fake_create(config):
+        configs.append(config)
+        return FakeEmbedder(config)
+
+    with patch("raglite.core.document.create_embedder", side_effect=fake_create):
+        yield configs
+
+
+def test_url_reindexes_when_page_content_changes(tmp_dir, embedder_configs):
+    page = {"text": "Refunds are issued within 30 days."}
+    opts = {"storeDir": tmp_dir, "logLevel": "silent"}
+    url = "https://example.com/policy"
+
+    with patch("raglite.loaders.web.WebLoader.load", side_effect=lambda: page["text"]):
+        assert Document(url, opts).build()["cached"] is False
+        assert Document(url, opts).build()["cached"] is True
+        page["text"] = "Refunds are issued within 60 days."
+        assert Document(url, opts).build()["cached"] is False
+
+
+def test_query_embedder_uses_index_provider_after_reload(tmp_dir, embedder_configs):
+    path = Path(tmp_dir) / "policy.txt"
+    path.write_text("Refunds are issued within 30 days.", encoding="utf-8")
+    opts = {"storeDir": tmp_dir, "logLevel": "silent"}
+
+    Document(str(path), opts).build(
+        embeddings=EmbeddingProviderConfig(provider="openai", model="text-embedding-3-small")
+    )
+    embedder_configs.clear()
+    Document(str(path), opts).search("refund")
+
+    assert len(embedder_configs) == 1
+    assert embedder_configs[0].provider == "openai"
+    assert embedder_configs[0].model == "text-embedding-3-small"
+    assert embedder_configs[0].apiKey is None
+
+
+def test_query_embedder_reuses_credentials_for_same_provider(tmp_dir, embedder_configs):
+    path = Path(tmp_dir) / "policy.txt"
+    path.write_text("Refunds are issued within 30 days.", encoding="utf-8")
+    opts = {
+        "storeDir": tmp_dir,
+        "logLevel": "silent",
+        "embeddings": {"provider": "openai", "apiKey": "sk-test"},
+    }
+
+    Document(str(path), opts).build()
+    embedder_configs.clear()
+    Document(str(path), opts).search("refund")
+
+    assert embedder_configs[0].apiKey == "sk-test"
+
+
+def _two_doc_collection(tmp_dir):
+    (Path(tmp_dir) / "policy.txt").write_text("Refunds within 30 days.", encoding="utf-8")
+    (Path(tmp_dir) / "shipping.txt").write_text("Delivery takes 2 days.", encoding="utf-8")
+    collection = DocumentCollection(
+        tmp_dir, {"storeDir": str(Path(tmp_dir) / ".raglite"), "logLevel": "silent"}
+    )
+    collection.build()
+    return collection
+
+
+def test_collection_search_survives_one_failing_document(tmp_dir, embedder_configs):
+    collection = _two_doc_collection(tmp_dir)
+    broken = next(iter(collection.documents.values()))
+    with patch.object(broken, "search", side_effect=RuntimeError("boom")):
+        hits = collection.search("refund", top_k=5)
+    assert len(hits) == 1
+
+
+def test_collection_search_raises_when_every_document_fails(tmp_dir, embedder_configs):
+    collection = _two_doc_collection(tmp_dir)
+    docs = list(collection.documents.values())
+    with patch.object(docs[0], "search", side_effect=RuntimeError("bad api key")), patch.object(
+        docs[1], "search", side_effect=RuntimeError("bad api key")
+    ):
+        with pytest.raises(RuntimeError, match="bad api key"):
+            collection.search("refund")