qdrant-haystack 3.3.0__tar.gz → 3.4.0__tar.gz

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qdrant-haystack
-Version: 3.3.0
+Version: 3.4.0
 Summary: An integration of Qdrant ANN vector database backend with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md

qdrant_haystack-3.4.0/examples/embedding_retrieval.py

@@ -0,0 +1,52 @@
+# Install required packages for this example, including qdrant-haystack and other libraries needed
+# for Markdown conversion and embeddings generation. Use the following command:
+# pip install qdrant-haystack markdown-it-py mdit_plain sentence-transformers
+
+# Download some Markdown files to index.
+# git clone https://github.com/anakin87/neural-search-pills
+
+import glob
+
+from haystack import Pipeline
+from haystack.components.converters import MarkdownToDocument
+from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack_integrations.components.retrievers.qdrant import QdrantEmbeddingRetriever
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+
+# Initialize QdrantDocumentStore: for simplicity, we use an in-memory store here.
+# You can also run a Qdrant instance using Docker or use Qdrant Cloud.
+document_store = QdrantDocumentStore(
+    ":memory:",
+    index="Document",
+    embedding_dim=768,
+    recreate_index=True,
+)
+
+# Create the indexing Pipeline and index some documents
+file_paths = glob.glob("neural-search-pills/pills/*.md")
+
+
+indexing = Pipeline()
+indexing.add_component("converter", MarkdownToDocument())
+indexing.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=2))
+indexing.add_component("embedder", SentenceTransformersDocumentEmbedder())
+indexing.add_component("writer", DocumentWriter(document_store))
+indexing.connect("converter", "splitter")
+indexing.connect("splitter", "embedder")
+indexing.connect("embedder", "writer")
+
+indexing.run({"converter": {"sources": file_paths}})
+
+# Create the querying Pipeline and try a query
+querying = Pipeline()
+querying.add_component("embedder", SentenceTransformersTextEmbedder())
+querying.add_component("retriever", QdrantEmbeddingRetriever(document_store=document_store, top_k=3))
+querying.connect("embedder", "retriever")
+
+results = querying.run({"embedder": {"text": "What is a cross-encoder?"}})
+
+for doc in results["retriever"]["documents"]:
+    print(doc)
+    print("-" * 10)

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/pydoc/config.yml

@@ -5,8 +5,7 @@ loaders:
       [
         "haystack_integrations.components.retrievers.qdrant.retriever",
         "haystack_integrations.document_stores.qdrant.document_store",
-        "haystack_integrations.document_stores.qdrant.converters",
-        "haystack_integrations.document_stores.qdrant.filters",
+        "haystack_integrations.document_stores.qdrant.migrate_to_sparse",
       ]
     ignore_when_discovered: ["__init__"]
 processors:

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/pyproject.toml

@@ -127,6 +127,8 @@ ban-relative-imports = "parents"
 [tool.ruff.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
+# examples can contain "print" commands
+"examples/**/*" = ["T201"]
 
 
 [tool.coverage.run]

qdrant_haystack-3.4.0/src/haystack_integrations/components/retrievers/qdrant/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .retriever import QdrantEmbeddingRetriever, QdrantSparseEmbeddingRetriever
+
+__all__ = ("QdrantEmbeddingRetriever", "QdrantSparseEmbeddingRetriever")

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/src/haystack_integrations/components/retrievers/qdrant/retriever.py

@@ -124,13 +124,13 @@ class QdrantEmbeddingRetriever:
 
 
 @component
-class QdrantSparseRetriever:
+class QdrantSparseEmbeddingRetriever:
     """
     A component for retrieving documents from an QdrantDocumentStore using sparse vectors.
 
     Usage example:
     ```python
-    from haystack_integrations.components.retrievers.qdrant import QdrantSparseRetriever
+    from haystack_integrations.components.retrievers.qdrant import QdrantSparseEmbeddingRetriever
     from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
     from haystack.dataclasses.sparse_embedding import SparseEmbedding
 
@@ -140,7 +140,7 @@ class QdrantSparseRetriever:
         return_embedding=True,
         wait_result_from_api=True,
     )
-    retriever = QdrantSparseRetriever(document_store=document_store)
+    retriever = QdrantSparseEmbeddingRetriever(document_store=document_store)
     sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
     retriever.run(query_sparse_embedding=sparse_embedding)
     ```
@@ -155,7 +155,7 @@ class QdrantSparseRetriever:
         return_embedding: bool = False,
     ):
         """
-        Create a QdrantSparseRetriever component.
+        Create a QdrantSparseEmbeddingRetriever component.
 
         :param document_store: An instance of QdrantDocumentStore.
         :param filters: A dictionary with filters to narrow down the search space. Default is None.

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/src/haystack_integrations/document_stores/qdrant/__init__.py

@@ -3,5 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from .document_store import QdrantDocumentStore
+from .migrate_to_sparse import migrate_to_sparse_embeddings_support
 
-__all__ = ("QdrantDocumentStore",)
+__all__ = ("QdrantDocumentStore", "migrate_to_sparse_embeddings_support")

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/src/haystack_integrations/document_stores/qdrant/document_store.py

@@ -453,7 +453,9 @@ class QdrantDocumentStore:
                 f"Collection '{collection_name}' already exists in Qdrant, "
                 f"but it has been originally created without sparse embedding vectors. "
                 f"If you want to use that collection, you can set `use_sparse_embeddings=False`. "
-                f"To use sparse embeddings, you need to recreate the collection or migrate the existing one."
+                f"To use sparse embeddings, you need to recreate the collection or migrate the existing one. "
+                f"See `migrate_to_sparse_embeddings_support` function in "
+                f"`haystack_integrations.document_stores.qdrant`."
             )
             raise QdrantStoreError(msg)
 

qdrant_haystack-3.4.0/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py

@@ -0,0 +1,127 @@
+import logging
+import time
+
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+from qdrant_client.http import models
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.INFO)
+
+
+def migrate_to_sparse_embeddings_support(old_document_store: QdrantDocumentStore, new_index: str):
+    """
+    Utility function to migrate an existing `QdrantDocumentStore` to a new one with support for sparse embeddings.
+
+    With qdrant-hasytack v3.3.0, support for sparse embeddings has been added to `QdrantDocumentStore`.
+    This feature is disabled by default and can be enabled by setting `use_sparse_embeddings=True` in the init
+    parameters. To store sparse embeddings, Document stores/collections created with this feature disabled must be
+    migrated to a new collection with the feature enabled.
+
+    This utility function applies to on-premise and cloud instances of Qdrant.
+    It does not work for local in-memory/disk-persisted instances.
+
+    The utility function merely migrates the existing documents so that they are ready to store sparse embeddings.
+    It does not compute sparse embeddings. To do this, you need to use a Sparse Embedder component.
+
+    Example usage:
+    ```python
+    from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+    from haystack_integrations.document_stores.qdrant import migrate_to_sparse_embeddings_support
+
+    old_document_store = QdrantDocumentStore(url="http://localhost:6333",
+                                             index="Document",
+                                             use_sparse_embeddings=False)
+    new_index = "Document_sparse"
+
+    migrate_to_sparse_embeddings_support(old_document_store, new_index)
+
+    # now you can use the new document store with sparse embeddings support
+    new_document_store = QdrantDocumentStore(url="http://localhost:6333",
+                                             index=new_index,
+                                             use_sparse_embeddings=True)
+    ```
+
+
+    :param old_document_store: The existing QdrantDocumentStore instance to migrate from.
+    :param new_index: The name of the new index/collection to create with sparse embeddings support.
+    """
+
+    start = time.time()
+
+    old_collection_name = old_document_store.index
+    total_points = old_document_store.count_documents()
+
+    # copy the init parameters of the old document to create a new document store
+    init_parameters = old_document_store.to_dict()["init_parameters"]
+    init_parameters["index"] = new_index
+    init_parameters["use_sparse_embeddings"] = True
+    init_parameters["recreate_index"] = True
+
+    new_document_store = QdrantDocumentStore(**init_parameters)
+
+    client = new_document_store.client
+
+    original_indexing_threshold = client.get_collection(
+        collection_name=new_index
+    ).config.optimizer_config.indexing_threshold
+
+    # disable indexing while adding points so it's faster
+    # https://qdrant.tech/documentation/concepts/collections/#update-collection-parameters
+    client.update_collection(
+        collection_name=new_index,
+        optimizer_config=models.OptimizersConfigDiff(indexing_threshold=0),
+    )
+
+    # migration loop
+    next_page_offset = "first"
+    offset = None
+    points_transmitted = 0
+
+    while next_page_offset:
+        if next_page_offset != "first":
+            offset = next_page_offset
+
+        # get the records
+        records = client.scroll(
+            collection_name=old_collection_name,
+            limit=100,
+            with_payload=True,
+            with_vectors=True,
+            offset=offset,
+        )
+
+        next_page_offset = records[1]
+        current_records = records[0]
+
+        points = []
+
+        for record in current_records:
+            vector = {}
+
+            vector["text-dense"] = record.vector
+
+            point = {"id": record.id, "payload": record.payload, "vector": vector}
+
+            embedding_point = models.PointStruct(**point)
+            points.append(embedding_point)
+
+        client.upsert(collection_name=new_index, points=points)
+
+        points_transmitted += len(points)
+        points_remaining = total_points - points_transmitted
+
+        message = (
+            f"Points transmitted: {points_transmitted}/{total_points}\n"
+            f"Percent done {points_transmitted/total_points*100:.2f}%\n"
+            f"Time elapsed: {time.time() - start:.2f} seconds\n"
+            f"Time remaining: {(((time.time() - start) / points_transmitted) * points_remaining) / 60:.2f} minutes\n"
+            f"Current offset: {next_page_offset}\n"
+        )
+        logger.info(message)
+
+    # restore the original indexing threshold (re-enable indexing)
+    client.update_collection(
+        collection_name=new_index,
+        optimizer_config=models.OptimizersConfigDiff(indexing_threshold=original_indexing_threshold),
+    )

{qdrant_haystack-3.3.0 → qdrant_haystack-3.4.0}/tests/test_retriever.py

@@ -8,7 +8,7 @@ from haystack.testing.document_store import (
 )
 from haystack_integrations.components.retrievers.qdrant import (
     QdrantEmbeddingRetriever,
-    QdrantSparseRetriever,
+    QdrantSparseEmbeddingRetriever,
 )
 from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 
@@ -135,10 +135,10 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
             assert document.embedding is None
 
 
-class TestQdrantSparseRetriever(FilterableDocsFixtureMixin):
+class TestQdrantSparseEmbeddingRetriever(FilterableDocsFixtureMixin):
     def test_init_default(self):
         document_store = QdrantDocumentStore(location=":memory:", index="test")
-        retriever = QdrantSparseRetriever(document_store=document_store)
+        retriever = QdrantSparseEmbeddingRetriever(document_store=document_store)
         assert retriever._document_store == document_store
         assert retriever._filters is None
         assert retriever._top_k == 10
@@ -146,10 +146,10 @@ class TestQdrantSparseRetriever(FilterableDocsFixtureMixin):
 
     def test_to_dict(self):
         document_store = QdrantDocumentStore(location=":memory:", index="test")
-        retriever = QdrantSparseRetriever(document_store=document_store)
+        retriever = QdrantSparseEmbeddingRetriever(document_store=document_store)
         res = retriever.to_dict()
         assert res == {
-            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseRetriever",
+            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseEmbeddingRetriever",
             "init_parameters": {
                 "document_store": {
                     "type": "haystack_integrations.document_stores.qdrant.document_store.QdrantDocumentStore",
@@ -202,7 +202,7 @@ class TestQdrantSparseRetriever(FilterableDocsFixtureMixin):
 
     def test_from_dict(self):
         data = {
-            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseRetriever",
+            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseEmbeddingRetriever",
             "init_parameters": {
                 "document_store": {
                     "init_parameters": {"location": ":memory:", "index": "test"},
@@ -214,7 +214,7 @@ class TestQdrantSparseRetriever(FilterableDocsFixtureMixin):
                 "return_embedding": True,
             },
         }
-        retriever = QdrantSparseRetriever.from_dict(data)
+        retriever = QdrantSparseEmbeddingRetriever.from_dict(data)
         assert isinstance(retriever._document_store, QdrantDocumentStore)
         assert retriever._document_store.index == "test"
         assert retriever._filters is None
@@ -241,7 +241,7 @@ class TestQdrantSparseRetriever(FilterableDocsFixtureMixin):
             doc.sparse_embedding = SparseEmbedding.from_dict(self._generate_mocked_sparse_embedding(1)[0])
 
         document_store.write_documents(filterable_docs)
-        retriever = QdrantSparseRetriever(document_store=document_store)
+        retriever = QdrantSparseEmbeddingRetriever(document_store=document_store)
         sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
 
         results: List[Document] = retriever.run(query_sparse_embedding=sparse_embedding)["documents"]

qdrant_haystack-3.3.0/src/haystack_integrations/components/retrievers/qdrant/__init__.py

@@ -1,7 +0,0 @@
-# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from .retriever import QdrantEmbeddingRetriever, QdrantSparseRetriever
-
-__all__ = ("QdrantEmbeddingRetriever", "QdrantSparseRetriever")