qdrant-haystack 3.3.0__py3-none-any.whl → 3.4.0__py3-none-any.whl

haystack_integrations/components/retrievers/qdrant/__init__.py

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from .retriever import QdrantEmbeddingRetriever, QdrantSparseRetriever
+from .retriever import QdrantEmbeddingRetriever, QdrantSparseEmbeddingRetriever
 
-__all__ = ("QdrantEmbeddingRetriever", "QdrantSparseRetriever")
+__all__ = ("QdrantEmbeddingRetriever", "QdrantSparseEmbeddingRetriever")

haystack_integrations/components/retrievers/qdrant/retriever.py

@@ -124,13 +124,13 @@ class QdrantEmbeddingRetriever:
 
 
 @component
-class QdrantSparseRetriever:
+class QdrantSparseEmbeddingRetriever:
     """
     A component for retrieving documents from an QdrantDocumentStore using sparse vectors.
 
     Usage example:
     ```python
-    from haystack_integrations.components.retrievers.qdrant import QdrantSparseRetriever
+    from haystack_integrations.components.retrievers.qdrant import QdrantSparseEmbeddingRetriever
     from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
     from haystack.dataclasses.sparse_embedding import SparseEmbedding
 
@@ -140,7 +140,7 @@ class QdrantSparseRetriever:
         return_embedding=True,
         wait_result_from_api=True,
     )
-    retriever = QdrantSparseRetriever(document_store=document_store)
+    retriever = QdrantSparseEmbeddingRetriever(document_store=document_store)
     sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
     retriever.run(query_sparse_embedding=sparse_embedding)
     ```
@@ -155,7 +155,7 @@ class QdrantSparseRetriever:
         return_embedding: bool = False,
     ):
         """
-        Create a QdrantSparseRetriever component.
+        Create a QdrantSparseEmbeddingRetriever component.
 
         :param document_store: An instance of QdrantDocumentStore.
         :param filters: A dictionary with filters to narrow down the search space. Default is None.

haystack_integrations/document_stores/qdrant/__init__.py

@@ -3,5 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from .document_store import QdrantDocumentStore
+from .migrate_to_sparse import migrate_to_sparse_embeddings_support
 
-__all__ = ("QdrantDocumentStore",)
+__all__ = ("QdrantDocumentStore", "migrate_to_sparse_embeddings_support")

haystack_integrations/document_stores/qdrant/document_store.py

@@ -453,7 +453,9 @@ class QdrantDocumentStore:
                 f"Collection '{collection_name}' already exists in Qdrant, "
                 f"but it has been originally created without sparse embedding vectors. "
                 f"If you want to use that collection, you can set `use_sparse_embeddings=False`. "
-                f"To use sparse embeddings, you need to recreate the collection or migrate the existing one."
+                f"To use sparse embeddings, you need to recreate the collection or migrate the existing one. "
+                f"See `migrate_to_sparse_embeddings_support` function in "
+                f"`haystack_integrations.document_stores.qdrant`."
             )
             raise QdrantStoreError(msg)
 

haystack_integrations/document_stores/qdrant/migrate_to_sparse.py

@@ -0,0 +1,127 @@
+import logging
+import time
+
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+from qdrant_client.http import models
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.INFO)
+
+
+def migrate_to_sparse_embeddings_support(old_document_store: QdrantDocumentStore, new_index: str):
+    """
+    Utility function to migrate an existing `QdrantDocumentStore` to a new one with support for sparse embeddings.
+
+    With qdrant-hasytack v3.3.0, support for sparse embeddings has been added to `QdrantDocumentStore`.
+    This feature is disabled by default and can be enabled by setting `use_sparse_embeddings=True` in the init
+    parameters. To store sparse embeddings, Document stores/collections created with this feature disabled must be
+    migrated to a new collection with the feature enabled.
+
+    This utility function applies to on-premise and cloud instances of Qdrant.
+    It does not work for local in-memory/disk-persisted instances.
+
+    The utility function merely migrates the existing documents so that they are ready to store sparse embeddings.
+    It does not compute sparse embeddings. To do this, you need to use a Sparse Embedder component.
+
+    Example usage:
+    ```python
+    from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+    from haystack_integrations.document_stores.qdrant import migrate_to_sparse_embeddings_support
+
+    old_document_store = QdrantDocumentStore(url="http://localhost:6333",
+                                             index="Document",
+                                             use_sparse_embeddings=False)
+    new_index = "Document_sparse"
+
+    migrate_to_sparse_embeddings_support(old_document_store, new_index)
+
+    # now you can use the new document store with sparse embeddings support
+    new_document_store = QdrantDocumentStore(url="http://localhost:6333",
+                                             index=new_index,
+                                             use_sparse_embeddings=True)
+    ```
+
+
+    :param old_document_store: The existing QdrantDocumentStore instance to migrate from.
+    :param new_index: The name of the new index/collection to create with sparse embeddings support.
+    """
+
+    start = time.time()
+
+    old_collection_name = old_document_store.index
+    total_points = old_document_store.count_documents()
+
+    # copy the init parameters of the old document to create a new document store
+    init_parameters = old_document_store.to_dict()["init_parameters"]
+    init_parameters["index"] = new_index
+    init_parameters["use_sparse_embeddings"] = True
+    init_parameters["recreate_index"] = True
+
+    new_document_store = QdrantDocumentStore(**init_parameters)
+
+    client = new_document_store.client
+
+    original_indexing_threshold = client.get_collection(
+        collection_name=new_index
+    ).config.optimizer_config.indexing_threshold
+
+    # disable indexing while adding points so it's faster
+    # https://qdrant.tech/documentation/concepts/collections/#update-collection-parameters
+    client.update_collection(
+        collection_name=new_index,
+        optimizer_config=models.OptimizersConfigDiff(indexing_threshold=0),
+    )
+
+    # migration loop
+    next_page_offset = "first"
+    offset = None
+    points_transmitted = 0
+
+    while next_page_offset:
+        if next_page_offset != "first":
+            offset = next_page_offset
+
+        # get the records
+        records = client.scroll(
+            collection_name=old_collection_name,
+            limit=100,
+            with_payload=True,
+            with_vectors=True,
+            offset=offset,
+        )
+
+        next_page_offset = records[1]
+        current_records = records[0]
+
+        points = []
+
+        for record in current_records:
+            vector = {}
+
+            vector["text-dense"] = record.vector
+
+            point = {"id": record.id, "payload": record.payload, "vector": vector}
+
+            embedding_point = models.PointStruct(**point)
+            points.append(embedding_point)
+
+        client.upsert(collection_name=new_index, points=points)
+
+        points_transmitted += len(points)
+        points_remaining = total_points - points_transmitted
+
+        message = (
+            f"Points transmitted: {points_transmitted}/{total_points}\n"
+            f"Percent done {points_transmitted/total_points*100:.2f}%\n"
+            f"Time elapsed: {time.time() - start:.2f} seconds\n"
+            f"Time remaining: {(((time.time() - start) / points_transmitted) * points_remaining) / 60:.2f} minutes\n"
+            f"Current offset: {next_page_offset}\n"
+        )
+        logger.info(message)
+
+    # restore the original indexing threshold (re-enable indexing)
+    client.update_collection(
+        collection_name=new_index,
+        optimizer_config=models.OptimizersConfigDiff(indexing_threshold=original_indexing_threshold),
+    )

{qdrant_haystack-3.3.0.dist-info → qdrant_haystack-3.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qdrant-haystack
-Version: 3.3.0
+Version: 3.4.0
 Summary: An integration of Qdrant ANN vector database backend with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md

qdrant_haystack-3.4.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+haystack_integrations/components/retrievers/qdrant/__init__.py,sha256=Paisl7uv_QHK9HdCOCLP3tW5PAI8XzoDKlCnPc1xvPk,265
+haystack_integrations/components/retrievers/qdrant/retriever.py,sha256=62t6YYSA7FX4ZvDasPERMrXKr6nk0NEqJLEEE0p1J7k,8853
+haystack_integrations/document_stores/qdrant/__init__.py,sha256=kUGc5uewqArhmVR-JqB_NmJ4kNkTIQIvYDNSoO2ELn0,302
+haystack_integrations/document_stores/qdrant/converters.py,sha256=oSO2YlsWEQbcw9CPlWfSg_HoTZlnkAhZw_6VlYWzKKs,2525
+haystack_integrations/document_stores/qdrant/document_store.py,sha256=uZkTKjHhFQFv27C4LQ6xJO3kFf0m_aNhlpAyEyuP8n4,22943
+haystack_integrations/document_stores/qdrant/filters.py,sha256=iNWOqv1otUaXTURXd8e9QOYg8sx3Qm_LOqOaxAP2xJI,8249
+haystack_integrations/document_stores/qdrant/migrate_to_sparse.py,sha256=i6wBC_9_JVzYZtqKm3dhHKTxhwNdcAdpgki8GABDp1c,4909
+qdrant_haystack-3.4.0.dist-info/METADATA,sha256=el2vEKE_LY7BerXSrZVwBdPJmaqeIW8mwdNM_U6zfE4,1799
+qdrant_haystack-3.4.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+qdrant_haystack-3.4.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
+qdrant_haystack-3.4.0.dist-info/RECORD,,

{qdrant_haystack-3.3.0.dist-info → qdrant_haystack-3.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.22.5
+Generator: hatchling 1.24.2
 Root-Is-Purelib: true
 Tag: py3-none-any

qdrant_haystack-3.3.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-haystack_integrations/components/retrievers/qdrant/__init__.py,sha256=5P4opz_iVPY-6ntpNdQgTu2MAS102_ki8fT-rm7aiJs,247
-haystack_integrations/components/retrievers/qdrant/retriever.py,sha256=M1_5suJnF9VDC0fPbHyfU8kx_YIYG7TEyAWkIYbLTN8,8817
-haystack_integrations/document_stores/qdrant/__init__.py,sha256=PuGxUj29V00f6UiCpTHRkzGufL8bJUML2iNwJnX2KwM,195
-haystack_integrations/document_stores/qdrant/converters.py,sha256=oSO2YlsWEQbcw9CPlWfSg_HoTZlnkAhZw_6VlYWzKKs,2525
-haystack_integrations/document_stores/qdrant/document_store.py,sha256=HEUkzkcxRlTSegx1aq95ay1lJFaEmQh8NyxMf3TAarQ,22800
-haystack_integrations/document_stores/qdrant/filters.py,sha256=iNWOqv1otUaXTURXd8e9QOYg8sx3Qm_LOqOaxAP2xJI,8249
-qdrant_haystack-3.3.0.dist-info/METADATA,sha256=dqi3IRTo3fqIASBVbFW8I8y1F9a14vU4z-dAMJKlUOU,1799
-qdrant_haystack-3.3.0.dist-info/WHEEL,sha256=as-1oFTWSeWBgyzh0O_qF439xqBe6AbBgt4MfYe5zwY,87
-qdrant_haystack-3.3.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
-qdrant_haystack-3.3.0.dist-info/RECORD,,