qdrant-haystack 3.2.1__tar.gz → 3.3.1__tar.gz

{qdrant_haystack-3.2.1 → qdrant_haystack-3.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qdrant-haystack
-Version: 3.2.1
+Version: 3.3.1
 Summary: An integration of Qdrant ANN vector database backend with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.8
-Requires-Dist: haystack-ai
+Requires-Dist: haystack-ai>=2.0.1
 Requires-Dist: qdrant-client
 Description-Content-Type: text/markdown
 

{qdrant_haystack-3.2.1 → qdrant_haystack-3.3.1}/pydoc/config.yml

@@ -5,8 +5,7 @@ loaders:
       [
         "haystack_integrations.components.retrievers.qdrant.retriever",
         "haystack_integrations.document_stores.qdrant.document_store",
-        "haystack_integrations.document_stores.qdrant.converters",
-        "haystack_integrations.document_stores.qdrant.filters",
+        "haystack_integrations.document_stores.qdrant.migrate_to_sparse",
       ]
     ignore_when_discovered: ["__init__"]
 processors:

{qdrant_haystack-3.2.1 → qdrant_haystack-3.3.1}/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai", "qdrant-client"]
+dependencies = ["haystack-ai>=2.0.1", "qdrant-client"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations"
@@ -103,6 +103,8 @@ ignore = [
   "B027",
   # Allow boolean positional values in function calls, like `dict.get(... True)`
   "FBT003",
+  # Allow boolean arguments in function definition
+  "FBT001", "FBT002",
   # Ignore checks for possible passwords
   "S105",
   "S106",

qdrant_haystack-3.3.1/src/haystack_integrations/components/retrievers/qdrant/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .retriever import QdrantEmbeddingRetriever, QdrantSparseRetriever
+
+__all__ = ("QdrantEmbeddingRetriever", "QdrantSparseRetriever")

qdrant_haystack-3.3.1/src/haystack_integrations/components/retrievers/qdrant/retriever.py

@@ -0,0 +1,241 @@
+from typing import Any, Dict, List, Optional
+
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.dataclasses.sparse_embedding import SparseEmbedding
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+
+
+@component
+class QdrantEmbeddingRetriever:
+    """
+    A component for retrieving documents from an QdrantDocumentStore using dense vectors.
+
+    Usage example:
+    ```python
+    from haystack_integrations.components.retrievers.qdrant import QdrantEmbeddingRetriever
+    from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+
+    document_store = QdrantDocumentStore(
+        ":memory:",
+        recreate_index=True,
+        return_embedding=True,
+        wait_result_from_api=True,
+    )
+    retriever = QdrantEmbeddingRetriever(document_store=document_store)
+
+    # using a fake vector to keep the example simple
+    retriever.run(query_embedding=[0.1]*768)
+    ```
+    """
+
+    def __init__(
+        self,
+        document_store: QdrantDocumentStore,
+        filters: Optional[Dict[str, Any]] = None,
+        top_k: int = 10,
+        scale_score: bool = True,
+        return_embedding: bool = False,
+    ):
+        """
+        Create a QdrantEmbeddingRetriever component.
+
+        :param document_store: An instance of QdrantDocumentStore.
+        :param filters: A dictionary with filters to narrow down the search space. Default is None.
+        :param top_k: The maximum number of documents to retrieve. Default is 10.
+        :param scale_score: Whether to scale the scores of the retrieved documents or not. Default is True.
+        :param return_embedding: Whether to return the embedding of the retrieved Documents. Default is False.
+
+        :raises ValueError: If 'document_store' is not an instance of QdrantDocumentStore.
+        """
+
+        if not isinstance(document_store, QdrantDocumentStore):
+            msg = "document_store must be an instance of QdrantDocumentStore"
+            raise ValueError(msg)
+
+        self._document_store = document_store
+        self._filters = filters
+        self._top_k = top_k
+        self._scale_score = scale_score
+        self._return_embedding = return_embedding
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        d = default_to_dict(
+            self,
+            document_store=self._document_store,
+            filters=self._filters,
+            top_k=self._top_k,
+            scale_score=self._scale_score,
+            return_embedding=self._return_embedding,
+        )
+        d["init_parameters"]["document_store"] = self._document_store.to_dict()
+
+        return d
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "QdrantEmbeddingRetriever":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
+        data["init_parameters"]["document_store"] = document_store
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=List[Document])
+    def run(
+        self,
+        query_embedding: List[float],
+        filters: Optional[Dict[str, Any]] = None,
+        top_k: Optional[int] = None,
+        scale_score: Optional[bool] = None,
+        return_embedding: Optional[bool] = None,
+    ):
+        """
+        Run the Embedding Retriever on the given input data.
+
+        :param query_embedding: Embedding of the query.
+        :param filters: A dictionary with filters to narrow down the search space.
+        :param top_k: The maximum number of documents to return.
+        :param scale_score: Whether to scale the scores of the retrieved documents or not.
+        :param return_embedding: Whether to return the embedding of the retrieved Documents.
+        :returns:
+            The retrieved documents.
+
+        """
+        docs = self._document_store.query_by_embedding(
+            query_embedding=query_embedding,
+            filters=filters or self._filters,
+            top_k=top_k or self._top_k,
+            scale_score=scale_score or self._scale_score,
+            return_embedding=return_embedding or self._return_embedding,
+        )
+
+        return {"documents": docs}
+
+
+@component
+class QdrantSparseRetriever:
+    """
+    A component for retrieving documents from an QdrantDocumentStore using sparse vectors.
+
+    Usage example:
+    ```python
+    from haystack_integrations.components.retrievers.qdrant import QdrantSparseRetriever
+    from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+    from haystack.dataclasses.sparse_embedding import SparseEmbedding
+
+    document_store = QdrantDocumentStore(
+        ":memory:",
+        recreate_index=True,
+        return_embedding=True,
+        wait_result_from_api=True,
+    )
+    retriever = QdrantSparseRetriever(document_store=document_store)
+    sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
+    retriever.run(query_sparse_embedding=sparse_embedding)
+    ```
+    """
+
+    def __init__(
+        self,
+        document_store: QdrantDocumentStore,
+        filters: Optional[Dict[str, Any]] = None,
+        top_k: int = 10,
+        scale_score: bool = True,
+        return_embedding: bool = False,
+    ):
+        """
+        Create a QdrantSparseRetriever component.
+
+        :param document_store: An instance of QdrantDocumentStore.
+        :param filters: A dictionary with filters to narrow down the search space. Default is None.
+        :param top_k: The maximum number of documents to retrieve. Default is 10.
+        :param scale_score: Whether to scale the scores of the retrieved documents or not. Default is True.
+        :param return_embedding: Whether to return the sparse embedding of the retrieved Documents. Default is False.
+
+        :raises ValueError: If 'document_store' is not an instance of QdrantDocumentStore.
+        """
+
+        if not isinstance(document_store, QdrantDocumentStore):
+            msg = "document_store must be an instance of QdrantDocumentStore"
+            raise ValueError(msg)
+
+        self._document_store = document_store
+        self._filters = filters
+        self._top_k = top_k
+        self._scale_score = scale_score
+        self._return_embedding = return_embedding
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        d = default_to_dict(
+            self,
+            document_store=self._document_store,
+            filters=self._filters,
+            top_k=self._top_k,
+            scale_score=self._scale_score,
+            return_embedding=self._return_embedding,
+        )
+        d["init_parameters"]["document_store"] = self._document_store.to_dict()
+
+        return d
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "QdrantEmbeddingRetriever":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
+        data["init_parameters"]["document_store"] = document_store
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=List[Document])
+    def run(
+        self,
+        query_sparse_embedding: SparseEmbedding,
+        filters: Optional[Dict[str, Any]] = None,
+        top_k: Optional[int] = None,
+        scale_score: Optional[bool] = None,
+        return_embedding: Optional[bool] = None,
+    ):
+        """
+        Run the Sparse Embedding Retriever on the given input data.
+
+        :param query_sparse_embedding: Sparse Embedding of the query.
+        :param filters: A dictionary with filters to narrow down the search space.
+        :param top_k: The maximum number of documents to return.
+        :param scale_score: Whether to scale the scores of the retrieved documents or not.
+        :param return_embedding: Whether to return the embedding of the retrieved Documents.
+        :returns:
+            The retrieved documents.
+
+        """
+        docs = self._document_store.query_by_sparse(
+            query_sparse_embedding=query_sparse_embedding,
+            filters=filters or self._filters,
+            top_k=top_k or self._top_k,
+            scale_score=scale_score or self._scale_score,
+            return_embedding=return_embedding or self._return_embedding,
+        )
+
+        return {"documents": docs}

{qdrant_haystack-3.2.1 → qdrant_haystack-3.3.1}/src/haystack_integrations/document_stores/qdrant/__init__.py

@@ -3,5 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from .document_store import QdrantDocumentStore
+from .migrate_to_sparse import migrate_to_sparse_embeddings_support
 
-__all__ = ("QdrantDocumentStore",)
+__all__ = ("QdrantDocumentStore", "migrate_to_sparse_embeddings_support")

qdrant_haystack-3.3.1/src/haystack_integrations/document_stores/qdrant/converters.py

@@ -0,0 +1,80 @@
+import logging
+import uuid
+from typing import List, Union
+
+from haystack.dataclasses import Document
+from qdrant_client.http import models as rest
+
+logger = logging.getLogger(__name__)
+
+DENSE_VECTORS_NAME = "text-dense"
+SPARSE_VECTORS_NAME = "text-sparse"
+
+
+UUID_NAMESPACE = uuid.UUID("3896d314-1e95-4a3a-b45a-945f9f0b541d")
+
+
+def convert_haystack_documents_to_qdrant_points(
+    documents: List[Document],
+    *,
+    embedding_field: str,
+    use_sparse_embeddings: bool,
+) -> List[rest.PointStruct]:
+    points = []
+    for document in documents:
+        payload = document.to_dict(flatten=False)
+        if use_sparse_embeddings:
+            vector = {}
+
+            dense_vector = payload.pop(embedding_field, None)
+            if dense_vector is not None:
+                vector[DENSE_VECTORS_NAME] = dense_vector
+
+            sparse_vector = payload.pop("sparse_embedding", None)
+            if sparse_vector is not None:
+                sparse_vector_instance = rest.SparseVector(**sparse_vector)
+                vector[SPARSE_VECTORS_NAME] = sparse_vector_instance
+
+        else:
+            vector = payload.pop(embedding_field) or {}
+        _id = convert_id(payload.get("id"))
+
+        point = rest.PointStruct(
+            payload=payload,
+            vector=vector,
+            id=_id,
+        )
+        points.append(point)
+    return points
+
+
+def convert_id(_id: str) -> str:
+    """
+    Converts any string into a UUID-like format in a deterministic way.
+
+    Qdrant does not accept any string as an id, so an internal id has to be
+    generated for each point. This is a deterministic way of doing so.
+    """
+    return uuid.uuid5(UUID_NAMESPACE, _id).hex
+
+
+QdrantPoint = Union[rest.ScoredPoint, rest.Record]
+
+
+def convert_qdrant_point_to_haystack_document(point: QdrantPoint, use_sparse_embeddings: bool) -> Document:
+    payload = {**point.payload}
+    payload["score"] = point.score if hasattr(point, "score") else None
+
+    if not use_sparse_embeddings:
+        payload["embedding"] = point.vector if hasattr(point, "vector") else None
+    elif hasattr(point, "vector") and point.vector is not None:
+        payload["embedding"] = point.vector.get(DENSE_VECTORS_NAME)
+
+        if SPARSE_VECTORS_NAME in point.vector:
+            parse_vector_dict = {
+                "indices": point.vector[SPARSE_VECTORS_NAME].indices,
+                "values": point.vector[SPARSE_VECTORS_NAME].values,
+            }
+            payload["sparse_embedding"] = parse_vector_dict
+
+    return Document.from_dict(payload)