qdrant-haystack 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

qdrant_haystack/document_stores/qdrant.py

@@ -37,6 +37,7 @@ class QdrantDocumentStore(BaseDocumentStore):
 
     def __init__(
         self,
+        location: Optional[str] = None,
         url: Optional[str] = None,
         port: int = 6333,
         grpc_port: int = 6334,
@@ -46,8 +47,10 @@ class QdrantDocumentStore(BaseDocumentStore):
         prefix: Optional[str] = None,
         timeout: Optional[float] = None,
         host: Optional[str] = None,
+        path: Optional[str] = None,
         index: str = "Document",
         embedding_dim: int = 768,
+        hnsw_config: Optional[Dict] = None,
         content_field: str = "content",
         name_field: str = "name",
         embedding_field: str = "vector",
@@ -61,6 +64,7 @@ class QdrantDocumentStore(BaseDocumentStore):
         super().__init__()
 
         self.client = qdrant_client.QdrantClient(
+            location=location,
             url=url,
             port=port,
             grpc_port=grpc_port,
@@ -70,15 +74,19 @@ class QdrantDocumentStore(BaseDocumentStore):
             prefix=prefix,
             timeout=timeout,
             host=host,
+            path=path,
             **kwargs,
         )
 
-        self._set_up_collection(index, embedding_dim, recreate_index, similarity)
+        self._set_up_collection(
+            index, embedding_dim, hnsw_config, recreate_index, similarity
+        )
 
         self.embedding_dim = embedding_dim
         self.content_field = content_field
         self.name_field = name_field
         self.embedding_field = embedding_field
+        self.hnsw_config = hnsw_config
         self.similarity = similarity
         self.index = index
         self.return_embedding = return_embedding
@@ -161,10 +169,11 @@ class QdrantDocumentStore(BaseDocumentStore):
 
         next_offset = None
         stop_scrolling = False
+        scroll_filter = self.qdrant_filter_converter.convert(None, ids)
         while not stop_scrolling:
             records, next_offset = self.client.scroll(
                 collection_name=index,
-                scroll_filter=self.qdrant_filter_converter.convert(None, ids),
+                scroll_filter=scroll_filter,
                 limit=batch_size,
                 offset=next_offset,
                 with_payload=True,
@@ -197,7 +206,10 @@ class QdrantDocumentStore(BaseDocumentStore):
                 count_filter=qdrant_filters,
             )
             return response.count
-        except UnexpectedResponse:
+        except (UnexpectedResponse, ValueError):
+            # Qdrant local raises ValueError if the collection is not found, but
+            # with the remote server UnexpectedResponse is raised. Until that's unified,
+            # we need to catch both.
             return 0
 
     def get_embedding_count(
@@ -247,7 +259,9 @@ class QdrantDocumentStore(BaseDocumentStore):
         headers: Optional[Dict[str, str]] = None,
     ):
         index = index or self.index
-        self._set_up_collection(index, self.embedding_dim, False, self.similarity)
+        self._set_up_collection(
+            index, self.embedding_dim, self.hnsw_config, False, self.similarity
+        )
         field_map = self._create_document_field_map()
 
         duplicate_documents = duplicate_documents or self.duplicate_documents
@@ -466,6 +480,7 @@ class QdrantDocumentStore(BaseDocumentStore):
         self,
         collection_name: str,
         embedding_dim: int,
+        hnsw_config: dict,
         recreate_collection: bool,
         similarity: str,
     ):
@@ -474,41 +489,54 @@ class QdrantDocumentStore(BaseDocumentStore):
         if recreate_collection:
             # There is no need to verify the current configuration of that
             # collection. It might be just recreated again.
-            self._recreate_collection(collection_name, distance, embedding_dim)
+            self._recreate_collection(
+                collection_name, distance, embedding_dim, hnsw_config
+            )
             return
 
         try:
             # Check if the collection already exists and validate its
             # current configuration with the parameters.
             collection_info = self.client.get_collection(collection_name)
-            current_distance = collection_info.config.params.vectors.distance
-            current_vector_size = collection_info.config.params.vectors.size
-
-            if current_distance != distance:
-                raise ValueError(
-                    f"Collection '{collection_name}' already exists in Qdrant, "
-                    f"but it is configured with a similarity '{current_distance.name}'. "
-                    f"If you want to use that collection, but with a different "
-                    f"similarity, please set `recreate_collection=True` argument."
-                )
-
-            if current_vector_size != embedding_dim:
-                raise ValueError(
-                    f"Collection '{collection_name}' already exists in Qdrant, "
-                    f"but it is configured with a vector size '{current_vector_size}'. "
-                    f"If you want to use that collection, but with a different "
-                    f"vector size, please set `recreate_collection=True` argument."
-                )
-        except (UnexpectedResponse, _InactiveRpcError):
+        except (UnexpectedResponse, _InactiveRpcError, ValueError):
             # That indicates the collection does not exist, so it can be
             # safely created with any configuration.
-            self._recreate_collection(collection_name, distance, embedding_dim)
+            #
+            # Qdrant local raises ValueError if the collection is not found, but
+            # with the remote server UnexpectedResponse / _InactiveRpcError is raised.
+            # Until that's unified, we need to catch both.
+            self._recreate_collection(
+                collection_name, distance, embedding_dim, hnsw_config
+            )
+            return
+
+        current_distance = collection_info.config.params.vectors.distance
+        current_vector_size = collection_info.config.params.vectors.size
+
+        if current_distance != distance:
+            raise ValueError(
+                f"Collection '{collection_name}' already exists in Qdrant, "
+                f"but it is configured with a similarity '{current_distance.name}'. "
+                f"If you want to use that collection, but with a different "
+                f"similarity, please set `recreate_collection=True` argument."
+            )
 
-    def _recreate_collection(self, collection_name, distance, embedding_dim):
+        if current_vector_size != embedding_dim:
+            raise ValueError(
+                f"Collection '{collection_name}' already exists in Qdrant, "
+                f"but it is configured with a vector size '{current_vector_size}'. "
+                f"If you want to use that collection, but with a different "
+                f"vector size, please set `recreate_collection=True` argument."
+            )
+
+    def _recreate_collection(
+        self, collection_name, distance, embedding_dim, hnsw_config
+    ):
         self.client.recreate_collection(
             collection_name=collection_name,
             vectors_config=rest.VectorParams(
                 size=embedding_dim,
                 distance=distance,
             ),
+            hnsw_config=hnsw_config,
         )

qdrant_haystack-0.0.5.dist-info/METADATA

@@ -0,0 +1,122 @@
+Metadata-Version: 2.1
+Name: qdrant-haystack
+Version: 0.0.5
+Summary: An integration of Qdrant ANN vector database backend with Haystack
+License: Apache 2.0
+Author: Kacper Łukawski
+Author-email: kacper.lukawski@qdrant.com
+Requires-Python: >=3.8.1,<=3.11
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: farm-haystack (>=1.13.0,<2.0.0)
+Requires-Dist: qdrant-client (>=1.1.2,<2.0.0)
+Description-Content-Type: text/markdown
+
+# qdrant-haystack
+
+An integration of [Qdrant](https://qdrant.tech) vector database with [Haystack](https://haystack.deepset.ai/)
+by [deepset](https://www.deepset.ai).
+
+The library finally allows using Qdrant as a document store, and provides an in-place replacement
+for any other vector embeddings store. Thus, you should expect any kind of application to be working
+smoothly just by changing the provider to `QdrantDocumentStore`.
+
+## Installation
+
+`qdrant-haystack` might be installed as any other Python library, using pip or poetry:
+
+```bash
+pip install qdrant-haystack
+```
+
+```bash
+poetry add qdrant-haystack
+```
+
+## Usage
+
+Once installed, you can already start using `QdrantDocumentStore` as any other store that supports
+embeddings.
+
+```python
+from qdrant_haystack import QdrantDocumentStore
+
+document_store = QdrantDocumentStore(
+    "localhost",
+    index="Document",
+    embedding_dim=512,
+    recreate_index=True,
+    hnsw_config={"m": 16, "ef_construct": 64}  # Optional
+)
+```
+
+The list of parameters accepted by `QdrantDocumentStore` is complementary to those used in the
+official [Python Qdrant client](https://github.com/qdrant/qdrant_client).
+
+### Using local in-memory / disk-persisted mode
+
+Qdrant Python client, from version 1.1.1, supports local in-memory/disk-persisted mode. That's
+a good choice for any test scenarios and quick experiments in which you do not plan to store
+lots of vectors. In such a case spinning a Docker container might be even not required.
+
+The local mode was also implemented in `qdrant-haystack` integration.
+
+#### In-memory storage
+
+In case you want to have a transient storage, for example in case of automated tests launched
+during your CI/CD pipeline, using Qdrant Local mode with in-memory storage might be a preferred
+option. It might be simply enabled by passing `:memory:` as first parameter, while creating an
+instance of `QdrantDocumentStore`.
+
+```python
+from qdrant_haystack import QdrantDocumentStore
+
+document_store = QdrantDocumentStore(
+    ":memory:",
+    index="Document",
+    embedding_dim=512,
+    recreate_index=True,
+    hnsw_config={"m": 16, "ef_construct": 64}  # Optional
+)
+```
+
+#### On disk storage
+
+However, if you prefer to keep the vectors between different runs of your application, it
+might be better to use on disk storage and pass the path that should be used to persist
+the data.
+
+```python
+from qdrant_haystack import QdrantDocumentStore
+
+document_store = QdrantDocumentStore(
+    path="/home/qdrant/storage_local",
+    index="Document",
+    embedding_dim=512,
+    recreate_index=True,
+    hnsw_config={"m": 16, "ef_construct": 64}  # Optional
+)
+```
+
+### Connecting to Qdrant Cloud cluster
+
+If you prefer not to manage your own Qdrant instance, [Qdrant Cloud](https://cloud.qdrant.io/)
+might be a better option.
+
+```python
+from qdrant_haystack import QdrantDocumentStore
+
+document_store = QdrantDocumentStore(
+    "https://YOUR-CLUSTER-URL.aws.cloud.qdrant.io",
+    index="Document",
+    api_key="<< YOUR QDRANT CLOUD API KEY >>",
+    embedding_dim=512,
+    recreate_index=True,
+)
+```
+
+There is no difference in terms of functionality between local instances and cloud clusters.
+

{qdrant_haystack-0.0.3.dist-info → qdrant_haystack-0.0.5.dist-info}/RECORD

@@ -2,9 +2,9 @@ qdrant_haystack/__init__.py,sha256=F7uhTaB7pul_sbPyBQNYIz5eTOVCW5f9qF3FjiN8CRM,1
 qdrant_haystack/document_stores/__init__.py,sha256=RbJ_JV8mXcmWI9LrHj0gPPEokPfoKGi6zFwkpXvHsjA,107
 qdrant_haystack/document_stores/converters.py,sha256=UkmoK2AMS_zEUy2C9B-04DdMdL0KxANB5d7QBOySzY0,2285
 qdrant_haystack/document_stores/filters.py,sha256=lZumUKgAz1vlWkI0x2jqITDaHDGkth8n3bw1uXTqmjM,8295
-qdrant_haystack/document_stores/qdrant.py,sha256=-ysi39v0lHS0XByh1usUFkOOr8h_LwXBf3qwcP3t44I,18121
+qdrant_haystack/document_stores/qdrant.py,sha256=hekVs25mnqHcjId9buWzlr7aIsMIq1YjkZbGEQrm3Uw,19054
 qdrant_haystack/utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-qdrant_haystack-0.0.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-qdrant_haystack-0.0.3.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-qdrant_haystack-0.0.3.dist-info/METADATA,sha256=AhstKT1BYXBABZCImGXtFJkT902ZVg5eZVIWSh1P2G0,1743
-qdrant_haystack-0.0.3.dist-info/RECORD,,
+qdrant_haystack-0.0.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+qdrant_haystack-0.0.5.dist-info/METADATA,sha256=LVmmp970YdLGka5aLH-ycUMttzzbr8bpjH83iJUgiDg,3797
+qdrant_haystack-0.0.5.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+qdrant_haystack-0.0.5.dist-info/RECORD,,

{qdrant_haystack-0.0.3.dist-info → qdrant_haystack-0.0.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.4.0
+Generator: poetry-core 1.5.2
 Root-Is-Purelib: true
 Tag: py3-none-any

qdrant_haystack-0.0.3.dist-info/METADATA

@@ -1,57 +0,0 @@
-Metadata-Version: 2.1
-Name: qdrant-haystack
-Version: 0.0.3
-Summary: An integration of Qdrant ANN vector database backend with Haystack
-License: Apache 2.0
-Author: Kacper Łukawski
-Author-email: kacper.lukawski@qdrant.com
-Requires-Python: >=3.8.1,<=3.11
-Classifier: License :: Other/Proprietary License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: farm-haystack (>=1.13.0,<2.0.0)
-Requires-Dist: qdrant-client (>=1.0.2,<2.0.0)
-Description-Content-Type: text/markdown
-
-# qdrant-haystack
-
-An integration of [Qdrant](https://qdrant.tech) vector database with [Haystack](https://haystack.deepset.ai/)
-by [deepset](https://www.deepset.ai).
-
-The library finally allows using Qdrant as a document store, and provides an in-place replacement
-for any other vector embeddings store. Thus, you should expect any kind of application to be working
-smoothly just by changing the provider to `QdrantDocumentStore`.
-
-## Installation
-
-`qdrant-haystack` might be installed as any other Python library, using pip or poetry:
-
-```bash
-pip install qdrant-haystack
-```
-
-```bash
-poetry add qdrant-haystack
-```
-
-## Usage
-
-Once installed, you can already start using `QdrantDocumentStore` as any other store that supports
-embeddings.
-
-```python
-from qdrant_haystack import QdrantDocumentStore
-
-document_store = QdrantDocumentStore(
-    host="localhost",
-    index="Document",
-    embedding_dim=512,
-    recreate_index=True,
-)
-```
-
-The list of parameters accepted by `QdrantDocumentStore` is complementary to those used in the
-official [Python Qdrant client](https://github.com/qdrant/qdrant_client).
-