qdrant-haystack 10.2.1__tar.gz → 10.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/CHANGELOG.md +27 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/PKG-INFO +4 -3
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/examples/embedding_retrieval.py +4 -1
- qdrant_haystack-10.3.1/pydoc/config_docusaurus.yml +15 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/pyproject.toml +29 -5
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/components/retrievers/qdrant/retriever.py +3 -2
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/document_stores/qdrant/converters.py +2 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/document_stores/qdrant/document_store.py +144 -47
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/document_stores/qdrant/filters.py +3 -1
- qdrant_haystack-10.3.1/tests/test_converters.py +144 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/test_document_store.py +332 -419
- qdrant_haystack-10.3.1/tests/test_document_store_async.py +311 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/test_embedding_retriever.py +101 -5
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/test_filters.py +80 -1
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/test_hybrid_retriever.py +26 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/test_sparse_embedding_retriever.py +74 -11
- qdrant_haystack-10.2.1/pydoc/config_docusaurus.yml +0 -30
- qdrant_haystack-10.2.1/tests/test_converters.py +0 -62
- qdrant_haystack-10.2.1/tests/test_document_store_async.py +0 -640
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/.gitignore +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/LICENSE.txt +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/README.md +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/components/retrievers/qdrant/__init__.py +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/document_stores/py.typed +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/document_stores/qdrant/__init__.py +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/__init__.py +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/conftest.py +0 -0
- {qdrant_haystack-10.2.1 → qdrant_haystack-10.3.1}/tests/test_dict_converters.py +0 -0
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/qdrant-v10.3.0] - 2026-03-23
|
|
4
|
+
|
|
5
|
+
### 📚 Documentation
|
|
6
|
+
|
|
7
|
+
- Simplify pydoc configs (#2855)
|
|
8
|
+
|
|
9
|
+
### 🧪 Testing
|
|
10
|
+
|
|
11
|
+
- Replacing each `DocumentStore` specific tests and used the generalised ones from `haystack.testing.document_store` (#2812)
|
|
12
|
+
- Test compatible integrations with python 3.14; update pyproject (#3001)
|
|
13
|
+
|
|
14
|
+
### 🧹 Chores
|
|
15
|
+
|
|
16
|
+
- Standardize author mentions (#2897)
|
|
17
|
+
- Add ANN ruff ruleset to optimum, paddleocr, pgvector, pinecone, pyversity, qdrant, ragas, snowflake (#2992)
|
|
18
|
+
|
|
19
|
+
### 🌀 Miscellaneous
|
|
20
|
+
|
|
21
|
+
- !test: `QdrantDocumentStore` use Mixin tests + updated signature `get_metadata_fields_info(self) -> dict[str, dict[str, str]]`: (#3004)
|
|
22
|
+
|
|
23
|
+
## [integrations/qdrant-v10.2.1] - 2026-02-02
|
|
24
|
+
|
|
25
|
+
### 📚 Documentation
|
|
26
|
+
|
|
27
|
+
- Fixing `QdrantDocumentStore` docstring parsing error (#2806)
|
|
28
|
+
|
|
29
|
+
|
|
3
30
|
## [integrations/qdrant-v10.2.0] - 2026-02-02
|
|
4
31
|
|
|
5
32
|
### 🌀 Miscellaneous
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: qdrant-haystack
|
|
3
|
-
Version: 10.
|
|
3
|
+
Version: 10.3.1
|
|
4
4
|
Summary: An integration of Qdrant ANN vector database backend with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md
|
|
7
7
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
8
|
-
Author-email: Kacper Łukawski <kacper.lukawski@qdrant.com>, Anush Shetty <anush.shetty@qdrant.com>
|
|
8
|
+
Author-email: deepset GmbH <info@deepset.ai>, Kacper Łukawski <kacper.lukawski@qdrant.com>, Anush Shetty <anush.shetty@qdrant.com>
|
|
9
9
|
License-Expression: Apache-2.0
|
|
10
10
|
License-File: LICENSE.txt
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -15,10 +15,11 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
21
|
Requires-Python: >=3.10
|
|
21
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.29.0
|
|
22
23
|
Requires-Dist: qdrant-client>=1.12.0
|
|
23
24
|
Description-Content-Type: text/markdown
|
|
24
25
|
|
|
@@ -9,9 +9,12 @@ import glob
|
|
|
9
9
|
|
|
10
10
|
from haystack import Pipeline
|
|
11
11
|
from haystack.components.converters import MarkdownToDocument
|
|
12
|
-
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
13
12
|
from haystack.components.preprocessors import DocumentSplitter
|
|
14
13
|
from haystack.components.writers import DocumentWriter
|
|
14
|
+
from haystack_integrations.components.embedders.sentence_transformers import (
|
|
15
|
+
SentenceTransformersDocumentEmbedder,
|
|
16
|
+
SentenceTransformersTextEmbedder,
|
|
17
|
+
)
|
|
15
18
|
|
|
16
19
|
from haystack_integrations.components.retrievers.qdrant import QdrantEmbeddingRetriever
|
|
17
20
|
from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- modules:
|
|
3
|
+
- haystack_integrations.components.retrievers.qdrant.retriever
|
|
4
|
+
- haystack_integrations.document_stores.qdrant.document_store
|
|
5
|
+
- haystack_integrations.document_stores.qdrant.migrate_to_sparse
|
|
6
|
+
search_path: [../src]
|
|
7
|
+
processors:
|
|
8
|
+
- type: filter
|
|
9
|
+
documented_only: true
|
|
10
|
+
skip_empty_modules: true
|
|
11
|
+
renderer:
|
|
12
|
+
description: Qdrant integration for Haystack
|
|
13
|
+
id: integrations-qdrant
|
|
14
|
+
filename: qdrant.md
|
|
15
|
+
title: Qdrant
|
|
@@ -11,6 +11,7 @@ requires-python = ">=3.10"
|
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
13
|
authors = [
|
|
14
|
+
{ name = "deepset GmbH", email = "info@deepset.ai" },
|
|
14
15
|
{ name = "Kacper Łukawski", email = "kacper.lukawski@qdrant.com" },
|
|
15
16
|
{ name = "Anush Shetty", email = "anush.shetty@qdrant.com" },
|
|
16
17
|
]
|
|
@@ -22,10 +23,14 @@ classifiers = [
|
|
|
22
23
|
"Programming Language :: Python :: 3.11",
|
|
23
24
|
"Programming Language :: Python :: 3.12",
|
|
24
25
|
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
25
27
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
26
28
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
27
29
|
]
|
|
28
|
-
dependencies = [
|
|
30
|
+
dependencies = [
|
|
31
|
+
"haystack-ai>=2.29.0",
|
|
32
|
+
"qdrant-client>=1.12.0"
|
|
33
|
+
]
|
|
29
34
|
|
|
30
35
|
[project.urls]
|
|
31
36
|
Source = "https://github.com/deepset-ai/haystack-core-integrations"
|
|
@@ -48,7 +53,7 @@ installer = "uv"
|
|
|
48
53
|
dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
49
54
|
|
|
50
55
|
[tool.hatch.envs.default.scripts]
|
|
51
|
-
docs = ["pydoc
|
|
56
|
+
docs = ["haystack-pydoc pydoc/config_docusaurus.yml"]
|
|
52
57
|
fmt = "ruff check --fix {args}; ruff format {args}"
|
|
53
58
|
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
54
59
|
|
|
@@ -66,7 +71,8 @@ dependencies = [
|
|
|
66
71
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
67
72
|
integration = 'pytest -m "integration" {args:tests}'
|
|
68
73
|
all = 'pytest {args:tests}'
|
|
69
|
-
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
74
|
+
unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
|
|
75
|
+
integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
|
|
70
76
|
|
|
71
77
|
types = """mypy -p haystack_integrations.document_stores.qdrant \
|
|
72
78
|
-p haystack_integrations.components.retrievers.qdrant {args}"""
|
|
@@ -84,9 +90,17 @@ line-length = 120
|
|
|
84
90
|
[tool.ruff.lint]
|
|
85
91
|
select = [
|
|
86
92
|
"A",
|
|
93
|
+
"ANN",
|
|
87
94
|
"ARG",
|
|
88
95
|
"B",
|
|
89
96
|
"C",
|
|
97
|
+
"D102", # Missing docstring in public method
|
|
98
|
+
"D103", # Missing docstring in public function
|
|
99
|
+
"D205", # 1 blank line required between summary line and description
|
|
100
|
+
"D209", # Closing triple quotes go to new line
|
|
101
|
+
"D213", # summary lines must be positioned on the second physical line of the docstring
|
|
102
|
+
"D417", # Missing argument descriptions in the docstring
|
|
103
|
+
"D419", # Docstring is empty
|
|
90
104
|
"DTZ",
|
|
91
105
|
"E",
|
|
92
106
|
"EM",
|
|
@@ -112,6 +126,8 @@ select = [
|
|
|
112
126
|
ignore = [
|
|
113
127
|
# Allow non-abstract empty methods in abstract base classes
|
|
114
128
|
"B027",
|
|
129
|
+
# Allow Any in type annotations at dynamic boundaries
|
|
130
|
+
"ANN401",
|
|
115
131
|
# Allow boolean positional values in function calls, like `dict.get(... True)`
|
|
116
132
|
"FBT003",
|
|
117
133
|
# Allow boolean arguments in function definition
|
|
@@ -136,14 +152,15 @@ ban-relative-imports = "parents"
|
|
|
136
152
|
|
|
137
153
|
[tool.ruff.lint.per-file-ignores]
|
|
138
154
|
# Tests can use magic values, assertions, and relative imports
|
|
139
|
-
"tests/**/*" = ["PLR2004", "S101", "TID252"]
|
|
155
|
+
"tests/**/*" = ["D", "PLR2004", "S101", "TID252", "ANN"]
|
|
140
156
|
# examples can contain "print" commands
|
|
141
|
-
"examples/**/*" = ["T201"]
|
|
157
|
+
"examples/**/*" = ["D", "T201"]
|
|
142
158
|
|
|
143
159
|
|
|
144
160
|
[tool.coverage.run]
|
|
145
161
|
source = ["haystack_integrations"]
|
|
146
162
|
branch = true
|
|
163
|
+
relative_files = true
|
|
147
164
|
parallel = false
|
|
148
165
|
|
|
149
166
|
|
|
@@ -155,3 +172,10 @@ omit = [
|
|
|
155
172
|
]
|
|
156
173
|
show_missing = true
|
|
157
174
|
exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
|
|
175
|
+
|
|
176
|
+
[tool.pytest.ini_options]
|
|
177
|
+
addopts = "--strict-markers"
|
|
178
|
+
markers = [
|
|
179
|
+
"integration: integration tests",
|
|
180
|
+
]
|
|
181
|
+
log_cli = true
|
|
@@ -482,8 +482,9 @@ class QdrantSparseEmbeddingRetriever:
|
|
|
482
482
|
@component
|
|
483
483
|
class QdrantHybridRetriever:
|
|
484
484
|
"""
|
|
485
|
-
A component for retrieving documents from
|
|
486
|
-
|
|
485
|
+
A component for retrieving documents from a QdrantDocumentStore using both dense and sparse vectors.
|
|
486
|
+
|
|
487
|
+
Fuses the results using Reciprocal Rank Fusion.
|
|
487
488
|
|
|
488
489
|
Usage example:
|
|
489
490
|
```python
|
|
@@ -18,6 +18,7 @@ def convert_haystack_documents_to_qdrant_points(
|
|
|
18
18
|
*,
|
|
19
19
|
use_sparse_embeddings: bool,
|
|
20
20
|
) -> list[rest.PointStruct]:
|
|
21
|
+
"""Convert a list of Haystack Document objects to Qdrant PointStruct objects."""
|
|
21
22
|
points = []
|
|
22
23
|
for document in documents:
|
|
23
24
|
payload = document.to_dict(flatten=False)
|
|
@@ -61,6 +62,7 @@ QdrantPoint = rest.ScoredPoint | rest.Record
|
|
|
61
62
|
|
|
62
63
|
|
|
63
64
|
def convert_qdrant_point_to_haystack_document(point: QdrantPoint, use_sparse_embeddings: bool) -> Document:
|
|
65
|
+
"""Convert a Qdrant ScoredPoint or Record to a Haystack Document object."""
|
|
64
66
|
payload = point.payload or {}
|
|
65
67
|
payload["score"] = point.score if hasattr(point, "score") else None
|
|
66
68
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
from collections.abc import AsyncGenerator, Generator
|
|
3
|
+
from dataclasses import replace
|
|
3
4
|
from itertools import islice
|
|
4
5
|
from typing import Any, ClassVar, cast
|
|
5
6
|
|
|
@@ -10,6 +11,7 @@ from haystack.dataclasses.sparse_embedding import SparseEmbedding
|
|
|
10
11
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
11
12
|
from haystack.document_stores.types import DuplicatePolicy
|
|
12
13
|
from haystack.utils import Secret, deserialize_secrets_inplace
|
|
14
|
+
from haystack.utils.misc import _normalize_metadata_field_name
|
|
13
15
|
from numpy import exp
|
|
14
16
|
from qdrant_client.http import models as rest
|
|
15
17
|
from qdrant_client.http.exceptions import UnexpectedResponse
|
|
@@ -53,8 +55,9 @@ def get_batches_from_generator(iterable: list, n: int) -> Generator:
|
|
|
53
55
|
|
|
54
56
|
class QdrantDocumentStore:
|
|
55
57
|
"""
|
|
56
|
-
A QdrantDocumentStore implementation that you can use with any Qdrant instance
|
|
57
|
-
|
|
58
|
+
A QdrantDocumentStore implementation that you can use with any Qdrant instance.
|
|
59
|
+
|
|
60
|
+
Supports in-memory, disk-persisted, Docker-based, and Qdrant Cloud Cluster deployments.
|
|
58
61
|
|
|
59
62
|
Usage example by creating an in-memory instance:
|
|
60
63
|
|
|
@@ -375,6 +378,7 @@ class QdrantDocumentStore:
|
|
|
375
378
|
) -> int:
|
|
376
379
|
"""
|
|
377
380
|
Writes documents to Qdrant using the specified policy.
|
|
381
|
+
|
|
378
382
|
The QdrantDocumentStore can handle duplicate documents based on the given policy.
|
|
379
383
|
The available policies are:
|
|
380
384
|
- `FAIL`: The operation will raise an error if any document already exists.
|
|
@@ -428,6 +432,7 @@ class QdrantDocumentStore:
|
|
|
428
432
|
) -> int:
|
|
429
433
|
"""
|
|
430
434
|
Asynchronously writes documents to Qdrant using the specified policy.
|
|
435
|
+
|
|
431
436
|
The QdrantDocumentStore can handle duplicate documents based on the given policy.
|
|
432
437
|
The available policies are:
|
|
433
438
|
- `FAIL`: The operation will raise an error if any document already exists.
|
|
@@ -603,14 +608,59 @@ class QdrantDocumentStore:
|
|
|
603
608
|
)
|
|
604
609
|
|
|
605
610
|
@staticmethod
|
|
606
|
-
def
|
|
607
|
-
"""
|
|
608
|
-
|
|
611
|
+
def _infer_type_from_value(value: Any) -> str:
|
|
612
|
+
"""
|
|
613
|
+
Infers the type from a metadata value for get_metadata_fields_info.
|
|
614
|
+
|
|
615
|
+
Returns types matching OpenSearch format for consistency:
|
|
616
|
+
- 'keyword' for strings
|
|
617
|
+
- 'long' for integers
|
|
618
|
+
- 'float' for floats
|
|
619
|
+
- 'boolean' for booleans
|
|
620
|
+
"""
|
|
621
|
+
if isinstance(value, bool):
|
|
622
|
+
return "boolean"
|
|
623
|
+
elif isinstance(value, int):
|
|
624
|
+
return "long"
|
|
625
|
+
elif isinstance(value, float):
|
|
626
|
+
return "float"
|
|
627
|
+
elif isinstance(value, str):
|
|
628
|
+
return "keyword"
|
|
629
|
+
else:
|
|
630
|
+
return "keyword"
|
|
631
|
+
|
|
632
|
+
@staticmethod
|
|
633
|
+
def _process_records_fields_info(records: list[Any], field_info: dict[str, dict[str, str]]) -> None:
|
|
634
|
+
"""
|
|
635
|
+
Update field_info from a batch of Qdrant records.
|
|
636
|
+
|
|
637
|
+
Used by get_metadata_fields_info (sync/async). Extracts metadata from
|
|
638
|
+
payload["meta"] and infers types for each field.
|
|
639
|
+
"""
|
|
640
|
+
for record in records:
|
|
641
|
+
if record.payload and "meta" in record.payload:
|
|
642
|
+
meta = record.payload["meta"]
|
|
643
|
+
for field_name, value in meta.items():
|
|
644
|
+
if value is not None and field_name not in field_info:
|
|
645
|
+
field_info[field_name] = {"type": QdrantDocumentStore._infer_type_from_value(value)}
|
|
646
|
+
|
|
647
|
+
@staticmethod
|
|
648
|
+
def _metadata_fields_info_from_schema(payload_schema: dict[str, Any]) -> dict[str, dict[str, str]]:
|
|
649
|
+
"""
|
|
650
|
+
Build field name -> {type: ...} dict from Qdrant payload_schema.
|
|
651
|
+
|
|
652
|
+
Used when payload_schema has indexed metadata fields (e.g. meta.category).
|
|
653
|
+
Returns empty dict when schema has no metadata field info.
|
|
654
|
+
"""
|
|
655
|
+
fields_info: dict[str, dict[str, str]] = {}
|
|
609
656
|
for field_name, field_config in payload_schema.items():
|
|
610
|
-
if
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
657
|
+
if field_name.startswith("meta."):
|
|
658
|
+
meta_field = field_name[5:]
|
|
659
|
+
if hasattr(field_config, "data_type"):
|
|
660
|
+
qdrant_type = str(field_config.data_type).lower()
|
|
661
|
+
fields_info[meta_field] = {"type": qdrant_type}
|
|
662
|
+
else:
|
|
663
|
+
fields_info[meta_field] = {"type": "unknown"}
|
|
614
664
|
return fields_info
|
|
615
665
|
|
|
616
666
|
@staticmethod
|
|
@@ -978,14 +1028,18 @@ class QdrantDocumentStore:
|
|
|
978
1028
|
logger.warning(f"Error {e} when calling QdrantDocumentStore.count_documents_by_filter_async()")
|
|
979
1029
|
return 0
|
|
980
1030
|
|
|
981
|
-
def get_metadata_fields_info(self) -> dict[str, str]:
|
|
1031
|
+
def get_metadata_fields_info(self) -> dict[str, dict[str, str]]:
|
|
982
1032
|
"""
|
|
983
|
-
Returns the information about the fields
|
|
1033
|
+
Returns the information about the metadata fields in the collection.
|
|
1034
|
+
|
|
1035
|
+
Since Qdrant may not have a payload schema for unindexed metadata,
|
|
1036
|
+
this method scrolls through documents to infer field types from
|
|
1037
|
+
payload["meta"].
|
|
984
1038
|
|
|
985
1039
|
:returns:
|
|
986
|
-
A dictionary mapping field names to their
|
|
1040
|
+
A dictionary mapping field names to their type information e.g.:
|
|
987
1041
|
```python
|
|
988
|
-
{"
|
|
1042
|
+
{"category": {"type": "keyword"}, "priority": {"type": "long"}}
|
|
989
1043
|
```
|
|
990
1044
|
"""
|
|
991
1045
|
self._initialize_client()
|
|
@@ -994,19 +1048,40 @@ class QdrantDocumentStore:
|
|
|
994
1048
|
try:
|
|
995
1049
|
collection_info = self._client.get_collection(self.index)
|
|
996
1050
|
payload_schema = collection_info.payload_schema or {}
|
|
997
|
-
|
|
1051
|
+
fields_info = self._metadata_fields_info_from_schema(payload_schema)
|
|
1052
|
+
|
|
1053
|
+
if not fields_info:
|
|
1054
|
+
next_offset = None
|
|
1055
|
+
while True:
|
|
1056
|
+
records, next_offset = self._client.scroll(
|
|
1057
|
+
collection_name=self.index,
|
|
1058
|
+
scroll_filter=None,
|
|
1059
|
+
limit=self.scroll_size,
|
|
1060
|
+
offset=next_offset,
|
|
1061
|
+
with_payload=True,
|
|
1062
|
+
with_vectors=False,
|
|
1063
|
+
)
|
|
1064
|
+
self._process_records_fields_info(records, fields_info)
|
|
1065
|
+
if self._check_stop_scrolling(next_offset):
|
|
1066
|
+
break
|
|
1067
|
+
|
|
1068
|
+
return fields_info
|
|
998
1069
|
except (UnexpectedResponse, ValueError) as e:
|
|
999
1070
|
logger.warning(f"Error {e} when calling QdrantDocumentStore.get_metadata_fields_info()")
|
|
1000
1071
|
return {}
|
|
1001
1072
|
|
|
1002
|
-
async def get_metadata_fields_info_async(self) -> dict[str, str]:
|
|
1073
|
+
async def get_metadata_fields_info_async(self) -> dict[str, dict[str, str]]:
|
|
1003
1074
|
"""
|
|
1004
|
-
Asynchronously returns the information about the fields
|
|
1075
|
+
Asynchronously returns the information about the metadata fields in the collection.
|
|
1076
|
+
|
|
1077
|
+
Since Qdrant may not have a payload schema for unindexed metadata,
|
|
1078
|
+
this method scrolls through documents to infer field types from
|
|
1079
|
+
payload["meta"].
|
|
1005
1080
|
|
|
1006
1081
|
:returns:
|
|
1007
|
-
A dictionary mapping field names to their
|
|
1082
|
+
A dictionary mapping field names to their type information e.g.:
|
|
1008
1083
|
```python
|
|
1009
|
-
{"
|
|
1084
|
+
{"category": {"type": "keyword"}, "priority": {"type": "long"}}
|
|
1010
1085
|
```
|
|
1011
1086
|
"""
|
|
1012
1087
|
await self._initialize_async_client()
|
|
@@ -1015,7 +1090,24 @@ class QdrantDocumentStore:
|
|
|
1015
1090
|
try:
|
|
1016
1091
|
collection_info = await self._async_client.get_collection(self.index)
|
|
1017
1092
|
payload_schema = collection_info.payload_schema or {}
|
|
1018
|
-
|
|
1093
|
+
fields_info = self._metadata_fields_info_from_schema(payload_schema)
|
|
1094
|
+
|
|
1095
|
+
if not fields_info:
|
|
1096
|
+
next_offset = None
|
|
1097
|
+
while True:
|
|
1098
|
+
records, next_offset = await self._async_client.scroll(
|
|
1099
|
+
collection_name=self.index,
|
|
1100
|
+
scroll_filter=None,
|
|
1101
|
+
limit=self.scroll_size,
|
|
1102
|
+
offset=next_offset,
|
|
1103
|
+
with_payload=True,
|
|
1104
|
+
with_vectors=False,
|
|
1105
|
+
)
|
|
1106
|
+
self._process_records_fields_info(records, fields_info)
|
|
1107
|
+
if self._check_stop_scrolling(next_offset):
|
|
1108
|
+
break
|
|
1109
|
+
|
|
1110
|
+
return fields_info
|
|
1019
1111
|
except (UnexpectedResponse, ValueError) as e:
|
|
1020
1112
|
logger.warning(f"Error {e} when calling QdrantDocumentStore.get_metadata_fields_info_async()")
|
|
1021
1113
|
return {}
|
|
@@ -1027,11 +1119,13 @@ class QdrantDocumentStore:
|
|
|
1027
1119
|
:param metadata_field: The metadata field key (inside ``meta``) to get the minimum and maximum values for.
|
|
1028
1120
|
|
|
1029
1121
|
:returns: A dictionary with the keys "min" and "max", where each value is the minimum or maximum value of the
|
|
1030
|
-
metadata field across all documents. Returns
|
|
1122
|
+
metadata field across all documents. Returns ``{"min": None, "max": None}`` if no documents have
|
|
1123
|
+
the field.
|
|
1031
1124
|
"""
|
|
1032
1125
|
self._initialize_client()
|
|
1033
1126
|
assert self._client is not None
|
|
1034
1127
|
|
|
1128
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
1035
1129
|
try:
|
|
1036
1130
|
min_value: Any = None
|
|
1037
1131
|
max_value: Any = None
|
|
@@ -1046,13 +1140,11 @@ class QdrantDocumentStore:
|
|
|
1046
1140
|
with_payload=True,
|
|
1047
1141
|
with_vectors=False,
|
|
1048
1142
|
)
|
|
1049
|
-
min_value, max_value = self._process_records_min_max(records,
|
|
1143
|
+
min_value, max_value = self._process_records_min_max(records, field_name, min_value, max_value)
|
|
1050
1144
|
if self._check_stop_scrolling(next_offset):
|
|
1051
1145
|
break
|
|
1052
1146
|
|
|
1053
|
-
|
|
1054
|
-
return {"min": min_value, "max": max_value}
|
|
1055
|
-
return {}
|
|
1147
|
+
return {"min": min_value, "max": max_value}
|
|
1056
1148
|
except Exception as e:
|
|
1057
1149
|
logger.warning(f"Error {e} when calling QdrantDocumentStore.get_metadata_field_min_max()")
|
|
1058
1150
|
return {}
|
|
@@ -1064,11 +1156,13 @@ class QdrantDocumentStore:
|
|
|
1064
1156
|
:param metadata_field: The metadata field key (inside ``meta``) to get the minimum and maximum values for.
|
|
1065
1157
|
|
|
1066
1158
|
:returns: A dictionary with the keys "min" and "max", where each value is the minimum or maximum value of the
|
|
1067
|
-
metadata field across all documents. Returns
|
|
1159
|
+
metadata field across all documents. Returns ``{"min": None, "max": None}`` if no documents have
|
|
1160
|
+
the field.
|
|
1068
1161
|
"""
|
|
1069
1162
|
await self._initialize_async_client()
|
|
1070
1163
|
assert self._async_client is not None
|
|
1071
1164
|
|
|
1165
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
1072
1166
|
try:
|
|
1073
1167
|
min_value: Any = None
|
|
1074
1168
|
max_value: Any = None
|
|
@@ -1083,13 +1177,11 @@ class QdrantDocumentStore:
|
|
|
1083
1177
|
with_payload=True,
|
|
1084
1178
|
with_vectors=False,
|
|
1085
1179
|
)
|
|
1086
|
-
min_value, max_value = self._process_records_min_max(records,
|
|
1180
|
+
min_value, max_value = self._process_records_min_max(records, field_name, min_value, max_value)
|
|
1087
1181
|
if self._check_stop_scrolling(next_offset):
|
|
1088
1182
|
break
|
|
1089
1183
|
|
|
1090
|
-
|
|
1091
|
-
return {"min": min_value, "max": max_value}
|
|
1092
|
-
return {}
|
|
1184
|
+
return {"min": min_value, "max": max_value}
|
|
1093
1185
|
except Exception as e:
|
|
1094
1186
|
logger.warning(f"Error {e} when calling QdrantDocumentStore.get_metadata_field_min_max_async()")
|
|
1095
1187
|
return {}
|
|
@@ -1135,8 +1227,9 @@ class QdrantDocumentStore:
|
|
|
1135
1227
|
self, filters: dict[str, Any], metadata_fields: list[str]
|
|
1136
1228
|
) -> dict[str, int]:
|
|
1137
1229
|
"""
|
|
1138
|
-
Asynchronously returns the number of unique values for each specified metadata field among documents
|
|
1139
|
-
|
|
1230
|
+
Asynchronously returns the number of unique values for each specified metadata field among documents.
|
|
1231
|
+
|
|
1232
|
+
Only documents that match the filters are considered.
|
|
1140
1233
|
|
|
1141
1234
|
:param filters: The filters to restrict the documents considered.
|
|
1142
1235
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1838,8 +1931,9 @@ class QdrantDocumentStore:
|
|
|
1838
1931
|
group_size: int | None = None,
|
|
1839
1932
|
) -> list[Document]:
|
|
1840
1933
|
"""
|
|
1841
|
-
Asynchronously retrieves documents based on dense and sparse embeddings
|
|
1842
|
-
|
|
1934
|
+
Asynchronously retrieves documents based on dense and sparse embeddings.
|
|
1935
|
+
|
|
1936
|
+
Fuses the results using Reciprocal Rank Fusion.
|
|
1843
1937
|
|
|
1844
1938
|
This method is not part of the public interface of `QdrantDocumentStore` and shouldn't be used directly.
|
|
1845
1939
|
Use the `QdrantHybridRetriever` instead.
|
|
@@ -2204,8 +2298,9 @@ class QdrantDocumentStore:
|
|
|
2204
2298
|
policy: DuplicatePolicy | None = None,
|
|
2205
2299
|
) -> list[Document]:
|
|
2206
2300
|
"""
|
|
2207
|
-
Checks whether any of the passed documents is already existing in the chosen index
|
|
2208
|
-
|
|
2301
|
+
Checks whether any of the passed documents is already existing in the chosen index.
|
|
2302
|
+
|
|
2303
|
+
Returns a list of documents that are not in the index yet.
|
|
2209
2304
|
|
|
2210
2305
|
:param documents: A list of Haystack Document objects.
|
|
2211
2306
|
:param policy: The duplicate policy to use when writing documents.
|
|
@@ -2231,9 +2326,9 @@ class QdrantDocumentStore:
|
|
|
2231
2326
|
policy: DuplicatePolicy | None = None,
|
|
2232
2327
|
) -> list[Document]:
|
|
2233
2328
|
"""
|
|
2234
|
-
Asynchronously checks whether any of the passed documents is already existing
|
|
2235
|
-
|
|
2236
|
-
documents that are not in the index yet.
|
|
2329
|
+
Asynchronously checks whether any of the passed documents is already existing in the chosen index.
|
|
2330
|
+
|
|
2331
|
+
Returns a list of documents that are not in the index yet.
|
|
2237
2332
|
|
|
2238
2333
|
:param documents: A list of Haystack Document objects.
|
|
2239
2334
|
:param policy: The duplicate policy to use when writing documents.
|
|
@@ -2380,15 +2475,17 @@ class QdrantDocumentStore:
|
|
|
2380
2475
|
]
|
|
2381
2476
|
|
|
2382
2477
|
if scale_score:
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
document
|
|
2478
|
+
documents = [
|
|
2479
|
+
replace(
|
|
2480
|
+
document,
|
|
2481
|
+
score=(document.score + 1) / 2
|
|
2482
|
+
if self.similarity == "cosine"
|
|
2483
|
+
else float(1 / (1 + exp(-document.score / 100))),
|
|
2484
|
+
)
|
|
2485
|
+
if document.score is not None
|
|
2486
|
+
else document
|
|
2487
|
+
for document in documents
|
|
2488
|
+
]
|
|
2392
2489
|
|
|
2393
2490
|
return documents
|
|
2394
2491
|
|
|
@@ -9,7 +9,8 @@ from qdrant_client.http import models
|
|
|
9
9
|
def convert_filters_to_qdrant(
|
|
10
10
|
filter_term: list[dict[str, Any]] | dict[str, Any] | models.Filter | None = None,
|
|
11
11
|
) -> models.Filter | None:
|
|
12
|
-
"""
|
|
12
|
+
"""
|
|
13
|
+
Converts Haystack filters to the format used by Qdrant.
|
|
13
14
|
|
|
14
15
|
:param filter_term: the haystack filter to be converted to qdrant.
|
|
15
16
|
:returns: a single Qdrant Filter or None.
|
|
@@ -228,6 +229,7 @@ def _build_gte_condition(key: str, value: str | float | int) -> models.Condition
|
|
|
228
229
|
|
|
229
230
|
|
|
230
231
|
def is_datetime_string(value: str) -> bool:
|
|
232
|
+
"""Return True if the given string can be parsed as an ISO 8601 datetime, False otherwise."""
|
|
231
233
|
try:
|
|
232
234
|
datetime.fromisoformat(value)
|
|
233
235
|
return True
|