weaviate-haystack 7.9.0__tar.gz → 8.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/CHANGELOG.md +15 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/PKG-INFO +2 -3
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/pyproject.toml +1 -2
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py +4 -4
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/document_stores/weaviate/_filters.py +57 -39
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/document_stores/weaviate/document_store.py +8 -2
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_bm25_retriever.py +20 -5
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_bm25_retriever_async.py +4 -5
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_document_store.py +126 -101
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_embedding_retriever.py +45 -7
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_embedding_retriever_async.py +29 -6
- weaviate_haystack-8.0.0/tests/test_filters.py +191 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_hybrid_retriever.py +32 -16
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_hybrid_retriever_async.py +7 -8
- weaviate_haystack-8.0.0/tests/test_retrieval.py +125 -0
- weaviate_haystack-7.9.0/tests/test_filters.py +0 -72
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/.gitignore +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/LICENSE.txt +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/README.md +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/docker-compose.yml +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/pydoc/config_docusaurus.yml +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/components/retrievers/weaviate/__init__.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/components/retrievers/weaviate/hybrid_retriever.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/document_stores/weaviate/__init__.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/src/haystack_integrations/document_stores/weaviate/auth.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/__init__.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/conftest.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_auth.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_document_store_async.py +0 -0
- {weaviate_haystack-7.9.0 → weaviate_haystack-8.0.0}/tests/test_files/robot1.jpg +0 -0
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/weaviate-v7.9.0] - 2026-09-21
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- Send X-Weaviate-Client-Integration telemetry header (#3954)
|
|
8
|
+
|
|
9
|
+
### ⚙️ CI
|
|
10
|
+
|
|
11
|
+
- Improve changelog generation; fix existing changelogs (#3883)
|
|
12
|
+
|
|
13
|
+
### 🧹 Chores
|
|
14
|
+
|
|
15
|
+
- Raise weaviate-client floor and test against Weaviate 1.39 (#3956)
|
|
16
|
+
|
|
17
|
+
|
|
3
18
|
## [integrations/weaviate-v7.8.0] - 2026-09-01
|
|
4
19
|
|
|
5
20
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: weaviate-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 8.0.0
|
|
4
4
|
Summary: An integration of Weaviate vector database with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md
|
|
@@ -11,14 +11,13 @@ License-File: LICENSE.txt
|
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: Programming Language :: Python
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.14
|
|
19
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
19
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
-
Requires-Python: >=3.
|
|
20
|
+
Requires-Python: >=3.11
|
|
22
21
|
Requires-Dist: haystack-ai>=2.28.0
|
|
23
22
|
Requires-Dist: python-dateutil>=2.7.0
|
|
24
23
|
Requires-Dist: weaviate-client>=4.23.0
|
|
@@ -7,7 +7,7 @@ name = "weaviate-haystack"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "An integration of Weaviate vector database with Haystack"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
13
|
authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
|
|
@@ -15,7 +15,6 @@ classifiers = [
|
|
|
15
15
|
"License :: OSI Approved :: Apache Software License",
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
17
17
|
"Programming Language :: Python",
|
|
18
|
-
"Programming Language :: Python :: 3.10",
|
|
19
18
|
"Programming Language :: Python :: 3.11",
|
|
20
19
|
"Programming Language :: Python :: 3.12",
|
|
21
20
|
"Programming Language :: Python :: 3.13",
|
|
@@ -143,8 +143,8 @@ class WeaviateEmbeddingRetriever:
|
|
|
143
143
|
filters = apply_filter_policy(self._filter_policy, self._filters, filters)
|
|
144
144
|
top_k = top_k or self._top_k
|
|
145
145
|
|
|
146
|
-
distance = distance
|
|
147
|
-
certainty = certainty
|
|
146
|
+
distance = self._distance if distance is None else distance
|
|
147
|
+
certainty = self._certainty if certainty is None else certainty
|
|
148
148
|
if distance is not None and certainty is not None:
|
|
149
149
|
msg = f"Can't use 'distance' ({distance}) and 'certainty' ({certainty}) parameters together"
|
|
150
150
|
raise ValueError(msg)
|
|
@@ -191,8 +191,8 @@ class WeaviateEmbeddingRetriever:
|
|
|
191
191
|
filters = apply_filter_policy(self._filter_policy, self._filters, filters)
|
|
192
192
|
top_k = top_k or self._top_k
|
|
193
193
|
|
|
194
|
-
distance = distance
|
|
195
|
-
certainty = certainty
|
|
194
|
+
distance = self._distance if distance is None else distance
|
|
195
|
+
certainty = self._certainty if certainty is None else certainty
|
|
196
196
|
if distance is not None and certainty is not None:
|
|
197
197
|
msg = f"Can't use 'distance' ({distance}) and 'certainty' ({certainty}) parameters together"
|
|
198
198
|
raise ValueError(msg)
|
|
@@ -33,6 +33,10 @@ def convert_filters(filters: dict[str, Any]) -> FilterReturn:
|
|
|
33
33
|
Note: The ``contains`` operator performs substring matching and is
|
|
34
34
|
**case-sensitive**. For case-insensitive matching, normalize the value
|
|
35
35
|
(e.g., lowercase) before building the filter.
|
|
36
|
+
|
|
37
|
+
Note: ``NOT`` is applied client-side by inverting its conditions (De Morgan), so it works
|
|
38
|
+
on any Weaviate version. The only exception is ``NOT`` over ``contains``, which has no
|
|
39
|
+
invertible counterpart and uses Weaviate's native ``NOT`` operator (Weaviate 1.33 or later).
|
|
36
40
|
"""
|
|
37
41
|
if not isinstance(filters, dict):
|
|
38
42
|
msg = "Filters must be a dictionary"
|
|
@@ -43,7 +47,13 @@ def convert_filters(filters: dict[str, Any]) -> FilterReturn:
|
|
|
43
47
|
return _parse_logical_condition(filters)
|
|
44
48
|
|
|
45
49
|
|
|
46
|
-
|
|
50
|
+
LOGICAL_OPERATORS = {
|
|
51
|
+
"AND": Filter.all_of,
|
|
52
|
+
"OR": Filter.any_of,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
COMPARISON_INVERSE = {
|
|
47
57
|
"==": "!=",
|
|
48
58
|
"!=": "==",
|
|
49
59
|
">": "<=",
|
|
@@ -52,38 +62,10 @@ OPERATOR_INVERSE = {
|
|
|
52
62
|
"<=": ">",
|
|
53
63
|
"in": "not in",
|
|
54
64
|
"not in": "in",
|
|
55
|
-
"AND": "OR",
|
|
56
|
-
"OR": "AND",
|
|
57
|
-
"NOT": "OR",
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def _invert_condition(filters: dict[str, Any]) -> dict[str, Any]:
|
|
62
|
-
"""
|
|
63
|
-
Invert condition recursively.
|
|
64
|
-
|
|
65
|
-
Weaviate doesn't support NOT filters so we need to invert them ourselves.
|
|
66
|
-
"""
|
|
67
|
-
inverted_condition = filters.copy()
|
|
68
|
-
if "operator" not in filters:
|
|
69
|
-
# Let's not handle this stuff in here, we'll fail later on anyway.
|
|
70
|
-
return inverted_condition
|
|
71
|
-
inverted_condition["operator"] = OPERATOR_INVERSE[filters["operator"]]
|
|
72
|
-
if "conditions" in filters:
|
|
73
|
-
inverted_condition["conditions"] = []
|
|
74
|
-
for condition in filters["conditions"]:
|
|
75
|
-
inverted_condition["conditions"].append(_invert_condition(condition))
|
|
76
|
-
|
|
77
|
-
return inverted_condition
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
LOGICAL_OPERATORS = {
|
|
81
|
-
"AND": Filter.all_of,
|
|
82
|
-
"OR": Filter.any_of,
|
|
83
65
|
}
|
|
84
66
|
|
|
85
67
|
|
|
86
|
-
def
|
|
68
|
+
def _validate_logical_condition(condition: dict[str, Any]) -> None:
|
|
87
69
|
if "operator" not in condition:
|
|
88
70
|
msg = f"'operator' key missing in {condition}"
|
|
89
71
|
raise FilterError(msg)
|
|
@@ -91,23 +73,55 @@ def _parse_logical_condition(condition: dict[str, Any]) -> FilterReturn:
|
|
|
91
73
|
msg = f"'conditions' key missing in {condition}"
|
|
92
74
|
raise FilterError(msg)
|
|
93
75
|
|
|
76
|
+
|
|
77
|
+
def _parse_logical_condition(condition: dict[str, Any]) -> FilterReturn:
|
|
78
|
+
_validate_logical_condition(condition)
|
|
79
|
+
|
|
94
80
|
operator = condition["operator"]
|
|
95
81
|
if operator in ["AND", "OR"]:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
82
|
+
return LOGICAL_OPERATORS[operator](_parse_operands(condition["conditions"]))
|
|
83
|
+
elif operator == "NOT":
|
|
84
|
+
# A NOT node negates the conjunction of its conditions: NOT(a AND b) == NOT(a) OR NOT(b).
|
|
85
|
+
return Filter.any_of([_negate(c) for c in condition["conditions"]])
|
|
86
|
+
else:
|
|
87
|
+
msg = f"Unknown logical operator '{operator}'"
|
|
88
|
+
raise FilterError(msg)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _negate(condition: dict[str, Any]) -> FilterReturn:
|
|
92
|
+
"""
|
|
93
|
+
Build the negation of a condition by applying De Morgan's laws.
|
|
94
|
+
|
|
95
|
+
Negation is done client-side so that it works on Weaviate versions without a native NOT operator.
|
|
96
|
+
"""
|
|
97
|
+
if "field" in condition:
|
|
98
|
+
operator = condition.get("operator")
|
|
99
|
+
if operator in COMPARISON_INVERSE:
|
|
100
|
+
return _parse_comparison_condition({**condition, "operator": COMPARISON_INVERSE[operator]})
|
|
101
|
+
# Operators like `contains` have no invertible counterpart, so we fall back to Weaviate's
|
|
102
|
+
# native NOT, which requires Weaviate 1.33 or later.
|
|
103
|
+
# Unknown operators raise a FilterError while parsing the condition.
|
|
104
|
+
return Filter.not_(_parse_comparison_condition(condition))
|
|
105
|
+
|
|
106
|
+
_validate_logical_condition(condition)
|
|
107
|
+
operator = condition["operator"]
|
|
108
|
+
conditions = condition["conditions"]
|
|
109
|
+
if operator == "AND":
|
|
110
|
+
return Filter.any_of([_negate(c) for c in conditions])
|
|
111
|
+
elif operator == "OR":
|
|
112
|
+
return Filter.all_of([_negate(c) for c in conditions])
|
|
103
113
|
elif operator == "NOT":
|
|
104
|
-
|
|
105
|
-
return
|
|
114
|
+
# The two negations cancel out, leaving the conjunction of the conditions.
|
|
115
|
+
return Filter.all_of(_parse_operands(conditions))
|
|
106
116
|
else:
|
|
107
117
|
msg = f"Unknown logical operator '{operator}'"
|
|
108
118
|
raise FilterError(msg)
|
|
109
119
|
|
|
110
120
|
|
|
121
|
+
def _parse_operands(conditions: list[dict[str, Any]]) -> list[FilterReturn]:
|
|
122
|
+
return [_parse_comparison_condition(c) if "field" in c else _parse_logical_condition(c) for c in conditions]
|
|
123
|
+
|
|
124
|
+
|
|
111
125
|
def _handle_date(value: Any) -> str:
|
|
112
126
|
if isinstance(value, str):
|
|
113
127
|
try:
|
|
@@ -282,6 +296,10 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> FilterReturn:
|
|
|
282
296
|
operator: str = condition["operator"]
|
|
283
297
|
value: Any = condition["value"]
|
|
284
298
|
|
|
299
|
+
if operator not in COMPARISON_OPERATORS:
|
|
300
|
+
msg = f"Unknown comparison operator '{operator}'. Valid operators are: {list(COMPARISON_OPERATORS.keys())}"
|
|
301
|
+
raise FilterError(msg)
|
|
302
|
+
|
|
285
303
|
return COMPARISON_OPERATORS[operator](field, value)
|
|
286
304
|
|
|
287
305
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
import base64
|
|
6
|
-
import datetime
|
|
6
|
+
import datetime as dt
|
|
7
7
|
import importlib.metadata
|
|
8
8
|
import json
|
|
9
9
|
from contextlib import suppress
|
|
@@ -876,7 +876,7 @@ class WeaviateDocumentStore:
|
|
|
876
876
|
document_data.pop("blob_mime_type", None)
|
|
877
877
|
|
|
878
878
|
for key, value in document_data.items():
|
|
879
|
-
if isinstance(value,
|
|
879
|
+
if isinstance(value, dt.datetime):
|
|
880
880
|
document_data[key] = value.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
881
881
|
|
|
882
882
|
if weaviate_meta := getattr(data, "metadata", None):
|
|
@@ -983,6 +983,9 @@ class WeaviateDocumentStore:
|
|
|
983
983
|
matching). For case-insensitive matching, normalize the value before
|
|
984
984
|
building the filter.
|
|
985
985
|
|
|
986
|
+
Note: ``NOT`` over the ``contains`` operator is translated to Weaviate's
|
|
987
|
+
native ``NOT``, which requires Weaviate 1.33 or later.
|
|
988
|
+
|
|
986
989
|
:param filters: The filters to apply to the document list.
|
|
987
990
|
:returns: A list of Documents that match the given filters.
|
|
988
991
|
"""
|
|
@@ -1006,6 +1009,9 @@ class WeaviateDocumentStore:
|
|
|
1006
1009
|
matching). For case-insensitive matching, normalize the value before
|
|
1007
1010
|
building the filter.
|
|
1008
1011
|
|
|
1012
|
+
Note: ``NOT`` over the ``contains`` operator is translated to Weaviate's
|
|
1013
|
+
native ``NOT``, which requires Weaviate 1.33 or later.
|
|
1014
|
+
|
|
1009
1015
|
:param filters: The filters to apply to the document list.
|
|
1010
1016
|
:returns: A list of Documents that match the given filters.
|
|
1011
1017
|
"""
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import
|
|
5
|
+
from unittest.mock import create_autospec, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
|
+
from haystack import Pipeline
|
|
8
9
|
from haystack.document_stores.types import FilterPolicy
|
|
9
10
|
|
|
10
11
|
from haystack_integrations.components.retrievers.weaviate import WeaviateBM25Retriever
|
|
@@ -12,7 +13,7 @@ from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
|
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
def test_close():
|
|
15
|
-
mock_document_store =
|
|
16
|
+
mock_document_store = create_autospec(WeaviateDocumentStore, instance=True)
|
|
16
17
|
retriever = WeaviateBM25Retriever(document_store=mock_document_store)
|
|
17
18
|
|
|
18
19
|
retriever.close()
|
|
@@ -22,7 +23,7 @@ def test_close():
|
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
def test_init_default():
|
|
25
|
-
mock_document_store =
|
|
26
|
+
mock_document_store = create_autospec(WeaviateDocumentStore, instance=True)
|
|
26
27
|
retriever = WeaviateBM25Retriever(document_store=mock_document_store)
|
|
27
28
|
assert retriever._document_store == mock_document_store
|
|
28
29
|
assert retriever._filters == {}
|
|
@@ -153,10 +154,24 @@ def test_from_dict_no_filter_policy(_mock_weaviate):
|
|
|
153
154
|
assert retriever._filter_policy == FilterPolicy.REPLACE
|
|
154
155
|
|
|
155
156
|
|
|
156
|
-
|
|
157
|
-
|
|
157
|
+
def test_run():
|
|
158
|
+
mock_document_store = create_autospec(WeaviateDocumentStore, instance=True)
|
|
158
159
|
retriever = WeaviateBM25Retriever(document_store=mock_document_store)
|
|
159
160
|
query = "some query"
|
|
160
161
|
filters = {"field": "content", "operator": "==", "value": "Some text"}
|
|
161
162
|
retriever.run(query=query, filters=filters, top_k=5)
|
|
162
163
|
mock_document_store._bm25_retrieval.assert_called_once_with(query=query, filters=filters, top_k=5)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_pipeline_serde():
|
|
167
|
+
"""A pipeline holding the retriever must survive a dumps/loads round trip, nested store included."""
|
|
168
|
+
document_store = WeaviateDocumentStore(url="http://localhost:8080")
|
|
169
|
+
pipeline = Pipeline()
|
|
170
|
+
pipeline.add_component("retriever", WeaviateBM25Retriever(document_store=document_store, top_k=3))
|
|
171
|
+
|
|
172
|
+
reloaded = Pipeline.loads(pipeline.dumps()).get_component("retriever")
|
|
173
|
+
|
|
174
|
+
assert isinstance(reloaded, WeaviateBM25Retriever)
|
|
175
|
+
assert isinstance(reloaded._document_store, WeaviateDocumentStore)
|
|
176
|
+
assert reloaded._document_store.to_dict() == document_store.to_dict()
|
|
177
|
+
assert reloaded._top_k == 3
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import
|
|
5
|
+
from unittest.mock import create_autospec
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
|
|
@@ -12,8 +12,7 @@ from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
|
|
|
12
12
|
|
|
13
13
|
@pytest.mark.asyncio
|
|
14
14
|
async def test_close_async():
|
|
15
|
-
mock_document_store =
|
|
16
|
-
mock_document_store.close_async = AsyncMock()
|
|
15
|
+
mock_document_store = create_autospec(WeaviateDocumentStore, instance=True)
|
|
17
16
|
retriever = WeaviateBM25Retriever(document_store=mock_document_store)
|
|
18
17
|
|
|
19
18
|
await retriever.close_async()
|
|
@@ -24,8 +23,8 @@ async def test_close_async():
|
|
|
24
23
|
|
|
25
24
|
@pytest.mark.asyncio
|
|
26
25
|
async def test_run_async_calls_async_retrieval():
|
|
27
|
-
mock_document_store =
|
|
28
|
-
mock_document_store._bm25_retrieval_async =
|
|
26
|
+
mock_document_store = create_autospec(WeaviateDocumentStore, instance=True)
|
|
27
|
+
mock_document_store._bm25_retrieval_async.return_value = []
|
|
29
28
|
|
|
30
29
|
retriever = WeaviateBM25Retriever(document_store=mock_document_store)
|
|
31
30
|
|
|
@@ -305,76 +305,8 @@ def test_close_is_exception_safe():
|
|
|
305
305
|
assert document_store._collection is None
|
|
306
306
|
|
|
307
307
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
DocumentStoreBaseExtendedTests,
|
|
311
|
-
CountDocumentsByFilterTest,
|
|
312
|
-
CountUniqueMetadataByFilterTest,
|
|
313
|
-
GetMetadataFieldsInfoTest,
|
|
314
|
-
GetMetadataFieldMinMaxTest,
|
|
315
|
-
GetMetadataFieldUniqueValuesTest,
|
|
316
|
-
):
|
|
317
|
-
@pytest.fixture
|
|
318
|
-
def document_store(self, request) -> Generator[WeaviateDocumentStore, None, None]:
|
|
319
|
-
# Use a different index for each test so we can run them in parallel
|
|
320
|
-
collection_settings = {
|
|
321
|
-
"class": f"{request.node.name}",
|
|
322
|
-
"invertedIndexConfig": {"indexNullState": True},
|
|
323
|
-
"properties": [
|
|
324
|
-
*DOCUMENT_COLLECTION_PROPERTIES,
|
|
325
|
-
{"name": "number", "dataType": ["int"]},
|
|
326
|
-
{"name": "date", "dataType": ["date"]},
|
|
327
|
-
{"name": "category", "dataType": ["text"]},
|
|
328
|
-
{"name": "status", "dataType": ["text"]},
|
|
329
|
-
],
|
|
330
|
-
}
|
|
331
|
-
store = WeaviateDocumentStore(
|
|
332
|
-
url="http://localhost:8080",
|
|
333
|
-
collection_settings=collection_settings,
|
|
334
|
-
)
|
|
335
|
-
yield store
|
|
336
|
-
store.client.collections.delete(collection_settings["class"])
|
|
337
|
-
store.close()
|
|
338
|
-
|
|
339
|
-
@pytest.fixture
|
|
340
|
-
def filterable_docs(self) -> list[Document]:
|
|
341
|
-
"""
|
|
342
|
-
This fixture has been copied from haystack/testing/document_store.py and modified to
|
|
343
|
-
use a different date format.
|
|
344
|
-
Weaviate forces RFC 3339 date strings.
|
|
345
|
-
The original fixture uses ISO 8601 date strings.
|
|
346
|
-
"""
|
|
347
|
-
documents = create_filterable_docs()
|
|
348
|
-
for i in range(len(documents)):
|
|
349
|
-
if date := documents[i].meta.get("date"):
|
|
350
|
-
documents[i].meta["date"] = f"{date}Z"
|
|
351
|
-
return documents
|
|
352
|
-
|
|
353
|
-
def assert_documents_are_equal(self, received: list[Document], expected: list[Document]):
|
|
354
|
-
assert len(received) == len(expected)
|
|
355
|
-
received = sorted(received, key=lambda doc: doc.id)
|
|
356
|
-
expected = sorted(expected, key=lambda doc: doc.id)
|
|
357
|
-
for received_doc, expected_doc in zip(received, expected, strict=True):
|
|
358
|
-
received_doc_dict = received_doc.to_dict(flatten=False)
|
|
359
|
-
expected_doc_dict = expected_doc.to_dict(flatten=False)
|
|
360
|
-
|
|
361
|
-
# Weaviate stores embeddings with lower precision floats so we handle that here.
|
|
362
|
-
assert np_array_equal(
|
|
363
|
-
np_array(received_doc_dict.pop("embedding", None), dtype=np_float32),
|
|
364
|
-
np_array(expected_doc_dict.pop("embedding", None), dtype=np_float32),
|
|
365
|
-
equal_nan=True,
|
|
366
|
-
)
|
|
367
|
-
|
|
368
|
-
received_meta = received_doc_dict.pop("meta", None)
|
|
369
|
-
expected_meta = expected_doc_dict.pop("meta", None)
|
|
370
|
-
|
|
371
|
-
assert received_doc_dict == expected_doc_dict
|
|
372
|
-
|
|
373
|
-
# If a meta field is not set in a saved document, it will be None when retrieved
|
|
374
|
-
# from Weaviate so we need to handle that.
|
|
375
|
-
meta_keys = set(received_meta.keys()).union(set(expected_meta.keys()))
|
|
376
|
-
for key in meta_keys:
|
|
377
|
-
assert received_meta.get(key) == expected_meta.get(key)
|
|
308
|
+
class TestWeaviateDocumentStoreSerde:
|
|
309
|
+
"""Unit tests that need no running Weaviate: serialization and document conversion."""
|
|
378
310
|
|
|
379
311
|
@patch("haystack_integrations.document_stores.weaviate.document_store.weaviate.WeaviateClient")
|
|
380
312
|
def test_connection(self, mock_weaviate_client_class, monkeypatch):
|
|
@@ -427,22 +359,6 @@ class TestWeaviateDocumentStore(
|
|
|
427
359
|
{"class": "My_collection", "properties": DOCUMENT_COLLECTION_PROPERTIES}
|
|
428
360
|
)
|
|
429
361
|
|
|
430
|
-
def test_close_and_reopen(self, document_store: WeaviateDocumentStore) -> None:
|
|
431
|
-
# Initialise client and collection
|
|
432
|
-
assert document_store.client is not None
|
|
433
|
-
assert document_store.collection is not None
|
|
434
|
-
|
|
435
|
-
document_store.close()
|
|
436
|
-
|
|
437
|
-
assert document_store._client is None
|
|
438
|
-
assert document_store._collection is None
|
|
439
|
-
|
|
440
|
-
# Initialise client and collection, then test it stills works after reopening
|
|
441
|
-
assert document_store.client is not None
|
|
442
|
-
assert document_store.collection is not None
|
|
443
|
-
|
|
444
|
-
assert document_store.count_documents() == 0
|
|
445
|
-
|
|
446
362
|
@patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
|
|
447
363
|
def test_to_dict(self, _mock_weaviate, monkeypatch):
|
|
448
364
|
monkeypatch.setenv("WEAVIATE_API_KEY", "my_api_key")
|
|
@@ -645,6 +561,109 @@ class TestWeaviateDocumentStore(
|
|
|
645
561
|
assert doc.score is None
|
|
646
562
|
assert doc.meta == {"key": "value"}
|
|
647
563
|
|
|
564
|
+
def test_schema_class_name_conversion_preserves_pascal_case(self):
|
|
565
|
+
collection_settings = {"class": "CaseDocument"}
|
|
566
|
+
doc_score = WeaviateDocumentStore(
|
|
567
|
+
url="http://localhost:8080",
|
|
568
|
+
collection_settings=collection_settings,
|
|
569
|
+
)
|
|
570
|
+
assert doc_score._collection_settings["class"] == "CaseDocument"
|
|
571
|
+
|
|
572
|
+
collection_settings = {"class": "lower_case_name"}
|
|
573
|
+
doc_score = WeaviateDocumentStore(
|
|
574
|
+
url="http://localhost:8080",
|
|
575
|
+
collection_settings=collection_settings,
|
|
576
|
+
)
|
|
577
|
+
assert doc_score._collection_settings["class"] == "Lower_case_name"
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
@pytest.mark.integration
|
|
581
|
+
class TestWeaviateDocumentStore(
|
|
582
|
+
DocumentStoreBaseExtendedTests,
|
|
583
|
+
CountDocumentsByFilterTest,
|
|
584
|
+
CountUniqueMetadataByFilterTest,
|
|
585
|
+
GetMetadataFieldsInfoTest,
|
|
586
|
+
GetMetadataFieldMinMaxTest,
|
|
587
|
+
GetMetadataFieldUniqueValuesTest,
|
|
588
|
+
):
|
|
589
|
+
@pytest.fixture
|
|
590
|
+
def document_store(self, request) -> Generator[WeaviateDocumentStore, None, None]:
|
|
591
|
+
# Use a different index for each test so we can run them in parallel
|
|
592
|
+
collection_settings = {
|
|
593
|
+
"class": f"{request.node.name}",
|
|
594
|
+
"invertedIndexConfig": {"indexNullState": True},
|
|
595
|
+
"properties": [
|
|
596
|
+
*DOCUMENT_COLLECTION_PROPERTIES,
|
|
597
|
+
{"name": "number", "dataType": ["int"]},
|
|
598
|
+
{"name": "date", "dataType": ["date"]},
|
|
599
|
+
{"name": "category", "dataType": ["text"]},
|
|
600
|
+
{"name": "status", "dataType": ["text"]},
|
|
601
|
+
],
|
|
602
|
+
}
|
|
603
|
+
store = WeaviateDocumentStore(
|
|
604
|
+
url="http://localhost:8080",
|
|
605
|
+
collection_settings=collection_settings,
|
|
606
|
+
)
|
|
607
|
+
yield store
|
|
608
|
+
store.client.collections.delete(collection_settings["class"])
|
|
609
|
+
store.close()
|
|
610
|
+
|
|
611
|
+
@pytest.fixture
|
|
612
|
+
def filterable_docs(self) -> list[Document]:
|
|
613
|
+
"""
|
|
614
|
+
This fixture has been copied from haystack/testing/document_store.py and modified to
|
|
615
|
+
use a different date format.
|
|
616
|
+
Weaviate forces RFC 3339 date strings.
|
|
617
|
+
The original fixture uses ISO 8601 date strings.
|
|
618
|
+
"""
|
|
619
|
+
documents = create_filterable_docs()
|
|
620
|
+
for i in range(len(documents)):
|
|
621
|
+
if date := documents[i].meta.get("date"):
|
|
622
|
+
documents[i].meta["date"] = f"{date}Z"
|
|
623
|
+
return documents
|
|
624
|
+
|
|
625
|
+
def assert_documents_are_equal(self, received: list[Document], expected: list[Document]):
|
|
626
|
+
assert len(received) == len(expected)
|
|
627
|
+
received = sorted(received, key=lambda doc: doc.id)
|
|
628
|
+
expected = sorted(expected, key=lambda doc: doc.id)
|
|
629
|
+
for received_doc, expected_doc in zip(received, expected, strict=True):
|
|
630
|
+
received_doc_dict = received_doc.to_dict(flatten=False)
|
|
631
|
+
expected_doc_dict = expected_doc.to_dict(flatten=False)
|
|
632
|
+
|
|
633
|
+
# Weaviate stores embeddings with lower precision floats so we handle that here.
|
|
634
|
+
assert np_array_equal(
|
|
635
|
+
np_array(received_doc_dict.pop("embedding", None), dtype=np_float32),
|
|
636
|
+
np_array(expected_doc_dict.pop("embedding", None), dtype=np_float32),
|
|
637
|
+
equal_nan=True,
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
received_meta = received_doc_dict.pop("meta", None)
|
|
641
|
+
expected_meta = expected_doc_dict.pop("meta", None)
|
|
642
|
+
|
|
643
|
+
assert received_doc_dict == expected_doc_dict
|
|
644
|
+
|
|
645
|
+
# If a meta field is not set in a saved document, it will be None when retrieved
|
|
646
|
+
# from Weaviate so we need to handle that.
|
|
647
|
+
meta_keys = set(received_meta.keys()).union(set(expected_meta.keys()))
|
|
648
|
+
for key in meta_keys:
|
|
649
|
+
assert received_meta.get(key) == expected_meta.get(key)
|
|
650
|
+
|
|
651
|
+
def test_close_and_reopen(self, document_store: WeaviateDocumentStore) -> None:
|
|
652
|
+
# Initialise client and collection
|
|
653
|
+
assert document_store.client is not None
|
|
654
|
+
assert document_store.collection is not None
|
|
655
|
+
|
|
656
|
+
document_store.close()
|
|
657
|
+
|
|
658
|
+
assert document_store._client is None
|
|
659
|
+
assert document_store._collection is None
|
|
660
|
+
|
|
661
|
+
# Initialise client and collection, then test it stills works after reopening
|
|
662
|
+
assert document_store.client is not None
|
|
663
|
+
assert document_store.collection is not None
|
|
664
|
+
|
|
665
|
+
assert document_store.count_documents() == 0
|
|
666
|
+
|
|
648
667
|
def test_write_documents(self, document_store):
|
|
649
668
|
"""
|
|
650
669
|
Test write_documents() with default policy overwrites existing documents.
|
|
@@ -756,6 +775,27 @@ class TestWeaviateDocumentStore(
|
|
|
756
775
|
],
|
|
757
776
|
)
|
|
758
777
|
|
|
778
|
+
def test_nested_not_operator(self, document_store, filterable_docs):
|
|
779
|
+
"""NOT(NOT(x)) must select the same Documents as x, matching `document_matches_filter`."""
|
|
780
|
+
document_store.write_documents(filterable_docs)
|
|
781
|
+
inner = {"field": "meta.number", "operator": "==", "value": 100}
|
|
782
|
+
nested_not = {"operator": "NOT", "conditions": [{"operator": "NOT", "conditions": [inner]}]}
|
|
783
|
+
|
|
784
|
+
result = document_store.filter_documents(nested_not)
|
|
785
|
+
|
|
786
|
+
self.assert_documents_are_equal(result, [d for d in filterable_docs if d.meta.get("number") == 100])
|
|
787
|
+
|
|
788
|
+
def test_not_operator_over_contains(self, document_store, filterable_docs):
|
|
789
|
+
"""`contains` has no inverted counterpart, so negating it used to raise a bare KeyError."""
|
|
790
|
+
document_store.write_documents(filterable_docs)
|
|
791
|
+
filters = {"operator": "NOT", "conditions": [{"field": "meta.name", "operator": "contains", "value": "name_0"}]}
|
|
792
|
+
|
|
793
|
+
result = document_store.filter_documents(filters)
|
|
794
|
+
|
|
795
|
+
self.assert_documents_are_equal(
|
|
796
|
+
result, [d for d in filterable_docs if "name_0" not in (d.meta.get("name") or "")]
|
|
797
|
+
)
|
|
798
|
+
|
|
759
799
|
def test_split_overlap_preserved(self, document_store):
|
|
760
800
|
"""Split overlap meta is written and read back correctly."""
|
|
761
801
|
overlap = [
|
|
@@ -1072,21 +1112,6 @@ class TestWeaviateDocumentStore(
|
|
|
1072
1112
|
with pytest.raises(DocumentStoreError):
|
|
1073
1113
|
document_store.filter_documents({"field": "content", "operator": "==", "value": "This is some content"})
|
|
1074
1114
|
|
|
1075
|
-
def test_schema_class_name_conversion_preserves_pascal_case(self):
|
|
1076
|
-
collection_settings = {"class": "CaseDocument"}
|
|
1077
|
-
doc_score = WeaviateDocumentStore(
|
|
1078
|
-
url="http://localhost:8080",
|
|
1079
|
-
collection_settings=collection_settings,
|
|
1080
|
-
)
|
|
1081
|
-
assert doc_score._collection_settings["class"] == "CaseDocument"
|
|
1082
|
-
|
|
1083
|
-
collection_settings = {"class": "lower_case_name"}
|
|
1084
|
-
doc_score = WeaviateDocumentStore(
|
|
1085
|
-
url="http://localhost:8080",
|
|
1086
|
-
collection_settings=collection_settings,
|
|
1087
|
-
)
|
|
1088
|
-
assert doc_score._collection_settings["class"] == "Lower_case_name"
|
|
1089
|
-
|
|
1090
1115
|
@pytest.mark.skipif(
|
|
1091
1116
|
not os.environ.get("WEAVIATE_API_KEY", None) and not os.environ.get("WEAVIATE_CLOUD_CLUSTER_URL", None),
|
|
1092
1117
|
reason="Both WEAVIATE_API_KEY and WEAVIATE_CLOUD_CLUSTER_URL are not set. Skipping test.",
|