opensearch-haystack 7.0.0__tar.gz → 7.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/CHANGELOG.md +7 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/PKG-INFO +3 -2
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/pyproject.toml +16 -3
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +1 -1
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +1 -1
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +1 -1
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +12 -6
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +1 -1
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/document_stores/opensearch/auth.py +1 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +65 -28
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_document_store.py +195 -158
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_document_store_async.py +48 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/.gitignore +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/LICENSE +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/README.md +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/docker-compose.yml +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/document_stores/opensearch/filters.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/__init__.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/conftest.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_auth.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_bm25_retriever.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_embedding_retriever.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_filters.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_sql_retriever.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.1.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -15,10 +15,11 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
21
|
Requires-Python: >=3.10
|
|
21
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.26.1
|
|
22
23
|
Requires-Dist: opensearch-py[async]>=3.0.0
|
|
23
24
|
Description-Content-Type: text/markdown
|
|
24
25
|
|
|
@@ -19,12 +19,13 @@ classifiers = [
|
|
|
19
19
|
"Programming Language :: Python :: 3.11",
|
|
20
20
|
"Programming Language :: Python :: 3.12",
|
|
21
21
|
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
22
23
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
23
24
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
24
25
|
]
|
|
25
26
|
|
|
26
27
|
dependencies = [
|
|
27
|
-
"haystack-ai>=2.
|
|
28
|
+
"haystack-ai>=2.26.1",
|
|
28
29
|
"opensearch-py[async]>=3.0.0"
|
|
29
30
|
]
|
|
30
31
|
|
|
@@ -70,7 +71,8 @@ dependencies = [
|
|
|
70
71
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
71
72
|
integration = 'pytest -m "integration" {args:tests}'
|
|
72
73
|
all = 'pytest {args:tests}'
|
|
73
|
-
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
74
|
+
unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
|
|
75
|
+
integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
|
|
74
76
|
|
|
75
77
|
types = """mypy -p haystack_integrations.document_stores.opensearch \
|
|
76
78
|
-p haystack_integrations.components.retrievers.opensearch {args}"""
|
|
@@ -89,9 +91,17 @@ line-length = 120
|
|
|
89
91
|
[tool.ruff.lint]
|
|
90
92
|
select = [
|
|
91
93
|
"A",
|
|
94
|
+
"ANN",
|
|
92
95
|
"ARG",
|
|
93
96
|
"B",
|
|
94
97
|
"C",
|
|
98
|
+
"D102", # Missing docstring in public method
|
|
99
|
+
"D103", # Missing docstring in public function
|
|
100
|
+
"D205", # 1 blank line required between summary line and description
|
|
101
|
+
"D209", # Closing triple quotes go to new line
|
|
102
|
+
"D213", # summary lines must be positioned on the second physical line of the docstring
|
|
103
|
+
"D417", # Missing argument descriptions in the docstring
|
|
104
|
+
"D419", # Docstring is empty
|
|
95
105
|
"DTZ",
|
|
96
106
|
"E",
|
|
97
107
|
"EM",
|
|
@@ -131,6 +141,8 @@ ignore = [
|
|
|
131
141
|
"PLR0915",
|
|
132
142
|
# Ignore asserts
|
|
133
143
|
"S101",
|
|
144
|
+
# Allow `Any` - used legitimately for dynamic types and SDK boundaries
|
|
145
|
+
"ANN401",
|
|
134
146
|
]
|
|
135
147
|
|
|
136
148
|
[tool.ruff.lint.isort]
|
|
@@ -141,11 +153,12 @@ ban-relative-imports = "parents"
|
|
|
141
153
|
|
|
142
154
|
[tool.ruff.lint.per-file-ignores]
|
|
143
155
|
# Tests can use magic values, assertions, and relative imports
|
|
144
|
-
"tests/**/*" = ["PLR2004", "S101", "TID252"]
|
|
156
|
+
"tests/**/*" = ["D", "PLR2004", "S101", "TID252", "ANN"]
|
|
145
157
|
|
|
146
158
|
[tool.coverage.run]
|
|
147
159
|
source = ["haystack_integrations"]
|
|
148
160
|
branch = true
|
|
161
|
+
relative_files = true
|
|
149
162
|
parallel = false
|
|
150
163
|
|
|
151
164
|
|
|
@@ -114,8 +114,9 @@ class OpenSearchHybridRetriever:
|
|
|
114
114
|
**kwargs: Any,
|
|
115
115
|
) -> None:
|
|
116
116
|
"""
|
|
117
|
-
Initialize the OpenSearchHybridRetriever
|
|
118
|
-
|
|
117
|
+
Initialize the OpenSearchHybridRetriever using both embedding-based and keyword-based retrieval methods.
|
|
118
|
+
|
|
119
|
+
This is a super component to retrieve documents from OpenSearch using both retrieval methods.
|
|
119
120
|
|
|
120
121
|
We don't explicitly define all the init parameters of the components in the constructor, for each
|
|
121
122
|
of the components, since that would be around 20+ parameters. Instead, we define the most important ones
|
|
@@ -242,7 +243,9 @@ class OpenSearchHybridRetriever:
|
|
|
242
243
|
|
|
243
244
|
if TYPE_CHECKING:
|
|
244
245
|
|
|
245
|
-
def warm_up(self) -> None:
|
|
246
|
+
def warm_up(self) -> None:
|
|
247
|
+
"""Warm up the underlying pipeline components."""
|
|
248
|
+
...
|
|
246
249
|
|
|
247
250
|
def run(
|
|
248
251
|
self,
|
|
@@ -251,7 +254,9 @@ class OpenSearchHybridRetriever:
|
|
|
251
254
|
filters_embedding: dict[str, Any] | None = None,
|
|
252
255
|
top_k_bm25: int | None = None,
|
|
253
256
|
top_k_embedding: int | None = None,
|
|
254
|
-
) -> dict[str, list[Document]]:
|
|
257
|
+
) -> dict[str, list[Document]]:
|
|
258
|
+
"""Run the hybrid retrieval pipeline and return retrieved documents."""
|
|
259
|
+
...
|
|
255
260
|
|
|
256
261
|
def _create_pipeline(self, data: dict[str, Any]) -> Pipeline:
|
|
257
262
|
"""
|
|
@@ -284,7 +289,7 @@ class OpenSearchHybridRetriever:
|
|
|
284
289
|
|
|
285
290
|
return hybrid_retrieval
|
|
286
291
|
|
|
287
|
-
def to_dict(self):
|
|
292
|
+
def to_dict(self) -> dict[str, Any]:
|
|
288
293
|
"""
|
|
289
294
|
Serialize OpenSearchHybridRetriever to a dictionary.
|
|
290
295
|
|
|
@@ -327,7 +332,8 @@ class OpenSearchHybridRetriever:
|
|
|
327
332
|
)
|
|
328
333
|
|
|
329
334
|
@classmethod
|
|
330
|
-
def from_dict(cls, data):
|
|
335
|
+
def from_dict(cls, data: dict[str, Any]) -> "OpenSearchHybridRetriever":
|
|
336
|
+
"""Deserialize an OpenSearchHybridRetriever from a dictionary."""
|
|
331
337
|
# deserialize the document store
|
|
332
338
|
doc_store = OpenSearchDocumentStore.from_dict(data["init_parameters"]["document_store"])
|
|
333
339
|
data["init_parameters"]["document_store"] = doc_store
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
# ruff: noqa: FBT001, FBT002 boolean-type-hint-positional-argument and boolean-default-value-positional-argument
|
|
6
6
|
|
|
7
|
-
from collections.abc import Mapping
|
|
7
|
+
from collections.abc import Generator, Mapping
|
|
8
8
|
from math import exp
|
|
9
9
|
from typing import Any, Literal
|
|
10
10
|
|
|
@@ -13,7 +13,8 @@ from haystack.dataclasses import Document
|
|
|
13
13
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
14
14
|
from haystack.document_stores.types import DuplicatePolicy
|
|
15
15
|
from haystack.utils.auth import Secret
|
|
16
|
-
from
|
|
16
|
+
from haystack.utils.misc import _normalize_metadata_field_name
|
|
17
|
+
from opensearchpy import AsyncHttpConnection, AsyncOpenSearch, OpenSearch, TransportError
|
|
17
18
|
from opensearchpy.helpers import async_bulk, bulk
|
|
18
19
|
|
|
19
20
|
from haystack_integrations.document_stores.opensearch.auth import AsyncAWSAuth, AWSAuth
|
|
@@ -274,7 +275,7 @@ class OpenSearchDocumentStore:
|
|
|
274
275
|
return self._http_auth
|
|
275
276
|
return None
|
|
276
277
|
|
|
277
|
-
def _ensure_initialized(self):
|
|
278
|
+
def _ensure_initialized(self) -> None:
|
|
278
279
|
# Ideally, we have a warm-up stage for document stores as well as components.
|
|
279
280
|
if not self._client:
|
|
280
281
|
self._client = OpenSearch(
|
|
@@ -289,7 +290,7 @@ class OpenSearchDocumentStore:
|
|
|
289
290
|
|
|
290
291
|
self._ensure_index_exists()
|
|
291
292
|
|
|
292
|
-
async def _ensure_initialized_async(self):
|
|
293
|
+
async def _ensure_initialized_async(self) -> None:
|
|
293
294
|
if not self._async_client:
|
|
294
295
|
resolved_auth = self._resolve_http_auth()
|
|
295
296
|
async_http_auth = AsyncAWSAuth(resolved_auth) if isinstance(resolved_auth, AWSAuth) else resolved_auth
|
|
@@ -307,7 +308,7 @@ class OpenSearchDocumentStore:
|
|
|
307
308
|
self._initialized = True
|
|
308
309
|
await self._ensure_index_exists_async()
|
|
309
310
|
|
|
310
|
-
async def _ensure_index_exists_async(self):
|
|
311
|
+
async def _ensure_index_exists_async(self) -> None:
|
|
311
312
|
assert self._async_client is not None
|
|
312
313
|
|
|
313
314
|
if await self._async_client.indices.exists(index=self._index):
|
|
@@ -321,7 +322,7 @@ class OpenSearchDocumentStore:
|
|
|
321
322
|
body = {"mappings": self._mappings, "settings": self._settings}
|
|
322
323
|
await self._async_client.indices.create(index=self._index, body=body)
|
|
323
324
|
|
|
324
|
-
def _ensure_index_exists(self):
|
|
325
|
+
def _ensure_index_exists(self) -> None:
|
|
325
326
|
assert self._client is not None
|
|
326
327
|
|
|
327
328
|
if self._client.indices.exists(index=self._index):
|
|
@@ -558,6 +559,7 @@ class OpenSearchDocumentStore:
|
|
|
558
559
|
def _deserialize_document(hit: dict[str, Any]) -> Document:
|
|
559
560
|
"""
|
|
560
561
|
Creates a Document from the search hit provided.
|
|
562
|
+
|
|
561
563
|
This is mostly useful in self.filter_documents().
|
|
562
564
|
"""
|
|
563
565
|
data = hit["_source"]
|
|
@@ -576,7 +578,7 @@ class OpenSearchDocumentStore:
|
|
|
576
578
|
refresh: Literal["wait_for", True, False],
|
|
577
579
|
routing: dict[str, str] | None = None,
|
|
578
580
|
) -> dict[str, Any]:
|
|
579
|
-
def action_generator():
|
|
581
|
+
def action_generator() -> Generator[dict[str, Any], None, None]:
|
|
580
582
|
for id_ in document_ids:
|
|
581
583
|
action = {"_op_type": "delete", "_id": id_}
|
|
582
584
|
# Add routing if provided for this document ID
|
|
@@ -977,7 +979,27 @@ class OpenSearchDocumentStore:
|
|
|
977
979
|
all_terms_must_match=all_terms_must_match,
|
|
978
980
|
custom_query=custom_query,
|
|
979
981
|
)
|
|
980
|
-
|
|
982
|
+
try:
|
|
983
|
+
documents = self._search_documents(search_params)
|
|
984
|
+
except TransportError as e:
|
|
985
|
+
if "too_many_clauses" in f"{e.info} {e.error}" and fuzziness not in (0, "0") and custom_query is None:
|
|
986
|
+
logger.warning(
|
|
987
|
+
"BM25 query with fuzziness='{fuzziness}' exceeded OpenSearch's clause limit. "
|
|
988
|
+
"Retrying with fuzziness=0 (exact matching). Consider reducing query length or "
|
|
989
|
+
"setting fuzziness=0 explicitly if this occurs frequently.",
|
|
990
|
+
fuzziness=fuzziness,
|
|
991
|
+
)
|
|
992
|
+
search_params = self._prepare_bm25_search_request(
|
|
993
|
+
query=query,
|
|
994
|
+
filters=filters,
|
|
995
|
+
fuzziness=0,
|
|
996
|
+
top_k=top_k,
|
|
997
|
+
all_terms_must_match=all_terms_must_match,
|
|
998
|
+
custom_query=custom_query,
|
|
999
|
+
)
|
|
1000
|
+
documents = self._search_documents(search_params)
|
|
1001
|
+
else:
|
|
1002
|
+
raise
|
|
981
1003
|
OpenSearchDocumentStore._postprocess_bm25_search_results(results=documents, scale_score=scale_score)
|
|
982
1004
|
return documents
|
|
983
1005
|
|
|
@@ -1017,7 +1039,27 @@ class OpenSearchDocumentStore:
|
|
|
1017
1039
|
all_terms_must_match=all_terms_must_match,
|
|
1018
1040
|
custom_query=custom_query,
|
|
1019
1041
|
)
|
|
1020
|
-
|
|
1042
|
+
try:
|
|
1043
|
+
documents = await self._search_documents_async(search_params)
|
|
1044
|
+
except TransportError as e:
|
|
1045
|
+
if "too_many_clauses" in f"{e.info} {e.error}" and fuzziness not in (0, "0") and custom_query is None:
|
|
1046
|
+
logger.warning(
|
|
1047
|
+
"BM25 query with fuzziness='{fuzziness}' exceeded OpenSearch's clause limit. "
|
|
1048
|
+
"Retrying with fuzziness=0 (exact matching). Consider reducing query length or "
|
|
1049
|
+
"setting fuzziness=0 explicitly if this occurs frequently.",
|
|
1050
|
+
fuzziness=fuzziness,
|
|
1051
|
+
)
|
|
1052
|
+
search_params = self._prepare_bm25_search_request(
|
|
1053
|
+
query=query,
|
|
1054
|
+
filters=filters,
|
|
1055
|
+
fuzziness=0,
|
|
1056
|
+
top_k=top_k,
|
|
1057
|
+
all_terms_must_match=all_terms_must_match,
|
|
1058
|
+
custom_query=custom_query,
|
|
1059
|
+
)
|
|
1060
|
+
documents = await self._search_documents_async(search_params)
|
|
1061
|
+
else:
|
|
1062
|
+
raise
|
|
1021
1063
|
OpenSearchDocumentStore._postprocess_bm25_search_results(results=documents, scale_score=scale_score)
|
|
1022
1064
|
return documents
|
|
1023
1065
|
|
|
@@ -1481,6 +1523,7 @@ class OpenSearchDocumentStore:
|
|
|
1481
1523
|
) -> list[Document]:
|
|
1482
1524
|
"""
|
|
1483
1525
|
Retrieves documents that are most similar to the query embedding using a vector similarity metric.
|
|
1526
|
+
|
|
1484
1527
|
It uses the OpenSearch's Approximate k-Nearest Neighbors search algorithm.
|
|
1485
1528
|
|
|
1486
1529
|
This method is not meant to be part of the public interface of
|
|
@@ -1512,8 +1555,9 @@ class OpenSearchDocumentStore:
|
|
|
1512
1555
|
search_kwargs: dict[str, Any] | None = None,
|
|
1513
1556
|
) -> list[Document]:
|
|
1514
1557
|
"""
|
|
1515
|
-
Asynchronously retrieves documents
|
|
1516
|
-
|
|
1558
|
+
Asynchronously retrieves documents most similar to the query embedding using a vector similarity metric.
|
|
1559
|
+
|
|
1560
|
+
It uses the OpenSearch's Approximate k-Nearest Neighbors search algorithm.
|
|
1517
1561
|
|
|
1518
1562
|
This method is not meant to be part of the public interface of
|
|
1519
1563
|
`OpenSearchDocumentStore` nor called directly.
|
|
@@ -1640,8 +1684,7 @@ class OpenSearchDocumentStore:
|
|
|
1640
1684
|
|
|
1641
1685
|
def count_unique_metadata_by_filter(self, filters: dict[str, Any], metadata_fields: list[str]) -> dict[str, int]:
|
|
1642
1686
|
"""
|
|
1643
|
-
Returns the number of unique values for each specified metadata field of the documents
|
|
1644
|
-
that match the provided filters.
|
|
1687
|
+
Returns the number of unique values for each specified metadata field of the documents that match the filters.
|
|
1645
1688
|
|
|
1646
1689
|
:param filters: The filters to apply to count documents.
|
|
1647
1690
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1659,7 +1702,7 @@ class OpenSearchDocumentStore:
|
|
|
1659
1702
|
index_mapping = mapping[self._index]["mappings"]["properties"]
|
|
1660
1703
|
|
|
1661
1704
|
# normalize field names
|
|
1662
|
-
normalized_metadata_fields = [
|
|
1705
|
+
normalized_metadata_fields = [_normalize_metadata_field_name(field) for field in metadata_fields]
|
|
1663
1706
|
# validate that all requested fields exist in the index mapping
|
|
1664
1707
|
missing_fields = [f for f in normalized_metadata_fields if f not in index_mapping]
|
|
1665
1708
|
if missing_fields:
|
|
@@ -1684,8 +1727,7 @@ class OpenSearchDocumentStore:
|
|
|
1684
1727
|
self, filters: dict[str, Any], metadata_fields: list[str]
|
|
1685
1728
|
) -> dict[str, int]:
|
|
1686
1729
|
"""
|
|
1687
|
-
Asynchronously returns the number of unique values for each specified metadata field
|
|
1688
|
-
that match the provided filters.
|
|
1730
|
+
Asynchronously returns the number of unique values for each specified metadata field matching the filters.
|
|
1689
1731
|
|
|
1690
1732
|
:param filters: The filters to apply to count documents.
|
|
1691
1733
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1703,7 +1745,7 @@ class OpenSearchDocumentStore:
|
|
|
1703
1745
|
index_mapping = mapping[self._index]["mappings"]["properties"]
|
|
1704
1746
|
|
|
1705
1747
|
# normalize field names
|
|
1706
|
-
normalized_metadata_fields = [
|
|
1748
|
+
normalized_metadata_fields = [_normalize_metadata_field_name(field) for field in metadata_fields]
|
|
1707
1749
|
# validate that all requested fields exist in the index mapping
|
|
1708
1750
|
missing_fields = [f for f in normalized_metadata_fields if f not in index_mapping]
|
|
1709
1751
|
if missing_fields:
|
|
@@ -1790,13 +1832,6 @@ class OpenSearchDocumentStore:
|
|
|
1790
1832
|
index_mapping = {k: v for k, v in index_mapping.items() if k not in SPECIAL_FIELDS}
|
|
1791
1833
|
return index_mapping
|
|
1792
1834
|
|
|
1793
|
-
@staticmethod
|
|
1794
|
-
def _normalize_metadata_field_name(metadata_field: str) -> str:
|
|
1795
|
-
"""
|
|
1796
|
-
Normalizes a metadata field name by removing the "meta." prefix if present.
|
|
1797
|
-
"""
|
|
1798
|
-
return metadata_field[5:] if metadata_field.startswith("meta.") else metadata_field
|
|
1799
|
-
|
|
1800
1835
|
@staticmethod
|
|
1801
1836
|
def _build_min_max_query_body(field_name: str) -> dict[str, Any]:
|
|
1802
1837
|
"""
|
|
@@ -1834,7 +1869,7 @@ class OpenSearchDocumentStore:
|
|
|
1834
1869
|
self._ensure_initialized()
|
|
1835
1870
|
assert self._client is not None
|
|
1836
1871
|
|
|
1837
|
-
field_name =
|
|
1872
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
1838
1873
|
body = self._build_min_max_query_body(field_name)
|
|
1839
1874
|
result = self._client.search(index=self._index, body=body)
|
|
1840
1875
|
stats = result.get("aggregations", {}).get("field_stats", {})
|
|
@@ -1852,7 +1887,7 @@ class OpenSearchDocumentStore:
|
|
|
1852
1887
|
await self._ensure_initialized_async()
|
|
1853
1888
|
assert self._async_client is not None
|
|
1854
1889
|
|
|
1855
|
-
field_name =
|
|
1890
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
1856
1891
|
body = self._build_min_max_query_body(field_name)
|
|
1857
1892
|
result = await self._async_client.search(index=self._index, body=body)
|
|
1858
1893
|
stats = result.get("aggregations", {}).get("field_stats", {})
|
|
@@ -1868,6 +1903,7 @@ class OpenSearchDocumentStore:
|
|
|
1868
1903
|
) -> tuple[list[str], dict[str, Any] | None]:
|
|
1869
1904
|
"""
|
|
1870
1905
|
Returns unique values for a metadata field, optionally filtered by a search term in the content.
|
|
1906
|
+
|
|
1871
1907
|
Uses composite aggregations for proper pagination beyond 10k results.
|
|
1872
1908
|
|
|
1873
1909
|
:param metadata_field: The metadata field to get unique values for.
|
|
@@ -1882,7 +1918,7 @@ class OpenSearchDocumentStore:
|
|
|
1882
1918
|
self._ensure_initialized()
|
|
1883
1919
|
assert self._client is not None
|
|
1884
1920
|
|
|
1885
|
-
field_name =
|
|
1921
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
1886
1922
|
|
|
1887
1923
|
# filter by search_term if provided
|
|
1888
1924
|
query: dict[str, Any] = {"match_all": {}}
|
|
@@ -1933,6 +1969,7 @@ class OpenSearchDocumentStore:
|
|
|
1933
1969
|
) -> tuple[list[str], dict[str, Any] | None]:
|
|
1934
1970
|
"""
|
|
1935
1971
|
Asynchronously returns unique values for a metadata field, optionally filtered by a search term in the content.
|
|
1972
|
+
|
|
1936
1973
|
Uses composite aggregations for proper pagination beyond 10k results.
|
|
1937
1974
|
|
|
1938
1975
|
:param metadata_field: The metadata field to get unique values for.
|
|
@@ -1947,7 +1984,7 @@ class OpenSearchDocumentStore:
|
|
|
1947
1984
|
await self._ensure_initialized_async()
|
|
1948
1985
|
assert self._async_client is not None
|
|
1949
1986
|
|
|
1950
|
-
field_name =
|
|
1987
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
1951
1988
|
|
|
1952
1989
|
# filter by search_term if provided
|
|
1953
1990
|
query: dict[str, Any] = {"match_all": {}}
|
|
@@ -3,14 +3,21 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
import random
|
|
6
|
-
from unittest.mock import patch
|
|
6
|
+
from unittest.mock import AsyncMock, MagicMock, patch
|
|
7
7
|
|
|
8
8
|
import pytest
|
|
9
9
|
from haystack.dataclasses.document import Document
|
|
10
10
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
11
11
|
from haystack.document_stores.types import DuplicatePolicy
|
|
12
|
-
from haystack.testing.document_store import
|
|
13
|
-
|
|
12
|
+
from haystack.testing.document_store import (
|
|
13
|
+
CountDocumentsByFilterTest,
|
|
14
|
+
CountUniqueMetadataByFilterTest,
|
|
15
|
+
DocumentStoreBaseExtendedTests,
|
|
16
|
+
GetMetadataFieldMinMaxTest,
|
|
17
|
+
GetMetadataFieldsInfoTest,
|
|
18
|
+
GetMetadataFieldUniqueValuesTest,
|
|
19
|
+
)
|
|
20
|
+
from opensearchpy.exceptions import RequestError, TransportError
|
|
14
21
|
|
|
15
22
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
16
23
|
from haystack_integrations.document_stores.opensearch.document_store import DEFAULT_MAX_CHUNK_BYTES
|
|
@@ -217,11 +224,166 @@ def test_routing_in_delete(mock_bulk, _mock_opensearch_client):
|
|
|
217
224
|
assert "_routing" not in actions[2]
|
|
218
225
|
|
|
219
226
|
|
|
227
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
228
|
+
def test_bm25_retrieval_retries_with_fuzziness_zero_on_too_many_clauses(_mock_opensearch_client, caplog):
|
|
229
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
230
|
+
store._client = MagicMock()
|
|
231
|
+
|
|
232
|
+
too_many_clauses_error = TransportError(
|
|
233
|
+
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
234
|
+
)
|
|
235
|
+
store._client.search.side_effect = [
|
|
236
|
+
too_many_clauses_error,
|
|
237
|
+
{"hits": {"hits": []}},
|
|
238
|
+
]
|
|
239
|
+
|
|
240
|
+
results = store._bm25_retrieval("a very long query", fuzziness="AUTO")
|
|
241
|
+
|
|
242
|
+
assert results == []
|
|
243
|
+
assert store._client.search.call_count == 2
|
|
244
|
+
# Verify the retry used fuzziness=0
|
|
245
|
+
second_call_body = store._client.search.call_args_list[1].kwargs["body"]
|
|
246
|
+
assert second_call_body["query"]["bool"]["must"][0]["multi_match"]["fuzziness"] == 0
|
|
247
|
+
assert "Retrying with fuzziness=0" in caplog.text
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
251
|
+
def test_bm25_retrieval_no_retry_when_fuzziness_already_zero(_mock_opensearch_client):
|
|
252
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
253
|
+
store._client = MagicMock()
|
|
254
|
+
|
|
255
|
+
too_many_clauses_error = TransportError(
|
|
256
|
+
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
257
|
+
)
|
|
258
|
+
store._client.search.side_effect = too_many_clauses_error
|
|
259
|
+
|
|
260
|
+
with pytest.raises(TransportError):
|
|
261
|
+
store._bm25_retrieval("a very long query", fuzziness=0)
|
|
262
|
+
|
|
263
|
+
assert store._client.search.call_count == 1
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
267
|
+
def test_bm25_retrieval_no_retry_with_custom_query(_mock_opensearch_client):
|
|
268
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
269
|
+
store._client = MagicMock()
|
|
270
|
+
|
|
271
|
+
too_many_clauses_error = TransportError(
|
|
272
|
+
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
273
|
+
)
|
|
274
|
+
store._client.search.side_effect = too_many_clauses_error
|
|
275
|
+
|
|
276
|
+
custom_query = {"query": {"match": {"content": "$query"}}}
|
|
277
|
+
with pytest.raises(TransportError):
|
|
278
|
+
store._bm25_retrieval("a very long query", fuzziness="AUTO", custom_query=custom_query)
|
|
279
|
+
|
|
280
|
+
assert store._client.search.call_count == 1
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
284
|
+
def test_bm25_retrieval_reraises_other_transport_errors(_mock_opensearch_client):
|
|
285
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
286
|
+
store._client = MagicMock()
|
|
287
|
+
|
|
288
|
+
other_error = TransportError(500, "parsing_exception", {"error": {"reason": "some other error"}})
|
|
289
|
+
store._client.search.side_effect = other_error
|
|
290
|
+
|
|
291
|
+
with pytest.raises(TransportError):
|
|
292
|
+
store._bm25_retrieval("some query", fuzziness="AUTO")
|
|
293
|
+
|
|
294
|
+
assert store._client.search.call_count == 1
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
@pytest.mark.asyncio
|
|
298
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
299
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
300
|
+
async def test_bm25_retrieval_async_retries_with_fuzziness_zero_on_too_many_clauses(
|
|
301
|
+
_mock_opensearch_client, _mock_async_client, caplog
|
|
302
|
+
):
|
|
303
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
304
|
+
store._async_client = AsyncMock()
|
|
305
|
+
|
|
306
|
+
too_many_clauses_error = TransportError(
|
|
307
|
+
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
308
|
+
)
|
|
309
|
+
store._async_client.search.side_effect = [
|
|
310
|
+
too_many_clauses_error,
|
|
311
|
+
{"hits": {"hits": []}},
|
|
312
|
+
]
|
|
313
|
+
|
|
314
|
+
results = await store._bm25_retrieval_async("a very long query", fuzziness="AUTO")
|
|
315
|
+
|
|
316
|
+
assert results == []
|
|
317
|
+
assert store._async_client.search.call_count == 2
|
|
318
|
+
second_call_body = store._async_client.search.call_args_list[1].kwargs["body"]
|
|
319
|
+
assert second_call_body["query"]["bool"]["must"][0]["multi_match"]["fuzziness"] == 0
|
|
320
|
+
assert "Retrying with fuzziness=0" in caplog.text
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
@pytest.mark.asyncio
|
|
324
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
325
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
326
|
+
async def test_bm25_retrieval_async_no_retry_when_fuzziness_already_zero(_mock_opensearch_client, _mock_async_client):
|
|
327
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
328
|
+
store._async_client = AsyncMock()
|
|
329
|
+
|
|
330
|
+
too_many_clauses_error = TransportError(
|
|
331
|
+
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
332
|
+
)
|
|
333
|
+
store._async_client.search.side_effect = too_many_clauses_error
|
|
334
|
+
|
|
335
|
+
with pytest.raises(TransportError):
|
|
336
|
+
await store._bm25_retrieval_async("a very long query", fuzziness=0)
|
|
337
|
+
|
|
338
|
+
assert store._async_client.search.call_count == 1
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
@pytest.mark.asyncio
|
|
342
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
343
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
344
|
+
async def test_bm25_retrieval_async_no_retry_with_custom_query(_mock_opensearch_client, _mock_async_client):
|
|
345
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
346
|
+
store._async_client = AsyncMock()
|
|
347
|
+
|
|
348
|
+
too_many_clauses_error = TransportError(
|
|
349
|
+
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
350
|
+
)
|
|
351
|
+
store._async_client.search.side_effect = too_many_clauses_error
|
|
352
|
+
|
|
353
|
+
custom_query = {"query": {"match": {"content": "$query"}}}
|
|
354
|
+
with pytest.raises(TransportError):
|
|
355
|
+
await store._bm25_retrieval_async("a very long query", fuzziness="AUTO", custom_query=custom_query)
|
|
356
|
+
|
|
357
|
+
assert store._async_client.search.call_count == 1
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
@pytest.mark.asyncio
|
|
361
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
362
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
363
|
+
async def test_bm25_retrieval_async_reraises_other_transport_errors(_mock_opensearch_client, _mock_async_client):
|
|
364
|
+
store = OpenSearchDocumentStore(hosts="testhost")
|
|
365
|
+
store._async_client = AsyncMock()
|
|
366
|
+
|
|
367
|
+
other_error = TransportError(500, "parsing_exception", {"error": {"reason": "some other error"}})
|
|
368
|
+
store._async_client.search.side_effect = other_error
|
|
369
|
+
|
|
370
|
+
with pytest.raises(TransportError):
|
|
371
|
+
await store._bm25_retrieval_async("some query", fuzziness="AUTO")
|
|
372
|
+
|
|
373
|
+
assert store._async_client.search.call_count == 1
|
|
374
|
+
|
|
375
|
+
|
|
220
376
|
@pytest.mark.integration
|
|
221
|
-
class TestDocumentStore(
|
|
377
|
+
class TestDocumentStore(
|
|
378
|
+
CountDocumentsByFilterTest,
|
|
379
|
+
CountUniqueMetadataByFilterTest,
|
|
380
|
+
DocumentStoreBaseExtendedTests,
|
|
381
|
+
GetMetadataFieldsInfoTest,
|
|
382
|
+
GetMetadataFieldMinMaxTest,
|
|
383
|
+
GetMetadataFieldUniqueValuesTest,
|
|
384
|
+
):
|
|
222
385
|
"""
|
|
223
|
-
Common test cases will be provided by `DocumentStoreBaseExtendedTests` but
|
|
224
|
-
you can add more to this class.
|
|
386
|
+
Common test cases will be provided by `DocumentStoreBaseExtendedTests` but you can add more to this class.
|
|
225
387
|
"""
|
|
226
388
|
|
|
227
389
|
@pytest.fixture
|
|
@@ -320,6 +482,33 @@ class TestDocumentStore(DocumentStoreBaseExtendedTests):
|
|
|
320
482
|
assert "functional" in res[1].content
|
|
321
483
|
assert "functional" in res[2].content
|
|
322
484
|
|
|
485
|
+
def test_bm25_retrieval_with_fuzziness_overflow(self, document_store: OpenSearchDocumentStore, caplog):
|
|
486
|
+
"""
|
|
487
|
+
Test that a long query with fuzziness="AUTO" that exceeds OpenSearch's maxClauseCount
|
|
488
|
+
is automatically retried with fuzziness=0 instead of raising an error.
|
|
489
|
+
"""
|
|
490
|
+
# Build an index vocabulary of similar 5-character words. With fuzziness="AUTO",
|
|
491
|
+
# 5-char words get edit distance 1, so each query term fuzzy-matches many similar
|
|
492
|
+
# indexed terms, causing clause expansion beyond the default maxClauseCount (1024).
|
|
493
|
+
# With fuzziness=0, each term produces exactly 1 clause, staying well under the limit.
|
|
494
|
+
words = [f"foo{chr(97 + i)}{chr(97 + j)}" for i in range(20) for j in range(26)] # 520 words
|
|
495
|
+
|
|
496
|
+
chunk_size = 52
|
|
497
|
+
docs = [
|
|
498
|
+
Document(content=" ".join(words[i : i + chunk_size]), id=str(idx))
|
|
499
|
+
for idx, i in enumerate(range(0, len(words), chunk_size))
|
|
500
|
+
]
|
|
501
|
+
document_store.write_documents(docs)
|
|
502
|
+
|
|
503
|
+
# Query with a subset of words. With fuzziness="AUTO", each 5-char term expands
|
|
504
|
+
# to match ~45 similar indexed terms, pushing total clauses well above 1024.
|
|
505
|
+
long_query = " ".join(words[:100])
|
|
506
|
+
|
|
507
|
+
# This should not raise: the too_many_clauses error is caught and retried with fuzziness=0
|
|
508
|
+
res = document_store._bm25_retrieval(long_query, top_k=3, fuzziness="AUTO")
|
|
509
|
+
assert isinstance(res, list)
|
|
510
|
+
assert "Retrying with fuzziness=0" in caplog.text
|
|
511
|
+
|
|
323
512
|
def test_bm25_retrieval_with_filters(self, document_store: OpenSearchDocumentStore, test_documents: list[Document]):
|
|
324
513
|
document_store.write_documents(test_documents)
|
|
325
514
|
res = document_store._bm25_retrieval(
|
|
@@ -607,158 +796,6 @@ class TestDocumentStore(DocumentStoreBaseExtendedTests):
|
|
|
607
796
|
assert len(results) == 1
|
|
608
797
|
assert results[0].content == "New document after delete all"
|
|
609
798
|
|
|
610
|
-
def test_count_documents_by_filter(self, document_store: OpenSearchDocumentStore):
|
|
611
|
-
docs = [
|
|
612
|
-
Document(content="Doc 1", meta={"category": "A", "status": "active"}),
|
|
613
|
-
Document(content="Doc 2", meta={"category": "B", "status": "active"}),
|
|
614
|
-
Document(content="Doc 3", meta={"category": "A", "status": "inactive"}),
|
|
615
|
-
Document(content="Doc 4", meta={"category": "A", "status": "active"}),
|
|
616
|
-
]
|
|
617
|
-
document_store.write_documents(docs)
|
|
618
|
-
assert document_store.count_documents() == 4
|
|
619
|
-
|
|
620
|
-
count_a = document_store.count_documents_by_filter(
|
|
621
|
-
filters={"field": "meta.category", "operator": "==", "value": "A"}
|
|
622
|
-
)
|
|
623
|
-
assert count_a == 3
|
|
624
|
-
|
|
625
|
-
count_a_active = document_store.count_documents_by_filter(
|
|
626
|
-
filters={
|
|
627
|
-
"operator": "AND",
|
|
628
|
-
"conditions": [
|
|
629
|
-
{"field": "meta.category", "operator": "==", "value": "A"},
|
|
630
|
-
{"field": "meta.status", "operator": "==", "value": "active"},
|
|
631
|
-
],
|
|
632
|
-
}
|
|
633
|
-
)
|
|
634
|
-
assert count_a_active == 2
|
|
635
|
-
|
|
636
|
-
def test_count_unique_metadata_by_filter(self, document_store: OpenSearchDocumentStore):
|
|
637
|
-
docs = [
|
|
638
|
-
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
|
|
639
|
-
Document(content="Doc 2", meta={"category": "B", "status": "active", "priority": 2}),
|
|
640
|
-
Document(content="Doc 3", meta={"category": "A", "status": "inactive", "priority": 1}),
|
|
641
|
-
Document(content="Doc 4", meta={"category": "A", "status": "active", "priority": 3}),
|
|
642
|
-
Document(content="Doc 5", meta={"category": "C", "status": "active", "priority": 2}),
|
|
643
|
-
]
|
|
644
|
-
document_store.write_documents(docs)
|
|
645
|
-
assert document_store.count_documents() == 5
|
|
646
|
-
|
|
647
|
-
# Count distinct values for all documents
|
|
648
|
-
distinct_counts = document_store.count_unique_metadata_by_filter(
|
|
649
|
-
filters={}, metadata_fields=["category", "status", "priority"]
|
|
650
|
-
)
|
|
651
|
-
assert distinct_counts["category"] == 3 # A, B, C
|
|
652
|
-
assert distinct_counts["status"] == 2 # active, inactive
|
|
653
|
-
assert distinct_counts["priority"] == 3 # 1, 2, 3
|
|
654
|
-
|
|
655
|
-
# Count distinct values for documents with category="A"
|
|
656
|
-
distinct_counts_a = document_store.count_unique_metadata_by_filter(
|
|
657
|
-
filters={"field": "meta.category", "operator": "==", "value": "A"},
|
|
658
|
-
metadata_fields=["category", "status", "priority"],
|
|
659
|
-
)
|
|
660
|
-
assert distinct_counts_a["category"] == 1 # Only A
|
|
661
|
-
assert distinct_counts_a["status"] == 2 # active, inactive
|
|
662
|
-
assert distinct_counts_a["priority"] == 2 # 1, 3
|
|
663
|
-
|
|
664
|
-
# Count distinct values for documents with status="active"
|
|
665
|
-
distinct_counts_active = document_store.count_unique_metadata_by_filter(
|
|
666
|
-
filters={"field": "meta.status", "operator": "==", "value": "active"},
|
|
667
|
-
metadata_fields=["category", "status", "priority"],
|
|
668
|
-
)
|
|
669
|
-
assert distinct_counts_active["category"] == 3 # A, B, C
|
|
670
|
-
assert distinct_counts_active["status"] == 1 # Only active
|
|
671
|
-
assert distinct_counts_active["priority"] == 3 # 1, 2, 3
|
|
672
|
-
|
|
673
|
-
# Count distinct values with complex filter (category="A" AND status="active")
|
|
674
|
-
distinct_counts_a_active = document_store.count_unique_metadata_by_filter(
|
|
675
|
-
filters={
|
|
676
|
-
"operator": "AND",
|
|
677
|
-
"conditions": [
|
|
678
|
-
{"field": "meta.category", "operator": "==", "value": "A"},
|
|
679
|
-
{"field": "meta.status", "operator": "==", "value": "active"},
|
|
680
|
-
],
|
|
681
|
-
},
|
|
682
|
-
metadata_fields=["category", "status", "priority"],
|
|
683
|
-
)
|
|
684
|
-
assert distinct_counts_a_active["category"] == 1 # Only A
|
|
685
|
-
assert distinct_counts_a_active["status"] == 1 # Only active
|
|
686
|
-
assert distinct_counts_a_active["priority"] == 2 # 1, 3
|
|
687
|
-
|
|
688
|
-
# Test with only a subset of fields
|
|
689
|
-
distinct_counts_subset = document_store.count_unique_metadata_by_filter(
|
|
690
|
-
filters={}, metadata_fields=["category", "status"]
|
|
691
|
-
)
|
|
692
|
-
assert distinct_counts_subset["category"] == 3
|
|
693
|
-
assert distinct_counts_subset["status"] == 2
|
|
694
|
-
assert "priority" not in distinct_counts_subset
|
|
695
|
-
|
|
696
|
-
# Test field name normalization (with "meta." prefix)
|
|
697
|
-
distinct_counts_normalized = document_store.count_unique_metadata_by_filter(
|
|
698
|
-
filters={}, metadata_fields=["meta.category", "status", "meta.priority"]
|
|
699
|
-
)
|
|
700
|
-
assert distinct_counts_normalized["category"] == 3
|
|
701
|
-
assert distinct_counts_normalized["status"] == 2
|
|
702
|
-
assert distinct_counts_normalized["priority"] == 3
|
|
703
|
-
|
|
704
|
-
# Test error handling when field doesn't exist
|
|
705
|
-
with pytest.raises(ValueError, match="Fields not found in index mapping"):
|
|
706
|
-
document_store.count_unique_metadata_by_filter(filters={}, metadata_fields=["nonexistent_field"])
|
|
707
|
-
|
|
708
|
-
def test_get_metadata_fields_info(self, document_store: OpenSearchDocumentStore):
|
|
709
|
-
docs = [
|
|
710
|
-
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
|
|
711
|
-
Document(content="Doc 2", meta={"category": "B", "status": "inactive"}),
|
|
712
|
-
]
|
|
713
|
-
document_store.write_documents(docs)
|
|
714
|
-
|
|
715
|
-
fields_info = document_store.get_metadata_fields_info()
|
|
716
|
-
|
|
717
|
-
# Verify that fields_info contains expected fields
|
|
718
|
-
assert "category" in fields_info
|
|
719
|
-
assert "status" in fields_info
|
|
720
|
-
assert "priority" in fields_info
|
|
721
|
-
|
|
722
|
-
assert fields_info["category"]["type"] == "keyword"
|
|
723
|
-
assert fields_info["status"]["type"] == "keyword"
|
|
724
|
-
assert fields_info["priority"]["type"] == "long"
|
|
725
|
-
|
|
726
|
-
def test_get_metadata_field_min_max(self, document_store: OpenSearchDocumentStore):
|
|
727
|
-
# Test with integer values
|
|
728
|
-
docs = [
|
|
729
|
-
Document(content="Doc 1", meta={"priority": 1, "age": 10}),
|
|
730
|
-
Document(content="Doc 2", meta={"priority": 5, "age": 20}),
|
|
731
|
-
Document(content="Doc 3", meta={"priority": 3, "age": 15}),
|
|
732
|
-
Document(content="Doc 4", meta={"priority": 10, "age": 5}),
|
|
733
|
-
Document(content="Doc 6", meta={"rating": 10.5}),
|
|
734
|
-
Document(content="Doc 7", meta={"rating": 20.3}),
|
|
735
|
-
Document(content="Doc 8", meta={"rating": 15.7}),
|
|
736
|
-
Document(content="Doc 9", meta={"rating": 5.2}),
|
|
737
|
-
]
|
|
738
|
-
document_store.write_documents(docs)
|
|
739
|
-
|
|
740
|
-
# Test with "meta." prefix for integer field
|
|
741
|
-
min_max_priority = document_store.get_metadata_field_min_max("meta.priority")
|
|
742
|
-
assert min_max_priority["min"] == 1
|
|
743
|
-
assert min_max_priority["max"] == 10
|
|
744
|
-
|
|
745
|
-
# Test with "meta." prefix for another integer field
|
|
746
|
-
min_max_rating = document_store.get_metadata_field_min_max("meta.age")
|
|
747
|
-
assert min_max_rating["min"] == 5
|
|
748
|
-
assert min_max_rating["max"] == 20
|
|
749
|
-
|
|
750
|
-
# Test with single value
|
|
751
|
-
single_doc = [Document(content="Doc 5", meta={"single_value": 42})]
|
|
752
|
-
document_store.write_documents(single_doc)
|
|
753
|
-
min_max_single = document_store.get_metadata_field_min_max("meta.single_value")
|
|
754
|
-
assert min_max_single["min"] == 42
|
|
755
|
-
assert min_max_single["max"] == 42
|
|
756
|
-
|
|
757
|
-
# Test with float values
|
|
758
|
-
min_max_score = document_store.get_metadata_field_min_max("meta.rating")
|
|
759
|
-
assert min_max_score["min"] == pytest.approx(5.2)
|
|
760
|
-
assert min_max_score["max"] == pytest.approx(20.3)
|
|
761
|
-
|
|
762
799
|
def test_get_metadata_field_unique_values(self, document_store: OpenSearchDocumentStore):
|
|
763
800
|
# Test with string values
|
|
764
801
|
docs = [
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
+
|
|
5
6
|
import pytest
|
|
6
7
|
from haystack.dataclasses import Document
|
|
7
8
|
from haystack.document_stores.errors import DocumentStoreError
|
|
@@ -60,6 +61,53 @@ class TestDocumentStoreAsync:
|
|
|
60
61
|
assert len(res) == 5
|
|
61
62
|
assert all("functional" in doc.content for doc in res)
|
|
62
63
|
|
|
64
|
+
@pytest.mark.asyncio
|
|
65
|
+
async def test_bm25_retrieval_with_fuzziness(
|
|
66
|
+
self, document_store: OpenSearchDocumentStore, test_documents: list[Document]
|
|
67
|
+
):
|
|
68
|
+
document_store.write_documents(test_documents)
|
|
69
|
+
|
|
70
|
+
query_with_typo = "functinal"
|
|
71
|
+
# Query without fuzziness to search for the exact match
|
|
72
|
+
res = await document_store._bm25_retrieval_async(query_with_typo, top_k=3, fuzziness="0")
|
|
73
|
+
# Nothing is found as the query contains a typo
|
|
74
|
+
assert res == []
|
|
75
|
+
|
|
76
|
+
# Query with fuzziness with the same query
|
|
77
|
+
res = await document_store._bm25_retrieval_async(query_with_typo, top_k=3, fuzziness="1")
|
|
78
|
+
assert len(res) == 3
|
|
79
|
+
assert "functional" in res[0].content
|
|
80
|
+
assert "functional" in res[1].content
|
|
81
|
+
assert "functional" in res[2].content
|
|
82
|
+
|
|
83
|
+
@pytest.mark.asyncio
|
|
84
|
+
async def test_bm25_retrieval_with_fuzziness_overflow(self, document_store: OpenSearchDocumentStore, caplog):
|
|
85
|
+
"""
|
|
86
|
+
Test that a long query with fuzziness="AUTO" that exceeds OpenSearch's maxClauseCount
|
|
87
|
+
is automatically retried with fuzziness=0 instead of raising an error.
|
|
88
|
+
"""
|
|
89
|
+
# Build an index vocabulary of similar 5-character words. With fuzziness="AUTO",
|
|
90
|
+
# 5-char words get edit distance 1, so each query term fuzzy-matches many similar
|
|
91
|
+
# indexed terms, causing clause expansion beyond the default maxClauseCount (1024).
|
|
92
|
+
# With fuzziness=0, each term produces exactly 1 clause, staying well under the limit.
|
|
93
|
+
words = [f"foo{chr(97 + i)}{chr(97 + j)}" for i in range(20) for j in range(26)] # 520 words
|
|
94
|
+
|
|
95
|
+
chunk_size = 52
|
|
96
|
+
docs = [
|
|
97
|
+
Document(content=" ".join(words[i : i + chunk_size]), id=str(idx))
|
|
98
|
+
for idx, i in enumerate(range(0, len(words), chunk_size))
|
|
99
|
+
]
|
|
100
|
+
document_store.write_documents(docs)
|
|
101
|
+
|
|
102
|
+
# Query with a subset of words. With fuzziness="AUTO", each 5-char term expands
|
|
103
|
+
# to match ~45 similar indexed terms, pushing total clauses well above 1024.
|
|
104
|
+
long_query = " ".join(words[:100])
|
|
105
|
+
|
|
106
|
+
# This should not raise: the too_many_clauses error is caught and retried with fuzziness=0
|
|
107
|
+
res = await document_store._bm25_retrieval_async(long_query, top_k=3, fuzziness="AUTO")
|
|
108
|
+
assert isinstance(res, list)
|
|
109
|
+
assert "Retrying with fuzziness=0" in caplog.text
|
|
110
|
+
|
|
63
111
|
@pytest.mark.asyncio
|
|
64
112
|
async def test_bm25_retrieval_with_filters(
|
|
65
113
|
self, document_store: OpenSearchDocumentStore, test_documents: list[Document]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_metadata_retriever_async.py
RENAMED
|
File without changes
|
{opensearch_haystack-7.0.0 → opensearch_haystack-7.1.0}/tests/test_open_search_hybrid_retriever.py
RENAMED
|
File without changes
|
|
File without changes
|