opensearch-haystack 7.2.1__tar.gz → 8.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/CHANGELOG.md +7 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/PKG-INFO +2 -2
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/pyproject.toml +1 -1
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +7 -17
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +4 -14
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +4 -8
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +1 -1
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +4 -14
- opensearch_haystack-8.0.0/src/haystack_integrations/components/retrievers/opensearch/utils.py +23 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +2 -2
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/conftest.py +45 -141
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_auth.py +18 -1
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_bm25_retriever.py +21 -5
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_document_store.py +2 -21
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_document_store_async.py +45 -410
- opensearch_haystack-8.0.0/tests/test_document_store_common.py +24 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_embedding_retriever.py +16 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_open_search_hybrid_retriever.py +11 -3
- opensearch_haystack-8.0.0/tests/test_utils.py +27 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/.gitignore +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/LICENSE +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/README.md +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/docker-compose.yml +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/filters.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/__init__.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_filters.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.0.0}/tests/test_sql_retriever.py +0 -0
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v7.2.1] - 2026-04-13
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- *(opensearch)* Pass positional args to transport.perform_request for ddtrace compatibility (#3132)
|
|
8
|
+
|
|
9
|
+
|
|
3
10
|
## [integrations/opensearch-v7.2.0] - 2026-04-10
|
|
4
11
|
|
|
5
12
|
### 🚀 Features
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 8.0.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.28.0
|
|
23
23
|
Requires-Dist: opensearch-py[async]>=3.0.0
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
25
25
|
|
|
@@ -13,6 +13,8 @@ from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
15
15
|
|
|
16
|
+
from .utils import _resolve_document_store
|
|
17
|
+
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
@@ -29,7 +31,7 @@ class OpenSearchBM25Retriever:
|
|
|
29
31
|
*,
|
|
30
32
|
document_store: OpenSearchDocumentStore,
|
|
31
33
|
filters: dict[str, Any] | None = None,
|
|
32
|
-
fuzziness: int | str =
|
|
34
|
+
fuzziness: int | str = 0,
|
|
33
35
|
top_k: int = 10,
|
|
34
36
|
scale_score: bool = False,
|
|
35
37
|
all_terms_must_match: bool = False,
|
|
@@ -47,8 +49,8 @@ class OpenSearchBM25Retriever:
|
|
|
47
49
|
required to transform one word into another. For example, the "fuzziness" between the words
|
|
48
50
|
"wined" and "wind" is 1 because only one edit is needed to match them.
|
|
49
51
|
|
|
50
|
-
Use "AUTO"
|
|
51
|
-
|
|
52
|
+
Defaults to `0` (exact matching). Use `"AUTO"` for automatic adjustment based on term length.
|
|
53
|
+
For detailed guidance, refer to the
|
|
52
54
|
[OpenSearch fuzzy query documentation](https://opensearch.org/docs/latest/query-dsl/term/fuzzy/).
|
|
53
55
|
:param top_k: Maximum number of documents to return.
|
|
54
56
|
:param scale_score: If `True`, scales the score of retrieved documents to a range between 0 and 1.
|
|
@@ -268,13 +270,7 @@ class OpenSearchBM25Retriever:
|
|
|
268
270
|
custom_query=custom_query,
|
|
269
271
|
)
|
|
270
272
|
|
|
271
|
-
|
|
272
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
273
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
274
|
-
raise ValueError(msg)
|
|
275
|
-
doc_store = document_store
|
|
276
|
-
else:
|
|
277
|
-
doc_store = self._document_store
|
|
273
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
278
274
|
|
|
279
275
|
try:
|
|
280
276
|
docs = doc_store._bm25_retrieval(**bm25_args) # example for BM25Retriever
|
|
@@ -335,13 +331,7 @@ class OpenSearchBM25Retriever:
|
|
|
335
331
|
custom_query=custom_query,
|
|
336
332
|
)
|
|
337
333
|
|
|
338
|
-
|
|
339
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
340
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
341
|
-
raise ValueError(msg)
|
|
342
|
-
doc_store = document_store
|
|
343
|
-
else:
|
|
344
|
-
doc_store = self._document_store
|
|
334
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
345
335
|
|
|
346
336
|
try:
|
|
347
337
|
docs = await doc_store._bm25_retrieval_async(**bm25_args)
|
|
@@ -13,6 +13,8 @@ from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
15
15
|
|
|
16
|
+
from .utils import _resolve_document_store
|
|
17
|
+
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
@@ -257,13 +259,7 @@ class OpenSearchEmbeddingRetriever:
|
|
|
257
259
|
|
|
258
260
|
docs: list[Document] = []
|
|
259
261
|
|
|
260
|
-
|
|
261
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
262
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
263
|
-
raise ValueError(msg)
|
|
264
|
-
doc_store = document_store
|
|
265
|
-
else:
|
|
266
|
-
doc_store = self._document_store
|
|
262
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
267
263
|
|
|
268
264
|
try:
|
|
269
265
|
docs = doc_store._embedding_retrieval(
|
|
@@ -383,13 +379,7 @@ class OpenSearchEmbeddingRetriever:
|
|
|
383
379
|
|
|
384
380
|
docs: list[Document] = []
|
|
385
381
|
|
|
386
|
-
|
|
387
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
388
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
389
|
-
raise ValueError(msg)
|
|
390
|
-
doc_store = document_store
|
|
391
|
-
else:
|
|
392
|
-
doc_store = self._document_store
|
|
382
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
393
383
|
|
|
394
384
|
try:
|
|
395
385
|
docs = await doc_store._embedding_retrieval_async(
|
|
@@ -9,6 +9,8 @@ from haystack import component, default_from_dict, default_to_dict, logging
|
|
|
9
9
|
|
|
10
10
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
11
11
|
|
|
12
|
+
from .utils import _resolve_document_store
|
|
13
|
+
|
|
12
14
|
logger = logging.getLogger(__name__)
|
|
13
15
|
|
|
14
16
|
|
|
@@ -252,10 +254,7 @@ class OpenSearchMetadataRetriever:
|
|
|
252
254
|
# Returns: {"metadata": [{"category": "Python", "status": "active", "priority": 1}]}
|
|
253
255
|
```
|
|
254
256
|
"""
|
|
255
|
-
doc_store = document_store
|
|
256
|
-
if not isinstance(doc_store, OpenSearchDocumentStore):
|
|
257
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
258
|
-
raise ValueError(msg)
|
|
257
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
259
258
|
|
|
260
259
|
fields_to_use = metadata_fields if metadata_fields is not None else self._metadata_fields
|
|
261
260
|
top_k_to_use = top_k if top_k is not None else self._top_k
|
|
@@ -366,10 +365,7 @@ class OpenSearchMetadataRetriever:
|
|
|
366
365
|
# Returns: {"metadata": [{"category": "Python", "status": "active", "priority": 1}]}
|
|
367
366
|
```
|
|
368
367
|
"""
|
|
369
|
-
doc_store = document_store
|
|
370
|
-
if not isinstance(doc_store, OpenSearchDocumentStore):
|
|
371
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
372
|
-
raise ValueError(msg)
|
|
368
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
373
369
|
|
|
374
370
|
fields_to_use = metadata_fields if metadata_fields is not None else self._metadata_fields
|
|
375
371
|
top_k_to_use = top_k if top_k is not None else self._top_k
|
|
@@ -93,7 +93,7 @@ class OpenSearchHybridRetriever:
|
|
|
93
93
|
embedder: TextEmbedder,
|
|
94
94
|
# OpenSearchBM25Retriever
|
|
95
95
|
filters_bm25: dict[str, Any] | None = None,
|
|
96
|
-
fuzziness: int | str =
|
|
96
|
+
fuzziness: int | str = 0,
|
|
97
97
|
top_k_bm25: int = 10,
|
|
98
98
|
scale_score: bool = False,
|
|
99
99
|
all_terms_must_match: bool = False,
|
|
@@ -8,6 +8,8 @@ from haystack import component, default_from_dict, default_to_dict, logging
|
|
|
8
8
|
|
|
9
9
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
10
10
|
|
|
11
|
+
from .utils import _resolve_document_store
|
|
12
|
+
|
|
11
13
|
logger = logging.getLogger(__name__)
|
|
12
14
|
|
|
13
15
|
|
|
@@ -108,13 +110,7 @@ class OpenSearchSQLRetriever:
|
|
|
108
110
|
# For aggregate queries: result["result"]["aggregations"] contains aggregations
|
|
109
111
|
```
|
|
110
112
|
"""
|
|
111
|
-
|
|
112
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
113
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
114
|
-
raise ValueError(msg)
|
|
115
|
-
doc_store = document_store
|
|
116
|
-
else:
|
|
117
|
-
doc_store = self._document_store
|
|
113
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
118
114
|
|
|
119
115
|
fetch_size = fetch_size if fetch_size is not None else self._fetch_size
|
|
120
116
|
|
|
@@ -163,13 +159,7 @@ class OpenSearchSQLRetriever:
|
|
|
163
159
|
# For aggregate queries: result["result"]["aggregations"] contains aggregations
|
|
164
160
|
```
|
|
165
161
|
"""
|
|
166
|
-
|
|
167
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
168
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
169
|
-
raise ValueError(msg)
|
|
170
|
-
doc_store = document_store
|
|
171
|
-
else:
|
|
172
|
-
doc_store = self._document_store
|
|
162
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
173
163
|
|
|
174
164
|
fetch_size = fetch_size if fetch_size is not None else self._fetch_size
|
|
175
165
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _resolve_document_store(
|
|
10
|
+
runtime_document_store: OpenSearchDocumentStore | None,
|
|
11
|
+
default_document_store: OpenSearchDocumentStore,
|
|
12
|
+
) -> OpenSearchDocumentStore:
|
|
13
|
+
"""
|
|
14
|
+
Return the runtime document store if provided and valid, otherwise the default one.
|
|
15
|
+
|
|
16
|
+
:raises ValueError: If `runtime_document_store` is not None and not an OpenSearchDocumentStore.
|
|
17
|
+
"""
|
|
18
|
+
if runtime_document_store is None:
|
|
19
|
+
return default_document_store
|
|
20
|
+
if not isinstance(runtime_document_store, OpenSearchDocumentStore):
|
|
21
|
+
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
22
|
+
raise ValueError(msg)
|
|
23
|
+
return runtime_document_store
|
|
@@ -1057,7 +1057,7 @@ class OpenSearchDocumentStore:
|
|
|
1057
1057
|
query: str,
|
|
1058
1058
|
*,
|
|
1059
1059
|
filters: dict[str, Any] | None = None,
|
|
1060
|
-
fuzziness: int | str =
|
|
1060
|
+
fuzziness: int | str = 0,
|
|
1061
1061
|
top_k: int = 10,
|
|
1062
1062
|
scale_score: bool = False,
|
|
1063
1063
|
all_terms_must_match: bool = False,
|
|
@@ -1115,7 +1115,7 @@ class OpenSearchDocumentStore:
|
|
|
1115
1115
|
query: str,
|
|
1116
1116
|
*,
|
|
1117
1117
|
filters: dict[str, Any] | None = None,
|
|
1118
|
-
fuzziness: str =
|
|
1118
|
+
fuzziness: int | str = 0,
|
|
1119
1119
|
top_k: int = 10,
|
|
1120
1120
|
scale_score: bool = False,
|
|
1121
1121
|
all_terms_must_match: bool = False,
|
|
@@ -6,6 +6,13 @@ from haystack import Document
|
|
|
6
6
|
|
|
7
7
|
from haystack_integrations.document_stores.opensearch.document_store import OpenSearchDocumentStore
|
|
8
8
|
|
|
9
|
+
COMMON_KWARGS = {
|
|
10
|
+
"hosts": ["https://localhost:9200"],
|
|
11
|
+
"http_auth": ("admin", "SecureHaystack!2026"),
|
|
12
|
+
"verify_certs": False,
|
|
13
|
+
}
|
|
14
|
+
DEFAULT_METHOD = {"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"}
|
|
15
|
+
|
|
9
16
|
|
|
10
17
|
def _get_unique_index_name() -> str:
|
|
11
18
|
"""
|
|
@@ -17,178 +24,75 @@ def _get_unique_index_name() -> str:
|
|
|
17
24
|
|
|
18
25
|
|
|
19
26
|
@pytest.fixture
|
|
20
|
-
def
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
44
|
-
asyncio.run(store._async_client.close())
|
|
27
|
+
def opensearch_store():
|
|
28
|
+
created: list[OpenSearchDocumentStore] = []
|
|
29
|
+
|
|
30
|
+
def _make(**overrides) -> OpenSearchDocumentStore:
|
|
31
|
+
kwargs = {
|
|
32
|
+
**COMMON_KWARGS,
|
|
33
|
+
"index": _get_unique_index_name(),
|
|
34
|
+
"embedding_dim": 768,
|
|
35
|
+
"return_embedding": True,
|
|
36
|
+
"method": DEFAULT_METHOD,
|
|
37
|
+
**overrides,
|
|
38
|
+
}
|
|
39
|
+
store = OpenSearchDocumentStore(**kwargs)
|
|
40
|
+
store._ensure_initialized()
|
|
41
|
+
created.append(store)
|
|
42
|
+
return store
|
|
43
|
+
|
|
44
|
+
yield _make
|
|
45
|
+
|
|
46
|
+
for store in created:
|
|
47
|
+
asyncio.run(store._ensure_initialized_async())
|
|
48
|
+
store._client.indices.delete(index=store._index, params={"ignore": [400, 404]})
|
|
49
|
+
asyncio.run(store._async_client.close())
|
|
45
50
|
|
|
46
51
|
|
|
47
52
|
@pytest.fixture
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
index = f"test_index_2_{_get_unique_index_name()}"
|
|
53
|
+
def document_store(opensearch_store):
|
|
54
|
+
return opensearch_store()
|
|
51
55
|
|
|
52
|
-
store = OpenSearchDocumentStore(
|
|
53
|
-
hosts=hosts,
|
|
54
|
-
index=index,
|
|
55
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
56
|
-
verify_certs=False,
|
|
57
|
-
embedding_dim=768,
|
|
58
|
-
return_embedding=False,
|
|
59
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
60
|
-
)
|
|
61
|
-
yield store
|
|
62
56
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
assert store._client
|
|
67
|
-
assert store._async_client
|
|
68
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
69
|
-
asyncio.run(store._async_client.close())
|
|
57
|
+
@pytest.fixture
|
|
58
|
+
def document_store_2(opensearch_store):
|
|
59
|
+
return opensearch_store(return_embedding=False)
|
|
70
60
|
|
|
71
61
|
|
|
72
62
|
@pytest.fixture
|
|
73
|
-
def document_store_readonly():
|
|
74
|
-
|
|
75
|
-
A document store that does not automatically create the underlying index.
|
|
76
|
-
"""
|
|
77
|
-
hosts = ["https://localhost:9200"]
|
|
78
|
-
# Use a different index for each test so we can run them in parallel
|
|
79
|
-
index = _get_unique_index_name()
|
|
80
|
-
|
|
81
|
-
store = OpenSearchDocumentStore(
|
|
82
|
-
hosts=hosts,
|
|
83
|
-
index=index,
|
|
84
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
85
|
-
verify_certs=False,
|
|
86
|
-
embedding_dim=768,
|
|
87
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
88
|
-
create_index=False,
|
|
89
|
-
)
|
|
90
|
-
store._ensure_initialized()
|
|
91
|
-
asyncio.run(store._ensure_initialized_async())
|
|
92
|
-
assert store._client
|
|
93
|
-
assert store._async_client
|
|
63
|
+
def document_store_readonly(opensearch_store):
|
|
64
|
+
store = opensearch_store(create_index=False)
|
|
94
65
|
store._client.cluster.put_settings(body={"transient": {"action.auto_create_index": False}})
|
|
95
66
|
yield store
|
|
96
|
-
|
|
97
67
|
store._client.cluster.put_settings(body={"transient": {"action.auto_create_index": True}})
|
|
98
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
99
68
|
|
|
100
69
|
|
|
101
70
|
@pytest.fixture
|
|
102
|
-
def document_store_embedding_dim_4_no_emb_returned():
|
|
103
|
-
|
|
104
|
-
A document store with embedding dimension 4 that does not return embeddings.
|
|
105
|
-
"""
|
|
106
|
-
hosts = ["https://localhost:9200"]
|
|
107
|
-
index = _get_unique_index_name()
|
|
108
|
-
|
|
109
|
-
store = OpenSearchDocumentStore(
|
|
110
|
-
hosts=hosts,
|
|
111
|
-
index=index,
|
|
112
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
113
|
-
verify_certs=False,
|
|
114
|
-
embedding_dim=4,
|
|
115
|
-
return_embedding=False,
|
|
116
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
117
|
-
)
|
|
118
|
-
yield store
|
|
119
|
-
|
|
120
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
71
|
+
def document_store_embedding_dim_4_no_emb_returned(opensearch_store):
|
|
72
|
+
return opensearch_store(embedding_dim=4, return_embedding=False)
|
|
121
73
|
|
|
122
74
|
|
|
123
75
|
@pytest.fixture
|
|
124
|
-
def document_store_embedding_dim_4_no_emb_returned_faiss():
|
|
76
|
+
def document_store_embedding_dim_4_no_emb_returned_faiss(opensearch_store):
|
|
125
77
|
"""
|
|
126
78
|
A document store with embedding dimension 4 that uses a FAISS engine with HNSW algorithm for vector search.
|
|
127
79
|
We use this document store for testing efficient k-NN filtering according to
|
|
128
80
|
https://opensearch.org/docs/latest/vector-search/filter-search-knn/efficient-knn-filtering/.
|
|
129
81
|
"""
|
|
130
|
-
|
|
131
|
-
index = _get_unique_index_name()
|
|
132
|
-
|
|
133
|
-
store = OpenSearchDocumentStore(
|
|
134
|
-
hosts=hosts,
|
|
135
|
-
index=index,
|
|
136
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
137
|
-
verify_certs=False,
|
|
82
|
+
return opensearch_store(
|
|
138
83
|
embedding_dim=4,
|
|
139
84
|
method={"space_type": "innerproduct", "engine": "faiss", "name": "hnsw"},
|
|
140
85
|
)
|
|
141
|
-
yield store
|
|
142
|
-
|
|
143
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
144
86
|
|
|
145
87
|
|
|
146
88
|
@pytest.fixture
|
|
147
|
-
def document_store_nested():
|
|
148
|
-
"""
|
|
149
|
-
OpenSearch document store with explicit nested fields.
|
|
150
|
-
"""
|
|
151
|
-
hosts = ["https://localhost:9200"]
|
|
152
|
-
index = _get_unique_index_name()
|
|
153
|
-
|
|
154
|
-
store = OpenSearchDocumentStore(
|
|
155
|
-
hosts=hosts,
|
|
156
|
-
index=index,
|
|
157
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
158
|
-
verify_certs=False,
|
|
159
|
-
embedding_dim=768,
|
|
160
|
-
return_embedding=False,
|
|
161
|
-
nested_fields=["refs", "tags"],
|
|
162
|
-
)
|
|
163
|
-
store._ensure_initialized()
|
|
164
|
-
yield store
|
|
165
|
-
|
|
166
|
-
assert store._client
|
|
167
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
89
|
+
def document_store_nested(opensearch_store):
|
|
90
|
+
return opensearch_store(return_embedding=False, nested_fields=["refs", "tags"])
|
|
168
91
|
|
|
169
92
|
|
|
170
93
|
@pytest.fixture
|
|
171
|
-
def document_store_wildcard_nested():
|
|
172
|
-
""
|
|
173
|
-
OpenSearch document store with wildcard nested field auto-detection.
|
|
174
|
-
"""
|
|
175
|
-
hosts = ["https://localhost:9200"]
|
|
176
|
-
index = _get_unique_index_name()
|
|
177
|
-
|
|
178
|
-
store = OpenSearchDocumentStore(
|
|
179
|
-
hosts=hosts,
|
|
180
|
-
index=index,
|
|
181
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
182
|
-
verify_certs=False,
|
|
183
|
-
embedding_dim=768,
|
|
184
|
-
return_embedding=False,
|
|
185
|
-
nested_fields="*",
|
|
186
|
-
)
|
|
187
|
-
store._ensure_initialized()
|
|
188
|
-
yield store
|
|
189
|
-
|
|
190
|
-
assert store._client
|
|
191
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
94
|
+
def document_store_wildcard_nested(opensearch_store):
|
|
95
|
+
return opensearch_store(return_embedding=False, nested_fields="*")
|
|
192
96
|
|
|
193
97
|
|
|
194
98
|
@pytest.fixture
|
|
@@ -5,10 +5,16 @@
|
|
|
5
5
|
from unittest.mock import Mock, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
|
+
from botocore.exceptions import BotoCoreError
|
|
8
9
|
from haystack.utils.auth import Secret
|
|
9
10
|
from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
|
|
10
11
|
|
|
11
|
-
from haystack_integrations.document_stores.opensearch.auth import
|
|
12
|
+
from haystack_integrations.document_stores.opensearch.auth import (
|
|
13
|
+
AsyncAWSAuth,
|
|
14
|
+
AWSAuth,
|
|
15
|
+
AWSConfigurationError,
|
|
16
|
+
_get_aws_session,
|
|
17
|
+
)
|
|
12
18
|
from haystack_integrations.document_stores.opensearch.document_store import (
|
|
13
19
|
DEFAULT_MAX_CHUNK_BYTES,
|
|
14
20
|
OpenSearchDocumentStore,
|
|
@@ -131,6 +137,17 @@ class TestAWSAuth:
|
|
|
131
137
|
async_aws_auth(method="GET", url="http://some.url", body="some body", headers={"Host": "localhost"})
|
|
132
138
|
signer_auth_mock.assert_called_once_with("GET", "http://some.url", "some body", {"Host": "localhost"})
|
|
133
139
|
|
|
140
|
+
def test_get_aws_session_wraps_boto_core_error(self, mock_boto3_session):
|
|
141
|
+
mock_boto3_session.side_effect = BotoCoreError()
|
|
142
|
+
with pytest.raises(AWSConfigurationError, match="Failed to initialize the session"):
|
|
143
|
+
_get_aws_session(aws_access_key_id="x", aws_secret_access_key="y")
|
|
144
|
+
|
|
145
|
+
@patch("haystack_integrations.document_stores.opensearch.auth.Urllib3AWSV4SignerAuth")
|
|
146
|
+
def test_get_aws_v4_signer_auth_wraps_exceptions(self, mock_signer):
|
|
147
|
+
mock_signer.side_effect = RuntimeError("signer creation failed")
|
|
148
|
+
with pytest.raises(AWSConfigurationError, match="Could not connect to AWS OpenSearch"):
|
|
149
|
+
AWSAuth()
|
|
150
|
+
|
|
134
151
|
def test_async_aws_auth_init(self):
|
|
135
152
|
data = {
|
|
136
153
|
"type": "haystack_integrations.document_stores.opensearch.auth.AWSAuth",
|
|
@@ -68,7 +68,7 @@ def test_to_dict(_mock_opensearch_client):
|
|
|
68
68
|
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
69
69
|
},
|
|
70
70
|
"filters": {},
|
|
71
|
-
"fuzziness":
|
|
71
|
+
"fuzziness": 0,
|
|
72
72
|
"top_k": 10,
|
|
73
73
|
"scale_score": False,
|
|
74
74
|
"filter_policy": "replace",
|
|
@@ -163,7 +163,7 @@ def test_run():
|
|
|
163
163
|
mock_store._bm25_retrieval.assert_called_once_with(
|
|
164
164
|
query="some query",
|
|
165
165
|
filters={},
|
|
166
|
-
fuzziness=
|
|
166
|
+
fuzziness=0,
|
|
167
167
|
top_k=10,
|
|
168
168
|
scale_score=False,
|
|
169
169
|
all_terms_must_match=False,
|
|
@@ -183,7 +183,7 @@ async def test_run_async():
|
|
|
183
183
|
mock_store._bm25_retrieval_async.assert_called_once_with(
|
|
184
184
|
query="some query",
|
|
185
185
|
filters={},
|
|
186
|
-
fuzziness=
|
|
186
|
+
fuzziness=0,
|
|
187
187
|
top_k=10,
|
|
188
188
|
scale_score=False,
|
|
189
189
|
all_terms_must_match=False,
|
|
@@ -326,6 +326,22 @@ def test_run_ignore_errors(caplog):
|
|
|
326
326
|
assert "Some error" in caplog.text
|
|
327
327
|
|
|
328
328
|
|
|
329
|
+
@pytest.mark.asyncio
|
|
330
|
+
async def test_run_async_ignore_errors(caplog):
|
|
331
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
332
|
+
mock_store._bm25_retrieval_async.side_effect = Exception("Some error")
|
|
333
|
+
retriever = OpenSearchBM25Retriever(document_store=mock_store, raise_on_failure=False)
|
|
334
|
+
res = await retriever.run_async(query="some query")
|
|
335
|
+
assert len(res) == 1
|
|
336
|
+
assert res["documents"] == []
|
|
337
|
+
assert "Some error" in caplog.text
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def test_init_raises_on_invalid_document_store():
|
|
341
|
+
with pytest.raises(ValueError, match="document_store must be an instance of OpenSearchDocumentStore"):
|
|
342
|
+
OpenSearchBM25Retriever(document_store="not a document store")
|
|
343
|
+
|
|
344
|
+
|
|
329
345
|
def test_run_with_runtime_document_store():
|
|
330
346
|
# initial document store
|
|
331
347
|
initial_store = Mock(spec=OpenSearchDocumentStore)
|
|
@@ -343,7 +359,7 @@ def test_run_with_runtime_document_store():
|
|
|
343
359
|
runtime_store._bm25_retrieval.assert_called_once_with(
|
|
344
360
|
query="some query",
|
|
345
361
|
filters={},
|
|
346
|
-
fuzziness=
|
|
362
|
+
fuzziness=0,
|
|
347
363
|
top_k=10,
|
|
348
364
|
scale_score=False,
|
|
349
365
|
all_terms_must_match=False,
|
|
@@ -374,7 +390,7 @@ async def test_run_async_with_runtime_document_store():
|
|
|
374
390
|
runtime_store._bm25_retrieval_async.assert_called_once_with(
|
|
375
391
|
query="some query",
|
|
376
392
|
filters={},
|
|
377
|
-
fuzziness=
|
|
393
|
+
fuzziness=0,
|
|
378
394
|
top_k=10,
|
|
379
395
|
scale_score=False,
|
|
380
396
|
all_terms_must_match=False,
|
|
@@ -21,6 +21,7 @@ from opensearchpy.exceptions import RequestError, TransportError
|
|
|
21
21
|
|
|
22
22
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
23
23
|
from haystack_integrations.document_stores.opensearch.document_store import DEFAULT_MAX_CHUNK_BYTES
|
|
24
|
+
from tests.test_document_store_common import OpenSearchDocumentStoreTestMixin
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
@@ -554,6 +555,7 @@ def test_explicit_nested_fields_no_detection_on_write(mock_bulk, _mock_opensearc
|
|
|
554
555
|
|
|
555
556
|
@pytest.mark.integration
|
|
556
557
|
class TestDocumentStore(
|
|
558
|
+
OpenSearchDocumentStoreTestMixin,
|
|
557
559
|
CountDocumentsByFilterTest,
|
|
558
560
|
CountUniqueMetadataByFilterTest,
|
|
559
561
|
DocumentStoreBaseExtendedTests,
|
|
@@ -570,27 +572,6 @@ class TestDocumentStore(
|
|
|
570
572
|
"""Override base class fixture to provide OpenSearch document store."""
|
|
571
573
|
yield document_store
|
|
572
574
|
|
|
573
|
-
def assert_documents_are_equal(self, received: list[Document], expected: list[Document]):
|
|
574
|
-
"""
|
|
575
|
-
The OpenSearchDocumentStore.filter_documents() method returns documents with their score set.
|
|
576
|
-
|
|
577
|
-
We don't want to compare the score, so we set it to None before comparing.
|
|
578
|
-
|
|
579
|
-
Embeddings are not exactly the same when retrieved from OpenSearch (float round-trip),
|
|
580
|
-
so we compare them approximately and then set both to None for the final equality check.
|
|
581
|
-
"""
|
|
582
|
-
assert len(received) == len(expected)
|
|
583
|
-
received = sorted(received, key=lambda x: x.id)
|
|
584
|
-
expected = sorted(expected, key=lambda x: x.id)
|
|
585
|
-
for received_doc, expected_doc in zip(received, expected, strict=True):
|
|
586
|
-
received_doc.score = None
|
|
587
|
-
if received_doc.embedding is None:
|
|
588
|
-
assert expected_doc.embedding is None
|
|
589
|
-
else:
|
|
590
|
-
assert received_doc.embedding == pytest.approx(expected_doc.embedding)
|
|
591
|
-
received_doc.embedding, expected_doc.embedding = None, None
|
|
592
|
-
assert received_doc == expected_doc
|
|
593
|
-
|
|
594
575
|
def test_write_documents(self, document_store: OpenSearchDocumentStore):
|
|
595
576
|
docs = [Document(id="1")]
|
|
596
577
|
assert document_store.write_documents(docs) == 1
|