opensearch-haystack 7.2.0__tar.gz → 8.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/CHANGELOG.md +22 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/PKG-INFO +2 -2
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/pyproject.toml +1 -1
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +7 -17
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +4 -14
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +4 -8
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +1 -1
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +4 -14
- opensearch_haystack-8.0.0/src/haystack_integrations/components/retrievers/opensearch/utils.py +23 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +14 -6
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/conftest.py +45 -141
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_auth.py +18 -1
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_bm25_retriever.py +21 -5
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_document_store.py +2 -21
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_document_store_async.py +45 -410
- opensearch_haystack-8.0.0/tests/test_document_store_common.py +24 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_embedding_retriever.py +16 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_open_search_hybrid_retriever.py +11 -3
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_sql_retriever.py +45 -1
- opensearch_haystack-8.0.0/tests/test_utils.py +27 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/.gitignore +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/LICENSE +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/README.md +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/docker-compose.yml +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/filters.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/__init__.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_filters.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-7.2.0 → opensearch_haystack-8.0.0}/tests/test_metadata_retriever_async.py +0 -0
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v7.2.1] - 2026-04-13
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- *(opensearch)* Pass positional args to transport.perform_request for ddtrace compatibility (#3132)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
## [integrations/opensearch-v7.2.0] - 2026-04-10
|
|
11
|
+
|
|
12
|
+
### 🚀 Features
|
|
13
|
+
|
|
14
|
+
- *(opensearch)* Add support for nested fields in OpenSearchDocumentStore (#3117)
|
|
15
|
+
|
|
16
|
+
### 🐛 Bug Fixes
|
|
17
|
+
|
|
18
|
+
- Replace in-place dataclass mutations in document stores (#3114)
|
|
19
|
+
|
|
20
|
+
### 🧪 Testing
|
|
21
|
+
|
|
22
|
+
- Better categorize some Document Stores tests (#3085)
|
|
23
|
+
|
|
24
|
+
|
|
3
25
|
## [integrations/opensearch-v7.1.0] - 2026-04-01
|
|
4
26
|
|
|
5
27
|
### 🚀 Features
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 8.0.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.28.0
|
|
23
23
|
Requires-Dist: opensearch-py[async]>=3.0.0
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
25
25
|
|
|
@@ -13,6 +13,8 @@ from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
15
15
|
|
|
16
|
+
from .utils import _resolve_document_store
|
|
17
|
+
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
@@ -29,7 +31,7 @@ class OpenSearchBM25Retriever:
|
|
|
29
31
|
*,
|
|
30
32
|
document_store: OpenSearchDocumentStore,
|
|
31
33
|
filters: dict[str, Any] | None = None,
|
|
32
|
-
fuzziness: int | str =
|
|
34
|
+
fuzziness: int | str = 0,
|
|
33
35
|
top_k: int = 10,
|
|
34
36
|
scale_score: bool = False,
|
|
35
37
|
all_terms_must_match: bool = False,
|
|
@@ -47,8 +49,8 @@ class OpenSearchBM25Retriever:
|
|
|
47
49
|
required to transform one word into another. For example, the "fuzziness" between the words
|
|
48
50
|
"wined" and "wind" is 1 because only one edit is needed to match them.
|
|
49
51
|
|
|
50
|
-
Use "AUTO"
|
|
51
|
-
|
|
52
|
+
Defaults to `0` (exact matching). Use `"AUTO"` for automatic adjustment based on term length.
|
|
53
|
+
For detailed guidance, refer to the
|
|
52
54
|
[OpenSearch fuzzy query documentation](https://opensearch.org/docs/latest/query-dsl/term/fuzzy/).
|
|
53
55
|
:param top_k: Maximum number of documents to return.
|
|
54
56
|
:param scale_score: If `True`, scales the score of retrieved documents to a range between 0 and 1.
|
|
@@ -268,13 +270,7 @@ class OpenSearchBM25Retriever:
|
|
|
268
270
|
custom_query=custom_query,
|
|
269
271
|
)
|
|
270
272
|
|
|
271
|
-
|
|
272
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
273
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
274
|
-
raise ValueError(msg)
|
|
275
|
-
doc_store = document_store
|
|
276
|
-
else:
|
|
277
|
-
doc_store = self._document_store
|
|
273
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
278
274
|
|
|
279
275
|
try:
|
|
280
276
|
docs = doc_store._bm25_retrieval(**bm25_args) # example for BM25Retriever
|
|
@@ -335,13 +331,7 @@ class OpenSearchBM25Retriever:
|
|
|
335
331
|
custom_query=custom_query,
|
|
336
332
|
)
|
|
337
333
|
|
|
338
|
-
|
|
339
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
340
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
341
|
-
raise ValueError(msg)
|
|
342
|
-
doc_store = document_store
|
|
343
|
-
else:
|
|
344
|
-
doc_store = self._document_store
|
|
334
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
345
335
|
|
|
346
336
|
try:
|
|
347
337
|
docs = await doc_store._bm25_retrieval_async(**bm25_args)
|
|
@@ -13,6 +13,8 @@ from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
15
15
|
|
|
16
|
+
from .utils import _resolve_document_store
|
|
17
|
+
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
@@ -257,13 +259,7 @@ class OpenSearchEmbeddingRetriever:
|
|
|
257
259
|
|
|
258
260
|
docs: list[Document] = []
|
|
259
261
|
|
|
260
|
-
|
|
261
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
262
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
263
|
-
raise ValueError(msg)
|
|
264
|
-
doc_store = document_store
|
|
265
|
-
else:
|
|
266
|
-
doc_store = self._document_store
|
|
262
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
267
263
|
|
|
268
264
|
try:
|
|
269
265
|
docs = doc_store._embedding_retrieval(
|
|
@@ -383,13 +379,7 @@ class OpenSearchEmbeddingRetriever:
|
|
|
383
379
|
|
|
384
380
|
docs: list[Document] = []
|
|
385
381
|
|
|
386
|
-
|
|
387
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
388
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
389
|
-
raise ValueError(msg)
|
|
390
|
-
doc_store = document_store
|
|
391
|
-
else:
|
|
392
|
-
doc_store = self._document_store
|
|
382
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
393
383
|
|
|
394
384
|
try:
|
|
395
385
|
docs = await doc_store._embedding_retrieval_async(
|
|
@@ -9,6 +9,8 @@ from haystack import component, default_from_dict, default_to_dict, logging
|
|
|
9
9
|
|
|
10
10
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
11
11
|
|
|
12
|
+
from .utils import _resolve_document_store
|
|
13
|
+
|
|
12
14
|
logger = logging.getLogger(__name__)
|
|
13
15
|
|
|
14
16
|
|
|
@@ -252,10 +254,7 @@ class OpenSearchMetadataRetriever:
|
|
|
252
254
|
# Returns: {"metadata": [{"category": "Python", "status": "active", "priority": 1}]}
|
|
253
255
|
```
|
|
254
256
|
"""
|
|
255
|
-
doc_store = document_store
|
|
256
|
-
if not isinstance(doc_store, OpenSearchDocumentStore):
|
|
257
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
258
|
-
raise ValueError(msg)
|
|
257
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
259
258
|
|
|
260
259
|
fields_to_use = metadata_fields if metadata_fields is not None else self._metadata_fields
|
|
261
260
|
top_k_to_use = top_k if top_k is not None else self._top_k
|
|
@@ -366,10 +365,7 @@ class OpenSearchMetadataRetriever:
|
|
|
366
365
|
# Returns: {"metadata": [{"category": "Python", "status": "active", "priority": 1}]}
|
|
367
366
|
```
|
|
368
367
|
"""
|
|
369
|
-
doc_store = document_store
|
|
370
|
-
if not isinstance(doc_store, OpenSearchDocumentStore):
|
|
371
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
372
|
-
raise ValueError(msg)
|
|
368
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
373
369
|
|
|
374
370
|
fields_to_use = metadata_fields if metadata_fields is not None else self._metadata_fields
|
|
375
371
|
top_k_to_use = top_k if top_k is not None else self._top_k
|
|
@@ -93,7 +93,7 @@ class OpenSearchHybridRetriever:
|
|
|
93
93
|
embedder: TextEmbedder,
|
|
94
94
|
# OpenSearchBM25Retriever
|
|
95
95
|
filters_bm25: dict[str, Any] | None = None,
|
|
96
|
-
fuzziness: int | str =
|
|
96
|
+
fuzziness: int | str = 0,
|
|
97
97
|
top_k_bm25: int = 10,
|
|
98
98
|
scale_score: bool = False,
|
|
99
99
|
all_terms_must_match: bool = False,
|
|
@@ -8,6 +8,8 @@ from haystack import component, default_from_dict, default_to_dict, logging
|
|
|
8
8
|
|
|
9
9
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
10
10
|
|
|
11
|
+
from .utils import _resolve_document_store
|
|
12
|
+
|
|
11
13
|
logger = logging.getLogger(__name__)
|
|
12
14
|
|
|
13
15
|
|
|
@@ -108,13 +110,7 @@ class OpenSearchSQLRetriever:
|
|
|
108
110
|
# For aggregate queries: result["result"]["aggregations"] contains aggregations
|
|
109
111
|
```
|
|
110
112
|
"""
|
|
111
|
-
|
|
112
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
113
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
114
|
-
raise ValueError(msg)
|
|
115
|
-
doc_store = document_store
|
|
116
|
-
else:
|
|
117
|
-
doc_store = self._document_store
|
|
113
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
118
114
|
|
|
119
115
|
fetch_size = fetch_size if fetch_size is not None else self._fetch_size
|
|
120
116
|
|
|
@@ -163,13 +159,7 @@ class OpenSearchSQLRetriever:
|
|
|
163
159
|
# For aggregate queries: result["result"]["aggregations"] contains aggregations
|
|
164
160
|
```
|
|
165
161
|
"""
|
|
166
|
-
|
|
167
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
168
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
169
|
-
raise ValueError(msg)
|
|
170
|
-
doc_store = document_store
|
|
171
|
-
else:
|
|
172
|
-
doc_store = self._document_store
|
|
162
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
173
163
|
|
|
174
164
|
fetch_size = fetch_size if fetch_size is not None else self._fetch_size
|
|
175
165
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _resolve_document_store(
|
|
10
|
+
runtime_document_store: OpenSearchDocumentStore | None,
|
|
11
|
+
default_document_store: OpenSearchDocumentStore,
|
|
12
|
+
) -> OpenSearchDocumentStore:
|
|
13
|
+
"""
|
|
14
|
+
Return the runtime document store if provided and valid, otherwise the default one.
|
|
15
|
+
|
|
16
|
+
:raises ValueError: If `runtime_document_store` is not None and not an OpenSearchDocumentStore.
|
|
17
|
+
"""
|
|
18
|
+
if runtime_document_store is None:
|
|
19
|
+
return default_document_store
|
|
20
|
+
if not isinstance(runtime_document_store, OpenSearchDocumentStore):
|
|
21
|
+
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
22
|
+
raise ValueError(msg)
|
|
23
|
+
return runtime_document_store
|
|
@@ -1057,7 +1057,7 @@ class OpenSearchDocumentStore:
|
|
|
1057
1057
|
query: str,
|
|
1058
1058
|
*,
|
|
1059
1059
|
filters: dict[str, Any] | None = None,
|
|
1060
|
-
fuzziness: int | str =
|
|
1060
|
+
fuzziness: int | str = 0,
|
|
1061
1061
|
top_k: int = 10,
|
|
1062
1062
|
scale_score: bool = False,
|
|
1063
1063
|
all_terms_must_match: bool = False,
|
|
@@ -1115,7 +1115,7 @@ class OpenSearchDocumentStore:
|
|
|
1115
1115
|
query: str,
|
|
1116
1116
|
*,
|
|
1117
1117
|
filters: dict[str, Any] | None = None,
|
|
1118
|
-
fuzziness: str =
|
|
1118
|
+
fuzziness: int | str = 0,
|
|
1119
1119
|
top_k: int = 10,
|
|
1120
1120
|
scale_score: bool = False,
|
|
1121
1121
|
all_terms_must_match: bool = False,
|
|
@@ -2160,9 +2160,13 @@ class OpenSearchDocumentStore:
|
|
|
2160
2160
|
if fetch_size is not None:
|
|
2161
2161
|
body["fetch_size"] = fetch_size
|
|
2162
2162
|
|
|
2163
|
+
# NOTE: method and url must be passed as positional args (not keyword args).
|
|
2164
|
+
# ddtrace's Elasticsearch instrumentation patch wraps perform_request and unpacks
|
|
2165
|
+
# the first two positional args as `method, target = args`. Passing them as
|
|
2166
|
+
# keyword args results in an empty `args` tuple and raises a ValueError.
|
|
2163
2167
|
response_data = self._client.transport.perform_request(
|
|
2164
|
-
|
|
2165
|
-
|
|
2168
|
+
"POST",
|
|
2169
|
+
"/_plugins/_sql",
|
|
2166
2170
|
body=body,
|
|
2167
2171
|
)
|
|
2168
2172
|
|
|
@@ -2193,9 +2197,13 @@ class OpenSearchDocumentStore:
|
|
|
2193
2197
|
if fetch_size is not None:
|
|
2194
2198
|
body["fetch_size"] = fetch_size
|
|
2195
2199
|
|
|
2200
|
+
# NOTE: method and url must be passed as positional args (not keyword args).
|
|
2201
|
+
# ddtrace's Elasticsearch instrumentation patch wraps perform_request and unpacks
|
|
2202
|
+
# the first two positional args as `method, target = args`. Passing them as
|
|
2203
|
+
# keyword args results in an empty `args` tuple and raises a ValueError.
|
|
2196
2204
|
response_data = await self._async_client.transport.perform_request(
|
|
2197
|
-
|
|
2198
|
-
|
|
2205
|
+
"POST",
|
|
2206
|
+
"/_plugins/_sql",
|
|
2199
2207
|
body=body,
|
|
2200
2208
|
)
|
|
2201
2209
|
|
|
@@ -6,6 +6,13 @@ from haystack import Document
|
|
|
6
6
|
|
|
7
7
|
from haystack_integrations.document_stores.opensearch.document_store import OpenSearchDocumentStore
|
|
8
8
|
|
|
9
|
+
COMMON_KWARGS = {
|
|
10
|
+
"hosts": ["https://localhost:9200"],
|
|
11
|
+
"http_auth": ("admin", "SecureHaystack!2026"),
|
|
12
|
+
"verify_certs": False,
|
|
13
|
+
}
|
|
14
|
+
DEFAULT_METHOD = {"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"}
|
|
15
|
+
|
|
9
16
|
|
|
10
17
|
def _get_unique_index_name() -> str:
|
|
11
18
|
"""
|
|
@@ -17,178 +24,75 @@ def _get_unique_index_name() -> str:
|
|
|
17
24
|
|
|
18
25
|
|
|
19
26
|
@pytest.fixture
|
|
20
|
-
def
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
44
|
-
asyncio.run(store._async_client.close())
|
|
27
|
+
def opensearch_store():
|
|
28
|
+
created: list[OpenSearchDocumentStore] = []
|
|
29
|
+
|
|
30
|
+
def _make(**overrides) -> OpenSearchDocumentStore:
|
|
31
|
+
kwargs = {
|
|
32
|
+
**COMMON_KWARGS,
|
|
33
|
+
"index": _get_unique_index_name(),
|
|
34
|
+
"embedding_dim": 768,
|
|
35
|
+
"return_embedding": True,
|
|
36
|
+
"method": DEFAULT_METHOD,
|
|
37
|
+
**overrides,
|
|
38
|
+
}
|
|
39
|
+
store = OpenSearchDocumentStore(**kwargs)
|
|
40
|
+
store._ensure_initialized()
|
|
41
|
+
created.append(store)
|
|
42
|
+
return store
|
|
43
|
+
|
|
44
|
+
yield _make
|
|
45
|
+
|
|
46
|
+
for store in created:
|
|
47
|
+
asyncio.run(store._ensure_initialized_async())
|
|
48
|
+
store._client.indices.delete(index=store._index, params={"ignore": [400, 404]})
|
|
49
|
+
asyncio.run(store._async_client.close())
|
|
45
50
|
|
|
46
51
|
|
|
47
52
|
@pytest.fixture
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
index = f"test_index_2_{_get_unique_index_name()}"
|
|
53
|
+
def document_store(opensearch_store):
|
|
54
|
+
return opensearch_store()
|
|
51
55
|
|
|
52
|
-
store = OpenSearchDocumentStore(
|
|
53
|
-
hosts=hosts,
|
|
54
|
-
index=index,
|
|
55
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
56
|
-
verify_certs=False,
|
|
57
|
-
embedding_dim=768,
|
|
58
|
-
return_embedding=False,
|
|
59
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
60
|
-
)
|
|
61
|
-
yield store
|
|
62
56
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
assert store._client
|
|
67
|
-
assert store._async_client
|
|
68
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
69
|
-
asyncio.run(store._async_client.close())
|
|
57
|
+
@pytest.fixture
|
|
58
|
+
def document_store_2(opensearch_store):
|
|
59
|
+
return opensearch_store(return_embedding=False)
|
|
70
60
|
|
|
71
61
|
|
|
72
62
|
@pytest.fixture
|
|
73
|
-
def document_store_readonly():
|
|
74
|
-
|
|
75
|
-
A document store that does not automatically create the underlying index.
|
|
76
|
-
"""
|
|
77
|
-
hosts = ["https://localhost:9200"]
|
|
78
|
-
# Use a different index for each test so we can run them in parallel
|
|
79
|
-
index = _get_unique_index_name()
|
|
80
|
-
|
|
81
|
-
store = OpenSearchDocumentStore(
|
|
82
|
-
hosts=hosts,
|
|
83
|
-
index=index,
|
|
84
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
85
|
-
verify_certs=False,
|
|
86
|
-
embedding_dim=768,
|
|
87
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
88
|
-
create_index=False,
|
|
89
|
-
)
|
|
90
|
-
store._ensure_initialized()
|
|
91
|
-
asyncio.run(store._ensure_initialized_async())
|
|
92
|
-
assert store._client
|
|
93
|
-
assert store._async_client
|
|
63
|
+
def document_store_readonly(opensearch_store):
|
|
64
|
+
store = opensearch_store(create_index=False)
|
|
94
65
|
store._client.cluster.put_settings(body={"transient": {"action.auto_create_index": False}})
|
|
95
66
|
yield store
|
|
96
|
-
|
|
97
67
|
store._client.cluster.put_settings(body={"transient": {"action.auto_create_index": True}})
|
|
98
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
99
68
|
|
|
100
69
|
|
|
101
70
|
@pytest.fixture
|
|
102
|
-
def document_store_embedding_dim_4_no_emb_returned():
|
|
103
|
-
|
|
104
|
-
A document store with embedding dimension 4 that does not return embeddings.
|
|
105
|
-
"""
|
|
106
|
-
hosts = ["https://localhost:9200"]
|
|
107
|
-
index = _get_unique_index_name()
|
|
108
|
-
|
|
109
|
-
store = OpenSearchDocumentStore(
|
|
110
|
-
hosts=hosts,
|
|
111
|
-
index=index,
|
|
112
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
113
|
-
verify_certs=False,
|
|
114
|
-
embedding_dim=4,
|
|
115
|
-
return_embedding=False,
|
|
116
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
117
|
-
)
|
|
118
|
-
yield store
|
|
119
|
-
|
|
120
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
71
|
+
def document_store_embedding_dim_4_no_emb_returned(opensearch_store):
|
|
72
|
+
return opensearch_store(embedding_dim=4, return_embedding=False)
|
|
121
73
|
|
|
122
74
|
|
|
123
75
|
@pytest.fixture
|
|
124
|
-
def document_store_embedding_dim_4_no_emb_returned_faiss():
|
|
76
|
+
def document_store_embedding_dim_4_no_emb_returned_faiss(opensearch_store):
|
|
125
77
|
"""
|
|
126
78
|
A document store with embedding dimension 4 that uses a FAISS engine with HNSW algorithm for vector search.
|
|
127
79
|
We use this document store for testing efficient k-NN filtering according to
|
|
128
80
|
https://opensearch.org/docs/latest/vector-search/filter-search-knn/efficient-knn-filtering/.
|
|
129
81
|
"""
|
|
130
|
-
|
|
131
|
-
index = _get_unique_index_name()
|
|
132
|
-
|
|
133
|
-
store = OpenSearchDocumentStore(
|
|
134
|
-
hosts=hosts,
|
|
135
|
-
index=index,
|
|
136
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
137
|
-
verify_certs=False,
|
|
82
|
+
return opensearch_store(
|
|
138
83
|
embedding_dim=4,
|
|
139
84
|
method={"space_type": "innerproduct", "engine": "faiss", "name": "hnsw"},
|
|
140
85
|
)
|
|
141
|
-
yield store
|
|
142
|
-
|
|
143
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
144
86
|
|
|
145
87
|
|
|
146
88
|
@pytest.fixture
|
|
147
|
-
def document_store_nested():
|
|
148
|
-
"""
|
|
149
|
-
OpenSearch document store with explicit nested fields.
|
|
150
|
-
"""
|
|
151
|
-
hosts = ["https://localhost:9200"]
|
|
152
|
-
index = _get_unique_index_name()
|
|
153
|
-
|
|
154
|
-
store = OpenSearchDocumentStore(
|
|
155
|
-
hosts=hosts,
|
|
156
|
-
index=index,
|
|
157
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
158
|
-
verify_certs=False,
|
|
159
|
-
embedding_dim=768,
|
|
160
|
-
return_embedding=False,
|
|
161
|
-
nested_fields=["refs", "tags"],
|
|
162
|
-
)
|
|
163
|
-
store._ensure_initialized()
|
|
164
|
-
yield store
|
|
165
|
-
|
|
166
|
-
assert store._client
|
|
167
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
89
|
+
def document_store_nested(opensearch_store):
|
|
90
|
+
return opensearch_store(return_embedding=False, nested_fields=["refs", "tags"])
|
|
168
91
|
|
|
169
92
|
|
|
170
93
|
@pytest.fixture
|
|
171
|
-
def document_store_wildcard_nested():
|
|
172
|
-
""
|
|
173
|
-
OpenSearch document store with wildcard nested field auto-detection.
|
|
174
|
-
"""
|
|
175
|
-
hosts = ["https://localhost:9200"]
|
|
176
|
-
index = _get_unique_index_name()
|
|
177
|
-
|
|
178
|
-
store = OpenSearchDocumentStore(
|
|
179
|
-
hosts=hosts,
|
|
180
|
-
index=index,
|
|
181
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
182
|
-
verify_certs=False,
|
|
183
|
-
embedding_dim=768,
|
|
184
|
-
return_embedding=False,
|
|
185
|
-
nested_fields="*",
|
|
186
|
-
)
|
|
187
|
-
store._ensure_initialized()
|
|
188
|
-
yield store
|
|
189
|
-
|
|
190
|
-
assert store._client
|
|
191
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
94
|
+
def document_store_wildcard_nested(opensearch_store):
|
|
95
|
+
return opensearch_store(return_embedding=False, nested_fields="*")
|
|
192
96
|
|
|
193
97
|
|
|
194
98
|
@pytest.fixture
|
|
@@ -5,10 +5,16 @@
|
|
|
5
5
|
from unittest.mock import Mock, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
|
+
from botocore.exceptions import BotoCoreError
|
|
8
9
|
from haystack.utils.auth import Secret
|
|
9
10
|
from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
|
|
10
11
|
|
|
11
|
-
from haystack_integrations.document_stores.opensearch.auth import
|
|
12
|
+
from haystack_integrations.document_stores.opensearch.auth import (
|
|
13
|
+
AsyncAWSAuth,
|
|
14
|
+
AWSAuth,
|
|
15
|
+
AWSConfigurationError,
|
|
16
|
+
_get_aws_session,
|
|
17
|
+
)
|
|
12
18
|
from haystack_integrations.document_stores.opensearch.document_store import (
|
|
13
19
|
DEFAULT_MAX_CHUNK_BYTES,
|
|
14
20
|
OpenSearchDocumentStore,
|
|
@@ -131,6 +137,17 @@ class TestAWSAuth:
|
|
|
131
137
|
async_aws_auth(method="GET", url="http://some.url", body="some body", headers={"Host": "localhost"})
|
|
132
138
|
signer_auth_mock.assert_called_once_with("GET", "http://some.url", "some body", {"Host": "localhost"})
|
|
133
139
|
|
|
140
|
+
def test_get_aws_session_wraps_boto_core_error(self, mock_boto3_session):
|
|
141
|
+
mock_boto3_session.side_effect = BotoCoreError()
|
|
142
|
+
with pytest.raises(AWSConfigurationError, match="Failed to initialize the session"):
|
|
143
|
+
_get_aws_session(aws_access_key_id="x", aws_secret_access_key="y")
|
|
144
|
+
|
|
145
|
+
@patch("haystack_integrations.document_stores.opensearch.auth.Urllib3AWSV4SignerAuth")
|
|
146
|
+
def test_get_aws_v4_signer_auth_wraps_exceptions(self, mock_signer):
|
|
147
|
+
mock_signer.side_effect = RuntimeError("signer creation failed")
|
|
148
|
+
with pytest.raises(AWSConfigurationError, match="Could not connect to AWS OpenSearch"):
|
|
149
|
+
AWSAuth()
|
|
150
|
+
|
|
134
151
|
def test_async_aws_auth_init(self):
|
|
135
152
|
data = {
|
|
136
153
|
"type": "haystack_integrations.document_stores.opensearch.auth.AWSAuth",
|
|
@@ -68,7 +68,7 @@ def test_to_dict(_mock_opensearch_client):
|
|
|
68
68
|
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
69
69
|
},
|
|
70
70
|
"filters": {},
|
|
71
|
-
"fuzziness":
|
|
71
|
+
"fuzziness": 0,
|
|
72
72
|
"top_k": 10,
|
|
73
73
|
"scale_score": False,
|
|
74
74
|
"filter_policy": "replace",
|
|
@@ -163,7 +163,7 @@ def test_run():
|
|
|
163
163
|
mock_store._bm25_retrieval.assert_called_once_with(
|
|
164
164
|
query="some query",
|
|
165
165
|
filters={},
|
|
166
|
-
fuzziness=
|
|
166
|
+
fuzziness=0,
|
|
167
167
|
top_k=10,
|
|
168
168
|
scale_score=False,
|
|
169
169
|
all_terms_must_match=False,
|
|
@@ -183,7 +183,7 @@ async def test_run_async():
|
|
|
183
183
|
mock_store._bm25_retrieval_async.assert_called_once_with(
|
|
184
184
|
query="some query",
|
|
185
185
|
filters={},
|
|
186
|
-
fuzziness=
|
|
186
|
+
fuzziness=0,
|
|
187
187
|
top_k=10,
|
|
188
188
|
scale_score=False,
|
|
189
189
|
all_terms_must_match=False,
|
|
@@ -326,6 +326,22 @@ def test_run_ignore_errors(caplog):
|
|
|
326
326
|
assert "Some error" in caplog.text
|
|
327
327
|
|
|
328
328
|
|
|
329
|
+
@pytest.mark.asyncio
|
|
330
|
+
async def test_run_async_ignore_errors(caplog):
|
|
331
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
332
|
+
mock_store._bm25_retrieval_async.side_effect = Exception("Some error")
|
|
333
|
+
retriever = OpenSearchBM25Retriever(document_store=mock_store, raise_on_failure=False)
|
|
334
|
+
res = await retriever.run_async(query="some query")
|
|
335
|
+
assert len(res) == 1
|
|
336
|
+
assert res["documents"] == []
|
|
337
|
+
assert "Some error" in caplog.text
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def test_init_raises_on_invalid_document_store():
|
|
341
|
+
with pytest.raises(ValueError, match="document_store must be an instance of OpenSearchDocumentStore"):
|
|
342
|
+
OpenSearchBM25Retriever(document_store="not a document store")
|
|
343
|
+
|
|
344
|
+
|
|
329
345
|
def test_run_with_runtime_document_store():
|
|
330
346
|
# initial document store
|
|
331
347
|
initial_store = Mock(spec=OpenSearchDocumentStore)
|
|
@@ -343,7 +359,7 @@ def test_run_with_runtime_document_store():
|
|
|
343
359
|
runtime_store._bm25_retrieval.assert_called_once_with(
|
|
344
360
|
query="some query",
|
|
345
361
|
filters={},
|
|
346
|
-
fuzziness=
|
|
362
|
+
fuzziness=0,
|
|
347
363
|
top_k=10,
|
|
348
364
|
scale_score=False,
|
|
349
365
|
all_terms_must_match=False,
|
|
@@ -374,7 +390,7 @@ async def test_run_async_with_runtime_document_store():
|
|
|
374
390
|
runtime_store._bm25_retrieval_async.assert_called_once_with(
|
|
375
391
|
query="some query",
|
|
376
392
|
filters={},
|
|
377
|
-
fuzziness=
|
|
393
|
+
fuzziness=0,
|
|
378
394
|
top_k=10,
|
|
379
395
|
scale_score=False,
|
|
380
396
|
all_terms_must_match=False,
|