opensearch-haystack 7.2.1__tar.gz → 8.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/CHANGELOG.md +22 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/PKG-INFO +2 -2
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/pyproject.toml +1 -1
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +7 -17
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +4 -14
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +4 -8
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +1 -1
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +4 -14
- opensearch_haystack-8.1.0/src/haystack_integrations/components/retrievers/opensearch/utils.py +23 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +10 -4
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/conftest.py +45 -141
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_auth.py +26 -1
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_bm25_retriever.py +21 -5
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_document_store.py +164 -21
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_document_store_async.py +71 -410
- opensearch_haystack-8.1.0/tests/test_document_store_common.py +24 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_embedding_retriever.py +16 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_open_search_hybrid_retriever.py +11 -3
- opensearch_haystack-8.1.0/tests/test_utils.py +27 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/.gitignore +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/LICENSE +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/README.md +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/docker-compose.yml +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/document_stores/opensearch/filters.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/__init__.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_filters.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-7.2.1 → opensearch_haystack-8.1.0}/tests/test_sql_retriever.py +0 -0
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v8.0.0] - 2026-04-29
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- [**breaking**] Change default fuzziness from AUTO to 0 in OpenSearch retrievers (#3198)
|
|
8
|
+
|
|
9
|
+
### 🚜 Refactor
|
|
10
|
+
|
|
11
|
+
- *(opensearch)* Use async DocumentStore mixin tests (#3224)
|
|
12
|
+
|
|
13
|
+
### 🧪 Testing
|
|
14
|
+
|
|
15
|
+
- Opensearch - add unit tests (#3182)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
## [integrations/opensearch-v7.2.1] - 2026-04-13
|
|
19
|
+
|
|
20
|
+
### 🐛 Bug Fixes
|
|
21
|
+
|
|
22
|
+
- *(opensearch)* Pass positional args to transport.perform_request for ddtrace compatibility (#3132)
|
|
23
|
+
|
|
24
|
+
|
|
3
25
|
## [integrations/opensearch-v7.2.0] - 2026-04-10
|
|
4
26
|
|
|
5
27
|
### 🚀 Features
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 8.1.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.28.0
|
|
23
23
|
Requires-Dist: opensearch-py[async]>=3.0.0
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
25
25
|
|
|
@@ -13,6 +13,8 @@ from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
15
15
|
|
|
16
|
+
from .utils import _resolve_document_store
|
|
17
|
+
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
@@ -29,7 +31,7 @@ class OpenSearchBM25Retriever:
|
|
|
29
31
|
*,
|
|
30
32
|
document_store: OpenSearchDocumentStore,
|
|
31
33
|
filters: dict[str, Any] | None = None,
|
|
32
|
-
fuzziness: int | str =
|
|
34
|
+
fuzziness: int | str = 0,
|
|
33
35
|
top_k: int = 10,
|
|
34
36
|
scale_score: bool = False,
|
|
35
37
|
all_terms_must_match: bool = False,
|
|
@@ -47,8 +49,8 @@ class OpenSearchBM25Retriever:
|
|
|
47
49
|
required to transform one word into another. For example, the "fuzziness" between the words
|
|
48
50
|
"wined" and "wind" is 1 because only one edit is needed to match them.
|
|
49
51
|
|
|
50
|
-
Use "AUTO"
|
|
51
|
-
|
|
52
|
+
Defaults to `0` (exact matching). Use `"AUTO"` for automatic adjustment based on term length.
|
|
53
|
+
For detailed guidance, refer to the
|
|
52
54
|
[OpenSearch fuzzy query documentation](https://opensearch.org/docs/latest/query-dsl/term/fuzzy/).
|
|
53
55
|
:param top_k: Maximum number of documents to return.
|
|
54
56
|
:param scale_score: If `True`, scales the score of retrieved documents to a range between 0 and 1.
|
|
@@ -268,13 +270,7 @@ class OpenSearchBM25Retriever:
|
|
|
268
270
|
custom_query=custom_query,
|
|
269
271
|
)
|
|
270
272
|
|
|
271
|
-
|
|
272
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
273
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
274
|
-
raise ValueError(msg)
|
|
275
|
-
doc_store = document_store
|
|
276
|
-
else:
|
|
277
|
-
doc_store = self._document_store
|
|
273
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
278
274
|
|
|
279
275
|
try:
|
|
280
276
|
docs = doc_store._bm25_retrieval(**bm25_args) # example for BM25Retriever
|
|
@@ -335,13 +331,7 @@ class OpenSearchBM25Retriever:
|
|
|
335
331
|
custom_query=custom_query,
|
|
336
332
|
)
|
|
337
333
|
|
|
338
|
-
|
|
339
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
340
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
341
|
-
raise ValueError(msg)
|
|
342
|
-
doc_store = document_store
|
|
343
|
-
else:
|
|
344
|
-
doc_store = self._document_store
|
|
334
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
345
335
|
|
|
346
336
|
try:
|
|
347
337
|
docs = await doc_store._bm25_retrieval_async(**bm25_args)
|
|
@@ -13,6 +13,8 @@ from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
15
15
|
|
|
16
|
+
from .utils import _resolve_document_store
|
|
17
|
+
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
17
19
|
|
|
18
20
|
|
|
@@ -257,13 +259,7 @@ class OpenSearchEmbeddingRetriever:
|
|
|
257
259
|
|
|
258
260
|
docs: list[Document] = []
|
|
259
261
|
|
|
260
|
-
|
|
261
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
262
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
263
|
-
raise ValueError(msg)
|
|
264
|
-
doc_store = document_store
|
|
265
|
-
else:
|
|
266
|
-
doc_store = self._document_store
|
|
262
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
267
263
|
|
|
268
264
|
try:
|
|
269
265
|
docs = doc_store._embedding_retrieval(
|
|
@@ -383,13 +379,7 @@ class OpenSearchEmbeddingRetriever:
|
|
|
383
379
|
|
|
384
380
|
docs: list[Document] = []
|
|
385
381
|
|
|
386
|
-
|
|
387
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
388
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
389
|
-
raise ValueError(msg)
|
|
390
|
-
doc_store = document_store
|
|
391
|
-
else:
|
|
392
|
-
doc_store = self._document_store
|
|
382
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
393
383
|
|
|
394
384
|
try:
|
|
395
385
|
docs = await doc_store._embedding_retrieval_async(
|
|
@@ -9,6 +9,8 @@ from haystack import component, default_from_dict, default_to_dict, logging
|
|
|
9
9
|
|
|
10
10
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
11
11
|
|
|
12
|
+
from .utils import _resolve_document_store
|
|
13
|
+
|
|
12
14
|
logger = logging.getLogger(__name__)
|
|
13
15
|
|
|
14
16
|
|
|
@@ -252,10 +254,7 @@ class OpenSearchMetadataRetriever:
|
|
|
252
254
|
# Returns: {"metadata": [{"category": "Python", "status": "active", "priority": 1}]}
|
|
253
255
|
```
|
|
254
256
|
"""
|
|
255
|
-
doc_store = document_store
|
|
256
|
-
if not isinstance(doc_store, OpenSearchDocumentStore):
|
|
257
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
258
|
-
raise ValueError(msg)
|
|
257
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
259
258
|
|
|
260
259
|
fields_to_use = metadata_fields if metadata_fields is not None else self._metadata_fields
|
|
261
260
|
top_k_to_use = top_k if top_k is not None else self._top_k
|
|
@@ -366,10 +365,7 @@ class OpenSearchMetadataRetriever:
|
|
|
366
365
|
# Returns: {"metadata": [{"category": "Python", "status": "active", "priority": 1}]}
|
|
367
366
|
```
|
|
368
367
|
"""
|
|
369
|
-
doc_store = document_store
|
|
370
|
-
if not isinstance(doc_store, OpenSearchDocumentStore):
|
|
371
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
372
|
-
raise ValueError(msg)
|
|
368
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
373
369
|
|
|
374
370
|
fields_to_use = metadata_fields if metadata_fields is not None else self._metadata_fields
|
|
375
371
|
top_k_to_use = top_k if top_k is not None else self._top_k
|
|
@@ -93,7 +93,7 @@ class OpenSearchHybridRetriever:
|
|
|
93
93
|
embedder: TextEmbedder,
|
|
94
94
|
# OpenSearchBM25Retriever
|
|
95
95
|
filters_bm25: dict[str, Any] | None = None,
|
|
96
|
-
fuzziness: int | str =
|
|
96
|
+
fuzziness: int | str = 0,
|
|
97
97
|
top_k_bm25: int = 10,
|
|
98
98
|
scale_score: bool = False,
|
|
99
99
|
all_terms_must_match: bool = False,
|
|
@@ -8,6 +8,8 @@ from haystack import component, default_from_dict, default_to_dict, logging
|
|
|
8
8
|
|
|
9
9
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
10
10
|
|
|
11
|
+
from .utils import _resolve_document_store
|
|
12
|
+
|
|
11
13
|
logger = logging.getLogger(__name__)
|
|
12
14
|
|
|
13
15
|
|
|
@@ -108,13 +110,7 @@ class OpenSearchSQLRetriever:
|
|
|
108
110
|
# For aggregate queries: result["result"]["aggregations"] contains aggregations
|
|
109
111
|
```
|
|
110
112
|
"""
|
|
111
|
-
|
|
112
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
113
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
114
|
-
raise ValueError(msg)
|
|
115
|
-
doc_store = document_store
|
|
116
|
-
else:
|
|
117
|
-
doc_store = self._document_store
|
|
113
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
118
114
|
|
|
119
115
|
fetch_size = fetch_size if fetch_size is not None else self._fetch_size
|
|
120
116
|
|
|
@@ -163,13 +159,7 @@ class OpenSearchSQLRetriever:
|
|
|
163
159
|
# For aggregate queries: result["result"]["aggregations"] contains aggregations
|
|
164
160
|
```
|
|
165
161
|
"""
|
|
166
|
-
|
|
167
|
-
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
168
|
-
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
169
|
-
raise ValueError(msg)
|
|
170
|
-
doc_store = document_store
|
|
171
|
-
else:
|
|
172
|
-
doc_store = self._document_store
|
|
162
|
+
doc_store = _resolve_document_store(document_store, self._document_store)
|
|
173
163
|
|
|
174
164
|
fetch_size = fetch_size if fetch_size is not None else self._fetch_size
|
|
175
165
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _resolve_document_store(
|
|
10
|
+
runtime_document_store: OpenSearchDocumentStore | None,
|
|
11
|
+
default_document_store: OpenSearchDocumentStore,
|
|
12
|
+
) -> OpenSearchDocumentStore:
|
|
13
|
+
"""
|
|
14
|
+
Return the runtime document store if provided and valid, otherwise the default one.
|
|
15
|
+
|
|
16
|
+
:raises ValueError: If `runtime_document_store` is not None and not an OpenSearchDocumentStore.
|
|
17
|
+
"""
|
|
18
|
+
if runtime_document_store is None:
|
|
19
|
+
return default_document_store
|
|
20
|
+
if not isinstance(runtime_document_store, OpenSearchDocumentStore):
|
|
21
|
+
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
22
|
+
raise ValueError(msg)
|
|
23
|
+
return runtime_document_store
|
|
@@ -400,7 +400,10 @@ class OpenSearchDocumentStore:
|
|
|
400
400
|
index=self._index,
|
|
401
401
|
)
|
|
402
402
|
mapping = await self._async_client.indices.get_mapping(index=self._index)
|
|
403
|
-
|
|
403
|
+
# get_mapping keys the response by the real index name; when self._index is an alias
|
|
404
|
+
# the key differs from the alias, so we use next(iter(...)) to handle both cases.
|
|
405
|
+
actual_index = next(iter(mapping))
|
|
406
|
+
properties = mapping[actual_index]["mappings"].get("properties", {})
|
|
404
407
|
self._populate_nested_fields_from_mapping(properties)
|
|
405
408
|
elif self._create_index:
|
|
406
409
|
# Create the index if it doesn't exist
|
|
@@ -417,7 +420,10 @@ class OpenSearchDocumentStore:
|
|
|
417
420
|
index=self._index,
|
|
418
421
|
)
|
|
419
422
|
mapping = self._client.indices.get_mapping(index=self._index)
|
|
420
|
-
|
|
423
|
+
# get_mapping keys the response by the real index name; when self._index is an alias
|
|
424
|
+
# the key differs from the alias, so we use next(iter(...)) to handle both cases.
|
|
425
|
+
actual_index = next(iter(mapping))
|
|
426
|
+
properties = mapping[actual_index]["mappings"].get("properties", {})
|
|
421
427
|
self._populate_nested_fields_from_mapping(properties)
|
|
422
428
|
elif self._create_index:
|
|
423
429
|
# Create the index if it doesn't exist
|
|
@@ -1057,7 +1063,7 @@ class OpenSearchDocumentStore:
|
|
|
1057
1063
|
query: str,
|
|
1058
1064
|
*,
|
|
1059
1065
|
filters: dict[str, Any] | None = None,
|
|
1060
|
-
fuzziness: int | str =
|
|
1066
|
+
fuzziness: int | str = 0,
|
|
1061
1067
|
top_k: int = 10,
|
|
1062
1068
|
scale_score: bool = False,
|
|
1063
1069
|
all_terms_must_match: bool = False,
|
|
@@ -1115,7 +1121,7 @@ class OpenSearchDocumentStore:
|
|
|
1115
1121
|
query: str,
|
|
1116
1122
|
*,
|
|
1117
1123
|
filters: dict[str, Any] | None = None,
|
|
1118
|
-
fuzziness: str =
|
|
1124
|
+
fuzziness: int | str = 0,
|
|
1119
1125
|
top_k: int = 10,
|
|
1120
1126
|
scale_score: bool = False,
|
|
1121
1127
|
all_terms_must_match: bool = False,
|
|
@@ -6,6 +6,13 @@ from haystack import Document
|
|
|
6
6
|
|
|
7
7
|
from haystack_integrations.document_stores.opensearch.document_store import OpenSearchDocumentStore
|
|
8
8
|
|
|
9
|
+
COMMON_KWARGS = {
|
|
10
|
+
"hosts": ["https://localhost:9200"],
|
|
11
|
+
"http_auth": ("admin", "SecureHaystack!2026"),
|
|
12
|
+
"verify_certs": False,
|
|
13
|
+
}
|
|
14
|
+
DEFAULT_METHOD = {"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"}
|
|
15
|
+
|
|
9
16
|
|
|
10
17
|
def _get_unique_index_name() -> str:
|
|
11
18
|
"""
|
|
@@ -17,178 +24,75 @@ def _get_unique_index_name() -> str:
|
|
|
17
24
|
|
|
18
25
|
|
|
19
26
|
@pytest.fixture
|
|
20
|
-
def
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
44
|
-
asyncio.run(store._async_client.close())
|
|
27
|
+
def opensearch_store():
|
|
28
|
+
created: list[OpenSearchDocumentStore] = []
|
|
29
|
+
|
|
30
|
+
def _make(**overrides) -> OpenSearchDocumentStore:
|
|
31
|
+
kwargs = {
|
|
32
|
+
**COMMON_KWARGS,
|
|
33
|
+
"index": _get_unique_index_name(),
|
|
34
|
+
"embedding_dim": 768,
|
|
35
|
+
"return_embedding": True,
|
|
36
|
+
"method": DEFAULT_METHOD,
|
|
37
|
+
**overrides,
|
|
38
|
+
}
|
|
39
|
+
store = OpenSearchDocumentStore(**kwargs)
|
|
40
|
+
store._ensure_initialized()
|
|
41
|
+
created.append(store)
|
|
42
|
+
return store
|
|
43
|
+
|
|
44
|
+
yield _make
|
|
45
|
+
|
|
46
|
+
for store in created:
|
|
47
|
+
asyncio.run(store._ensure_initialized_async())
|
|
48
|
+
store._client.indices.delete(index=store._index, params={"ignore": [400, 404]})
|
|
49
|
+
asyncio.run(store._async_client.close())
|
|
45
50
|
|
|
46
51
|
|
|
47
52
|
@pytest.fixture
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
index = f"test_index_2_{_get_unique_index_name()}"
|
|
53
|
+
def document_store(opensearch_store):
|
|
54
|
+
return opensearch_store()
|
|
51
55
|
|
|
52
|
-
store = OpenSearchDocumentStore(
|
|
53
|
-
hosts=hosts,
|
|
54
|
-
index=index,
|
|
55
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
56
|
-
verify_certs=False,
|
|
57
|
-
embedding_dim=768,
|
|
58
|
-
return_embedding=False,
|
|
59
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
60
|
-
)
|
|
61
|
-
yield store
|
|
62
56
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
assert store._client
|
|
67
|
-
assert store._async_client
|
|
68
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
69
|
-
asyncio.run(store._async_client.close())
|
|
57
|
+
@pytest.fixture
|
|
58
|
+
def document_store_2(opensearch_store):
|
|
59
|
+
return opensearch_store(return_embedding=False)
|
|
70
60
|
|
|
71
61
|
|
|
72
62
|
@pytest.fixture
|
|
73
|
-
def document_store_readonly():
|
|
74
|
-
|
|
75
|
-
A document store that does not automatically create the underlying index.
|
|
76
|
-
"""
|
|
77
|
-
hosts = ["https://localhost:9200"]
|
|
78
|
-
# Use a different index for each test so we can run them in parallel
|
|
79
|
-
index = _get_unique_index_name()
|
|
80
|
-
|
|
81
|
-
store = OpenSearchDocumentStore(
|
|
82
|
-
hosts=hosts,
|
|
83
|
-
index=index,
|
|
84
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
85
|
-
verify_certs=False,
|
|
86
|
-
embedding_dim=768,
|
|
87
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
88
|
-
create_index=False,
|
|
89
|
-
)
|
|
90
|
-
store._ensure_initialized()
|
|
91
|
-
asyncio.run(store._ensure_initialized_async())
|
|
92
|
-
assert store._client
|
|
93
|
-
assert store._async_client
|
|
63
|
+
def document_store_readonly(opensearch_store):
|
|
64
|
+
store = opensearch_store(create_index=False)
|
|
94
65
|
store._client.cluster.put_settings(body={"transient": {"action.auto_create_index": False}})
|
|
95
66
|
yield store
|
|
96
|
-
|
|
97
67
|
store._client.cluster.put_settings(body={"transient": {"action.auto_create_index": True}})
|
|
98
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
99
68
|
|
|
100
69
|
|
|
101
70
|
@pytest.fixture
|
|
102
|
-
def document_store_embedding_dim_4_no_emb_returned():
|
|
103
|
-
|
|
104
|
-
A document store with embedding dimension 4 that does not return embeddings.
|
|
105
|
-
"""
|
|
106
|
-
hosts = ["https://localhost:9200"]
|
|
107
|
-
index = _get_unique_index_name()
|
|
108
|
-
|
|
109
|
-
store = OpenSearchDocumentStore(
|
|
110
|
-
hosts=hosts,
|
|
111
|
-
index=index,
|
|
112
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
113
|
-
verify_certs=False,
|
|
114
|
-
embedding_dim=4,
|
|
115
|
-
return_embedding=False,
|
|
116
|
-
method={"space_type": "cosinesimil", "engine": "lucene", "name": "hnsw"},
|
|
117
|
-
)
|
|
118
|
-
yield store
|
|
119
|
-
|
|
120
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
71
|
+
def document_store_embedding_dim_4_no_emb_returned(opensearch_store):
|
|
72
|
+
return opensearch_store(embedding_dim=4, return_embedding=False)
|
|
121
73
|
|
|
122
74
|
|
|
123
75
|
@pytest.fixture
|
|
124
|
-
def document_store_embedding_dim_4_no_emb_returned_faiss():
|
|
76
|
+
def document_store_embedding_dim_4_no_emb_returned_faiss(opensearch_store):
|
|
125
77
|
"""
|
|
126
78
|
A document store with embedding dimension 4 that uses a FAISS engine with HNSW algorithm for vector search.
|
|
127
79
|
We use this document store for testing efficient k-NN filtering according to
|
|
128
80
|
https://opensearch.org/docs/latest/vector-search/filter-search-knn/efficient-knn-filtering/.
|
|
129
81
|
"""
|
|
130
|
-
|
|
131
|
-
index = _get_unique_index_name()
|
|
132
|
-
|
|
133
|
-
store = OpenSearchDocumentStore(
|
|
134
|
-
hosts=hosts,
|
|
135
|
-
index=index,
|
|
136
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
137
|
-
verify_certs=False,
|
|
82
|
+
return opensearch_store(
|
|
138
83
|
embedding_dim=4,
|
|
139
84
|
method={"space_type": "innerproduct", "engine": "faiss", "name": "hnsw"},
|
|
140
85
|
)
|
|
141
|
-
yield store
|
|
142
|
-
|
|
143
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
144
86
|
|
|
145
87
|
|
|
146
88
|
@pytest.fixture
|
|
147
|
-
def document_store_nested():
|
|
148
|
-
"""
|
|
149
|
-
OpenSearch document store with explicit nested fields.
|
|
150
|
-
"""
|
|
151
|
-
hosts = ["https://localhost:9200"]
|
|
152
|
-
index = _get_unique_index_name()
|
|
153
|
-
|
|
154
|
-
store = OpenSearchDocumentStore(
|
|
155
|
-
hosts=hosts,
|
|
156
|
-
index=index,
|
|
157
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
158
|
-
verify_certs=False,
|
|
159
|
-
embedding_dim=768,
|
|
160
|
-
return_embedding=False,
|
|
161
|
-
nested_fields=["refs", "tags"],
|
|
162
|
-
)
|
|
163
|
-
store._ensure_initialized()
|
|
164
|
-
yield store
|
|
165
|
-
|
|
166
|
-
assert store._client
|
|
167
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
89
|
+
def document_store_nested(opensearch_store):
|
|
90
|
+
return opensearch_store(return_embedding=False, nested_fields=["refs", "tags"])
|
|
168
91
|
|
|
169
92
|
|
|
170
93
|
@pytest.fixture
|
|
171
|
-
def document_store_wildcard_nested():
|
|
172
|
-
""
|
|
173
|
-
OpenSearch document store with wildcard nested field auto-detection.
|
|
174
|
-
"""
|
|
175
|
-
hosts = ["https://localhost:9200"]
|
|
176
|
-
index = _get_unique_index_name()
|
|
177
|
-
|
|
178
|
-
store = OpenSearchDocumentStore(
|
|
179
|
-
hosts=hosts,
|
|
180
|
-
index=index,
|
|
181
|
-
http_auth=("admin", "SecureHaystack!2026"),
|
|
182
|
-
verify_certs=False,
|
|
183
|
-
embedding_dim=768,
|
|
184
|
-
return_embedding=False,
|
|
185
|
-
nested_fields="*",
|
|
186
|
-
)
|
|
187
|
-
store._ensure_initialized()
|
|
188
|
-
yield store
|
|
189
|
-
|
|
190
|
-
assert store._client
|
|
191
|
-
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
94
|
+
def document_store_wildcard_nested(opensearch_store):
|
|
95
|
+
return opensearch_store(return_embedding=False, nested_fields="*")
|
|
192
96
|
|
|
193
97
|
|
|
194
98
|
@pytest.fixture
|
|
@@ -5,10 +5,16 @@
|
|
|
5
5
|
from unittest.mock import Mock, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
|
+
from botocore.exceptions import BotoCoreError
|
|
8
9
|
from haystack.utils.auth import Secret
|
|
9
10
|
from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
|
|
10
11
|
|
|
11
|
-
from haystack_integrations.document_stores.opensearch.auth import
|
|
12
|
+
from haystack_integrations.document_stores.opensearch.auth import (
|
|
13
|
+
AsyncAWSAuth,
|
|
14
|
+
AWSAuth,
|
|
15
|
+
AWSConfigurationError,
|
|
16
|
+
_get_aws_session,
|
|
17
|
+
)
|
|
12
18
|
from haystack_integrations.document_stores.opensearch.document_store import (
|
|
13
19
|
DEFAULT_MAX_CHUNK_BYTES,
|
|
14
20
|
OpenSearchDocumentStore,
|
|
@@ -131,6 +137,17 @@ class TestAWSAuth:
|
|
|
131
137
|
async_aws_auth(method="GET", url="http://some.url", body="some body", headers={"Host": "localhost"})
|
|
132
138
|
signer_auth_mock.assert_called_once_with("GET", "http://some.url", "some body", {"Host": "localhost"})
|
|
133
139
|
|
|
140
|
+
def test_get_aws_session_wraps_boto_core_error(self, mock_boto3_session):
|
|
141
|
+
mock_boto3_session.side_effect = BotoCoreError()
|
|
142
|
+
with pytest.raises(AWSConfigurationError, match="Failed to initialize the session"):
|
|
143
|
+
_get_aws_session(aws_access_key_id="x", aws_secret_access_key="y")
|
|
144
|
+
|
|
145
|
+
@patch("haystack_integrations.document_stores.opensearch.auth.Urllib3AWSV4SignerAuth")
|
|
146
|
+
def test_get_aws_v4_signer_auth_wraps_exceptions(self, mock_signer):
|
|
147
|
+
mock_signer.side_effect = RuntimeError("signer creation failed")
|
|
148
|
+
with pytest.raises(AWSConfigurationError, match="Could not connect to AWS OpenSearch"):
|
|
149
|
+
AWSAuth()
|
|
150
|
+
|
|
134
151
|
def test_async_aws_auth_init(self):
|
|
135
152
|
data = {
|
|
136
153
|
"type": "haystack_integrations.document_stores.opensearch.auth.AWSAuth",
|
|
@@ -209,6 +226,7 @@ class TestDocumentStoreWithAuth:
|
|
|
209
226
|
|
|
210
227
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
211
228
|
def test_ds_init_with_basic_auth(self, _mock_opensearch_client):
|
|
229
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
212
230
|
document_store = OpenSearchDocumentStore(hosts="testhost", http_auth=("user", "pw"))
|
|
213
231
|
document_store._ensure_initialized()
|
|
214
232
|
assert document_store._client
|
|
@@ -217,6 +235,7 @@ class TestDocumentStoreWithAuth:
|
|
|
217
235
|
|
|
218
236
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
219
237
|
def test_ds_init_without_auth(self, _mock_opensearch_client):
|
|
238
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
220
239
|
document_store = OpenSearchDocumentStore(hosts="testhost")
|
|
221
240
|
document_store._ensure_initialized()
|
|
222
241
|
assert document_store._client
|
|
@@ -225,6 +244,7 @@ class TestDocumentStoreWithAuth:
|
|
|
225
244
|
|
|
226
245
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
227
246
|
def test_ds_init_aws_auth(self, _mock_opensearch_client):
|
|
247
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
228
248
|
document_store = OpenSearchDocumentStore(
|
|
229
249
|
hosts="testhost",
|
|
230
250
|
http_auth=AWSAuth(aws_region_name=Secret.from_token("dummy-region")),
|
|
@@ -240,6 +260,7 @@ class TestDocumentStoreWithAuth:
|
|
|
240
260
|
|
|
241
261
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
242
262
|
def test_ds_from_dict_basic_auth(self, _mock_opensearch_client):
|
|
263
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
243
264
|
document_store = OpenSearchDocumentStore.from_dict(
|
|
244
265
|
{
|
|
245
266
|
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
@@ -258,6 +279,7 @@ class TestDocumentStoreWithAuth:
|
|
|
258
279
|
|
|
259
280
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
260
281
|
def test_ds_from_dict_aws_auth(self, _mock_opensearch_client, monkeypatch: pytest.MonkeyPatch):
|
|
282
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
261
283
|
monkeypatch.setenv("AWS_DEFAULT_REGION", "dummy-region")
|
|
262
284
|
document_store = OpenSearchDocumentStore.from_dict(
|
|
263
285
|
{
|
|
@@ -362,6 +384,7 @@ class TestDocumentStoreWithAuth:
|
|
|
362
384
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
363
385
|
def test_ds_init_with_env_var_secrets(self, _mock_opensearch_client, monkeypatch):
|
|
364
386
|
"""Test the default initialization using environment variables"""
|
|
387
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
365
388
|
monkeypatch.setenv("OPENSEARCH_USERNAME", "user")
|
|
366
389
|
monkeypatch.setenv("OPENSEARCH_PASSWORD", "pass")
|
|
367
390
|
|
|
@@ -374,6 +397,7 @@ class TestDocumentStoreWithAuth:
|
|
|
374
397
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
375
398
|
def test_ds_init_with_missing_env_vars(self, _mock_opensearch_client):
|
|
376
399
|
"""Test that auth is None when environment variables are missing"""
|
|
400
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
377
401
|
document_store = OpenSearchDocumentStore(hosts="testhost")
|
|
378
402
|
document_store._ensure_initialized()
|
|
379
403
|
assert document_store._client
|
|
@@ -402,6 +426,7 @@ class TestDocumentStoreWithAuth:
|
|
|
402
426
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
403
427
|
def test_ds_from_dict_with_env_var_secrets(self, _mock_opensearch_client, monkeypatch):
|
|
404
428
|
"""Test deserialization with environment variables"""
|
|
429
|
+
_mock_opensearch_client.return_value.indices.exists.return_value = False
|
|
405
430
|
# Set environment variables so the secrets resolve properly
|
|
406
431
|
monkeypatch.setenv("OPENSEARCH_USERNAME", "user")
|
|
407
432
|
monkeypatch.setenv("OPENSEARCH_PASSWORD", "pass")
|