opensearch-haystack 8.1.2__tar.gz → 8.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/CHANGELOG.md +7 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/PKG-INFO +1 -1
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +12 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +12 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +12 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +15 -2
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +12 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +19 -4
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_bm25_retriever.py +23 -1
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_document_store.py +35 -3
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_document_store_async.py +40 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_embedding_retriever.py +23 -1
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_metadata_retriever.py +23 -1
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_open_search_hybrid_retriever.py +102 -92
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_sql_retriever.py +22 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/.gitignore +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/LICENSE +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/README.md +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/docker-compose.yml +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/pyproject.toml +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/document_stores/opensearch/filters.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/__init__.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/conftest.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_auth.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_document_store_common.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_filters.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version: 8.
|
|
3
|
+
Version: 8.2.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -157,6 +157,18 @@ class OpenSearchBM25Retriever:
|
|
|
157
157
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
|
|
158
158
|
return default_from_dict(cls, data)
|
|
159
159
|
|
|
160
|
+
def close(self) -> None:
|
|
161
|
+
"""
|
|
162
|
+
Release the synchronous resources of the underlying Document Store.
|
|
163
|
+
"""
|
|
164
|
+
self._document_store.close()
|
|
165
|
+
|
|
166
|
+
async def close_async(self) -> None:
|
|
167
|
+
"""
|
|
168
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
169
|
+
"""
|
|
170
|
+
await self._document_store.close_async()
|
|
171
|
+
|
|
160
172
|
def _prepare_bm25_args(
|
|
161
173
|
self,
|
|
162
174
|
*,
|
|
@@ -163,6 +163,18 @@ class OpenSearchEmbeddingRetriever:
|
|
|
163
163
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
|
|
164
164
|
return default_from_dict(cls, data)
|
|
165
165
|
|
|
166
|
+
def close(self) -> None:
|
|
167
|
+
"""
|
|
168
|
+
Release the synchronous resources of the underlying Document Store.
|
|
169
|
+
"""
|
|
170
|
+
self._document_store.close()
|
|
171
|
+
|
|
172
|
+
async def close_async(self) -> None:
|
|
173
|
+
"""
|
|
174
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
175
|
+
"""
|
|
176
|
+
await self._document_store.close_async()
|
|
177
|
+
|
|
166
178
|
@component.output_types(documents=list[Document])
|
|
167
179
|
def run(
|
|
168
180
|
self,
|
|
@@ -181,6 +181,18 @@ class OpenSearchMetadataRetriever:
|
|
|
181
181
|
)
|
|
182
182
|
return default_from_dict(cls, data)
|
|
183
183
|
|
|
184
|
+
def close(self) -> None:
|
|
185
|
+
"""
|
|
186
|
+
Release the synchronous resources of the underlying Document Store.
|
|
187
|
+
"""
|
|
188
|
+
self._document_store.close()
|
|
189
|
+
|
|
190
|
+
async def close_async(self) -> None:
|
|
191
|
+
"""
|
|
192
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
193
|
+
"""
|
|
194
|
+
await self._document_store.close_async()
|
|
195
|
+
|
|
184
196
|
@component.output_types(metadata=list[dict[str, Any]])
|
|
185
197
|
def run(
|
|
186
198
|
self,
|
|
@@ -37,7 +37,9 @@ class OpenSearchHybridRetriever:
|
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
39
|
from haystack import Document
|
|
40
|
-
|
|
40
|
+
# Requires: pip install sentence-transformers-haystack
|
|
41
|
+
from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder
|
|
42
|
+
from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder
|
|
41
43
|
from haystack_integrations.components.retrievers.opensearch import OpenSearchHybridRetriever
|
|
42
44
|
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
43
45
|
|
|
@@ -59,7 +61,6 @@ class OpenSearchHybridRetriever:
|
|
|
59
61
|
|
|
60
62
|
# Embed the documents and add them to the document store
|
|
61
63
|
doc_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
|
|
62
|
-
doc_embedder.warm_up()
|
|
63
64
|
docs = doc_embedder.run(docs)
|
|
64
65
|
doc_store.write_documents(docs['documents'])
|
|
65
66
|
|
|
@@ -356,3 +357,15 @@ class OpenSearchHybridRetriever:
|
|
|
356
357
|
data["init_parameters"]["join_mode"] = join_mode
|
|
357
358
|
|
|
358
359
|
return default_from_dict(cls, data)
|
|
360
|
+
|
|
361
|
+
def close(self) -> None:
|
|
362
|
+
"""
|
|
363
|
+
Release the synchronous resources of the underlying Document Store.
|
|
364
|
+
"""
|
|
365
|
+
self.document_store.close()
|
|
366
|
+
|
|
367
|
+
async def close_async(self) -> None:
|
|
368
|
+
"""
|
|
369
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
370
|
+
"""
|
|
371
|
+
await self.document_store.close_async()
|
|
@@ -80,6 +80,18 @@ class OpenSearchSQLRetriever:
|
|
|
80
80
|
)
|
|
81
81
|
return default_from_dict(cls, data)
|
|
82
82
|
|
|
83
|
+
def close(self) -> None:
|
|
84
|
+
"""
|
|
85
|
+
Release the synchronous resources of the underlying Document Store.
|
|
86
|
+
"""
|
|
87
|
+
self._document_store.close()
|
|
88
|
+
|
|
89
|
+
async def close_async(self) -> None:
|
|
90
|
+
"""
|
|
91
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
92
|
+
"""
|
|
93
|
+
await self._document_store.close_async()
|
|
94
|
+
|
|
83
95
|
@component.output_types(result=dict[str, Any])
|
|
84
96
|
def run(
|
|
85
97
|
self,
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
# ruff: noqa: FBT001, FBT002 boolean-type-hint-positional-argument and boolean-default-value-positional-argument
|
|
6
6
|
|
|
7
7
|
from collections.abc import Generator, Mapping
|
|
8
|
+
from contextlib import suppress
|
|
8
9
|
from dataclasses import replace
|
|
9
10
|
from math import exp
|
|
10
11
|
from typing import Any, Literal
|
|
@@ -162,7 +163,6 @@ class OpenSearchDocumentStore:
|
|
|
162
163
|
# the document store is instantiated.
|
|
163
164
|
self._client: OpenSearch | None = None
|
|
164
165
|
self._async_client: AsyncOpenSearch | None = None
|
|
165
|
-
self._initialized = False
|
|
166
166
|
|
|
167
167
|
def _get_default_mappings(self) -> dict[str, Any]:
|
|
168
168
|
default_mappings: dict[str, Any] = {
|
|
@@ -303,8 +303,6 @@ class OpenSearchDocumentStore:
|
|
|
303
303
|
timeout=self._timeout,
|
|
304
304
|
**self._kwargs,
|
|
305
305
|
)
|
|
306
|
-
self._initialized = True
|
|
307
|
-
|
|
308
306
|
self._ensure_index_exists()
|
|
309
307
|
|
|
310
308
|
async def _ensure_initialized_async(self) -> None:
|
|
@@ -322,9 +320,26 @@ class OpenSearchDocumentStore:
|
|
|
322
320
|
connection_class=AsyncHttpConnection,
|
|
323
321
|
**self._kwargs,
|
|
324
322
|
)
|
|
325
|
-
self._initialized = True
|
|
326
323
|
await self._ensure_index_exists_async()
|
|
327
324
|
|
|
325
|
+
def close(self) -> None:
|
|
326
|
+
"""
|
|
327
|
+
Release the associated synchronous resources.
|
|
328
|
+
"""
|
|
329
|
+
if self._client is not None:
|
|
330
|
+
with suppress(Exception):
|
|
331
|
+
self._client.close()
|
|
332
|
+
self._client = None
|
|
333
|
+
|
|
334
|
+
async def close_async(self) -> None:
|
|
335
|
+
"""
|
|
336
|
+
Release the associated asynchronous resources.
|
|
337
|
+
"""
|
|
338
|
+
if self._async_client is not None:
|
|
339
|
+
with suppress(Exception):
|
|
340
|
+
await self._async_client.close()
|
|
341
|
+
self._async_client = None
|
|
342
|
+
|
|
328
343
|
@staticmethod
|
|
329
344
|
def _extract_nested_fields_from_mapping(mapping_properties: dict[str, Any]) -> set[str]:
|
|
330
345
|
return {name for name, defn in mapping_properties.items() if defn.get("type") == "nested"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import Mock, patch
|
|
5
|
+
from unittest.mock import AsyncMock, Mock, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
from haystack.dataclasses import Document
|
|
@@ -159,6 +159,28 @@ def test_from_dict_not_defaults(_mock_opensearch_client):
|
|
|
159
159
|
assert retriever._raise_on_failure is True
|
|
160
160
|
|
|
161
161
|
|
|
162
|
+
def test_close():
|
|
163
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
164
|
+
retriever = OpenSearchBM25Retriever(document_store=mock_store)
|
|
165
|
+
|
|
166
|
+
retriever.close()
|
|
167
|
+
|
|
168
|
+
mock_store.close.assert_called_once_with()
|
|
169
|
+
assert retriever._document_store is mock_store
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@pytest.mark.asyncio
|
|
173
|
+
async def test_close_async():
|
|
174
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
175
|
+
mock_store.close_async = AsyncMock()
|
|
176
|
+
retriever = OpenSearchBM25Retriever(document_store=mock_store)
|
|
177
|
+
|
|
178
|
+
await retriever.close_async()
|
|
179
|
+
|
|
180
|
+
mock_store.close_async.assert_awaited_once_with()
|
|
181
|
+
assert retriever._document_store is mock_store
|
|
182
|
+
|
|
183
|
+
|
|
162
184
|
def test_run():
|
|
163
185
|
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
164
186
|
mock_store._bm25_retrieval.return_value = [Document(content="Test doc")]
|
|
@@ -497,7 +497,6 @@ def test_wildcard_nested_fields_detected_on_write(mock_bulk, _mock_opensearch_cl
|
|
|
497
497
|
mock_bulk.return_value = (2, [])
|
|
498
498
|
store = OpenSearchDocumentStore(hosts="testhost", nested_fields="*")
|
|
499
499
|
store._client = MagicMock()
|
|
500
|
-
store._initialized = True
|
|
501
500
|
|
|
502
501
|
docs = [
|
|
503
502
|
Document(
|
|
@@ -526,7 +525,6 @@ def test_wildcard_nested_fields_incremental_detection(mock_bulk, _mock_opensearc
|
|
|
526
525
|
mock_bulk.return_value = (1, [])
|
|
527
526
|
store = OpenSearchDocumentStore(hosts="testhost", nested_fields="*")
|
|
528
527
|
store._client = MagicMock()
|
|
529
|
-
store._initialized = True
|
|
530
528
|
|
|
531
529
|
# First batch: detects "refs"
|
|
532
530
|
store.write_documents([Document(content="d1", meta={"refs": [{"law": "bgb"}]})])
|
|
@@ -545,7 +543,6 @@ def test_explicit_nested_fields_no_detection_on_write(mock_bulk, _mock_opensearc
|
|
|
545
543
|
mock_bulk.return_value = (1, [])
|
|
546
544
|
store = OpenSearchDocumentStore(hosts="testhost", nested_fields=["refs"])
|
|
547
545
|
store._client = MagicMock()
|
|
548
|
-
store._initialized = True
|
|
549
546
|
|
|
550
547
|
store.write_documents([Document(content="d1", meta={"tags": [{"name": "x"}]})])
|
|
551
548
|
|
|
@@ -755,6 +752,31 @@ async def test_delete_all_documents_async_recreate_works_for_concrete_index(_moc
|
|
|
755
752
|
mock_client.indices.create.assert_called_once()
|
|
756
753
|
|
|
757
754
|
|
|
755
|
+
def test_close():
|
|
756
|
+
store = OpenSearchDocumentStore(hosts="testhost", http_auth=("a", "b"))
|
|
757
|
+
mock_client = MagicMock()
|
|
758
|
+
store._client = mock_client
|
|
759
|
+
|
|
760
|
+
store.close()
|
|
761
|
+
|
|
762
|
+
mock_client.close.assert_called_once()
|
|
763
|
+
assert store._client is None
|
|
764
|
+
|
|
765
|
+
store.close()
|
|
766
|
+
mock_client.close.assert_called_once()
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
def test_close_is_exception_safe():
|
|
770
|
+
store = OpenSearchDocumentStore(hosts="testhost", http_auth=("a", "b"))
|
|
771
|
+
mock_client = MagicMock()
|
|
772
|
+
mock_client.close.side_effect = RuntimeError("boom")
|
|
773
|
+
store._client = mock_client
|
|
774
|
+
|
|
775
|
+
store.close()
|
|
776
|
+
|
|
777
|
+
assert store._client is None
|
|
778
|
+
|
|
779
|
+
|
|
758
780
|
@pytest.mark.integration
|
|
759
781
|
class TestDocumentStore(
|
|
760
782
|
OpenSearchDocumentStoreTestMixin,
|
|
@@ -774,6 +796,16 @@ class TestDocumentStore(
|
|
|
774
796
|
"""Override base class fixture to provide OpenSearch document store."""
|
|
775
797
|
yield document_store
|
|
776
798
|
|
|
799
|
+
def test_close_and_reopen(self, document_store: OpenSearchDocumentStore):
|
|
800
|
+
assert document_store.count_documents() == 0
|
|
801
|
+
assert document_store._client is not None
|
|
802
|
+
|
|
803
|
+
document_store.close()
|
|
804
|
+
assert document_store._client is None
|
|
805
|
+
|
|
806
|
+
assert document_store.count_documents() == 0
|
|
807
|
+
assert document_store._client is not None
|
|
808
|
+
|
|
777
809
|
def test_write_documents(self, document_store: OpenSearchDocumentStore):
|
|
778
810
|
docs = [Document(id="1")]
|
|
779
811
|
assert document_store.write_documents(docs) == 1
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
|
|
6
|
+
from unittest.mock import AsyncMock
|
|
7
|
+
|
|
6
8
|
import pytest
|
|
7
9
|
from haystack.dataclasses import Document
|
|
8
10
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
@@ -26,6 +28,33 @@ from haystack_integrations.document_stores.opensearch.document_store import Open
|
|
|
26
28
|
from tests.test_document_store_common import OpenSearchDocumentStoreTestMixin
|
|
27
29
|
|
|
28
30
|
|
|
31
|
+
@pytest.mark.asyncio
|
|
32
|
+
async def test_close_async():
|
|
33
|
+
store = OpenSearchDocumentStore(hosts="testhost", http_auth=("a", "b"))
|
|
34
|
+
mock_client = AsyncMock()
|
|
35
|
+
store._async_client = mock_client
|
|
36
|
+
|
|
37
|
+
await store.close_async()
|
|
38
|
+
|
|
39
|
+
mock_client.close.assert_awaited_once()
|
|
40
|
+
assert store._async_client is None
|
|
41
|
+
|
|
42
|
+
await store.close_async()
|
|
43
|
+
mock_client.close.assert_awaited_once()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@pytest.mark.asyncio
|
|
47
|
+
async def test_close_async_is_exception_safe():
|
|
48
|
+
store = OpenSearchDocumentStore(hosts="testhost", http_auth=("a", "b"))
|
|
49
|
+
mock_client = AsyncMock()
|
|
50
|
+
mock_client.close.side_effect = RuntimeError("boom")
|
|
51
|
+
store._async_client = mock_client
|
|
52
|
+
|
|
53
|
+
await store.close_async()
|
|
54
|
+
|
|
55
|
+
assert store._async_client is None
|
|
56
|
+
|
|
57
|
+
|
|
29
58
|
@pytest.mark.integration
|
|
30
59
|
class TestDocumentStoreAsync(
|
|
31
60
|
OpenSearchDocumentStoreTestMixin,
|
|
@@ -42,6 +71,17 @@ class TestDocumentStoreAsync(
|
|
|
42
71
|
GetMetadataFieldMinMaxAsyncTest,
|
|
43
72
|
GetMetadataFieldUniqueValuesAsyncTest,
|
|
44
73
|
):
|
|
74
|
+
@pytest.mark.asyncio
|
|
75
|
+
async def test_close_async_and_reopen(self, document_store: OpenSearchDocumentStore):
|
|
76
|
+
assert await document_store.count_documents_async() == 0
|
|
77
|
+
assert document_store._async_client is not None
|
|
78
|
+
|
|
79
|
+
await document_store.close_async()
|
|
80
|
+
assert document_store._async_client is None
|
|
81
|
+
|
|
82
|
+
assert await document_store.count_documents_async() == 0
|
|
83
|
+
assert document_store._async_client is not None
|
|
84
|
+
|
|
45
85
|
@pytest.mark.asyncio
|
|
46
86
|
async def test_write_documents_async(self, document_store: OpenSearchDocumentStore):
|
|
47
87
|
docs = [Document(id="1")]
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import Mock, patch
|
|
5
|
+
from unittest.mock import AsyncMock, Mock, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
from haystack.dataclasses import Document
|
|
@@ -139,6 +139,28 @@ def test_from_dict(_mock_opensearch_client):
|
|
|
139
139
|
assert retriever._filter_policy == FilterPolicy.REPLACE
|
|
140
140
|
|
|
141
141
|
|
|
142
|
+
def test_close():
|
|
143
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
144
|
+
retriever = OpenSearchEmbeddingRetriever(document_store=mock_store)
|
|
145
|
+
|
|
146
|
+
retriever.close()
|
|
147
|
+
|
|
148
|
+
mock_store.close.assert_called_once_with()
|
|
149
|
+
assert retriever._document_store is mock_store
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@pytest.mark.asyncio
|
|
153
|
+
async def test_close_async():
|
|
154
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
155
|
+
mock_store.close_async = AsyncMock()
|
|
156
|
+
retriever = OpenSearchEmbeddingRetriever(document_store=mock_store)
|
|
157
|
+
|
|
158
|
+
await retriever.close_async()
|
|
159
|
+
|
|
160
|
+
mock_store.close_async.assert_awaited_once_with()
|
|
161
|
+
assert retriever._document_store is mock_store
|
|
162
|
+
|
|
163
|
+
|
|
142
164
|
def test_run():
|
|
143
165
|
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
144
166
|
mock_store._embedding_retrieval.return_value = [Document(content="Test doc", embedding=[0.1, 0.2])]
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import Mock, patch
|
|
5
|
+
from unittest.mock import AsyncMock, Mock, patch
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
from haystack.dataclasses import Document
|
|
@@ -119,6 +119,28 @@ def test_from_dict(_mock_opensearch_client):
|
|
|
119
119
|
assert retriever_from_dict._jaccard_n == 5
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
def test_close():
|
|
123
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
124
|
+
retriever = OpenSearchMetadataRetriever(document_store=mock_store, metadata_fields=["category"])
|
|
125
|
+
|
|
126
|
+
retriever.close()
|
|
127
|
+
|
|
128
|
+
mock_store.close.assert_called_once_with()
|
|
129
|
+
assert retriever._document_store is mock_store
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@pytest.mark.asyncio
|
|
133
|
+
async def test_close_async():
|
|
134
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
135
|
+
mock_store.close_async = AsyncMock()
|
|
136
|
+
retriever = OpenSearchMetadataRetriever(document_store=mock_store, metadata_fields=["category"])
|
|
137
|
+
|
|
138
|
+
await retriever.close_async()
|
|
139
|
+
|
|
140
|
+
mock_store.close_async.assert_awaited_once_with()
|
|
141
|
+
assert retriever._document_store is mock_store
|
|
142
|
+
|
|
143
|
+
|
|
122
144
|
def test_run_with_runtime_document_store():
|
|
123
145
|
mock_store1 = Mock(spec=OpenSearchDocumentStore)
|
|
124
146
|
mock_store2 = Mock(spec=OpenSearchDocumentStore)
|
{opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_open_search_hybrid_retriever.py
RENAMED
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
|
|
5
5
|
from copy import deepcopy
|
|
6
6
|
from typing import Any
|
|
7
|
-
from unittest.mock import Mock
|
|
7
|
+
from unittest.mock import AsyncMock, Mock
|
|
8
8
|
|
|
9
9
|
import pytest
|
|
10
10
|
from haystack import Document, Pipeline
|
|
11
|
-
from haystack.components.embedders import
|
|
11
|
+
from haystack.components.embedders import OpenAITextEmbedder
|
|
12
12
|
from haystack.core.component import component
|
|
13
13
|
|
|
14
14
|
from haystack_integrations.components.retrievers.opensearch import OpenSearchHybridRetriever
|
|
@@ -23,133 +23,143 @@ class MockedTextEmbedder:
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
class TestOpenSearchHybridRetriever:
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
"
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
26
|
+
@pytest.fixture(autouse=True)
|
|
27
|
+
def openai_api_key(self, monkeypatch):
|
|
28
|
+
# the serde tests build a real OpenAITextEmbedder; haystack-ai 2.x resolves the key at init
|
|
29
|
+
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
|
|
30
|
+
|
|
31
|
+
@pytest.fixture
|
|
32
|
+
def serialised(self):
|
|
33
|
+
return {
|
|
34
|
+
"type": "haystack_integrations.components.retrievers.opensearch.open_search_hybrid_retriever.OpenSearchHybridRetriever", # noqa: E501
|
|
35
|
+
"init_parameters": {
|
|
36
|
+
"document_store": {
|
|
37
|
+
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
38
|
+
"init_parameters": {
|
|
39
|
+
"hosts": None,
|
|
40
|
+
"index": "default",
|
|
41
|
+
"max_chunk_bytes": 104857600,
|
|
42
|
+
"embedding_dim": 768,
|
|
43
|
+
"method": None,
|
|
44
|
+
"mappings": {
|
|
45
|
+
"properties": {
|
|
46
|
+
"embedding": {"type": "knn_vector", "index": True, "dimension": 768},
|
|
47
|
+
"content": {"type": "text"},
|
|
48
|
+
},
|
|
49
|
+
"dynamic_templates": [
|
|
50
|
+
{"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}
|
|
51
|
+
],
|
|
41
52
|
},
|
|
42
|
-
"
|
|
43
|
-
|
|
53
|
+
"settings": {"index.knn": True},
|
|
54
|
+
"create_index": True,
|
|
55
|
+
"return_embedding": False,
|
|
56
|
+
"http_auth": [
|
|
57
|
+
{"type": "env_var", "env_vars": ["OPENSEARCH_USERNAME"], "strict": False},
|
|
58
|
+
{"type": "env_var", "env_vars": ["OPENSEARCH_PASSWORD"], "strict": False},
|
|
44
59
|
],
|
|
60
|
+
"use_ssl": None,
|
|
61
|
+
"verify_certs": None,
|
|
62
|
+
"timeout": None,
|
|
63
|
+
"nested_fields": None,
|
|
45
64
|
},
|
|
46
|
-
"settings": {"index.knn": True},
|
|
47
|
-
"create_index": True,
|
|
48
|
-
"return_embedding": False,
|
|
49
|
-
"http_auth": [
|
|
50
|
-
{"type": "env_var", "env_vars": ["OPENSEARCH_USERNAME"], "strict": False},
|
|
51
|
-
{"type": "env_var", "env_vars": ["OPENSEARCH_PASSWORD"], "strict": False},
|
|
52
|
-
],
|
|
53
|
-
"use_ssl": None,
|
|
54
|
-
"verify_certs": None,
|
|
55
|
-
"timeout": None,
|
|
56
|
-
"nested_fields": None,
|
|
57
65
|
},
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
"model_kwargs": None,
|
|
73
|
-
"tokenizer_kwargs": None,
|
|
74
|
-
"config_kwargs": None,
|
|
75
|
-
"precision": "float32",
|
|
76
|
-
"encode_kwargs": None,
|
|
77
|
-
"backend": "torch",
|
|
66
|
+
"embedder": {
|
|
67
|
+
"type": "haystack.components.embedders.openai_text_embedder.OpenAITextEmbedder",
|
|
68
|
+
"init_parameters": {
|
|
69
|
+
"api_key": {"type": "env_var", "env_vars": ["OPENAI_API_KEY"], "strict": True},
|
|
70
|
+
"model": "text-embedding-ada-002",
|
|
71
|
+
"dimensions": None,
|
|
72
|
+
"api_base_url": None,
|
|
73
|
+
"organization": None,
|
|
74
|
+
"prefix": "",
|
|
75
|
+
"suffix": "",
|
|
76
|
+
"timeout": None,
|
|
77
|
+
"max_retries": None,
|
|
78
|
+
"http_client_kwargs": None,
|
|
79
|
+
},
|
|
78
80
|
},
|
|
81
|
+
"filters_bm25": None,
|
|
82
|
+
"fuzziness": 0,
|
|
83
|
+
"top_k_bm25": 10,
|
|
84
|
+
"scale_score": False,
|
|
85
|
+
"all_terms_must_match": False,
|
|
86
|
+
"filter_policy_bm25": "replace",
|
|
87
|
+
"custom_query_bm25": None,
|
|
88
|
+
"filters_embedding": None,
|
|
89
|
+
"top_k_embedding": 10,
|
|
90
|
+
"filter_policy_embedding": "replace",
|
|
91
|
+
"custom_query_embedding": None,
|
|
92
|
+
"join_mode": "reciprocal_rank_fusion",
|
|
93
|
+
"weights": None,
|
|
94
|
+
"top_k": None,
|
|
95
|
+
"sort_by_score": True,
|
|
96
|
+
"search_kwargs_embedding": None,
|
|
79
97
|
},
|
|
80
|
-
|
|
81
|
-
"fuzziness": 0,
|
|
82
|
-
"top_k_bm25": 10,
|
|
83
|
-
"scale_score": False,
|
|
84
|
-
"all_terms_must_match": False,
|
|
85
|
-
"filter_policy_bm25": "replace",
|
|
86
|
-
"custom_query_bm25": None,
|
|
87
|
-
"filters_embedding": None,
|
|
88
|
-
"top_k_embedding": 10,
|
|
89
|
-
"filter_policy_embedding": "replace",
|
|
90
|
-
"custom_query_embedding": None,
|
|
91
|
-
"join_mode": "reciprocal_rank_fusion",
|
|
92
|
-
"weights": None,
|
|
93
|
-
"top_k": None,
|
|
94
|
-
"sort_by_score": True,
|
|
95
|
-
"search_kwargs_embedding": None,
|
|
96
|
-
},
|
|
97
|
-
}
|
|
98
|
+
}
|
|
98
99
|
|
|
99
100
|
@pytest.fixture
|
|
100
101
|
def mock_embedder(self):
|
|
101
102
|
return MockedTextEmbedder()
|
|
102
103
|
|
|
103
|
-
def test_to_dict(self) -> None:
|
|
104
|
+
def test_to_dict(self, serialised) -> None:
|
|
104
105
|
doc_store = OpenSearchDocumentStore()
|
|
105
|
-
embedder =
|
|
106
|
+
embedder = OpenAITextEmbedder() # we use actual embedder here for the de/serialization
|
|
106
107
|
hybrid_retriever = OpenSearchHybridRetriever(document_store=doc_store, embedder=embedder)
|
|
107
108
|
result = hybrid_retriever.to_dict()
|
|
108
|
-
result
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
if "revision" in result["init_parameters"]["embedder"]["init_parameters"]:
|
|
113
|
-
data["init_parameters"]["embedder"]["init_parameters"]["revision"] = None
|
|
114
|
-
assert result == data
|
|
115
|
-
|
|
116
|
-
def test_from_dict(self):
|
|
117
|
-
data = deepcopy(self.serialised)
|
|
109
|
+
assert result == serialised
|
|
110
|
+
|
|
111
|
+
def test_from_dict(self, serialised):
|
|
112
|
+
data = deepcopy(serialised)
|
|
118
113
|
super_component = OpenSearchHybridRetriever.from_dict(data)
|
|
119
114
|
assert isinstance(super_component, OpenSearchHybridRetriever)
|
|
120
115
|
assert super_component.to_dict()
|
|
121
116
|
|
|
122
|
-
def test_to_dict_with_extra_args(self):
|
|
117
|
+
def test_to_dict_with_extra_args(self, serialised):
|
|
123
118
|
doc_store = OpenSearchDocumentStore()
|
|
124
|
-
embedder =
|
|
119
|
+
embedder = OpenAITextEmbedder() # an actual embedder here for the de/serialization
|
|
125
120
|
hybrid_retriever = OpenSearchHybridRetriever(
|
|
126
121
|
document_store=doc_store, embedder=embedder, embedding_retriever={"raise_on_failure": True}
|
|
127
122
|
)
|
|
128
123
|
result = hybrid_retriever.to_dict()
|
|
129
|
-
expected = deepcopy(
|
|
124
|
+
expected = deepcopy(serialised)
|
|
130
125
|
expected["init_parameters"]["embedding_retriever"] = {"raise_on_failure": True}
|
|
131
|
-
# We add revision to the expected dict if it exists in the result for comparison
|
|
132
|
-
# This was added in PR https://github.com/deepset-ai/haystack/pull/10003 and released in Haystack 2.20.0
|
|
133
|
-
if "revision" in result["init_parameters"]["embedder"]["init_parameters"]:
|
|
134
|
-
expected["init_parameters"]["embedder"]["init_parameters"]["revision"] = None
|
|
135
|
-
result["init_parameters"]["embedder"]["init_parameters"].pop("device") # remove device info for comparison
|
|
136
126
|
assert result == expected
|
|
137
127
|
|
|
138
|
-
def test_from_dict_with_extra_args(self):
|
|
139
|
-
data = deepcopy(
|
|
128
|
+
def test_from_dict_with_extra_args(self, serialised):
|
|
129
|
+
data = deepcopy(serialised)
|
|
140
130
|
data["init_parameters"]["embedding_retriever"] = {"raise_on_failure": True}
|
|
141
131
|
hybrid = OpenSearchHybridRetriever.from_dict(data)
|
|
142
132
|
assert isinstance(hybrid, OpenSearchHybridRetriever)
|
|
143
133
|
assert hybrid.to_dict()
|
|
144
134
|
|
|
145
|
-
def test_from_dict_without_optional_keys(self):
|
|
146
|
-
data = deepcopy(
|
|
135
|
+
def test_from_dict_without_optional_keys(self, serialised):
|
|
136
|
+
data = deepcopy(serialised)
|
|
147
137
|
del data["init_parameters"]["filter_policy_bm25"]
|
|
148
138
|
del data["init_parameters"]["filter_policy_embedding"]
|
|
149
139
|
del data["init_parameters"]["join_mode"]
|
|
150
140
|
hybrid = OpenSearchHybridRetriever.from_dict(data)
|
|
151
141
|
assert isinstance(hybrid, OpenSearchHybridRetriever)
|
|
152
142
|
|
|
143
|
+
def test_close(self, mock_embedder):
|
|
144
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
145
|
+
retriever = OpenSearchHybridRetriever(document_store=mock_store, embedder=mock_embedder)
|
|
146
|
+
|
|
147
|
+
retriever.close()
|
|
148
|
+
|
|
149
|
+
mock_store.close.assert_called_once_with()
|
|
150
|
+
assert retriever.document_store is mock_store
|
|
151
|
+
|
|
152
|
+
@pytest.mark.asyncio
|
|
153
|
+
async def test_close_async(self, mock_embedder):
|
|
154
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
155
|
+
mock_store.close_async = AsyncMock()
|
|
156
|
+
retriever = OpenSearchHybridRetriever(document_store=mock_store, embedder=mock_embedder)
|
|
157
|
+
|
|
158
|
+
await retriever.close_async()
|
|
159
|
+
|
|
160
|
+
mock_store.close_async.assert_awaited_once_with()
|
|
161
|
+
assert retriever.document_store is mock_store
|
|
162
|
+
|
|
153
163
|
def test_run(self, mock_embedder):
|
|
154
164
|
# mocked document store
|
|
155
165
|
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
@@ -50,6 +50,28 @@ def test_from_dict(_mock_opensearch_client):
|
|
|
50
50
|
assert retriever_from_dict._raise_on_failure is True
|
|
51
51
|
|
|
52
52
|
|
|
53
|
+
def test_close():
|
|
54
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
55
|
+
retriever = OpenSearchSQLRetriever(document_store=mock_store)
|
|
56
|
+
|
|
57
|
+
retriever.close()
|
|
58
|
+
|
|
59
|
+
mock_store.close.assert_called_once_with()
|
|
60
|
+
assert retriever._document_store is mock_store
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@pytest.mark.asyncio
|
|
64
|
+
async def test_close_async():
|
|
65
|
+
mock_store = Mock(spec=OpenSearchDocumentStore)
|
|
66
|
+
mock_store.close_async = AsyncMock()
|
|
67
|
+
retriever = OpenSearchSQLRetriever(document_store=mock_store)
|
|
68
|
+
|
|
69
|
+
await retriever.close_async()
|
|
70
|
+
|
|
71
|
+
mock_store.close_async.assert_awaited_once_with()
|
|
72
|
+
assert retriever._document_store is mock_store
|
|
73
|
+
|
|
74
|
+
|
|
53
75
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
54
76
|
def test_query_sql_uses_positional_args(_mock_opensearch_class):
|
|
55
77
|
"""Verify perform_request is called with positional method/url args (ddtrace compatibility)."""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{opensearch_haystack-8.1.2 → opensearch_haystack-8.2.0}/tests/test_metadata_retriever_async.py
RENAMED
|
File without changes
|
|
File without changes
|