opensearch-haystack 8.3.0__tar.gz → 9.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/CHANGELOG.md +39 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/PKG-INFO +4 -4
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/pydoc/config_docusaurus.yml +1 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/pyproject.toml +3 -3
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +2 -0
- opensearch_haystack-9.1.0/src/haystack_integrations/components/retrievers/opensearch/multivector_retriever.py +225 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +115 -10
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +11 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_auth.py +2 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_bm25_retriever.py +1 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_document_store.py +103 -16
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_document_store_async.py +14 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_embedding_retriever.py +1 -0
- opensearch_haystack-9.1.0/tests/test_multivector_retriever.py +168 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_open_search_hybrid_retriever.py +1 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/.gitignore +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/LICENSE.txt +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/README.md +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/docker-compose.yml +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/document_stores/opensearch/filters.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/__init__.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/conftest.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_document_store_common.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_filters.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_sql_retriever.py +0 -0
- {opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_utils.py +0 -0
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v9.0.0] - 2026-10-16
|
|
4
|
+
|
|
5
|
+
### 🧪 Testing
|
|
6
|
+
|
|
7
|
+
- Test compatible integrations with Python 3.15; update pyproject (#4104)
|
|
8
|
+
|
|
9
|
+
### 🧹 Chores
|
|
10
|
+
|
|
11
|
+
- [**breaking**] Drop Python 3.10 support (#4105)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
## [integrations/opensearch-v8.3.0] - 2026-10-08
|
|
15
|
+
|
|
16
|
+
### 🚀 Features
|
|
17
|
+
|
|
18
|
+
- `OpenSearchDocumentStore` emulating offset with cursor (#3692)
|
|
19
|
+
|
|
20
|
+
### 🐛 Bug Fixes
|
|
21
|
+
|
|
22
|
+
- `OpenSearchDocumentStore` adding filter to unique metadata values (#3754)
|
|
23
|
+
- Standardize license files (#3771)
|
|
24
|
+
- *(elasticsearch,opensearch)* Keep every bound when merging same-field ranges (#3984)
|
|
25
|
+
- OpenSearch fixing painless script code injection (#4067)
|
|
26
|
+
|
|
27
|
+
### 🧪 Testing
|
|
28
|
+
|
|
29
|
+
- `OpenSearchDocumentStore` removing duplicated tests already covered by haystack Mixin tests (#3780)
|
|
30
|
+
- Elastisearch/OpenSearch - explicitly set the embedding model in Hybrid Retriever tests (#4053)
|
|
31
|
+
|
|
32
|
+
### ⚙️ CI
|
|
33
|
+
|
|
34
|
+
- Improve changelog generation; fix existing changelogs (#3883)
|
|
35
|
+
|
|
36
|
+
### 🧹 Chores
|
|
37
|
+
|
|
38
|
+
- Re-standardize READMEs (#3768)
|
|
39
|
+
- Drop redundant deserialize_secrets_inplace calls in integrations requiring haystack-ai>=2.23 (#4058)
|
|
40
|
+
|
|
41
|
+
|
|
3
42
|
## [integrations/opensearch-v8.2.1] - 2026-08-03
|
|
4
43
|
|
|
5
44
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 9.1.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -11,15 +11,15 @@ License-File: LICENSE.txt
|
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: Programming Language :: Python
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
-
Requires-Python: >=3.
|
|
22
|
-
Requires-Dist: haystack-ai>=
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Requires-Dist: haystack-ai>=3.5.0
|
|
23
23
|
Requires-Dist: opensearch-py[async]>=3.0.0
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
25
25
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
loaders:
|
|
2
2
|
- modules:
|
|
3
3
|
- haystack_integrations.components.retrievers.opensearch.bm25_retriever
|
|
4
|
+
- haystack_integrations.components.retrievers.opensearch.multivector_retriever
|
|
4
5
|
- haystack_integrations.components.retrievers.opensearch.embedding_retriever
|
|
5
6
|
- haystack_integrations.components.retrievers.opensearch.metadata_retriever
|
|
6
7
|
- haystack_integrations.components.retrievers.opensearch.open_search_hybrid_retriever
|
|
@@ -7,7 +7,7 @@ name = "opensearch-haystack"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = 'Haystack 2.x Document Store for OpenSearch'
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
13
|
authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
|
|
@@ -15,17 +15,17 @@ classifiers = [
|
|
|
15
15
|
"License :: OSI Approved :: Apache Software License",
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
17
17
|
"Programming Language :: Python",
|
|
18
|
-
"Programming Language :: Python :: 3.10",
|
|
19
18
|
"Programming Language :: Python :: 3.11",
|
|
20
19
|
"Programming Language :: Python :: 3.12",
|
|
21
20
|
"Programming Language :: Python :: 3.13",
|
|
22
21
|
"Programming Language :: Python :: 3.14",
|
|
22
|
+
"Programming Language :: Python :: 3.15",
|
|
23
23
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
24
24
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
25
25
|
]
|
|
26
26
|
|
|
27
27
|
dependencies = [
|
|
28
|
-
"haystack-ai>=
|
|
28
|
+
"haystack-ai>=3.5.0",
|
|
29
29
|
"opensearch-py[async]>=3.0.0"
|
|
30
30
|
]
|
|
31
31
|
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
from .bm25_retriever import OpenSearchBM25Retriever
|
|
6
6
|
from .embedding_retriever import OpenSearchEmbeddingRetriever
|
|
7
7
|
from .metadata_retriever import OpenSearchMetadataRetriever
|
|
8
|
+
from .multivector_retriever import OpenSearchMultivectorEmbeddingRetriever
|
|
8
9
|
from .open_search_hybrid_retriever import OpenSearchHybridRetriever
|
|
9
10
|
from .sql_retriever import OpenSearchSQLRetriever
|
|
10
11
|
|
|
@@ -13,5 +14,6 @@ __all__ = [
|
|
|
13
14
|
"OpenSearchEmbeddingRetriever",
|
|
14
15
|
"OpenSearchHybridRetriever",
|
|
15
16
|
"OpenSearchMetadataRetriever",
|
|
17
|
+
"OpenSearchMultivectorEmbeddingRetriever",
|
|
16
18
|
"OpenSearchSQLRetriever",
|
|
17
19
|
]
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from haystack import component, default_from_dict, default_to_dict, logging
|
|
8
|
+
from haystack.dataclasses import Document
|
|
9
|
+
from haystack.document_stores.types import FilterPolicy
|
|
10
|
+
from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
11
|
+
|
|
12
|
+
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
13
|
+
|
|
14
|
+
from .utils import _resolve_document_store
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@component
|
|
20
|
+
class OpenSearchMultivectorEmbeddingRetriever:
|
|
21
|
+
"""
|
|
22
|
+
Retrieves documents from the OpenSearchDocumentStore using multivector embeddings.
|
|
23
|
+
|
|
24
|
+
Documents are ranked using OpenSearch's `lateInteractionScore` with `innerproduct` similarity.
|
|
25
|
+
Documents without multivector embeddings are excluded. Requires OpenSearch 3.3 or later.
|
|
26
|
+
Direct retrieval scans the filtered documents and can be computationally expensive on large collections.
|
|
27
|
+
To limit MaxSim scoring to a smaller set of candidates, use `OpenSearchEmbeddingRetriever` with
|
|
28
|
+
`return_embedding=True` on the Document Store, followed by `MaxSimRanker`.
|
|
29
|
+
|
|
30
|
+
### Usage example
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from haystack import Document
|
|
34
|
+
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
35
|
+
from haystack_integrations.components.retrievers.opensearch import OpenSearchMultivectorEmbeddingRetriever
|
|
36
|
+
|
|
37
|
+
store = OpenSearchDocumentStore(hosts="http://localhost:9200", multivector_embedding_dim=2)
|
|
38
|
+
store.write_documents([
|
|
39
|
+
Document(id="solar", content="Solar energy.", multivector_embedding=[[1.0, 0.0], [0.0, 1.0]]),
|
|
40
|
+
Document(id="wind", content="Wind energy.", multivector_embedding=[[0.5, 0.0]]),
|
|
41
|
+
])
|
|
42
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(document_store=store, top_k=1)
|
|
43
|
+
result = retriever.run(query_multivector_embedding=[[1.0, 0.0], [0.0, 1.0]])
|
|
44
|
+
print(result["documents"][0].id, result["documents"][0].score)
|
|
45
|
+
# >> solar 4.0
|
|
46
|
+
```
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
*,
|
|
52
|
+
document_store: OpenSearchDocumentStore,
|
|
53
|
+
filters: dict[str, Any] | None = None,
|
|
54
|
+
top_k: int = 10,
|
|
55
|
+
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
56
|
+
raise_on_failure: bool = True,
|
|
57
|
+
) -> None:
|
|
58
|
+
"""
|
|
59
|
+
Create the OpenSearchMultivectorEmbeddingRetriever component.
|
|
60
|
+
|
|
61
|
+
:param document_store: An instance of OpenSearchDocumentStore with `multivector_embedding_dim` configured.
|
|
62
|
+
Its `return_embedding` setting controls whether embeddings are returned.
|
|
63
|
+
:param filters: Filters applied when fetching documents from the Document Store.
|
|
64
|
+
Filters restrict the documents scored by OpenSearch.
|
|
65
|
+
:param top_k: Maximum number of documents to return. Must be greater than zero.
|
|
66
|
+
:param filter_policy: Policy to determine how filters are applied. Possible options:
|
|
67
|
+
- `merge`: Runtime filters are merged with initialization filters.
|
|
68
|
+
- `replace`: Runtime filters replace initialization filters. Use this policy to change the filtering scope.
|
|
69
|
+
:param raise_on_failure: Whether to raise retrieval errors. If False, logs a warning and returns an empty list.
|
|
70
|
+
|
|
71
|
+
:raises TypeError: `document_store` is not an instance of `OpenSearchDocumentStore`.
|
|
72
|
+
:raises ValueError: Multivector support is disabled, `top_k` is not positive, or the filter policy is invalid.
|
|
73
|
+
"""
|
|
74
|
+
if not isinstance(document_store, OpenSearchDocumentStore):
|
|
75
|
+
msg = "document_store must be an instance of OpenSearchDocumentStore"
|
|
76
|
+
raise TypeError(msg)
|
|
77
|
+
if document_store._multivector_embedding_dim is None:
|
|
78
|
+
msg = "Set multivector_embedding_dim on the document store before creating this retriever."
|
|
79
|
+
raise ValueError(msg)
|
|
80
|
+
if top_k <= 0:
|
|
81
|
+
msg = "top_k must be greater than zero."
|
|
82
|
+
raise ValueError(msg)
|
|
83
|
+
self._document_store = document_store
|
|
84
|
+
self._filters = filters or {}
|
|
85
|
+
self._raise_on_failure = raise_on_failure
|
|
86
|
+
self._top_k = top_k
|
|
87
|
+
self._filter_policy = FilterPolicy.from_str(filter_policy) if isinstance(filter_policy, str) else filter_policy
|
|
88
|
+
|
|
89
|
+
def to_dict(self) -> dict[str, Any]:
|
|
90
|
+
"""
|
|
91
|
+
Serializes the component to a dictionary.
|
|
92
|
+
|
|
93
|
+
:returns:
|
|
94
|
+
Dictionary with serialized data.
|
|
95
|
+
"""
|
|
96
|
+
return default_to_dict(
|
|
97
|
+
self,
|
|
98
|
+
document_store=self._document_store.to_dict(),
|
|
99
|
+
filters=self._filters,
|
|
100
|
+
top_k=self._top_k,
|
|
101
|
+
filter_policy=self._filter_policy.value,
|
|
102
|
+
raise_on_failure=self._raise_on_failure,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def from_dict(cls, data: dict[str, Any]) -> "OpenSearchMultivectorEmbeddingRetriever":
|
|
107
|
+
"""
|
|
108
|
+
Deserializes the component from a dictionary.
|
|
109
|
+
|
|
110
|
+
:param data:
|
|
111
|
+
Dictionary to deserialize from.
|
|
112
|
+
|
|
113
|
+
:returns:
|
|
114
|
+
Deserialized component.
|
|
115
|
+
"""
|
|
116
|
+
data["init_parameters"]["document_store"] = OpenSearchDocumentStore.from_dict(
|
|
117
|
+
data["init_parameters"]["document_store"]
|
|
118
|
+
)
|
|
119
|
+
return default_from_dict(cls, data)
|
|
120
|
+
|
|
121
|
+
def close(self) -> None:
|
|
122
|
+
"""
|
|
123
|
+
Release the synchronous resources of the underlying Document Store.
|
|
124
|
+
"""
|
|
125
|
+
self._document_store.close()
|
|
126
|
+
|
|
127
|
+
async def close_async(self) -> None:
|
|
128
|
+
"""
|
|
129
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
130
|
+
"""
|
|
131
|
+
await self._document_store.close_async()
|
|
132
|
+
|
|
133
|
+
@component.output_types(documents=list[Document])
|
|
134
|
+
def run(
|
|
135
|
+
self,
|
|
136
|
+
query_multivector_embedding: list[list[float]],
|
|
137
|
+
*,
|
|
138
|
+
filters: dict[str, Any] | None = None,
|
|
139
|
+
top_k: int | None = None,
|
|
140
|
+
document_store: OpenSearchDocumentStore | None = None,
|
|
141
|
+
) -> dict[str, list[Document]]:
|
|
142
|
+
"""
|
|
143
|
+
Retrieve documents using multivector embeddings.
|
|
144
|
+
|
|
145
|
+
:param query_multivector_embedding: Multivector embedding of the query. Must be non-empty and match
|
|
146
|
+
the Document Store's per-vector dimension.
|
|
147
|
+
:param filters: Filters applied when fetching documents from the Document Store.
|
|
148
|
+
The way runtime filters are applied depends on the `filter_policy` selected when initializing the Retriever.
|
|
149
|
+
:param top_k: Maximum number of documents to return. Must be greater than zero.
|
|
150
|
+
If None, the initialization value is used.
|
|
151
|
+
:param document_store: Optional instance of OpenSearchDocumentStore to use with the Retriever.
|
|
152
|
+
If None, the Document Store provided at initialization is used.
|
|
153
|
+
:returns:
|
|
154
|
+
Dictionary with key "documents" containing the retrieved Documents.
|
|
155
|
+
- documents: List of Documents sorted by descending native OpenSearch score.
|
|
156
|
+
|
|
157
|
+
:raises ValueError: The runtime document store is invalid, or `raise_on_failure` is True and the embedding,
|
|
158
|
+
multivector configuration, or `top_k` is invalid.
|
|
159
|
+
"""
|
|
160
|
+
store = _resolve_document_store(document_store, self._document_store)
|
|
161
|
+
filters = apply_filter_policy(self._filter_policy, self._filters, filters)
|
|
162
|
+
try:
|
|
163
|
+
documents = store._multivector_retrieval(
|
|
164
|
+
query_multivector_embedding=query_multivector_embedding,
|
|
165
|
+
filters=filters,
|
|
166
|
+
top_k=self._top_k if top_k is None else top_k,
|
|
167
|
+
)
|
|
168
|
+
except Exception as error:
|
|
169
|
+
if self._raise_on_failure:
|
|
170
|
+
raise
|
|
171
|
+
logger.warning(
|
|
172
|
+
"An error during multivector retrieval occurred and will be ignored by returning empty results: "
|
|
173
|
+
"{error}",
|
|
174
|
+
error=str(error),
|
|
175
|
+
exc_info=True,
|
|
176
|
+
)
|
|
177
|
+
documents = []
|
|
178
|
+
return {"documents": documents}
|
|
179
|
+
|
|
180
|
+
@component.output_types(documents=list[Document])
|
|
181
|
+
async def run_async(
|
|
182
|
+
self,
|
|
183
|
+
query_multivector_embedding: list[list[float]],
|
|
184
|
+
*,
|
|
185
|
+
filters: dict[str, Any] | None = None,
|
|
186
|
+
top_k: int | None = None,
|
|
187
|
+
document_store: OpenSearchDocumentStore | None = None,
|
|
188
|
+
) -> dict[str, list[Document]]:
|
|
189
|
+
"""
|
|
190
|
+
Asynchronously retrieve documents using multivector embeddings.
|
|
191
|
+
|
|
192
|
+
:param query_multivector_embedding: Multivector embedding of the query. Must be non-empty and match
|
|
193
|
+
the Document Store's per-vector dimension.
|
|
194
|
+
:param filters: Filters applied when fetching documents from the Document Store.
|
|
195
|
+
The way runtime filters are applied depends on the `filter_policy` selected when initializing the Retriever.
|
|
196
|
+
:param top_k: Maximum number of documents to return. Must be greater than zero.
|
|
197
|
+
If None, the initialization value is used.
|
|
198
|
+
:param document_store: Optional instance of OpenSearchDocumentStore to use with the Retriever.
|
|
199
|
+
If None, the Document Store provided at initialization is used.
|
|
200
|
+
:returns:
|
|
201
|
+
Dictionary with key "documents" containing the retrieved Documents.
|
|
202
|
+
- documents: List of Documents sorted by descending native OpenSearch score.
|
|
203
|
+
|
|
204
|
+
:raises ValueError: The runtime document store is invalid, or `raise_on_failure` is True and the embedding,
|
|
205
|
+
multivector configuration, or `top_k` is invalid.
|
|
206
|
+
"""
|
|
207
|
+
store = _resolve_document_store(document_store, self._document_store)
|
|
208
|
+
filters = apply_filter_policy(self._filter_policy, self._filters, filters)
|
|
209
|
+
try:
|
|
210
|
+
documents = await store._multivector_retrieval_async(
|
|
211
|
+
query_multivector_embedding=query_multivector_embedding,
|
|
212
|
+
filters=filters,
|
|
213
|
+
top_k=self._top_k if top_k is None else top_k,
|
|
214
|
+
)
|
|
215
|
+
except Exception as error:
|
|
216
|
+
if self._raise_on_failure:
|
|
217
|
+
raise
|
|
218
|
+
logger.warning(
|
|
219
|
+
"An error during multivector retrieval occurred and will be ignored by returning empty results: "
|
|
220
|
+
"{error}",
|
|
221
|
+
error=str(error),
|
|
222
|
+
exc_info=True,
|
|
223
|
+
)
|
|
224
|
+
documents = []
|
|
225
|
+
return {"documents": documents}
|
|
@@ -21,7 +21,10 @@ from opensearchpy.helpers import async_bulk, bulk
|
|
|
21
21
|
|
|
22
22
|
from haystack_integrations.document_stores.opensearch.auth import AsyncAWSAuth, AWSAuth
|
|
23
23
|
from haystack_integrations.document_stores.opensearch.filters import normalize_filters
|
|
24
|
-
from haystack_integrations.document_stores.opensearch.opensearch_scripts import
|
|
24
|
+
from haystack_integrations.document_stores.opensearch.opensearch_scripts import (
|
|
25
|
+
METADATA_SEARCH_JACCARD_SCRIPT,
|
|
26
|
+
MULTIVECTOR_SEARCH_SCRIPT,
|
|
27
|
+
)
|
|
25
28
|
|
|
26
29
|
logger = logging.getLogger(__name__)
|
|
27
30
|
|
|
@@ -32,7 +35,7 @@ UPDATE_SCRIPT = """
|
|
|
32
35
|
}
|
|
33
36
|
"""
|
|
34
37
|
|
|
35
|
-
SPECIAL_FIELDS = {"content", "embedding", "id", "score", "sparse_embedding", "blob"}
|
|
38
|
+
SPECIAL_FIELDS = {"content", "embedding", "id", "score", "sparse_embedding", "multivector_embedding", "blob"}
|
|
36
39
|
|
|
37
40
|
Hosts = str | list[str | Mapping[str, str | int]]
|
|
38
41
|
|
|
@@ -100,6 +103,7 @@ class OpenSearchDocumentStore:
|
|
|
100
103
|
verify_certs: bool | None = None,
|
|
101
104
|
timeout: int | None = None,
|
|
102
105
|
nested_fields: list[str] | Literal["*"] | None = None,
|
|
106
|
+
multivector_embedding_dim: int | None = None,
|
|
103
107
|
**kwargs: Any,
|
|
104
108
|
) -> None:
|
|
105
109
|
"""
|
|
@@ -115,8 +119,8 @@ class OpenSearchDocumentStore:
|
|
|
115
119
|
:param max_chunk_bytes: Maximum size of the requests in bytes. Defaults to 100MB
|
|
116
120
|
:param embedding_dim: Dimension of the embeddings. Defaults to 768
|
|
117
121
|
:param return_embedding:
|
|
118
|
-
Whether to return
|
|
119
|
-
`filter_documents` and `filter_documents_async
|
|
122
|
+
Whether to return `embedding` and `multivector_embedding` on retrieved Documents. This also applies to
|
|
123
|
+
`filter_documents` and `filter_documents_async`.
|
|
120
124
|
:param method: The method definition of the underlying configuration of the approximate k-NN algorithm. Please
|
|
121
125
|
see the [official OpenSearch docs](https://opensearch.org/docs/latest/search-plugins/knn/knn-index/#method-definitions)
|
|
122
126
|
for more information. Defaults to None
|
|
@@ -143,9 +147,16 @@ class OpenSearchDocumentStore:
|
|
|
143
147
|
the first `write_documents` batch.
|
|
144
148
|
When the index already exists, nested fields are discovered from the live mapping.
|
|
145
149
|
Defaults to None (no nested support).
|
|
150
|
+
:param multivector_embedding_dim: Dimension of each vector in a multivector embedding. Set to enable
|
|
151
|
+
multivector support.
|
|
146
152
|
:param **kwargs: Optional arguments that ``OpenSearch`` takes. For the full list of supported kwargs,
|
|
147
153
|
see the [official OpenSearch reference](https://opensearch-project.github.io/opensearch-py/api-ref/clients/opensearch_client.html)
|
|
154
|
+
:raises ValueError: `multivector_embedding_dim` is not positive or the multivector mapping is incompatible.
|
|
148
155
|
"""
|
|
156
|
+
if multivector_embedding_dim is not None and multivector_embedding_dim <= 0:
|
|
157
|
+
msg = "multivector_embedding_dim must be greater than zero."
|
|
158
|
+
raise ValueError(msg)
|
|
159
|
+
self._multivector_embedding_dim = multivector_embedding_dim
|
|
149
160
|
self._hosts = hosts
|
|
150
161
|
self._index = index
|
|
151
162
|
self._max_chunk_bytes = max_chunk_bytes
|
|
@@ -154,6 +165,7 @@ class OpenSearchDocumentStore:
|
|
|
154
165
|
self._method = method
|
|
155
166
|
self._nested_fields = nested_fields
|
|
156
167
|
self._mappings = mappings or self._get_default_mappings()
|
|
168
|
+
self._validate_multivector_mapping(self._mappings)
|
|
157
169
|
self._settings = settings
|
|
158
170
|
self._create_index = create_index
|
|
159
171
|
self._http_auth = http_auth
|
|
@@ -186,6 +198,8 @@ class OpenSearchDocumentStore:
|
|
|
186
198
|
}
|
|
187
199
|
],
|
|
188
200
|
}
|
|
201
|
+
if self._multivector_embedding_dim is not None:
|
|
202
|
+
default_mappings["properties"]["multivector_embedding"] = {"type": "object", "enabled": False}
|
|
189
203
|
if self._method:
|
|
190
204
|
default_mappings["properties"]["embedding"]["method"] = self._method
|
|
191
205
|
if self._nested_fields and self._nested_fields != "*":
|
|
@@ -193,6 +207,28 @@ class OpenSearchDocumentStore:
|
|
|
193
207
|
default_mappings["properties"][field] = {"type": "nested"}
|
|
194
208
|
return default_mappings
|
|
195
209
|
|
|
210
|
+
def _validate_multivector_mapping(self, mappings: dict[str, Any]) -> None:
|
|
211
|
+
"""Validate the multivector field mapping and check that `_source` is enabled."""
|
|
212
|
+
if self._multivector_embedding_dim is None:
|
|
213
|
+
return
|
|
214
|
+
field = mappings.get("properties", {}).get("multivector_embedding", {})
|
|
215
|
+
if field.get("type") != "object" or field.get("enabled") is not False:
|
|
216
|
+
msg = (
|
|
217
|
+
"The index is not configured for multivector embeddings. "
|
|
218
|
+
"Use a new index name with multivector_embedding_dim set so Haystack "
|
|
219
|
+
"can create the required mapping. "
|
|
220
|
+
'If you provide custom mappings, set "multivector_embedding" to '
|
|
221
|
+
'{"type": "object", "enabled": false} under "properties".'
|
|
222
|
+
)
|
|
223
|
+
raise ValueError(msg)
|
|
224
|
+
if mappings.get("_source", {}).get("enabled") is False:
|
|
225
|
+
msg = (
|
|
226
|
+
"Multivector retrieval requires the original document data in _source. "
|
|
227
|
+
"Use a new index with the default mappings and write your documents again. "
|
|
228
|
+
'If you provide custom mappings, remove "_source": {"enabled": false}.'
|
|
229
|
+
)
|
|
230
|
+
raise ValueError(msg)
|
|
231
|
+
|
|
196
232
|
def create_index(
|
|
197
233
|
self,
|
|
198
234
|
index: str | None = None,
|
|
@@ -210,16 +246,16 @@ class OpenSearchDocumentStore:
|
|
|
210
246
|
:param settings: The settings of the index to be created. Please see the [official OpenSearch docs](https://opensearch.org/docs/latest/search-plugins/knn/knn-index/#index-settings)
|
|
211
247
|
for more information. If None, the settings from the constructor are used.
|
|
212
248
|
"""
|
|
213
|
-
self._ensure_initialized()
|
|
214
|
-
assert self._client is not None
|
|
215
|
-
|
|
216
249
|
if not index:
|
|
217
250
|
index = self._index
|
|
218
251
|
if not mappings:
|
|
219
252
|
mappings = self._mappings
|
|
253
|
+
self._validate_multivector_mapping(mappings)
|
|
220
254
|
if not settings:
|
|
221
255
|
settings = self._settings
|
|
222
256
|
|
|
257
|
+
self._ensure_initialized()
|
|
258
|
+
assert self._client is not None
|
|
223
259
|
if not self._client.indices.exists(index=index):
|
|
224
260
|
self._client.indices.create(index=index, body={"mappings": mappings, "settings": settings})
|
|
225
261
|
|
|
@@ -248,6 +284,7 @@ class OpenSearchDocumentStore:
|
|
|
248
284
|
index=self._index,
|
|
249
285
|
max_chunk_bytes=self._max_chunk_bytes,
|
|
250
286
|
embedding_dim=self._embedding_dim,
|
|
287
|
+
multivector_embedding_dim=self._multivector_embedding_dim,
|
|
251
288
|
method=self._method,
|
|
252
289
|
mappings=self._mappings,
|
|
253
290
|
settings=self._settings,
|
|
@@ -426,6 +463,7 @@ class OpenSearchDocumentStore:
|
|
|
426
463
|
# the key differs from the alias, so we use next(iter(...)) to handle both cases.
|
|
427
464
|
actual_index = next(iter(mapping))
|
|
428
465
|
properties = mapping[actual_index]["mappings"].get("properties", {})
|
|
466
|
+
self._validate_multivector_mapping(mapping[actual_index]["mappings"])
|
|
429
467
|
self._populate_nested_fields_from_mapping(properties)
|
|
430
468
|
elif self._create_index:
|
|
431
469
|
# Create the index if it doesn't exist
|
|
@@ -446,6 +484,7 @@ class OpenSearchDocumentStore:
|
|
|
446
484
|
# the key differs from the alias, so we use next(iter(...)) to handle both cases.
|
|
447
485
|
actual_index = next(iter(mapping))
|
|
448
486
|
properties = mapping[actual_index]["mappings"].get("properties", {})
|
|
487
|
+
self._validate_multivector_mapping(mapping[actual_index]["mappings"])
|
|
449
488
|
self._populate_nested_fields_from_mapping(properties)
|
|
450
489
|
elif self._create_index:
|
|
451
490
|
# Create the index if it doesn't exist
|
|
@@ -492,7 +531,7 @@ class OpenSearchDocumentStore:
|
|
|
492
531
|
# For some applications not returning the embedding can save a lot of bandwidth
|
|
493
532
|
# if you don't need this data not retrieving it can be a good idea
|
|
494
533
|
if not self._return_embedding:
|
|
495
|
-
search_kwargs["_source"] = {"excludes": ["embedding"]}
|
|
534
|
+
search_kwargs["_source"] = {"excludes": ["embedding", "multivector_embedding"]}
|
|
496
535
|
return search_kwargs
|
|
497
536
|
|
|
498
537
|
def _search_documents(self, request_body: dict[str, Any]) -> list[Document]:
|
|
@@ -532,6 +571,15 @@ class OpenSearchDocumentStore:
|
|
|
532
571
|
|
|
533
572
|
return await self._search_documents_async(self._prepare_filter_search_request(filters))
|
|
534
573
|
|
|
574
|
+
def _validate_multivector_embedding(self, embedding: list[list[float]]) -> None:
|
|
575
|
+
"""Check that the embedding is non-empty and each vector matches the configured dimension."""
|
|
576
|
+
if self._multivector_embedding_dim is None:
|
|
577
|
+
msg = "Set multivector_embedding_dim before writing or querying multivector embeddings."
|
|
578
|
+
raise ValueError(msg)
|
|
579
|
+
if not embedding or any(len(vector) != self._multivector_embedding_dim for vector in embedding):
|
|
580
|
+
msg = "A multivector embedding must be non-empty and match multivector_embedding_dim."
|
|
581
|
+
raise ValueError(msg)
|
|
582
|
+
|
|
535
583
|
def _prepare_bulk_write_request(
|
|
536
584
|
self,
|
|
537
585
|
*,
|
|
@@ -550,6 +598,8 @@ class OpenSearchDocumentStore:
|
|
|
550
598
|
action = "index" if policy == DuplicatePolicy.OVERWRITE else "create"
|
|
551
599
|
opensearch_actions = []
|
|
552
600
|
for doc in documents:
|
|
601
|
+
if doc.multivector_embedding is not None:
|
|
602
|
+
self._validate_multivector_embedding(doc.multivector_embedding)
|
|
553
603
|
doc_dict = doc.to_dict()
|
|
554
604
|
|
|
555
605
|
# Extract routing from document metadata
|
|
@@ -1070,7 +1120,7 @@ class OpenSearchDocumentStore:
|
|
|
1070
1120
|
# For some applications not returning the embedding can save a lot of bandwidth
|
|
1071
1121
|
# if you don't need this data not retrieving it can be a good idea
|
|
1072
1122
|
if not self._return_embedding:
|
|
1073
|
-
body["_source"] = {"excludes": ["embedding"]}
|
|
1123
|
+
body["_source"] = {"excludes": ["embedding", "multivector_embedding"]}
|
|
1074
1124
|
|
|
1075
1125
|
return body
|
|
1076
1126
|
|
|
@@ -1594,6 +1644,61 @@ class OpenSearchDocumentStore:
|
|
|
1594
1644
|
# Process and return results
|
|
1595
1645
|
return self._process_metadata_search_results(hit_list, fields, top_k)
|
|
1596
1646
|
|
|
1647
|
+
def _prepare_multivector_search_request(
|
|
1648
|
+
self,
|
|
1649
|
+
query_multivector_embedding: list[list[float]],
|
|
1650
|
+
*,
|
|
1651
|
+
filters: dict[str, Any] | None,
|
|
1652
|
+
top_k: int,
|
|
1653
|
+
) -> dict[str, Any]:
|
|
1654
|
+
"""Build the OpenSearch script_score query for MaxSim retrieval with optional filters."""
|
|
1655
|
+
self._validate_multivector_embedding(query_multivector_embedding)
|
|
1656
|
+
if top_k <= 0:
|
|
1657
|
+
msg = "top_k must be greater than zero."
|
|
1658
|
+
raise ValueError(msg)
|
|
1659
|
+
body = self._prepare_filter_search_request(filters)
|
|
1660
|
+
body["size"] = top_k
|
|
1661
|
+
body["query"] = {
|
|
1662
|
+
"script_score": {
|
|
1663
|
+
"query": body.get("query", {"match_all": {}}),
|
|
1664
|
+
"script": {
|
|
1665
|
+
"source": MULTIVECTOR_SEARCH_SCRIPT,
|
|
1666
|
+
"params": {"query_vectors": query_multivector_embedding},
|
|
1667
|
+
},
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
return body
|
|
1671
|
+
|
|
1672
|
+
def _multivector_retrieval(
|
|
1673
|
+
self,
|
|
1674
|
+
query_multivector_embedding: list[list[float]],
|
|
1675
|
+
*,
|
|
1676
|
+
filters: dict[str, Any] | None = None,
|
|
1677
|
+
top_k: int = 10,
|
|
1678
|
+
) -> list[Document]:
|
|
1679
|
+
"""Retrieve documents with multivector embeddings ranked by their MaxSim score."""
|
|
1680
|
+
self._ensure_initialized()
|
|
1681
|
+
body = self._prepare_multivector_search_request(query_multivector_embedding, filters=filters, top_k=top_k)
|
|
1682
|
+
documents = self._search_documents(body)
|
|
1683
|
+
# Our MULTIVECTOR_SEARCH_SCRIPT assigns zero to documents without multivectors
|
|
1684
|
+
# We exclude them from results, similarly to how dense embedding retrieval works on OpenSearch.
|
|
1685
|
+
return [doc for doc in documents if doc.score is not None and doc.score > 0]
|
|
1686
|
+
|
|
1687
|
+
async def _multivector_retrieval_async(
|
|
1688
|
+
self,
|
|
1689
|
+
query_multivector_embedding: list[list[float]],
|
|
1690
|
+
*,
|
|
1691
|
+
filters: dict[str, Any] | None = None,
|
|
1692
|
+
top_k: int = 10,
|
|
1693
|
+
) -> list[Document]:
|
|
1694
|
+
"""Asynchronously retrieve documents with multivector embeddings ranked by their MaxSim score."""
|
|
1695
|
+
await self._ensure_initialized_async()
|
|
1696
|
+
body = self._prepare_multivector_search_request(query_multivector_embedding, filters=filters, top_k=top_k)
|
|
1697
|
+
documents = await self._search_documents_async(body)
|
|
1698
|
+
# Our MULTIVECTOR_SEARCH_SCRIPT assigns zero to documents without multivectors
|
|
1699
|
+
# We exclude them from results, similarly to how dense embedding retrieval works on OpenSearch.
|
|
1700
|
+
return [doc for doc in documents if doc.score is not None and doc.score > 0]
|
|
1701
|
+
|
|
1597
1702
|
def _prepare_embedding_search_request(
|
|
1598
1703
|
self,
|
|
1599
1704
|
*,
|
|
@@ -1654,7 +1759,7 @@ class OpenSearchDocumentStore:
|
|
|
1654
1759
|
# For some applications not returning the embedding can save a lot of bandwidth
|
|
1655
1760
|
# if you don't need this data not retrieving it can be a good idea
|
|
1656
1761
|
if not self._return_embedding:
|
|
1657
|
-
body["_source"] = {"excludes": ["embedding"]}
|
|
1762
|
+
body["_source"] = {"excludes": ["embedding", "multivector_embedding"]}
|
|
1658
1763
|
|
|
1659
1764
|
return body
|
|
1660
1765
|
|
|
@@ -47,3 +47,14 @@ if (union == 0) return 0.0;
|
|
|
47
47
|
|
|
48
48
|
return inter / (double) union;
|
|
49
49
|
"""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# We use innerproduct, which produces positive scores:
|
|
53
|
+
# https://docs.opensearch.org/latest/mappings/supported-field-types/knn-spaces/
|
|
54
|
+
# We assign zero to documents without multivectors so we can exclude them later in the document store.
|
|
55
|
+
MULTIVECTOR_SEARCH_SCRIPT = """
|
|
56
|
+
if (params._source.multivector_embedding == null || params._source.multivector_embedding.isEmpty()) {
|
|
57
|
+
return 0.0;
|
|
58
|
+
}
|
|
59
|
+
return lateInteractionScore(params.query_vectors, 'multivector_embedding', params._source, 'innerproduct');
|
|
60
|
+
"""
|
|
@@ -331,6 +331,7 @@ class TestDocumentStoreWithAuth:
|
|
|
331
331
|
"verify_certs": None,
|
|
332
332
|
"timeout": None,
|
|
333
333
|
"nested_fields": None,
|
|
334
|
+
"multivector_embedding_dim": None,
|
|
334
335
|
},
|
|
335
336
|
}
|
|
336
337
|
|
|
@@ -378,6 +379,7 @@ class TestDocumentStoreWithAuth:
|
|
|
378
379
|
"verify_certs": None,
|
|
379
380
|
"timeout": None,
|
|
380
381
|
"nested_fields": None,
|
|
382
|
+
"multivector_embedding_dim": None,
|
|
381
383
|
},
|
|
382
384
|
}
|
|
383
385
|
|
|
@@ -64,6 +64,7 @@ def test_to_dict(_mock_opensearch_client):
|
|
|
64
64
|
"verify_certs": None,
|
|
65
65
|
"timeout": None,
|
|
66
66
|
"nested_fields": None,
|
|
67
|
+
"multivector_embedding_dim": None,
|
|
67
68
|
},
|
|
68
69
|
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
69
70
|
},
|
|
@@ -26,7 +26,7 @@ from tests.test_document_store_common import OpenSearchDocumentStoreTestMixin
|
|
|
26
26
|
|
|
27
27
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
28
28
|
def test_to_dict(_mock_opensearch_client):
|
|
29
|
-
document_store = OpenSearchDocumentStore(hosts="some hosts")
|
|
29
|
+
document_store = OpenSearchDocumentStore(hosts="some hosts", multivector_embedding_dim=2)
|
|
30
30
|
res = document_store.to_dict()
|
|
31
31
|
assert res == {
|
|
32
32
|
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
@@ -38,6 +38,7 @@ def test_to_dict(_mock_opensearch_client):
|
|
|
38
38
|
"dynamic_templates": [{"strings": {"mapping": {"type": "keyword"}, "match_mapping_type": "string"}}],
|
|
39
39
|
"properties": {
|
|
40
40
|
"content": {"type": "text"},
|
|
41
|
+
"multivector_embedding": {"type": "object", "enabled": False},
|
|
41
42
|
"embedding": {"dimension": 768, "index": True, "type": "knn_vector"},
|
|
42
43
|
},
|
|
43
44
|
},
|
|
@@ -54,6 +55,7 @@ def test_to_dict(_mock_opensearch_client):
|
|
|
54
55
|
"verify_certs": None,
|
|
55
56
|
"timeout": None,
|
|
56
57
|
"nested_fields": None,
|
|
58
|
+
"multivector_embedding_dim": 2,
|
|
57
59
|
},
|
|
58
60
|
}
|
|
59
61
|
|
|
@@ -77,6 +79,7 @@ def test_from_dict(_mock_opensearch_client):
|
|
|
77
79
|
"index": "default",
|
|
78
80
|
"max_chunk_bytes": 1000,
|
|
79
81
|
"embedding_dim": 1536,
|
|
82
|
+
"multivector_embedding_dim": 2,
|
|
80
83
|
"create_index": False,
|
|
81
84
|
"return_embedding": True,
|
|
82
85
|
"aws_service": "es",
|
|
@@ -91,10 +94,12 @@ def test_from_dict(_mock_opensearch_client):
|
|
|
91
94
|
assert document_store._index == "default"
|
|
92
95
|
assert document_store._max_chunk_bytes == 1000
|
|
93
96
|
assert document_store._embedding_dim == 1536
|
|
97
|
+
assert document_store._multivector_embedding_dim == 2
|
|
94
98
|
assert document_store._method is None
|
|
95
99
|
assert document_store._mappings == {
|
|
96
100
|
"properties": {
|
|
97
101
|
"embedding": {"type": "knn_vector", "index": True, "dimension": 1536},
|
|
102
|
+
"multivector_embedding": {"type": "object", "enabled": False},
|
|
98
103
|
"content": {"type": "text"},
|
|
99
104
|
},
|
|
100
105
|
"dynamic_templates": [
|
|
@@ -169,9 +174,15 @@ def test_init_is_lazy(_mock_opensearch_client):
|
|
|
169
174
|
_mock_opensearch_client.assert_not_called()
|
|
170
175
|
|
|
171
176
|
|
|
177
|
+
def test_init_invalid_multivector_embedding_dim():
|
|
178
|
+
with pytest.raises(ValueError, match="multivector_embedding_dim must be greater than zero"):
|
|
179
|
+
OpenSearchDocumentStore(multivector_embedding_dim=0)
|
|
180
|
+
|
|
181
|
+
|
|
172
182
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
173
183
|
def test_get_default_mappings(_mock_opensearch_client):
|
|
174
184
|
store = OpenSearchDocumentStore(hosts="testhost", embedding_dim=1536, method={"name": "hnsw"})
|
|
185
|
+
assert "multivector_embedding" not in store._mappings["properties"]
|
|
175
186
|
assert store._mappings["properties"]["embedding"] == {
|
|
176
187
|
"type": "knn_vector",
|
|
177
188
|
"index": True,
|
|
@@ -180,6 +191,37 @@ def test_get_default_mappings(_mock_opensearch_client):
|
|
|
180
191
|
}
|
|
181
192
|
|
|
182
193
|
|
|
194
|
+
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
195
|
+
def test_create_index_with_multivector_embedding_dim(mock_client):
|
|
196
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2, create_index=False)
|
|
197
|
+
indices = mock_client.return_value.indices
|
|
198
|
+
indices.exists.return_value = False
|
|
199
|
+
|
|
200
|
+
store.create_index(index="documents")
|
|
201
|
+
|
|
202
|
+
request = indices.create.call_args.kwargs
|
|
203
|
+
assert request["index"] == "documents"
|
|
204
|
+
assert request["body"]["mappings"]["properties"]["multivector_embedding"] == {"type": "object", "enabled": False}
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_create_index_rejects_disabled_source_for_multivectors():
|
|
208
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
209
|
+
mappings = {**store._mappings, "_source": {"enabled": False}}
|
|
210
|
+
|
|
211
|
+
with pytest.raises(ValueError, match="requires the original document data"):
|
|
212
|
+
store.create_index(mappings=mappings)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@pytest.mark.parametrize("multivector_embedding_dim", [None, 3])
|
|
216
|
+
def test_write_documents_requires_matching_multivector_embedding_dim(multivector_embedding_dim):
|
|
217
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=multivector_embedding_dim)
|
|
218
|
+
store._client = MagicMock()
|
|
219
|
+
document = Document(multivector_embedding=[[1.0, 0.0]])
|
|
220
|
+
|
|
221
|
+
with pytest.raises(ValueError, match="multivector_embedding_dim"):
|
|
222
|
+
store.write_documents([document])
|
|
223
|
+
|
|
224
|
+
|
|
183
225
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
184
226
|
@patch("haystack_integrations.document_stores.opensearch.document_store.bulk")
|
|
185
227
|
def test_routing_extracted_from_metadata(mock_bulk, _mock_opensearch_client):
|
|
@@ -298,6 +340,30 @@ def test_bm25_retrieval_reraises_other_transport_errors(_mock_opensearch_client)
|
|
|
298
340
|
assert store._client.search.call_count == 1
|
|
299
341
|
|
|
300
342
|
|
|
343
|
+
def test_embedding_retrieval_excludes_embeddings_from_source_by_default():
|
|
344
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
345
|
+
store._client = MagicMock()
|
|
346
|
+
store._client.search.return_value = {"hits": {"hits": []}}
|
|
347
|
+
|
|
348
|
+
store._embedding_retrieval(query_embedding=[1.0, 0.0])
|
|
349
|
+
|
|
350
|
+
body = store._client.search.call_args.kwargs["body"]
|
|
351
|
+
assert body["_source"]["excludes"] == ["embedding", "multivector_embedding"]
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@pytest.mark.parametrize("query_embedding", [[], [[1.0, 0.0], [1.0]]])
|
|
355
|
+
def test_prepare_multivector_search_request_invalid_embedding(query_embedding):
|
|
356
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
357
|
+
with pytest.raises(ValueError, match="non-empty and match multivector_embedding_dim"):
|
|
358
|
+
store._prepare_multivector_search_request(query_embedding, filters=None, top_k=1)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def test_prepare_multivector_search_request_invalid_top_k():
|
|
362
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
363
|
+
with pytest.raises(ValueError, match="top_k"):
|
|
364
|
+
store._prepare_multivector_search_request([[1.0, 0.0]], filters=None, top_k=0)
|
|
365
|
+
|
|
366
|
+
|
|
301
367
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
302
368
|
def test_get_metadata_field_unique_values_no_search_term_uses_match_all_query(_mock_opensearch_client):
|
|
303
369
|
"""Composite aggregation terms sources don't support `include`/`exclude`, so with no search_term
|
|
@@ -554,6 +620,15 @@ def test_ensure_index_exists_direct_index(_mock_opensearch_client):
|
|
|
554
620
|
mock_client.indices.create.assert_not_called()
|
|
555
621
|
|
|
556
622
|
|
|
623
|
+
def test_ensure_index_exists_rejects_missing_multivector_mapping():
|
|
624
|
+
store = OpenSearchDocumentStore(index="existing", multivector_embedding_dim=2)
|
|
625
|
+
store._client = MagicMock()
|
|
626
|
+
store._client.indices.get_mapping.return_value = {"existing": {"mappings": {"properties": {}}}}
|
|
627
|
+
|
|
628
|
+
with pytest.raises(ValueError, match="not configured for multivector embeddings"):
|
|
629
|
+
store._ensure_index_exists()
|
|
630
|
+
|
|
631
|
+
|
|
557
632
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
558
633
|
def test_ensure_index_exists_with_alias(_mock_opensearch_client):
|
|
559
634
|
"""When self._index is an alias, get_mapping keys the response by the real index name.
|
|
@@ -857,6 +932,19 @@ class TestDocumentStore(
|
|
|
857
932
|
assert results[0].content == "Most similar document"
|
|
858
933
|
assert results[1].content == "2nd best document"
|
|
859
934
|
|
|
935
|
+
def test_multivector_retrieval(self, opensearch_store):
|
|
936
|
+
store = opensearch_store(multivector_embedding_dim=2, return_embedding=True)
|
|
937
|
+
documents = [
|
|
938
|
+
Document(id="a", multivector_embedding=[[1.0, 0.0], [0.0, 1.0]]),
|
|
939
|
+
Document(id="b", multivector_embedding=[[0.5, 0.0]]),
|
|
940
|
+
Document(id="missing"),
|
|
941
|
+
]
|
|
942
|
+
store.write_documents(documents)
|
|
943
|
+
result = store._multivector_retrieval([[1.0, 0.0], [0.0, 1.0]])
|
|
944
|
+
assert [doc.id for doc in result] == ["a", "b"]
|
|
945
|
+
assert [doc.score for doc in result] == pytest.approx([4.0, 2.5])
|
|
946
|
+
assert result[0].multivector_embedding == documents[0].multivector_embedding
|
|
947
|
+
|
|
860
948
|
def test_embedding_retrieval_with_filters(
|
|
861
949
|
self, document_store_embedding_dim_4_no_emb_returned: OpenSearchDocumentStore
|
|
862
950
|
):
|
|
@@ -990,20 +1078,17 @@ class TestDocumentStore(
|
|
|
990
1078
|
|
|
991
1079
|
assert mock_bulk.call_args.kwargs["max_chunk_bytes"] == DEFAULT_MAX_CHUNK_BYTES
|
|
992
1080
|
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
)
|
|
1005
|
-
assert len(results) == 2
|
|
1006
|
-
assert results[0].embedding is None
|
|
1081
|
+
@pytest.mark.parametrize("return_embedding", [False, True])
|
|
1082
|
+
def test_embedding_retrieval_respects_return_embedding(self, opensearch_store, return_embedding):
|
|
1083
|
+
store = opensearch_store(embedding_dim=2, multivector_embedding_dim=2, return_embedding=return_embedding)
|
|
1084
|
+
document = Document(embedding=[1.0, 0.0], multivector_embedding=[[1.0, 0.0], [0.0, 1.0]])
|
|
1085
|
+
store.write_documents([document])
|
|
1086
|
+
|
|
1087
|
+
(result,) = store._embedding_retrieval(query_embedding=[1.0, 0.0])
|
|
1088
|
+
|
|
1089
|
+
assert result.id == document.id
|
|
1090
|
+
assert result.embedding == (document.embedding if return_embedding else None)
|
|
1091
|
+
assert result.multivector_embedding == (document.multivector_embedding if return_embedding else None)
|
|
1007
1092
|
|
|
1008
1093
|
def test_embedding_retrieval_but_dont_return_embeddings_for_bm25_retrieval(
|
|
1009
1094
|
self, document_store_embedding_dim_4_no_emb_returned: OpenSearchDocumentStore
|
|
@@ -1686,8 +1771,9 @@ class TestDocumentStore(
|
|
|
1686
1771
|
finally:
|
|
1687
1772
|
client.indices.delete_alias(index=document_store._index, name=alias_name)
|
|
1688
1773
|
|
|
1689
|
-
def test_get_metadata_fields_info_with_alias(self,
|
|
1774
|
+
def test_get_metadata_fields_info_with_alias(self, opensearch_store):
|
|
1690
1775
|
"""get_metadata_fields_info reads the mapping via alias without KeyError."""
|
|
1776
|
+
document_store = opensearch_store(multivector_embedding_dim=2)
|
|
1691
1777
|
alias_name = f"alias_mfi_{document_store._index}"
|
|
1692
1778
|
client = document_store._client
|
|
1693
1779
|
client.indices.put_alias(index=document_store._index, name=alias_name)
|
|
@@ -1703,6 +1789,7 @@ class TestDocumentStore(
|
|
|
1703
1789
|
alias_store.write_documents([Document(content="doc", meta={"category": "A", "count": 1})])
|
|
1704
1790
|
|
|
1705
1791
|
fields_info = alias_store.get_metadata_fields_info()
|
|
1792
|
+
assert "multivector_embedding" not in fields_info
|
|
1706
1793
|
assert "category" in fields_info
|
|
1707
1794
|
assert fields_info["category"]["type"] == "keyword"
|
|
1708
1795
|
assert "count" in fields_info
|
|
@@ -397,6 +397,20 @@ class TestDocumentStoreAsync(
|
|
|
397
397
|
assert results[0].content == "Most similar document"
|
|
398
398
|
assert results[1].content == "2nd best document"
|
|
399
399
|
|
|
400
|
+
@pytest.mark.asyncio
|
|
401
|
+
async def test_multivector_retrieval(self, opensearch_store):
|
|
402
|
+
store = opensearch_store(multivector_embedding_dim=2, return_embedding=True)
|
|
403
|
+
documents = [
|
|
404
|
+
Document(id="a", multivector_embedding=[[1.0, 0.0], [0.0, 1.0]]),
|
|
405
|
+
Document(id="b", multivector_embedding=[[0.5, 0.0]]),
|
|
406
|
+
Document(id="missing"),
|
|
407
|
+
]
|
|
408
|
+
await store.write_documents_async(documents)
|
|
409
|
+
result = await store._multivector_retrieval_async([[1.0, 0.0], [0.0, 1.0]])
|
|
410
|
+
assert [doc.id for doc in result] == ["a", "b"]
|
|
411
|
+
assert [doc.score for doc in result] == pytest.approx([4.0, 2.5])
|
|
412
|
+
assert result[0].multivector_embedding == documents[0].multivector_embedding
|
|
413
|
+
|
|
400
414
|
@pytest.mark.asyncio
|
|
401
415
|
async def test_embedding_retrieval_with_filters(
|
|
402
416
|
self, document_store_embedding_dim_4_no_emb_returned: OpenSearchDocumentStore
|
|
@@ -80,6 +80,7 @@ def test_to_dict(_mock_opensearch_client):
|
|
|
80
80
|
"verify_certs": None,
|
|
81
81
|
"timeout": None,
|
|
82
82
|
"nested_fields": None,
|
|
83
|
+
"multivector_embedding_dim": None,
|
|
83
84
|
},
|
|
84
85
|
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
85
86
|
},
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from unittest.mock import AsyncMock, Mock
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from haystack import Document
|
|
9
|
+
from haystack.document_stores.types import FilterPolicy
|
|
10
|
+
|
|
11
|
+
from haystack_integrations.components.retrievers.opensearch import OpenSearchMultivectorEmbeddingRetriever
|
|
12
|
+
from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_init_invalid_document_store():
|
|
16
|
+
with pytest.raises(TypeError, match="OpenSearchDocumentStore"):
|
|
17
|
+
OpenSearchMultivectorEmbeddingRetriever(document_store=None)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_init_requires_multivector_support():
|
|
21
|
+
with pytest.raises(ValueError, match="multivector_embedding_dim"):
|
|
22
|
+
OpenSearchMultivectorEmbeddingRetriever(document_store=OpenSearchDocumentStore())
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_init_invalid_top_k():
|
|
26
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
27
|
+
with pytest.raises(ValueError, match="top_k"):
|
|
28
|
+
OpenSearchMultivectorEmbeddingRetriever(document_store=store, top_k=0)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_to_dict():
|
|
32
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2, return_embedding=True)
|
|
33
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(
|
|
34
|
+
document_store=store, top_k=3, filter_policy="merge", raise_on_failure=False
|
|
35
|
+
)
|
|
36
|
+
assert retriever.to_dict() == {
|
|
37
|
+
"type": (
|
|
38
|
+
"haystack_integrations.components.retrievers.opensearch.multivector_retriever."
|
|
39
|
+
"OpenSearchMultivectorEmbeddingRetriever"
|
|
40
|
+
),
|
|
41
|
+
"init_parameters": {
|
|
42
|
+
"document_store": store.to_dict(),
|
|
43
|
+
"filters": {},
|
|
44
|
+
"top_k": 3,
|
|
45
|
+
"filter_policy": "merge",
|
|
46
|
+
"raise_on_failure": False,
|
|
47
|
+
},
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_from_dict():
|
|
52
|
+
data = {
|
|
53
|
+
"type": (
|
|
54
|
+
"haystack_integrations.components.retrievers.opensearch.multivector_retriever."
|
|
55
|
+
"OpenSearchMultivectorEmbeddingRetriever"
|
|
56
|
+
),
|
|
57
|
+
"init_parameters": {
|
|
58
|
+
"document_store": {
|
|
59
|
+
"type": "haystack_integrations.document_stores.opensearch.document_store.OpenSearchDocumentStore",
|
|
60
|
+
"init_parameters": {"multivector_embedding_dim": 2, "return_embedding": True},
|
|
61
|
+
},
|
|
62
|
+
"filters": {},
|
|
63
|
+
"top_k": 3,
|
|
64
|
+
"filter_policy": "merge",
|
|
65
|
+
"raise_on_failure": False,
|
|
66
|
+
},
|
|
67
|
+
}
|
|
68
|
+
restored = OpenSearchMultivectorEmbeddingRetriever.from_dict(data)
|
|
69
|
+
assert restored._raise_on_failure is False
|
|
70
|
+
assert restored._top_k == 3
|
|
71
|
+
assert restored._filter_policy.value == "merge"
|
|
72
|
+
assert restored._document_store._multivector_embedding_dim == 2
|
|
73
|
+
assert restored._document_store._return_embedding is True
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_close():
|
|
77
|
+
store = Mock(spec=OpenSearchDocumentStore, _multivector_embedding_dim=2)
|
|
78
|
+
OpenSearchMultivectorEmbeddingRetriever(document_store=store).close()
|
|
79
|
+
store.close.assert_called_once_with()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@pytest.mark.asyncio
|
|
83
|
+
async def test_close_async():
|
|
84
|
+
store = Mock(spec=OpenSearchDocumentStore, _multivector_embedding_dim=2)
|
|
85
|
+
await OpenSearchMultivectorEmbeddingRetriever(document_store=store).close_async()
|
|
86
|
+
store.close_async.assert_awaited_once_with()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_run():
|
|
90
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
91
|
+
store._client = Mock()
|
|
92
|
+
store._client.search.return_value = {
|
|
93
|
+
"hits": {
|
|
94
|
+
"hits": [
|
|
95
|
+
{"_source": {"id": "match"}, "_score": 2.0},
|
|
96
|
+
{"_source": {"id": "missing"}, "_score": 0.0},
|
|
97
|
+
]
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(
|
|
101
|
+
document_store=store,
|
|
102
|
+
filters={"field": "meta.kind", "operator": "==", "value": "article"},
|
|
103
|
+
filter_policy=FilterPolicy.MERGE,
|
|
104
|
+
)
|
|
105
|
+
result = retriever.run(
|
|
106
|
+
query_multivector_embedding=[[1.0, 0.0]], top_k=2, filters={"field": "meta.page", "operator": "==", "value": 2}
|
|
107
|
+
)
|
|
108
|
+
assert result["documents"] == [Document(id="match", score=2.0)]
|
|
109
|
+
body = store._client.search.call_args.kwargs["body"]
|
|
110
|
+
assert body["size"] == 2
|
|
111
|
+
script_score = body["query"]["script_score"]
|
|
112
|
+
assert script_score["script"]["params"]["query_vectors"] == [[1.0, 0.0]]
|
|
113
|
+
conditions = script_score["query"]["bool"]["filter"]["bool"]["must"]
|
|
114
|
+
assert {"term": {"kind": "article"}} in conditions
|
|
115
|
+
assert {"term": {"page": 2}} in conditions
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@pytest.mark.parametrize("raise_on_failure", [True, False])
|
|
119
|
+
def test_run_raise_on_failure(raise_on_failure, caplog):
|
|
120
|
+
store = Mock(spec=OpenSearchDocumentStore, _multivector_embedding_dim=2)
|
|
121
|
+
store._multivector_retrieval.side_effect = RuntimeError("Search failed")
|
|
122
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(document_store=store, raise_on_failure=raise_on_failure)
|
|
123
|
+
|
|
124
|
+
if raise_on_failure:
|
|
125
|
+
with pytest.raises(RuntimeError, match="Search failed"):
|
|
126
|
+
retriever.run(query_multivector_embedding=[[1.0, 0.0]])
|
|
127
|
+
else:
|
|
128
|
+
result = retriever.run(query_multivector_embedding=[[1.0, 0.0]])
|
|
129
|
+
assert result["documents"] == []
|
|
130
|
+
assert "Search failed" in caplog.text
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_run_with_runtime_document_store():
|
|
134
|
+
initial_store = Mock(spec=OpenSearchDocumentStore, _multivector_embedding_dim=2)
|
|
135
|
+
runtime_store = Mock(spec=OpenSearchDocumentStore, _multivector_embedding_dim=2)
|
|
136
|
+
runtime_store._multivector_retrieval.return_value = [Document(id="runtime")]
|
|
137
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(document_store=initial_store)
|
|
138
|
+
|
|
139
|
+
result = retriever.run(query_multivector_embedding=[[1.0, 0.0]], document_store=runtime_store)
|
|
140
|
+
|
|
141
|
+
assert result["documents"] == [Document(id="runtime")]
|
|
142
|
+
initial_store._multivector_retrieval.assert_not_called()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@pytest.mark.asyncio
|
|
146
|
+
async def test_run_async():
|
|
147
|
+
store = OpenSearchDocumentStore(multivector_embedding_dim=2)
|
|
148
|
+
store._async_client = AsyncMock()
|
|
149
|
+
store._async_client.search.return_value = {"hits": {"hits": [{"_source": {"id": "match"}, "_score": 2.0}]}}
|
|
150
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(document_store=store)
|
|
151
|
+
result = await retriever.run_async(query_multivector_embedding=[[1.0, 0.0]])
|
|
152
|
+
assert result["documents"] == [Document(id="match", score=2.0)]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@pytest.mark.asyncio
|
|
156
|
+
@pytest.mark.parametrize("raise_on_failure", [True, False])
|
|
157
|
+
async def test_run_async_raise_on_failure(raise_on_failure, caplog):
|
|
158
|
+
store = Mock(spec=OpenSearchDocumentStore, _multivector_embedding_dim=2)
|
|
159
|
+
store._multivector_retrieval_async.side_effect = RuntimeError("Search failed")
|
|
160
|
+
retriever = OpenSearchMultivectorEmbeddingRetriever(document_store=store, raise_on_failure=raise_on_failure)
|
|
161
|
+
|
|
162
|
+
if raise_on_failure:
|
|
163
|
+
with pytest.raises(RuntimeError, match="Search failed"):
|
|
164
|
+
await retriever.run_async(query_multivector_embedding=[[1.0, 0.0]])
|
|
165
|
+
else:
|
|
166
|
+
result = await retriever.run_async(query_multivector_embedding=[[1.0, 0.0]])
|
|
167
|
+
assert result["documents"] == []
|
|
168
|
+
assert "Search failed" in caplog.text
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{opensearch_haystack-8.3.0 → opensearch_haystack-9.1.0}/tests/test_metadata_retriever_async.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|