opensearch-haystack 8.2.0__tar.gz → 8.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/CHANGELOG.md +19 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/PKG-INFO +1 -1
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/pyproject.toml +1 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/document_stores/opensearch/document_store.py +49 -21
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/document_stores/opensearch/filters.py +5 -2
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_document_store.py +76 -62
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_document_store_async.py +110 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_filters.py +53 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/.gitignore +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/LICENSE +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/README.md +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/docker-compose.yml +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/__init__.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/conftest.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_auth.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_bm25_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_document_store_common.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_embedding_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_sql_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_utils.py +0 -0
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v8.2.0] - 2026-07-22
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- Opensearch - add methods to close resources (#3640)
|
|
8
|
+
|
|
9
|
+
### 🐛 Bug Fixes
|
|
10
|
+
|
|
11
|
+
- Adapt optimum, ragas, elasticsearch, opensearch to APIs removed in Haystack 3.0 (#3538)
|
|
12
|
+
|
|
13
|
+
### 🚜 Refactor
|
|
14
|
+
|
|
15
|
+
- Elasticsearch - split sync/async paths (#3633)
|
|
16
|
+
|
|
17
|
+
### 📚 Documentation
|
|
18
|
+
|
|
19
|
+
- Replace old haystack core imports with haystack_integrations paths (#3545)
|
|
20
|
+
|
|
21
|
+
|
|
3
22
|
## [integrations/opensearch-v8.1.2] - 2026-06-24
|
|
4
23
|
|
|
5
24
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version: 8.2.
|
|
3
|
+
Version: 8.2.1
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -2053,16 +2053,19 @@ class OpenSearchDocumentStore:
|
|
|
2053
2053
|
self,
|
|
2054
2054
|
metadata_field: str,
|
|
2055
2055
|
search_term: str | None = None,
|
|
2056
|
-
size: int | None =
|
|
2056
|
+
size: int | None = 10,
|
|
2057
2057
|
after: dict[str, Any] | None = None,
|
|
2058
2058
|
) -> tuple[list[str], dict[str, Any] | None]:
|
|
2059
2059
|
"""
|
|
2060
|
-
Returns unique values for a metadata field, optionally filtered by a search term
|
|
2060
|
+
Returns unique values for a metadata field, optionally filtered by a search term.
|
|
2061
2061
|
|
|
2062
2062
|
Uses composite aggregations for proper pagination beyond 10k results.
|
|
2063
2063
|
|
|
2064
2064
|
:param metadata_field: The metadata field to get unique values for.
|
|
2065
|
-
:param search_term: Optional
|
|
2065
|
+
:param search_term: Optional term to filter the returned values by, matching as a case-insensitive substring
|
|
2066
|
+
of the metadata field's own value (not the document content). NOTE: The matching is done with a server-side
|
|
2067
|
+
script to accomplish the substring matching on the value of the field and this operation is quite expensive
|
|
2068
|
+
for a large corpus.
|
|
2066
2069
|
:param size: The number of unique values to return per page. Defaults to 10000.
|
|
2067
2070
|
:param after: Optional pagination key from the previous response. Use None for the first page.
|
|
2068
2071
|
For subsequent pages, pass the `after_key` from the previous response.
|
|
@@ -2075,12 +2078,6 @@ class OpenSearchDocumentStore:
|
|
|
2075
2078
|
|
|
2076
2079
|
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2077
2080
|
|
|
2078
|
-
# filter by search_term if provided
|
|
2079
|
-
query: dict[str, Any] = {"match_all": {}}
|
|
2080
|
-
if search_term:
|
|
2081
|
-
# Use match_phrase for exact phrase matching to avoid tokenization issues
|
|
2082
|
-
query = {"match_phrase": {"content": search_term}}
|
|
2083
|
-
|
|
2084
2081
|
# Build composite aggregation for proper pagination
|
|
2085
2082
|
composite_agg: dict[str, Any] = {
|
|
2086
2083
|
"size": size,
|
|
@@ -2089,8 +2086,7 @@ class OpenSearchDocumentStore:
|
|
|
2089
2086
|
if after is not None:
|
|
2090
2087
|
composite_agg["after"] = after
|
|
2091
2088
|
|
|
2092
|
-
body = {
|
|
2093
|
-
"query": query,
|
|
2089
|
+
body: dict[str, Any] = {
|
|
2094
2090
|
"aggs": {
|
|
2095
2091
|
"unique_values": {
|
|
2096
2092
|
"composite": composite_agg,
|
|
@@ -2098,6 +2094,24 @@ class OpenSearchDocumentStore:
|
|
|
2098
2094
|
},
|
|
2099
2095
|
"size": 0, # we only need aggregations, not documents
|
|
2100
2096
|
}
|
|
2097
|
+
if search_term:
|
|
2098
|
+
# Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
|
|
2099
|
+
# standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
|
|
2100
|
+
# numeric ones. A doc-value script query works uniformly across field types by stringifying the
|
|
2101
|
+
# field's value, matching the case-insensitive substring semantics documented above. The term is
|
|
2102
|
+
# lower-cased once here rather than per-document in the script.
|
|
2103
|
+
body["query"] = {
|
|
2104
|
+
"script": {
|
|
2105
|
+
"script": {
|
|
2106
|
+
"source": (
|
|
2107
|
+
"def v = doc[params.field]; "
|
|
2108
|
+
"if (v.size() == 0) { return false; } "
|
|
2109
|
+
"return v.value.toString().toLowerCase().contains(params.term)"
|
|
2110
|
+
),
|
|
2111
|
+
"params": {"field": field_name, "term": search_term.lower()},
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
}
|
|
2101
2115
|
|
|
2102
2116
|
result = self._client.search(index=self._index, body=body)
|
|
2103
2117
|
aggregations = result.get("aggregations", {})
|
|
@@ -2123,12 +2137,15 @@ class OpenSearchDocumentStore:
|
|
|
2123
2137
|
after: dict[str, Any] | None = None,
|
|
2124
2138
|
) -> tuple[list[str], dict[str, Any] | None]:
|
|
2125
2139
|
"""
|
|
2126
|
-
Asynchronously returns unique values for a metadata field, optionally filtered by a search term
|
|
2140
|
+
Asynchronously returns unique values for a metadata field, optionally filtered by a search term.
|
|
2127
2141
|
|
|
2128
2142
|
Uses composite aggregations for proper pagination beyond 10k results.
|
|
2129
2143
|
|
|
2130
2144
|
:param metadata_field: The metadata field to get unique values for.
|
|
2131
|
-
:param search_term: Optional
|
|
2145
|
+
:param search_term: Optional term to filter the returned values by, matching as a case-insensitive substring
|
|
2146
|
+
of the metadata field's own value (not the document content). NOTE: The matching is done with a server-side
|
|
2147
|
+
script to accomplish the substring matching on the value of the field and this operation is quite expensive
|
|
2148
|
+
for a large corpus.
|
|
2132
2149
|
:param size: The number of unique values to return per page. Defaults to 10000.
|
|
2133
2150
|
:param after: Optional pagination key from the previous response. Use None for the first page.
|
|
2134
2151
|
For subsequent pages, pass the `after_key` from the previous response.
|
|
@@ -2141,12 +2158,6 @@ class OpenSearchDocumentStore:
|
|
|
2141
2158
|
|
|
2142
2159
|
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2143
2160
|
|
|
2144
|
-
# filter by search_term if provided
|
|
2145
|
-
query: dict[str, Any] = {"match_all": {}}
|
|
2146
|
-
if search_term:
|
|
2147
|
-
# Use match_phrase for exact phrase matching to avoid tokenization issues
|
|
2148
|
-
query = {"match_phrase": {"content": search_term}}
|
|
2149
|
-
|
|
2150
2161
|
# Build composite aggregation for proper pagination
|
|
2151
2162
|
composite_agg: dict[str, Any] = {
|
|
2152
2163
|
"size": size,
|
|
@@ -2155,8 +2166,7 @@ class OpenSearchDocumentStore:
|
|
|
2155
2166
|
if after is not None:
|
|
2156
2167
|
composite_agg["after"] = after
|
|
2157
2168
|
|
|
2158
|
-
body = {
|
|
2159
|
-
"query": query,
|
|
2169
|
+
body: dict[str, Any] = {
|
|
2160
2170
|
"aggs": {
|
|
2161
2171
|
"unique_values": {
|
|
2162
2172
|
"composite": composite_agg,
|
|
@@ -2164,6 +2174,24 @@ class OpenSearchDocumentStore:
|
|
|
2164
2174
|
},
|
|
2165
2175
|
"size": 0, # we only need aggregations, not documents
|
|
2166
2176
|
}
|
|
2177
|
+
if search_term:
|
|
2178
|
+
# Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
|
|
2179
|
+
# standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
|
|
2180
|
+
# numeric ones. A doc-value script query works uniformly across field types by stringifying the
|
|
2181
|
+
# field's value, matching the case-insensitive substring semantics documented above. The term is
|
|
2182
|
+
# lower-cased once here rather than per-document in the script.
|
|
2183
|
+
body["query"] = {
|
|
2184
|
+
"script": {
|
|
2185
|
+
"script": {
|
|
2186
|
+
"source": (
|
|
2187
|
+
"def v = doc[params.field]; "
|
|
2188
|
+
"if (v.size() == 0) { return false; } "
|
|
2189
|
+
"return v.value.toString().toLowerCase().contains(params.term)"
|
|
2190
|
+
),
|
|
2191
|
+
"params": {"field": field_name, "term": search_term.lower()},
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2167
2195
|
|
|
2168
2196
|
result = await self._async_client.search(index=self._index, body=body)
|
|
2169
2197
|
aggregations = result.get("aggregations", {})
|
|
@@ -93,7 +93,7 @@ def _group_nested_conditions(
|
|
|
93
93
|
inner.append(_parse_logical_condition(c, nested_fields=None))
|
|
94
94
|
else:
|
|
95
95
|
inner.append(_parse_comparison_condition(c, nested_fields))
|
|
96
|
-
if len(inner) > 1:
|
|
96
|
+
if len(inner) > 1 and operator != "OR":
|
|
97
97
|
inner = _normalize_ranges(inner)
|
|
98
98
|
if len(inner) == 1:
|
|
99
99
|
conditions.append({"nested": {"path": path, "query": inner[0]}})
|
|
@@ -120,7 +120,10 @@ def _parse_logical_condition(condition: dict[str, Any], nested_fields: set[str]
|
|
|
120
120
|
else:
|
|
121
121
|
conditions = [_parse_comparison_condition(c, nested_fields) for c in condition["conditions"]]
|
|
122
122
|
|
|
123
|
-
|
|
123
|
+
# Range merging is only valid when the conditions are AND-combined (AND, and the
|
|
124
|
+
# inner `must` of NOT). Merging same-field ranges under OR would collapse e.g.
|
|
125
|
+
# `price < 10 OR price > 100` into a single impossible range, matching nothing.
|
|
126
|
+
if len(conditions) > 1 and operator != "OR":
|
|
124
127
|
conditions = _normalize_ranges(conditions)
|
|
125
128
|
if operator == "AND":
|
|
126
129
|
return {"bool": {"must": conditions}}
|
|
@@ -298,83 +298,67 @@ def test_bm25_retrieval_reraises_other_transport_errors(_mock_opensearch_client)
|
|
|
298
298
|
assert store._client.search.call_count == 1
|
|
299
299
|
|
|
300
300
|
|
|
301
|
-
@pytest.mark.asyncio
|
|
302
|
-
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
303
301
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
302
|
+
def test_get_metadata_field_unique_values_no_search_term_builds_query_without_include(_mock_opensearch_client):
|
|
303
|
+
"""Composite aggregation terms sources don't support `include`/`exclude`, so with no search_term
|
|
304
|
+
the request body must have neither a `query` filter nor an `include` clause on the terms source."""
|
|
307
305
|
store = OpenSearchDocumentStore(hosts="testhost")
|
|
308
|
-
store.
|
|
306
|
+
store._client = MagicMock()
|
|
307
|
+
store._client.search.return_value = {"aggregations": {"unique_values": {"buckets": []}}}
|
|
309
308
|
|
|
310
|
-
|
|
311
|
-
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
312
|
-
)
|
|
313
|
-
store._async_client.search.side_effect = [
|
|
314
|
-
too_many_clauses_error,
|
|
315
|
-
{"hits": {"hits": []}},
|
|
316
|
-
]
|
|
309
|
+
store.get_metadata_field_unique_values("category", None, 10)
|
|
317
310
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
assert
|
|
322
|
-
|
|
323
|
-
assert second_call_body["query"]["bool"]["must"][0]["multi_match"]["fuzziness"] == 0
|
|
324
|
-
assert "Retrying with fuzziness=0" in caplog.text
|
|
311
|
+
body = store._client.search.call_args.kwargs["body"]
|
|
312
|
+
assert "query" not in body
|
|
313
|
+
terms_source = body["aggs"]["unique_values"]["composite"]["sources"][0]["category"]["terms"]
|
|
314
|
+
assert "include" not in terms_source
|
|
315
|
+
assert terms_source["field"] == "category"
|
|
325
316
|
|
|
326
317
|
|
|
327
|
-
@pytest.mark.asyncio
|
|
328
|
-
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
329
318
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
330
|
-
|
|
319
|
+
def test_get_metadata_field_unique_values_search_term_filters_on_field_value_not_content(_mock_opensearch_client):
|
|
331
320
|
store = OpenSearchDocumentStore(hosts="testhost")
|
|
332
|
-
store.
|
|
333
|
-
|
|
334
|
-
too_many_clauses_error = TransportError(
|
|
335
|
-
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
336
|
-
)
|
|
337
|
-
store._async_client.search.side_effect = too_many_clauses_error
|
|
321
|
+
store._client = MagicMock()
|
|
322
|
+
store._client.search.return_value = {"aggregations": {"unique_values": {"buckets": []}}}
|
|
338
323
|
|
|
339
|
-
|
|
340
|
-
await store._bm25_retrieval_async("a very long query", fuzziness=0)
|
|
324
|
+
store.get_metadata_field_unique_values("category", "needle", 10)
|
|
341
325
|
|
|
342
|
-
|
|
326
|
+
body = store._client.search.call_args.kwargs["body"]
|
|
327
|
+
# Composite aggregation terms sources don't support `include`/`exclude`, so the substring match
|
|
328
|
+
# against the aggregated field's own value is applied as a query-level doc-value script filter instead.
|
|
329
|
+
terms_source = body["aggs"]["unique_values"]["composite"]["sources"][0]["category"]["terms"]
|
|
330
|
+
assert "include" not in terms_source
|
|
331
|
+
script = body["query"]["script"]["script"]
|
|
332
|
+
assert script["params"] == {"field": "category", "term": "needle"}
|
|
333
|
+
assert "contains(params.term)" in script["source"]
|
|
343
334
|
|
|
344
335
|
|
|
345
|
-
@pytest.mark.asyncio
|
|
346
|
-
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
347
336
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
348
|
-
|
|
337
|
+
def test_get_metadata_field_unique_values_search_term_is_lowercased_for_case_insensitivity(_mock_opensearch_client):
|
|
349
338
|
store = OpenSearchDocumentStore(hosts="testhost")
|
|
350
|
-
store.
|
|
351
|
-
|
|
352
|
-
too_many_clauses_error = TransportError(
|
|
353
|
-
500, "search_phase_execution_exception", "too_many_clauses: maxClauseCount is set to 1024"
|
|
354
|
-
)
|
|
355
|
-
store._async_client.search.side_effect = too_many_clauses_error
|
|
339
|
+
store._client = MagicMock()
|
|
340
|
+
store._client.search.return_value = {"aggregations": {"unique_values": {"buckets": []}}}
|
|
356
341
|
|
|
357
|
-
|
|
358
|
-
with pytest.raises(TransportError):
|
|
359
|
-
await store._bm25_retrieval_async("a very long query", fuzziness="AUTO", custom_query=custom_query)
|
|
342
|
+
store.get_metadata_field_unique_values("category", "NeEdLe", 10)
|
|
360
343
|
|
|
361
|
-
|
|
344
|
+
body = store._client.search.call_args.kwargs["body"]
|
|
345
|
+
script = body["query"]["script"]["script"]
|
|
346
|
+
assert script["params"]["term"] == "needle"
|
|
347
|
+
assert "toLowerCase().contains(params.term)" in script["source"]
|
|
362
348
|
|
|
363
349
|
|
|
364
|
-
@pytest.mark.asyncio
|
|
365
|
-
@patch("haystack_integrations.document_stores.opensearch.document_store.AsyncOpenSearch")
|
|
366
350
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
367
|
-
|
|
351
|
+
def test_get_metadata_field_unique_values_search_term_with_regex_metacharacters(_mock_opensearch_client):
|
|
352
|
+
"""search_term is matched as a literal substring (via a doc-value script), so regex metacharacters
|
|
353
|
+
in the term must be passed through as-is, not treated as a regex pattern."""
|
|
368
354
|
store = OpenSearchDocumentStore(hosts="testhost")
|
|
369
|
-
store.
|
|
370
|
-
|
|
371
|
-
other_error = TransportError(500, "parsing_exception", {"error": {"reason": "some other error"}})
|
|
372
|
-
store._async_client.search.side_effect = other_error
|
|
355
|
+
store._client = MagicMock()
|
|
356
|
+
store._client.search.return_value = {"aggregations": {"unique_values": {"buckets": []}}}
|
|
373
357
|
|
|
374
|
-
|
|
375
|
-
await store._bm25_retrieval_async("some query", fuzziness="AUTO")
|
|
358
|
+
store.get_metadata_field_unique_values("category", "a.b*c", 10)
|
|
376
359
|
|
|
377
|
-
|
|
360
|
+
body = store._client.search.call_args.kwargs["body"]
|
|
361
|
+
assert body["query"]["script"]["script"]["params"]["term"] == "a.b*c"
|
|
378
362
|
|
|
379
363
|
|
|
380
364
|
@patch("haystack_integrations.document_stores.opensearch.document_store.OpenSearch")
|
|
@@ -1228,13 +1212,20 @@ class TestDocumentStore(
|
|
|
1228
1212
|
# Should have no more results
|
|
1229
1213
|
assert after_key_page2 is None
|
|
1230
1214
|
|
|
1231
|
-
# Test with search term - filter by
|
|
1215
|
+
# Test with search term - filter by the metadata field's own VALUE matching "Python"
|
|
1216
|
+
# ("language" values are "Python"/"Java"/"JavaScript", so searching "Python" against
|
|
1217
|
+
# the "category" field's values ("A"/"B"/"C") should match nothing)
|
|
1232
1218
|
unique_values_filtered, _ = document_store.get_metadata_field_unique_values("meta.category", "Python", 10)
|
|
1233
|
-
assert set(unique_values_filtered) ==
|
|
1219
|
+
assert set(unique_values_filtered) == set()
|
|
1234
1220
|
|
|
1235
|
-
#
|
|
1236
|
-
|
|
1237
|
-
|
|
1221
|
+
# Searching "language" values themselves for the substring "Java" must match both
|
|
1222
|
+
# "Java" and "JavaScript" (substring match on the field's own value).
|
|
1223
|
+
unique_languages_filtered, _ = document_store.get_metadata_field_unique_values("meta.language", "Java", 10)
|
|
1224
|
+
assert set(unique_languages_filtered) == {"Java", "JavaScript"}
|
|
1225
|
+
|
|
1226
|
+
# Case-insensitivity: a lowercase search term must still match the differently-cased values above.
|
|
1227
|
+
unique_languages_lower, _ = document_store.get_metadata_field_unique_values("meta.language", "java", 10)
|
|
1228
|
+
assert set(unique_languages_lower) == {"Java", "JavaScript"}
|
|
1238
1229
|
|
|
1239
1230
|
# Test with integer values
|
|
1240
1231
|
int_docs = [
|
|
@@ -1247,9 +1238,32 @@ class TestDocumentStore(
|
|
|
1247
1238
|
unique_priorities, _ = document_store.get_metadata_field_unique_values("meta.priority", None, 10)
|
|
1248
1239
|
assert set(unique_priorities) == {"1", "2", "3"}
|
|
1249
1240
|
|
|
1250
|
-
#
|
|
1241
|
+
# search_term now matches against the field's own value, not the content, so searching
|
|
1242
|
+
# for content text ("Doc 1") against the "priority" field's values ("1"/"2"/"3") matches nothing.
|
|
1251
1243
|
unique_priorities_filtered, _ = document_store.get_metadata_field_unique_values("meta.priority", "Doc 1", 10)
|
|
1252
|
-
assert set(unique_priorities_filtered) ==
|
|
1244
|
+
assert set(unique_priorities_filtered) == set()
|
|
1245
|
+
|
|
1246
|
+
# search_term matching the field's own value (e.g. "1") does match.
|
|
1247
|
+
unique_priorities_by_value, _ = document_store.get_metadata_field_unique_values("meta.priority", "1", 10)
|
|
1248
|
+
assert set(unique_priorities_by_value) == {"1"}
|
|
1249
|
+
|
|
1250
|
+
# Prove the semantic change explicitly with a document whose CONTENT contains the search
|
|
1251
|
+
# term but whose target metadata field value does NOT: it must now be EXCLUDED.
|
|
1252
|
+
content_match_docs = [
|
|
1253
|
+
Document(content="This mentions needle in the text", meta={"topic": "unrelated"}),
|
|
1254
|
+
]
|
|
1255
|
+
document_store.write_documents(content_match_docs)
|
|
1256
|
+
unique_topics_content_only, _ = document_store.get_metadata_field_unique_values("meta.topic", "needle", 10)
|
|
1257
|
+
assert set(unique_topics_content_only) == set()
|
|
1258
|
+
|
|
1259
|
+
# And a document whose metadata field VALUE contains the search term but whose content does
|
|
1260
|
+
# NOT: it must now be INCLUDED.
|
|
1261
|
+
value_match_docs = [
|
|
1262
|
+
Document(content="Nothing special here", meta={"topic": "needle-in-haystack"}),
|
|
1263
|
+
]
|
|
1264
|
+
document_store.write_documents(value_match_docs)
|
|
1265
|
+
unique_topics_value_only, _ = document_store.get_metadata_field_unique_values("meta.topic", "needle", 10)
|
|
1266
|
+
assert set(unique_topics_value_only) == {"needle-in-haystack"}
|
|
1253
1267
|
|
|
1254
1268
|
def test_write_with_routing(self, document_store: OpenSearchDocumentStore):
|
|
1255
1269
|
"""Test writing documents with routing metadata"""
|
|
@@ -807,3 +807,113 @@ class TestDocumentStoreAsync(
|
|
|
807
807
|
client.indices.delete_alias(index=document_store._index, name=alias_name)
|
|
808
808
|
if alias_store._async_client:
|
|
809
809
|
await alias_store._async_client.close()
|
|
810
|
+
|
|
811
|
+
@pytest.mark.asyncio
|
|
812
|
+
async def test_get_metadata_field_unique_values_async(self, document_store: OpenSearchDocumentStore):
|
|
813
|
+
# Test with string values
|
|
814
|
+
docs = [
|
|
815
|
+
Document(content="Python programming", meta={"category": "A", "language": "Python"}),
|
|
816
|
+
Document(content="Java programming", meta={"category": "B", "language": "Java"}),
|
|
817
|
+
Document(content="Python scripting", meta={"category": "A", "language": "Python"}),
|
|
818
|
+
Document(content="JavaScript development", meta={"category": "C", "language": "JavaScript"}),
|
|
819
|
+
Document(content="Python data science", meta={"category": "A", "language": "Python"}),
|
|
820
|
+
Document(content="Java backend", meta={"category": "B", "language": "Java"}),
|
|
821
|
+
]
|
|
822
|
+
await document_store.write_documents_async(docs)
|
|
823
|
+
|
|
824
|
+
# Test getting all unique values without search term
|
|
825
|
+
unique_values, after_key = await document_store.get_metadata_field_unique_values_async(
|
|
826
|
+
"meta.category", None, 10
|
|
827
|
+
)
|
|
828
|
+
assert set(unique_values) == {"A", "B", "C"}
|
|
829
|
+
# after_key should be None when all results are returned
|
|
830
|
+
assert after_key is None
|
|
831
|
+
|
|
832
|
+
# Test with "meta." prefix
|
|
833
|
+
unique_languages, _ = await document_store.get_metadata_field_unique_values_async("meta.language", None, 10)
|
|
834
|
+
assert set(unique_languages) == {"Python", "Java", "JavaScript"}
|
|
835
|
+
|
|
836
|
+
# Test pagination - first page
|
|
837
|
+
unique_values_page1, after_key_page1 = await document_store.get_metadata_field_unique_values_async(
|
|
838
|
+
"meta.category", None, 2
|
|
839
|
+
)
|
|
840
|
+
assert len(unique_values_page1) == 2
|
|
841
|
+
assert all(val in ["A", "B", "C"] for val in unique_values_page1)
|
|
842
|
+
# Should have an after_key for pagination
|
|
843
|
+
assert after_key_page1 is not None
|
|
844
|
+
|
|
845
|
+
# Test pagination - second page using after_key
|
|
846
|
+
unique_values_page2, after_key_page2 = await document_store.get_metadata_field_unique_values_async(
|
|
847
|
+
"meta.category", None, 2, after=after_key_page1
|
|
848
|
+
)
|
|
849
|
+
assert len(unique_values_page2) == 1
|
|
850
|
+
assert unique_values_page2[0] in ["A", "B", "C"]
|
|
851
|
+
# Should have no more results
|
|
852
|
+
assert after_key_page2 is None
|
|
853
|
+
|
|
854
|
+
# Test with search term - filter by the metadata field's own VALUE matching "Python"
|
|
855
|
+
# ("language" values are "Python"/"Java"/"JavaScript", so searching "Python" against
|
|
856
|
+
# the "category" field's values ("A"/"B"/"C") should match nothing)
|
|
857
|
+
unique_values_filtered, _ = await document_store.get_metadata_field_unique_values_async(
|
|
858
|
+
"meta.category", "Python", 10
|
|
859
|
+
)
|
|
860
|
+
assert set(unique_values_filtered) == set()
|
|
861
|
+
|
|
862
|
+
# Searching "language" values themselves for the substring "Java" must match both
|
|
863
|
+
# "Java" and "JavaScript" (substring match on the field's own value).
|
|
864
|
+
unique_languages_filtered, _ = await document_store.get_metadata_field_unique_values_async(
|
|
865
|
+
"meta.language", "Java", 10
|
|
866
|
+
)
|
|
867
|
+
assert set(unique_languages_filtered) == {"Java", "JavaScript"}
|
|
868
|
+
|
|
869
|
+
# Case-insensitivity: a lowercase search term must still match the differently-cased values above.
|
|
870
|
+
unique_languages_lower, _ = await document_store.get_metadata_field_unique_values_async(
|
|
871
|
+
"meta.language", "java", 10
|
|
872
|
+
)
|
|
873
|
+
assert set(unique_languages_lower) == {"Java", "JavaScript"}
|
|
874
|
+
|
|
875
|
+
# Test with integer values
|
|
876
|
+
int_docs = [
|
|
877
|
+
Document(content="Doc 1", meta={"priority": 1}),
|
|
878
|
+
Document(content="Doc 2", meta={"priority": 2}),
|
|
879
|
+
Document(content="Doc 3", meta={"priority": 1}),
|
|
880
|
+
Document(content="Doc 4", meta={"priority": 3}),
|
|
881
|
+
]
|
|
882
|
+
await document_store.write_documents_async(int_docs)
|
|
883
|
+
unique_priorities, _ = await document_store.get_metadata_field_unique_values_async("meta.priority", None, 10)
|
|
884
|
+
assert set(unique_priorities) == {"1", "2", "3"}
|
|
885
|
+
|
|
886
|
+
# search_term now matches against the field's own value, not the content, so searching
|
|
887
|
+
# for content text ("Doc 1") against the "priority" field's values ("1"/"2"/"3") matches nothing.
|
|
888
|
+
unique_priorities_filtered, _ = await document_store.get_metadata_field_unique_values_async(
|
|
889
|
+
"meta.priority", "Doc 1", 10
|
|
890
|
+
)
|
|
891
|
+
assert set(unique_priorities_filtered) == set()
|
|
892
|
+
|
|
893
|
+
# search_term matching the field's own value (e.g. "1") does match.
|
|
894
|
+
unique_priorities_by_value, _ = await document_store.get_metadata_field_unique_values_async(
|
|
895
|
+
"meta.priority", "1", 10
|
|
896
|
+
)
|
|
897
|
+
assert set(unique_priorities_by_value) == {"1"}
|
|
898
|
+
|
|
899
|
+
# Prove the semantic change explicitly with a document whose CONTENT contains the search
|
|
900
|
+
# term but whose target metadata field value does NOT: it must now be EXCLUDED.
|
|
901
|
+
content_match_docs = [
|
|
902
|
+
Document(content="This mentions needle in the text", meta={"topic": "unrelated"}),
|
|
903
|
+
]
|
|
904
|
+
await document_store.write_documents_async(content_match_docs)
|
|
905
|
+
unique_topics_content_only, _ = await document_store.get_metadata_field_unique_values_async(
|
|
906
|
+
"meta.topic", "needle", 10
|
|
907
|
+
)
|
|
908
|
+
assert set(unique_topics_content_only) == set()
|
|
909
|
+
|
|
910
|
+
# And a document whose metadata field VALUE contains the search term but whose content does
|
|
911
|
+
# NOT: it must now be INCLUDED.
|
|
912
|
+
value_match_docs = [
|
|
913
|
+
Document(content="Nothing special here", meta={"topic": "needle-in-haystack"}),
|
|
914
|
+
]
|
|
915
|
+
await document_store.write_documents_async(value_match_docs)
|
|
916
|
+
unique_topics_value_only, _ = await document_store.get_metadata_field_unique_values_async(
|
|
917
|
+
"meta.topic", "needle", 10
|
|
918
|
+
)
|
|
919
|
+
assert set(unique_topics_value_only) == {"needle-in-haystack"}
|
|
@@ -500,6 +500,59 @@ def test_normalize_ranges():
|
|
|
500
500
|
]
|
|
501
501
|
|
|
502
502
|
|
|
503
|
+
def test_or_of_ranges_on_same_field_are_not_merged():
|
|
504
|
+
# `price < 10 OR price > 100` must stay two separate range clauses under `should`.
|
|
505
|
+
# Merging them into a single {"lt": 10, "gt": 100} range means price < 10 AND
|
|
506
|
+
# price > 100, which matches no document.
|
|
507
|
+
filters = {
|
|
508
|
+
"operator": "OR",
|
|
509
|
+
"conditions": [
|
|
510
|
+
{"field": "meta.price", "operator": "<", "value": 10},
|
|
511
|
+
{"field": "meta.price", "operator": ">", "value": 100},
|
|
512
|
+
],
|
|
513
|
+
}
|
|
514
|
+
assert normalize_filters(filters) == {
|
|
515
|
+
"bool": {
|
|
516
|
+
"should": [
|
|
517
|
+
{"range": {"price": {"lt": 10}}},
|
|
518
|
+
{"range": {"price": {"gt": 100}}},
|
|
519
|
+
]
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def test_or_of_ranges_on_same_nested_field_are_not_merged():
|
|
525
|
+
# Same range-merge bug inside a nested-field logical group: an OR of two ranges on
|
|
526
|
+
# the same nested sub-field must stay two separate range clauses, not collapse into
|
|
527
|
+
# a single impossible range.
|
|
528
|
+
filters = {
|
|
529
|
+
"operator": "OR",
|
|
530
|
+
"conditions": [
|
|
531
|
+
{"field": "meta.refs.score", "operator": "<", "value": 10},
|
|
532
|
+
{"field": "meta.refs.score", "operator": ">", "value": 100},
|
|
533
|
+
],
|
|
534
|
+
}
|
|
535
|
+
assert normalize_filters(filters, nested_fields={"refs"}) == {
|
|
536
|
+
"bool": {
|
|
537
|
+
"should": [
|
|
538
|
+
{
|
|
539
|
+
"nested": {
|
|
540
|
+
"path": "refs",
|
|
541
|
+
"query": {
|
|
542
|
+
"bool": {
|
|
543
|
+
"should": [
|
|
544
|
+
{"range": {"refs.score": {"lt": 10}}},
|
|
545
|
+
{"range": {"refs.score": {"gt": 100}}},
|
|
546
|
+
]
|
|
547
|
+
}
|
|
548
|
+
},
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
]
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
|
|
503
556
|
@pytest.mark.parametrize("filters, nested_fields, expected", nested_filters_data)
|
|
504
557
|
def test_normalize_filters_with_nested_fields(filters, nested_fields, expected):
|
|
505
558
|
result = normalize_filters(filters, nested_fields=nested_fields)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_metadata_retriever_async.py
RENAMED
|
File without changes
|
{opensearch_haystack-8.2.0 → opensearch_haystack-8.2.1}/tests/test_open_search_hybrid_retriever.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|