opensearch-haystack 7.1.0__tar.gz → 7.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/CHANGELOG.md +37 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/PKG-INFO +1 -1
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/document_stores/opensearch/document_store.py +147 -27
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/document_stores/opensearch/filters.py +97 -7
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/conftest.py +48 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_auth.py +2 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_bm25_retriever.py +1 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_document_store.py +403 -10
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_document_store_async.py +57 -9
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_embedding_retriever.py +1 -0
- opensearch_haystack-7.2.1/tests/test_filters.py +605 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_open_search_hybrid_retriever.py +1 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_sql_retriever.py +45 -1
- opensearch_haystack-7.1.0/tests/test_filters.py +0 -247
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/.gitignore +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/LICENSE +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/README.md +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/docker-compose.yml +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/pyproject.toml +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/document_stores/opensearch/auth.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/__init__.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-7.1.0 → opensearch_haystack-7.2.1}/tests/test_metadata_retriever_async.py +0 -0
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v7.2.0] - 2026-04-10
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- *(opensearch)* Add support for nested fields in OpenSearchDocumentStore (#3117)
|
|
8
|
+
|
|
9
|
+
### 🐛 Bug Fixes
|
|
10
|
+
|
|
11
|
+
- Replace in-place dataclass mutations in document stores (#3114)
|
|
12
|
+
|
|
13
|
+
### 🧪 Testing
|
|
14
|
+
|
|
15
|
+
- Better categorize some Document Stores tests (#3085)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
## [integrations/opensearch-v7.1.0] - 2026-04-01
|
|
19
|
+
|
|
20
|
+
### 🚀 Features
|
|
21
|
+
|
|
22
|
+
- *(opensearch)* Handle errors due to clause overflow caused by fuzziness (#3068)
|
|
23
|
+
|
|
24
|
+
### 🚜 Refactor
|
|
25
|
+
|
|
26
|
+
- Weaviate, chroma, elasticsearch, opensearch, azure_ai_search use `_normalize_metadata_field_name` from haystack.utils (#2953)
|
|
27
|
+
|
|
28
|
+
### 🧪 Testing
|
|
29
|
+
|
|
30
|
+
- `OpensearchDocumentStore` use more Mixin tests (#3002)
|
|
31
|
+
- Test compatible integrations with python 3.14; update pyproject (#3001)
|
|
32
|
+
- Track test coverage for all integrations (#3065)
|
|
33
|
+
|
|
34
|
+
### 🧹 Chores
|
|
35
|
+
|
|
36
|
+
- Add ANN ruff ruleset to llama_cpp, llama_stack, mcp, meta_llama, mistral, mongodb_atlas, nvidia, ollama, openrouter, opensearch (#2991)
|
|
37
|
+
- Enforce ruff docstring rules in integrations 31-40 (openrouter, opensearch, optimum, paddleocr, pgvector, pinecone, pyversity, qdrant, ragas, snowflake) (#3011)
|
|
38
|
+
|
|
39
|
+
|
|
3
40
|
## [integrations/opensearch-v7.0.0] - 2026-03-19
|
|
4
41
|
|
|
5
42
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version: 7.1
|
|
3
|
+
Version: 7.2.1
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
# ruff: noqa: FBT001, FBT002 boolean-type-hint-positional-argument and boolean-default-value-positional-argument
|
|
6
6
|
|
|
7
7
|
from collections.abc import Generator, Mapping
|
|
8
|
+
from dataclasses import replace
|
|
8
9
|
from math import exp
|
|
9
10
|
from typing import Any, Literal
|
|
10
11
|
|
|
@@ -14,7 +15,7 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumen
|
|
|
14
15
|
from haystack.document_stores.types import DuplicatePolicy
|
|
15
16
|
from haystack.utils.auth import Secret
|
|
16
17
|
from haystack.utils.misc import _normalize_metadata_field_name
|
|
17
|
-
from opensearchpy import AsyncHttpConnection, AsyncOpenSearch, OpenSearch, TransportError
|
|
18
|
+
from opensearchpy import AsyncHttpConnection, AsyncOpenSearch, OpenSearch, RequestError, TransportError
|
|
18
19
|
from opensearchpy.helpers import async_bulk, bulk
|
|
19
20
|
|
|
20
21
|
from haystack_integrations.document_stores.opensearch.auth import AsyncAWSAuth, AWSAuth
|
|
@@ -90,6 +91,7 @@ class OpenSearchDocumentStore:
|
|
|
90
91
|
use_ssl: bool | None = None,
|
|
91
92
|
verify_certs: bool | None = None,
|
|
92
93
|
timeout: int | None = None,
|
|
94
|
+
nested_fields: list[str] | Literal["*"] | None = None,
|
|
93
95
|
**kwargs: Any,
|
|
94
96
|
) -> None:
|
|
95
97
|
"""
|
|
@@ -127,6 +129,12 @@ class OpenSearchDocumentStore:
|
|
|
127
129
|
:param use_ssl: Whether to use SSL. Defaults to None
|
|
128
130
|
:param verify_certs: Whether to verify certificates. Defaults to None
|
|
129
131
|
:param timeout: Timeout in seconds. Defaults to None
|
|
132
|
+
:param nested_fields: List of metadata field paths (without the `meta.` prefix) that should be mapped
|
|
133
|
+
as OpenSearch `nested` type, enabling multi-condition filtering on array-of-objects fields.
|
|
134
|
+
Pass `"*"` to auto-detect `list[dict]` fields and map them as nested from
|
|
135
|
+
the first `write_documents` batch.
|
|
136
|
+
When the index already exists, nested fields are discovered from the live mapping.
|
|
137
|
+
Defaults to None (no nested support).
|
|
130
138
|
:param **kwargs: Optional arguments that ``OpenSearch`` takes. For the full list of supported kwargs,
|
|
131
139
|
see the [official OpenSearch reference](https://opensearch-project.github.io/opensearch-py/api-ref/clients/opensearch_client.html)
|
|
132
140
|
"""
|
|
@@ -136,6 +144,7 @@ class OpenSearchDocumentStore:
|
|
|
136
144
|
self._embedding_dim = embedding_dim
|
|
137
145
|
self._return_embedding = return_embedding
|
|
138
146
|
self._method = method
|
|
147
|
+
self._nested_fields = nested_fields
|
|
139
148
|
self._mappings = mappings or self._get_default_mappings()
|
|
140
149
|
self._settings = settings
|
|
141
150
|
self._create_index = create_index
|
|
@@ -145,6 +154,10 @@ class OpenSearchDocumentStore:
|
|
|
145
154
|
self._timeout = timeout
|
|
146
155
|
self._kwargs = kwargs
|
|
147
156
|
|
|
157
|
+
self._resolved_nested_fields: set[str] = set()
|
|
158
|
+
if nested_fields and nested_fields != "*":
|
|
159
|
+
self._resolved_nested_fields = set(nested_fields)
|
|
160
|
+
|
|
148
161
|
# Client is initialized lazily to prevent side effects when
|
|
149
162
|
# the document store is instantiated.
|
|
150
163
|
self._client: OpenSearch | None = None
|
|
@@ -168,6 +181,9 @@ class OpenSearchDocumentStore:
|
|
|
168
181
|
}
|
|
169
182
|
if self._method:
|
|
170
183
|
default_mappings["properties"]["embedding"]["method"] = self._method
|
|
184
|
+
if self._nested_fields and self._nested_fields != "*":
|
|
185
|
+
for field in self._nested_fields:
|
|
186
|
+
default_mappings["properties"][field] = {"type": "nested"}
|
|
171
187
|
return default_mappings
|
|
172
188
|
|
|
173
189
|
def create_index(
|
|
@@ -234,6 +250,7 @@ class OpenSearchDocumentStore:
|
|
|
234
250
|
use_ssl=self._use_ssl,
|
|
235
251
|
verify_certs=self._verify_certs,
|
|
236
252
|
timeout=self._timeout,
|
|
253
|
+
nested_fields=self._nested_fields,
|
|
237
254
|
**self._kwargs,
|
|
238
255
|
)
|
|
239
256
|
|
|
@@ -308,6 +325,71 @@ class OpenSearchDocumentStore:
|
|
|
308
325
|
self._initialized = True
|
|
309
326
|
await self._ensure_index_exists_async()
|
|
310
327
|
|
|
328
|
+
@staticmethod
|
|
329
|
+
def _extract_nested_fields_from_mapping(mapping_properties: dict[str, Any]) -> set[str]:
|
|
330
|
+
return {name for name, defn in mapping_properties.items() if defn.get("type") == "nested"}
|
|
331
|
+
|
|
332
|
+
def _populate_nested_fields_from_mapping(self, mapping_properties: dict[str, Any]) -> None:
|
|
333
|
+
live_nested = self._extract_nested_fields_from_mapping(mapping_properties)
|
|
334
|
+
self._resolved_nested_fields.update(live_nested)
|
|
335
|
+
|
|
336
|
+
if self._nested_fields and self._nested_fields != "*":
|
|
337
|
+
declared_but_not_nested = set(self._nested_fields) - live_nested
|
|
338
|
+
for field in declared_but_not_nested:
|
|
339
|
+
logger.warning(
|
|
340
|
+
"Field '{field}' was declared as nested but is not mapped as nested in the existing "
|
|
341
|
+
"index '{index}'. A full re-index is required to change a field from object to nested.",
|
|
342
|
+
field=field,
|
|
343
|
+
index=self._index,
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
@staticmethod
|
|
347
|
+
def _detect_nested_fields_from_documents(documents: list[Document]) -> set[str]:
|
|
348
|
+
nested: set[str] = set()
|
|
349
|
+
for doc in documents:
|
|
350
|
+
if not doc.meta:
|
|
351
|
+
continue
|
|
352
|
+
for key, value in doc.meta.items():
|
|
353
|
+
if isinstance(value, list) and value and all(isinstance(elem, dict) for elem in value):
|
|
354
|
+
nested.add(key)
|
|
355
|
+
return nested
|
|
356
|
+
|
|
357
|
+
def _update_mapping_for_nested_fields(self, new_nested_fields: set[str]) -> None:
|
|
358
|
+
assert self._client is not None
|
|
359
|
+
for field in new_nested_fields:
|
|
360
|
+
try:
|
|
361
|
+
self._client.indices.put_mapping(
|
|
362
|
+
index=self._index,
|
|
363
|
+
body={"properties": {field: {"type": "nested"}}},
|
|
364
|
+
)
|
|
365
|
+
self._resolved_nested_fields.add(field)
|
|
366
|
+
except RequestError as e:
|
|
367
|
+
logger.warning(
|
|
368
|
+
"Could not map field '{field}' as nested in index '{index}': {error}. "
|
|
369
|
+
"If it was previously mapped as object, a full re-index is required.",
|
|
370
|
+
field=field,
|
|
371
|
+
index=self._index,
|
|
372
|
+
error=str(e),
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
async def _update_mapping_for_nested_fields_async(self, new_nested_fields: set[str]) -> None:
|
|
376
|
+
assert self._async_client is not None
|
|
377
|
+
for field in new_nested_fields:
|
|
378
|
+
try:
|
|
379
|
+
await self._async_client.indices.put_mapping(
|
|
380
|
+
index=self._index,
|
|
381
|
+
body={"properties": {field: {"type": "nested"}}},
|
|
382
|
+
)
|
|
383
|
+
self._resolved_nested_fields.add(field)
|
|
384
|
+
except RequestError as e:
|
|
385
|
+
logger.warning(
|
|
386
|
+
"Could not map field '{field}' as nested in index '{index}': {error}. "
|
|
387
|
+
"If it was previously mapped as object, a full re-index is required.",
|
|
388
|
+
field=field,
|
|
389
|
+
index=self._index,
|
|
390
|
+
error=str(e),
|
|
391
|
+
)
|
|
392
|
+
|
|
311
393
|
async def _ensure_index_exists_async(self) -> None:
|
|
312
394
|
assert self._async_client is not None
|
|
313
395
|
|
|
@@ -317,6 +399,9 @@ class OpenSearchDocumentStore:
|
|
|
317
399
|
"`settings` values will be ignored.",
|
|
318
400
|
index=self._index,
|
|
319
401
|
)
|
|
402
|
+
mapping = await self._async_client.indices.get_mapping(index=self._index)
|
|
403
|
+
properties = mapping[self._index]["mappings"].get("properties", {})
|
|
404
|
+
self._populate_nested_fields_from_mapping(properties)
|
|
320
405
|
elif self._create_index:
|
|
321
406
|
# Create the index if it doesn't exist
|
|
322
407
|
body = {"mappings": self._mappings, "settings": self._settings}
|
|
@@ -331,6 +416,9 @@ class OpenSearchDocumentStore:
|
|
|
331
416
|
"`settings` values will be ignored.",
|
|
332
417
|
index=self._index,
|
|
333
418
|
)
|
|
419
|
+
mapping = self._client.indices.get_mapping(index=self._index)
|
|
420
|
+
properties = mapping[self._index]["mappings"].get("properties", {})
|
|
421
|
+
self._populate_nested_fields_from_mapping(properties)
|
|
334
422
|
elif self._create_index:
|
|
335
423
|
# Create the index if it doesn't exist
|
|
336
424
|
body = {"mappings": self._mappings, "settings": self._settings}
|
|
@@ -369,7 +457,9 @@ class OpenSearchDocumentStore:
|
|
|
369
457
|
def _prepare_filter_search_request(self, filters: dict[str, Any] | None) -> dict[str, Any]:
|
|
370
458
|
search_kwargs: dict[str, Any] = {"size": 10_000}
|
|
371
459
|
if filters:
|
|
372
|
-
search_kwargs["query"] = {
|
|
460
|
+
search_kwargs["query"] = {
|
|
461
|
+
"bool": {"filter": normalize_filters(filters, nested_fields=self._resolved_nested_fields)}
|
|
462
|
+
}
|
|
373
463
|
|
|
374
464
|
# For some applications not returning the embedding can save a lot of bandwidth
|
|
375
465
|
# if you don't need this data not retrieving it can be a good idea
|
|
@@ -520,6 +610,12 @@ class OpenSearchDocumentStore:
|
|
|
520
610
|
"""
|
|
521
611
|
self._ensure_initialized()
|
|
522
612
|
|
|
613
|
+
if self._nested_fields == "*" and documents:
|
|
614
|
+
detected_nested_fields = self._detect_nested_fields_from_documents(documents)
|
|
615
|
+
new_nested_fields = detected_nested_fields - self._resolved_nested_fields
|
|
616
|
+
if new_nested_fields:
|
|
617
|
+
self._update_mapping_for_nested_fields(new_nested_fields)
|
|
618
|
+
|
|
523
619
|
bulk_params = self._prepare_bulk_write_request(
|
|
524
620
|
documents=documents, policy=policy, is_async=False, refresh=refresh
|
|
525
621
|
)
|
|
@@ -547,6 +643,13 @@ class OpenSearchDocumentStore:
|
|
|
547
643
|
"""
|
|
548
644
|
await self._ensure_initialized_async()
|
|
549
645
|
assert self._async_client is not None
|
|
646
|
+
|
|
647
|
+
if self._nested_fields == "*" and documents:
|
|
648
|
+
detected_nested_fields = self._detect_nested_fields_from_documents(documents)
|
|
649
|
+
new_nested_fields = detected_nested_fields - self._resolved_nested_fields
|
|
650
|
+
if new_nested_fields:
|
|
651
|
+
await self._update_mapping_for_nested_fields_async(new_nested_fields)
|
|
652
|
+
|
|
550
653
|
bulk_params = self._prepare_bulk_write_request(
|
|
551
654
|
documents=documents, policy=policy, is_async=True, refresh=refresh
|
|
552
655
|
)
|
|
@@ -756,7 +859,7 @@ class OpenSearchDocumentStore:
|
|
|
756
859
|
assert self._client is not None
|
|
757
860
|
|
|
758
861
|
try:
|
|
759
|
-
normalized_filters = normalize_filters(filters)
|
|
862
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
760
863
|
body = {"query": {"bool": {"filter": normalized_filters}}}
|
|
761
864
|
|
|
762
865
|
result = self._client.delete_by_query(index=self._index, body=body, refresh=refresh)
|
|
@@ -786,7 +889,7 @@ class OpenSearchDocumentStore:
|
|
|
786
889
|
assert self._async_client is not None
|
|
787
890
|
|
|
788
891
|
try:
|
|
789
|
-
normalized_filters = normalize_filters(filters)
|
|
892
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
790
893
|
body = {"query": {"bool": {"filter": normalized_filters}}}
|
|
791
894
|
result = await self._async_client.delete_by_query(index=self._index, body=body, refresh=refresh)
|
|
792
895
|
deleted_count = result.get("deleted", 0)
|
|
@@ -816,7 +919,7 @@ class OpenSearchDocumentStore:
|
|
|
816
919
|
assert self._client is not None
|
|
817
920
|
|
|
818
921
|
try:
|
|
819
|
-
normalized_filters = normalize_filters(filters)
|
|
922
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
820
923
|
# Build the update script to modify metadata fields
|
|
821
924
|
# Documents are stored with flattened metadata, so update fields directly in ctx._source
|
|
822
925
|
update_script_lines = []
|
|
@@ -856,7 +959,7 @@ class OpenSearchDocumentStore:
|
|
|
856
959
|
assert self._async_client is not None
|
|
857
960
|
|
|
858
961
|
try:
|
|
859
|
-
normalized_filters = normalize_filters(filters)
|
|
962
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
860
963
|
# Build the update script to modify metadata fields
|
|
861
964
|
# Documents are stored with flattened metadata, so update fields directly in ctx._source
|
|
862
965
|
update_script_lines = []
|
|
@@ -893,14 +996,16 @@ class OpenSearchDocumentStore:
|
|
|
893
996
|
if not query:
|
|
894
997
|
body: dict[str, Any] = {"query": {"bool": {"must": {"match_all": {}}}}}
|
|
895
998
|
if filters:
|
|
896
|
-
body["query"]["bool"]["filter"] = normalize_filters(filters)
|
|
999
|
+
body["query"]["bool"]["filter"] = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
897
1000
|
|
|
898
1001
|
if isinstance(custom_query, dict):
|
|
899
1002
|
body = self._render_custom_query(
|
|
900
1003
|
custom_query,
|
|
901
1004
|
{
|
|
902
1005
|
"$query": query,
|
|
903
|
-
"$filters": normalize_filters(filters)
|
|
1006
|
+
"$filters": normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1007
|
+
if filters
|
|
1008
|
+
else None,
|
|
904
1009
|
},
|
|
905
1010
|
)
|
|
906
1011
|
|
|
@@ -924,7 +1029,7 @@ class OpenSearchDocumentStore:
|
|
|
924
1029
|
}
|
|
925
1030
|
|
|
926
1031
|
if filters:
|
|
927
|
-
body["query"]["bool"]["filter"] = normalize_filters(filters)
|
|
1032
|
+
body["query"]["bool"]["filter"] = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
928
1033
|
|
|
929
1034
|
body["size"] = top_k
|
|
930
1035
|
|
|
@@ -940,10 +1045,12 @@ class OpenSearchDocumentStore:
|
|
|
940
1045
|
if not scale_score:
|
|
941
1046
|
return
|
|
942
1047
|
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
1048
|
+
results = [
|
|
1049
|
+
replace(doc, score=float(1 / (1 + exp(-(doc.score / float(BM25_SCALING_FACTOR))))))
|
|
1050
|
+
if doc.score is not None
|
|
1051
|
+
else doc
|
|
1052
|
+
for doc in results
|
|
1053
|
+
]
|
|
947
1054
|
|
|
948
1055
|
def _bm25_retrieval(
|
|
949
1056
|
self,
|
|
@@ -1327,7 +1434,7 @@ class OpenSearchDocumentStore:
|
|
|
1327
1434
|
|
|
1328
1435
|
# Add filters if provided
|
|
1329
1436
|
if filters:
|
|
1330
|
-
normalized_filters = normalize_filters(filters)
|
|
1437
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1331
1438
|
self._apply_metadata_search_filters(os_query, [normalized_filters], mode)
|
|
1332
1439
|
|
|
1333
1440
|
body = {"size": 1000, "query": os_query}
|
|
@@ -1432,7 +1539,7 @@ class OpenSearchDocumentStore:
|
|
|
1432
1539
|
|
|
1433
1540
|
# Add filters if provided
|
|
1434
1541
|
if filters:
|
|
1435
|
-
normalized_filters = normalize_filters(filters)
|
|
1542
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1436
1543
|
self._apply_metadata_search_filters(os_query, [normalized_filters], mode)
|
|
1437
1544
|
|
|
1438
1545
|
body = {"size": 1000, "query": os_query}
|
|
@@ -1473,7 +1580,9 @@ class OpenSearchDocumentStore:
|
|
|
1473
1580
|
custom_query,
|
|
1474
1581
|
{
|
|
1475
1582
|
"$query_embedding": query_embedding,
|
|
1476
|
-
"$filters": normalize_filters(filters)
|
|
1583
|
+
"$filters": normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1584
|
+
if filters
|
|
1585
|
+
else None,
|
|
1477
1586
|
},
|
|
1478
1587
|
)
|
|
1479
1588
|
|
|
@@ -1498,9 +1607,13 @@ class OpenSearchDocumentStore:
|
|
|
1498
1607
|
|
|
1499
1608
|
if filters:
|
|
1500
1609
|
if efficient_filtering:
|
|
1501
|
-
body["query"]["bool"]["must"][0]["knn"]["embedding"]["filter"] = normalize_filters(
|
|
1610
|
+
body["query"]["bool"]["must"][0]["knn"]["embedding"]["filter"] = normalize_filters(
|
|
1611
|
+
filters, nested_fields=self._resolved_nested_fields
|
|
1612
|
+
)
|
|
1502
1613
|
else:
|
|
1503
|
-
body["query"]["bool"]["filter"] = normalize_filters(
|
|
1614
|
+
body["query"]["bool"]["filter"] = normalize_filters(
|
|
1615
|
+
filters, nested_fields=self._resolved_nested_fields
|
|
1616
|
+
)
|
|
1504
1617
|
|
|
1505
1618
|
body["size"] = top_k
|
|
1506
1619
|
|
|
@@ -1606,7 +1719,7 @@ class OpenSearchDocumentStore:
|
|
|
1606
1719
|
self._ensure_initialized()
|
|
1607
1720
|
assert self._client is not None
|
|
1608
1721
|
|
|
1609
|
-
normalized_filters = normalize_filters(filters)
|
|
1722
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1610
1723
|
body = {"query": {"bool": {"filter": normalized_filters}}}
|
|
1611
1724
|
return self._client.count(index=self._index, body=body)["count"]
|
|
1612
1725
|
|
|
@@ -1621,7 +1734,7 @@ class OpenSearchDocumentStore:
|
|
|
1621
1734
|
await self._ensure_initialized_async()
|
|
1622
1735
|
assert self._async_client is not None
|
|
1623
1736
|
|
|
1624
|
-
normalized_filters = normalize_filters(filters)
|
|
1737
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1625
1738
|
body = {"query": {"bool": {"filter": normalized_filters}}}
|
|
1626
1739
|
return (await self._async_client.count(index=self._index, body=body))["count"]
|
|
1627
1740
|
|
|
@@ -1642,13 +1755,12 @@ class OpenSearchDocumentStore:
|
|
|
1642
1755
|
aggs[f"{field_name}_cardinality"] = {"cardinality": {"field": field_name}}
|
|
1643
1756
|
return aggs
|
|
1644
1757
|
|
|
1645
|
-
|
|
1646
|
-
def _build_distinct_values_query_body(filters: dict[str, Any] | None, aggs: dict[str, Any]) -> dict[str, Any]:
|
|
1758
|
+
def _build_distinct_values_query_body(self, filters: dict[str, Any] | None, aggs: dict[str, Any]) -> dict[str, Any]:
|
|
1647
1759
|
"""
|
|
1648
1760
|
Builds the query body for distinct values counting with filters and aggregations.
|
|
1649
1761
|
"""
|
|
1650
1762
|
if filters:
|
|
1651
|
-
normalized_filters = normalize_filters(filters)
|
|
1763
|
+
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1652
1764
|
return {
|
|
1653
1765
|
"query": {"bool": {"filter": normalized_filters}},
|
|
1654
1766
|
"aggs": aggs,
|
|
@@ -2048,9 +2160,13 @@ class OpenSearchDocumentStore:
|
|
|
2048
2160
|
if fetch_size is not None:
|
|
2049
2161
|
body["fetch_size"] = fetch_size
|
|
2050
2162
|
|
|
2163
|
+
# NOTE: method and url must be passed as positional args (not keyword args).
|
|
2164
|
+
# ddtrace's Elasticsearch instrumentation patch wraps perform_request and unpacks
|
|
2165
|
+
# the first two positional args as `method, target = args`. Passing them as
|
|
2166
|
+
# keyword args results in an empty `args` tuple and raises a ValueError.
|
|
2051
2167
|
response_data = self._client.transport.perform_request(
|
|
2052
|
-
|
|
2053
|
-
|
|
2168
|
+
"POST",
|
|
2169
|
+
"/_plugins/_sql",
|
|
2054
2170
|
body=body,
|
|
2055
2171
|
)
|
|
2056
2172
|
|
|
@@ -2081,9 +2197,13 @@ class OpenSearchDocumentStore:
|
|
|
2081
2197
|
if fetch_size is not None:
|
|
2082
2198
|
body["fetch_size"] = fetch_size
|
|
2083
2199
|
|
|
2200
|
+
# NOTE: method and url must be passed as positional args (not keyword args).
|
|
2201
|
+
# ddtrace's Elasticsearch instrumentation patch wraps perform_request and unpacks
|
|
2202
|
+
# the first two positional args as `method, target = args`. Passing them as
|
|
2203
|
+
# keyword args results in an empty `args` tuple and raises a ValueError.
|
|
2084
2204
|
response_data = await self._async_client.transport.perform_request(
|
|
2085
|
-
|
|
2086
|
-
|
|
2205
|
+
"POST",
|
|
2206
|
+
"/_plugins/_sql",
|
|
2087
2207
|
body=body,
|
|
2088
2208
|
)
|
|
2089
2209
|
|
|
@@ -7,20 +7,105 @@ from typing import Any
|
|
|
7
7
|
from haystack.errors import FilterError
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def normalize_filters(filters: dict[str, Any]) -> dict[str, Any]:
|
|
10
|
+
def normalize_filters(filters: dict[str, Any], nested_fields: set[str] | None = None) -> dict[str, Any]:
|
|
11
11
|
"""
|
|
12
12
|
Converts Haystack filters in OpenSearch compatible filters.
|
|
13
|
+
|
|
14
|
+
:param filters: Haystack filter dictionary.
|
|
15
|
+
:param nested_fields: Set of metadata field paths that are mapped as `nested` type in OpenSearch.
|
|
16
|
+
When provided, conditions targeting sub-fields of these paths are wrapped in `nested` queries.
|
|
13
17
|
"""
|
|
14
18
|
if not isinstance(filters, dict):
|
|
15
19
|
msg = "Filters must be a dictionary"
|
|
16
20
|
raise FilterError(msg)
|
|
17
21
|
|
|
18
22
|
if "field" in filters:
|
|
19
|
-
|
|
20
|
-
|
|
23
|
+
parsed = _parse_comparison_condition(filters, nested_fields)
|
|
24
|
+
if nested_fields:
|
|
25
|
+
nested_path = _get_nested_path(filters, nested_fields)
|
|
26
|
+
if nested_path:
|
|
27
|
+
return {"bool": {"must": {"nested": {"path": nested_path, "query": parsed}}}}
|
|
28
|
+
return {"bool": {"must": parsed}}
|
|
29
|
+
return _parse_logical_condition(filters, nested_fields)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _get_nested_path(condition: dict[str, Any], nested_fields: set[str]) -> str | None:
|
|
33
|
+
"""Returns the nested path for a comparison condition, or None."""
|
|
34
|
+
if not (field := condition.get("field")):
|
|
35
|
+
return None
|
|
36
|
+
if field.startswith("meta."):
|
|
37
|
+
field = field[5:]
|
|
38
|
+
parts = field.split(".")
|
|
39
|
+
for i in range(1, len(parts)):
|
|
40
|
+
prefix = ".".join(parts[:i])
|
|
41
|
+
if prefix in nested_fields:
|
|
42
|
+
return prefix
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _get_logical_condition_nested_path(condition: dict[str, Any], nested_fields: set[str]) -> str | None:
|
|
47
|
+
"""Returns the common nested path if all leaf comparisons in a logical sub-group share one, else None."""
|
|
48
|
+
paths: set[str | None] = set()
|
|
49
|
+
for c in condition.get("conditions", []):
|
|
50
|
+
if "field" in c:
|
|
51
|
+
paths.add(_get_nested_path(c, nested_fields))
|
|
52
|
+
elif "operator" in c and "conditions" in c:
|
|
53
|
+
paths.add(_get_logical_condition_nested_path(c, nested_fields))
|
|
54
|
+
else:
|
|
55
|
+
return None
|
|
56
|
+
if len(paths) == 1:
|
|
57
|
+
return next(iter(paths))
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _group_nested_conditions(
|
|
62
|
+
raw_conditions: list[dict[str, Any]],
|
|
63
|
+
nested_fields: set[str],
|
|
64
|
+
operator: str,
|
|
65
|
+
) -> list[dict[str, Any]]:
|
|
66
|
+
"""
|
|
67
|
+
Group conditions by nested path and wrap each group in a `nested` query.
|
|
21
68
|
|
|
69
|
+
Both direct comparisons and logical sub-groups whose leaves all target the
|
|
70
|
+
same nested path are absorbed into the group so that they match within the
|
|
71
|
+
same array element.
|
|
72
|
+
"""
|
|
73
|
+
nested_groups: dict[str, list[dict[str, Any]]] = {}
|
|
74
|
+
flat_raw: list[dict[str, Any]] = []
|
|
75
|
+
|
|
76
|
+
for c in raw_conditions:
|
|
77
|
+
nested_path = _get_nested_path(c, nested_fields)
|
|
78
|
+
if nested_path is None and "operator" in c and "conditions" in c:
|
|
79
|
+
nested_path = _get_logical_condition_nested_path(c, nested_fields)
|
|
80
|
+
if nested_path:
|
|
81
|
+
nested_groups.setdefault(nested_path, []).append(c)
|
|
82
|
+
else:
|
|
83
|
+
flat_raw.append(c)
|
|
84
|
+
|
|
85
|
+
conditions = [_parse_comparison_condition(c, nested_fields) for c in flat_raw]
|
|
86
|
+
|
|
87
|
+
for path, group in nested_groups.items():
|
|
88
|
+
inner = []
|
|
89
|
+
for c in group:
|
|
90
|
+
if "operator" in c and "conditions" in c:
|
|
91
|
+
# Logical sub-group: parse without nested awareness to avoid
|
|
92
|
+
# redundant nested wrapping — the outer code handles that.
|
|
93
|
+
inner.append(_parse_logical_condition(c, nested_fields=None))
|
|
94
|
+
else:
|
|
95
|
+
inner.append(_parse_comparison_condition(c, nested_fields))
|
|
96
|
+
if len(inner) > 1:
|
|
97
|
+
inner = _normalize_ranges(inner)
|
|
98
|
+
if len(inner) == 1:
|
|
99
|
+
conditions.append({"nested": {"path": path, "query": inner[0]}})
|
|
100
|
+
elif operator == "OR":
|
|
101
|
+
conditions.append({"nested": {"path": path, "query": {"bool": {"should": inner}}}})
|
|
102
|
+
else:
|
|
103
|
+
conditions.append({"nested": {"path": path, "query": {"bool": {"must": inner}}}})
|
|
22
104
|
|
|
23
|
-
|
|
105
|
+
return conditions
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _parse_logical_condition(condition: dict[str, Any], nested_fields: set[str] | None = None) -> dict[str, Any]:
|
|
24
109
|
if "operator" not in condition:
|
|
25
110
|
msg = f"'operator' key missing in {condition}"
|
|
26
111
|
raise FilterError(msg)
|
|
@@ -29,7 +114,12 @@ def _parse_logical_condition(condition: dict[str, Any]) -> dict[str, Any]:
|
|
|
29
114
|
raise FilterError(msg)
|
|
30
115
|
|
|
31
116
|
operator = condition["operator"]
|
|
32
|
-
|
|
117
|
+
|
|
118
|
+
if nested_fields:
|
|
119
|
+
conditions = _group_nested_conditions(condition["conditions"], nested_fields, operator)
|
|
120
|
+
else:
|
|
121
|
+
conditions = [_parse_comparison_condition(c, nested_fields) for c in condition["conditions"]]
|
|
122
|
+
|
|
33
123
|
if len(conditions) > 1:
|
|
34
124
|
conditions = _normalize_ranges(conditions)
|
|
35
125
|
if operator == "AND":
|
|
@@ -189,11 +279,11 @@ COMPARISON_OPERATORS = {
|
|
|
189
279
|
}
|
|
190
280
|
|
|
191
281
|
|
|
192
|
-
def _parse_comparison_condition(condition: dict[str, Any]) -> dict[str, Any]:
|
|
282
|
+
def _parse_comparison_condition(condition: dict[str, Any], nested_fields: set[str] | None = None) -> dict[str, Any]:
|
|
193
283
|
if "field" not in condition:
|
|
194
284
|
# 'field' key is only found in comparison dictionaries.
|
|
195
285
|
# We assume this is a logic dictionary since it's not present.
|
|
196
|
-
return _parse_logical_condition(condition)
|
|
286
|
+
return _parse_logical_condition(condition, nested_fields)
|
|
197
287
|
field: str = condition["field"]
|
|
198
288
|
|
|
199
289
|
if field.startswith("meta."):
|
|
@@ -143,6 +143,54 @@ def document_store_embedding_dim_4_no_emb_returned_faiss():
|
|
|
143
143
|
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
144
144
|
|
|
145
145
|
|
|
146
|
+
@pytest.fixture
|
|
147
|
+
def document_store_nested():
|
|
148
|
+
"""
|
|
149
|
+
OpenSearch document store with explicit nested fields.
|
|
150
|
+
"""
|
|
151
|
+
hosts = ["https://localhost:9200"]
|
|
152
|
+
index = _get_unique_index_name()
|
|
153
|
+
|
|
154
|
+
store = OpenSearchDocumentStore(
|
|
155
|
+
hosts=hosts,
|
|
156
|
+
index=index,
|
|
157
|
+
http_auth=("admin", "SecureHaystack!2026"),
|
|
158
|
+
verify_certs=False,
|
|
159
|
+
embedding_dim=768,
|
|
160
|
+
return_embedding=False,
|
|
161
|
+
nested_fields=["refs", "tags"],
|
|
162
|
+
)
|
|
163
|
+
store._ensure_initialized()
|
|
164
|
+
yield store
|
|
165
|
+
|
|
166
|
+
assert store._client
|
|
167
|
+
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@pytest.fixture
|
|
171
|
+
def document_store_wildcard_nested():
|
|
172
|
+
"""
|
|
173
|
+
OpenSearch document store with wildcard nested field auto-detection.
|
|
174
|
+
"""
|
|
175
|
+
hosts = ["https://localhost:9200"]
|
|
176
|
+
index = _get_unique_index_name()
|
|
177
|
+
|
|
178
|
+
store = OpenSearchDocumentStore(
|
|
179
|
+
hosts=hosts,
|
|
180
|
+
index=index,
|
|
181
|
+
http_auth=("admin", "SecureHaystack!2026"),
|
|
182
|
+
verify_certs=False,
|
|
183
|
+
embedding_dim=768,
|
|
184
|
+
return_embedding=False,
|
|
185
|
+
nested_fields="*",
|
|
186
|
+
)
|
|
187
|
+
store._ensure_initialized()
|
|
188
|
+
yield store
|
|
189
|
+
|
|
190
|
+
assert store._client
|
|
191
|
+
store._client.indices.delete(index=index, params={"ignore": [400, 404]})
|
|
192
|
+
|
|
193
|
+
|
|
146
194
|
@pytest.fixture
|
|
147
195
|
def test_documents():
|
|
148
196
|
return [
|
|
@@ -308,6 +308,7 @@ class TestDocumentStoreWithAuth:
|
|
|
308
308
|
"use_ssl": None,
|
|
309
309
|
"verify_certs": None,
|
|
310
310
|
"timeout": None,
|
|
311
|
+
"nested_fields": None,
|
|
311
312
|
},
|
|
312
313
|
}
|
|
313
314
|
|
|
@@ -354,6 +355,7 @@ class TestDocumentStoreWithAuth:
|
|
|
354
355
|
"use_ssl": None,
|
|
355
356
|
"verify_certs": None,
|
|
356
357
|
"timeout": None,
|
|
358
|
+
"nested_fields": None,
|
|
357
359
|
},
|
|
358
360
|
}
|
|
359
361
|
|