opensearch-haystack 8.2.0__tar.gz → 8.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/CHANGELOG.md +94 -28
- opensearch_haystack-8.2.0/LICENSE → opensearch_haystack-8.3.0/LICENSE.txt +1 -1
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/PKG-INFO +5 -5
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/README.md +2 -2
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/docker-compose.yml +1 -1
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/pyproject.toml +1 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/auth.py +1 -6
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +176 -114
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/filters.py +31 -9
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_document_store.py +166 -213
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_document_store_async.py +189 -1
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_filters.py +186 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_open_search_hybrid_retriever.py +3 -3
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/.gitignore +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/__init__.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/conftest.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_auth.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_bm25_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_document_store_common.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_embedding_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_sql_retriever.py +0 -0
- {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_utils.py +0 -0
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v8.2.1] - 2026-08-03
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Fix new issues raised by ruff 0.16.0 (#3670)
|
|
8
|
+
- `OpensearchDocumentStore` match search_term against metadata field value, not content (#3647)
|
|
9
|
+
- *(opensearch)* Don't merge same-field ranges under OR (#3589)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
## [integrations/opensearch-v8.2.0] - 2026-07-22
|
|
13
|
+
|
|
14
|
+
### 🚀 Features
|
|
15
|
+
|
|
16
|
+
- Opensearch - add methods to close resources (#3640)
|
|
17
|
+
|
|
18
|
+
### 🐛 Bug Fixes
|
|
19
|
+
|
|
20
|
+
- Adapt optimum, ragas, elasticsearch, opensearch to APIs removed in Haystack 3.0 (#3538)
|
|
21
|
+
|
|
22
|
+
### 🚜 Refactor
|
|
23
|
+
|
|
24
|
+
- Elasticsearch - split sync/async paths (#3633)
|
|
25
|
+
|
|
26
|
+
### 📚 Documentation
|
|
27
|
+
|
|
28
|
+
- Replace old haystack core imports with haystack_integrations paths (#3545)
|
|
29
|
+
|
|
30
|
+
|
|
3
31
|
## [integrations/opensearch-v8.1.2] - 2026-06-24
|
|
4
32
|
|
|
5
33
|
### 🐛 Bug Fixes
|
|
@@ -315,6 +343,13 @@
|
|
|
315
343
|
|
|
316
344
|
## [integrations/opensearch-v3.0.0] - 2025-03-11
|
|
317
345
|
|
|
346
|
+
|
|
347
|
+
### 🌀 Miscellaneous
|
|
348
|
+
|
|
349
|
+
- Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
|
|
350
|
+
|
|
351
|
+
## [integrations/opensearch-v2.2.0] - 2025-03-07
|
|
352
|
+
|
|
318
353
|
### 🐛 Bug Fixes
|
|
319
354
|
|
|
320
355
|
- OpenSearchDocumentStore depends on async opensearch-py (#1438)
|
|
@@ -327,10 +362,6 @@
|
|
|
327
362
|
|
|
328
363
|
- Remove Python 3.8 support (#1421)
|
|
329
364
|
|
|
330
|
-
### 🌀 Miscellaneous
|
|
331
|
-
|
|
332
|
-
- Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
|
|
333
|
-
|
|
334
365
|
## [integrations/opensearch-v2.1.0] - 2025-02-18
|
|
335
366
|
|
|
336
367
|
### 🚀 Features
|
|
@@ -340,10 +371,6 @@
|
|
|
340
371
|
|
|
341
372
|
## [integrations/opensearch-v2.0.0] - 2025-02-14
|
|
342
373
|
|
|
343
|
-
### 🚀 Features
|
|
344
|
-
|
|
345
|
-
- Add Secret handling in OpenSearchDocumentStore (#1288)
|
|
346
|
-
|
|
347
374
|
### 🧹 Chores
|
|
348
375
|
|
|
349
376
|
- Inherit from `FilterDocumentsTestWithDataframe` in Document Stores (#1290)
|
|
@@ -353,6 +380,13 @@
|
|
|
353
380
|
|
|
354
381
|
- Chore: OpenSearch - manually fix changelog (#1299)
|
|
355
382
|
|
|
383
|
+
## [integrations/opensearch-v1.3.0] - 2025-01-16
|
|
384
|
+
|
|
385
|
+
### 🚀 Features
|
|
386
|
+
|
|
387
|
+
- Add Secret handling in OpenSearchDocumentStore (#1288)
|
|
388
|
+
|
|
389
|
+
|
|
356
390
|
## [integrations/opensearch-v1.2.0] - 2024-12-12
|
|
357
391
|
|
|
358
392
|
### 🧹 Chores
|
|
@@ -366,6 +400,17 @@
|
|
|
366
400
|
|
|
367
401
|
- Efficient knn filtering support for OpenSearch (#1134)
|
|
368
402
|
|
|
403
|
+
### ⚙️ CI
|
|
404
|
+
|
|
405
|
+
- Adopt uv as installer (#1142)
|
|
406
|
+
|
|
407
|
+
### 🧹 Chores
|
|
408
|
+
|
|
409
|
+
- Update changelog after removing legacy filters (#1083)
|
|
410
|
+
- Update ruff linting scripts and settings (#1105)
|
|
411
|
+
|
|
412
|
+
## [integrations/opensearch-v1.0.0] - 2024-09-12
|
|
413
|
+
|
|
369
414
|
### 📚 Documentation
|
|
370
415
|
|
|
371
416
|
- Update opensearch retriever docstrings (#1035)
|
|
@@ -374,15 +419,9 @@
|
|
|
374
419
|
|
|
375
420
|
- Do not retry tests in `hatch run test` command (#954)
|
|
376
421
|
|
|
377
|
-
### ⚙️ CI
|
|
378
|
-
|
|
379
|
-
- Adopt uv as installer (#1142)
|
|
380
|
-
|
|
381
422
|
### 🧹 Chores
|
|
382
423
|
|
|
383
424
|
- OpenSearch - remove legacy filter support (#1067)
|
|
384
|
-
- Update changelog after removing legacy filters (#1083)
|
|
385
|
-
- Update ruff linting scripts and settings (#1105)
|
|
386
425
|
|
|
387
426
|
### 🌀 Miscellaneous
|
|
388
427
|
|
|
@@ -399,15 +438,17 @@
|
|
|
399
438
|
|
|
400
439
|
## [integrations/opensearch-v0.8.1] - 2024-07-15
|
|
401
440
|
|
|
441
|
+
### 🐛 Bug Fixes
|
|
442
|
+
|
|
443
|
+
- `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
|
|
444
|
+
|
|
445
|
+
## [integrations/opensearch-v0.8.0] - 2024-07-15
|
|
446
|
+
|
|
402
447
|
### 🚀 Features
|
|
403
448
|
|
|
404
449
|
- Add raise_on_failure param to OpenSearch retrievers (#852)
|
|
405
450
|
- Add filter_policy to opensearch integration (#822)
|
|
406
451
|
|
|
407
|
-
### 🐛 Bug Fixes
|
|
408
|
-
|
|
409
|
-
- `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
|
|
410
|
-
|
|
411
452
|
### 🧹 Chores
|
|
412
453
|
|
|
413
454
|
- Update ruff invocation to include check parameter (#853)
|
|
@@ -428,12 +469,44 @@
|
|
|
428
469
|
|
|
429
470
|
### 🚀 Features
|
|
430
471
|
|
|
431
|
-
- Defer the database connection to when it's needed (#753)
|
|
432
|
-
- Improve `OpenSearchDocumentStore.__init__` arguments (#739)
|
|
433
|
-
- Return_embeddings flag for opensearch (#784)
|
|
434
472
|
- Add create_index option to OpenSearchDocumentStore (#840)
|
|
435
473
|
- Add custom_query param to OpenSearch retrievers (#841)
|
|
436
474
|
|
|
475
|
+
### ⚙️ CI
|
|
476
|
+
|
|
477
|
+
- Retry tests to reduce flakyness (#836)
|
|
478
|
+
|
|
479
|
+
### 🌀 Miscellaneous
|
|
480
|
+
|
|
481
|
+
- Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
|
|
482
|
+
|
|
483
|
+
## [integrations/opensearch-v0.6.0] - 2024-06-07
|
|
484
|
+
|
|
485
|
+
### 🚀 Features
|
|
486
|
+
|
|
487
|
+
- Return_embeddings flag for opensearch (#784)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
## [integrations/opensearch-v0.5.0] - 2024-05-30
|
|
491
|
+
|
|
492
|
+
### 🚀 Features
|
|
493
|
+
|
|
494
|
+
- Improve `OpenSearchDocumentStore.__init__` arguments (#739)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
## [integrations/opensearch-v0.4.0] - 2024-05-24
|
|
498
|
+
|
|
499
|
+
### 🚀 Features
|
|
500
|
+
|
|
501
|
+
- Defer the database connection to when it's needed (#753)
|
|
502
|
+
|
|
503
|
+
### 🌀 Miscellaneous
|
|
504
|
+
|
|
505
|
+
- Chore: add license classifiers (#680)
|
|
506
|
+
- Chore: change the pydoc renderer class (#718)
|
|
507
|
+
|
|
508
|
+
## [integrations/opensearch-v0.3.0] - 2024-04-04
|
|
509
|
+
|
|
437
510
|
### 🐛 Bug Fixes
|
|
438
511
|
|
|
439
512
|
- Fix order of API docs (#447)
|
|
@@ -445,10 +518,6 @@
|
|
|
445
518
|
- Small consistency improvements (#536)
|
|
446
519
|
- Disable-class-def (#556)
|
|
447
520
|
|
|
448
|
-
### ⚙️ CI
|
|
449
|
-
|
|
450
|
-
- Retry tests to reduce flakyness (#836)
|
|
451
|
-
|
|
452
521
|
### 🌀 Miscellaneous
|
|
453
522
|
|
|
454
523
|
- Generate API docs (#324)
|
|
@@ -457,9 +526,6 @@
|
|
|
457
526
|
- Fix opensearch errors bulk write (#594)
|
|
458
527
|
- Remove references to Python 3.7 (#601)
|
|
459
528
|
- [Elasticsearch] fix: Filters not working with metadata that contain a space or capitalization (#639)
|
|
460
|
-
- Chore: add license classifiers (#680)
|
|
461
|
-
- Chore: change the pydoc renderer class (#718)
|
|
462
|
-
- Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
|
|
463
529
|
|
|
464
530
|
## [integrations/opensearch-v0.2.0] - 2024-01-17
|
|
465
531
|
|
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2023 deepset GmbH
|
|
189
|
+
Copyright 2023-present deepset GmbH
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version: 8.
|
|
3
|
+
Version: 8.3.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
7
7
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch
|
|
8
8
|
Author-email: deepset GmbH <info@deepset.ai>
|
|
9
9
|
License-Expression: Apache-2.0
|
|
10
|
-
License-File: LICENSE
|
|
10
|
+
License-File: LICENSE.txt
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: Programming Language :: Python
|
|
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
[](https://pypi.org/project/opensearch-haystack)
|
|
29
29
|
[](https://pypi.org/project/opensearch-haystack)
|
|
30
30
|
|
|
31
|
-
- [Integration page](https://haystack.deepset.ai/integrations/opensearch)
|
|
31
|
+
- [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
|
|
32
32
|
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
|
|
33
33
|
|
|
34
34
|
---
|
|
@@ -38,4 +38,4 @@ Description-Content-Type: text/markdown
|
|
|
38
38
|
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
39
39
|
|
|
40
40
|
To run integration tests locally, you need a Docker container running OpenSearch.
|
|
41
|
-
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
41
|
+
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](https://pypi.org/project/opensearch-haystack)
|
|
4
4
|
[](https://pypi.org/project/opensearch-haystack)
|
|
5
5
|
|
|
6
|
-
- [Integration page](https://haystack.deepset.ai/integrations/opensearch)
|
|
6
|
+
- [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
|
|
7
7
|
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
|
|
8
8
|
|
|
9
9
|
---
|
|
@@ -13,4 +13,4 @@
|
|
|
13
13
|
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
14
14
|
|
|
15
15
|
To run integration tests locally, you need a Docker container running OpenSearch.
|
|
16
|
-
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
16
|
+
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
@@ -10,7 +10,7 @@ services:
|
|
|
10
10
|
- "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
|
|
11
11
|
- "OPENSEARCH_INITIAL_ADMIN_PASSWORD=SecureHaystack!2026"
|
|
12
12
|
healthcheck:
|
|
13
|
-
test: curl --fail https://localhost:9200/_cat/health -ku admin:
|
|
13
|
+
test: curl --fail https://localhost:9200/_cat/health -ku admin:SecureHaystack!2026 || exit 1
|
|
14
14
|
interval: 10s
|
|
15
15
|
timeout: 1s
|
|
16
16
|
retries: 10
|
|
@@ -4,7 +4,7 @@ from typing import Any, Optional, TypeVar
|
|
|
4
4
|
from haystack import default_from_dict, default_to_dict
|
|
5
5
|
from haystack.document_stores.errors import DocumentStoreError
|
|
6
6
|
from haystack.lazy_imports import LazyImport
|
|
7
|
-
from haystack.utils.auth import Secret
|
|
7
|
+
from haystack.utils.auth import Secret
|
|
8
8
|
from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
|
|
9
9
|
|
|
10
10
|
with LazyImport("Run 'pip install \"boto3\"' to install boto3.") as boto3_import:
|
|
@@ -119,11 +119,6 @@ class AWSAuth:
|
|
|
119
119
|
"""
|
|
120
120
|
Converts a dictionary representation to an AWSAuth object.
|
|
121
121
|
"""
|
|
122
|
-
init_parameters = data.get("init_parameters", {})
|
|
123
|
-
deserialize_secrets_inplace(
|
|
124
|
-
init_parameters,
|
|
125
|
-
["aws_access_key_id", "aws_secret_access_key", "aws_session_token", "aws_region_name", "aws_profile_name"],
|
|
126
|
-
)
|
|
127
122
|
return default_from_dict(cls, data)
|
|
128
123
|
|
|
129
124
|
def __call__(
|
|
@@ -25,6 +25,13 @@ from haystack_integrations.document_stores.opensearch.opensearch_scripts import
|
|
|
25
25
|
|
|
26
26
|
logger = logging.getLogger(__name__)
|
|
27
27
|
|
|
28
|
+
# Meta keys are passed only as params and never interpolated into the script source, so they can't inject code.
|
|
29
|
+
UPDATE_SCRIPT = """
|
|
30
|
+
for (entry in params.entrySet()) {
|
|
31
|
+
ctx._source[entry.getKey()] = entry.getValue();
|
|
32
|
+
}
|
|
33
|
+
"""
|
|
34
|
+
|
|
28
35
|
SPECIAL_FIELDS = {"content", "embedding", "id", "score", "sparse_embedding", "blob"}
|
|
29
36
|
|
|
30
37
|
Hosts = str | list[str | Mapping[str, str | int]]
|
|
@@ -959,16 +966,10 @@ class OpenSearchDocumentStore:
|
|
|
959
966
|
|
|
960
967
|
try:
|
|
961
968
|
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
962
|
-
# Build the update script to modify metadata fields
|
|
963
969
|
# Documents are stored with flattened metadata, so update fields directly in ctx._source
|
|
964
|
-
update_script_lines = []
|
|
965
|
-
for key in meta.keys():
|
|
966
|
-
update_script_lines.append(f"ctx._source.{key} = params.{key};")
|
|
967
|
-
update_script = " ".join(update_script_lines)
|
|
968
|
-
|
|
969
970
|
body = {
|
|
970
971
|
"query": {"bool": {"filter": normalized_filters}},
|
|
971
|
-
"script": {"source":
|
|
972
|
+
"script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
|
|
972
973
|
}
|
|
973
974
|
result = self._client.update_by_query(index=self._index, body=body, refresh=refresh)
|
|
974
975
|
updated_count = result.get("updated", 0)
|
|
@@ -999,16 +1000,10 @@ class OpenSearchDocumentStore:
|
|
|
999
1000
|
|
|
1000
1001
|
try:
|
|
1001
1002
|
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1002
|
-
# Build the update script to modify metadata fields
|
|
1003
1003
|
# Documents are stored with flattened metadata, so update fields directly in ctx._source
|
|
1004
|
-
update_script_lines = []
|
|
1005
|
-
for key in meta.keys():
|
|
1006
|
-
update_script_lines.append(f"ctx._source.{key} = params.{key};")
|
|
1007
|
-
update_script = " ".join(update_script_lines)
|
|
1008
|
-
|
|
1009
1004
|
body = {
|
|
1010
1005
|
"query": {"bool": {"filter": normalized_filters}},
|
|
1011
|
-
"script": {"source":
|
|
1006
|
+
"script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
|
|
1012
1007
|
}
|
|
1013
1008
|
result = await self._async_client.update_by_query(index=self._index, body=body, refresh=refresh)
|
|
1014
1009
|
updated_count = result.get("updated", 0)
|
|
@@ -2049,39 +2044,53 @@ class OpenSearchDocumentStore:
|
|
|
2049
2044
|
|
|
2050
2045
|
return self._extract_min_max_from_stats(stats)
|
|
2051
2046
|
|
|
2052
|
-
def
|
|
2053
|
-
self,
|
|
2054
|
-
|
|
2055
|
-
search_term: str | None = None,
|
|
2056
|
-
size: int | None = 10000,
|
|
2057
|
-
after: dict[str, Any] | None = None,
|
|
2058
|
-
) -> tuple[list[str], dict[str, Any] | None]:
|
|
2047
|
+
def _build_unique_values_field_query(
|
|
2048
|
+
self, field_name: str, search_term: str | None, filters: dict[str, Any] | None = None
|
|
2049
|
+
) -> dict[str, Any]:
|
|
2059
2050
|
"""
|
|
2060
|
-
|
|
2051
|
+
Builds a query matching docs whose metadata field's contains `search_term`, optionally restricted by `filters`.
|
|
2061
2052
|
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
:param metadata_field: The metadata field to get unique values for.
|
|
2065
|
-
:param search_term: Optional search term to filter documents by matching in the content field.
|
|
2066
|
-
:param size: The number of unique values to return per page. Defaults to 10000.
|
|
2067
|
-
:param after: Optional pagination key from the previous response. Use None for the first page.
|
|
2068
|
-
For subsequent pages, pass the `after_key` from the previous response.
|
|
2069
|
-
:returns: A tuple containing (list of unique values, after_key for pagination).
|
|
2070
|
-
The after_key is None when there are no more results. Use it in the `after` parameter
|
|
2071
|
-
for the next page.
|
|
2053
|
+
Matching is a case-insensitive substring match (not against the document content).
|
|
2072
2054
|
"""
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2077
|
-
|
|
2078
|
-
# filter by search_term if provided
|
|
2079
|
-
query: dict[str, Any] = {"match_all": {}}
|
|
2055
|
+
clauses: list[dict[str, Any]] = []
|
|
2056
|
+
if filters:
|
|
2057
|
+
clauses.append({"bool": {"filter": normalize_filters(filters, nested_fields=self._resolved_nested_fields)}})
|
|
2080
2058
|
if search_term:
|
|
2081
|
-
#
|
|
2082
|
-
query
|
|
2059
|
+
# Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
|
|
2060
|
+
# standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
|
|
2061
|
+
# numeric ones. A doc-value script query works uniformly across field types by stringifying the
|
|
2062
|
+
# field's value, matching the case-insensitive substring semantics documented above. The term is
|
|
2063
|
+
# lower-cased once here rather than per-document in the script.
|
|
2064
|
+
clauses.append(
|
|
2065
|
+
{
|
|
2066
|
+
"script": {
|
|
2067
|
+
"script": {
|
|
2068
|
+
"source": (
|
|
2069
|
+
"def v = doc[params.field]; "
|
|
2070
|
+
"if (v.size() == 0) { return false; } "
|
|
2071
|
+
"return v.value.toString().toLowerCase().contains(params.term)"
|
|
2072
|
+
),
|
|
2073
|
+
"params": {"field": field_name, "term": search_term.lower()},
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
2077
|
+
)
|
|
2078
|
+
|
|
2079
|
+
if not clauses:
|
|
2080
|
+
return {"match_all": {}}
|
|
2081
|
+
if len(clauses) == 1:
|
|
2082
|
+
return clauses[0]
|
|
2083
|
+
return {"bool": {"filter": clauses}}
|
|
2083
2084
|
|
|
2084
|
-
|
|
2085
|
+
@staticmethod
|
|
2086
|
+
def _build_composite_agg_body(
|
|
2087
|
+
field_name: str,
|
|
2088
|
+
query: dict[str, Any],
|
|
2089
|
+
size: int,
|
|
2090
|
+
after: dict[str, Any] | None,
|
|
2091
|
+
*,
|
|
2092
|
+
with_count: bool,
|
|
2093
|
+
) -> dict[str, Any]:
|
|
2085
2094
|
composite_agg: dict[str, Any] = {
|
|
2086
2095
|
"size": size,
|
|
2087
2096
|
"sources": [{field_name: {"terms": {"field": field_name}}}],
|
|
@@ -2089,97 +2098,150 @@ class OpenSearchDocumentStore:
|
|
|
2089
2098
|
if after is not None:
|
|
2090
2099
|
composite_agg["after"] = after
|
|
2091
2100
|
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
}
|
|
2098
|
-
},
|
|
2099
|
-
"size": 0, # we only need aggregations, not documents
|
|
2100
|
-
}
|
|
2101
|
+
aggs: dict[str, Any] = {"unique_values": {"composite": composite_agg}}
|
|
2102
|
+
if with_count:
|
|
2103
|
+
# cardinality is a single-pass, approximate distinct count - computed alongside the
|
|
2104
|
+
# page fetch at no extra round trip, unlike walking the composite agg to exhaustion.
|
|
2105
|
+
aggs["unique_values_count"] = {"cardinality": {"field": field_name}}
|
|
2101
2106
|
|
|
2102
|
-
|
|
2103
|
-
aggregations = result.get("aggregations", {})
|
|
2107
|
+
return {"query": query, "aggs": aggs, "size": 0}
|
|
2104
2108
|
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
+
@staticmethod
|
|
2110
|
+
def _extract_unique_values_and_count(result: dict[str, Any], field_name: str) -> tuple[list[Any], int]:
|
|
2111
|
+
aggregations = result.get("aggregations", {})
|
|
2112
|
+
buckets = aggregations.get("unique_values", {}).get("buckets", [])
|
|
2113
|
+
unique_values = [bucket["key"][field_name] for bucket in buckets]
|
|
2114
|
+
total_count = int(aggregations.get("unique_values_count", {}).get("value", 0))
|
|
2115
|
+
return unique_values, total_count
|
|
2109
2116
|
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2117
|
+
def _skip_unique_values(
|
|
2118
|
+
self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
|
|
2119
|
+
) -> dict[str, Any] | None:
|
|
2120
|
+
"""
|
|
2121
|
+
Walks composite aggregation pages to reach `offset`, discarding buckets along the way.
|
|
2115
2122
|
|
|
2116
|
-
|
|
2123
|
+
Composite aggregations only support cursor-based iteration (no native offset), so this
|
|
2124
|
+
replay is the only way to honor an arbitrary `from_` - cost scales with `offset`, not `size`.
|
|
2125
|
+
"""
|
|
2126
|
+
assert self._client is not None
|
|
2127
|
+
after_key = None
|
|
2128
|
+
remaining = offset
|
|
2129
|
+
while remaining > 0:
|
|
2130
|
+
step = min(batch_size, remaining)
|
|
2131
|
+
body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
|
|
2132
|
+
result = self._client.search(index=self._index, body=body)
|
|
2133
|
+
buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
|
|
2134
|
+
if not buckets:
|
|
2135
|
+
return after_key
|
|
2136
|
+
after_key = result["aggregations"]["unique_values"].get("after_key")
|
|
2137
|
+
remaining -= len(buckets)
|
|
2138
|
+
if len(buckets) < step:
|
|
2139
|
+
return after_key
|
|
2140
|
+
return after_key
|
|
2141
|
+
|
|
2142
|
+
async def _skip_unique_values_async(
|
|
2143
|
+
self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
|
|
2144
|
+
) -> dict[str, Any] | None:
|
|
2145
|
+
"""
|
|
2146
|
+
Async counterpart of `_skip_unique_values`. See that method for the cost trade-off explanation.
|
|
2147
|
+
"""
|
|
2148
|
+
assert self._async_client is not None
|
|
2149
|
+
after_key = None
|
|
2150
|
+
remaining = offset
|
|
2151
|
+
while remaining > 0:
|
|
2152
|
+
step = min(batch_size, remaining)
|
|
2153
|
+
body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
|
|
2154
|
+
result = await self._async_client.search(index=self._index, body=body)
|
|
2155
|
+
buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
|
|
2156
|
+
if not buckets:
|
|
2157
|
+
return after_key
|
|
2158
|
+
after_key = result["aggregations"]["unique_values"].get("after_key")
|
|
2159
|
+
remaining -= len(buckets)
|
|
2160
|
+
if len(buckets) < step:
|
|
2161
|
+
return after_key
|
|
2162
|
+
return after_key
|
|
2117
2163
|
|
|
2118
|
-
|
|
2164
|
+
def get_metadata_field_unique_values(
|
|
2119
2165
|
self,
|
|
2120
2166
|
metadata_field: str,
|
|
2121
2167
|
search_term: str | None = None,
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2168
|
+
from_: int = 0,
|
|
2169
|
+
size: int = 10,
|
|
2170
|
+
filters: dict[str, Any] | None = None,
|
|
2171
|
+
) -> tuple[list[Any], int]:
|
|
2172
|
+
"""
|
|
2173
|
+
Returns unique values for a metadata field, optionally filtered by a search term.
|
|
2174
|
+
|
|
2175
|
+
Internally still backed by composite aggregations, which only support cursor-based iteration.
|
|
2176
|
+
Reaching offset `from_` therefore requires walking and discarding the first `from_` buckets -
|
|
2177
|
+
cost scales with `from_`, not `size`.
|
|
2178
|
+
|
|
2179
|
+
**Note**: To keep this signature uniform across document stores, offset-based pagination is
|
|
2180
|
+
emulated on top of the cursor by re-fetching and discarding every bucket before `from_` on each
|
|
2181
|
+
call, requiring additional search round-trips proportional to `from_`.
|
|
2182
|
+
Performance may degrade for high-cardinality metadata fields and large from_ values.
|
|
2183
|
+
|
|
2184
|
+
**Note**: `total_count` is computed via an approximate cardinality aggregation; for fields with
|
|
2185
|
+
very high cardinality it may not be exact.
|
|
2186
|
+
|
|
2187
|
+
:param metadata_field: The metadata field to get unique values for. Can include or omit the
|
|
2188
|
+
"meta." prefix.
|
|
2189
|
+
:param search_term: Optional case-insensitive substring to filter the returned values by, matched
|
|
2190
|
+
against the metadata field's own value (not the document content). NOTE: The matching is done
|
|
2191
|
+
with a server-side script and is quite expensive for a large corpus.
|
|
2192
|
+
:param from_: Offset to start returning values from. Defaults to 0.
|
|
2193
|
+
:param size: The number of unique values to return per page. Defaults to 10.
|
|
2194
|
+
:param filters: Optional filters to restrict the documents considered.
|
|
2195
|
+
:returns: A tuple of (list of unique values in their original type, total count of distinct values
|
|
2196
|
+
for the field matching `search_term`). Note that filters also narrows down the number of documents
|
|
2197
|
+
against which the search term is matched.
|
|
2125
2198
|
"""
|
|
2126
|
-
|
|
2199
|
+
self._ensure_initialized()
|
|
2200
|
+
assert self._client is not None
|
|
2127
2201
|
|
|
2128
|
-
|
|
2202
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2203
|
+
query = self._build_unique_values_field_query(field_name, search_term, filters)
|
|
2204
|
+
|
|
2205
|
+
after_key = self._skip_unique_values(field_name, query, from_) if from_ > 0 else None
|
|
2129
2206
|
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2207
|
+
body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
|
|
2208
|
+
result = self._client.search(index=self._index, body=body)
|
|
2209
|
+
return self._extract_unique_values_and_count(dict(result), field_name)
|
|
2210
|
+
|
|
2211
|
+
async def get_metadata_field_unique_values_async(
|
|
2212
|
+
self,
|
|
2213
|
+
metadata_field: str,
|
|
2214
|
+
search_term: str | None = None,
|
|
2215
|
+
from_: int = 0,
|
|
2216
|
+
size: int = 10,
|
|
2217
|
+
filters: dict[str, Any] | None = None,
|
|
2218
|
+
) -> tuple[list[Any], int]:
|
|
2219
|
+
"""
|
|
2220
|
+
Asynchronous counterpart of `get_metadata_field_unique_values`.
|
|
2221
|
+
|
|
2222
|
+
:param metadata_field: The metadata field to get unique values for. Can include or omit the
|
|
2223
|
+
"meta." prefix.
|
|
2224
|
+
:param search_term: Optional case-insensitive substring to filter the returned values by, matched
|
|
2225
|
+
against the metadata field's own value (not the document content). NOTE: The matching is done
|
|
2226
|
+
with a server-side script and is quite expensive for a large corpus.
|
|
2227
|
+
:param from_: Offset to start returning values from. Defaults to 0.
|
|
2228
|
+
:param size: The number of unique values to return per page. Defaults to 10.
|
|
2229
|
+
:param filters: Optional filters to restrict the documents considered.
|
|
2230
|
+
:returns: A tuple of (list of unique values in their original type, total count of distinct values
|
|
2231
|
+
for the field matching `search_term`). Note that filters also narrows down the number of documents
|
|
2232
|
+
against which the search term is matched.
|
|
2138
2233
|
"""
|
|
2139
2234
|
await self._ensure_initialized_async()
|
|
2140
2235
|
assert self._async_client is not None
|
|
2141
2236
|
|
|
2142
2237
|
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2238
|
+
query = self._build_unique_values_field_query(field_name, search_term, filters)
|
|
2143
2239
|
|
|
2144
|
-
|
|
2145
|
-
query: dict[str, Any] = {"match_all": {}}
|
|
2146
|
-
if search_term:
|
|
2147
|
-
# Use match_phrase for exact phrase matching to avoid tokenization issues
|
|
2148
|
-
query = {"match_phrase": {"content": search_term}}
|
|
2149
|
-
|
|
2150
|
-
# Build composite aggregation for proper pagination
|
|
2151
|
-
composite_agg: dict[str, Any] = {
|
|
2152
|
-
"size": size,
|
|
2153
|
-
"sources": [{field_name: {"terms": {"field": field_name}}}],
|
|
2154
|
-
}
|
|
2155
|
-
if after is not None:
|
|
2156
|
-
composite_agg["after"] = after
|
|
2157
|
-
|
|
2158
|
-
body = {
|
|
2159
|
-
"query": query,
|
|
2160
|
-
"aggs": {
|
|
2161
|
-
"unique_values": {
|
|
2162
|
-
"composite": composite_agg,
|
|
2163
|
-
}
|
|
2164
|
-
},
|
|
2165
|
-
"size": 0, # we only need aggregations, not documents
|
|
2166
|
-
}
|
|
2240
|
+
after_key = await self._skip_unique_values_async(field_name, query, from_) if from_ > 0 else None
|
|
2167
2241
|
|
|
2242
|
+
body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
|
|
2168
2243
|
result = await self._async_client.search(index=self._index, body=body)
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
# Extract unique values from composite aggregation buckets
|
|
2172
|
-
unique_values_agg = aggregations.get("unique_values", {})
|
|
2173
|
-
unique_values_buckets = unique_values_agg.get("buckets", [])
|
|
2174
|
-
unique_values = [str(bucket["key"][field_name]) for bucket in unique_values_buckets]
|
|
2175
|
-
|
|
2176
|
-
# Extract after_key for pagination
|
|
2177
|
-
# If we got fewer results than requested, we've reached the end
|
|
2178
|
-
after_key = unique_values_agg.get("after_key")
|
|
2179
|
-
if after_key is not None and size is not None and len(unique_values_buckets) < size:
|
|
2180
|
-
after_key = None
|
|
2181
|
-
|
|
2182
|
-
return unique_values, after_key
|
|
2244
|
+
return self._extract_unique_values_and_count(dict(result), field_name)
|
|
2183
2245
|
|
|
2184
2246
|
def _query_sql(self, query: str, fetch_size: int | None = None) -> dict[str, Any]:
|
|
2185
2247
|
"""
|