opensearch-haystack 8.2.1__tar.gz → 8.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/CHANGELOG.md +75 -28
- opensearch_haystack-8.2.1/LICENSE → opensearch_haystack-8.3.0/LICENSE.txt +1 -1
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/PKG-INFO +5 -5
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/README.md +2 -2
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/docker-compose.yml +1 -1
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/auth.py +1 -6
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +177 -143
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/filters.py +26 -7
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_document_store.py +135 -196
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_document_store_async.py +176 -98
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_filters.py +133 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_open_search_hybrid_retriever.py +3 -3
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/.gitignore +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/pydoc/config_docusaurus.yml +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/pyproject.toml +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/__init__.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/conftest.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_auth.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_bm25_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_document_store_common.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_embedding_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever_async.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_sql_retriever.py +0 -0
- {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_utils.py +0 -0
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/opensearch-v8.2.1] - 2026-08-03
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Fix new issues raised by ruff 0.16.0 (#3670)
|
|
8
|
+
- `OpensearchDocumentStore` match search_term against metadata field value, not content (#3647)
|
|
9
|
+
- *(opensearch)* Don't merge same-field ranges under OR (#3589)
|
|
10
|
+
|
|
11
|
+
|
|
3
12
|
## [integrations/opensearch-v8.2.0] - 2026-07-22
|
|
4
13
|
|
|
5
14
|
### 🚀 Features
|
|
@@ -334,6 +343,13 @@
|
|
|
334
343
|
|
|
335
344
|
## [integrations/opensearch-v3.0.0] - 2025-03-11
|
|
336
345
|
|
|
346
|
+
|
|
347
|
+
### 🌀 Miscellaneous
|
|
348
|
+
|
|
349
|
+
- Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
|
|
350
|
+
|
|
351
|
+
## [integrations/opensearch-v2.2.0] - 2025-03-07
|
|
352
|
+
|
|
337
353
|
### 🐛 Bug Fixes
|
|
338
354
|
|
|
339
355
|
- OpenSearchDocumentStore depends on async opensearch-py (#1438)
|
|
@@ -346,10 +362,6 @@
|
|
|
346
362
|
|
|
347
363
|
- Remove Python 3.8 support (#1421)
|
|
348
364
|
|
|
349
|
-
### 🌀 Miscellaneous
|
|
350
|
-
|
|
351
|
-
- Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
|
|
352
|
-
|
|
353
365
|
## [integrations/opensearch-v2.1.0] - 2025-02-18
|
|
354
366
|
|
|
355
367
|
### 🚀 Features
|
|
@@ -359,10 +371,6 @@
|
|
|
359
371
|
|
|
360
372
|
## [integrations/opensearch-v2.0.0] - 2025-02-14
|
|
361
373
|
|
|
362
|
-
### 🚀 Features
|
|
363
|
-
|
|
364
|
-
- Add Secret handling in OpenSearchDocumentStore (#1288)
|
|
365
|
-
|
|
366
374
|
### 🧹 Chores
|
|
367
375
|
|
|
368
376
|
- Inherit from `FilterDocumentsTestWithDataframe` in Document Stores (#1290)
|
|
@@ -372,6 +380,13 @@
|
|
|
372
380
|
|
|
373
381
|
- Chore: OpenSearch - manually fix changelog (#1299)
|
|
374
382
|
|
|
383
|
+
## [integrations/opensearch-v1.3.0] - 2025-01-16
|
|
384
|
+
|
|
385
|
+
### 🚀 Features
|
|
386
|
+
|
|
387
|
+
- Add Secret handling in OpenSearchDocumentStore (#1288)
|
|
388
|
+
|
|
389
|
+
|
|
375
390
|
## [integrations/opensearch-v1.2.0] - 2024-12-12
|
|
376
391
|
|
|
377
392
|
### 🧹 Chores
|
|
@@ -385,6 +400,17 @@
|
|
|
385
400
|
|
|
386
401
|
- Efficient knn filtering support for OpenSearch (#1134)
|
|
387
402
|
|
|
403
|
+
### ⚙️ CI
|
|
404
|
+
|
|
405
|
+
- Adopt uv as installer (#1142)
|
|
406
|
+
|
|
407
|
+
### 🧹 Chores
|
|
408
|
+
|
|
409
|
+
- Update changelog after removing legacy filters (#1083)
|
|
410
|
+
- Update ruff linting scripts and settings (#1105)
|
|
411
|
+
|
|
412
|
+
## [integrations/opensearch-v1.0.0] - 2024-09-12
|
|
413
|
+
|
|
388
414
|
### 📚 Documentation
|
|
389
415
|
|
|
390
416
|
- Update opensearch retriever docstrings (#1035)
|
|
@@ -393,15 +419,9 @@
|
|
|
393
419
|
|
|
394
420
|
- Do not retry tests in `hatch run test` command (#954)
|
|
395
421
|
|
|
396
|
-
### ⚙️ CI
|
|
397
|
-
|
|
398
|
-
- Adopt uv as installer (#1142)
|
|
399
|
-
|
|
400
422
|
### 🧹 Chores
|
|
401
423
|
|
|
402
424
|
- OpenSearch - remove legacy filter support (#1067)
|
|
403
|
-
- Update changelog after removing legacy filters (#1083)
|
|
404
|
-
- Update ruff linting scripts and settings (#1105)
|
|
405
425
|
|
|
406
426
|
### 🌀 Miscellaneous
|
|
407
427
|
|
|
@@ -418,15 +438,17 @@
|
|
|
418
438
|
|
|
419
439
|
## [integrations/opensearch-v0.8.1] - 2024-07-15
|
|
420
440
|
|
|
441
|
+
### 🐛 Bug Fixes
|
|
442
|
+
|
|
443
|
+
- `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
|
|
444
|
+
|
|
445
|
+
## [integrations/opensearch-v0.8.0] - 2024-07-15
|
|
446
|
+
|
|
421
447
|
### 🚀 Features
|
|
422
448
|
|
|
423
449
|
- Add raise_on_failure param to OpenSearch retrievers (#852)
|
|
424
450
|
- Add filter_policy to opensearch integration (#822)
|
|
425
451
|
|
|
426
|
-
### 🐛 Bug Fixes
|
|
427
|
-
|
|
428
|
-
- `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
|
|
429
|
-
|
|
430
452
|
### 🧹 Chores
|
|
431
453
|
|
|
432
454
|
- Update ruff invocation to include check parameter (#853)
|
|
@@ -447,12 +469,44 @@
|
|
|
447
469
|
|
|
448
470
|
### 🚀 Features
|
|
449
471
|
|
|
450
|
-
- Defer the database connection to when it's needed (#753)
|
|
451
|
-
- Improve `OpenSearchDocumentStore.__init__` arguments (#739)
|
|
452
|
-
- Return_embeddings flag for opensearch (#784)
|
|
453
472
|
- Add create_index option to OpenSearchDocumentStore (#840)
|
|
454
473
|
- Add custom_query param to OpenSearch retrievers (#841)
|
|
455
474
|
|
|
475
|
+
### ⚙️ CI
|
|
476
|
+
|
|
477
|
+
- Retry tests to reduce flakyness (#836)
|
|
478
|
+
|
|
479
|
+
### 🌀 Miscellaneous
|
|
480
|
+
|
|
481
|
+
- Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
|
|
482
|
+
|
|
483
|
+
## [integrations/opensearch-v0.6.0] - 2024-06-07
|
|
484
|
+
|
|
485
|
+
### 🚀 Features
|
|
486
|
+
|
|
487
|
+
- Return_embeddings flag for opensearch (#784)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
## [integrations/opensearch-v0.5.0] - 2024-05-30
|
|
491
|
+
|
|
492
|
+
### 🚀 Features
|
|
493
|
+
|
|
494
|
+
- Improve `OpenSearchDocumentStore.__init__` arguments (#739)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
## [integrations/opensearch-v0.4.0] - 2024-05-24
|
|
498
|
+
|
|
499
|
+
### 🚀 Features
|
|
500
|
+
|
|
501
|
+
- Defer the database connection to when it's needed (#753)
|
|
502
|
+
|
|
503
|
+
### 🌀 Miscellaneous
|
|
504
|
+
|
|
505
|
+
- Chore: add license classifiers (#680)
|
|
506
|
+
- Chore: change the pydoc renderer class (#718)
|
|
507
|
+
|
|
508
|
+
## [integrations/opensearch-v0.3.0] - 2024-04-04
|
|
509
|
+
|
|
456
510
|
### 🐛 Bug Fixes
|
|
457
511
|
|
|
458
512
|
- Fix order of API docs (#447)
|
|
@@ -464,10 +518,6 @@
|
|
|
464
518
|
- Small consistency improvements (#536)
|
|
465
519
|
- Disable-class-def (#556)
|
|
466
520
|
|
|
467
|
-
### ⚙️ CI
|
|
468
|
-
|
|
469
|
-
- Retry tests to reduce flakyness (#836)
|
|
470
|
-
|
|
471
521
|
### 🌀 Miscellaneous
|
|
472
522
|
|
|
473
523
|
- Generate API docs (#324)
|
|
@@ -476,9 +526,6 @@
|
|
|
476
526
|
- Fix opensearch errors bulk write (#594)
|
|
477
527
|
- Remove references to Python 3.7 (#601)
|
|
478
528
|
- [Elasticsearch] fix: Filters not working with metadata that contain a space or capitalization (#639)
|
|
479
|
-
- Chore: add license classifiers (#680)
|
|
480
|
-
- Chore: change the pydoc renderer class (#718)
|
|
481
|
-
- Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
|
|
482
529
|
|
|
483
530
|
## [integrations/opensearch-v0.2.0] - 2024-01-17
|
|
484
531
|
|
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2023 deepset GmbH
|
|
189
|
+
Copyright 2023-present deepset GmbH
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: opensearch-haystack
|
|
3
|
-
Version: 8.
|
|
3
|
+
Version: 8.3.0
|
|
4
4
|
Summary: Haystack 2.x Document Store for OpenSearch
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
7
7
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch
|
|
8
8
|
Author-email: deepset GmbH <info@deepset.ai>
|
|
9
9
|
License-Expression: Apache-2.0
|
|
10
|
-
License-File: LICENSE
|
|
10
|
+
License-File: LICENSE.txt
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: Programming Language :: Python
|
|
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
[](https://pypi.org/project/opensearch-haystack)
|
|
29
29
|
[](https://pypi.org/project/opensearch-haystack)
|
|
30
30
|
|
|
31
|
-
- [Integration page](https://haystack.deepset.ai/integrations/opensearch)
|
|
31
|
+
- [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
|
|
32
32
|
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
|
|
33
33
|
|
|
34
34
|
---
|
|
@@ -38,4 +38,4 @@ Description-Content-Type: text/markdown
|
|
|
38
38
|
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
39
39
|
|
|
40
40
|
To run integration tests locally, you need a Docker container running OpenSearch.
|
|
41
|
-
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
41
|
+
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](https://pypi.org/project/opensearch-haystack)
|
|
4
4
|
[](https://pypi.org/project/opensearch-haystack)
|
|
5
5
|
|
|
6
|
-
- [Integration page](https://haystack.deepset.ai/integrations/opensearch)
|
|
6
|
+
- [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
|
|
7
7
|
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
|
|
8
8
|
|
|
9
9
|
---
|
|
@@ -13,4 +13,4 @@
|
|
|
13
13
|
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
14
14
|
|
|
15
15
|
To run integration tests locally, you need a Docker container running OpenSearch.
|
|
16
|
-
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
16
|
+
Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
|
|
@@ -10,7 +10,7 @@ services:
|
|
|
10
10
|
- "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
|
|
11
11
|
- "OPENSEARCH_INITIAL_ADMIN_PASSWORD=SecureHaystack!2026"
|
|
12
12
|
healthcheck:
|
|
13
|
-
test: curl --fail https://localhost:9200/_cat/health -ku admin:
|
|
13
|
+
test: curl --fail https://localhost:9200/_cat/health -ku admin:SecureHaystack!2026 || exit 1
|
|
14
14
|
interval: 10s
|
|
15
15
|
timeout: 1s
|
|
16
16
|
retries: 10
|
|
@@ -4,7 +4,7 @@ from typing import Any, Optional, TypeVar
|
|
|
4
4
|
from haystack import default_from_dict, default_to_dict
|
|
5
5
|
from haystack.document_stores.errors import DocumentStoreError
|
|
6
6
|
from haystack.lazy_imports import LazyImport
|
|
7
|
-
from haystack.utils.auth import Secret
|
|
7
|
+
from haystack.utils.auth import Secret
|
|
8
8
|
from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
|
|
9
9
|
|
|
10
10
|
with LazyImport("Run 'pip install \"boto3\"' to install boto3.") as boto3_import:
|
|
@@ -119,11 +119,6 @@ class AWSAuth:
|
|
|
119
119
|
"""
|
|
120
120
|
Converts a dictionary representation to an AWSAuth object.
|
|
121
121
|
"""
|
|
122
|
-
init_parameters = data.get("init_parameters", {})
|
|
123
|
-
deserialize_secrets_inplace(
|
|
124
|
-
init_parameters,
|
|
125
|
-
["aws_access_key_id", "aws_secret_access_key", "aws_session_token", "aws_region_name", "aws_profile_name"],
|
|
126
|
-
)
|
|
127
122
|
return default_from_dict(cls, data)
|
|
128
123
|
|
|
129
124
|
def __call__(
|
|
@@ -25,6 +25,13 @@ from haystack_integrations.document_stores.opensearch.opensearch_scripts import
|
|
|
25
25
|
|
|
26
26
|
logger = logging.getLogger(__name__)
|
|
27
27
|
|
|
28
|
+
# Meta keys are passed only as params and never interpolated into the script source, so they can't inject code.
|
|
29
|
+
UPDATE_SCRIPT = """
|
|
30
|
+
for (entry in params.entrySet()) {
|
|
31
|
+
ctx._source[entry.getKey()] = entry.getValue();
|
|
32
|
+
}
|
|
33
|
+
"""
|
|
34
|
+
|
|
28
35
|
SPECIAL_FIELDS = {"content", "embedding", "id", "score", "sparse_embedding", "blob"}
|
|
29
36
|
|
|
30
37
|
Hosts = str | list[str | Mapping[str, str | int]]
|
|
@@ -959,16 +966,10 @@ class OpenSearchDocumentStore:
|
|
|
959
966
|
|
|
960
967
|
try:
|
|
961
968
|
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
962
|
-
# Build the update script to modify metadata fields
|
|
963
969
|
# Documents are stored with flattened metadata, so update fields directly in ctx._source
|
|
964
|
-
update_script_lines = []
|
|
965
|
-
for key in meta.keys():
|
|
966
|
-
update_script_lines.append(f"ctx._source.{key} = params.{key};")
|
|
967
|
-
update_script = " ".join(update_script_lines)
|
|
968
|
-
|
|
969
970
|
body = {
|
|
970
971
|
"query": {"bool": {"filter": normalized_filters}},
|
|
971
|
-
"script": {"source":
|
|
972
|
+
"script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
|
|
972
973
|
}
|
|
973
974
|
result = self._client.update_by_query(index=self._index, body=body, refresh=refresh)
|
|
974
975
|
updated_count = result.get("updated", 0)
|
|
@@ -999,16 +1000,10 @@ class OpenSearchDocumentStore:
|
|
|
999
1000
|
|
|
1000
1001
|
try:
|
|
1001
1002
|
normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
|
|
1002
|
-
# Build the update script to modify metadata fields
|
|
1003
1003
|
# Documents are stored with flattened metadata, so update fields directly in ctx._source
|
|
1004
|
-
update_script_lines = []
|
|
1005
|
-
for key in meta.keys():
|
|
1006
|
-
update_script_lines.append(f"ctx._source.{key} = params.{key};")
|
|
1007
|
-
update_script = " ".join(update_script_lines)
|
|
1008
|
-
|
|
1009
1004
|
body = {
|
|
1010
1005
|
"query": {"bool": {"filter": normalized_filters}},
|
|
1011
|
-
"script": {"source":
|
|
1006
|
+
"script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
|
|
1012
1007
|
}
|
|
1013
1008
|
result = await self._async_client.update_by_query(index=self._index, body=body, refresh=refresh)
|
|
1014
1009
|
updated_count = result.get("updated", 0)
|
|
@@ -2049,165 +2044,204 @@ class OpenSearchDocumentStore:
|
|
|
2049
2044
|
|
|
2050
2045
|
return self._extract_min_max_from_stats(stats)
|
|
2051
2046
|
|
|
2052
|
-
def
|
|
2053
|
-
self,
|
|
2054
|
-
|
|
2055
|
-
search_term: str | None = None,
|
|
2056
|
-
size: int | None = 10,
|
|
2057
|
-
after: dict[str, Any] | None = None,
|
|
2058
|
-
) -> tuple[list[str], dict[str, Any] | None]:
|
|
2047
|
+
def _build_unique_values_field_query(
|
|
2048
|
+
self, field_name: str, search_term: str | None, filters: dict[str, Any] | None = None
|
|
2049
|
+
) -> dict[str, Any]:
|
|
2059
2050
|
"""
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
Uses composite aggregations for proper pagination beyond 10k results.
|
|
2051
|
+
Builds a query matching docs whose metadata field's contains `search_term`, optionally restricted by `filters`.
|
|
2063
2052
|
|
|
2064
|
-
|
|
2065
|
-
:param search_term: Optional term to filter the returned values by, matching as a case-insensitive substring
|
|
2066
|
-
of the metadata field's own value (not the document content). NOTE: The matching is done with a server-side
|
|
2067
|
-
script to accomplish the substring matching on the value of the field and this operation is quite expensive
|
|
2068
|
-
for a large corpus.
|
|
2069
|
-
:param size: The number of unique values to return per page. Defaults to 10000.
|
|
2070
|
-
:param after: Optional pagination key from the previous response. Use None for the first page.
|
|
2071
|
-
For subsequent pages, pass the `after_key` from the previous response.
|
|
2072
|
-
:returns: A tuple containing (list of unique values, after_key for pagination).
|
|
2073
|
-
The after_key is None when there are no more results. Use it in the `after` parameter
|
|
2074
|
-
for the next page.
|
|
2053
|
+
Matching is a case-insensitive substring match (not against the document content).
|
|
2075
2054
|
"""
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2080
|
-
|
|
2081
|
-
# Build composite aggregation for proper pagination
|
|
2082
|
-
composite_agg: dict[str, Any] = {
|
|
2083
|
-
"size": size,
|
|
2084
|
-
"sources": [{field_name: {"terms": {"field": field_name}}}],
|
|
2085
|
-
}
|
|
2086
|
-
if after is not None:
|
|
2087
|
-
composite_agg["after"] = after
|
|
2088
|
-
|
|
2089
|
-
body: dict[str, Any] = {
|
|
2090
|
-
"aggs": {
|
|
2091
|
-
"unique_values": {
|
|
2092
|
-
"composite": composite_agg,
|
|
2093
|
-
}
|
|
2094
|
-
},
|
|
2095
|
-
"size": 0, # we only need aggregations, not documents
|
|
2096
|
-
}
|
|
2055
|
+
clauses: list[dict[str, Any]] = []
|
|
2056
|
+
if filters:
|
|
2057
|
+
clauses.append({"bool": {"filter": normalize_filters(filters, nested_fields=self._resolved_nested_fields)}})
|
|
2097
2058
|
if search_term:
|
|
2098
2059
|
# Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
|
|
2099
2060
|
# standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
|
|
2100
2061
|
# numeric ones. A doc-value script query works uniformly across field types by stringifying the
|
|
2101
2062
|
# field's value, matching the case-insensitive substring semantics documented above. The term is
|
|
2102
2063
|
# lower-cased once here rather than per-document in the script.
|
|
2103
|
-
|
|
2104
|
-
|
|
2064
|
+
clauses.append(
|
|
2065
|
+
{
|
|
2105
2066
|
"script": {
|
|
2106
|
-
"
|
|
2107
|
-
"
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2067
|
+
"script": {
|
|
2068
|
+
"source": (
|
|
2069
|
+
"def v = doc[params.field]; "
|
|
2070
|
+
"if (v.size() == 0) { return false; } "
|
|
2071
|
+
"return v.value.toString().toLowerCase().contains(params.term)"
|
|
2072
|
+
),
|
|
2073
|
+
"params": {"field": field_name, "term": search_term.lower()},
|
|
2074
|
+
}
|
|
2112
2075
|
}
|
|
2113
2076
|
}
|
|
2114
|
-
|
|
2077
|
+
)
|
|
2115
2078
|
|
|
2116
|
-
|
|
2079
|
+
if not clauses:
|
|
2080
|
+
return {"match_all": {}}
|
|
2081
|
+
if len(clauses) == 1:
|
|
2082
|
+
return clauses[0]
|
|
2083
|
+
return {"bool": {"filter": clauses}}
|
|
2084
|
+
|
|
2085
|
+
@staticmethod
|
|
2086
|
+
def _build_composite_agg_body(
|
|
2087
|
+
field_name: str,
|
|
2088
|
+
query: dict[str, Any],
|
|
2089
|
+
size: int,
|
|
2090
|
+
after: dict[str, Any] | None,
|
|
2091
|
+
*,
|
|
2092
|
+
with_count: bool,
|
|
2093
|
+
) -> dict[str, Any]:
|
|
2094
|
+
composite_agg: dict[str, Any] = {
|
|
2095
|
+
"size": size,
|
|
2096
|
+
"sources": [{field_name: {"terms": {"field": field_name}}}],
|
|
2097
|
+
}
|
|
2098
|
+
if after is not None:
|
|
2099
|
+
composite_agg["after"] = after
|
|
2100
|
+
|
|
2101
|
+
aggs: dict[str, Any] = {"unique_values": {"composite": composite_agg}}
|
|
2102
|
+
if with_count:
|
|
2103
|
+
# cardinality is a single-pass, approximate distinct count - computed alongside the
|
|
2104
|
+
# page fetch at no extra round trip, unlike walking the composite agg to exhaustion.
|
|
2105
|
+
aggs["unique_values_count"] = {"cardinality": {"field": field_name}}
|
|
2106
|
+
|
|
2107
|
+
return {"query": query, "aggs": aggs, "size": 0}
|
|
2108
|
+
|
|
2109
|
+
@staticmethod
|
|
2110
|
+
def _extract_unique_values_and_count(result: dict[str, Any], field_name: str) -> tuple[list[Any], int]:
|
|
2117
2111
|
aggregations = result.get("aggregations", {})
|
|
2112
|
+
buckets = aggregations.get("unique_values", {}).get("buckets", [])
|
|
2113
|
+
unique_values = [bucket["key"][field_name] for bucket in buckets]
|
|
2114
|
+
total_count = int(aggregations.get("unique_values_count", {}).get("value", 0))
|
|
2115
|
+
return unique_values, total_count
|
|
2118
2116
|
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2117
|
+
def _skip_unique_values(
|
|
2118
|
+
self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
|
|
2119
|
+
) -> dict[str, Any] | None:
|
|
2120
|
+
"""
|
|
2121
|
+
Walks composite aggregation pages to reach `offset`, discarding buckets along the way.
|
|
2122
|
+
|
|
2123
|
+
Composite aggregations only support cursor-based iteration (no native offset), so this
|
|
2124
|
+
replay is the only way to honor an arbitrary `from_` - cost scales with `offset`, not `size`.
|
|
2125
|
+
"""
|
|
2126
|
+
assert self._client is not None
|
|
2127
|
+
after_key = None
|
|
2128
|
+
remaining = offset
|
|
2129
|
+
while remaining > 0:
|
|
2130
|
+
step = min(batch_size, remaining)
|
|
2131
|
+
body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
|
|
2132
|
+
result = self._client.search(index=self._index, body=body)
|
|
2133
|
+
buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
|
|
2134
|
+
if not buckets:
|
|
2135
|
+
return after_key
|
|
2136
|
+
after_key = result["aggregations"]["unique_values"].get("after_key")
|
|
2137
|
+
remaining -= len(buckets)
|
|
2138
|
+
if len(buckets) < step:
|
|
2139
|
+
return after_key
|
|
2140
|
+
return after_key
|
|
2141
|
+
|
|
2142
|
+
async def _skip_unique_values_async(
|
|
2143
|
+
self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
|
|
2144
|
+
) -> dict[str, Any] | None:
|
|
2145
|
+
"""
|
|
2146
|
+
Async counterpart of `_skip_unique_values`. See that method for the cost trade-off explanation.
|
|
2147
|
+
"""
|
|
2148
|
+
assert self._async_client is not None
|
|
2149
|
+
after_key = None
|
|
2150
|
+
remaining = offset
|
|
2151
|
+
while remaining > 0:
|
|
2152
|
+
step = min(batch_size, remaining)
|
|
2153
|
+
body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
|
|
2154
|
+
result = await self._async_client.search(index=self._index, body=body)
|
|
2155
|
+
buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
|
|
2156
|
+
if not buckets:
|
|
2157
|
+
return after_key
|
|
2158
|
+
after_key = result["aggregations"]["unique_values"].get("after_key")
|
|
2159
|
+
remaining -= len(buckets)
|
|
2160
|
+
if len(buckets) < step:
|
|
2161
|
+
return after_key
|
|
2162
|
+
return after_key
|
|
2163
|
+
|
|
2164
|
+
def get_metadata_field_unique_values(
|
|
2165
|
+
self,
|
|
2166
|
+
metadata_field: str,
|
|
2167
|
+
search_term: str | None = None,
|
|
2168
|
+
from_: int = 0,
|
|
2169
|
+
size: int = 10,
|
|
2170
|
+
filters: dict[str, Any] | None = None,
|
|
2171
|
+
) -> tuple[list[Any], int]:
|
|
2172
|
+
"""
|
|
2173
|
+
Returns unique values for a metadata field, optionally filtered by a search term.
|
|
2174
|
+
|
|
2175
|
+
Internally still backed by composite aggregations, which only support cursor-based iteration.
|
|
2176
|
+
Reaching offset `from_` therefore requires walking and discarding the first `from_` buckets -
|
|
2177
|
+
cost scales with `from_`, not `size`.
|
|
2178
|
+
|
|
2179
|
+
**Note**: To keep this signature uniform across document stores, offset-based pagination is
|
|
2180
|
+
emulated on top of the cursor by re-fetching and discarding every bucket before `from_` on each
|
|
2181
|
+
call, requiring additional search round-trips proportional to `from_`.
|
|
2182
|
+
Performance may degrade for high-cardinality metadata fields and large from_ values.
|
|
2183
|
+
|
|
2184
|
+
**Note**: `total_count` is computed via an approximate cardinality aggregation; for fields with
|
|
2185
|
+
very high cardinality it may not be exact.
|
|
2186
|
+
|
|
2187
|
+
:param metadata_field: The metadata field to get unique values for. Can include or omit the
|
|
2188
|
+
"meta." prefix.
|
|
2189
|
+
:param search_term: Optional case-insensitive substring to filter the returned values by, matched
|
|
2190
|
+
against the metadata field's own value (not the document content). NOTE: The matching is done
|
|
2191
|
+
with a server-side script and is quite expensive for a large corpus.
|
|
2192
|
+
:param from_: Offset to start returning values from. Defaults to 0.
|
|
2193
|
+
:param size: The number of unique values to return per page. Defaults to 10.
|
|
2194
|
+
:param filters: Optional filters to restrict the documents considered.
|
|
2195
|
+
:returns: A tuple of (list of unique values in their original type, total count of distinct values
|
|
2196
|
+
for the field matching `search_term`). Note that filters also narrows down the number of documents
|
|
2197
|
+
against which the search term is matched.
|
|
2198
|
+
"""
|
|
2199
|
+
self._ensure_initialized()
|
|
2200
|
+
assert self._client is not None
|
|
2123
2201
|
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
after_key = None
|
|
2202
|
+
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2203
|
+
query = self._build_unique_values_field_query(field_name, search_term, filters)
|
|
2204
|
+
|
|
2205
|
+
after_key = self._skip_unique_values(field_name, query, from_) if from_ > 0 else None
|
|
2129
2206
|
|
|
2130
|
-
|
|
2207
|
+
body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
|
|
2208
|
+
result = self._client.search(index=self._index, body=body)
|
|
2209
|
+
return self._extract_unique_values_and_count(dict(result), field_name)
|
|
2131
2210
|
|
|
2132
2211
|
async def get_metadata_field_unique_values_async(
|
|
2133
2212
|
self,
|
|
2134
2213
|
metadata_field: str,
|
|
2135
2214
|
search_term: str | None = None,
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
:param search_term: Optional
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
:param size: The number of unique values to return per page. Defaults to
|
|
2150
|
-
:param
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
for the next page.
|
|
2215
|
+
from_: int = 0,
|
|
2216
|
+
size: int = 10,
|
|
2217
|
+
filters: dict[str, Any] | None = None,
|
|
2218
|
+
) -> tuple[list[Any], int]:
|
|
2219
|
+
"""
|
|
2220
|
+
Asynchronous counterpart of `get_metadata_field_unique_values`.
|
|
2221
|
+
|
|
2222
|
+
:param metadata_field: The metadata field to get unique values for. Can include or omit the
|
|
2223
|
+
"meta." prefix.
|
|
2224
|
+
:param search_term: Optional case-insensitive substring to filter the returned values by, matched
|
|
2225
|
+
against the metadata field's own value (not the document content). NOTE: The matching is done
|
|
2226
|
+
with a server-side script and is quite expensive for a large corpus.
|
|
2227
|
+
:param from_: Offset to start returning values from. Defaults to 0.
|
|
2228
|
+
:param size: The number of unique values to return per page. Defaults to 10.
|
|
2229
|
+
:param filters: Optional filters to restrict the documents considered.
|
|
2230
|
+
:returns: A tuple of (list of unique values in their original type, total count of distinct values
|
|
2231
|
+
for the field matching `search_term`). Note that filters also narrows down the number of documents
|
|
2232
|
+
against which the search term is matched.
|
|
2155
2233
|
"""
|
|
2156
2234
|
await self._ensure_initialized_async()
|
|
2157
2235
|
assert self._async_client is not None
|
|
2158
2236
|
|
|
2159
2237
|
field_name = _normalize_metadata_field_name(metadata_field)
|
|
2238
|
+
query = self._build_unique_values_field_query(field_name, search_term, filters)
|
|
2160
2239
|
|
|
2161
|
-
|
|
2162
|
-
composite_agg: dict[str, Any] = {
|
|
2163
|
-
"size": size,
|
|
2164
|
-
"sources": [{field_name: {"terms": {"field": field_name}}}],
|
|
2165
|
-
}
|
|
2166
|
-
if after is not None:
|
|
2167
|
-
composite_agg["after"] = after
|
|
2168
|
-
|
|
2169
|
-
body: dict[str, Any] = {
|
|
2170
|
-
"aggs": {
|
|
2171
|
-
"unique_values": {
|
|
2172
|
-
"composite": composite_agg,
|
|
2173
|
-
}
|
|
2174
|
-
},
|
|
2175
|
-
"size": 0, # we only need aggregations, not documents
|
|
2176
|
-
}
|
|
2177
|
-
if search_term:
|
|
2178
|
-
# Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
|
|
2179
|
-
# standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
|
|
2180
|
-
# numeric ones. A doc-value script query works uniformly across field types by stringifying the
|
|
2181
|
-
# field's value, matching the case-insensitive substring semantics documented above. The term is
|
|
2182
|
-
# lower-cased once here rather than per-document in the script.
|
|
2183
|
-
body["query"] = {
|
|
2184
|
-
"script": {
|
|
2185
|
-
"script": {
|
|
2186
|
-
"source": (
|
|
2187
|
-
"def v = doc[params.field]; "
|
|
2188
|
-
"if (v.size() == 0) { return false; } "
|
|
2189
|
-
"return v.value.toString().toLowerCase().contains(params.term)"
|
|
2190
|
-
),
|
|
2191
|
-
"params": {"field": field_name, "term": search_term.lower()},
|
|
2192
|
-
}
|
|
2193
|
-
}
|
|
2194
|
-
}
|
|
2240
|
+
after_key = await self._skip_unique_values_async(field_name, query, from_) if from_ > 0 else None
|
|
2195
2241
|
|
|
2242
|
+
body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
|
|
2196
2243
|
result = await self._async_client.search(index=self._index, body=body)
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
# Extract unique values from composite aggregation buckets
|
|
2200
|
-
unique_values_agg = aggregations.get("unique_values", {})
|
|
2201
|
-
unique_values_buckets = unique_values_agg.get("buckets", [])
|
|
2202
|
-
unique_values = [str(bucket["key"][field_name]) for bucket in unique_values_buckets]
|
|
2203
|
-
|
|
2204
|
-
# Extract after_key for pagination
|
|
2205
|
-
# If we got fewer results than requested, we've reached the end
|
|
2206
|
-
after_key = unique_values_agg.get("after_key")
|
|
2207
|
-
if after_key is not None and size is not None and len(unique_values_buckets) < size:
|
|
2208
|
-
after_key = None
|
|
2209
|
-
|
|
2210
|
-
return unique_values, after_key
|
|
2244
|
+
return self._extract_unique_values_and_count(dict(result), field_name)
|
|
2211
2245
|
|
|
2212
2246
|
def _query_sql(self, query: str, fetch_size: int | None = None) -> dict[str, Any]:
|
|
2213
2247
|
"""
|