opensearch-haystack 8.2.1__tar.gz → 8.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/CHANGELOG.md +75 -28
  2. opensearch_haystack-8.2.1/LICENSE → opensearch_haystack-8.3.0/LICENSE.txt +1 -1
  3. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/PKG-INFO +5 -5
  4. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/README.md +2 -2
  5. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/docker-compose.yml +1 -1
  6. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/auth.py +1 -6
  7. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +177 -143
  8. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/filters.py +26 -7
  9. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_document_store.py +135 -196
  10. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_document_store_async.py +176 -98
  11. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_filters.py +133 -0
  12. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_open_search_hybrid_retriever.py +3 -3
  13. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/.gitignore +0 -0
  14. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/pydoc/config_docusaurus.yml +0 -0
  15. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/pyproject.toml +0 -0
  16. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
  17. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
  18. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
  19. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
  20. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
  21. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
  22. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
  23. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
  24. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
  25. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
  26. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/py.typed +0 -0
  27. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/__init__.py +0 -0
  28. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/conftest.py +0 -0
  29. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_auth.py +0 -0
  30. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_bm25_retriever.py +0 -0
  31. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_document_store_common.py +0 -0
  32. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_embedding_retriever.py +0 -0
  33. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever.py +0 -0
  34. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever_async.py +0 -0
  35. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_sql_retriever.py +0 -0
  36. {opensearch_haystack-8.2.1 → opensearch_haystack-8.3.0}/tests/test_utils.py +0 -0
@@ -1,5 +1,14 @@
1
1
  # Changelog
2
2
 
3
+ ## [integrations/opensearch-v8.2.1] - 2026-08-03
4
+
5
+ ### 🐛 Bug Fixes
6
+
7
+ - Fix new issues raised by ruff 0.16.0 (#3670)
8
+ - `OpensearchDocumentStore` match search_term against metadata field value, not content (#3647)
9
+ - *(opensearch)* Don't merge same-field ranges under OR (#3589)
10
+
11
+
3
12
  ## [integrations/opensearch-v8.2.0] - 2026-07-22
4
13
 
5
14
  ### 🚀 Features
@@ -334,6 +343,13 @@
334
343
 
335
344
  ## [integrations/opensearch-v3.0.0] - 2025-03-11
336
345
 
346
+
347
+ ### 🌀 Miscellaneous
348
+
349
+ - Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
350
+
351
+ ## [integrations/opensearch-v2.2.0] - 2025-03-07
352
+
337
353
  ### 🐛 Bug Fixes
338
354
 
339
355
  - OpenSearchDocumentStore depends on async opensearch-py (#1438)
@@ -346,10 +362,6 @@
346
362
 
347
363
  - Remove Python 3.8 support (#1421)
348
364
 
349
- ### 🌀 Miscellaneous
350
-
351
- - Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
352
-
353
365
  ## [integrations/opensearch-v2.1.0] - 2025-02-18
354
366
 
355
367
  ### 🚀 Features
@@ -359,10 +371,6 @@
359
371
 
360
372
  ## [integrations/opensearch-v2.0.0] - 2025-02-14
361
373
 
362
- ### 🚀 Features
363
-
364
- - Add Secret handling in OpenSearchDocumentStore (#1288)
365
-
366
374
  ### 🧹 Chores
367
375
 
368
376
  - Inherit from `FilterDocumentsTestWithDataframe` in Document Stores (#1290)
@@ -372,6 +380,13 @@
372
380
 
373
381
  - Chore: OpenSearch - manually fix changelog (#1299)
374
382
 
383
+ ## [integrations/opensearch-v1.3.0] - 2025-01-16
384
+
385
+ ### 🚀 Features
386
+
387
+ - Add Secret handling in OpenSearchDocumentStore (#1288)
388
+
389
+
375
390
  ## [integrations/opensearch-v1.2.0] - 2024-12-12
376
391
 
377
392
  ### 🧹 Chores
@@ -385,6 +400,17 @@
385
400
 
386
401
  - Efficient knn filtering support for OpenSearch (#1134)
387
402
 
403
+ ### ⚙️ CI
404
+
405
+ - Adopt uv as installer (#1142)
406
+
407
+ ### 🧹 Chores
408
+
409
+ - Update changelog after removing legacy filters (#1083)
410
+ - Update ruff linting scripts and settings (#1105)
411
+
412
+ ## [integrations/opensearch-v1.0.0] - 2024-09-12
413
+
388
414
  ### 📚 Documentation
389
415
 
390
416
  - Update opensearch retriever docstrings (#1035)
@@ -393,15 +419,9 @@
393
419
 
394
420
  - Do not retry tests in `hatch run test` command (#954)
395
421
 
396
- ### ⚙️ CI
397
-
398
- - Adopt uv as installer (#1142)
399
-
400
422
  ### 🧹 Chores
401
423
 
402
424
  - OpenSearch - remove legacy filter support (#1067)
403
- - Update changelog after removing legacy filters (#1083)
404
- - Update ruff linting scripts and settings (#1105)
405
425
 
406
426
  ### 🌀 Miscellaneous
407
427
 
@@ -418,15 +438,17 @@
418
438
 
419
439
  ## [integrations/opensearch-v0.8.1] - 2024-07-15
420
440
 
441
+ ### 🐛 Bug Fixes
442
+
443
+ - `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
444
+
445
+ ## [integrations/opensearch-v0.8.0] - 2024-07-15
446
+
421
447
  ### 🚀 Features
422
448
 
423
449
  - Add raise_on_failure param to OpenSearch retrievers (#852)
424
450
  - Add filter_policy to opensearch integration (#822)
425
451
 
426
- ### 🐛 Bug Fixes
427
-
428
- - `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
429
-
430
452
  ### 🧹 Chores
431
453
 
432
454
  - Update ruff invocation to include check parameter (#853)
@@ -447,12 +469,44 @@
447
469
 
448
470
  ### 🚀 Features
449
471
 
450
- - Defer the database connection to when it's needed (#753)
451
- - Improve `OpenSearchDocumentStore.__init__` arguments (#739)
452
- - Return_embeddings flag for opensearch (#784)
453
472
  - Add create_index option to OpenSearchDocumentStore (#840)
454
473
  - Add custom_query param to OpenSearch retrievers (#841)
455
474
 
475
+ ### ⚙️ CI
476
+
477
+ - Retry tests to reduce flakyness (#836)
478
+
479
+ ### 🌀 Miscellaneous
480
+
481
+ - Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
482
+
483
+ ## [integrations/opensearch-v0.6.0] - 2024-06-07
484
+
485
+ ### 🚀 Features
486
+
487
+ - Return_embeddings flag for opensearch (#784)
488
+
489
+
490
+ ## [integrations/opensearch-v0.5.0] - 2024-05-30
491
+
492
+ ### 🚀 Features
493
+
494
+ - Improve `OpenSearchDocumentStore.__init__` arguments (#739)
495
+
496
+
497
+ ## [integrations/opensearch-v0.4.0] - 2024-05-24
498
+
499
+ ### 🚀 Features
500
+
501
+ - Defer the database connection to when it's needed (#753)
502
+
503
+ ### 🌀 Miscellaneous
504
+
505
+ - Chore: add license classifiers (#680)
506
+ - Chore: change the pydoc renderer class (#718)
507
+
508
+ ## [integrations/opensearch-v0.3.0] - 2024-04-04
509
+
456
510
  ### 🐛 Bug Fixes
457
511
 
458
512
  - Fix order of API docs (#447)
@@ -464,10 +518,6 @@
464
518
  - Small consistency improvements (#536)
465
519
  - Disable-class-def (#556)
466
520
 
467
- ### ⚙️ CI
468
-
469
- - Retry tests to reduce flakyness (#836)
470
-
471
521
  ### 🌀 Miscellaneous
472
522
 
473
523
  - Generate API docs (#324)
@@ -476,9 +526,6 @@
476
526
  - Fix opensearch errors bulk write (#594)
477
527
  - Remove references to Python 3.7 (#601)
478
528
  - [Elasticsearch] fix: Filters not working with metadata that contain a space or capitalization (#639)
479
- - Chore: add license classifiers (#680)
480
- - Chore: change the pydoc renderer class (#718)
481
- - Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
482
529
 
483
530
  ## [integrations/opensearch-v0.2.0] - 2024-01-17
484
531
 
@@ -186,7 +186,7 @@
186
186
  same "printed page" as the copyright notice for easier
187
187
  identification within third-party archives.
188
188
 
189
- Copyright 2023 deepset GmbH
189
+ Copyright 2023-present deepset GmbH
190
190
 
191
191
  Licensed under the Apache License, Version 2.0 (the "License");
192
192
  you may not use this file except in compliance with the License.
@@ -1,13 +1,13 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: opensearch-haystack
3
- Version: 8.2.1
3
+ Version: 8.3.0
4
4
  Summary: Haystack 2.x Document Store for OpenSearch
5
5
  Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
6
6
  Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
7
7
  Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch
8
8
  Author-email: deepset GmbH <info@deepset.ai>
9
9
  License-Expression: Apache-2.0
10
- License-File: LICENSE
10
+ License-File: LICENSE.txt
11
11
  Classifier: Development Status :: 4 - Beta
12
12
  Classifier: License :: OSI Approved :: Apache Software License
13
13
  Classifier: Programming Language :: Python
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
28
28
  [![PyPI - Version](https://img.shields.io/pypi/v/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
29
29
  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
30
30
 
31
- - [Integration page](https://haystack.deepset.ai/integrations/opensearch)
31
+ - [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
32
32
  - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
33
33
 
34
34
  ---
@@ -38,4 +38,4 @@ Description-Content-Type: text/markdown
38
38
  Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
39
39
 
40
40
  To run integration tests locally, you need a Docker container running OpenSearch.
41
- Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
41
+ Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
@@ -3,7 +3,7 @@
3
3
  [![PyPI - Version](https://img.shields.io/pypi/v/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
4
4
  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
5
5
 
6
- - [Integration page](https://haystack.deepset.ai/integrations/opensearch)
6
+ - [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
7
7
  - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
8
8
 
9
9
  ---
@@ -13,4 +13,4 @@
13
13
  Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
14
14
 
15
15
  To run integration tests locally, you need a Docker container running OpenSearch.
16
- Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
16
+ Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
@@ -10,7 +10,7 @@ services:
10
10
  - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
11
11
  - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=SecureHaystack!2026"
12
12
  healthcheck:
13
- test: curl --fail https://localhost:9200/_cat/health -ku admin:admin || exit 1
13
+ test: curl --fail https://localhost:9200/_cat/health -ku admin:SecureHaystack!2026 || exit 1
14
14
  interval: 10s
15
15
  timeout: 1s
16
16
  retries: 10
@@ -4,7 +4,7 @@ from typing import Any, Optional, TypeVar
4
4
  from haystack import default_from_dict, default_to_dict
5
5
  from haystack.document_stores.errors import DocumentStoreError
6
6
  from haystack.lazy_imports import LazyImport
7
- from haystack.utils.auth import Secret, deserialize_secrets_inplace
7
+ from haystack.utils.auth import Secret
8
8
  from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
9
9
 
10
10
  with LazyImport("Run 'pip install \"boto3\"' to install boto3.") as boto3_import:
@@ -119,11 +119,6 @@ class AWSAuth:
119
119
  """
120
120
  Converts a dictionary representation to an AWSAuth object.
121
121
  """
122
- init_parameters = data.get("init_parameters", {})
123
- deserialize_secrets_inplace(
124
- init_parameters,
125
- ["aws_access_key_id", "aws_secret_access_key", "aws_session_token", "aws_region_name", "aws_profile_name"],
126
- )
127
122
  return default_from_dict(cls, data)
128
123
 
129
124
  def __call__(
@@ -25,6 +25,13 @@ from haystack_integrations.document_stores.opensearch.opensearch_scripts import
25
25
 
26
26
  logger = logging.getLogger(__name__)
27
27
 
28
+ # Meta keys are passed only as params and never interpolated into the script source, so they can't inject code.
29
+ UPDATE_SCRIPT = """
30
+ for (entry in params.entrySet()) {
31
+ ctx._source[entry.getKey()] = entry.getValue();
32
+ }
33
+ """
34
+
28
35
  SPECIAL_FIELDS = {"content", "embedding", "id", "score", "sparse_embedding", "blob"}
29
36
 
30
37
  Hosts = str | list[str | Mapping[str, str | int]]
@@ -959,16 +966,10 @@ class OpenSearchDocumentStore:
959
966
 
960
967
  try:
961
968
  normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
962
- # Build the update script to modify metadata fields
963
969
  # Documents are stored with flattened metadata, so update fields directly in ctx._source
964
- update_script_lines = []
965
- for key in meta.keys():
966
- update_script_lines.append(f"ctx._source.{key} = params.{key};")
967
- update_script = " ".join(update_script_lines)
968
-
969
970
  body = {
970
971
  "query": {"bool": {"filter": normalized_filters}},
971
- "script": {"source": update_script, "params": meta, "lang": "painless"},
972
+ "script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
972
973
  }
973
974
  result = self._client.update_by_query(index=self._index, body=body, refresh=refresh)
974
975
  updated_count = result.get("updated", 0)
@@ -999,16 +1000,10 @@ class OpenSearchDocumentStore:
999
1000
 
1000
1001
  try:
1001
1002
  normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
1002
- # Build the update script to modify metadata fields
1003
1003
  # Documents are stored with flattened metadata, so update fields directly in ctx._source
1004
- update_script_lines = []
1005
- for key in meta.keys():
1006
- update_script_lines.append(f"ctx._source.{key} = params.{key};")
1007
- update_script = " ".join(update_script_lines)
1008
-
1009
1004
  body = {
1010
1005
  "query": {"bool": {"filter": normalized_filters}},
1011
- "script": {"source": update_script, "params": meta, "lang": "painless"},
1006
+ "script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
1012
1007
  }
1013
1008
  result = await self._async_client.update_by_query(index=self._index, body=body, refresh=refresh)
1014
1009
  updated_count = result.get("updated", 0)
@@ -2049,165 +2044,204 @@ class OpenSearchDocumentStore:
2049
2044
 
2050
2045
  return self._extract_min_max_from_stats(stats)
2051
2046
 
2052
- def get_metadata_field_unique_values(
2053
- self,
2054
- metadata_field: str,
2055
- search_term: str | None = None,
2056
- size: int | None = 10,
2057
- after: dict[str, Any] | None = None,
2058
- ) -> tuple[list[str], dict[str, Any] | None]:
2047
+ def _build_unique_values_field_query(
2048
+ self, field_name: str, search_term: str | None, filters: dict[str, Any] | None = None
2049
+ ) -> dict[str, Any]:
2059
2050
  """
2060
- Returns unique values for a metadata field, optionally filtered by a search term.
2061
-
2062
- Uses composite aggregations for proper pagination beyond 10k results.
2051
+ Builds a query matching docs whose metadata field's contains `search_term`, optionally restricted by `filters`.
2063
2052
 
2064
- :param metadata_field: The metadata field to get unique values for.
2065
- :param search_term: Optional term to filter the returned values by, matching as a case-insensitive substring
2066
- of the metadata field's own value (not the document content). NOTE: The matching is done with a server-side
2067
- script to accomplish the substring matching on the value of the field and this operation is quite expensive
2068
- for a large corpus.
2069
- :param size: The number of unique values to return per page. Defaults to 10000.
2070
- :param after: Optional pagination key from the previous response. Use None for the first page.
2071
- For subsequent pages, pass the `after_key` from the previous response.
2072
- :returns: A tuple containing (list of unique values, after_key for pagination).
2073
- The after_key is None when there are no more results. Use it in the `after` parameter
2074
- for the next page.
2053
+ Matching is a case-insensitive substring match (not against the document content).
2075
2054
  """
2076
- self._ensure_initialized()
2077
- assert self._client is not None
2078
-
2079
- field_name = _normalize_metadata_field_name(metadata_field)
2080
-
2081
- # Build composite aggregation for proper pagination
2082
- composite_agg: dict[str, Any] = {
2083
- "size": size,
2084
- "sources": [{field_name: {"terms": {"field": field_name}}}],
2085
- }
2086
- if after is not None:
2087
- composite_agg["after"] = after
2088
-
2089
- body: dict[str, Any] = {
2090
- "aggs": {
2091
- "unique_values": {
2092
- "composite": composite_agg,
2093
- }
2094
- },
2095
- "size": 0, # we only need aggregations, not documents
2096
- }
2055
+ clauses: list[dict[str, Any]] = []
2056
+ if filters:
2057
+ clauses.append({"bool": {"filter": normalize_filters(filters, nested_fields=self._resolved_nested_fields)}})
2097
2058
  if search_term:
2098
2059
  # Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
2099
2060
  # standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
2100
2061
  # numeric ones. A doc-value script query works uniformly across field types by stringifying the
2101
2062
  # field's value, matching the case-insensitive substring semantics documented above. The term is
2102
2063
  # lower-cased once here rather than per-document in the script.
2103
- body["query"] = {
2104
- "script": {
2064
+ clauses.append(
2065
+ {
2105
2066
  "script": {
2106
- "source": (
2107
- "def v = doc[params.field]; "
2108
- "if (v.size() == 0) { return false; } "
2109
- "return v.value.toString().toLowerCase().contains(params.term)"
2110
- ),
2111
- "params": {"field": field_name, "term": search_term.lower()},
2067
+ "script": {
2068
+ "source": (
2069
+ "def v = doc[params.field]; "
2070
+ "if (v.size() == 0) { return false; } "
2071
+ "return v.value.toString().toLowerCase().contains(params.term)"
2072
+ ),
2073
+ "params": {"field": field_name, "term": search_term.lower()},
2074
+ }
2112
2075
  }
2113
2076
  }
2114
- }
2077
+ )
2115
2078
 
2116
- result = self._client.search(index=self._index, body=body)
2079
+ if not clauses:
2080
+ return {"match_all": {}}
2081
+ if len(clauses) == 1:
2082
+ return clauses[0]
2083
+ return {"bool": {"filter": clauses}}
2084
+
2085
+ @staticmethod
2086
+ def _build_composite_agg_body(
2087
+ field_name: str,
2088
+ query: dict[str, Any],
2089
+ size: int,
2090
+ after: dict[str, Any] | None,
2091
+ *,
2092
+ with_count: bool,
2093
+ ) -> dict[str, Any]:
2094
+ composite_agg: dict[str, Any] = {
2095
+ "size": size,
2096
+ "sources": [{field_name: {"terms": {"field": field_name}}}],
2097
+ }
2098
+ if after is not None:
2099
+ composite_agg["after"] = after
2100
+
2101
+ aggs: dict[str, Any] = {"unique_values": {"composite": composite_agg}}
2102
+ if with_count:
2103
+ # cardinality is a single-pass, approximate distinct count - computed alongside the
2104
+ # page fetch at no extra round trip, unlike walking the composite agg to exhaustion.
2105
+ aggs["unique_values_count"] = {"cardinality": {"field": field_name}}
2106
+
2107
+ return {"query": query, "aggs": aggs, "size": 0}
2108
+
2109
+ @staticmethod
2110
+ def _extract_unique_values_and_count(result: dict[str, Any], field_name: str) -> tuple[list[Any], int]:
2117
2111
  aggregations = result.get("aggregations", {})
2112
+ buckets = aggregations.get("unique_values", {}).get("buckets", [])
2113
+ unique_values = [bucket["key"][field_name] for bucket in buckets]
2114
+ total_count = int(aggregations.get("unique_values_count", {}).get("value", 0))
2115
+ return unique_values, total_count
2118
2116
 
2119
- # Extract unique values from composite aggregation buckets
2120
- unique_values_agg = aggregations.get("unique_values", {})
2121
- unique_values_buckets = unique_values_agg.get("buckets", [])
2122
- unique_values = [str(bucket["key"][field_name]) for bucket in unique_values_buckets]
2117
+ def _skip_unique_values(
2118
+ self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
2119
+ ) -> dict[str, Any] | None:
2120
+ """
2121
+ Walks composite aggregation pages to reach `offset`, discarding buckets along the way.
2122
+
2123
+ Composite aggregations only support cursor-based iteration (no native offset), so this
2124
+ replay is the only way to honor an arbitrary `from_` - cost scales with `offset`, not `size`.
2125
+ """
2126
+ assert self._client is not None
2127
+ after_key = None
2128
+ remaining = offset
2129
+ while remaining > 0:
2130
+ step = min(batch_size, remaining)
2131
+ body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
2132
+ result = self._client.search(index=self._index, body=body)
2133
+ buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
2134
+ if not buckets:
2135
+ return after_key
2136
+ after_key = result["aggregations"]["unique_values"].get("after_key")
2137
+ remaining -= len(buckets)
2138
+ if len(buckets) < step:
2139
+ return after_key
2140
+ return after_key
2141
+
2142
+ async def _skip_unique_values_async(
2143
+ self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
2144
+ ) -> dict[str, Any] | None:
2145
+ """
2146
+ Async counterpart of `_skip_unique_values`. See that method for the cost trade-off explanation.
2147
+ """
2148
+ assert self._async_client is not None
2149
+ after_key = None
2150
+ remaining = offset
2151
+ while remaining > 0:
2152
+ step = min(batch_size, remaining)
2153
+ body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
2154
+ result = await self._async_client.search(index=self._index, body=body)
2155
+ buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
2156
+ if not buckets:
2157
+ return after_key
2158
+ after_key = result["aggregations"]["unique_values"].get("after_key")
2159
+ remaining -= len(buckets)
2160
+ if len(buckets) < step:
2161
+ return after_key
2162
+ return after_key
2163
+
2164
+ def get_metadata_field_unique_values(
2165
+ self,
2166
+ metadata_field: str,
2167
+ search_term: str | None = None,
2168
+ from_: int = 0,
2169
+ size: int = 10,
2170
+ filters: dict[str, Any] | None = None,
2171
+ ) -> tuple[list[Any], int]:
2172
+ """
2173
+ Returns unique values for a metadata field, optionally filtered by a search term.
2174
+
2175
+ Internally still backed by composite aggregations, which only support cursor-based iteration.
2176
+ Reaching offset `from_` therefore requires walking and discarding the first `from_` buckets -
2177
+ cost scales with `from_`, not `size`.
2178
+
2179
+ **Note**: To keep this signature uniform across document stores, offset-based pagination is
2180
+ emulated on top of the cursor by re-fetching and discarding every bucket before `from_` on each
2181
+ call, requiring additional search round-trips proportional to `from_`.
2182
+ Performance may degrade for high-cardinality metadata fields and large from_ values.
2183
+
2184
+ **Note**: `total_count` is computed via an approximate cardinality aggregation; for fields with
2185
+ very high cardinality it may not be exact.
2186
+
2187
+ :param metadata_field: The metadata field to get unique values for. Can include or omit the
2188
+ "meta." prefix.
2189
+ :param search_term: Optional case-insensitive substring to filter the returned values by, matched
2190
+ against the metadata field's own value (not the document content). NOTE: The matching is done
2191
+ with a server-side script and is quite expensive for a large corpus.
2192
+ :param from_: Offset to start returning values from. Defaults to 0.
2193
+ :param size: The number of unique values to return per page. Defaults to 10.
2194
+ :param filters: Optional filters to restrict the documents considered.
2195
+ :returns: A tuple of (list of unique values in their original type, total count of distinct values
2196
+ for the field matching `search_term`). Note that filters also narrows down the number of documents
2197
+ against which the search term is matched.
2198
+ """
2199
+ self._ensure_initialized()
2200
+ assert self._client is not None
2123
2201
 
2124
- # Extract after_key for pagination
2125
- # If we got fewer results than requested, we've reached the end
2126
- after_key = unique_values_agg.get("after_key")
2127
- if after_key is not None and size is not None and len(unique_values_buckets) < size:
2128
- after_key = None
2202
+ field_name = _normalize_metadata_field_name(metadata_field)
2203
+ query = self._build_unique_values_field_query(field_name, search_term, filters)
2204
+
2205
+ after_key = self._skip_unique_values(field_name, query, from_) if from_ > 0 else None
2129
2206
 
2130
- return unique_values, after_key
2207
+ body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
2208
+ result = self._client.search(index=self._index, body=body)
2209
+ return self._extract_unique_values_and_count(dict(result), field_name)
2131
2210
 
2132
2211
  async def get_metadata_field_unique_values_async(
2133
2212
  self,
2134
2213
  metadata_field: str,
2135
2214
  search_term: str | None = None,
2136
- size: int | None = 10000,
2137
- after: dict[str, Any] | None = None,
2138
- ) -> tuple[list[str], dict[str, Any] | None]:
2139
- """
2140
- Asynchronously returns unique values for a metadata field, optionally filtered by a search term.
2141
-
2142
- Uses composite aggregations for proper pagination beyond 10k results.
2143
-
2144
- :param metadata_field: The metadata field to get unique values for.
2145
- :param search_term: Optional term to filter the returned values by, matching as a case-insensitive substring
2146
- of the metadata field's own value (not the document content). NOTE: The matching is done with a server-side
2147
- script to accomplish the substring matching on the value of the field and this operation is quite expensive
2148
- for a large corpus.
2149
- :param size: The number of unique values to return per page. Defaults to 10000.
2150
- :param after: Optional pagination key from the previous response. Use None for the first page.
2151
- For subsequent pages, pass the `after_key` from the previous response.
2152
- :returns: A tuple containing (list of unique values, after_key for pagination).
2153
- The after_key is None when there are no more results. Use it in the `after` parameter
2154
- for the next page.
2215
+ from_: int = 0,
2216
+ size: int = 10,
2217
+ filters: dict[str, Any] | None = None,
2218
+ ) -> tuple[list[Any], int]:
2219
+ """
2220
+ Asynchronous counterpart of `get_metadata_field_unique_values`.
2221
+
2222
+ :param metadata_field: The metadata field to get unique values for. Can include or omit the
2223
+ "meta." prefix.
2224
+ :param search_term: Optional case-insensitive substring to filter the returned values by, matched
2225
+ against the metadata field's own value (not the document content). NOTE: The matching is done
2226
+ with a server-side script and is quite expensive for a large corpus.
2227
+ :param from_: Offset to start returning values from. Defaults to 0.
2228
+ :param size: The number of unique values to return per page. Defaults to 10.
2229
+ :param filters: Optional filters to restrict the documents considered.
2230
+ :returns: A tuple of (list of unique values in their original type, total count of distinct values
2231
+ for the field matching `search_term`). Note that filters also narrows down the number of documents
2232
+ against which the search term is matched.
2155
2233
  """
2156
2234
  await self._ensure_initialized_async()
2157
2235
  assert self._async_client is not None
2158
2236
 
2159
2237
  field_name = _normalize_metadata_field_name(metadata_field)
2238
+ query = self._build_unique_values_field_query(field_name, search_term, filters)
2160
2239
 
2161
- # Build composite aggregation for proper pagination
2162
- composite_agg: dict[str, Any] = {
2163
- "size": size,
2164
- "sources": [{field_name: {"terms": {"field": field_name}}}],
2165
- }
2166
- if after is not None:
2167
- composite_agg["after"] = after
2168
-
2169
- body: dict[str, Any] = {
2170
- "aggs": {
2171
- "unique_values": {
2172
- "composite": composite_agg,
2173
- }
2174
- },
2175
- "size": 0, # we only need aggregations, not documents
2176
- }
2177
- if search_term:
2178
- # Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
2179
- # standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
2180
- # numeric ones. A doc-value script query works uniformly across field types by stringifying the
2181
- # field's value, matching the case-insensitive substring semantics documented above. The term is
2182
- # lower-cased once here rather than per-document in the script.
2183
- body["query"] = {
2184
- "script": {
2185
- "script": {
2186
- "source": (
2187
- "def v = doc[params.field]; "
2188
- "if (v.size() == 0) { return false; } "
2189
- "return v.value.toString().toLowerCase().contains(params.term)"
2190
- ),
2191
- "params": {"field": field_name, "term": search_term.lower()},
2192
- }
2193
- }
2194
- }
2240
+ after_key = await self._skip_unique_values_async(field_name, query, from_) if from_ > 0 else None
2195
2241
 
2242
+ body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
2196
2243
  result = await self._async_client.search(index=self._index, body=body)
2197
- aggregations = result.get("aggregations", {})
2198
-
2199
- # Extract unique values from composite aggregation buckets
2200
- unique_values_agg = aggregations.get("unique_values", {})
2201
- unique_values_buckets = unique_values_agg.get("buckets", [])
2202
- unique_values = [str(bucket["key"][field_name]) for bucket in unique_values_buckets]
2203
-
2204
- # Extract after_key for pagination
2205
- # If we got fewer results than requested, we've reached the end
2206
- after_key = unique_values_agg.get("after_key")
2207
- if after_key is not None and size is not None and len(unique_values_buckets) < size:
2208
- after_key = None
2209
-
2210
- return unique_values, after_key
2244
+ return self._extract_unique_values_and_count(dict(result), field_name)
2211
2245
 
2212
2246
  def _query_sql(self, query: str, fetch_size: int | None = None) -> dict[str, Any]:
2213
2247
  """