opensearch-haystack 8.2.0__tar.gz → 8.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/CHANGELOG.md +94 -28
  2. opensearch_haystack-8.2.0/LICENSE → opensearch_haystack-8.3.0/LICENSE.txt +1 -1
  3. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/PKG-INFO +5 -5
  4. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/README.md +2 -2
  5. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/docker-compose.yml +1 -1
  6. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/pyproject.toml +1 -0
  7. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/auth.py +1 -6
  8. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/document_store.py +176 -114
  9. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/filters.py +31 -9
  10. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_document_store.py +166 -213
  11. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_document_store_async.py +189 -1
  12. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_filters.py +186 -0
  13. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_open_search_hybrid_retriever.py +3 -3
  14. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/.gitignore +0 -0
  15. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/pydoc/config_docusaurus.yml +0 -0
  16. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/__init__.py +0 -0
  17. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/bm25_retriever.py +0 -0
  18. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/embedding_retriever.py +0 -0
  19. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/metadata_retriever.py +0 -0
  20. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/open_search_hybrid_retriever.py +0 -0
  21. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/sql_retriever.py +0 -0
  22. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/opensearch/utils.py +0 -0
  23. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
  24. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/__init__.py +0 -0
  25. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/opensearch/opensearch_scripts.py +0 -0
  26. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/src/haystack_integrations/document_stores/py.typed +0 -0
  27. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/__init__.py +0 -0
  28. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/conftest.py +0 -0
  29. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_auth.py +0 -0
  30. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_bm25_retriever.py +0 -0
  31. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_document_store_common.py +0 -0
  32. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_embedding_retriever.py +0 -0
  33. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever.py +0 -0
  34. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_metadata_retriever_async.py +0 -0
  35. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_sql_retriever.py +0 -0
  36. {opensearch_haystack-8.2.0 → opensearch_haystack-8.3.0}/tests/test_utils.py +0 -0
@@ -1,5 +1,33 @@
1
1
  # Changelog
2
2
 
3
+ ## [integrations/opensearch-v8.2.1] - 2026-08-03
4
+
5
+ ### 🐛 Bug Fixes
6
+
7
+ - Fix new issues raised by ruff 0.16.0 (#3670)
8
+ - `OpensearchDocumentStore` match search_term against metadata field value, not content (#3647)
9
+ - *(opensearch)* Don't merge same-field ranges under OR (#3589)
10
+
11
+
12
+ ## [integrations/opensearch-v8.2.0] - 2026-07-22
13
+
14
+ ### 🚀 Features
15
+
16
+ - Opensearch - add methods to close resources (#3640)
17
+
18
+ ### 🐛 Bug Fixes
19
+
20
+ - Adapt optimum, ragas, elasticsearch, opensearch to APIs removed in Haystack 3.0 (#3538)
21
+
22
+ ### 🚜 Refactor
23
+
24
+ - Elasticsearch - split sync/async paths (#3633)
25
+
26
+ ### 📚 Documentation
27
+
28
+ - Replace old haystack core imports with haystack_integrations paths (#3545)
29
+
30
+
3
31
  ## [integrations/opensearch-v8.1.2] - 2026-06-24
4
32
 
5
33
  ### 🐛 Bug Fixes
@@ -315,6 +343,13 @@
315
343
 
316
344
  ## [integrations/opensearch-v3.0.0] - 2025-03-11
317
345
 
346
+
347
+ ### 🌀 Miscellaneous
348
+
349
+ - Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
350
+
351
+ ## [integrations/opensearch-v2.2.0] - 2025-03-07
352
+
318
353
  ### 🐛 Bug Fixes
319
354
 
320
355
  - OpenSearchDocumentStore depends on async opensearch-py (#1438)
@@ -327,10 +362,6 @@
327
362
 
328
363
  - Remove Python 3.8 support (#1421)
329
364
 
330
- ### 🌀 Miscellaneous
331
-
332
- - Chore: OpenSearch - pin haystack and remove dataframe checks (#1513)
333
-
334
365
  ## [integrations/opensearch-v2.1.0] - 2025-02-18
335
366
 
336
367
  ### 🚀 Features
@@ -340,10 +371,6 @@
340
371
 
341
372
  ## [integrations/opensearch-v2.0.0] - 2025-02-14
342
373
 
343
- ### 🚀 Features
344
-
345
- - Add Secret handling in OpenSearchDocumentStore (#1288)
346
-
347
374
  ### 🧹 Chores
348
375
 
349
376
  - Inherit from `FilterDocumentsTestWithDataframe` in Document Stores (#1290)
@@ -353,6 +380,13 @@
353
380
 
354
381
  - Chore: OpenSearch - manually fix changelog (#1299)
355
382
 
383
+ ## [integrations/opensearch-v1.3.0] - 2025-01-16
384
+
385
+ ### 🚀 Features
386
+
387
+ - Add Secret handling in OpenSearchDocumentStore (#1288)
388
+
389
+
356
390
  ## [integrations/opensearch-v1.2.0] - 2024-12-12
357
391
 
358
392
  ### 🧹 Chores
@@ -366,6 +400,17 @@
366
400
 
367
401
  - Efficient knn filtering support for OpenSearch (#1134)
368
402
 
403
+ ### ⚙️ CI
404
+
405
+ - Adopt uv as installer (#1142)
406
+
407
+ ### 🧹 Chores
408
+
409
+ - Update changelog after removing legacy filters (#1083)
410
+ - Update ruff linting scripts and settings (#1105)
411
+
412
+ ## [integrations/opensearch-v1.0.0] - 2024-09-12
413
+
369
414
  ### 📚 Documentation
370
415
 
371
416
  - Update opensearch retriever docstrings (#1035)
@@ -374,15 +419,9 @@
374
419
 
375
420
  - Do not retry tests in `hatch run test` command (#954)
376
421
 
377
- ### ⚙️ CI
378
-
379
- - Adopt uv as installer (#1142)
380
-
381
422
  ### 🧹 Chores
382
423
 
383
424
  - OpenSearch - remove legacy filter support (#1067)
384
- - Update changelog after removing legacy filters (#1083)
385
- - Update ruff linting scripts and settings (#1105)
386
425
 
387
426
  ### 🌀 Miscellaneous
388
427
 
@@ -399,15 +438,17 @@
399
438
 
400
439
  ## [integrations/opensearch-v0.8.1] - 2024-07-15
401
440
 
441
+ ### 🐛 Bug Fixes
442
+
443
+ - `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
444
+
445
+ ## [integrations/opensearch-v0.8.0] - 2024-07-15
446
+
402
447
  ### 🚀 Features
403
448
 
404
449
  - Add raise_on_failure param to OpenSearch retrievers (#852)
405
450
  - Add filter_policy to opensearch integration (#822)
406
451
 
407
- ### 🐛 Bug Fixes
408
-
409
- - `OpenSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#895)
410
-
411
452
  ### 🧹 Chores
412
453
 
413
454
  - Update ruff invocation to include check parameter (#853)
@@ -428,12 +469,44 @@
428
469
 
429
470
  ### 🚀 Features
430
471
 
431
- - Defer the database connection to when it's needed (#753)
432
- - Improve `OpenSearchDocumentStore.__init__` arguments (#739)
433
- - Return_embeddings flag for opensearch (#784)
434
472
  - Add create_index option to OpenSearchDocumentStore (#840)
435
473
  - Add custom_query param to OpenSearch retrievers (#841)
436
474
 
475
+ ### ⚙️ CI
476
+
477
+ - Retry tests to reduce flakyness (#836)
478
+
479
+ ### 🌀 Miscellaneous
480
+
481
+ - Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
482
+
483
+ ## [integrations/opensearch-v0.6.0] - 2024-06-07
484
+
485
+ ### 🚀 Features
486
+
487
+ - Return_embeddings flag for opensearch (#784)
488
+
489
+
490
+ ## [integrations/opensearch-v0.5.0] - 2024-05-30
491
+
492
+ ### 🚀 Features
493
+
494
+ - Improve `OpenSearchDocumentStore.__init__` arguments (#739)
495
+
496
+
497
+ ## [integrations/opensearch-v0.4.0] - 2024-05-24
498
+
499
+ ### 🚀 Features
500
+
501
+ - Defer the database connection to when it's needed (#753)
502
+
503
+ ### 🌀 Miscellaneous
504
+
505
+ - Chore: add license classifiers (#680)
506
+ - Chore: change the pydoc renderer class (#718)
507
+
508
+ ## [integrations/opensearch-v0.3.0] - 2024-04-04
509
+
437
510
  ### 🐛 Bug Fixes
438
511
 
439
512
  - Fix order of API docs (#447)
@@ -445,10 +518,6 @@
445
518
  - Small consistency improvements (#536)
446
519
  - Disable-class-def (#556)
447
520
 
448
- ### ⚙️ CI
449
-
450
- - Retry tests to reduce flakyness (#836)
451
-
452
521
  ### 🌀 Miscellaneous
453
522
 
454
523
  - Generate API docs (#324)
@@ -457,9 +526,6 @@
457
526
  - Fix opensearch errors bulk write (#594)
458
527
  - Remove references to Python 3.7 (#601)
459
528
  - [Elasticsearch] fix: Filters not working with metadata that contain a space or capitalization (#639)
460
- - Chore: add license classifiers (#680)
461
- - Chore: change the pydoc renderer class (#718)
462
- - Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
463
529
 
464
530
  ## [integrations/opensearch-v0.2.0] - 2024-01-17
465
531
 
@@ -186,7 +186,7 @@
186
186
  same "printed page" as the copyright notice for easier
187
187
  identification within third-party archives.
188
188
 
189
- Copyright 2023 deepset GmbH
189
+ Copyright 2023-present deepset GmbH
190
190
 
191
191
  Licensed under the Apache License, Version 2.0 (the "License");
192
192
  you may not use this file except in compliance with the License.
@@ -1,13 +1,13 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: opensearch-haystack
3
- Version: 8.2.0
3
+ Version: 8.3.0
4
4
  Summary: Haystack 2.x Document Store for OpenSearch
5
5
  Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch#readme
6
6
  Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
7
7
  Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/opensearch
8
8
  Author-email: deepset GmbH <info@deepset.ai>
9
9
  License-Expression: Apache-2.0
10
- License-File: LICENSE
10
+ License-File: LICENSE.txt
11
11
  Classifier: Development Status :: 4 - Beta
12
12
  Classifier: License :: OSI Approved :: Apache Software License
13
13
  Classifier: Programming Language :: Python
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
28
28
  [![PyPI - Version](https://img.shields.io/pypi/v/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
29
29
  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
30
30
 
31
- - [Integration page](https://haystack.deepset.ai/integrations/opensearch)
31
+ - [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
32
32
  - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
33
33
 
34
34
  ---
@@ -38,4 +38,4 @@ Description-Content-Type: text/markdown
38
38
  Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
39
39
 
40
40
  To run integration tests locally, you need a Docker container running OpenSearch.
41
- Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
41
+ Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
@@ -3,7 +3,7 @@
3
3
  [![PyPI - Version](https://img.shields.io/pypi/v/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
4
4
  [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)
5
5
 
6
- - [Integration page](https://haystack.deepset.ai/integrations/opensearch)
6
+ - [Integration page](https://haystack.deepset.ai/integrations/opensearch-document-store)
7
7
  - [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/opensearch/CHANGELOG.md)
8
8
 
9
9
  ---
@@ -13,4 +13,4 @@
13
13
  Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
14
14
 
15
15
  To run integration tests locally, you need a Docker container running OpenSearch.
16
- Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
16
+ Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.
@@ -10,7 +10,7 @@ services:
10
10
  - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
11
11
  - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=SecureHaystack!2026"
12
12
  healthcheck:
13
- test: curl --fail https://localhost:9200/_cat/health -ku admin:admin || exit 1
13
+ test: curl --fail https://localhost:9200/_cat/health -ku admin:SecureHaystack!2026 || exit 1
14
14
  interval: 10s
15
15
  timeout: 1s
16
16
  retries: 10
@@ -139,6 +139,7 @@ ignore = [
139
139
  "PLR0912",
140
140
  "PLR0913",
141
141
  "PLR0915",
142
+ "PLR0917",
142
143
  # Ignore asserts
143
144
  "S101",
144
145
  # Allow `Any` - used legitimately for dynamic types and SDK boundaries
@@ -4,7 +4,7 @@ from typing import Any, Optional, TypeVar
4
4
  from haystack import default_from_dict, default_to_dict
5
5
  from haystack.document_stores.errors import DocumentStoreError
6
6
  from haystack.lazy_imports import LazyImport
7
- from haystack.utils.auth import Secret, deserialize_secrets_inplace
7
+ from haystack.utils.auth import Secret
8
8
  from opensearchpy import AWSV4SignerAsyncAuth, Urllib3AWSV4SignerAuth
9
9
 
10
10
  with LazyImport("Run 'pip install \"boto3\"' to install boto3.") as boto3_import:
@@ -119,11 +119,6 @@ class AWSAuth:
119
119
  """
120
120
  Converts a dictionary representation to an AWSAuth object.
121
121
  """
122
- init_parameters = data.get("init_parameters", {})
123
- deserialize_secrets_inplace(
124
- init_parameters,
125
- ["aws_access_key_id", "aws_secret_access_key", "aws_session_token", "aws_region_name", "aws_profile_name"],
126
- )
127
122
  return default_from_dict(cls, data)
128
123
 
129
124
  def __call__(
@@ -25,6 +25,13 @@ from haystack_integrations.document_stores.opensearch.opensearch_scripts import
25
25
 
26
26
  logger = logging.getLogger(__name__)
27
27
 
28
+ # Meta keys are passed only as params and never interpolated into the script source, so they can't inject code.
29
+ UPDATE_SCRIPT = """
30
+ for (entry in params.entrySet()) {
31
+ ctx._source[entry.getKey()] = entry.getValue();
32
+ }
33
+ """
34
+
28
35
  SPECIAL_FIELDS = {"content", "embedding", "id", "score", "sparse_embedding", "blob"}
29
36
 
30
37
  Hosts = str | list[str | Mapping[str, str | int]]
@@ -959,16 +966,10 @@ class OpenSearchDocumentStore:
959
966
 
960
967
  try:
961
968
  normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
962
- # Build the update script to modify metadata fields
963
969
  # Documents are stored with flattened metadata, so update fields directly in ctx._source
964
- update_script_lines = []
965
- for key in meta.keys():
966
- update_script_lines.append(f"ctx._source.{key} = params.{key};")
967
- update_script = " ".join(update_script_lines)
968
-
969
970
  body = {
970
971
  "query": {"bool": {"filter": normalized_filters}},
971
- "script": {"source": update_script, "params": meta, "lang": "painless"},
972
+ "script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
972
973
  }
973
974
  result = self._client.update_by_query(index=self._index, body=body, refresh=refresh)
974
975
  updated_count = result.get("updated", 0)
@@ -999,16 +1000,10 @@ class OpenSearchDocumentStore:
999
1000
 
1000
1001
  try:
1001
1002
  normalized_filters = normalize_filters(filters, nested_fields=self._resolved_nested_fields)
1002
- # Build the update script to modify metadata fields
1003
1003
  # Documents are stored with flattened metadata, so update fields directly in ctx._source
1004
- update_script_lines = []
1005
- for key in meta.keys():
1006
- update_script_lines.append(f"ctx._source.{key} = params.{key};")
1007
- update_script = " ".join(update_script_lines)
1008
-
1009
1004
  body = {
1010
1005
  "query": {"bool": {"filter": normalized_filters}},
1011
- "script": {"source": update_script, "params": meta, "lang": "painless"},
1006
+ "script": {"source": UPDATE_SCRIPT, "params": meta, "lang": "painless"},
1012
1007
  }
1013
1008
  result = await self._async_client.update_by_query(index=self._index, body=body, refresh=refresh)
1014
1009
  updated_count = result.get("updated", 0)
@@ -2049,39 +2044,53 @@ class OpenSearchDocumentStore:
2049
2044
 
2050
2045
  return self._extract_min_max_from_stats(stats)
2051
2046
 
2052
- def get_metadata_field_unique_values(
2053
- self,
2054
- metadata_field: str,
2055
- search_term: str | None = None,
2056
- size: int | None = 10000,
2057
- after: dict[str, Any] | None = None,
2058
- ) -> tuple[list[str], dict[str, Any] | None]:
2047
+ def _build_unique_values_field_query(
2048
+ self, field_name: str, search_term: str | None, filters: dict[str, Any] | None = None
2049
+ ) -> dict[str, Any]:
2059
2050
  """
2060
- Returns unique values for a metadata field, optionally filtered by a search term in the content.
2051
+ Builds a query matching docs whose metadata field's contains `search_term`, optionally restricted by `filters`.
2061
2052
 
2062
- Uses composite aggregations for proper pagination beyond 10k results.
2063
-
2064
- :param metadata_field: The metadata field to get unique values for.
2065
- :param search_term: Optional search term to filter documents by matching in the content field.
2066
- :param size: The number of unique values to return per page. Defaults to 10000.
2067
- :param after: Optional pagination key from the previous response. Use None for the first page.
2068
- For subsequent pages, pass the `after_key` from the previous response.
2069
- :returns: A tuple containing (list of unique values, after_key for pagination).
2070
- The after_key is None when there are no more results. Use it in the `after` parameter
2071
- for the next page.
2053
+ Matching is a case-insensitive substring match (not against the document content).
2072
2054
  """
2073
- self._ensure_initialized()
2074
- assert self._client is not None
2075
-
2076
- field_name = _normalize_metadata_field_name(metadata_field)
2077
-
2078
- # filter by search_term if provided
2079
- query: dict[str, Any] = {"match_all": {}}
2055
+ clauses: list[dict[str, Any]] = []
2056
+ if filters:
2057
+ clauses.append({"bool": {"filter": normalize_filters(filters, nested_fields=self._resolved_nested_fields)}})
2080
2058
  if search_term:
2081
- # Use match_phrase for exact phrase matching to avoid tokenization issues
2082
- query = {"match_phrase": {"content": search_term}}
2059
+ # Composite aggregation terms sources don't support `include`/`exclude` (that's only valid on
2060
+ # standalone `terms` aggregations), and a `regexp` query only works on keyword/text fields, not
2061
+ # numeric ones. A doc-value script query works uniformly across field types by stringifying the
2062
+ # field's value, matching the case-insensitive substring semantics documented above. The term is
2063
+ # lower-cased once here rather than per-document in the script.
2064
+ clauses.append(
2065
+ {
2066
+ "script": {
2067
+ "script": {
2068
+ "source": (
2069
+ "def v = doc[params.field]; "
2070
+ "if (v.size() == 0) { return false; } "
2071
+ "return v.value.toString().toLowerCase().contains(params.term)"
2072
+ ),
2073
+ "params": {"field": field_name, "term": search_term.lower()},
2074
+ }
2075
+ }
2076
+ }
2077
+ )
2078
+
2079
+ if not clauses:
2080
+ return {"match_all": {}}
2081
+ if len(clauses) == 1:
2082
+ return clauses[0]
2083
+ return {"bool": {"filter": clauses}}
2083
2084
 
2084
- # Build composite aggregation for proper pagination
2085
+ @staticmethod
2086
+ def _build_composite_agg_body(
2087
+ field_name: str,
2088
+ query: dict[str, Any],
2089
+ size: int,
2090
+ after: dict[str, Any] | None,
2091
+ *,
2092
+ with_count: bool,
2093
+ ) -> dict[str, Any]:
2085
2094
  composite_agg: dict[str, Any] = {
2086
2095
  "size": size,
2087
2096
  "sources": [{field_name: {"terms": {"field": field_name}}}],
@@ -2089,97 +2098,150 @@ class OpenSearchDocumentStore:
2089
2098
  if after is not None:
2090
2099
  composite_agg["after"] = after
2091
2100
 
2092
- body = {
2093
- "query": query,
2094
- "aggs": {
2095
- "unique_values": {
2096
- "composite": composite_agg,
2097
- }
2098
- },
2099
- "size": 0, # we only need aggregations, not documents
2100
- }
2101
+ aggs: dict[str, Any] = {"unique_values": {"composite": composite_agg}}
2102
+ if with_count:
2103
+ # cardinality is a single-pass, approximate distinct count - computed alongside the
2104
+ # page fetch at no extra round trip, unlike walking the composite agg to exhaustion.
2105
+ aggs["unique_values_count"] = {"cardinality": {"field": field_name}}
2101
2106
 
2102
- result = self._client.search(index=self._index, body=body)
2103
- aggregations = result.get("aggregations", {})
2107
+ return {"query": query, "aggs": aggs, "size": 0}
2104
2108
 
2105
- # Extract unique values from composite aggregation buckets
2106
- unique_values_agg = aggregations.get("unique_values", {})
2107
- unique_values_buckets = unique_values_agg.get("buckets", [])
2108
- unique_values = [str(bucket["key"][field_name]) for bucket in unique_values_buckets]
2109
+ @staticmethod
2110
+ def _extract_unique_values_and_count(result: dict[str, Any], field_name: str) -> tuple[list[Any], int]:
2111
+ aggregations = result.get("aggregations", {})
2112
+ buckets = aggregations.get("unique_values", {}).get("buckets", [])
2113
+ unique_values = [bucket["key"][field_name] for bucket in buckets]
2114
+ total_count = int(aggregations.get("unique_values_count", {}).get("value", 0))
2115
+ return unique_values, total_count
2109
2116
 
2110
- # Extract after_key for pagination
2111
- # If we got fewer results than requested, we've reached the end
2112
- after_key = unique_values_agg.get("after_key")
2113
- if after_key is not None and size is not None and len(unique_values_buckets) < size:
2114
- after_key = None
2117
+ def _skip_unique_values(
2118
+ self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
2119
+ ) -> dict[str, Any] | None:
2120
+ """
2121
+ Walks composite aggregation pages to reach `offset`, discarding buckets along the way.
2115
2122
 
2116
- return unique_values, after_key
2123
+ Composite aggregations only support cursor-based iteration (no native offset), so this
2124
+ replay is the only way to honor an arbitrary `from_` - cost scales with `offset`, not `size`.
2125
+ """
2126
+ assert self._client is not None
2127
+ after_key = None
2128
+ remaining = offset
2129
+ while remaining > 0:
2130
+ step = min(batch_size, remaining)
2131
+ body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
2132
+ result = self._client.search(index=self._index, body=body)
2133
+ buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
2134
+ if not buckets:
2135
+ return after_key
2136
+ after_key = result["aggregations"]["unique_values"].get("after_key")
2137
+ remaining -= len(buckets)
2138
+ if len(buckets) < step:
2139
+ return after_key
2140
+ return after_key
2141
+
2142
+ async def _skip_unique_values_async(
2143
+ self, field_name: str, query: dict[str, Any], offset: int, batch_size: int = 10000
2144
+ ) -> dict[str, Any] | None:
2145
+ """
2146
+ Async counterpart of `_skip_unique_values`. See that method for the cost trade-off explanation.
2147
+ """
2148
+ assert self._async_client is not None
2149
+ after_key = None
2150
+ remaining = offset
2151
+ while remaining > 0:
2152
+ step = min(batch_size, remaining)
2153
+ body = self._build_composite_agg_body(field_name, query, step, after_key, with_count=False)
2154
+ result = await self._async_client.search(index=self._index, body=body)
2155
+ buckets = result.get("aggregations", {}).get("unique_values", {}).get("buckets", [])
2156
+ if not buckets:
2157
+ return after_key
2158
+ after_key = result["aggregations"]["unique_values"].get("after_key")
2159
+ remaining -= len(buckets)
2160
+ if len(buckets) < step:
2161
+ return after_key
2162
+ return after_key
2117
2163
 
2118
- async def get_metadata_field_unique_values_async(
2164
+ def get_metadata_field_unique_values(
2119
2165
  self,
2120
2166
  metadata_field: str,
2121
2167
  search_term: str | None = None,
2122
- size: int | None = 10000,
2123
- after: dict[str, Any] | None = None,
2124
- ) -> tuple[list[str], dict[str, Any] | None]:
2168
+ from_: int = 0,
2169
+ size: int = 10,
2170
+ filters: dict[str, Any] | None = None,
2171
+ ) -> tuple[list[Any], int]:
2172
+ """
2173
+ Returns unique values for a metadata field, optionally filtered by a search term.
2174
+
2175
+ Internally still backed by composite aggregations, which only support cursor-based iteration.
2176
+ Reaching offset `from_` therefore requires walking and discarding the first `from_` buckets -
2177
+ cost scales with `from_`, not `size`.
2178
+
2179
+ **Note**: To keep this signature uniform across document stores, offset-based pagination is
2180
+ emulated on top of the cursor by re-fetching and discarding every bucket before `from_` on each
2181
+ call, requiring additional search round-trips proportional to `from_`.
2182
+ Performance may degrade for high-cardinality metadata fields and large from_ values.
2183
+
2184
+ **Note**: `total_count` is computed via an approximate cardinality aggregation; for fields with
2185
+ very high cardinality it may not be exact.
2186
+
2187
+ :param metadata_field: The metadata field to get unique values for. Can include or omit the
2188
+ "meta." prefix.
2189
+ :param search_term: Optional case-insensitive substring to filter the returned values by, matched
2190
+ against the metadata field's own value (not the document content). NOTE: The matching is done
2191
+ with a server-side script and is quite expensive for a large corpus.
2192
+ :param from_: Offset to start returning values from. Defaults to 0.
2193
+ :param size: The number of unique values to return per page. Defaults to 10.
2194
+ :param filters: Optional filters to restrict the documents considered.
2195
+ :returns: A tuple of (list of unique values in their original type, total count of distinct values
2196
+ for the field matching `search_term`). Note that filters also narrows down the number of documents
2197
+ against which the search term is matched.
2125
2198
  """
2126
- Asynchronously returns unique values for a metadata field, optionally filtered by a search term in the content.
2199
+ self._ensure_initialized()
2200
+ assert self._client is not None
2127
2201
 
2128
- Uses composite aggregations for proper pagination beyond 10k results.
2202
+ field_name = _normalize_metadata_field_name(metadata_field)
2203
+ query = self._build_unique_values_field_query(field_name, search_term, filters)
2204
+
2205
+ after_key = self._skip_unique_values(field_name, query, from_) if from_ > 0 else None
2129
2206
 
2130
- :param metadata_field: The metadata field to get unique values for.
2131
- :param search_term: Optional search term to filter documents by matching in the content field.
2132
- :param size: The number of unique values to return per page. Defaults to 10000.
2133
- :param after: Optional pagination key from the previous response. Use None for the first page.
2134
- For subsequent pages, pass the `after_key` from the previous response.
2135
- :returns: A tuple containing (list of unique values, after_key for pagination).
2136
- The after_key is None when there are no more results. Use it in the `after` parameter
2137
- for the next page.
2207
+ body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
2208
+ result = self._client.search(index=self._index, body=body)
2209
+ return self._extract_unique_values_and_count(dict(result), field_name)
2210
+
2211
+ async def get_metadata_field_unique_values_async(
2212
+ self,
2213
+ metadata_field: str,
2214
+ search_term: str | None = None,
2215
+ from_: int = 0,
2216
+ size: int = 10,
2217
+ filters: dict[str, Any] | None = None,
2218
+ ) -> tuple[list[Any], int]:
2219
+ """
2220
+ Asynchronous counterpart of `get_metadata_field_unique_values`.
2221
+
2222
+ :param metadata_field: The metadata field to get unique values for. Can include or omit the
2223
+ "meta." prefix.
2224
+ :param search_term: Optional case-insensitive substring to filter the returned values by, matched
2225
+ against the metadata field's own value (not the document content). NOTE: The matching is done
2226
+ with a server-side script and is quite expensive for a large corpus.
2227
+ :param from_: Offset to start returning values from. Defaults to 0.
2228
+ :param size: The number of unique values to return per page. Defaults to 10.
2229
+ :param filters: Optional filters to restrict the documents considered.
2230
+ :returns: A tuple of (list of unique values in their original type, total count of distinct values
2231
+ for the field matching `search_term`). Note that filters also narrows down the number of documents
2232
+ against which the search term is matched.
2138
2233
  """
2139
2234
  await self._ensure_initialized_async()
2140
2235
  assert self._async_client is not None
2141
2236
 
2142
2237
  field_name = _normalize_metadata_field_name(metadata_field)
2238
+ query = self._build_unique_values_field_query(field_name, search_term, filters)
2143
2239
 
2144
- # filter by search_term if provided
2145
- query: dict[str, Any] = {"match_all": {}}
2146
- if search_term:
2147
- # Use match_phrase for exact phrase matching to avoid tokenization issues
2148
- query = {"match_phrase": {"content": search_term}}
2149
-
2150
- # Build composite aggregation for proper pagination
2151
- composite_agg: dict[str, Any] = {
2152
- "size": size,
2153
- "sources": [{field_name: {"terms": {"field": field_name}}}],
2154
- }
2155
- if after is not None:
2156
- composite_agg["after"] = after
2157
-
2158
- body = {
2159
- "query": query,
2160
- "aggs": {
2161
- "unique_values": {
2162
- "composite": composite_agg,
2163
- }
2164
- },
2165
- "size": 0, # we only need aggregations, not documents
2166
- }
2240
+ after_key = await self._skip_unique_values_async(field_name, query, from_) if from_ > 0 else None
2167
2241
 
2242
+ body = self._build_composite_agg_body(field_name, query, size, after_key, with_count=True)
2168
2243
  result = await self._async_client.search(index=self._index, body=body)
2169
- aggregations = result.get("aggregations", {})
2170
-
2171
- # Extract unique values from composite aggregation buckets
2172
- unique_values_agg = aggregations.get("unique_values", {})
2173
- unique_values_buckets = unique_values_agg.get("buckets", [])
2174
- unique_values = [str(bucket["key"][field_name]) for bucket in unique_values_buckets]
2175
-
2176
- # Extract after_key for pagination
2177
- # If we got fewer results than requested, we've reached the end
2178
- after_key = unique_values_agg.get("after_key")
2179
- if after_key is not None and size is not None and len(unique_values_buckets) < size:
2180
- after_key = None
2181
-
2182
- return unique_values, after_key
2244
+ return self._extract_unique_values_and_count(dict(result), field_name)
2183
2245
 
2184
2246
  def _query_sql(self, query: str, fetch_size: int | None = None) -> dict[str, Any]:
2185
2247
  """