gllm-datastore-binary 0.5.50__cp312-cp312-macosx_13_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. gllm_datastore/__init__.pyi +0 -0
  2. gllm_datastore/cache/__init__.pyi +4 -0
  3. gllm_datastore/cache/base.pyi +84 -0
  4. gllm_datastore/cache/cache.pyi +137 -0
  5. gllm_datastore/cache/hybrid_cache/__init__.pyi +5 -0
  6. gllm_datastore/cache/hybrid_cache/file_system_hybrid_cache.pyi +50 -0
  7. gllm_datastore/cache/hybrid_cache/hybrid_cache.pyi +115 -0
  8. gllm_datastore/cache/hybrid_cache/in_memory_hybrid_cache.pyi +29 -0
  9. gllm_datastore/cache/hybrid_cache/key_matcher/__init__.pyi +5 -0
  10. gllm_datastore/cache/hybrid_cache/key_matcher/exact_key_matcher.pyi +44 -0
  11. gllm_datastore/cache/hybrid_cache/key_matcher/fuzzy_key_matcher.pyi +70 -0
  12. gllm_datastore/cache/hybrid_cache/key_matcher/key_matcher.pyi +60 -0
  13. gllm_datastore/cache/hybrid_cache/key_matcher/semantic_key_matcher.pyi +93 -0
  14. gllm_datastore/cache/hybrid_cache/redis_hybrid_cache.pyi +34 -0
  15. gllm_datastore/cache/hybrid_cache/utils.pyi +36 -0
  16. gllm_datastore/cache/utils.pyi +34 -0
  17. gllm_datastore/cache/vector_cache/__init__.pyi +0 -0
  18. gllm_datastore/cache/vector_cache/eviction_manager/__init__.pyi +0 -0
  19. gllm_datastore/cache/vector_cache/eviction_manager/asyncio_eviction_manager.pyi +48 -0
  20. gllm_datastore/cache/vector_cache/eviction_manager/eviction_manager.pyi +38 -0
  21. gllm_datastore/cache/vector_cache/eviction_strategy/__init__.pyi +0 -0
  22. gllm_datastore/cache/vector_cache/eviction_strategy/eviction_strategy.pyi +34 -0
  23. gllm_datastore/cache/vector_cache/eviction_strategy/ttl_eviction_strategy.pyi +34 -0
  24. gllm_datastore/cache/vector_cache/vector_cache.pyi +99 -0
  25. gllm_datastore/constants.pyi +66 -0
  26. gllm_datastore/core/__init__.pyi +7 -0
  27. gllm_datastore/core/capabilities/__init__.pyi +7 -0
  28. gllm_datastore/core/capabilities/encryption_capability.pyi +21 -0
  29. gllm_datastore/core/capabilities/fulltext_capability.pyi +73 -0
  30. gllm_datastore/core/capabilities/graph_capability.pyi +70 -0
  31. gllm_datastore/core/capabilities/hybrid_capability.pyi +184 -0
  32. gllm_datastore/core/capabilities/vector_capability.pyi +90 -0
  33. gllm_datastore/core/filters/__init__.pyi +4 -0
  34. gllm_datastore/core/filters/filter.pyi +340 -0
  35. gllm_datastore/core/filters/schema.pyi +149 -0
  36. gllm_datastore/data_store/__init__.pyi +8 -0
  37. gllm_datastore/data_store/_elastic_core/__init__.pyi +0 -0
  38. gllm_datastore/data_store/_elastic_core/client_factory.pyi +66 -0
  39. gllm_datastore/data_store/_elastic_core/constants.pyi +27 -0
  40. gllm_datastore/data_store/_elastic_core/elastic_like_core.pyi +115 -0
  41. gllm_datastore/data_store/_elastic_core/index_manager.pyi +37 -0
  42. gllm_datastore/data_store/_elastic_core/query_translator.pyi +89 -0
  43. gllm_datastore/data_store/base.pyi +176 -0
  44. gllm_datastore/data_store/chroma/__init__.pyi +4 -0
  45. gllm_datastore/data_store/chroma/_chroma_import.pyi +13 -0
  46. gllm_datastore/data_store/chroma/data_store.pyi +201 -0
  47. gllm_datastore/data_store/chroma/fulltext.pyi +134 -0
  48. gllm_datastore/data_store/chroma/query.pyi +266 -0
  49. gllm_datastore/data_store/chroma/query_translator.pyi +41 -0
  50. gllm_datastore/data_store/chroma/vector.pyi +197 -0
  51. gllm_datastore/data_store/elasticsearch/__init__.pyi +5 -0
  52. gllm_datastore/data_store/elasticsearch/data_store.pyi +147 -0
  53. gllm_datastore/data_store/elasticsearch/fulltext.pyi +238 -0
  54. gllm_datastore/data_store/elasticsearch/query.pyi +118 -0
  55. gllm_datastore/data_store/elasticsearch/query_translator.pyi +18 -0
  56. gllm_datastore/data_store/elasticsearch/vector.pyi +180 -0
  57. gllm_datastore/data_store/exceptions.pyi +35 -0
  58. gllm_datastore/data_store/in_memory/__init__.pyi +5 -0
  59. gllm_datastore/data_store/in_memory/data_store.pyi +71 -0
  60. gllm_datastore/data_store/in_memory/fulltext.pyi +131 -0
  61. gllm_datastore/data_store/in_memory/query.pyi +175 -0
  62. gllm_datastore/data_store/in_memory/vector.pyi +174 -0
  63. gllm_datastore/data_store/opensearch/__init__.pyi +5 -0
  64. gllm_datastore/data_store/opensearch/data_store.pyi +160 -0
  65. gllm_datastore/data_store/opensearch/fulltext.pyi +240 -0
  66. gllm_datastore/data_store/opensearch/query.pyi +89 -0
  67. gllm_datastore/data_store/opensearch/query_translator.pyi +18 -0
  68. gllm_datastore/data_store/opensearch/vector.pyi +211 -0
  69. gllm_datastore/data_store/redis/__init__.pyi +5 -0
  70. gllm_datastore/data_store/redis/data_store.pyi +153 -0
  71. gllm_datastore/data_store/redis/fulltext.pyi +128 -0
  72. gllm_datastore/data_store/redis/query.pyi +428 -0
  73. gllm_datastore/data_store/redis/query_translator.pyi +37 -0
  74. gllm_datastore/data_store/redis/vector.pyi +131 -0
  75. gllm_datastore/data_store/sql/__init__.pyi +4 -0
  76. gllm_datastore/data_store/sql/constants.pyi +5 -0
  77. gllm_datastore/data_store/sql/data_store.pyi +201 -0
  78. gllm_datastore/data_store/sql/fulltext.pyi +164 -0
  79. gllm_datastore/data_store/sql/query.pyi +81 -0
  80. gllm_datastore/data_store/sql/query_translator.pyi +51 -0
  81. gllm_datastore/data_store/sql/schema.pyi +16 -0
  82. gllm_datastore/encryptor/__init__.pyi +4 -0
  83. gllm_datastore/encryptor/aes_gcm_encryptor.pyi +45 -0
  84. gllm_datastore/encryptor/capability/__init__.pyi +3 -0
  85. gllm_datastore/encryptor/capability/mixin.pyi +32 -0
  86. gllm_datastore/encryptor/encryptor.pyi +52 -0
  87. gllm_datastore/encryptor/key_ring/__init__.pyi +3 -0
  88. gllm_datastore/encryptor/key_ring/in_memory_key_ring.pyi +52 -0
  89. gllm_datastore/encryptor/key_ring/key_ring.pyi +45 -0
  90. gllm_datastore/encryptor/key_rotating_encryptor.pyi +60 -0
  91. gllm_datastore/graph_data_store/__init__.pyi +6 -0
  92. gllm_datastore/graph_data_store/graph_data_store.pyi +151 -0
  93. gllm_datastore/graph_data_store/graph_rag_data_store.pyi +29 -0
  94. gllm_datastore/graph_data_store/light_rag_data_store.pyi +93 -0
  95. gllm_datastore/graph_data_store/light_rag_postgres_data_store.pyi +96 -0
  96. gllm_datastore/graph_data_store/llama_index_graph_rag_data_store.pyi +49 -0
  97. gllm_datastore/graph_data_store/llama_index_neo4j_graph_rag_data_store.pyi +78 -0
  98. gllm_datastore/graph_data_store/mixins/__init__.pyi +3 -0
  99. gllm_datastore/graph_data_store/mixins/agentic_graph_tools_mixin.pyi +175 -0
  100. gllm_datastore/graph_data_store/nebula_graph_data_store.pyi +206 -0
  101. gllm_datastore/graph_data_store/neo4j_graph_data_store.pyi +182 -0
  102. gllm_datastore/graph_data_store/schema.pyi +27 -0
  103. gllm_datastore/graph_data_store/utils/__init__.pyi +6 -0
  104. gllm_datastore/graph_data_store/utils/constants.pyi +21 -0
  105. gllm_datastore/graph_data_store/utils/light_rag_em_invoker_adapter.pyi +56 -0
  106. gllm_datastore/graph_data_store/utils/light_rag_lm_invoker_adapter.pyi +43 -0
  107. gllm_datastore/graph_data_store/utils/llama_index_em_invoker_adapter.pyi +45 -0
  108. gllm_datastore/graph_data_store/utils/llama_index_lm_invoker_adapter.pyi +169 -0
  109. gllm_datastore/signature/__init__.pyi +0 -0
  110. gllm_datastore/signature/webhook_signature.pyi +31 -0
  111. gllm_datastore/sql_data_store/__init__.pyi +4 -0
  112. gllm_datastore/sql_data_store/adapter/__init__.pyi +0 -0
  113. gllm_datastore/sql_data_store/adapter/sqlalchemy_adapter.pyi +38 -0
  114. gllm_datastore/sql_data_store/constants.pyi +6 -0
  115. gllm_datastore/sql_data_store/sql_data_store.pyi +86 -0
  116. gllm_datastore/sql_data_store/sqlalchemy_sql_data_store.pyi +216 -0
  117. gllm_datastore/sql_data_store/types.pyi +31 -0
  118. gllm_datastore/utils/__init__.pyi +6 -0
  119. gllm_datastore/utils/converter.pyi +51 -0
  120. gllm_datastore/utils/dict.pyi +21 -0
  121. gllm_datastore/utils/ttl.pyi +25 -0
  122. gllm_datastore/utils/types.pyi +32 -0
  123. gllm_datastore/vector_data_store/__init__.pyi +6 -0
  124. gllm_datastore/vector_data_store/chroma_vector_data_store.pyi +259 -0
  125. gllm_datastore/vector_data_store/elasticsearch_vector_data_store.pyi +357 -0
  126. gllm_datastore/vector_data_store/in_memory_vector_data_store.pyi +179 -0
  127. gllm_datastore/vector_data_store/mixin/__init__.pyi +0 -0
  128. gllm_datastore/vector_data_store/mixin/cache_compatible_mixin.pyi +145 -0
  129. gllm_datastore/vector_data_store/redis_vector_data_store.pyi +191 -0
  130. gllm_datastore/vector_data_store/vector_data_store.pyi +146 -0
  131. gllm_datastore.build/.gitignore +1 -0
  132. gllm_datastore.cpython-312-darwin.so +0 -0
  133. gllm_datastore.pyi +178 -0
  134. gllm_datastore_binary-0.5.50.dist-info/METADATA +185 -0
  135. gllm_datastore_binary-0.5.50.dist-info/RECORD +137 -0
  136. gllm_datastore_binary-0.5.50.dist-info/WHEEL +5 -0
  137. gllm_datastore_binary-0.5.50.dist-info/top_level.txt +1 -0
@@ -0,0 +1,197 @@
1
+ from _typeshed import Incomplete
2
+ from chromadb import ClientAPI
3
+ from gllm_core.schema import Chunk
4
+ from gllm_datastore.constants import CHUNK_KEYS as CHUNK_KEYS, DEFAULT_TOP_K as DEFAULT_TOP_K, METADATA_KEYS as METADATA_KEYS
5
+ from gllm_datastore.core.filters import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
6
+ from gllm_datastore.data_store.chroma._chroma_import import safe_import_chromadb as safe_import_chromadb
7
+ from gllm_datastore.data_store.chroma.query import ChromaCollectionKeys as ChromaCollectionKeys, DEFAULT_NUM_CANDIDATES as DEFAULT_NUM_CANDIDATES, build_chroma_delete_kwargs as build_chroma_delete_kwargs, build_chroma_get_kwargs as build_chroma_get_kwargs
8
+ from gllm_datastore.data_store.chroma.query_translator import ChromaQueryTranslator as ChromaQueryTranslator
9
+ from gllm_datastore.utils.converter import from_langchain as from_langchain, l2_distance_to_similarity_score as l2_distance_to_similarity_score, to_langchain as to_langchain
10
+ from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
11
+ from gllm_inference.schema import Vector
12
+ from typing import Any
13
+
14
+ chromadb: Incomplete
15
+
16
+ class ChromaVectorCapability:
17
+ """ChromaDB implementation of VectorCapability protocol.
18
+
19
+ This class provides document CRUD operations and vector search using ChromaDB.
20
+
21
+ Attributes:
22
+ collection_name (str): The name of the ChromaDB collection.
23
+ collection (Collection): The ChromaDB collection instance.
24
+ vector_store (Chroma): The langchain Chroma vector store instance.
25
+ num_candidates (int): The maximum number of candidates to consider during search.
26
+ """
27
+ collection_name: Incomplete
28
+ client: Incomplete
29
+ collection: Incomplete
30
+ num_candidates: Incomplete
31
+ vector_store: Incomplete
32
+ def __init__(self, collection_name: str, em_invoker: BaseEMInvoker, client: ClientAPI, num_candidates: int = ...) -> None:
33
+ """Initialize the ChromaDB vector capability.
34
+
35
+ Args:
36
+ collection_name (str): The name of the ChromaDB collection.
37
+ em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
38
+ client (ClientAPI): The ChromaDB client instance.
39
+ num_candidates (int, optional): Maximum number of candidates to consider during search.
40
+ Defaults to 50.
41
+ """
42
+ @property
43
+ def em_invoker(self) -> BaseEMInvoker:
44
+ """Returns the EM Invoker instance.
45
+
46
+ Returns:
47
+ BaseEMInvoker: The EM Invoker instance.
48
+ """
49
+ async def ensure_index(self) -> None:
50
+ """Ensure ChromaDB collection exists, creating it if necessary.
51
+
52
+ This method is idempotent - if the collection already exists, it will return
53
+ the existing collection. The collection is automatically created during initialization,
54
+ but this method can be called explicitly to ensure it exists.
55
+
56
+ Raises:
57
+ RuntimeError: If collection creation fails.
58
+ """
59
+ async def create(self, data: Chunk | list[Chunk], **kwargs: Any) -> None:
60
+ """Add chunks to the vector store with automatic embedding generation.
61
+
62
+ Args:
63
+ data (Chunk | list[Chunk]): Single chunk or list of chunks to add.
64
+ **kwargs: Backend-specific parameters.
65
+ """
66
+ async def create_from_vector(self, chunk_vectors: list[tuple[Chunk, Vector]], **kwargs: Any) -> None:
67
+ '''Add pre-computed embeddings directly.
68
+
69
+ Examples:
70
+ ```python
71
+ await datastore.vector.create_from_vector(chunk_vectors=[
72
+ (Chunk(content="text1", metadata={"source": "source1"}, id="id1"), [0.1, 0.2, 0.3]),
73
+ (Chunk(content="text2", metadata={"source": "source2"}, id="id2"), [0.4, 0.5, 0.6]),
74
+ ])
75
+ ```
76
+
77
+ Args:
78
+ chunk_vectors (list[tuple[Chunk, Vector]]): List of tuples containing chunks and their
79
+ corresponding vectors.
80
+ **kwargs: Datastore-specific parameters.
81
+ '''
82
+ async def retrieve(self, query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
83
+ '''Semantic search using text query converted to vector.
84
+
85
+ Examples:
86
+ ```python
87
+ from gllm_datastore.core.filters import filter as F
88
+
89
+ # Direct FilterClause usage
90
+ await datastore.vector.retrieve(
91
+ query="What is the capital of France?",
92
+ filters=F.eq("metadata.category", "tech")
93
+ )
94
+
95
+ # Multiple filters
96
+ filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
97
+ await datastore.vector.retrieve(query="What is the capital of France?", filters=filters)
98
+ ```
99
+ This will retrieve the top 10 chunks by similarity score from the vector store
100
+ that match the query and the filters. The chunks will be sorted by score in descending order.
101
+
102
+ Args:
103
+ query (str): Text query to embed and search for.
104
+ filters (FilterClause | QueryFilter | None, optional): Filters to apply to the search.
105
+ FilterClause objects are automatically converted to QueryFilter internally.
106
+ Defaults to None.
107
+ options (QueryOptions | None, optional): Options to apply to the search. Defaults to None.
108
+
109
+ Returns:
110
+ list[Chunk]: List of chunks ordered by relevance score.
111
+ '''
112
+ async def retrieve_by_vector(self, vector: Vector, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
113
+ '''Direct vector similarity search.
114
+
115
+ Examples:
116
+ ```python
117
+ from gllm_datastore.core.filters import filter as F
118
+
119
+ # Direct FilterClause usage
120
+ await datastore.vector.retrieve_by_vector(
121
+ vector=[0.1, 0.2, 0.3],
122
+ filters=F.eq("metadata.category", "tech")
123
+ )
124
+
125
+ # Multiple filters
126
+ filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
127
+ await datastore.vector.retrieve_by_vector(vector=[0.1, 0.2, 0.3], filters=filters)
128
+ ```
129
+ This will retrieve the top 10 chunks by similarity score from the vector store
130
+ that match the vector and the filters. The chunks will be sorted by score in descending order.
131
+
132
+ Args:
133
+ vector (Vector): Query embedding vector.
134
+ filters (FilterClause | QueryFilter | None, optional): Filters to apply to the search.
135
+ FilterClause objects are automatically converted to QueryFilter internally.
136
+ Defaults to None.
137
+ options (QueryOptions | None, optional): Options to apply to the search. Defaults to None.
138
+
139
+ Returns:
140
+ list[Chunk]: List of chunks ordered by similarity score.
141
+ '''
142
+ async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None) -> None:
143
+ '''Update existing records in the datastore.
144
+
145
+ Examples:
146
+ ```python
147
+ from gllm_datastore.core.filters import filter as F
148
+
149
+ # Direct FilterClause usage
150
+ await datastore.vector.update(
151
+ update_values={"metadata": {"status": "published"}},
152
+ filters=F.eq("metadata.category", "tech"),
153
+ )
154
+
155
+ # Multiple filters
156
+ filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
157
+ await datastore.vector.update(
158
+ update_values={"metadata": {"status": "published"}},
159
+ filters=filters,
160
+ )
161
+ ```
162
+ This will update the metadata of the chunks that match the filters to "published".
163
+
164
+ Args:
165
+ update_values (dict[str, Any]): Values to update.
166
+ filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
167
+ FilterClause objects are automatically converted to QueryFilter internally.
168
+ Defaults to None, in which case no operation is performed (no-op).
169
+
170
+ Note:
171
+ ChromaDB doesn\'t support direct update operations. This method requires
172
+ filters to identify records and will update matching records.
173
+ '''
174
+ async def delete(self, filters: FilterClause | QueryFilter | None = None, **kwargs: Any) -> None:
175
+ '''Delete records from the datastore.
176
+
177
+ Examples:
178
+ ```python
179
+ from gllm_datastore.core.filters import filter as F
180
+
181
+ # Direct FilterClause usage
182
+ await datastore.vector.delete(filters=F.eq("metadata.category", "tech"))
183
+
184
+ # Multiple filters
185
+ filters = F.and_(F.eq("metadata.source", "wikipedia"), F.eq("metadata.category", "tech"))
186
+ await datastore.vector.delete(filters=filters)
187
+ ```
188
+ This will delete all chunks from the vector store that match the filters.
189
+
190
+ Args:
191
+ filters (FilterClause | QueryFilter | None, optional): Filters to select records to delete.
192
+ FilterClause objects are automatically converted to QueryFilter internally.
193
+ Defaults to None, in which case no operation is performed (no-op).
194
+ **kwargs: Datastore-specific parameters.
195
+ '''
196
+ async def clear(self) -> None:
197
+ """Clear all records from the datastore."""
@@ -0,0 +1,5 @@
1
+ from gllm_datastore.data_store.elasticsearch.data_store import ElasticsearchDataStore as ElasticsearchDataStore
2
+ from gllm_datastore.data_store.elasticsearch.fulltext import ElasticsearchFulltextCapability as ElasticsearchFulltextCapability
3
+ from gllm_datastore.data_store.elasticsearch.vector import ElasticsearchVectorCapability as ElasticsearchVectorCapability
4
+
5
+ __all__ = ['ElasticsearchDataStore', 'ElasticsearchFulltextCapability', 'ElasticsearchVectorCapability']
@@ -0,0 +1,147 @@
1
+ from _typeshed import Incomplete
2
+ from elasticsearch import AsyncElasticsearch
3
+ from gllm_datastore.constants import DEFAULT_REQUEST_TIMEOUT as DEFAULT_REQUEST_TIMEOUT
4
+ from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter
5
+ from gllm_datastore.data_store._elastic_core.client_factory import EngineType as EngineType, create_client as create_client
6
+ from gllm_datastore.data_store.base import BaseDataStore as BaseDataStore, CapabilityType as CapabilityType
7
+ from gllm_datastore.data_store.elasticsearch.fulltext import ElasticsearchFulltextCapability as ElasticsearchFulltextCapability
8
+ from gllm_datastore.data_store.elasticsearch.query_translator import ElasticsearchQueryTranslator as ElasticsearchQueryTranslator
9
+ from gllm_datastore.data_store.elasticsearch.vector import ElasticsearchVectorCapability as ElasticsearchVectorCapability
10
+ from gllm_inference.em_invoker.em_invoker import BaseEMInvoker
11
+ from langchain_elasticsearch.vectorstores import AsyncRetrievalStrategy
12
+ from typing import Any
13
+
14
+ class ElasticsearchDataStore(BaseDataStore):
15
+ '''Elasticsearch data store with multiple capability support.
16
+
17
+ This is the explicit public API for Elasticsearch. Users know they\'re
18
+ using Elasticsearch, not a generic "elastic-like" datastore.
19
+
20
+ Attributes:
21
+ engine (str): Always "elasticsearch" for explicit identification.
22
+ This attribute ensures users know they\'re using Elasticsearch, not a generic
23
+ "elastic-like" datastore.
24
+ index_name (str): The name of the Elasticsearch index.
25
+ client (AsyncElasticsearch): AsyncElasticsearch client.
26
+ '''
27
+ engine: str
28
+ client: Incomplete
29
+ index_name: Incomplete
30
+ def __init__(self, index_name: str, client: AsyncElasticsearch | None = None, url: str | None = None, cloud_id: str | None = None, api_key: str | None = None, username: str | None = None, password: str | None = None, request_timeout: int = ...) -> None:
31
+ '''Initialize the Elasticsearch data store.
32
+
33
+ Args:
34
+ index_name (str): The name of the Elasticsearch index to use for operations.
35
+ This index name will be used for all queries and operations.
36
+ client (AsyncElasticsearch | None, optional): Pre-configured Elasticsearch client instance.
37
+ If provided, it will be used instead of creating a new client from url/cloud_id.
38
+ Must be an instance of AsyncElasticsearch. Defaults to None.
39
+ url (str | None, optional): The URL of the Elasticsearch server.
40
+ For example, "http://localhost:9200". Either url or cloud_id must be provided
41
+ if client is None. Defaults to None.
42
+ cloud_id (str | None, optional): The cloud ID of the Elasticsearch cluster.
43
+ Used for Elastic Cloud connections. Either url or cloud_id must be provided
44
+ if client is None. Defaults to None.
45
+ api_key (str | None, optional): The API key for authentication.
46
+ If provided, will be used for authentication. Mutually exclusive with username/password.
47
+ Defaults to None.
48
+ username (str | None, optional): The username for basic authentication.
49
+ Must be provided together with password. Mutually exclusive with api_key.
50
+ Defaults to None.
51
+ password (str | None, optional): The password for basic authentication.
52
+ Must be provided together with username. Mutually exclusive with api_key.
53
+ Defaults to None.
54
+ request_timeout (int, optional): The request timeout in seconds.
55
+ Defaults to DEFAULT_REQUEST_TIMEOUT.
56
+
57
+ Raises:
58
+ ValueError: If neither url nor cloud_id is provided when client is None.
59
+ TypeError: If client is provided but is not an instance of AsyncElasticsearch.
60
+ '''
61
+ @property
62
+ def supported_capabilities(self) -> list[str]:
63
+ """Return list of currently supported capabilities.
64
+
65
+ Returns:
66
+ list[str]: List of capability names that are supported.
67
+ """
68
+ @property
69
+ def fulltext(self) -> ElasticsearchFulltextCapability:
70
+ """Access fulltext capability if supported.
71
+
72
+ This method uses the logic of its parent class to return the fulltext capability handler.
73
+ This method overrides the parent class to return the ElasticsearchFulltextCapability handler for better
74
+ type hinting.
75
+
76
+ Returns:
77
+ ElasticsearchFulltextCapability: Fulltext capability handler.
78
+
79
+ Raises:
80
+ NotSupportedException: If fulltext capability is not supported.
81
+ """
82
+ @property
83
+ def vector(self) -> ElasticsearchVectorCapability:
84
+ """Access vector capability if supported.
85
+
86
+ This method uses the logic of its parent class to return the vector capability handler.
87
+ This method overrides the parent class to return the ElasticsearchVectorCapability handler for better
88
+ type hinting.
89
+
90
+ Returns:
91
+ ElasticsearchVectorCapability: Vector capability handler.
92
+
93
+ Raises:
94
+ NotSupportedException: If vector capability is not supported.
95
+ """
96
+ def with_fulltext(self, index_name: str | None = None, query_field: str = 'text') -> ElasticsearchDataStore:
97
+ '''Configure fulltext capability and return datastore instance.
98
+
99
+ This method uses the logic of its parent class to configure the fulltext capability.
100
+ This method overrides the parent class for better type hinting.
101
+
102
+ Args:
103
+ index_name (str | None, optional): The name of the Elasticsearch index to use for fulltext operations.
104
+ If None, uses the default index_name from the datastore instance.
105
+ Defaults to None.
106
+ query_field (str, optional): The field name to use for text content in queries.
107
+ This field will be used for BM25 and other text search operations.
108
+ Defaults to "text".
109
+
110
+ Returns:
111
+ ElasticsearchDataStore: Self for method chaining.
112
+ '''
113
+ def with_vector(self, em_invoker: BaseEMInvoker, index_name: str | None = None, query_field: str = 'text', vector_query_field: str = 'vector', retrieval_strategy: AsyncRetrievalStrategy | None = None, distance_strategy: str | None = None) -> ElasticsearchDataStore:
114
+ '''Configure vector capability and return datastore instance.
115
+
116
+ This method uses the logic of its parent class to configure the vector capability.
117
+ This method overrides the parent class for better type hinting.
118
+
119
+ Args:
120
+ em_invoker (BaseEMInvoker): The embedding model to perform vectorization.
121
+ index_name (str | None, optional): The name of the Elasticsearch index. Defaults to None,
122
+ in which case the default class attribute will be utilized.
123
+ query_field (str, optional): The field name for text queries. Defaults to "text".
124
+ vector_query_field (str, optional): The field name for vector queries. Defaults to "vector".
125
+ retrieval_strategy (AsyncRetrievalStrategy | None, optional): The retrieval strategy for retrieval.
126
+ Defaults to None, in which case DenseVectorStrategy() is used.
127
+ distance_strategy (str | None, optional): The distance strategy for retrieval. Defaults to None.
128
+
129
+ Returns:
130
+ Self: Self for method chaining.
131
+ '''
132
+ @classmethod
133
+ def translate_query_filter(cls, query_filter: FilterClause | QueryFilter | None) -> dict[str, Any] | None:
134
+ """Translate QueryFilter or FilterClause to Elasticsearch native filter syntax.
135
+
136
+ This method delegates to the ElasticsearchQueryTranslator and returns the result as a dictionary.
137
+
138
+ Args:
139
+ query_filter (FilterClause | QueryFilter | None): The filter to translate.
140
+ Can be a single FilterClause, a QueryFilter with multiple clauses and logical conditions,
141
+ or None for empty filters. FilterClause objects are automatically converted to QueryFilter.
142
+
143
+ Returns:
144
+ dict[str, Any] | None: The translated filter as an Elasticsearch DSL dictionary.
145
+ Returns None for empty filters or when query_filter is None.
146
+ The dictionary format matches Elasticsearch Query DSL syntax.
147
+ """
@@ -0,0 +1,238 @@
1
+ from _typeshed import Incomplete
2
+ from elasticsearch import AsyncElasticsearch
3
+ from enum import StrEnum
4
+ from gllm_core.schema import Chunk
5
+ from gllm_datastore.constants import METADATA_KEYS as METADATA_KEYS
6
+ from gllm_datastore.core.filters.schema import FilterClause as FilterClause, QueryFilter as QueryFilter, QueryOptions as QueryOptions
7
+ from gllm_datastore.data_store._elastic_core.elastic_like_core import ElasticLikeCore as ElasticLikeCore
8
+ from gllm_datastore.data_store._elastic_core.query_translator import convert_filter_clause as convert_filter_clause
9
+ from gllm_datastore.data_store.elasticsearch.query import apply_filter_query_to_search as apply_filter_query_to_search, apply_filters_and_options as apply_filters_and_options, create_search_with_filters as create_search_with_filters, delete_by_id as delete_by_id, delete_by_query as delete_by_query, safe_execute as safe_execute, translate_filter as translate_filter, update_by_query as update_by_query, validate_query_length as validate_query_length
10
+ from typing import Any, Literal, overload
11
+
12
+ class SupportedQueryMethods(StrEnum):
13
+ """Supported query methods for Elasticsearch fulltext capability."""
14
+ AUTOCOMPLETE: str
15
+ AUTOSUGGEST: str
16
+ BM25: str
17
+ BY_FIELD: str
18
+ SHINGLES: str
19
+
20
+ QUERY_REQUIRED_STRATEGIES: Incomplete
21
+
22
+ class ElasticsearchFulltextCapability:
23
+ """Elasticsearch implementation of FulltextCapability protocol.
24
+
25
+ This class provides document CRUD operations and flexible querying using Elasticsearch.
26
+
27
+ Attributes:
28
+ index_name (str): The name of the Elasticsearch index.
29
+ client (AsyncElasticsearch): AsyncElasticsearch client.
30
+ query_field (str): The field name to use for text content.
31
+
32
+ """
33
+ index_name: Incomplete
34
+ client: Incomplete
35
+ query_field: Incomplete
36
+ def __init__(self, index_name: str, client: AsyncElasticsearch, query_field: str = 'text') -> None:
37
+ '''Initialize the Elasticsearch fulltext capability.
38
+
39
+ Args:
40
+ index_name (str): The name of the Elasticsearch index.
41
+ client (AsyncElasticsearch): The Elasticsearch client.
42
+ query_field (str, optional): The field name to use for text content. Defaults to "text".
43
+ '''
44
+ async def get_size(self) -> int:
45
+ """Returns the total number of documents in the index.
46
+
47
+ Returns:
48
+ int: The total number of documents.
49
+ """
50
+ async def create(self, data: Chunk | list[Chunk], **kwargs: Any) -> None:
51
+ """Create new records in the datastore.
52
+
53
+ Args:
54
+ data (Chunk | list[Chunk]): Data to create (single item or collection).
55
+ **kwargs: Backend-specific parameters forwarded to Elasticsearch bulk API.
56
+
57
+ Raises:
58
+ ValueError: If data structure is invalid.
59
+ """
60
+ @overload
61
+ async def retrieve(self, strategy: Literal[SupportedQueryMethods.BY_FIELD] | None = ..., query: str | None = None, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, **kwargs: Any) -> list[Chunk]: ...
62
+ @overload
63
+ async def retrieve(self, strategy: Literal[SupportedQueryMethods.BM25], query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, k1: float | None = None, b: float | None = None, **kwargs: Any) -> list[Chunk]: ...
64
+ @overload
65
+ async def retrieve(self, strategy: Literal[SupportedQueryMethods.AUTOCOMPLETE], query: str, field: str, size: int = 20, fuzzy_tolerance: int = 1, min_prefix_length: int = 3, filter_query: dict[str, Any] | None = None, **kwargs: Any) -> list[str]: ...
66
+ @overload
67
+ async def retrieve(self, strategy: Literal[SupportedQueryMethods.AUTOSUGGEST], query: str, search_fields: list[str], autocomplete_field: str, size: int = 20, min_length: int = 3, filter_query: dict[str, Any] | None = None, **kwargs: Any) -> list[str]: ...
68
+ @overload
69
+ async def retrieve(self, strategy: Literal[SupportedQueryMethods.SHINGLES], query: str, field: str, size: int = 20, min_length: int = 3, max_length: int = 30, filter_query: dict[str, Any] | None = None, **kwargs: Any) -> list[str]: ...
70
+ async def retrieve_by_field(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
71
+ """Retrieve records from the datastore based on metadata field filtering.
72
+
73
+ This method filters and returns stored chunks based on metadata values
74
+ rather than text content. It is particularly useful for structured lookups,
75
+ such as retrieving all chunks from a certain source, tagged with a specific label,
76
+ or authored by a particular user.
77
+
78
+ Args:
79
+ filters (FilterClause | QueryFilter | None, optional): Query filters to apply.
80
+ FilterClause objects are automatically converted to QueryFilter internally.
81
+ Defaults to None.
82
+ options (QueryOptions | None, optional): Query options (sorting, pagination, etc.).
83
+ Defaults to None.
84
+
85
+ Returns:
86
+ list[Chunk]: The filtered results as Chunk objects.
87
+ """
88
+ async def retrieve_bm25(self, query: str, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None, k1: float | None = None, b: float | None = None) -> list[Chunk]:
89
+ '''Queries the Elasticsearch data store using BM25 algorithm for keyword-based search.
90
+
91
+ Args:
92
+ query (str): The query string.
93
+ filters (FilterClause | QueryFilter | None, optional): Optional metadata filter to apply to the search.
94
+ FilterClause objects are automatically converted to QueryFilter internally.
95
+ Use filter builder functions like `F.eq()`, `F.and_()`, etc. Defaults to None.
96
+ options (QueryOptions | None, optional): Query options including fields, limit, order_by, etc.
97
+ For example, `QueryOptions(fields=["title", "content"], limit=10, order_by="score", order_desc=True)`.
98
+ If fields is None, defaults to ["text"]. For multiple fields, uses multi_match query. Defaults to None.
99
+ k1 (float | None, optional): BM25 parameter controlling term frequency saturation.
100
+ Higher values mean term frequency has more impact before diminishing returns.
101
+ Typical values: 1.2-2.0. If None, uses Elasticsearch default (~1.2). Defaults to None.
102
+ b (float | None, optional): BM25 parameter controlling document length normalization.
103
+ 0.0 = no length normalization, 1.0 = full normalization.
104
+ Typical values: 0.75. If None, uses Elasticsearch default (~0.75). Defaults to None.
105
+
106
+ Examples:
107
+ ```python
108
+ from gllm_datastore.core.filters import filter as F
109
+
110
+ # Basic BM25 query on the \'text\' field
111
+ results = await data_store.query_bm25("machine learning")
112
+
113
+ # BM25 query on specific fields with query options
114
+ results = await data_store.query_bm25(
115
+ "natural language",
116
+ options=QueryOptions(fields=["title", "abstract"], limit=5)
117
+ )
118
+
119
+ # BM25 query with direct FilterClause
120
+ results = await data_store.query_bm25(
121
+ "deep learning",
122
+ filters=F.eq("metadata.category", "AI")
123
+ )
124
+
125
+ # BM25 query with multiple filters
126
+ results = await data_store.query_bm25(
127
+ "deep learning",
128
+ filters=F.and_(F.eq("metadata.category", "AI"), F.eq("metadata.status", "published"))
129
+ )
130
+
131
+ # BM25 query with custom BM25 parameters for more aggressive term frequency weighting
132
+ results = await data_store.query_bm25(
133
+ "artificial intelligence",
134
+ k1=2.0,
135
+ b=0.5
136
+ )
137
+
138
+ # BM25 query with fields, filters, and options
139
+ results = await data_store.query_bm25(
140
+ "data science applications",
141
+ filters=F.and_(
142
+ F.eq("metadata.author_id", "user123"),
143
+ F.in_("metadata.publication_year", [2022, 2023])
144
+ ),
145
+ options=QueryOptions(fields=["content", "tags"], limit=10, order_by="score", order_desc=True),
146
+ k1=1.5,
147
+ b=0.9
148
+ )
149
+ ```
150
+
151
+ Returns:
152
+ list[Chunk]: A list of Chunk objects representing the retrieved documents.
153
+ '''
154
+ async def retrieve_autocomplete(self, query: str, field: str, size: int = 20, fuzzy_tolerance: int = 1, min_prefix_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
155
+ """Provides suggestions based on a prefix query for a specific field.
156
+
157
+ Args:
158
+ query (str): The query string.
159
+ field (str): The field name for autocomplete.
160
+ size (int, optional): The number of suggestions to retrieve. Defaults to 20.
161
+ fuzzy_tolerance (int, optional): The level of fuzziness for suggestions. Defaults to 1.
162
+ min_prefix_length (int, optional): The minimum prefix length to trigger fuzzy matching. Defaults to 3.
163
+ filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
164
+
165
+ Returns:
166
+ list[str]: A list of suggestions.
167
+ """
168
+ async def retrieve_autosuggest(self, query: str, search_fields: list[str], autocomplete_field: str, size: int = 20, min_length: int = 3, filters: QueryFilter | None = None) -> list[str]:
169
+ """Generates suggestions across multiple fields using a multi_match query to broaden the search criteria.
170
+
171
+ Args:
172
+ query (str): The query string.
173
+ search_fields (list[str]): The fields to search for.
174
+ autocomplete_field (str): The field name for autocomplete.
175
+ size (int, optional): The number of suggestions to retrieve. Defaults to 20.
176
+ min_length (int, optional): The minimum length of the query. Defaults to 3.
177
+ filters (QueryFilter | None, optional): The filter query. Defaults to None.
178
+
179
+ Returns:
180
+ list[str]: A list of suggestions.
181
+ """
182
+ async def retrieve_shingles(self, query: str, field: str, size: int = 20, min_length: int = 3, max_length: int = 30, filters: QueryFilter | None = None) -> list[str]:
183
+ """Searches using shingles for prefix and fuzzy matching.
184
+
185
+ Args:
186
+ query (str): The query string.
187
+ field (str): The field name for autocomplete.
188
+ size (int, optional): The number of suggestions to retrieve. Defaults to 20.
189
+ min_length (int, optional): The minimum length of the query.
190
+ Queries shorter than this limit will return an empty list. Defaults to 3.
191
+ max_length (int, optional): The maximum length of the query.
192
+ Queries exceeding this limit will return an empty list. Defaults to 30.
193
+ filters (QueryFilter | None, optional): The filter query. Defaults to None.
194
+
195
+ Returns:
196
+ list[str]: A list of suggestions.
197
+ """
198
+ async def retrieve_fuzzy(self, query: str, max_distance: int = 2, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> list[Chunk]:
199
+ """Find records that fuzzy match the query within distance threshold.
200
+
201
+ Args:
202
+ query (str): Text to fuzzy match against.
203
+ max_distance (int): Maximum edit distance for matches. Defaults to 2.
204
+ filters (FilterClause | QueryFilter | None, optional): Optional metadata filters to apply.
205
+ FilterClause objects are automatically converted to QueryFilter internally.
206
+ Defaults to None.
207
+ options (QueryOptions | None, optional): Query options (limit, sorting, etc.). Defaults to None.
208
+
209
+ Returns:
210
+ list[Chunk]: Matched chunks.
211
+ """
212
+ async def update(self, update_values: dict[str, Any], filters: FilterClause | QueryFilter | None = None) -> None:
213
+ """Update existing records in the datastore.
214
+
215
+ Args:
216
+ update_values (dict[str, Any]): Values to update.
217
+ filters (FilterClause | QueryFilter | None, optional): Filters to select records to update.
218
+ FilterClause objects are automatically converted to QueryFilter internally.
219
+ Defaults to None.
220
+ """
221
+ async def delete(self, filters: FilterClause | QueryFilter | None = None, options: QueryOptions | None = None) -> None:
222
+ """Delete records from the data store using filters and optional options.
223
+
224
+ Args:
225
+ filters (FilterClause | QueryFilter | None, optional): Filters to select records for deletion.
226
+ FilterClause objects are automatically converted to QueryFilter internally.
227
+ Defaults to None.
228
+ options (QueryOptions | None, optional): Query options supporting limit and sorting
229
+ for eviction-like operations. Defaults to None.
230
+ """
231
+ async def delete_by_id(self, id_: str | list[str]) -> None:
232
+ """Deletes records from the data store based on IDs.
233
+
234
+ Args:
235
+ id_ (str | list[str]): ID or list of IDs to delete.
236
+ """
237
+ async def clear(self) -> None:
238
+ """Clear all records from the datastore."""