dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,372 @@
1
+ """
2
+ DXMT Haystack Retrievers
3
+
4
+ Retriever components for using DXMT in Haystack pipelines.
5
+ Provides both BM25 (full-text) and embedding (vector) retrievers.
6
+ """
7
+
8
+ from typing import Any, Dict, List, Optional
9
+
10
+ try:
11
+ from haystack import Document, component, default_from_dict, default_to_dict
12
+ from haystack.document_stores.types import FilterPolicy
13
+ HAYSTACK_AVAILABLE = True
14
+ except ImportError:
15
+ HAYSTACK_AVAILABLE = False
16
+ # Minimal stubs
17
+ Document = Any
18
+
19
+ class FilterPolicy:
20
+ REPLACE = "replace"
21
+ MERGE = "merge"
22
+
23
+ @classmethod
24
+ def from_str(cls, value):
25
+ return value
26
+
27
+ def component(cls):
28
+ return cls
29
+
30
+ def default_to_dict(*args, **kwargs):
31
+ return {}
32
+
33
+ def default_from_dict(cls, data):
34
+ return cls(**data.get("init_parameters", {}))
35
+
36
+ from dxmt.haystack import DXMTDocumentStore
37
+
38
+
39
+ @component
40
+ class DXMTEmbeddingRetriever:
41
+ """
42
+ Retriever for semantic search using DXMT vector similarity.
43
+
44
+ This retriever performs approximate nearest neighbor search on document embeddings
45
+ stored in DXMT. It leverages DXMT's tiered storage and adaptive index selection
46
+ for cost-efficient, high-performance vector search.
47
+
48
+ Usage example:
49
+ ```python
50
+ from haystack import Pipeline
51
+ from haystack.components.embedders import SentenceTransformersTextEmbedder
52
+ from dxmt import Client as DXMTClient
53
+ from dxmt.haystack import DXMTDocumentStore
54
+ from dxmt.haystack_retrievers import DXMTEmbeddingRetriever
55
+
56
+ # Initialize
57
+ client = DXMTClient(host="localhost", port=9999)
58
+ doc_store = DXMTDocumentStore(client=client, embedding_dim=384)
59
+ retriever = DXMTEmbeddingRetriever(document_store=doc_store, top_k=10)
60
+
61
+ # Use in pipeline
62
+ pipeline = Pipeline()
63
+ pipeline.add_component("embedder", SentenceTransformersTextEmbedder())
64
+ pipeline.add_component("retriever", retriever)
65
+ pipeline.connect("embedder.embedding", "retriever.query_embedding")
66
+
67
+ result = pipeline.run({"embedder": {"text": "What is vector search?"}})
68
+ ```
69
+ """
70
+
71
+ def __init__(
72
+ self,
73
+ document_store: DXMTDocumentStore,
74
+ filters: Optional[Dict[str, Any]] = None,
75
+ top_k: int = 10,
76
+ scale_score: bool = False,
77
+ return_embedding: bool = False,
78
+ filter_policy: "FilterPolicy" = None,
79
+ ):
80
+ """
81
+ Initialize the DXMT Embedding Retriever.
82
+
83
+ Args:
84
+ document_store: DXMTDocumentStore instance
85
+ filters: Default filters to apply
86
+ top_k: Number of documents to retrieve (default: 10)
87
+ scale_score: Scale scores to [0, 1] range (default: False)
88
+ return_embedding: Return embeddings with documents (default: False)
89
+ filter_policy: How to merge runtime and init filters (default: REPLACE)
90
+
91
+ Raises:
92
+ ValueError: If document_store is not a DXMTDocumentStore instance
93
+ ImportError: If haystack-ai is not installed (only when used in real pipelines)
94
+
95
+ Note:
96
+ While haystack-ai is recommended, this works without it for testing purposes.
97
+ """
98
+ # Note: We don't enforce HAYSTACK_AVAILABLE to allow mock testing
99
+
100
+ if not isinstance(document_store, DXMTDocumentStore):
101
+ raise ValueError("document_store must be an instance of DXMTDocumentStore")
102
+
103
+ if top_k <= 0:
104
+ raise ValueError(f"top_k must be greater than 0, got {top_k}")
105
+
106
+ self.document_store = document_store
107
+ self.filters = filters
108
+ self.top_k = top_k
109
+ self.scale_score = scale_score
110
+ self.return_embedding = return_embedding
111
+ self.filter_policy = filter_policy or FilterPolicy.REPLACE
112
+
113
+ def to_dict(self) -> Dict[str, Any]:
114
+ """Serialize the retriever to a dictionary."""
115
+ return default_to_dict(
116
+ self,
117
+ document_store=self.document_store,
118
+ filters=self.filters,
119
+ top_k=self.top_k,
120
+ scale_score=self.scale_score,
121
+ return_embedding=self.return_embedding,
122
+ filter_policy=self.filter_policy.value if hasattr(self.filter_policy, "value") else str(self.filter_policy),
123
+ )
124
+
125
+ @classmethod
126
+ def from_dict(cls, data: Dict[str, Any]) -> "DXMTEmbeddingRetriever":
127
+ """Deserialize the retriever from a dictionary."""
128
+ init_params = data.get("init_parameters", {})
129
+ if "filter_policy" in init_params and HAYSTACK_AVAILABLE:
130
+ init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"])
131
+ return default_from_dict(cls, data)
132
+
133
+ def _get_telemetry_data(self) -> Dict[str, Any]:
134
+ """Telemetry data for usage analytics."""
135
+ return {"document_store": type(self.document_store).__name__}
136
+
137
+ @component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
138
+ def run(
139
+ self,
140
+ query_embedding: List[float],
141
+ filters: Optional[Dict[str, Any]] = None,
142
+ top_k: Optional[int] = None,
143
+ scale_score: Optional[bool] = None,
144
+ return_embedding: Optional[bool] = None,
145
+ ) -> Dict[str, Any]:
146
+ """
147
+ Run embedding retrieval.
148
+
149
+ Args:
150
+ query_embedding: Query embedding vector
151
+ filters: Runtime filters to apply
152
+ top_k: Override default top_k
153
+ scale_score: Override default scale_score
154
+ return_embedding: Override default return_embedding
155
+
156
+ Returns:
157
+ Dictionary with "documents" key containing retrieved documents
158
+ """
159
+ # Merge filters based on policy
160
+ if self.filter_policy == FilterPolicy.MERGE and filters:
161
+ filters = {**(self.filters or {}), **filters}
162
+ else:
163
+ filters = filters or self.filters
164
+
165
+ # Use provided values or defaults
166
+ top_k = top_k if top_k is not None else self.top_k
167
+ scale_score = scale_score if scale_score is not None else self.scale_score
168
+ return_embedding = return_embedding if return_embedding is not None else self.return_embedding
169
+
170
+ # Perform retrieval
171
+ docs = self.document_store.embedding_retrieval(
172
+ query_embedding=query_embedding,
173
+ filters=filters,
174
+ top_k=top_k,
175
+ scale_score=scale_score,
176
+ return_embedding=return_embedding,
177
+ )
178
+
179
+ return {"documents": docs}
180
+
181
+ @component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
182
+ async def run_async(
183
+ self,
184
+ query_embedding: List[float],
185
+ filters: Optional[Dict[str, Any]] = None,
186
+ top_k: Optional[int] = None,
187
+ scale_score: Optional[bool] = None,
188
+ return_embedding: Optional[bool] = None,
189
+ ) -> Dict[str, Any]:
190
+ """Async version of run."""
191
+ # Merge filters
192
+ if self.filter_policy == FilterPolicy.MERGE and filters:
193
+ filters = {**(self.filters or {}), **filters}
194
+ else:
195
+ filters = filters or self.filters
196
+
197
+ # Use provided values or defaults
198
+ top_k = top_k if top_k is not None else self.top_k
199
+ scale_score = scale_score if scale_score is not None else self.scale_score
200
+ return_embedding = return_embedding if return_embedding is not None else self.return_embedding
201
+
202
+ # Perform async retrieval
203
+ docs = await self.document_store.embedding_retrieval_async(
204
+ query_embedding=query_embedding,
205
+ filters=filters,
206
+ top_k=top_k,
207
+ scale_score=scale_score,
208
+ return_embedding=return_embedding,
209
+ )
210
+
211
+ return {"documents": docs}
212
+
213
+
214
+ @component
215
+ class DXMTBM25Retriever:
216
+ """
217
+ Retriever for full-text search using BM25 ranking in DXMT.
218
+
219
+ This retriever performs BM25-ranked full-text search on document content
220
+ stored in DXMT. It uses DXMT's built-in full-text search capabilities
221
+ powered by Tantivy.
222
+
223
+ Usage example:
224
+ ```python
225
+ from haystack import Pipeline
226
+ from dxmt import Client as DXMTClient
227
+ from dxmt.haystack import DXMTDocumentStore
228
+ from dxmt.haystack_retrievers import DXMTBM25Retriever
229
+
230
+ # Initialize
231
+ client = DXMTClient(host="localhost", port=9999)
232
+ doc_store = DXMTDocumentStore(client=client)
233
+ retriever = DXMTBM25Retriever(document_store=doc_store, top_k=10)
234
+
235
+ # Use in pipeline
236
+ pipeline = Pipeline()
237
+ pipeline.add_component("retriever", retriever)
238
+
239
+ result = pipeline.run({"retriever": {"query": "machine learning"}})
240
+ ```
241
+ """
242
+
243
+ def __init__(
244
+ self,
245
+ document_store: DXMTDocumentStore,
246
+ filters: Optional[Dict[str, Any]] = None,
247
+ top_k: int = 10,
248
+ scale_score: bool = False,
249
+ filter_policy: "FilterPolicy" = None,
250
+ ):
251
+ """
252
+ Initialize the DXMT BM25 Retriever.
253
+
254
+ Args:
255
+ document_store: DXMTDocumentStore instance
256
+ filters: Default filters to apply
257
+ top_k: Number of documents to retrieve (default: 10)
258
+ scale_score: Scale scores to [0, 1] range (default: False)
259
+ filter_policy: How to merge runtime and init filters (default: REPLACE)
260
+
261
+ Raises:
262
+ ValueError: If document_store is not a DXMTDocumentStore instance
263
+ ImportError: If haystack-ai is not installed (only when used in real pipelines)
264
+
265
+ Note:
266
+ While haystack-ai is recommended, this works without it for testing purposes.
267
+ """
268
+ # Note: We don't enforce HAYSTACK_AVAILABLE to allow mock testing
269
+
270
+ if not isinstance(document_store, DXMTDocumentStore):
271
+ raise ValueError("document_store must be an instance of DXMTDocumentStore")
272
+
273
+ if top_k <= 0:
274
+ raise ValueError(f"top_k must be greater than 0, got {top_k}")
275
+
276
+ self.document_store = document_store
277
+ self.filters = filters
278
+ self.top_k = top_k
279
+ self.scale_score = scale_score
280
+ self.filter_policy = filter_policy or FilterPolicy.REPLACE
281
+
282
+ def to_dict(self) -> Dict[str, Any]:
283
+ """Serialize the retriever to a dictionary."""
284
+ return default_to_dict(
285
+ self,
286
+ document_store=self.document_store,
287
+ filters=self.filters,
288
+ top_k=self.top_k,
289
+ scale_score=self.scale_score,
290
+ filter_policy=self.filter_policy.value if hasattr(self.filter_policy, "value") else str(self.filter_policy),
291
+ )
292
+
293
+ @classmethod
294
+ def from_dict(cls, data: Dict[str, Any]) -> "DXMTBM25Retriever":
295
+ """Deserialize the retriever from a dictionary."""
296
+ init_params = data.get("init_parameters", {})
297
+ if "filter_policy" in init_params and HAYSTACK_AVAILABLE:
298
+ init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"])
299
+ return default_from_dict(cls, data)
300
+
301
+ def _get_telemetry_data(self) -> Dict[str, Any]:
302
+ """Telemetry data for usage analytics."""
303
+ return {"document_store": type(self.document_store).__name__}
304
+
305
+ @component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
306
+ def run(
307
+ self,
308
+ query: str,
309
+ filters: Optional[Dict[str, Any]] = None,
310
+ top_k: Optional[int] = None,
311
+ scale_score: Optional[bool] = None,
312
+ ) -> Dict[str, Any]:
313
+ """
314
+ Run BM25 retrieval.
315
+
316
+ Args:
317
+ query: Search query string
318
+ filters: Runtime filters to apply
319
+ top_k: Override default top_k
320
+ scale_score: Override default scale_score
321
+
322
+ Returns:
323
+ Dictionary with "documents" key containing retrieved documents
324
+ """
325
+ # Merge filters based on policy
326
+ if self.filter_policy == FilterPolicy.MERGE and filters:
327
+ filters = {**(self.filters or {}), **filters}
328
+ else:
329
+ filters = filters or self.filters
330
+
331
+ # Use provided values or defaults
332
+ top_k = top_k if top_k is not None else self.top_k
333
+ scale_score = scale_score if scale_score is not None else self.scale_score
334
+
335
+ # Perform retrieval
336
+ docs = self.document_store.bm25_retrieval(
337
+ query=query,
338
+ filters=filters,
339
+ top_k=top_k,
340
+ scale_score=scale_score,
341
+ )
342
+
343
+ return {"documents": docs}
344
+
345
+ @component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
346
+ async def run_async(
347
+ self,
348
+ query: str,
349
+ filters: Optional[Dict[str, Any]] = None,
350
+ top_k: Optional[int] = None,
351
+ scale_score: Optional[bool] = None,
352
+ ) -> Dict[str, Any]:
353
+ """Async version of run."""
354
+ # Merge filters
355
+ if self.filter_policy == FilterPolicy.MERGE and filters:
356
+ filters = {**(self.filters or {}), **filters}
357
+ else:
358
+ filters = filters or self.filters
359
+
360
+ # Use provided values or defaults
361
+ top_k = top_k if top_k is not None else self.top_k
362
+ scale_score = scale_score if scale_score is not None else self.scale_score
363
+
364
+ # Perform async retrieval
365
+ docs = await self.document_store.bm25_retrieval_async(
366
+ query=query,
367
+ filters=filters,
368
+ top_k=top_k,
369
+ scale_score=scale_score,
370
+ )
371
+
372
+ return {"documents": docs}