dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DXMT Haystack Retrievers
|
|
3
|
+
|
|
4
|
+
Retriever components for using DXMT in Haystack pipelines.
|
|
5
|
+
Provides both BM25 (full-text) and embedding (vector) retrievers.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Any, Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from haystack import Document, component, default_from_dict, default_to_dict
|
|
12
|
+
from haystack.document_stores.types import FilterPolicy
|
|
13
|
+
HAYSTACK_AVAILABLE = True
|
|
14
|
+
except ImportError:
|
|
15
|
+
HAYSTACK_AVAILABLE = False
|
|
16
|
+
# Minimal stubs
|
|
17
|
+
Document = Any
|
|
18
|
+
|
|
19
|
+
class FilterPolicy:
|
|
20
|
+
REPLACE = "replace"
|
|
21
|
+
MERGE = "merge"
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def from_str(cls, value):
|
|
25
|
+
return value
|
|
26
|
+
|
|
27
|
+
def component(cls):
|
|
28
|
+
return cls
|
|
29
|
+
|
|
30
|
+
def default_to_dict(*args, **kwargs):
|
|
31
|
+
return {}
|
|
32
|
+
|
|
33
|
+
def default_from_dict(cls, data):
|
|
34
|
+
return cls(**data.get("init_parameters", {}))
|
|
35
|
+
|
|
36
|
+
from dxmt.haystack import DXMTDocumentStore
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@component
|
|
40
|
+
class DXMTEmbeddingRetriever:
|
|
41
|
+
"""
|
|
42
|
+
Retriever for semantic search using DXMT vector similarity.
|
|
43
|
+
|
|
44
|
+
This retriever performs approximate nearest neighbor search on document embeddings
|
|
45
|
+
stored in DXMT. It leverages DXMT's tiered storage and adaptive index selection
|
|
46
|
+
for cost-efficient, high-performance vector search.
|
|
47
|
+
|
|
48
|
+
Usage example:
|
|
49
|
+
```python
|
|
50
|
+
from haystack import Pipeline
|
|
51
|
+
from haystack.components.embedders import SentenceTransformersTextEmbedder
|
|
52
|
+
from dxmt import Client as DXMTClient
|
|
53
|
+
from dxmt.haystack import DXMTDocumentStore
|
|
54
|
+
from dxmt.haystack_retrievers import DXMTEmbeddingRetriever
|
|
55
|
+
|
|
56
|
+
# Initialize
|
|
57
|
+
client = DXMTClient(host="localhost", port=9999)
|
|
58
|
+
doc_store = DXMTDocumentStore(client=client, embedding_dim=384)
|
|
59
|
+
retriever = DXMTEmbeddingRetriever(document_store=doc_store, top_k=10)
|
|
60
|
+
|
|
61
|
+
# Use in pipeline
|
|
62
|
+
pipeline = Pipeline()
|
|
63
|
+
pipeline.add_component("embedder", SentenceTransformersTextEmbedder())
|
|
64
|
+
pipeline.add_component("retriever", retriever)
|
|
65
|
+
pipeline.connect("embedder.embedding", "retriever.query_embedding")
|
|
66
|
+
|
|
67
|
+
result = pipeline.run({"embedder": {"text": "What is vector search?"}})
|
|
68
|
+
```
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(
|
|
72
|
+
self,
|
|
73
|
+
document_store: DXMTDocumentStore,
|
|
74
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
75
|
+
top_k: int = 10,
|
|
76
|
+
scale_score: bool = False,
|
|
77
|
+
return_embedding: bool = False,
|
|
78
|
+
filter_policy: "FilterPolicy" = None,
|
|
79
|
+
):
|
|
80
|
+
"""
|
|
81
|
+
Initialize the DXMT Embedding Retriever.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
document_store: DXMTDocumentStore instance
|
|
85
|
+
filters: Default filters to apply
|
|
86
|
+
top_k: Number of documents to retrieve (default: 10)
|
|
87
|
+
scale_score: Scale scores to [0, 1] range (default: False)
|
|
88
|
+
return_embedding: Return embeddings with documents (default: False)
|
|
89
|
+
filter_policy: How to merge runtime and init filters (default: REPLACE)
|
|
90
|
+
|
|
91
|
+
Raises:
|
|
92
|
+
ValueError: If document_store is not a DXMTDocumentStore instance
|
|
93
|
+
ImportError: If haystack-ai is not installed (only when used in real pipelines)
|
|
94
|
+
|
|
95
|
+
Note:
|
|
96
|
+
While haystack-ai is recommended, this works without it for testing purposes.
|
|
97
|
+
"""
|
|
98
|
+
# Note: We don't enforce HAYSTACK_AVAILABLE to allow mock testing
|
|
99
|
+
|
|
100
|
+
if not isinstance(document_store, DXMTDocumentStore):
|
|
101
|
+
raise ValueError("document_store must be an instance of DXMTDocumentStore")
|
|
102
|
+
|
|
103
|
+
if top_k <= 0:
|
|
104
|
+
raise ValueError(f"top_k must be greater than 0, got {top_k}")
|
|
105
|
+
|
|
106
|
+
self.document_store = document_store
|
|
107
|
+
self.filters = filters
|
|
108
|
+
self.top_k = top_k
|
|
109
|
+
self.scale_score = scale_score
|
|
110
|
+
self.return_embedding = return_embedding
|
|
111
|
+
self.filter_policy = filter_policy or FilterPolicy.REPLACE
|
|
112
|
+
|
|
113
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
114
|
+
"""Serialize the retriever to a dictionary."""
|
|
115
|
+
return default_to_dict(
|
|
116
|
+
self,
|
|
117
|
+
document_store=self.document_store,
|
|
118
|
+
filters=self.filters,
|
|
119
|
+
top_k=self.top_k,
|
|
120
|
+
scale_score=self.scale_score,
|
|
121
|
+
return_embedding=self.return_embedding,
|
|
122
|
+
filter_policy=self.filter_policy.value if hasattr(self.filter_policy, "value") else str(self.filter_policy),
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
@classmethod
|
|
126
|
+
def from_dict(cls, data: Dict[str, Any]) -> "DXMTEmbeddingRetriever":
|
|
127
|
+
"""Deserialize the retriever from a dictionary."""
|
|
128
|
+
init_params = data.get("init_parameters", {})
|
|
129
|
+
if "filter_policy" in init_params and HAYSTACK_AVAILABLE:
|
|
130
|
+
init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"])
|
|
131
|
+
return default_from_dict(cls, data)
|
|
132
|
+
|
|
133
|
+
def _get_telemetry_data(self) -> Dict[str, Any]:
|
|
134
|
+
"""Telemetry data for usage analytics."""
|
|
135
|
+
return {"document_store": type(self.document_store).__name__}
|
|
136
|
+
|
|
137
|
+
@component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
|
|
138
|
+
def run(
|
|
139
|
+
self,
|
|
140
|
+
query_embedding: List[float],
|
|
141
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
142
|
+
top_k: Optional[int] = None,
|
|
143
|
+
scale_score: Optional[bool] = None,
|
|
144
|
+
return_embedding: Optional[bool] = None,
|
|
145
|
+
) -> Dict[str, Any]:
|
|
146
|
+
"""
|
|
147
|
+
Run embedding retrieval.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
query_embedding: Query embedding vector
|
|
151
|
+
filters: Runtime filters to apply
|
|
152
|
+
top_k: Override default top_k
|
|
153
|
+
scale_score: Override default scale_score
|
|
154
|
+
return_embedding: Override default return_embedding
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
Dictionary with "documents" key containing retrieved documents
|
|
158
|
+
"""
|
|
159
|
+
# Merge filters based on policy
|
|
160
|
+
if self.filter_policy == FilterPolicy.MERGE and filters:
|
|
161
|
+
filters = {**(self.filters or {}), **filters}
|
|
162
|
+
else:
|
|
163
|
+
filters = filters or self.filters
|
|
164
|
+
|
|
165
|
+
# Use provided values or defaults
|
|
166
|
+
top_k = top_k if top_k is not None else self.top_k
|
|
167
|
+
scale_score = scale_score if scale_score is not None else self.scale_score
|
|
168
|
+
return_embedding = return_embedding if return_embedding is not None else self.return_embedding
|
|
169
|
+
|
|
170
|
+
# Perform retrieval
|
|
171
|
+
docs = self.document_store.embedding_retrieval(
|
|
172
|
+
query_embedding=query_embedding,
|
|
173
|
+
filters=filters,
|
|
174
|
+
top_k=top_k,
|
|
175
|
+
scale_score=scale_score,
|
|
176
|
+
return_embedding=return_embedding,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
return {"documents": docs}
|
|
180
|
+
|
|
181
|
+
@component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
|
|
182
|
+
async def run_async(
|
|
183
|
+
self,
|
|
184
|
+
query_embedding: List[float],
|
|
185
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
186
|
+
top_k: Optional[int] = None,
|
|
187
|
+
scale_score: Optional[bool] = None,
|
|
188
|
+
return_embedding: Optional[bool] = None,
|
|
189
|
+
) -> Dict[str, Any]:
|
|
190
|
+
"""Async version of run."""
|
|
191
|
+
# Merge filters
|
|
192
|
+
if self.filter_policy == FilterPolicy.MERGE and filters:
|
|
193
|
+
filters = {**(self.filters or {}), **filters}
|
|
194
|
+
else:
|
|
195
|
+
filters = filters or self.filters
|
|
196
|
+
|
|
197
|
+
# Use provided values or defaults
|
|
198
|
+
top_k = top_k if top_k is not None else self.top_k
|
|
199
|
+
scale_score = scale_score if scale_score is not None else self.scale_score
|
|
200
|
+
return_embedding = return_embedding if return_embedding is not None else self.return_embedding
|
|
201
|
+
|
|
202
|
+
# Perform async retrieval
|
|
203
|
+
docs = await self.document_store.embedding_retrieval_async(
|
|
204
|
+
query_embedding=query_embedding,
|
|
205
|
+
filters=filters,
|
|
206
|
+
top_k=top_k,
|
|
207
|
+
scale_score=scale_score,
|
|
208
|
+
return_embedding=return_embedding,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
return {"documents": docs}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@component
|
|
215
|
+
class DXMTBM25Retriever:
|
|
216
|
+
"""
|
|
217
|
+
Retriever for full-text search using BM25 ranking in DXMT.
|
|
218
|
+
|
|
219
|
+
This retriever performs BM25-ranked full-text search on document content
|
|
220
|
+
stored in DXMT. It uses DXMT's built-in full-text search capabilities
|
|
221
|
+
powered by Tantivy.
|
|
222
|
+
|
|
223
|
+
Usage example:
|
|
224
|
+
```python
|
|
225
|
+
from haystack import Pipeline
|
|
226
|
+
from dxmt import Client as DXMTClient
|
|
227
|
+
from dxmt.haystack import DXMTDocumentStore
|
|
228
|
+
from dxmt.haystack_retrievers import DXMTBM25Retriever
|
|
229
|
+
|
|
230
|
+
# Initialize
|
|
231
|
+
client = DXMTClient(host="localhost", port=9999)
|
|
232
|
+
doc_store = DXMTDocumentStore(client=client)
|
|
233
|
+
retriever = DXMTBM25Retriever(document_store=doc_store, top_k=10)
|
|
234
|
+
|
|
235
|
+
# Use in pipeline
|
|
236
|
+
pipeline = Pipeline()
|
|
237
|
+
pipeline.add_component("retriever", retriever)
|
|
238
|
+
|
|
239
|
+
result = pipeline.run({"retriever": {"query": "machine learning"}})
|
|
240
|
+
```
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
def __init__(
|
|
244
|
+
self,
|
|
245
|
+
document_store: DXMTDocumentStore,
|
|
246
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
247
|
+
top_k: int = 10,
|
|
248
|
+
scale_score: bool = False,
|
|
249
|
+
filter_policy: "FilterPolicy" = None,
|
|
250
|
+
):
|
|
251
|
+
"""
|
|
252
|
+
Initialize the DXMT BM25 Retriever.
|
|
253
|
+
|
|
254
|
+
Args:
|
|
255
|
+
document_store: DXMTDocumentStore instance
|
|
256
|
+
filters: Default filters to apply
|
|
257
|
+
top_k: Number of documents to retrieve (default: 10)
|
|
258
|
+
scale_score: Scale scores to [0, 1] range (default: False)
|
|
259
|
+
filter_policy: How to merge runtime and init filters (default: REPLACE)
|
|
260
|
+
|
|
261
|
+
Raises:
|
|
262
|
+
ValueError: If document_store is not a DXMTDocumentStore instance
|
|
263
|
+
ImportError: If haystack-ai is not installed (only when used in real pipelines)
|
|
264
|
+
|
|
265
|
+
Note:
|
|
266
|
+
While haystack-ai is recommended, this works without it for testing purposes.
|
|
267
|
+
"""
|
|
268
|
+
# Note: We don't enforce HAYSTACK_AVAILABLE to allow mock testing
|
|
269
|
+
|
|
270
|
+
if not isinstance(document_store, DXMTDocumentStore):
|
|
271
|
+
raise ValueError("document_store must be an instance of DXMTDocumentStore")
|
|
272
|
+
|
|
273
|
+
if top_k <= 0:
|
|
274
|
+
raise ValueError(f"top_k must be greater than 0, got {top_k}")
|
|
275
|
+
|
|
276
|
+
self.document_store = document_store
|
|
277
|
+
self.filters = filters
|
|
278
|
+
self.top_k = top_k
|
|
279
|
+
self.scale_score = scale_score
|
|
280
|
+
self.filter_policy = filter_policy or FilterPolicy.REPLACE
|
|
281
|
+
|
|
282
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
283
|
+
"""Serialize the retriever to a dictionary."""
|
|
284
|
+
return default_to_dict(
|
|
285
|
+
self,
|
|
286
|
+
document_store=self.document_store,
|
|
287
|
+
filters=self.filters,
|
|
288
|
+
top_k=self.top_k,
|
|
289
|
+
scale_score=self.scale_score,
|
|
290
|
+
filter_policy=self.filter_policy.value if hasattr(self.filter_policy, "value") else str(self.filter_policy),
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
@classmethod
|
|
294
|
+
def from_dict(cls, data: Dict[str, Any]) -> "DXMTBM25Retriever":
|
|
295
|
+
"""Deserialize the retriever from a dictionary."""
|
|
296
|
+
init_params = data.get("init_parameters", {})
|
|
297
|
+
if "filter_policy" in init_params and HAYSTACK_AVAILABLE:
|
|
298
|
+
init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"])
|
|
299
|
+
return default_from_dict(cls, data)
|
|
300
|
+
|
|
301
|
+
def _get_telemetry_data(self) -> Dict[str, Any]:
|
|
302
|
+
"""Telemetry data for usage analytics."""
|
|
303
|
+
return {"document_store": type(self.document_store).__name__}
|
|
304
|
+
|
|
305
|
+
@component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
|
|
306
|
+
def run(
|
|
307
|
+
self,
|
|
308
|
+
query: str,
|
|
309
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
310
|
+
top_k: Optional[int] = None,
|
|
311
|
+
scale_score: Optional[bool] = None,
|
|
312
|
+
) -> Dict[str, Any]:
|
|
313
|
+
"""
|
|
314
|
+
Run BM25 retrieval.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
query: Search query string
|
|
318
|
+
filters: Runtime filters to apply
|
|
319
|
+
top_k: Override default top_k
|
|
320
|
+
scale_score: Override default scale_score
|
|
321
|
+
|
|
322
|
+
Returns:
|
|
323
|
+
Dictionary with "documents" key containing retrieved documents
|
|
324
|
+
"""
|
|
325
|
+
# Merge filters based on policy
|
|
326
|
+
if self.filter_policy == FilterPolicy.MERGE and filters:
|
|
327
|
+
filters = {**(self.filters or {}), **filters}
|
|
328
|
+
else:
|
|
329
|
+
filters = filters or self.filters
|
|
330
|
+
|
|
331
|
+
# Use provided values or defaults
|
|
332
|
+
top_k = top_k if top_k is not None else self.top_k
|
|
333
|
+
scale_score = scale_score if scale_score is not None else self.scale_score
|
|
334
|
+
|
|
335
|
+
# Perform retrieval
|
|
336
|
+
docs = self.document_store.bm25_retrieval(
|
|
337
|
+
query=query,
|
|
338
|
+
filters=filters,
|
|
339
|
+
top_k=top_k,
|
|
340
|
+
scale_score=scale_score,
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
return {"documents": docs}
|
|
344
|
+
|
|
345
|
+
@component.output_types(documents=List[Document]) if HAYSTACK_AVAILABLE else lambda x: x
|
|
346
|
+
async def run_async(
|
|
347
|
+
self,
|
|
348
|
+
query: str,
|
|
349
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
350
|
+
top_k: Optional[int] = None,
|
|
351
|
+
scale_score: Optional[bool] = None,
|
|
352
|
+
) -> Dict[str, Any]:
|
|
353
|
+
"""Async version of run."""
|
|
354
|
+
# Merge filters
|
|
355
|
+
if self.filter_policy == FilterPolicy.MERGE and filters:
|
|
356
|
+
filters = {**(self.filters or {}), **filters}
|
|
357
|
+
else:
|
|
358
|
+
filters = filters or self.filters
|
|
359
|
+
|
|
360
|
+
# Use provided values or defaults
|
|
361
|
+
top_k = top_k if top_k is not None else self.top_k
|
|
362
|
+
scale_score = scale_score if scale_score is not None else self.scale_score
|
|
363
|
+
|
|
364
|
+
# Perform async retrieval
|
|
365
|
+
docs = await self.document_store.bm25_retrieval_async(
|
|
366
|
+
query=query,
|
|
367
|
+
filters=filters,
|
|
368
|
+
top_k=top_k,
|
|
369
|
+
scale_score=scale_score,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
return {"documents": docs}
|