dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""DXMT Retriever for MetaGPT.
|
|
2
|
+
|
|
3
|
+
This module provides a MetaGPT-compatible retriever implementation for DXMT VectorDB.
|
|
4
|
+
|
|
5
|
+
Example:
|
|
6
|
+
```python
|
|
7
|
+
from llama_index.core import VectorStoreIndex, StorageContext
|
|
8
|
+
from llama_index.core.schema import Document
|
|
9
|
+
from dxmt import DXMTClient
|
|
10
|
+
from dxmt.metagpt_vector_store import DXMTVectorStore
|
|
11
|
+
from dxmt.metagpt_retriever import DXMTRetriever
|
|
12
|
+
|
|
13
|
+
# Create documents
|
|
14
|
+
documents = [Document(text="DXMT is a cost-efficient vector database")]
|
|
15
|
+
|
|
16
|
+
# Create DXMT vector store
|
|
17
|
+
client = DXMTClient(host="localhost", port=9999)
|
|
18
|
+
vector_store = DXMTVectorStore(
|
|
19
|
+
client=client,
|
|
20
|
+
table_name="metagpt_docs",
|
|
21
|
+
dimension=1536
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# Create index
|
|
25
|
+
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
|
26
|
+
index = VectorStoreIndex.from_documents(
|
|
27
|
+
documents,
|
|
28
|
+
storage_context=storage_context
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Create DXMT retriever
|
|
32
|
+
retriever = DXMTRetriever(index=index, similarity_top_k=5)
|
|
33
|
+
|
|
34
|
+
# Use in MetaGPT
|
|
35
|
+
nodes = retriever.retrieve("What is DXMT?")
|
|
36
|
+
```
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import logging
|
|
42
|
+
from typing import Any, List, Optional
|
|
43
|
+
|
|
44
|
+
from llama_index.core.retrievers import VectorIndexRetriever
|
|
45
|
+
from llama_index.core.schema import BaseNode, NodeWithScore
|
|
46
|
+
|
|
47
|
+
from .metagpt_vector_store import DXMTVectorStore
|
|
48
|
+
|
|
49
|
+
logger = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class DXMTRetriever(VectorIndexRetriever):
|
|
53
|
+
"""DXMT Retriever for MetaGPT.
|
|
54
|
+
|
|
55
|
+
Implements MetaGPT's retriever interface for DXMT VectorDB,
|
|
56
|
+
providing vector similarity search with additional utility methods.
|
|
57
|
+
|
|
58
|
+
This retriever extends LlamaIndex's VectorIndexRetriever with:
|
|
59
|
+
- add_nodes: Add new documents to the index
|
|
60
|
+
- persist: Save the index to disk
|
|
61
|
+
- query_total_count: Get total number of documents
|
|
62
|
+
- clear: Remove all documents
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
index: VectorStoreIndex using DXMTVectorStore.
|
|
66
|
+
similarity_top_k: Number of top results to return.
|
|
67
|
+
**kwargs: Additional arguments passed to VectorIndexRetriever.
|
|
68
|
+
|
|
69
|
+
Example:
|
|
70
|
+
```python
|
|
71
|
+
from llama_index.core import VectorStoreIndex
|
|
72
|
+
from dxmt.metagpt_retriever import DXMTRetriever
|
|
73
|
+
|
|
74
|
+
# Assuming index is already created with DXMTVectorStore
|
|
75
|
+
retriever = DXMTRetriever(
|
|
76
|
+
index=index,
|
|
77
|
+
similarity_top_k=5
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# Retrieve documents
|
|
81
|
+
results = retriever.retrieve("search query")
|
|
82
|
+
|
|
83
|
+
# Add new documents
|
|
84
|
+
from llama_index.core.schema import Document
|
|
85
|
+
new_docs = [Document(text="New document")]
|
|
86
|
+
retriever.add_documents(new_docs)
|
|
87
|
+
|
|
88
|
+
# Get total count
|
|
89
|
+
count = retriever.query_total_count()
|
|
90
|
+
```
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(
|
|
94
|
+
self,
|
|
95
|
+
index: Any,
|
|
96
|
+
similarity_top_k: int = 5,
|
|
97
|
+
**kwargs: Any,
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Initialize DXMT retriever."""
|
|
100
|
+
super().__init__(
|
|
101
|
+
index=index,
|
|
102
|
+
similarity_top_k=similarity_top_k,
|
|
103
|
+
**kwargs,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def vector_store(self) -> DXMTVectorStore:
|
|
108
|
+
"""Get the DXMT vector store.
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
DXMTVectorStore instance.
|
|
112
|
+
|
|
113
|
+
Raises:
|
|
114
|
+
TypeError: If vector store is not DXMTVectorStore.
|
|
115
|
+
"""
|
|
116
|
+
vector_store = self._index.vector_store
|
|
117
|
+
if not isinstance(vector_store, DXMTVectorStore):
|
|
118
|
+
raise TypeError(
|
|
119
|
+
f"Expected DXMTVectorStore, got {type(vector_store).__name__}"
|
|
120
|
+
)
|
|
121
|
+
return vector_store
|
|
122
|
+
|
|
123
|
+
def add_nodes(self, nodes: List[BaseNode], **kwargs: Any) -> None:
|
|
124
|
+
"""Add nodes to the index.
|
|
125
|
+
|
|
126
|
+
This method is required by MetaGPT's ModifiableRAGRetriever interface.
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
nodes: List of nodes to add.
|
|
130
|
+
**kwargs: Additional arguments.
|
|
131
|
+
|
|
132
|
+
Example:
|
|
133
|
+
```python
|
|
134
|
+
from llama_index.core.schema import TextNode
|
|
135
|
+
|
|
136
|
+
nodes = [
|
|
137
|
+
TextNode(text="Document 1", embedding=[...]),
|
|
138
|
+
TextNode(text="Document 2", embedding=[...]),
|
|
139
|
+
]
|
|
140
|
+
retriever.add_nodes(nodes)
|
|
141
|
+
```
|
|
142
|
+
"""
|
|
143
|
+
self._index.insert_nodes(nodes, **kwargs)
|
|
144
|
+
logger.info(f"Added {len(nodes)} nodes to DXMT index")
|
|
145
|
+
|
|
146
|
+
def add_documents(self, documents: List[Any], **kwargs: Any) -> None:
|
|
147
|
+
"""Add documents to the index.
|
|
148
|
+
|
|
149
|
+
Convenience method for adding raw documents.
|
|
150
|
+
|
|
151
|
+
Args:
|
|
152
|
+
documents: List of Document objects.
|
|
153
|
+
**kwargs: Additional arguments.
|
|
154
|
+
|
|
155
|
+
Example:
|
|
156
|
+
```python
|
|
157
|
+
from llama_index.core.schema import Document
|
|
158
|
+
|
|
159
|
+
docs = [
|
|
160
|
+
Document(text="Document 1"),
|
|
161
|
+
Document(text="Document 2"),
|
|
162
|
+
]
|
|
163
|
+
retriever.add_documents(docs)
|
|
164
|
+
```
|
|
165
|
+
"""
|
|
166
|
+
# Convert documents to nodes and insert
|
|
167
|
+
for doc in documents:
|
|
168
|
+
self._index.insert(doc, **kwargs)
|
|
169
|
+
logger.info(f"Added {len(documents)} documents to DXMT index")
|
|
170
|
+
|
|
171
|
+
def persist(self, persist_dir: str, **kwargs: Any) -> None:
|
|
172
|
+
"""Persist the index.
|
|
173
|
+
|
|
174
|
+
This method is required by MetaGPT's PersistableRAGRetriever interface.
|
|
175
|
+
|
|
176
|
+
Note: DXMT automatically persists data to its backend storage
|
|
177
|
+
(tiered: Memory → SSD → S3), so this method primarily saves the
|
|
178
|
+
index metadata.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
persist_dir: Directory to persist to.
|
|
182
|
+
**kwargs: Additional arguments.
|
|
183
|
+
|
|
184
|
+
Example:
|
|
185
|
+
```python
|
|
186
|
+
retriever.persist("./storage")
|
|
187
|
+
```
|
|
188
|
+
"""
|
|
189
|
+
# DXMT automatically persists vector data
|
|
190
|
+
# We only need to persist the index metadata
|
|
191
|
+
self._index.storage_context.persist(persist_dir)
|
|
192
|
+
logger.info(f"Persisted DXMT index metadata to {persist_dir}")
|
|
193
|
+
|
|
194
|
+
def query_total_count(self) -> int:
|
|
195
|
+
"""Query total number of documents in the index.
|
|
196
|
+
|
|
197
|
+
This method is required by MetaGPT's QueryableRAGRetriever interface.
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
Total number of documents.
|
|
201
|
+
|
|
202
|
+
Example:
|
|
203
|
+
```python
|
|
204
|
+
count = retriever.query_total_count()
|
|
205
|
+
print(f"Total documents: {count}")
|
|
206
|
+
```
|
|
207
|
+
"""
|
|
208
|
+
try:
|
|
209
|
+
# Get table info from DXMT
|
|
210
|
+
table_info = self.vector_store.client.get_table(
|
|
211
|
+
self.vector_store.table_name
|
|
212
|
+
)
|
|
213
|
+
# Try to get record count from table info
|
|
214
|
+
count = table_info.get("record_count", 0)
|
|
215
|
+
if count == 0:
|
|
216
|
+
# Fallback: count documents in docstore
|
|
217
|
+
count = len(self._index.docstore.docs)
|
|
218
|
+
return count
|
|
219
|
+
except Exception as e:
|
|
220
|
+
logger.warning(f"Error getting total count: {e}, falling back to docstore")
|
|
221
|
+
# Fallback to docstore count
|
|
222
|
+
return len(self._index.docstore.docs)
|
|
223
|
+
|
|
224
|
+
def clear(self, **kwargs: Any) -> None:
|
|
225
|
+
"""Clear all documents from the index.
|
|
226
|
+
|
|
227
|
+
This method is required by MetaGPT's DeletableRAGRetriever interface.
|
|
228
|
+
|
|
229
|
+
Warning: This operation is irreversible and will delete all data
|
|
230
|
+
in the DXMT table.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
**kwargs: Additional arguments.
|
|
234
|
+
|
|
235
|
+
Example:
|
|
236
|
+
```python
|
|
237
|
+
retriever.clear() # Deletes all documents
|
|
238
|
+
```
|
|
239
|
+
"""
|
|
240
|
+
self.vector_store.clear()
|
|
241
|
+
logger.info("Cleared all documents from DXMT index")
|
|
242
|
+
|
|
243
|
+
async def _aretrieve(self, query_str: str) -> List[NodeWithScore]:
|
|
244
|
+
"""Async retrieve nodes.
|
|
245
|
+
|
|
246
|
+
Required by MetaGPT's RAGRetriever interface.
|
|
247
|
+
|
|
248
|
+
Args:
|
|
249
|
+
query_str: Query string.
|
|
250
|
+
|
|
251
|
+
Returns:
|
|
252
|
+
List of nodes with scores.
|
|
253
|
+
"""
|
|
254
|
+
# Use the sync retrieve method
|
|
255
|
+
# LlamaIndex VectorIndexRetriever doesn't have async by default
|
|
256
|
+
return self.retrieve(query_str)
|
|
257
|
+
|
|
258
|
+
def _retrieve(self, query_str: str) -> List[NodeWithScore]:
|
|
259
|
+
"""Sync retrieve nodes.
|
|
260
|
+
|
|
261
|
+
Args:
|
|
262
|
+
query_str: Query string.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
List of nodes with scores.
|
|
266
|
+
"""
|
|
267
|
+
# Delegate to parent's retrieve method
|
|
268
|
+
return self.retrieve(query_str)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
# Convenience function for creating DXMT retriever
|
|
272
|
+
def create_dxmt_retriever(
|
|
273
|
+
documents: List[Any],
|
|
274
|
+
table_name: str = "metagpt_docs",
|
|
275
|
+
dimension: int = 1536,
|
|
276
|
+
similarity_top_k: int = 5,
|
|
277
|
+
host: str = "localhost",
|
|
278
|
+
port: int = 9999,
|
|
279
|
+
embed_model: Optional[Any] = None,
|
|
280
|
+
api_key: Optional[str] = None,
|
|
281
|
+
**kwargs: Any,
|
|
282
|
+
) -> DXMTRetriever:
|
|
283
|
+
"""Create a DXMT retriever from documents.
|
|
284
|
+
|
|
285
|
+
Convenience function that creates the full pipeline:
|
|
286
|
+
1. Creates DXMTVectorStore
|
|
287
|
+
2. Creates VectorStoreIndex
|
|
288
|
+
3. Creates DXMTRetriever
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
documents: List of documents to index.
|
|
292
|
+
table_name: DXMT table name.
|
|
293
|
+
dimension: Vector dimension.
|
|
294
|
+
similarity_top_k: Number of results to return.
|
|
295
|
+
host: DXMT server host.
|
|
296
|
+
port: DXMT server port.
|
|
297
|
+
embed_model: Embedding model (optional).
|
|
298
|
+
**kwargs: Additional arguments.
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
Configured DXMTRetriever instance.
|
|
302
|
+
|
|
303
|
+
Example:
|
|
304
|
+
```python
|
|
305
|
+
from llama_index.core.schema import Document
|
|
306
|
+
from dxmt.metagpt_retriever import create_dxmt_retriever
|
|
307
|
+
|
|
308
|
+
documents = [
|
|
309
|
+
Document(text="DXMT is a vector database"),
|
|
310
|
+
Document(text="MetaGPT is an AI agent framework"),
|
|
311
|
+
]
|
|
312
|
+
|
|
313
|
+
retriever = create_dxmt_retriever(
|
|
314
|
+
documents=documents,
|
|
315
|
+
table_name="my_docs",
|
|
316
|
+
similarity_top_k=3
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
# Use the retriever
|
|
320
|
+
results = retriever.retrieve("What is DXMT?")
|
|
321
|
+
```
|
|
322
|
+
"""
|
|
323
|
+
from llama_index.core import StorageContext, VectorStoreIndex
|
|
324
|
+
|
|
325
|
+
from .client import DXMTClient
|
|
326
|
+
from .metagpt_vector_store import DXMTVectorStore
|
|
327
|
+
|
|
328
|
+
# Create DXMT client and vector store
|
|
329
|
+
client = DXMTClient(host=host, port=port, api_key=api_key)
|
|
330
|
+
vector_store = DXMTVectorStore(
|
|
331
|
+
client=client,
|
|
332
|
+
table_name=table_name,
|
|
333
|
+
dimension=dimension,
|
|
334
|
+
create_if_not_exists=True,
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
# Create storage context and index
|
|
338
|
+
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
|
339
|
+
|
|
340
|
+
index_kwargs = {
|
|
341
|
+
"storage_context": storage_context,
|
|
342
|
+
"show_progress": True,
|
|
343
|
+
}
|
|
344
|
+
if embed_model is not None:
|
|
345
|
+
index_kwargs["embed_model"] = embed_model
|
|
346
|
+
|
|
347
|
+
index = VectorStoreIndex.from_documents(documents, **index_kwargs)
|
|
348
|
+
|
|
349
|
+
# Create and return retriever
|
|
350
|
+
retriever = DXMTRetriever(
|
|
351
|
+
index=index,
|
|
352
|
+
similarity_top_k=similarity_top_k,
|
|
353
|
+
**kwargs,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
logger.info(
|
|
357
|
+
f"Created DXMT retriever with {len(documents)} documents "
|
|
358
|
+
f"in table '{table_name}'"
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
return retriever
|