dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,361 @@
1
+ """DXMT Retriever for MetaGPT.
2
+
3
+ This module provides a MetaGPT-compatible retriever implementation for DXMT VectorDB.
4
+
5
+ Example:
6
+ ```python
7
+ from llama_index.core import VectorStoreIndex, StorageContext
8
+ from llama_index.core.schema import Document
9
+ from dxmt import DXMTClient
10
+ from dxmt.metagpt_vector_store import DXMTVectorStore
11
+ from dxmt.metagpt_retriever import DXMTRetriever
12
+
13
+ # Create documents
14
+ documents = [Document(text="DXMT is a cost-efficient vector database")]
15
+
16
+ # Create DXMT vector store
17
+ client = DXMTClient(host="localhost", port=9999)
18
+ vector_store = DXMTVectorStore(
19
+ client=client,
20
+ table_name="metagpt_docs",
21
+ dimension=1536
22
+ )
23
+
24
+ # Create index
25
+ storage_context = StorageContext.from_defaults(vector_store=vector_store)
26
+ index = VectorStoreIndex.from_documents(
27
+ documents,
28
+ storage_context=storage_context
29
+ )
30
+
31
+ # Create DXMT retriever
32
+ retriever = DXMTRetriever(index=index, similarity_top_k=5)
33
+
34
+ # Use in MetaGPT
35
+ nodes = retriever.retrieve("What is DXMT?")
36
+ ```
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ import logging
42
+ from typing import Any, List, Optional
43
+
44
+ from llama_index.core.retrievers import VectorIndexRetriever
45
+ from llama_index.core.schema import BaseNode, NodeWithScore
46
+
47
+ from .metagpt_vector_store import DXMTVectorStore
48
+
49
+ logger = logging.getLogger(__name__)
50
+
51
+
52
+ class DXMTRetriever(VectorIndexRetriever):
53
+ """DXMT Retriever for MetaGPT.
54
+
55
+ Implements MetaGPT's retriever interface for DXMT VectorDB,
56
+ providing vector similarity search with additional utility methods.
57
+
58
+ This retriever extends LlamaIndex's VectorIndexRetriever with:
59
+ - add_nodes: Add new documents to the index
60
+ - persist: Save the index to disk
61
+ - query_total_count: Get total number of documents
62
+ - clear: Remove all documents
63
+
64
+ Args:
65
+ index: VectorStoreIndex using DXMTVectorStore.
66
+ similarity_top_k: Number of top results to return.
67
+ **kwargs: Additional arguments passed to VectorIndexRetriever.
68
+
69
+ Example:
70
+ ```python
71
+ from llama_index.core import VectorStoreIndex
72
+ from dxmt.metagpt_retriever import DXMTRetriever
73
+
74
+ # Assuming index is already created with DXMTVectorStore
75
+ retriever = DXMTRetriever(
76
+ index=index,
77
+ similarity_top_k=5
78
+ )
79
+
80
+ # Retrieve documents
81
+ results = retriever.retrieve("search query")
82
+
83
+ # Add new documents
84
+ from llama_index.core.schema import Document
85
+ new_docs = [Document(text="New document")]
86
+ retriever.add_documents(new_docs)
87
+
88
+ # Get total count
89
+ count = retriever.query_total_count()
90
+ ```
91
+ """
92
+
93
+ def __init__(
94
+ self,
95
+ index: Any,
96
+ similarity_top_k: int = 5,
97
+ **kwargs: Any,
98
+ ) -> None:
99
+ """Initialize DXMT retriever."""
100
+ super().__init__(
101
+ index=index,
102
+ similarity_top_k=similarity_top_k,
103
+ **kwargs,
104
+ )
105
+
106
+ @property
107
+ def vector_store(self) -> DXMTVectorStore:
108
+ """Get the DXMT vector store.
109
+
110
+ Returns:
111
+ DXMTVectorStore instance.
112
+
113
+ Raises:
114
+ TypeError: If vector store is not DXMTVectorStore.
115
+ """
116
+ vector_store = self._index.vector_store
117
+ if not isinstance(vector_store, DXMTVectorStore):
118
+ raise TypeError(
119
+ f"Expected DXMTVectorStore, got {type(vector_store).__name__}"
120
+ )
121
+ return vector_store
122
+
123
+ def add_nodes(self, nodes: List[BaseNode], **kwargs: Any) -> None:
124
+ """Add nodes to the index.
125
+
126
+ This method is required by MetaGPT's ModifiableRAGRetriever interface.
127
+
128
+ Args:
129
+ nodes: List of nodes to add.
130
+ **kwargs: Additional arguments.
131
+
132
+ Example:
133
+ ```python
134
+ from llama_index.core.schema import TextNode
135
+
136
+ nodes = [
137
+ TextNode(text="Document 1", embedding=[...]),
138
+ TextNode(text="Document 2", embedding=[...]),
139
+ ]
140
+ retriever.add_nodes(nodes)
141
+ ```
142
+ """
143
+ self._index.insert_nodes(nodes, **kwargs)
144
+ logger.info(f"Added {len(nodes)} nodes to DXMT index")
145
+
146
+ def add_documents(self, documents: List[Any], **kwargs: Any) -> None:
147
+ """Add documents to the index.
148
+
149
+ Convenience method for adding raw documents.
150
+
151
+ Args:
152
+ documents: List of Document objects.
153
+ **kwargs: Additional arguments.
154
+
155
+ Example:
156
+ ```python
157
+ from llama_index.core.schema import Document
158
+
159
+ docs = [
160
+ Document(text="Document 1"),
161
+ Document(text="Document 2"),
162
+ ]
163
+ retriever.add_documents(docs)
164
+ ```
165
+ """
166
+ # Convert documents to nodes and insert
167
+ for doc in documents:
168
+ self._index.insert(doc, **kwargs)
169
+ logger.info(f"Added {len(documents)} documents to DXMT index")
170
+
171
+ def persist(self, persist_dir: str, **kwargs: Any) -> None:
172
+ """Persist the index.
173
+
174
+ This method is required by MetaGPT's PersistableRAGRetriever interface.
175
+
176
+ Note: DXMT automatically persists data to its backend storage
177
+ (tiered: Memory → SSD → S3), so this method primarily saves the
178
+ index metadata.
179
+
180
+ Args:
181
+ persist_dir: Directory to persist to.
182
+ **kwargs: Additional arguments.
183
+
184
+ Example:
185
+ ```python
186
+ retriever.persist("./storage")
187
+ ```
188
+ """
189
+ # DXMT automatically persists vector data
190
+ # We only need to persist the index metadata
191
+ self._index.storage_context.persist(persist_dir)
192
+ logger.info(f"Persisted DXMT index metadata to {persist_dir}")
193
+
194
+ def query_total_count(self) -> int:
195
+ """Query total number of documents in the index.
196
+
197
+ This method is required by MetaGPT's QueryableRAGRetriever interface.
198
+
199
+ Returns:
200
+ Total number of documents.
201
+
202
+ Example:
203
+ ```python
204
+ count = retriever.query_total_count()
205
+ print(f"Total documents: {count}")
206
+ ```
207
+ """
208
+ try:
209
+ # Get table info from DXMT
210
+ table_info = self.vector_store.client.get_table(
211
+ self.vector_store.table_name
212
+ )
213
+ # Try to get record count from table info
214
+ count = table_info.get("record_count", 0)
215
+ if count == 0:
216
+ # Fallback: count documents in docstore
217
+ count = len(self._index.docstore.docs)
218
+ return count
219
+ except Exception as e:
220
+ logger.warning(f"Error getting total count: {e}, falling back to docstore")
221
+ # Fallback to docstore count
222
+ return len(self._index.docstore.docs)
223
+
224
+ def clear(self, **kwargs: Any) -> None:
225
+ """Clear all documents from the index.
226
+
227
+ This method is required by MetaGPT's DeletableRAGRetriever interface.
228
+
229
+ Warning: This operation is irreversible and will delete all data
230
+ in the DXMT table.
231
+
232
+ Args:
233
+ **kwargs: Additional arguments.
234
+
235
+ Example:
236
+ ```python
237
+ retriever.clear() # Deletes all documents
238
+ ```
239
+ """
240
+ self.vector_store.clear()
241
+ logger.info("Cleared all documents from DXMT index")
242
+
243
+ async def _aretrieve(self, query_str: str) -> List[NodeWithScore]:
244
+ """Async retrieve nodes.
245
+
246
+ Required by MetaGPT's RAGRetriever interface.
247
+
248
+ Args:
249
+ query_str: Query string.
250
+
251
+ Returns:
252
+ List of nodes with scores.
253
+ """
254
+ # Use the sync retrieve method
255
+ # LlamaIndex VectorIndexRetriever doesn't have async by default
256
+ return self.retrieve(query_str)
257
+
258
+ def _retrieve(self, query_str: str) -> List[NodeWithScore]:
259
+ """Sync retrieve nodes.
260
+
261
+ Args:
262
+ query_str: Query string.
263
+
264
+ Returns:
265
+ List of nodes with scores.
266
+ """
267
+ # Delegate to parent's retrieve method
268
+ return self.retrieve(query_str)
269
+
270
+
271
+ # Convenience function for creating DXMT retriever
272
+ def create_dxmt_retriever(
273
+ documents: List[Any],
274
+ table_name: str = "metagpt_docs",
275
+ dimension: int = 1536,
276
+ similarity_top_k: int = 5,
277
+ host: str = "localhost",
278
+ port: int = 9999,
279
+ embed_model: Optional[Any] = None,
280
+ api_key: Optional[str] = None,
281
+ **kwargs: Any,
282
+ ) -> DXMTRetriever:
283
+ """Create a DXMT retriever from documents.
284
+
285
+ Convenience function that creates the full pipeline:
286
+ 1. Creates DXMTVectorStore
287
+ 2. Creates VectorStoreIndex
288
+ 3. Creates DXMTRetriever
289
+
290
+ Args:
291
+ documents: List of documents to index.
292
+ table_name: DXMT table name.
293
+ dimension: Vector dimension.
294
+ similarity_top_k: Number of results to return.
295
+ host: DXMT server host.
296
+ port: DXMT server port.
297
+ embed_model: Embedding model (optional).
298
+ **kwargs: Additional arguments.
299
+
300
+ Returns:
301
+ Configured DXMTRetriever instance.
302
+
303
+ Example:
304
+ ```python
305
+ from llama_index.core.schema import Document
306
+ from dxmt.metagpt_retriever import create_dxmt_retriever
307
+
308
+ documents = [
309
+ Document(text="DXMT is a vector database"),
310
+ Document(text="MetaGPT is an AI agent framework"),
311
+ ]
312
+
313
+ retriever = create_dxmt_retriever(
314
+ documents=documents,
315
+ table_name="my_docs",
316
+ similarity_top_k=3
317
+ )
318
+
319
+ # Use the retriever
320
+ results = retriever.retrieve("What is DXMT?")
321
+ ```
322
+ """
323
+ from llama_index.core import StorageContext, VectorStoreIndex
324
+
325
+ from .client import DXMTClient
326
+ from .metagpt_vector_store import DXMTVectorStore
327
+
328
+ # Create DXMT client and vector store
329
+ client = DXMTClient(host=host, port=port, api_key=api_key)
330
+ vector_store = DXMTVectorStore(
331
+ client=client,
332
+ table_name=table_name,
333
+ dimension=dimension,
334
+ create_if_not_exists=True,
335
+ )
336
+
337
+ # Create storage context and index
338
+ storage_context = StorageContext.from_defaults(vector_store=vector_store)
339
+
340
+ index_kwargs = {
341
+ "storage_context": storage_context,
342
+ "show_progress": True,
343
+ }
344
+ if embed_model is not None:
345
+ index_kwargs["embed_model"] = embed_model
346
+
347
+ index = VectorStoreIndex.from_documents(documents, **index_kwargs)
348
+
349
+ # Create and return retriever
350
+ retriever = DXMTRetriever(
351
+ index=index,
352
+ similarity_top_k=similarity_top_k,
353
+ **kwargs,
354
+ )
355
+
356
+ logger.info(
357
+ f"Created DXMT retriever with {len(documents)} documents "
358
+ f"in table '{table_name}'"
359
+ )
360
+
361
+ return retriever