dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/__init__.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""DXMT Python SDK.
|
|
2
|
+
|
|
3
|
+
Python SDK for DXMT, the open-source memory database for personal agents: agent
|
|
4
|
+
memory (remember / recall / forget), vectors and full text, served by one Rust binary.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from dxmt import Client
|
|
8
|
+
>>> client = Client(host="localhost", port=9999)
|
|
9
|
+
>>> client.create_table(
|
|
10
|
+
... name="my_table",
|
|
11
|
+
... dimension=1536,
|
|
12
|
+
... distance_metric="cosine"
|
|
13
|
+
... )
|
|
14
|
+
>>> client.insert_records(
|
|
15
|
+
... table_name="my_table",
|
|
16
|
+
... records=[{
|
|
17
|
+
... "id": "doc1",
|
|
18
|
+
... "vector": [0.1, 0.2, ...],
|
|
19
|
+
... "metadata": {"text": "content"}
|
|
20
|
+
... }]
|
|
21
|
+
... )
|
|
22
|
+
>>> results = client.search(
|
|
23
|
+
... table_name="my_table",
|
|
24
|
+
... vector=[0.1, 0.2, ...],
|
|
25
|
+
... limit=10
|
|
26
|
+
... )
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from .client import DXMTClient, UploadFailure, UploadResult
|
|
30
|
+
from .client import DXMTClient as Client # Alias for backward compatibility
|
|
31
|
+
from .exceptions import (
|
|
32
|
+
DXMTConnectionError,
|
|
33
|
+
DXMTError,
|
|
34
|
+
DXMTRecordError,
|
|
35
|
+
DXMTSearchError,
|
|
36
|
+
DXMTTableError,
|
|
37
|
+
DXMTTimeoutError,
|
|
38
|
+
DXMTValidationError,
|
|
39
|
+
)
|
|
40
|
+
from .filters import Field
|
|
41
|
+
from .types import (
|
|
42
|
+
DenseVector,
|
|
43
|
+
DistanceMetric,
|
|
44
|
+
Filter,
|
|
45
|
+
IndexType,
|
|
46
|
+
InsertMode,
|
|
47
|
+
QuantizationType,
|
|
48
|
+
Record,
|
|
49
|
+
SearchResult,
|
|
50
|
+
SparseVector,
|
|
51
|
+
TableConfig,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
# Optional LangGraph integration (requires langgraph package)
|
|
55
|
+
try:
|
|
56
|
+
from .langgraph_store import DXMTLangGraphStore
|
|
57
|
+
LANGGRAPH_AVAILABLE = True
|
|
58
|
+
except (ImportError, RuntimeError, Exception):
|
|
59
|
+
# Catch all exceptions during import (including pydantic validation errors)
|
|
60
|
+
LANGGRAPH_AVAILABLE = False
|
|
61
|
+
DXMTLangGraphStore = None # type: ignore
|
|
62
|
+
|
|
63
|
+
# Optional mem0 integration (requires mem0ai package)
|
|
64
|
+
try:
|
|
65
|
+
from .mem0 import DXMTMemoryStore
|
|
66
|
+
MEM0_AVAILABLE = True
|
|
67
|
+
except ImportError:
|
|
68
|
+
MEM0_AVAILABLE = False
|
|
69
|
+
DXMTMemoryStore = None # type: ignore
|
|
70
|
+
|
|
71
|
+
# Optional Haystack integration (requires haystack-ai package)
|
|
72
|
+
try:
|
|
73
|
+
from .haystack import DXMTDocumentStore
|
|
74
|
+
from .haystack_retrievers import DXMTEmbeddingRetriever, DXMTBM25Retriever
|
|
75
|
+
HAYSTACK_AVAILABLE = True
|
|
76
|
+
except ImportError:
|
|
77
|
+
HAYSTACK_AVAILABLE = False
|
|
78
|
+
DXMTDocumentStore = None # type: ignore
|
|
79
|
+
DXMTEmbeddingRetriever = None # type: ignore
|
|
80
|
+
DXMTBM25Retriever = None # type: ignore
|
|
81
|
+
|
|
82
|
+
# Optional CrewAI integration (requires crewai package)
|
|
83
|
+
try:
|
|
84
|
+
from .crewai_tool import DXMTVectorSearchTool, DXMTToolConfig
|
|
85
|
+
from .crewai_storage import DXMTStorage, DXMTStorageConfig
|
|
86
|
+
CREWAI_AVAILABLE = True
|
|
87
|
+
except ImportError:
|
|
88
|
+
CREWAI_AVAILABLE = False
|
|
89
|
+
DXMTVectorSearchTool = None # type: ignore
|
|
90
|
+
DXMTToolConfig = None # type: ignore
|
|
91
|
+
DXMTStorage = None # type: ignore
|
|
92
|
+
DXMTStorageConfig = None # type: ignore
|
|
93
|
+
|
|
94
|
+
# Optional MetaGPT/LlamaIndex integration (requires llama-index package)
|
|
95
|
+
try:
|
|
96
|
+
from .metagpt_vector_store import DXMTVectorStore as DXMTLlamaIndexVectorStore
|
|
97
|
+
from .metagpt_retriever import DXMTRetriever, create_dxmt_retriever
|
|
98
|
+
METAGPT_AVAILABLE = True
|
|
99
|
+
except ImportError:
|
|
100
|
+
METAGPT_AVAILABLE = False
|
|
101
|
+
DXMTLlamaIndexVectorStore = None # type: ignore
|
|
102
|
+
DXMTRetriever = None # type: ignore
|
|
103
|
+
create_dxmt_retriever = None # type: ignore
|
|
104
|
+
|
|
105
|
+
# Optional Bisheng compatibility layer (requires langchain). For LangChain itself use the
|
|
106
|
+
# `langchain-dxmt` package; see the docstring of `langchain_vectorstore.py`.
|
|
107
|
+
try:
|
|
108
|
+
from .langchain_vectorstore import DXMTVectorStore as DXMTLangChainVectorStore
|
|
109
|
+
from .bisheng_integration import initialize_dxmt, initial_dxmt
|
|
110
|
+
LANGCHAIN_AVAILABLE = True
|
|
111
|
+
except ImportError:
|
|
112
|
+
LANGCHAIN_AVAILABLE = False
|
|
113
|
+
DXMTLangChainVectorStore = None # type: ignore
|
|
114
|
+
initialize_dxmt = None # type: ignore
|
|
115
|
+
initial_dxmt = None # type: ignore
|
|
116
|
+
|
|
117
|
+
__version__ = "1.6.0"
|
|
118
|
+
|
|
119
|
+
__all__ = [
|
|
120
|
+
"UploadResult",
|
|
121
|
+
"UploadFailure",
|
|
122
|
+
# Main client
|
|
123
|
+
"Client",
|
|
124
|
+
"DXMTClient", # Also export under original name
|
|
125
|
+
# Exceptions
|
|
126
|
+
"DXMTError",
|
|
127
|
+
"DXMTConnectionError",
|
|
128
|
+
"DXMTTableError",
|
|
129
|
+
"DXMTRecordError",
|
|
130
|
+
"DXMTSearchError",
|
|
131
|
+
"DXMTValidationError",
|
|
132
|
+
"DXMTTimeoutError",
|
|
133
|
+
# Types
|
|
134
|
+
"DenseVector",
|
|
135
|
+
"SparseVector",
|
|
136
|
+
"DistanceMetric",
|
|
137
|
+
"IndexType",
|
|
138
|
+
"InsertMode",
|
|
139
|
+
"QuantizationType",
|
|
140
|
+
"Record",
|
|
141
|
+
"SearchResult",
|
|
142
|
+
"TableConfig",
|
|
143
|
+
"Filter",
|
|
144
|
+
"Field",
|
|
145
|
+
# LangGraph integration (optional)
|
|
146
|
+
"DXMTLangGraphStore",
|
|
147
|
+
# mem0 integration (optional)
|
|
148
|
+
"DXMTMemoryStore",
|
|
149
|
+
# Haystack integration (optional)
|
|
150
|
+
"DXMTDocumentStore",
|
|
151
|
+
"DXMTEmbeddingRetriever",
|
|
152
|
+
"DXMTBM25Retriever",
|
|
153
|
+
# CrewAI integration (optional)
|
|
154
|
+
"DXMTVectorSearchTool",
|
|
155
|
+
"DXMTToolConfig",
|
|
156
|
+
"DXMTStorage",
|
|
157
|
+
"DXMTStorageConfig",
|
|
158
|
+
# MetaGPT/LlamaIndex integration (optional)
|
|
159
|
+
"DXMTLlamaIndexVectorStore",
|
|
160
|
+
"DXMTRetriever",
|
|
161
|
+
"create_dxmt_retriever",
|
|
162
|
+
# Bisheng/LangChain integration (optional)
|
|
163
|
+
"DXMTLangChainVectorStore",
|
|
164
|
+
"initialize_dxmt",
|
|
165
|
+
"initial_dxmt",
|
|
166
|
+
]
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""DXMT + Bisheng Integration Module.
|
|
2
|
+
|
|
3
|
+
This module provides the initialization function for DXMT VectorStore
|
|
4
|
+
to be used with Bisheng's vector store system.
|
|
5
|
+
|
|
6
|
+
Bisheng Integration Pattern:
|
|
7
|
+
Bisheng uses a factory pattern with initialization functions for each
|
|
8
|
+
vector store. This module provides the initialization function that
|
|
9
|
+
Bisheng can call to create DXMT vector store instances.
|
|
10
|
+
|
|
11
|
+
Usage in Bisheng:
|
|
12
|
+
Add to bisheng/interface/initialize/vector_store.py:
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from dxmt.bisheng_integration import initialize_dxmt
|
|
16
|
+
|
|
17
|
+
vecstore_initializer = {
|
|
18
|
+
# ... other vector stores ...
|
|
19
|
+
'DXMT': initialize_dxmt,
|
|
20
|
+
}
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Example:
|
|
24
|
+
```python
|
|
25
|
+
from langchain.embeddings import OpenAIEmbeddings
|
|
26
|
+
from dxmt.bisheng_integration import initialize_dxmt
|
|
27
|
+
from dxmt.langchain_vectorstore import DXMTVectorStore
|
|
28
|
+
|
|
29
|
+
# Bisheng-style params
|
|
30
|
+
params = {
|
|
31
|
+
'embedding': OpenAIEmbeddings(),
|
|
32
|
+
'collection_name': 'test_docs',
|
|
33
|
+
'host': 'localhost',
|
|
34
|
+
'port': 9999,
|
|
35
|
+
'documents': [...], # Optional initial documents
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
# Initialize via Bisheng pattern
|
|
39
|
+
vector_store = initialize_dxmt(DXMTVectorStore, params, {})
|
|
40
|
+
```
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import logging
|
|
46
|
+
from typing import Any, Dict, Type
|
|
47
|
+
|
|
48
|
+
try:
|
|
49
|
+
from langchain_core.documents import Document
|
|
50
|
+
LANGCHAIN_AVAILABLE = True
|
|
51
|
+
except ImportError:
|
|
52
|
+
LANGCHAIN_AVAILABLE = False
|
|
53
|
+
Document = Any # type: ignore
|
|
54
|
+
|
|
55
|
+
from .langchain_vectorstore import DXMTVectorStore
|
|
56
|
+
|
|
57
|
+
logger = logging.getLogger(__name__)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def docs_in_params(params: dict) -> bool:
|
|
61
|
+
"""Check if params has documents OR texts.
|
|
62
|
+
|
|
63
|
+
This is a helper function following Bisheng's pattern to determine
|
|
64
|
+
if we should create a new index with documents or return an existing index.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
params: Parameters dict.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
True if documents or texts are present and not empty.
|
|
71
|
+
"""
|
|
72
|
+
return ('documents' in params and params['documents']) or ('texts' in params and params['texts'])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def initialize_dxmt(
|
|
76
|
+
class_object: Type[DXMTVectorStore],
|
|
77
|
+
params: Dict[str, Any],
|
|
78
|
+
search_kwargs: Dict[str, Any]
|
|
79
|
+
) -> DXMTVectorStore:
|
|
80
|
+
"""Initialize DXMT vector store for Bisheng.
|
|
81
|
+
|
|
82
|
+
This function follows Bisheng's initialization pattern for vector stores.
|
|
83
|
+
It handles both creating a new DXMT collection with documents and
|
|
84
|
+
connecting to an existing collection.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
class_object: The DXMTVectorStore class (passed by Bisheng).
|
|
88
|
+
params: Initialization parameters from Bisheng, including:
|
|
89
|
+
- embedding: Embeddings instance (required)
|
|
90
|
+
- collection_name: Name of the DXMT collection (default: "bisheng_collection")
|
|
91
|
+
- host: DXMT server host (default: "localhost")
|
|
92
|
+
- port: DXMT server port (default: 9999)
|
|
93
|
+
- dimension: Vector dimension (optional, auto-detected)
|
|
94
|
+
- distance_metric: Distance metric (default: "cosine")
|
|
95
|
+
- documents: Optional list of Document objects to add
|
|
96
|
+
- texts: Optional list of text strings to add
|
|
97
|
+
- metadatas: Optional list of metadata dicts (for texts)
|
|
98
|
+
- collection_id: Optional Bisheng knowledge base ID
|
|
99
|
+
search_kwargs: Additional search parameters (currently unused).
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
Initialized DXMTVectorStore instance.
|
|
103
|
+
|
|
104
|
+
Raises:
|
|
105
|
+
ValueError: If required parameters are missing.
|
|
106
|
+
DXMTError: If initialization fails.
|
|
107
|
+
|
|
108
|
+
Example:
|
|
109
|
+
```python
|
|
110
|
+
from langchain.embeddings import OpenAIEmbeddings
|
|
111
|
+
from dxmt.bisheng_integration import initialize_dxmt
|
|
112
|
+
from dxmt.langchain_vectorstore import DXMTVectorStore
|
|
113
|
+
|
|
114
|
+
params = {
|
|
115
|
+
'embedding': OpenAIEmbeddings(),
|
|
116
|
+
'collection_name': 'my_docs',
|
|
117
|
+
'host': 'localhost',
|
|
118
|
+
'port': 9999,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
# Without initial documents (connect to existing)
|
|
122
|
+
vector_store = initialize_dxmt(DXMTVectorStore, params, {})
|
|
123
|
+
|
|
124
|
+
# With initial documents (create new index)
|
|
125
|
+
params['documents'] = [Document(page_content="doc1", metadata={})]
|
|
126
|
+
vector_store = initialize_dxmt(DXMTVectorStore, params, {})
|
|
127
|
+
```
|
|
128
|
+
"""
|
|
129
|
+
if not LANGCHAIN_AVAILABLE:
|
|
130
|
+
raise ImportError(
|
|
131
|
+
"LangChain is required for Bisheng integration. "
|
|
132
|
+
"Install with: pip install langchain langchain-core"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# Validate required parameters
|
|
136
|
+
if 'embedding' not in params:
|
|
137
|
+
raise ValueError('Embedding must be provided in params')
|
|
138
|
+
|
|
139
|
+
# Extract DXMT-specific parameters
|
|
140
|
+
collection_name = params.pop('collection_name', 'bisheng_collection')
|
|
141
|
+
host = params.pop('host', 'localhost')
|
|
142
|
+
port = params.pop('port', 9999)
|
|
143
|
+
dimension = params.pop('dimension', None)
|
|
144
|
+
distance_metric = params.pop('distance_metric', 'cosine')
|
|
145
|
+
collection_id = params.pop('collection_id', None) # Bisheng knowledge base ID
|
|
146
|
+
|
|
147
|
+
logger.info(
|
|
148
|
+
f"Initializing DXMT vector store: collection={collection_name}, "
|
|
149
|
+
f"host={host}, port={port}, dimension={dimension}"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# Handle Bisheng knowledge base ID mapping
|
|
153
|
+
if collection_id:
|
|
154
|
+
logger.info(f"Using Bisheng knowledge base ID: {collection_id}")
|
|
155
|
+
# Bisheng knowledge bases use collection_id to identify collections
|
|
156
|
+
# You could implement custom logic here to map collection_id to collection_name
|
|
157
|
+
|
|
158
|
+
# Check if we have initial documents to add
|
|
159
|
+
if not docs_in_params(params):
|
|
160
|
+
# No documents - connect to existing collection
|
|
161
|
+
params.pop('documents', None)
|
|
162
|
+
params.pop('texts', None)
|
|
163
|
+
params.pop('metadatas', None)
|
|
164
|
+
|
|
165
|
+
logger.info(f"Connecting to existing DXMT collection: {collection_name}")
|
|
166
|
+
|
|
167
|
+
# Create vector store instance (connects to existing table)
|
|
168
|
+
vector_store = class_object(
|
|
169
|
+
embedding=params['embedding'],
|
|
170
|
+
collection_name=collection_name,
|
|
171
|
+
host=host,
|
|
172
|
+
port=port,
|
|
173
|
+
dimension=dimension,
|
|
174
|
+
distance_metric=distance_metric,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
return vector_store
|
|
178
|
+
|
|
179
|
+
# Has documents - create new index with documents
|
|
180
|
+
logger.info(f"Creating new DXMT collection with documents: {collection_name}")
|
|
181
|
+
|
|
182
|
+
# Handle both 'documents' and 'texts' formats
|
|
183
|
+
if 'texts' in params:
|
|
184
|
+
# Convert texts to documents
|
|
185
|
+
texts = params.pop('texts')
|
|
186
|
+
metadatas = params.pop('metadatas', None)
|
|
187
|
+
|
|
188
|
+
# Use from_texts class method
|
|
189
|
+
vector_store = class_object.from_texts(
|
|
190
|
+
texts=texts,
|
|
191
|
+
embedding=params.pop('embedding'),
|
|
192
|
+
metadatas=metadatas,
|
|
193
|
+
collection_name=collection_name,
|
|
194
|
+
host=host,
|
|
195
|
+
port=port,
|
|
196
|
+
dimension=dimension,
|
|
197
|
+
distance_metric=distance_metric,
|
|
198
|
+
)
|
|
199
|
+
else:
|
|
200
|
+
# Use documents directly
|
|
201
|
+
documents = params.pop('documents')
|
|
202
|
+
|
|
203
|
+
# Use from_documents class method
|
|
204
|
+
vector_store = class_object.from_documents(
|
|
205
|
+
documents=documents,
|
|
206
|
+
embedding=params.pop('embedding'),
|
|
207
|
+
collection_name=collection_name,
|
|
208
|
+
host=host,
|
|
209
|
+
port=port,
|
|
210
|
+
dimension=dimension,
|
|
211
|
+
distance_metric=distance_metric,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
logger.info(f"Successfully created DXMT vector store: {collection_name}")
|
|
215
|
+
return vector_store
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# Alias for consistency with Bisheng naming
|
|
219
|
+
initial_dxmt = initialize_dxmt
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
__all__ = [
|
|
223
|
+
'initialize_dxmt',
|
|
224
|
+
'initial_dxmt',
|
|
225
|
+
'docs_in_params',
|
|
226
|
+
]
|