dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/crewai_tool.py
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
"""DXMT VectorDB Tool for CrewAI.
|
|
2
|
+
|
|
3
|
+
This module provides a CrewAI BaseTool implementation for DXMT VectorDB,
|
|
4
|
+
enabling vector similarity search within CrewAI agents and workflows.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
```python
|
|
8
|
+
from crewai import Agent, Task, Crew
|
|
9
|
+
from dxmt.crewai_tool import DXMTVectorSearchTool, DXMTToolConfig
|
|
10
|
+
|
|
11
|
+
# Configure DXMT tool
|
|
12
|
+
config = DXMTToolConfig(
|
|
13
|
+
host="localhost",
|
|
14
|
+
port=9999,
|
|
15
|
+
table_name="documents",
|
|
16
|
+
limit=5,
|
|
17
|
+
score_threshold=0.35
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# Create tool
|
|
21
|
+
search_tool = DXMTVectorSearchTool(dxmt_config=config)
|
|
22
|
+
|
|
23
|
+
# Create agent with tool
|
|
24
|
+
agent = Agent(
|
|
25
|
+
role="Research Assistant",
|
|
26
|
+
goal="Find relevant information",
|
|
27
|
+
tools=[search_tool],
|
|
28
|
+
verbose=True
|
|
29
|
+
)
|
|
30
|
+
```
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import json
|
|
36
|
+
import logging
|
|
37
|
+
import os
|
|
38
|
+
from typing import Any, Callable, Optional
|
|
39
|
+
|
|
40
|
+
from pydantic import BaseModel, ConfigDict, Field, SecretStr
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
from crewai.tools import BaseTool, EnvVar
|
|
44
|
+
CREWAI_AVAILABLE = True
|
|
45
|
+
except ImportError:
|
|
46
|
+
# CrewAI not installed, define dummy classes
|
|
47
|
+
CREWAI_AVAILABLE = False
|
|
48
|
+
|
|
49
|
+
class BaseTool: # type: ignore
|
|
50
|
+
"""Dummy BaseTool when CrewAI is not installed."""
|
|
51
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
52
|
+
"""Initialize dummy BaseTool."""
|
|
53
|
+
for key, value in kwargs.items():
|
|
54
|
+
setattr(self, key, value)
|
|
55
|
+
|
|
56
|
+
class EnvVar: # type: ignore
|
|
57
|
+
"""Dummy EnvVar when CrewAI is not installed."""
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
from .client import DXMTClient
|
|
61
|
+
from .exceptions import DXMTError
|
|
62
|
+
|
|
63
|
+
logger = logging.getLogger(__name__)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class DXMTToolSchema(BaseModel):
|
|
67
|
+
"""Input schema for DXMT Vector Search Tool."""
|
|
68
|
+
|
|
69
|
+
query: str = Field(
|
|
70
|
+
...,
|
|
71
|
+
description="Query to search in DXMT VectorDB - always required."
|
|
72
|
+
)
|
|
73
|
+
filter_by: str | None = Field(
|
|
74
|
+
default=None,
|
|
75
|
+
description="Metadata field to filter by. When filtering, use with filter_value."
|
|
76
|
+
)
|
|
77
|
+
filter_value: Any | None = Field(
|
|
78
|
+
default=None,
|
|
79
|
+
description="Value to filter by. When filtering, use with filter_by."
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class DXMTToolConfig(BaseModel):
|
|
84
|
+
"""Configuration for DXMT Vector Search Tool."""
|
|
85
|
+
|
|
86
|
+
host: str = Field(
|
|
87
|
+
default="localhost",
|
|
88
|
+
description="DXMT server hostname"
|
|
89
|
+
)
|
|
90
|
+
port: int = Field(
|
|
91
|
+
default=9999,
|
|
92
|
+
description="DXMT server port"
|
|
93
|
+
)
|
|
94
|
+
api_key: SecretStr | None = Field(
|
|
95
|
+
default=None,
|
|
96
|
+
description="The server's API key, if it was started with DXMT_API_KEY",
|
|
97
|
+
# A credential: hidden in repr (SecretStr) and left out of model_dump(), so it cannot
|
|
98
|
+
# travel wherever this config is logged or serialised.
|
|
99
|
+
exclude=True,
|
|
100
|
+
)
|
|
101
|
+
table_name: str = Field(
|
|
102
|
+
...,
|
|
103
|
+
description="Name of the DXMT table to search"
|
|
104
|
+
)
|
|
105
|
+
limit: int = Field(
|
|
106
|
+
default=5,
|
|
107
|
+
description="Maximum number of results to return"
|
|
108
|
+
)
|
|
109
|
+
score_threshold: float = Field(
|
|
110
|
+
default=0.35,
|
|
111
|
+
description="Minimum similarity score threshold (0-1)"
|
|
112
|
+
)
|
|
113
|
+
filter: dict[str, Any] | None = Field(
|
|
114
|
+
default=None,
|
|
115
|
+
description="DXMT Filter for advanced filtering"
|
|
116
|
+
)
|
|
117
|
+
distance_metric: str = Field(
|
|
118
|
+
default="cosine",
|
|
119
|
+
description="Distance metric (cosine, euclidean, dot)"
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class DXMTVectorSearchTool(BaseTool):
|
|
124
|
+
"""Vector search tool for DXMT VectorDB.
|
|
125
|
+
|
|
126
|
+
This tool enables CrewAI agents to search DXMT VectorDB for relevant
|
|
127
|
+
documents using vector similarity search. It supports filtering,
|
|
128
|
+
custom embedding functions, and configurable search parameters.
|
|
129
|
+
|
|
130
|
+
Features:
|
|
131
|
+
- Vector similarity search with cosine/euclidean/dot metrics
|
|
132
|
+
- Metadata filtering (simple and advanced)
|
|
133
|
+
- Custom embedding function support
|
|
134
|
+
- Score threshold filtering
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
dxmt_config: Configuration for DXMT connection and search
|
|
138
|
+
custom_embedding_fn: Optional custom embedding function
|
|
139
|
+
client: Optional pre-initialized DXMT client
|
|
140
|
+
|
|
141
|
+
Example:
|
|
142
|
+
```python
|
|
143
|
+
from dxmt.crewai_tool import DXMTVectorSearchTool, DXMTToolConfig
|
|
144
|
+
|
|
145
|
+
config = DXMTToolConfig(
|
|
146
|
+
host="localhost",
|
|
147
|
+
port=9999,
|
|
148
|
+
table_name="documents",
|
|
149
|
+
limit=5
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
tool = DXMTVectorSearchTool(dxmt_config=config)
|
|
153
|
+
|
|
154
|
+
# Use in agent
|
|
155
|
+
agent = Agent(
|
|
156
|
+
role="Researcher",
|
|
157
|
+
tools=[tool]
|
|
158
|
+
)
|
|
159
|
+
```
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
163
|
+
|
|
164
|
+
# --- Metadata ---
|
|
165
|
+
name: str = "DXMTVectorSearchTool"
|
|
166
|
+
description: str = (
|
|
167
|
+
"Search DXMT VectorDB for relevant documents using vector similarity search. "
|
|
168
|
+
"Supports filtering by metadata fields and returns top-k most similar results."
|
|
169
|
+
)
|
|
170
|
+
args_schema: type[BaseModel] = DXMTToolSchema
|
|
171
|
+
package_dependencies: list[str] = Field(
|
|
172
|
+
default_factory=lambda: ["dxmt-python-sdk"]
|
|
173
|
+
)
|
|
174
|
+
env_vars: list[EnvVar] = Field(
|
|
175
|
+
default_factory=lambda: [
|
|
176
|
+
EnvVar(
|
|
177
|
+
name="OPENAI_API_KEY",
|
|
178
|
+
description="API key for OpenAI embeddings",
|
|
179
|
+
required=True
|
|
180
|
+
)
|
|
181
|
+
]
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
# --- Configuration ---
|
|
185
|
+
dxmt_config: DXMTToolConfig
|
|
186
|
+
custom_embedding_fn: Callable[[str], list[float]] | None = Field(
|
|
187
|
+
default=None,
|
|
188
|
+
description="Optional custom embedding function (query_text -> vector)"
|
|
189
|
+
)
|
|
190
|
+
client: DXMTClient | None = Field(
|
|
191
|
+
default=None,
|
|
192
|
+
description="DXMT client instance (auto-created if None)"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def __init__(self, **kwargs):
|
|
196
|
+
"""Initialize DXMT Vector Search Tool."""
|
|
197
|
+
super().__init__(**kwargs)
|
|
198
|
+
|
|
199
|
+
if not CREWAI_AVAILABLE:
|
|
200
|
+
logger.warning(
|
|
201
|
+
"CrewAI is not installed. Install with: pip install crewai"
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
# Initialize DXMT client if not provided
|
|
205
|
+
if not self.client:
|
|
206
|
+
key = self.dxmt_config.api_key
|
|
207
|
+
self.client = DXMTClient(
|
|
208
|
+
host=self.dxmt_config.host,
|
|
209
|
+
port=self.dxmt_config.port,
|
|
210
|
+
api_key=key.get_secret_value() if key else None,
|
|
211
|
+
)
|
|
212
|
+
logger.info(
|
|
213
|
+
f"Initialized DXMT client: {self.dxmt_config.host}:{self.dxmt_config.port}"
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
def _run(
|
|
217
|
+
self,
|
|
218
|
+
query: str,
|
|
219
|
+
filter_by: str | None = None,
|
|
220
|
+
filter_value: Any | None = None,
|
|
221
|
+
) -> str:
|
|
222
|
+
"""Perform vector similarity search.
|
|
223
|
+
|
|
224
|
+
Args:
|
|
225
|
+
query: Search query text
|
|
226
|
+
filter_by: Optional metadata field to filter by
|
|
227
|
+
filter_value: Optional value for filter_by field
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
JSON string containing search results with score, metadata, and context
|
|
231
|
+
|
|
232
|
+
Raises:
|
|
233
|
+
DXMTError: If search fails
|
|
234
|
+
"""
|
|
235
|
+
try:
|
|
236
|
+
# Build filter if provided
|
|
237
|
+
search_filter = None
|
|
238
|
+
if filter_by and filter_value is not None:
|
|
239
|
+
# Simple equality filter
|
|
240
|
+
search_filter = {
|
|
241
|
+
"must": [{
|
|
242
|
+
"key": filter_by,
|
|
243
|
+
"operation": {"type": "Equal", "value": filter_value}
|
|
244
|
+
}]
|
|
245
|
+
}
|
|
246
|
+
logger.debug(f"Applied filter: {filter_by}={filter_value}")
|
|
247
|
+
elif self.dxmt_config.filter:
|
|
248
|
+
# Use pre-configured filter
|
|
249
|
+
search_filter = self.dxmt_config.filter
|
|
250
|
+
|
|
251
|
+
# Get query embedding
|
|
252
|
+
if self.custom_embedding_fn:
|
|
253
|
+
query_vector = self.custom_embedding_fn(query)
|
|
254
|
+
logger.debug("Used custom embedding function")
|
|
255
|
+
else:
|
|
256
|
+
# Use OpenAI embeddings
|
|
257
|
+
query_vector = self._get_openai_embedding(query)
|
|
258
|
+
logger.debug("Used OpenAI embedding")
|
|
259
|
+
|
|
260
|
+
# Perform search
|
|
261
|
+
results = self.client.search(
|
|
262
|
+
table_name=self.dxmt_config.table_name,
|
|
263
|
+
vector=query_vector,
|
|
264
|
+
k=self.dxmt_config.limit,
|
|
265
|
+
filter=search_filter
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
# Filter by score threshold
|
|
269
|
+
filtered_results = [
|
|
270
|
+
r for r in results
|
|
271
|
+
if r.get("score", 0.0) >= self.dxmt_config.score_threshold
|
|
272
|
+
]
|
|
273
|
+
|
|
274
|
+
logger.info(
|
|
275
|
+
f"Search returned {len(filtered_results)}/{len(results)} results "
|
|
276
|
+
f"above threshold {self.dxmt_config.score_threshold}"
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
# Format results for CrewAI
|
|
280
|
+
formatted_results = []
|
|
281
|
+
for result in filtered_results:
|
|
282
|
+
formatted_results.append({
|
|
283
|
+
"distance": 1.0 - result.get("score", 0.0), # Convert similarity to distance
|
|
284
|
+
"score": result.get("score", 0.0),
|
|
285
|
+
"metadata": result.get("metadata", {}),
|
|
286
|
+
"context": result.get("metadata", {}).get("text", "") or result.get("text", ""),
|
|
287
|
+
"id": result.get("id", "")
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
return json.dumps(formatted_results, indent=2)
|
|
291
|
+
|
|
292
|
+
except Exception as e:
|
|
293
|
+
logger.error(f"DXMT search error: {e}")
|
|
294
|
+
raise DXMTError(f"Vector search failed: {e}")
|
|
295
|
+
|
|
296
|
+
def _get_openai_embedding(self, text: str) -> list[float]:
|
|
297
|
+
"""Get OpenAI embedding for text.
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
text: Text to embed
|
|
301
|
+
|
|
302
|
+
Returns:
|
|
303
|
+
Embedding vector
|
|
304
|
+
|
|
305
|
+
Raises:
|
|
306
|
+
ImportError: If openai package is not installed
|
|
307
|
+
ValueError: If OPENAI_API_KEY is not set
|
|
308
|
+
"""
|
|
309
|
+
try:
|
|
310
|
+
import openai
|
|
311
|
+
except ImportError:
|
|
312
|
+
raise ImportError(
|
|
313
|
+
"OpenAI package required for default embeddings. "
|
|
314
|
+
"Install with: pip install openai"
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
api_key = os.getenv("OPENAI_API_KEY")
|
|
318
|
+
if not api_key:
|
|
319
|
+
raise ValueError(
|
|
320
|
+
"OPENAI_API_KEY environment variable is required when using "
|
|
321
|
+
"default embedding function"
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
client = openai.Client(api_key=api_key)
|
|
325
|
+
response = client.embeddings.create(
|
|
326
|
+
input=[text],
|
|
327
|
+
model="text-embedding-3-large"
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
return response.data[0].embedding
|
|
331
|
+
|
|
332
|
+
def add_documents(
|
|
333
|
+
self,
|
|
334
|
+
texts: list[str],
|
|
335
|
+
metadatas: list[dict[str, Any]] | None = None,
|
|
336
|
+
ids: list[str] | None = None
|
|
337
|
+
) -> list[str]:
|
|
338
|
+
"""Add documents to DXMT table.
|
|
339
|
+
|
|
340
|
+
This is a utility method for adding documents with automatic embedding.
|
|
341
|
+
Useful for preparing the vector database before using the search tool.
|
|
342
|
+
|
|
343
|
+
Args:
|
|
344
|
+
texts: List of document texts to add
|
|
345
|
+
metadatas: Optional list of metadata dicts
|
|
346
|
+
ids: Optional list of document IDs
|
|
347
|
+
|
|
348
|
+
Returns:
|
|
349
|
+
List of inserted document IDs
|
|
350
|
+
|
|
351
|
+
Example:
|
|
352
|
+
```python
|
|
353
|
+
tool = DXMTVectorSearchTool(dxmt_config=config)
|
|
354
|
+
tool.add_documents(
|
|
355
|
+
texts=["Document 1 content", "Document 2 content"],
|
|
356
|
+
metadatas=[{"source": "wiki"}, {"source": "docs"}]
|
|
357
|
+
)
|
|
358
|
+
```
|
|
359
|
+
"""
|
|
360
|
+
import uuid
|
|
361
|
+
|
|
362
|
+
if metadatas is None:
|
|
363
|
+
metadatas = [{}] * len(texts)
|
|
364
|
+
|
|
365
|
+
if ids is None:
|
|
366
|
+
ids = [str(uuid.uuid4()) for _ in range(len(texts))]
|
|
367
|
+
|
|
368
|
+
if len(texts) != len(metadatas) != len(ids):
|
|
369
|
+
raise ValueError("texts, metadatas, and ids must have the same length")
|
|
370
|
+
|
|
371
|
+
# Get embeddings
|
|
372
|
+
if self.custom_embedding_fn:
|
|
373
|
+
vectors = [self.custom_embedding_fn(text) for text in texts]
|
|
374
|
+
else:
|
|
375
|
+
# Batch embed with OpenAI
|
|
376
|
+
vectors = self._batch_embed_openai(texts)
|
|
377
|
+
|
|
378
|
+
# Prepare records
|
|
379
|
+
records = []
|
|
380
|
+
for i, (text, metadata, doc_id) in enumerate(zip(texts, metadatas, ids)):
|
|
381
|
+
# Add text to metadata for retrieval
|
|
382
|
+
metadata_with_text = {**metadata, "text": text}
|
|
383
|
+
|
|
384
|
+
records.append({
|
|
385
|
+
"id": doc_id,
|
|
386
|
+
"vector": vectors[i],
|
|
387
|
+
"metadata": metadata_with_text
|
|
388
|
+
})
|
|
389
|
+
|
|
390
|
+
# Insert into DXMT
|
|
391
|
+
self.client.insert_records(
|
|
392
|
+
table_name=self.dxmt_config.table_name,
|
|
393
|
+
records=records
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
logger.info(f"Added {len(records)} documents to {self.dxmt_config.table_name}")
|
|
397
|
+
return ids
|
|
398
|
+
|
|
399
|
+
def _batch_embed_openai(self, texts: list[str]) -> list[list[float]]:
|
|
400
|
+
"""Batch embed texts using OpenAI.
|
|
401
|
+
|
|
402
|
+
Args:
|
|
403
|
+
texts: List of texts to embed
|
|
404
|
+
|
|
405
|
+
Returns:
|
|
406
|
+
List of embedding vectors
|
|
407
|
+
"""
|
|
408
|
+
try:
|
|
409
|
+
import openai
|
|
410
|
+
except ImportError:
|
|
411
|
+
raise ImportError("openai package required. Install with: pip install openai")
|
|
412
|
+
|
|
413
|
+
api_key = os.getenv("OPENAI_API_KEY")
|
|
414
|
+
if not api_key:
|
|
415
|
+
raise ValueError("OPENAI_API_KEY environment variable required")
|
|
416
|
+
|
|
417
|
+
client = openai.Client(api_key=api_key)
|
|
418
|
+
response = client.embeddings.create(
|
|
419
|
+
input=texts,
|
|
420
|
+
model="text-embedding-3-large"
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
return [data.embedding for data in response.data]
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
__all__ = [
|
|
427
|
+
"DXMTVectorSearchTool",
|
|
428
|
+
"DXMTToolConfig",
|
|
429
|
+
"DXMTToolSchema",
|
|
430
|
+
]
|
dxmt/exceptions.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""DXMT Python SDK Exceptions.
|
|
2
|
+
|
|
3
|
+
This module defines all custom exceptions used by the DXMT Python SDK.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DXMTError(Exception):
|
|
8
|
+
"""Base exception for all DXMT errors."""
|
|
9
|
+
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DXMTConnectionError(DXMTError):
|
|
14
|
+
"""Raised when connection to DXMT server fails."""
|
|
15
|
+
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DXMTTableError(DXMTError):
|
|
20
|
+
"""Raised when a table operation fails."""
|
|
21
|
+
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DXMTRecordError(DXMTError):
|
|
26
|
+
"""Raised when a record operation fails."""
|
|
27
|
+
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DXMTSearchError(DXMTError):
|
|
32
|
+
"""Raised when a search operation fails."""
|
|
33
|
+
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class DXMTValidationError(DXMTError):
|
|
38
|
+
"""Raised when input validation fails."""
|
|
39
|
+
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class DXMTTimeoutError(DXMTError):
|
|
44
|
+
"""Raised when a request times out."""
|
|
45
|
+
|
|
46
|
+
pass
|