dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dxmt/crewai_tool.py ADDED
@@ -0,0 +1,430 @@
1
+ """DXMT VectorDB Tool for CrewAI.
2
+
3
+ This module provides a CrewAI BaseTool implementation for DXMT VectorDB,
4
+ enabling vector similarity search within CrewAI agents and workflows.
5
+
6
+ Example:
7
+ ```python
8
+ from crewai import Agent, Task, Crew
9
+ from dxmt.crewai_tool import DXMTVectorSearchTool, DXMTToolConfig
10
+
11
+ # Configure DXMT tool
12
+ config = DXMTToolConfig(
13
+ host="localhost",
14
+ port=9999,
15
+ table_name="documents",
16
+ limit=5,
17
+ score_threshold=0.35
18
+ )
19
+
20
+ # Create tool
21
+ search_tool = DXMTVectorSearchTool(dxmt_config=config)
22
+
23
+ # Create agent with tool
24
+ agent = Agent(
25
+ role="Research Assistant",
26
+ goal="Find relevant information",
27
+ tools=[search_tool],
28
+ verbose=True
29
+ )
30
+ ```
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import json
36
+ import logging
37
+ import os
38
+ from typing import Any, Callable, Optional
39
+
40
+ from pydantic import BaseModel, ConfigDict, Field, SecretStr
41
+
42
+ try:
43
+ from crewai.tools import BaseTool, EnvVar
44
+ CREWAI_AVAILABLE = True
45
+ except ImportError:
46
+ # CrewAI not installed, define dummy classes
47
+ CREWAI_AVAILABLE = False
48
+
49
+ class BaseTool: # type: ignore
50
+ """Dummy BaseTool when CrewAI is not installed."""
51
+ def __init__(self, **kwargs: Any) -> None:
52
+ """Initialize dummy BaseTool."""
53
+ for key, value in kwargs.items():
54
+ setattr(self, key, value)
55
+
56
+ class EnvVar: # type: ignore
57
+ """Dummy EnvVar when CrewAI is not installed."""
58
+ pass
59
+
60
+ from .client import DXMTClient
61
+ from .exceptions import DXMTError
62
+
63
+ logger = logging.getLogger(__name__)
64
+
65
+
66
+ class DXMTToolSchema(BaseModel):
67
+ """Input schema for DXMT Vector Search Tool."""
68
+
69
+ query: str = Field(
70
+ ...,
71
+ description="Query to search in DXMT VectorDB - always required."
72
+ )
73
+ filter_by: str | None = Field(
74
+ default=None,
75
+ description="Metadata field to filter by. When filtering, use with filter_value."
76
+ )
77
+ filter_value: Any | None = Field(
78
+ default=None,
79
+ description="Value to filter by. When filtering, use with filter_by."
80
+ )
81
+
82
+
83
+ class DXMTToolConfig(BaseModel):
84
+ """Configuration for DXMT Vector Search Tool."""
85
+
86
+ host: str = Field(
87
+ default="localhost",
88
+ description="DXMT server hostname"
89
+ )
90
+ port: int = Field(
91
+ default=9999,
92
+ description="DXMT server port"
93
+ )
94
+ api_key: SecretStr | None = Field(
95
+ default=None,
96
+ description="The server's API key, if it was started with DXMT_API_KEY",
97
+ # A credential: hidden in repr (SecretStr) and left out of model_dump(), so it cannot
98
+ # travel wherever this config is logged or serialised.
99
+ exclude=True,
100
+ )
101
+ table_name: str = Field(
102
+ ...,
103
+ description="Name of the DXMT table to search"
104
+ )
105
+ limit: int = Field(
106
+ default=5,
107
+ description="Maximum number of results to return"
108
+ )
109
+ score_threshold: float = Field(
110
+ default=0.35,
111
+ description="Minimum similarity score threshold (0-1)"
112
+ )
113
+ filter: dict[str, Any] | None = Field(
114
+ default=None,
115
+ description="DXMT Filter for advanced filtering"
116
+ )
117
+ distance_metric: str = Field(
118
+ default="cosine",
119
+ description="Distance metric (cosine, euclidean, dot)"
120
+ )
121
+
122
+
123
+ class DXMTVectorSearchTool(BaseTool):
124
+ """Vector search tool for DXMT VectorDB.
125
+
126
+ This tool enables CrewAI agents to search DXMT VectorDB for relevant
127
+ documents using vector similarity search. It supports filtering,
128
+ custom embedding functions, and configurable search parameters.
129
+
130
+ Features:
131
+ - Vector similarity search with cosine/euclidean/dot metrics
132
+ - Metadata filtering (simple and advanced)
133
+ - Custom embedding function support
134
+ - Score threshold filtering
135
+
136
+ Args:
137
+ dxmt_config: Configuration for DXMT connection and search
138
+ custom_embedding_fn: Optional custom embedding function
139
+ client: Optional pre-initialized DXMT client
140
+
141
+ Example:
142
+ ```python
143
+ from dxmt.crewai_tool import DXMTVectorSearchTool, DXMTToolConfig
144
+
145
+ config = DXMTToolConfig(
146
+ host="localhost",
147
+ port=9999,
148
+ table_name="documents",
149
+ limit=5
150
+ )
151
+
152
+ tool = DXMTVectorSearchTool(dxmt_config=config)
153
+
154
+ # Use in agent
155
+ agent = Agent(
156
+ role="Researcher",
157
+ tools=[tool]
158
+ )
159
+ ```
160
+ """
161
+
162
+ model_config = ConfigDict(arbitrary_types_allowed=True)
163
+
164
+ # --- Metadata ---
165
+ name: str = "DXMTVectorSearchTool"
166
+ description: str = (
167
+ "Search DXMT VectorDB for relevant documents using vector similarity search. "
168
+ "Supports filtering by metadata fields and returns top-k most similar results."
169
+ )
170
+ args_schema: type[BaseModel] = DXMTToolSchema
171
+ package_dependencies: list[str] = Field(
172
+ default_factory=lambda: ["dxmt-python-sdk"]
173
+ )
174
+ env_vars: list[EnvVar] = Field(
175
+ default_factory=lambda: [
176
+ EnvVar(
177
+ name="OPENAI_API_KEY",
178
+ description="API key for OpenAI embeddings",
179
+ required=True
180
+ )
181
+ ]
182
+ )
183
+
184
+ # --- Configuration ---
185
+ dxmt_config: DXMTToolConfig
186
+ custom_embedding_fn: Callable[[str], list[float]] | None = Field(
187
+ default=None,
188
+ description="Optional custom embedding function (query_text -> vector)"
189
+ )
190
+ client: DXMTClient | None = Field(
191
+ default=None,
192
+ description="DXMT client instance (auto-created if None)"
193
+ )
194
+
195
+ def __init__(self, **kwargs):
196
+ """Initialize DXMT Vector Search Tool."""
197
+ super().__init__(**kwargs)
198
+
199
+ if not CREWAI_AVAILABLE:
200
+ logger.warning(
201
+ "CrewAI is not installed. Install with: pip install crewai"
202
+ )
203
+
204
+ # Initialize DXMT client if not provided
205
+ if not self.client:
206
+ key = self.dxmt_config.api_key
207
+ self.client = DXMTClient(
208
+ host=self.dxmt_config.host,
209
+ port=self.dxmt_config.port,
210
+ api_key=key.get_secret_value() if key else None,
211
+ )
212
+ logger.info(
213
+ f"Initialized DXMT client: {self.dxmt_config.host}:{self.dxmt_config.port}"
214
+ )
215
+
216
+ def _run(
217
+ self,
218
+ query: str,
219
+ filter_by: str | None = None,
220
+ filter_value: Any | None = None,
221
+ ) -> str:
222
+ """Perform vector similarity search.
223
+
224
+ Args:
225
+ query: Search query text
226
+ filter_by: Optional metadata field to filter by
227
+ filter_value: Optional value for filter_by field
228
+
229
+ Returns:
230
+ JSON string containing search results with score, metadata, and context
231
+
232
+ Raises:
233
+ DXMTError: If search fails
234
+ """
235
+ try:
236
+ # Build filter if provided
237
+ search_filter = None
238
+ if filter_by and filter_value is not None:
239
+ # Simple equality filter
240
+ search_filter = {
241
+ "must": [{
242
+ "key": filter_by,
243
+ "operation": {"type": "Equal", "value": filter_value}
244
+ }]
245
+ }
246
+ logger.debug(f"Applied filter: {filter_by}={filter_value}")
247
+ elif self.dxmt_config.filter:
248
+ # Use pre-configured filter
249
+ search_filter = self.dxmt_config.filter
250
+
251
+ # Get query embedding
252
+ if self.custom_embedding_fn:
253
+ query_vector = self.custom_embedding_fn(query)
254
+ logger.debug("Used custom embedding function")
255
+ else:
256
+ # Use OpenAI embeddings
257
+ query_vector = self._get_openai_embedding(query)
258
+ logger.debug("Used OpenAI embedding")
259
+
260
+ # Perform search
261
+ results = self.client.search(
262
+ table_name=self.dxmt_config.table_name,
263
+ vector=query_vector,
264
+ k=self.dxmt_config.limit,
265
+ filter=search_filter
266
+ )
267
+
268
+ # Filter by score threshold
269
+ filtered_results = [
270
+ r for r in results
271
+ if r.get("score", 0.0) >= self.dxmt_config.score_threshold
272
+ ]
273
+
274
+ logger.info(
275
+ f"Search returned {len(filtered_results)}/{len(results)} results "
276
+ f"above threshold {self.dxmt_config.score_threshold}"
277
+ )
278
+
279
+ # Format results for CrewAI
280
+ formatted_results = []
281
+ for result in filtered_results:
282
+ formatted_results.append({
283
+ "distance": 1.0 - result.get("score", 0.0), # Convert similarity to distance
284
+ "score": result.get("score", 0.0),
285
+ "metadata": result.get("metadata", {}),
286
+ "context": result.get("metadata", {}).get("text", "") or result.get("text", ""),
287
+ "id": result.get("id", "")
288
+ })
289
+
290
+ return json.dumps(formatted_results, indent=2)
291
+
292
+ except Exception as e:
293
+ logger.error(f"DXMT search error: {e}")
294
+ raise DXMTError(f"Vector search failed: {e}")
295
+
296
+ def _get_openai_embedding(self, text: str) -> list[float]:
297
+ """Get OpenAI embedding for text.
298
+
299
+ Args:
300
+ text: Text to embed
301
+
302
+ Returns:
303
+ Embedding vector
304
+
305
+ Raises:
306
+ ImportError: If openai package is not installed
307
+ ValueError: If OPENAI_API_KEY is not set
308
+ """
309
+ try:
310
+ import openai
311
+ except ImportError:
312
+ raise ImportError(
313
+ "OpenAI package required for default embeddings. "
314
+ "Install with: pip install openai"
315
+ )
316
+
317
+ api_key = os.getenv("OPENAI_API_KEY")
318
+ if not api_key:
319
+ raise ValueError(
320
+ "OPENAI_API_KEY environment variable is required when using "
321
+ "default embedding function"
322
+ )
323
+
324
+ client = openai.Client(api_key=api_key)
325
+ response = client.embeddings.create(
326
+ input=[text],
327
+ model="text-embedding-3-large"
328
+ )
329
+
330
+ return response.data[0].embedding
331
+
332
+ def add_documents(
333
+ self,
334
+ texts: list[str],
335
+ metadatas: list[dict[str, Any]] | None = None,
336
+ ids: list[str] | None = None
337
+ ) -> list[str]:
338
+ """Add documents to DXMT table.
339
+
340
+ This is a utility method for adding documents with automatic embedding.
341
+ Useful for preparing the vector database before using the search tool.
342
+
343
+ Args:
344
+ texts: List of document texts to add
345
+ metadatas: Optional list of metadata dicts
346
+ ids: Optional list of document IDs
347
+
348
+ Returns:
349
+ List of inserted document IDs
350
+
351
+ Example:
352
+ ```python
353
+ tool = DXMTVectorSearchTool(dxmt_config=config)
354
+ tool.add_documents(
355
+ texts=["Document 1 content", "Document 2 content"],
356
+ metadatas=[{"source": "wiki"}, {"source": "docs"}]
357
+ )
358
+ ```
359
+ """
360
+ import uuid
361
+
362
+ if metadatas is None:
363
+ metadatas = [{}] * len(texts)
364
+
365
+ if ids is None:
366
+ ids = [str(uuid.uuid4()) for _ in range(len(texts))]
367
+
368
+ if len(texts) != len(metadatas) != len(ids):
369
+ raise ValueError("texts, metadatas, and ids must have the same length")
370
+
371
+ # Get embeddings
372
+ if self.custom_embedding_fn:
373
+ vectors = [self.custom_embedding_fn(text) for text in texts]
374
+ else:
375
+ # Batch embed with OpenAI
376
+ vectors = self._batch_embed_openai(texts)
377
+
378
+ # Prepare records
379
+ records = []
380
+ for i, (text, metadata, doc_id) in enumerate(zip(texts, metadatas, ids)):
381
+ # Add text to metadata for retrieval
382
+ metadata_with_text = {**metadata, "text": text}
383
+
384
+ records.append({
385
+ "id": doc_id,
386
+ "vector": vectors[i],
387
+ "metadata": metadata_with_text
388
+ })
389
+
390
+ # Insert into DXMT
391
+ self.client.insert_records(
392
+ table_name=self.dxmt_config.table_name,
393
+ records=records
394
+ )
395
+
396
+ logger.info(f"Added {len(records)} documents to {self.dxmt_config.table_name}")
397
+ return ids
398
+
399
+ def _batch_embed_openai(self, texts: list[str]) -> list[list[float]]:
400
+ """Batch embed texts using OpenAI.
401
+
402
+ Args:
403
+ texts: List of texts to embed
404
+
405
+ Returns:
406
+ List of embedding vectors
407
+ """
408
+ try:
409
+ import openai
410
+ except ImportError:
411
+ raise ImportError("openai package required. Install with: pip install openai")
412
+
413
+ api_key = os.getenv("OPENAI_API_KEY")
414
+ if not api_key:
415
+ raise ValueError("OPENAI_API_KEY environment variable required")
416
+
417
+ client = openai.Client(api_key=api_key)
418
+ response = client.embeddings.create(
419
+ input=texts,
420
+ model="text-embedding-3-large"
421
+ )
422
+
423
+ return [data.embedding for data in response.data]
424
+
425
+
426
+ __all__ = [
427
+ "DXMTVectorSearchTool",
428
+ "DXMTToolConfig",
429
+ "DXMTToolSchema",
430
+ ]
dxmt/exceptions.py ADDED
@@ -0,0 +1,46 @@
1
+ """DXMT Python SDK Exceptions.
2
+
3
+ This module defines all custom exceptions used by the DXMT Python SDK.
4
+ """
5
+
6
+
7
+ class DXMTError(Exception):
8
+ """Base exception for all DXMT errors."""
9
+
10
+ pass
11
+
12
+
13
+ class DXMTConnectionError(DXMTError):
14
+ """Raised when connection to DXMT server fails."""
15
+
16
+ pass
17
+
18
+
19
+ class DXMTTableError(DXMTError):
20
+ """Raised when a table operation fails."""
21
+
22
+ pass
23
+
24
+
25
+ class DXMTRecordError(DXMTError):
26
+ """Raised when a record operation fails."""
27
+
28
+ pass
29
+
30
+
31
+ class DXMTSearchError(DXMTError):
32
+ """Raised when a search operation fails."""
33
+
34
+ pass
35
+
36
+
37
+ class DXMTValidationError(DXMTError):
38
+ """Raised when input validation fails."""
39
+
40
+ pass
41
+
42
+
43
+ class DXMTTimeoutError(DXMTError):
44
+ """Raised when a request times out."""
45
+
46
+ pass