dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dxmt/crewai_storage.py ADDED
@@ -0,0 +1,524 @@
1
+ """DXMT VectorDB Storage for CrewAI Memory.
2
+
3
+ This module provides a CrewAI Storage implementation backed by DXMT VectorDB,
4
+ enabling persistent memory storage for CrewAI agents with cost-efficient
5
+ tiered storage (Memory → SSD → S3).
6
+
7
+ Example:
8
+ ```python
9
+ from crewai import Agent, Crew
10
+ from dxmt.crewai_storage import DXMTStorage
11
+
12
+ # Configure DXMT storage for long-term memory
13
+ storage_config = {
14
+ "host": "localhost",
15
+ "port": 9999,
16
+ "table_name": "crew_memory",
17
+ "user_id": "user_123",
18
+ "agent_id": "research_agent"
19
+ }
20
+
21
+ # Create storage
22
+ storage = DXMTStorage(
23
+ type="long_term",
24
+ config=storage_config
25
+ )
26
+
27
+ # Use with Crew
28
+ crew = Crew(
29
+ agents=[agent],
30
+ memory=True,
31
+ long_term_memory=storage
32
+ )
33
+ ```
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import logging
39
+ import os
40
+ import re
41
+ from collections import defaultdict
42
+ from typing import Any, Optional
43
+
44
+ from pydantic import BaseModel, Field, SecretStr
45
+
46
+ try:
47
+ from crewai.memory.storage.interface import Storage
48
+ CREWAI_STORAGE_AVAILABLE = True
49
+ except ImportError:
50
+ # CrewAI not installed, define dummy base class
51
+ CREWAI_STORAGE_AVAILABLE = False
52
+
53
+ class Storage: # type: ignore
54
+ """Dummy Storage when CrewAI is not installed."""
55
+ pass
56
+
57
+ from .client import DXMTClient
58
+ from .exceptions import DXMTError
59
+
60
+ logger = logging.getLogger(__name__)
61
+
62
+
63
+ class DXMTStorageConfig(BaseModel):
64
+ """Configuration for DXMT Storage."""
65
+
66
+ host: str = Field(
67
+ default="localhost",
68
+ description="DXMT server hostname"
69
+ )
70
+ port: int = Field(
71
+ default=9999,
72
+ description="DXMT server port"
73
+ )
74
+ api_key: SecretStr | None = Field(
75
+ default=None,
76
+ description="The server's API key, if it was started with DXMT_API_KEY",
77
+ # A credential: hidden in repr (SecretStr) and left out of model_dump(), so it cannot
78
+ # travel wherever this config is logged or serialised.
79
+ exclude=True,
80
+ )
81
+ table_name: str = Field(
82
+ default="crew_memory",
83
+ description="Name of the DXMT table for memory storage"
84
+ )
85
+ user_id: str | None = Field(
86
+ default=None,
87
+ description="User ID for memory scoping"
88
+ )
89
+ agent_id: str | None = Field(
90
+ default=None,
91
+ description="Agent ID for memory scoping"
92
+ )
93
+ run_id: str | None = Field(
94
+ default=None,
95
+ description="Run ID for short-term memory scoping"
96
+ )
97
+ distance_metric: str = Field(
98
+ default="cosine",
99
+ description="Distance metric (cosine, euclidean, dot)"
100
+ )
101
+ dimension: int = Field(
102
+ default=3072,
103
+ description="Vector dimension (3072 for text-embedding-3-large)"
104
+ )
105
+
106
+
107
+ class DXMTStorage(Storage):
108
+ """CrewAI Storage implementation using DXMT VectorDB.
109
+
110
+ Implements the CrewAI Storage interface with DXMT backend, providing
111
+ persistent memory storage on a single-node vector database.
112
+
113
+ Features:
114
+ - Multi-scope memory (user_id, agent_id, run_id)
115
+ - High-performance vector search (<100ms P99)
116
+ - Support for all memory types (short_term, long_term, entities, external)
117
+ - Automatic embedding generation
118
+
119
+ Args:
120
+ type: Memory type - "short_term", "long_term", "entities", or "external"
121
+ crew: Optional Crew instance for agent name extraction
122
+ config: Configuration dict with DXMT connection parameters
123
+
124
+ Example:
125
+ ```python
126
+ storage = DXMTStorage(
127
+ type="long_term",
128
+ config={
129
+ "host": "localhost",
130
+ "port": 9999,
131
+ "table_name": "crew_memory",
132
+ "user_id": "user_123"
133
+ }
134
+ )
135
+
136
+ # Save memory
137
+ storage.save(
138
+ "User prefers Python over Java",
139
+ {"source": "conversation", "timestamp": "2024-01-28"}
140
+ )
141
+
142
+ # Search memory
143
+ results = storage.search("programming preferences", limit=5)
144
+ ```
145
+ """
146
+
147
+ def __init__(
148
+ self,
149
+ type: str,
150
+ crew=None,
151
+ config: dict[str, Any] | None = None
152
+ ):
153
+ """Initialize DXMT Storage."""
154
+ super().__init__()
155
+
156
+ if not CREWAI_STORAGE_AVAILABLE:
157
+ logger.warning(
158
+ "CrewAI storage interface not installed. "
159
+ "Install with: pip install crewai"
160
+ )
161
+
162
+ self._validate_type(type)
163
+ self.memory_type = type
164
+ self.crew = crew
165
+ self.config = config or {}
166
+
167
+ # Parse configuration
168
+ self._parse_config()
169
+
170
+ # Initialize DXMT client
171
+ self._initialize_client()
172
+
173
+ logger.info(
174
+ f"Initialized DXMTStorage: type={type}, "
175
+ f"table={self.storage_config.table_name}"
176
+ )
177
+
178
+ def _validate_type(self, type: str):
179
+ """Validate memory type."""
180
+ supported_types = {"short_term", "long_term", "entities", "external"}
181
+ if type not in supported_types:
182
+ raise ValueError(
183
+ f"Invalid type '{type}' for DXMTStorage. "
184
+ f"Must be one of: {', '.join(supported_types)}"
185
+ )
186
+
187
+ def _parse_config(self):
188
+ """Parse configuration from config dict."""
189
+ self.storage_config = DXMTStorageConfig(**self.config)
190
+
191
+ def _initialize_client(self):
192
+ """Initialize DXMT client and create table if needed."""
193
+ key = self.storage_config.api_key
194
+ self.client = DXMTClient(
195
+ host=self.storage_config.host,
196
+ port=self.storage_config.port,
197
+ api_key=key.get_secret_value() if key else None,
198
+ )
199
+
200
+ # Check if table exists, create if not
201
+ try:
202
+ self.client.get_table(self.storage_config.table_name)
203
+ logger.debug(f"Table {self.storage_config.table_name} already exists")
204
+ except DXMTError:
205
+ # Table doesn't exist, create it
206
+ logger.info(f"Creating table {self.storage_config.table_name}")
207
+ self.client.create_table(
208
+ name=self.storage_config.table_name,
209
+ dimension=self.storage_config.dimension,
210
+ distance_metric=self.storage_config.distance_metric,
211
+ description=f"CrewAI {self.memory_type} memory storage"
212
+ )
213
+
214
+ def save(self, value: Any, metadata: dict[str, Any]) -> None:
215
+ """Save memory to DXMT.
216
+
217
+ Args:
218
+ value: Memory content (string or conversation)
219
+ metadata: Additional metadata (messages, timestamp, etc.)
220
+
221
+ Example:
222
+ ```python
223
+ storage.save(
224
+ "User prefers dark mode",
225
+ {"source": "conversation", "confidence": 0.95}
226
+ )
227
+ ```
228
+ """
229
+ try:
230
+ # Extract content from value
231
+ if isinstance(value, str):
232
+ content = value
233
+ else:
234
+ content = str(value)
235
+
236
+ # Extract messages if present
237
+ messages = metadata.pop("messages", None)
238
+ if messages:
239
+ # Extract last user and assistant messages
240
+ content = self._extract_conversation_content(messages)
241
+
242
+ # Generate embedding
243
+ vector = self._get_embedding(content)
244
+
245
+ # Prepare metadata
246
+ memory_metadata = {
247
+ "type": self.memory_type,
248
+ "content": content,
249
+ **metadata
250
+ }
251
+
252
+ # Add scoping IDs
253
+ if self.storage_config.user_id:
254
+ memory_metadata["user_id"] = self.storage_config.user_id
255
+
256
+ if self.storage_config.agent_id:
257
+ memory_metadata["agent_id"] = self.storage_config.agent_id
258
+ elif self.crew:
259
+ memory_metadata["agent_id"] = self._get_agent_name()
260
+
261
+ if self.memory_type == "short_term" and self.storage_config.run_id:
262
+ memory_metadata["run_id"] = self.storage_config.run_id
263
+
264
+ # Insert into DXMT
265
+ import uuid
266
+ record = {
267
+ "id": str(uuid.uuid4()),
268
+ "vector": vector,
269
+ "metadata": memory_metadata
270
+ }
271
+
272
+ self.client.insert_records(
273
+ table_name=self.storage_config.table_name,
274
+ records=[record]
275
+ )
276
+
277
+ logger.debug(f"Saved memory: {content[:100]}...")
278
+
279
+ except Exception as e:
280
+ logger.error(f"Error saving memory: {e}")
281
+ raise DXMTError(f"Failed to save memory: {e}")
282
+
283
+ def search(
284
+ self,
285
+ query: str,
286
+ limit: int = 5,
287
+ score_threshold: float = 0.6
288
+ ) -> list[Any]:
289
+ """Search for relevant memories.
290
+
291
+ Args:
292
+ query: Search query text
293
+ limit: Maximum number of results
294
+ score_threshold: Minimum similarity score (0-1)
295
+
296
+ Returns:
297
+ List of memory results with content and metadata
298
+
299
+ Example:
300
+ ```python
301
+ results = storage.search(
302
+ "programming preferences",
303
+ limit=5,
304
+ score_threshold=0.7
305
+ )
306
+
307
+ for result in results:
308
+ print(result["content"])
309
+ print(result["score"])
310
+ ```
311
+ """
312
+ try:
313
+ # Generate query embedding
314
+ query_vector = self._get_embedding(query)
315
+
316
+ # Build filter
317
+ search_filter = self._build_search_filter()
318
+
319
+ # Search DXMT
320
+ results = self.client.search(
321
+ table_name=self.storage_config.table_name,
322
+ vector=query_vector,
323
+ k=limit,
324
+ filter=search_filter
325
+ )
326
+
327
+ # Filter by score threshold and format results
328
+ formatted_results = []
329
+ for result in results:
330
+ score = result.get("score", 0.0)
331
+ if score >= score_threshold:
332
+ metadata = result.get("metadata", {})
333
+ formatted_results.append({
334
+ "content": metadata.get("content", ""),
335
+ "memory": metadata.get("content", ""), # CrewAI compatibility
336
+ "score": score,
337
+ "metadata": metadata,
338
+ "id": result.get("id", "")
339
+ })
340
+
341
+ logger.debug(
342
+ f"Search returned {len(formatted_results)}/{len(results)} results "
343
+ f"above threshold {score_threshold}"
344
+ )
345
+
346
+ # Wrap in results dict for CrewAI compatibility
347
+ return {"results": formatted_results}
348
+
349
+ except Exception as e:
350
+ logger.error(f"Error searching memory: {e}")
351
+ return {"results": []}
352
+
353
+ def reset(self) -> None:
354
+ """Reset storage by clearing all memories.
355
+
356
+ Warning: This will delete all memories in the table!
357
+ """
358
+ try:
359
+ logger.warning(f"Resetting storage: {self.storage_config.table_name}")
360
+
361
+ # Delete and recreate table
362
+ self.client.delete_table(self.storage_config.table_name)
363
+ self.client.create_table(
364
+ name=self.storage_config.table_name,
365
+ dimension=self.storage_config.dimension,
366
+ distance_metric=self.storage_config.distance_metric,
367
+ description=f"CrewAI {self.memory_type} memory storage"
368
+ )
369
+
370
+ logger.info(f"Storage reset complete: {self.storage_config.table_name}")
371
+
372
+ except Exception as e:
373
+ logger.error(f"Error resetting storage: {e}")
374
+ raise DXMTError(f"Failed to reset storage: {e}")
375
+
376
+ # Helper methods
377
+
378
+ def _build_search_filter(self) -> dict[str, Any] | None:
379
+ """Build DXMT filter for search based on scoping."""
380
+ conditions = []
381
+
382
+ # Add memory type filter
383
+ conditions.append({
384
+ "key": "type",
385
+ "operation": {"type": "Equal", "value": self.memory_type}
386
+ })
387
+
388
+ # Add scoping filters
389
+ if self.memory_type == "short_term" and self.storage_config.run_id:
390
+ conditions.append({
391
+ "key": "run_id",
392
+ "operation": {"type": "Equal", "value": self.storage_config.run_id}
393
+ })
394
+ else:
395
+ if self.storage_config.user_id and self.storage_config.agent_id:
396
+ # User OR agent filter
397
+ return {
398
+ "should": [
399
+ {
400
+ "must": [
401
+ {
402
+ "key": "user_id",
403
+ "operation": {"type": "Equal", "value": self.storage_config.user_id}
404
+ }
405
+ ]
406
+ },
407
+ {
408
+ "must": [
409
+ {
410
+ "key": "agent_id",
411
+ "operation": {"type": "Equal", "value": self.storage_config.agent_id}
412
+ }
413
+ ]
414
+ }
415
+ ]
416
+ }
417
+ elif self.storage_config.user_id:
418
+ conditions.append({
419
+ "key": "user_id",
420
+ "operation": {"type": "Equal", "value": self.storage_config.user_id}
421
+ })
422
+ elif self.storage_config.agent_id:
423
+ conditions.append({
424
+ "key": "agent_id",
425
+ "operation": {"type": "Equal", "value": self.storage_config.agent_id}
426
+ })
427
+
428
+ return {"must": conditions} if conditions else None
429
+
430
+ def _get_embedding(self, text: str) -> list[float]:
431
+ """Get OpenAI embedding for text.
432
+
433
+ Args:
434
+ text: Text to embed
435
+
436
+ Returns:
437
+ Embedding vector
438
+ """
439
+ try:
440
+ import openai
441
+ except ImportError:
442
+ raise ImportError(
443
+ "openai package required for embeddings. "
444
+ "Install with: pip install openai"
445
+ )
446
+
447
+ api_key = os.getenv("OPENAI_API_KEY")
448
+ if not api_key:
449
+ raise ValueError("OPENAI_API_KEY environment variable required")
450
+
451
+ client = openai.Client(api_key=api_key)
452
+ response = client.embeddings.create(
453
+ input=[text],
454
+ model="text-embedding-3-large"
455
+ )
456
+
457
+ return response.data[0].embedding
458
+
459
+ def _extract_conversation_content(self, messages: list[dict]) -> str:
460
+ """Extract content from conversation messages.
461
+
462
+ Args:
463
+ messages: List of message dicts with role and content
464
+
465
+ Returns:
466
+ Combined content string
467
+ """
468
+ def _last_content(messages, role):
469
+ for msg in reversed(messages):
470
+ if msg.get("role") == role:
471
+ return msg.get("content", "")
472
+ return ""
473
+
474
+ last_user = _last_content(messages, "user")
475
+ last_assistant = _last_content(messages, "assistant")
476
+
477
+ # Extract user message
478
+ user_msg = self._extract_user_message(last_user)
479
+
480
+ # Extract assistant message
481
+ assistant_msg = self._extract_assistant_message(last_assistant)
482
+
483
+ # Combine
484
+ parts = []
485
+ if user_msg:
486
+ parts.append(f"User: {user_msg}")
487
+ if assistant_msg:
488
+ parts.append(f"Assistant: {assistant_msg}")
489
+
490
+ return " ".join(parts) if parts else last_user or last_assistant
491
+
492
+ def _extract_user_message(self, text: str) -> str:
493
+ """Extract user message from text."""
494
+ pattern = r"User message:\s*(.*)"
495
+ match = re.search(pattern, text)
496
+ if match:
497
+ return match.group(1).strip()
498
+ return text
499
+
500
+ def _extract_assistant_message(self, text: str) -> str:
501
+ """Extract assistant message from text."""
502
+ marker = "Final Answer:"
503
+ if marker in text:
504
+ return text.split(marker, 1)[1].strip()
505
+ return text
506
+
507
+ def _get_agent_name(self) -> str:
508
+ """Get sanitized agent name from crew."""
509
+ if not self.crew:
510
+ return ""
511
+
512
+ agents = self.crew.agents
513
+ agent_roles = [self._sanitize_role(agent.role) for agent in agents]
514
+ return "_".join(agent_roles)
515
+
516
+ def _sanitize_role(self, role: str) -> str:
517
+ """Sanitize agent role for use in identifiers."""
518
+ return role.replace("\n", "").replace(" ", "_").replace("/", "_")
519
+
520
+
521
+ __all__ = [
522
+ "DXMTStorage",
523
+ "DXMTStorageConfig",
524
+ ]