dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/mem0.py
ADDED
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
"""DXMT mem0 VectorStore Adapter.
|
|
2
|
+
|
|
3
|
+
This module provides a mem0-compatible VectorStore implementation backed by DXMT VectorDB.
|
|
4
|
+
mem0 (https://mem0.ai) is an intelligent memory layer for AI assistants and agents.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> from mem0 import Memory
|
|
8
|
+
>>> from dxmt import Client
|
|
9
|
+
>>> from dxmt.mem0 import DXMTMemoryStore
|
|
10
|
+
>>>
|
|
11
|
+
>>> # Initialize DXMT memory store
|
|
12
|
+
>>> client = Client(host="localhost", port=9999)
|
|
13
|
+
>>> vector_store = DXMTMemoryStore(
|
|
14
|
+
... collection_name="user_memories",
|
|
15
|
+
... client=client,
|
|
16
|
+
... embedding_model_dims=768
|
|
17
|
+
... )
|
|
18
|
+
>>>
|
|
19
|
+
>>> # Configure mem0
|
|
20
|
+
>>> config = {
|
|
21
|
+
... "vector_store": {
|
|
22
|
+
... "provider": "custom",
|
|
23
|
+
... "config": {"custom_vector_store": vector_store}
|
|
24
|
+
... }
|
|
25
|
+
... }
|
|
26
|
+
>>> m = Memory.from_config(config)
|
|
27
|
+
>>>
|
|
28
|
+
>>> # Store memories
|
|
29
|
+
>>> m.add("I love Python programming", user_id="user_123")
|
|
30
|
+
>>>
|
|
31
|
+
>>> # Search memories
|
|
32
|
+
>>> results = m.search("programming preferences", user_id="user_123")
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import logging
|
|
38
|
+
import math
|
|
39
|
+
import uuid
|
|
40
|
+
from datetime import datetime
|
|
41
|
+
from typing import Any, Dict, List, Optional
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
from mem0.vector_stores.base import VectorStoreBase
|
|
45
|
+
MEM0_AVAILABLE = True
|
|
46
|
+
except ImportError:
|
|
47
|
+
# mem0 not installed, define dummy base class
|
|
48
|
+
MEM0_AVAILABLE = False
|
|
49
|
+
|
|
50
|
+
class VectorStoreBase: # type: ignore
|
|
51
|
+
"""Dummy base class when mem0 is not installed."""
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
from .client import DXMTClient
|
|
55
|
+
from .exceptions import DXMTError, DXMTTableError
|
|
56
|
+
from .filters import convert_simple_filter_to_dxmt
|
|
57
|
+
|
|
58
|
+
logger = logging.getLogger(__name__)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class OutputData:
|
|
62
|
+
"""What mem0 reads off a result: `.id`, `.score`, `.payload`.
|
|
63
|
+
|
|
64
|
+
mem0 indexes results by attribute (`memory.id`, `memory.score`) and every backend in its
|
|
65
|
+
tree returns a small model with these three fields. This adapter returned dicts, which
|
|
66
|
+
worked while mem0 went through `provider: "custom"` and stopped when it did not:
|
|
67
|
+
`'dict' object has no attribute 'id'`. A plain class rather than a pydantic model, so
|
|
68
|
+
the SDK does not gain a dependency for three attributes.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
__slots__ = ("id", "score", "payload")
|
|
72
|
+
|
|
73
|
+
def __init__(self, id, score, payload):
|
|
74
|
+
self.id = id
|
|
75
|
+
self.score = score
|
|
76
|
+
self.payload = payload
|
|
77
|
+
|
|
78
|
+
# Subscripting kept working for anything that still reads these as dicts.
|
|
79
|
+
def __getitem__(self, key):
|
|
80
|
+
return getattr(self, key)
|
|
81
|
+
|
|
82
|
+
def get(self, key, default=None):
|
|
83
|
+
return getattr(self, key, default)
|
|
84
|
+
|
|
85
|
+
def __repr__(self):
|
|
86
|
+
return f"OutputData(id={self.id!r}, score={self.score!r}, payload={self.payload!r})"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class DXMTMemoryStore(VectorStoreBase):
|
|
90
|
+
"""mem0-compatible VectorStore implementation using DXMT VectorDB.
|
|
91
|
+
|
|
92
|
+
Implements mem0's VectorStoreBase interface with DXMT backend.
|
|
93
|
+
Supports multi-scope memory storage (user_id, agent_id, run_id) and
|
|
94
|
+
efficient semantic search with filtering.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
collection_name (str): Name of the memory collection (table).
|
|
98
|
+
client (DXMTClient): DXMT client instance.
|
|
99
|
+
embedding_model_dims (int): Dimension of embedding vectors.
|
|
100
|
+
distance_metric (str): Distance metric (cosine, euclidean, manhattan, dot).
|
|
101
|
+
Defaults to "cosine".
|
|
102
|
+
|
|
103
|
+
Example:
|
|
104
|
+
>>> client = DXMTClient(host="localhost", port=9999)
|
|
105
|
+
>>> store = DXMTMemoryStore(
|
|
106
|
+
... collection_name="user_memories",
|
|
107
|
+
... client=client,
|
|
108
|
+
... embedding_model_dims=768
|
|
109
|
+
... )
|
|
110
|
+
>>>
|
|
111
|
+
>>> # Insert memories
|
|
112
|
+
>>> store.insert(
|
|
113
|
+
... vectors=[[0.1, 0.2, ...]],
|
|
114
|
+
... payloads=[{"data": "User likes Python", "user_id": "user_1"}],
|
|
115
|
+
... ids=["mem_1"]
|
|
116
|
+
... )
|
|
117
|
+
>>>
|
|
118
|
+
>>> # Search memories
|
|
119
|
+
>>> results = store.search(
|
|
120
|
+
... query="",
|
|
121
|
+
... vectors=[[0.1, 0.2, ...]],
|
|
122
|
+
... limit=5,
|
|
123
|
+
... filters={"user_id": "user_1"}
|
|
124
|
+
... )
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
def __init__(
|
|
128
|
+
self,
|
|
129
|
+
collection_name: str,
|
|
130
|
+
client: DXMTClient,
|
|
131
|
+
embedding_model_dims: int,
|
|
132
|
+
distance_metric: str = "cosine",
|
|
133
|
+
):
|
|
134
|
+
"""Initialize DXMT memory store."""
|
|
135
|
+
if not MEM0_AVAILABLE:
|
|
136
|
+
logger.warning(
|
|
137
|
+
"mem0 is not installed. Install with: pip install mem0ai"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
self.collection_name = collection_name
|
|
141
|
+
self.client = client
|
|
142
|
+
self.embedding_model_dims = embedding_model_dims
|
|
143
|
+
self.distance_metric = distance_metric
|
|
144
|
+
|
|
145
|
+
# Create collection (table) if not exists
|
|
146
|
+
self.create_col(collection_name, embedding_model_dims, distance_metric)
|
|
147
|
+
|
|
148
|
+
def create_col(
|
|
149
|
+
self, name: str, vector_size: int, distance: str = "cosine"
|
|
150
|
+
) -> None:
|
|
151
|
+
"""Create a new collection (table).
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
name (str): Collection name.
|
|
155
|
+
vector_size (int): Vector embedding dimension.
|
|
156
|
+
distance (str): Distance metric (cosine, euclidean, manhattan, dot).
|
|
157
|
+
|
|
158
|
+
Raises:
|
|
159
|
+
DXMTTableError: If table creation fails.
|
|
160
|
+
"""
|
|
161
|
+
try:
|
|
162
|
+
# Check if table exists
|
|
163
|
+
self.client.get_table(name)
|
|
164
|
+
logger.info(f"Collection {name} already exists, skipping creation")
|
|
165
|
+
except DXMTError:
|
|
166
|
+
# Table doesn't exist, create it
|
|
167
|
+
logger.info(f"Creating collection {name} with dimension {vector_size}")
|
|
168
|
+
self.client.create_table(
|
|
169
|
+
name=name,
|
|
170
|
+
dimension=vector_size,
|
|
171
|
+
distance_metric=distance,
|
|
172
|
+
index_type="hnsw", # Use HNSW for good recall
|
|
173
|
+
description=f"mem0 memory collection: {name}",
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
def insert(
|
|
177
|
+
self,
|
|
178
|
+
vectors: List[List[float]],
|
|
179
|
+
payloads: Optional[List[Dict[str, Any]]] = None,
|
|
180
|
+
ids: Optional[List[str]] = None,
|
|
181
|
+
) -> None:
|
|
182
|
+
"""Insert vectors with payloads into the collection.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
vectors (List[List[float]]): List of embedding vectors.
|
|
186
|
+
payloads (Optional[List[Dict[str, Any]]]): List of metadata payloads.
|
|
187
|
+
ids (Optional[List[str]]): List of vector IDs. Generated if not provided.
|
|
188
|
+
|
|
189
|
+
Raises:
|
|
190
|
+
ValueError: If vector dimensions don't match.
|
|
191
|
+
DXMTError: If insertion fails.
|
|
192
|
+
"""
|
|
193
|
+
if not vectors:
|
|
194
|
+
logger.warning("insert called with empty vectors list")
|
|
195
|
+
return
|
|
196
|
+
|
|
197
|
+
# Validate dimensions
|
|
198
|
+
for i, vector in enumerate(vectors):
|
|
199
|
+
if len(vector) != self.embedding_model_dims:
|
|
200
|
+
raise ValueError(
|
|
201
|
+
f"Vector {i} dimension mismatch: expected {self.embedding_model_dims}, "
|
|
202
|
+
f"got {len(vector)}"
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
# Generate defaults
|
|
206
|
+
if payloads is None:
|
|
207
|
+
payloads = [{}] * len(vectors)
|
|
208
|
+
|
|
209
|
+
if ids is None:
|
|
210
|
+
ids = [str(uuid.uuid4()) for _ in range(len(vectors))]
|
|
211
|
+
|
|
212
|
+
# Build records for batch insert
|
|
213
|
+
records = []
|
|
214
|
+
for i, (vector, payload, record_id) in enumerate(zip(vectors, payloads, ids)):
|
|
215
|
+
# Add timestamps if not present
|
|
216
|
+
if "created_at" not in payload:
|
|
217
|
+
payload["created_at"] = datetime.utcnow().isoformat() + "Z"
|
|
218
|
+
if "updated_at" not in payload:
|
|
219
|
+
payload["updated_at"] = payload["created_at"]
|
|
220
|
+
|
|
221
|
+
records.append({
|
|
222
|
+
"id": record_id,
|
|
223
|
+
"vector": vector,
|
|
224
|
+
"payload": payload,
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
# Batch insert
|
|
228
|
+
logger.debug(f"Inserting {len(records)} records into {self.collection_name}")
|
|
229
|
+
self.client.insert_records(self.collection_name, records)
|
|
230
|
+
|
|
231
|
+
def search(
|
|
232
|
+
self,
|
|
233
|
+
query: str, # Not used, kept for mem0 interface compatibility
|
|
234
|
+
vectors: List[List[float]],
|
|
235
|
+
top_k: int = 5,
|
|
236
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
237
|
+
limit: Optional[int] = None,
|
|
238
|
+
) -> List[Dict[str, Any]]:
|
|
239
|
+
"""Search for similar vectors.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
query (str): Query string (not used, vectors are used instead).
|
|
243
|
+
vectors (List[List[float]]): Query vector(s). Only first vector is used.
|
|
244
|
+
limit (int): Maximum number of results to return.
|
|
245
|
+
filters (Optional[Dict[str, Any]]): Metadata filters (user_id, agent_id, etc.).
|
|
246
|
+
|
|
247
|
+
Returns:
|
|
248
|
+
List[Dict[str, Any]]: Search results with id, score, and payload.
|
|
249
|
+
|
|
250
|
+
Raises:
|
|
251
|
+
ValueError: If no vectors provided.
|
|
252
|
+
DXMTError: If search fails.
|
|
253
|
+
"""
|
|
254
|
+
if not vectors or len(vectors) == 0:
|
|
255
|
+
raise ValueError("At least one query vector is required")
|
|
256
|
+
|
|
257
|
+
# `VectorStoreBase.search` spells this `top_k`; this adapter called it `limit` and so
|
|
258
|
+
# raised `unexpected keyword argument 'top_k'` the moment mem0 called it. Both are
|
|
259
|
+
# accepted, `limit` winning when given, so a caller written against either signature
|
|
260
|
+
# keeps working.
|
|
261
|
+
if limit is not None:
|
|
262
|
+
top_k = limit
|
|
263
|
+
|
|
264
|
+
# `vectors` is one query embedding, flat, in current mem0 -- `search(vectors=
|
|
265
|
+
# query_embedding, ...)` in `Memory._add_to_vector_store`. It used to be a list of
|
|
266
|
+
# them, and this adapter took `vectors[0]`, which on a flat list is a float and
|
|
267
|
+
# produced `object of type 'float' has no len()` two lines later. Both shapes are
|
|
268
|
+
# read here rather than one being chosen, because the caller is not ours to pin.
|
|
269
|
+
query_vector = vectors if vectors and isinstance(vectors[0], (int, float)) else vectors[0]
|
|
270
|
+
|
|
271
|
+
# Validate dimension
|
|
272
|
+
if len(query_vector) != self.embedding_model_dims:
|
|
273
|
+
raise ValueError(
|
|
274
|
+
f"Query vector dimension mismatch: expected {self.embedding_model_dims}, "
|
|
275
|
+
f"got {len(query_vector)}"
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
# Convert mem0 filters to DXMT format
|
|
279
|
+
dxmt_filter = self._convert_filters(filters) if filters else None
|
|
280
|
+
|
|
281
|
+
# Perform search
|
|
282
|
+
logger.debug(
|
|
283
|
+
f"Searching {self.collection_name} with top_k={top_k}, filters={filters}"
|
|
284
|
+
)
|
|
285
|
+
results = self.client.search(
|
|
286
|
+
table_name=self.collection_name,
|
|
287
|
+
vector=query_vector,
|
|
288
|
+
k=top_k,
|
|
289
|
+
filter=dxmt_filter,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# Normalize scores and format output
|
|
293
|
+
output = []
|
|
294
|
+
for hit in results:
|
|
295
|
+
output.append(OutputData(
|
|
296
|
+
id=hit["id"],
|
|
297
|
+
score=self._normalize_score(hit["score"]),
|
|
298
|
+
payload=hit.get("metadata", hit.get("payload", {})),
|
|
299
|
+
))
|
|
300
|
+
|
|
301
|
+
return output
|
|
302
|
+
|
|
303
|
+
def delete(self, vector_id: str) -> None:
|
|
304
|
+
"""Delete a vector by ID.
|
|
305
|
+
|
|
306
|
+
Args:
|
|
307
|
+
vector_id (str): Vector ID to delete.
|
|
308
|
+
|
|
309
|
+
Raises:
|
|
310
|
+
DXMTError: If deletion fails.
|
|
311
|
+
"""
|
|
312
|
+
logger.debug(f"Deleting vector {vector_id} from {self.collection_name}")
|
|
313
|
+
self.client.delete_records(self.collection_name, [vector_id])
|
|
314
|
+
|
|
315
|
+
def update(
|
|
316
|
+
self,
|
|
317
|
+
vector_id: str,
|
|
318
|
+
vector: Optional[List[float]] = None,
|
|
319
|
+
payload: Optional[Dict[str, Any]] = None,
|
|
320
|
+
) -> None:
|
|
321
|
+
"""Update a vector and its payload.
|
|
322
|
+
|
|
323
|
+
Args:
|
|
324
|
+
vector_id (str): Vector ID to update.
|
|
325
|
+
vector (Optional[List[float]]): New vector. If None, vector is not updated.
|
|
326
|
+
payload (Optional[Dict[str, Any]]): New payload. Merged with existing.
|
|
327
|
+
|
|
328
|
+
Raises:
|
|
329
|
+
ValueError: If vector dimension doesn't match.
|
|
330
|
+
DXMTError: If update fails.
|
|
331
|
+
"""
|
|
332
|
+
# Validate dimension if vector provided
|
|
333
|
+
if vector is not None and len(vector) != self.embedding_model_dims:
|
|
334
|
+
raise ValueError(
|
|
335
|
+
f"Vector dimension mismatch: expected {self.embedding_model_dims}, "
|
|
336
|
+
f"got {len(vector)}"
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
# Add updated_at timestamp
|
|
340
|
+
if payload is not None:
|
|
341
|
+
payload["updated_at"] = datetime.utcnow().isoformat() + "Z"
|
|
342
|
+
|
|
343
|
+
logger.debug(f"Updating vector {vector_id} in {self.collection_name}")
|
|
344
|
+
self.client.update_record(
|
|
345
|
+
table_name=self.collection_name,
|
|
346
|
+
record_id=vector_id,
|
|
347
|
+
vector=vector,
|
|
348
|
+
payload=payload,
|
|
349
|
+
merge_mode="merge", # Preserve existing fields
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
def get(self, vector_id: str) -> Optional[Dict[str, Any]]:
|
|
353
|
+
"""Retrieve a vector by ID.
|
|
354
|
+
|
|
355
|
+
Args:
|
|
356
|
+
vector_id (str): Vector ID to retrieve.
|
|
357
|
+
|
|
358
|
+
Returns:
|
|
359
|
+
Optional[Dict[str, Any]]: Record with id, score (None), and payload.
|
|
360
|
+
Returns None if not found.
|
|
361
|
+
|
|
362
|
+
Raises:
|
|
363
|
+
DXMTError: If retrieval fails.
|
|
364
|
+
"""
|
|
365
|
+
try:
|
|
366
|
+
records = self.client.get_records(self.collection_name, [vector_id])
|
|
367
|
+
if not records:
|
|
368
|
+
return None
|
|
369
|
+
|
|
370
|
+
record = records[0]
|
|
371
|
+
return OutputData(
|
|
372
|
+
id=record["id"],
|
|
373
|
+
score=None, # No score for direct retrieval
|
|
374
|
+
payload=record.get("metadata", record.get("payload", {})),
|
|
375
|
+
)
|
|
376
|
+
except DXMTError as e:
|
|
377
|
+
logger.error(f"Error getting vector {vector_id}: {e}")
|
|
378
|
+
return None
|
|
379
|
+
|
|
380
|
+
def list_cols(self) -> List[str]:
|
|
381
|
+
"""List all collections (tables).
|
|
382
|
+
|
|
383
|
+
Returns:
|
|
384
|
+
List[str]: List of collection names.
|
|
385
|
+
|
|
386
|
+
Raises:
|
|
387
|
+
DXMTError: If listing fails.
|
|
388
|
+
"""
|
|
389
|
+
tables = self.client.list_tables()
|
|
390
|
+
return [table["name"] for table in tables]
|
|
391
|
+
|
|
392
|
+
def delete_col(self) -> None:
|
|
393
|
+
"""Delete the collection (table).
|
|
394
|
+
|
|
395
|
+
Raises:
|
|
396
|
+
DXMTTableError: If deletion fails.
|
|
397
|
+
"""
|
|
398
|
+
logger.info(f"Deleting collection {self.collection_name}")
|
|
399
|
+
self.client.delete_table(self.collection_name)
|
|
400
|
+
|
|
401
|
+
def col_info(self) -> Dict[str, Any]:
|
|
402
|
+
"""Get collection (table) information.
|
|
403
|
+
|
|
404
|
+
Returns:
|
|
405
|
+
Dict[str, Any]: Collection metadata including name, dimension, count, etc.
|
|
406
|
+
|
|
407
|
+
Raises:
|
|
408
|
+
DXMTError: If retrieval fails.
|
|
409
|
+
"""
|
|
410
|
+
return self.client.get_table(self.collection_name)
|
|
411
|
+
|
|
412
|
+
def list(
|
|
413
|
+
self,
|
|
414
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
415
|
+
top_k: Optional[int] = None,
|
|
416
|
+
limit: Optional[int] = None,
|
|
417
|
+
) -> List[List["OutputData"]]:
|
|
418
|
+
"""List all vectors in the collection.
|
|
419
|
+
|
|
420
|
+
Args:
|
|
421
|
+
filters (Optional[Dict[str, Any]]): Metadata filters to apply.
|
|
422
|
+
limit (int): Maximum number of results to return.
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
List[Dict[str, Any]]: List of records with id, score (None), and payload.
|
|
426
|
+
|
|
427
|
+
Raises:
|
|
428
|
+
DXMTError: If listing fails.
|
|
429
|
+
|
|
430
|
+
Note:
|
|
431
|
+
DXMT doesn't have a direct "list all" endpoint, so we use search
|
|
432
|
+
with a dummy vector. For large collections, this may be inefficient.
|
|
433
|
+
"""
|
|
434
|
+
# Create a dummy vector for search
|
|
435
|
+
dummy_vector = [0.0] * self.embedding_model_dims
|
|
436
|
+
|
|
437
|
+
# Convert filters
|
|
438
|
+
dxmt_filter = self._convert_filters(filters) if filters else None
|
|
439
|
+
|
|
440
|
+
# Search with dummy vector to get records
|
|
441
|
+
# `VectorStoreBase.list` spells it `top_k`, as `search` does. Both accepted, and
|
|
442
|
+
# neither given means the old default of 100 rather than everything.
|
|
443
|
+
top_k = limit if limit is not None else (top_k if top_k is not None else 100)
|
|
444
|
+
logger.debug(f"Listing records from {self.collection_name} with top_k={top_k}")
|
|
445
|
+
results = self.client.search(
|
|
446
|
+
table_name=self.collection_name,
|
|
447
|
+
vector=dummy_vector,
|
|
448
|
+
k=top_k,
|
|
449
|
+
filter=dxmt_filter,
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
# Format output
|
|
453
|
+
output = []
|
|
454
|
+
for hit in results:
|
|
455
|
+
output.append(OutputData(
|
|
456
|
+
id=hit["id"],
|
|
457
|
+
score=None, # Not relevant for listing
|
|
458
|
+
payload=hit.get("metadata", hit.get("payload", {})),
|
|
459
|
+
))
|
|
460
|
+
|
|
461
|
+
# Wrapped in a list, which looks wrong and is the contract. mem0 reads this as
|
|
462
|
+
# `self.vector_store.list(...)[0]` -- see `Memory.delete_all` -- and every backend in
|
|
463
|
+
# its own tree ends with `return [self._parse_output(results)]`. Returning the flat
|
|
464
|
+
# list made `delete_all` raise `IndexError: list index out of range` on an empty
|
|
465
|
+
# result and iterate a single `OutputData` on a non-empty one.
|
|
466
|
+
return [output]
|
|
467
|
+
|
|
468
|
+
def reset(self) -> None:
|
|
469
|
+
"""Reset the collection by deleting and recreating it.
|
|
470
|
+
|
|
471
|
+
Raises:
|
|
472
|
+
DXMTError: If reset fails.
|
|
473
|
+
"""
|
|
474
|
+
logger.info(f"Resetting collection {self.collection_name}")
|
|
475
|
+
try:
|
|
476
|
+
self.delete_col()
|
|
477
|
+
except DXMTError as e:
|
|
478
|
+
logger.warning(f"Error deleting collection during reset: {e}")
|
|
479
|
+
|
|
480
|
+
# Recreate collection
|
|
481
|
+
self.create_col(
|
|
482
|
+
self.collection_name, self.embedding_model_dims, self.distance_metric
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
# Helper methods
|
|
486
|
+
|
|
487
|
+
def _convert_filters(self, mem0_filters: Dict[str, Any]) -> Dict[str, Any]:
|
|
488
|
+
"""Convert mem0 filters to DXMT filter format.
|
|
489
|
+
|
|
490
|
+
mem0 filters support both simple equality and operator-based syntax:
|
|
491
|
+
{"user_id": "user_1"}
|
|
492
|
+
{"user_id": "user_1", "agent_id": "agent_1"}
|
|
493
|
+
{"created_at": {"gte": "2024-01-01T00:00:00Z", "lte": "2024-12-31T23:59:59Z"}}
|
|
494
|
+
|
|
495
|
+
DXMT expects structured Filter format:
|
|
496
|
+
{"must": [{"key": "user_id", "operation": {"type": "Equal", "value": "user_1"}}]}
|
|
497
|
+
|
|
498
|
+
Args:
|
|
499
|
+
mem0_filters (Dict[str, Any]): mem0 filter dictionary.
|
|
500
|
+
|
|
501
|
+
Returns:
|
|
502
|
+
Dict[str, Any]: DXMT filter format.
|
|
503
|
+
|
|
504
|
+
Raises:
|
|
505
|
+
ValueError: If filter format is invalid.
|
|
506
|
+
"""
|
|
507
|
+
if not mem0_filters:
|
|
508
|
+
return None
|
|
509
|
+
|
|
510
|
+
conditions = []
|
|
511
|
+
|
|
512
|
+
for key, value in mem0_filters.items():
|
|
513
|
+
if isinstance(value, dict):
|
|
514
|
+
# Handle operators: {"created_at": {"gte": "...", "lte": "..."}}
|
|
515
|
+
for op, op_value in value.items():
|
|
516
|
+
if op == "eq":
|
|
517
|
+
conditions.append({
|
|
518
|
+
"key": key,
|
|
519
|
+
"operation": {"type": "Equal", "value": op_value}
|
|
520
|
+
})
|
|
521
|
+
elif op == "ne":
|
|
522
|
+
conditions.append({
|
|
523
|
+
"key": key,
|
|
524
|
+
"operation": {"type": "NotEqual", "value": op_value}
|
|
525
|
+
})
|
|
526
|
+
elif op in ["gt", "gte", "lt", "lte"]:
|
|
527
|
+
# Map mem0 operators to DXMT operators
|
|
528
|
+
op_map = {
|
|
529
|
+
"gt": "GreaterThan",
|
|
530
|
+
"gte": "GreaterThanOrEqual",
|
|
531
|
+
"lt": "LessThan",
|
|
532
|
+
"lte": "LessThanOrEqual",
|
|
533
|
+
}
|
|
534
|
+
conditions.append({
|
|
535
|
+
"key": key,
|
|
536
|
+
"operation": {"type": op_map[op], "value": op_value}
|
|
537
|
+
})
|
|
538
|
+
elif op == "in":
|
|
539
|
+
conditions.append({
|
|
540
|
+
"key": key,
|
|
541
|
+
"operation": {"type": "In", "value": op_value}
|
|
542
|
+
})
|
|
543
|
+
elif op == "nin":
|
|
544
|
+
conditions.append({
|
|
545
|
+
"key": key,
|
|
546
|
+
"operation": {"type": "NotIn", "value": op_value}
|
|
547
|
+
})
|
|
548
|
+
else:
|
|
549
|
+
logger.warning(f"Unsupported operator: {op}, treating as equality")
|
|
550
|
+
conditions.append({
|
|
551
|
+
"key": key,
|
|
552
|
+
"operation": {"type": "Equal", "value": value}
|
|
553
|
+
})
|
|
554
|
+
break
|
|
555
|
+
else:
|
|
556
|
+
# Simple equality: {"user_id": "user_1"}
|
|
557
|
+
conditions.append({
|
|
558
|
+
"key": key,
|
|
559
|
+
"operation": {"type": "Equal", "value": value}
|
|
560
|
+
})
|
|
561
|
+
|
|
562
|
+
return {"must": conditions} if conditions else None
|
|
563
|
+
|
|
564
|
+
def _normalize_score(self, score: float) -> float:
|
|
565
|
+
"""Pass through DXMT similarity score to mem0.
|
|
566
|
+
|
|
567
|
+
DXMT already returns a similarity score in [0, 1] for cosine and dot
|
|
568
|
+
metrics (higher = more similar), and a non-negative distance for
|
|
569
|
+
euclidean. mem0 expects similarity where 1.0 is most similar.
|
|
570
|
+
|
|
571
|
+
Args:
|
|
572
|
+
score (float): DXMT score from search result.
|
|
573
|
+
|
|
574
|
+
Returns:
|
|
575
|
+
float: Similarity score suitable for mem0 (higher = more similar).
|
|
576
|
+
"""
|
|
577
|
+
if self.distance_metric in ("cosine", "dot"):
|
|
578
|
+
# DXMT returns cosine/dot similarity directly in [0, 1]; pass through.
|
|
579
|
+
return max(0.0, min(1.0, score))
|
|
580
|
+
|
|
581
|
+
elif self.distance_metric == "euclidean":
|
|
582
|
+
# DXMT returns L2 distance (0 = identical); convert to similarity.
|
|
583
|
+
return math.exp(-score)
|
|
584
|
+
|
|
585
|
+
elif self.distance_metric == "manhattan":
|
|
586
|
+
return max(0.0, min(1.0, 1.0 / (1.0 + abs(score))))
|
|
587
|
+
|
|
588
|
+
else:
|
|
589
|
+
logger.warning(f"Unknown distance metric {self.distance_metric}, returning raw score")
|
|
590
|
+
return score
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def register_with_mem0(provider: str = "dxmt") -> str:
|
|
594
|
+
"""Make `{"vector_store": {"provider": "dxmt", ...}}` a thing mem0 accepts.
|
|
595
|
+
|
|
596
|
+
mem0 used to take any `VectorStoreBase` through `provider: "custom"` and a
|
|
597
|
+
`custom_vector_store` in the config. That door is closed: current mem0 validates the
|
|
598
|
+
provider against a fixed table of twenty-five named backends, and a config for an
|
|
599
|
+
unlisted one fails in pydantic before any of this code runs --
|
|
600
|
+
|
|
601
|
+
ValidationError: Unsupported vector store provider: custom
|
|
602
|
+
|
|
603
|
+
-- which is what every DXMT + mem0 example in this repository hit. The adapter below was
|
|
604
|
+
never the problem; the way it was plugged in stopped existing.
|
|
605
|
+
|
|
606
|
+
There are two tables to join, because mem0 validates the config and builds the store in
|
|
607
|
+
different places, and the validator reaches the config class by importing
|
|
608
|
+
`mem0.configs.vector_stores.<provider>`. So this registers a module under that name as
|
|
609
|
+
well as the two dictionary entries. Idempotent, and it touches nothing but the entries it
|
|
610
|
+
adds.
|
|
611
|
+
|
|
612
|
+
Returns the provider name, so a caller can write
|
|
613
|
+
|
|
614
|
+
config = {"vector_store": {"provider": register_with_mem0(), "config": {...}}}
|
|
615
|
+
|
|
616
|
+
and not repeat the string. `config` then takes this class's own arguments --
|
|
617
|
+
`collection_name`, `client`, `embedding_model_dims`, `distance_metric` -- rather than a
|
|
618
|
+
prebuilt instance, because mem0 constructs the store itself.
|
|
619
|
+
"""
|
|
620
|
+
import sys
|
|
621
|
+
import types
|
|
622
|
+
|
|
623
|
+
from pydantic import BaseModel, Field
|
|
624
|
+
|
|
625
|
+
from mem0.utils.factory import VectorStoreFactory
|
|
626
|
+
from mem0.vector_stores.configs import VectorStoreConfig
|
|
627
|
+
|
|
628
|
+
VectorStoreFactory.provider_to_class[provider] = "dxmt.mem0.DXMTMemoryStore"
|
|
629
|
+
|
|
630
|
+
module_name = f"mem0.configs.vector_stores.{provider}"
|
|
631
|
+
if module_name not in sys.modules:
|
|
632
|
+
class DXMTVectorStoreConfig(BaseModel):
|
|
633
|
+
collection_name: str = Field("mem0", description="table name")
|
|
634
|
+
client: Any = Field(None, description="a dxmt.Client")
|
|
635
|
+
embedding_model_dims: int = Field(1536, description="vector width")
|
|
636
|
+
distance_metric: str = Field("cosine", description="cosine|euclidean|manhattan|dot")
|
|
637
|
+
|
|
638
|
+
model_config = {"arbitrary_types_allowed": True}
|
|
639
|
+
|
|
640
|
+
module = types.ModuleType(module_name)
|
|
641
|
+
module.DXMTVectorStoreConfig = DXMTVectorStoreConfig
|
|
642
|
+
sys.modules[module_name] = module
|
|
643
|
+
|
|
644
|
+
# `_provider_configs` is a pydantic private attribute, so reading it off the class gives
|
|
645
|
+
# the descriptor rather than the dict. The dict is its default, and is the object the
|
|
646
|
+
# validator actually consults.
|
|
647
|
+
VectorStoreConfig.__private_attributes__["_provider_configs"].default[provider] = (
|
|
648
|
+
"DXMTVectorStoreConfig"
|
|
649
|
+
)
|
|
650
|
+
return provider
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
__all__ = ["DXMTMemoryStore", "OutputData", "register_with_mem0"]
|