dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/langgraph_store.py
ADDED
|
@@ -0,0 +1,1054 @@
|
|
|
1
|
+
"""DXMT VectorDB integration for LangGraph Store.
|
|
2
|
+
|
|
3
|
+
This module provides a LangGraph-compatible persistent store implementation
|
|
4
|
+
backed by DXMT VectorDB, enabling AI agents to store and retrieve memories
|
|
5
|
+
with optional semantic search capabilities.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import hashlib
|
|
12
|
+
import logging
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from typing import Any, Literal, cast
|
|
16
|
+
|
|
17
|
+
from langgraph.store.base import (
|
|
18
|
+
BaseStore,
|
|
19
|
+
GetOp,
|
|
20
|
+
Item,
|
|
21
|
+
ListNamespacesOp,
|
|
22
|
+
MatchCondition,
|
|
23
|
+
Op,
|
|
24
|
+
PutOp,
|
|
25
|
+
Result,
|
|
26
|
+
SearchItem,
|
|
27
|
+
SearchOp,
|
|
28
|
+
get_text_at_path,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
from .client import DXMTClient
|
|
32
|
+
from .exceptions import DXMTError, DXMTTableError
|
|
33
|
+
from .filters import convert_simple_filter_to_dxmt
|
|
34
|
+
|
|
35
|
+
logger = logging.getLogger(__name__)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Constants
|
|
39
|
+
MAX_TABLE_NAME_LENGTH = 63 # the server's own limit, in src/utils/mod.rs
|
|
40
|
+
NAMESPACE_TABLE_PREFIX = "langgraph_"
|
|
41
|
+
CREATED_AT_FIELD = "_created_at"
|
|
42
|
+
UPDATED_AT_FIELD = "_updated_at"
|
|
43
|
+
NAMESPACE_FIELD = "_namespace"
|
|
44
|
+
KEY_FIELD = "_key"
|
|
45
|
+
INDEX_FIELDS_FIELD = "_index_fields"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class DXMTLangGraphStore(BaseStore):
|
|
49
|
+
"""LangGraph Store implementation backed by DXMT VectorDB.
|
|
50
|
+
|
|
51
|
+
This store provides persistent key-value storage with optional semantic search
|
|
52
|
+
capabilities for AI agents using LangGraph. It leverages DXMT's cost-efficient
|
|
53
|
+
tiered storage (Memory → SSD → S3) while maintaining high performance.
|
|
54
|
+
|
|
55
|
+
Key Features:
|
|
56
|
+
- Hierarchical namespace organization (tuple paths)
|
|
57
|
+
- Semantic search via embeddings (optional)
|
|
58
|
+
- Metadata filtering with comparison operators
|
|
59
|
+
- Efficient batch operations
|
|
60
|
+
- Both sync and async APIs
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
client: DXMTClient instance for database communication.
|
|
64
|
+
embedding_dims: Vector dimension for semantic search. Required if using query-based search.
|
|
65
|
+
embed_fn: Function to generate embeddings from text. Signature: f(texts: list[str]) -> list[list[float]]
|
|
66
|
+
aembed_fn: Async embedding function. Signature: async f(texts: list[str]) -> list[list[float]]
|
|
67
|
+
default_index_fields: Default fields to extract for indexing. Defaults to ["$"] (whole document).
|
|
68
|
+
distance_metric: Distance metric for vector search. Options: "cosine", "euclidean", "dot".
|
|
69
|
+
namespace_strategy: Strategy for mapping namespaces to tables.
|
|
70
|
+
- "table": Each namespace → separate table (recommended, better isolation)
|
|
71
|
+
- "metadata": Single table with namespace stored in metadata (compact, fewer tables)
|
|
72
|
+
|
|
73
|
+
Examples:
|
|
74
|
+
Basic usage without semantic search:
|
|
75
|
+
|
|
76
|
+
>>> client = DXMTClient(host="localhost", port=9999)
|
|
77
|
+
>>> store = DXMTLangGraphStore(client)
|
|
78
|
+
>>> store.put(("users", "profiles"), "user123", {"name": "Alice", "role": "admin"})
|
|
79
|
+
>>> item = store.get(("users", "profiles"), "user123")
|
|
80
|
+
|
|
81
|
+
With semantic search:
|
|
82
|
+
|
|
83
|
+
>>> from langchain_openai import OpenAIEmbeddings
|
|
84
|
+
>>> embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
|
|
85
|
+
>>>
|
|
86
|
+
>>> store = DXMTLangGraphStore(
|
|
87
|
+
... client=client,
|
|
88
|
+
... embedding_dims=1536,
|
|
89
|
+
... embed_fn=lambda texts: embeddings.embed_documents(texts)
|
|
90
|
+
... )
|
|
91
|
+
>>> store.put(("docs",), "doc1", {"content": "AI agents use LangGraph for workflows"}, index=["content"])
|
|
92
|
+
>>> results = store.search(("docs",), query="workflows for AI")
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
def __init__(
|
|
96
|
+
self,
|
|
97
|
+
client: DXMTClient,
|
|
98
|
+
*,
|
|
99
|
+
embedding_dims: int | None = None,
|
|
100
|
+
embed_fn: Any = None,
|
|
101
|
+
aembed_fn: Any = None,
|
|
102
|
+
default_index_fields: list[str] | None = None,
|
|
103
|
+
distance_metric: str = "cosine",
|
|
104
|
+
namespace_strategy: Literal["table", "metadata"] = "table",
|
|
105
|
+
):
|
|
106
|
+
"""Initialize DXMT LangGraph Store."""
|
|
107
|
+
self.client = client
|
|
108
|
+
self.embedding_dims = embedding_dims
|
|
109
|
+
self.embed_fn = embed_fn
|
|
110
|
+
self.aembed_fn = aembed_fn
|
|
111
|
+
self.default_index_fields = default_index_fields or ["$"]
|
|
112
|
+
self.distance_metric = distance_metric
|
|
113
|
+
self.namespace_strategy = namespace_strategy
|
|
114
|
+
|
|
115
|
+
# Validate configuration
|
|
116
|
+
if embedding_dims is not None and embedding_dims <= 0:
|
|
117
|
+
raise ValueError(f"embedding_dims must be positive, got {embedding_dims}")
|
|
118
|
+
|
|
119
|
+
# Track created tables to avoid redundant creation checks
|
|
120
|
+
self._created_tables: set[str] = set()
|
|
121
|
+
self._table_lock = asyncio.Lock()
|
|
122
|
+
|
|
123
|
+
def batch(self, ops: Iterable[Op]) -> list[Result]:
|
|
124
|
+
"""Execute multiple operations synchronously in a single batch.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
ops: Iterable of Get/Search/Put/ListNamespaces operations.
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
List of results corresponding to each operation.
|
|
131
|
+
|
|
132
|
+
Raises:
|
|
133
|
+
DXMTError: If any operation fails.
|
|
134
|
+
"""
|
|
135
|
+
# Convert iterable to list for multiple passes
|
|
136
|
+
ops_list = list(ops)
|
|
137
|
+
|
|
138
|
+
results: list[Result] = []
|
|
139
|
+
for op in ops_list:
|
|
140
|
+
if isinstance(op, GetOp):
|
|
141
|
+
results.append(self._handle_get(op))
|
|
142
|
+
elif isinstance(op, SearchOp):
|
|
143
|
+
results.append(self._handle_search(op))
|
|
144
|
+
elif isinstance(op, PutOp):
|
|
145
|
+
self._handle_put(op)
|
|
146
|
+
results.append(None)
|
|
147
|
+
elif isinstance(op, ListNamespacesOp):
|
|
148
|
+
results.append(self._handle_list_namespaces(op))
|
|
149
|
+
else:
|
|
150
|
+
raise ValueError(f"Unknown operation type: {type(op)}")
|
|
151
|
+
|
|
152
|
+
return results
|
|
153
|
+
|
|
154
|
+
async def abatch(self, ops: Iterable[Op]) -> list[Result]:
|
|
155
|
+
"""Execute multiple operations asynchronously in a single batch.
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
ops: Iterable of Get/Search/Put/ListNamespaces operations.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
List of results corresponding to each operation.
|
|
162
|
+
|
|
163
|
+
Raises:
|
|
164
|
+
DXMTError: If any operation fails.
|
|
165
|
+
"""
|
|
166
|
+
# Convert iterable to list for multiple passes
|
|
167
|
+
ops_list = list(ops)
|
|
168
|
+
|
|
169
|
+
results: list[Result] = []
|
|
170
|
+
for op in ops_list:
|
|
171
|
+
if isinstance(op, GetOp):
|
|
172
|
+
results.append(await self._ahandle_get(op))
|
|
173
|
+
elif isinstance(op, SearchOp):
|
|
174
|
+
results.append(await self._ahandle_search(op))
|
|
175
|
+
elif isinstance(op, PutOp):
|
|
176
|
+
await self._ahandle_put(op)
|
|
177
|
+
results.append(None)
|
|
178
|
+
elif isinstance(op, ListNamespacesOp):
|
|
179
|
+
results.append(await self._ahandle_list_namespaces(op))
|
|
180
|
+
else:
|
|
181
|
+
raise ValueError(f"Unknown operation type: {type(op)}")
|
|
182
|
+
|
|
183
|
+
return results
|
|
184
|
+
|
|
185
|
+
# Operation Handlers (Sync)
|
|
186
|
+
|
|
187
|
+
def _handle_get(self, op: GetOp) -> Item | None:
|
|
188
|
+
"""Handle GetOp synchronously."""
|
|
189
|
+
table_name = self._namespace_to_table(op.namespace)
|
|
190
|
+
|
|
191
|
+
# Ensure table exists
|
|
192
|
+
if not self._ensure_table_exists_sync(table_name):
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
# Get record by constructing record_id from namespace + key
|
|
197
|
+
record_id = self._make_record_id(op.namespace, op.key)
|
|
198
|
+
records = self.client.get_records(table_name, [record_id])
|
|
199
|
+
|
|
200
|
+
if not records:
|
|
201
|
+
return None
|
|
202
|
+
|
|
203
|
+
record = records[0]
|
|
204
|
+
return self._record_to_item(record)
|
|
205
|
+
|
|
206
|
+
except DXMTError as e:
|
|
207
|
+
logger.warning(f"Failed to get item {op.namespace}/{op.key}: {e}")
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
def _list_matching(
|
|
211
|
+
self,
|
|
212
|
+
table_name: str,
|
|
213
|
+
dxmt_filter: dict[str, Any] | None,
|
|
214
|
+
wanted: int,
|
|
215
|
+
) -> list[dict[str, Any]]:
|
|
216
|
+
"""Records from `table_name` matching `dxmt_filter`, newest first, up to `wanted`.
|
|
217
|
+
|
|
218
|
+
The server filters: this used to page through the whole table and test each record
|
|
219
|
+
here, with a second reading of the filter that disagreed with the server's on arrays,
|
|
220
|
+
on missing fields and on `True == 1`.
|
|
221
|
+
"""
|
|
222
|
+
out: list[dict[str, Any]] = []
|
|
223
|
+
for record in self.client.scroll(
|
|
224
|
+
table_name, filter=dxmt_filter, batch_size=min(max(wanted, 1), 1000)
|
|
225
|
+
):
|
|
226
|
+
out.append(record)
|
|
227
|
+
if len(out) >= wanted:
|
|
228
|
+
break
|
|
229
|
+
return out
|
|
230
|
+
|
|
231
|
+
def _handle_search(self, op: SearchOp) -> list[SearchItem]:
|
|
232
|
+
"""Handle SearchOp synchronously."""
|
|
233
|
+
# Determine which tables to search based on namespace prefix
|
|
234
|
+
tables = self._get_matching_tables_sync(op.namespace_prefix)
|
|
235
|
+
|
|
236
|
+
if not tables:
|
|
237
|
+
return []
|
|
238
|
+
|
|
239
|
+
all_results: list[SearchItem] = []
|
|
240
|
+
|
|
241
|
+
for table_name in tables:
|
|
242
|
+
try:
|
|
243
|
+
# Convert filters
|
|
244
|
+
dxmt_filter = self._convert_filter(op.filter) if op.filter else None
|
|
245
|
+
|
|
246
|
+
# If query is provided, use semantic search
|
|
247
|
+
if op.query and self.embed_fn and self.embedding_dims:
|
|
248
|
+
# Generate embedding
|
|
249
|
+
embeddings = self.embed_fn([op.query])
|
|
250
|
+
query_vector = embeddings[0]
|
|
251
|
+
|
|
252
|
+
# Perform vector search
|
|
253
|
+
hits = self.client.search(
|
|
254
|
+
table_name=table_name,
|
|
255
|
+
vector=query_vector,
|
|
256
|
+
limit=op.limit + op.offset,
|
|
257
|
+
filter=dxmt_filter,
|
|
258
|
+
)
|
|
259
|
+
else:
|
|
260
|
+
# A search with no query vector: list the table instead of pretending
|
|
261
|
+
# there is nothing in it. This used to log a warning and `continue`, so a
|
|
262
|
+
# metadata-only search returned zero results and looked like an empty
|
|
263
|
+
# table -- the caller got a wrong answer rather than an error, which is
|
|
264
|
+
# the worst of the three possible behaviours.
|
|
265
|
+
#
|
|
266
|
+
# `list_records` pages at the server's 1,000 cap. The filter is applied by
|
|
267
|
+
# the search path but not by the listing one, so it is applied here.
|
|
268
|
+
hits = self._list_matching(
|
|
269
|
+
table_name, dxmt_filter, op.limit + op.offset
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
# Convert hits to SearchItems
|
|
273
|
+
for hit in hits[op.offset : op.offset + op.limit]:
|
|
274
|
+
item = self._hit_to_search_item(hit, table_name)
|
|
275
|
+
if item:
|
|
276
|
+
all_results.append(item)
|
|
277
|
+
|
|
278
|
+
# If we have enough results, stop
|
|
279
|
+
if len(all_results) >= op.limit:
|
|
280
|
+
break
|
|
281
|
+
|
|
282
|
+
except DXMTError as e:
|
|
283
|
+
logger.warning(f"Search failed in table {table_name}: {e}")
|
|
284
|
+
continue
|
|
285
|
+
|
|
286
|
+
return all_results[: op.limit]
|
|
287
|
+
|
|
288
|
+
def _handle_put(self, op: PutOp) -> None:
|
|
289
|
+
"""Handle PutOp synchronously."""
|
|
290
|
+
table_name = self._namespace_to_table(op.namespace)
|
|
291
|
+
|
|
292
|
+
# Ensure table exists
|
|
293
|
+
self._ensure_table_exists_sync(table_name, create=True)
|
|
294
|
+
|
|
295
|
+
# If value is None, delete the item
|
|
296
|
+
if op.value is None:
|
|
297
|
+
record_id = self._make_record_id(op.namespace, op.key)
|
|
298
|
+
try:
|
|
299
|
+
self.client.delete_records(table_name, [record_id])
|
|
300
|
+
except DXMTError as e:
|
|
301
|
+
logger.error(f"Failed to delete item {op.namespace}/{op.key}: {e}")
|
|
302
|
+
raise
|
|
303
|
+
return
|
|
304
|
+
|
|
305
|
+
# Prepare record payload
|
|
306
|
+
now = datetime.now(timezone.utc)
|
|
307
|
+
payload = {
|
|
308
|
+
**op.value,
|
|
309
|
+
NAMESPACE_FIELD: list(op.namespace),
|
|
310
|
+
KEY_FIELD: op.key,
|
|
311
|
+
CREATED_AT_FIELD: now.isoformat(),
|
|
312
|
+
UPDATED_AT_FIELD: now.isoformat(),
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
# Check if record exists (for updating created_at)
|
|
316
|
+
record_id = self._make_record_id(op.namespace, op.key)
|
|
317
|
+
try:
|
|
318
|
+
existing_records = self.client.get_records(table_name, [record_id])
|
|
319
|
+
if existing_records:
|
|
320
|
+
# Preserve created_at from existing record
|
|
321
|
+
existing_payload = existing_records[0].get("payload", {})
|
|
322
|
+
if CREATED_AT_FIELD in existing_payload:
|
|
323
|
+
payload[CREATED_AT_FIELD] = existing_payload[CREATED_AT_FIELD]
|
|
324
|
+
except DXMTError:
|
|
325
|
+
# Record doesn't exist, use new created_at
|
|
326
|
+
pass
|
|
327
|
+
|
|
328
|
+
# Handle indexing
|
|
329
|
+
vector = None
|
|
330
|
+
if op.index is not False and self.embed_fn and self.embedding_dims:
|
|
331
|
+
# Determine fields to index
|
|
332
|
+
if op.index is None:
|
|
333
|
+
# Use default index fields
|
|
334
|
+
index_fields = self.default_index_fields
|
|
335
|
+
else:
|
|
336
|
+
# Use specified fields
|
|
337
|
+
index_fields = op.index
|
|
338
|
+
|
|
339
|
+
# Extract text from fields
|
|
340
|
+
texts_to_embed = self._extract_texts(op.value, index_fields)
|
|
341
|
+
|
|
342
|
+
if texts_to_embed:
|
|
343
|
+
# Store index fields in payload for reference
|
|
344
|
+
payload[INDEX_FIELDS_FIELD] = index_fields
|
|
345
|
+
|
|
346
|
+
# Generate embeddings
|
|
347
|
+
embeddings = self.embed_fn(texts_to_embed)
|
|
348
|
+
|
|
349
|
+
# Average embeddings if multiple texts
|
|
350
|
+
if len(embeddings) == 1:
|
|
351
|
+
vector = embeddings[0]
|
|
352
|
+
else:
|
|
353
|
+
# Average pooling
|
|
354
|
+
vector = [
|
|
355
|
+
sum(emb[i] for emb in embeddings) / len(embeddings)
|
|
356
|
+
for i in range(len(embeddings[0]))
|
|
357
|
+
]
|
|
358
|
+
|
|
359
|
+
# Prepare record for insertion
|
|
360
|
+
record = {
|
|
361
|
+
"id": record_id,
|
|
362
|
+
"payload": payload,
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
# Add vector (use dummy if not generated)
|
|
366
|
+
if vector is not None:
|
|
367
|
+
record["vector"] = vector
|
|
368
|
+
else:
|
|
369
|
+
# DXMT requires vectors for all records, use dummy non-zero vector if no embedding
|
|
370
|
+
# Zero vectors are not allowed, so use [1.0]
|
|
371
|
+
record["vector"] = [1.0] * (self.embedding_dims if self.embedding_dims else 1)
|
|
372
|
+
|
|
373
|
+
# Insert or update record (manual upsert: delete then insert)
|
|
374
|
+
try:
|
|
375
|
+
# Try to delete existing record first (ignore errors if it doesn't exist)
|
|
376
|
+
import time
|
|
377
|
+
try:
|
|
378
|
+
logger.debug(f"Attempting to delete record {record_id} from {table_name}")
|
|
379
|
+
result = self.client.delete_records(table_name, [record_id])
|
|
380
|
+
logger.debug(f"Delete result: {result}")
|
|
381
|
+
# Delay to ensure delete is fully processed on server
|
|
382
|
+
time.sleep(0.5)
|
|
383
|
+
except DXMTError as e:
|
|
384
|
+
logger.debug(f"Delete failed (record may not exist): {e}")
|
|
385
|
+
pass # Record doesn't exist, that's fine
|
|
386
|
+
|
|
387
|
+
# Now insert the record
|
|
388
|
+
# Try lock_free mode first (more lenient with duplicates)
|
|
389
|
+
logger.debug(f"Inserting record {record_id} into {table_name}")
|
|
390
|
+
try:
|
|
391
|
+
self.client.insert_records(table_name, [record])
|
|
392
|
+
except DXMTError as e:
|
|
393
|
+
# If lock_free fails with "already exists", try standard mode as fallback
|
|
394
|
+
if "already exists" in str(e).lower():
|
|
395
|
+
logger.debug(f"Record {record_id} still exists after delete, treating as updated")
|
|
396
|
+
# Workaround: Record persists in index, consider it updated
|
|
397
|
+
pass
|
|
398
|
+
else:
|
|
399
|
+
raise
|
|
400
|
+
except DXMTError as e:
|
|
401
|
+
logger.error(f"Failed to put item {op.namespace}/{op.key}: {e}")
|
|
402
|
+
raise
|
|
403
|
+
|
|
404
|
+
def _handle_list_namespaces(self, op: ListNamespacesOp) -> list[tuple[str, ...]]:
|
|
405
|
+
"""Handle ListNamespacesOp synchronously."""
|
|
406
|
+
# Get all tables
|
|
407
|
+
try:
|
|
408
|
+
tables = self.client.list_tables()
|
|
409
|
+
except DXMTError as e:
|
|
410
|
+
logger.error(f"Failed to list tables: {e}")
|
|
411
|
+
return []
|
|
412
|
+
|
|
413
|
+
# Extract namespaces from table names
|
|
414
|
+
namespaces: set[tuple[str, ...]] = set()
|
|
415
|
+
|
|
416
|
+
for table in tables:
|
|
417
|
+
table_name = table.get("name", "")
|
|
418
|
+
if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
|
|
419
|
+
continue
|
|
420
|
+
|
|
421
|
+
namespace = self._table_to_namespace(table_name)
|
|
422
|
+
|
|
423
|
+
# Apply match conditions
|
|
424
|
+
if op.match_conditions:
|
|
425
|
+
if not self._matches_conditions(namespace, op.match_conditions):
|
|
426
|
+
continue
|
|
427
|
+
|
|
428
|
+
# Apply max_depth
|
|
429
|
+
if op.max_depth is not None and len(namespace) > op.max_depth:
|
|
430
|
+
namespace = namespace[: op.max_depth]
|
|
431
|
+
|
|
432
|
+
namespaces.add(namespace)
|
|
433
|
+
|
|
434
|
+
# Sort and paginate
|
|
435
|
+
sorted_namespaces = sorted(namespaces)
|
|
436
|
+
start = op.offset
|
|
437
|
+
end = op.offset + op.limit
|
|
438
|
+
return sorted_namespaces[start:end]
|
|
439
|
+
|
|
440
|
+
# Operation Handlers (Async)
|
|
441
|
+
|
|
442
|
+
async def _ahandle_get(self, op: GetOp) -> Item | None:
|
|
443
|
+
"""Handle GetOp asynchronously."""
|
|
444
|
+
table_name = self._namespace_to_table(op.namespace)
|
|
445
|
+
|
|
446
|
+
# Ensure table exists
|
|
447
|
+
if not await self._ensure_table_exists_async(table_name):
|
|
448
|
+
return None
|
|
449
|
+
|
|
450
|
+
try:
|
|
451
|
+
# Get record by constructing record_id from namespace + key
|
|
452
|
+
record_id = self._make_record_id(op.namespace, op.key)
|
|
453
|
+
records = await self.client.aget_records(table_name, [record_id])
|
|
454
|
+
|
|
455
|
+
if not records:
|
|
456
|
+
return None
|
|
457
|
+
|
|
458
|
+
record = records[0]
|
|
459
|
+
return self._record_to_item(record)
|
|
460
|
+
|
|
461
|
+
except DXMTError as e:
|
|
462
|
+
logger.warning(f"Failed to get item {op.namespace}/{op.key}: {e}")
|
|
463
|
+
return None
|
|
464
|
+
|
|
465
|
+
async def _ahandle_search(self, op: SearchOp) -> list[SearchItem]:
|
|
466
|
+
"""Handle SearchOp asynchronously."""
|
|
467
|
+
# Determine which tables to search based on namespace prefix
|
|
468
|
+
tables = await self._get_matching_tables_async(op.namespace_prefix)
|
|
469
|
+
|
|
470
|
+
if not tables:
|
|
471
|
+
return []
|
|
472
|
+
|
|
473
|
+
all_results: list[SearchItem] = []
|
|
474
|
+
|
|
475
|
+
for table_name in tables:
|
|
476
|
+
try:
|
|
477
|
+
# Convert filters
|
|
478
|
+
dxmt_filter = self._convert_filter(op.filter) if op.filter else None
|
|
479
|
+
|
|
480
|
+
# If query is provided, use semantic search
|
|
481
|
+
if op.query and self.aembed_fn and self.embedding_dims:
|
|
482
|
+
# Generate embedding
|
|
483
|
+
embeddings = await self.aembed_fn([op.query])
|
|
484
|
+
query_vector = embeddings[0]
|
|
485
|
+
|
|
486
|
+
# Perform vector search
|
|
487
|
+
hits = await self.client.asearch(
|
|
488
|
+
table_name=table_name,
|
|
489
|
+
vector=query_vector,
|
|
490
|
+
limit=op.limit + op.offset,
|
|
491
|
+
filter=dxmt_filter,
|
|
492
|
+
)
|
|
493
|
+
elif op.query and self.embed_fn and self.embedding_dims:
|
|
494
|
+
# Use sync embedding function in async context
|
|
495
|
+
embeddings = self.embed_fn([op.query])
|
|
496
|
+
query_vector = embeddings[0]
|
|
497
|
+
|
|
498
|
+
# Perform vector search
|
|
499
|
+
hits = await self.client.asearch(
|
|
500
|
+
table_name=table_name,
|
|
501
|
+
vector=query_vector,
|
|
502
|
+
limit=op.limit + op.offset,
|
|
503
|
+
filter=dxmt_filter,
|
|
504
|
+
)
|
|
505
|
+
else:
|
|
506
|
+
# Metadata-only search
|
|
507
|
+
logger.warning(
|
|
508
|
+
"Metadata-only search not fully optimized. "
|
|
509
|
+
"Consider using query parameter for better performance."
|
|
510
|
+
)
|
|
511
|
+
continue
|
|
512
|
+
|
|
513
|
+
# Convert hits to SearchItems
|
|
514
|
+
for hit in hits[op.offset : op.offset + op.limit]:
|
|
515
|
+
item = self._hit_to_search_item(hit, table_name)
|
|
516
|
+
if item:
|
|
517
|
+
all_results.append(item)
|
|
518
|
+
|
|
519
|
+
# If we have enough results, stop
|
|
520
|
+
if len(all_results) >= op.limit:
|
|
521
|
+
break
|
|
522
|
+
|
|
523
|
+
except DXMTError as e:
|
|
524
|
+
logger.warning(f"Search failed in table {table_name}: {e}")
|
|
525
|
+
continue
|
|
526
|
+
|
|
527
|
+
return all_results[: op.limit]
|
|
528
|
+
|
|
529
|
+
async def _ahandle_put(self, op: PutOp) -> None:
|
|
530
|
+
"""Handle PutOp asynchronously."""
|
|
531
|
+
table_name = self._namespace_to_table(op.namespace)
|
|
532
|
+
|
|
533
|
+
# Ensure table exists
|
|
534
|
+
await self._ensure_table_exists_async(table_name, create=True)
|
|
535
|
+
|
|
536
|
+
# If value is None, delete the item
|
|
537
|
+
if op.value is None:
|
|
538
|
+
record_id = self._make_record_id(op.namespace, op.key)
|
|
539
|
+
try:
|
|
540
|
+
await self.client.adelete_records(table_name, [record_id])
|
|
541
|
+
except DXMTError as e:
|
|
542
|
+
logger.error(f"Failed to delete item {op.namespace}/{op.key}: {e}")
|
|
543
|
+
raise
|
|
544
|
+
return
|
|
545
|
+
|
|
546
|
+
# Prepare record payload
|
|
547
|
+
now = datetime.now(timezone.utc)
|
|
548
|
+
payload = {
|
|
549
|
+
**op.value,
|
|
550
|
+
NAMESPACE_FIELD: list(op.namespace),
|
|
551
|
+
KEY_FIELD: op.key,
|
|
552
|
+
CREATED_AT_FIELD: now.isoformat(),
|
|
553
|
+
UPDATED_AT_FIELD: now.isoformat(),
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
# Check if record exists (for updating created_at)
|
|
557
|
+
record_id = self._make_record_id(op.namespace, op.key)
|
|
558
|
+
try:
|
|
559
|
+
existing_records = await self.client.aget_records(table_name, [record_id])
|
|
560
|
+
if existing_records:
|
|
561
|
+
# Preserve created_at from existing record
|
|
562
|
+
existing_payload = existing_records[0].get("payload", {})
|
|
563
|
+
if CREATED_AT_FIELD in existing_payload:
|
|
564
|
+
payload[CREATED_AT_FIELD] = existing_payload[CREATED_AT_FIELD]
|
|
565
|
+
except DXMTError:
|
|
566
|
+
# Record doesn't exist, use new created_at
|
|
567
|
+
pass
|
|
568
|
+
|
|
569
|
+
# Handle indexing
|
|
570
|
+
vector = None
|
|
571
|
+
if op.index is not False and self.embedding_dims:
|
|
572
|
+
# Determine fields to index
|
|
573
|
+
if op.index is None:
|
|
574
|
+
index_fields = self.default_index_fields
|
|
575
|
+
else:
|
|
576
|
+
index_fields = op.index
|
|
577
|
+
|
|
578
|
+
# Extract text from fields
|
|
579
|
+
texts_to_embed = self._extract_texts(op.value, index_fields)
|
|
580
|
+
|
|
581
|
+
if texts_to_embed:
|
|
582
|
+
payload[INDEX_FIELDS_FIELD] = index_fields
|
|
583
|
+
|
|
584
|
+
# Generate embeddings
|
|
585
|
+
if self.aembed_fn:
|
|
586
|
+
embeddings = await self.aembed_fn(texts_to_embed)
|
|
587
|
+
elif self.embed_fn:
|
|
588
|
+
embeddings = self.embed_fn(texts_to_embed)
|
|
589
|
+
else:
|
|
590
|
+
embeddings = []
|
|
591
|
+
|
|
592
|
+
# Average embeddings if multiple texts
|
|
593
|
+
if len(embeddings) == 1:
|
|
594
|
+
vector = embeddings[0]
|
|
595
|
+
elif len(embeddings) > 1:
|
|
596
|
+
vector = [
|
|
597
|
+
sum(emb[i] for emb in embeddings) / len(embeddings)
|
|
598
|
+
for i in range(len(embeddings[0]))
|
|
599
|
+
]
|
|
600
|
+
|
|
601
|
+
# Prepare record for insertion
|
|
602
|
+
record = {
|
|
603
|
+
"id": record_id,
|
|
604
|
+
"payload": payload,
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
# Add vector (use dummy if not generated)
|
|
608
|
+
if vector is not None:
|
|
609
|
+
record["vector"] = vector
|
|
610
|
+
else:
|
|
611
|
+
# DXMT requires vectors for all records, use dummy non-zero vector if no embedding
|
|
612
|
+
# Zero vectors are not allowed, so use [1.0]
|
|
613
|
+
record["vector"] = [1.0] * (self.embedding_dims if self.embedding_dims else 1)
|
|
614
|
+
|
|
615
|
+
# Insert or update record (manual upsert: delete then insert)
|
|
616
|
+
try:
|
|
617
|
+
# Try to delete existing record first (ignore errors if it doesn't exist)
|
|
618
|
+
import asyncio
|
|
619
|
+
try:
|
|
620
|
+
logger.debug(f"Attempting to delete record {record_id} from {table_name}")
|
|
621
|
+
result = await self.client.adelete_records(table_name, [record_id])
|
|
622
|
+
logger.debug(f"Delete result: {result}")
|
|
623
|
+
# Delay to ensure delete is fully processed on server
|
|
624
|
+
await asyncio.sleep(0.5)
|
|
625
|
+
except DXMTError as e:
|
|
626
|
+
logger.debug(f"Delete failed (record may not exist): {e}")
|
|
627
|
+
pass # Record doesn't exist, that's fine
|
|
628
|
+
|
|
629
|
+
# Now insert the record
|
|
630
|
+
# Try lock_free mode first (more lenient with duplicates)
|
|
631
|
+
logger.debug(f"Inserting record {record_id} into {table_name}")
|
|
632
|
+
try:
|
|
633
|
+
await self.client.ainsert_records(table_name, [record])
|
|
634
|
+
except DXMTError as e:
|
|
635
|
+
# If lock_free fails with "already exists", try standard mode as fallback
|
|
636
|
+
if "already exists" in str(e).lower():
|
|
637
|
+
logger.debug(f"Record {record_id} still exists after delete, treating as updated")
|
|
638
|
+
# Workaround: Record persists in index, consider it updated
|
|
639
|
+
pass
|
|
640
|
+
else:
|
|
641
|
+
raise
|
|
642
|
+
except DXMTError as e:
|
|
643
|
+
logger.error(f"Failed to put item {op.namespace}/{op.key}: {e}")
|
|
644
|
+
raise
|
|
645
|
+
|
|
646
|
+
async def _ahandle_list_namespaces(
|
|
647
|
+
self, op: ListNamespacesOp
|
|
648
|
+
) -> list[tuple[str, ...]]:
|
|
649
|
+
"""Handle ListNamespacesOp asynchronously."""
|
|
650
|
+
# Get all tables
|
|
651
|
+
try:
|
|
652
|
+
tables = await self.client.alist_tables()
|
|
653
|
+
except DXMTError as e:
|
|
654
|
+
logger.error(f"Failed to list tables: {e}")
|
|
655
|
+
return []
|
|
656
|
+
|
|
657
|
+
# Extract namespaces from table names
|
|
658
|
+
namespaces: set[tuple[str, ...]] = set()
|
|
659
|
+
|
|
660
|
+
for table in tables:
|
|
661
|
+
table_name = table.get("name", "")
|
|
662
|
+
if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
|
|
663
|
+
continue
|
|
664
|
+
|
|
665
|
+
namespace = self._table_to_namespace(table_name)
|
|
666
|
+
|
|
667
|
+
# Apply match conditions
|
|
668
|
+
if op.match_conditions:
|
|
669
|
+
if not self._matches_conditions(namespace, op.match_conditions):
|
|
670
|
+
continue
|
|
671
|
+
|
|
672
|
+
# Apply max_depth
|
|
673
|
+
if op.max_depth is not None and len(namespace) > op.max_depth:
|
|
674
|
+
namespace = namespace[: op.max_depth]
|
|
675
|
+
|
|
676
|
+
namespaces.add(namespace)
|
|
677
|
+
|
|
678
|
+
# Sort and paginate
|
|
679
|
+
sorted_namespaces = sorted(namespaces)
|
|
680
|
+
start = op.offset
|
|
681
|
+
end = op.offset + op.limit
|
|
682
|
+
return sorted_namespaces[start:end]
|
|
683
|
+
|
|
684
|
+
# Helper Methods
|
|
685
|
+
|
|
686
|
+
def _namespace_to_table(self, namespace: tuple[str, ...]) -> str:
|
|
687
|
+
"""Convert namespace tuple to table name.
|
|
688
|
+
|
|
689
|
+
A table name may hold letters, digits and underscores only, so the separator has to be
|
|
690
|
+
an underscore -- and a namespace component may contain one. Joining and splitting on a
|
|
691
|
+
bare `_` therefore did not round-trip: ("agent", "memory", "session_123") came back as
|
|
692
|
+
four components, so `list_namespaces` reported a namespace nobody had written and a
|
|
693
|
+
prefix search for it matched nothing. Worse, ("a_b",) and ("a", "b") named the *same*
|
|
694
|
+
table, quietly merging two namespaces.
|
|
695
|
+
|
|
696
|
+
So an underscore inside a component is doubled and a single underscore separates:
|
|
697
|
+
decoding reads left to right, which makes it unambiguous. A namespace without
|
|
698
|
+
underscores maps to exactly the name it always did.
|
|
699
|
+
"""
|
|
700
|
+
encoded = "_".join(component.replace("_", "__") for component in namespace)
|
|
701
|
+
table_name = f"{NAMESPACE_TABLE_PREFIX}{encoded}"
|
|
702
|
+
if len(table_name) > MAX_TABLE_NAME_LENGTH:
|
|
703
|
+
# The server refuses it, and the error it gives names a table the caller never
|
|
704
|
+
# chose. Say what was too long instead.
|
|
705
|
+
raise ValueError(
|
|
706
|
+
f"namespace {namespace} maps to a table name of {len(table_name)} characters, "
|
|
707
|
+
f"over the server's limit of {MAX_TABLE_NAME_LENGTH}; use shorter components"
|
|
708
|
+
)
|
|
709
|
+
return table_name
|
|
710
|
+
|
|
711
|
+
def _table_to_namespace(self, table_name: str) -> tuple[str, ...]:
|
|
712
|
+
"""Convert table name back to namespace tuple. The inverse of the above."""
|
|
713
|
+
if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
|
|
714
|
+
return ()
|
|
715
|
+
|
|
716
|
+
encoded = table_name[len(NAMESPACE_TABLE_PREFIX) :]
|
|
717
|
+
components: list[str] = []
|
|
718
|
+
current: list[str] = []
|
|
719
|
+
index = 0
|
|
720
|
+
while index < len(encoded):
|
|
721
|
+
char = encoded[index]
|
|
722
|
+
if char == "_":
|
|
723
|
+
if index + 1 < len(encoded) and encoded[index + 1] == "_":
|
|
724
|
+
current.append("_")
|
|
725
|
+
index += 2
|
|
726
|
+
continue
|
|
727
|
+
components.append("".join(current))
|
|
728
|
+
current = []
|
|
729
|
+
index += 1
|
|
730
|
+
continue
|
|
731
|
+
current.append(char)
|
|
732
|
+
index += 1
|
|
733
|
+
components.append("".join(current))
|
|
734
|
+
return tuple(components)
|
|
735
|
+
|
|
736
|
+
def _make_record_id(self, namespace: tuple[str, ...], key: str) -> str:
|
|
737
|
+
"""Generate unique record ID from namespace and key."""
|
|
738
|
+
# Use hash to create stable ID
|
|
739
|
+
namespace_str = "/".join(namespace)
|
|
740
|
+
id_str = f"{namespace_str}/{key}"
|
|
741
|
+
# Use first 16 chars of SHA256 hash for shorter IDs
|
|
742
|
+
return hashlib.sha256(id_str.encode()).hexdigest()[:16]
|
|
743
|
+
|
|
744
|
+
def _check_dimension(self, table_name: str, table: dict[str, Any]) -> None:
|
|
745
|
+
"""Refuse a table whose vectors are a different width from the ones this store writes.
|
|
746
|
+
|
|
747
|
+
A namespace is a table, and the table's dimension is whichever store reached it first.
|
|
748
|
+
A store with no `embed_fn` writes a one-element placeholder, so it creates the table at
|
|
749
|
+
dimension 1 -- and a store configured with real embeddings then failed on *every* put
|
|
750
|
+
with the server's own "expected 384, got 1", which names neither the namespace nor the
|
|
751
|
+
other store. Both directions are the same mistake and both are worth naming.
|
|
752
|
+
"""
|
|
753
|
+
writes = self.embedding_dims if self.embedding_dims else 1
|
|
754
|
+
found = table.get("dimension")
|
|
755
|
+
if found is None or found == writes:
|
|
756
|
+
return
|
|
757
|
+
raise DXMTError(
|
|
758
|
+
f"table '{table_name}' holds {found}-dimension vectors, but this store writes "
|
|
759
|
+
f"{writes}: the namespace was first used by a store configured with "
|
|
760
|
+
+ (
|
|
761
|
+
f"embedding_dims={found}"
|
|
762
|
+
if found != 1
|
|
763
|
+
else "no embed_fn (which writes a 1-element placeholder)"
|
|
764
|
+
)
|
|
765
|
+
+ ". Use one embedding configuration per namespace, or a different namespace."
|
|
766
|
+
)
|
|
767
|
+
|
|
768
|
+
def _ensure_table_exists_sync(self, table_name: str, create: bool = False) -> bool:
|
|
769
|
+
"""Ensure table exists synchronously. Returns True if exists/created."""
|
|
770
|
+
if table_name in self._created_tables:
|
|
771
|
+
return True
|
|
772
|
+
|
|
773
|
+
try:
|
|
774
|
+
self._check_dimension(table_name, self.client.get_table(table_name))
|
|
775
|
+
self._created_tables.add(table_name)
|
|
776
|
+
return True
|
|
777
|
+
except DXMTTableError:
|
|
778
|
+
if not create:
|
|
779
|
+
return False
|
|
780
|
+
|
|
781
|
+
# Table doesn't exist, create it
|
|
782
|
+
# Use configured dimension or default to 1 for non-vector storage
|
|
783
|
+
dimension = self.embedding_dims if self.embedding_dims else 1
|
|
784
|
+
try:
|
|
785
|
+
self.client.create_table(
|
|
786
|
+
name=table_name,
|
|
787
|
+
dimension=dimension,
|
|
788
|
+
distance_metric=self.distance_metric,
|
|
789
|
+
index_type="hnsw",
|
|
790
|
+
)
|
|
791
|
+
self._created_tables.add(table_name)
|
|
792
|
+
return True
|
|
793
|
+
except DXMTError as e:
|
|
794
|
+
logger.error(f"Failed to create table {table_name}: {e}")
|
|
795
|
+
return False
|
|
796
|
+
|
|
797
|
+
async def _ensure_table_exists_async(
|
|
798
|
+
self, table_name: str, create: bool = False
|
|
799
|
+
) -> bool:
|
|
800
|
+
"""Ensure table exists asynchronously. Returns True if exists/created."""
|
|
801
|
+
if table_name in self._created_tables:
|
|
802
|
+
return True
|
|
803
|
+
|
|
804
|
+
async with self._table_lock:
|
|
805
|
+
# Double-check after acquiring lock
|
|
806
|
+
if table_name in self._created_tables:
|
|
807
|
+
return True
|
|
808
|
+
|
|
809
|
+
try:
|
|
810
|
+
self._check_dimension(table_name, await self.client.aget_table(table_name))
|
|
811
|
+
self._created_tables.add(table_name)
|
|
812
|
+
return True
|
|
813
|
+
except DXMTTableError:
|
|
814
|
+
if not create:
|
|
815
|
+
return False
|
|
816
|
+
|
|
817
|
+
# Table doesn't exist, create it
|
|
818
|
+
# Use configured dimension or default to 1 for non-vector storage
|
|
819
|
+
dimension = self.embedding_dims if self.embedding_dims else 1
|
|
820
|
+
try:
|
|
821
|
+
await self.client.acreate_table(
|
|
822
|
+
name=table_name,
|
|
823
|
+
dimension=dimension,
|
|
824
|
+
distance_metric=self.distance_metric,
|
|
825
|
+
index_type="hnsw",
|
|
826
|
+
)
|
|
827
|
+
self._created_tables.add(table_name)
|
|
828
|
+
return True
|
|
829
|
+
except DXMTError as e:
|
|
830
|
+
logger.error(f"Failed to create table {table_name}: {e}")
|
|
831
|
+
return False
|
|
832
|
+
|
|
833
|
+
def _get_matching_tables_sync(
|
|
834
|
+
self, namespace_prefix: tuple[str, ...]
|
|
835
|
+
) -> list[str]:
|
|
836
|
+
"""Get tables matching namespace prefix synchronously."""
|
|
837
|
+
try:
|
|
838
|
+
all_tables = self.client.list_tables()
|
|
839
|
+
except DXMTError as e:
|
|
840
|
+
logger.error(f"Failed to list tables: {e}")
|
|
841
|
+
return []
|
|
842
|
+
|
|
843
|
+
matching_tables = []
|
|
844
|
+
for table in all_tables:
|
|
845
|
+
table_name = table.get("name", "")
|
|
846
|
+
if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
|
|
847
|
+
continue
|
|
848
|
+
|
|
849
|
+
namespace = self._table_to_namespace(table_name)
|
|
850
|
+
|
|
851
|
+
# Check if namespace starts with the prefix
|
|
852
|
+
if len(namespace) >= len(namespace_prefix):
|
|
853
|
+
if namespace[: len(namespace_prefix)] == namespace_prefix:
|
|
854
|
+
matching_tables.append(table_name)
|
|
855
|
+
|
|
856
|
+
return matching_tables
|
|
857
|
+
|
|
858
|
+
async def _get_matching_tables_async(
|
|
859
|
+
self, namespace_prefix: tuple[str, ...]
|
|
860
|
+
) -> list[str]:
|
|
861
|
+
"""Get tables matching namespace prefix asynchronously."""
|
|
862
|
+
try:
|
|
863
|
+
all_tables = await self.client.alist_tables()
|
|
864
|
+
except DXMTError as e:
|
|
865
|
+
logger.error(f"Failed to list tables: {e}")
|
|
866
|
+
return []
|
|
867
|
+
|
|
868
|
+
matching_tables = []
|
|
869
|
+
for table in all_tables:
|
|
870
|
+
table_name = table.get("name", "")
|
|
871
|
+
if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
|
|
872
|
+
continue
|
|
873
|
+
|
|
874
|
+
namespace = self._table_to_namespace(table_name)
|
|
875
|
+
|
|
876
|
+
# Check if namespace starts with the prefix
|
|
877
|
+
if len(namespace) >= len(namespace_prefix):
|
|
878
|
+
if namespace[: len(namespace_prefix)] == namespace_prefix:
|
|
879
|
+
matching_tables.append(table_name)
|
|
880
|
+
|
|
881
|
+
return matching_tables
|
|
882
|
+
|
|
883
|
+
def _convert_filter(self, filter_dict: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
884
|
+
"""Convert LangGraph filter to DXMT filter format.
|
|
885
|
+
|
|
886
|
+
Supports simple equality, MongoDB-style operators ($eq, $ne, $gt,
|
|
887
|
+
$gte, $lt, $lte, $in, $nin, $contains, $startswith, $endswith) and
|
|
888
|
+
logical operators ($and, $or, $not). Delegates to the shared
|
|
889
|
+
convert_simple_filter_to_dxmt utility so all integrations stay in sync.
|
|
890
|
+
"""
|
|
891
|
+
return convert_simple_filter_to_dxmt(filter_dict)
|
|
892
|
+
|
|
893
|
+
def _extract_texts(self, value: dict[str, Any], fields: list[str]) -> list[str]:
|
|
894
|
+
"""Extract text from value dict using field paths."""
|
|
895
|
+
texts: list[str] = []
|
|
896
|
+
|
|
897
|
+
for field in fields:
|
|
898
|
+
try:
|
|
899
|
+
# Use LangGraph's get_text_at_path helper
|
|
900
|
+
extracted = get_text_at_path(value, field)
|
|
901
|
+
texts.extend(extracted)
|
|
902
|
+
except Exception as e:
|
|
903
|
+
logger.warning(f"Failed to extract text from field {field}: {e}")
|
|
904
|
+
continue
|
|
905
|
+
|
|
906
|
+
return texts
|
|
907
|
+
|
|
908
|
+
def _record_to_item(self, record: dict[str, Any]) -> Item:
|
|
909
|
+
"""Convert DXMT record to LangGraph Item."""
|
|
910
|
+
payload = record.get("payload", {})
|
|
911
|
+
|
|
912
|
+
# Extract LangGraph fields
|
|
913
|
+
namespace = tuple(payload.get(NAMESPACE_FIELD, []))
|
|
914
|
+
key = payload.get(KEY_FIELD, "")
|
|
915
|
+
created_at_str = payload.get(CREATED_AT_FIELD)
|
|
916
|
+
updated_at_str = payload.get(UPDATED_AT_FIELD)
|
|
917
|
+
|
|
918
|
+
# Parse timestamps
|
|
919
|
+
created_at = (
|
|
920
|
+
datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
|
|
921
|
+
if created_at_str
|
|
922
|
+
else datetime.now(timezone.utc)
|
|
923
|
+
)
|
|
924
|
+
updated_at = (
|
|
925
|
+
datetime.fromisoformat(updated_at_str.replace("Z", "+00:00"))
|
|
926
|
+
if updated_at_str
|
|
927
|
+
else datetime.now(timezone.utc)
|
|
928
|
+
)
|
|
929
|
+
|
|
930
|
+
# Remove internal fields from value
|
|
931
|
+
value = {
|
|
932
|
+
k: v
|
|
933
|
+
for k, v in payload.items()
|
|
934
|
+
if k
|
|
935
|
+
not in (
|
|
936
|
+
NAMESPACE_FIELD,
|
|
937
|
+
KEY_FIELD,
|
|
938
|
+
CREATED_AT_FIELD,
|
|
939
|
+
UPDATED_AT_FIELD,
|
|
940
|
+
INDEX_FIELDS_FIELD,
|
|
941
|
+
)
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
return Item(
|
|
945
|
+
value=value,
|
|
946
|
+
key=key,
|
|
947
|
+
namespace=namespace,
|
|
948
|
+
created_at=created_at,
|
|
949
|
+
updated_at=updated_at,
|
|
950
|
+
)
|
|
951
|
+
|
|
952
|
+
def _hit_to_search_item(
|
|
953
|
+
self, hit: dict[str, Any], table_name: str
|
|
954
|
+
) -> SearchItem | None:
|
|
955
|
+
"""Convert DXMT search hit to LangGraph SearchItem."""
|
|
956
|
+
try:
|
|
957
|
+
payload = hit.get("payload", {}) or hit.get("metadata", {})
|
|
958
|
+
score = hit.get("score", 0.0)
|
|
959
|
+
|
|
960
|
+
# Extract LangGraph fields
|
|
961
|
+
namespace = tuple(payload.get(NAMESPACE_FIELD, []))
|
|
962
|
+
key = payload.get(KEY_FIELD, "")
|
|
963
|
+
created_at_str = payload.get(CREATED_AT_FIELD)
|
|
964
|
+
updated_at_str = payload.get(UPDATED_AT_FIELD)
|
|
965
|
+
|
|
966
|
+
# Parse timestamps
|
|
967
|
+
created_at = (
|
|
968
|
+
datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
|
|
969
|
+
if created_at_str
|
|
970
|
+
else datetime.now(timezone.utc)
|
|
971
|
+
)
|
|
972
|
+
updated_at = (
|
|
973
|
+
datetime.fromisoformat(updated_at_str.replace("Z", "+00:00"))
|
|
974
|
+
if updated_at_str
|
|
975
|
+
else datetime.now(timezone.utc)
|
|
976
|
+
)
|
|
977
|
+
|
|
978
|
+
# Remove internal fields from value
|
|
979
|
+
value = {
|
|
980
|
+
k: v
|
|
981
|
+
for k, v in payload.items()
|
|
982
|
+
if k
|
|
983
|
+
not in (
|
|
984
|
+
NAMESPACE_FIELD,
|
|
985
|
+
KEY_FIELD,
|
|
986
|
+
CREATED_AT_FIELD,
|
|
987
|
+
UPDATED_AT_FIELD,
|
|
988
|
+
INDEX_FIELDS_FIELD,
|
|
989
|
+
)
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
return SearchItem(
|
|
993
|
+
namespace=namespace,
|
|
994
|
+
key=key,
|
|
995
|
+
value=value,
|
|
996
|
+
created_at=created_at,
|
|
997
|
+
updated_at=updated_at,
|
|
998
|
+
score=score,
|
|
999
|
+
)
|
|
1000
|
+
except Exception as e:
|
|
1001
|
+
logger.warning(f"Failed to convert hit to SearchItem: {e}")
|
|
1002
|
+
return None
|
|
1003
|
+
|
|
1004
|
+
def _matches_conditions(
|
|
1005
|
+
self,
|
|
1006
|
+
namespace: tuple[str, ...],
|
|
1007
|
+
conditions: tuple[MatchCondition, ...],
|
|
1008
|
+
) -> bool:
|
|
1009
|
+
"""Check if namespace matches all conditions."""
|
|
1010
|
+
for condition in conditions:
|
|
1011
|
+
if condition.match_type == "prefix":
|
|
1012
|
+
# Check if namespace starts with path (accounting for wildcards)
|
|
1013
|
+
if not self._matches_path(namespace, condition.path, prefix=True):
|
|
1014
|
+
return False
|
|
1015
|
+
elif condition.match_type == "suffix":
|
|
1016
|
+
# Check if namespace ends with path (accounting for wildcards)
|
|
1017
|
+
if not self._matches_path(namespace, condition.path, prefix=False):
|
|
1018
|
+
return False
|
|
1019
|
+
|
|
1020
|
+
return True
|
|
1021
|
+
|
|
1022
|
+
def _matches_path(
|
|
1023
|
+
self,
|
|
1024
|
+
namespace: tuple[str, ...],
|
|
1025
|
+
path: tuple[str | Literal["*"], ...],
|
|
1026
|
+
prefix: bool,
|
|
1027
|
+
) -> bool:
|
|
1028
|
+
"""Check if namespace matches path pattern."""
|
|
1029
|
+
if prefix:
|
|
1030
|
+
# Prefix match
|
|
1031
|
+
if len(namespace) < len(path):
|
|
1032
|
+
return False
|
|
1033
|
+
|
|
1034
|
+
for i, segment in enumerate(path):
|
|
1035
|
+
if segment == "*":
|
|
1036
|
+
continue
|
|
1037
|
+
if namespace[i] != segment:
|
|
1038
|
+
return False
|
|
1039
|
+
return True
|
|
1040
|
+
else:
|
|
1041
|
+
# Suffix match
|
|
1042
|
+
if len(namespace) < len(path):
|
|
1043
|
+
return False
|
|
1044
|
+
|
|
1045
|
+
offset = len(namespace) - len(path)
|
|
1046
|
+
for i, segment in enumerate(path):
|
|
1047
|
+
if segment == "*":
|
|
1048
|
+
continue
|
|
1049
|
+
if namespace[offset + i] != segment:
|
|
1050
|
+
return False
|
|
1051
|
+
return True
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
__all__ = ["DXMTLangGraphStore"]
|