dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1054 @@
1
+ """DXMT VectorDB integration for LangGraph Store.
2
+
3
+ This module provides a LangGraph-compatible persistent store implementation
4
+ backed by DXMT VectorDB, enabling AI agents to store and retrieve memories
5
+ with optional semantic search capabilities.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ import hashlib
12
+ import logging
13
+ from collections.abc import Iterable
14
+ from datetime import datetime, timezone
15
+ from typing import Any, Literal, cast
16
+
17
+ from langgraph.store.base import (
18
+ BaseStore,
19
+ GetOp,
20
+ Item,
21
+ ListNamespacesOp,
22
+ MatchCondition,
23
+ Op,
24
+ PutOp,
25
+ Result,
26
+ SearchItem,
27
+ SearchOp,
28
+ get_text_at_path,
29
+ )
30
+
31
+ from .client import DXMTClient
32
+ from .exceptions import DXMTError, DXMTTableError
33
+ from .filters import convert_simple_filter_to_dxmt
34
+
35
+ logger = logging.getLogger(__name__)
36
+
37
+
38
+ # Constants
39
+ MAX_TABLE_NAME_LENGTH = 63 # the server's own limit, in src/utils/mod.rs
40
+ NAMESPACE_TABLE_PREFIX = "langgraph_"
41
+ CREATED_AT_FIELD = "_created_at"
42
+ UPDATED_AT_FIELD = "_updated_at"
43
+ NAMESPACE_FIELD = "_namespace"
44
+ KEY_FIELD = "_key"
45
+ INDEX_FIELDS_FIELD = "_index_fields"
46
+
47
+
48
+ class DXMTLangGraphStore(BaseStore):
49
+ """LangGraph Store implementation backed by DXMT VectorDB.
50
+
51
+ This store provides persistent key-value storage with optional semantic search
52
+ capabilities for AI agents using LangGraph. It leverages DXMT's cost-efficient
53
+ tiered storage (Memory → SSD → S3) while maintaining high performance.
54
+
55
+ Key Features:
56
+ - Hierarchical namespace organization (tuple paths)
57
+ - Semantic search via embeddings (optional)
58
+ - Metadata filtering with comparison operators
59
+ - Efficient batch operations
60
+ - Both sync and async APIs
61
+
62
+ Args:
63
+ client: DXMTClient instance for database communication.
64
+ embedding_dims: Vector dimension for semantic search. Required if using query-based search.
65
+ embed_fn: Function to generate embeddings from text. Signature: f(texts: list[str]) -> list[list[float]]
66
+ aembed_fn: Async embedding function. Signature: async f(texts: list[str]) -> list[list[float]]
67
+ default_index_fields: Default fields to extract for indexing. Defaults to ["$"] (whole document).
68
+ distance_metric: Distance metric for vector search. Options: "cosine", "euclidean", "dot".
69
+ namespace_strategy: Strategy for mapping namespaces to tables.
70
+ - "table": Each namespace → separate table (recommended, better isolation)
71
+ - "metadata": Single table with namespace stored in metadata (compact, fewer tables)
72
+
73
+ Examples:
74
+ Basic usage without semantic search:
75
+
76
+ >>> client = DXMTClient(host="localhost", port=9999)
77
+ >>> store = DXMTLangGraphStore(client)
78
+ >>> store.put(("users", "profiles"), "user123", {"name": "Alice", "role": "admin"})
79
+ >>> item = store.get(("users", "profiles"), "user123")
80
+
81
+ With semantic search:
82
+
83
+ >>> from langchain_openai import OpenAIEmbeddings
84
+ >>> embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
85
+ >>>
86
+ >>> store = DXMTLangGraphStore(
87
+ ... client=client,
88
+ ... embedding_dims=1536,
89
+ ... embed_fn=lambda texts: embeddings.embed_documents(texts)
90
+ ... )
91
+ >>> store.put(("docs",), "doc1", {"content": "AI agents use LangGraph for workflows"}, index=["content"])
92
+ >>> results = store.search(("docs",), query="workflows for AI")
93
+ """
94
+
95
+ def __init__(
96
+ self,
97
+ client: DXMTClient,
98
+ *,
99
+ embedding_dims: int | None = None,
100
+ embed_fn: Any = None,
101
+ aembed_fn: Any = None,
102
+ default_index_fields: list[str] | None = None,
103
+ distance_metric: str = "cosine",
104
+ namespace_strategy: Literal["table", "metadata"] = "table",
105
+ ):
106
+ """Initialize DXMT LangGraph Store."""
107
+ self.client = client
108
+ self.embedding_dims = embedding_dims
109
+ self.embed_fn = embed_fn
110
+ self.aembed_fn = aembed_fn
111
+ self.default_index_fields = default_index_fields or ["$"]
112
+ self.distance_metric = distance_metric
113
+ self.namespace_strategy = namespace_strategy
114
+
115
+ # Validate configuration
116
+ if embedding_dims is not None and embedding_dims <= 0:
117
+ raise ValueError(f"embedding_dims must be positive, got {embedding_dims}")
118
+
119
+ # Track created tables to avoid redundant creation checks
120
+ self._created_tables: set[str] = set()
121
+ self._table_lock = asyncio.Lock()
122
+
123
+ def batch(self, ops: Iterable[Op]) -> list[Result]:
124
+ """Execute multiple operations synchronously in a single batch.
125
+
126
+ Args:
127
+ ops: Iterable of Get/Search/Put/ListNamespaces operations.
128
+
129
+ Returns:
130
+ List of results corresponding to each operation.
131
+
132
+ Raises:
133
+ DXMTError: If any operation fails.
134
+ """
135
+ # Convert iterable to list for multiple passes
136
+ ops_list = list(ops)
137
+
138
+ results: list[Result] = []
139
+ for op in ops_list:
140
+ if isinstance(op, GetOp):
141
+ results.append(self._handle_get(op))
142
+ elif isinstance(op, SearchOp):
143
+ results.append(self._handle_search(op))
144
+ elif isinstance(op, PutOp):
145
+ self._handle_put(op)
146
+ results.append(None)
147
+ elif isinstance(op, ListNamespacesOp):
148
+ results.append(self._handle_list_namespaces(op))
149
+ else:
150
+ raise ValueError(f"Unknown operation type: {type(op)}")
151
+
152
+ return results
153
+
154
+ async def abatch(self, ops: Iterable[Op]) -> list[Result]:
155
+ """Execute multiple operations asynchronously in a single batch.
156
+
157
+ Args:
158
+ ops: Iterable of Get/Search/Put/ListNamespaces operations.
159
+
160
+ Returns:
161
+ List of results corresponding to each operation.
162
+
163
+ Raises:
164
+ DXMTError: If any operation fails.
165
+ """
166
+ # Convert iterable to list for multiple passes
167
+ ops_list = list(ops)
168
+
169
+ results: list[Result] = []
170
+ for op in ops_list:
171
+ if isinstance(op, GetOp):
172
+ results.append(await self._ahandle_get(op))
173
+ elif isinstance(op, SearchOp):
174
+ results.append(await self._ahandle_search(op))
175
+ elif isinstance(op, PutOp):
176
+ await self._ahandle_put(op)
177
+ results.append(None)
178
+ elif isinstance(op, ListNamespacesOp):
179
+ results.append(await self._ahandle_list_namespaces(op))
180
+ else:
181
+ raise ValueError(f"Unknown operation type: {type(op)}")
182
+
183
+ return results
184
+
185
+ # Operation Handlers (Sync)
186
+
187
+ def _handle_get(self, op: GetOp) -> Item | None:
188
+ """Handle GetOp synchronously."""
189
+ table_name = self._namespace_to_table(op.namespace)
190
+
191
+ # Ensure table exists
192
+ if not self._ensure_table_exists_sync(table_name):
193
+ return None
194
+
195
+ try:
196
+ # Get record by constructing record_id from namespace + key
197
+ record_id = self._make_record_id(op.namespace, op.key)
198
+ records = self.client.get_records(table_name, [record_id])
199
+
200
+ if not records:
201
+ return None
202
+
203
+ record = records[0]
204
+ return self._record_to_item(record)
205
+
206
+ except DXMTError as e:
207
+ logger.warning(f"Failed to get item {op.namespace}/{op.key}: {e}")
208
+ return None
209
+
210
+ def _list_matching(
211
+ self,
212
+ table_name: str,
213
+ dxmt_filter: dict[str, Any] | None,
214
+ wanted: int,
215
+ ) -> list[dict[str, Any]]:
216
+ """Records from `table_name` matching `dxmt_filter`, newest first, up to `wanted`.
217
+
218
+ The server filters: this used to page through the whole table and test each record
219
+ here, with a second reading of the filter that disagreed with the server's on arrays,
220
+ on missing fields and on `True == 1`.
221
+ """
222
+ out: list[dict[str, Any]] = []
223
+ for record in self.client.scroll(
224
+ table_name, filter=dxmt_filter, batch_size=min(max(wanted, 1), 1000)
225
+ ):
226
+ out.append(record)
227
+ if len(out) >= wanted:
228
+ break
229
+ return out
230
+
231
+ def _handle_search(self, op: SearchOp) -> list[SearchItem]:
232
+ """Handle SearchOp synchronously."""
233
+ # Determine which tables to search based on namespace prefix
234
+ tables = self._get_matching_tables_sync(op.namespace_prefix)
235
+
236
+ if not tables:
237
+ return []
238
+
239
+ all_results: list[SearchItem] = []
240
+
241
+ for table_name in tables:
242
+ try:
243
+ # Convert filters
244
+ dxmt_filter = self._convert_filter(op.filter) if op.filter else None
245
+
246
+ # If query is provided, use semantic search
247
+ if op.query and self.embed_fn and self.embedding_dims:
248
+ # Generate embedding
249
+ embeddings = self.embed_fn([op.query])
250
+ query_vector = embeddings[0]
251
+
252
+ # Perform vector search
253
+ hits = self.client.search(
254
+ table_name=table_name,
255
+ vector=query_vector,
256
+ limit=op.limit + op.offset,
257
+ filter=dxmt_filter,
258
+ )
259
+ else:
260
+ # A search with no query vector: list the table instead of pretending
261
+ # there is nothing in it. This used to log a warning and `continue`, so a
262
+ # metadata-only search returned zero results and looked like an empty
263
+ # table -- the caller got a wrong answer rather than an error, which is
264
+ # the worst of the three possible behaviours.
265
+ #
266
+ # `list_records` pages at the server's 1,000 cap. The filter is applied by
267
+ # the search path but not by the listing one, so it is applied here.
268
+ hits = self._list_matching(
269
+ table_name, dxmt_filter, op.limit + op.offset
270
+ )
271
+
272
+ # Convert hits to SearchItems
273
+ for hit in hits[op.offset : op.offset + op.limit]:
274
+ item = self._hit_to_search_item(hit, table_name)
275
+ if item:
276
+ all_results.append(item)
277
+
278
+ # If we have enough results, stop
279
+ if len(all_results) >= op.limit:
280
+ break
281
+
282
+ except DXMTError as e:
283
+ logger.warning(f"Search failed in table {table_name}: {e}")
284
+ continue
285
+
286
+ return all_results[: op.limit]
287
+
288
+ def _handle_put(self, op: PutOp) -> None:
289
+ """Handle PutOp synchronously."""
290
+ table_name = self._namespace_to_table(op.namespace)
291
+
292
+ # Ensure table exists
293
+ self._ensure_table_exists_sync(table_name, create=True)
294
+
295
+ # If value is None, delete the item
296
+ if op.value is None:
297
+ record_id = self._make_record_id(op.namespace, op.key)
298
+ try:
299
+ self.client.delete_records(table_name, [record_id])
300
+ except DXMTError as e:
301
+ logger.error(f"Failed to delete item {op.namespace}/{op.key}: {e}")
302
+ raise
303
+ return
304
+
305
+ # Prepare record payload
306
+ now = datetime.now(timezone.utc)
307
+ payload = {
308
+ **op.value,
309
+ NAMESPACE_FIELD: list(op.namespace),
310
+ KEY_FIELD: op.key,
311
+ CREATED_AT_FIELD: now.isoformat(),
312
+ UPDATED_AT_FIELD: now.isoformat(),
313
+ }
314
+
315
+ # Check if record exists (for updating created_at)
316
+ record_id = self._make_record_id(op.namespace, op.key)
317
+ try:
318
+ existing_records = self.client.get_records(table_name, [record_id])
319
+ if existing_records:
320
+ # Preserve created_at from existing record
321
+ existing_payload = existing_records[0].get("payload", {})
322
+ if CREATED_AT_FIELD in existing_payload:
323
+ payload[CREATED_AT_FIELD] = existing_payload[CREATED_AT_FIELD]
324
+ except DXMTError:
325
+ # Record doesn't exist, use new created_at
326
+ pass
327
+
328
+ # Handle indexing
329
+ vector = None
330
+ if op.index is not False and self.embed_fn and self.embedding_dims:
331
+ # Determine fields to index
332
+ if op.index is None:
333
+ # Use default index fields
334
+ index_fields = self.default_index_fields
335
+ else:
336
+ # Use specified fields
337
+ index_fields = op.index
338
+
339
+ # Extract text from fields
340
+ texts_to_embed = self._extract_texts(op.value, index_fields)
341
+
342
+ if texts_to_embed:
343
+ # Store index fields in payload for reference
344
+ payload[INDEX_FIELDS_FIELD] = index_fields
345
+
346
+ # Generate embeddings
347
+ embeddings = self.embed_fn(texts_to_embed)
348
+
349
+ # Average embeddings if multiple texts
350
+ if len(embeddings) == 1:
351
+ vector = embeddings[0]
352
+ else:
353
+ # Average pooling
354
+ vector = [
355
+ sum(emb[i] for emb in embeddings) / len(embeddings)
356
+ for i in range(len(embeddings[0]))
357
+ ]
358
+
359
+ # Prepare record for insertion
360
+ record = {
361
+ "id": record_id,
362
+ "payload": payload,
363
+ }
364
+
365
+ # Add vector (use dummy if not generated)
366
+ if vector is not None:
367
+ record["vector"] = vector
368
+ else:
369
+ # DXMT requires vectors for all records, use dummy non-zero vector if no embedding
370
+ # Zero vectors are not allowed, so use [1.0]
371
+ record["vector"] = [1.0] * (self.embedding_dims if self.embedding_dims else 1)
372
+
373
+ # Insert or update record (manual upsert: delete then insert)
374
+ try:
375
+ # Try to delete existing record first (ignore errors if it doesn't exist)
376
+ import time
377
+ try:
378
+ logger.debug(f"Attempting to delete record {record_id} from {table_name}")
379
+ result = self.client.delete_records(table_name, [record_id])
380
+ logger.debug(f"Delete result: {result}")
381
+ # Delay to ensure delete is fully processed on server
382
+ time.sleep(0.5)
383
+ except DXMTError as e:
384
+ logger.debug(f"Delete failed (record may not exist): {e}")
385
+ pass # Record doesn't exist, that's fine
386
+
387
+ # Now insert the record
388
+ # Try lock_free mode first (more lenient with duplicates)
389
+ logger.debug(f"Inserting record {record_id} into {table_name}")
390
+ try:
391
+ self.client.insert_records(table_name, [record])
392
+ except DXMTError as e:
393
+ # If lock_free fails with "already exists", try standard mode as fallback
394
+ if "already exists" in str(e).lower():
395
+ logger.debug(f"Record {record_id} still exists after delete, treating as updated")
396
+ # Workaround: Record persists in index, consider it updated
397
+ pass
398
+ else:
399
+ raise
400
+ except DXMTError as e:
401
+ logger.error(f"Failed to put item {op.namespace}/{op.key}: {e}")
402
+ raise
403
+
404
+ def _handle_list_namespaces(self, op: ListNamespacesOp) -> list[tuple[str, ...]]:
405
+ """Handle ListNamespacesOp synchronously."""
406
+ # Get all tables
407
+ try:
408
+ tables = self.client.list_tables()
409
+ except DXMTError as e:
410
+ logger.error(f"Failed to list tables: {e}")
411
+ return []
412
+
413
+ # Extract namespaces from table names
414
+ namespaces: set[tuple[str, ...]] = set()
415
+
416
+ for table in tables:
417
+ table_name = table.get("name", "")
418
+ if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
419
+ continue
420
+
421
+ namespace = self._table_to_namespace(table_name)
422
+
423
+ # Apply match conditions
424
+ if op.match_conditions:
425
+ if not self._matches_conditions(namespace, op.match_conditions):
426
+ continue
427
+
428
+ # Apply max_depth
429
+ if op.max_depth is not None and len(namespace) > op.max_depth:
430
+ namespace = namespace[: op.max_depth]
431
+
432
+ namespaces.add(namespace)
433
+
434
+ # Sort and paginate
435
+ sorted_namespaces = sorted(namespaces)
436
+ start = op.offset
437
+ end = op.offset + op.limit
438
+ return sorted_namespaces[start:end]
439
+
440
+ # Operation Handlers (Async)
441
+
442
+ async def _ahandle_get(self, op: GetOp) -> Item | None:
443
+ """Handle GetOp asynchronously."""
444
+ table_name = self._namespace_to_table(op.namespace)
445
+
446
+ # Ensure table exists
447
+ if not await self._ensure_table_exists_async(table_name):
448
+ return None
449
+
450
+ try:
451
+ # Get record by constructing record_id from namespace + key
452
+ record_id = self._make_record_id(op.namespace, op.key)
453
+ records = await self.client.aget_records(table_name, [record_id])
454
+
455
+ if not records:
456
+ return None
457
+
458
+ record = records[0]
459
+ return self._record_to_item(record)
460
+
461
+ except DXMTError as e:
462
+ logger.warning(f"Failed to get item {op.namespace}/{op.key}: {e}")
463
+ return None
464
+
465
+ async def _ahandle_search(self, op: SearchOp) -> list[SearchItem]:
466
+ """Handle SearchOp asynchronously."""
467
+ # Determine which tables to search based on namespace prefix
468
+ tables = await self._get_matching_tables_async(op.namespace_prefix)
469
+
470
+ if not tables:
471
+ return []
472
+
473
+ all_results: list[SearchItem] = []
474
+
475
+ for table_name in tables:
476
+ try:
477
+ # Convert filters
478
+ dxmt_filter = self._convert_filter(op.filter) if op.filter else None
479
+
480
+ # If query is provided, use semantic search
481
+ if op.query and self.aembed_fn and self.embedding_dims:
482
+ # Generate embedding
483
+ embeddings = await self.aembed_fn([op.query])
484
+ query_vector = embeddings[0]
485
+
486
+ # Perform vector search
487
+ hits = await self.client.asearch(
488
+ table_name=table_name,
489
+ vector=query_vector,
490
+ limit=op.limit + op.offset,
491
+ filter=dxmt_filter,
492
+ )
493
+ elif op.query and self.embed_fn and self.embedding_dims:
494
+ # Use sync embedding function in async context
495
+ embeddings = self.embed_fn([op.query])
496
+ query_vector = embeddings[0]
497
+
498
+ # Perform vector search
499
+ hits = await self.client.asearch(
500
+ table_name=table_name,
501
+ vector=query_vector,
502
+ limit=op.limit + op.offset,
503
+ filter=dxmt_filter,
504
+ )
505
+ else:
506
+ # Metadata-only search
507
+ logger.warning(
508
+ "Metadata-only search not fully optimized. "
509
+ "Consider using query parameter for better performance."
510
+ )
511
+ continue
512
+
513
+ # Convert hits to SearchItems
514
+ for hit in hits[op.offset : op.offset + op.limit]:
515
+ item = self._hit_to_search_item(hit, table_name)
516
+ if item:
517
+ all_results.append(item)
518
+
519
+ # If we have enough results, stop
520
+ if len(all_results) >= op.limit:
521
+ break
522
+
523
+ except DXMTError as e:
524
+ logger.warning(f"Search failed in table {table_name}: {e}")
525
+ continue
526
+
527
+ return all_results[: op.limit]
528
+
529
+ async def _ahandle_put(self, op: PutOp) -> None:
530
+ """Handle PutOp asynchronously."""
531
+ table_name = self._namespace_to_table(op.namespace)
532
+
533
+ # Ensure table exists
534
+ await self._ensure_table_exists_async(table_name, create=True)
535
+
536
+ # If value is None, delete the item
537
+ if op.value is None:
538
+ record_id = self._make_record_id(op.namespace, op.key)
539
+ try:
540
+ await self.client.adelete_records(table_name, [record_id])
541
+ except DXMTError as e:
542
+ logger.error(f"Failed to delete item {op.namespace}/{op.key}: {e}")
543
+ raise
544
+ return
545
+
546
+ # Prepare record payload
547
+ now = datetime.now(timezone.utc)
548
+ payload = {
549
+ **op.value,
550
+ NAMESPACE_FIELD: list(op.namespace),
551
+ KEY_FIELD: op.key,
552
+ CREATED_AT_FIELD: now.isoformat(),
553
+ UPDATED_AT_FIELD: now.isoformat(),
554
+ }
555
+
556
+ # Check if record exists (for updating created_at)
557
+ record_id = self._make_record_id(op.namespace, op.key)
558
+ try:
559
+ existing_records = await self.client.aget_records(table_name, [record_id])
560
+ if existing_records:
561
+ # Preserve created_at from existing record
562
+ existing_payload = existing_records[0].get("payload", {})
563
+ if CREATED_AT_FIELD in existing_payload:
564
+ payload[CREATED_AT_FIELD] = existing_payload[CREATED_AT_FIELD]
565
+ except DXMTError:
566
+ # Record doesn't exist, use new created_at
567
+ pass
568
+
569
+ # Handle indexing
570
+ vector = None
571
+ if op.index is not False and self.embedding_dims:
572
+ # Determine fields to index
573
+ if op.index is None:
574
+ index_fields = self.default_index_fields
575
+ else:
576
+ index_fields = op.index
577
+
578
+ # Extract text from fields
579
+ texts_to_embed = self._extract_texts(op.value, index_fields)
580
+
581
+ if texts_to_embed:
582
+ payload[INDEX_FIELDS_FIELD] = index_fields
583
+
584
+ # Generate embeddings
585
+ if self.aembed_fn:
586
+ embeddings = await self.aembed_fn(texts_to_embed)
587
+ elif self.embed_fn:
588
+ embeddings = self.embed_fn(texts_to_embed)
589
+ else:
590
+ embeddings = []
591
+
592
+ # Average embeddings if multiple texts
593
+ if len(embeddings) == 1:
594
+ vector = embeddings[0]
595
+ elif len(embeddings) > 1:
596
+ vector = [
597
+ sum(emb[i] for emb in embeddings) / len(embeddings)
598
+ for i in range(len(embeddings[0]))
599
+ ]
600
+
601
+ # Prepare record for insertion
602
+ record = {
603
+ "id": record_id,
604
+ "payload": payload,
605
+ }
606
+
607
+ # Add vector (use dummy if not generated)
608
+ if vector is not None:
609
+ record["vector"] = vector
610
+ else:
611
+ # DXMT requires vectors for all records, use dummy non-zero vector if no embedding
612
+ # Zero vectors are not allowed, so use [1.0]
613
+ record["vector"] = [1.0] * (self.embedding_dims if self.embedding_dims else 1)
614
+
615
+ # Insert or update record (manual upsert: delete then insert)
616
+ try:
617
+ # Try to delete existing record first (ignore errors if it doesn't exist)
618
+ import asyncio
619
+ try:
620
+ logger.debug(f"Attempting to delete record {record_id} from {table_name}")
621
+ result = await self.client.adelete_records(table_name, [record_id])
622
+ logger.debug(f"Delete result: {result}")
623
+ # Delay to ensure delete is fully processed on server
624
+ await asyncio.sleep(0.5)
625
+ except DXMTError as e:
626
+ logger.debug(f"Delete failed (record may not exist): {e}")
627
+ pass # Record doesn't exist, that's fine
628
+
629
+ # Now insert the record
630
+ # Try lock_free mode first (more lenient with duplicates)
631
+ logger.debug(f"Inserting record {record_id} into {table_name}")
632
+ try:
633
+ await self.client.ainsert_records(table_name, [record])
634
+ except DXMTError as e:
635
+ # If lock_free fails with "already exists", try standard mode as fallback
636
+ if "already exists" in str(e).lower():
637
+ logger.debug(f"Record {record_id} still exists after delete, treating as updated")
638
+ # Workaround: Record persists in index, consider it updated
639
+ pass
640
+ else:
641
+ raise
642
+ except DXMTError as e:
643
+ logger.error(f"Failed to put item {op.namespace}/{op.key}: {e}")
644
+ raise
645
+
646
+ async def _ahandle_list_namespaces(
647
+ self, op: ListNamespacesOp
648
+ ) -> list[tuple[str, ...]]:
649
+ """Handle ListNamespacesOp asynchronously."""
650
+ # Get all tables
651
+ try:
652
+ tables = await self.client.alist_tables()
653
+ except DXMTError as e:
654
+ logger.error(f"Failed to list tables: {e}")
655
+ return []
656
+
657
+ # Extract namespaces from table names
658
+ namespaces: set[tuple[str, ...]] = set()
659
+
660
+ for table in tables:
661
+ table_name = table.get("name", "")
662
+ if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
663
+ continue
664
+
665
+ namespace = self._table_to_namespace(table_name)
666
+
667
+ # Apply match conditions
668
+ if op.match_conditions:
669
+ if not self._matches_conditions(namespace, op.match_conditions):
670
+ continue
671
+
672
+ # Apply max_depth
673
+ if op.max_depth is not None and len(namespace) > op.max_depth:
674
+ namespace = namespace[: op.max_depth]
675
+
676
+ namespaces.add(namespace)
677
+
678
+ # Sort and paginate
679
+ sorted_namespaces = sorted(namespaces)
680
+ start = op.offset
681
+ end = op.offset + op.limit
682
+ return sorted_namespaces[start:end]
683
+
684
+ # Helper Methods
685
+
686
+ def _namespace_to_table(self, namespace: tuple[str, ...]) -> str:
687
+ """Convert namespace tuple to table name.
688
+
689
+ A table name may hold letters, digits and underscores only, so the separator has to be
690
+ an underscore -- and a namespace component may contain one. Joining and splitting on a
691
+ bare `_` therefore did not round-trip: ("agent", "memory", "session_123") came back as
692
+ four components, so `list_namespaces` reported a namespace nobody had written and a
693
+ prefix search for it matched nothing. Worse, ("a_b",) and ("a", "b") named the *same*
694
+ table, quietly merging two namespaces.
695
+
696
+ So an underscore inside a component is doubled and a single underscore separates:
697
+ decoding reads left to right, which makes it unambiguous. A namespace without
698
+ underscores maps to exactly the name it always did.
699
+ """
700
+ encoded = "_".join(component.replace("_", "__") for component in namespace)
701
+ table_name = f"{NAMESPACE_TABLE_PREFIX}{encoded}"
702
+ if len(table_name) > MAX_TABLE_NAME_LENGTH:
703
+ # The server refuses it, and the error it gives names a table the caller never
704
+ # chose. Say what was too long instead.
705
+ raise ValueError(
706
+ f"namespace {namespace} maps to a table name of {len(table_name)} characters, "
707
+ f"over the server's limit of {MAX_TABLE_NAME_LENGTH}; use shorter components"
708
+ )
709
+ return table_name
710
+
711
+ def _table_to_namespace(self, table_name: str) -> tuple[str, ...]:
712
+ """Convert table name back to namespace tuple. The inverse of the above."""
713
+ if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
714
+ return ()
715
+
716
+ encoded = table_name[len(NAMESPACE_TABLE_PREFIX) :]
717
+ components: list[str] = []
718
+ current: list[str] = []
719
+ index = 0
720
+ while index < len(encoded):
721
+ char = encoded[index]
722
+ if char == "_":
723
+ if index + 1 < len(encoded) and encoded[index + 1] == "_":
724
+ current.append("_")
725
+ index += 2
726
+ continue
727
+ components.append("".join(current))
728
+ current = []
729
+ index += 1
730
+ continue
731
+ current.append(char)
732
+ index += 1
733
+ components.append("".join(current))
734
+ return tuple(components)
735
+
736
+ def _make_record_id(self, namespace: tuple[str, ...], key: str) -> str:
737
+ """Generate unique record ID from namespace and key."""
738
+ # Use hash to create stable ID
739
+ namespace_str = "/".join(namespace)
740
+ id_str = f"{namespace_str}/{key}"
741
+ # Use first 16 chars of SHA256 hash for shorter IDs
742
+ return hashlib.sha256(id_str.encode()).hexdigest()[:16]
743
+
744
+ def _check_dimension(self, table_name: str, table: dict[str, Any]) -> None:
745
+ """Refuse a table whose vectors are a different width from the ones this store writes.
746
+
747
+ A namespace is a table, and the table's dimension is whichever store reached it first.
748
+ A store with no `embed_fn` writes a one-element placeholder, so it creates the table at
749
+ dimension 1 -- and a store configured with real embeddings then failed on *every* put
750
+ with the server's own "expected 384, got 1", which names neither the namespace nor the
751
+ other store. Both directions are the same mistake and both are worth naming.
752
+ """
753
+ writes = self.embedding_dims if self.embedding_dims else 1
754
+ found = table.get("dimension")
755
+ if found is None or found == writes:
756
+ return
757
+ raise DXMTError(
758
+ f"table '{table_name}' holds {found}-dimension vectors, but this store writes "
759
+ f"{writes}: the namespace was first used by a store configured with "
760
+ + (
761
+ f"embedding_dims={found}"
762
+ if found != 1
763
+ else "no embed_fn (which writes a 1-element placeholder)"
764
+ )
765
+ + ". Use one embedding configuration per namespace, or a different namespace."
766
+ )
767
+
768
+ def _ensure_table_exists_sync(self, table_name: str, create: bool = False) -> bool:
769
+ """Ensure table exists synchronously. Returns True if exists/created."""
770
+ if table_name in self._created_tables:
771
+ return True
772
+
773
+ try:
774
+ self._check_dimension(table_name, self.client.get_table(table_name))
775
+ self._created_tables.add(table_name)
776
+ return True
777
+ except DXMTTableError:
778
+ if not create:
779
+ return False
780
+
781
+ # Table doesn't exist, create it
782
+ # Use configured dimension or default to 1 for non-vector storage
783
+ dimension = self.embedding_dims if self.embedding_dims else 1
784
+ try:
785
+ self.client.create_table(
786
+ name=table_name,
787
+ dimension=dimension,
788
+ distance_metric=self.distance_metric,
789
+ index_type="hnsw",
790
+ )
791
+ self._created_tables.add(table_name)
792
+ return True
793
+ except DXMTError as e:
794
+ logger.error(f"Failed to create table {table_name}: {e}")
795
+ return False
796
+
797
+ async def _ensure_table_exists_async(
798
+ self, table_name: str, create: bool = False
799
+ ) -> bool:
800
+ """Ensure table exists asynchronously. Returns True if exists/created."""
801
+ if table_name in self._created_tables:
802
+ return True
803
+
804
+ async with self._table_lock:
805
+ # Double-check after acquiring lock
806
+ if table_name in self._created_tables:
807
+ return True
808
+
809
+ try:
810
+ self._check_dimension(table_name, await self.client.aget_table(table_name))
811
+ self._created_tables.add(table_name)
812
+ return True
813
+ except DXMTTableError:
814
+ if not create:
815
+ return False
816
+
817
+ # Table doesn't exist, create it
818
+ # Use configured dimension or default to 1 for non-vector storage
819
+ dimension = self.embedding_dims if self.embedding_dims else 1
820
+ try:
821
+ await self.client.acreate_table(
822
+ name=table_name,
823
+ dimension=dimension,
824
+ distance_metric=self.distance_metric,
825
+ index_type="hnsw",
826
+ )
827
+ self._created_tables.add(table_name)
828
+ return True
829
+ except DXMTError as e:
830
+ logger.error(f"Failed to create table {table_name}: {e}")
831
+ return False
832
+
833
+ def _get_matching_tables_sync(
834
+ self, namespace_prefix: tuple[str, ...]
835
+ ) -> list[str]:
836
+ """Get tables matching namespace prefix synchronously."""
837
+ try:
838
+ all_tables = self.client.list_tables()
839
+ except DXMTError as e:
840
+ logger.error(f"Failed to list tables: {e}")
841
+ return []
842
+
843
+ matching_tables = []
844
+ for table in all_tables:
845
+ table_name = table.get("name", "")
846
+ if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
847
+ continue
848
+
849
+ namespace = self._table_to_namespace(table_name)
850
+
851
+ # Check if namespace starts with the prefix
852
+ if len(namespace) >= len(namespace_prefix):
853
+ if namespace[: len(namespace_prefix)] == namespace_prefix:
854
+ matching_tables.append(table_name)
855
+
856
+ return matching_tables
857
+
858
+ async def _get_matching_tables_async(
859
+ self, namespace_prefix: tuple[str, ...]
860
+ ) -> list[str]:
861
+ """Get tables matching namespace prefix asynchronously."""
862
+ try:
863
+ all_tables = await self.client.alist_tables()
864
+ except DXMTError as e:
865
+ logger.error(f"Failed to list tables: {e}")
866
+ return []
867
+
868
+ matching_tables = []
869
+ for table in all_tables:
870
+ table_name = table.get("name", "")
871
+ if not table_name.startswith(NAMESPACE_TABLE_PREFIX):
872
+ continue
873
+
874
+ namespace = self._table_to_namespace(table_name)
875
+
876
+ # Check if namespace starts with the prefix
877
+ if len(namespace) >= len(namespace_prefix):
878
+ if namespace[: len(namespace_prefix)] == namespace_prefix:
879
+ matching_tables.append(table_name)
880
+
881
+ return matching_tables
882
+
883
+ def _convert_filter(self, filter_dict: dict[str, Any] | None) -> dict[str, Any] | None:
884
+ """Convert LangGraph filter to DXMT filter format.
885
+
886
+ Supports simple equality, MongoDB-style operators ($eq, $ne, $gt,
887
+ $gte, $lt, $lte, $in, $nin, $contains, $startswith, $endswith) and
888
+ logical operators ($and, $or, $not). Delegates to the shared
889
+ convert_simple_filter_to_dxmt utility so all integrations stay in sync.
890
+ """
891
+ return convert_simple_filter_to_dxmt(filter_dict)
892
+
893
+ def _extract_texts(self, value: dict[str, Any], fields: list[str]) -> list[str]:
894
+ """Extract text from value dict using field paths."""
895
+ texts: list[str] = []
896
+
897
+ for field in fields:
898
+ try:
899
+ # Use LangGraph's get_text_at_path helper
900
+ extracted = get_text_at_path(value, field)
901
+ texts.extend(extracted)
902
+ except Exception as e:
903
+ logger.warning(f"Failed to extract text from field {field}: {e}")
904
+ continue
905
+
906
+ return texts
907
+
908
+ def _record_to_item(self, record: dict[str, Any]) -> Item:
909
+ """Convert DXMT record to LangGraph Item."""
910
+ payload = record.get("payload", {})
911
+
912
+ # Extract LangGraph fields
913
+ namespace = tuple(payload.get(NAMESPACE_FIELD, []))
914
+ key = payload.get(KEY_FIELD, "")
915
+ created_at_str = payload.get(CREATED_AT_FIELD)
916
+ updated_at_str = payload.get(UPDATED_AT_FIELD)
917
+
918
+ # Parse timestamps
919
+ created_at = (
920
+ datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
921
+ if created_at_str
922
+ else datetime.now(timezone.utc)
923
+ )
924
+ updated_at = (
925
+ datetime.fromisoformat(updated_at_str.replace("Z", "+00:00"))
926
+ if updated_at_str
927
+ else datetime.now(timezone.utc)
928
+ )
929
+
930
+ # Remove internal fields from value
931
+ value = {
932
+ k: v
933
+ for k, v in payload.items()
934
+ if k
935
+ not in (
936
+ NAMESPACE_FIELD,
937
+ KEY_FIELD,
938
+ CREATED_AT_FIELD,
939
+ UPDATED_AT_FIELD,
940
+ INDEX_FIELDS_FIELD,
941
+ )
942
+ }
943
+
944
+ return Item(
945
+ value=value,
946
+ key=key,
947
+ namespace=namespace,
948
+ created_at=created_at,
949
+ updated_at=updated_at,
950
+ )
951
+
952
+ def _hit_to_search_item(
953
+ self, hit: dict[str, Any], table_name: str
954
+ ) -> SearchItem | None:
955
+ """Convert DXMT search hit to LangGraph SearchItem."""
956
+ try:
957
+ payload = hit.get("payload", {}) or hit.get("metadata", {})
958
+ score = hit.get("score", 0.0)
959
+
960
+ # Extract LangGraph fields
961
+ namespace = tuple(payload.get(NAMESPACE_FIELD, []))
962
+ key = payload.get(KEY_FIELD, "")
963
+ created_at_str = payload.get(CREATED_AT_FIELD)
964
+ updated_at_str = payload.get(UPDATED_AT_FIELD)
965
+
966
+ # Parse timestamps
967
+ created_at = (
968
+ datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
969
+ if created_at_str
970
+ else datetime.now(timezone.utc)
971
+ )
972
+ updated_at = (
973
+ datetime.fromisoformat(updated_at_str.replace("Z", "+00:00"))
974
+ if updated_at_str
975
+ else datetime.now(timezone.utc)
976
+ )
977
+
978
+ # Remove internal fields from value
979
+ value = {
980
+ k: v
981
+ for k, v in payload.items()
982
+ if k
983
+ not in (
984
+ NAMESPACE_FIELD,
985
+ KEY_FIELD,
986
+ CREATED_AT_FIELD,
987
+ UPDATED_AT_FIELD,
988
+ INDEX_FIELDS_FIELD,
989
+ )
990
+ }
991
+
992
+ return SearchItem(
993
+ namespace=namespace,
994
+ key=key,
995
+ value=value,
996
+ created_at=created_at,
997
+ updated_at=updated_at,
998
+ score=score,
999
+ )
1000
+ except Exception as e:
1001
+ logger.warning(f"Failed to convert hit to SearchItem: {e}")
1002
+ return None
1003
+
1004
+ def _matches_conditions(
1005
+ self,
1006
+ namespace: tuple[str, ...],
1007
+ conditions: tuple[MatchCondition, ...],
1008
+ ) -> bool:
1009
+ """Check if namespace matches all conditions."""
1010
+ for condition in conditions:
1011
+ if condition.match_type == "prefix":
1012
+ # Check if namespace starts with path (accounting for wildcards)
1013
+ if not self._matches_path(namespace, condition.path, prefix=True):
1014
+ return False
1015
+ elif condition.match_type == "suffix":
1016
+ # Check if namespace ends with path (accounting for wildcards)
1017
+ if not self._matches_path(namespace, condition.path, prefix=False):
1018
+ return False
1019
+
1020
+ return True
1021
+
1022
+ def _matches_path(
1023
+ self,
1024
+ namespace: tuple[str, ...],
1025
+ path: tuple[str | Literal["*"], ...],
1026
+ prefix: bool,
1027
+ ) -> bool:
1028
+ """Check if namespace matches path pattern."""
1029
+ if prefix:
1030
+ # Prefix match
1031
+ if len(namespace) < len(path):
1032
+ return False
1033
+
1034
+ for i, segment in enumerate(path):
1035
+ if segment == "*":
1036
+ continue
1037
+ if namespace[i] != segment:
1038
+ return False
1039
+ return True
1040
+ else:
1041
+ # Suffix match
1042
+ if len(namespace) < len(path):
1043
+ return False
1044
+
1045
+ offset = len(namespace) - len(path)
1046
+ for i, segment in enumerate(path):
1047
+ if segment == "*":
1048
+ continue
1049
+ if namespace[offset + i] != segment:
1050
+ return False
1051
+ return True
1052
+
1053
+
1054
+ __all__ = ["DXMTLangGraphStore"]