dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,527 @@
1
+ """DXMT VectorStore for LangChain -- the Bisheng compatibility layer.
2
+
3
+ **New LangChain code should use the `langchain-dxmt` package** (`clients/langchain-dxmt/`,
4
+ `from langchain_dxmt import DXMTVectorStore`). That is the adapter kept against LangChain's
5
+ standard test suite and against a live server in CI; it also carries MMR, server-side
6
+ embedding and the full filter translation.
7
+
8
+ This module stays because Bisheng (`dxmt.bisheng_integration`) imports it by this path and
9
+ constructs it with these keyword arguments. It is a plain synchronous `VectorStore`: the
10
+ `a*` methods run the sync ones on a thread, nothing more is wrapped, and it is tested with a
11
+ mocked client only.
12
+
13
+ Example:
14
+ ```python
15
+ from langchain.embeddings import OpenAIEmbeddings
16
+ from langchain.schema import Document
17
+ from dxmt.langchain_vectorstore import DXMTVectorStore
18
+
19
+ # Create embeddings
20
+ embeddings = OpenAIEmbeddings()
21
+
22
+ # Create DXMT vector store
23
+ vector_store = DXMTVectorStore(
24
+ embedding=embeddings,
25
+ collection_name="documents",
26
+ host="localhost",
27
+ port=9999
28
+ )
29
+
30
+ # Add documents
31
+ documents = [Document(page_content="Sample document", metadata={"source": "test"})]
32
+ vector_store.add_documents(documents)
33
+
34
+ # Search
35
+ results = vector_store.similarity_search("query", k=5)
36
+ ```
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ import logging
42
+ import uuid
43
+ from typing import Any, Iterable, List, Optional, Tuple
44
+
45
+ try:
46
+ from langchain_core.documents import Document
47
+ from langchain_core.embeddings import Embeddings
48
+ from langchain_core.vectorstores import VectorStore
49
+ LANGCHAIN_AVAILABLE = True
50
+ except ImportError:
51
+ LANGCHAIN_AVAILABLE = False
52
+ # Define dummy classes
53
+ Document = Any # type: ignore
54
+ Embeddings = Any # type: ignore
55
+ VectorStore = object # type: ignore
56
+
57
+ from .client import DXMTClient
58
+ from .exceptions import DXMTError
59
+ from .filters import convert_simple_filter_to_dxmt
60
+
61
+ logger = logging.getLogger(__name__)
62
+
63
+
64
+ class DXMTVectorStore(VectorStore):
65
+ """DXMT VectorStore for LangChain.
66
+
67
+ Implements LangChain's VectorStore interface to enable DXMT VectorDB
68
+ integration with LangChain and Bisheng.
69
+
70
+ Args:
71
+ embedding: Embeddings instance for encoding text.
72
+ collection_name: Name of the DXMT table/collection.
73
+ host: DXMT server host.
74
+ port: DXMT server port.
75
+ dimension: Vector dimension (auto-detected from embeddings if not provided).
76
+ distance_metric: Distance metric (cosine, euclidean, dot, manhattan).
77
+ client: Optional DXMTClient instance (created if not provided).
78
+ **kwargs: Additional arguments.
79
+
80
+ Attributes:
81
+ embeddings: The embeddings instance.
82
+ collection_name: Collection name in DXMT.
83
+ client: DXMT client instance.
84
+
85
+ Example:
86
+ ```python
87
+ from langchain.embeddings import OpenAIEmbeddings
88
+ from dxmt.langchain_vectorstore import DXMTVectorStore
89
+
90
+ embeddings = OpenAIEmbeddings()
91
+ vector_store = DXMTVectorStore(
92
+ embedding=embeddings,
93
+ collection_name="my_docs",
94
+ host="localhost",
95
+ port=9999
96
+ )
97
+
98
+ # Add documents
99
+ vector_store.add_texts(
100
+ texts=["Document 1", "Document 2"],
101
+ metadatas=[{"source": "doc1"}, {"source": "doc2"}]
102
+ )
103
+
104
+ # Search
105
+ results = vector_store.similarity_search("query text", k=3)
106
+ ```
107
+ """
108
+
109
+ def __init__(
110
+ self,
111
+ embedding: Embeddings,
112
+ collection_name: str = "langchain_collection",
113
+ host: str = "localhost",
114
+ port: int = 9999,
115
+ dimension: Optional[int] = None,
116
+ distance_metric: str = "cosine",
117
+ client: Optional[DXMTClient] = None,
118
+ api_key: Optional[str] = None,
119
+ **kwargs: Any,
120
+ ) -> None:
121
+ """Initialize DXMT vector store. ``api_key`` is ignored when ``client`` is given."""
122
+ if not LANGCHAIN_AVAILABLE:
123
+ raise ImportError(
124
+ "LangChain is required for DXMTVectorStore. "
125
+ "Install with: pip install langchain langchain-core"
126
+ )
127
+
128
+ self._embeddings = embedding
129
+ self.collection_name = collection_name
130
+ self.distance_metric = distance_metric
131
+ self._dimension = dimension
132
+
133
+ # Create or use provided client
134
+ if client is None:
135
+ self.client = DXMTClient(host=host, port=port, api_key=api_key)
136
+ else:
137
+ self.client = client
138
+
139
+ # Ensure table exists
140
+ self._ensure_collection_exists()
141
+
142
+ def _ensure_collection_exists(self) -> None:
143
+ """Ensure the DXMT collection/table exists."""
144
+ try:
145
+ # Try to get existing table
146
+ self.client.get_table(self.collection_name)
147
+ logger.info(f"Collection {self.collection_name} already exists")
148
+ except DXMTError:
149
+ # Table doesn't exist, create it
150
+ # Determine dimension from embeddings if not provided
151
+ if self._dimension is None:
152
+ # Get dimension by embedding a test string
153
+ test_embedding = self._embeddings.embed_query("test")
154
+ self._dimension = len(test_embedding)
155
+
156
+ logger.info(f"Creating collection {self.collection_name} with dimension {self._dimension}")
157
+ self.client.create_table(
158
+ name=self.collection_name,
159
+ dimension=self._dimension,
160
+ distance_metric=self.distance_metric,
161
+ description=f"LangChain collection (dimension={self._dimension})"
162
+ )
163
+
164
+ @property
165
+ def embeddings(self) -> Embeddings:
166
+ """Get the embeddings instance."""
167
+ return self._embeddings
168
+
169
+ @property
170
+ def embedding_function(self) -> Embeddings:
171
+ """Get the embeddings function."""
172
+ return self._embeddings
173
+
174
+ def add_texts(
175
+ self,
176
+ texts: Iterable[str],
177
+ metadatas: Optional[List[dict]] = None,
178
+ ids: Optional[List[str]] = None,
179
+ **kwargs: Any,
180
+ ) -> List[str]:
181
+ """Add texts to the vector store.
182
+
183
+ Args:
184
+ texts: Texts to add to the vector store.
185
+ metadatas: Optional list of metadata dicts for each text.
186
+ ids: Optional list of IDs for each text.
187
+ **kwargs: Additional arguments.
188
+
189
+ Returns:
190
+ List of IDs for the added texts.
191
+ """
192
+ texts_list = list(texts)
193
+
194
+ # Generate IDs if not provided
195
+ if ids is None:
196
+ ids = [str(uuid.uuid4()) for _ in texts_list]
197
+
198
+ # Prepare metadatas
199
+ if metadatas is None:
200
+ metadatas = [{} for _ in texts_list]
201
+ elif len(metadatas) != len(texts_list):
202
+ raise ValueError("Number of metadatas must match number of texts")
203
+
204
+ # Generate embeddings
205
+ embeddings = self.embeddings.embed_documents(texts_list)
206
+
207
+ # Prepare records for DXMT
208
+ records = []
209
+ for i, (text, embedding, metadata) in enumerate(zip(texts_list, embeddings, metadatas)):
210
+ # Add text to metadata
211
+ metadata_with_text = metadata.copy()
212
+ metadata_with_text["text"] = text
213
+
214
+ record = {
215
+ "id": ids[i],
216
+ "vector": embedding,
217
+ "metadata": metadata_with_text
218
+ }
219
+ records.append(record)
220
+
221
+ # Insert into DXMT
222
+ try:
223
+ self.client.insert_records(
224
+ table_name=self.collection_name,
225
+ records=records
226
+ )
227
+ logger.info(f"Added {len(records)} texts to {self.collection_name}")
228
+ return ids
229
+ except Exception as e:
230
+ logger.error(f"Error adding texts: {e}")
231
+ raise DXMTError(f"Failed to add texts: {e}")
232
+
233
+ def similarity_search(
234
+ self,
235
+ query: str,
236
+ k: int = 4,
237
+ filter: Optional[dict] = None,
238
+ **kwargs: Any,
239
+ ) -> List[Document]:
240
+ """Search for similar documents.
241
+
242
+ Args:
243
+ query: Query text.
244
+ k: Number of results to return.
245
+ filter: Optional metadata filter.
246
+ **kwargs: Additional arguments.
247
+
248
+ Returns:
249
+ List of similar documents.
250
+ """
251
+ # Generate query embedding
252
+ query_embedding = self.embeddings.embed_query(query)
253
+
254
+ # Convert filter to DXMT format
255
+ dxmt_filter = self._convert_filter(filter) if filter is not None else None
256
+
257
+ # Search DXMT
258
+ try:
259
+ results = self.client.search(
260
+ table_name=self.collection_name,
261
+ vector=query_embedding,
262
+ k=k,
263
+ filter=dxmt_filter
264
+ )
265
+ except Exception as e:
266
+ logger.error(f"Error searching: {e}")
267
+ raise DXMTError(f"Failed to search: {e}")
268
+
269
+ # Convert results to Documents
270
+ documents = []
271
+ for result in results:
272
+ metadata = result.get("metadata", {})
273
+ text = metadata.pop("text", "")
274
+
275
+ doc = Document(
276
+ page_content=text,
277
+ metadata=metadata
278
+ )
279
+ documents.append(doc)
280
+
281
+ return documents
282
+
283
+ def similarity_search_with_score(
284
+ self,
285
+ query: str,
286
+ k: int = 4,
287
+ filter: Optional[dict] = None,
288
+ **kwargs: Any,
289
+ ) -> List[Tuple[Document, float]]:
290
+ """Search for similar documents with scores.
291
+
292
+ Args:
293
+ query: Query text.
294
+ k: Number of results to return.
295
+ filter: Optional metadata filter.
296
+ **kwargs: Additional arguments.
297
+
298
+ Returns:
299
+ List of tuples (document, score).
300
+ """
301
+ # Generate query embedding
302
+ query_embedding = self.embeddings.embed_query(query)
303
+
304
+ # Convert filter to DXMT format
305
+ dxmt_filter = self._convert_filter(filter) if filter is not None else None
306
+
307
+ # Search DXMT
308
+ try:
309
+ results = self.client.search(
310
+ table_name=self.collection_name,
311
+ vector=query_embedding,
312
+ k=k,
313
+ filter=dxmt_filter
314
+ )
315
+ except Exception as e:
316
+ logger.error(f"Error searching with score: {e}")
317
+ raise DXMTError(f"Failed to search: {e}")
318
+
319
+ # Convert results to Documents with scores
320
+ documents_with_scores = []
321
+ for result in results:
322
+ metadata = result.get("metadata", {})
323
+ text = metadata.pop("text", "")
324
+ score = result.get("score", 0.0)
325
+
326
+ doc = Document(
327
+ page_content=text,
328
+ metadata=metadata
329
+ )
330
+ documents_with_scores.append((doc, score))
331
+
332
+ return documents_with_scores
333
+
334
+ def _convert_filter(self, filter: dict) -> Optional[dict]:
335
+ """Convert LangChain filter to DXMT format.
336
+
337
+ Supports simple equality dicts and MongoDB-style operators
338
+ ($eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $contains, $startswith,
339
+ $endswith) as well as logical operators ($and, $or, $not).
340
+
341
+ Args:
342
+ filter: LangChain filter dict.
343
+
344
+ Returns:
345
+ DXMT filter dict.
346
+ """
347
+ return convert_simple_filter_to_dxmt(filter)
348
+
349
+ @classmethod
350
+ def from_texts(
351
+ cls,
352
+ texts: List[str],
353
+ embedding: Embeddings,
354
+ metadatas: Optional[List[dict]] = None,
355
+ ids: Optional[List[str]] = None,
356
+ collection_name: str = "langchain_collection",
357
+ host: str = "localhost",
358
+ port: int = 9999,
359
+ **kwargs: Any,
360
+ ) -> "DXMTVectorStore":
361
+ """Create a DXMT vector store from texts.
362
+
363
+ Args:
364
+ texts: List of texts to add.
365
+ embedding: Embeddings instance.
366
+ metadatas: Optional list of metadata dicts.
367
+ ids: Optional list of IDs.
368
+ collection_name: Collection name.
369
+ host: DXMT server host.
370
+ port: DXMT server port.
371
+ **kwargs: Additional arguments.
372
+
373
+ Returns:
374
+ DXMTVectorStore instance.
375
+ """
376
+ vector_store = cls(
377
+ embedding=embedding,
378
+ collection_name=collection_name,
379
+ host=host,
380
+ port=port,
381
+ **kwargs,
382
+ )
383
+ vector_store.add_texts(texts=texts, metadatas=metadatas, ids=ids)
384
+ return vector_store
385
+
386
+ @classmethod
387
+ def from_documents(
388
+ cls,
389
+ documents: List[Document],
390
+ embedding: Embeddings,
391
+ ids: Optional[List[str]] = None,
392
+ collection_name: str = "langchain_collection",
393
+ host: str = "localhost",
394
+ port: int = 9999,
395
+ **kwargs: Any,
396
+ ) -> "DXMTVectorStore":
397
+ """Create a DXMT vector store from documents.
398
+
399
+ Args:
400
+ documents: List of Document objects.
401
+ embedding: Embeddings instance.
402
+ ids: Optional list of IDs.
403
+ collection_name: Collection name.
404
+ host: DXMT server host.
405
+ port: DXMT server port.
406
+ **kwargs: Additional arguments.
407
+
408
+ Returns:
409
+ DXMTVectorStore instance.
410
+ """
411
+ texts = [doc.page_content for doc in documents]
412
+ metadatas = [doc.metadata for doc in documents]
413
+
414
+ return cls.from_texts(
415
+ texts=texts,
416
+ embedding=embedding,
417
+ metadatas=metadatas,
418
+ ids=ids,
419
+ collection_name=collection_name,
420
+ host=host,
421
+ port=port,
422
+ **kwargs,
423
+ )
424
+
425
+ def similarity_search_by_vector(
426
+ self,
427
+ embedding: List[float],
428
+ k: int = 4,
429
+ filter: Optional[dict] = None,
430
+ **kwargs: Any,
431
+ ) -> List[Document]:
432
+ """Search using a pre-computed embedding vector.
433
+
434
+ Args:
435
+ embedding: Query embedding vector.
436
+ k: Number of results to return.
437
+ filter: Optional metadata filter.
438
+ **kwargs: Additional arguments.
439
+
440
+ Returns:
441
+ List of similar documents.
442
+ """
443
+ dxmt_filter = self._convert_filter(filter) if filter is not None else None
444
+ try:
445
+ results = self.client.search(
446
+ table_name=self.collection_name,
447
+ vector=embedding,
448
+ k=k,
449
+ filter=dxmt_filter,
450
+ )
451
+ except Exception as e:
452
+ logger.error(f"Error searching by vector: {e}")
453
+ raise DXMTError(f"Failed to search: {e}")
454
+
455
+ documents = []
456
+ for result in results:
457
+ metadata = result.get("metadata", {})
458
+ text = metadata.pop("text", "")
459
+ documents.append(Document(page_content=text, metadata=metadata))
460
+ return documents
461
+
462
+ def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]:
463
+ """Delete by IDs.
464
+
465
+ Args:
466
+ ids: List of IDs to delete.
467
+ **kwargs: Additional arguments.
468
+
469
+ Returns:
470
+ True if successful.
471
+ """
472
+ if ids is None:
473
+ return None
474
+
475
+ try:
476
+ self.client.delete_records(
477
+ table_name=self.collection_name,
478
+ ids=ids,
479
+ )
480
+ logger.info(f"Deleted {len(ids)} documents from {self.collection_name}")
481
+ return True
482
+ except Exception as e:
483
+ logger.error(f"Error deleting documents: {e}")
484
+ raise DXMTError(f"Failed to delete: {e}")
485
+
486
+ async def aadd_texts(
487
+ self,
488
+ texts: Iterable[str],
489
+ metadatas: Optional[List[dict]] = None,
490
+ ids: Optional[List[str]] = None,
491
+ **kwargs: Any,
492
+ ) -> List[str]:
493
+ """Async add texts — delegates to sync add_texts via thread executor."""
494
+ import asyncio
495
+ loop = asyncio.get_event_loop()
496
+ return await loop.run_in_executor(
497
+ None, lambda: self.add_texts(texts, metadatas=metadatas, ids=ids, **kwargs)
498
+ )
499
+
500
+ async def asimilarity_search(
501
+ self,
502
+ query: str,
503
+ k: int = 4,
504
+ filter: Optional[dict] = None,
505
+ **kwargs: Any,
506
+ ) -> List[Document]:
507
+ """Async similarity search."""
508
+ import asyncio
509
+ loop = asyncio.get_event_loop()
510
+ return await loop.run_in_executor(
511
+ None, lambda: self.similarity_search(query, k=k, filter=filter, **kwargs)
512
+ )
513
+
514
+ async def asimilarity_search_with_score(
515
+ self,
516
+ query: str,
517
+ k: int = 4,
518
+ filter: Optional[dict] = None,
519
+ **kwargs: Any,
520
+ ) -> List[Tuple[Document, float]]:
521
+ """Async similarity search with scores."""
522
+ import asyncio
523
+ loop = asyncio.get_event_loop()
524
+ return await loop.run_in_executor(
525
+ None,
526
+ lambda: self.similarity_search_with_score(query, k=k, filter=filter, **kwargs),
527
+ )