dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/haystack.py
ADDED
|
@@ -0,0 +1,773 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DXMT Haystack Integration
|
|
3
|
+
|
|
4
|
+
This module provides a Haystack-compatible DocumentStore implementation backed by DXMT VectorDB.
|
|
5
|
+
Enables using DXMT's tiered storage (Memory → SSD → S3) in Haystack pipelines for RAG and search.
|
|
6
|
+
|
|
7
|
+
Example usage:
|
|
8
|
+
```python
|
|
9
|
+
from haystack import Pipeline, Document
|
|
10
|
+
from haystack.components.embedders import SentenceTransformersDocumentEmbedder
|
|
11
|
+
from haystack.components.retrievers import InMemoryEmbeddingRetriever
|
|
12
|
+
from dxmt import Client as DXMTClient
|
|
13
|
+
from dxmt.haystack import DXMTDocumentStore
|
|
14
|
+
|
|
15
|
+
# Initialize DXMT document store
|
|
16
|
+
client = DXMTClient(host="localhost", port=9999)
|
|
17
|
+
document_store = DXMTDocumentStore(
|
|
18
|
+
client=client,
|
|
19
|
+
table_name="documents",
|
|
20
|
+
embedding_dim=384,
|
|
21
|
+
distance_metric="cosine"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# Use in Haystack pipelines
|
|
25
|
+
documents = [
|
|
26
|
+
Document(content="DXMT is a vector database"),
|
|
27
|
+
Document(content="Haystack is a framework for LLM applications"),
|
|
28
|
+
]
|
|
29
|
+
document_store.write_documents(documents)
|
|
30
|
+
|
|
31
|
+
# Search
|
|
32
|
+
results = document_store.filter_documents(filters={"field": "meta.type", "operator": "==", "value": "doc"})
|
|
33
|
+
```
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import asyncio
|
|
37
|
+
import hashlib
|
|
38
|
+
import uuid
|
|
39
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
40
|
+
from datetime import datetime
|
|
41
|
+
from typing import Any, List, Dict, Optional, Literal
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
from haystack.dataclasses import Document
|
|
45
|
+
from haystack.document_stores.errors import DuplicateDocumentError, DocumentStoreError
|
|
46
|
+
from haystack.document_stores.types import DuplicatePolicy
|
|
47
|
+
HAYSTACK_AVAILABLE = True
|
|
48
|
+
except ImportError:
|
|
49
|
+
HAYSTACK_AVAILABLE = False
|
|
50
|
+
# Define minimal stubs for type hints
|
|
51
|
+
DuplicateDocumentError = Exception
|
|
52
|
+
DocumentStoreError = Exception
|
|
53
|
+
|
|
54
|
+
# Stub Document class
|
|
55
|
+
class Document:
|
|
56
|
+
def __init__(self, id=None, content=None, meta=None, embedding=None, dataframe=None, blob=None, **kwargs):
|
|
57
|
+
self.id = id
|
|
58
|
+
self.content = content
|
|
59
|
+
self.meta = meta or {}
|
|
60
|
+
self.embedding = embedding
|
|
61
|
+
self.dataframe = dataframe
|
|
62
|
+
self.blob = blob
|
|
63
|
+
self.score = None
|
|
64
|
+
|
|
65
|
+
def to_dict(self, **kwargs):
|
|
66
|
+
return {
|
|
67
|
+
"id": self.id,
|
|
68
|
+
"content": self.content,
|
|
69
|
+
"meta": self.meta,
|
|
70
|
+
"embedding": self.embedding,
|
|
71
|
+
"dataframe": self.dataframe,
|
|
72
|
+
"blob": self.blob,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
def from_dict(cls, data):
|
|
77
|
+
return cls(**data)
|
|
78
|
+
|
|
79
|
+
# Stub DuplicatePolicy enum
|
|
80
|
+
class DuplicatePolicy:
|
|
81
|
+
NONE = "none"
|
|
82
|
+
FAIL = "fail"
|
|
83
|
+
SKIP = "skip"
|
|
84
|
+
OVERWRITE = "overwrite"
|
|
85
|
+
|
|
86
|
+
from dxmt.client import DXMTClient
|
|
87
|
+
from dxmt.exceptions import DXMTError
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class DXMTDocumentStore:
|
|
91
|
+
"""
|
|
92
|
+
Haystack DocumentStore implementation using DXMT VectorDB as backend.
|
|
93
|
+
|
|
94
|
+
This store leverages DXMT's tiered storage architecture (Memory → SSD → S3) for
|
|
95
|
+
cost-efficient document storage and retrieval at scale.
|
|
96
|
+
|
|
97
|
+
Features:
|
|
98
|
+
- Full-text search with BM25 ranking
|
|
99
|
+
- Vector similarity search (cosine, euclidean, dot product)
|
|
100
|
+
- Metadata filtering with complex boolean logic
|
|
101
|
+
- High-performance search (3.56M QPS capability)
|
|
102
|
+
- Automatic embedding storage and retrieval
|
|
103
|
+
- Async operation support
|
|
104
|
+
|
|
105
|
+
Attributes:
|
|
106
|
+
client: DXMT client instance for backend communication
|
|
107
|
+
table_name: Name of the DXMT table storing documents
|
|
108
|
+
embedding_dim: Dimensionality of document embeddings
|
|
109
|
+
distance_metric: Similarity metric for vector search
|
|
110
|
+
return_embedding: Whether to return embeddings with documents
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
def __init__(
|
|
114
|
+
self,
|
|
115
|
+
client: DXMTClient,
|
|
116
|
+
table_name: str = "haystack_documents",
|
|
117
|
+
embedding_dim: int = 768,
|
|
118
|
+
distance_metric: Literal["cosine", "euclidean", "dot"] = "cosine",
|
|
119
|
+
return_embedding: bool = False,
|
|
120
|
+
async_executor: Optional[ThreadPoolExecutor] = None,
|
|
121
|
+
):
|
|
122
|
+
"""
|
|
123
|
+
Initialize the DXMT Document Store.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
client: DXMT client instance
|
|
127
|
+
table_name: Name for the document table (default: "haystack_documents")
|
|
128
|
+
embedding_dim: Dimension of document embeddings (default: 768)
|
|
129
|
+
distance_metric: Similarity metric - "cosine", "euclidean", or "dot" (default: "cosine")
|
|
130
|
+
return_embedding: Whether to return embeddings with documents (default: False)
|
|
131
|
+
async_executor: Optional executor for async operations (default: None, creates single-threaded executor)
|
|
132
|
+
|
|
133
|
+
Raises:
|
|
134
|
+
ImportError: If haystack-ai is not installed
|
|
135
|
+
DXMTError: If table creation fails
|
|
136
|
+
|
|
137
|
+
Note:
|
|
138
|
+
While haystack-ai is recommended, this class can work without it for testing.
|
|
139
|
+
However, real usage requires haystack-ai to be installed.
|
|
140
|
+
"""
|
|
141
|
+
# Note: We don't enforce HAYSTACK_AVAILABLE here to allow mock testing
|
|
142
|
+
# Users will get errors when trying to use real Haystack features without it
|
|
143
|
+
|
|
144
|
+
self.client = client
|
|
145
|
+
self.table_name = table_name
|
|
146
|
+
self.embedding_dim = embedding_dim
|
|
147
|
+
self.distance_metric = distance_metric
|
|
148
|
+
self.return_embedding = return_embedding
|
|
149
|
+
|
|
150
|
+
# Async executor setup
|
|
151
|
+
self._owns_executor = async_executor is None
|
|
152
|
+
self.executor = (
|
|
153
|
+
ThreadPoolExecutor(
|
|
154
|
+
thread_name_prefix=f"async-dxmt-docstore-executor-{id(self)}",
|
|
155
|
+
max_workers=1
|
|
156
|
+
)
|
|
157
|
+
if async_executor is None
|
|
158
|
+
else async_executor
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Create table if not exists
|
|
162
|
+
self._ensure_table()
|
|
163
|
+
|
|
164
|
+
def __del__(self):
|
|
165
|
+
"""Cleanup executor when instance is destroyed."""
|
|
166
|
+
if hasattr(self, "_owns_executor") and self._owns_executor and hasattr(self, "executor"):
|
|
167
|
+
self.executor.shutdown(wait=True)
|
|
168
|
+
|
|
169
|
+
def shutdown(self):
|
|
170
|
+
"""Explicitly shutdown the async executor."""
|
|
171
|
+
if self._owns_executor:
|
|
172
|
+
self.executor.shutdown(wait=True)
|
|
173
|
+
|
|
174
|
+
def _ensure_table(self) -> None:
|
|
175
|
+
"""Create table if it doesn't exist."""
|
|
176
|
+
try:
|
|
177
|
+
self.client.get_table(self.table_name)
|
|
178
|
+
except DXMTError:
|
|
179
|
+
# Table doesn't exist, create it
|
|
180
|
+
try:
|
|
181
|
+
self.client.create_table(
|
|
182
|
+
name=self.table_name,
|
|
183
|
+
dimension=self.embedding_dim,
|
|
184
|
+
distance_metric=self.distance_metric,
|
|
185
|
+
enable_full_text_search=True, # Enable BM25 search
|
|
186
|
+
)
|
|
187
|
+
except Exception as e:
|
|
188
|
+
raise DocumentStoreError(f"Failed to create table {self.table_name}: {e}")
|
|
189
|
+
|
|
190
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
191
|
+
"""
|
|
192
|
+
Serialize the document store to a dictionary.
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
Dictionary with store configuration
|
|
196
|
+
"""
|
|
197
|
+
return {
|
|
198
|
+
"type": "dxmt.haystack.DXMTDocumentStore",
|
|
199
|
+
"init_parameters": {
|
|
200
|
+
"table_name": self.table_name,
|
|
201
|
+
"embedding_dim": self.embedding_dim,
|
|
202
|
+
"distance_metric": self.distance_metric,
|
|
203
|
+
"return_embedding": self.return_embedding,
|
|
204
|
+
},
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
@classmethod
|
|
208
|
+
def from_dict(cls, data: Dict[str, Any]) -> "DXMTDocumentStore":
|
|
209
|
+
"""
|
|
210
|
+
Deserialize the document store from a dictionary.
|
|
211
|
+
|
|
212
|
+
Note: Requires a DXMT client to be provided separately as it cannot be serialized.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
data: Serialized store configuration
|
|
216
|
+
|
|
217
|
+
Returns:
|
|
218
|
+
DXMTDocumentStore instance
|
|
219
|
+
|
|
220
|
+
Raises:
|
|
221
|
+
ValueError: If client is not provided
|
|
222
|
+
"""
|
|
223
|
+
init_params = data.get("init_parameters", {})
|
|
224
|
+
if "client" not in init_params:
|
|
225
|
+
raise ValueError(
|
|
226
|
+
"DXMT client must be provided to deserialize DXMTDocumentStore. "
|
|
227
|
+
"Create a client and pass it as client=DXMTClient(...)"
|
|
228
|
+
)
|
|
229
|
+
return cls(**init_params)
|
|
230
|
+
|
|
231
|
+
def count_documents(self) -> int:
|
|
232
|
+
"""
|
|
233
|
+
Count the total number of documents in the store.
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
Number of documents
|
|
237
|
+
"""
|
|
238
|
+
try:
|
|
239
|
+
table_info = self.client.get_table(self.table_name)
|
|
240
|
+
return table_info.get("record_count", 0)
|
|
241
|
+
except Exception as e:
|
|
242
|
+
raise DocumentStoreError(f"Failed to count documents: {e}")
|
|
243
|
+
|
|
244
|
+
def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
|
|
245
|
+
"""
|
|
246
|
+
Retrieve documents matching the provided filters.
|
|
247
|
+
|
|
248
|
+
Haystack filter format is already compatible with DXMT's filter structure:
|
|
249
|
+
```python
|
|
250
|
+
filters = {
|
|
251
|
+
"operator": "AND",
|
|
252
|
+
"conditions": [
|
|
253
|
+
{"field": "meta.type", "operator": "==", "value": "article"},
|
|
254
|
+
{"field": "meta.rating", "operator": ">=", "value": 3}
|
|
255
|
+
]
|
|
256
|
+
}
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Args:
|
|
260
|
+
filters: Filter dict with operator + conditions or field + operator + value
|
|
261
|
+
|
|
262
|
+
Returns:
|
|
263
|
+
List of Documents matching the filters
|
|
264
|
+
"""
|
|
265
|
+
try:
|
|
266
|
+
# DXMT uses the same filter format as Haystack, can pass through directly
|
|
267
|
+
dxmt_filter = self._convert_haystack_filters(filters) if filters else None
|
|
268
|
+
|
|
269
|
+
# Get all matching records
|
|
270
|
+
# Note: DXMT doesn't have a direct filter_all endpoint, so we use search with zero vector
|
|
271
|
+
# This is a workaround - in production you may want to add a scan endpoint to DXMT
|
|
272
|
+
results = self.client.search(
|
|
273
|
+
table_name=self.table_name,
|
|
274
|
+
vector=[0.0] * self.embedding_dim, # Dummy vector
|
|
275
|
+
k=10000, # Large number to get all matches
|
|
276
|
+
filter=dxmt_filter,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
# Convert DXMT records to Haystack Documents
|
|
280
|
+
documents = []
|
|
281
|
+
for result in results:
|
|
282
|
+
doc = self._dxmt_to_haystack_doc(result)
|
|
283
|
+
if not self.return_embedding:
|
|
284
|
+
doc.embedding = None
|
|
285
|
+
documents.append(doc)
|
|
286
|
+
|
|
287
|
+
return documents
|
|
288
|
+
|
|
289
|
+
except Exception as e:
|
|
290
|
+
raise DocumentStoreError(f"Failed to filter documents: {e}")
|
|
291
|
+
|
|
292
|
+
def write_documents(
|
|
293
|
+
self,
|
|
294
|
+
documents: List[Document],
|
|
295
|
+
policy: DuplicatePolicy = DuplicatePolicy.NONE
|
|
296
|
+
) -> int:
|
|
297
|
+
"""
|
|
298
|
+
Write documents to the store.
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
documents: List of Haystack Document objects
|
|
302
|
+
policy: How to handle duplicates (NONE, FAIL, SKIP, OVERWRITE)
|
|
303
|
+
|
|
304
|
+
Returns:
|
|
305
|
+
Number of documents written
|
|
306
|
+
|
|
307
|
+
Raises:
|
|
308
|
+
DuplicateDocumentError: If policy is FAIL and duplicate exists
|
|
309
|
+
DocumentStoreError: If write operation fails
|
|
310
|
+
"""
|
|
311
|
+
if not isinstance(documents, list):
|
|
312
|
+
raise ValueError("documents must be a list of Document objects")
|
|
313
|
+
|
|
314
|
+
if not documents:
|
|
315
|
+
return 0
|
|
316
|
+
|
|
317
|
+
if policy == DuplicatePolicy.NONE:
|
|
318
|
+
policy = DuplicatePolicy.FAIL
|
|
319
|
+
|
|
320
|
+
written_count = 0
|
|
321
|
+
|
|
322
|
+
try:
|
|
323
|
+
for doc in documents:
|
|
324
|
+
# Check for duplicates
|
|
325
|
+
if policy != DuplicatePolicy.OVERWRITE:
|
|
326
|
+
existing = self._get_document_by_id(doc.id)
|
|
327
|
+
if existing:
|
|
328
|
+
if policy == DuplicatePolicy.FAIL:
|
|
329
|
+
raise DuplicateDocumentError(f"Document with id '{doc.id}' already exists")
|
|
330
|
+
if policy == DuplicatePolicy.SKIP:
|
|
331
|
+
continue
|
|
332
|
+
|
|
333
|
+
# Convert Haystack Document to DXMT record
|
|
334
|
+
record = self._haystack_to_dxmt_record(doc)
|
|
335
|
+
|
|
336
|
+
# Insert or update
|
|
337
|
+
if policy == DuplicatePolicy.OVERWRITE and self._get_document_by_id(doc.id):
|
|
338
|
+
self.client.update_record(
|
|
339
|
+
table_name=self.table_name,
|
|
340
|
+
record_id=doc.id,
|
|
341
|
+
vector=doc.embedding if doc.embedding else None,
|
|
342
|
+
payload=record["payload"],
|
|
343
|
+
)
|
|
344
|
+
else:
|
|
345
|
+
self.client.insert_records(
|
|
346
|
+
table_name=self.table_name,
|
|
347
|
+
records=[record],
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
written_count += 1
|
|
351
|
+
|
|
352
|
+
except DuplicateDocumentError:
|
|
353
|
+
raise
|
|
354
|
+
except Exception as e:
|
|
355
|
+
raise DocumentStoreError(f"Failed to write documents: {e}")
|
|
356
|
+
|
|
357
|
+
return written_count
|
|
358
|
+
|
|
359
|
+
def delete_documents(self, document_ids: List[str]) -> None:
|
|
360
|
+
"""
|
|
361
|
+
Delete documents by their IDs.
|
|
362
|
+
|
|
363
|
+
Args:
|
|
364
|
+
document_ids: List of document IDs to delete
|
|
365
|
+
"""
|
|
366
|
+
if not document_ids:
|
|
367
|
+
return
|
|
368
|
+
|
|
369
|
+
try:
|
|
370
|
+
self.client.delete_records(
|
|
371
|
+
table_name=self.table_name,
|
|
372
|
+
ids=document_ids,
|
|
373
|
+
)
|
|
374
|
+
except Exception as e:
|
|
375
|
+
raise DocumentStoreError(f"Failed to delete documents: {e}")
|
|
376
|
+
|
|
377
|
+
def bm25_retrieval(
|
|
378
|
+
self,
|
|
379
|
+
query: str,
|
|
380
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
381
|
+
top_k: int = 10,
|
|
382
|
+
scale_score: bool = False,
|
|
383
|
+
) -> List[Document]:
|
|
384
|
+
"""
|
|
385
|
+
Retrieve documents using BM25 full-text search.
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
query: Search query string
|
|
389
|
+
filters: Optional metadata filters
|
|
390
|
+
top_k: Number of top documents to return
|
|
391
|
+
scale_score: Whether to scale scores to [0, 1] range
|
|
392
|
+
|
|
393
|
+
Returns:
|
|
394
|
+
List of Documents ranked by BM25 relevance
|
|
395
|
+
"""
|
|
396
|
+
if not query:
|
|
397
|
+
raise ValueError("Query must be a non-empty string")
|
|
398
|
+
|
|
399
|
+
try:
|
|
400
|
+
# Use DXMT's full-text search
|
|
401
|
+
dxmt_filter = self._convert_haystack_filters(filters) if filters else None
|
|
402
|
+
|
|
403
|
+
results = self.client.full_text_search(
|
|
404
|
+
table_name=self.table_name,
|
|
405
|
+
query=query,
|
|
406
|
+
k=top_k,
|
|
407
|
+
filter=dxmt_filter,
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
# Convert to Haystack Documents with scores
|
|
411
|
+
documents = []
|
|
412
|
+
for result in results:
|
|
413
|
+
doc = self._dxmt_to_haystack_doc(result)
|
|
414
|
+
# BM25 score from DXMT
|
|
415
|
+
score = result.get("score", 0.0)
|
|
416
|
+
if scale_score:
|
|
417
|
+
# Scale using expit function like Haystack's InMemoryDocumentStore
|
|
418
|
+
score = self._expit(score / 8.0) # BM25_SCALING_FACTOR = 8
|
|
419
|
+
doc.score = score
|
|
420
|
+
|
|
421
|
+
if not self.return_embedding:
|
|
422
|
+
doc.embedding = None
|
|
423
|
+
|
|
424
|
+
documents.append(doc)
|
|
425
|
+
|
|
426
|
+
return documents
|
|
427
|
+
|
|
428
|
+
except Exception as e:
|
|
429
|
+
raise DocumentStoreError(f"BM25 retrieval failed: {e}")
|
|
430
|
+
|
|
431
|
+
def embedding_retrieval(
|
|
432
|
+
self,
|
|
433
|
+
query_embedding: List[float],
|
|
434
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
435
|
+
top_k: int = 10,
|
|
436
|
+
scale_score: bool = False,
|
|
437
|
+
return_embedding: Optional[bool] = None,
|
|
438
|
+
) -> List[Document]:
|
|
439
|
+
"""
|
|
440
|
+
Retrieve documents by embedding similarity.
|
|
441
|
+
|
|
442
|
+
Args:
|
|
443
|
+
query_embedding: Query embedding vector
|
|
444
|
+
filters: Optional metadata filters
|
|
445
|
+
top_k: Number of top documents to return
|
|
446
|
+
scale_score: Whether to scale scores to [0, 1] range
|
|
447
|
+
return_embedding: Override return_embedding setting
|
|
448
|
+
|
|
449
|
+
Returns:
|
|
450
|
+
List of Documents ranked by vector similarity
|
|
451
|
+
"""
|
|
452
|
+
if not query_embedding or not isinstance(query_embedding[0], float):
|
|
453
|
+
raise ValueError("query_embedding must be a non-empty list of floats")
|
|
454
|
+
|
|
455
|
+
if len(query_embedding) != self.embedding_dim:
|
|
456
|
+
raise ValueError(
|
|
457
|
+
f"Query embedding dimension ({len(query_embedding)}) "
|
|
458
|
+
f"does not match store dimension ({self.embedding_dim})"
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
try:
|
|
462
|
+
# Convert Haystack filters to DXMT format
|
|
463
|
+
dxmt_filter = self._convert_haystack_filters(filters) if filters else None
|
|
464
|
+
|
|
465
|
+
# Add filter to exclude documents without real embeddings
|
|
466
|
+
# This matches Haystack's behavior of skipping documents without embeddings
|
|
467
|
+
real_embedding_filter = {"_has_real_embedding": True}
|
|
468
|
+
|
|
469
|
+
if dxmt_filter:
|
|
470
|
+
# Combine user filter with real embedding filter using AND
|
|
471
|
+
combined_filter = {
|
|
472
|
+
"$and": [
|
|
473
|
+
dxmt_filter,
|
|
474
|
+
real_embedding_filter
|
|
475
|
+
]
|
|
476
|
+
}
|
|
477
|
+
else:
|
|
478
|
+
combined_filter = real_embedding_filter
|
|
479
|
+
|
|
480
|
+
results = self.client.search(
|
|
481
|
+
table_name=self.table_name,
|
|
482
|
+
vector=query_embedding,
|
|
483
|
+
k=top_k,
|
|
484
|
+
filter=combined_filter,
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
# Determine whether to return embeddings
|
|
488
|
+
resolved_return_embedding = (
|
|
489
|
+
self.return_embedding if return_embedding is None else return_embedding
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
# Convert to Haystack Documents
|
|
493
|
+
documents = []
|
|
494
|
+
for result in results:
|
|
495
|
+
doc = self._dxmt_to_haystack_doc(result)
|
|
496
|
+
# Convert distance to similarity score
|
|
497
|
+
distance = result.get("score", 0.0)
|
|
498
|
+
score = self._normalize_score(distance, scale_score)
|
|
499
|
+
doc.score = score
|
|
500
|
+
|
|
501
|
+
if not resolved_return_embedding:
|
|
502
|
+
doc.embedding = None
|
|
503
|
+
|
|
504
|
+
documents.append(doc)
|
|
505
|
+
|
|
506
|
+
return documents
|
|
507
|
+
|
|
508
|
+
except Exception as e:
|
|
509
|
+
raise DocumentStoreError(f"Embedding retrieval failed: {e}")
|
|
510
|
+
|
|
511
|
+
# Async methods
|
|
512
|
+
async def count_documents_async(self) -> int:
|
|
513
|
+
"""Async version of count_documents."""
|
|
514
|
+
return await asyncio.get_event_loop().run_in_executor(
|
|
515
|
+
self.executor, self.count_documents
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
async def filter_documents_async(
|
|
519
|
+
self, filters: Optional[Dict[str, Any]] = None
|
|
520
|
+
) -> List[Document]:
|
|
521
|
+
"""Async version of filter_documents."""
|
|
522
|
+
return await asyncio.get_event_loop().run_in_executor(
|
|
523
|
+
self.executor, lambda: self.filter_documents(filters=filters)
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
async def write_documents_async(
|
|
527
|
+
self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
|
|
528
|
+
) -> int:
|
|
529
|
+
"""Async version of write_documents."""
|
|
530
|
+
return await asyncio.get_event_loop().run_in_executor(
|
|
531
|
+
self.executor, lambda: self.write_documents(documents=documents, policy=policy)
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
async def delete_documents_async(self, document_ids: List[str]) -> None:
|
|
535
|
+
"""Async version of delete_documents."""
|
|
536
|
+
await asyncio.get_event_loop().run_in_executor(
|
|
537
|
+
self.executor, lambda: self.delete_documents(document_ids=document_ids)
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
async def bm25_retrieval_async(
|
|
541
|
+
self,
|
|
542
|
+
query: str,
|
|
543
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
544
|
+
top_k: int = 10,
|
|
545
|
+
scale_score: bool = False,
|
|
546
|
+
) -> List[Document]:
|
|
547
|
+
"""Async version of bm25_retrieval."""
|
|
548
|
+
return await asyncio.get_event_loop().run_in_executor(
|
|
549
|
+
self.executor,
|
|
550
|
+
lambda: self.bm25_retrieval(
|
|
551
|
+
query=query, filters=filters, top_k=top_k, scale_score=scale_score
|
|
552
|
+
),
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
async def embedding_retrieval_async(
|
|
556
|
+
self,
|
|
557
|
+
query_embedding: List[float],
|
|
558
|
+
filters: Optional[Dict[str, Any]] = None,
|
|
559
|
+
top_k: int = 10,
|
|
560
|
+
scale_score: bool = False,
|
|
561
|
+
return_embedding: bool = False,
|
|
562
|
+
) -> List[Document]:
|
|
563
|
+
"""Async version of embedding_retrieval."""
|
|
564
|
+
return await asyncio.get_event_loop().run_in_executor(
|
|
565
|
+
self.executor,
|
|
566
|
+
lambda: self.embedding_retrieval(
|
|
567
|
+
query_embedding=query_embedding,
|
|
568
|
+
filters=filters,
|
|
569
|
+
top_k=top_k,
|
|
570
|
+
scale_score=scale_score,
|
|
571
|
+
return_embedding=return_embedding,
|
|
572
|
+
),
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
# Helper methods
|
|
576
|
+
def _convert_haystack_filters(self, filters: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
577
|
+
"""
|
|
578
|
+
Convert Haystack filters to DXMT MongoDB-style format.
|
|
579
|
+
|
|
580
|
+
Haystack format:
|
|
581
|
+
{"field": "meta.type", "operator": "==", "value": "programming"}
|
|
582
|
+
|
|
583
|
+
DXMT MongoDB format:
|
|
584
|
+
{"type": "programming"}
|
|
585
|
+
or {"type": {"$eq": "programming"}}
|
|
586
|
+
"""
|
|
587
|
+
if not filters:
|
|
588
|
+
return None
|
|
589
|
+
|
|
590
|
+
# Operator mapping from Haystack to MongoDB
|
|
591
|
+
OPERATOR_MAP = {
|
|
592
|
+
"==": "$eq",
|
|
593
|
+
"!=": "$ne",
|
|
594
|
+
">": "$gt",
|
|
595
|
+
">=": "$gte",
|
|
596
|
+
"<": "$lt",
|
|
597
|
+
"<=": "$lte",
|
|
598
|
+
"in": "$in",
|
|
599
|
+
"not in": "$nin",
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
def convert_condition(condition: Dict[str, Any]) -> Dict[str, Any]:
|
|
603
|
+
if "field" in condition:
|
|
604
|
+
# Single comparison condition
|
|
605
|
+
field = condition["field"]
|
|
606
|
+
# Strip "meta." prefix if present
|
|
607
|
+
if field.startswith("meta."):
|
|
608
|
+
field = field[5:]
|
|
609
|
+
|
|
610
|
+
operator = condition["operator"]
|
|
611
|
+
value = condition["value"]
|
|
612
|
+
|
|
613
|
+
# For equality, use simple form
|
|
614
|
+
if operator == "==":
|
|
615
|
+
return {field: value}
|
|
616
|
+
else:
|
|
617
|
+
# For other operators, use MongoDB operator form
|
|
618
|
+
mongo_op = OPERATOR_MAP.get(operator, operator)
|
|
619
|
+
return {field: {mongo_op: value}}
|
|
620
|
+
|
|
621
|
+
elif "operator" in condition and "conditions" in condition:
|
|
622
|
+
# Logical operator (AND, OR, NOT)
|
|
623
|
+
logical_op = condition["operator"].upper()
|
|
624
|
+
sub_conditions = condition["conditions"]
|
|
625
|
+
|
|
626
|
+
if logical_op == "AND":
|
|
627
|
+
converted = [convert_condition(c) for c in sub_conditions]
|
|
628
|
+
return {"$and": converted}
|
|
629
|
+
elif logical_op == "OR":
|
|
630
|
+
converted = [convert_condition(c) for c in sub_conditions]
|
|
631
|
+
return {"$or": converted}
|
|
632
|
+
elif logical_op == "NOT":
|
|
633
|
+
# NOT typically has one condition
|
|
634
|
+
converted = [convert_condition(c) for c in sub_conditions]
|
|
635
|
+
return {"$not": converted}
|
|
636
|
+
else:
|
|
637
|
+
# Fallback: treat as AND
|
|
638
|
+
converted = [convert_condition(c) for c in sub_conditions]
|
|
639
|
+
return {"$and": converted}
|
|
640
|
+
else:
|
|
641
|
+
# Unknown format, return as-is
|
|
642
|
+
return condition
|
|
643
|
+
|
|
644
|
+
return convert_condition(filters)
|
|
645
|
+
|
|
646
|
+
def _haystack_to_dxmt_record(self, doc: Document) -> Dict[str, Any]:
|
|
647
|
+
"""Convert Haystack Document to DXMT record format."""
|
|
648
|
+
# Generate ID if not present
|
|
649
|
+
doc_id = doc.id if doc.id else str(uuid.uuid4())
|
|
650
|
+
|
|
651
|
+
# Prepare payload (metadata + content)
|
|
652
|
+
payload = dict(doc.meta) if doc.meta else {}
|
|
653
|
+
payload["content"] = doc.content
|
|
654
|
+
|
|
655
|
+
# Optional fields - only add if present
|
|
656
|
+
if hasattr(doc, 'dataframe') and doc.dataframe is not None:
|
|
657
|
+
payload["dataframe"] = doc.dataframe
|
|
658
|
+
if hasattr(doc, 'blob') and doc.blob is not None:
|
|
659
|
+
payload["blob"] = doc.blob
|
|
660
|
+
|
|
661
|
+
# Add content hash for deduplication
|
|
662
|
+
if doc.content:
|
|
663
|
+
content_hash = hashlib.md5(doc.content.encode()).hexdigest()
|
|
664
|
+
payload["content_hash"] = content_hash
|
|
665
|
+
|
|
666
|
+
# Prepare record
|
|
667
|
+
record = {
|
|
668
|
+
"id": doc_id,
|
|
669
|
+
"payload": payload,
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
# Add embedding if present, otherwise use dummy vector
|
|
673
|
+
if doc.embedding:
|
|
674
|
+
if len(doc.embedding) != self.embedding_dim:
|
|
675
|
+
raise ValueError(
|
|
676
|
+
f"Document embedding dimension ({len(doc.embedding)}) "
|
|
677
|
+
f"does not match store dimension ({self.embedding_dim})"
|
|
678
|
+
)
|
|
679
|
+
record["vector"] = doc.embedding
|
|
680
|
+
# Mark that this document has a real embedding (not a dummy)
|
|
681
|
+
payload["_has_real_embedding"] = True
|
|
682
|
+
else:
|
|
683
|
+
# DXMT requires a non-zero vector field
|
|
684
|
+
# Use a small constant vector for BM25-only documents
|
|
685
|
+
# Mark this as a dummy vector so we can filter it out during embedding retrieval
|
|
686
|
+
record["vector"] = [0.001] * self.embedding_dim
|
|
687
|
+
payload["_has_real_embedding"] = False
|
|
688
|
+
|
|
689
|
+
return record
|
|
690
|
+
|
|
691
|
+
def _dxmt_to_haystack_doc(self, result: Dict[str, Any]) -> Document:
|
|
692
|
+
"""Convert DXMT search result to Haystack Document."""
|
|
693
|
+
metadata = result.get("metadata", {})
|
|
694
|
+
|
|
695
|
+
# Extract core fields
|
|
696
|
+
content = metadata.pop("content", None)
|
|
697
|
+
dataframe = metadata.pop("dataframe", None)
|
|
698
|
+
blob = metadata.pop("blob", None)
|
|
699
|
+
content_hash = metadata.pop("content_hash", None)
|
|
700
|
+
# Remove internal field (not part of user metadata)
|
|
701
|
+
metadata.pop("_has_real_embedding", None)
|
|
702
|
+
|
|
703
|
+
# Get embedding if present
|
|
704
|
+
embedding = result.get("vector", None)
|
|
705
|
+
|
|
706
|
+
# Build Document with only supported fields
|
|
707
|
+
doc_kwargs = {
|
|
708
|
+
"id": result.get("id"),
|
|
709
|
+
"content": content,
|
|
710
|
+
"meta": metadata,
|
|
711
|
+
"embedding": embedding,
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
# Only add optional fields if they exist and Document supports them
|
|
715
|
+
if dataframe is not None:
|
|
716
|
+
try:
|
|
717
|
+
doc_kwargs["dataframe"] = dataframe
|
|
718
|
+
except:
|
|
719
|
+
pass # Skip if Document doesn't support dataframe
|
|
720
|
+
|
|
721
|
+
if blob is not None:
|
|
722
|
+
try:
|
|
723
|
+
doc_kwargs["blob"] = blob
|
|
724
|
+
except:
|
|
725
|
+
pass # Skip if Document doesn't support blob
|
|
726
|
+
|
|
727
|
+
doc = Document(**doc_kwargs)
|
|
728
|
+
|
|
729
|
+
return doc
|
|
730
|
+
|
|
731
|
+
def _get_document_by_id(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
|
732
|
+
"""Get a document by ID, return None if not found."""
|
|
733
|
+
try:
|
|
734
|
+
results = self.client.get_records(
|
|
735
|
+
table_name=self.table_name,
|
|
736
|
+
ids=[doc_id],
|
|
737
|
+
)
|
|
738
|
+
return results[0] if results else None
|
|
739
|
+
except:
|
|
740
|
+
return None
|
|
741
|
+
|
|
742
|
+
def _normalize_score(self, distance: float, scale: bool) -> float:
|
|
743
|
+
"""
|
|
744
|
+
Convert distance to similarity score.
|
|
745
|
+
|
|
746
|
+
Args:
|
|
747
|
+
distance: Distance value from DXMT
|
|
748
|
+
scale: Whether to scale to [0, 1] range
|
|
749
|
+
|
|
750
|
+
Returns:
|
|
751
|
+
Similarity score
|
|
752
|
+
"""
|
|
753
|
+
if not scale:
|
|
754
|
+
return distance
|
|
755
|
+
|
|
756
|
+
# Convert distance to similarity based on metric
|
|
757
|
+
if self.distance_metric == "cosine":
|
|
758
|
+
# Cosine distance is in [0, 2], convert to similarity [0, 1]
|
|
759
|
+
# 0 distance = 1 similarity, 2 distance = 0 similarity
|
|
760
|
+
return (distance + 1) / 2
|
|
761
|
+
elif self.distance_metric == "dot":
|
|
762
|
+
# Dot product can be negative, scale using expit
|
|
763
|
+
return self._expit(distance / 100.0)
|
|
764
|
+
else:
|
|
765
|
+
# Euclidean/other: use exponential decay
|
|
766
|
+
import math
|
|
767
|
+
return math.exp(-distance)
|
|
768
|
+
|
|
769
|
+
@staticmethod
|
|
770
|
+
def _expit(x: float) -> float:
|
|
771
|
+
"""Expit function (inverse of logit), maps R to [0, 1]."""
|
|
772
|
+
import math
|
|
773
|
+
return 1 / (1 + math.exp(-x))
|