dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dxmt/haystack.py ADDED
@@ -0,0 +1,773 @@
1
+ """
2
+ DXMT Haystack Integration
3
+
4
+ This module provides a Haystack-compatible DocumentStore implementation backed by DXMT VectorDB.
5
+ Enables using DXMT's tiered storage (Memory → SSD → S3) in Haystack pipelines for RAG and search.
6
+
7
+ Example usage:
8
+ ```python
9
+ from haystack import Pipeline, Document
10
+ from haystack.components.embedders import SentenceTransformersDocumentEmbedder
11
+ from haystack.components.retrievers import InMemoryEmbeddingRetriever
12
+ from dxmt import Client as DXMTClient
13
+ from dxmt.haystack import DXMTDocumentStore
14
+
15
+ # Initialize DXMT document store
16
+ client = DXMTClient(host="localhost", port=9999)
17
+ document_store = DXMTDocumentStore(
18
+ client=client,
19
+ table_name="documents",
20
+ embedding_dim=384,
21
+ distance_metric="cosine"
22
+ )
23
+
24
+ # Use in Haystack pipelines
25
+ documents = [
26
+ Document(content="DXMT is a vector database"),
27
+ Document(content="Haystack is a framework for LLM applications"),
28
+ ]
29
+ document_store.write_documents(documents)
30
+
31
+ # Search
32
+ results = document_store.filter_documents(filters={"field": "meta.type", "operator": "==", "value": "doc"})
33
+ ```
34
+ """
35
+
36
+ import asyncio
37
+ import hashlib
38
+ import uuid
39
+ from concurrent.futures import ThreadPoolExecutor
40
+ from datetime import datetime
41
+ from typing import Any, List, Dict, Optional, Literal
42
+
43
+ try:
44
+ from haystack.dataclasses import Document
45
+ from haystack.document_stores.errors import DuplicateDocumentError, DocumentStoreError
46
+ from haystack.document_stores.types import DuplicatePolicy
47
+ HAYSTACK_AVAILABLE = True
48
+ except ImportError:
49
+ HAYSTACK_AVAILABLE = False
50
+ # Define minimal stubs for type hints
51
+ DuplicateDocumentError = Exception
52
+ DocumentStoreError = Exception
53
+
54
+ # Stub Document class
55
+ class Document:
56
+ def __init__(self, id=None, content=None, meta=None, embedding=None, dataframe=None, blob=None, **kwargs):
57
+ self.id = id
58
+ self.content = content
59
+ self.meta = meta or {}
60
+ self.embedding = embedding
61
+ self.dataframe = dataframe
62
+ self.blob = blob
63
+ self.score = None
64
+
65
+ def to_dict(self, **kwargs):
66
+ return {
67
+ "id": self.id,
68
+ "content": self.content,
69
+ "meta": self.meta,
70
+ "embedding": self.embedding,
71
+ "dataframe": self.dataframe,
72
+ "blob": self.blob,
73
+ }
74
+
75
+ @classmethod
76
+ def from_dict(cls, data):
77
+ return cls(**data)
78
+
79
+ # Stub DuplicatePolicy enum
80
+ class DuplicatePolicy:
81
+ NONE = "none"
82
+ FAIL = "fail"
83
+ SKIP = "skip"
84
+ OVERWRITE = "overwrite"
85
+
86
+ from dxmt.client import DXMTClient
87
+ from dxmt.exceptions import DXMTError
88
+
89
+
90
+ class DXMTDocumentStore:
91
+ """
92
+ Haystack DocumentStore implementation using DXMT VectorDB as backend.
93
+
94
+ This store leverages DXMT's tiered storage architecture (Memory → SSD → S3) for
95
+ cost-efficient document storage and retrieval at scale.
96
+
97
+ Features:
98
+ - Full-text search with BM25 ranking
99
+ - Vector similarity search (cosine, euclidean, dot product)
100
+ - Metadata filtering with complex boolean logic
101
+ - High-performance search (3.56M QPS capability)
102
+ - Automatic embedding storage and retrieval
103
+ - Async operation support
104
+
105
+ Attributes:
106
+ client: DXMT client instance for backend communication
107
+ table_name: Name of the DXMT table storing documents
108
+ embedding_dim: Dimensionality of document embeddings
109
+ distance_metric: Similarity metric for vector search
110
+ return_embedding: Whether to return embeddings with documents
111
+ """
112
+
113
+ def __init__(
114
+ self,
115
+ client: DXMTClient,
116
+ table_name: str = "haystack_documents",
117
+ embedding_dim: int = 768,
118
+ distance_metric: Literal["cosine", "euclidean", "dot"] = "cosine",
119
+ return_embedding: bool = False,
120
+ async_executor: Optional[ThreadPoolExecutor] = None,
121
+ ):
122
+ """
123
+ Initialize the DXMT Document Store.
124
+
125
+ Args:
126
+ client: DXMT client instance
127
+ table_name: Name for the document table (default: "haystack_documents")
128
+ embedding_dim: Dimension of document embeddings (default: 768)
129
+ distance_metric: Similarity metric - "cosine", "euclidean", or "dot" (default: "cosine")
130
+ return_embedding: Whether to return embeddings with documents (default: False)
131
+ async_executor: Optional executor for async operations (default: None, creates single-threaded executor)
132
+
133
+ Raises:
134
+ ImportError: If haystack-ai is not installed
135
+ DXMTError: If table creation fails
136
+
137
+ Note:
138
+ While haystack-ai is recommended, this class can work without it for testing.
139
+ However, real usage requires haystack-ai to be installed.
140
+ """
141
+ # Note: We don't enforce HAYSTACK_AVAILABLE here to allow mock testing
142
+ # Users will get errors when trying to use real Haystack features without it
143
+
144
+ self.client = client
145
+ self.table_name = table_name
146
+ self.embedding_dim = embedding_dim
147
+ self.distance_metric = distance_metric
148
+ self.return_embedding = return_embedding
149
+
150
+ # Async executor setup
151
+ self._owns_executor = async_executor is None
152
+ self.executor = (
153
+ ThreadPoolExecutor(
154
+ thread_name_prefix=f"async-dxmt-docstore-executor-{id(self)}",
155
+ max_workers=1
156
+ )
157
+ if async_executor is None
158
+ else async_executor
159
+ )
160
+
161
+ # Create table if not exists
162
+ self._ensure_table()
163
+
164
+ def __del__(self):
165
+ """Cleanup executor when instance is destroyed."""
166
+ if hasattr(self, "_owns_executor") and self._owns_executor and hasattr(self, "executor"):
167
+ self.executor.shutdown(wait=True)
168
+
169
+ def shutdown(self):
170
+ """Explicitly shutdown the async executor."""
171
+ if self._owns_executor:
172
+ self.executor.shutdown(wait=True)
173
+
174
+ def _ensure_table(self) -> None:
175
+ """Create table if it doesn't exist."""
176
+ try:
177
+ self.client.get_table(self.table_name)
178
+ except DXMTError:
179
+ # Table doesn't exist, create it
180
+ try:
181
+ self.client.create_table(
182
+ name=self.table_name,
183
+ dimension=self.embedding_dim,
184
+ distance_metric=self.distance_metric,
185
+ enable_full_text_search=True, # Enable BM25 search
186
+ )
187
+ except Exception as e:
188
+ raise DocumentStoreError(f"Failed to create table {self.table_name}: {e}")
189
+
190
+ def to_dict(self) -> Dict[str, Any]:
191
+ """
192
+ Serialize the document store to a dictionary.
193
+
194
+ Returns:
195
+ Dictionary with store configuration
196
+ """
197
+ return {
198
+ "type": "dxmt.haystack.DXMTDocumentStore",
199
+ "init_parameters": {
200
+ "table_name": self.table_name,
201
+ "embedding_dim": self.embedding_dim,
202
+ "distance_metric": self.distance_metric,
203
+ "return_embedding": self.return_embedding,
204
+ },
205
+ }
206
+
207
+ @classmethod
208
+ def from_dict(cls, data: Dict[str, Any]) -> "DXMTDocumentStore":
209
+ """
210
+ Deserialize the document store from a dictionary.
211
+
212
+ Note: Requires a DXMT client to be provided separately as it cannot be serialized.
213
+
214
+ Args:
215
+ data: Serialized store configuration
216
+
217
+ Returns:
218
+ DXMTDocumentStore instance
219
+
220
+ Raises:
221
+ ValueError: If client is not provided
222
+ """
223
+ init_params = data.get("init_parameters", {})
224
+ if "client" not in init_params:
225
+ raise ValueError(
226
+ "DXMT client must be provided to deserialize DXMTDocumentStore. "
227
+ "Create a client and pass it as client=DXMTClient(...)"
228
+ )
229
+ return cls(**init_params)
230
+
231
+ def count_documents(self) -> int:
232
+ """
233
+ Count the total number of documents in the store.
234
+
235
+ Returns:
236
+ Number of documents
237
+ """
238
+ try:
239
+ table_info = self.client.get_table(self.table_name)
240
+ return table_info.get("record_count", 0)
241
+ except Exception as e:
242
+ raise DocumentStoreError(f"Failed to count documents: {e}")
243
+
244
+ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
245
+ """
246
+ Retrieve documents matching the provided filters.
247
+
248
+ Haystack filter format is already compatible with DXMT's filter structure:
249
+ ```python
250
+ filters = {
251
+ "operator": "AND",
252
+ "conditions": [
253
+ {"field": "meta.type", "operator": "==", "value": "article"},
254
+ {"field": "meta.rating", "operator": ">=", "value": 3}
255
+ ]
256
+ }
257
+ ```
258
+
259
+ Args:
260
+ filters: Filter dict with operator + conditions or field + operator + value
261
+
262
+ Returns:
263
+ List of Documents matching the filters
264
+ """
265
+ try:
266
+ # DXMT uses the same filter format as Haystack, can pass through directly
267
+ dxmt_filter = self._convert_haystack_filters(filters) if filters else None
268
+
269
+ # Get all matching records
270
+ # Note: DXMT doesn't have a direct filter_all endpoint, so we use search with zero vector
271
+ # This is a workaround - in production you may want to add a scan endpoint to DXMT
272
+ results = self.client.search(
273
+ table_name=self.table_name,
274
+ vector=[0.0] * self.embedding_dim, # Dummy vector
275
+ k=10000, # Large number to get all matches
276
+ filter=dxmt_filter,
277
+ )
278
+
279
+ # Convert DXMT records to Haystack Documents
280
+ documents = []
281
+ for result in results:
282
+ doc = self._dxmt_to_haystack_doc(result)
283
+ if not self.return_embedding:
284
+ doc.embedding = None
285
+ documents.append(doc)
286
+
287
+ return documents
288
+
289
+ except Exception as e:
290
+ raise DocumentStoreError(f"Failed to filter documents: {e}")
291
+
292
+ def write_documents(
293
+ self,
294
+ documents: List[Document],
295
+ policy: DuplicatePolicy = DuplicatePolicy.NONE
296
+ ) -> int:
297
+ """
298
+ Write documents to the store.
299
+
300
+ Args:
301
+ documents: List of Haystack Document objects
302
+ policy: How to handle duplicates (NONE, FAIL, SKIP, OVERWRITE)
303
+
304
+ Returns:
305
+ Number of documents written
306
+
307
+ Raises:
308
+ DuplicateDocumentError: If policy is FAIL and duplicate exists
309
+ DocumentStoreError: If write operation fails
310
+ """
311
+ if not isinstance(documents, list):
312
+ raise ValueError("documents must be a list of Document objects")
313
+
314
+ if not documents:
315
+ return 0
316
+
317
+ if policy == DuplicatePolicy.NONE:
318
+ policy = DuplicatePolicy.FAIL
319
+
320
+ written_count = 0
321
+
322
+ try:
323
+ for doc in documents:
324
+ # Check for duplicates
325
+ if policy != DuplicatePolicy.OVERWRITE:
326
+ existing = self._get_document_by_id(doc.id)
327
+ if existing:
328
+ if policy == DuplicatePolicy.FAIL:
329
+ raise DuplicateDocumentError(f"Document with id '{doc.id}' already exists")
330
+ if policy == DuplicatePolicy.SKIP:
331
+ continue
332
+
333
+ # Convert Haystack Document to DXMT record
334
+ record = self._haystack_to_dxmt_record(doc)
335
+
336
+ # Insert or update
337
+ if policy == DuplicatePolicy.OVERWRITE and self._get_document_by_id(doc.id):
338
+ self.client.update_record(
339
+ table_name=self.table_name,
340
+ record_id=doc.id,
341
+ vector=doc.embedding if doc.embedding else None,
342
+ payload=record["payload"],
343
+ )
344
+ else:
345
+ self.client.insert_records(
346
+ table_name=self.table_name,
347
+ records=[record],
348
+ )
349
+
350
+ written_count += 1
351
+
352
+ except DuplicateDocumentError:
353
+ raise
354
+ except Exception as e:
355
+ raise DocumentStoreError(f"Failed to write documents: {e}")
356
+
357
+ return written_count
358
+
359
+ def delete_documents(self, document_ids: List[str]) -> None:
360
+ """
361
+ Delete documents by their IDs.
362
+
363
+ Args:
364
+ document_ids: List of document IDs to delete
365
+ """
366
+ if not document_ids:
367
+ return
368
+
369
+ try:
370
+ self.client.delete_records(
371
+ table_name=self.table_name,
372
+ ids=document_ids,
373
+ )
374
+ except Exception as e:
375
+ raise DocumentStoreError(f"Failed to delete documents: {e}")
376
+
377
+ def bm25_retrieval(
378
+ self,
379
+ query: str,
380
+ filters: Optional[Dict[str, Any]] = None,
381
+ top_k: int = 10,
382
+ scale_score: bool = False,
383
+ ) -> List[Document]:
384
+ """
385
+ Retrieve documents using BM25 full-text search.
386
+
387
+ Args:
388
+ query: Search query string
389
+ filters: Optional metadata filters
390
+ top_k: Number of top documents to return
391
+ scale_score: Whether to scale scores to [0, 1] range
392
+
393
+ Returns:
394
+ List of Documents ranked by BM25 relevance
395
+ """
396
+ if not query:
397
+ raise ValueError("Query must be a non-empty string")
398
+
399
+ try:
400
+ # Use DXMT's full-text search
401
+ dxmt_filter = self._convert_haystack_filters(filters) if filters else None
402
+
403
+ results = self.client.full_text_search(
404
+ table_name=self.table_name,
405
+ query=query,
406
+ k=top_k,
407
+ filter=dxmt_filter,
408
+ )
409
+
410
+ # Convert to Haystack Documents with scores
411
+ documents = []
412
+ for result in results:
413
+ doc = self._dxmt_to_haystack_doc(result)
414
+ # BM25 score from DXMT
415
+ score = result.get("score", 0.0)
416
+ if scale_score:
417
+ # Scale using expit function like Haystack's InMemoryDocumentStore
418
+ score = self._expit(score / 8.0) # BM25_SCALING_FACTOR = 8
419
+ doc.score = score
420
+
421
+ if not self.return_embedding:
422
+ doc.embedding = None
423
+
424
+ documents.append(doc)
425
+
426
+ return documents
427
+
428
+ except Exception as e:
429
+ raise DocumentStoreError(f"BM25 retrieval failed: {e}")
430
+
431
+ def embedding_retrieval(
432
+ self,
433
+ query_embedding: List[float],
434
+ filters: Optional[Dict[str, Any]] = None,
435
+ top_k: int = 10,
436
+ scale_score: bool = False,
437
+ return_embedding: Optional[bool] = None,
438
+ ) -> List[Document]:
439
+ """
440
+ Retrieve documents by embedding similarity.
441
+
442
+ Args:
443
+ query_embedding: Query embedding vector
444
+ filters: Optional metadata filters
445
+ top_k: Number of top documents to return
446
+ scale_score: Whether to scale scores to [0, 1] range
447
+ return_embedding: Override return_embedding setting
448
+
449
+ Returns:
450
+ List of Documents ranked by vector similarity
451
+ """
452
+ if not query_embedding or not isinstance(query_embedding[0], float):
453
+ raise ValueError("query_embedding must be a non-empty list of floats")
454
+
455
+ if len(query_embedding) != self.embedding_dim:
456
+ raise ValueError(
457
+ f"Query embedding dimension ({len(query_embedding)}) "
458
+ f"does not match store dimension ({self.embedding_dim})"
459
+ )
460
+
461
+ try:
462
+ # Convert Haystack filters to DXMT format
463
+ dxmt_filter = self._convert_haystack_filters(filters) if filters else None
464
+
465
+ # Add filter to exclude documents without real embeddings
466
+ # This matches Haystack's behavior of skipping documents without embeddings
467
+ real_embedding_filter = {"_has_real_embedding": True}
468
+
469
+ if dxmt_filter:
470
+ # Combine user filter with real embedding filter using AND
471
+ combined_filter = {
472
+ "$and": [
473
+ dxmt_filter,
474
+ real_embedding_filter
475
+ ]
476
+ }
477
+ else:
478
+ combined_filter = real_embedding_filter
479
+
480
+ results = self.client.search(
481
+ table_name=self.table_name,
482
+ vector=query_embedding,
483
+ k=top_k,
484
+ filter=combined_filter,
485
+ )
486
+
487
+ # Determine whether to return embeddings
488
+ resolved_return_embedding = (
489
+ self.return_embedding if return_embedding is None else return_embedding
490
+ )
491
+
492
+ # Convert to Haystack Documents
493
+ documents = []
494
+ for result in results:
495
+ doc = self._dxmt_to_haystack_doc(result)
496
+ # Convert distance to similarity score
497
+ distance = result.get("score", 0.0)
498
+ score = self._normalize_score(distance, scale_score)
499
+ doc.score = score
500
+
501
+ if not resolved_return_embedding:
502
+ doc.embedding = None
503
+
504
+ documents.append(doc)
505
+
506
+ return documents
507
+
508
+ except Exception as e:
509
+ raise DocumentStoreError(f"Embedding retrieval failed: {e}")
510
+
511
+ # Async methods
512
+ async def count_documents_async(self) -> int:
513
+ """Async version of count_documents."""
514
+ return await asyncio.get_event_loop().run_in_executor(
515
+ self.executor, self.count_documents
516
+ )
517
+
518
+ async def filter_documents_async(
519
+ self, filters: Optional[Dict[str, Any]] = None
520
+ ) -> List[Document]:
521
+ """Async version of filter_documents."""
522
+ return await asyncio.get_event_loop().run_in_executor(
523
+ self.executor, lambda: self.filter_documents(filters=filters)
524
+ )
525
+
526
+ async def write_documents_async(
527
+ self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
528
+ ) -> int:
529
+ """Async version of write_documents."""
530
+ return await asyncio.get_event_loop().run_in_executor(
531
+ self.executor, lambda: self.write_documents(documents=documents, policy=policy)
532
+ )
533
+
534
+ async def delete_documents_async(self, document_ids: List[str]) -> None:
535
+ """Async version of delete_documents."""
536
+ await asyncio.get_event_loop().run_in_executor(
537
+ self.executor, lambda: self.delete_documents(document_ids=document_ids)
538
+ )
539
+
540
+ async def bm25_retrieval_async(
541
+ self,
542
+ query: str,
543
+ filters: Optional[Dict[str, Any]] = None,
544
+ top_k: int = 10,
545
+ scale_score: bool = False,
546
+ ) -> List[Document]:
547
+ """Async version of bm25_retrieval."""
548
+ return await asyncio.get_event_loop().run_in_executor(
549
+ self.executor,
550
+ lambda: self.bm25_retrieval(
551
+ query=query, filters=filters, top_k=top_k, scale_score=scale_score
552
+ ),
553
+ )
554
+
555
+ async def embedding_retrieval_async(
556
+ self,
557
+ query_embedding: List[float],
558
+ filters: Optional[Dict[str, Any]] = None,
559
+ top_k: int = 10,
560
+ scale_score: bool = False,
561
+ return_embedding: bool = False,
562
+ ) -> List[Document]:
563
+ """Async version of embedding_retrieval."""
564
+ return await asyncio.get_event_loop().run_in_executor(
565
+ self.executor,
566
+ lambda: self.embedding_retrieval(
567
+ query_embedding=query_embedding,
568
+ filters=filters,
569
+ top_k=top_k,
570
+ scale_score=scale_score,
571
+ return_embedding=return_embedding,
572
+ ),
573
+ )
574
+
575
+ # Helper methods
576
+ def _convert_haystack_filters(self, filters: Dict[str, Any]) -> Optional[Dict[str, Any]]:
577
+ """
578
+ Convert Haystack filters to DXMT MongoDB-style format.
579
+
580
+ Haystack format:
581
+ {"field": "meta.type", "operator": "==", "value": "programming"}
582
+
583
+ DXMT MongoDB format:
584
+ {"type": "programming"}
585
+ or {"type": {"$eq": "programming"}}
586
+ """
587
+ if not filters:
588
+ return None
589
+
590
+ # Operator mapping from Haystack to MongoDB
591
+ OPERATOR_MAP = {
592
+ "==": "$eq",
593
+ "!=": "$ne",
594
+ ">": "$gt",
595
+ ">=": "$gte",
596
+ "<": "$lt",
597
+ "<=": "$lte",
598
+ "in": "$in",
599
+ "not in": "$nin",
600
+ }
601
+
602
+ def convert_condition(condition: Dict[str, Any]) -> Dict[str, Any]:
603
+ if "field" in condition:
604
+ # Single comparison condition
605
+ field = condition["field"]
606
+ # Strip "meta." prefix if present
607
+ if field.startswith("meta."):
608
+ field = field[5:]
609
+
610
+ operator = condition["operator"]
611
+ value = condition["value"]
612
+
613
+ # For equality, use simple form
614
+ if operator == "==":
615
+ return {field: value}
616
+ else:
617
+ # For other operators, use MongoDB operator form
618
+ mongo_op = OPERATOR_MAP.get(operator, operator)
619
+ return {field: {mongo_op: value}}
620
+
621
+ elif "operator" in condition and "conditions" in condition:
622
+ # Logical operator (AND, OR, NOT)
623
+ logical_op = condition["operator"].upper()
624
+ sub_conditions = condition["conditions"]
625
+
626
+ if logical_op == "AND":
627
+ converted = [convert_condition(c) for c in sub_conditions]
628
+ return {"$and": converted}
629
+ elif logical_op == "OR":
630
+ converted = [convert_condition(c) for c in sub_conditions]
631
+ return {"$or": converted}
632
+ elif logical_op == "NOT":
633
+ # NOT typically has one condition
634
+ converted = [convert_condition(c) for c in sub_conditions]
635
+ return {"$not": converted}
636
+ else:
637
+ # Fallback: treat as AND
638
+ converted = [convert_condition(c) for c in sub_conditions]
639
+ return {"$and": converted}
640
+ else:
641
+ # Unknown format, return as-is
642
+ return condition
643
+
644
+ return convert_condition(filters)
645
+
646
+ def _haystack_to_dxmt_record(self, doc: Document) -> Dict[str, Any]:
647
+ """Convert Haystack Document to DXMT record format."""
648
+ # Generate ID if not present
649
+ doc_id = doc.id if doc.id else str(uuid.uuid4())
650
+
651
+ # Prepare payload (metadata + content)
652
+ payload = dict(doc.meta) if doc.meta else {}
653
+ payload["content"] = doc.content
654
+
655
+ # Optional fields - only add if present
656
+ if hasattr(doc, 'dataframe') and doc.dataframe is not None:
657
+ payload["dataframe"] = doc.dataframe
658
+ if hasattr(doc, 'blob') and doc.blob is not None:
659
+ payload["blob"] = doc.blob
660
+
661
+ # Add content hash for deduplication
662
+ if doc.content:
663
+ content_hash = hashlib.md5(doc.content.encode()).hexdigest()
664
+ payload["content_hash"] = content_hash
665
+
666
+ # Prepare record
667
+ record = {
668
+ "id": doc_id,
669
+ "payload": payload,
670
+ }
671
+
672
+ # Add embedding if present, otherwise use dummy vector
673
+ if doc.embedding:
674
+ if len(doc.embedding) != self.embedding_dim:
675
+ raise ValueError(
676
+ f"Document embedding dimension ({len(doc.embedding)}) "
677
+ f"does not match store dimension ({self.embedding_dim})"
678
+ )
679
+ record["vector"] = doc.embedding
680
+ # Mark that this document has a real embedding (not a dummy)
681
+ payload["_has_real_embedding"] = True
682
+ else:
683
+ # DXMT requires a non-zero vector field
684
+ # Use a small constant vector for BM25-only documents
685
+ # Mark this as a dummy vector so we can filter it out during embedding retrieval
686
+ record["vector"] = [0.001] * self.embedding_dim
687
+ payload["_has_real_embedding"] = False
688
+
689
+ return record
690
+
691
+ def _dxmt_to_haystack_doc(self, result: Dict[str, Any]) -> Document:
692
+ """Convert DXMT search result to Haystack Document."""
693
+ metadata = result.get("metadata", {})
694
+
695
+ # Extract core fields
696
+ content = metadata.pop("content", None)
697
+ dataframe = metadata.pop("dataframe", None)
698
+ blob = metadata.pop("blob", None)
699
+ content_hash = metadata.pop("content_hash", None)
700
+ # Remove internal field (not part of user metadata)
701
+ metadata.pop("_has_real_embedding", None)
702
+
703
+ # Get embedding if present
704
+ embedding = result.get("vector", None)
705
+
706
+ # Build Document with only supported fields
707
+ doc_kwargs = {
708
+ "id": result.get("id"),
709
+ "content": content,
710
+ "meta": metadata,
711
+ "embedding": embedding,
712
+ }
713
+
714
+ # Only add optional fields if they exist and Document supports them
715
+ if dataframe is not None:
716
+ try:
717
+ doc_kwargs["dataframe"] = dataframe
718
+ except:
719
+ pass # Skip if Document doesn't support dataframe
720
+
721
+ if blob is not None:
722
+ try:
723
+ doc_kwargs["blob"] = blob
724
+ except:
725
+ pass # Skip if Document doesn't support blob
726
+
727
+ doc = Document(**doc_kwargs)
728
+
729
+ return doc
730
+
731
+ def _get_document_by_id(self, doc_id: str) -> Optional[Dict[str, Any]]:
732
+ """Get a document by ID, return None if not found."""
733
+ try:
734
+ results = self.client.get_records(
735
+ table_name=self.table_name,
736
+ ids=[doc_id],
737
+ )
738
+ return results[0] if results else None
739
+ except:
740
+ return None
741
+
742
+ def _normalize_score(self, distance: float, scale: bool) -> float:
743
+ """
744
+ Convert distance to similarity score.
745
+
746
+ Args:
747
+ distance: Distance value from DXMT
748
+ scale: Whether to scale to [0, 1] range
749
+
750
+ Returns:
751
+ Similarity score
752
+ """
753
+ if not scale:
754
+ return distance
755
+
756
+ # Convert distance to similarity based on metric
757
+ if self.distance_metric == "cosine":
758
+ # Cosine distance is in [0, 2], convert to similarity [0, 1]
759
+ # 0 distance = 1 similarity, 2 distance = 0 similarity
760
+ return (distance + 1) / 2
761
+ elif self.distance_metric == "dot":
762
+ # Dot product can be negative, scale using expit
763
+ return self._expit(distance / 100.0)
764
+ else:
765
+ # Euclidean/other: use exponential decay
766
+ import math
767
+ return math.exp(-distance)
768
+
769
+ @staticmethod
770
+ def _expit(x: float) -> float:
771
+ """Expit function (inverse of logit), maps R to [0, 1]."""
772
+ import math
773
+ return 1 / (1 + math.exp(-x))