dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
"""DXMT VectorStore for LlamaIndex.
|
|
2
|
+
|
|
3
|
+
This module provides a LlamaIndex VectorStore implementation for DXMT VectorDB,
|
|
4
|
+
enabling integration with LlamaIndex-based frameworks like MetaGPT.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
```python
|
|
8
|
+
from llama_index.core import VectorStoreIndex, StorageContext
|
|
9
|
+
from dxmt import DXMTClient
|
|
10
|
+
from dxmt.metagpt_vector_store import DXMTVectorStore
|
|
11
|
+
|
|
12
|
+
# Create DXMT vector store
|
|
13
|
+
client = DXMTClient(host="localhost", port=9999)
|
|
14
|
+
vector_store = DXMTVectorStore(
|
|
15
|
+
client=client,
|
|
16
|
+
table_name="documents",
|
|
17
|
+
dimension=1536
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# Create storage context and index
|
|
21
|
+
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
|
22
|
+
index = VectorStoreIndex.from_documents(
|
|
23
|
+
documents,
|
|
24
|
+
storage_context=storage_context
|
|
25
|
+
)
|
|
26
|
+
```
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import logging
|
|
32
|
+
import uuid
|
|
33
|
+
from typing import Any, List, Optional
|
|
34
|
+
|
|
35
|
+
from llama_index.core.schema import BaseNode, TextNode
|
|
36
|
+
from llama_index.core.vector_stores.types import (
|
|
37
|
+
BasePydanticVectorStore,
|
|
38
|
+
VectorStoreQuery,
|
|
39
|
+
VectorStoreQueryResult,
|
|
40
|
+
)
|
|
41
|
+
from pydantic import ConfigDict, PrivateAttr
|
|
42
|
+
|
|
43
|
+
from llama_index.core.vector_stores.utils import (
|
|
44
|
+
metadata_dict_to_node,
|
|
45
|
+
node_to_metadata_dict,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
from .client import DXMTClient
|
|
49
|
+
from .exceptions import DXMTError
|
|
50
|
+
from .filters import convert_simple_filter_to_dxmt
|
|
51
|
+
|
|
52
|
+
logger = logging.getLogger(__name__)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DXMTVectorStore(BasePydanticVectorStore):
|
|
56
|
+
"""DXMT Vector Store for LlamaIndex.
|
|
57
|
+
|
|
58
|
+
Implements LlamaIndex's BasePydanticVectorStore interface to enable
|
|
59
|
+
DXMT VectorDB integration with LlamaIndex and MetaGPT.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
client: DXMTClient instance. If None, creates with default settings.
|
|
63
|
+
table_name: Name of the DXMT table to use.
|
|
64
|
+
dimension: Vector dimension. Required if table doesn't exist.
|
|
65
|
+
distance_metric: Distance metric (cosine, euclidean, dot, manhattan).
|
|
66
|
+
create_if_not_exists: Create table if it doesn't exist.
|
|
67
|
+
host: DXMT server host (used if client is None).
|
|
68
|
+
port: DXMT server port (used if client is None).
|
|
69
|
+
|
|
70
|
+
Attributes:
|
|
71
|
+
stores_text: Always True - DXMT stores text in metadata.
|
|
72
|
+
is_embedding_query: Always True - queries use embeddings.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
stores_text: bool = True
|
|
76
|
+
is_embedding_query: bool = True
|
|
77
|
+
|
|
78
|
+
# Pydantic fields. `client` is not one of them: `BasePydanticVectorStore` declares it as an
|
|
79
|
+
# abstract property, and a field of the same name shadows it -- pydantic said so in a
|
|
80
|
+
# warning printed on every `import dxmt` once llama-index was installed. It is a private
|
|
81
|
+
# attribute with the property the interface asks for, which is what the standalone
|
|
82
|
+
# `llama-index-vector-stores-dxmt` package has always done.
|
|
83
|
+
_client: Optional[DXMTClient] = PrivateAttr(default=None)
|
|
84
|
+
table_name: str = "documents"
|
|
85
|
+
dimension: int = 1536
|
|
86
|
+
distance_metric: str = "cosine"
|
|
87
|
+
create_if_not_exists: bool = True
|
|
88
|
+
host: str = "localhost"
|
|
89
|
+
port: int = 9999
|
|
90
|
+
|
|
91
|
+
_initialized: bool = PrivateAttr(default=False)
|
|
92
|
+
|
|
93
|
+
# A class-based `Config` is the pydantic v1 spelling, deprecated since v2 and removed in
|
|
94
|
+
# v3; it warned on every import of this module.
|
|
95
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
96
|
+
|
|
97
|
+
def __init__(
|
|
98
|
+
self,
|
|
99
|
+
client: Optional[DXMTClient] = None,
|
|
100
|
+
table_name: str = "documents",
|
|
101
|
+
dimension: int = 1536,
|
|
102
|
+
distance_metric: str = "cosine",
|
|
103
|
+
create_if_not_exists: bool = True,
|
|
104
|
+
host: str = "localhost",
|
|
105
|
+
port: int = 9999,
|
|
106
|
+
api_key: Optional[str] = None,
|
|
107
|
+
**kwargs: Any,
|
|
108
|
+
) -> None:
|
|
109
|
+
"""Initialize DXMT vector store.
|
|
110
|
+
|
|
111
|
+
``api_key`` goes to the client and is not kept as a field: the other arguments are
|
|
112
|
+
pydantic fields and come back out of ``repr()`` and serialisation.
|
|
113
|
+
"""
|
|
114
|
+
super().__init__(
|
|
115
|
+
table_name=table_name,
|
|
116
|
+
dimension=dimension,
|
|
117
|
+
distance_metric=distance_metric,
|
|
118
|
+
create_if_not_exists=create_if_not_exists,
|
|
119
|
+
host=host,
|
|
120
|
+
port=port,
|
|
121
|
+
**kwargs,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# Create client if not provided
|
|
125
|
+
self._client = client or DXMTClient(host=self.host, port=self.port, api_key=api_key)
|
|
126
|
+
|
|
127
|
+
# Initialize table
|
|
128
|
+
if create_if_not_exists and not self._initialized:
|
|
129
|
+
self._ensure_table_exists()
|
|
130
|
+
self._initialized = True
|
|
131
|
+
|
|
132
|
+
def _ensure_table_exists(self) -> None:
|
|
133
|
+
"""Ensure the DXMT table exists."""
|
|
134
|
+
try:
|
|
135
|
+
# Try to get table
|
|
136
|
+
self.client.get_table(self.table_name)
|
|
137
|
+
logger.info(f"Table {self.table_name} already exists")
|
|
138
|
+
except DXMTError:
|
|
139
|
+
# Table doesn't exist, create it
|
|
140
|
+
logger.info(f"Creating table {self.table_name}")
|
|
141
|
+
self.client.create_table(
|
|
142
|
+
name=self.table_name,
|
|
143
|
+
dimension=self.dimension,
|
|
144
|
+
distance_metric=self.distance_metric,
|
|
145
|
+
description=f"DXMT vector store for LlamaIndex (dimension={self.dimension})"
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def client(self) -> DXMTClient:
|
|
150
|
+
"""The underlying DXMT client, as `BasePydanticVectorStore` requires."""
|
|
151
|
+
assert self._client is not None # set in __init__, which is the only way in
|
|
152
|
+
return self._client
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def client_instance(self) -> DXMTClient:
|
|
156
|
+
"""The client, under the name this class used before `client` became a property."""
|
|
157
|
+
return self.client
|
|
158
|
+
|
|
159
|
+
def add(
|
|
160
|
+
self,
|
|
161
|
+
nodes: List[BaseNode],
|
|
162
|
+
**kwargs: Any,
|
|
163
|
+
) -> List[str]:
|
|
164
|
+
"""Add nodes to DXMT.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
nodes: List of nodes to add.
|
|
168
|
+
**kwargs: Additional arguments.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
List of node IDs.
|
|
172
|
+
"""
|
|
173
|
+
if not nodes:
|
|
174
|
+
return []
|
|
175
|
+
|
|
176
|
+
# Convert nodes to DXMT records
|
|
177
|
+
records = []
|
|
178
|
+
for node in nodes:
|
|
179
|
+
# Get embedding
|
|
180
|
+
embedding = node.get_embedding()
|
|
181
|
+
if embedding is None:
|
|
182
|
+
logger.warning(f"Node {node.node_id} has no embedding, skipping")
|
|
183
|
+
continue
|
|
184
|
+
|
|
185
|
+
# Convert node to metadata dict
|
|
186
|
+
metadata = node_to_metadata_dict(node, remove_text=False, flat_metadata=False)
|
|
187
|
+
|
|
188
|
+
# Add text content explicitly
|
|
189
|
+
if hasattr(node, 'text') and node.text:
|
|
190
|
+
metadata['text'] = node.text
|
|
191
|
+
|
|
192
|
+
# Create record
|
|
193
|
+
record = {
|
|
194
|
+
"id": node.node_id,
|
|
195
|
+
"vector": embedding,
|
|
196
|
+
"metadata": metadata
|
|
197
|
+
}
|
|
198
|
+
records.append(record)
|
|
199
|
+
|
|
200
|
+
if not records:
|
|
201
|
+
return []
|
|
202
|
+
|
|
203
|
+
# Insert into DXMT
|
|
204
|
+
try:
|
|
205
|
+
self.client.insert_records(
|
|
206
|
+
table_name=self.table_name,
|
|
207
|
+
records=records
|
|
208
|
+
)
|
|
209
|
+
logger.info(f"Added {len(records)} nodes to {self.table_name}")
|
|
210
|
+
return [r["id"] for r in records]
|
|
211
|
+
except Exception as e:
|
|
212
|
+
logger.error(f"Error adding nodes: {e}")
|
|
213
|
+
raise DXMTError(f"Failed to add nodes: {e}")
|
|
214
|
+
|
|
215
|
+
def delete(self, ref_doc_id: str, **kwargs: Any) -> None:
|
|
216
|
+
"""Delete nodes by reference document ID.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
ref_doc_id: Reference document ID.
|
|
220
|
+
**kwargs: Additional arguments.
|
|
221
|
+
"""
|
|
222
|
+
try:
|
|
223
|
+
# Search for all nodes with this ref_doc_id using a filter-only scan.
|
|
224
|
+
# A dummy zero vector is used because DXMT has no filter-scan endpoint;
|
|
225
|
+
# results are restricted by the ref_doc_id filter, not by proximity.
|
|
226
|
+
results = self.client.search(
|
|
227
|
+
table_name=self.table_name,
|
|
228
|
+
vector=[0.0] * self.dimension,
|
|
229
|
+
k=10000,
|
|
230
|
+
filter={
|
|
231
|
+
"must": [
|
|
232
|
+
{
|
|
233
|
+
"key": "ref_doc_id",
|
|
234
|
+
"operation": {"type": "Equal", "value": ref_doc_id},
|
|
235
|
+
}
|
|
236
|
+
]
|
|
237
|
+
},
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
node_ids = [r.get("id") for r in results if r.get("id")]
|
|
241
|
+
if node_ids:
|
|
242
|
+
self.client.delete_records(
|
|
243
|
+
table_name=self.table_name,
|
|
244
|
+
ids=node_ids,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
logger.info(f"Deleted {len(node_ids)} nodes with ref_doc_id={ref_doc_id}")
|
|
248
|
+
except Exception as e:
|
|
249
|
+
logger.error(f"Error deleting nodes: {e}")
|
|
250
|
+
raise DXMTError(f"Failed to delete nodes: {e}")
|
|
251
|
+
|
|
252
|
+
def query(
|
|
253
|
+
self,
|
|
254
|
+
query: VectorStoreQuery,
|
|
255
|
+
**kwargs: Any,
|
|
256
|
+
) -> VectorStoreQueryResult:
|
|
257
|
+
"""Query DXMT vector store.
|
|
258
|
+
|
|
259
|
+
Respects VectorStoreQuery.mode:
|
|
260
|
+
- DEFAULT / None → dense vector search
|
|
261
|
+
- HYBRID → hybrid_search (vector + text)
|
|
262
|
+
- SEMANTIC_HYBRID → the same; `alpha`, when the query carries one, is the
|
|
263
|
+
vector side's share and selects weighted fusion
|
|
264
|
+
- SPARSE / TEXT_SEARCH → full_text_search (BM25 only)
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
query: Vector store query.
|
|
268
|
+
**kwargs: Additional arguments.
|
|
269
|
+
|
|
270
|
+
Returns:
|
|
271
|
+
Query results with nodes and similarities.
|
|
272
|
+
"""
|
|
273
|
+
try:
|
|
274
|
+
from llama_index.core.vector_stores.types import VectorStoreQueryMode
|
|
275
|
+
_MODE = VectorStoreQueryMode
|
|
276
|
+
except ImportError:
|
|
277
|
+
_MODE = None
|
|
278
|
+
|
|
279
|
+
dxmt_filter = self._convert_filters(query.filters) if query.filters else None
|
|
280
|
+
top_k = query.similarity_top_k or 5
|
|
281
|
+
mode = getattr(query, "mode", None)
|
|
282
|
+
|
|
283
|
+
try:
|
|
284
|
+
# --- HYBRID modes ---
|
|
285
|
+
if _MODE and mode in (_MODE.HYBRID, _MODE.SEMANTIC_HYBRID):
|
|
286
|
+
# Pass the query's alpha through rather than inventing one: a weight selects
|
|
287
|
+
# weighted fusion, and a caller who set none should get the server's default
|
|
288
|
+
# (`dbsf`), which needs no weight.
|
|
289
|
+
results = self.client.hybrid_search(
|
|
290
|
+
table_name=self.table_name,
|
|
291
|
+
vector=query.query_embedding,
|
|
292
|
+
text=getattr(query, "query_str", None),
|
|
293
|
+
alpha=getattr(query, "alpha", None),
|
|
294
|
+
k=top_k,
|
|
295
|
+
filter=dxmt_filter,
|
|
296
|
+
)
|
|
297
|
+
# --- SPARSE / TEXT_SEARCH modes ---
|
|
298
|
+
elif _MODE and mode in (_MODE.SPARSE, _MODE.TEXT_SEARCH):
|
|
299
|
+
query_str = getattr(query, "query_str", None)
|
|
300
|
+
if not query_str:
|
|
301
|
+
raise ValueError("query_str is required for SPARSE/TEXT_SEARCH mode")
|
|
302
|
+
results = self.client.full_text_search(
|
|
303
|
+
table_name=self.table_name,
|
|
304
|
+
query=query_str,
|
|
305
|
+
k=top_k,
|
|
306
|
+
filter=dxmt_filter,
|
|
307
|
+
)
|
|
308
|
+
# --- DEFAULT: dense vector search ---
|
|
309
|
+
else:
|
|
310
|
+
if query.query_embedding is None:
|
|
311
|
+
raise ValueError("query_embedding is required for DEFAULT mode")
|
|
312
|
+
results = self.client.search(
|
|
313
|
+
table_name=self.table_name,
|
|
314
|
+
vector=query.query_embedding,
|
|
315
|
+
k=top_k,
|
|
316
|
+
filter=dxmt_filter,
|
|
317
|
+
)
|
|
318
|
+
except (ValueError, AttributeError):
|
|
319
|
+
raise
|
|
320
|
+
except Exception as e:
|
|
321
|
+
logger.error(f"Error querying DXMT: {e}")
|
|
322
|
+
raise DXMTError(f"Failed to query: {e}")
|
|
323
|
+
|
|
324
|
+
# Convert results to LlamaIndex format
|
|
325
|
+
nodes = []
|
|
326
|
+
similarities = []
|
|
327
|
+
ids = []
|
|
328
|
+
|
|
329
|
+
for result in results:
|
|
330
|
+
# Extract metadata
|
|
331
|
+
metadata = result.get("metadata", {})
|
|
332
|
+
|
|
333
|
+
# Reconstruct node from metadata
|
|
334
|
+
try:
|
|
335
|
+
node = metadata_dict_to_node(metadata)
|
|
336
|
+
node.id_ = result.get("id")
|
|
337
|
+
except Exception:
|
|
338
|
+
# Fallback: create TextNode manually
|
|
339
|
+
text = metadata.get("text", "")
|
|
340
|
+
node = TextNode(
|
|
341
|
+
text=text,
|
|
342
|
+
id_=result.get("id"),
|
|
343
|
+
metadata=metadata,
|
|
344
|
+
embedding=result.get("vector")
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
nodes.append(node)
|
|
348
|
+
|
|
349
|
+
# Get similarity score
|
|
350
|
+
score = result.get("score", 0.0)
|
|
351
|
+
similarities.append(score)
|
|
352
|
+
|
|
353
|
+
# Get ID
|
|
354
|
+
ids.append(result.get("id"))
|
|
355
|
+
|
|
356
|
+
return VectorStoreQueryResult(
|
|
357
|
+
nodes=nodes,
|
|
358
|
+
similarities=similarities,
|
|
359
|
+
ids=ids,
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
def persist(
|
|
363
|
+
self,
|
|
364
|
+
persist_path: str,
|
|
365
|
+
fs: Optional[Any] = None,
|
|
366
|
+
) -> None:
|
|
367
|
+
"""Persist the vector store.
|
|
368
|
+
|
|
369
|
+
DXMT automatically persists data, so this is a no-op.
|
|
370
|
+
|
|
371
|
+
Args:
|
|
372
|
+
persist_path: Path to persist to (unused).
|
|
373
|
+
fs: Filesystem (unused).
|
|
374
|
+
"""
|
|
375
|
+
logger.info("DXMT automatically persists data")
|
|
376
|
+
|
|
377
|
+
def _convert_filters(self, filters: Any) -> Optional[dict]:
|
|
378
|
+
"""Convert LlamaIndex MetadataFilters (or plain dict) to DXMT format.
|
|
379
|
+
|
|
380
|
+
Handles:
|
|
381
|
+
- LlamaIndex MetadataFilters object (with .filters list of
|
|
382
|
+
MetadataFilter, each having .key / .value / .operator)
|
|
383
|
+
- Plain dict (passed through convert_simple_filter_to_dxmt which
|
|
384
|
+
supports MongoDB-style operators: $eq, $ne, $gt, $gte, $lt,
|
|
385
|
+
$lte, $in, $nin, $contains, $startswith, $endswith, $and, $or)
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
filters: LlamaIndex filters or plain dict.
|
|
389
|
+
|
|
390
|
+
Returns:
|
|
391
|
+
DXMT filter dict, or None.
|
|
392
|
+
"""
|
|
393
|
+
if filters is None:
|
|
394
|
+
return None
|
|
395
|
+
|
|
396
|
+
# LlamaIndex MetadataFilters object
|
|
397
|
+
if hasattr(filters, "filters") and hasattr(filters, "condition"):
|
|
398
|
+
from llama_index.core.vector_stores.types import FilterOperator, FilterCondition
|
|
399
|
+
_OP_MAP = {
|
|
400
|
+
FilterOperator.EQ: "Equal",
|
|
401
|
+
FilterOperator.NE: "NotEqual",
|
|
402
|
+
FilterOperator.GT: "GreaterThan",
|
|
403
|
+
FilterOperator.GTE: "GreaterThanOrEqual",
|
|
404
|
+
FilterOperator.LT: "LessThan",
|
|
405
|
+
FilterOperator.LTE: "LessThanOrEqual",
|
|
406
|
+
FilterOperator.IN: "In",
|
|
407
|
+
FilterOperator.NIN: "NotIn",
|
|
408
|
+
FilterOperator.CONTAINS: "Contains",
|
|
409
|
+
}
|
|
410
|
+
conditions = []
|
|
411
|
+
for f in filters.filters:
|
|
412
|
+
op_type = _OP_MAP.get(f.operator, "Equal")
|
|
413
|
+
conditions.append({
|
|
414
|
+
"key": f.key,
|
|
415
|
+
"operation": {"type": op_type, "value": f.value},
|
|
416
|
+
})
|
|
417
|
+
if not conditions:
|
|
418
|
+
return None
|
|
419
|
+
cond_key = "must"
|
|
420
|
+
if hasattr(FilterCondition, "OR") and filters.condition == FilterCondition.OR:
|
|
421
|
+
cond_key = "should"
|
|
422
|
+
return {cond_key: conditions}
|
|
423
|
+
|
|
424
|
+
# Plain dict — delegate to shared filter utility
|
|
425
|
+
if isinstance(filters, dict):
|
|
426
|
+
return convert_simple_filter_to_dxmt(filters)
|
|
427
|
+
|
|
428
|
+
return None
|
|
429
|
+
|
|
430
|
+
def clear(self) -> None:
|
|
431
|
+
"""Clear all nodes from the table.
|
|
432
|
+
|
|
433
|
+
Deletes and recreates the table.
|
|
434
|
+
"""
|
|
435
|
+
try:
|
|
436
|
+
self.client.delete_table(self.table_name)
|
|
437
|
+
logger.info(f"Deleted table {self.table_name}")
|
|
438
|
+
|
|
439
|
+
# Recreate table
|
|
440
|
+
self._ensure_table_exists()
|
|
441
|
+
logger.info(f"Recreated table {self.table_name}")
|
|
442
|
+
except Exception as e:
|
|
443
|
+
logger.error(f"Error clearing vector store: {e}")
|
|
444
|
+
raise DXMTError(f"Failed to clear: {e}")
|
|
445
|
+
|
|
446
|
+
async def async_add(self, nodes: List[BaseNode], **kwargs: Any) -> List[str]:
|
|
447
|
+
"""Async add nodes."""
|
|
448
|
+
import asyncio
|
|
449
|
+
loop = asyncio.get_event_loop()
|
|
450
|
+
return await loop.run_in_executor(None, lambda: self.add(nodes, **kwargs))
|
|
451
|
+
|
|
452
|
+
async def aquery(
|
|
453
|
+
self,
|
|
454
|
+
query: VectorStoreQuery,
|
|
455
|
+
**kwargs: Any,
|
|
456
|
+
) -> VectorStoreQueryResult:
|
|
457
|
+
"""Async query."""
|
|
458
|
+
import asyncio
|
|
459
|
+
loop = asyncio.get_event_loop()
|
|
460
|
+
return await loop.run_in_executor(None, lambda: self.query(query, **kwargs))
|
|
461
|
+
|
|
462
|
+
async def adelete(self, ref_doc_id: str, **kwargs: Any) -> None:
|
|
463
|
+
"""Async delete by ref_doc_id."""
|
|
464
|
+
import asyncio
|
|
465
|
+
loop = asyncio.get_event_loop()
|
|
466
|
+
await loop.run_in_executor(None, lambda: self.delete(ref_doc_id, **kwargs))
|
dxmt/migrate/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Moving an existing store into DXMT. `python -m dxmt.migrate.chroma --help`."""
|
dxmt/migrate/chroma.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Copy Chroma collections into DXMT tables, vectors included, so nothing is re-embedded.
|
|
2
|
+
|
|
3
|
+
pip install 'dxmt-python-sdk[chroma]'
|
|
4
|
+
python -m dxmt.migrate.chroma --path ./chroma_db # every collection
|
|
5
|
+
python -m dxmt.migrate.chroma --path ./chroma_db --collection notes --table notes
|
|
6
|
+
|
|
7
|
+
Each collection becomes one table: the same ids, the stored embeddings, the collection's
|
|
8
|
+
distance (`l2` -> `euclidean`, `ip` -> `dot`, `cosine`), each metadata dict as the payload and
|
|
9
|
+
the document under `text`, with a full-text index on it so text and hybrid search work on the
|
|
10
|
+
copy. A record is written by id, so running it again over the same table rewrites the same
|
|
11
|
+
records rather than duplicating them -- an interrupted run is finished by running it again.
|
|
12
|
+
|
|
13
|
+
Chroma's embedding function does not come along: the copy is searched with vectors (or with
|
|
14
|
+
text through the full-text index). To search it with text through a model, create the table
|
|
15
|
+
with `embedding` naming the same model the collection used and migrate into it with `--table`.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import sys
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from typing import Any, Callable, Iterable, Optional
|
|
26
|
+
|
|
27
|
+
from ..client import DXMTClient
|
|
28
|
+
|
|
29
|
+
_SPACES = {"l2": "euclidean", "cosine": "cosine", "ip": "dot"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Migrated:
|
|
34
|
+
collection: str
|
|
35
|
+
table: str
|
|
36
|
+
records: int
|
|
37
|
+
skipped: Optional[str] = None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def table_name_for(collection: str) -> str:
|
|
41
|
+
"""A collection name as a table name: DXMT allows letters, digits, `_` and `-`."""
|
|
42
|
+
name = re.sub(r"[^A-Za-z0-9_-]", "_", collection)
|
|
43
|
+
if len(name) > 63:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"collection {collection!r} makes a table name over 63 characters; "
|
|
46
|
+
"name the table with --table"
|
|
47
|
+
)
|
|
48
|
+
return name
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def distance_of(collection: Any) -> str:
|
|
52
|
+
"""The collection's distance as a DXMT metric. Chroma 1.x keeps it in `configuration`,
|
|
53
|
+
older versions in the `hnsw:space` metadata key; the default is `l2`."""
|
|
54
|
+
space = None
|
|
55
|
+
config = getattr(collection, "configuration", None)
|
|
56
|
+
if isinstance(config, dict):
|
|
57
|
+
space = (config.get("hnsw") or {}).get("space") or (config.get("spann") or {}).get(
|
|
58
|
+
"space"
|
|
59
|
+
)
|
|
60
|
+
if space is None:
|
|
61
|
+
space = (collection.metadata or {}).get("hnsw:space", "l2")
|
|
62
|
+
if space not in _SPACES:
|
|
63
|
+
raise ValueError(f"collection {collection.name!r} uses distance {space!r}, which DXMT lacks")
|
|
64
|
+
return _SPACES[space]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _vector(embedding: Any) -> list[float]:
|
|
68
|
+
return embedding.tolist() if hasattr(embedding, "tolist") else list(embedding)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _existing_dimension(client: DXMTClient, table: str) -> Optional[int]:
|
|
72
|
+
for t in client.list_tables():
|
|
73
|
+
if t.get("name") == table:
|
|
74
|
+
return int(t.get("dimension") or 0)
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def migrate_collection(
|
|
79
|
+
collection: Any,
|
|
80
|
+
client: DXMTClient,
|
|
81
|
+
table: Optional[str] = None,
|
|
82
|
+
*,
|
|
83
|
+
text_field: str = "text",
|
|
84
|
+
analyzer: str = "standard",
|
|
85
|
+
batch_size: int = 500,
|
|
86
|
+
progress: Optional[Callable[[str, int, int], None]] = None,
|
|
87
|
+
) -> Migrated:
|
|
88
|
+
"""Copy one Chroma collection into a DXMT table, creating the table if it does not exist."""
|
|
89
|
+
table = table or table_name_for(collection.name)
|
|
90
|
+
total = collection.count()
|
|
91
|
+
if total == 0:
|
|
92
|
+
return Migrated(collection.name, table, 0, skipped="empty: no vectors to size a table by")
|
|
93
|
+
|
|
94
|
+
first = collection.get(limit=1, include=["embeddings"])
|
|
95
|
+
dimension = len(_vector(first["embeddings"][0]))
|
|
96
|
+
existing = _existing_dimension(client, table)
|
|
97
|
+
if existing is None:
|
|
98
|
+
client.create_table(
|
|
99
|
+
name=table,
|
|
100
|
+
dimension=dimension,
|
|
101
|
+
distance_metric=distance_of(collection),
|
|
102
|
+
enable_full_text_search=True,
|
|
103
|
+
text_fields=[
|
|
104
|
+
{
|
|
105
|
+
"name": "content",
|
|
106
|
+
"source": {"metadata_field": text_field},
|
|
107
|
+
"analyzer": analyzer,
|
|
108
|
+
}
|
|
109
|
+
],
|
|
110
|
+
)
|
|
111
|
+
elif existing and existing != dimension:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"table {table!r} holds {existing}-dimension vectors and collection "
|
|
114
|
+
f"{collection.name!r} {dimension}-dimension ones"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
copied = 0
|
|
118
|
+
while copied < total:
|
|
119
|
+
page = collection.get(
|
|
120
|
+
limit=batch_size,
|
|
121
|
+
offset=copied,
|
|
122
|
+
include=["embeddings", "documents", "metadatas"],
|
|
123
|
+
)
|
|
124
|
+
ids = page["ids"]
|
|
125
|
+
if not ids:
|
|
126
|
+
break
|
|
127
|
+
documents = page.get("documents") or [None] * len(ids)
|
|
128
|
+
metadatas = page.get("metadatas") or [None] * len(ids)
|
|
129
|
+
records = []
|
|
130
|
+
for rid, embedding, document, metadata in zip(
|
|
131
|
+
ids, page["embeddings"], documents, metadatas
|
|
132
|
+
):
|
|
133
|
+
payload = dict(metadata or {})
|
|
134
|
+
if document is not None:
|
|
135
|
+
if text_field in payload and payload[text_field] != document:
|
|
136
|
+
raise ValueError(
|
|
137
|
+
f"record {rid!r} in {collection.name!r} has a {text_field!r} metadata "
|
|
138
|
+
"field as well as a document; choose another field with --text-field"
|
|
139
|
+
)
|
|
140
|
+
payload[text_field] = document
|
|
141
|
+
records.append({"id": rid, "vector": _vector(embedding), "payload": payload})
|
|
142
|
+
client.insert_records(table, records)
|
|
143
|
+
copied += len(ids)
|
|
144
|
+
if progress:
|
|
145
|
+
progress(collection.name, copied, total)
|
|
146
|
+
return Migrated(collection.name, table, copied)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def migrate(
|
|
150
|
+
chroma: Any,
|
|
151
|
+
client: DXMTClient,
|
|
152
|
+
collections: Optional[Iterable[str]] = None,
|
|
153
|
+
**options: Any,
|
|
154
|
+
) -> list[Migrated]:
|
|
155
|
+
"""Copy the named collections, or every one, from a Chroma client into DXMT."""
|
|
156
|
+
names = list(collections) if collections else [c if isinstance(c, str) else c.name
|
|
157
|
+
for c in chroma.list_collections()]
|
|
158
|
+
# The embedding function is not needed to read stored vectors, and asking for the
|
|
159
|
+
# collection's own would load its model.
|
|
160
|
+
return [
|
|
161
|
+
migrate_collection(chroma.get_collection(name), client, **options) for name in names
|
|
162
|
+
]
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def main(argv: Optional[list[str]] = None) -> int:
|
|
166
|
+
parser = argparse.ArgumentParser(
|
|
167
|
+
prog="python -m dxmt.migrate.chroma",
|
|
168
|
+
description="Copy Chroma collections, with their vectors, into DXMT tables.",
|
|
169
|
+
)
|
|
170
|
+
source = parser.add_mutually_exclusive_group(required=True)
|
|
171
|
+
source.add_argument("--path", help="a Chroma persistent directory")
|
|
172
|
+
source.add_argument("--chroma-host", help="a Chroma server, host[:port]")
|
|
173
|
+
parser.add_argument("--collection", action="append", help="copy only this one (repeatable)")
|
|
174
|
+
parser.add_argument("--table", help="the table to copy into (one --collection only)")
|
|
175
|
+
parser.add_argument("--url", default=os.environ.get("DXMT_URL", "http://localhost:9999"))
|
|
176
|
+
parser.add_argument("--text-field", default="text", help="payload field for the document")
|
|
177
|
+
parser.add_argument("--analyzer", default="standard", help="full-text analyzer, e.g. chinese")
|
|
178
|
+
parser.add_argument("--batch-size", type=int, default=500)
|
|
179
|
+
args = parser.parse_args(argv)
|
|
180
|
+
|
|
181
|
+
if args.table and len(args.collection or []) != 1:
|
|
182
|
+
parser.error("--table needs exactly one --collection")
|
|
183
|
+
try:
|
|
184
|
+
import chromadb
|
|
185
|
+
except ImportError:
|
|
186
|
+
print("chromadb is not installed: pip install 'dxmt-python-sdk[chroma]'", file=sys.stderr)
|
|
187
|
+
return 2
|
|
188
|
+
|
|
189
|
+
if args.path:
|
|
190
|
+
chroma = chromadb.PersistentClient(path=args.path)
|
|
191
|
+
else:
|
|
192
|
+
host, _, port = args.chroma_host.partition(":")
|
|
193
|
+
chroma = chromadb.HttpClient(host=host, port=int(port or 8000))
|
|
194
|
+
|
|
195
|
+
match = re.match(r"^(https?)://([^:/]+)(?::(\d+))?", args.url)
|
|
196
|
+
if not match:
|
|
197
|
+
parser.error(f"--url {args.url!r} is not http(s)://host[:port]")
|
|
198
|
+
scheme, host, port = match.groups()
|
|
199
|
+
client = DXMTClient(
|
|
200
|
+
host=host,
|
|
201
|
+
port=int(port or (443 if scheme == "https" else 9999)),
|
|
202
|
+
ssl=scheme == "https",
|
|
203
|
+
api_key=os.environ.get("DXMT_API_KEY"),
|
|
204
|
+
timeout=120.0,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
def progress(name: str, done: int, total: int) -> None:
|
|
208
|
+
print(f"\r{name}: {done}/{total}", end="", file=sys.stderr, flush=True)
|
|
209
|
+
|
|
210
|
+
names = args.collection
|
|
211
|
+
if names is None:
|
|
212
|
+
names = [c if isinstance(c, str) else c.name for c in chroma.list_collections()]
|
|
213
|
+
failed = False
|
|
214
|
+
for name in names:
|
|
215
|
+
try:
|
|
216
|
+
result = migrate_collection(
|
|
217
|
+
chroma.get_collection(name),
|
|
218
|
+
client,
|
|
219
|
+
args.table,
|
|
220
|
+
text_field=args.text_field,
|
|
221
|
+
analyzer=args.analyzer,
|
|
222
|
+
batch_size=args.batch_size,
|
|
223
|
+
progress=progress,
|
|
224
|
+
)
|
|
225
|
+
except Exception as e: # one collection failing does not stop the others
|
|
226
|
+
print(f"\n{name}: failed: {e}", file=sys.stderr)
|
|
227
|
+
failed = True
|
|
228
|
+
continue
|
|
229
|
+
if result.skipped:
|
|
230
|
+
print(f"{name}: skipped ({result.skipped})", file=sys.stderr)
|
|
231
|
+
else:
|
|
232
|
+
print(f"\n{name} -> {result.table}: {result.records} records", file=sys.stderr)
|
|
233
|
+
return 1 if failed else 0
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
if __name__ == "__main__":
|
|
237
|
+
sys.exit(main())
|