dxmt-python-sdk 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,466 @@
1
+ """DXMT VectorStore for LlamaIndex.
2
+
3
+ This module provides a LlamaIndex VectorStore implementation for DXMT VectorDB,
4
+ enabling integration with LlamaIndex-based frameworks like MetaGPT.
5
+
6
+ Example:
7
+ ```python
8
+ from llama_index.core import VectorStoreIndex, StorageContext
9
+ from dxmt import DXMTClient
10
+ from dxmt.metagpt_vector_store import DXMTVectorStore
11
+
12
+ # Create DXMT vector store
13
+ client = DXMTClient(host="localhost", port=9999)
14
+ vector_store = DXMTVectorStore(
15
+ client=client,
16
+ table_name="documents",
17
+ dimension=1536
18
+ )
19
+
20
+ # Create storage context and index
21
+ storage_context = StorageContext.from_defaults(vector_store=vector_store)
22
+ index = VectorStoreIndex.from_documents(
23
+ documents,
24
+ storage_context=storage_context
25
+ )
26
+ ```
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import logging
32
+ import uuid
33
+ from typing import Any, List, Optional
34
+
35
+ from llama_index.core.schema import BaseNode, TextNode
36
+ from llama_index.core.vector_stores.types import (
37
+ BasePydanticVectorStore,
38
+ VectorStoreQuery,
39
+ VectorStoreQueryResult,
40
+ )
41
+ from pydantic import ConfigDict, PrivateAttr
42
+
43
+ from llama_index.core.vector_stores.utils import (
44
+ metadata_dict_to_node,
45
+ node_to_metadata_dict,
46
+ )
47
+
48
+ from .client import DXMTClient
49
+ from .exceptions import DXMTError
50
+ from .filters import convert_simple_filter_to_dxmt
51
+
52
+ logger = logging.getLogger(__name__)
53
+
54
+
55
+ class DXMTVectorStore(BasePydanticVectorStore):
56
+ """DXMT Vector Store for LlamaIndex.
57
+
58
+ Implements LlamaIndex's BasePydanticVectorStore interface to enable
59
+ DXMT VectorDB integration with LlamaIndex and MetaGPT.
60
+
61
+ Args:
62
+ client: DXMTClient instance. If None, creates with default settings.
63
+ table_name: Name of the DXMT table to use.
64
+ dimension: Vector dimension. Required if table doesn't exist.
65
+ distance_metric: Distance metric (cosine, euclidean, dot, manhattan).
66
+ create_if_not_exists: Create table if it doesn't exist.
67
+ host: DXMT server host (used if client is None).
68
+ port: DXMT server port (used if client is None).
69
+
70
+ Attributes:
71
+ stores_text: Always True - DXMT stores text in metadata.
72
+ is_embedding_query: Always True - queries use embeddings.
73
+ """
74
+
75
+ stores_text: bool = True
76
+ is_embedding_query: bool = True
77
+
78
+ # Pydantic fields. `client` is not one of them: `BasePydanticVectorStore` declares it as an
79
+ # abstract property, and a field of the same name shadows it -- pydantic said so in a
80
+ # warning printed on every `import dxmt` once llama-index was installed. It is a private
81
+ # attribute with the property the interface asks for, which is what the standalone
82
+ # `llama-index-vector-stores-dxmt` package has always done.
83
+ _client: Optional[DXMTClient] = PrivateAttr(default=None)
84
+ table_name: str = "documents"
85
+ dimension: int = 1536
86
+ distance_metric: str = "cosine"
87
+ create_if_not_exists: bool = True
88
+ host: str = "localhost"
89
+ port: int = 9999
90
+
91
+ _initialized: bool = PrivateAttr(default=False)
92
+
93
+ # A class-based `Config` is the pydantic v1 spelling, deprecated since v2 and removed in
94
+ # v3; it warned on every import of this module.
95
+ model_config = ConfigDict(arbitrary_types_allowed=True)
96
+
97
+ def __init__(
98
+ self,
99
+ client: Optional[DXMTClient] = None,
100
+ table_name: str = "documents",
101
+ dimension: int = 1536,
102
+ distance_metric: str = "cosine",
103
+ create_if_not_exists: bool = True,
104
+ host: str = "localhost",
105
+ port: int = 9999,
106
+ api_key: Optional[str] = None,
107
+ **kwargs: Any,
108
+ ) -> None:
109
+ """Initialize DXMT vector store.
110
+
111
+ ``api_key`` goes to the client and is not kept as a field: the other arguments are
112
+ pydantic fields and come back out of ``repr()`` and serialisation.
113
+ """
114
+ super().__init__(
115
+ table_name=table_name,
116
+ dimension=dimension,
117
+ distance_metric=distance_metric,
118
+ create_if_not_exists=create_if_not_exists,
119
+ host=host,
120
+ port=port,
121
+ **kwargs,
122
+ )
123
+
124
+ # Create client if not provided
125
+ self._client = client or DXMTClient(host=self.host, port=self.port, api_key=api_key)
126
+
127
+ # Initialize table
128
+ if create_if_not_exists and not self._initialized:
129
+ self._ensure_table_exists()
130
+ self._initialized = True
131
+
132
+ def _ensure_table_exists(self) -> None:
133
+ """Ensure the DXMT table exists."""
134
+ try:
135
+ # Try to get table
136
+ self.client.get_table(self.table_name)
137
+ logger.info(f"Table {self.table_name} already exists")
138
+ except DXMTError:
139
+ # Table doesn't exist, create it
140
+ logger.info(f"Creating table {self.table_name}")
141
+ self.client.create_table(
142
+ name=self.table_name,
143
+ dimension=self.dimension,
144
+ distance_metric=self.distance_metric,
145
+ description=f"DXMT vector store for LlamaIndex (dimension={self.dimension})"
146
+ )
147
+
148
+ @property
149
+ def client(self) -> DXMTClient:
150
+ """The underlying DXMT client, as `BasePydanticVectorStore` requires."""
151
+ assert self._client is not None # set in __init__, which is the only way in
152
+ return self._client
153
+
154
+ @property
155
+ def client_instance(self) -> DXMTClient:
156
+ """The client, under the name this class used before `client` became a property."""
157
+ return self.client
158
+
159
+ def add(
160
+ self,
161
+ nodes: List[BaseNode],
162
+ **kwargs: Any,
163
+ ) -> List[str]:
164
+ """Add nodes to DXMT.
165
+
166
+ Args:
167
+ nodes: List of nodes to add.
168
+ **kwargs: Additional arguments.
169
+
170
+ Returns:
171
+ List of node IDs.
172
+ """
173
+ if not nodes:
174
+ return []
175
+
176
+ # Convert nodes to DXMT records
177
+ records = []
178
+ for node in nodes:
179
+ # Get embedding
180
+ embedding = node.get_embedding()
181
+ if embedding is None:
182
+ logger.warning(f"Node {node.node_id} has no embedding, skipping")
183
+ continue
184
+
185
+ # Convert node to metadata dict
186
+ metadata = node_to_metadata_dict(node, remove_text=False, flat_metadata=False)
187
+
188
+ # Add text content explicitly
189
+ if hasattr(node, 'text') and node.text:
190
+ metadata['text'] = node.text
191
+
192
+ # Create record
193
+ record = {
194
+ "id": node.node_id,
195
+ "vector": embedding,
196
+ "metadata": metadata
197
+ }
198
+ records.append(record)
199
+
200
+ if not records:
201
+ return []
202
+
203
+ # Insert into DXMT
204
+ try:
205
+ self.client.insert_records(
206
+ table_name=self.table_name,
207
+ records=records
208
+ )
209
+ logger.info(f"Added {len(records)} nodes to {self.table_name}")
210
+ return [r["id"] for r in records]
211
+ except Exception as e:
212
+ logger.error(f"Error adding nodes: {e}")
213
+ raise DXMTError(f"Failed to add nodes: {e}")
214
+
215
+ def delete(self, ref_doc_id: str, **kwargs: Any) -> None:
216
+ """Delete nodes by reference document ID.
217
+
218
+ Args:
219
+ ref_doc_id: Reference document ID.
220
+ **kwargs: Additional arguments.
221
+ """
222
+ try:
223
+ # Search for all nodes with this ref_doc_id using a filter-only scan.
224
+ # A dummy zero vector is used because DXMT has no filter-scan endpoint;
225
+ # results are restricted by the ref_doc_id filter, not by proximity.
226
+ results = self.client.search(
227
+ table_name=self.table_name,
228
+ vector=[0.0] * self.dimension,
229
+ k=10000,
230
+ filter={
231
+ "must": [
232
+ {
233
+ "key": "ref_doc_id",
234
+ "operation": {"type": "Equal", "value": ref_doc_id},
235
+ }
236
+ ]
237
+ },
238
+ )
239
+
240
+ node_ids = [r.get("id") for r in results if r.get("id")]
241
+ if node_ids:
242
+ self.client.delete_records(
243
+ table_name=self.table_name,
244
+ ids=node_ids,
245
+ )
246
+
247
+ logger.info(f"Deleted {len(node_ids)} nodes with ref_doc_id={ref_doc_id}")
248
+ except Exception as e:
249
+ logger.error(f"Error deleting nodes: {e}")
250
+ raise DXMTError(f"Failed to delete nodes: {e}")
251
+
252
+ def query(
253
+ self,
254
+ query: VectorStoreQuery,
255
+ **kwargs: Any,
256
+ ) -> VectorStoreQueryResult:
257
+ """Query DXMT vector store.
258
+
259
+ Respects VectorStoreQuery.mode:
260
+ - DEFAULT / None → dense vector search
261
+ - HYBRID → hybrid_search (vector + text)
262
+ - SEMANTIC_HYBRID → the same; `alpha`, when the query carries one, is the
263
+ vector side's share and selects weighted fusion
264
+ - SPARSE / TEXT_SEARCH → full_text_search (BM25 only)
265
+
266
+ Args:
267
+ query: Vector store query.
268
+ **kwargs: Additional arguments.
269
+
270
+ Returns:
271
+ Query results with nodes and similarities.
272
+ """
273
+ try:
274
+ from llama_index.core.vector_stores.types import VectorStoreQueryMode
275
+ _MODE = VectorStoreQueryMode
276
+ except ImportError:
277
+ _MODE = None
278
+
279
+ dxmt_filter = self._convert_filters(query.filters) if query.filters else None
280
+ top_k = query.similarity_top_k or 5
281
+ mode = getattr(query, "mode", None)
282
+
283
+ try:
284
+ # --- HYBRID modes ---
285
+ if _MODE and mode in (_MODE.HYBRID, _MODE.SEMANTIC_HYBRID):
286
+ # Pass the query's alpha through rather than inventing one: a weight selects
287
+ # weighted fusion, and a caller who set none should get the server's default
288
+ # (`dbsf`), which needs no weight.
289
+ results = self.client.hybrid_search(
290
+ table_name=self.table_name,
291
+ vector=query.query_embedding,
292
+ text=getattr(query, "query_str", None),
293
+ alpha=getattr(query, "alpha", None),
294
+ k=top_k,
295
+ filter=dxmt_filter,
296
+ )
297
+ # --- SPARSE / TEXT_SEARCH modes ---
298
+ elif _MODE and mode in (_MODE.SPARSE, _MODE.TEXT_SEARCH):
299
+ query_str = getattr(query, "query_str", None)
300
+ if not query_str:
301
+ raise ValueError("query_str is required for SPARSE/TEXT_SEARCH mode")
302
+ results = self.client.full_text_search(
303
+ table_name=self.table_name,
304
+ query=query_str,
305
+ k=top_k,
306
+ filter=dxmt_filter,
307
+ )
308
+ # --- DEFAULT: dense vector search ---
309
+ else:
310
+ if query.query_embedding is None:
311
+ raise ValueError("query_embedding is required for DEFAULT mode")
312
+ results = self.client.search(
313
+ table_name=self.table_name,
314
+ vector=query.query_embedding,
315
+ k=top_k,
316
+ filter=dxmt_filter,
317
+ )
318
+ except (ValueError, AttributeError):
319
+ raise
320
+ except Exception as e:
321
+ logger.error(f"Error querying DXMT: {e}")
322
+ raise DXMTError(f"Failed to query: {e}")
323
+
324
+ # Convert results to LlamaIndex format
325
+ nodes = []
326
+ similarities = []
327
+ ids = []
328
+
329
+ for result in results:
330
+ # Extract metadata
331
+ metadata = result.get("metadata", {})
332
+
333
+ # Reconstruct node from metadata
334
+ try:
335
+ node = metadata_dict_to_node(metadata)
336
+ node.id_ = result.get("id")
337
+ except Exception:
338
+ # Fallback: create TextNode manually
339
+ text = metadata.get("text", "")
340
+ node = TextNode(
341
+ text=text,
342
+ id_=result.get("id"),
343
+ metadata=metadata,
344
+ embedding=result.get("vector")
345
+ )
346
+
347
+ nodes.append(node)
348
+
349
+ # Get similarity score
350
+ score = result.get("score", 0.0)
351
+ similarities.append(score)
352
+
353
+ # Get ID
354
+ ids.append(result.get("id"))
355
+
356
+ return VectorStoreQueryResult(
357
+ nodes=nodes,
358
+ similarities=similarities,
359
+ ids=ids,
360
+ )
361
+
362
+ def persist(
363
+ self,
364
+ persist_path: str,
365
+ fs: Optional[Any] = None,
366
+ ) -> None:
367
+ """Persist the vector store.
368
+
369
+ DXMT automatically persists data, so this is a no-op.
370
+
371
+ Args:
372
+ persist_path: Path to persist to (unused).
373
+ fs: Filesystem (unused).
374
+ """
375
+ logger.info("DXMT automatically persists data")
376
+
377
+ def _convert_filters(self, filters: Any) -> Optional[dict]:
378
+ """Convert LlamaIndex MetadataFilters (or plain dict) to DXMT format.
379
+
380
+ Handles:
381
+ - LlamaIndex MetadataFilters object (with .filters list of
382
+ MetadataFilter, each having .key / .value / .operator)
383
+ - Plain dict (passed through convert_simple_filter_to_dxmt which
384
+ supports MongoDB-style operators: $eq, $ne, $gt, $gte, $lt,
385
+ $lte, $in, $nin, $contains, $startswith, $endswith, $and, $or)
386
+
387
+ Args:
388
+ filters: LlamaIndex filters or plain dict.
389
+
390
+ Returns:
391
+ DXMT filter dict, or None.
392
+ """
393
+ if filters is None:
394
+ return None
395
+
396
+ # LlamaIndex MetadataFilters object
397
+ if hasattr(filters, "filters") and hasattr(filters, "condition"):
398
+ from llama_index.core.vector_stores.types import FilterOperator, FilterCondition
399
+ _OP_MAP = {
400
+ FilterOperator.EQ: "Equal",
401
+ FilterOperator.NE: "NotEqual",
402
+ FilterOperator.GT: "GreaterThan",
403
+ FilterOperator.GTE: "GreaterThanOrEqual",
404
+ FilterOperator.LT: "LessThan",
405
+ FilterOperator.LTE: "LessThanOrEqual",
406
+ FilterOperator.IN: "In",
407
+ FilterOperator.NIN: "NotIn",
408
+ FilterOperator.CONTAINS: "Contains",
409
+ }
410
+ conditions = []
411
+ for f in filters.filters:
412
+ op_type = _OP_MAP.get(f.operator, "Equal")
413
+ conditions.append({
414
+ "key": f.key,
415
+ "operation": {"type": op_type, "value": f.value},
416
+ })
417
+ if not conditions:
418
+ return None
419
+ cond_key = "must"
420
+ if hasattr(FilterCondition, "OR") and filters.condition == FilterCondition.OR:
421
+ cond_key = "should"
422
+ return {cond_key: conditions}
423
+
424
+ # Plain dict — delegate to shared filter utility
425
+ if isinstance(filters, dict):
426
+ return convert_simple_filter_to_dxmt(filters)
427
+
428
+ return None
429
+
430
+ def clear(self) -> None:
431
+ """Clear all nodes from the table.
432
+
433
+ Deletes and recreates the table.
434
+ """
435
+ try:
436
+ self.client.delete_table(self.table_name)
437
+ logger.info(f"Deleted table {self.table_name}")
438
+
439
+ # Recreate table
440
+ self._ensure_table_exists()
441
+ logger.info(f"Recreated table {self.table_name}")
442
+ except Exception as e:
443
+ logger.error(f"Error clearing vector store: {e}")
444
+ raise DXMTError(f"Failed to clear: {e}")
445
+
446
+ async def async_add(self, nodes: List[BaseNode], **kwargs: Any) -> List[str]:
447
+ """Async add nodes."""
448
+ import asyncio
449
+ loop = asyncio.get_event_loop()
450
+ return await loop.run_in_executor(None, lambda: self.add(nodes, **kwargs))
451
+
452
+ async def aquery(
453
+ self,
454
+ query: VectorStoreQuery,
455
+ **kwargs: Any,
456
+ ) -> VectorStoreQueryResult:
457
+ """Async query."""
458
+ import asyncio
459
+ loop = asyncio.get_event_loop()
460
+ return await loop.run_in_executor(None, lambda: self.query(query, **kwargs))
461
+
462
+ async def adelete(self, ref_doc_id: str, **kwargs: Any) -> None:
463
+ """Async delete by ref_doc_id."""
464
+ import asyncio
465
+ loop = asyncio.get_event_loop()
466
+ await loop.run_in_executor(None, lambda: self.delete(ref_doc_id, **kwargs))
@@ -0,0 +1 @@
1
+ """Moving an existing store into DXMT. `python -m dxmt.migrate.chroma --help`."""
dxmt/migrate/chroma.py ADDED
@@ -0,0 +1,237 @@
1
+ """Copy Chroma collections into DXMT tables, vectors included, so nothing is re-embedded.
2
+
3
+ pip install 'dxmt-python-sdk[chroma]'
4
+ python -m dxmt.migrate.chroma --path ./chroma_db # every collection
5
+ python -m dxmt.migrate.chroma --path ./chroma_db --collection notes --table notes
6
+
7
+ Each collection becomes one table: the same ids, the stored embeddings, the collection's
8
+ distance (`l2` -> `euclidean`, `ip` -> `dot`, `cosine`), each metadata dict as the payload and
9
+ the document under `text`, with a full-text index on it so text and hybrid search work on the
10
+ copy. A record is written by id, so running it again over the same table rewrites the same
11
+ records rather than duplicating them -- an interrupted run is finished by running it again.
12
+
13
+ Chroma's embedding function does not come along: the copy is searched with vectors (or with
14
+ text through the full-text index). To search it with text through a model, create the table
15
+ with `embedding` naming the same model the collection used and migrate into it with `--table`.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import argparse
21
+ import os
22
+ import re
23
+ import sys
24
+ from dataclasses import dataclass
25
+ from typing import Any, Callable, Iterable, Optional
26
+
27
+ from ..client import DXMTClient
28
+
29
+ _SPACES = {"l2": "euclidean", "cosine": "cosine", "ip": "dot"}
30
+
31
+
32
+ @dataclass
33
+ class Migrated:
34
+ collection: str
35
+ table: str
36
+ records: int
37
+ skipped: Optional[str] = None
38
+
39
+
40
+ def table_name_for(collection: str) -> str:
41
+ """A collection name as a table name: DXMT allows letters, digits, `_` and `-`."""
42
+ name = re.sub(r"[^A-Za-z0-9_-]", "_", collection)
43
+ if len(name) > 63:
44
+ raise ValueError(
45
+ f"collection {collection!r} makes a table name over 63 characters; "
46
+ "name the table with --table"
47
+ )
48
+ return name
49
+
50
+
51
+ def distance_of(collection: Any) -> str:
52
+ """The collection's distance as a DXMT metric. Chroma 1.x keeps it in `configuration`,
53
+ older versions in the `hnsw:space` metadata key; the default is `l2`."""
54
+ space = None
55
+ config = getattr(collection, "configuration", None)
56
+ if isinstance(config, dict):
57
+ space = (config.get("hnsw") or {}).get("space") or (config.get("spann") or {}).get(
58
+ "space"
59
+ )
60
+ if space is None:
61
+ space = (collection.metadata or {}).get("hnsw:space", "l2")
62
+ if space not in _SPACES:
63
+ raise ValueError(f"collection {collection.name!r} uses distance {space!r}, which DXMT lacks")
64
+ return _SPACES[space]
65
+
66
+
67
+ def _vector(embedding: Any) -> list[float]:
68
+ return embedding.tolist() if hasattr(embedding, "tolist") else list(embedding)
69
+
70
+
71
+ def _existing_dimension(client: DXMTClient, table: str) -> Optional[int]:
72
+ for t in client.list_tables():
73
+ if t.get("name") == table:
74
+ return int(t.get("dimension") or 0)
75
+ return None
76
+
77
+
78
+ def migrate_collection(
79
+ collection: Any,
80
+ client: DXMTClient,
81
+ table: Optional[str] = None,
82
+ *,
83
+ text_field: str = "text",
84
+ analyzer: str = "standard",
85
+ batch_size: int = 500,
86
+ progress: Optional[Callable[[str, int, int], None]] = None,
87
+ ) -> Migrated:
88
+ """Copy one Chroma collection into a DXMT table, creating the table if it does not exist."""
89
+ table = table or table_name_for(collection.name)
90
+ total = collection.count()
91
+ if total == 0:
92
+ return Migrated(collection.name, table, 0, skipped="empty: no vectors to size a table by")
93
+
94
+ first = collection.get(limit=1, include=["embeddings"])
95
+ dimension = len(_vector(first["embeddings"][0]))
96
+ existing = _existing_dimension(client, table)
97
+ if existing is None:
98
+ client.create_table(
99
+ name=table,
100
+ dimension=dimension,
101
+ distance_metric=distance_of(collection),
102
+ enable_full_text_search=True,
103
+ text_fields=[
104
+ {
105
+ "name": "content",
106
+ "source": {"metadata_field": text_field},
107
+ "analyzer": analyzer,
108
+ }
109
+ ],
110
+ )
111
+ elif existing and existing != dimension:
112
+ raise ValueError(
113
+ f"table {table!r} holds {existing}-dimension vectors and collection "
114
+ f"{collection.name!r} {dimension}-dimension ones"
115
+ )
116
+
117
+ copied = 0
118
+ while copied < total:
119
+ page = collection.get(
120
+ limit=batch_size,
121
+ offset=copied,
122
+ include=["embeddings", "documents", "metadatas"],
123
+ )
124
+ ids = page["ids"]
125
+ if not ids:
126
+ break
127
+ documents = page.get("documents") or [None] * len(ids)
128
+ metadatas = page.get("metadatas") or [None] * len(ids)
129
+ records = []
130
+ for rid, embedding, document, metadata in zip(
131
+ ids, page["embeddings"], documents, metadatas
132
+ ):
133
+ payload = dict(metadata or {})
134
+ if document is not None:
135
+ if text_field in payload and payload[text_field] != document:
136
+ raise ValueError(
137
+ f"record {rid!r} in {collection.name!r} has a {text_field!r} metadata "
138
+ "field as well as a document; choose another field with --text-field"
139
+ )
140
+ payload[text_field] = document
141
+ records.append({"id": rid, "vector": _vector(embedding), "payload": payload})
142
+ client.insert_records(table, records)
143
+ copied += len(ids)
144
+ if progress:
145
+ progress(collection.name, copied, total)
146
+ return Migrated(collection.name, table, copied)
147
+
148
+
149
+ def migrate(
150
+ chroma: Any,
151
+ client: DXMTClient,
152
+ collections: Optional[Iterable[str]] = None,
153
+ **options: Any,
154
+ ) -> list[Migrated]:
155
+ """Copy the named collections, or every one, from a Chroma client into DXMT."""
156
+ names = list(collections) if collections else [c if isinstance(c, str) else c.name
157
+ for c in chroma.list_collections()]
158
+ # The embedding function is not needed to read stored vectors, and asking for the
159
+ # collection's own would load its model.
160
+ return [
161
+ migrate_collection(chroma.get_collection(name), client, **options) for name in names
162
+ ]
163
+
164
+
165
+ def main(argv: Optional[list[str]] = None) -> int:
166
+ parser = argparse.ArgumentParser(
167
+ prog="python -m dxmt.migrate.chroma",
168
+ description="Copy Chroma collections, with their vectors, into DXMT tables.",
169
+ )
170
+ source = parser.add_mutually_exclusive_group(required=True)
171
+ source.add_argument("--path", help="a Chroma persistent directory")
172
+ source.add_argument("--chroma-host", help="a Chroma server, host[:port]")
173
+ parser.add_argument("--collection", action="append", help="copy only this one (repeatable)")
174
+ parser.add_argument("--table", help="the table to copy into (one --collection only)")
175
+ parser.add_argument("--url", default=os.environ.get("DXMT_URL", "http://localhost:9999"))
176
+ parser.add_argument("--text-field", default="text", help="payload field for the document")
177
+ parser.add_argument("--analyzer", default="standard", help="full-text analyzer, e.g. chinese")
178
+ parser.add_argument("--batch-size", type=int, default=500)
179
+ args = parser.parse_args(argv)
180
+
181
+ if args.table and len(args.collection or []) != 1:
182
+ parser.error("--table needs exactly one --collection")
183
+ try:
184
+ import chromadb
185
+ except ImportError:
186
+ print("chromadb is not installed: pip install 'dxmt-python-sdk[chroma]'", file=sys.stderr)
187
+ return 2
188
+
189
+ if args.path:
190
+ chroma = chromadb.PersistentClient(path=args.path)
191
+ else:
192
+ host, _, port = args.chroma_host.partition(":")
193
+ chroma = chromadb.HttpClient(host=host, port=int(port or 8000))
194
+
195
+ match = re.match(r"^(https?)://([^:/]+)(?::(\d+))?", args.url)
196
+ if not match:
197
+ parser.error(f"--url {args.url!r} is not http(s)://host[:port]")
198
+ scheme, host, port = match.groups()
199
+ client = DXMTClient(
200
+ host=host,
201
+ port=int(port or (443 if scheme == "https" else 9999)),
202
+ ssl=scheme == "https",
203
+ api_key=os.environ.get("DXMT_API_KEY"),
204
+ timeout=120.0,
205
+ )
206
+
207
+ def progress(name: str, done: int, total: int) -> None:
208
+ print(f"\r{name}: {done}/{total}", end="", file=sys.stderr, flush=True)
209
+
210
+ names = args.collection
211
+ if names is None:
212
+ names = [c if isinstance(c, str) else c.name for c in chroma.list_collections()]
213
+ failed = False
214
+ for name in names:
215
+ try:
216
+ result = migrate_collection(
217
+ chroma.get_collection(name),
218
+ client,
219
+ args.table,
220
+ text_field=args.text_field,
221
+ analyzer=args.analyzer,
222
+ batch_size=args.batch_size,
223
+ progress=progress,
224
+ )
225
+ except Exception as e: # one collection failing does not stop the others
226
+ print(f"\n{name}: failed: {e}", file=sys.stderr)
227
+ failed = True
228
+ continue
229
+ if result.skipped:
230
+ print(f"{name}: skipped ({result.skipped})", file=sys.stderr)
231
+ else:
232
+ print(f"\n{name} -> {result.table}: {result.records} records", file=sys.stderr)
233
+ return 1 if failed else 0
234
+
235
+
236
+ if __name__ == "__main__":
237
+ sys.exit(main())