dxmt-python-sdk 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dxmt/__init__.py +166 -0
- dxmt/bisheng_integration.py +226 -0
- dxmt/client.py +2510 -0
- dxmt/crewai_storage.py +524 -0
- dxmt/crewai_tool.py +430 -0
- dxmt/exceptions.py +46 -0
- dxmt/filters.py +360 -0
- dxmt/haystack.py +773 -0
- dxmt/haystack_retrievers.py +372 -0
- dxmt/langchain_vectorstore.py +527 -0
- dxmt/langgraph_store.py +1054 -0
- dxmt/mem0.py +653 -0
- dxmt/metagpt_retriever.py +361 -0
- dxmt/metagpt_vector_store.py +466 -0
- dxmt/migrate/__init__.py +1 -0
- dxmt/migrate/chroma.py +237 -0
- dxmt/py.typed +0 -0
- dxmt/types.py +51 -0
- dxmt_python_sdk-1.6.0.dist-info/METADATA +736 -0
- dxmt_python_sdk-1.6.0.dist-info/RECORD +23 -0
- dxmt_python_sdk-1.6.0.dist-info/WHEEL +5 -0
- dxmt_python_sdk-1.6.0.dist-info/licenses/LICENSE +201 -0
- dxmt_python_sdk-1.6.0.dist-info/top_level.txt +1 -0
dxmt/client.py
ADDED
|
@@ -0,0 +1,2510 @@
|
|
|
1
|
+
"""DXMT Python SDK HTTP Client.
|
|
2
|
+
|
|
3
|
+
This module provides sync and async HTTP clients for interacting with
|
|
4
|
+
the DXMT VectorDB REST API.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import time
|
|
12
|
+
import warnings
|
|
13
|
+
from collections import deque
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any, AsyncIterator, Iterable, Iterator, Optional
|
|
17
|
+
|
|
18
|
+
from urllib.parse import quote
|
|
19
|
+
|
|
20
|
+
import httpx
|
|
21
|
+
|
|
22
|
+
from .exceptions import (
|
|
23
|
+
DXMTConnectionError,
|
|
24
|
+
DXMTError,
|
|
25
|
+
DXMTRecordError,
|
|
26
|
+
DXMTSearchError,
|
|
27
|
+
DXMTTableError,
|
|
28
|
+
)
|
|
29
|
+
from .filters import convert_simple_filter_to_dxmt
|
|
30
|
+
from .types import Filter
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
try: # pragma: no cover - exercised by whichever branch is installed
|
|
35
|
+
import orjson
|
|
36
|
+
|
|
37
|
+
_HAS_ORJSON = True
|
|
38
|
+
except ImportError: # pragma: no cover
|
|
39
|
+
orjson = None
|
|
40
|
+
_HAS_ORJSON = False
|
|
41
|
+
|
|
42
|
+
# Content-Type has to be set by hand when a body is passed as bytes rather than through
|
|
43
|
+
# httpx's `json=`. httpx merges per-request headers over the client's, so auth survives.
|
|
44
|
+
_JSON_HEADERS = {"Content-Type": "application/json"}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _encode_json(payload: Any) -> bytes:
|
|
48
|
+
"""Serialise a request body, preferring orjson when it is installed.
|
|
49
|
+
|
|
50
|
+
An insert is mostly floats -- 100 records of 768 dimensions is 76,800 of them per
|
|
51
|
+
request -- and `json.dumps` spends that entirely in Python holding the GIL, so
|
|
52
|
+
concurrent writer threads serialise one at a time no matter how many there are.
|
|
53
|
+
Measured on this SDK's own insert path, that capped a four-thread load at about a
|
|
54
|
+
third of what the server could absorb.
|
|
55
|
+
|
|
56
|
+
Measured here on a 100 x 768 body: the standard library encodes 1,215 rows/s on one
|
|
57
|
+
thread and *1,124 on four* -- it does not scale at all -- while orjson does about
|
|
58
|
+
7,900 either way, 7.3x faster. The four-thread figure is the one that matters: it is
|
|
59
|
+
within noise of the 1,262 rows/s a four-worker VectorDBBench load measured against
|
|
60
|
+
this SDK, which is to say that benchmark was measuring this function.
|
|
61
|
+
|
|
62
|
+
orjson is optional and the fallback is what httpx itself would have done. The two
|
|
63
|
+
bodies are not byte-identical -- the standard library writes small magnitudes in
|
|
64
|
+
scientific notation (8.859986183940016e-05) where orjson writes them out
|
|
65
|
+
(0.00008859986183940016) -- but they parse to the same values, so this is a
|
|
66
|
+
formatting difference and not a protocol change. One real behavioural difference:
|
|
67
|
+
`allow_nan=False` makes the fallback raise on a NaN or infinity, while orjson writes
|
|
68
|
+
`null` and lets the server reject it. A NaN in a vector is a bug either way; it just
|
|
69
|
+
surfaces one hop later.
|
|
70
|
+
"""
|
|
71
|
+
if _HAS_ORJSON:
|
|
72
|
+
return orjson.dumps(payload, option=orjson.OPT_SERIALIZE_NUMPY)
|
|
73
|
+
return json.dumps(
|
|
74
|
+
payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
|
75
|
+
).encode("utf-8")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _seg(value: str) -> str:
|
|
80
|
+
"""Percent-encode one URL path segment.
|
|
81
|
+
|
|
82
|
+
A table name or a record id goes into the path, and ids carry characters the path reserves.
|
|
83
|
+
`doc.md#3` stops at the `#` -- a fragment delimiter -- so the request reaches
|
|
84
|
+
`/records/doc.md` and the server answers about a record nobody asked for; `s3://bucket/key`
|
|
85
|
+
turns one segment into three. Both shapes are routine in a RAG corpus. Characters that need
|
|
86
|
+
no encoding pass through unchanged, so this is safe on every id.
|
|
87
|
+
"""
|
|
88
|
+
return quote(str(value), safe="")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
#: Statuses a retry can fix: the server is busy (429, 503) or momentarily wrong (5xx). A 4xx
|
|
92
|
+
#: other than 429 says the request itself is wrong, and sending it again changes nothing.
|
|
93
|
+
_RETRYABLE = frozenset({429, 500, 502, 503, 504})
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class UploadFailure:
|
|
98
|
+
"""One batch that did not go in, after every retry it was worth."""
|
|
99
|
+
|
|
100
|
+
ids: list[str]
|
|
101
|
+
error: str
|
|
102
|
+
status: Optional[int] = None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class UploadResult:
|
|
107
|
+
"""What `upload` did. `failed` is empty when everything went in."""
|
|
108
|
+
|
|
109
|
+
uploaded: int = 0
|
|
110
|
+
batches: int = 0
|
|
111
|
+
retries: int = 0
|
|
112
|
+
failed: list[UploadFailure] = field(default_factory=list)
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def ok(self) -> bool:
|
|
116
|
+
return not self.failed
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _payload_records(records: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
120
|
+
"""`metadata` renamed to `payload`, copying only the records that need it."""
|
|
121
|
+
out = []
|
|
122
|
+
for record in records:
|
|
123
|
+
if "metadata" in record and "payload" not in record:
|
|
124
|
+
record = dict(record)
|
|
125
|
+
record["payload"] = record.pop("metadata")
|
|
126
|
+
out.append(record)
|
|
127
|
+
return out
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _batches(records: Iterable[dict[str, Any]], size: int) -> Iterator[list[dict[str, Any]]]:
|
|
131
|
+
"""Chunks of `size`, drawn lazily — so a generator of ten million records is never held."""
|
|
132
|
+
batch: list[dict[str, Any]] = []
|
|
133
|
+
for record in records:
|
|
134
|
+
batch.append(record)
|
|
135
|
+
if len(batch) == size:
|
|
136
|
+
yield batch
|
|
137
|
+
batch = []
|
|
138
|
+
if batch:
|
|
139
|
+
yield batch
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class DXMTClient:
|
|
143
|
+
"""HTTP client for DXMT VectorDB API.
|
|
144
|
+
|
|
145
|
+
Handles all communication with DXMT server via REST API.
|
|
146
|
+
Supports both sync and async operations.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
host: DXMT server hostname. Default is 'localhost'.
|
|
150
|
+
port: DXMT server port. Default is 9999.
|
|
151
|
+
ssl: Whether to use HTTPS. Default is False.
|
|
152
|
+
api_key: The server's API key, when it was started with ``DXMT_API_KEY``. Sent as the
|
|
153
|
+
``api-key`` header on every request. Default is None: the server's key is off by
|
|
154
|
+
default, and without one the client sends nothing.
|
|
155
|
+
headers: Additional HTTP headers.
|
|
156
|
+
timeout: Request timeout in seconds. Default is 30.0.
|
|
157
|
+
**kwargs: Additional httpx client parameters.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
#: Hosts where sending a key over plain HTTP does not cross a network.
|
|
161
|
+
_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
|
|
162
|
+
|
|
163
|
+
def __init__(
|
|
164
|
+
self,
|
|
165
|
+
host: str = "localhost",
|
|
166
|
+
port: int = 9999,
|
|
167
|
+
*,
|
|
168
|
+
ssl: bool = False,
|
|
169
|
+
api_key: Optional[str] = None,
|
|
170
|
+
headers: Optional[dict[str, str]] = None,
|
|
171
|
+
timeout: float = 30.0,
|
|
172
|
+
**kwargs: Any,
|
|
173
|
+
) -> None:
|
|
174
|
+
"""Initialize DXMT client."""
|
|
175
|
+
protocol = "https" if ssl else "http"
|
|
176
|
+
self.base_url = f"{protocol}://{host}:{port}/api/v1"
|
|
177
|
+
# Copied, so adding the key does not write into a dict the caller still holds.
|
|
178
|
+
self._headers = dict(headers or {})
|
|
179
|
+
if api_key:
|
|
180
|
+
# The explicit argument wins over an `api-key` the caller also put in `headers`, the
|
|
181
|
+
# way qdrant-client resolves the same collision.
|
|
182
|
+
self._headers["api-key"] = api_key
|
|
183
|
+
if not ssl and host not in self._LOCAL_HOSTS:
|
|
184
|
+
# A key over plain HTTP reaches every hop between here and the server, which is
|
|
185
|
+
# worse than sending none. qdrant-client switches to HTTPS when given a key; that
|
|
186
|
+
# would break a server that has no TLS in front of it, so this only says so.
|
|
187
|
+
warnings.warn(
|
|
188
|
+
f"sending an API key to {host} over plain HTTP; pass ssl=True, or put TLS "
|
|
189
|
+
"in front of the server",
|
|
190
|
+
stacklevel=2,
|
|
191
|
+
)
|
|
192
|
+
self._timeout = timeout
|
|
193
|
+
self._client_kwargs = kwargs
|
|
194
|
+
|
|
195
|
+
# Create sync client
|
|
196
|
+
self._client = httpx.Client(
|
|
197
|
+
base_url=self.base_url,
|
|
198
|
+
headers=self._headers,
|
|
199
|
+
timeout=self._timeout,
|
|
200
|
+
**self._client_kwargs,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
# Async client is created lazily
|
|
204
|
+
self._async_client: Optional[httpx.AsyncClient] = None
|
|
205
|
+
|
|
206
|
+
def __enter__(self) -> DXMTClient:
|
|
207
|
+
"""Context manager entry."""
|
|
208
|
+
return self
|
|
209
|
+
|
|
210
|
+
def __exit__(self, *args: Any) -> None:
|
|
211
|
+
"""Context manager exit."""
|
|
212
|
+
self.close()
|
|
213
|
+
|
|
214
|
+
async def __aenter__(self) -> DXMTClient:
|
|
215
|
+
"""Async context manager entry."""
|
|
216
|
+
return self
|
|
217
|
+
|
|
218
|
+
async def __aexit__(self, *args: Any) -> None:
|
|
219
|
+
"""Async context manager exit."""
|
|
220
|
+
await self.aclose()
|
|
221
|
+
|
|
222
|
+
def close(self) -> None:
|
|
223
|
+
"""Close the HTTP client."""
|
|
224
|
+
self._client.close()
|
|
225
|
+
|
|
226
|
+
async def aclose(self) -> None:
|
|
227
|
+
"""Async close the HTTP clients."""
|
|
228
|
+
self._client.close()
|
|
229
|
+
if self._async_client:
|
|
230
|
+
await self._async_client.aclose()
|
|
231
|
+
|
|
232
|
+
async def ahealth_check(self) -> dict[str, Any]:
|
|
233
|
+
"""Async variant of :meth:`health_check`."""
|
|
234
|
+
try:
|
|
235
|
+
response = await self.async_client.get("/health")
|
|
236
|
+
response.raise_for_status()
|
|
237
|
+
data = self._response_data(response.json())
|
|
238
|
+
return data if isinstance(data, dict) else {}
|
|
239
|
+
except httpx.HTTPStatusError as e:
|
|
240
|
+
raise DXMTConnectionError(f"Health check failed: {e.response.text}") from e
|
|
241
|
+
except httpx.RequestError as e:
|
|
242
|
+
raise DXMTConnectionError(f"Connection error during health check: {e}") from e
|
|
243
|
+
|
|
244
|
+
async def ahealth_metrics(self) -> dict[str, Any]:
|
|
245
|
+
"""Async variant of :meth:`health_metrics`."""
|
|
246
|
+
try:
|
|
247
|
+
response = await self.async_client.get("/health/metrics")
|
|
248
|
+
response.raise_for_status()
|
|
249
|
+
data = self._response_data(response.json())
|
|
250
|
+
return data if isinstance(data, dict) else {}
|
|
251
|
+
except httpx.HTTPStatusError as e:
|
|
252
|
+
raise DXMTConnectionError(f"Health metrics failed: {e.response.text}") from e
|
|
253
|
+
except httpx.RequestError as e:
|
|
254
|
+
raise DXMTConnectionError(f"Connection error during health metrics: {e}") from e
|
|
255
|
+
|
|
256
|
+
def health_check(self) -> dict[str, Any]:
|
|
257
|
+
"""Return the server's readiness and durability health snapshot."""
|
|
258
|
+
try:
|
|
259
|
+
response = self._client.get("/health")
|
|
260
|
+
response.raise_for_status()
|
|
261
|
+
data = self._response_data(response.json())
|
|
262
|
+
return data if isinstance(data, dict) else {}
|
|
263
|
+
except httpx.HTTPStatusError as e:
|
|
264
|
+
raise DXMTConnectionError(f"Health check failed: {e.response.text}") from e
|
|
265
|
+
except httpx.RequestError as e:
|
|
266
|
+
raise DXMTConnectionError(f"Connection error during health check: {e}") from e
|
|
267
|
+
|
|
268
|
+
def health_metrics(self) -> dict[str, Any]:
|
|
269
|
+
"""Return detailed server and storage health metrics."""
|
|
270
|
+
try:
|
|
271
|
+
response = self._client.get("/health/metrics")
|
|
272
|
+
response.raise_for_status()
|
|
273
|
+
data = self._response_data(response.json())
|
|
274
|
+
return data if isinstance(data, dict) else {}
|
|
275
|
+
except httpx.HTTPStatusError as e:
|
|
276
|
+
raise DXMTConnectionError(f"Health metrics failed: {e.response.text}") from e
|
|
277
|
+
except httpx.RequestError as e:
|
|
278
|
+
raise DXMTConnectionError(f"Connection error during health metrics: {e}") from e
|
|
279
|
+
|
|
280
|
+
@property
|
|
281
|
+
def async_client(self) -> httpx.AsyncClient:
|
|
282
|
+
"""Get or create async client."""
|
|
283
|
+
if self._async_client is None:
|
|
284
|
+
self._async_client = httpx.AsyncClient(
|
|
285
|
+
base_url=self.base_url,
|
|
286
|
+
headers=self._headers,
|
|
287
|
+
timeout=self._timeout,
|
|
288
|
+
**self._client_kwargs,
|
|
289
|
+
)
|
|
290
|
+
return self._async_client
|
|
291
|
+
|
|
292
|
+
@staticmethod
|
|
293
|
+
def _response_data(response: Any) -> Any:
|
|
294
|
+
"""Return the payload from a DXMT response envelope.
|
|
295
|
+
|
|
296
|
+
The API normally returns ``{"success": true, "data": ...}``, while
|
|
297
|
+
a couple of compatibility endpoints return the payload directly. SDK
|
|
298
|
+
methods should not need to know which form a route used.
|
|
299
|
+
"""
|
|
300
|
+
if (
|
|
301
|
+
isinstance(response, dict)
|
|
302
|
+
and "data" in response
|
|
303
|
+
and (response.get("success") is True or set(response) == {"data"})
|
|
304
|
+
):
|
|
305
|
+
return response.get("data")
|
|
306
|
+
return response
|
|
307
|
+
|
|
308
|
+
@classmethod
|
|
309
|
+
def _normalize_record(cls, record: Any) -> dict[str, Any]:
|
|
310
|
+
"""Normalize Rust record/search-hit fields to the public SDK shape."""
|
|
311
|
+
if not isinstance(record, dict):
|
|
312
|
+
return {}
|
|
313
|
+
normalized = dict(record)
|
|
314
|
+
|
|
315
|
+
# ``Record`` uses payload_json on the wire to avoid bincode issues;
|
|
316
|
+
# search hits already expose a decoded payload object.
|
|
317
|
+
payload = normalized.get("payload")
|
|
318
|
+
if payload is None and normalized.get("payload_json"):
|
|
319
|
+
try:
|
|
320
|
+
payload = json.loads(normalized["payload_json"])
|
|
321
|
+
except (json.JSONDecodeError, TypeError):
|
|
322
|
+
payload = {}
|
|
323
|
+
normalized["payload"] = payload
|
|
324
|
+
|
|
325
|
+
if "payload" in normalized and "metadata" not in normalized:
|
|
326
|
+
normalized["metadata"] = normalized["payload"]
|
|
327
|
+
return normalized
|
|
328
|
+
|
|
329
|
+
@classmethod
|
|
330
|
+
def _normalize_hits(cls, response: Any) -> list[dict[str, Any]]:
|
|
331
|
+
"""Extract and normalize a search response's hits."""
|
|
332
|
+
data = cls._response_data(response)
|
|
333
|
+
if not isinstance(data, dict):
|
|
334
|
+
return []
|
|
335
|
+
hits = data.get("hits", [])
|
|
336
|
+
if not isinstance(hits, list):
|
|
337
|
+
return []
|
|
338
|
+
return [cls._normalize_record(hit) for hit in hits]
|
|
339
|
+
|
|
340
|
+
# Table Management
|
|
341
|
+
|
|
342
|
+
def create_table(
|
|
343
|
+
self,
|
|
344
|
+
name: str,
|
|
345
|
+
dimension: Optional[int] = None,
|
|
346
|
+
distance_metric: str = "cosine",
|
|
347
|
+
index_type: str = "hnsw",
|
|
348
|
+
description: Optional[str] = None,
|
|
349
|
+
enable_full_text_search: bool = False,
|
|
350
|
+
text_fields: Optional[list[dict[str, Any]]] = None,
|
|
351
|
+
memory_optimization: Optional[bool] = None,
|
|
352
|
+
hnsw: Optional[dict[str, int]] = None,
|
|
353
|
+
quantization: Optional[dict[str, int]] = None,
|
|
354
|
+
payload_index: Optional[dict[str, list[str]]] = None,
|
|
355
|
+
embedding: Optional[dict[str, Any]] = None,
|
|
356
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
357
|
+
) -> dict[str, Any]:
|
|
358
|
+
"""Create a new table (collection).
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
name: Table name.
|
|
362
|
+
dimension: Vector dimension, fixed for the life of the table. Optional only
|
|
363
|
+
when `embedding` is given: the server then asks the model how wide its vectors
|
|
364
|
+
are, which is the number most often got wrong (1536 for text-embedding-3-small,
|
|
365
|
+
1024 for bge-m3, 384 for all-MiniLM-L6-v2) and whose cost is a table that
|
|
366
|
+
accepts no writes.
|
|
367
|
+
distance_metric: Distance metric (cosine, euclidean, manhattan, dot).
|
|
368
|
+
index_type: Index type (hnsw, ivf, ivf_pq, lsh, brute_force).
|
|
369
|
+
description: Optional table description.
|
|
370
|
+
hnsw: Graph parameters for this table: `m`, `ef_construction`, `ef_search`. They
|
|
371
|
+
decide the graph and cannot be recovered afterwards -- searching harder does
|
|
372
|
+
not add edges the build never made -- and they are per table because one
|
|
373
|
+
server holds a small lookup table beside a large corpus.
|
|
374
|
+
quantization: `num_subvectors`, `num_centroids`, `quantize_at_rows`,
|
|
375
|
+
`fast_scan`. Setting `quantize_at_rows` above the table's eventual size
|
|
376
|
+
keeps it exact; `fast_scan: False` leaves out the 4-bit block-scan codes
|
|
377
|
+
that speed up filtered search over large match sets (default on).
|
|
378
|
+
payload_index: Which payload fields to index for filtering: `{"only": [...]}` or
|
|
379
|
+
`{"except": [...]}`. Every field is indexed by default, which is what makes a
|
|
380
|
+
filter fast without declaring anything; a field nobody filters on pays memory
|
|
381
|
+
and rebuild time for nothing. A filter on an excluded field is refused, not
|
|
382
|
+
answered empty.
|
|
383
|
+
embedding: Server-side embedding for this table, e.g.
|
|
384
|
+
`{"model": "text-embedding-3-small"}` or
|
|
385
|
+
`{"model": "BAAI/bge-m3", "base_url": "http://10.0.0.7:8080/v1"}`. Records may
|
|
386
|
+
then be inserted with `text` in place of `vector`, and `search(text=...)`
|
|
387
|
+
embeds the query with this same model. The key is read from the server's
|
|
388
|
+
environment (`api_key_env`, default `OPENAI_API_KEY`), never sent from here.
|
|
389
|
+
enable_full_text_search: Enable full-text search.
|
|
390
|
+
text_fields: Text field configurations.
|
|
391
|
+
memory_optimization: Enable memory optimization (optional).
|
|
392
|
+
- None: Use global config (DXMT_MEMORY_OPTIMIZATION)
|
|
393
|
+
- True: Enable memory optimization (Phase 4 Hybrid), saves 85-90% memory, 87-95% recall
|
|
394
|
+
- False: High accuracy mode (HNSW/Phase 3), maintains 95-99% recall
|
|
395
|
+
|
|
396
|
+
Returns:
|
|
397
|
+
Response data from server.
|
|
398
|
+
|
|
399
|
+
Raises:
|
|
400
|
+
DXMTTableError: If table creation fails.
|
|
401
|
+
"""
|
|
402
|
+
payload = {
|
|
403
|
+
"name": name,
|
|
404
|
+
"distance_metric": distance_metric,
|
|
405
|
+
"index_type": index_type,
|
|
406
|
+
"description": description,
|
|
407
|
+
"enable_full_text_search": enable_full_text_search,
|
|
408
|
+
"text_fields": text_fields,
|
|
409
|
+
"memory_optimization": memory_optimization,
|
|
410
|
+
}
|
|
411
|
+
# Omitted rather than sent as null: the server refuses fields it does not have, and
|
|
412
|
+
# would have to decide what a null means where None already means "the default".
|
|
413
|
+
if hnsw is not None:
|
|
414
|
+
payload["hnsw"] = hnsw
|
|
415
|
+
if quantization is not None:
|
|
416
|
+
payload["quantization"] = quantization
|
|
417
|
+
# Omitted rather than null when the model is to supply it: the server reads its
|
|
418
|
+
# absence as "ask the model how wide its vectors are", and a null would be a value.
|
|
419
|
+
if dimension is not None:
|
|
420
|
+
payload["dimension"] = dimension
|
|
421
|
+
if payload_index is not None:
|
|
422
|
+
payload["payload_index"] = payload_index
|
|
423
|
+
# The model the server embeds text with: `{"model": "text-embedding-3-small"}`, plus
|
|
424
|
+
# an optional `base_url` for anything else speaking the OpenAI /embeddings protocol.
|
|
425
|
+
# With it, records and searches may carry `text` instead of a vector. The API key is
|
|
426
|
+
# not in here -- `api_key_env` names an environment variable of the *server*.
|
|
427
|
+
if embedding is not None:
|
|
428
|
+
payload["embedding"] = embedding
|
|
429
|
+
if rerank is not None:
|
|
430
|
+
# A cross-encoder the server calls when a search asks for `rerank=...`:
|
|
431
|
+
# `{"model": ..., "base_url": ..., "format": "cohere" | "tei", "api_key_env": ...}`.
|
|
432
|
+
# The key is read from the server's environment, never sent from here.
|
|
433
|
+
payload["rerank"] = rerank
|
|
434
|
+
try:
|
|
435
|
+
response = self._client.post("/tables", json=payload)
|
|
436
|
+
response.raise_for_status()
|
|
437
|
+
return response.json()
|
|
438
|
+
except httpx.HTTPStatusError as e:
|
|
439
|
+
msg = f"Failed to create table {name}: {e.response.text}"
|
|
440
|
+
raise DXMTTableError(msg) from e
|
|
441
|
+
except httpx.RequestError as e:
|
|
442
|
+
msg = f"Connection error creating table {name}: {e}"
|
|
443
|
+
raise DXMTConnectionError(msg) from e
|
|
444
|
+
|
|
445
|
+
async def acreate_table(
|
|
446
|
+
self,
|
|
447
|
+
name: str,
|
|
448
|
+
dimension: Optional[int] = None,
|
|
449
|
+
distance_metric: str = "cosine",
|
|
450
|
+
index_type: str = "hnsw",
|
|
451
|
+
description: Optional[str] = None,
|
|
452
|
+
enable_full_text_search: bool = False,
|
|
453
|
+
text_fields: Optional[list[dict[str, Any]]] = None,
|
|
454
|
+
memory_optimization: Optional[bool] = None,
|
|
455
|
+
hnsw: Optional[dict[str, int]] = None,
|
|
456
|
+
quantization: Optional[dict[str, int]] = None,
|
|
457
|
+
payload_index: Optional[dict[str, list[str]]] = None,
|
|
458
|
+
embedding: Optional[dict[str, Any]] = None,
|
|
459
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
460
|
+
) -> dict[str, Any]:
|
|
461
|
+
"""Async create a new table. See `create_table` for what each argument does."""
|
|
462
|
+
payload = {
|
|
463
|
+
"name": name,
|
|
464
|
+
"distance_metric": distance_metric,
|
|
465
|
+
"index_type": index_type,
|
|
466
|
+
"description": description,
|
|
467
|
+
"enable_full_text_search": enable_full_text_search,
|
|
468
|
+
"text_fields": text_fields,
|
|
469
|
+
"memory_optimization": memory_optimization,
|
|
470
|
+
}
|
|
471
|
+
if hnsw is not None:
|
|
472
|
+
payload["hnsw"] = hnsw
|
|
473
|
+
if quantization is not None:
|
|
474
|
+
payload["quantization"] = quantization
|
|
475
|
+
# Omitted rather than null when the model is to supply it: the server reads its
|
|
476
|
+
# absence as "ask the model how wide its vectors are", and a null would be a value.
|
|
477
|
+
if dimension is not None:
|
|
478
|
+
payload["dimension"] = dimension
|
|
479
|
+
if payload_index is not None:
|
|
480
|
+
payload["payload_index"] = payload_index
|
|
481
|
+
# The model the server embeds text with: `{"model": "text-embedding-3-small"}`, plus
|
|
482
|
+
# an optional `base_url` for anything else speaking the OpenAI /embeddings protocol.
|
|
483
|
+
# With it, records and searches may carry `text` instead of a vector. The API key is
|
|
484
|
+
# not in here -- `api_key_env` names an environment variable of the *server*.
|
|
485
|
+
if embedding is not None:
|
|
486
|
+
payload["embedding"] = embedding
|
|
487
|
+
if rerank is not None:
|
|
488
|
+
# A cross-encoder the server calls when a search asks for `rerank=...`:
|
|
489
|
+
# `{"model": ..., "base_url": ..., "format": "cohere" | "tei", "api_key_env": ...}`.
|
|
490
|
+
# The key is read from the server's environment, never sent from here.
|
|
491
|
+
payload["rerank"] = rerank
|
|
492
|
+
try:
|
|
493
|
+
response = await self.async_client.post("/tables", json=payload)
|
|
494
|
+
response.raise_for_status()
|
|
495
|
+
return response.json()
|
|
496
|
+
except httpx.HTTPStatusError as e:
|
|
497
|
+
msg = f"Failed to create table {name}: {e.response.text}"
|
|
498
|
+
raise DXMTTableError(msg) from e
|
|
499
|
+
except httpx.RequestError as e:
|
|
500
|
+
msg = f"Connection error creating table {name}: {e}"
|
|
501
|
+
raise DXMTConnectionError(msg) from e
|
|
502
|
+
|
|
503
|
+
def get_table(self, name: str) -> dict[str, Any]:
|
|
504
|
+
"""Get table information.
|
|
505
|
+
|
|
506
|
+
Args:
|
|
507
|
+
name: Table name.
|
|
508
|
+
|
|
509
|
+
Returns:
|
|
510
|
+
Table information.
|
|
511
|
+
|
|
512
|
+
Raises:
|
|
513
|
+
DXMTTableError: If table not found or request fails.
|
|
514
|
+
"""
|
|
515
|
+
try:
|
|
516
|
+
response = self._client.get(f"/tables/{name}")
|
|
517
|
+
response.raise_for_status()
|
|
518
|
+
data = self._response_data(response.json())
|
|
519
|
+
# API returns {"data": {"table": {...}, "record_count": N}}
|
|
520
|
+
if isinstance(data, dict) and isinstance(data.get("table"), dict):
|
|
521
|
+
table = dict(data["table"])
|
|
522
|
+
if "record_count" in data:
|
|
523
|
+
table["record_count"] = data["record_count"]
|
|
524
|
+
return table
|
|
525
|
+
return data if isinstance(data, dict) else {}
|
|
526
|
+
except httpx.HTTPStatusError as e:
|
|
527
|
+
msg = f"Failed to get table {name}: {e.response.text}"
|
|
528
|
+
raise DXMTTableError(msg) from e
|
|
529
|
+
except httpx.RequestError as e:
|
|
530
|
+
msg = f"Connection error getting table {name}: {e}"
|
|
531
|
+
raise DXMTConnectionError(msg) from e
|
|
532
|
+
|
|
533
|
+
async def aget_table(self, name: str) -> dict[str, Any]:
|
|
534
|
+
"""Async get table information."""
|
|
535
|
+
try:
|
|
536
|
+
response = await self.async_client.get(f"/tables/{name}")
|
|
537
|
+
response.raise_for_status()
|
|
538
|
+
data = self._response_data(response.json())
|
|
539
|
+
# API returns {"data": {"table": {...}, "record_count": N}}
|
|
540
|
+
if isinstance(data, dict) and isinstance(data.get("table"), dict):
|
|
541
|
+
table = dict(data["table"])
|
|
542
|
+
if "record_count" in data:
|
|
543
|
+
table["record_count"] = data["record_count"]
|
|
544
|
+
return table
|
|
545
|
+
return data if isinstance(data, dict) else {}
|
|
546
|
+
except httpx.HTTPStatusError as e:
|
|
547
|
+
msg = f"Failed to get table {name}: {e.response.text}"
|
|
548
|
+
raise DXMTTableError(msg) from e
|
|
549
|
+
except httpx.RequestError as e:
|
|
550
|
+
msg = f"Connection error getting table {name}: {e}"
|
|
551
|
+
raise DXMTConnectionError(msg) from e
|
|
552
|
+
|
|
553
|
+
def delete_table(self, name: str) -> dict[str, Any]:
|
|
554
|
+
"""Delete a table.
|
|
555
|
+
|
|
556
|
+
Args:
|
|
557
|
+
name: Table name.
|
|
558
|
+
|
|
559
|
+
Returns:
|
|
560
|
+
Response data from server.
|
|
561
|
+
|
|
562
|
+
Raises:
|
|
563
|
+
DXMTTableError: If deletion fails.
|
|
564
|
+
"""
|
|
565
|
+
try:
|
|
566
|
+
response = self._client.delete(f"/tables/{name}")
|
|
567
|
+
response.raise_for_status()
|
|
568
|
+
return response.json()
|
|
569
|
+
except httpx.HTTPStatusError as e:
|
|
570
|
+
msg = f"Failed to delete table {name}: {e.response.text}"
|
|
571
|
+
raise DXMTTableError(msg) from e
|
|
572
|
+
except httpx.RequestError as e:
|
|
573
|
+
msg = f"Connection error deleting table {name}: {e}"
|
|
574
|
+
raise DXMTConnectionError(msg) from e
|
|
575
|
+
|
|
576
|
+
async def adelete_table(self, name: str) -> dict[str, Any]:
|
|
577
|
+
"""Async delete a table."""
|
|
578
|
+
try:
|
|
579
|
+
response = await self.async_client.delete(f"/tables/{name}")
|
|
580
|
+
response.raise_for_status()
|
|
581
|
+
return response.json()
|
|
582
|
+
except httpx.HTTPStatusError as e:
|
|
583
|
+
msg = f"Failed to delete table {name}: {e.response.text}"
|
|
584
|
+
raise DXMTTableError(msg) from e
|
|
585
|
+
except httpx.RequestError as e:
|
|
586
|
+
msg = f"Connection error deleting table {name}: {e}"
|
|
587
|
+
raise DXMTConnectionError(msg) from e
|
|
588
|
+
|
|
589
|
+
def list_tables(self) -> list[dict[str, Any]]:
|
|
590
|
+
"""List all tables.
|
|
591
|
+
|
|
592
|
+
Returns:
|
|
593
|
+
List of table information dicts.
|
|
594
|
+
|
|
595
|
+
Raises:
|
|
596
|
+
DXMTTableError: If listing fails.
|
|
597
|
+
"""
|
|
598
|
+
try:
|
|
599
|
+
response = self._client.get("/tables")
|
|
600
|
+
response.raise_for_status()
|
|
601
|
+
data = self._response_data(response.json())
|
|
602
|
+
# API returns {"data": {"tables": [...], "total": N, ...}}
|
|
603
|
+
return data.get("tables", []) if isinstance(data, dict) else []
|
|
604
|
+
except httpx.HTTPStatusError as e:
|
|
605
|
+
msg = f"Failed to list tables: {e.response.text}"
|
|
606
|
+
raise DXMTTableError(msg) from e
|
|
607
|
+
except httpx.RequestError as e:
|
|
608
|
+
msg = f"Connection error listing tables: {e}"
|
|
609
|
+
raise DXMTConnectionError(msg) from e
|
|
610
|
+
|
|
611
|
+
async def alist_tables(self) -> list[dict[str, Any]]:
|
|
612
|
+
"""Async list all tables."""
|
|
613
|
+
try:
|
|
614
|
+
response = await self.async_client.get("/tables")
|
|
615
|
+
response.raise_for_status()
|
|
616
|
+
data = self._response_data(response.json())
|
|
617
|
+
# API returns {"data": {"tables": [...], "total": N, ...}}
|
|
618
|
+
return data.get("tables", []) if isinstance(data, dict) else []
|
|
619
|
+
except httpx.HTTPStatusError as e:
|
|
620
|
+
msg = f"Failed to list tables: {e.response.text}"
|
|
621
|
+
raise DXMTTableError(msg) from e
|
|
622
|
+
except httpx.RequestError as e:
|
|
623
|
+
msg = f"Connection error listing tables: {e}"
|
|
624
|
+
raise DXMTConnectionError(msg) from e
|
|
625
|
+
|
|
626
|
+
# Document Operations
|
|
627
|
+
|
|
628
|
+
def scroll(
|
|
629
|
+
self,
|
|
630
|
+
table_name: str,
|
|
631
|
+
*,
|
|
632
|
+
filter: Optional[Filter] = None,
|
|
633
|
+
batch_size: int = 100,
|
|
634
|
+
with_vector: bool = False,
|
|
635
|
+
) -> Iterator[dict[str, Any]]:
|
|
636
|
+
"""Every record of a table, newest first, fetched a page at a time.
|
|
637
|
+
|
|
638
|
+
Pages by cursor, not offset: every record present for the whole scroll comes back
|
|
639
|
+
exactly once however the table is written to meanwhile, where an offset shifts with each
|
|
640
|
+
insert and repeats or skips records. For exporting a table, re-indexing it elsewhere, or
|
|
641
|
+
anything else that has to see all of it.
|
|
642
|
+
|
|
643
|
+
Args:
|
|
644
|
+
table_name: Name of the table.
|
|
645
|
+
filter: Only records matching this filter.
|
|
646
|
+
batch_size: Records per request, at most 1,000.
|
|
647
|
+
with_vector: Include each record's vector.
|
|
648
|
+
|
|
649
|
+
Yields:
|
|
650
|
+
Records, normalised the way `list_records` returns them.
|
|
651
|
+
|
|
652
|
+
Raises:
|
|
653
|
+
DXMTRecordError: If a page fails, or the server does not page by cursor.
|
|
654
|
+
DXMTConnectionError: On a connection error.
|
|
655
|
+
"""
|
|
656
|
+
cursor: Optional[str] = None
|
|
657
|
+
while True:
|
|
658
|
+
body = self._scroll_body(filter, batch_size, with_vector, cursor)
|
|
659
|
+
try:
|
|
660
|
+
response = self._client.post(f"/tables/{_seg(table_name)}/records/list", json=body)
|
|
661
|
+
response.raise_for_status()
|
|
662
|
+
except httpx.HTTPStatusError as e:
|
|
663
|
+
raise DXMTRecordError(f"Failed to scroll '{table_name}': {e.response.text}") from e
|
|
664
|
+
except httpx.RequestError as e:
|
|
665
|
+
raise DXMTConnectionError(f"Connection error while scrolling: {e}") from e
|
|
666
|
+
records, cursor = self._scroll_page(response.json(), batch_size)
|
|
667
|
+
yield from records
|
|
668
|
+
if cursor is None:
|
|
669
|
+
return
|
|
670
|
+
|
|
671
|
+
async def ascroll(
|
|
672
|
+
self,
|
|
673
|
+
table_name: str,
|
|
674
|
+
*,
|
|
675
|
+
filter: Optional[Filter] = None,
|
|
676
|
+
batch_size: int = 100,
|
|
677
|
+
with_vector: bool = False,
|
|
678
|
+
) -> AsyncIterator[dict[str, Any]]:
|
|
679
|
+
"""Async version of :meth:`scroll`."""
|
|
680
|
+
cursor: Optional[str] = None
|
|
681
|
+
while True:
|
|
682
|
+
body = self._scroll_body(filter, batch_size, with_vector, cursor)
|
|
683
|
+
try:
|
|
684
|
+
response = await self.async_client.post(f"/tables/{_seg(table_name)}/records/list", json=body)
|
|
685
|
+
response.raise_for_status()
|
|
686
|
+
except httpx.HTTPStatusError as e:
|
|
687
|
+
raise DXMTRecordError(f"Failed to scroll '{table_name}': {e.response.text}") from e
|
|
688
|
+
except httpx.RequestError as e:
|
|
689
|
+
raise DXMTConnectionError(f"Connection error while scrolling: {e}") from e
|
|
690
|
+
records, cursor = self._scroll_page(response.json(), batch_size)
|
|
691
|
+
for record in records:
|
|
692
|
+
yield record
|
|
693
|
+
if cursor is None:
|
|
694
|
+
return
|
|
695
|
+
|
|
696
|
+
@staticmethod
|
|
697
|
+
def _scroll_body(filter, batch_size, with_vector, cursor) -> dict[str, Any]:
|
|
698
|
+
body: dict[str, Any] = {"limit": batch_size, "with_vector": with_vector}
|
|
699
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
700
|
+
if converted is not None:
|
|
701
|
+
body["filter"] = converted
|
|
702
|
+
if cursor is not None:
|
|
703
|
+
body["cursor"] = cursor
|
|
704
|
+
return body
|
|
705
|
+
|
|
706
|
+
def _scroll_page(self, raw: Any, batch_size: int) -> tuple[list[dict[str, Any]], Optional[str]]:
|
|
707
|
+
data = self._response_data(raw)
|
|
708
|
+
records = data.get("records") or []
|
|
709
|
+
# A server from before cursors answers without the field. Taking that as "no next page"
|
|
710
|
+
# would end every scroll after one page, silently; a full page without it is an error.
|
|
711
|
+
if "next_cursor" not in data and len(records) >= batch_size:
|
|
712
|
+
raise DXMTRecordError("this server does not page by cursor; upgrade it to scroll")
|
|
713
|
+
return [self._normalize_record(r) for r in records], data.get("next_cursor")
|
|
714
|
+
|
|
715
|
+
def upload(
|
|
716
|
+
self,
|
|
717
|
+
table_name: str,
|
|
718
|
+
records: Iterable[dict[str, Any]],
|
|
719
|
+
*,
|
|
720
|
+
batch_size: int = 256,
|
|
721
|
+
parallel: int = 1,
|
|
722
|
+
max_retries: int = 3,
|
|
723
|
+
backoff: float = 0.5,
|
|
724
|
+
) -> UploadResult:
|
|
725
|
+
"""Insert many records, in batches, retrying what a retry can fix.
|
|
726
|
+
|
|
727
|
+
`records` may be any iterable, a generator included; it is drawn a batch at a time, so a
|
|
728
|
+
load larger than memory never has to be held at once. A batch that fails for a reason a
|
|
729
|
+
retry can fix — a dropped connection, a timeout, 429, a 5xx — is sent again after an
|
|
730
|
+
exponential backoff; one that fails because the request is wrong (a dimension mismatch,
|
|
731
|
+
a missing table) is not, and is reported. Inserts are upserts, so a retried batch that
|
|
732
|
+
had in fact landed does not duplicate anything.
|
|
733
|
+
|
|
734
|
+
It does not raise for a failed batch: every batch is attempted, and `result.failed`
|
|
735
|
+
says which ids did not go in and why, so one bad record does not abandon the rest.
|
|
736
|
+
|
|
737
|
+
Args:
|
|
738
|
+
table_name: Name of the table.
|
|
739
|
+
records: Records with `id`, `vector` and optional `payload` (or `metadata`).
|
|
740
|
+
batch_size: Records per request.
|
|
741
|
+
parallel: Requests in flight at once.
|
|
742
|
+
max_retries: Further attempts per batch after the first.
|
|
743
|
+
backoff: Seconds before the first retry, doubling after each.
|
|
744
|
+
|
|
745
|
+
Returns:
|
|
746
|
+
How many records went in, how many batches and retries it took, and what failed.
|
|
747
|
+
"""
|
|
748
|
+
result = UploadResult()
|
|
749
|
+
path = f"/tables/{_seg(table_name)}/records"
|
|
750
|
+
|
|
751
|
+
def send(batch: list[dict[str, Any]]) -> tuple[int, Optional[UploadFailure], int]:
|
|
752
|
+
body = _encode_json({"records": _payload_records(batch)})
|
|
753
|
+
retries = 0
|
|
754
|
+
while True:
|
|
755
|
+
status: Optional[int] = None
|
|
756
|
+
try:
|
|
757
|
+
response = self._client.post(path, content=body, headers=_JSON_HEADERS)
|
|
758
|
+
if response.is_success:
|
|
759
|
+
return len(batch), None, retries
|
|
760
|
+
status, error = response.status_code, response.text
|
|
761
|
+
except httpx.RequestError as e:
|
|
762
|
+
error = f"connection error: {e}"
|
|
763
|
+
retryable = status is None or status in _RETRYABLE
|
|
764
|
+
if not retryable or retries >= max_retries:
|
|
765
|
+
ids = [str(r.get("id")) for r in batch]
|
|
766
|
+
return 0, UploadFailure(ids=ids, error=error[:500], status=status), retries
|
|
767
|
+
time.sleep(backoff * (2 ** retries))
|
|
768
|
+
retries += 1
|
|
769
|
+
|
|
770
|
+
def record(outcome: tuple[int, Optional[UploadFailure], int]) -> None:
|
|
771
|
+
uploaded, failure, retries = outcome
|
|
772
|
+
result.uploaded += uploaded
|
|
773
|
+
result.batches += 1
|
|
774
|
+
result.retries += retries
|
|
775
|
+
if failure is not None:
|
|
776
|
+
result.failed.append(failure)
|
|
777
|
+
|
|
778
|
+
if parallel <= 1:
|
|
779
|
+
for batch in _batches(records, batch_size):
|
|
780
|
+
record(send(batch))
|
|
781
|
+
return result
|
|
782
|
+
|
|
783
|
+
# Bounded in flight, so a generator is still drawn only as fast as batches are sent.
|
|
784
|
+
with ThreadPoolExecutor(max_workers=parallel) as pool:
|
|
785
|
+
pending: deque = deque()
|
|
786
|
+
for batch in _batches(records, batch_size):
|
|
787
|
+
pending.append(pool.submit(send, batch))
|
|
788
|
+
if len(pending) >= parallel * 2:
|
|
789
|
+
record(pending.popleft().result())
|
|
790
|
+
while pending:
|
|
791
|
+
record(pending.popleft().result())
|
|
792
|
+
return result
|
|
793
|
+
|
|
794
|
+
def insert_records(
|
|
795
|
+
self,
|
|
796
|
+
table_name: str,
|
|
797
|
+
records: list[dict[str, Any]],
|
|
798
|
+
) -> dict[str, Any]:
|
|
799
|
+
"""Insert records into a table.
|
|
800
|
+
|
|
801
|
+
Args:
|
|
802
|
+
table_name: Name of the table.
|
|
803
|
+
records: List of records with id, vector, and metadata.
|
|
804
|
+
|
|
805
|
+
Returns:
|
|
806
|
+
Response data from server.
|
|
807
|
+
|
|
808
|
+
Raises:
|
|
809
|
+
DXMTError: If insertion fails.
|
|
810
|
+
"""
|
|
811
|
+
# Normalize records: map 'metadata' to 'payload' if present. Only the records that
|
|
812
|
+
# actually need remapping are copied -- this used to copy every record on every
|
|
813
|
+
# insert, which on a bulk load is a dict allocation per row for nothing.
|
|
814
|
+
normalized_records = []
|
|
815
|
+
for record in records:
|
|
816
|
+
if "metadata" in record and "payload" not in record:
|
|
817
|
+
normalized = dict(record)
|
|
818
|
+
normalized["payload"] = normalized.pop("metadata")
|
|
819
|
+
normalized_records.append(normalized)
|
|
820
|
+
else:
|
|
821
|
+
normalized_records.append(record)
|
|
822
|
+
|
|
823
|
+
payload = {"records": normalized_records}
|
|
824
|
+
try:
|
|
825
|
+
response = self._client.post(
|
|
826
|
+
f"/tables/{_seg(table_name)}/records",
|
|
827
|
+
content=_encode_json(payload),
|
|
828
|
+
headers=_JSON_HEADERS,
|
|
829
|
+
)
|
|
830
|
+
response.raise_for_status()
|
|
831
|
+
return response.json()
|
|
832
|
+
except httpx.HTTPStatusError as e:
|
|
833
|
+
msg = f"Failed to insert records: {e.response.text}"
|
|
834
|
+
raise DXMTError(msg) from e
|
|
835
|
+
except httpx.RequestError as e:
|
|
836
|
+
msg = f"Connection error inserting records: {e}"
|
|
837
|
+
raise DXMTConnectionError(msg) from e
|
|
838
|
+
|
|
839
|
+
async def ainsert_records(
|
|
840
|
+
self,
|
|
841
|
+
table_name: str,
|
|
842
|
+
records: list[dict[str, Any]],
|
|
843
|
+
) -> dict[str, Any]:
|
|
844
|
+
"""Async insert records into a table."""
|
|
845
|
+
# Normalize records: map 'metadata' to 'payload' if present
|
|
846
|
+
normalized_records = []
|
|
847
|
+
for record in records:
|
|
848
|
+
normalized = record.copy()
|
|
849
|
+
if "metadata" in normalized and "payload" not in normalized:
|
|
850
|
+
normalized["payload"] = normalized.pop("metadata")
|
|
851
|
+
normalized_records.append(normalized)
|
|
852
|
+
|
|
853
|
+
payload = {"records": normalized_records}
|
|
854
|
+
try:
|
|
855
|
+
response = await self.async_client.post(
|
|
856
|
+
f"/tables/{_seg(table_name)}/records",
|
|
857
|
+
json=payload,
|
|
858
|
+
)
|
|
859
|
+
response.raise_for_status()
|
|
860
|
+
return response.json()
|
|
861
|
+
except httpx.HTTPStatusError as e:
|
|
862
|
+
msg = f"Failed to insert records: {e.response.text}"
|
|
863
|
+
raise DXMTError(msg) from e
|
|
864
|
+
except httpx.RequestError as e:
|
|
865
|
+
msg = f"Connection error inserting records: {e}"
|
|
866
|
+
raise DXMTConnectionError(msg) from e
|
|
867
|
+
|
|
868
|
+
def update_record(
|
|
869
|
+
self,
|
|
870
|
+
table_name: str,
|
|
871
|
+
record_id: str,
|
|
872
|
+
vector: Optional[list[float]] = None,
|
|
873
|
+
payload: Optional[dict[str, Any]] = None,
|
|
874
|
+
merge_mode: str = "replace",
|
|
875
|
+
upsert: bool = False,
|
|
876
|
+
) -> dict[str, Any]:
|
|
877
|
+
"""Update a record in a table.
|
|
878
|
+
|
|
879
|
+
Args:
|
|
880
|
+
table_name: Name of the table.
|
|
881
|
+
record_id: ID of the record to update.
|
|
882
|
+
vector: New vector for the record (optional).
|
|
883
|
+
payload: New metadata/payload for the record (optional).
|
|
884
|
+
merge_mode: "replace" (default) or "merge".
|
|
885
|
+
- "replace": Replace entire payload (may lose fields)
|
|
886
|
+
- "merge": Merge with existing payload (preserves fields)
|
|
887
|
+
upsert: If True, create record if it doesn't exist. Default False.
|
|
888
|
+
|
|
889
|
+
Returns:
|
|
890
|
+
Response data from server.
|
|
891
|
+
|
|
892
|
+
Raises:
|
|
893
|
+
DXMTError: If update fails.
|
|
894
|
+
|
|
895
|
+
Examples:
|
|
896
|
+
# Replace mode (default) - replaces entire payload
|
|
897
|
+
client.update_record("docs", "doc1", payload={"views": 100})
|
|
898
|
+
# Result: payload = {"views": 100} only
|
|
899
|
+
|
|
900
|
+
# Merge mode - preserves existing fields
|
|
901
|
+
client.update_record("docs", "doc1",
|
|
902
|
+
payload={"views": 150},
|
|
903
|
+
merge_mode="merge")
|
|
904
|
+
# Result: payload = {...existing fields..., "views": 150}
|
|
905
|
+
|
|
906
|
+
# Upsert - create if not exists
|
|
907
|
+
client.update_record("docs", "new_doc",
|
|
908
|
+
vector=embedding,
|
|
909
|
+
payload={"content": "..."},
|
|
910
|
+
upsert=True)
|
|
911
|
+
"""
|
|
912
|
+
request_payload = {}
|
|
913
|
+
if vector is not None:
|
|
914
|
+
request_payload["vector"] = vector
|
|
915
|
+
if payload is not None:
|
|
916
|
+
request_payload["payload"] = payload
|
|
917
|
+
request_payload["merge_mode"] = merge_mode
|
|
918
|
+
request_payload["upsert"] = upsert
|
|
919
|
+
|
|
920
|
+
try:
|
|
921
|
+
response = self._client.put(
|
|
922
|
+
f"/tables/{_seg(table_name)}/records/{_seg(record_id)}",
|
|
923
|
+
json=request_payload,
|
|
924
|
+
)
|
|
925
|
+
response.raise_for_status()
|
|
926
|
+
return response.json()
|
|
927
|
+
except httpx.HTTPStatusError as e:
|
|
928
|
+
msg = f"Failed to update record {record_id}: {e.response.text}"
|
|
929
|
+
raise DXMTError(msg) from e
|
|
930
|
+
except httpx.RequestError as e:
|
|
931
|
+
msg = f"Connection error updating record {record_id}: {e}"
|
|
932
|
+
raise DXMTConnectionError(msg) from e
|
|
933
|
+
|
|
934
|
+
async def aupdate_record(
|
|
935
|
+
self,
|
|
936
|
+
table_name: str,
|
|
937
|
+
record_id: str,
|
|
938
|
+
vector: Optional[list[float]] = None,
|
|
939
|
+
payload: Optional[dict[str, Any]] = None,
|
|
940
|
+
merge_mode: str = "replace",
|
|
941
|
+
upsert: bool = False,
|
|
942
|
+
) -> dict[str, Any]:
|
|
943
|
+
"""Async update a record in a table.
|
|
944
|
+
|
|
945
|
+
Args:
|
|
946
|
+
table_name: Name of the table.
|
|
947
|
+
record_id: ID of the record to update.
|
|
948
|
+
vector: New vector for the record (optional).
|
|
949
|
+
payload: New metadata/payload for the record (optional).
|
|
950
|
+
|
|
951
|
+
Returns:
|
|
952
|
+
Response data from server.
|
|
953
|
+
|
|
954
|
+
Raises:
|
|
955
|
+
DXMTError: If update fails.
|
|
956
|
+
"""
|
|
957
|
+
request_payload = {}
|
|
958
|
+
if vector is not None:
|
|
959
|
+
request_payload["vector"] = vector
|
|
960
|
+
if payload is not None:
|
|
961
|
+
request_payload["payload"] = payload
|
|
962
|
+
request_payload["merge_mode"] = merge_mode
|
|
963
|
+
request_payload["upsert"] = upsert
|
|
964
|
+
|
|
965
|
+
try:
|
|
966
|
+
response = await self.async_client.put(
|
|
967
|
+
f"/tables/{_seg(table_name)}/records/{_seg(record_id)}",
|
|
968
|
+
json=request_payload,
|
|
969
|
+
)
|
|
970
|
+
response.raise_for_status()
|
|
971
|
+
return response.json()
|
|
972
|
+
except httpx.HTTPStatusError as e:
|
|
973
|
+
msg = f"Failed to update record {record_id}: {e.response.text}"
|
|
974
|
+
raise DXMTError(msg) from e
|
|
975
|
+
except httpx.RequestError as e:
|
|
976
|
+
msg = f"Connection error updating record {record_id}: {e}"
|
|
977
|
+
raise DXMTConnectionError(msg) from e
|
|
978
|
+
|
|
979
|
+
def sparse_search(
|
|
980
|
+
self,
|
|
981
|
+
table_name: str,
|
|
982
|
+
sparse_vector: dict[str, Any],
|
|
983
|
+
*,
|
|
984
|
+
limit: int = 10,
|
|
985
|
+
filter: Optional[Filter] = None,
|
|
986
|
+
with_payload: bool = True,
|
|
987
|
+
with_vector: bool = False,
|
|
988
|
+
payload_fields: Optional[list[str]] = None,
|
|
989
|
+
score_threshold: Optional[float] = None,
|
|
990
|
+
) -> list[dict[str, Any]]:
|
|
991
|
+
"""Retrieve by sparse vector alone, ranked by dot product.
|
|
992
|
+
|
|
993
|
+
Sparse vectors could be stored and indexed here long before they could be searched:
|
|
994
|
+
the only path into the sparse index was the hybrid endpoint, which refuses a body with
|
|
995
|
+
no dense vector and no text, and needs a `text-bm25` build. This needs neither.
|
|
996
|
+
|
|
997
|
+
Args:
|
|
998
|
+
table_name: Name of the table.
|
|
999
|
+
sparse_vector: `{"indices": [...], "values": [...], "dimension": n}`.
|
|
1000
|
+
limit: How many hits to return.
|
|
1001
|
+
filter: Applied to the candidates the sparse index returns.
|
|
1002
|
+
with_payload: Include payloads.
|
|
1003
|
+
with_vector: Include the dense vectors.
|
|
1004
|
+
payload_fields: Return only these payload keys.
|
|
1005
|
+
score_threshold: Floor on the dot product; higher is nearer, and it only removes.
|
|
1006
|
+
|
|
1007
|
+
Returns:
|
|
1008
|
+
The hits, best first.
|
|
1009
|
+
|
|
1010
|
+
Raises:
|
|
1011
|
+
DXMTError: If the request fails.
|
|
1012
|
+
"""
|
|
1013
|
+
body: dict[str, Any] = {
|
|
1014
|
+
"sparse_vector": sparse_vector,
|
|
1015
|
+
"limit": limit,
|
|
1016
|
+
"with_payload": with_payload,
|
|
1017
|
+
"with_vector": with_vector,
|
|
1018
|
+
}
|
|
1019
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1020
|
+
if converted is not None:
|
|
1021
|
+
body["filter"] = converted
|
|
1022
|
+
if payload_fields is not None:
|
|
1023
|
+
body["payload_fields"] = payload_fields
|
|
1024
|
+
if score_threshold is not None:
|
|
1025
|
+
body["score_threshold"] = score_threshold
|
|
1026
|
+
response = self._client.post(
|
|
1027
|
+
f"/tables/{quote(table_name, safe='')}/search/sparse", json=body
|
|
1028
|
+
)
|
|
1029
|
+
response.raise_for_status()
|
|
1030
|
+
return response.json().get("data", {}).get("hits", [])
|
|
1031
|
+
|
|
1032
|
+
@staticmethod
|
|
1033
|
+
def _sparse_body(
|
|
1034
|
+
sparse_vector: dict[str, Any],
|
|
1035
|
+
limit: int,
|
|
1036
|
+
filter: Optional[Filter],
|
|
1037
|
+
with_payload: bool,
|
|
1038
|
+
with_vector: bool,
|
|
1039
|
+
payload_fields: Optional[list[str]],
|
|
1040
|
+
score_threshold: Optional[float],
|
|
1041
|
+
) -> dict[str, Any]:
|
|
1042
|
+
"""The request body for `/search/sparse`, shared by the sync and async pair.
|
|
1043
|
+
|
|
1044
|
+
One function, because the two used to be the place where a field gets added to one
|
|
1045
|
+
call and not the other, and nothing notices until a caller on the async path finds
|
|
1046
|
+
an option that silently does nothing.
|
|
1047
|
+
"""
|
|
1048
|
+
body: dict[str, Any] = {
|
|
1049
|
+
"sparse_vector": sparse_vector,
|
|
1050
|
+
"limit": limit,
|
|
1051
|
+
"with_payload": with_payload,
|
|
1052
|
+
"with_vector": with_vector,
|
|
1053
|
+
}
|
|
1054
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1055
|
+
if converted is not None:
|
|
1056
|
+
body["filter"] = converted
|
|
1057
|
+
if payload_fields is not None:
|
|
1058
|
+
body["payload_fields"] = payload_fields
|
|
1059
|
+
if score_threshold is not None:
|
|
1060
|
+
body["score_threshold"] = score_threshold
|
|
1061
|
+
return body
|
|
1062
|
+
|
|
1063
|
+
async def asparse_search(
|
|
1064
|
+
self,
|
|
1065
|
+
table_name: str,
|
|
1066
|
+
sparse_vector: dict[str, Any],
|
|
1067
|
+
*,
|
|
1068
|
+
limit: int = 10,
|
|
1069
|
+
filter: Optional[Filter] = None,
|
|
1070
|
+
with_payload: bool = True,
|
|
1071
|
+
with_vector: bool = False,
|
|
1072
|
+
payload_fields: Optional[list[str]] = None,
|
|
1073
|
+
score_threshold: Optional[float] = None,
|
|
1074
|
+
) -> list[dict[str, Any]]:
|
|
1075
|
+
"""`sparse_search` on the event loop. See it for what the arguments mean.
|
|
1076
|
+
|
|
1077
|
+
It was the one search endpoint with no async twin, which made
|
|
1078
|
+
`VectorStoreQueryMode.SPARSE` the one mode the LlamaIndex integration could not
|
|
1079
|
+
answer without blocking.
|
|
1080
|
+
"""
|
|
1081
|
+
body = self._sparse_body(
|
|
1082
|
+
sparse_vector,
|
|
1083
|
+
limit,
|
|
1084
|
+
filter,
|
|
1085
|
+
with_payload,
|
|
1086
|
+
with_vector,
|
|
1087
|
+
payload_fields,
|
|
1088
|
+
score_threshold,
|
|
1089
|
+
)
|
|
1090
|
+
try:
|
|
1091
|
+
response = await self.async_client.post(
|
|
1092
|
+
f"/tables/{_seg(table_name)}/search/sparse", json=body
|
|
1093
|
+
)
|
|
1094
|
+
response.raise_for_status()
|
|
1095
|
+
return response.json().get("data", {}).get("hits", [])
|
|
1096
|
+
except httpx.HTTPStatusError as e:
|
|
1097
|
+
msg = f"Sparse search failed: {e.response.text}"
|
|
1098
|
+
raise DXMTSearchError(msg) from e
|
|
1099
|
+
except httpx.RequestError as e:
|
|
1100
|
+
msg = f"Connection error during sparse search: {e}"
|
|
1101
|
+
raise DXMTConnectionError(msg) from e
|
|
1102
|
+
|
|
1103
|
+
def search_groups(
|
|
1104
|
+
self,
|
|
1105
|
+
table_name: str,
|
|
1106
|
+
vector: list[float],
|
|
1107
|
+
group_by: str,
|
|
1108
|
+
*,
|
|
1109
|
+
groups: int = 10,
|
|
1110
|
+
group_size: int = 3,
|
|
1111
|
+
filter: Optional[Filter] = None,
|
|
1112
|
+
with_payload: bool = True,
|
|
1113
|
+
payload_fields: Optional[list[str]] = None,
|
|
1114
|
+
score_threshold: Optional[float] = None,
|
|
1115
|
+
search_mode: Optional[str] = None,
|
|
1116
|
+
) -> dict[str, Any]:
|
|
1117
|
+
"""Nearest neighbours grouped by a payload field.
|
|
1118
|
+
|
|
1119
|
+
A document chunked for retrieval takes a whole top-10 by itself: the ten nearest
|
|
1120
|
+
vectors are ten pieces of the same text. Grouping by the document's id with
|
|
1121
|
+
`group_size=1` returns ten documents instead.
|
|
1122
|
+
|
|
1123
|
+
Groups are filled from a window of nearest neighbours that widens until enough groups
|
|
1124
|
+
appear or the server's cap is reached, so `candidates_fetched` in the answer is worth
|
|
1125
|
+
reading: a short result means either the data ran out or the cap did.
|
|
1126
|
+
|
|
1127
|
+
Args:
|
|
1128
|
+
table_name: Name of the table.
|
|
1129
|
+
vector: Query vector.
|
|
1130
|
+
group_by: Payload field whose value defines a group. A record with no value there
|
|
1131
|
+
is in no group and is counted in `ungrouped`.
|
|
1132
|
+
groups: How many groups to return.
|
|
1133
|
+
group_size: How many hits to keep per group, best first.
|
|
1134
|
+
filter: Applied before grouping, as in a plain search.
|
|
1135
|
+
with_payload: Include payloads in the hits.
|
|
1136
|
+
payload_fields: Return only these payload keys.
|
|
1137
|
+
score_threshold: Similarity floor; higher is nearer.
|
|
1138
|
+
search_mode: fast | normal | accurate | auto | exact.
|
|
1139
|
+
|
|
1140
|
+
Returns:
|
|
1141
|
+
`{"groups": [...], "candidates_fetched": n, "ungrouped": n, "tookms": n}`.
|
|
1142
|
+
|
|
1143
|
+
Raises:
|
|
1144
|
+
DXMTError: If the request fails.
|
|
1145
|
+
"""
|
|
1146
|
+
body: dict[str, Any] = {
|
|
1147
|
+
"vector": vector,
|
|
1148
|
+
"group_by": group_by,
|
|
1149
|
+
"groups": groups,
|
|
1150
|
+
"group_size": group_size,
|
|
1151
|
+
"with_payload": with_payload,
|
|
1152
|
+
}
|
|
1153
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1154
|
+
if converted is not None:
|
|
1155
|
+
body["filter"] = converted
|
|
1156
|
+
if payload_fields is not None:
|
|
1157
|
+
body["payload_fields"] = payload_fields
|
|
1158
|
+
if score_threshold is not None:
|
|
1159
|
+
body["score_threshold"] = score_threshold
|
|
1160
|
+
if search_mode is not None:
|
|
1161
|
+
body["search_mode"] = search_mode
|
|
1162
|
+
response = self._client.post(
|
|
1163
|
+
f"/tables/{quote(table_name, safe='')}/search/groups", json=body
|
|
1164
|
+
)
|
|
1165
|
+
response.raise_for_status()
|
|
1166
|
+
return response.json().get("data", {})
|
|
1167
|
+
|
|
1168
|
+
def aliases(self) -> dict[str, str]:
|
|
1169
|
+
"""Every alias and the table it points at.
|
|
1170
|
+
|
|
1171
|
+
Returns:
|
|
1172
|
+
`{alias: table}`.
|
|
1173
|
+
|
|
1174
|
+
Raises:
|
|
1175
|
+
DXMTError: If the request fails.
|
|
1176
|
+
"""
|
|
1177
|
+
response = self._client.get("/aliases")
|
|
1178
|
+
response.raise_for_status()
|
|
1179
|
+
return response.json().get("data", {}).get("aliases", {})
|
|
1180
|
+
|
|
1181
|
+
def set_alias(self, alias: str, table_name: str) -> dict[str, Any]:
|
|
1182
|
+
"""Point an alias at a table, creating it or moving it.
|
|
1183
|
+
|
|
1184
|
+
This is how a table is rebuilt without a client deploy: build the new one under its own
|
|
1185
|
+
name, load it, then move the alias. Every reader that arrives during the move sees one
|
|
1186
|
+
table or the other, never neither. An alias may be used anywhere a table name is.
|
|
1187
|
+
|
|
1188
|
+
Args:
|
|
1189
|
+
alias: The alias name. It may not be the name of an existing table.
|
|
1190
|
+
table_name: The table it resolves to.
|
|
1191
|
+
|
|
1192
|
+
Returns:
|
|
1193
|
+
`{"alias", "table"}`.
|
|
1194
|
+
|
|
1195
|
+
Raises:
|
|
1196
|
+
DXMTError: If the table does not exist, or a table already has that name.
|
|
1197
|
+
"""
|
|
1198
|
+
response = self._client.put(
|
|
1199
|
+
f"/aliases/{quote(alias, safe='')}", json={"table": table_name}
|
|
1200
|
+
)
|
|
1201
|
+
response.raise_for_status()
|
|
1202
|
+
return response.json().get("data", {})
|
|
1203
|
+
|
|
1204
|
+
def delete_alias(self, alias: str) -> dict[str, Any]:
|
|
1205
|
+
"""Remove an alias. The table it pointed at is untouched.
|
|
1206
|
+
|
|
1207
|
+
Args:
|
|
1208
|
+
alias: The alias name.
|
|
1209
|
+
|
|
1210
|
+
Returns:
|
|
1211
|
+
The server's message.
|
|
1212
|
+
|
|
1213
|
+
Raises:
|
|
1214
|
+
DXMTError: If there is no such alias.
|
|
1215
|
+
"""
|
|
1216
|
+
response = self._client.delete(f"/aliases/{quote(alias, safe='')}")
|
|
1217
|
+
response.raise_for_status()
|
|
1218
|
+
return response.json().get("data", {})
|
|
1219
|
+
|
|
1220
|
+
def export_table(self, table_name: str, path: str) -> dict[str, Any]:
|
|
1221
|
+
"""Write one table -- configuration and every record, vectors included -- to a file.
|
|
1222
|
+
|
|
1223
|
+
This is the unit `POST /backup` is not: a backup is the whole instance and restoring it
|
|
1224
|
+
means stopping the server and replacing its directory. The file is written **by the
|
|
1225
|
+
server**, on the server's filesystem, inside its `DXMT_BACKUP_ROOT`; `path` is not a
|
|
1226
|
+
path on this machine unless the two are the same machine.
|
|
1227
|
+
|
|
1228
|
+
The graph is not in the file. `import_table` rebuilds it from the records, which is the
|
|
1229
|
+
same cost a restore from the WAL pays.
|
|
1230
|
+
|
|
1231
|
+
Args:
|
|
1232
|
+
table_name: Table to export.
|
|
1233
|
+
path: Destination file on the server, inside `DXMT_BACKUP_ROOT`.
|
|
1234
|
+
|
|
1235
|
+
Returns:
|
|
1236
|
+
`{"table", "path", "records", "bytes", "records_without_vectors"}`.
|
|
1237
|
+
`records_without_vectors` is zero on a healthy table; anything else is a record
|
|
1238
|
+
whose vector was neither in memory nor readable from the row file.
|
|
1239
|
+
|
|
1240
|
+
Raises:
|
|
1241
|
+
DXMTError: If the path is outside the root, or the table does not exist.
|
|
1242
|
+
"""
|
|
1243
|
+
response = self._client.post(
|
|
1244
|
+
f"/tables/{quote(table_name, safe='')}/export", json={"path": path}
|
|
1245
|
+
)
|
|
1246
|
+
response.raise_for_status()
|
|
1247
|
+
return response.json().get("data", {})
|
|
1248
|
+
|
|
1249
|
+
def import_table(
|
|
1250
|
+
self,
|
|
1251
|
+
path: str,
|
|
1252
|
+
*,
|
|
1253
|
+
name: Optional[str] = None,
|
|
1254
|
+
overwrite: bool = False,
|
|
1255
|
+
) -> dict[str, Any]:
|
|
1256
|
+
"""Read a table back out of a file written by `export_table`.
|
|
1257
|
+
|
|
1258
|
+
Records keep their ids, payloads, versions and `created_at`: they are applied the way
|
|
1259
|
+
WAL replay applies them rather than re-inserted through the API, so an import does not
|
|
1260
|
+
restamp every row and reorder listings.
|
|
1261
|
+
|
|
1262
|
+
Args:
|
|
1263
|
+
path: The file on the server, inside `DXMT_BACKUP_ROOT`.
|
|
1264
|
+
name: Import under this name instead of the one in the file -- which is how a table
|
|
1265
|
+
is copied beside itself, to try a different index on the same data.
|
|
1266
|
+
overwrite: Replace a table of that name if one exists. Without it an import onto a
|
|
1267
|
+
live name is refused, because a merge and a replacement are different
|
|
1268
|
+
operations and neither is the obvious reading of "import".
|
|
1269
|
+
|
|
1270
|
+
Returns:
|
|
1271
|
+
`{"table", "path", "records", "records_in_file", "replaced"}`.
|
|
1272
|
+
|
|
1273
|
+
Raises:
|
|
1274
|
+
DXMTError: If the file is unreadable, outside the root, or the name is in use.
|
|
1275
|
+
"""
|
|
1276
|
+
body: dict[str, Any] = {"path": path, "overwrite": overwrite}
|
|
1277
|
+
if name is not None:
|
|
1278
|
+
body["name"] = name
|
|
1279
|
+
response = self._client.post("/tables/import", json=body)
|
|
1280
|
+
response.raise_for_status()
|
|
1281
|
+
return response.json().get("data", {})
|
|
1282
|
+
|
|
1283
|
+
def payload_index(self, table_name: str) -> dict[str, Any]:
|
|
1284
|
+
"""What the payload (filter) index holds for each field, and what it costs.
|
|
1285
|
+
|
|
1286
|
+
Every payload field is indexed by default, so this is the answer to "what is that
|
|
1287
|
+
costing me": distinct values, postings and an estimate of the bytes, worst field
|
|
1288
|
+
first. It does not build the index in order to describe it -- on a table that has not
|
|
1289
|
+
filtered since it was restored, `built` is false and `fields` is empty.
|
|
1290
|
+
|
|
1291
|
+
Args:
|
|
1292
|
+
table_name: Name of the table.
|
|
1293
|
+
|
|
1294
|
+
Returns:
|
|
1295
|
+
The report, as the server sends it.
|
|
1296
|
+
|
|
1297
|
+
Raises:
|
|
1298
|
+
DXMTError: If the request fails.
|
|
1299
|
+
"""
|
|
1300
|
+
response = self._client.get(f"/tables/{quote(table_name, safe='')}/payload_index")
|
|
1301
|
+
response.raise_for_status()
|
|
1302
|
+
return response.json().get("data", {})
|
|
1303
|
+
|
|
1304
|
+
def index_payload_field(self, table_name: str, field: str) -> dict[str, Any]:
|
|
1305
|
+
"""Index a payload field for filtering.
|
|
1306
|
+
|
|
1307
|
+
Idempotent, and a no-op on a table that indexes every field -- which is the default,
|
|
1308
|
+
so code written against a database where indexes must be declared works unchanged.
|
|
1309
|
+
The index is rebuilt from the records, so the field is filterable over rows written
|
|
1310
|
+
before it was indexed.
|
|
1311
|
+
|
|
1312
|
+
Args:
|
|
1313
|
+
table_name: Name of the table.
|
|
1314
|
+
field: Payload field name.
|
|
1315
|
+
|
|
1316
|
+
Returns:
|
|
1317
|
+
The server's message and the table's resulting policy.
|
|
1318
|
+
|
|
1319
|
+
Raises:
|
|
1320
|
+
DXMTError: If the request fails.
|
|
1321
|
+
"""
|
|
1322
|
+
response = self._client.put(
|
|
1323
|
+
f"/tables/{quote(table_name, safe='')}/payload_index/{quote(field, safe='')}"
|
|
1324
|
+
)
|
|
1325
|
+
response.raise_for_status()
|
|
1326
|
+
return response.json().get("data", {})
|
|
1327
|
+
|
|
1328
|
+
def drop_payload_index_field(self, table_name: str, field: str) -> dict[str, Any]:
|
|
1329
|
+
"""Stop indexing a payload field, freeing what its postings cost.
|
|
1330
|
+
|
|
1331
|
+
The price is filtering on it: a filter naming an unindexed field is refused with a
|
|
1332
|
+
400, because this engine answers a filter only from the index and the only other
|
|
1333
|
+
honest answer would be an empty page -- which looks exactly like "nothing matched".
|
|
1334
|
+
|
|
1335
|
+
Args:
|
|
1336
|
+
table_name: Name of the table.
|
|
1337
|
+
field: Payload field name.
|
|
1338
|
+
|
|
1339
|
+
Returns:
|
|
1340
|
+
The server's message and the table's resulting policy.
|
|
1341
|
+
|
|
1342
|
+
Raises:
|
|
1343
|
+
DXMTError: If the request fails.
|
|
1344
|
+
"""
|
|
1345
|
+
response = self._client.delete(
|
|
1346
|
+
f"/tables/{quote(table_name, safe='')}/payload_index/{quote(field, safe='')}"
|
|
1347
|
+
)
|
|
1348
|
+
response.raise_for_status()
|
|
1349
|
+
return response.json().get("data", {})
|
|
1350
|
+
|
|
1351
|
+
def count(
|
|
1352
|
+
self,
|
|
1353
|
+
table_name: str,
|
|
1354
|
+
*,
|
|
1355
|
+
filter: Optional[Filter] = None,
|
|
1356
|
+
) -> int:
|
|
1357
|
+
"""How many records a filter matches, or how many the table holds.
|
|
1358
|
+
|
|
1359
|
+
The count is computed server-side from the same index that answers the filter: before
|
|
1360
|
+
this existed the only way to get it was to page every match and count them here, which
|
|
1361
|
+
put every matching record on the wire to produce one integer.
|
|
1362
|
+
|
|
1363
|
+
Args:
|
|
1364
|
+
table_name: Name of the table.
|
|
1365
|
+
filter: Count only what this matches. Omitted, it counts the table.
|
|
1366
|
+
|
|
1367
|
+
Returns:
|
|
1368
|
+
The number of matching records.
|
|
1369
|
+
|
|
1370
|
+
Raises:
|
|
1371
|
+
DXMTError: If the request fails.
|
|
1372
|
+
"""
|
|
1373
|
+
body: dict[str, Any] = {}
|
|
1374
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1375
|
+
if converted is not None:
|
|
1376
|
+
body["filter"] = converted
|
|
1377
|
+
try:
|
|
1378
|
+
response = self._client.post(
|
|
1379
|
+
f"/tables/{_seg(table_name)}/records/count", json=body
|
|
1380
|
+
)
|
|
1381
|
+
response.raise_for_status()
|
|
1382
|
+
return int(self._response_data(response.json())["count"])
|
|
1383
|
+
except httpx.HTTPStatusError as e:
|
|
1384
|
+
msg = f"Failed to count records: {e.response.text}"
|
|
1385
|
+
raise DXMTError(msg) from e
|
|
1386
|
+
except httpx.RequestError as e:
|
|
1387
|
+
msg = f"Connection error while counting: {e}"
|
|
1388
|
+
raise DXMTConnectionError(msg) from e
|
|
1389
|
+
|
|
1390
|
+
def facets(
|
|
1391
|
+
self,
|
|
1392
|
+
table_name: str,
|
|
1393
|
+
key: str,
|
|
1394
|
+
*,
|
|
1395
|
+
filter: Optional[Filter] = None,
|
|
1396
|
+
limit: int = 20,
|
|
1397
|
+
) -> dict[str, Any]:
|
|
1398
|
+
"""How many records hold each value of a payload field.
|
|
1399
|
+
|
|
1400
|
+
Computed from the payload index, which is already a value-to-records map, so this
|
|
1401
|
+
does not page the table to count it.
|
|
1402
|
+
|
|
1403
|
+
**Read `covered` against `matched` before drawing anything.** A facet built from an
|
|
1404
|
+
index describes what the index holds, and what it does not hold — a record with no
|
|
1405
|
+
value for the field, or one whose value is an object — is missing from every bucket
|
|
1406
|
+
rather than having a bucket of its own. Their difference is how many records this
|
|
1407
|
+
answer cannot describe, and `hits` alone will happily show a corpus that is not the
|
|
1408
|
+
corpus.
|
|
1409
|
+
|
|
1410
|
+
Args:
|
|
1411
|
+
table_name: Name of the table.
|
|
1412
|
+
key: The payload field to count values of. A dotted path works, as in a filter.
|
|
1413
|
+
filter: Count only within what this matches. Omitted, it counts the table.
|
|
1414
|
+
limit: How many of the most common values to return; `distinct_values` in the
|
|
1415
|
+
result says how many there were.
|
|
1416
|
+
|
|
1417
|
+
Returns:
|
|
1418
|
+
`{"hits": [{"value": ..., "count": n}, ...], "distinct_values": n,
|
|
1419
|
+
"covered": n, "matched": n}`.
|
|
1420
|
+
|
|
1421
|
+
Raises:
|
|
1422
|
+
DXMTError: If the field is not indexed, or the request fails.
|
|
1423
|
+
"""
|
|
1424
|
+
body: dict[str, Any] = {"key": key, "limit": limit}
|
|
1425
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1426
|
+
if converted is not None:
|
|
1427
|
+
body["filter"] = converted
|
|
1428
|
+
try:
|
|
1429
|
+
response = self._client.post(f"/tables/{_seg(table_name)}/facets", json=body)
|
|
1430
|
+
response.raise_for_status()
|
|
1431
|
+
return self._response_data(response.json())
|
|
1432
|
+
except httpx.HTTPStatusError as e:
|
|
1433
|
+
msg = f"Failed to facet '{key}': {e.response.text}"
|
|
1434
|
+
raise DXMTError(msg) from e
|
|
1435
|
+
except httpx.RequestError as e:
|
|
1436
|
+
msg = f"Connection error while faceting: {e}"
|
|
1437
|
+
raise DXMTConnectionError(msg) from e
|
|
1438
|
+
|
|
1439
|
+
def delete_records(
|
|
1440
|
+
self,
|
|
1441
|
+
table_name: str,
|
|
1442
|
+
ids: Optional[list[str]] = None,
|
|
1443
|
+
*,
|
|
1444
|
+
filter: Optional[Filter] = None,
|
|
1445
|
+
) -> dict[str, Any]:
|
|
1446
|
+
"""Delete records by id, or everything a filter matches.
|
|
1447
|
+
|
|
1448
|
+
Exactly one of `ids` and `filter`. Deleting by filter is one request whatever it
|
|
1449
|
+
matches: before it existed, "remove everything belonging to tenant X" meant paging the
|
|
1450
|
+
table, collecting ids and sending them back, a round trip per page and at the mercy of
|
|
1451
|
+
anything written meanwhile.
|
|
1452
|
+
|
|
1453
|
+
Args:
|
|
1454
|
+
table_name: Name of the table.
|
|
1455
|
+
ids: Record IDs to delete.
|
|
1456
|
+
filter: Delete every record this matches instead (a `Field` expression or a dict).
|
|
1457
|
+
|
|
1458
|
+
Returns:
|
|
1459
|
+
Response data from server, including `deleted_count`.
|
|
1460
|
+
|
|
1461
|
+
Raises:
|
|
1462
|
+
ValueError: If both or neither selector is given.
|
|
1463
|
+
DXMTError: If deletion fails.
|
|
1464
|
+
"""
|
|
1465
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1466
|
+
if (ids is None) == (converted is None):
|
|
1467
|
+
raise ValueError("give either ids or filter, not both and not neither")
|
|
1468
|
+
payload = {"ids": ids} if ids is not None else {"filter": converted}
|
|
1469
|
+
try:
|
|
1470
|
+
response = self._client.request(
|
|
1471
|
+
"DELETE",
|
|
1472
|
+
f"/tables/{_seg(table_name)}/records",
|
|
1473
|
+
json=payload,
|
|
1474
|
+
)
|
|
1475
|
+
response.raise_for_status()
|
|
1476
|
+
return response.json()
|
|
1477
|
+
except httpx.HTTPStatusError as e:
|
|
1478
|
+
msg = f"Failed to delete records: {e.response.text}"
|
|
1479
|
+
raise DXMTError(msg) from e
|
|
1480
|
+
except httpx.RequestError as e:
|
|
1481
|
+
msg = f"Connection error deleting records: {e}"
|
|
1482
|
+
raise DXMTConnectionError(msg) from e
|
|
1483
|
+
|
|
1484
|
+
async def adelete_records(
|
|
1485
|
+
self,
|
|
1486
|
+
table_name: str,
|
|
1487
|
+
ids: Optional[list[str]] = None,
|
|
1488
|
+
*,
|
|
1489
|
+
filter: Optional[Filter] = None,
|
|
1490
|
+
) -> dict[str, Any]:
|
|
1491
|
+
"""Async version of :meth:`delete_records`; same one-selector rule."""
|
|
1492
|
+
converted = convert_simple_filter_to_dxmt(filter)
|
|
1493
|
+
if (ids is None) == (converted is None):
|
|
1494
|
+
raise ValueError("give either ids or filter, not both and not neither")
|
|
1495
|
+
payload = {"ids": ids} if ids is not None else {"filter": converted}
|
|
1496
|
+
try:
|
|
1497
|
+
response = await self.async_client.request(
|
|
1498
|
+
"DELETE",
|
|
1499
|
+
f"/tables/{_seg(table_name)}/records",
|
|
1500
|
+
json=payload,
|
|
1501
|
+
)
|
|
1502
|
+
response.raise_for_status()
|
|
1503
|
+
return response.json()
|
|
1504
|
+
except httpx.HTTPStatusError as e:
|
|
1505
|
+
msg = f"Failed to delete records: {e.response.text}"
|
|
1506
|
+
raise DXMTError(msg) from e
|
|
1507
|
+
except httpx.RequestError as e:
|
|
1508
|
+
msg = f"Connection error deleting records: {e}"
|
|
1509
|
+
raise DXMTConnectionError(msg) from e
|
|
1510
|
+
|
|
1511
|
+
def list_records(
|
|
1512
|
+
self,
|
|
1513
|
+
table_name: str,
|
|
1514
|
+
limit: int = 100,
|
|
1515
|
+
offset: int = 0,
|
|
1516
|
+
) -> list[dict[str, Any]]:
|
|
1517
|
+
"""List a table's records, without a query vector.
|
|
1518
|
+
|
|
1519
|
+
The server has had `POST /tables/{t}/records/list` all along; this client did not
|
|
1520
|
+
expose it, and three call sites worked around the gap in ways that are worse than the
|
|
1521
|
+
round trip: `mem0.py` and `metagpt_vector_store.py` search with a zero vector and a
|
|
1522
|
+
large `k` to mean "everything", and `langgraph_store.py` logged a warning and returned
|
|
1523
|
+
nothing at all for a metadata-only search.
|
|
1524
|
+
|
|
1525
|
+
`limit` is capped at 1,000 by the server. Page with `offset`.
|
|
1526
|
+
|
|
1527
|
+
Args:
|
|
1528
|
+
table_name: Name of the table.
|
|
1529
|
+
limit: Records per page, at most 1,000.
|
|
1530
|
+
offset: Records to skip.
|
|
1531
|
+
|
|
1532
|
+
Returns:
|
|
1533
|
+
The page, normalised the way search hits and single-record reads are.
|
|
1534
|
+
|
|
1535
|
+
Raises:
|
|
1536
|
+
DXMTError: If the listing fails.
|
|
1537
|
+
"""
|
|
1538
|
+
try:
|
|
1539
|
+
response = self._client.post(
|
|
1540
|
+
f"/tables/{_seg(table_name)}/records/list",
|
|
1541
|
+
json={"limit": limit, "offset": offset},
|
|
1542
|
+
)
|
|
1543
|
+
response.raise_for_status()
|
|
1544
|
+
data = self._response_data(response.json())
|
|
1545
|
+
records = data.get("records") or []
|
|
1546
|
+
return [self._normalize_record(r) for r in records]
|
|
1547
|
+
except httpx.HTTPStatusError as e:
|
|
1548
|
+
msg = f"Failed to list records in '{table_name}': {e.response.text}"
|
|
1549
|
+
raise DXMTRecordError(msg) from e
|
|
1550
|
+
except httpx.RequestError as e:
|
|
1551
|
+
msg = f"Connection error while listing records: {e}"
|
|
1552
|
+
raise DXMTConnectionError(msg) from e
|
|
1553
|
+
|
|
1554
|
+
async def alist_records(
|
|
1555
|
+
self,
|
|
1556
|
+
table_name: str,
|
|
1557
|
+
limit: int = 100,
|
|
1558
|
+
offset: int = 0,
|
|
1559
|
+
) -> list[dict[str, Any]]:
|
|
1560
|
+
"""Async twin of :meth:`list_records`."""
|
|
1561
|
+
try:
|
|
1562
|
+
response = await self.async_client.post(
|
|
1563
|
+
f"/tables/{_seg(table_name)}/records/list",
|
|
1564
|
+
json={"limit": limit, "offset": offset},
|
|
1565
|
+
)
|
|
1566
|
+
response.raise_for_status()
|
|
1567
|
+
data = self._response_data(response.json())
|
|
1568
|
+
records = data.get("records") or []
|
|
1569
|
+
return [self._normalize_record(r) for r in records]
|
|
1570
|
+
except httpx.HTTPStatusError as e:
|
|
1571
|
+
msg = f"Failed to list records in '{table_name}': {e.response.text}"
|
|
1572
|
+
raise DXMTRecordError(msg) from e
|
|
1573
|
+
except httpx.RequestError as e:
|
|
1574
|
+
msg = f"Connection error while listing records: {e}"
|
|
1575
|
+
raise DXMTConnectionError(msg) from e
|
|
1576
|
+
|
|
1577
|
+
def get_records(
|
|
1578
|
+
self,
|
|
1579
|
+
table_name: str,
|
|
1580
|
+
ids: list[str],
|
|
1581
|
+
) -> list[dict[str, Any]]:
|
|
1582
|
+
"""Get records by IDs.
|
|
1583
|
+
|
|
1584
|
+
Args:
|
|
1585
|
+
table_name: Name of the table.
|
|
1586
|
+
ids: List of record IDs to retrieve.
|
|
1587
|
+
|
|
1588
|
+
Returns:
|
|
1589
|
+
List of records.
|
|
1590
|
+
|
|
1591
|
+
Raises:
|
|
1592
|
+
DXMTError: If retrieval fails.
|
|
1593
|
+
"""
|
|
1594
|
+
# Note: No batch get endpoint, so we fetch individually
|
|
1595
|
+
records = []
|
|
1596
|
+
for record_id in ids:
|
|
1597
|
+
try:
|
|
1598
|
+
response = self._client.get(
|
|
1599
|
+
f"/tables/{_seg(table_name)}/records/{_seg(record_id)}"
|
|
1600
|
+
)
|
|
1601
|
+
response.raise_for_status()
|
|
1602
|
+
data = self._response_data(response.json())
|
|
1603
|
+
# Extract record from response envelope
|
|
1604
|
+
record = self._normalize_record(data)
|
|
1605
|
+
if record:
|
|
1606
|
+
records.append(record)
|
|
1607
|
+
except httpx.HTTPStatusError as e:
|
|
1608
|
+
if e.response.status_code == 404:
|
|
1609
|
+
# Skip not found records
|
|
1610
|
+
continue
|
|
1611
|
+
msg = f"Failed to get record {record_id}: {e.response.text}"
|
|
1612
|
+
raise DXMTError(msg) from e
|
|
1613
|
+
except httpx.RequestError as e:
|
|
1614
|
+
msg = f"Connection error getting record {record_id}: {e}"
|
|
1615
|
+
raise DXMTConnectionError(msg) from e
|
|
1616
|
+
return records
|
|
1617
|
+
|
|
1618
|
+
async def aget_records(
|
|
1619
|
+
self,
|
|
1620
|
+
table_name: str,
|
|
1621
|
+
ids: list[str],
|
|
1622
|
+
) -> list[dict[str, Any]]:
|
|
1623
|
+
"""Async get records by IDs."""
|
|
1624
|
+
# Note: No batch get endpoint, so we fetch individually
|
|
1625
|
+
records = []
|
|
1626
|
+
for record_id in ids:
|
|
1627
|
+
try:
|
|
1628
|
+
response = await self.async_client.get(
|
|
1629
|
+
f"/tables/{_seg(table_name)}/records/{_seg(record_id)}"
|
|
1630
|
+
)
|
|
1631
|
+
response.raise_for_status()
|
|
1632
|
+
data = self._response_data(response.json())
|
|
1633
|
+
# Extract record from response envelope
|
|
1634
|
+
record = self._normalize_record(data)
|
|
1635
|
+
if record:
|
|
1636
|
+
records.append(record)
|
|
1637
|
+
except httpx.HTTPStatusError as e:
|
|
1638
|
+
if e.response.status_code == 404:
|
|
1639
|
+
# Skip not found records
|
|
1640
|
+
continue
|
|
1641
|
+
msg = f"Failed to get record {record_id}: {e.response.text}"
|
|
1642
|
+
raise DXMTError(msg) from e
|
|
1643
|
+
except httpx.RequestError as e:
|
|
1644
|
+
msg = f"Connection error getting record {record_id}: {e}"
|
|
1645
|
+
raise DXMTConnectionError(msg) from e
|
|
1646
|
+
return records
|
|
1647
|
+
|
|
1648
|
+
# Documents: a long text stored as overlapping chunks, one record each
|
|
1649
|
+
|
|
1650
|
+
@staticmethod
|
|
1651
|
+
def _document_body(text, metadata, chunk_size, chunk_overlap) -> dict[str, Any]:
|
|
1652
|
+
body: dict[str, Any] = {"text": text}
|
|
1653
|
+
if metadata is not None:
|
|
1654
|
+
body["metadata"] = metadata
|
|
1655
|
+
if chunk_size is not None:
|
|
1656
|
+
body["chunk_size"] = chunk_size
|
|
1657
|
+
if chunk_overlap is not None:
|
|
1658
|
+
body["chunk_overlap"] = chunk_overlap
|
|
1659
|
+
return body
|
|
1660
|
+
|
|
1661
|
+
def put_document(
|
|
1662
|
+
self,
|
|
1663
|
+
table_name: str,
|
|
1664
|
+
doc_id: str,
|
|
1665
|
+
text: str,
|
|
1666
|
+
metadata: Optional[dict[str, Any]] = None,
|
|
1667
|
+
chunk_size: Optional[int] = None,
|
|
1668
|
+
chunk_overlap: Optional[int] = None,
|
|
1669
|
+
) -> dict[str, Any]:
|
|
1670
|
+
"""Store ``text`` as the document ``doc_id``, replacing any previous version.
|
|
1671
|
+
|
|
1672
|
+
The server splits it into overlapping chunks (500 characters, 50 shared, by default)
|
|
1673
|
+
on paragraph and sentence boundaries and embeds each with the table's model, so the
|
|
1674
|
+
table needs one -- the ``kb`` table has the server's default. Each chunk is a record
|
|
1675
|
+
``{doc_id}#{n}`` whose payload carries ``doc_id``, ``chunk_index``, ``chunk_count``
|
|
1676
|
+
and ``metadata``; search returns chunks.
|
|
1677
|
+
|
|
1678
|
+
Returns:
|
|
1679
|
+
``{"doc_id", "chunks", "removed"}``: chunks stored, and chunks of the previous
|
|
1680
|
+
version the new one no longer has.
|
|
1681
|
+
"""
|
|
1682
|
+
try:
|
|
1683
|
+
response = self._client.put(
|
|
1684
|
+
f"/tables/{_seg(table_name)}/documents/{_seg(doc_id)}",
|
|
1685
|
+
json=self._document_body(text, metadata, chunk_size, chunk_overlap),
|
|
1686
|
+
)
|
|
1687
|
+
response.raise_for_status()
|
|
1688
|
+
return self._response_data(response.json())
|
|
1689
|
+
except httpx.HTTPStatusError as e:
|
|
1690
|
+
msg = f"Failed to store document {doc_id}: {e.response.text}"
|
|
1691
|
+
raise DXMTRecordError(msg) from e
|
|
1692
|
+
except httpx.RequestError as e:
|
|
1693
|
+
msg = f"Connection error storing document {doc_id}: {e}"
|
|
1694
|
+
raise DXMTConnectionError(msg) from e
|
|
1695
|
+
|
|
1696
|
+
async def aput_document(
|
|
1697
|
+
self,
|
|
1698
|
+
table_name: str,
|
|
1699
|
+
doc_id: str,
|
|
1700
|
+
text: str,
|
|
1701
|
+
metadata: Optional[dict[str, Any]] = None,
|
|
1702
|
+
chunk_size: Optional[int] = None,
|
|
1703
|
+
chunk_overlap: Optional[int] = None,
|
|
1704
|
+
) -> dict[str, Any]:
|
|
1705
|
+
"""Async version of :meth:`put_document`."""
|
|
1706
|
+
try:
|
|
1707
|
+
response = await self.async_client.put(
|
|
1708
|
+
f"/tables/{_seg(table_name)}/documents/{_seg(doc_id)}",
|
|
1709
|
+
json=self._document_body(text, metadata, chunk_size, chunk_overlap),
|
|
1710
|
+
)
|
|
1711
|
+
response.raise_for_status()
|
|
1712
|
+
return self._response_data(response.json())
|
|
1713
|
+
except httpx.HTTPStatusError as e:
|
|
1714
|
+
msg = f"Failed to store document {doc_id}: {e.response.text}"
|
|
1715
|
+
raise DXMTRecordError(msg) from e
|
|
1716
|
+
except httpx.RequestError as e:
|
|
1717
|
+
msg = f"Connection error storing document {doc_id}: {e}"
|
|
1718
|
+
raise DXMTConnectionError(msg) from e
|
|
1719
|
+
|
|
1720
|
+
def get_document(self, table_name: str, doc_id: str) -> Optional[dict[str, Any]]:
|
|
1721
|
+
"""A document's chunks in order, with its metadata; ``None`` if there is none.
|
|
1722
|
+
|
|
1723
|
+
Returns:
|
|
1724
|
+
``{"doc_id", "metadata", "chunks": [{"id", "chunk_index", "text"}, ...]}``.
|
|
1725
|
+
"""
|
|
1726
|
+
try:
|
|
1727
|
+
response = self._client.get(
|
|
1728
|
+
f"/tables/{_seg(table_name)}/documents/{_seg(doc_id)}"
|
|
1729
|
+
)
|
|
1730
|
+
# Both a missing table and a missing document are 404 NOT_FOUND; only the second
|
|
1731
|
+
# is an answer. The server says "no document '<id>'" for it.
|
|
1732
|
+
if response.status_code == 404 and "no document" in response.text:
|
|
1733
|
+
return None
|
|
1734
|
+
response.raise_for_status()
|
|
1735
|
+
return self._response_data(response.json())
|
|
1736
|
+
except httpx.HTTPStatusError as e:
|
|
1737
|
+
msg = f"Failed to get document {doc_id}: {e.response.text}"
|
|
1738
|
+
raise DXMTRecordError(msg) from e
|
|
1739
|
+
except httpx.RequestError as e:
|
|
1740
|
+
msg = f"Connection error getting document {doc_id}: {e}"
|
|
1741
|
+
raise DXMTConnectionError(msg) from e
|
|
1742
|
+
|
|
1743
|
+
async def aget_document(self, table_name: str, doc_id: str) -> Optional[dict[str, Any]]:
|
|
1744
|
+
"""Async version of :meth:`get_document`."""
|
|
1745
|
+
try:
|
|
1746
|
+
response = await self.async_client.get(
|
|
1747
|
+
f"/tables/{_seg(table_name)}/documents/{_seg(doc_id)}"
|
|
1748
|
+
)
|
|
1749
|
+
# Both a missing table and a missing document are 404 NOT_FOUND; only the second
|
|
1750
|
+
# is an answer. The server says "no document '<id>'" for it.
|
|
1751
|
+
if response.status_code == 404 and "no document" in response.text:
|
|
1752
|
+
return None
|
|
1753
|
+
response.raise_for_status()
|
|
1754
|
+
return self._response_data(response.json())
|
|
1755
|
+
except httpx.HTTPStatusError as e:
|
|
1756
|
+
msg = f"Failed to get document {doc_id}: {e.response.text}"
|
|
1757
|
+
raise DXMTRecordError(msg) from e
|
|
1758
|
+
except httpx.RequestError as e:
|
|
1759
|
+
msg = f"Connection error getting document {doc_id}: {e}"
|
|
1760
|
+
raise DXMTConnectionError(msg) from e
|
|
1761
|
+
|
|
1762
|
+
def delete_document(self, table_name: str, doc_id: str) -> int:
|
|
1763
|
+
"""Delete every chunk of ``doc_id``; returns how many there were (0 if none)."""
|
|
1764
|
+
try:
|
|
1765
|
+
response = self._client.delete(
|
|
1766
|
+
f"/tables/{_seg(table_name)}/documents/{_seg(doc_id)}"
|
|
1767
|
+
)
|
|
1768
|
+
response.raise_for_status()
|
|
1769
|
+
return int(self._response_data(response.json())["deleted_chunks"])
|
|
1770
|
+
except httpx.HTTPStatusError as e:
|
|
1771
|
+
msg = f"Failed to delete document {doc_id}: {e.response.text}"
|
|
1772
|
+
raise DXMTRecordError(msg) from e
|
|
1773
|
+
except httpx.RequestError as e:
|
|
1774
|
+
msg = f"Connection error deleting document {doc_id}: {e}"
|
|
1775
|
+
raise DXMTConnectionError(msg) from e
|
|
1776
|
+
|
|
1777
|
+
async def adelete_document(self, table_name: str, doc_id: str) -> int:
|
|
1778
|
+
"""Async version of :meth:`delete_document`."""
|
|
1779
|
+
try:
|
|
1780
|
+
response = await self.async_client.delete(
|
|
1781
|
+
f"/tables/{_seg(table_name)}/documents/{_seg(doc_id)}"
|
|
1782
|
+
)
|
|
1783
|
+
response.raise_for_status()
|
|
1784
|
+
return int(self._response_data(response.json())["deleted_chunks"])
|
|
1785
|
+
except httpx.HTTPStatusError as e:
|
|
1786
|
+
msg = f"Failed to delete document {doc_id}: {e.response.text}"
|
|
1787
|
+
raise DXMTRecordError(msg) from e
|
|
1788
|
+
except httpx.RequestError as e:
|
|
1789
|
+
msg = f"Connection error deleting document {doc_id}: {e}"
|
|
1790
|
+
raise DXMTConnectionError(msg) from e
|
|
1791
|
+
|
|
1792
|
+
# -- Memory: what an agent keeps about its user and its work ------------------------------
|
|
1793
|
+
#
|
|
1794
|
+
# One table on the server (``memory``), made on first use with the server's embedding
|
|
1795
|
+
# model. Recall searches by meaning and by keyword; a memory that ``supersedes`` others
|
|
1796
|
+
# leaves them as history and out of recall. Every agent reaching this database shares it.
|
|
1797
|
+
|
|
1798
|
+
@staticmethod
|
|
1799
|
+
def _remember_body(
|
|
1800
|
+
text: str,
|
|
1801
|
+
kind: Optional[str],
|
|
1802
|
+
tags: Optional[list[str]],
|
|
1803
|
+
source: Optional[str],
|
|
1804
|
+
supersedes: Optional[list[str]],
|
|
1805
|
+
memory_id: Optional[str],
|
|
1806
|
+
metadata: Optional[dict[str, Any]],
|
|
1807
|
+
) -> dict[str, Any]:
|
|
1808
|
+
body: dict[str, Any] = {"text": text}
|
|
1809
|
+
for key, value in (
|
|
1810
|
+
("kind", kind),
|
|
1811
|
+
("tags", tags),
|
|
1812
|
+
("source", source),
|
|
1813
|
+
("supersedes", supersedes),
|
|
1814
|
+
("id", memory_id),
|
|
1815
|
+
("metadata", metadata),
|
|
1816
|
+
):
|
|
1817
|
+
if value is not None:
|
|
1818
|
+
body[key] = value
|
|
1819
|
+
return body
|
|
1820
|
+
|
|
1821
|
+
@staticmethod
|
|
1822
|
+
def _recall_body(
|
|
1823
|
+
query: str,
|
|
1824
|
+
limit: Optional[int],
|
|
1825
|
+
kind: Optional[str],
|
|
1826
|
+
tags: Optional[list[str]],
|
|
1827
|
+
since: Optional[Any],
|
|
1828
|
+
include_superseded: bool,
|
|
1829
|
+
) -> dict[str, Any]:
|
|
1830
|
+
body: dict[str, Any] = {"query": query}
|
|
1831
|
+
for key, value in (("limit", limit), ("kind", kind), ("tags", tags), ("since", since)):
|
|
1832
|
+
if value is not None:
|
|
1833
|
+
body[key] = value
|
|
1834
|
+
if include_superseded:
|
|
1835
|
+
body["include_superseded"] = True
|
|
1836
|
+
return body
|
|
1837
|
+
|
|
1838
|
+
def _memory(self, method: str, path: str, action: str, **kwargs: Any) -> dict[str, Any]:
|
|
1839
|
+
try:
|
|
1840
|
+
response = self._client.request(method, path, **kwargs)
|
|
1841
|
+
response.raise_for_status()
|
|
1842
|
+
return self._response_data(response.json())
|
|
1843
|
+
except httpx.HTTPStatusError as e:
|
|
1844
|
+
raise DXMTRecordError(f"Failed to {action}: {e.response.text}") from e
|
|
1845
|
+
except httpx.RequestError as e:
|
|
1846
|
+
raise DXMTConnectionError(f"Connection error trying to {action}: {e}") from e
|
|
1847
|
+
|
|
1848
|
+
async def _amemory(self, method: str, path: str, action: str, **kwargs: Any) -> dict[str, Any]:
|
|
1849
|
+
try:
|
|
1850
|
+
response = await self.async_client.request(method, path, **kwargs)
|
|
1851
|
+
response.raise_for_status()
|
|
1852
|
+
return self._response_data(response.json())
|
|
1853
|
+
except httpx.HTTPStatusError as e:
|
|
1854
|
+
raise DXMTRecordError(f"Failed to {action}: {e.response.text}") from e
|
|
1855
|
+
except httpx.RequestError as e:
|
|
1856
|
+
raise DXMTConnectionError(f"Connection error trying to {action}: {e}") from e
|
|
1857
|
+
|
|
1858
|
+
def remember(
|
|
1859
|
+
self,
|
|
1860
|
+
text: str,
|
|
1861
|
+
kind: Optional[str] = None,
|
|
1862
|
+
tags: Optional[list[str]] = None,
|
|
1863
|
+
source: Optional[str] = None,
|
|
1864
|
+
supersedes: Optional[list[str]] = None,
|
|
1865
|
+
memory_id: Optional[str] = None,
|
|
1866
|
+
metadata: Optional[dict[str, Any]] = None,
|
|
1867
|
+
) -> dict[str, Any]:
|
|
1868
|
+
"""Keep a memory -- a fact, preference, event, instruction or note (``kind``).
|
|
1869
|
+
|
|
1870
|
+
``supersedes`` names memories this one replaces (they are kept as history and no longer
|
|
1871
|
+
recalled); ``memory_id`` corrects an existing memory in place.
|
|
1872
|
+
|
|
1873
|
+
Saying what is already remembered refreshes that memory instead of adding a copy.
|
|
1874
|
+
|
|
1875
|
+
Returns:
|
|
1876
|
+
``{"id", "superseded"}``, plus ``"duplicate": True`` when the memory was already held.
|
|
1877
|
+
"""
|
|
1878
|
+
body = self._remember_body(text, kind, tags, source, supersedes, memory_id, metadata)
|
|
1879
|
+
return self._memory("POST", "/memories", "remember", json=body)
|
|
1880
|
+
|
|
1881
|
+
async def aremember(
|
|
1882
|
+
self,
|
|
1883
|
+
text: str,
|
|
1884
|
+
kind: Optional[str] = None,
|
|
1885
|
+
tags: Optional[list[str]] = None,
|
|
1886
|
+
source: Optional[str] = None,
|
|
1887
|
+
supersedes: Optional[list[str]] = None,
|
|
1888
|
+
memory_id: Optional[str] = None,
|
|
1889
|
+
metadata: Optional[dict[str, Any]] = None,
|
|
1890
|
+
) -> dict[str, Any]:
|
|
1891
|
+
"""Async version of :meth:`remember`."""
|
|
1892
|
+
body = self._remember_body(text, kind, tags, source, supersedes, memory_id, metadata)
|
|
1893
|
+
return await self._amemory("POST", "/memories", "remember", json=body)
|
|
1894
|
+
|
|
1895
|
+
def recall(
|
|
1896
|
+
self,
|
|
1897
|
+
query: str,
|
|
1898
|
+
limit: Optional[int] = None,
|
|
1899
|
+
kind: Optional[str] = None,
|
|
1900
|
+
tags: Optional[list[str]] = None,
|
|
1901
|
+
since: Optional[Any] = None,
|
|
1902
|
+
include_superseded: bool = False,
|
|
1903
|
+
) -> list[dict[str, Any]]:
|
|
1904
|
+
"""The memories that best match ``query``, by meaning and keyword, best first.
|
|
1905
|
+
|
|
1906
|
+
``since`` is Unix seconds or an RFC 3339 time. Each memory carries ``id``, ``text``,
|
|
1907
|
+
``kind``, ``tags``, ``source``, ``at`` and ``score``.
|
|
1908
|
+
"""
|
|
1909
|
+
body = self._recall_body(query, limit, kind, tags, since, include_superseded)
|
|
1910
|
+
return self._memory("POST", "/memories/recall", "recall", json=body)["memories"]
|
|
1911
|
+
|
|
1912
|
+
async def arecall(
|
|
1913
|
+
self,
|
|
1914
|
+
query: str,
|
|
1915
|
+
limit: Optional[int] = None,
|
|
1916
|
+
kind: Optional[str] = None,
|
|
1917
|
+
tags: Optional[list[str]] = None,
|
|
1918
|
+
since: Optional[Any] = None,
|
|
1919
|
+
include_superseded: bool = False,
|
|
1920
|
+
) -> list[dict[str, Any]]:
|
|
1921
|
+
"""Async version of :meth:`recall`."""
|
|
1922
|
+
body = self._recall_body(query, limit, kind, tags, since, include_superseded)
|
|
1923
|
+
data = await self._amemory("POST", "/memories/recall", "recall", json=body)
|
|
1924
|
+
return data["memories"]
|
|
1925
|
+
|
|
1926
|
+
def forget(self, memory_id: str) -> int:
|
|
1927
|
+
"""Delete a memory for good; returns 1, or 0 when there was none."""
|
|
1928
|
+
data = self._memory("DELETE", f"/memories/{_seg(memory_id)}", "forget")
|
|
1929
|
+
return int(data["forgotten"])
|
|
1930
|
+
|
|
1931
|
+
async def aforget(self, memory_id: str) -> int:
|
|
1932
|
+
"""Async version of :meth:`forget`."""
|
|
1933
|
+
data = await self._amemory("DELETE", f"/memories/{_seg(memory_id)}", "forget")
|
|
1934
|
+
return int(data["forgotten"])
|
|
1935
|
+
|
|
1936
|
+
def list_memories(
|
|
1937
|
+
self, kind: Optional[str] = None, limit: int = 50, offset: int = 0
|
|
1938
|
+
) -> dict[str, Any]:
|
|
1939
|
+
"""Memories newest first: ``{"memories": [...], "total": n}``."""
|
|
1940
|
+
params: dict[str, Any] = {"limit": limit, "offset": offset}
|
|
1941
|
+
if kind is not None:
|
|
1942
|
+
params["kind"] = kind
|
|
1943
|
+
return self._memory("GET", "/memories", "list memories", params=params)
|
|
1944
|
+
|
|
1945
|
+
async def alist_memories(
|
|
1946
|
+
self, kind: Optional[str] = None, limit: int = 50, offset: int = 0
|
|
1947
|
+
) -> dict[str, Any]:
|
|
1948
|
+
"""Async version of :meth:`list_memories`."""
|
|
1949
|
+
params: dict[str, Any] = {"limit": limit, "offset": offset}
|
|
1950
|
+
if kind is not None:
|
|
1951
|
+
params["kind"] = kind
|
|
1952
|
+
return await self._amemory("GET", "/memories", "list memories", params=params)
|
|
1953
|
+
|
|
1954
|
+
# Search Operations
|
|
1955
|
+
|
|
1956
|
+
def search(
|
|
1957
|
+
self,
|
|
1958
|
+
table_name: str,
|
|
1959
|
+
vector: Optional[list[float]] = None,
|
|
1960
|
+
k: Optional[int] = None,
|
|
1961
|
+
limit: Optional[int] = None,
|
|
1962
|
+
filter: Optional[Filter] = None,
|
|
1963
|
+
search_mode: Optional[str] = None,
|
|
1964
|
+
score_threshold: Optional[float] = None,
|
|
1965
|
+
payload_fields: Optional[list[str]] = None,
|
|
1966
|
+
offset: int = 0,
|
|
1967
|
+
mmr: Optional[dict[str, Any]] = None,
|
|
1968
|
+
text: Optional[str] = None,
|
|
1969
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
1970
|
+
) -> list[dict[str, Any]]:
|
|
1971
|
+
"""Vector similarity search.
|
|
1972
|
+
|
|
1973
|
+
Args:
|
|
1974
|
+
table_name: Name of the table.
|
|
1975
|
+
vector: Query vector. Omit it when passing `text`.
|
|
1976
|
+
text: Query text, embedded by the table's own model -- which is the only model it
|
|
1977
|
+
may be embedded with, since a query vector from a different model than the
|
|
1978
|
+
corpus is not a worse answer but an unrelated one. Requires the table to have
|
|
1979
|
+
been created with `embedding=...`.
|
|
1980
|
+
k: Number of results to return (preferred parameter name).
|
|
1981
|
+
limit: Number of results to return (alias for k, for compatibility).
|
|
1982
|
+
filter: Optional metadata filter.
|
|
1983
|
+
search_mode: How hard to search -- "fast", "normal" (default), or "accurate",
|
|
1984
|
+
which scale the graph's ef_search by roughly 1x, 1x and 4x. The server has
|
|
1985
|
+
accepted this on every search since SearchMode existed; the SDK simply never
|
|
1986
|
+
passed it, so no caller could reach the recall/latency trade it offers.
|
|
1987
|
+
Omitted from the request when None, leaving the server's default.
|
|
1988
|
+
|
|
1989
|
+
offset: Skip this many hits before the page starts -- page two of a search,
|
|
1990
|
+
without fetching page one again. Not free: the server asks the index for
|
|
1991
|
+
`offset + k` and drops the prefix, because an approximate index cannot start in
|
|
1992
|
+
the middle of a ranking it has not computed. To walk a whole table, `scroll`
|
|
1993
|
+
pages by cursor and stays flat.
|
|
1994
|
+
|
|
1995
|
+
Returns:
|
|
1996
|
+
List of search results.
|
|
1997
|
+
|
|
1998
|
+
Raises:
|
|
1999
|
+
DXMTSearchError: If search fails.
|
|
2000
|
+
ValueError: If neither k nor limit is provided.
|
|
2001
|
+
"""
|
|
2002
|
+
# Support both k and limit parameters (limit is an alias for k)
|
|
2003
|
+
if k is None and limit is None:
|
|
2004
|
+
k = 4 # Default value
|
|
2005
|
+
elif k is None:
|
|
2006
|
+
k = limit
|
|
2007
|
+
elif limit is not None and limit != k:
|
|
2008
|
+
# Both provided but different - prefer k
|
|
2009
|
+
pass
|
|
2010
|
+
|
|
2011
|
+
# `text` is embedded by the table's own model, server-side. The two are exclusive
|
|
2012
|
+
# and the server refuses a body carrying both, rather than quietly preferring one --
|
|
2013
|
+
# which is how a caller ends up measuring the wrong query for a week.
|
|
2014
|
+
if (vector is None) == (text is None):
|
|
2015
|
+
msg = "search takes either a vector or text, not both and not neither"
|
|
2016
|
+
raise ValueError(msg)
|
|
2017
|
+
payload: dict[str, Any] = {
|
|
2018
|
+
"top_k": k,
|
|
2019
|
+
"limit": k,
|
|
2020
|
+
"filter": convert_simple_filter_to_dxmt(filter),
|
|
2021
|
+
}
|
|
2022
|
+
if text is not None:
|
|
2023
|
+
payload["text"] = text
|
|
2024
|
+
else:
|
|
2025
|
+
payload["vector"] = vector
|
|
2026
|
+
if search_mode is not None:
|
|
2027
|
+
payload["search_mode"] = search_mode
|
|
2028
|
+
# A floor on the similarity the server returns (higher is nearer), not on a distance.
|
|
2029
|
+
if score_threshold is not None:
|
|
2030
|
+
payload["score_threshold"] = score_threshold
|
|
2031
|
+
# Only these payload keys come back. A payload is often mostly the text the embedding
|
|
2032
|
+
# came from, so a page of hits can be a megabyte of JSON to deliver two fields.
|
|
2033
|
+
if payload_fields is not None:
|
|
2034
|
+
payload["payload_fields"] = payload_fields
|
|
2035
|
+
if offset:
|
|
2036
|
+
# Page two of a search. Not free: the server asks the index for `offset + k` and
|
|
2037
|
+
# drops the prefix, because an approximate index cannot start in the middle of a
|
|
2038
|
+
# ranking it has not computed. To walk a whole table, `scroll` stays flat.
|
|
2039
|
+
payload["offset"] = offset
|
|
2040
|
+
if mmr is not None:
|
|
2041
|
+
# Diversity reranking, done by the server, which already holds the vectors:
|
|
2042
|
+
# `{"lambda_mult": 0.5, "fetch_k": 20}`. 1.0 is the plain ranking, 0.0 ignores the
|
|
2043
|
+
# query. Only candidates that were fetched can contribute diversity, so `fetch_k`
|
|
2044
|
+
# is what it costs.
|
|
2045
|
+
payload["mmr"] = mmr
|
|
2046
|
+
if rerank is not None:
|
|
2047
|
+
# Re-score the page with the table's rerank provider: `{}` for the defaults,
|
|
2048
|
+
# `{"top_n": 50, "field": "body"}` to size the pool or name the text field. Needs
|
|
2049
|
+
# `text`, and a table created with `rerank=...`; each hit gains `rerank_score`.
|
|
2050
|
+
payload["rerank"] = rerank
|
|
2051
|
+
try:
|
|
2052
|
+
response = self._client.post(
|
|
2053
|
+
f"/tables/{_seg(table_name)}/search",
|
|
2054
|
+
content=_encode_json(payload),
|
|
2055
|
+
headers=_JSON_HEADERS,
|
|
2056
|
+
)
|
|
2057
|
+
response.raise_for_status()
|
|
2058
|
+
return self._normalize_hits(response.json())
|
|
2059
|
+
except httpx.HTTPStatusError as e:
|
|
2060
|
+
msg = f"Search failed: {e.response.text}"
|
|
2061
|
+
raise DXMTSearchError(msg) from e
|
|
2062
|
+
except httpx.RequestError as e:
|
|
2063
|
+
msg = f"Connection error during search: {e}"
|
|
2064
|
+
raise DXMTConnectionError(msg) from e
|
|
2065
|
+
|
|
2066
|
+
async def asearch(
|
|
2067
|
+
self,
|
|
2068
|
+
table_name: str,
|
|
2069
|
+
vector: Optional[list[float]] = None,
|
|
2070
|
+
k: Optional[int] = None,
|
|
2071
|
+
limit: Optional[int] = None,
|
|
2072
|
+
filter: Optional[Filter] = None,
|
|
2073
|
+
search_mode: Optional[str] = None,
|
|
2074
|
+
score_threshold: Optional[float] = None,
|
|
2075
|
+
payload_fields: Optional[list[str]] = None,
|
|
2076
|
+
offset: int = 0,
|
|
2077
|
+
mmr: Optional[dict[str, Any]] = None,
|
|
2078
|
+
text: Optional[str] = None,
|
|
2079
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
2080
|
+
) -> list[dict[str, Any]]:
|
|
2081
|
+
"""Async vector similarity search.
|
|
2082
|
+
|
|
2083
|
+
Args:
|
|
2084
|
+
table_name: Name of the table.
|
|
2085
|
+
vector: Query vector. Omit it when passing `text`.
|
|
2086
|
+
text: Query text, embedded by the table's own model -- which is the only model it
|
|
2087
|
+
may be embedded with, since a query vector from a different model than the
|
|
2088
|
+
corpus is not a worse answer but an unrelated one. Requires the table to have
|
|
2089
|
+
been created with `embedding=...`.
|
|
2090
|
+
k: Number of results to return (preferred parameter name).
|
|
2091
|
+
limit: Number of results to return (alias for k, for compatibility).
|
|
2092
|
+
filter: Optional metadata filter.
|
|
2093
|
+
|
|
2094
|
+
Returns:
|
|
2095
|
+
List of search results.
|
|
2096
|
+
"""
|
|
2097
|
+
# Support both k and limit parameters
|
|
2098
|
+
if k is None and limit is None:
|
|
2099
|
+
k = 4
|
|
2100
|
+
elif k is None:
|
|
2101
|
+
k = limit
|
|
2102
|
+
|
|
2103
|
+
# `text` is embedded by the table's own model, server-side. The two are exclusive
|
|
2104
|
+
# and the server refuses a body carrying both, rather than quietly preferring one --
|
|
2105
|
+
# which is how a caller ends up measuring the wrong query for a week.
|
|
2106
|
+
if (vector is None) == (text is None):
|
|
2107
|
+
msg = "search takes either a vector or text, not both and not neither"
|
|
2108
|
+
raise ValueError(msg)
|
|
2109
|
+
payload: dict[str, Any] = {
|
|
2110
|
+
"top_k": k,
|
|
2111
|
+
"limit": k,
|
|
2112
|
+
"filter": convert_simple_filter_to_dxmt(filter),
|
|
2113
|
+
}
|
|
2114
|
+
if text is not None:
|
|
2115
|
+
payload["text"] = text
|
|
2116
|
+
else:
|
|
2117
|
+
payload["vector"] = vector
|
|
2118
|
+
if search_mode is not None:
|
|
2119
|
+
payload["search_mode"] = search_mode
|
|
2120
|
+
# A floor on the similarity the server returns (higher is nearer), not on a distance.
|
|
2121
|
+
if score_threshold is not None:
|
|
2122
|
+
payload["score_threshold"] = score_threshold
|
|
2123
|
+
# Only these payload keys come back. A payload is often mostly the text the embedding
|
|
2124
|
+
# came from, so a page of hits can be a megabyte of JSON to deliver two fields.
|
|
2125
|
+
if payload_fields is not None:
|
|
2126
|
+
payload["payload_fields"] = payload_fields
|
|
2127
|
+
if offset:
|
|
2128
|
+
payload["offset"] = offset
|
|
2129
|
+
if mmr is not None:
|
|
2130
|
+
payload["mmr"] = mmr
|
|
2131
|
+
if rerank is not None:
|
|
2132
|
+
# Re-score the page with the table's rerank provider: `{}` for the defaults,
|
|
2133
|
+
# `{"top_n": 50, "field": "body"}` to size the pool or name the text field. Needs
|
|
2134
|
+
# `text`, and a table created with `rerank=...`; each hit gains `rerank_score`.
|
|
2135
|
+
payload["rerank"] = rerank
|
|
2136
|
+
try:
|
|
2137
|
+
response = await self.async_client.post(
|
|
2138
|
+
f"/tables/{_seg(table_name)}/search",
|
|
2139
|
+
json=payload,
|
|
2140
|
+
)
|
|
2141
|
+
response.raise_for_status()
|
|
2142
|
+
return self._normalize_hits(response.json())
|
|
2143
|
+
except httpx.HTTPStatusError as e:
|
|
2144
|
+
msg = f"Search failed: {e.response.text}"
|
|
2145
|
+
raise DXMTSearchError(msg) from e
|
|
2146
|
+
except httpx.RequestError as e:
|
|
2147
|
+
msg = f"Connection error during search: {e}"
|
|
2148
|
+
raise DXMTConnectionError(msg) from e
|
|
2149
|
+
|
|
2150
|
+
@staticmethod
|
|
2151
|
+
def _fusion_body(vector_weight, alpha, fusion_method) -> dict[str, Any]:
|
|
2152
|
+
"""The fusion half of a hybrid-search body.
|
|
2153
|
+
|
|
2154
|
+
`dbsf` normalises each side by its own distribution and **ignores `vector_weight`**,
|
|
2155
|
+
and it was the server's default for every request until 2026-10-13 (now: `weighted_sum`
|
|
2156
|
+
at 0.6, or `dbsf` when there is a sparse vector). This SDK sent `vector_weight` and never
|
|
2157
|
+
a `fusion_method`, so every weight a
|
|
2158
|
+
caller passed, and the 0.7 default it invented for them, went to a method that does not
|
|
2159
|
+
read it: `vector_weight=0.0` and `vector_weight=1.0` returned the same rows with the
|
|
2160
|
+
same scores. Measured, not assumed.
|
|
2161
|
+
|
|
2162
|
+
So a weight now implies a method that reads one (`weighted_sum`, min-max normalisation
|
|
2163
|
+
then a weighted sum, which is what "70% vector" means to a reader), and asking for no
|
|
2164
|
+
weight sends neither field and leaves the choice to the server's default.
|
|
2165
|
+
"""
|
|
2166
|
+
if vector_weight is None:
|
|
2167
|
+
vector_weight = alpha
|
|
2168
|
+
body: dict[str, Any] = {}
|
|
2169
|
+
if fusion_method is not None:
|
|
2170
|
+
body["fusion_method"] = fusion_method
|
|
2171
|
+
if vector_weight is not None:
|
|
2172
|
+
body["vector_weight"] = vector_weight
|
|
2173
|
+
elif vector_weight is not None:
|
|
2174
|
+
body["vector_weight"] = vector_weight
|
|
2175
|
+
body["fusion_method"] = "weighted_sum"
|
|
2176
|
+
return body
|
|
2177
|
+
|
|
2178
|
+
def hybrid_search(
|
|
2179
|
+
self,
|
|
2180
|
+
table_name: str,
|
|
2181
|
+
vector: Optional[list[float]] = None,
|
|
2182
|
+
text: Optional[str] = None,
|
|
2183
|
+
text_query: Optional[str] = None, # Alias for text
|
|
2184
|
+
vector_weight: Optional[float] = None,
|
|
2185
|
+
alpha: Optional[float] = None, # Alias for vector_weight
|
|
2186
|
+
fusion_method: Optional[str] = None,
|
|
2187
|
+
k: Optional[int] = None,
|
|
2188
|
+
limit: Optional[int] = None, # Alias for k
|
|
2189
|
+
filter: Optional[Filter] = None,
|
|
2190
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
2191
|
+
) -> list[dict[str, Any]]:
|
|
2192
|
+
"""Hybrid search (vector + text).
|
|
2193
|
+
|
|
2194
|
+
Args:
|
|
2195
|
+
table_name: Name of the table.
|
|
2196
|
+
vector: Query vector (optional if text provided).
|
|
2197
|
+
text: Query text (optional if vector provided).
|
|
2198
|
+
text_query: Alias for text parameter (for compatibility).
|
|
2199
|
+
vector_weight: Weight for vector similarity (0-1). Sending one selects
|
|
2200
|
+
`weighted_sum` fusion, which is the method that reads it; leaving it unset
|
|
2201
|
+
leaves the server's default (`weighted_sum` at 0.6).
|
|
2202
|
+
alpha: Alias for vector_weight (for compatibility).
|
|
2203
|
+
fusion_method: "weighted_sum" (the server's default), "dbsf" (its default with a
|
|
2204
|
+
sparse vector) or "reciprocal_rank_fusion".
|
|
2205
|
+
k: Number of results to return (preferred parameter name).
|
|
2206
|
+
limit: Number of results to return (alias for k).
|
|
2207
|
+
filter: Optional metadata filter.
|
|
2208
|
+
|
|
2209
|
+
Returns:
|
|
2210
|
+
List of search results.
|
|
2211
|
+
|
|
2212
|
+
Raises:
|
|
2213
|
+
DXMTSearchError: If search fails.
|
|
2214
|
+
"""
|
|
2215
|
+
# Support text_query as alias for text
|
|
2216
|
+
if text is None and text_query is not None:
|
|
2217
|
+
text = text_query
|
|
2218
|
+
|
|
2219
|
+
fusion = self._fusion_body(vector_weight, alpha, fusion_method)
|
|
2220
|
+
|
|
2221
|
+
# Support limit as alias for k
|
|
2222
|
+
if k is None and limit is None:
|
|
2223
|
+
k = 4
|
|
2224
|
+
elif k is None:
|
|
2225
|
+
k = limit
|
|
2226
|
+
|
|
2227
|
+
payload = {
|
|
2228
|
+
"vector": vector,
|
|
2229
|
+
"text": text,
|
|
2230
|
+
"limit": k,
|
|
2231
|
+
"filter": convert_simple_filter_to_dxmt(filter),
|
|
2232
|
+
**fusion,
|
|
2233
|
+
}
|
|
2234
|
+
if rerank is not None:
|
|
2235
|
+
# The fused page, re-scored by the table's rerank provider against `text`.
|
|
2236
|
+
payload["rerank"] = rerank
|
|
2237
|
+
try:
|
|
2238
|
+
response = self._client.post(
|
|
2239
|
+
f"/tables/{_seg(table_name)}/search/hybrid",
|
|
2240
|
+
json=payload,
|
|
2241
|
+
)
|
|
2242
|
+
response.raise_for_status()
|
|
2243
|
+
return self._normalize_hits(response.json())
|
|
2244
|
+
except httpx.HTTPStatusError as e:
|
|
2245
|
+
msg = f"Hybrid search failed: {e.response.text}"
|
|
2246
|
+
raise DXMTSearchError(msg) from e
|
|
2247
|
+
except httpx.RequestError as e:
|
|
2248
|
+
msg = f"Connection error during hybrid search: {e}"
|
|
2249
|
+
raise DXMTConnectionError(msg) from e
|
|
2250
|
+
|
|
2251
|
+
async def ahybrid_search(
|
|
2252
|
+
self,
|
|
2253
|
+
table_name: str,
|
|
2254
|
+
vector: Optional[list[float]] = None,
|
|
2255
|
+
text: Optional[str] = None,
|
|
2256
|
+
text_query: Optional[str] = None, # Alias for text
|
|
2257
|
+
vector_weight: Optional[float] = None,
|
|
2258
|
+
alpha: Optional[float] = None, # Alias for vector_weight
|
|
2259
|
+
fusion_method: Optional[str] = None,
|
|
2260
|
+
k: Optional[int] = None,
|
|
2261
|
+
limit: Optional[int] = None, # Alias for k
|
|
2262
|
+
filter: Optional[Filter] = None,
|
|
2263
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
2264
|
+
) -> list[dict[str, Any]]:
|
|
2265
|
+
"""Async hybrid search.
|
|
2266
|
+
|
|
2267
|
+
Args:
|
|
2268
|
+
table_name: Name of the table.
|
|
2269
|
+
vector: Query vector (optional if text provided).
|
|
2270
|
+
text: Query text (optional if vector provided).
|
|
2271
|
+
text_query: Alias for text parameter (for compatibility).
|
|
2272
|
+
vector_weight: Weight for vector similarity (0-1). Sending one selects
|
|
2273
|
+
`weighted_sum` fusion, which is the method that reads it; leaving it unset
|
|
2274
|
+
leaves the server's default (`weighted_sum` at 0.6).
|
|
2275
|
+
alpha: Alias for vector_weight (for compatibility).
|
|
2276
|
+
fusion_method: "weighted_sum" (the server's default), "dbsf" (its default with a
|
|
2277
|
+
sparse vector) or "reciprocal_rank_fusion".
|
|
2278
|
+
k: Number of results to return (preferred parameter name).
|
|
2279
|
+
limit: Number of results to return (alias for k).
|
|
2280
|
+
filter: Optional metadata filter.
|
|
2281
|
+
|
|
2282
|
+
Returns:
|
|
2283
|
+
List of search results.
|
|
2284
|
+
"""
|
|
2285
|
+
# Support text_query as alias for text
|
|
2286
|
+
if text is None and text_query is not None:
|
|
2287
|
+
text = text_query
|
|
2288
|
+
|
|
2289
|
+
fusion = self._fusion_body(vector_weight, alpha, fusion_method)
|
|
2290
|
+
|
|
2291
|
+
# Support limit as alias for k
|
|
2292
|
+
if k is None and limit is None:
|
|
2293
|
+
k = 4
|
|
2294
|
+
elif k is None:
|
|
2295
|
+
k = limit
|
|
2296
|
+
|
|
2297
|
+
payload = {
|
|
2298
|
+
"vector": vector,
|
|
2299
|
+
"text": text,
|
|
2300
|
+
"limit": k,
|
|
2301
|
+
"filter": convert_simple_filter_to_dxmt(filter),
|
|
2302
|
+
**fusion,
|
|
2303
|
+
}
|
|
2304
|
+
if rerank is not None:
|
|
2305
|
+
# The fused page, re-scored by the table's rerank provider against `text`.
|
|
2306
|
+
payload["rerank"] = rerank
|
|
2307
|
+
try:
|
|
2308
|
+
response = await self.async_client.post(
|
|
2309
|
+
f"/tables/{_seg(table_name)}/search/hybrid",
|
|
2310
|
+
json=payload,
|
|
2311
|
+
)
|
|
2312
|
+
response.raise_for_status()
|
|
2313
|
+
return self._normalize_hits(response.json())
|
|
2314
|
+
except httpx.HTTPStatusError as e:
|
|
2315
|
+
msg = f"Hybrid search failed: {e.response.text}"
|
|
2316
|
+
raise DXMTSearchError(msg) from e
|
|
2317
|
+
except httpx.RequestError as e:
|
|
2318
|
+
msg = f"Connection error during hybrid search: {e}"
|
|
2319
|
+
raise DXMTConnectionError(msg) from e
|
|
2320
|
+
|
|
2321
|
+
def full_text_search(
|
|
2322
|
+
self,
|
|
2323
|
+
table_name: str,
|
|
2324
|
+
query: Optional[str] = None,
|
|
2325
|
+
text_query: Optional[str] = None, # Alias for query
|
|
2326
|
+
k: Optional[int] = None,
|
|
2327
|
+
limit: Optional[int] = None, # Alias for k
|
|
2328
|
+
fields: Optional[list[str]] = None,
|
|
2329
|
+
highlight: bool = False,
|
|
2330
|
+
filter: Optional[Filter] = None,
|
|
2331
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
2332
|
+
) -> list[dict[str, Any]]:
|
|
2333
|
+
"""Full-text search.
|
|
2334
|
+
|
|
2335
|
+
Args:
|
|
2336
|
+
table_name: Name of the table.
|
|
2337
|
+
query: Search query text (preferred parameter name).
|
|
2338
|
+
text_query: Alias for query parameter (for compatibility).
|
|
2339
|
+
k: Number of results to return (preferred parameter name).
|
|
2340
|
+
limit: Number of results to return (alias for k).
|
|
2341
|
+
fields: Optional list of fields to search.
|
|
2342
|
+
highlight: Whether to include highlights.
|
|
2343
|
+
filter: Optional metadata filter conditions.
|
|
2344
|
+
|
|
2345
|
+
Returns:
|
|
2346
|
+
List of search results.
|
|
2347
|
+
|
|
2348
|
+
Raises:
|
|
2349
|
+
DXMTSearchError: If search fails.
|
|
2350
|
+
ValueError: If neither query nor text_query is provided.
|
|
2351
|
+
"""
|
|
2352
|
+
# Support text_query as alias for query
|
|
2353
|
+
if query is None and text_query is not None:
|
|
2354
|
+
query = text_query
|
|
2355
|
+
elif query is None:
|
|
2356
|
+
msg = "Either 'query' or 'text_query' parameter is required"
|
|
2357
|
+
raise ValueError(msg)
|
|
2358
|
+
|
|
2359
|
+
# Support limit as alias for k
|
|
2360
|
+
if k is None and limit is None:
|
|
2361
|
+
k = 4
|
|
2362
|
+
elif k is None:
|
|
2363
|
+
k = limit
|
|
2364
|
+
|
|
2365
|
+
payload = {
|
|
2366
|
+
"query": query,
|
|
2367
|
+
"limit": k,
|
|
2368
|
+
"fields": fields,
|
|
2369
|
+
"highlight": highlight,
|
|
2370
|
+
# Converted, like every other search method. This was passed through raw, so a
|
|
2371
|
+
# filter written the documented way -- {"year": {"$gte": 2022}} -- worked on
|
|
2372
|
+
# `search` and `hybrid_search` and reached the server unconverted here.
|
|
2373
|
+
"filter": convert_simple_filter_to_dxmt(filter),
|
|
2374
|
+
}
|
|
2375
|
+
if rerank is not None:
|
|
2376
|
+
# The BM25 page, re-scored by the table's rerank provider against `query`.
|
|
2377
|
+
payload["rerank"] = rerank
|
|
2378
|
+
try:
|
|
2379
|
+
response = self._client.post(
|
|
2380
|
+
f"/tables/{_seg(table_name)}/search/text",
|
|
2381
|
+
json=payload,
|
|
2382
|
+
)
|
|
2383
|
+
response.raise_for_status()
|
|
2384
|
+
return self._normalize_hits(response.json())
|
|
2385
|
+
except httpx.HTTPStatusError as e:
|
|
2386
|
+
msg = f"Full-text search failed: {e.response.text}"
|
|
2387
|
+
raise DXMTSearchError(msg) from e
|
|
2388
|
+
except httpx.RequestError as e:
|
|
2389
|
+
msg = f"Connection error during full-text search: {e}"
|
|
2390
|
+
raise DXMTConnectionError(msg) from e
|
|
2391
|
+
|
|
2392
|
+
def query(
|
|
2393
|
+
self,
|
|
2394
|
+
sql: str,
|
|
2395
|
+
) -> dict[str, Any]:
|
|
2396
|
+
"""Execute a DXQL query.
|
|
2397
|
+
|
|
2398
|
+
Args:
|
|
2399
|
+
sql: DXQL query string (SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, DROP TABLE).
|
|
2400
|
+
|
|
2401
|
+
Returns:
|
|
2402
|
+
Dict with 'records' list and 'metadata' dict.
|
|
2403
|
+
|
|
2404
|
+
Raises:
|
|
2405
|
+
DXMTError: If query fails.
|
|
2406
|
+
|
|
2407
|
+
Example:
|
|
2408
|
+
client.query("SELECT * FROM my_table WHERE category = 'AI' LIMIT 10")
|
|
2409
|
+
"""
|
|
2410
|
+
try:
|
|
2411
|
+
response = self._client.post(
|
|
2412
|
+
"/query",
|
|
2413
|
+
json={"query": sql},
|
|
2414
|
+
)
|
|
2415
|
+
response.raise_for_status()
|
|
2416
|
+
data = self._response_data(response.json())
|
|
2417
|
+
return data if isinstance(data, dict) else {}
|
|
2418
|
+
except httpx.HTTPStatusError as e:
|
|
2419
|
+
msg = f"DXQL query failed: {e.response.text}"
|
|
2420
|
+
raise DXMTError(msg) from e
|
|
2421
|
+
except httpx.RequestError as e:
|
|
2422
|
+
msg = f"Connection error during DXQL query: {e}"
|
|
2423
|
+
raise DXMTConnectionError(msg) from e
|
|
2424
|
+
|
|
2425
|
+
async def aquery(
|
|
2426
|
+
self,
|
|
2427
|
+
sql: str,
|
|
2428
|
+
) -> dict[str, Any]:
|
|
2429
|
+
"""Async execute a DXQL query."""
|
|
2430
|
+
try:
|
|
2431
|
+
response = await self.async_client.post(
|
|
2432
|
+
"/query",
|
|
2433
|
+
json={"query": sql},
|
|
2434
|
+
)
|
|
2435
|
+
response.raise_for_status()
|
|
2436
|
+
data = self._response_data(response.json())
|
|
2437
|
+
return data if isinstance(data, dict) else {}
|
|
2438
|
+
except httpx.HTTPStatusError as e:
|
|
2439
|
+
msg = f"DXQL query failed: {e.response.text}"
|
|
2440
|
+
raise DXMTError(msg) from e
|
|
2441
|
+
except httpx.RequestError as e:
|
|
2442
|
+
msg = f"Connection error during DXQL query: {e}"
|
|
2443
|
+
raise DXMTConnectionError(msg) from e
|
|
2444
|
+
|
|
2445
|
+
async def afull_text_search(
|
|
2446
|
+
self,
|
|
2447
|
+
table_name: str,
|
|
2448
|
+
query: Optional[str] = None,
|
|
2449
|
+
text_query: Optional[str] = None, # Alias for query
|
|
2450
|
+
k: Optional[int] = None,
|
|
2451
|
+
limit: Optional[int] = None, # Alias for k
|
|
2452
|
+
fields: Optional[list[str]] = None,
|
|
2453
|
+
highlight: bool = False,
|
|
2454
|
+
filter: Optional[Filter] = None,
|
|
2455
|
+
rerank: Optional[dict[str, Any]] = None,
|
|
2456
|
+
) -> list[dict[str, Any]]:
|
|
2457
|
+
"""Async full-text search.
|
|
2458
|
+
|
|
2459
|
+
Args:
|
|
2460
|
+
table_name: Name of the table.
|
|
2461
|
+
query: Search query text (preferred parameter name).
|
|
2462
|
+
text_query: Alias for query parameter (for compatibility).
|
|
2463
|
+
k: Number of results to return (preferred parameter name).
|
|
2464
|
+
limit: Number of results to return (alias for k).
|
|
2465
|
+
fields: Optional list of fields to search.
|
|
2466
|
+
highlight: Whether to include highlights.
|
|
2467
|
+
filter: Optional metadata filter conditions.
|
|
2468
|
+
|
|
2469
|
+
Returns:
|
|
2470
|
+
List of search results.
|
|
2471
|
+
"""
|
|
2472
|
+
# Support text_query as alias for query
|
|
2473
|
+
if query is None and text_query is not None:
|
|
2474
|
+
query = text_query
|
|
2475
|
+
elif query is None:
|
|
2476
|
+
msg = "Either 'query' or 'text_query' parameter is required"
|
|
2477
|
+
raise ValueError(msg)
|
|
2478
|
+
|
|
2479
|
+
# Support limit as alias for k
|
|
2480
|
+
if k is None and limit is None:
|
|
2481
|
+
k = 4
|
|
2482
|
+
elif k is None:
|
|
2483
|
+
k = limit
|
|
2484
|
+
|
|
2485
|
+
payload = {
|
|
2486
|
+
"query": query,
|
|
2487
|
+
"limit": k,
|
|
2488
|
+
"fields": fields,
|
|
2489
|
+
"highlight": highlight,
|
|
2490
|
+
# Converted, like every other search method. This was passed through raw, so a
|
|
2491
|
+
# filter written the documented way -- {"year": {"$gte": 2022}} -- worked on
|
|
2492
|
+
# `search` and `hybrid_search` and reached the server unconverted here.
|
|
2493
|
+
"filter": convert_simple_filter_to_dxmt(filter),
|
|
2494
|
+
}
|
|
2495
|
+
if rerank is not None:
|
|
2496
|
+
# The BM25 page, re-scored by the table's rerank provider against `query`.
|
|
2497
|
+
payload["rerank"] = rerank
|
|
2498
|
+
try:
|
|
2499
|
+
response = await self.async_client.post(
|
|
2500
|
+
f"/tables/{_seg(table_name)}/search/text",
|
|
2501
|
+
json=payload,
|
|
2502
|
+
)
|
|
2503
|
+
response.raise_for_status()
|
|
2504
|
+
return self._normalize_hits(response.json())
|
|
2505
|
+
except httpx.HTTPStatusError as e:
|
|
2506
|
+
msg = f"Full-text search failed: {e.response.text}"
|
|
2507
|
+
raise DXMTSearchError(msg) from e
|
|
2508
|
+
except httpx.RequestError as e:
|
|
2509
|
+
msg = f"Connection error during full-text search: {e}"
|
|
2510
|
+
raise DXMTConnectionError(msg) from e
|