vlite-client 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vlite_client/__init__.py +1009 -0
- vlite_client/integrations/__init__.py +13 -0
- vlite_client/integrations/_common.py +44 -0
- vlite_client/integrations/langchain.py +110 -0
- vlite_client/integrations/llamaindex.py +122 -0
- vlite_client/integrations/test_integrations.py +188 -0
- vlite_client/py.typed +0 -0
- vlite_client-0.3.0.dist-info/METADATA +26 -0
- vlite_client-0.3.0.dist-info/RECORD +12 -0
- vlite_client-0.3.0.dist-info/WHEEL +5 -0
- vlite_client-0.3.0.dist-info/licenses/LICENSE +21 -0
- vlite_client-0.3.0.dist-info/top_level.txt +1 -0
vlite_client/__init__.py
ADDED
|
@@ -0,0 +1,1009 @@
|
|
|
1
|
+
"""Synchronous VLite REST client. No retries or redirect following."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from http.client import HTTPException
|
|
5
|
+
import math
|
|
6
|
+
from os import PathLike
|
|
7
|
+
import re
|
|
8
|
+
from typing import Literal, NotRequired, TypedDict
|
|
9
|
+
from urllib.error import HTTPError, URLError
|
|
10
|
+
from urllib.parse import urlencode, urlsplit
|
|
11
|
+
from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class StringValue(TypedDict):
|
|
15
|
+
type: Literal["string"]
|
|
16
|
+
value: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class IntegerValue(TypedDict):
|
|
20
|
+
type: Literal["integer"]
|
|
21
|
+
value: str
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class FloatValue(TypedDict):
|
|
25
|
+
type: Literal["float"]
|
|
26
|
+
value: float
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class BooleanValue(TypedDict):
|
|
30
|
+
type: Literal["boolean"]
|
|
31
|
+
value: bool
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
MetadataValue = StringValue | IntegerValue | FloatValue | BooleanValue
|
|
35
|
+
Metadata = dict[str, MetadataValue]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Put(TypedDict):
|
|
39
|
+
vector: list[float]
|
|
40
|
+
payload_base64: NotRequired[str]
|
|
41
|
+
metadata: NotRequired[Metadata]
|
|
42
|
+
expires_at_ms: NotRequired[int | None]
|
|
43
|
+
ttl_ms: NotRequired[int]
|
|
44
|
+
ttl: NotRequired[int]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Record(TypedDict):
|
|
48
|
+
id: str
|
|
49
|
+
vector: list[float]
|
|
50
|
+
payload_base64: str
|
|
51
|
+
metadata: Metadata
|
|
52
|
+
# Per-record revision (decimal u64 string); absent on older servers.
|
|
53
|
+
revision: NotRequired[str]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class VersionedRecord(TypedDict):
|
|
57
|
+
record: Record
|
|
58
|
+
etag: str
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Summary(TypedDict):
|
|
62
|
+
id: str
|
|
63
|
+
metadata: Metadata
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Page(TypedDict):
|
|
67
|
+
items: list[Summary]
|
|
68
|
+
next_after: str | None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Equal(TypedDict):
|
|
72
|
+
op: Literal["eq"]
|
|
73
|
+
field: str
|
|
74
|
+
value: MetadataValue
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Range(TypedDict):
|
|
78
|
+
op: Literal["range"]
|
|
79
|
+
field: str
|
|
80
|
+
min: NotRequired[IntegerValue | FloatValue | None]
|
|
81
|
+
max: NotRequired[IntegerValue | FloatValue | None]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Fusion(TypedDict, total=False):
|
|
85
|
+
"""Hybrid RRF options: rrf_k 1..1000 (60), weights finite 0..10 not both 0
|
|
86
|
+
(1.0), candidates k..1000 (max(k, 50))."""
|
|
87
|
+
|
|
88
|
+
rrf_k: int
|
|
89
|
+
vector_weight: float
|
|
90
|
+
text_weight: float
|
|
91
|
+
candidates: int
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class Search(TypedDict):
|
|
95
|
+
"""One request shape for every mode. Approximate modes support squared_l2 and
|
|
96
|
+
cosine only. ``filter`` is an AND of typed conditions (None/[] keeps every
|
|
97
|
+
record), valid in exact mode and, on servers advertising
|
|
98
|
+
``features.filtered_ann``, in both approximate modes; those return at most k
|
|
99
|
+
eligible records and may return fewer. ``ef_search`` (approximate only) then
|
|
100
|
+
controls eligible-candidate retention, not a visit/time cap.
|
|
101
|
+
|
|
102
|
+
``auto`` lets the server choose exact (scan or metadata-index candidates) or a
|
|
103
|
+
*ready* ANN index; it never builds. ``hybrid`` fuses BM25 over declared text
|
|
104
|
+
fields with a vector leg (``exact`` or ``approximate_ready``, never a lazy
|
|
105
|
+
build); ``vector`` may be omitted only in hybrid mode. ``explain`` adds
|
|
106
|
+
``plan``/``stats`` to the response (use ``search_detailed``)."""
|
|
107
|
+
|
|
108
|
+
vector: NotRequired[list[float]]
|
|
109
|
+
k: int
|
|
110
|
+
metric: NotRequired[Literal["squared_l2", "cosine", "inner_product"]]
|
|
111
|
+
mode: NotRequired[Literal["exact", "approximate", "approximate_ready", "auto", "hybrid"]]
|
|
112
|
+
ef_search: NotRequired[int]
|
|
113
|
+
filter: NotRequired[list[Equal | Range] | None]
|
|
114
|
+
text: NotRequired[str]
|
|
115
|
+
fusion: NotRequired[Fusion]
|
|
116
|
+
vector_leg: NotRequired[Literal["exact", "approximate_ready"]]
|
|
117
|
+
quantization: NotRequired[Literal["none", "scalar_u8"]]
|
|
118
|
+
rerank_multiplier: NotRequired[int]
|
|
119
|
+
config: NotRequired["IndexConfig"]
|
|
120
|
+
explain: NotRequired[bool]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class BatchPut(Put):
|
|
124
|
+
op: Literal["put"]
|
|
125
|
+
id: str
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class BatchDelete(TypedDict):
|
|
129
|
+
op: Literal["delete"]
|
|
130
|
+
id: str
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class Hit(TypedDict):
|
|
134
|
+
id: str
|
|
135
|
+
distance: float
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class HybridHit(TypedDict):
|
|
139
|
+
"""Ordered by score descending, then numeric ID."""
|
|
140
|
+
|
|
141
|
+
id: str
|
|
142
|
+
score: float
|
|
143
|
+
distance: float | None
|
|
144
|
+
bm25: float | None
|
|
145
|
+
vector_rank: int | None
|
|
146
|
+
text_rank: int | None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class QueryPlan(TypedDict):
|
|
150
|
+
strategy: Literal["exact_scan", "exact_indexed", "filtered_ann", "ann"]
|
|
151
|
+
estimated_candidates: int | str | None
|
|
152
|
+
live_records: int | str
|
|
153
|
+
index_field: str | None
|
|
154
|
+
reason: str
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class SearchStats(TypedDict):
|
|
158
|
+
records_scanned: int
|
|
159
|
+
distance_evaluations: int
|
|
160
|
+
quantized_distance_evaluations: int
|
|
161
|
+
graph_nodes_visited: int
|
|
162
|
+
buffered_records_merged: int
|
|
163
|
+
reranked_candidates: int
|
|
164
|
+
index_candidates: int
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class SearchResponse(TypedDict):
|
|
168
|
+
hits: list[Hit] | list[HybridHit]
|
|
169
|
+
# Present only with explain; plan is None outside auto mode.
|
|
170
|
+
plan: NotRequired[QueryPlan | None]
|
|
171
|
+
stats: NotRequired[SearchStats]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class MetadataIndex(TypedDict):
|
|
175
|
+
field: str
|
|
176
|
+
kind: Literal["eq", "range", "eq_range"]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class PlannerPolicy(TypedDict, total=False):
|
|
180
|
+
exact_max_candidates: int
|
|
181
|
+
exact_max_selectivity_ppm: int
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class HnswSettings(TypedDict, total=False):
|
|
185
|
+
m: int
|
|
186
|
+
ef_construction: int
|
|
187
|
+
seed: str
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class CollectionSettings(TypedDict, total=False):
|
|
191
|
+
"""Per-collection settings. In requests every field is optional (a settings
|
|
192
|
+
update keeps omitted fields); responses carry all fields."""
|
|
193
|
+
|
|
194
|
+
metric: Literal["squared_l2", "cosine", "inner_product"]
|
|
195
|
+
hnsw: HnswSettings
|
|
196
|
+
quantization: Literal["none", "scalar_u8"]
|
|
197
|
+
rerank_multiplier: int
|
|
198
|
+
planner: PlannerPolicy
|
|
199
|
+
metadata_indexes: list[MetadataIndex]
|
|
200
|
+
text_fields: list[str]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class CollectionInfo(TypedDict):
|
|
204
|
+
id: int
|
|
205
|
+
name: str
|
|
206
|
+
dimensions: int
|
|
207
|
+
live_records: int
|
|
208
|
+
settings: CollectionSettings
|
|
209
|
+
settings_persisted: bool
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class TextIndexStatus(TypedDict):
|
|
213
|
+
declared_fields: int
|
|
214
|
+
available: bool
|
|
215
|
+
documents: int
|
|
216
|
+
terms: int
|
|
217
|
+
postings: int
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class CollectionDetail(CollectionInfo):
|
|
221
|
+
text_index: TextIndexStatus
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class CollectionList(TypedDict):
|
|
225
|
+
items: list[CollectionInfo]
|
|
226
|
+
format_version: int
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class SettingsUpdate(TypedDict):
|
|
230
|
+
collection: CollectionInfo
|
|
231
|
+
persisted: bool
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class AnnMemory(TypedDict):
|
|
235
|
+
"""Component byte counts only; this is not total process memory."""
|
|
236
|
+
|
|
237
|
+
graph_nodes: int
|
|
238
|
+
graph_link_bytes: int
|
|
239
|
+
code_bytes: int
|
|
240
|
+
codebook_bytes: int
|
|
241
|
+
buffer_vector_bytes: int
|
|
242
|
+
resident_vector_bytes: int
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class CollectionIndexStatus(TypedDict):
|
|
246
|
+
"""Superset of the legacy IndexStatus (also returned by newer servers on
|
|
247
|
+
the legacy route). ``settings_ready``: the built graph matches the
|
|
248
|
+
collection's current settings."""
|
|
249
|
+
|
|
250
|
+
ready: bool
|
|
251
|
+
config: dict
|
|
252
|
+
metric: Literal["squared_l2", "cosine"] | None
|
|
253
|
+
quantization: Literal["none", "scalar_u8"]
|
|
254
|
+
rerank_multiplier: int
|
|
255
|
+
buffered_ids: int
|
|
256
|
+
buffer_vector_bytes: int
|
|
257
|
+
graph_nodes: int
|
|
258
|
+
stale_nodes: int
|
|
259
|
+
memory: AnnMemory | None
|
|
260
|
+
max_buffer_ids: int
|
|
261
|
+
max_buffer_vector_bytes: int
|
|
262
|
+
settings_ready: bool
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
class QualityOptions(TypedDict, total=False):
|
|
266
|
+
config: dict
|
|
267
|
+
metric: Literal["squared_l2", "cosine"]
|
|
268
|
+
quantization: Literal["none", "scalar_u8"]
|
|
269
|
+
rerank_multiplier: int
|
|
270
|
+
samples: int
|
|
271
|
+
k: int
|
|
272
|
+
ef_search: int
|
|
273
|
+
seed: str
|
|
274
|
+
max_distance_evaluations: int
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
class AnnQuality(TypedDict):
|
|
278
|
+
recall_at_k: float | None
|
|
279
|
+
samples_used: int
|
|
280
|
+
distance_evaluations: int
|
|
281
|
+
skipped_reason: Literal["budget", "empty"] | None
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
ImportPolicy = Literal["reject_chunk", "skip_invalid"]
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class InvalidRow(TypedDict):
|
|
288
|
+
row: int
|
|
289
|
+
code: str
|
|
290
|
+
message: str
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
class ImportChunkReport(TypedDict):
|
|
294
|
+
"""``outcome`` is explicit: ``committed`` or ``already_committed`` (an
|
|
295
|
+
identical resubmission of the last chunk; nothing written). A rejected chunk
|
|
296
|
+
raises ImportRejectedError whose ``report`` has outcome ``rejected``."""
|
|
297
|
+
|
|
298
|
+
outcome: Literal["committed", "already_committed", "rejected"]
|
|
299
|
+
job: str
|
|
300
|
+
chunk: int | str
|
|
301
|
+
rows_committed: int
|
|
302
|
+
rows_skipped: int
|
|
303
|
+
invalid_rows: int
|
|
304
|
+
invalid: list[InvalidRow]
|
|
305
|
+
revision: str | None
|
|
306
|
+
status: NotRequired["ImportStatus"]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class ImportStatus(TypedDict):
|
|
310
|
+
job: str
|
|
311
|
+
collection: str
|
|
312
|
+
collection_dropped: bool
|
|
313
|
+
state: Literal["open", "finished", "aborted"]
|
|
314
|
+
last_chunk: int | str
|
|
315
|
+
rows_committed: int | str
|
|
316
|
+
rows_skipped: int | str
|
|
317
|
+
last_digest: str | None
|
|
318
|
+
# False while a reserved job has no committed chunk (it would not survive a restart).
|
|
319
|
+
durable: bool
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
Operation = Literal["read", "search", "search_build", "write", "import", "index",
|
|
323
|
+
"collections", "diagnostics", "backup"]
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
class Credential(TypedDict):
|
|
327
|
+
"""Scoped credential description. Secrets and hashes are never returned.
|
|
328
|
+
Scopes are bound to collection identity and database lineage: a recreated
|
|
329
|
+
collection or another database fails closed (403 forbidden_scope)."""
|
|
330
|
+
|
|
331
|
+
id: str
|
|
332
|
+
name: str
|
|
333
|
+
collections: list[str]
|
|
334
|
+
operations: list[Operation]
|
|
335
|
+
lineage: str | None
|
|
336
|
+
created_unix_ms: int
|
|
337
|
+
expires_unix_ms: int
|
|
338
|
+
revoked_unix_ms: int | None
|
|
339
|
+
rotated_from: str | None
|
|
340
|
+
state: Literal["active", "expired", "revoked"]
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class NewCredential(TypedDict):
|
|
344
|
+
name: str # 1..64 characters
|
|
345
|
+
collections: list[str] # ["*"] or collection names
|
|
346
|
+
operations: list[Operation] # nonempty
|
|
347
|
+
expires_unix_ms: int # mandatory, at most now + 365 days
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class IssuedCredential(TypedDict):
|
|
351
|
+
credential: Credential
|
|
352
|
+
token: str # returned once; never log or store in plaintext
|
|
353
|
+
previous: NotRequired[Credential]
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
class DatabaseInfo(TypedDict):
|
|
357
|
+
dimensions: int
|
|
358
|
+
format_version: int
|
|
359
|
+
records: int
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
class IndexConfig(TypedDict):
|
|
363
|
+
m: int
|
|
364
|
+
ef_construction: int
|
|
365
|
+
seed: int
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
class IndexStatus(TypedDict):
|
|
369
|
+
metric: NotRequired[Literal["squared_l2", "cosine"] | None]
|
|
370
|
+
ready: bool
|
|
371
|
+
buffered_ids: int
|
|
372
|
+
buffer_vector_bytes: int
|
|
373
|
+
max_buffer_ids: int
|
|
374
|
+
max_buffer_vector_bytes: int
|
|
375
|
+
config: IndexConfig
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
class VLiteError(Exception):
|
|
379
|
+
"""HTTP failure; status, server code and request ID are available."""
|
|
380
|
+
|
|
381
|
+
def __init__(self, status: int, code: str, message: str, request_id: str | None):
|
|
382
|
+
self.status, self.code, self.request_id = status, code, request_id
|
|
383
|
+
super().__init__(f"HTTP {status}: {code}: {message}")
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class TransportError(Exception):
|
|
387
|
+
"""Network/timeout/invalid response. A mutation may already be committed."""
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
class UnsupportedError(Exception):
|
|
391
|
+
"""The server does not advertise conditional writes; nothing was mutated."""
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
class ImportRejectedError(VLiteError):
|
|
395
|
+
"""HTTP 422 import_rows_invalid: the chunk had invalid rows under the
|
|
396
|
+
reject_chunk policy. Nothing was written and the job did not advance;
|
|
397
|
+
``report`` lists the invalid rows (outcome ``rejected``)."""
|
|
398
|
+
|
|
399
|
+
def __init__(self, status: int, code: str, message: str, request_id: str | None, report: ImportChunkReport):
|
|
400
|
+
super().__init__(status, code, message, request_id)
|
|
401
|
+
self.report = report
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class _NoRedirect(HTTPRedirectHandler):
|
|
405
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
406
|
+
return None
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _id(value: str) -> str:
|
|
410
|
+
if not isinstance(value, str) or not re.fullmatch(r"0|[1-9][0-9]{0,19}", value) or int(value) > 18446744073709551615:
|
|
411
|
+
raise ValueError("ID must be a canonical decimal u64 string")
|
|
412
|
+
return value
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
# One strong RFC 9110 entity tag of visible ASCII: no W/ prefix, wildcard,
|
|
416
|
+
# whitespace, control characters, inner quotes or comma-separated lists.
|
|
417
|
+
_ETAG = re.compile(r'"[\x21\x23-\x7e]*"')
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _etag(value: str) -> str:
|
|
421
|
+
if not isinstance(value, str) or not _ETAG.fullmatch(value):
|
|
422
|
+
raise ValueError("if_match must be one exact strong entity tag returned by get_versioned")
|
|
423
|
+
return value
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _collection(value: str) -> str:
|
|
427
|
+
if not isinstance(value, str) or not re.fullmatch(r"[a-z][a-z0-9_-]{0,62}", value):
|
|
428
|
+
raise ValueError("collection name must match ^[a-z][a-z0-9_-]{0,62}$")
|
|
429
|
+
return value
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _credential(value: str) -> str:
|
|
433
|
+
if not isinstance(value, str) or not re.fullmatch(r"[a-z2-7]{12}", value):
|
|
434
|
+
raise ValueError("credential ID must be 12 characters of [a-z2-7]")
|
|
435
|
+
return value
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _backup(value: str) -> str:
|
|
439
|
+
# Generated snapshot names only; never a path, traversal or URL syntax.
|
|
440
|
+
if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,95}", value) or ".." in value:
|
|
441
|
+
raise ValueError("backup name must be a generated snapshot name")
|
|
442
|
+
return value
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _int(value, low: int, high: int, name: str) -> int:
|
|
446
|
+
if type(value) is not int or not low <= value <= high:
|
|
447
|
+
raise ValueError(f"{name} must be an integer in {low}..{high}")
|
|
448
|
+
return value
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _object(value, name: str) -> dict:
|
|
452
|
+
if not isinstance(value, dict):
|
|
453
|
+
raise ValueError(f"{name} must be an object")
|
|
454
|
+
return value
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _policy(value: str) -> str:
|
|
458
|
+
if value not in ("reject_chunk", "skip_invalid"):
|
|
459
|
+
raise ValueError("policy must be reject_chunk or skip_invalid")
|
|
460
|
+
return value
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _ndjson_chunks(file, size: int):
|
|
464
|
+
rows = []
|
|
465
|
+
for line_number, line in enumerate(file, 1):
|
|
466
|
+
if not line.strip():
|
|
467
|
+
continue
|
|
468
|
+
try:
|
|
469
|
+
row = json.loads(line)
|
|
470
|
+
except ValueError:
|
|
471
|
+
raise ValueError(f"line {line_number} is not valid JSON") from None
|
|
472
|
+
if not isinstance(row, dict):
|
|
473
|
+
raise ValueError(f"line {line_number} must be a JSON object")
|
|
474
|
+
rows.append(row)
|
|
475
|
+
if len(rows) == size:
|
|
476
|
+
yield rows
|
|
477
|
+
rows = []
|
|
478
|
+
if rows:
|
|
479
|
+
yield rows
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
class _Records:
|
|
483
|
+
"""Record, batch, search and index calls under one path prefix: ``/v1`` (the
|
|
484
|
+
legacy routes, which act on the ``default`` collection with historical
|
|
485
|
+
defaults) or ``/v1/collections/{name}`` (omitted metric/config/quantization
|
|
486
|
+
default to that collection's settings)."""
|
|
487
|
+
|
|
488
|
+
_prefix: str
|
|
489
|
+
|
|
490
|
+
def _transport(self) -> "Client":
|
|
491
|
+
raise NotImplementedError
|
|
492
|
+
|
|
493
|
+
def get(self, id: str) -> Record:
|
|
494
|
+
return self._transport()._request("GET", self._prefix + "/vectors/" + _id(id))
|
|
495
|
+
|
|
496
|
+
def get_versioned(self, id: str) -> VersionedRecord:
|
|
497
|
+
"""Return {"record": ..., "etag": ...}. The tag is opaque; pass it back
|
|
498
|
+
unchanged. Servers advertising ``features.per_record_etags`` invalidate it
|
|
499
|
+
only when this record changes (and, for v1-v3 files, on restart); older
|
|
500
|
+
servers invalidate it on any committed write or restart. A missing/invalid
|
|
501
|
+
ETag (for example, an older server) raises TransportError."""
|
|
502
|
+
record, tag = self._transport()._request("GET", self._prefix + "/vectors/" + _id(id), etag=True)
|
|
503
|
+
return {"record": record, "etag": tag}
|
|
504
|
+
|
|
505
|
+
def put(self, id: str, record: Put, *, if_match: str | None = None) -> None:
|
|
506
|
+
"""Replace a record. With if_match, send it unchanged as If-Match; a stale
|
|
507
|
+
tag raises VLiteError 412 precondition_failed and nothing is retried. A
|
|
508
|
+
conditional write first checks /v1/capabilities and raises UnsupportedError
|
|
509
|
+
on servers that do not advertise conditional_writes."""
|
|
510
|
+
path = self._prefix + "/vectors/" + _id(id)
|
|
511
|
+
client = self._transport()
|
|
512
|
+
client._request("PUT", path, record, if_match=client._conditional(if_match))
|
|
513
|
+
|
|
514
|
+
def delete(self, id: str, *, if_match: str | None = None) -> bool:
|
|
515
|
+
"""Delete a record. With if_match, a stale tag or missing record raises
|
|
516
|
+
VLiteError 412 precondition_failed and nothing is retried. Conditional deletes
|
|
517
|
+
check capabilities first, like put."""
|
|
518
|
+
path = self._prefix + "/vectors/" + _id(id)
|
|
519
|
+
client = self._transport()
|
|
520
|
+
return client._request("DELETE", path, if_match=client._conditional(if_match))["deleted"]
|
|
521
|
+
|
|
522
|
+
def list_vectors(self, *, limit: int = 25, after: str | None = None) -> Page:
|
|
523
|
+
if type(limit) is not int or not 1 <= limit <= 100:
|
|
524
|
+
raise ValueError("limit must be an integer in 1..100")
|
|
525
|
+
query = {"limit": str(limit)}
|
|
526
|
+
if after is not None:
|
|
527
|
+
query["after"] = _id(after)
|
|
528
|
+
return self._transport()._request("GET", self._prefix + "/vectors?" + urlencode(query))
|
|
529
|
+
|
|
530
|
+
def batch(self, operations: list[BatchPut | BatchDelete]) -> None:
|
|
531
|
+
for operation in operations:
|
|
532
|
+
_id(operation["id"])
|
|
533
|
+
self._transport()._request("POST", self._prefix + "/batches", {"operations": operations})
|
|
534
|
+
|
|
535
|
+
def search(self, request: Search) -> list[Hit] | list[HybridHit]:
|
|
536
|
+
"""Send the request unchanged and return the hits. There is no exact
|
|
537
|
+
fallback: an older server's 422 (approximate filter, unknown mode/field) or
|
|
538
|
+
a 409 from a ready-only mode is raised. Hybrid mode returns HybridHit
|
|
539
|
+
entries ordered by score descending; other modes order by distance."""
|
|
540
|
+
return self.search_detailed(request)["hits"]
|
|
541
|
+
|
|
542
|
+
def search_detailed(self, request: Search) -> SearchResponse:
|
|
543
|
+
"""Like search, but return the whole response. With ``explain: True`` it
|
|
544
|
+
carries ``stats`` and ``plan`` (``plan`` is null outside ``auto`` mode)."""
|
|
545
|
+
return self._transport()._request("POST", self._prefix + "/search", request)
|
|
546
|
+
|
|
547
|
+
def _feature_prefix(self) -> str:
|
|
548
|
+
return "/v1/collections/default" if self._prefix == "/v1" else self._prefix
|
|
549
|
+
|
|
550
|
+
def embed_text(self, text: str, *, provider: str = "hashing") -> dict:
|
|
551
|
+
if not isinstance(text, str) or len(text.encode("utf-8")) > 16384:
|
|
552
|
+
raise ValueError("text must be at most 16384 UTF-8 bytes")
|
|
553
|
+
if provider not in {"hashing", "http"}:
|
|
554
|
+
raise ValueError("provider must be hashing or http")
|
|
555
|
+
return self._transport()._request(
|
|
556
|
+
"POST", self._feature_prefix() + "/embed", {"text": text, "provider": provider}
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
def update_metadata(self, id: str, metadata: Metadata, *, if_match: str | None = None) -> None:
|
|
560
|
+
"""Replace metadata without rewriting vector or payload; requires V5."""
|
|
561
|
+
if not isinstance(metadata, dict):
|
|
562
|
+
raise ValueError("metadata must be an object")
|
|
563
|
+
self._transport()._request("PATCH", self._feature_prefix() + "/vectors/" + _id(id) + "/metadata",
|
|
564
|
+
{"metadata": metadata}, if_match=self._transport()._conditional(if_match))
|
|
565
|
+
|
|
566
|
+
def grouped_search(self, request: dict) -> dict:
|
|
567
|
+
return self._transport()._request("POST", self._feature_prefix() + "/search/grouped", _object(request, "request"))
|
|
568
|
+
|
|
569
|
+
def recommend(self, request: dict) -> dict:
|
|
570
|
+
return self._transport()._request("POST", self._feature_prefix() + "/recommend", _object(request, "request"))
|
|
571
|
+
|
|
572
|
+
def facets(self, field: str, *, filter: list | None = None) -> dict:
|
|
573
|
+
return self._transport()._request("POST", self._feature_prefix() + "/facets", {"field": field, "filter": filter or []})
|
|
574
|
+
|
|
575
|
+
def numeric_stats(self, field: str, *, filter: list | None = None) -> dict:
|
|
576
|
+
return self._transport()._request("POST", self._feature_prefix() + "/stats/numeric", {"field": field, "filter": filter or []})
|
|
577
|
+
|
|
578
|
+
def geo_search(self, request: dict) -> dict:
|
|
579
|
+
return self._transport()._request("POST", self._feature_prefix() + "/search/geo", _object(request, "request"))
|
|
580
|
+
|
|
581
|
+
def configure_vector(self, name: str, *, dimensions: int, metric: str = "squared_l2", active_version: str = "v1", history: int = 0) -> dict:
|
|
582
|
+
return self._transport()._request("POST", self._feature_prefix() + "/vectors/configure", {"name": name, "dimensions": dimensions, "metric": metric, "active_version": active_version, "history": history})
|
|
583
|
+
|
|
584
|
+
def put_named_vectors(self, id: str, *, dense: dict | None = None, sparse: dict | None = None) -> dict:
|
|
585
|
+
return self._transport()._request("PUT", self._feature_prefix() + "/vectors/" + _id(id) + "/named", {"dense": dense or {}, "sparse": sparse or {}})
|
|
586
|
+
|
|
587
|
+
def search_named(self, request: dict) -> dict:
|
|
588
|
+
return self._transport()._request("POST", self._feature_prefix() + "/search/named", _object(request, "request"))
|
|
589
|
+
|
|
590
|
+
def search_sparse(self, request: dict) -> dict:
|
|
591
|
+
return self._transport()._request("POST", self._feature_prefix() + "/search/sparse", _object(request, "request"))
|
|
592
|
+
|
|
593
|
+
def search_dense_sparse(self, request: dict) -> dict:
|
|
594
|
+
return self._transport()._request("POST", self._feature_prefix() + "/search/dense-sparse", _object(request, "request"))
|
|
595
|
+
|
|
596
|
+
def query(self, query: str, vectors: dict | None = None) -> dict:
|
|
597
|
+
return self._transport()._request("POST", self._feature_prefix() + "/query", {"query": query, "vectors": vectors or {}})
|
|
598
|
+
|
|
599
|
+
def build_index(self, *, metric: Literal["squared_l2", "cosine"] | None = None,
|
|
600
|
+
quantization: Literal["none", "scalar_u8"] | None = None,
|
|
601
|
+
rerank_multiplier: int | None = None) -> None:
|
|
602
|
+
"""Synchronously build the HNSW index. Arguments left as None are omitted:
|
|
603
|
+
legacy routes then use squared_l2 without quantization, collection routes
|
|
604
|
+
use the collection's settings."""
|
|
605
|
+
body = self._index_body(metric, quantization, rerank_multiplier)
|
|
606
|
+
self._transport()._request("POST", self._prefix + "/index/hnsw", body)
|
|
607
|
+
|
|
608
|
+
def rebuild_status(self) -> dict:
|
|
609
|
+
return self._transport()._request("GET", self._prefix + "/index/hnsw/rebuild")
|
|
610
|
+
|
|
611
|
+
def start_rebuild(self, request: dict | None = None) -> dict:
|
|
612
|
+
"""Start a background rebuild (202). ``request`` is sent unchanged (for
|
|
613
|
+
example ``metric``, ``config``, ``limits``, ``quantization``,
|
|
614
|
+
``rerank_multiplier``); a busy rebuild is a 409, never queued or retried."""
|
|
615
|
+
body = {} if request is None else _object(request, "request")
|
|
616
|
+
return self._transport()._request("POST", self._prefix + "/index/hnsw/rebuild", body)
|
|
617
|
+
|
|
618
|
+
def poll_rebuild(self) -> dict:
|
|
619
|
+
return self._transport()._request("POST", self._prefix + "/index/hnsw/rebuild/poll")
|
|
620
|
+
|
|
621
|
+
def cancel_rebuild(self) -> dict:
|
|
622
|
+
return self._transport()._request("POST", self._prefix + "/index/hnsw/rebuild/cancel")
|
|
623
|
+
|
|
624
|
+
def maintenance_status(self) -> dict:
|
|
625
|
+
"""``{policy, status}`` (collection routes add ``identity``)."""
|
|
626
|
+
return self._transport()._request("GET", self._prefix + "/index/hnsw/maintenance")
|
|
627
|
+
|
|
628
|
+
def maintenance_tick(self) -> dict:
|
|
629
|
+
"""Run one bounded maintenance step: ``{outcome, started, status}``."""
|
|
630
|
+
return self._transport()._request("POST", self._prefix + "/index/hnsw/maintenance/tick")
|
|
631
|
+
|
|
632
|
+
def _index_body(self, metric, quantization, rerank_multiplier) -> dict:
|
|
633
|
+
body: dict = {}
|
|
634
|
+
if metric is not None:
|
|
635
|
+
body["metric"] = metric
|
|
636
|
+
if quantization is not None:
|
|
637
|
+
body["quantization"] = quantization
|
|
638
|
+
if rerank_multiplier is not None:
|
|
639
|
+
body["rerank_multiplier"] = _int(rerank_multiplier, 1, 16, "rerank_multiplier")
|
|
640
|
+
return body
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
class Collection(_Records):
|
|
644
|
+
"""Record, search and index operations scoped to one named collection.
|
|
645
|
+
Obtain with ``Client.collection(name)``; it shares the client's connection.
|
|
646
|
+
An unknown collection raises VLiteError 404 ``collection_not_found``; nothing
|
|
647
|
+
falls back to ``default``."""
|
|
648
|
+
|
|
649
|
+
def __init__(self, client: "Client", name: str):
|
|
650
|
+
self._client, self.name = client, _collection(name)
|
|
651
|
+
self._prefix = "/v1/collections/" + name
|
|
652
|
+
|
|
653
|
+
def _transport(self) -> "Client":
|
|
654
|
+
return self._client
|
|
655
|
+
|
|
656
|
+
def info(self) -> CollectionDetail:
|
|
657
|
+
return self._client.get_collection(self.name)
|
|
658
|
+
|
|
659
|
+
def index_status(self) -> CollectionIndexStatus:
|
|
660
|
+
"""Readiness, identity (config/metric/quantization/rerank_multiplier),
|
|
661
|
+
graph/stale-node counts and component memory bytes (``memory``, null
|
|
662
|
+
without a graph; not process RSS)."""
|
|
663
|
+
return self._client._request("GET", self._prefix + "/index/hnsw")
|
|
664
|
+
|
|
665
|
+
def ann_quality(self, options: QualityOptions | None = None) -> AnnQuality:
|
|
666
|
+
"""Sampled recall@k of the *ready* index against exact search (409
|
|
667
|
+
index_not_ready otherwise; never builds). ``skipped_reason`` is "budget"
|
|
668
|
+
or "empty" when no sample ran."""
|
|
669
|
+
body = {} if options is None else _object(options, "options")
|
|
670
|
+
return self._client._request("POST", self._prefix + "/index/hnsw/quality", body)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
class Client(_Records):
|
|
674
|
+
_prefix = "/v1"
|
|
675
|
+
|
|
676
|
+
def __init__(self, base_url: str, token: str, *, timeout: float = 30):
|
|
677
|
+
url = urlsplit(base_url)
|
|
678
|
+
if url.scheme not in ("http", "https") or not url.hostname or url.username is not None or url.password is not None or url.query or url.fragment or url.path not in ("", "/"):
|
|
679
|
+
raise ValueError("base_url must be an HTTP(S) origin without credentials, path, query or fragment")
|
|
680
|
+
# Bootstrap tokens and scoped vlk_<id>_<secret> credentials share this alphabet.
|
|
681
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{32,256}", token):
|
|
682
|
+
raise ValueError("token must contain 32..256 base64url characters")
|
|
683
|
+
if not math.isfinite(timeout) or timeout <= 0:
|
|
684
|
+
raise ValueError("timeout must be finite and positive")
|
|
685
|
+
self._base, self._token, self._timeout = base_url.rstrip("/"), token, timeout
|
|
686
|
+
# Do not send the bearer token through implicitly configured environment proxies.
|
|
687
|
+
self._opener = build_opener(ProxyHandler({}), _NoRedirect())
|
|
688
|
+
|
|
689
|
+
def _transport(self) -> "Client":
|
|
690
|
+
return self
|
|
691
|
+
|
|
692
|
+
def _request(self, method: str, path: str, body=None, *, if_match: str | None = None, etag: bool = False,
|
|
693
|
+
text: bool = False):
|
|
694
|
+
data = None if body is None else json.dumps(body, allow_nan=False).encode("utf-8")
|
|
695
|
+
headers = {"Authorization": "Bearer " + self._token, "Accept": "text/plain" if text else "application/json"}
|
|
696
|
+
if data is not None:
|
|
697
|
+
headers["Content-Type"] = "application/json"
|
|
698
|
+
if if_match is not None:
|
|
699
|
+
headers["If-Match"] = if_match
|
|
700
|
+
request = Request(self._base + path, data=data, headers=headers, method=method)
|
|
701
|
+
try:
|
|
702
|
+
with self._opener.open(request, timeout=self._timeout) as response:
|
|
703
|
+
raw = response.read()
|
|
704
|
+
if text:
|
|
705
|
+
if not (response.headers.get("Content-Type") or "").startswith("text/plain"):
|
|
706
|
+
raise ValueError("expected a text/plain response")
|
|
707
|
+
return raw.decode("utf-8")
|
|
708
|
+
value = json.loads(raw) if raw else None
|
|
709
|
+
if not etag:
|
|
710
|
+
return value
|
|
711
|
+
tags = response.headers.get_all("ETag") or []
|
|
712
|
+
if len(tags) != 1 or not _ETAG.fullmatch(tags[0]):
|
|
713
|
+
raise ValueError("response must carry exactly one strong ETag")
|
|
714
|
+
return value, tags[0]
|
|
715
|
+
except HTTPError as error:
|
|
716
|
+
with error:
|
|
717
|
+
raw = error.read()
|
|
718
|
+
try:
|
|
719
|
+
document = json.loads(raw)
|
|
720
|
+
detail = document["error"]
|
|
721
|
+
code, message = detail["code"], detail["message"]
|
|
722
|
+
except (ValueError, KeyError, TypeError):
|
|
723
|
+
document, code, message = None, "http_error", "Server returned a nonstandard error response"
|
|
724
|
+
request_id = error.headers.get("X-Request-ID")
|
|
725
|
+
if error.code == 422 and code == "import_rows_invalid" and isinstance(document, dict):
|
|
726
|
+
report = {key: value for key, value in document.items() if key != "error"}
|
|
727
|
+
raise ImportRejectedError(error.code, code, message, request_id, dict(report, outcome="rejected")) from None
|
|
728
|
+
raise VLiteError(error.code, code, message, request_id) from None
|
|
729
|
+
except (URLError, OSError, HTTPException, ValueError) as error:
|
|
730
|
+
raise TransportError("Request failed or response was invalid; mutation outcome may be unknown; no retry performed") from error
|
|
731
|
+
|
|
732
|
+
def health(self) -> dict[str, str]:
|
|
733
|
+
return self._request("GET", "/health")
|
|
734
|
+
|
|
735
|
+
def ready(self) -> dict[str, str]:
|
|
736
|
+
return self._request("GET", "/ready")
|
|
737
|
+
|
|
738
|
+
def database(self) -> DatabaseInfo:
|
|
739
|
+
return self._request("GET", "/v1/database")
|
|
740
|
+
|
|
741
|
+
def metrics(self) -> dict[str, object]:
|
|
742
|
+
return self._request("GET", "/metrics")
|
|
743
|
+
|
|
744
|
+
def prometheus_metrics(self) -> str:
|
|
745
|
+
"""GET /metrics?format=prometheus as text (text/plain; version=0.0.4).
|
|
746
|
+
Any other content type raises TransportError."""
|
|
747
|
+
return self._request("GET", "/metrics?format=prometheus", text=True)
|
|
748
|
+
|
|
749
|
+
def capabilities(self) -> dict:
|
|
750
|
+
"""Fresh GET /v1/capabilities (never cached): role, format_version,
|
|
751
|
+
``features`` flags and, for scoped tokens, ``scope``."""
|
|
752
|
+
return self._request("GET", "/v1/capabilities")
|
|
753
|
+
|
|
754
|
+
def _require_conditional_writes(self) -> None:
|
|
755
|
+
# Older servers ignore If-Match; a fresh check per write avoids silently
|
|
756
|
+
# downgrading to an unconditional mutation (never cached across restarts).
|
|
757
|
+
try:
|
|
758
|
+
caps = self._request("GET", "/v1/capabilities")
|
|
759
|
+
except VLiteError as error:
|
|
760
|
+
if error.status != 404:
|
|
761
|
+
raise
|
|
762
|
+
caps = None
|
|
763
|
+
features = caps.get("features") if isinstance(caps, dict) else None
|
|
764
|
+
if not (isinstance(caps, dict) and type(caps.get("api_version")) is int and caps["api_version"] == 1
|
|
765
|
+
and isinstance(features, dict) and features.get("conditional_writes") is True):
|
|
766
|
+
raise UnsupportedError("Server does not advertise conditional_writes (API version 1); no write was sent")
|
|
767
|
+
|
|
768
|
+
def _conditional(self, if_match: str | None) -> str | None:
|
|
769
|
+
if if_match is None:
|
|
770
|
+
return None
|
|
771
|
+
_etag(if_match)
|
|
772
|
+
self._require_conditional_writes()
|
|
773
|
+
return if_match
|
|
774
|
+
|
|
775
|
+
def index_status(self) -> IndexStatus:
|
|
776
|
+
return self._request("GET", "/v1/index/hnsw")
|
|
777
|
+
|
|
778
|
+
def build_index(self, *, metric: Literal["squared_l2", "cosine"] | None = "squared_l2",
|
|
779
|
+
quantization: Literal["none", "scalar_u8"] | None = None,
|
|
780
|
+
rerank_multiplier: int | None = None) -> None:
|
|
781
|
+
"""Synchronously build the default collection's index (legacy route)."""
|
|
782
|
+
body = self._index_body(None if metric == "squared_l2" else metric, quantization, rerank_multiplier)
|
|
783
|
+
self._request("POST", "/v1/index/hnsw", body)
|
|
784
|
+
|
|
785
|
+
# Collections (v4 databases; v1-v3 return 409 requires_format_v4).
|
|
786
|
+
|
|
787
|
+
def collection(self, name: str) -> Collection:
|
|
788
|
+
"""Return a handle for one collection's record/search/index routes. No
|
|
789
|
+
request is made; ``default`` addresses collection 0 explicitly."""
|
|
790
|
+
return Collection(self, name)
|
|
791
|
+
|
|
792
|
+
def list_collections(self) -> CollectionList:
|
|
793
|
+
return self._request("GET", "/v1/collections")
|
|
794
|
+
|
|
795
|
+
def create_collection(self, name: str, dimensions: int, settings: CollectionSettings | None = None) -> CollectionInfo:
|
|
796
|
+
"""Create a collection (201). Omitted settings use server defaults.
|
|
797
|
+
Errors: 409 requires_format_v4 / collection_exists / collection_limit,
|
|
798
|
+
422 invalid_input."""
|
|
799
|
+
body: dict = {"name": _collection(name), "dimensions": _int(dimensions, 1, 2**32 - 1, "dimensions")}
|
|
800
|
+
if settings is not None:
|
|
801
|
+
body["settings"] = _object(settings, "settings")
|
|
802
|
+
return self._request("POST", "/v1/collections", body)
|
|
803
|
+
|
|
804
|
+
def get_collection(self, name: str) -> CollectionDetail:
|
|
805
|
+
return self._request("GET", "/v1/collections/" + _collection(name))
|
|
806
|
+
|
|
807
|
+
def drop_collection(self, name: str) -> bool:
|
|
808
|
+
"""Drop an empty collection (409 collection_not_empty otherwise)."""
|
|
809
|
+
return self._request("DELETE", "/v1/collections/" + _collection(name))["deleted"]
|
|
810
|
+
|
|
811
|
+
def set_collection_settings(self, name: str, settings: CollectionSettings) -> SettingsUpdate:
|
|
812
|
+
"""Replace the given settings fields (omitted fields keep their values).
|
|
813
|
+
``persisted`` is False when the change is memory-only (``default`` on a
|
|
814
|
+
v1-v3 file)."""
|
|
815
|
+
path = "/v1/collections/" + _collection(name) + "/settings"
|
|
816
|
+
return self._request("PUT", path, _object(settings, "settings"))
|
|
817
|
+
|
|
818
|
+
# Resumable imports (v4).
|
|
819
|
+
|
|
820
|
+
def begin_import(self, collection: str) -> ImportStatus:
|
|
821
|
+
"""Reserve a job. It becomes durable with its first committed chunk; a job
|
|
822
|
+
without a durable chunk does not survive a restart."""
|
|
823
|
+
return self._request("POST", "/v1/collections/" + _collection(collection) + "/imports")
|
|
824
|
+
|
|
825
|
+
def import_chunk(self, job: str, chunk: str, operations: list[BatchPut | BatchDelete], *,
|
|
826
|
+
policy: ImportPolicy | None = None) -> ImportChunkReport:
|
|
827
|
+
"""Commit one chunk (<= 4096 operations) atomically with its checkpoint.
|
|
828
|
+
Returns outcome ``committed`` or ``already_committed`` (identical resubmission
|
|
829
|
+
of the last chunk; nothing written). Invalid rows under ``reject_chunk``
|
|
830
|
+
raise ImportRejectedError (422; nothing written, job not advanced).
|
|
831
|
+
A different payload for a committed chunk, or a gap, is 409 import_conflict."""
|
|
832
|
+
body: dict = {"operations": operations}
|
|
833
|
+
if policy is not None:
|
|
834
|
+
body["policy"] = _policy(policy)
|
|
835
|
+
return self._request("PUT", f"/v1/imports/{_id(job)}/chunks/{_id(chunk)}", body)
|
|
836
|
+
|
|
837
|
+
def import_status(self, job: str) -> ImportStatus:
|
|
838
|
+
return self._request("GET", "/v1/imports/" + _id(job))
|
|
839
|
+
|
|
840
|
+
def finish_import(self, job: str) -> ImportStatus:
|
|
841
|
+
return self._request("POST", f"/v1/imports/{_id(job)}/finish")
|
|
842
|
+
|
|
843
|
+
def abort_import(self, job: str) -> ImportStatus:
|
|
844
|
+
"""Close the job; already committed chunks are not rolled back."""
|
|
845
|
+
return self._request("POST", f"/v1/imports/{_id(job)}/abort")
|
|
846
|
+
|
|
847
|
+
def import_ndjson(self, path: str | PathLike, collection: str, *, chunk_size: int = 1024,
|
|
848
|
+
policy: ImportPolicy = "reject_chunk", job: str | None = None) -> ImportStatus:
|
|
849
|
+
"""Import a file of batch operations (one JSON object per line; blank lines
|
|
850
|
+
ignored) in chunks of ``chunk_size`` (1..4096) and finish the job.
|
|
851
|
+
|
|
852
|
+
With ``job``, resume: the job's ``last_chunk`` is read from the server,
|
|
853
|
+
earlier chunks are skipped, the last committed chunk is resubmitted and must
|
|
854
|
+
be confirmed ``already_committed`` (proving the same file, chunk size and
|
|
855
|
+
policy), and the rest are sent. A finished job returns its status without
|
|
856
|
+
sending anything. Nothing is retried: a TransportError or VLiteError stops
|
|
857
|
+
the import and the job can be resumed; ImportRejectedError reports invalid
|
|
858
|
+
rows of the chunk that was not written.
|
|
859
|
+
|
|
860
|
+
The whole file is read and validated before a job is reserved or resumed,
|
|
861
|
+
so a missing, unreadable, empty or malformed file sends nothing. Any later
|
|
862
|
+
failure re-raises the original exception with ``import_job`` set to the
|
|
863
|
+
job ID; the job is never aborted automatically, so committed chunks stay
|
|
864
|
+
resumable (pass ``job=error.import_job``) or can be aborted explicitly."""
|
|
865
|
+
_collection(collection)
|
|
866
|
+
_int(chunk_size, 1, 4096, "chunk_size")
|
|
867
|
+
_policy(policy)
|
|
868
|
+
if job is not None:
|
|
869
|
+
_id(job)
|
|
870
|
+
with open(path, encoding="utf-8") as file:
|
|
871
|
+
chunks = sum(1 for _ in _ndjson_chunks(file, chunk_size))
|
|
872
|
+
if job is None and chunks == 0:
|
|
873
|
+
raise ValueError("file contains no operations")
|
|
874
|
+
if job is None:
|
|
875
|
+
status = self.begin_import(collection)
|
|
876
|
+
else:
|
|
877
|
+
status = self.import_status(job)
|
|
878
|
+
if status["collection"] != collection or status.get("collection_dropped"):
|
|
879
|
+
raise ValueError("import job belongs to a different collection")
|
|
880
|
+
if status["state"] == "finished":
|
|
881
|
+
return status
|
|
882
|
+
if status["state"] != "open":
|
|
883
|
+
raise ValueError("import job is not open")
|
|
884
|
+
job = _id(status["job"])
|
|
885
|
+
try:
|
|
886
|
+
last = status["last_chunk"]
|
|
887
|
+
if isinstance(last, str) and last.isascii() and last.isdigit():
|
|
888
|
+
last = int(last)
|
|
889
|
+
if isinstance(last, bool) or not isinstance(last, int) or last < 0:
|
|
890
|
+
raise TransportError("import status has an invalid last_chunk")
|
|
891
|
+
if chunks < last:
|
|
892
|
+
raise ValueError("file has fewer chunks than the job already committed")
|
|
893
|
+
count = 0
|
|
894
|
+
with open(path, encoding="utf-8") as file:
|
|
895
|
+
for count, rows in enumerate(_ndjson_chunks(file, chunk_size), 1):
|
|
896
|
+
if count < last:
|
|
897
|
+
continue
|
|
898
|
+
report = self.import_chunk(job, str(count), rows, policy=policy)
|
|
899
|
+
expected = "already_committed" if count == last else "committed"
|
|
900
|
+
if report.get("outcome") != expected:
|
|
901
|
+
raise TransportError(f"import chunk {count}: expected outcome {expected}; no retry performed")
|
|
902
|
+
if count != chunks:
|
|
903
|
+
raise ValueError("file changed during import")
|
|
904
|
+
return self.finish_import(job)
|
|
905
|
+
except Exception as error:
|
|
906
|
+
error.import_job = job
|
|
907
|
+
raise
|
|
908
|
+
|
|
909
|
+
# Operations (admin).
|
|
910
|
+
|
|
911
|
+
def changes(self, *, cursor: str | None = None, limit: int = 1000) -> dict:
|
|
912
|
+
if type(limit) is not int or not 1 <= limit <= 10000:
|
|
913
|
+
raise ValueError("limit must be an integer in 1..10000")
|
|
914
|
+
query = {"limit": str(limit)}
|
|
915
|
+
if cursor is not None:
|
|
916
|
+
query["cursor"] = cursor
|
|
917
|
+
return self._request("GET", "/v1/changes?" + urlencode(query))
|
|
918
|
+
|
|
919
|
+
def list_webhooks(self) -> dict:
|
|
920
|
+
return self._request("GET", "/v1/admin/webhooks")
|
|
921
|
+
|
|
922
|
+
def create_webhook(self, request: dict) -> dict:
|
|
923
|
+
return self._request("POST", "/v1/admin/webhooks", _object(request, "request"))
|
|
924
|
+
|
|
925
|
+
def webhook_status(self, webhook_id: str) -> dict:
|
|
926
|
+
if not re.fullmatch(r"[a-f0-9]{24}", webhook_id):
|
|
927
|
+
raise ValueError("invalid webhook ID")
|
|
928
|
+
return self._request("GET", "/v1/admin/webhooks/" + webhook_id)
|
|
929
|
+
|
|
930
|
+
def delete_webhook(self, webhook_id: str) -> bool:
|
|
931
|
+
if not re.fullmatch(r"[a-f0-9]{24}", webhook_id):
|
|
932
|
+
raise ValueError("invalid webhook ID")
|
|
933
|
+
return self._request("DELETE", "/v1/admin/webhooks/" + webhook_id)["deleted"]
|
|
934
|
+
|
|
935
|
+
def auto_maintenance(self) -> dict:
|
|
936
|
+
"""``{"config": {..., "interval_ms", ...}, "state": {...}}``."""
|
|
937
|
+
return self._request("GET", "/v1/admin/maintenance/auto")
|
|
938
|
+
|
|
939
|
+
def set_auto_maintenance(self, policy: dict) -> dict:
|
|
940
|
+
"""Merge ``enabled``, ``interval_secs`` (10..3600), ``stale_ratio``,
|
|
941
|
+
``rebuild_on_low_recall``, ``min_recall``, ``quality_samples``, ``limits``.
|
|
942
|
+
Memory-only (restart restores the environment policy)."""
|
|
943
|
+
return self._request("PUT", "/v1/admin/maintenance/auto", _object(policy, "policy"))
|
|
944
|
+
|
|
945
|
+
def tick_auto_maintenance(self) -> dict:
|
|
946
|
+
"""Run exactly one bounded auto-maintenance tick now."""
|
|
947
|
+
return self._request("POST", "/v1/admin/maintenance/auto/tick")
|
|
948
|
+
|
|
949
|
+
def backup_schedule(self) -> dict:
|
|
950
|
+
"""``{"config": {...}, "state": {...}}``."""
|
|
951
|
+
return self._request("GET", "/v1/admin/backups/schedule")
|
|
952
|
+
|
|
953
|
+
def run_backup_schedule(self) -> dict:
|
|
954
|
+
"""Run one scheduled backup cycle (backup, retention, drill) now:
|
|
955
|
+
``{"name", "manifest", "retention_deleted"}``."""
|
|
956
|
+
return self._request("POST", "/v1/admin/backups/schedule/run")
|
|
957
|
+
|
|
958
|
+
def set_backup_schedule(self, schedule: dict) -> dict:
|
|
959
|
+
"""Memory-only schedule change; retention deletes only with
|
|
960
|
+
``retention_enabled: True``."""
|
|
961
|
+
return self._request("PUT", "/v1/admin/backups/schedule", _object(schedule, "schedule"))
|
|
962
|
+
|
|
963
|
+
def list_backups(self) -> dict:
|
|
964
|
+
return self._request("GET", "/v1/admin/backups")
|
|
965
|
+
|
|
966
|
+
def create_backup(self) -> dict:
|
|
967
|
+
return self._request("POST", "/v1/admin/backups")
|
|
968
|
+
|
|
969
|
+
def verify_backup(self, name: str) -> dict:
|
|
970
|
+
"""Verify a backup against its manifest (422 verification_failed, also when
|
|
971
|
+
the manifest or its metadata was altered)."""
|
|
972
|
+
return self._request("POST", f"/v1/admin/backups/{_backup(name)}/verify")
|
|
973
|
+
|
|
974
|
+
def drill_backup(self, name: str) -> dict:
|
|
975
|
+
"""Restore to a scratch file, validate and query it, then delete it; the
|
|
976
|
+
active database is never touched (422 drill_failed)."""
|
|
977
|
+
return self._request("POST", f"/v1/admin/backups/{_backup(name)}/drill")
|
|
978
|
+
|
|
979
|
+
def slow_queries(self) -> dict:
|
|
980
|
+
return self._request("GET", "/v1/admin/diagnostics/slow-queries")
|
|
981
|
+
|
|
982
|
+
# Scoped credentials (bootstrap admin token only).
|
|
983
|
+
|
|
984
|
+
def list_credentials(self) -> dict:
|
|
985
|
+
"""{"items": [Credential], "capacity": 256}; secrets are never returned."""
|
|
986
|
+
return self._request("GET", "/v1/admin/credentials")
|
|
987
|
+
|
|
988
|
+
def create_credential(self, request: NewCredential) -> IssuedCredential:
|
|
989
|
+
"""Returns {"credential", "token"}. The token is shown only once: hand it to
|
|
990
|
+
its holder and never log or persist it in plaintext. 503
|
|
991
|
+
``credentials_commit_indeterminate`` (also on rotate/revoke) means the
|
|
992
|
+
change is in effect but may not be durable and no token is returned:
|
|
993
|
+
re-list and revoke or re-issue; it is not "no change"."""
|
|
994
|
+
return self._request("POST", "/v1/admin/credentials", _object(request, "request"))
|
|
995
|
+
|
|
996
|
+
def rotate_credential(self, id: str, *, grace_seconds: int | None = None) -> IssuedCredential:
|
|
997
|
+
"""Issue a replacement with identical scopes; the old credential expires at
|
|
998
|
+
min(its expiry, now + grace_seconds) (server default 3600)."""
|
|
999
|
+
body = {} if grace_seconds is None else {"grace_seconds": _int(grace_seconds, 0, 86400, "grace_seconds")}
|
|
1000
|
+
return self._request("POST", f"/v1/admin/credentials/{_credential(id)}/rotate", body)
|
|
1001
|
+
|
|
1002
|
+
def revoke_credential(self, id: str) -> dict:
|
|
1003
|
+
"""Revoke immediately (effective on the next request)."""
|
|
1004
|
+
return self._request("POST", f"/v1/admin/credentials/{_credential(id)}/revoke")
|
|
1005
|
+
|
|
1006
|
+
def persistent_audit(self, *, limit: int = 100) -> dict:
|
|
1007
|
+
"""``{"items": [...], "enabled": bool}``."""
|
|
1008
|
+
query = urlencode({"limit": str(_int(limit, 1, 1000, "limit"))})
|
|
1009
|
+
return self._request("GET", "/v1/admin/audit/persistent?" + query)
|