vlite-embedded 0.3.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vlite_embedded/__init__.py +380 -0
- vlite_embedded/py.typed +0 -0
- vlite_embedded/vlite_ffi.dll +0 -0
- vlite_embedded-0.3.0.dist-info/METADATA +130 -0
- vlite_embedded-0.3.0.dist-info/RECORD +8 -0
- vlite_embedded-0.3.0.dist-info/WHEEL +5 -0
- vlite_embedded-0.3.0.dist-info/licenses/LICENSE +21 -0
- vlite_embedded-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
"""Embedded VLite using ctypes and an ABI-v1 native shared library."""
|
|
2
|
+
import base64
|
|
3
|
+
import ctypes
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import threading
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class EmbeddedError(Exception):
|
|
12
|
+
def __init__(self, code: str, message: str):
|
|
13
|
+
self.code = code
|
|
14
|
+
super().__init__(f"{code}: {message}")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class NativeLibrary:
|
|
18
|
+
"""Load the bundled library, or an explicit ABI-v1 native library path."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, path: str | os.PathLike[str] | None = None):
|
|
21
|
+
if path is None:
|
|
22
|
+
name = "vlite_ffi.dll" if sys.platform == "win32" else "libvlite_ffi.dylib" if sys.platform == "darwin" else "libvlite_ffi.so"
|
|
23
|
+
path = Path(__file__).parent / name
|
|
24
|
+
if not path.is_file():
|
|
25
|
+
raise FileNotFoundError("Bundled VLite library is missing; install a platform wheel or supply an explicit native library path")
|
|
26
|
+
self._pid = os.getpid()
|
|
27
|
+
self._lib = ctypes.CDLL(str(Path(path).resolve(strict=True)))
|
|
28
|
+
self._lib.vlite_abi_version.argtypes = []
|
|
29
|
+
self._lib.vlite_abi_version.restype = ctypes.c_uint32
|
|
30
|
+
if self._lib.vlite_abi_version() != 1:
|
|
31
|
+
raise RuntimeError("Unsupported VLite ABI version")
|
|
32
|
+
self._lib.vlite_execute.argtypes = [ctypes.c_uint64, ctypes.c_void_p, ctypes.c_size_t]
|
|
33
|
+
self._lib.vlite_execute.restype = ctypes.c_void_p
|
|
34
|
+
self._lib.vlite_response_free.argtypes = [ctypes.c_void_p]
|
|
35
|
+
self._lib.vlite_response_free.restype = None
|
|
36
|
+
|
|
37
|
+
def _execute(self, handle: int, request: dict):
|
|
38
|
+
if os.getpid() != self._pid:
|
|
39
|
+
raise RuntimeError("VLite cannot be used after fork; use a fresh spawned process")
|
|
40
|
+
data = json.dumps(request, allow_nan=False).encode("utf-8")
|
|
41
|
+
buffer = ctypes.create_string_buffer(data)
|
|
42
|
+
pointer = self._lib.vlite_execute(handle, buffer, len(data))
|
|
43
|
+
if not pointer:
|
|
44
|
+
raise RuntimeError("Native library returned a null response; mutation outcome unknown")
|
|
45
|
+
try:
|
|
46
|
+
response = json.loads(ctypes.string_at(pointer))
|
|
47
|
+
finally:
|
|
48
|
+
self._lib.vlite_response_free(pointer)
|
|
49
|
+
if not response["ok"]:
|
|
50
|
+
error = response["error"]
|
|
51
|
+
raise EmbeddedError(error["code"], error["message"])
|
|
52
|
+
return response["result"]
|
|
53
|
+
|
|
54
|
+
def create(self, path: str | os.PathLike[str], dimensions: int, *, format: int | None = None,
|
|
55
|
+
key: bytes | None = None, passphrase: str | None = None,
|
|
56
|
+
commit_timestamps: bool = False) -> "Database":
|
|
57
|
+
"""Create without overwrite. Default format is 3; 5 enables all new surfaces."""
|
|
58
|
+
if format == 5 or key is not None or passphrase is not None or commit_timestamps:
|
|
59
|
+
return self._create_feature(str(Path(path)), dimensions, format, key, passphrase, commit_timestamps)
|
|
60
|
+
request = {"op": "create", "path": str(Path(path)), "dimensions": dimensions}
|
|
61
|
+
if format is not None:
|
|
62
|
+
request["format"] = format
|
|
63
|
+
return Database(self, int(self._execute(0, request)["handle"]))
|
|
64
|
+
|
|
65
|
+
def _feature(self, operation: str, **fields):
|
|
66
|
+
return self._execute(0, {"op": "features", "request": {"op": operation, **fields}})
|
|
67
|
+
|
|
68
|
+
def _create_feature(self, path, dimensions, format, key, passphrase, commit_timestamps):
|
|
69
|
+
options = {"credentials": _credentials(key, passphrase), "commit_timestamps": commit_timestamps,
|
|
70
|
+
**_given(format=format)}
|
|
71
|
+
result = self._feature("create", path=path, dimensions=dimensions, options=options)
|
|
72
|
+
return Database(self, int(result["handle"]))
|
|
73
|
+
|
|
74
|
+
def create_in_memory(self, dimensions: int, *, format: int | None = None, key: bytes | None = None,
|
|
75
|
+
passphrase: str | None = None, commit_timestamps: bool = False) -> "Database":
|
|
76
|
+
"""Host-owned byte-image database; persist with Database.to_bytes()."""
|
|
77
|
+
return self._create_feature(None, dimensions, format, key, passphrase, commit_timestamps)
|
|
78
|
+
|
|
79
|
+
def open(self, path: str | os.PathLike[str], *, key: bytes | None = None,
|
|
80
|
+
passphrase: str | None = None) -> "Database":
|
|
81
|
+
if key is None and passphrase is None:
|
|
82
|
+
result = self._execute(0, {"op": "open", "path": str(Path(path))})
|
|
83
|
+
else:
|
|
84
|
+
result = self._feature("open", path=str(Path(path)), credentials=_credentials(key, passphrase))
|
|
85
|
+
return Database(self, int(result["handle"]))
|
|
86
|
+
|
|
87
|
+
def open_from_bytes(self, data: bytes, *, key: bytes | None = None,
|
|
88
|
+
passphrase: str | None = None) -> "Database":
|
|
89
|
+
result = self._feature("open_from_bytes", bytes_base64=_bytes_base64(data),
|
|
90
|
+
credentials=_credentials(key, passphrase))
|
|
91
|
+
return Database(self, int(result["handle"]))
|
|
92
|
+
|
|
93
|
+
def restore_to(self, log_path: str | os.PathLike[str], destination: str | os.PathLike[str], *,
|
|
94
|
+
sequence: str | int | None = None, timestamp_ms: str | int | None = None,
|
|
95
|
+
base_backup: str | os.PathLike[str] | None = None,
|
|
96
|
+
key: bytes | None = None, passphrase: str | None = None) -> dict:
|
|
97
|
+
"""Restore a closed log into a new path; choose exactly one sequence or timestamp."""
|
|
98
|
+
return self._feature("restore_to", log_path=str(Path(log_path)), destination=str(Path(destination)),
|
|
99
|
+
**_given(sequence=None if sequence is None else str(sequence),
|
|
100
|
+
timestamp_ms=None if timestamp_ms is None else str(timestamp_ms),
|
|
101
|
+
base_backup=None if base_backup is None else str(Path(base_backup))),
|
|
102
|
+
credentials=_credentials(key, passphrase))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class Database:
|
|
106
|
+
def __init__(self, library: NativeLibrary, handle: int):
|
|
107
|
+
self._library, self._handle = library, handle
|
|
108
|
+
self._lock = threading.Lock()
|
|
109
|
+
|
|
110
|
+
def __enter__(self) -> "Database":
|
|
111
|
+
return self
|
|
112
|
+
|
|
113
|
+
def __exit__(self, *_):
|
|
114
|
+
self.close()
|
|
115
|
+
|
|
116
|
+
def _execute(self, operation: str, **fields):
|
|
117
|
+
if os.getpid() != self._library._pid:
|
|
118
|
+
raise RuntimeError("VLite cannot be used after fork; use a fresh spawned process")
|
|
119
|
+
with self._lock:
|
|
120
|
+
if self._handle is None:
|
|
121
|
+
raise EmbeddedError("invalid_handle", "Database is closed")
|
|
122
|
+
return self._library._execute(self._handle, {"op": operation, **fields})
|
|
123
|
+
|
|
124
|
+
def close(self) -> None:
|
|
125
|
+
if os.getpid() != self._library._pid:
|
|
126
|
+
raise RuntimeError("VLite cannot be used after fork; use a fresh spawned process")
|
|
127
|
+
with self._lock:
|
|
128
|
+
if self._handle is not None:
|
|
129
|
+
handle = self._handle
|
|
130
|
+
self._handle = None
|
|
131
|
+
self._library._execute(handle, {"op": "close"})
|
|
132
|
+
|
|
133
|
+
def info(self) -> dict:
|
|
134
|
+
return self._execute("info")
|
|
135
|
+
|
|
136
|
+
def get(self, id: str, *, collection: str | None = None) -> dict:
|
|
137
|
+
"""Return the record, including its current ``revision`` as a decimal string."""
|
|
138
|
+
return self._execute("get", id=id, **_given(collection=collection))
|
|
139
|
+
|
|
140
|
+
def get_many(self, ids: list[str], *, collection: str | None = None) -> list[dict | None]:
|
|
141
|
+
return self._execute("get_many", ids=ids, **_given(collection=collection))["records"]
|
|
142
|
+
|
|
143
|
+
def put(self, id: str, vector: list[float], *, payload_base64: str = "", metadata: dict | None = None,
|
|
144
|
+
collection: str | None = None, expected_revision: str | None = None,
|
|
145
|
+
expires_at_ms: int | str | None = None) -> str | None:
|
|
146
|
+
"""Write a record. With ``expected_revision`` (``"absent"`` or a decimal revision string)
|
|
147
|
+
the write is compare-and-set and returns the new revision; otherwise it returns None."""
|
|
148
|
+
if expires_at_ms is not None:
|
|
149
|
+
if expected_revision is not None:
|
|
150
|
+
raise ValueError("expiry writes do not support expected_revision")
|
|
151
|
+
self.put_with_expiry(id, vector, expires_at_ms=expires_at_ms, payload_base64=payload_base64,
|
|
152
|
+
metadata=metadata, collection=collection)
|
|
153
|
+
return None
|
|
154
|
+
result = self._execute("put", id=id, vector=vector, payload_base64=payload_base64,
|
|
155
|
+
metadata={} if metadata is None else metadata,
|
|
156
|
+
**_given(collection=collection, expected_revision=expected_revision))
|
|
157
|
+
return None if expected_revision is None else result["revision"]
|
|
158
|
+
|
|
159
|
+
def delete(self, id: str, *, collection: str | None = None, expected_revision: str | None = None) -> bool:
|
|
160
|
+
return self._execute("delete", id=id,
|
|
161
|
+
**_given(collection=collection, expected_revision=expected_revision))["deleted"]
|
|
162
|
+
|
|
163
|
+
def batch(self, operations: list[dict], *, collection: str | None = None) -> None:
|
|
164
|
+
self._execute("batch", operations=operations, **_given(collection=collection))
|
|
165
|
+
|
|
166
|
+
def revision(self, id: str, *, collection: str | None = None) -> str | None:
|
|
167
|
+
return self._execute("revision", id=id, **_given(collection=collection))["revision"]
|
|
168
|
+
|
|
169
|
+
def search(self, vector: list[float], k: int, *, filter: list[dict] | None = None, metric: str | None = None,
|
|
170
|
+
collection: str | None = None, explain: bool = False) -> list[dict] | dict:
|
|
171
|
+
"""Exact search. Returns the hit list, or ``{"hits", "stats"}`` when ``explain`` is true."""
|
|
172
|
+
result = self._execute("search", vector=vector, k=k, filter=[] if filter is None else filter,
|
|
173
|
+
**_given(metric=metric, collection=collection, explain=explain or None))
|
|
174
|
+
return result if explain else result["hits"]
|
|
175
|
+
|
|
176
|
+
def search_auto(self, vector: list[float], k: int, *, filter: list[dict] | None = None, metric: str | None = None,
|
|
177
|
+
collection: str | None = None, ef_search: int | None = None, quantization: str | None = None,
|
|
178
|
+
rerank_multiplier: int | None = None, explain: bool = False) -> dict:
|
|
179
|
+
"""Planned search; ``plan`` and ``stats`` are included only when ``explain`` is true."""
|
|
180
|
+
return self._execute("search", mode="auto", vector=vector, k=k, explain=explain,
|
|
181
|
+
filter=[] if filter is None else filter,
|
|
182
|
+
**_given(metric=metric, collection=collection, ef_search=ef_search,
|
|
183
|
+
quantization=quantization, rerank_multiplier=rerank_multiplier))
|
|
184
|
+
|
|
185
|
+
def search_hybrid(self, text: str, k: int, *, vector: list[float] | None = None, filter: list[dict] | None = None,
|
|
186
|
+
metric: str | None = None, collection: str | None = None, fusion: dict | None = None,
|
|
187
|
+
vector_leg: str | None = None, ef_search: int | None = None, quantization: str | None = None,
|
|
188
|
+
rerank_multiplier: int | None = None, explain: bool = False) -> dict:
|
|
189
|
+
"""BM25 over the collection's declared text fields, fused with an optional vector leg."""
|
|
190
|
+
return self._execute("search", mode="hybrid", text=text, k=k, explain=explain,
|
|
191
|
+
filter=[] if filter is None else filter,
|
|
192
|
+
**_given(vector=vector, metric=metric, collection=collection, fusion=fusion,
|
|
193
|
+
vector_leg=vector_leg, ef_search=ef_search, quantization=quantization,
|
|
194
|
+
rerank_multiplier=rerank_multiplier))
|
|
195
|
+
|
|
196
|
+
def build_index(self, *, metric: str | None = None, collection: str | None = None, quantization: str | None = None,
|
|
197
|
+
rerank_multiplier: int | None = None) -> None:
|
|
198
|
+
self._execute("build_index", **_given(metric=metric, collection=collection, quantization=quantization,
|
|
199
|
+
rerank_multiplier=rerank_multiplier))
|
|
200
|
+
|
|
201
|
+
def search_approximate_ready(self, vector: list[float], k: int, *, ef_search: int = 100, metric: str | None = None,
|
|
202
|
+
collection: str | None = None, quantization: str | None = None,
|
|
203
|
+
rerank_multiplier: int | None = None) -> list[dict]:
|
|
204
|
+
return self._execute("search_approximate_ready", vector=vector, k=k, ef_search=ef_search,
|
|
205
|
+
**_given(metric=metric, collection=collection, quantization=quantization,
|
|
206
|
+
rerank_multiplier=rerank_multiplier))["hits"]
|
|
207
|
+
|
|
208
|
+
def list_collections(self) -> list[dict]:
|
|
209
|
+
return self._execute("list_collections")["collections"]
|
|
210
|
+
|
|
211
|
+
def create_collection(self, name: str, dimensions: int, *, settings: dict | None = None) -> dict:
|
|
212
|
+
return self._execute("create_collection", name=name, dimensions=dimensions, **_given(settings=settings))
|
|
213
|
+
|
|
214
|
+
def drop_collection(self, name: str) -> None:
|
|
215
|
+
self._execute("drop_collection", name=name)
|
|
216
|
+
|
|
217
|
+
def set_collection_settings(self, name: str, settings: dict) -> dict:
|
|
218
|
+
"""Full replacement: omitted settings revert to engine defaults. Returns ``{"persisted": bool}``."""
|
|
219
|
+
return self._execute("set_collection_settings", name=name, settings=settings)
|
|
220
|
+
|
|
221
|
+
def next_import_job_id(self) -> str:
|
|
222
|
+
return self._execute("next_import_job_id")["job"]
|
|
223
|
+
|
|
224
|
+
def import_chunk(self, job: str, chunk: str, operations: list[dict], *, collection: str | None = None,
|
|
225
|
+
policy: str | None = None) -> dict:
|
|
226
|
+
return self._execute("import_chunk", job=job, chunk=chunk, operations=operations,
|
|
227
|
+
**_given(collection=collection, policy=policy))
|
|
228
|
+
|
|
229
|
+
def import_status(self, job: str) -> dict:
|
|
230
|
+
return self._execute("import_status", job=job)
|
|
231
|
+
|
|
232
|
+
def import_finish(self, job: str, *, collection: str | None = None) -> dict:
|
|
233
|
+
return self._execute("import_finish", job=job, **_given(collection=collection))
|
|
234
|
+
|
|
235
|
+
def import_abort(self, job: str, *, collection: str | None = None) -> dict:
|
|
236
|
+
return self._execute("import_abort", job=job, **_given(collection=collection))
|
|
237
|
+
|
|
238
|
+
def ann_quality(self, *, collection: str | None = None, metric: str | None = None, quantization: str | None = None,
|
|
239
|
+
rerank_multiplier: int | None = None, samples: int | None = None, k: int | None = None,
|
|
240
|
+
ef_search: int | None = None, seed: str | None = None,
|
|
241
|
+
max_distance_evaluations: int | None = None) -> dict:
|
|
242
|
+
return self._execute("ann_quality", **_given(
|
|
243
|
+
collection=collection, metric=metric, quantization=quantization, rerank_multiplier=rerank_multiplier,
|
|
244
|
+
samples=samples, k=k, ef_search=ef_search, seed=seed, max_distance_evaluations=max_distance_evaluations))
|
|
245
|
+
|
|
246
|
+
def ann_memory(self, *, collection: str | None = None, metric: str | None = None, quantization: str | None = None,
|
|
247
|
+
rerank_multiplier: int | None = None) -> dict:
|
|
248
|
+
return self._execute("ann_memory", **_given(collection=collection, metric=metric, quantization=quantization,
|
|
249
|
+
rerank_multiplier=rerank_multiplier))
|
|
250
|
+
|
|
251
|
+
def _feature(self, operation: str, **fields):
|
|
252
|
+
return self._execute("features", request={"op": operation, **fields})
|
|
253
|
+
|
|
254
|
+
def put_with_expiry(self, id: str, vector: list[float], *, expires_at_ms: int | str | None = None,
|
|
255
|
+
payload_base64: str = "", metadata: dict | None = None,
|
|
256
|
+
collection: str | None = None) -> str:
|
|
257
|
+
"""Put with absolute Unix-ms expiry (None means no expiry); returns revision. Requires V5."""
|
|
258
|
+
return self._feature("put_with_expiry", id=id, vector=vector, payload_base64=payload_base64,
|
|
259
|
+
metadata={} if metadata is None else metadata,
|
|
260
|
+
expires_at_ms=None if expires_at_ms is None else str(expires_at_ms),
|
|
261
|
+
**_given(collection=collection))["revision"]
|
|
262
|
+
|
|
263
|
+
def update_metadata(self, id: str, metadata: dict, *, collection: str | None = None) -> str:
|
|
264
|
+
"""Replace metadata, retaining vectors, payload, and expiry; returns revision. Requires V5."""
|
|
265
|
+
return self._feature("update_metadata", id=id, metadata=metadata, **_given(collection=collection))["revision"]
|
|
266
|
+
|
|
267
|
+
def search_grouped(self, vector: list[float], groups: int, hits_per_group: int, field: str, *,
|
|
268
|
+
filter: list[dict] | None = None, metric: str | None = None,
|
|
269
|
+
collection: str | None = None) -> list[dict]:
|
|
270
|
+
return self._feature("search_grouped", vector=vector, groups=groups, hits_per_group=hits_per_group,
|
|
271
|
+
field=field, filter=[] if filter is None else filter, **_given(metric=metric, collection=collection))
|
|
272
|
+
|
|
273
|
+
def recommend(self, k: int, *, positive_ids: list[str] | None = None, negative_ids: list[str] | None = None,
|
|
274
|
+
positive_vectors: list[list[float]] | None = None, negative_vectors: list[list[float]] | None = None,
|
|
275
|
+
filter: list[dict] | None = None, metric: str | None = None, collection: str | None = None) -> list[dict]:
|
|
276
|
+
return self._feature("recommend", k=k, positive_ids=[] if positive_ids is None else positive_ids,
|
|
277
|
+
negative_ids=[] if negative_ids is None else negative_ids,
|
|
278
|
+
positive_vectors=[] if positive_vectors is None else positive_vectors,
|
|
279
|
+
negative_vectors=[] if negative_vectors is None else negative_vectors,
|
|
280
|
+
filter=[] if filter is None else filter, **_given(metric=metric, collection=collection))
|
|
281
|
+
|
|
282
|
+
def facets(self, field: str, *, filter: list[dict] | None = None, collection: str | None = None) -> list[dict]:
|
|
283
|
+
return self._feature("facets", field=field, filter=[] if filter is None else filter, **_given(collection=collection))
|
|
284
|
+
|
|
285
|
+
def numeric_stats(self, field: str, *, filter: list[dict] | None = None, collection: str | None = None) -> dict:
|
|
286
|
+
return self._feature("numeric_stats", field=field, filter=[] if filter is None else filter, **_given(collection=collection))
|
|
287
|
+
|
|
288
|
+
def search_geo(self, vector: list[float], k: int, field: str, latitude: float, longitude: float,
|
|
289
|
+
radius_meters: float, *, sort: str = "vector_distance", filter: list[dict] | None = None,
|
|
290
|
+
metric: str | None = None, collection: str | None = None) -> list[dict]:
|
|
291
|
+
return self._feature("search_geo", vector=vector, k=k, field=field, latitude=latitude, longitude=longitude,
|
|
292
|
+
radius_meters=radius_meters, sort=sort, filter=[] if filter is None else filter,
|
|
293
|
+
**_given(metric=metric, collection=collection))
|
|
294
|
+
|
|
295
|
+
def configure_vector(self, name: str, dimensions: int, *, active_version: str, metric: str = "squared_l2",
|
|
296
|
+
history: int = 0, collection: str | None = None) -> None:
|
|
297
|
+
self._feature("configure_vector", name=name, dimensions=dimensions, active_version=active_version,
|
|
298
|
+
metric=metric, history=history, **_given(collection=collection))
|
|
299
|
+
|
|
300
|
+
def vector_specs(self, *, collection: str | None = None) -> dict:
|
|
301
|
+
return self._feature("vector_specs", **_given(collection=collection))
|
|
302
|
+
|
|
303
|
+
def record_vectors(self, id: str, *, collection: str | None = None) -> dict | None:
|
|
304
|
+
return self._feature("record_vectors", id=id, **_given(collection=collection))
|
|
305
|
+
|
|
306
|
+
def put_vectors(self, id: str, *, dense: dict | None = None, sparse: dict | None = None,
|
|
307
|
+
collection: str | None = None) -> None:
|
|
308
|
+
"""Replace all additional vectors: dense name -> [{version, vector}], sparse name -> [(index, weight)]."""
|
|
309
|
+
self._feature("put_vectors", id=id, dense={} if dense is None else dense, sparse={} if sparse is None else sparse,
|
|
310
|
+
**_given(collection=collection))
|
|
311
|
+
|
|
312
|
+
def put_vector_version(self, id: str, name: str, version: str, vector: list[float], *, collection: str | None = None) -> None:
|
|
313
|
+
self._feature("put_vector_version", id=id, name=name, version=version, vector=vector, **_given(collection=collection))
|
|
314
|
+
|
|
315
|
+
def prune_vector_history(self, name: str, history: int, *, collection: str | None = None) -> None:
|
|
316
|
+
self._feature("prune_vector_history", name=name, history=history, **_given(collection=collection))
|
|
317
|
+
|
|
318
|
+
def search_named(self, name: str, vector: list[float], k: int, *, version: str | None = None,
|
|
319
|
+
collection: str | None = None) -> list[dict]:
|
|
320
|
+
return self._feature("search_named", name=name, vector=vector, k=k, **_given(version=version, collection=collection))
|
|
321
|
+
|
|
322
|
+
def search_sparse(self, name: str, vector: list[tuple[int, float]], k: int, *, collection: str | None = None) -> list[dict]:
|
|
323
|
+
return self._feature("search_sparse", name=name, vector=vector, k=k, **_given(collection=collection))
|
|
324
|
+
|
|
325
|
+
def search_dense_sparse(self, dense_name: str, dense: list[float], sparse_name: str, sparse: list[tuple[int, float]],
|
|
326
|
+
k: int, *, version: str | None = None, fusion: dict | None = None,
|
|
327
|
+
collection: str | None = None) -> list[dict]:
|
|
328
|
+
return self._feature("search_dense_sparse", dense_name=dense_name, dense=dense, sparse_name=sparse_name,
|
|
329
|
+
sparse=sparse, k=k, **_given(version=version, fusion=fusion, collection=collection))
|
|
330
|
+
|
|
331
|
+
def rerank(self, vector: list[float], candidates: list[dict], limit: int, *, method: str = "exact",
|
|
332
|
+
metric: str | None = None, lambda_: float | None = None, collection: str | None = None) -> list[dict]:
|
|
333
|
+
return self._feature("rerank", vector=vector, candidates=candidates, limit=limit, method=method,
|
|
334
|
+
**_given(metric=metric, collection=collection, **{"lambda": lambda_}))
|
|
335
|
+
|
|
336
|
+
def query(self, query: str, parameters: dict[str, list[float]] | None = None, *, collection: str | None = None) -> dict:
|
|
337
|
+
return self._feature("query", query=query, parameters={} if parameters is None else parameters, **_given(collection=collection))
|
|
338
|
+
|
|
339
|
+
def feed_cursor(self) -> str:
|
|
340
|
+
return self._feature("feed_cursor")
|
|
341
|
+
|
|
342
|
+
def changes_after(self, cursor: str | None = None, max_events: int = 1000) -> dict:
|
|
343
|
+
"""Read commits after cursor, or from the beginning when cursor is None. Does not split batch commits."""
|
|
344
|
+
return self._feature("changes_after", cursor=cursor, max_events=max_events)
|
|
345
|
+
|
|
346
|
+
def encryption_info(self) -> dict:
|
|
347
|
+
return self._feature("encryption_info")
|
|
348
|
+
|
|
349
|
+
def is_encrypted(self) -> bool:
|
|
350
|
+
return self.encryption_info()["is_encrypted"]
|
|
351
|
+
|
|
352
|
+
def encryption_key_id(self) -> str | None:
|
|
353
|
+
return self.encryption_info()["key_id"]
|
|
354
|
+
|
|
355
|
+
def build_disk_ann(self, path: str | os.PathLike[str], *, metric: str | None = None, collection: str | None = None) -> dict:
|
|
356
|
+
return self._feature("build_disk_ann", path=str(Path(path)), **_given(metric=metric, collection=collection))
|
|
357
|
+
|
|
358
|
+
def search_disk_ann(self, path: str | os.PathLike[str], vector: list[float], k: int, *, ef_search: int = 100,
|
|
359
|
+
cache_bytes: int = 8 * 1024 * 1024) -> list[dict]:
|
|
360
|
+
return self._feature("search_disk_ann", path=str(Path(path)), vector=vector, k=k, ef_search=ef_search, cache_bytes=cache_bytes)
|
|
361
|
+
|
|
362
|
+
def to_bytes(self) -> bytes:
|
|
363
|
+
return base64.b64decode(self._feature("to_bytes")["bytes_base64"], validate=True)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _given(**fields) -> dict:
|
|
367
|
+
"""Drop omitted options so the native layer applies the collection's declared defaults."""
|
|
368
|
+
return {name: value for name, value in fields.items() if value is not None}
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _bytes_base64(data: bytes) -> str:
|
|
372
|
+
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
373
|
+
raise TypeError("expected bytes-like data")
|
|
374
|
+
return base64.b64encode(data).decode("ascii")
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _credentials(key: bytes | None, passphrase: str | None) -> dict:
|
|
378
|
+
if key is not None and passphrase is not None:
|
|
379
|
+
raise ValueError("provide key or passphrase, not both")
|
|
380
|
+
return _given(key_base64=None if key is None else _bytes_base64(key), passphrase=passphrase)
|
vlite_embedded/py.typed
ADDED
|
File without changes
|
|
Binary file
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vlite-embedded
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: In-process Python binding for the VLite native database library
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: license-file
|
|
10
|
+
|
|
11
|
+
# VLite embedded Python
|
|
12
|
+
|
|
13
|
+
In-process binding for Python 3.11+, using ctypes and a bundled VLite shared
|
|
14
|
+
library. Platform wheels require no server, HTTP, Docker or Rust installation
|
|
15
|
+
at runtime. Install the wheel matching your operating system and architecture,
|
|
16
|
+
then use `NativeLibrary()` to load the included library automatically.
|
|
17
|
+
An explicit native library path is still supported.
|
|
18
|
+
|
|
19
|
+
`native.create(path, dims)` creates format 3; `native.create(path, dims,
|
|
20
|
+
format=4)` enables named collections (`create_collection`, `list_collections`,
|
|
21
|
+
`set_collection_settings`, `drop_collection`) and resumable imports
|
|
22
|
+
(`next_import_job_id`, `import_chunk`, `import_status`, `import_finish`,
|
|
23
|
+
`import_abort`). Record methods accept `collection=` and
|
|
24
|
+
`expected_revision=` (`"absent"` or a revision string); conditional `put`
|
|
25
|
+
returns the new revision. `search(..., explain=True)`, `search_auto` and
|
|
26
|
+
`search_hybrid` expose plans, work statistics and BM25 fusion; `ann_quality` and
|
|
27
|
+
`ann_memory` report ANN recall and memory, including `quantization="scalar_u8"`.
|
|
28
|
+
Omitted metric, quantization and rerank multiplier use the collection's
|
|
29
|
+
settings. Server-scheduled maintenance, backups, metrics and authentication
|
|
30
|
+
remain server-only.
|
|
31
|
+
Offline point-in-time restore and encryption are available below.
|
|
32
|
+
|
|
33
|
+
See the [embedded deployment guide](https://github.com/mahpate/Vlite/blob/main/bindings/README.md)
|
|
34
|
+
for builds, installation, examples, ABI ownership and limits.
|
|
35
|
+
Licensed under MIT.
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
## V5 embedded feature API
|
|
39
|
+
|
|
40
|
+
These bindings use ctypes, not PyO3/maturin. The wheel bundles the same ABI-1
|
|
41
|
+
Rust library as the Node package; new operations need a matching new library.
|
|
42
|
+
Default creation remains format 3. Pass `format=5` to enable TTL/metadata-only
|
|
43
|
+
writes and named/sparse/model-versioned vectors. Encryption and commit timestamps
|
|
44
|
+
imply format 5 if no explicit incompatible format is supplied.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from vlite_embedded import NativeLibrary
|
|
48
|
+
|
|
49
|
+
native = NativeLibrary() # or NativeLibrary("absolute/path/to/native-library")
|
|
50
|
+
with native.create_in_memory(2, format=5) as db:
|
|
51
|
+
cursor = db.feed_cursor()
|
|
52
|
+
db.put_with_expiry("1", [1, 0], expires_at_ms=2_000_000_000_000,
|
|
53
|
+
metadata={"tag": {"type": "string", "value": "example"}})
|
|
54
|
+
db.update_metadata("1", {"tag": {"type": "string", "value": "updated"}})
|
|
55
|
+
print(db.search_grouped([1, 0], 5, 2, "tag"))
|
|
56
|
+
db.configure_vector("text", 2, active_version="model-v1", history=1)
|
|
57
|
+
db.put_vectors("1", dense={"text": [{"version": "model-v1", "vector": [1, 0]}]},
|
|
58
|
+
sparse={"terms": [(7, 0.8)]})
|
|
59
|
+
print(db.search_named("text", [1, 0], 5))
|
|
60
|
+
print(db.search_dense_sparse("text", [1, 0], "terms", [(7, 1.0)], 5))
|
|
61
|
+
print(db.query("SEARCH default NEAR $q K 5", {"q": [1, 0]}))
|
|
62
|
+
print(db.changes_after(cursor))
|
|
63
|
+
image = db.to_bytes() # persist these bytes in host-owned storage
|
|
64
|
+
with native.open_from_bytes(image) as reopened:
|
|
65
|
+
print(reopened.get("1"))
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Additional methods (all accept `collection=` where meaningful):
|
|
69
|
+
|
|
70
|
+
- `put(..., expires_at_ms=...)` is an unconditional TTL write; the explicit
|
|
71
|
+
`put_with_expiry` also accepts None for no expiry and returns a revision string.
|
|
72
|
+
`update_metadata` replaces metadata only and returns its revision. TTL is an
|
|
73
|
+
absolute Unix-ms timestamp, not a duration; revision preconditions are rejected
|
|
74
|
+
for expiry writes. Base writes clear additional vectors; metadata-only writes
|
|
75
|
+
preserve vectors, payload and expiry.
|
|
76
|
+
- `recommend(k, positive_ids=..., negative_ids=..., positive_vectors=...,
|
|
77
|
+
negative_vectors=...)`, `facets(field)`, `numeric_stats(field)`,
|
|
78
|
+
`search_geo(vector, k, field, latitude, longitude, radius_meters,
|
|
79
|
+
sort="geo_distance")`; retrieval accepts optional AND `filter=` and `metric=`.
|
|
80
|
+
Geo metadata is `{"type":"geopoint","value":{"latitude":0,"longitude":0}}`.
|
|
81
|
+
- `vector_specs`, `record_vectors(id)`, `put_vector_version(id,name,version,vector)`,
|
|
82
|
+
`prune_vector_history(name,history)`, `search_sparse(name,pairs,k)`;
|
|
83
|
+
`search_named(...,version="old-model")` selects retained history. `put_vectors`
|
|
84
|
+
replaces all additional dense/sparse vectors, not a partial merge.
|
|
85
|
+
- `rerank(vector, candidates, limit, method="exact", metric=...)` or
|
|
86
|
+
`method="mmr", lambda_=0.5` (always cosine; omit metric). Candidates are hit
|
|
87
|
+
dictionaries with IDs and optional distances. The engine fetches live records.
|
|
88
|
+
- `feed_cursor()` starts at head; `changes_after(cursor=None,max_events=1000)`
|
|
89
|
+
starts at the beginning when omitted and returns `{changes,next,has_more,head}`.
|
|
90
|
+
Persist `next`; changes never split a committed batch. Expiry itself is not
|
|
91
|
+
an event. Query syntax is documented in `docs/VECTORS.md`.
|
|
92
|
+
- `native.create/open(...,key=bytes_of_length_32)` or `passphrase="..."` enables
|
|
93
|
+
encrypted creation/opening. `is_encrypted()`, `encryption_key_id()` and
|
|
94
|
+
`encryption_info()` expose non-secret status. Credentials also work with
|
|
95
|
+
in-memory create and byte-image open; never supply both credential kinds.
|
|
96
|
+
- `native.restore_to(closed_log,new_path,sequence="42")` or
|
|
97
|
+
`timestamp_ms=...`, optionally `base_backup=...`, `key=`/`passphrase=`.
|
|
98
|
+
Timestamp restore needs creation with `commit_timestamps=True`. Sources must
|
|
99
|
+
be closed/copied; restore publishes a new file and never overwrites a source.
|
|
100
|
+
- `build_disk_ann(path,metric=...)`, `search_disk_ann(path,vector,k,
|
|
101
|
+
ef_search=100,cache_bytes=...)`. Each search opens/closes its own bounded cache;
|
|
102
|
+
every database mutation makes the sidecar stale. Disk ANN does not filter
|
|
103
|
+
expiry/metadata, and encrypted databases refuse plaintext sidecars.
|
|
104
|
+
|
|
105
|
+
IDs/revisions are decimal strings. Absolute input timestamps also accept Python
|
|
106
|
+
integers (serialized losslessly as decimal strings). Feed timestamps/head,
|
|
107
|
+
lineage, key IDs and restore revisions are returned as strings. Metadata and
|
|
108
|
+
filter values retain the existing explicit typed JSON shape. Responses are
|
|
109
|
+
ordinary dictionaries/lists; byte images are `bytes`. New methods carry inline
|
|
110
|
+
Python type annotations; no separate stub file is required.
|
|
111
|
+
|
|
112
|
+
JSON requests/responses are capped at 8 MiB including base64 overhead, so large
|
|
113
|
+
byte images need a different persistence surface. In-memory images are durable
|
|
114
|
+
only when the host saves them. Encryption protects contents, not access patterns
|
|
115
|
+
or process memory; Python/JSON key copies cannot be reliably wiped. No provider
|
|
116
|
+
callbacks, GPU, read replica, re-embedding worker or named-ANN wrapper is exposed.
|
|
117
|
+
See `bindings/c/FEATURES.md` for the complete result/error contract and limits.
|
|
118
|
+
|
|
119
|
+
Build and test from the repository root (Rust 1.89+, Python 3.11+):
|
|
120
|
+
|
|
121
|
+
```text
|
|
122
|
+
cargo build --locked -p vlite-ffi
|
|
123
|
+
python -m unittest discover -s bindings/python -p test_features.py -v
|
|
124
|
+
python bindings/build_wheel.py --output embedded-wheels
|
|
125
|
+
python tests/embedded_wheel.py embedded-wheels
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The direct feature suite defaults to `target/debug` and accepts
|
|
129
|
+
`VLITE_TEST_LIBRARY` to validate an exact alternate/bundled library. It creates
|
|
130
|
+
and cleans its test files below the binding directory.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
vlite_embedded/__init__.py,sha256=UVJhK45CyC2bt7gAAq5Xo17_v5eQh2PoB2Jgu0PF38U,24359
|
|
2
|
+
vlite_embedded/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
vlite_embedded/vlite_ffi.dll,sha256=aA-_9E8udg8ExWIAy0dJ3DurBLWojwS8j2O_GGQdJ6M,2728960
|
|
4
|
+
vlite_embedded-0.3.0.dist-info/licenses/LICENSE,sha256=IKMI973KhvSjf70sno4TzZnsyq0LVf1CkZQ6VgFv5nM,1096
|
|
5
|
+
vlite_embedded-0.3.0.dist-info/METADATA,sha256=0A_jz6UjbGHEOBqq4MkUpaFigHcVycHufNZusREWhHk,7360
|
|
6
|
+
vlite_embedded-0.3.0.dist-info/WHEEL,sha256=79BCzrXBJOa2YV6ugKrK59dqWifMGjAkgVd_6AQrqbU,98
|
|
7
|
+
vlite_embedded-0.3.0.dist-info/top_level.txt,sha256=MPmbL_sT_UicHS8OWlfsw11DZ5EsaxMRG3FfM9D64kc,15
|
|
8
|
+
vlite_embedded-0.3.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VLite contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
vlite_embedded
|