arcadedb-embedded-native 26.10.2.dev1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arcadedb_embedded_native-26.10.2.dev1.dist-info/METADATA +114 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/RECORD +41 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/WHEEL +5 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/LICENSE +201 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/NOTICE +25 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/THIRD_PARTY_NOTICES.md +3629 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/licenses/Apache-2.0.txt +73 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/licenses/GraalVM-CE-LICENSE.txt +430 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/licenses/GraalVM-THIRD_PARTY_LICENSE.txt +7357 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/licenses/licenses/GraalVM-native-image-LICENSE.txt +379 -0
- arcadedb_embedded_native-26.10.2.dev1.dist-info/top_level.txt +1 -0
- arcadedb_native/__init__.py +128 -0
- arcadedb_native/_build_info.py +5 -0
- arcadedb_native/_jpype_stub.py +44 -0
- arcadedb_native/_logging.py +23 -0
- arcadedb_native/_native/__init__.py +31 -0
- arcadedb_native/_native/bulk.py +96 -0
- arcadedb_native/_native/codec.py +229 -0
- arcadedb_native/_native/columns.py +167 -0
- arcadedb_native/_native/database.py +1356 -0
- arcadedb_native/_native/extras.py +819 -0
- arcadedb_native/_native/lib.py +338 -0
- arcadedb_native/_native/libarcadedb.dll +4 -0
- arcadedb_native/_native/records.py +204 -0
- arcadedb_native/_native/schema_objects.py +379 -0
- arcadedb_native/_native/server.py +118 -0
- arcadedb_native/async_executor.py +1114 -0
- arcadedb_native/core.py +2189 -0
- arcadedb_native/dataframe.py +479 -0
- arcadedb_native/exceptions.py +147 -0
- arcadedb_native/exporter.py +309 -0
- arcadedb_native/graph.py +302 -0
- arcadedb_native/graph_batch.py +527 -0
- arcadedb_native/importer.py +284 -0
- arcadedb_native/jvm.py +632 -0
- arcadedb_native/results.py +1369 -0
- arcadedb_native/schema.py +758 -0
- arcadedb_native/server.py +347 -0
- arcadedb_native/transactions.py +38 -0
- arcadedb_native/type_conversion.py +707 -0
- arcadedb_native/vector.py +800 -0
|
@@ -0,0 +1,800 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ArcadeDB Python Bindings - Vector Search
|
|
3
|
+
|
|
4
|
+
Vector index and array conversion utilities for similarity search.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from arcadedb_native import _jpype_stub as jpype
|
|
8
|
+
from arcadedb_native import _jpype_stub as jtypes
|
|
9
|
+
from .exceptions import ArcadeDBError
|
|
10
|
+
|
|
11
|
+
try: # optional; hoisted to module scope to keep it out of per-call hot paths
|
|
12
|
+
import numpy as _np
|
|
13
|
+
except ImportError: # pragma: no cover - numpy is an optional dependency
|
|
14
|
+
_np = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _quote_identifier(identifier: str) -> str:
|
|
18
|
+
if not identifier:
|
|
19
|
+
raise ArcadeDBError("SQL identifier cannot be empty")
|
|
20
|
+
|
|
21
|
+
return "`" + identifier.replace("`", "``") + "`"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# float[] class, built on first use (it needs the JVM): jtypes.JArray(jtypes.JFloat)
|
|
25
|
+
# costs about 0.6 us a call, as much as the buffer copy it is followed by.
|
|
26
|
+
_FLOAT_ARRAY = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _native_enabled():
|
|
30
|
+
from . import _native
|
|
31
|
+
|
|
32
|
+
return _native.enabled()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def to_java_float_array(vector):
|
|
36
|
+
"""
|
|
37
|
+
Convert a Python array-like object to a Java float array.
|
|
38
|
+
|
|
39
|
+
Accepts:
|
|
40
|
+
- Python lists: [0.1, 0.2, 0.3]
|
|
41
|
+
- NumPy arrays: np.array([0.1, 0.2, 0.3])
|
|
42
|
+
- Any array-like object with __iter__
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
vector: Array-like object containing float values
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Java float array compatible with ArcadeDB vector indexes
|
|
49
|
+
"""
|
|
50
|
+
if _native_enabled():
|
|
51
|
+
return _np.asarray(
|
|
52
|
+
vector if hasattr(vector, "__array__") else list(vector), dtype=_np.float32
|
|
53
|
+
)
|
|
54
|
+
global _FLOAT_ARRAY
|
|
55
|
+
float_array = _FLOAT_ARRAY
|
|
56
|
+
if float_array is None:
|
|
57
|
+
float_array = _FLOAT_ARRAY = jtypes.JArray(jtypes.JFloat)
|
|
58
|
+
|
|
59
|
+
# Handle NumPy arrays
|
|
60
|
+
if _np is not None and isinstance(vector, _np.ndarray):
|
|
61
|
+
return float_array(vector)
|
|
62
|
+
|
|
63
|
+
# Convert to Python list if needed
|
|
64
|
+
if not isinstance(vector, list):
|
|
65
|
+
vector = list(vector)
|
|
66
|
+
|
|
67
|
+
# Create Java float array
|
|
68
|
+
return float_array(vector)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def to_java_int_array(vector):
|
|
72
|
+
"""
|
|
73
|
+
Convert a Python array-like object to a Java int array.
|
|
74
|
+
|
|
75
|
+
Accepts:
|
|
76
|
+
- Python lists: [3, 17, 4096]
|
|
77
|
+
- NumPy arrays of any integer dtype: np.array([3, 17], dtype=np.int32)
|
|
78
|
+
- Any array-like object with __iter__
|
|
79
|
+
|
|
80
|
+
The natural use is the token-index side of a sparse vector, whose weights
|
|
81
|
+
go through :func:`to_java_float_array`.
|
|
82
|
+
|
|
83
|
+
Prefer passing a NumPy array. JPype copies one through the buffer protocol
|
|
84
|
+
in a single crossing, while a Python list is marshalled element by element:
|
|
85
|
+
measured at 150 non-zeros, 2.6 us from an array against 6.7 us from a list,
|
|
86
|
+
2.6x, and the gap widens with length. The array's dtype does not matter --
|
|
87
|
+
int64, NumPy's default, converts as fast as int32 -- so there is no reason
|
|
88
|
+
to cast before calling this.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
vector: Array-like object containing integer values
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
Java int array
|
|
95
|
+
"""
|
|
96
|
+
if _np is not None and isinstance(vector, _np.ndarray):
|
|
97
|
+
if vector.dtype.kind in "iu" and vector.size:
|
|
98
|
+
bounds = _np.iinfo(_np.int32)
|
|
99
|
+
if vector.min() < bounds.min or vector.max() > bounds.max:
|
|
100
|
+
# A list raises this from JPype; an int64 array used to wrap
|
|
101
|
+
# silently (2**31 became -2**31).
|
|
102
|
+
raise OverflowError(
|
|
103
|
+
"Cannot convert value to Java int: an element does not fit in 32 bits"
|
|
104
|
+
)
|
|
105
|
+
return jtypes.JArray(jtypes.JInt)(vector)
|
|
106
|
+
|
|
107
|
+
if not isinstance(vector, list):
|
|
108
|
+
vector = list(vector)
|
|
109
|
+
|
|
110
|
+
return jtypes.JArray(jtypes.JInt)(vector)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def to_java_byte_array(vector):
|
|
114
|
+
"""
|
|
115
|
+
Convert a Python byte-like or integer array-like object to a Java byte array.
|
|
116
|
+
|
|
117
|
+
Accepts:
|
|
118
|
+
- Python lists of integers: [127, -12, 0]
|
|
119
|
+
- Python bytes / bytearray
|
|
120
|
+
- NumPy arrays
|
|
121
|
+
- Any array-like object with __iter__
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
vector: Array-like object containing byte values
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
Java byte array compatible with INT8-encoded ArcadeDB vector indexes
|
|
128
|
+
"""
|
|
129
|
+
try:
|
|
130
|
+
import numpy as np
|
|
131
|
+
|
|
132
|
+
if isinstance(vector, np.ndarray):
|
|
133
|
+
if vector.dtype in (np.int8, np.uint8):
|
|
134
|
+
return jtypes.JArray(jtypes.JByte)(vector.tobytes())
|
|
135
|
+
vector = vector.tolist()
|
|
136
|
+
except ImportError:
|
|
137
|
+
pass
|
|
138
|
+
|
|
139
|
+
if isinstance(vector, (bytes, bytearray)):
|
|
140
|
+
return jtypes.JArray(jtypes.JByte)(vector)
|
|
141
|
+
elif not isinstance(vector, list):
|
|
142
|
+
vector = list(vector)
|
|
143
|
+
|
|
144
|
+
normalized = []
|
|
145
|
+
for value in vector:
|
|
146
|
+
byte_value = int(value)
|
|
147
|
+
if byte_value < -128 or byte_value > 255:
|
|
148
|
+
raise ArcadeDBError(
|
|
149
|
+
"Byte vectors must contain values in the signed byte range "
|
|
150
|
+
"[-128, 127] or unsigned byte range [0, 255]"
|
|
151
|
+
)
|
|
152
|
+
if byte_value > 127:
|
|
153
|
+
byte_value -= 256
|
|
154
|
+
normalized.append(byte_value)
|
|
155
|
+
|
|
156
|
+
return jtypes.JArray(jtypes.JByte)(normalized)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def to_python_array(java_vector, use_numpy=True):
|
|
160
|
+
"""
|
|
161
|
+
Convert a Java array or ArrayList to a Python array.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
java_vector: Java array or ArrayList of floats
|
|
165
|
+
use_numpy: If True and NumPy is available, return np.array.
|
|
166
|
+
If False or NumPy unavailable, return Python list.
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
NumPy array if use_numpy=True and NumPy is available, else Python list
|
|
170
|
+
"""
|
|
171
|
+
# Convert to Python list first
|
|
172
|
+
if hasattr(java_vector, "__iter__"):
|
|
173
|
+
py_list = list(java_vector)
|
|
174
|
+
else:
|
|
175
|
+
py_list = [java_vector]
|
|
176
|
+
|
|
177
|
+
# Try to return NumPy array if requested
|
|
178
|
+
if use_numpy:
|
|
179
|
+
try:
|
|
180
|
+
import numpy as np
|
|
181
|
+
|
|
182
|
+
return np.array(py_list, dtype=np.float32)
|
|
183
|
+
except ImportError:
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
return py_list
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class VectorIndex:
|
|
190
|
+
"""
|
|
191
|
+
Wrapper for ArcadeDB vector index (JVector-based implementation).
|
|
192
|
+
|
|
193
|
+
This is the default and recommended implementation for vector search.
|
|
194
|
+
It uses JVector (a graph index combining HNSW hierarchy with Vamana/DiskANN
|
|
195
|
+
algorithms) which provides:
|
|
196
|
+
- No max_items limit (grows dynamically)
|
|
197
|
+
- Faster index construction
|
|
198
|
+
- Automatic indexing of existing records
|
|
199
|
+
- Concurrent construction support
|
|
200
|
+
|
|
201
|
+
Provides a Pythonic interface for creating and searching vector indexes,
|
|
202
|
+
with native support for NumPy arrays.
|
|
203
|
+
|
|
204
|
+
Distance Calculation:
|
|
205
|
+
The metric used depends on the `distance_function` parameter during index creation:
|
|
206
|
+
|
|
207
|
+
``create_vector_index`` defaults to COSINE.
|
|
208
|
+
|
|
209
|
+
1. **EUCLIDEAN**:
|
|
210
|
+
- Returns **Squared Euclidean Distance** (Lower is better).
|
|
211
|
+
- Formula: $d^2$ where $d$ is the Euclidean distance.
|
|
212
|
+
- Range: [0.0, +inf)
|
|
213
|
+
- 0.0: Identical vectors
|
|
214
|
+
- Larger values: more distant vectors
|
|
215
|
+
|
|
216
|
+
2. **COSINE**:
|
|
217
|
+
- Returns **Cosine Distance** (Lower is better).
|
|
218
|
+
- Formula: $1 - \\cos(\\theta)$
|
|
219
|
+
- Range: [0.0, 2.0]
|
|
220
|
+
- 0.0: Identical vectors (angle 0)
|
|
221
|
+
- 1.0: Orthogonal vectors (angle 90)
|
|
222
|
+
- 2.0: Opposite vectors (angle 180)
|
|
223
|
+
|
|
224
|
+
3. **DOT_PRODUCT**:
|
|
225
|
+
- Returns **-(1 + A.B) / 2** (Lower is better).
|
|
226
|
+
- Formula: $-(1 + A \\cdot B) / 2$
|
|
227
|
+
- Range: [-1.0, 0.0] for unit-length vectors, which the engine
|
|
228
|
+
expects (it logs a warning when sampled vectors are not).
|
|
229
|
+
- Lower values indicate higher similarity (larger positive dot product).
|
|
230
|
+
|
|
231
|
+
Quantization:
|
|
232
|
+
The index supports vector quantization to reduce memory usage and speed up search:
|
|
233
|
+
- **NONE** (Default): Full precision (float32).
|
|
234
|
+
- **INT8**: 8-bit integer quantization.
|
|
235
|
+
- **BINARY**: 1-bit quantization (for high-dimensional binary vectors).
|
|
236
|
+
"""
|
|
237
|
+
|
|
238
|
+
def __init__(self, java_index, database):
|
|
239
|
+
"""
|
|
240
|
+
Initialize VectorIndex wrapper.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
java_index: Java LSMVectorIndex object
|
|
244
|
+
database: Parent Database object
|
|
245
|
+
"""
|
|
246
|
+
self._java_index = java_index
|
|
247
|
+
self._database = database
|
|
248
|
+
# Resolved once per wrapper: the underlying LSM indexes and the
|
|
249
|
+
# PRODUCT-quantization readiness check are stable for the lifetime of
|
|
250
|
+
# this wrapper, but were previously re-derived on every search (several
|
|
251
|
+
# JPype crossings per query).
|
|
252
|
+
self._lsm_indexes = None
|
|
253
|
+
self._pq_ready = False
|
|
254
|
+
|
|
255
|
+
def _iter_lsm_indexes(self):
|
|
256
|
+
if self._lsm_indexes is None:
|
|
257
|
+
self._lsm_indexes = self._resolve_lsm_indexes()
|
|
258
|
+
return iter(self._lsm_indexes)
|
|
259
|
+
|
|
260
|
+
def _resolve_lsm_indexes(self):
|
|
261
|
+
if "LSMVectorIndex" in self._java_index.getClass().getName():
|
|
262
|
+
return [self._java_index]
|
|
263
|
+
|
|
264
|
+
resolved = []
|
|
265
|
+
if "TypeIndex" in self._java_index.getClass().getName():
|
|
266
|
+
sub_indexes = self._java_index.getSubIndexes()
|
|
267
|
+
if sub_indexes and not sub_indexes.isEmpty():
|
|
268
|
+
for sub in sub_indexes:
|
|
269
|
+
if "LSMVectorIndex" in sub.getClass().getName():
|
|
270
|
+
resolved.append(sub)
|
|
271
|
+
return resolved
|
|
272
|
+
|
|
273
|
+
def _get_primary_lsm_index(self):
|
|
274
|
+
for index in self._iter_lsm_indexes():
|
|
275
|
+
return index
|
|
276
|
+
return None
|
|
277
|
+
|
|
278
|
+
def _ensure_product_quantization_ready(self):
|
|
279
|
+
# Once the check confirms enough vectors it stays valid (indexed vectors
|
|
280
|
+
# only accumulate), so don't repeat the metadata/countEntries crossings
|
|
281
|
+
# on every search. An empty PRODUCT bucket is inconclusive (data may
|
|
282
|
+
# arrive later), so it does not memoize.
|
|
283
|
+
if self._pq_ready:
|
|
284
|
+
return
|
|
285
|
+
all_confirmed = True
|
|
286
|
+
for index in self._iter_lsm_indexes():
|
|
287
|
+
meta = index.getMetadata()
|
|
288
|
+
if str(meta.quantizationType) != "PRODUCT":
|
|
289
|
+
continue
|
|
290
|
+
|
|
291
|
+
vector_count = int(index.countEntries())
|
|
292
|
+
pq_clusters = int(meta.pqClusters)
|
|
293
|
+
|
|
294
|
+
if 0 < vector_count < pq_clusters:
|
|
295
|
+
raise ArcadeDBError(
|
|
296
|
+
"PRODUCT quantization in this ArcadeDB version requires at least "
|
|
297
|
+
"pq_clusters indexed vectors per bucket before graph build. "
|
|
298
|
+
f"Current bucket has {vector_count} vector(s) but "
|
|
299
|
+
f"pq_clusters={pq_clusters}. "
|
|
300
|
+
"Use a smaller pq_clusters value, insert more vectors, "
|
|
301
|
+
"or use INT8/BINARY/NONE."
|
|
302
|
+
)
|
|
303
|
+
if vector_count == 0:
|
|
304
|
+
all_confirmed = False
|
|
305
|
+
self._pq_ready = all_confirmed
|
|
306
|
+
|
|
307
|
+
def _normalize_ef_search(self, ef_search):
|
|
308
|
+
if ef_search is None:
|
|
309
|
+
return None
|
|
310
|
+
|
|
311
|
+
if isinstance(ef_search, bool):
|
|
312
|
+
raise ArcadeDBError("ef_search must be a positive integer")
|
|
313
|
+
|
|
314
|
+
try:
|
|
315
|
+
normalized = int(ef_search)
|
|
316
|
+
except (TypeError, ValueError) as exc:
|
|
317
|
+
raise ArcadeDBError("ef_search must be a positive integer") from exc
|
|
318
|
+
|
|
319
|
+
if normalized < 1 or normalized != ef_search:
|
|
320
|
+
raise ArcadeDBError("ef_search must be a positive integer")
|
|
321
|
+
|
|
322
|
+
return normalized
|
|
323
|
+
|
|
324
|
+
def _get_primary_metadata(self):
|
|
325
|
+
index = self._get_primary_lsm_index()
|
|
326
|
+
if index is None:
|
|
327
|
+
raise ArcadeDBError("Underlying index does not expose vector metadata")
|
|
328
|
+
return index.getMetadata()
|
|
329
|
+
|
|
330
|
+
def _get_type_name(self):
|
|
331
|
+
index = self._get_primary_lsm_index()
|
|
332
|
+
if index is None:
|
|
333
|
+
raise ArcadeDBError("Underlying index does not expose vector type metadata")
|
|
334
|
+
return str(index.getTypeName())
|
|
335
|
+
|
|
336
|
+
def _get_vector_property_name(self):
|
|
337
|
+
index = self._get_primary_lsm_index()
|
|
338
|
+
if index is None:
|
|
339
|
+
raise ArcadeDBError(
|
|
340
|
+
"Underlying index does not expose vector property metadata"
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
property_names = index.getPropertyNames()
|
|
344
|
+
if hasattr(property_names, "get"):
|
|
345
|
+
return str(property_names.get(0))
|
|
346
|
+
return str(property_names[0])
|
|
347
|
+
|
|
348
|
+
def _get_id_property_name(self):
|
|
349
|
+
index = self._get_primary_lsm_index()
|
|
350
|
+
if index is None:
|
|
351
|
+
raise ArcadeDBError("Underlying index does not expose id-property metadata")
|
|
352
|
+
return str(index.getIdPropertyName())
|
|
353
|
+
|
|
354
|
+
def _lookup_query_vector_by_key(self, key):
|
|
355
|
+
vector_property = self._get_vector_property_name()
|
|
356
|
+
type_name = self._get_type_name()
|
|
357
|
+
id_property = self._get_id_property_name()
|
|
358
|
+
quoted_vector_property = _quote_identifier(vector_property)
|
|
359
|
+
quoted_type_name = _quote_identifier(type_name)
|
|
360
|
+
quoted_id_property = _quote_identifier(id_property)
|
|
361
|
+
|
|
362
|
+
result = None
|
|
363
|
+
try:
|
|
364
|
+
result = self._database.lookup_by_key(type_name, [id_property], [key])
|
|
365
|
+
except ArcadeDBError:
|
|
366
|
+
result = None
|
|
367
|
+
|
|
368
|
+
if result is None:
|
|
369
|
+
# Identifiers are quoted via _quote_identifier(); the user-supplied
|
|
370
|
+
# `key` is passed as a `?` parameter, so this is not SQL injection.
|
|
371
|
+
result = self._database.query(
|
|
372
|
+
"sql",
|
|
373
|
+
(
|
|
374
|
+
f"SELECT {quoted_vector_property} AS `query_vector` FROM {quoted_type_name} " # nosec B608
|
|
375
|
+
f"WHERE {quoted_id_property} = ? LIMIT 1"
|
|
376
|
+
),
|
|
377
|
+
key,
|
|
378
|
+
).first()
|
|
379
|
+
|
|
380
|
+
if result is None:
|
|
381
|
+
raise ArcadeDBError(
|
|
382
|
+
f"No record found in type '{type_name}' where {id_property} = {key!r}"
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
query_vector = result.get("query_vector")
|
|
386
|
+
if query_vector is None:
|
|
387
|
+
raise ArcadeDBError(
|
|
388
|
+
f"Record found for {id_property} = {key!r} "
|
|
389
|
+
f"but property '{vector_property}' is null"
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
return query_vector
|
|
393
|
+
|
|
394
|
+
def _build_allowed_rids_set(self, allowed_rids):
|
|
395
|
+
if not allowed_rids:
|
|
396
|
+
return None
|
|
397
|
+
|
|
398
|
+
HashSet = jpype.JClass("java.util.HashSet")
|
|
399
|
+
|
|
400
|
+
allowed_rids_set = HashSet()
|
|
401
|
+
for rid_value in allowed_rids:
|
|
402
|
+
allowed_rids_set.add(self._database.to_java_rid(rid_value))
|
|
403
|
+
return allowed_rids_set
|
|
404
|
+
|
|
405
|
+
def _lookup_record_by_rid(self, rid):
|
|
406
|
+
# `rid` is already a Java RID from the search result: pass it straight
|
|
407
|
+
# through instead of stringifying and re-parsing it (previously one
|
|
408
|
+
# str() crossing + a new Java RID allocation per hit).
|
|
409
|
+
record = self._database._lookup_by_java_rid(rid)
|
|
410
|
+
if record is None:
|
|
411
|
+
raise ArcadeDBError(f"Vector search returned missing RID: {rid}")
|
|
412
|
+
return record
|
|
413
|
+
|
|
414
|
+
def _wrap_pair_results(self, pairs):
|
|
415
|
+
from .core import _wrap_java_record
|
|
416
|
+
from .graph import _graph_calls
|
|
417
|
+
|
|
418
|
+
calls = _graph_calls()
|
|
419
|
+
if calls is not None:
|
|
420
|
+
# every hit's record and score in one crossing instead of
|
|
421
|
+
# getFirst, getSecond, and the record lookup each
|
|
422
|
+
hits = calls.hits(self._database._java_db, pairs)
|
|
423
|
+
records, scores = hits[0], hits[1]
|
|
424
|
+
wrapped = []
|
|
425
|
+
for i, record in enumerate(records):
|
|
426
|
+
if record is None:
|
|
427
|
+
# name the hit exactly as the per-hit path does
|
|
428
|
+
self._lookup_record_by_rid(list(pairs)[i].getFirst())
|
|
429
|
+
wrapped.append(
|
|
430
|
+
(_wrap_java_record(record, self._database), float(scores[i]))
|
|
431
|
+
)
|
|
432
|
+
return wrapped
|
|
433
|
+
|
|
434
|
+
wrapped_results = []
|
|
435
|
+
for pair in pairs:
|
|
436
|
+
rid = pair.getFirst()
|
|
437
|
+
score = pair.getSecond()
|
|
438
|
+
wrapped_results.append((self._lookup_record_by_rid(rid), float(score)))
|
|
439
|
+
|
|
440
|
+
return wrapped_results
|
|
441
|
+
|
|
442
|
+
def _find_neighbor_pairs(
|
|
443
|
+
self,
|
|
444
|
+
idx,
|
|
445
|
+
*,
|
|
446
|
+
java_vector,
|
|
447
|
+
k,
|
|
448
|
+
allowed_rids_set,
|
|
449
|
+
approximate,
|
|
450
|
+
ef_search=None,
|
|
451
|
+
):
|
|
452
|
+
if approximate:
|
|
453
|
+
if allowed_rids_set is not None:
|
|
454
|
+
return idx.findNeighborsFromVectorApproximate(
|
|
455
|
+
java_vector, k, allowed_rids_set
|
|
456
|
+
)
|
|
457
|
+
return idx.findNeighborsFromVectorApproximate(java_vector, k)
|
|
458
|
+
|
|
459
|
+
if allowed_rids_set is not None and ef_search is not None:
|
|
460
|
+
return idx.findNeighborsFromVector(
|
|
461
|
+
java_vector, k, ef_search, allowed_rids_set
|
|
462
|
+
)
|
|
463
|
+
if allowed_rids_set is not None:
|
|
464
|
+
return idx.findNeighborsFromVector(java_vector, k, allowed_rids_set)
|
|
465
|
+
if ef_search is not None:
|
|
466
|
+
return idx.findNeighborsFromVector(java_vector, k, ef_search)
|
|
467
|
+
return idx.findNeighborsFromVector(java_vector, k)
|
|
468
|
+
|
|
469
|
+
def _sort_results(self, results):
|
|
470
|
+
# All similarity functions are exposed as distances (lower is better) by the
|
|
471
|
+
# engine's findNeighborsFromVector, so the best matches sort ascending by score.
|
|
472
|
+
results.sort(key=lambda item: item[1])
|
|
473
|
+
return results
|
|
474
|
+
|
|
475
|
+
def _collect_search_results(
|
|
476
|
+
self,
|
|
477
|
+
*,
|
|
478
|
+
java_vector,
|
|
479
|
+
k,
|
|
480
|
+
allowed_rids_set,
|
|
481
|
+
approximate,
|
|
482
|
+
ef_search=None,
|
|
483
|
+
):
|
|
484
|
+
all_results = []
|
|
485
|
+
|
|
486
|
+
for idx in self._iter_lsm_indexes():
|
|
487
|
+
pairs = self._find_neighbor_pairs(
|
|
488
|
+
idx,
|
|
489
|
+
java_vector=java_vector,
|
|
490
|
+
k=k,
|
|
491
|
+
allowed_rids_set=allowed_rids_set,
|
|
492
|
+
approximate=approximate,
|
|
493
|
+
ef_search=ef_search,
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
all_results.extend(self._wrap_pair_results(pairs))
|
|
497
|
+
|
|
498
|
+
return self._sort_results(all_results)[:k]
|
|
499
|
+
|
|
500
|
+
def _run_search(
|
|
501
|
+
self,
|
|
502
|
+
*,
|
|
503
|
+
query_vector,
|
|
504
|
+
k,
|
|
505
|
+
allowed_rids,
|
|
506
|
+
approximate,
|
|
507
|
+
ef_search=None,
|
|
508
|
+
):
|
|
509
|
+
java_vector = to_java_float_array(query_vector)
|
|
510
|
+
allowed_rids_set = self._build_allowed_rids_set(allowed_rids)
|
|
511
|
+
|
|
512
|
+
return self._collect_search_results(
|
|
513
|
+
java_vector=java_vector,
|
|
514
|
+
k=k,
|
|
515
|
+
allowed_rids_set=allowed_rids_set,
|
|
516
|
+
approximate=approximate,
|
|
517
|
+
ef_search=ef_search,
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
def find_nearest(
|
|
521
|
+
self,
|
|
522
|
+
query_vector,
|
|
523
|
+
k=10,
|
|
524
|
+
ef_search=None,
|
|
525
|
+
allowed_rids=None,
|
|
526
|
+
):
|
|
527
|
+
"""
|
|
528
|
+
Find k nearest neighbors to the query vector.
|
|
529
|
+
|
|
530
|
+
Args:
|
|
531
|
+
query_vector: Query vector as Python list, NumPy array, or array-like
|
|
532
|
+
k: Number of nearest neighbors to return (final k). Default is 10.
|
|
533
|
+
ef_search: Optional search beam width override for exact graph search. None
|
|
534
|
+
uses the Java index default/adaptive behavior.
|
|
535
|
+
allowed_rids: Optional list of RID strings (e.g. ["#1:0", "#2:5"]) to
|
|
536
|
+
restrict search
|
|
537
|
+
|
|
538
|
+
Returns:
|
|
539
|
+
List of tuples: [(record, score), ...]
|
|
540
|
+
- record: The matching ArcadeDB record (Vertex, Document, or Edge)
|
|
541
|
+
- score: The similarity score/distance
|
|
542
|
+
"""
|
|
543
|
+
effective_ef_search = self._normalize_ef_search(ef_search)
|
|
544
|
+
|
|
545
|
+
try:
|
|
546
|
+
self._ensure_product_quantization_ready()
|
|
547
|
+
return self._run_search(
|
|
548
|
+
query_vector=query_vector,
|
|
549
|
+
k=k,
|
|
550
|
+
allowed_rids=allowed_rids,
|
|
551
|
+
approximate=False,
|
|
552
|
+
ef_search=effective_ef_search,
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
except ArcadeDBError:
|
|
556
|
+
raise
|
|
557
|
+
except Exception as e:
|
|
558
|
+
raise ArcadeDBError(f"Vector search failed: {e}") from e
|
|
559
|
+
|
|
560
|
+
def get_size(self):
|
|
561
|
+
"""
|
|
562
|
+
Get the current number of items in the index.
|
|
563
|
+
|
|
564
|
+
Returns:
|
|
565
|
+
int: Number of items currently indexed
|
|
566
|
+
"""
|
|
567
|
+
try:
|
|
568
|
+
# Both TypeIndex and LSMVectorIndex support countEntries()
|
|
569
|
+
return self._java_index.countEntries()
|
|
570
|
+
except Exception as e:
|
|
571
|
+
raise ArcadeDBError(f"Failed to get index size: {e}") from e
|
|
572
|
+
|
|
573
|
+
def get_metadata(self):
|
|
574
|
+
"""
|
|
575
|
+
Get stable metadata for the vector index.
|
|
576
|
+
|
|
577
|
+
Returns:
|
|
578
|
+
dict: Index metadata including names, dimensions, similarity function,
|
|
579
|
+
id property, quantization, and selected tuning settings.
|
|
580
|
+
"""
|
|
581
|
+
try:
|
|
582
|
+
index = self._get_primary_lsm_index()
|
|
583
|
+
meta = self._get_primary_metadata()
|
|
584
|
+
|
|
585
|
+
return {
|
|
586
|
+
"index_name": str(self._java_index.getName()),
|
|
587
|
+
"bucket_index_name": str(index.getName()),
|
|
588
|
+
"type_name": str(index.getTypeName()),
|
|
589
|
+
"vector_property": self._get_vector_property_name(),
|
|
590
|
+
"dimensions": int(meta.dimensions),
|
|
591
|
+
"similarity_function": str(meta.similarityFunction),
|
|
592
|
+
"id_property": str(meta.idPropertyName),
|
|
593
|
+
"quantization": str(meta.quantizationType),
|
|
594
|
+
"max_connections": int(meta.maxConnections),
|
|
595
|
+
"beam_width": int(meta.beamWidth),
|
|
596
|
+
"ef_search": int(meta.efSearch),
|
|
597
|
+
# location_cache_size is absent on purpose: the engine dropped
|
|
598
|
+
# the field in #5559/#5568, so reading it here would raise on
|
|
599
|
+
# every get_metadata() call.
|
|
600
|
+
"graph_build_cache_size": int(meta.graphBuildCacheSize),
|
|
601
|
+
"mutations_before_rebuild": int(meta.mutationsBeforeRebuild),
|
|
602
|
+
"store_vectors_in_graph": bool(meta.storeVectorsInGraph),
|
|
603
|
+
"add_hierarchy": bool(meta.addHierarchy),
|
|
604
|
+
"build_state": str(meta.buildState),
|
|
605
|
+
"pq_subspaces": int(meta.pqSubspaces),
|
|
606
|
+
"pq_clusters": int(meta.pqClusters),
|
|
607
|
+
"pq_center_globally": bool(meta.pqCenterGlobally),
|
|
608
|
+
"pq_training_limit": int(meta.pqTrainingLimit),
|
|
609
|
+
}
|
|
610
|
+
except Exception as e:
|
|
611
|
+
raise ArcadeDBError(f"Failed to read vector index metadata: {e}") from e
|
|
612
|
+
|
|
613
|
+
def get_stats(self):
|
|
614
|
+
"""
|
|
615
|
+
Get live runtime counters for the vector index.
|
|
616
|
+
|
|
617
|
+
Unlike :meth:`get_metadata`, which returns the index's stable
|
|
618
|
+
configuration, this returns counters that change as the index is
|
|
619
|
+
built and queried: graph state, cache capacities and hit/miss
|
|
620
|
+
counts, and where vectors were read from (index pages, graph, or
|
|
621
|
+
the documents themselves). Keys follow the engine and may grow
|
|
622
|
+
between releases.
|
|
623
|
+
|
|
624
|
+
Useful for sizing caches and diagnosing build cost, for example
|
|
625
|
+
``vectorFetchFromDocuments`` being non-zero after a build means
|
|
626
|
+
the build cache did not hold the whole vector set.
|
|
627
|
+
|
|
628
|
+
Returns:
|
|
629
|
+
dict: Counter name to value, for the primary bucket index
|
|
630
|
+
(see ``get_metadata()["bucket_index_name"]``).
|
|
631
|
+
|
|
632
|
+
Examples:
|
|
633
|
+
>>> stats = index.get_stats() # doctest: +SKIP
|
|
634
|
+
>>> stats["vectorCacheHits"], stats["vectorCacheMisses"]
|
|
635
|
+
(10399017, 1840948)
|
|
636
|
+
"""
|
|
637
|
+
try:
|
|
638
|
+
raw = self._get_primary_lsm_index().getStats()
|
|
639
|
+
except Exception as e:
|
|
640
|
+
raise ArcadeDBError(f"Failed to read vector index stats: {e}") from e
|
|
641
|
+
|
|
642
|
+
stats = {}
|
|
643
|
+
for java_key in raw.keySet():
|
|
644
|
+
key = str(java_key)
|
|
645
|
+
value = raw.get(java_key)
|
|
646
|
+
if value is None:
|
|
647
|
+
stats[key] = None
|
|
648
|
+
continue
|
|
649
|
+
try:
|
|
650
|
+
kind = str(value.getClass().getSimpleName())
|
|
651
|
+
except Exception:
|
|
652
|
+
kind = ""
|
|
653
|
+
if kind in ("Long", "Integer", "Short", "Byte", "BigInteger"):
|
|
654
|
+
stats[key] = int(value)
|
|
655
|
+
elif kind in ("Double", "Float"):
|
|
656
|
+
stats[key] = float(value)
|
|
657
|
+
elif kind == "Boolean":
|
|
658
|
+
stats[key] = bool(value)
|
|
659
|
+
else:
|
|
660
|
+
stats[key] = str(value)
|
|
661
|
+
return stats
|
|
662
|
+
|
|
663
|
+
def find_nearest_by_key(
|
|
664
|
+
self,
|
|
665
|
+
key,
|
|
666
|
+
k=10,
|
|
667
|
+
ef_search=None,
|
|
668
|
+
allowed_rids=None,
|
|
669
|
+
):
|
|
670
|
+
"""
|
|
671
|
+
Find nearest neighbors using the indexed vector from an existing record.
|
|
672
|
+
|
|
673
|
+
Args:
|
|
674
|
+
key: Value of the configured id property for the source record.
|
|
675
|
+
k: Number of nearest neighbors to return (default: 10).
|
|
676
|
+
ef_search: Optional search beam width override for exact graph search.
|
|
677
|
+
allowed_rids: Optional RID whitelist to restrict the final search.
|
|
678
|
+
|
|
679
|
+
Returns:
|
|
680
|
+
List of tuples: [(record, score), ...]
|
|
681
|
+
"""
|
|
682
|
+
query_vector = self._lookup_query_vector_by_key(key)
|
|
683
|
+
return self.find_nearest(
|
|
684
|
+
query_vector,
|
|
685
|
+
k=k,
|
|
686
|
+
ef_search=ef_search,
|
|
687
|
+
allowed_rids=allowed_rids,
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
def find_nearest_approximate(
|
|
691
|
+
self,
|
|
692
|
+
query_vector,
|
|
693
|
+
k=10,
|
|
694
|
+
allowed_rids=None,
|
|
695
|
+
):
|
|
696
|
+
"""
|
|
697
|
+
Find k nearest neighbors using Product Quantization (PQ) approximate search.
|
|
698
|
+
|
|
699
|
+
Args:
|
|
700
|
+
query_vector: Query vector as Python list, NumPy array, or array-like
|
|
701
|
+
k: Number of nearest neighbors to return (final k)
|
|
702
|
+
allowed_rids: Optional list of RID strings (e.g. ["#1:0", "#2:5"]) to restrict search
|
|
703
|
+
|
|
704
|
+
Returns:
|
|
705
|
+
List of tuples: [(record, score), ...]
|
|
706
|
+
"""
|
|
707
|
+
try:
|
|
708
|
+
quantization = self.get_quantization()
|
|
709
|
+
if quantization != "PRODUCT":
|
|
710
|
+
raise ArcadeDBError(
|
|
711
|
+
"Approximate search requires quantization=PRODUCT (PQ)"
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
self._ensure_product_quantization_ready()
|
|
715
|
+
|
|
716
|
+
return self._run_search(
|
|
717
|
+
query_vector=query_vector,
|
|
718
|
+
k=k,
|
|
719
|
+
allowed_rids=allowed_rids,
|
|
720
|
+
approximate=True,
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
except ArcadeDBError:
|
|
724
|
+
raise
|
|
725
|
+
except Exception as e:
|
|
726
|
+
raise ArcadeDBError(f"Approximate search failed: {e}") from e
|
|
727
|
+
|
|
728
|
+
def get_quantization(self):
|
|
729
|
+
"""
|
|
730
|
+
Get the quantization type of the index.
|
|
731
|
+
|
|
732
|
+
Returns:
|
|
733
|
+
str: "NONE", "INT8", "BINARY", or "PRODUCT"
|
|
734
|
+
"""
|
|
735
|
+
try:
|
|
736
|
+
idx_to_check = self._get_primary_lsm_index()
|
|
737
|
+
|
|
738
|
+
if idx_to_check:
|
|
739
|
+
return str(idx_to_check.getMetadata().quantizationType)
|
|
740
|
+
return "NONE"
|
|
741
|
+
except Exception:
|
|
742
|
+
return "NONE"
|
|
743
|
+
|
|
744
|
+
def warm_up(self):
|
|
745
|
+
"""
|
|
746
|
+
Load the vector graph now instead of on the first search.
|
|
747
|
+
|
|
748
|
+
After a database is opened, the index loads its persisted graph lazily,
|
|
749
|
+
on the first search, and that search pays the load (ArcadeData/arcadedb#8852:
|
|
750
|
+
about 1 s at 1M 64-dimension vectors on 26.10.1, growing with the index, and
|
|
751
|
+
more on an index with deletions since its graph was saved, which still
|
|
752
|
+
re-reads every vector's document). Calling this right after
|
|
753
|
+
opening the database pays it before the first query arrives, which is
|
|
754
|
+
what a service that restarts wants. It loads the existing graph and does
|
|
755
|
+
not rebuild it; a no-op when the graph is already in memory, and safe to
|
|
756
|
+
call while other threads search. For TypeIndex wrappers, warms every
|
|
757
|
+
underlying LSMVectorIndex.
|
|
758
|
+
|
|
759
|
+
The alternative on engines before 26.10.1 is one throwaway search.
|
|
760
|
+
|
|
761
|
+
Raises:
|
|
762
|
+
ArcadeDBError: the index cannot be warmed, including on an engine
|
|
763
|
+
older than 26.10.1, which has no ``warmUp()``.
|
|
764
|
+
"""
|
|
765
|
+
try:
|
|
766
|
+
warmed_any = False
|
|
767
|
+
for index in self._iter_lsm_indexes():
|
|
768
|
+
index.warmUp()
|
|
769
|
+
warmed_any = True
|
|
770
|
+
if warmed_any:
|
|
771
|
+
return
|
|
772
|
+
raise ArcadeDBError("Underlying index is not an LSM vector index")
|
|
773
|
+
except ArcadeDBError:
|
|
774
|
+
raise
|
|
775
|
+
except Exception as e:
|
|
776
|
+
raise ArcadeDBError(f"Failed to warm up vector index: {e}") from e
|
|
777
|
+
|
|
778
|
+
def build_graph_now(self):
|
|
779
|
+
"""
|
|
780
|
+
Trigger an immediate rebuild of the underlying vector graph.
|
|
781
|
+
|
|
782
|
+
Useful after bulk inserts/updates to avoid waiting for a search-triggered lazy
|
|
783
|
+
build. For TypeIndex wrappers, rebuilds all underlying LSMVectorIndex instances.
|
|
784
|
+
"""
|
|
785
|
+
try:
|
|
786
|
+
self._ensure_product_quantization_ready()
|
|
787
|
+
|
|
788
|
+
built_any = False
|
|
789
|
+
for index in self._iter_lsm_indexes():
|
|
790
|
+
index.buildVectorGraphNow()
|
|
791
|
+
built_any = True
|
|
792
|
+
|
|
793
|
+
if built_any:
|
|
794
|
+
return
|
|
795
|
+
|
|
796
|
+
raise ArcadeDBError(
|
|
797
|
+
"Underlying index does not support vector graph rebuild"
|
|
798
|
+
)
|
|
799
|
+
except Exception as e:
|
|
800
|
+
raise ArcadeDBError(f"Failed to rebuild vector graph: {e}") from e
|