deglib 0.2.5__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deglib-0.2.5 → deglib-0.2.7}/CMakeLists.txt +11 -2
- {deglib-0.2.5/src/deglib.egg-info → deglib-0.2.7}/PKG-INFO +47 -24
- {deglib-0.2.5 → deglib-0.2.7}/README.md +46 -23
- {deglib-0.2.5 → deglib-0.2.7}/lib/CMakeLists.txt +6 -7
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/benchmark.h +37 -1
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/build.h +9 -7
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/dataset.h +201 -32
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/repository.h +51 -33
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/analysis.h +140 -13
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/builder.h +254 -85
- deglib-0.2.7/lib/deglib/include/deglib/deglib.h +128 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/evp_ip.h +147 -208
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp16.h +143 -1
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp16_ip.h +30 -2
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp16_l2.h +32 -1
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp32_ip.h +33 -2
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp32_l2.h +32 -2
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/int8_ip.h +76 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/uint8_ip.h +84 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distances.h +1 -1
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/dynamic_graph.h +227 -328
- deglib-0.2.7/lib/deglib/include/deglib/graph/internal_graph.h +1097 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/mutable_graph.h +2 -0
- deglib-0.2.7/lib/deglib/include/deglib/graph/readonly_graph.h +290 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/sizebounded_graph.h +237 -410
- deglib-0.2.7/lib/deglib/include/deglib/graph.h +1349 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/flas/junker_volgenant_solver.h +4 -2
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/pruning.h +48 -48
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization.h +13 -140
- deglib-0.2.7/lib/deglib/include/deglib/search/kmeans.h +358 -0
- deglib-0.2.7/lib/deglib/include/deglib/search/reranker.h +603 -0
- deglib-0.2.7/lib/deglib/include/deglib/search/result_list.h +273 -0
- deglib-0.2.7/lib/deglib/include/deglib/search/searcher.h +698 -0
- deglib-0.2.7/lib/deglib/include/deglib/search.h +14 -0
- deglib-0.2.7/lib/deglib/include/deglib/serialization.h +247 -0
- deglib-0.2.7/lib/deglib/include/deglib/transformation/evp.h +850 -0
- deglib-0.2.7/lib/deglib/include/deglib/transformation/mips.h +202 -0
- deglib-0.2.7/lib/deglib/include/deglib/transformation/normalize.h +131 -0
- deglib-0.2.7/lib/deglib/include/deglib/transformation/scalar.h +1362 -0
- deglib-0.2.7/lib/deglib/include/deglib/transformation.h +266 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/test/CMakeLists.txt +17 -6
- {deglib-0.2.5 → deglib-0.2.7}/lib/test/src/common/test_helpers.h +6 -3
- deglib-0.2.7/lib/test/src/unit/graph/test_dynamic_graph_common.h +22 -0
- {deglib-0.2.5 → deglib-0.2.7}/setup.py +28 -24
- deglib-0.2.7/src/deg_cpp/deglib_cpp.cpp +2540 -0
- deglib-0.2.7/src/deglib/__init__.py +83 -0
- deglib-0.2.7/src/deglib/analysis.py +181 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib/builder.py +104 -21
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib/distances.py +1 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib/graph.py +220 -73
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib/optimization.py +13 -88
- deglib-0.2.7/src/deglib/search.py +327 -0
- deglib-0.2.7/src/deglib/transformation.py +96 -0
- {deglib-0.2.5 → deglib-0.2.7/src/deglib.egg-info}/PKG-INFO +47 -24
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/SOURCES.txt +15 -5
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_analysis.py +70 -0
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_builder.py +79 -9
- deglib-0.2.7/tests/test_evp_quantizer.py +132 -0
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_filter.py +3 -3
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_graph.py +211 -35
- deglib-0.2.7/tests/test_l2_normalize.py +58 -0
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_rerank.py +85 -0
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_scalar_quantizer.py +109 -16
- deglib-0.2.7/tests/test_searcher.py +326 -0
- deglib-0.2.5/tests/test_transform.py → deglib-0.2.7/tests/test_transformation.py +1 -1
- deglib-0.2.7/tests/test_transformation_graph.py +350 -0
- deglib-0.2.5/lib/bench/include/stats.h +0 -341
- deglib-0.2.5/lib/deglib/include/deglib/deglib.h +0 -17
- deglib-0.2.5/lib/deglib/include/deglib/graph/internal_graph.h +0 -528
- deglib-0.2.5/lib/deglib/include/deglib/graph/readonly_graph.h +0 -289
- deglib-0.2.5/lib/deglib/include/deglib/graph.h +0 -266
- deglib-0.2.5/lib/deglib/include/deglib/optimization/quantization/evp_quantize.h +0 -460
- deglib-0.2.5/lib/deglib/include/deglib/optimization/quantization/scalar_quantize.h +0 -714
- deglib-0.2.5/lib/deglib/include/deglib/optimization/transform.h +0 -119
- deglib-0.2.5/lib/deglib/include/deglib/search/searcher.h +0 -493
- deglib-0.2.5/lib/deglib/include/deglib/search.h +0 -253
- deglib-0.2.5/src/deg_cpp/deglib_cpp.cpp +0 -1865
- deglib-0.2.5/src/deglib/__init__.py +0 -46
- deglib-0.2.5/src/deglib/analysis.py +0 -152
- deglib-0.2.5/src/deglib/search.py +0 -184
- deglib-0.2.5/tests/test_searcher.py +0 -166
- {deglib-0.2.5 → deglib-0.2.7}/LICENSE +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/MANIFEST.in +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/CMakeLists.txt +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/file_io.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/logging.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/stopwatch.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/cmake_modules/DetectCPUFeatures.cmake +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/CMakeLists.txt +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/concurrent.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/config.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp32.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/int8.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/int8_l2.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/residual_mode.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/uint8.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/uint8_l2.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/filter.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/intrusive_list.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/visited_list_pool.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/flas/fast_linear_assignment_sorter.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/flas/fast_linear_assignment_sorter_mt.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/utils/cpu.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/utils/memory.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/utils/random.h +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/pyproject.toml +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/setup.cfg +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib/cpu.py +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib/utils.py +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/dependency_links.txt +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/requires.txt +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/top_level.txt +0 -0
- {deglib-0.2.5 → deglib-0.2.7}/tests/test_presort.py +0 -0
|
@@ -15,11 +15,20 @@ find_package(pybind11 CONFIG REQUIRED)
|
|
|
15
15
|
set(ENABLE_BENCHMARKS OFF CACHE BOOL "Disable benchmarks" FORCE)
|
|
16
16
|
set(ENABLE_TESTING OFF CACHE BOOL "Disable tests" FORCE)
|
|
17
17
|
|
|
18
|
+
# Find deglib cpp source: prefer ../cpp in repository checkouts, fallback to local lib/ (PyPI sdist)
|
|
19
|
+
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/CMakeLists.txt")
|
|
20
|
+
set(DEGLIB_CPP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp")
|
|
21
|
+
elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/lib/CMakeLists.txt")
|
|
22
|
+
set(DEGLIB_CPP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib")
|
|
23
|
+
else()
|
|
24
|
+
message(FATAL_ERROR "Could not find deglib cpp source directory (checked ../cpp/ and lib/)")
|
|
25
|
+
endif()
|
|
26
|
+
|
|
18
27
|
# Add cpp-deglib
|
|
19
|
-
add_subdirectory(${
|
|
28
|
+
add_subdirectory(${DEGLIB_CPP_DIR} deglib_cpp_build)
|
|
20
29
|
pybind11_add_module(deglib_cpp src/deg_cpp/deglib_cpp.cpp)
|
|
21
30
|
|
|
22
31
|
# Specify the include directories
|
|
23
|
-
include_directories(${
|
|
32
|
+
include_directories(${DEGLIB_CPP_DIR}/deglib/include)
|
|
24
33
|
|
|
25
34
|
target_link_libraries(deglib_cpp PRIVATE compile-options)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deglib
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.7
|
|
4
4
|
Summary: Python bindings for the Dynamic Exploration Graph library by Nico Hezel
|
|
5
5
|
Author: Nico Hezel
|
|
6
6
|
Author-email: Bruno Schilling <bruno.schilling@htw-berlin.de>
|
|
@@ -47,7 +47,7 @@ Python bindings for the high-performance C++ Dynamic Exploration Graph (DEG) lib
|
|
|
47
47
|
- [Graph Optimization, Quantization & Reranking Pipeline](#graph-optimization-quantization--reranking-pipeline)
|
|
48
48
|
- [Concepts & Parameters](#concepts--parameters)
|
|
49
49
|
- [OptimizationTarget](#optimizationtarget)
|
|
50
|
-
- [Search Parameter `
|
|
50
|
+
- [Unified Search Parameter `eps_or_ef`](#unified-search-parameter-eps_or_ef)
|
|
51
51
|
- [Supported Metrics & Data Types](#supported-metrics--data-types)
|
|
52
52
|
- [Example Projects](#example-projects)
|
|
53
53
|
- [API Reference](#api-reference)
|
|
@@ -68,8 +68,10 @@ To build and install the package directly from the repository:
|
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
70
|
cd python/
|
|
71
|
-
|
|
72
|
-
pip install .
|
|
71
|
+
uv venv
|
|
72
|
+
uv pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0 wheel==0.48.0
|
|
73
|
+
uv run python setup.py copy_build_files
|
|
74
|
+
uv pip install . --no-build-isolation
|
|
73
75
|
```
|
|
74
76
|
|
|
75
77
|
---
|
|
@@ -89,7 +91,7 @@ data = np.random.random((num_samples, dims)).astype(np.float32)
|
|
|
89
91
|
query = np.random.random(dims).astype(np.float32)
|
|
90
92
|
|
|
91
93
|
# 2. Build index directly from data (multithreaded by default)
|
|
92
|
-
graph = deglib.builder.build_from_data(data, edges_per_vertex=32,
|
|
94
|
+
graph = deglib.builder.build_from_data(data, edges_per_vertex=32, show_progress=True)
|
|
93
95
|
|
|
94
96
|
# 3. Query top-k nearest neighbors
|
|
95
97
|
indices, distances = graph.search(query, k=10, eps=0.1)
|
|
@@ -106,7 +108,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
|
|
|
106
108
|
|
|
107
109
|
1. **Fixed-Capacity Mutable (`SizeBoundedGraph`)**: Memory is preallocated for a fixed maximum capacity. Fast and memory-efficient for static/batch datasets.
|
|
108
110
|
```python
|
|
109
|
-
space = deglib.FloatSpace.create(
|
|
111
|
+
space = deglib.FloatSpace.create(dim=128, metric=deglib.Metric.FP32_L2)
|
|
110
112
|
graph = deglib.create_empty(capacity=10_000, feature_space=space, edges_per_vertex=32)
|
|
111
113
|
```
|
|
112
114
|
|
|
@@ -125,7 +127,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
|
|
|
125
127
|
### Saving and Loading Graphs
|
|
126
128
|
|
|
127
129
|
```python
|
|
128
|
-
# Save
|
|
130
|
+
# Save graph to disk (supports both mutable and read-only graphs)
|
|
129
131
|
graph.save_graph("index.deg")
|
|
130
132
|
|
|
131
133
|
# Load as compact, optimized read-only graph for production search
|
|
@@ -184,7 +186,7 @@ from deglib.search import Filter, rerank
|
|
|
184
186
|
allowed_ids = np.array([1, 5, 10, 42, 99], dtype=np.int32)
|
|
185
187
|
search_filter = Filter(allowed_ids)
|
|
186
188
|
|
|
187
|
-
indices, distances = graph.search(query, k=5,
|
|
189
|
+
indices, distances = graph.search(query, k=5, eps_or_ef=0.1, filter_labels=search_filter)
|
|
188
190
|
|
|
189
191
|
# Exact distance reranking across candidates
|
|
190
192
|
queries = np.random.random((10, dims)).astype(np.float32)
|
|
@@ -205,11 +207,19 @@ top_indices, top_distances = rerank(
|
|
|
205
207
|
|
|
206
208
|
### Exploratory Search & Graph Navigation
|
|
207
209
|
|
|
208
|
-
DEG supports exploratory search directly from existing vertex labels:
|
|
210
|
+
DEG supports exploratory search directly from existing vertex labels (single or batch):
|
|
209
211
|
|
|
210
212
|
```python
|
|
211
213
|
# Explore graph starting from entry vertex label 105
|
|
212
|
-
explored_labels, distances = graph.explore(entry_external_label=105, k=10,
|
|
214
|
+
explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps_or_ef=0.1, include_entry=False)
|
|
215
|
+
|
|
216
|
+
# Batch exploration from multiple entry vertices in parallel
|
|
217
|
+
batch_entries = np.array([105, 210, 315], dtype=np.uint32)
|
|
218
|
+
batch_labels, batch_distances = graph.explore(batch_entries, k=10, eps_or_ef=0.1, threads=4)
|
|
219
|
+
|
|
220
|
+
# Retrieve graph neighbors directly
|
|
221
|
+
neighbors = graph.get_neighbors(105)
|
|
222
|
+
batch_neighbors = graph.get_neighbors_batch(batch_entries, threads=4) # shape: (3, edges_per_vertex)
|
|
213
223
|
```
|
|
214
224
|
|
|
215
225
|
---
|
|
@@ -221,7 +231,8 @@ A complete end-to-end pipeline demonstrating **FLAS pre-sorting**, **multithread
|
|
|
221
231
|
```python
|
|
222
232
|
import numpy as np
|
|
223
233
|
import deglib
|
|
224
|
-
from deglib.optimization import presort,
|
|
234
|
+
from deglib.optimization import presort, prune_non_mrng_edges
|
|
235
|
+
from deglib.transformation import ScalarQuantizerInt8
|
|
225
236
|
from deglib.search import create_searcher
|
|
226
237
|
|
|
227
238
|
num_vectors, dims = 10_000, 128
|
|
@@ -229,7 +240,7 @@ data = np.random.randn(num_vectors, dims).astype(np.float32)
|
|
|
229
240
|
query = np.random.randn(dims).astype(np.float32)
|
|
230
241
|
|
|
231
242
|
# 1. Pre-sort vectors using FLAS for improved memory locality and index construction speed
|
|
232
|
-
perm = presort(data, metric=deglib.Metric.FP32_InnerProduct,
|
|
243
|
+
perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, show_progress=True)
|
|
233
244
|
sorted_data = data[perm]
|
|
234
245
|
|
|
235
246
|
# 2. Build exploration graph on unquantized/original data
|
|
@@ -237,11 +248,11 @@ graph = deglib.builder.build_from_data(
|
|
|
237
248
|
sorted_data,
|
|
238
249
|
metric=deglib.Metric.FP32_InnerProduct,
|
|
239
250
|
edges_per_vertex=32,
|
|
240
|
-
|
|
251
|
+
show_progress=True,
|
|
241
252
|
)
|
|
242
253
|
|
|
243
|
-
# 3. Prune redundant non-
|
|
244
|
-
pruned_count =
|
|
254
|
+
# 3. Prune redundant non-MRNG edges to optimize graph topology
|
|
255
|
+
pruned_count = prune_non_mrng_edges(graph)
|
|
245
256
|
print(f"Pruned {pruned_count} redundant edges.")
|
|
246
257
|
|
|
247
258
|
# 4. Fit Int8 quantizer on base dataset and quantize features for memory reduction and ultra-fast search
|
|
@@ -251,19 +262,20 @@ quant_int8_data = quantizer.quantize(sorted_data)
|
|
|
251
262
|
int8_space = deglib.FloatSpace.create(dims, deglib.Metric.Int8_InnerProduct)
|
|
252
263
|
|
|
253
264
|
# 5. Convert mutable graph to a compact ReadOnlyGraph equipped with the quantized features
|
|
254
|
-
readonly_graph = graph.to_readonly(
|
|
265
|
+
readonly_graph = graph.to_readonly(
|
|
266
|
+
feature_space=int8_space, custom_features=quant_int8_data, transformations=[quantizer]
|
|
267
|
+
)
|
|
255
268
|
|
|
256
269
|
# 6. Create high-performance Searcher equipped with base vectors for exact candidate reranking
|
|
257
270
|
rerank_space = deglib.FloatSpace.create(dims, deglib.Metric.FP32_InnerProduct)
|
|
258
271
|
searcher = create_searcher(
|
|
259
272
|
graph=readonly_graph,
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
refine_data=sorted_data,
|
|
273
|
+
rerank_space=rerank_space,
|
|
274
|
+
rerank_data=sorted_data,
|
|
263
275
|
)
|
|
264
276
|
|
|
265
277
|
# 7. Query search (query is automatically quantized and top candidates are reranked)
|
|
266
|
-
indices, distances = searcher.search(query, k=10,
|
|
278
|
+
indices, distances = searcher.search(query, k=10, eps_or_ef=0.1, rerank_factor=1.5, return_distances=True)
|
|
267
279
|
print("Top-10 nearest neighbor indices:", indices)
|
|
268
280
|
print("Top-10 exact distances:", distances)
|
|
269
281
|
```
|
|
@@ -278,10 +290,21 @@ Controls the topology optimization strategy:
|
|
|
278
290
|
- `OptimizationTarget.HighLID`: Optimized for datasets with high local intrinsic dimensionality (supports multithreaded building).
|
|
279
291
|
- `OptimizationTarget.StreamingData`: Optimized for continuous dynamic additions and deletions.
|
|
280
292
|
|
|
281
|
-
### Search Parameter `
|
|
282
|
-
|
|
283
|
-
-
|
|
284
|
-
-
|
|
293
|
+
### Unified Search Parameter: `eps_or_ef`
|
|
294
|
+
`Searcher` provides a unified exploration parameter `eps_or_ef`:
|
|
295
|
+
- **Relative Distance Margin (`eps_or_ef < 1.0`)**: Standard DEG dynamic distance exploration bound $(1 + \epsilon) \cdot d_{\text{cur}}$.
|
|
296
|
+
- Small values (e.g. `eps_or_ef=0.001` or `0.01`): Faster query execution.
|
|
297
|
+
- Higher values (e.g. `eps_or_ef=0.1` to `0.3`): Higher recall rate.
|
|
298
|
+
- **Fixed Candidate Pool (`eps_or_ef >= 1.0`)**: HNSW-style search with a fixed candidate pool size of `round(eps_or_ef)` during exploration.
|
|
299
|
+
- Typical values: `64`, `96`, `128`, `200`, `400`.
|
|
300
|
+
|
|
301
|
+
```python
|
|
302
|
+
# Mode 1: Relative distance margin (default: eps_or_ef=0.1)
|
|
303
|
+
indices, distances = searcher.search(query, k=10, eps_or_ef=0.1)
|
|
304
|
+
|
|
305
|
+
# Mode 2: Fixed candidate pool (HNSW-style)
|
|
306
|
+
indices, distances = searcher.search(query, k=10, eps_or_ef=128)
|
|
307
|
+
```
|
|
285
308
|
|
|
286
309
|
### Supported Metrics & Data Types
|
|
287
310
|
- `Metric.FP32_L2`: Euclidean distance (`np.float32`)
|
|
@@ -19,7 +19,7 @@ Python bindings for the high-performance C++ Dynamic Exploration Graph (DEG) lib
|
|
|
19
19
|
- [Graph Optimization, Quantization & Reranking Pipeline](#graph-optimization-quantization--reranking-pipeline)
|
|
20
20
|
- [Concepts & Parameters](#concepts--parameters)
|
|
21
21
|
- [OptimizationTarget](#optimizationtarget)
|
|
22
|
-
- [Search Parameter `
|
|
22
|
+
- [Unified Search Parameter `eps_or_ef`](#unified-search-parameter-eps_or_ef)
|
|
23
23
|
- [Supported Metrics & Data Types](#supported-metrics--data-types)
|
|
24
24
|
- [Example Projects](#example-projects)
|
|
25
25
|
- [API Reference](#api-reference)
|
|
@@ -40,8 +40,10 @@ To build and install the package directly from the repository:
|
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
42
|
cd python/
|
|
43
|
-
|
|
44
|
-
pip install .
|
|
43
|
+
uv venv
|
|
44
|
+
uv pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0 wheel==0.48.0
|
|
45
|
+
uv run python setup.py copy_build_files
|
|
46
|
+
uv pip install . --no-build-isolation
|
|
45
47
|
```
|
|
46
48
|
|
|
47
49
|
---
|
|
@@ -61,7 +63,7 @@ data = np.random.random((num_samples, dims)).astype(np.float32)
|
|
|
61
63
|
query = np.random.random(dims).astype(np.float32)
|
|
62
64
|
|
|
63
65
|
# 2. Build index directly from data (multithreaded by default)
|
|
64
|
-
graph = deglib.builder.build_from_data(data, edges_per_vertex=32,
|
|
66
|
+
graph = deglib.builder.build_from_data(data, edges_per_vertex=32, show_progress=True)
|
|
65
67
|
|
|
66
68
|
# 3. Query top-k nearest neighbors
|
|
67
69
|
indices, distances = graph.search(query, k=10, eps=0.1)
|
|
@@ -78,7 +80,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
|
|
|
78
80
|
|
|
79
81
|
1. **Fixed-Capacity Mutable (`SizeBoundedGraph`)**: Memory is preallocated for a fixed maximum capacity. Fast and memory-efficient for static/batch datasets.
|
|
80
82
|
```python
|
|
81
|
-
space = deglib.FloatSpace.create(
|
|
83
|
+
space = deglib.FloatSpace.create(dim=128, metric=deglib.Metric.FP32_L2)
|
|
82
84
|
graph = deglib.create_empty(capacity=10_000, feature_space=space, edges_per_vertex=32)
|
|
83
85
|
```
|
|
84
86
|
|
|
@@ -97,7 +99,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
|
|
|
97
99
|
### Saving and Loading Graphs
|
|
98
100
|
|
|
99
101
|
```python
|
|
100
|
-
# Save
|
|
102
|
+
# Save graph to disk (supports both mutable and read-only graphs)
|
|
101
103
|
graph.save_graph("index.deg")
|
|
102
104
|
|
|
103
105
|
# Load as compact, optimized read-only graph for production search
|
|
@@ -156,7 +158,7 @@ from deglib.search import Filter, rerank
|
|
|
156
158
|
allowed_ids = np.array([1, 5, 10, 42, 99], dtype=np.int32)
|
|
157
159
|
search_filter = Filter(allowed_ids)
|
|
158
160
|
|
|
159
|
-
indices, distances = graph.search(query, k=5,
|
|
161
|
+
indices, distances = graph.search(query, k=5, eps_or_ef=0.1, filter_labels=search_filter)
|
|
160
162
|
|
|
161
163
|
# Exact distance reranking across candidates
|
|
162
164
|
queries = np.random.random((10, dims)).astype(np.float32)
|
|
@@ -177,11 +179,19 @@ top_indices, top_distances = rerank(
|
|
|
177
179
|
|
|
178
180
|
### Exploratory Search & Graph Navigation
|
|
179
181
|
|
|
180
|
-
DEG supports exploratory search directly from existing vertex labels:
|
|
182
|
+
DEG supports exploratory search directly from existing vertex labels (single or batch):
|
|
181
183
|
|
|
182
184
|
```python
|
|
183
185
|
# Explore graph starting from entry vertex label 105
|
|
184
|
-
explored_labels, distances = graph.explore(entry_external_label=105, k=10,
|
|
186
|
+
explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps_or_ef=0.1, include_entry=False)
|
|
187
|
+
|
|
188
|
+
# Batch exploration from multiple entry vertices in parallel
|
|
189
|
+
batch_entries = np.array([105, 210, 315], dtype=np.uint32)
|
|
190
|
+
batch_labels, batch_distances = graph.explore(batch_entries, k=10, eps_or_ef=0.1, threads=4)
|
|
191
|
+
|
|
192
|
+
# Retrieve graph neighbors directly
|
|
193
|
+
neighbors = graph.get_neighbors(105)
|
|
194
|
+
batch_neighbors = graph.get_neighbors_batch(batch_entries, threads=4) # shape: (3, edges_per_vertex)
|
|
185
195
|
```
|
|
186
196
|
|
|
187
197
|
---
|
|
@@ -193,7 +203,8 @@ A complete end-to-end pipeline demonstrating **FLAS pre-sorting**, **multithread
|
|
|
193
203
|
```python
|
|
194
204
|
import numpy as np
|
|
195
205
|
import deglib
|
|
196
|
-
from deglib.optimization import presort,
|
|
206
|
+
from deglib.optimization import presort, prune_non_mrng_edges
|
|
207
|
+
from deglib.transformation import ScalarQuantizerInt8
|
|
197
208
|
from deglib.search import create_searcher
|
|
198
209
|
|
|
199
210
|
num_vectors, dims = 10_000, 128
|
|
@@ -201,7 +212,7 @@ data = np.random.randn(num_vectors, dims).astype(np.float32)
|
|
|
201
212
|
query = np.random.randn(dims).astype(np.float32)
|
|
202
213
|
|
|
203
214
|
# 1. Pre-sort vectors using FLAS for improved memory locality and index construction speed
|
|
204
|
-
perm = presort(data, metric=deglib.Metric.FP32_InnerProduct,
|
|
215
|
+
perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, show_progress=True)
|
|
205
216
|
sorted_data = data[perm]
|
|
206
217
|
|
|
207
218
|
# 2. Build exploration graph on unquantized/original data
|
|
@@ -209,11 +220,11 @@ graph = deglib.builder.build_from_data(
|
|
|
209
220
|
sorted_data,
|
|
210
221
|
metric=deglib.Metric.FP32_InnerProduct,
|
|
211
222
|
edges_per_vertex=32,
|
|
212
|
-
|
|
223
|
+
show_progress=True,
|
|
213
224
|
)
|
|
214
225
|
|
|
215
|
-
# 3. Prune redundant non-
|
|
216
|
-
pruned_count =
|
|
226
|
+
# 3. Prune redundant non-MRNG edges to optimize graph topology
|
|
227
|
+
pruned_count = prune_non_mrng_edges(graph)
|
|
217
228
|
print(f"Pruned {pruned_count} redundant edges.")
|
|
218
229
|
|
|
219
230
|
# 4. Fit Int8 quantizer on base dataset and quantize features for memory reduction and ultra-fast search
|
|
@@ -223,19 +234,20 @@ quant_int8_data = quantizer.quantize(sorted_data)
|
|
|
223
234
|
int8_space = deglib.FloatSpace.create(dims, deglib.Metric.Int8_InnerProduct)
|
|
224
235
|
|
|
225
236
|
# 5. Convert mutable graph to a compact ReadOnlyGraph equipped with the quantized features
|
|
226
|
-
readonly_graph = graph.to_readonly(
|
|
237
|
+
readonly_graph = graph.to_readonly(
|
|
238
|
+
feature_space=int8_space, custom_features=quant_int8_data, transformations=[quantizer]
|
|
239
|
+
)
|
|
227
240
|
|
|
228
241
|
# 6. Create high-performance Searcher equipped with base vectors for exact candidate reranking
|
|
229
242
|
rerank_space = deglib.FloatSpace.create(dims, deglib.Metric.FP32_InnerProduct)
|
|
230
243
|
searcher = create_searcher(
|
|
231
244
|
graph=readonly_graph,
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
refine_data=sorted_data,
|
|
245
|
+
rerank_space=rerank_space,
|
|
246
|
+
rerank_data=sorted_data,
|
|
235
247
|
)
|
|
236
248
|
|
|
237
249
|
# 7. Query search (query is automatically quantized and top candidates are reranked)
|
|
238
|
-
indices, distances = searcher.search(query, k=10,
|
|
250
|
+
indices, distances = searcher.search(query, k=10, eps_or_ef=0.1, rerank_factor=1.5, return_distances=True)
|
|
239
251
|
print("Top-10 nearest neighbor indices:", indices)
|
|
240
252
|
print("Top-10 exact distances:", distances)
|
|
241
253
|
```
|
|
@@ -250,10 +262,21 @@ Controls the topology optimization strategy:
|
|
|
250
262
|
- `OptimizationTarget.HighLID`: Optimized for datasets with high local intrinsic dimensionality (supports multithreaded building).
|
|
251
263
|
- `OptimizationTarget.StreamingData`: Optimized for continuous dynamic additions and deletions.
|
|
252
264
|
|
|
253
|
-
### Search Parameter `
|
|
254
|
-
|
|
255
|
-
-
|
|
256
|
-
-
|
|
265
|
+
### Unified Search Parameter: `eps_or_ef`
|
|
266
|
+
`Searcher` provides a unified exploration parameter `eps_or_ef`:
|
|
267
|
+
- **Relative Distance Margin (`eps_or_ef < 1.0`)**: Standard DEG dynamic distance exploration bound $(1 + \epsilon) \cdot d_{\text{cur}}$.
|
|
268
|
+
- Small values (e.g. `eps_or_ef=0.001` or `0.01`): Faster query execution.
|
|
269
|
+
- Higher values (e.g. `eps_or_ef=0.1` to `0.3`): Higher recall rate.
|
|
270
|
+
- **Fixed Candidate Pool (`eps_or_ef >= 1.0`)**: HNSW-style search with a fixed candidate pool size of `round(eps_or_ef)` during exploration.
|
|
271
|
+
- Typical values: `64`, `96`, `128`, `200`, `400`.
|
|
272
|
+
|
|
273
|
+
```python
|
|
274
|
+
# Mode 1: Relative distance margin (default: eps_or_ef=0.1)
|
|
275
|
+
indices, distances = searcher.search(query, k=10, eps_or_ef=0.1)
|
|
276
|
+
|
|
277
|
+
# Mode 2: Fixed candidate pool (HNSW-style)
|
|
278
|
+
indices, distances = searcher.search(query, k=10, eps_or_ef=128)
|
|
279
|
+
```
|
|
257
280
|
|
|
258
281
|
### Supported Metrics & Data Types
|
|
259
282
|
- `Metric.FP32_L2`: Euclidean distance (`np.float32`)
|
|
@@ -48,7 +48,7 @@ if (FORCE_AVX2)
|
|
|
48
48
|
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
49
49
|
check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
|
|
50
50
|
if(COMPILER_SUPPORTS_AVX2)
|
|
51
|
-
target_compile_options(compile-options INTERFACE -O3 -mavx2 -mfma -fpic -w -pthread -ftree-vectorize)
|
|
51
|
+
target_compile_options(compile-options INTERFACE -O3 -mavx2 -mfma -mf16c -fpic -w -pthread -ftree-vectorize)
|
|
52
52
|
else()
|
|
53
53
|
target_compile_options(compile-options INTERFACE -O3 -fpic -w -pthread -ftree-vectorize)
|
|
54
54
|
endif()
|
|
@@ -57,7 +57,7 @@ if (FORCE_AVX2)
|
|
|
57
57
|
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
|
|
58
58
|
check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
|
|
59
59
|
if(COMPILER_SUPPORTS_AVX2)
|
|
60
|
-
target_compile_options(compile-options INTERFACE -mavx2 -mfma)
|
|
60
|
+
target_compile_options(compile-options INTERFACE -mavx2 -mfma -mf16c)
|
|
61
61
|
endif()
|
|
62
62
|
endif()
|
|
63
63
|
else()
|
|
@@ -70,12 +70,11 @@ else() # Native build
|
|
|
70
70
|
|
|
71
71
|
# setup compiler flags
|
|
72
72
|
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
73
|
+
# Portable native build: generic code is compiled for the build machine's
|
|
74
|
+
# baseline ISA only (no explicit AVX-512), so the module imports cleanly on
|
|
75
|
+
# CPUs without AVX-512. AVX-512 kernels are per-function target attributes
|
|
76
|
+
# (DEGLIB_TARGET_AVX512*), gated at runtime by deglib::cpu::select_dist().
|
|
73
77
|
target_compile_options(compile-options INTERFACE -O3 -march=native -fpic -w -pthread -ftree-vectorize)
|
|
74
|
-
if(SUPPORT_AVX512F)
|
|
75
|
-
target_compile_options(compile-options INTERFACE -mavx512f -mfma)
|
|
76
|
-
elseif(SUPPORT_AVX2)
|
|
77
|
-
target_compile_options(compile-options INTERFACE -mavx2 -mfma)
|
|
78
|
-
endif()
|
|
79
78
|
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
|
|
80
79
|
target_compile_options(compile-options INTERFACE -O2 -DNDEBUG -march=native -fpic)
|
|
81
80
|
elseif(MSVC)
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
#include "dataset.h"
|
|
9
9
|
#include "file_io.h"
|
|
10
10
|
#include "logging.h"
|
|
11
|
-
#include "stats.h"
|
|
12
11
|
#include "stopwatch.h"
|
|
13
12
|
|
|
14
13
|
#include <deglib/deglib.h>
|
|
@@ -17,15 +16,52 @@
|
|
|
17
16
|
#include <fmt/ostream.h>
|
|
18
17
|
#include <fmt/ranges.h>
|
|
19
18
|
|
|
19
|
+
#include <algorithm>
|
|
20
20
|
#include <atomic>
|
|
21
21
|
#include <filesystem>
|
|
22
22
|
#include <fstream>
|
|
23
23
|
#include <limits>
|
|
24
24
|
#include <random>
|
|
25
25
|
#include <span>
|
|
26
|
+
#include <vector>
|
|
26
27
|
|
|
27
28
|
namespace deglib::benchmark {
|
|
28
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Analyze a search graph, compute all statistics via deglib::analysis, and log them.
|
|
32
|
+
*
|
|
33
|
+
* @param graph The search graph to analyze
|
|
34
|
+
* @param base_gt Optional contiguous BaseGroundTruth buffer
|
|
35
|
+
* @return GraphStats with all computed statistics
|
|
36
|
+
*/
|
|
37
|
+
inline deglib::analysis::GraphStats analyze_graph(
|
|
38
|
+
const deglib::graph::InternalGraph& graph,
|
|
39
|
+
const BaseGroundTruth& base_gt = {}
|
|
40
|
+
) {
|
|
41
|
+
auto stats = deglib::analysis::analyze_graph(graph, base_gt.data(), base_gt.rows, base_gt.k);
|
|
42
|
+
|
|
43
|
+
log("Graph Statistics:\n");
|
|
44
|
+
log(" Vertices: {}\n", stats.vertex_count);
|
|
45
|
+
log(" Total edges: {}\n", stats.edge_count);
|
|
46
|
+
log(" Feature dimensions: {}\n", stats.feature_dims);
|
|
47
|
+
log(" Edges per vertex (k): {}\n", stats.edges_per_vertex);
|
|
48
|
+
log(" Out-degree: avg={:.2f}, min={}, max={}\n", stats.avg_out_degree, stats.min_out_degree, stats.max_out_degree);
|
|
49
|
+
log(" In-degree: avg={:.2f}, min={}, max={}, source_vertices={}\n", stats.avg_in_degree, stats.min_in_degree, stats.max_in_degree, stats.source_vertices);
|
|
50
|
+
|
|
51
|
+
if (stats.graph_quality >= 0.0f) {
|
|
52
|
+
log(" Graph Quality: {:.2f}%\n", stats.graph_quality * 100.0f);
|
|
53
|
+
}
|
|
54
|
+
if (stats.search_reachability >= 0.0f) {
|
|
55
|
+
log(" Search Reachability: {:.2f}%\n", stats.search_reachability * 100.0f);
|
|
56
|
+
}
|
|
57
|
+
if (stats.exploration_reachability >= 0.0f) {
|
|
58
|
+
log(" Exploration Reachability: {:.2f}%\n", stats.exploration_reachability * 100.0f);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
log(" Estimated memory: {:.2f} MB\n", stats.memory_bytes / (1024.0 * 1024.0));
|
|
62
|
+
return stats;
|
|
63
|
+
}
|
|
64
|
+
|
|
29
65
|
/**
|
|
30
66
|
* @brief Compute baseline time per query using linear distance computation.
|
|
31
67
|
*/
|
|
@@ -24,10 +24,10 @@ namespace deglib::benchmark {
|
|
|
24
24
|
enum DataStreamType { AddAll, AddHalf, AddAllRemoveHalf, AddHalfRemoveAndAddOneAtATime };
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
|
-
* Create a random exploration graph from the given
|
|
27
|
+
* Create a random exploration graph from the given FeatureRepository.
|
|
28
28
|
*/
|
|
29
29
|
inline deglib::graph::SizeBoundedGraph create_random_graph(
|
|
30
|
-
const deglib::
|
|
30
|
+
const deglib::FeatureRepository& repository,
|
|
31
31
|
deglib::distances::Metric metric,
|
|
32
32
|
const uint8_t k,
|
|
33
33
|
const uint32_t max_size = 0,
|
|
@@ -41,11 +41,13 @@ inline deglib::graph::SizeBoundedGraph create_random_graph(
|
|
|
41
41
|
const uint32_t vertex_count = (max_size > 0 && max_size < repository.size()) ? max_size : uint32_t(repository.size());
|
|
42
42
|
const auto feature_data = repository.getFeature(0);
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
auto graph = deglib::graph::SizeBoundedGraph(vertex_count, k, feature_space);
|
|
45
|
+
deglib::builder::populate_random_graph(graph, feature_data, vertex_count);
|
|
46
|
+
return graph;
|
|
45
47
|
}
|
|
46
48
|
|
|
47
49
|
inline void create_graph(
|
|
48
|
-
const deglib::
|
|
50
|
+
const deglib::FeatureRepository& repository,
|
|
49
51
|
const DataStreamType data_stream_type,
|
|
50
52
|
const std::string& graph_file,
|
|
51
53
|
deglib::distances::Metric metric,
|
|
@@ -163,7 +165,7 @@ inline void create_graph(
|
|
|
163
165
|
duration_ms / 1000);
|
|
164
166
|
|
|
165
167
|
graph.saveGraph(graph_file.c_str());
|
|
166
|
-
log("The graph contains {} non-
|
|
168
|
+
log("The graph contains {} non-MRNG edges\n", deglib::analysis::calc_non_mrng_edges(graph));
|
|
167
169
|
}
|
|
168
170
|
|
|
169
171
|
inline void optimize_graph(
|
|
@@ -209,8 +211,8 @@ inline void optimize_graph(
|
|
|
209
211
|
|
|
210
212
|
builder.build(improvement_callback, true);
|
|
211
213
|
|
|
212
|
-
log("Optimization complete. Final AEW: {:.2f}, non-
|
|
213
|
-
deglib::analysis::
|
|
214
|
+
log("Optimization complete. Final AEW: {:.2f}, non-MRNG edges: {}\n", deglib::analysis::calc_avg_edge_weight(graph, scale),
|
|
215
|
+
deglib::analysis::calc_non_mrng_edges(graph));
|
|
214
216
|
}
|
|
215
217
|
|
|
216
218
|
} // namespace deglib::benchmark
|