deglib 0.2.5__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. {deglib-0.2.5 → deglib-0.2.7}/CMakeLists.txt +11 -2
  2. {deglib-0.2.5/src/deglib.egg-info → deglib-0.2.7}/PKG-INFO +47 -24
  3. {deglib-0.2.5 → deglib-0.2.7}/README.md +46 -23
  4. {deglib-0.2.5 → deglib-0.2.7}/lib/CMakeLists.txt +6 -7
  5. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/benchmark.h +37 -1
  6. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/build.h +9 -7
  7. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/dataset.h +201 -32
  8. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/repository.h +51 -33
  9. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/analysis.h +140 -13
  10. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/builder.h +254 -85
  11. deglib-0.2.7/lib/deglib/include/deglib/deglib.h +128 -0
  12. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/evp_ip.h +147 -208
  13. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp16.h +143 -1
  14. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp16_ip.h +30 -2
  15. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp16_l2.h +32 -1
  16. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp32_ip.h +33 -2
  17. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp32_l2.h +32 -2
  18. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/int8_ip.h +76 -0
  19. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/uint8_ip.h +84 -0
  20. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distances.h +1 -1
  21. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/dynamic_graph.h +227 -328
  22. deglib-0.2.7/lib/deglib/include/deglib/graph/internal_graph.h +1097 -0
  23. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/mutable_graph.h +2 -0
  24. deglib-0.2.7/lib/deglib/include/deglib/graph/readonly_graph.h +290 -0
  25. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/sizebounded_graph.h +237 -410
  26. deglib-0.2.7/lib/deglib/include/deglib/graph.h +1349 -0
  27. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/flas/junker_volgenant_solver.h +4 -2
  28. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/pruning.h +48 -48
  29. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization.h +13 -140
  30. deglib-0.2.7/lib/deglib/include/deglib/search/kmeans.h +358 -0
  31. deglib-0.2.7/lib/deglib/include/deglib/search/reranker.h +603 -0
  32. deglib-0.2.7/lib/deglib/include/deglib/search/result_list.h +273 -0
  33. deglib-0.2.7/lib/deglib/include/deglib/search/searcher.h +698 -0
  34. deglib-0.2.7/lib/deglib/include/deglib/search.h +14 -0
  35. deglib-0.2.7/lib/deglib/include/deglib/serialization.h +247 -0
  36. deglib-0.2.7/lib/deglib/include/deglib/transformation/evp.h +850 -0
  37. deglib-0.2.7/lib/deglib/include/deglib/transformation/mips.h +202 -0
  38. deglib-0.2.7/lib/deglib/include/deglib/transformation/normalize.h +131 -0
  39. deglib-0.2.7/lib/deglib/include/deglib/transformation/scalar.h +1362 -0
  40. deglib-0.2.7/lib/deglib/include/deglib/transformation.h +266 -0
  41. {deglib-0.2.5 → deglib-0.2.7}/lib/test/CMakeLists.txt +17 -6
  42. {deglib-0.2.5 → deglib-0.2.7}/lib/test/src/common/test_helpers.h +6 -3
  43. deglib-0.2.7/lib/test/src/unit/graph/test_dynamic_graph_common.h +22 -0
  44. {deglib-0.2.5 → deglib-0.2.7}/setup.py +28 -24
  45. deglib-0.2.7/src/deg_cpp/deglib_cpp.cpp +2540 -0
  46. deglib-0.2.7/src/deglib/__init__.py +83 -0
  47. deglib-0.2.7/src/deglib/analysis.py +181 -0
  48. {deglib-0.2.5 → deglib-0.2.7}/src/deglib/builder.py +104 -21
  49. {deglib-0.2.5 → deglib-0.2.7}/src/deglib/distances.py +1 -0
  50. {deglib-0.2.5 → deglib-0.2.7}/src/deglib/graph.py +220 -73
  51. {deglib-0.2.5 → deglib-0.2.7}/src/deglib/optimization.py +13 -88
  52. deglib-0.2.7/src/deglib/search.py +327 -0
  53. deglib-0.2.7/src/deglib/transformation.py +96 -0
  54. {deglib-0.2.5 → deglib-0.2.7/src/deglib.egg-info}/PKG-INFO +47 -24
  55. {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/SOURCES.txt +15 -5
  56. {deglib-0.2.5 → deglib-0.2.7}/tests/test_analysis.py +70 -0
  57. {deglib-0.2.5 → deglib-0.2.7}/tests/test_builder.py +79 -9
  58. deglib-0.2.7/tests/test_evp_quantizer.py +132 -0
  59. {deglib-0.2.5 → deglib-0.2.7}/tests/test_filter.py +3 -3
  60. {deglib-0.2.5 → deglib-0.2.7}/tests/test_graph.py +211 -35
  61. deglib-0.2.7/tests/test_l2_normalize.py +58 -0
  62. {deglib-0.2.5 → deglib-0.2.7}/tests/test_rerank.py +85 -0
  63. {deglib-0.2.5 → deglib-0.2.7}/tests/test_scalar_quantizer.py +109 -16
  64. deglib-0.2.7/tests/test_searcher.py +326 -0
  65. deglib-0.2.5/tests/test_transform.py → deglib-0.2.7/tests/test_transformation.py +1 -1
  66. deglib-0.2.7/tests/test_transformation_graph.py +350 -0
  67. deglib-0.2.5/lib/bench/include/stats.h +0 -341
  68. deglib-0.2.5/lib/deglib/include/deglib/deglib.h +0 -17
  69. deglib-0.2.5/lib/deglib/include/deglib/graph/internal_graph.h +0 -528
  70. deglib-0.2.5/lib/deglib/include/deglib/graph/readonly_graph.h +0 -289
  71. deglib-0.2.5/lib/deglib/include/deglib/graph.h +0 -266
  72. deglib-0.2.5/lib/deglib/include/deglib/optimization/quantization/evp_quantize.h +0 -460
  73. deglib-0.2.5/lib/deglib/include/deglib/optimization/quantization/scalar_quantize.h +0 -714
  74. deglib-0.2.5/lib/deglib/include/deglib/optimization/transform.h +0 -119
  75. deglib-0.2.5/lib/deglib/include/deglib/search/searcher.h +0 -493
  76. deglib-0.2.5/lib/deglib/include/deglib/search.h +0 -253
  77. deglib-0.2.5/src/deg_cpp/deglib_cpp.cpp +0 -1865
  78. deglib-0.2.5/src/deglib/__init__.py +0 -46
  79. deglib-0.2.5/src/deglib/analysis.py +0 -152
  80. deglib-0.2.5/src/deglib/search.py +0 -184
  81. deglib-0.2.5/tests/test_searcher.py +0 -166
  82. {deglib-0.2.5 → deglib-0.2.7}/LICENSE +0 -0
  83. {deglib-0.2.5 → deglib-0.2.7}/MANIFEST.in +0 -0
  84. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/CMakeLists.txt +0 -0
  85. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/file_io.h +0 -0
  86. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/logging.h +0 -0
  87. {deglib-0.2.5 → deglib-0.2.7}/lib/bench/include/stopwatch.h +0 -0
  88. {deglib-0.2.5 → deglib-0.2.7}/lib/cmake_modules/DetectCPUFeatures.cmake +0 -0
  89. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/CMakeLists.txt +0 -0
  90. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/concurrent.h +0 -0
  91. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/config.h +0 -0
  92. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/fp32.h +0 -0
  93. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/int8.h +0 -0
  94. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/int8_l2.h +0 -0
  95. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/residual_mode.h +0 -0
  96. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/uint8.h +0 -0
  97. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/distance/uint8_l2.h +0 -0
  98. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/filter.h +0 -0
  99. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/intrusive_list.h +0 -0
  100. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/graph/visited_list_pool.h +0 -0
  101. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/flas/fast_linear_assignment_sorter.h +0 -0
  102. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/optimization/flas/fast_linear_assignment_sorter_mt.h +0 -0
  103. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/utils/cpu.h +0 -0
  104. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/utils/memory.h +0 -0
  105. {deglib-0.2.5 → deglib-0.2.7}/lib/deglib/include/deglib/utils/random.h +0 -0
  106. {deglib-0.2.5 → deglib-0.2.7}/pyproject.toml +0 -0
  107. {deglib-0.2.5 → deglib-0.2.7}/setup.cfg +0 -0
  108. {deglib-0.2.5 → deglib-0.2.7}/src/deglib/cpu.py +0 -0
  109. {deglib-0.2.5 → deglib-0.2.7}/src/deglib/utils.py +0 -0
  110. {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/dependency_links.txt +0 -0
  111. {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/requires.txt +0 -0
  112. {deglib-0.2.5 → deglib-0.2.7}/src/deglib.egg-info/top_level.txt +0 -0
  113. {deglib-0.2.5 → deglib-0.2.7}/tests/test_presort.py +0 -0
@@ -15,11 +15,20 @@ find_package(pybind11 CONFIG REQUIRED)
15
15
  set(ENABLE_BENCHMARKS OFF CACHE BOOL "Disable benchmarks" FORCE)
16
16
  set(ENABLE_TESTING OFF CACHE BOOL "Disable tests" FORCE)
17
17
 
18
+ # Find deglib cpp source: prefer ../cpp in repository checkouts, fallback to local lib/ (PyPI sdist)
19
+ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/CMakeLists.txt")
20
+ set(DEGLIB_CPP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp")
21
+ elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/lib/CMakeLists.txt")
22
+ set(DEGLIB_CPP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib")
23
+ else()
24
+ message(FATAL_ERROR "Could not find deglib cpp source directory (checked ../cpp/ and lib/)")
25
+ endif()
26
+
18
27
  # Add cpp-deglib
19
- add_subdirectory(${CMAKE_SOURCE_DIR}/lib/)
28
+ add_subdirectory(${DEGLIB_CPP_DIR} deglib_cpp_build)
20
29
  pybind11_add_module(deglib_cpp src/deg_cpp/deglib_cpp.cpp)
21
30
 
22
31
  # Specify the include directories
23
- include_directories(${CMAKE_SOURCE_DIR}/lib/deglib/include)
32
+ include_directories(${DEGLIB_CPP_DIR}/deglib/include)
24
33
 
25
34
  target_link_libraries(deglib_cpp PRIVATE compile-options)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: deglib
3
- Version: 0.2.5
3
+ Version: 0.2.7
4
4
  Summary: Python bindings for the Dynamic Exploration Graph library by Nico Hezel
5
5
  Author: Nico Hezel
6
6
  Author-email: Bruno Schilling <bruno.schilling@htw-berlin.de>
@@ -47,7 +47,7 @@ Python bindings for the high-performance C++ Dynamic Exploration Graph (DEG) lib
47
47
  - [Graph Optimization, Quantization & Reranking Pipeline](#graph-optimization-quantization--reranking-pipeline)
48
48
  - [Concepts & Parameters](#concepts--parameters)
49
49
  - [OptimizationTarget](#optimizationtarget)
50
- - [Search Parameter `eps`](#search-parameter-eps)
50
+ - [Unified Search Parameter `eps_or_ef`](#unified-search-parameter-eps_or_ef)
51
51
  - [Supported Metrics & Data Types](#supported-metrics--data-types)
52
52
  - [Example Projects](#example-projects)
53
53
  - [API Reference](#api-reference)
@@ -68,8 +68,10 @@ To build and install the package directly from the repository:
68
68
 
69
69
  ```bash
70
70
  cd python/
71
- python setup.py copy_build_files
72
- pip install .
71
+ uv venv
72
+ uv pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0 wheel==0.48.0
73
+ uv run python setup.py copy_build_files
74
+ uv pip install . --no-build-isolation
73
75
  ```
74
76
 
75
77
  ---
@@ -89,7 +91,7 @@ data = np.random.random((num_samples, dims)).astype(np.float32)
89
91
  query = np.random.random(dims).astype(np.float32)
90
92
 
91
93
  # 2. Build index directly from data (multithreaded by default)
92
- graph = deglib.builder.build_from_data(data, edges_per_vertex=32, callback="progress")
94
+ graph = deglib.builder.build_from_data(data, edges_per_vertex=32, show_progress=True)
93
95
 
94
96
  # 3. Query top-k nearest neighbors
95
97
  indices, distances = graph.search(query, k=10, eps=0.1)
@@ -106,7 +108,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
106
108
 
107
109
  1. **Fixed-Capacity Mutable (`SizeBoundedGraph`)**: Memory is preallocated for a fixed maximum capacity. Fast and memory-efficient for static/batch datasets.
108
110
  ```python
109
- space = deglib.FloatSpace.create(dims=128, metric=deglib.Metric.FP32_L2)
111
+ space = deglib.FloatSpace.create(dim=128, metric=deglib.Metric.FP32_L2)
110
112
  graph = deglib.create_empty(capacity=10_000, feature_space=space, edges_per_vertex=32)
111
113
  ```
112
114
 
@@ -125,7 +127,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
125
127
  ### Saving and Loading Graphs
126
128
 
127
129
  ```python
128
- # Save mutable graph to disk
130
+ # Save graph to disk (supports both mutable and read-only graphs)
129
131
  graph.save_graph("index.deg")
130
132
 
131
133
  # Load as compact, optimized read-only graph for production search
@@ -184,7 +186,7 @@ from deglib.search import Filter, rerank
184
186
  allowed_ids = np.array([1, 5, 10, 42, 99], dtype=np.int32)
185
187
  search_filter = Filter(allowed_ids)
186
188
 
187
- indices, distances = graph.search(query, k=5, eps=0.1, filter_labels=search_filter)
189
+ indices, distances = graph.search(query, k=5, eps_or_ef=0.1, filter_labels=search_filter)
188
190
 
189
191
  # Exact distance reranking across candidates
190
192
  queries = np.random.random((10, dims)).astype(np.float32)
@@ -205,11 +207,19 @@ top_indices, top_distances = rerank(
205
207
 
206
208
  ### Exploratory Search & Graph Navigation
207
209
 
208
- DEG supports exploratory search directly from existing vertex labels:
210
+ DEG supports exploratory search directly from existing vertex labels (single or batch):
209
211
 
210
212
  ```python
211
213
  # Explore graph starting from entry vertex label 105
212
- explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps=0.1, include_entry=False)
214
+ explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps_or_ef=0.1, include_entry=False)
215
+
216
+ # Batch exploration from multiple entry vertices in parallel
217
+ batch_entries = np.array([105, 210, 315], dtype=np.uint32)
218
+ batch_labels, batch_distances = graph.explore(batch_entries, k=10, eps_or_ef=0.1, threads=4)
219
+
220
+ # Retrieve graph neighbors directly
221
+ neighbors = graph.get_neighbors(105)
222
+ batch_neighbors = graph.get_neighbors_batch(batch_entries, threads=4) # shape: (3, edges_per_vertex)
213
223
  ```
214
224
 
215
225
  ---
@@ -221,7 +231,8 @@ A complete end-to-end pipeline demonstrating **FLAS pre-sorting**, **multithread
221
231
  ```python
222
232
  import numpy as np
223
233
  import deglib
224
- from deglib.optimization import presort, prune_non_rng_edges, ScalarQuantizerInt8
234
+ from deglib.optimization import presort, prune_non_mrng_edges
235
+ from deglib.transformation import ScalarQuantizerInt8
225
236
  from deglib.search import create_searcher
226
237
 
227
238
  num_vectors, dims = 10_000, 128
@@ -229,7 +240,7 @@ data = np.random.randn(num_vectors, dims).astype(np.float32)
229
240
  query = np.random.randn(dims).astype(np.float32)
230
241
 
231
242
  # 1. Pre-sort vectors using FLAS for improved memory locality and index construction speed
232
- perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, callback="progress")
243
+ perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, show_progress=True)
233
244
  sorted_data = data[perm]
234
245
 
235
246
  # 2. Build exploration graph on unquantized/original data
@@ -237,11 +248,11 @@ graph = deglib.builder.build_from_data(
237
248
  sorted_data,
238
249
  metric=deglib.Metric.FP32_InnerProduct,
239
250
  edges_per_vertex=32,
240
- callback="progress",
251
+ show_progress=True,
241
252
  )
242
253
 
243
- # 3. Prune redundant non-RNG edges to optimize graph topology
244
- pruned_count = prune_non_rng_edges(graph)
254
+ # 3. Prune redundant non-MRNG edges to optimize graph topology
255
+ pruned_count = prune_non_mrng_edges(graph)
245
256
  print(f"Pruned {pruned_count} redundant edges.")
246
257
 
247
258
  # 4. Fit Int8 quantizer on base dataset and quantize features for memory reduction and ultra-fast search
@@ -251,19 +262,20 @@ quant_int8_data = quantizer.quantize(sorted_data)
251
262
  int8_space = deglib.FloatSpace.create(dims, deglib.Metric.Int8_InnerProduct)
252
263
 
253
264
  # 5. Convert mutable graph to a compact ReadOnlyGraph equipped with the quantized features
254
- readonly_graph = graph.to_readonly(feature_space=int8_space, custom_features=quant_int8_data)
265
+ readonly_graph = graph.to_readonly(
266
+ feature_space=int8_space, custom_features=quant_int8_data, transformations=[quantizer]
267
+ )
255
268
 
256
269
  # 6. Create high-performance Searcher equipped with base vectors for exact candidate reranking
257
270
  rerank_space = deglib.FloatSpace.create(dims, deglib.Metric.FP32_InnerProduct)
258
271
  searcher = create_searcher(
259
272
  graph=readonly_graph,
260
- quantizer=quantizer,
261
- refine_space=rerank_space,
262
- refine_data=sorted_data,
273
+ rerank_space=rerank_space,
274
+ rerank_data=sorted_data,
263
275
  )
264
276
 
265
277
  # 7. Query search (query is automatically quantized and top candidates are reranked)
266
- indices, distances = searcher.search(query, k=10, eps=0.1, rerank_factor=1.5, return_distances=True)
278
+ indices, distances = searcher.search(query, k=10, eps_or_ef=0.1, rerank_factor=1.5, return_distances=True)
267
279
  print("Top-10 nearest neighbor indices:", indices)
268
280
  print("Top-10 exact distances:", distances)
269
281
  ```
@@ -278,10 +290,21 @@ Controls the topology optimization strategy:
278
290
  - `OptimizationTarget.HighLID`: Optimized for datasets with high local intrinsic dimensionality (supports multithreaded building).
279
291
  - `OptimizationTarget.StreamingData`: Optimized for continuous dynamic additions and deletions.
280
292
 
281
- ### Search Parameter `eps`
282
- - The epsilon parameter expands the search priority queue during graph exploration.
283
- - Small values (e.g. `eps=0.001` or `eps=0.01`): Faster query execution.
284
- - Higher values (e.g. `eps=0.1` to `eps=0.3`): Higher recall rate.
293
+ ### Unified Search Parameter: `eps_or_ef`
294
+ `Searcher` provides a unified exploration parameter `eps_or_ef`:
295
+ - **Relative Distance Margin (`eps_or_ef < 1.0`)**: Standard DEG dynamic distance exploration bound $(1 + \epsilon) \cdot d_{\text{cur}}$.
296
+ - Small values (e.g. `eps_or_ef=0.001` or `0.01`): Faster query execution.
297
+ - Higher values (e.g. `eps_or_ef=0.1` to `0.3`): Higher recall rate.
298
+ - **Fixed Candidate Pool (`eps_or_ef >= 1.0`)**: HNSW-style search with a fixed candidate pool size of `round(eps_or_ef)` during exploration.
299
+ - Typical values: `64`, `96`, `128`, `200`, `400`.
300
+
301
+ ```python
302
+ # Mode 1: Relative distance margin (default: eps_or_ef=0.1)
303
+ indices, distances = searcher.search(query, k=10, eps_or_ef=0.1)
304
+
305
+ # Mode 2: Fixed candidate pool (HNSW-style)
306
+ indices, distances = searcher.search(query, k=10, eps_or_ef=128)
307
+ ```
285
308
 
286
309
  ### Supported Metrics & Data Types
287
310
  - `Metric.FP32_L2`: Euclidean distance (`np.float32`)
@@ -19,7 +19,7 @@ Python bindings for the high-performance C++ Dynamic Exploration Graph (DEG) lib
19
19
  - [Graph Optimization, Quantization & Reranking Pipeline](#graph-optimization-quantization--reranking-pipeline)
20
20
  - [Concepts & Parameters](#concepts--parameters)
21
21
  - [OptimizationTarget](#optimizationtarget)
22
- - [Search Parameter `eps`](#search-parameter-eps)
22
+ - [Unified Search Parameter `eps_or_ef`](#unified-search-parameter-eps_or_ef)
23
23
  - [Supported Metrics & Data Types](#supported-metrics--data-types)
24
24
  - [Example Projects](#example-projects)
25
25
  - [API Reference](#api-reference)
@@ -40,8 +40,10 @@ To build and install the package directly from the repository:
40
40
 
41
41
  ```bash
42
42
  cd python/
43
- python setup.py copy_build_files
44
- pip install .
43
+ uv venv
44
+ uv pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0 wheel==0.48.0
45
+ uv run python setup.py copy_build_files
46
+ uv pip install . --no-build-isolation
45
47
  ```
46
48
 
47
49
  ---
@@ -61,7 +63,7 @@ data = np.random.random((num_samples, dims)).astype(np.float32)
61
63
  query = np.random.random(dims).astype(np.float32)
62
64
 
63
65
  # 2. Build index directly from data (multithreaded by default)
64
- graph = deglib.builder.build_from_data(data, edges_per_vertex=32, callback="progress")
66
+ graph = deglib.builder.build_from_data(data, edges_per_vertex=32, show_progress=True)
65
67
 
66
68
  # 3. Query top-k nearest neighbors
67
69
  indices, distances = graph.search(query, k=10, eps=0.1)
@@ -78,7 +80,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
78
80
 
79
81
  1. **Fixed-Capacity Mutable (`SizeBoundedGraph`)**: Memory is preallocated for a fixed maximum capacity. Fast and memory-efficient for static/batch datasets.
80
82
  ```python
81
- space = deglib.FloatSpace.create(dims=128, metric=deglib.Metric.FP32_L2)
83
+ space = deglib.FloatSpace.create(dim=128, metric=deglib.Metric.FP32_L2)
82
84
  graph = deglib.create_empty(capacity=10_000, feature_space=space, edges_per_vertex=32)
83
85
  ```
84
86
 
@@ -97,7 +99,7 @@ All search graphs are represented by `DynamicExplorationGraph`, backed by one of
97
99
  ### Saving and Loading Graphs
98
100
 
99
101
  ```python
100
- # Save mutable graph to disk
102
+ # Save graph to disk (supports both mutable and read-only graphs)
101
103
  graph.save_graph("index.deg")
102
104
 
103
105
  # Load as compact, optimized read-only graph for production search
@@ -156,7 +158,7 @@ from deglib.search import Filter, rerank
156
158
  allowed_ids = np.array([1, 5, 10, 42, 99], dtype=np.int32)
157
159
  search_filter = Filter(allowed_ids)
158
160
 
159
- indices, distances = graph.search(query, k=5, eps=0.1, filter_labels=search_filter)
161
+ indices, distances = graph.search(query, k=5, eps_or_ef=0.1, filter_labels=search_filter)
160
162
 
161
163
  # Exact distance reranking across candidates
162
164
  queries = np.random.random((10, dims)).astype(np.float32)
@@ -177,11 +179,19 @@ top_indices, top_distances = rerank(
177
179
 
178
180
  ### Exploratory Search & Graph Navigation
179
181
 
180
- DEG supports exploratory search directly from existing vertex labels:
182
+ DEG supports exploratory search directly from existing vertex labels (single or batch):
181
183
 
182
184
  ```python
183
185
  # Explore graph starting from entry vertex label 105
184
- explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps=0.1, include_entry=False)
186
+ explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps_or_ef=0.1, include_entry=False)
187
+
188
+ # Batch exploration from multiple entry vertices in parallel
189
+ batch_entries = np.array([105, 210, 315], dtype=np.uint32)
190
+ batch_labels, batch_distances = graph.explore(batch_entries, k=10, eps_or_ef=0.1, threads=4)
191
+
192
+ # Retrieve graph neighbors directly
193
+ neighbors = graph.get_neighbors(105)
194
+ batch_neighbors = graph.get_neighbors_batch(batch_entries, threads=4) # shape: (3, edges_per_vertex)
185
195
  ```
186
196
 
187
197
  ---
@@ -193,7 +203,8 @@ A complete end-to-end pipeline demonstrating **FLAS pre-sorting**, **multithread
193
203
  ```python
194
204
  import numpy as np
195
205
  import deglib
196
- from deglib.optimization import presort, prune_non_rng_edges, ScalarQuantizerInt8
206
+ from deglib.optimization import presort, prune_non_mrng_edges
207
+ from deglib.transformation import ScalarQuantizerInt8
197
208
  from deglib.search import create_searcher
198
209
 
199
210
  num_vectors, dims = 10_000, 128
@@ -201,7 +212,7 @@ data = np.random.randn(num_vectors, dims).astype(np.float32)
201
212
  query = np.random.randn(dims).astype(np.float32)
202
213
 
203
214
  # 1. Pre-sort vectors using FLAS for improved memory locality and index construction speed
204
- perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, callback="progress")
215
+ perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, show_progress=True)
205
216
  sorted_data = data[perm]
206
217
 
207
218
  # 2. Build exploration graph on unquantized/original data
@@ -209,11 +220,11 @@ graph = deglib.builder.build_from_data(
209
220
  sorted_data,
210
221
  metric=deglib.Metric.FP32_InnerProduct,
211
222
  edges_per_vertex=32,
212
- callback="progress",
223
+ show_progress=True,
213
224
  )
214
225
 
215
- # 3. Prune redundant non-RNG edges to optimize graph topology
216
- pruned_count = prune_non_rng_edges(graph)
226
+ # 3. Prune redundant non-MRNG edges to optimize graph topology
227
+ pruned_count = prune_non_mrng_edges(graph)
217
228
  print(f"Pruned {pruned_count} redundant edges.")
218
229
 
219
230
  # 4. Fit Int8 quantizer on base dataset and quantize features for memory reduction and ultra-fast search
@@ -223,19 +234,20 @@ quant_int8_data = quantizer.quantize(sorted_data)
223
234
  int8_space = deglib.FloatSpace.create(dims, deglib.Metric.Int8_InnerProduct)
224
235
 
225
236
  # 5. Convert mutable graph to a compact ReadOnlyGraph equipped with the quantized features
226
- readonly_graph = graph.to_readonly(feature_space=int8_space, custom_features=quant_int8_data)
237
+ readonly_graph = graph.to_readonly(
238
+ feature_space=int8_space, custom_features=quant_int8_data, transformations=[quantizer]
239
+ )
227
240
 
228
241
  # 6. Create high-performance Searcher equipped with base vectors for exact candidate reranking
229
242
  rerank_space = deglib.FloatSpace.create(dims, deglib.Metric.FP32_InnerProduct)
230
243
  searcher = create_searcher(
231
244
  graph=readonly_graph,
232
- quantizer=quantizer,
233
- refine_space=rerank_space,
234
- refine_data=sorted_data,
245
+ rerank_space=rerank_space,
246
+ rerank_data=sorted_data,
235
247
  )
236
248
 
237
249
  # 7. Query search (query is automatically quantized and top candidates are reranked)
238
- indices, distances = searcher.search(query, k=10, eps=0.1, rerank_factor=1.5, return_distances=True)
250
+ indices, distances = searcher.search(query, k=10, eps_or_ef=0.1, rerank_factor=1.5, return_distances=True)
239
251
  print("Top-10 nearest neighbor indices:", indices)
240
252
  print("Top-10 exact distances:", distances)
241
253
  ```
@@ -250,10 +262,21 @@ Controls the topology optimization strategy:
250
262
  - `OptimizationTarget.HighLID`: Optimized for datasets with high local intrinsic dimensionality (supports multithreaded building).
251
263
  - `OptimizationTarget.StreamingData`: Optimized for continuous dynamic additions and deletions.
252
264
 
253
- ### Search Parameter `eps`
254
- - The epsilon parameter expands the search priority queue during graph exploration.
255
- - Small values (e.g. `eps=0.001` or `eps=0.01`): Faster query execution.
256
- - Higher values (e.g. `eps=0.1` to `eps=0.3`): Higher recall rate.
265
+ ### Unified Search Parameter: `eps_or_ef`
266
+ `Searcher` provides a unified exploration parameter `eps_or_ef`:
267
+ - **Relative Distance Margin (`eps_or_ef < 1.0`)**: Standard DEG dynamic distance exploration bound $(1 + \epsilon) \cdot d_{\text{cur}}$.
268
+ - Small values (e.g. `eps_or_ef=0.001` or `0.01`): Faster query execution.
269
+ - Higher values (e.g. `eps_or_ef=0.1` to `0.3`): Higher recall rate.
270
+ - **Fixed Candidate Pool (`eps_or_ef >= 1.0`)**: HNSW-style search with a fixed candidate pool size of `round(eps_or_ef)` during exploration.
271
+ - Typical values: `64`, `96`, `128`, `200`, `400`.
272
+
273
+ ```python
274
+ # Mode 1: Relative distance margin (default: eps_or_ef=0.1)
275
+ indices, distances = searcher.search(query, k=10, eps_or_ef=0.1)
276
+
277
+ # Mode 2: Fixed candidate pool (HNSW-style)
278
+ indices, distances = searcher.search(query, k=10, eps_or_ef=128)
279
+ ```
257
280
 
258
281
  ### Supported Metrics & Data Types
259
282
  - `Metric.FP32_L2`: Euclidean distance (`np.float32`)
@@ -48,7 +48,7 @@ if (FORCE_AVX2)
48
48
  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
49
49
  check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
50
50
  if(COMPILER_SUPPORTS_AVX2)
51
- target_compile_options(compile-options INTERFACE -O3 -mavx2 -mfma -fpic -w -pthread -ftree-vectorize)
51
+ target_compile_options(compile-options INTERFACE -O3 -mavx2 -mfma -mf16c -fpic -w -pthread -ftree-vectorize)
52
52
  else()
53
53
  target_compile_options(compile-options INTERFACE -O3 -fpic -w -pthread -ftree-vectorize)
54
54
  endif()
@@ -57,7 +57,7 @@ if (FORCE_AVX2)
57
57
  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
58
58
  check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
59
59
  if(COMPILER_SUPPORTS_AVX2)
60
- target_compile_options(compile-options INTERFACE -mavx2 -mfma)
60
+ target_compile_options(compile-options INTERFACE -mavx2 -mfma -mf16c)
61
61
  endif()
62
62
  endif()
63
63
  else()
@@ -70,12 +70,11 @@ else() # Native build
70
70
 
71
71
  # setup compiler flags
72
72
  if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
73
+ # Portable native build: generic code is compiled for the build machine's
74
+ # baseline ISA only (no explicit AVX-512), so the module imports cleanly on
75
+ # CPUs without AVX-512. AVX-512 kernels are per-function target attributes
76
+ # (DEGLIB_TARGET_AVX512*), gated at runtime by deglib::cpu::select_dist().
73
77
  target_compile_options(compile-options INTERFACE -O3 -march=native -fpic -w -pthread -ftree-vectorize)
74
- if(SUPPORT_AVX512F)
75
- target_compile_options(compile-options INTERFACE -mavx512f -mfma)
76
- elseif(SUPPORT_AVX2)
77
- target_compile_options(compile-options INTERFACE -mavx2 -mfma)
78
- endif()
79
78
  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
80
79
  target_compile_options(compile-options INTERFACE -O2 -DNDEBUG -march=native -fpic)
81
80
  elseif(MSVC)
@@ -8,7 +8,6 @@
8
8
  #include "dataset.h"
9
9
  #include "file_io.h"
10
10
  #include "logging.h"
11
- #include "stats.h"
12
11
  #include "stopwatch.h"
13
12
 
14
13
  #include <deglib/deglib.h>
@@ -17,15 +16,52 @@
17
16
  #include <fmt/ostream.h>
18
17
  #include <fmt/ranges.h>
19
18
 
19
+ #include <algorithm>
20
20
  #include <atomic>
21
21
  #include <filesystem>
22
22
  #include <fstream>
23
23
  #include <limits>
24
24
  #include <random>
25
25
  #include <span>
26
+ #include <vector>
26
27
 
27
28
  namespace deglib::benchmark {
28
29
 
30
+ /**
31
+ * Analyze a search graph, compute all statistics via deglib::analysis, and log them.
32
+ *
33
+ * @param graph The search graph to analyze
34
+ * @param base_gt Optional contiguous BaseGroundTruth buffer
35
+ * @return GraphStats with all computed statistics
36
+ */
37
+ inline deglib::analysis::GraphStats analyze_graph(
38
+ const deglib::graph::InternalGraph& graph,
39
+ const BaseGroundTruth& base_gt = {}
40
+ ) {
41
+ auto stats = deglib::analysis::analyze_graph(graph, base_gt.data(), base_gt.rows, base_gt.k);
42
+
43
+ log("Graph Statistics:\n");
44
+ log(" Vertices: {}\n", stats.vertex_count);
45
+ log(" Total edges: {}\n", stats.edge_count);
46
+ log(" Feature dimensions: {}\n", stats.feature_dims);
47
+ log(" Edges per vertex (k): {}\n", stats.edges_per_vertex);
48
+ log(" Out-degree: avg={:.2f}, min={}, max={}\n", stats.avg_out_degree, stats.min_out_degree, stats.max_out_degree);
49
+ log(" In-degree: avg={:.2f}, min={}, max={}, source_vertices={}\n", stats.avg_in_degree, stats.min_in_degree, stats.max_in_degree, stats.source_vertices);
50
+
51
+ if (stats.graph_quality >= 0.0f) {
52
+ log(" Graph Quality: {:.2f}%\n", stats.graph_quality * 100.0f);
53
+ }
54
+ if (stats.search_reachability >= 0.0f) {
55
+ log(" Search Reachability: {:.2f}%\n", stats.search_reachability * 100.0f);
56
+ }
57
+ if (stats.exploration_reachability >= 0.0f) {
58
+ log(" Exploration Reachability: {:.2f}%\n", stats.exploration_reachability * 100.0f);
59
+ }
60
+
61
+ log(" Estimated memory: {:.2f} MB\n", stats.memory_bytes / (1024.0 * 1024.0));
62
+ return stats;
63
+ }
64
+
29
65
  /**
30
66
  * @brief Compute baseline time per query using linear distance computation.
31
67
  */
@@ -24,10 +24,10 @@ namespace deglib::benchmark {
24
24
  enum DataStreamType { AddAll, AddHalf, AddAllRemoveHalf, AddHalfRemoveAndAddOneAtATime };
25
25
 
26
26
  /**
27
- * Create a random exploration graph from the given StaticFeatureRepository.
27
+ * Create a random exploration graph from the given FeatureRepository.
28
28
  */
29
29
  inline deglib::graph::SizeBoundedGraph create_random_graph(
30
- const deglib::StaticFeatureRepository& repository,
30
+ const deglib::FeatureRepository& repository,
31
31
  deglib::distances::Metric metric,
32
32
  const uint8_t k,
33
33
  const uint32_t max_size = 0,
@@ -41,11 +41,13 @@ inline deglib::graph::SizeBoundedGraph create_random_graph(
41
41
  const uint32_t vertex_count = (max_size > 0 && max_size < repository.size()) ? max_size : uint32_t(repository.size());
42
42
  const auto feature_data = repository.getFeature(0);
43
43
 
44
- return deglib::graph::SizeBoundedGraph::create_random_graph(feature_data, vertex_count, k, feature_space);
44
+ auto graph = deglib::graph::SizeBoundedGraph(vertex_count, k, feature_space);
45
+ deglib::builder::populate_random_graph(graph, feature_data, vertex_count);
46
+ return graph;
45
47
  }
46
48
 
47
49
  inline void create_graph(
48
- const deglib::StaticFeatureRepository& repository,
50
+ const deglib::FeatureRepository& repository,
49
51
  const DataStreamType data_stream_type,
50
52
  const std::string& graph_file,
51
53
  deglib::distances::Metric metric,
@@ -163,7 +165,7 @@ inline void create_graph(
163
165
  duration_ms / 1000);
164
166
 
165
167
  graph.saveGraph(graph_file.c_str());
166
- log("The graph contains {} non-RNG edges\n", deglib::analysis::calc_non_rng_edges(graph));
168
+ log("The graph contains {} non-MRNG edges\n", deglib::analysis::calc_non_mrng_edges(graph));
167
169
  }
168
170
 
169
171
  inline void optimize_graph(
@@ -209,8 +211,8 @@ inline void optimize_graph(
209
211
 
210
212
  builder.build(improvement_callback, true);
211
213
 
212
- log("Optimization complete. Final AEW: {:.2f}, non-RNG edges: {}\n", deglib::analysis::calc_avg_edge_weight(graph, scale),
213
- deglib::analysis::calc_non_rng_edges(graph));
214
+ log("Optimization complete. Final AEW: {:.2f}, non-MRNG edges: {}\n", deglib::analysis::calc_avg_edge_weight(graph, scale),
215
+ deglib::analysis::calc_non_mrng_edges(graph));
214
216
  }
215
217
 
216
218
  } // namespace deglib::benchmark