deglib 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. deglib-0.2.2/PKG-INFO +345 -0
  2. deglib-0.2.2/README.md +317 -0
  3. {deglib-0.2.0 → deglib-0.2.2}/lib/CMakeLists.txt +3 -3
  4. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/config.h +6 -9
  5. deglib-0.2.0/lib/deglib/include/deglib/distance/evp_inner_product.h → deglib-0.2.2/lib/deglib/include/deglib/distance/evp_ip.h +13 -19
  6. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/fp16_ip.h +104 -122
  7. deglib-0.2.2/lib/deglib/include/deglib/distance/fp16_l2.h +413 -0
  8. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/fp32_ip.h +92 -99
  9. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/fp32_l2.h +94 -105
  10. deglib-0.2.2/lib/deglib/include/deglib/distance/int8.h +10 -0
  11. deglib-0.2.2/lib/deglib/include/deglib/distance/int8_ip.h +735 -0
  12. deglib-0.2.2/lib/deglib/include/deglib/distance/int8_l2.h +412 -0
  13. deglib-0.2.2/lib/deglib/include/deglib/distance/uint8_ip.h +687 -0
  14. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/uint8_l2.h +118 -108
  15. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distances.h +36 -8
  16. deglib-0.2.2/lib/deglib/include/deglib/optimization/quantization/scalar_quantize.h +743 -0
  17. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization.h +93 -0
  18. deglib-0.2.2/lib/deglib/include/deglib/utils/cpu.h +139 -0
  19. {deglib-0.2.0 → deglib-0.2.2}/lib/test/CMakeLists.txt +7 -0
  20. {deglib-0.2.0 → deglib-0.2.2}/lib/test/src/common/test_helpers.h +160 -57
  21. {deglib-0.2.0 → deglib-0.2.2}/pyproject.toml +3 -1
  22. {deglib-0.2.0 → deglib-0.2.2}/src/deg_cpp/deglib_cpp.cpp +414 -6
  23. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/__init__.py +1 -1
  24. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/cpu.py +23 -1
  25. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/distances.py +12 -3
  26. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/optimization.py +35 -15
  27. deglib-0.2.2/src/deglib.egg-info/PKG-INFO +345 -0
  28. {deglib-0.2.0 → deglib-0.2.2}/src/deglib.egg-info/SOURCES.txt +8 -1
  29. {deglib-0.2.0 → deglib-0.2.2}/tests/test_graph.py +63 -15
  30. {deglib-0.2.0 → deglib-0.2.2}/tests/test_presort.py +30 -4
  31. deglib-0.2.2/tests/test_scalar_quantizer.py +222 -0
  32. deglib-0.2.0/PKG-INFO +0 -265
  33. deglib-0.2.0/README.md +0 -240
  34. deglib-0.2.0/lib/deglib/include/deglib/utils/cpu.h +0 -69
  35. deglib-0.2.0/src/deglib.egg-info/PKG-INFO +0 -265
  36. {deglib-0.2.0 → deglib-0.2.2}/CMakeLists.txt +0 -0
  37. {deglib-0.2.0 → deglib-0.2.2}/LICENSE +0 -0
  38. {deglib-0.2.0 → deglib-0.2.2}/MANIFEST.in +0 -0
  39. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/CMakeLists.txt +0 -0
  40. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/benchmark.h +0 -0
  41. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/build.h +0 -0
  42. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/dataset.h +0 -0
  43. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/file_io.h +0 -0
  44. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/logging.h +0 -0
  45. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/repository.h +0 -0
  46. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/stats.h +0 -0
  47. {deglib-0.2.0 → deglib-0.2.2}/lib/bench/include/stopwatch.h +0 -0
  48. {deglib-0.2.0 → deglib-0.2.2}/lib/cmake_modules/DetectCPUFeatures.cmake +0 -0
  49. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/CMakeLists.txt +0 -0
  50. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/analysis.h +0 -0
  51. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/builder.h +0 -0
  52. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/concurrent.h +0 -0
  53. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/deglib.h +0 -0
  54. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/fp16.h +0 -0
  55. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/fp32.h +0 -0
  56. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/residual_mode.h +0 -0
  57. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/distance/uint8.h +0 -0
  58. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/filter.h +0 -0
  59. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/dynamic_graph.h +0 -0
  60. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/internal_graph.h +0 -0
  61. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/intrusive_list.h +0 -0
  62. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/mutable_graph.h +0 -0
  63. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/readonly_graph.h +0 -0
  64. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/sizebounded_graph.h +0 -0
  65. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph/visited_list_pool.h +0 -0
  66. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/graph.h +0 -0
  67. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization/flas/fast_linear_assignment_sorter.h +0 -0
  68. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization/flas/fast_linear_assignment_sorter_mt.h +0 -0
  69. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization/flas/junker_volgenant_solver.h +0 -0
  70. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization/pruning.h +0 -0
  71. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization/quantization/evp_quantize.h +0 -0
  72. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/optimization/transform.h +0 -0
  73. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/search.h +0 -0
  74. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/utils/memory.h +0 -0
  75. {deglib-0.2.0 → deglib-0.2.2}/lib/deglib/include/deglib/utils/random.h +0 -0
  76. {deglib-0.2.0 → deglib-0.2.2}/setup.cfg +0 -0
  77. {deglib-0.2.0 → deglib-0.2.2}/setup.py +0 -0
  78. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/analysis.py +0 -0
  79. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/builder.py +0 -0
  80. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/graph.py +0 -0
  81. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/search.py +0 -0
  82. {deglib-0.2.0 → deglib-0.2.2}/src/deglib/utils.py +0 -0
  83. {deglib-0.2.0 → deglib-0.2.2}/src/deglib.egg-info/dependency_links.txt +0 -0
  84. {deglib-0.2.0 → deglib-0.2.2}/src/deglib.egg-info/requires.txt +0 -0
  85. {deglib-0.2.0 → deglib-0.2.2}/src/deglib.egg-info/top_level.txt +0 -0
  86. {deglib-0.2.0 → deglib-0.2.2}/tests/test_analysis.py +0 -0
  87. {deglib-0.2.0 → deglib-0.2.2}/tests/test_builder.py +0 -0
  88. {deglib-0.2.0 → deglib-0.2.2}/tests/test_filter.py +0 -0
  89. {deglib-0.2.0 → deglib-0.2.2}/tests/test_rerank.py +0 -0
  90. {deglib-0.2.0 → deglib-0.2.2}/tests/test_transform.py +0 -0
deglib-0.2.2/PKG-INFO ADDED
@@ -0,0 +1,345 @@
1
+ Metadata-Version: 2.4
2
+ Name: deglib
3
+ Version: 0.2.2
4
+ Summary: Python bindings for the Dynamic Exploration Graph library by Nico Hezel
5
+ Author: Nico Hezel
6
+ Author-email: Bruno Schilling <bruno.schilling@htw-berlin.de>
7
+ Project-URL: Homepage, https://github.com/Visual-Computing/DynamicExplorationGraph
8
+ Project-URL: Documentation, https://dynamic-exploration-graph.readthedocs.io/
9
+ Project-URL: Repository, https://github.com/Visual-Computing/DynamicExplorationGraph
10
+ Project-URL: Changelog, https://github.com/Visual-Computing/DynamicExplorationGraph/blob/main/RELEASE_NOTES.md
11
+ Keywords: anns-search,graph,python
12
+ Classifier: Development Status :: 2 - Pre-Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Requires-Python: >=3.10
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: numpy>=2.2
24
+ Requires-Dist: psutil
25
+ Provides-Extra: test
26
+ Requires-Dist: pytest>=8.0; extra == "test"
27
+ Dynamic: license-file
28
+
29
+ # deglib: Python Bindings for the Dynamic Exploration Graph
30
+
31
+ Python bindings for the high-performance C++ Dynamic Exploration Graph (DEG) library, enabling approximate nearest neighbor search (ANNS) and graph exploration with state-of-the-art recall vs. QPS trade-offs.
32
+
33
+ ---
34
+
35
+ ## Table of Contents
36
+
37
+ - [Installation](#installation)
38
+ - [From PyPI](#from-pypi)
39
+ - [Build from Source](#build-from-source)
40
+ - [Quickstart & Examples](#quickstart--examples)
41
+ - [Basic Usage](#basic-usage)
42
+ - [Graph Types & Lifecycles](#graph-types--lifecycles)
43
+ - [Saving and Loading Graphs](#saving-and-loading-graphs)
44
+ - [Incremental / Streaming Graph Construction](#incremental--streaming-graph-construction)
45
+ - [Filtered Search & Candidate Reranking](#filtered-search--candidate-reranking)
46
+ - [Exploratory Search & Graph Navigation](#exploratory-search--graph-navigation)
47
+ - [Graph Optimization, Quantization & Reranking Pipeline](#graph-optimization-quantization--reranking-pipeline)
48
+ - [Concepts & Parameters](#concepts--parameters)
49
+ - [OptimizationTarget](#optimizationtarget)
50
+ - [Search Parameter `eps`](#search-parameter-eps)
51
+ - [Supported Metrics & Data Types](#supported-metrics--data-types)
52
+ - [Example Projects](#example-projects)
53
+ - [API Reference](#api-reference)
54
+ - [Development Setup](#development-setup)
55
+ ---
56
+
57
+ ## Installation
58
+
59
+ ### From PyPI
60
+
61
+ ```bash
62
+ pip install deglib
63
+ ```
64
+
65
+ ### Build from Source
66
+
67
+ To build and install the package directly from the repository:
68
+
69
+ ```bash
70
+ cd python/
71
+ python setup.py copy_build_files
72
+ pip install .
73
+ ```
74
+
75
+ ---
76
+
77
+ ## Quickstart & Examples
78
+
79
+ ### Basic Usage
80
+
81
+ ```python
82
+ import numpy as np
83
+ import deglib
84
+
85
+ num_samples, dims = 10_000, 128
86
+
87
+ # 1. Create random feature dataset and query vector
88
+ data = np.random.random((num_samples, dims)).astype(np.float32)
89
+ query = np.random.random(dims).astype(np.float32)
90
+
91
+ # 2. Build index directly from data (multithreaded by default)
92
+ graph = deglib.builder.build_from_data(data, edges_per_vertex=32, callback="progress")
93
+
94
+ # 3. Query top-k nearest neighbors
95
+ indices, distances = graph.search(query, k=10, eps=0.1)
96
+
97
+ print("Nearest neighbors:", indices)
98
+ print("Distances:", distances)
99
+ ```
100
+
101
+ ---
102
+
103
+ ### Graph Types & Lifecycles
104
+
105
+ All search graphs are represented by `DynamicExplorationGraph`, backed by one of three internal graph engines:
106
+
107
+ 1. **Fixed-Capacity Mutable (`SizeBoundedGraph`)**: Memory is preallocated for a fixed maximum capacity. Fast and memory-efficient for static/batch datasets.
108
+ ```python
109
+ space = deglib.FloatSpace.create(dims=128, metric=deglib.Metric.FP32_L2)
110
+ graph = deglib.create_empty(capacity=10_000, feature_space=space, edges_per_vertex=32)
111
+ ```
112
+
113
+ 2. **Chunk-Allocated Mutable (`DynamicGraph`)**: Dynamically allocates memory in chunks (e.g. 1024 vertices per chunk). Ideal for streaming datasets where total capacity is unknown.
114
+ ```python
115
+ graph = deglib.create_dynamic_empty(feature_space=space, edges_per_vertex=32, chunk_size=1024)
116
+ ```
117
+
118
+ 3. **Read-Only Deployment (`ReadOnlyGraph`)**: Stripped of mutation structures for minimal memory footprint and maximum query throughput.
119
+ ```python
120
+ readonly_graph = graph.to_readonly()
121
+ ```
122
+
123
+ ---
124
+
125
+ ### Saving and Loading Graphs
126
+
127
+ ```python
128
+ # Save mutable graph to disk
129
+ graph.save_graph("index.deg")
130
+
131
+ # Load as compact, optimized read-only graph for production search
132
+ readonly_graph = deglib.load_readonly_graph("index.deg")
133
+
134
+ # Load as dynamic graph supporting further insertions and deletions
135
+ dynamic_graph = deglib.load_dynamic_graph("index.deg")
136
+
137
+ # Load as fixed-capacity mutable graph
138
+ mutable_graph = deglib.load_mutable_graph("index.deg")
139
+ ```
140
+
141
+ ---
142
+
143
+ ### Incremental / Streaming Graph Construction
144
+
145
+ For continuous additions and removals, use `GraphBuilder`:
146
+
147
+ ```python
148
+ import numpy as np
149
+ import deglib
150
+ from deglib.builder import GraphBuilder, OptimizationTarget
151
+ from deglib.distances import FloatSpace, Metric
152
+
153
+ dims = 128
154
+ max_capacity = 10_000
155
+ edges_per_vertex = 32
156
+
157
+ # 1. Create feature space and empty mutable graph
158
+ space = FloatSpace.create(dims, metric=Metric.FP32_L2)
159
+ graph = deglib.create_empty(max_capacity, space, edges_per_vertex)
160
+
161
+ # 2. Initialize builder
162
+ builder = GraphBuilder(graph, optimization_target=OptimizationTarget.StreamingData, seed=42)
163
+
164
+ # 3. Add entries (individually or in batches)
165
+ labels = np.arange(100, dtype=np.uint32)
166
+ features = np.random.random((100, dims)).astype(np.float32)
167
+ builder.add_entry(labels, features)
168
+
169
+ # 4. Remove entries by external label
170
+ builder.remove_entry(42)
171
+
172
+ # 5. Build / optimize
173
+ builder.build()
174
+ ```
175
+
176
+ ---
177
+
178
+ ### Filtered Search & Candidate Reranking
179
+
180
+ ```python
181
+ from deglib.search import Filter, rerank
182
+
183
+ # Search only within allowed external labels
184
+ allowed_ids = np.array([1, 5, 10, 42, 99], dtype=np.int32)
185
+ search_filter = Filter(allowed_ids)
186
+
187
+ indices, distances = graph.search(query, k=5, eps=0.1, filter_labels=search_filter)
188
+
189
+ # Exact distance reranking across candidates
190
+ queries = np.random.random((10, dims)).astype(np.float32)
191
+ candidates = np.random.randint(0, 1000, size=(10, 50), dtype=np.uint32)
192
+ base_vectors = np.random.random((1000, dims)).astype(np.float32)
193
+
194
+ top_indices, top_distances = rerank(
195
+ space=graph.get_feature_space(),
196
+ queries=queries,
197
+ candidate_indices=candidates,
198
+ base_vectors=base_vectors,
199
+ k_top=10,
200
+ return_distances=True,
201
+ )
202
+ ```
203
+
204
+ ---
205
+
206
+ ### Exploratory Search & Graph Navigation
207
+
208
+ DEG supports exploratory search directly from existing vertex labels:
209
+
210
+ ```python
211
+ # Explore graph starting from entry vertex label 105
212
+ explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps=0.1, include_entry=False)
213
+ ```
214
+
215
+ ---
216
+
217
+ ### Graph Optimization, Quantization & Reranking Pipeline
218
+
219
+ A complete end-to-end pipeline demonstrating **FLAS pre-sorting**, **multithreaded graph building**, **RNG edge pruning**, **Int8 quantization**, **ReadOnlyGraph conversion**, and **exact FP32 candidate reranking**:
220
+
221
+ ```python
222
+ import numpy as np
223
+ import deglib
224
+ from deglib.optimization import presort, prune_non_rng_edges, ScalarQuantizerInt8
225
+ from deglib.search import rerank
226
+
227
+ num_vectors, dims = 10_000, 128
228
+ data = np.random.randn(num_vectors, dims).astype(np.float32)
229
+ query = np.random.randn(1, dims).astype(np.float32)
230
+
231
+ # 1. Pre-sort vectors using FLAS for improved memory locality and index construction speed
232
+ perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, callback="progress")
233
+ sorted_data = data[perm]
234
+
235
+ # 2. Build exploration graph on unquantized/original data
236
+ graph = deglib.builder.build_from_data(
237
+ sorted_data,
238
+ metric=deglib.Metric.FP32_InnerProduct,
239
+ edges_per_vertex=32,
240
+ callback="progress",
241
+ )
242
+
243
+ # 3. Prune redundant non-RNG edges to optimize graph topology
244
+ pruned_count = prune_non_rng_edges(graph)
245
+ print(f"Pruned {pruned_count} redundant edges.")
246
+
247
+ # 4. Fit Int8 quantizer on base dataset and quantize features for memory reduction and ultra-fast search
248
+ quantizer = ScalarQuantizerInt8()
249
+ quantizer.fit(sorted_data)
250
+ quant_int8_data = quantizer.quantize(sorted_data)
251
+ int8_space = deglib.FloatSpace.create(dims, deglib.Metric.Int8_InnerProduct)
252
+
253
+ # 5. Convert mutable graph to a compact ReadOnlyGraph equipped with the quantized features
254
+ readonly_graph = graph.to_readonly(feature_space=int8_space, custom_features=quant_int8_data)
255
+
256
+ # 6. Quantize query using the EXACT SAME calibrated scale, then search
257
+ quant_query = quantizer.quantize(query)
258
+ candidate_indices = readonly_graph.search(quant_query, k=20, eps=0.1, return_distances=False)
259
+
260
+ # 7. Exact distance reranking of top candidates on original float32 data
261
+ fp32_space = deglib.FloatSpace.create(dims, deglib.Metric.FP32_InnerProduct)
262
+ final_top_indices, distances = rerank(
263
+ space=fp32_space,
264
+ queries=query,
265
+ candidate_indices=candidate_indices,
266
+ base_vectors=sorted_data,
267
+ k_top=10,
268
+ return_distances=True,
269
+ )
270
+
271
+ print("Top-10 nearest neighbor indices:", final_top_indices[0])
272
+ print("Top-10 exact distances:", distances[0])
273
+ ```
274
+
275
+ ---
276
+
277
+ ## Concepts & Parameters
278
+
279
+ ### `OptimizationTarget`
280
+ Controls the topology optimization strategy:
281
+ - `OptimizationTarget.LowLID`: Default for datasets with low local intrinsic dimensionality (supports multithreaded building).
282
+ - `OptimizationTarget.HighLID`: Optimized for datasets with high local intrinsic dimensionality (supports multithreaded building).
283
+ - `OptimizationTarget.StreamingData`: Optimized for continuous dynamic additions and deletions.
284
+
285
+ ### Search Parameter `eps`
286
+ - The epsilon parameter expands the search priority queue during graph exploration.
287
+ - Small values (e.g. `eps=0.001` or `eps=0.01`): Faster query execution.
288
+ - Higher values (e.g. `eps=0.1` to `eps=0.3`): Higher recall rate.
289
+
290
+ ### Supported Metrics & Data Types
291
+ - `Metric.FP32_L2`: Euclidean distance (`np.float32`)
292
+ - `Metric.FP32_InnerProduct`: Inner product / cosine distance (`np.float32`)
293
+ - `Metric.Uint8_L2`: 8-bit unsigned integer Euclidean distance (`np.uint8`)
294
+ - `Metric.Uint8_InnerProduct`: 8-bit unsigned integer inner product (`np.uint8`)
295
+ - `Metric.FP16_L2`: 16-bit half-precision Euclidean distance (`np.uint16`)
296
+ - `Metric.FP16_InnerProduct`: 16-bit half-precision inner product (`np.uint16`)
297
+ - `Metric.EVP_InnerProduct`: Quantized Extreme Value Property bit-packed vectors (`np.uint8`)
298
+ - `Metric.Int8_InnerProduct`: 8-bit signed integer inner product (`np.int8`)
299
+ - `Metric.Int8_L2`: 8-bit signed integer Euclidean distance (`np.int8`)
300
+
301
+ ---
302
+
303
+ ## API Reference
304
+
305
+ For a complete overview of all Python modules, classes, and function signatures, see [API.md](API.md) or the [Official Documentation](https://dynamic-exploration-graph.readthedocs.io/).
306
+
307
+ ---
308
+
309
+ ## Example Projects
310
+
311
+ Ready-to-run example scripts with visual progress and evaluation are located in the [examples/](../examples/) directory:
312
+
313
+ - [examples/knng/](../examples/knng/): k-NN graph construction benchmark using EVP quantization and FP16 reranking (SISAP 2026 Challenge Task 1).
314
+ - [examples/mips/](../examples/mips/): Maximum Inner Product Search (MIPS) benchmark using $(d+1)$-dimensional $L_2$ transformation, FLAS pre-sorting, and SIMD FP16 inner products (SISAP 2026 Challenge Task 2).
315
+ - [examples/static_data/](../examples/static_data/): Static dataset indexing and ANNS benchmark.
316
+ - [examples/vibe/](../examples/vibe/): VIBE benchmark for modern embedding datasets with scalar quantization and interactive Plotly visualizations.
317
+ - [examples/dynamic_data/](../examples/dynamic_data/): Dynamic streaming additions and deletions.
318
+ - [examples/sliding_window/](../examples/sliding_window/): Sliding window continuous update benchmark against CleANN.
319
+
320
+
321
+ ---
322
+
323
+ ## Development Setup
324
+
325
+ If you want to contribute, modify C++ bindings, run tests, or build release packages:
326
+
327
+ ```bash
328
+ # 1. Setup virtual environment and dependencies (using uv)
329
+ cd python/
330
+ uv venv
331
+ uv pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0 wheel==0.48.0
332
+ uv run python setup.py copy_build_files
333
+
334
+ # 2. Install in editable mode
335
+ uv pip install -e . --no-build-isolation --verbose
336
+
337
+ # 3. Run test suite
338
+ uv run pytest
339
+
340
+ # 4. Format code
341
+ uv run ruff format .
342
+
343
+ # 5. Build distribution packages (sdist and binary wheels)
344
+ uv run python -m build
345
+ ```
deglib-0.2.2/README.md ADDED
@@ -0,0 +1,317 @@
1
+ # deglib: Python Bindings for the Dynamic Exploration Graph
2
+
3
+ Python bindings for the high-performance C++ Dynamic Exploration Graph (DEG) library, enabling approximate nearest neighbor search (ANNS) and graph exploration with state-of-the-art recall vs. QPS trade-offs.
4
+
5
+ ---
6
+
7
+ ## Table of Contents
8
+
9
+ - [Installation](#installation)
10
+ - [From PyPI](#from-pypi)
11
+ - [Build from Source](#build-from-source)
12
+ - [Quickstart & Examples](#quickstart--examples)
13
+ - [Basic Usage](#basic-usage)
14
+ - [Graph Types & Lifecycles](#graph-types--lifecycles)
15
+ - [Saving and Loading Graphs](#saving-and-loading-graphs)
16
+ - [Incremental / Streaming Graph Construction](#incremental--streaming-graph-construction)
17
+ - [Filtered Search & Candidate Reranking](#filtered-search--candidate-reranking)
18
+ - [Exploratory Search & Graph Navigation](#exploratory-search--graph-navigation)
19
+ - [Graph Optimization, Quantization & Reranking Pipeline](#graph-optimization-quantization--reranking-pipeline)
20
+ - [Concepts & Parameters](#concepts--parameters)
21
+ - [OptimizationTarget](#optimizationtarget)
22
+ - [Search Parameter `eps`](#search-parameter-eps)
23
+ - [Supported Metrics & Data Types](#supported-metrics--data-types)
24
+ - [Example Projects](#example-projects)
25
+ - [API Reference](#api-reference)
26
+ - [Development Setup](#development-setup)
27
+ ---
28
+
29
+ ## Installation
30
+
31
+ ### From PyPI
32
+
33
+ ```bash
34
+ pip install deglib
35
+ ```
36
+
37
+ ### Build from Source
38
+
39
+ To build and install the package directly from the repository:
40
+
41
+ ```bash
42
+ cd python/
43
+ python setup.py copy_build_files
44
+ pip install .
45
+ ```
46
+
47
+ ---
48
+
49
+ ## Quickstart & Examples
50
+
51
+ ### Basic Usage
52
+
53
+ ```python
54
+ import numpy as np
55
+ import deglib
56
+
57
+ num_samples, dims = 10_000, 128
58
+
59
+ # 1. Create random feature dataset and query vector
60
+ data = np.random.random((num_samples, dims)).astype(np.float32)
61
+ query = np.random.random(dims).astype(np.float32)
62
+
63
+ # 2. Build index directly from data (multithreaded by default)
64
+ graph = deglib.builder.build_from_data(data, edges_per_vertex=32, callback="progress")
65
+
66
+ # 3. Query top-k nearest neighbors
67
+ indices, distances = graph.search(query, k=10, eps=0.1)
68
+
69
+ print("Nearest neighbors:", indices)
70
+ print("Distances:", distances)
71
+ ```
72
+
73
+ ---
74
+
75
+ ### Graph Types & Lifecycles
76
+
77
+ All search graphs are represented by `DynamicExplorationGraph`, backed by one of three internal graph engines:
78
+
79
+ 1. **Fixed-Capacity Mutable (`SizeBoundedGraph`)**: Memory is preallocated for a fixed maximum capacity. Fast and memory-efficient for static/batch datasets.
80
+ ```python
81
+ space = deglib.FloatSpace.create(dims=128, metric=deglib.Metric.FP32_L2)
82
+ graph = deglib.create_empty(capacity=10_000, feature_space=space, edges_per_vertex=32)
83
+ ```
84
+
85
+ 2. **Chunk-Allocated Mutable (`DynamicGraph`)**: Dynamically allocates memory in chunks (e.g. 1024 vertices per chunk). Ideal for streaming datasets where total capacity is unknown.
86
+ ```python
87
+ graph = deglib.create_dynamic_empty(feature_space=space, edges_per_vertex=32, chunk_size=1024)
88
+ ```
89
+
90
+ 3. **Read-Only Deployment (`ReadOnlyGraph`)**: Stripped of mutation structures for minimal memory footprint and maximum query throughput.
91
+ ```python
92
+ readonly_graph = graph.to_readonly()
93
+ ```
94
+
95
+ ---
96
+
97
+ ### Saving and Loading Graphs
98
+
99
+ ```python
100
+ # Save mutable graph to disk
101
+ graph.save_graph("index.deg")
102
+
103
+ # Load as compact, optimized read-only graph for production search
104
+ readonly_graph = deglib.load_readonly_graph("index.deg")
105
+
106
+ # Load as dynamic graph supporting further insertions and deletions
107
+ dynamic_graph = deglib.load_dynamic_graph("index.deg")
108
+
109
+ # Load as fixed-capacity mutable graph
110
+ mutable_graph = deglib.load_mutable_graph("index.deg")
111
+ ```
112
+
113
+ ---
114
+
115
+ ### Incremental / Streaming Graph Construction
116
+
117
+ For continuous additions and removals, use `GraphBuilder`:
118
+
119
+ ```python
120
+ import numpy as np
121
+ import deglib
122
+ from deglib.builder import GraphBuilder, OptimizationTarget
123
+ from deglib.distances import FloatSpace, Metric
124
+
125
+ dims = 128
126
+ max_capacity = 10_000
127
+ edges_per_vertex = 32
128
+
129
+ # 1. Create feature space and empty mutable graph
130
+ space = FloatSpace.create(dims, metric=Metric.FP32_L2)
131
+ graph = deglib.create_empty(max_capacity, space, edges_per_vertex)
132
+
133
+ # 2. Initialize builder
134
+ builder = GraphBuilder(graph, optimization_target=OptimizationTarget.StreamingData, seed=42)
135
+
136
+ # 3. Add entries (individually or in batches)
137
+ labels = np.arange(100, dtype=np.uint32)
138
+ features = np.random.random((100, dims)).astype(np.float32)
139
+ builder.add_entry(labels, features)
140
+
141
+ # 4. Remove entries by external label
142
+ builder.remove_entry(42)
143
+
144
+ # 5. Build / optimize
145
+ builder.build()
146
+ ```
147
+
148
+ ---
149
+
150
+ ### Filtered Search & Candidate Reranking
151
+
152
+ ```python
153
+ from deglib.search import Filter, rerank
154
+
155
+ # Search only within allowed external labels
156
+ allowed_ids = np.array([1, 5, 10, 42, 99], dtype=np.int32)
157
+ search_filter = Filter(allowed_ids)
158
+
159
+ indices, distances = graph.search(query, k=5, eps=0.1, filter_labels=search_filter)
160
+
161
+ # Exact distance reranking across candidates
162
+ queries = np.random.random((10, dims)).astype(np.float32)
163
+ candidates = np.random.randint(0, 1000, size=(10, 50), dtype=np.uint32)
164
+ base_vectors = np.random.random((1000, dims)).astype(np.float32)
165
+
166
+ top_indices, top_distances = rerank(
167
+ space=graph.get_feature_space(),
168
+ queries=queries,
169
+ candidate_indices=candidates,
170
+ base_vectors=base_vectors,
171
+ k_top=10,
172
+ return_distances=True,
173
+ )
174
+ ```
175
+
176
+ ---
177
+
178
+ ### Exploratory Search & Graph Navigation
179
+
180
+ DEG supports exploratory search directly from existing vertex labels:
181
+
182
+ ```python
183
+ # Explore graph starting from entry vertex label 105
184
+ explored_labels, distances = graph.explore(entry_external_label=105, k=10, eps=0.1, include_entry=False)
185
+ ```
186
+
187
+ ---
188
+
189
+ ### Graph Optimization, Quantization & Reranking Pipeline
190
+
191
+ A complete end-to-end pipeline demonstrating **FLAS pre-sorting**, **multithreaded graph building**, **RNG edge pruning**, **Int8 quantization**, **ReadOnlyGraph conversion**, and **exact FP32 candidate reranking**:
192
+
193
+ ```python
194
+ import numpy as np
195
+ import deglib
196
+ from deglib.optimization import presort, prune_non_rng_edges, ScalarQuantizerInt8
197
+ from deglib.search import rerank
198
+
199
+ num_vectors, dims = 10_000, 128
200
+ data = np.random.randn(num_vectors, dims).astype(np.float32)
201
+ query = np.random.randn(1, dims).astype(np.float32)
202
+
203
+ # 1. Pre-sort vectors using FLAS for improved memory locality and index construction speed
204
+ perm = presort(data, metric=deglib.Metric.FP32_InnerProduct, callback="progress")
205
+ sorted_data = data[perm]
206
+
207
+ # 2. Build exploration graph on unquantized/original data
208
+ graph = deglib.builder.build_from_data(
209
+ sorted_data,
210
+ metric=deglib.Metric.FP32_InnerProduct,
211
+ edges_per_vertex=32,
212
+ callback="progress",
213
+ )
214
+
215
+ # 3. Prune redundant non-RNG edges to optimize graph topology
216
+ pruned_count = prune_non_rng_edges(graph)
217
+ print(f"Pruned {pruned_count} redundant edges.")
218
+
219
+ # 4. Fit Int8 quantizer on base dataset and quantize features for memory reduction and ultra-fast search
220
+ quantizer = ScalarQuantizerInt8()
221
+ quantizer.fit(sorted_data)
222
+ quant_int8_data = quantizer.quantize(sorted_data)
223
+ int8_space = deglib.FloatSpace.create(dims, deglib.Metric.Int8_InnerProduct)
224
+
225
+ # 5. Convert mutable graph to a compact ReadOnlyGraph equipped with the quantized features
226
+ readonly_graph = graph.to_readonly(feature_space=int8_space, custom_features=quant_int8_data)
227
+
228
+ # 6. Quantize query using the EXACT SAME calibrated scale, then search
229
+ quant_query = quantizer.quantize(query)
230
+ candidate_indices = readonly_graph.search(quant_query, k=20, eps=0.1, return_distances=False)
231
+
232
+ # 7. Exact distance reranking of top candidates on original float32 data
233
+ fp32_space = deglib.FloatSpace.create(dims, deglib.Metric.FP32_InnerProduct)
234
+ final_top_indices, distances = rerank(
235
+ space=fp32_space,
236
+ queries=query,
237
+ candidate_indices=candidate_indices,
238
+ base_vectors=sorted_data,
239
+ k_top=10,
240
+ return_distances=True,
241
+ )
242
+
243
+ print("Top-10 nearest neighbor indices:", final_top_indices[0])
244
+ print("Top-10 exact distances:", distances[0])
245
+ ```
246
+
247
+ ---
248
+
249
+ ## Concepts & Parameters
250
+
251
+ ### `OptimizationTarget`
252
+ Controls the topology optimization strategy:
253
+ - `OptimizationTarget.LowLID`: Default for datasets with low local intrinsic dimensionality (supports multithreaded building).
254
+ - `OptimizationTarget.HighLID`: Optimized for datasets with high local intrinsic dimensionality (supports multithreaded building).
255
+ - `OptimizationTarget.StreamingData`: Optimized for continuous dynamic additions and deletions.
256
+
257
+ ### Search Parameter `eps`
258
+ - The epsilon parameter expands the search priority queue during graph exploration.
259
+ - Small values (e.g. `eps=0.001` or `eps=0.01`): Faster query execution.
260
+ - Higher values (e.g. `eps=0.1` to `eps=0.3`): Higher recall rate.
261
+
262
+ ### Supported Metrics & Data Types
263
+ - `Metric.FP32_L2`: Euclidean distance (`np.float32`)
264
+ - `Metric.FP32_InnerProduct`: Inner product / cosine distance (`np.float32`)
265
+ - `Metric.Uint8_L2`: 8-bit unsigned integer Euclidean distance (`np.uint8`)
266
+ - `Metric.Uint8_InnerProduct`: 8-bit unsigned integer inner product (`np.uint8`)
267
+ - `Metric.FP16_L2`: 16-bit half-precision Euclidean distance (`np.uint16`)
268
+ - `Metric.FP16_InnerProduct`: 16-bit half-precision inner product (`np.uint16`)
269
+ - `Metric.EVP_InnerProduct`: Quantized Extreme Value Property bit-packed vectors (`np.uint8`)
270
+ - `Metric.Int8_InnerProduct`: 8-bit signed integer inner product (`np.int8`)
271
+ - `Metric.Int8_L2`: 8-bit signed integer Euclidean distance (`np.int8`)
272
+
273
+ ---
274
+
275
+ ## API Reference
276
+
277
+ For a complete overview of all Python modules, classes, and function signatures, see [API.md](API.md) or the [Official Documentation](https://dynamic-exploration-graph.readthedocs.io/).
278
+
279
+ ---
280
+
281
+ ## Example Projects
282
+
283
+ Ready-to-run example scripts with visual progress and evaluation are located in the [examples/](../examples/) directory:
284
+
285
+ - [examples/knng/](../examples/knng/): k-NN graph construction benchmark using EVP quantization and FP16 reranking (SISAP 2026 Challenge Task 1).
286
+ - [examples/mips/](../examples/mips/): Maximum Inner Product Search (MIPS) benchmark using $(d+1)$-dimensional $L_2$ transformation, FLAS pre-sorting, and SIMD FP16 inner products (SISAP 2026 Challenge Task 2).
287
+ - [examples/static_data/](../examples/static_data/): Static dataset indexing and ANNS benchmark.
288
+ - [examples/vibe/](../examples/vibe/): VIBE benchmark for modern embedding datasets with scalar quantization and interactive Plotly visualizations.
289
+ - [examples/dynamic_data/](../examples/dynamic_data/): Dynamic streaming additions and deletions.
290
+ - [examples/sliding_window/](../examples/sliding_window/): Sliding window continuous update benchmark against CleANN.
291
+
292
+
293
+ ---
294
+
295
+ ## Development Setup
296
+
297
+ If you want to contribute, modify C++ bindings, run tests, or build release packages:
298
+
299
+ ```bash
300
+ # 1. Setup virtual environment and dependencies (using uv)
301
+ cd python/
302
+ uv venv
303
+ uv pip install setuptools==83.0.0 pybind11==3.0.4 build==1.5.0 wheel==0.48.0
304
+ uv run python setup.py copy_build_files
305
+
306
+ # 2. Install in editable mode
307
+ uv pip install -e . --no-build-isolation --verbose
308
+
309
+ # 3. Run test suite
310
+ uv run pytest
311
+
312
+ # 4. Format code
313
+ uv run ruff format .
314
+
315
+ # 5. Build distribution packages (sdist and binary wheels)
316
+ uv run python -m build
317
+ ```
@@ -41,9 +41,9 @@ if (FORCE_AVX2)
41
41
  if(MSVC)
42
42
  check_cxx_compiler_flag("/arch:AVX2" COMPILER_SUPPORTS_AVX2)
43
43
  if(COMPILER_SUPPORTS_AVX2)
44
- target_compile_options(compile-options INTERFACE /O2 /arch:AVX2 /W1 /EHsc /fp:precise)
44
+ target_compile_options(compile-options INTERFACE /O2 /arch:AVX2 /W1 /EHsc /fp:precise /MP /bigobj)
45
45
  else()
46
- target_compile_options(compile-options INTERFACE /O2 /W1 /EHsc /fp:precise)
46
+ target_compile_options(compile-options INTERFACE /O2 /W1 /EHsc /fp:precise /MP /bigobj)
47
47
  endif()
48
48
  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
49
49
  check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
@@ -79,7 +79,7 @@ else() # Native build
79
79
  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
80
80
  target_compile_options(compile-options INTERFACE -O2 -DNDEBUG -march=native -fpic)
81
81
  elseif(MSVC)
82
- target_compile_options(compile-options INTERFACE /W4 /EHsc /fp:precise $<$<CONFIG:Release>:/O2>)
82
+ target_compile_options(compile-options INTERFACE /W4 /EHsc /fp:precise /MP /bigobj $<$<CONFIG:Release>:/O2>)
83
83
  if(SUPPORT_AVX512F)
84
84
  target_compile_options(compile-options INTERFACE /arch:AVX512)
85
85
  elseif(SUPPORT_AVX2)