deglib 0.1.0__tar.gz

deglib-0.1.0/CMakeLists.txt

@@ -0,0 +1,24 @@
+# python-bindings/CMakeLists.txt
+cmake_minimum_required(VERSION 3.19)
+project(deglib_cpp)
+
+# Add python executable
+if(DEFINED PYTHON_EXECUTABLE)
+    set(Python_EXECUTABLE ${PYTHON_EXECUTABLE})
+endif()
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+
+# Add pybind11
+find_package(pybind11 CONFIG REQUIRED)
+
+# disable benchmark building
+set(ENABLE_BENCHMARKS OFF CACHE BOOL "Disable benchmarks" FORCE)
+
+# Add cpp-deglib
+add_subdirectory(${CMAKE_SOURCE_DIR}/lib/)
+pybind11_add_module(deglib_cpp src/deg_cpp/deglib_cpp.cpp)
+
+# Specify the include directories
+include_directories(${CMAKE_SOURCE_DIR}/lib/deglib/include)
+
+target_link_libraries(deglib_cpp PRIVATE compile-options)

deglib-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Visual Computing Group
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

deglib-0.1.0/MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE
+include README.md
+recursive-include src *.cpp
+recursive-include lib *.h
+global-include CMakeLists.txt *.cmake

deglib-0.1.0/PKG-INFO

@@ -0,0 +1,203 @@
+Metadata-Version: 2.1
+Name: deglib
+Version: 0.1.0
+Summary: Python bindings for the Dynamic Exploration Graph library by Nico Hezel
+Author: Nico Hezel
+Author-email: Bruno Schilling <bruno.schilling@htw-berlin.de>
+License: MIT License
+        
+        Copyright (c) 2024 Visual Computing Group
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/Visual-Computing/DynamicExplorationGraph
+Keywords: anns-search,graph,python
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.21
+Requires-Dist: psutil
+Provides-Extra: test
+Requires-Dist: pytest>=8.0; extra == "test"
+
+# deglib: Python bindings for the Dynamic Exploration Graph
+
+Python bindings for the C++ library Dynamic Exploration Graph used in the paper:
+[Fast Approximate Nearest Neighbor Search with a Dynamic Exploration Graph using Continuous Refinement](https://arxiv.org/abs/2307.10479)
+
+## Table of Contents
+- [Installation](#installation)
+- [Examples](#examples)
+- [Naming](#naming)
+- [Limitations](#limitations)
+- [Troubleshooting](#troubleshooting)
+
+## Installation
+
+### Using pip
+```shell
+pip install deglib
+```
+This will install a source package, that needs to compile the C++ code in order to create an optimized version for your system.
+
+### Compiling from Source
+
+**Create Virtual Environment**
+```shell
+# create virtualenv with virtualenvwrapper or venv
+mkvirtualenv deglib
+# or
+python -m venv /path/to/deglib_env && . /path/to/deglib_env/bin/activate
+```
+
+**Get the Source**
+```shell
+# clone git repository
+git clone git@github.com:Visual-Computing/DynamicExplorationGraph.git
+cd DynamicExplorationGraph/python
+```
+
+**Install the Package from Source**
+```shell
+pip install setuptools pybind11 build
+python setup.py copy_build_files  # copy c++ library to ./lib/
+pip install .
+```
+This will compile the C++ code and install deglib into your virtual environment, so it may take a while.
+
+**Testing**
+
+To execute all tests.
+```shell
+pytest
+```
+
+**Building Packages**
+
+Build packages (sdist and wheels):
+```shell
+python -m build
+```
+
+Note: If you want to publish linux wheels to pypi you have to convert
+the wheel to musllinux-/manylinux-wheels.
+This can be easily done using `cibuildwheel` (if docker is installed):
+
+```shell
+cibuildwheel --archs auto64 --output-dir dist
+```
+
+## Examples
+### Loading Data
+To load a dataset formatted like the [TexMex-Datasets](http://corpus-texmex.irisa.fr/):
+```python
+import deglib
+import numpy as np
+
+dataset: np.ndarray = deglib.repository.fvecs_read("path/to/data.fvecs")
+num_samples, dims = dataset.shape
+```
+The dataset is a numpy array with shape (N, D), where N is the number of feature
+vectors and D is the number of dimensions of each feature vector.
+
+### Building a Graph
+
+```python
+graph = deglib.builder.build_from_data(dataset, edges_per_vertex=32, callback="progress")
+graph.save_graph("/path/to/graph.deg")
+rd_graph = deglib.graph.load_readonly_graph("/path/to/graph.deg")
+```
+
+### Searching the Graph
+```python
+# query can have shape (D,) or (Q, D), where
+# D is the dimensionality of the dataset and
+# Q is the number of queries.
+query = np.random.random((dims,)).astype(np.float32)
+result, dists = graph.search(query, eps=0.1, k=10)  # get 10 nearest features to query
+print('best dataset index:', result[0])
+best_match = dataset[result[0]]
+```
+
+For more examples see [tests](tests).
+
+### Referencing C++ memory
+Consider the following example:
+```python
+feature_vector = graph.get_feature_vector(42)
+del graph
+print(feature_vector)
+```
+This will crash as `feature_vector` is holding a reference to memory that is owned by `graph`. This can lead to undefined behaviour (most likely segmentation fault).
+Be careful to keep objects in memory that are referenced. If you need it use the `copy=True` option:
+
+```python
+feature_vector = graph.get_feature_vector(10, copy=True)
+del graph
+print(feature_vector)  # no problem
+```
+
+Copying feature vectors will be slower.
+
+## Naming
+### Vertex = Feature Vector
+Each vertex in the graph corresponds to a feature vector of the dataset.
+
+### Internal Index vs External Label
+There are two kinds of indices used in a graph: `internal_index` and `external_label`. Both are integers and specify
+a vertex in a graph.
+
+Internal Indices are dense, which means that every `internal_index < len(graph)` can be used.
+For example: If you add 100 vertices and remove the vertex with internal_index 42, the last vertex in the graph will
+be moved to index 42.
+
+In contrast, external label is a user defined identifier for each added vertex
+(see `builder.add_entry(external_label, feature_vector)`). Adding or Removing vertices to the graph will keep the
+connection between external labels and associated feature vector.
+
+When you create the external labels by starting with `0` and increasing it for each entry by `1` and don't remove
+elements from the graph, external labels and internal indices are equal.
+
+```python
+# as long as no elements are removed
+# external labels and internal indices are equal
+for i, vec in enumerate(data):
+    builder.add_entry(i, vec)
+```
+
+### Eps
+The eps-search-parameter controls how many nodes are checked during search.
+Lower eps values like 0.001 are faster but less accurate.
+Higher eps values like 0.1 are slower but more accurate. Should always be greater 0.
+
+### Relative Neighborhood Graph / RNG-conform
+TODO
+
+## Limitations
+- The python wrapper at the moment only supports `float32` and `uint8` feature vectors.
+
+## Troubleshooting
+
+### BuildError: `pybind11/typing.h:104:58: error: ‘copy_n’ is not a member of ‘std’`
+
+This is a pybind11 bug, that occurs when compiling it with gcc-14. Change the pybind version to 2.12.
+

deglib-0.1.0/README.md

@@ -0,0 +1,162 @@
+# deglib: Python bindings for the Dynamic Exploration Graph
+
+Python bindings for the C++ library Dynamic Exploration Graph used in the paper:
+[Fast Approximate Nearest Neighbor Search with a Dynamic Exploration Graph using Continuous Refinement](https://arxiv.org/abs/2307.10479)
+
+## Table of Contents
+- [Installation](#installation)
+- [Examples](#examples)
+- [Naming](#naming)
+- [Limitations](#limitations)
+- [Troubleshooting](#troubleshooting)
+
+## Installation
+
+### Using pip
+```shell
+pip install deglib
+```
+This will install a source package, that needs to compile the C++ code in order to create an optimized version for your system.
+
+### Compiling from Source
+
+**Create Virtual Environment**
+```shell
+# create virtualenv with virtualenvwrapper or venv
+mkvirtualenv deglib
+# or
+python -m venv /path/to/deglib_env && . /path/to/deglib_env/bin/activate
+```
+
+**Get the Source**
+```shell
+# clone git repository
+git clone git@github.com:Visual-Computing/DynamicExplorationGraph.git
+cd DynamicExplorationGraph/python
+```
+
+**Install the Package from Source**
+```shell
+pip install setuptools pybind11 build
+python setup.py copy_build_files  # copy c++ library to ./lib/
+pip install .
+```
+This will compile the C++ code and install deglib into your virtual environment, so it may take a while.
+
+**Testing**
+
+To execute all tests.
+```shell
+pytest
+```
+
+**Building Packages**
+
+Build packages (sdist and wheels):
+```shell
+python -m build
+```
+
+Note: If you want to publish linux wheels to pypi you have to convert
+the wheel to musllinux-/manylinux-wheels.
+This can be easily done using `cibuildwheel` (if docker is installed):
+
+```shell
+cibuildwheel --archs auto64 --output-dir dist
+```
+
+## Examples
+### Loading Data
+To load a dataset formatted like the [TexMex-Datasets](http://corpus-texmex.irisa.fr/):
+```python
+import deglib
+import numpy as np
+
+dataset: np.ndarray = deglib.repository.fvecs_read("path/to/data.fvecs")
+num_samples, dims = dataset.shape
+```
+The dataset is a numpy array with shape (N, D), where N is the number of feature
+vectors and D is the number of dimensions of each feature vector.
+
+### Building a Graph
+
+```python
+graph = deglib.builder.build_from_data(dataset, edges_per_vertex=32, callback="progress")
+graph.save_graph("/path/to/graph.deg")
+rd_graph = deglib.graph.load_readonly_graph("/path/to/graph.deg")
+```
+
+### Searching the Graph
+```python
+# query can have shape (D,) or (Q, D), where
+# D is the dimensionality of the dataset and
+# Q is the number of queries.
+query = np.random.random((dims,)).astype(np.float32)
+result, dists = graph.search(query, eps=0.1, k=10)  # get 10 nearest features to query
+print('best dataset index:', result[0])
+best_match = dataset[result[0]]
+```
+
+For more examples see [tests](tests).
+
+### Referencing C++ memory
+Consider the following example:
+```python
+feature_vector = graph.get_feature_vector(42)
+del graph
+print(feature_vector)
+```
+This will crash as `feature_vector` is holding a reference to memory that is owned by `graph`. This can lead to undefined behaviour (most likely segmentation fault).
+Be careful to keep objects in memory that are referenced. If you need it use the `copy=True` option:
+
+```python
+feature_vector = graph.get_feature_vector(10, copy=True)
+del graph
+print(feature_vector)  # no problem
+```
+
+Copying feature vectors will be slower.
+
+## Naming
+### Vertex = Feature Vector
+Each vertex in the graph corresponds to a feature vector of the dataset.
+
+### Internal Index vs External Label
+There are two kinds of indices used in a graph: `internal_index` and `external_label`. Both are integers and specify
+a vertex in a graph.
+
+Internal Indices are dense, which means that every `internal_index < len(graph)` can be used.
+For example: If you add 100 vertices and remove the vertex with internal_index 42, the last vertex in the graph will
+be moved to index 42.
+
+In contrast, external label is a user defined identifier for each added vertex
+(see `builder.add_entry(external_label, feature_vector)`). Adding or Removing vertices to the graph will keep the
+connection between external labels and associated feature vector.
+
+When you create the external labels by starting with `0` and increasing it for each entry by `1` and don't remove
+elements from the graph, external labels and internal indices are equal.
+
+```python
+# as long as no elements are removed
+# external labels and internal indices are equal
+for i, vec in enumerate(data):
+    builder.add_entry(i, vec)
+```
+
+### Eps
+The eps-search-parameter controls how many nodes are checked during search.
+Lower eps values like 0.001 are faster but less accurate.
+Higher eps values like 0.1 are slower but more accurate. Should always be greater 0.
+
+### Relative Neighborhood Graph / RNG-conform
+TODO
+
+## Limitations
+- The python wrapper at the moment only supports `float32` and `uint8` feature vectors.
+
+## Troubleshooting
+
+### BuildError: `pybind11/typing.h:104:58: error: ‘copy_n’ is not a member of ‘std’`
+
+This is a pybind11 bug, that occurs when compiling it with gcc-14. Change the pybind version to 2.12.
+

deglib-0.1.0/lib/CMakeLists.txt

@@ -0,0 +1,126 @@
+cmake_minimum_required(VERSION 3.19)
+
+# Only interpret if() arguments as variables or keywords when unquoted. More
+# details run "cmake --help-policy CMP0054"
+cmake_policy(SET CMP0054 NEW)
+
+project(
+  deg_lib
+  HOMEPAGE_URL "https://github.com/Neiko2002/DEG"
+  DESCRIPTION
+    "Dynamic Exploration Graphs for fast approximate nearest neighbor search and navigation in large image datasets"
+  VERSION 0.0.1
+  LANGUAGES CXX
+  )
+
+# build options
+option(ENABLE_BENCHMARKS "compile benchmarks" ON)
+option(FORCE_AVX2 "compile for avx2 support. Otherwise compile for this machine." OFF)
+
+# https://cmake.org/cmake/help/latest/prop_tgt/CXX_STANDARD.html
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+add_library(compile-options INTERFACE)
+target_compile_features(compile-options INTERFACE cxx_std_20)
+target_compile_definitions(compile-options INTERFACE $<$<PLATFORM_ID:Windows>:NOMINMAX>)
+
+# detecting CPU instruction support
+# https://github.com/lemire/FastPFor/blob/master/CMakeLists.txt
+if (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
+  message(WARNING "You are compiling on an Apple M chip. At the moment deglib does not support ARM optimizations, so using deglib will be slow!")
+endif()
+
+if (FORCE_AVX2)
+  message(NOTICE "Compiling for avx2, ignoring native optimizations.")
+
+  # setup compiler flags
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    target_compile_options(
+      compile-options
+      INTERFACE -O3
+                -mavx2
+                -mfma
+                -fpic
+                -w
+                -fopenmp
+                -pthread
+                -ftree-vectorize
+                -ftree-vectorizer-verbose=0
+                )
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+    target_compile_options(
+      compile-options
+      INTERFACE -O2
+                -mavx2
+                -mfma
+                -fpic)
+  elseif(MSVC)
+    target_compile_options(
+	  compile-options
+	  INTERFACE /O2
+				/arch:AVX2
+				/W1
+				/openmp
+				/EHsc)
+  else()
+    message(WARNING "Unknown compiler for AVX2 Build: ${CMAKE_CXX_COMPILER_ID}")
+  endif()
+
+else()  # Native build
+  list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
+  include(DetectCPUFeatures)
+
+  # setup compiler flags
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    target_compile_options(
+      compile-options
+      INTERFACE -O3
+                -march=native
+                -mavx2
+                -mfma
+                -fpic
+                -w
+                -fopenmp
+                -pthread
+                -ftree-vectorize
+                -ftree-vectorizer-verbose=0)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+    target_compile_options(
+      compile-options
+      INTERFACE -O2
+                -DNDEBUG
+                -march=native
+                -fpic)
+  elseif(MSVC)
+    # warning level 4 (show all)
+    target_compile_options(compile-options INTERFACE /W4)
+
+    # detecting SUPPORT_AVX2 or SSE2 support
+    cmake_host_system_information(RESULT SUPPORT_SSE2 QUERY HAS_SSE2)
+    if(SUPPORT_AVX2)
+      target_compile_options(compile-options INTERFACE /arch:AVX2)
+    elseif(SUPPORT_SSE2)
+      target_compile_options(compile-options INTERFACE /arch:SSE2)
+    else()
+      target_compile_options(compile-options INTERFACE /arch:native)
+    endif()
+
+    # disable string optimizations and function level linking
+    # https://stackoverflow.com/questions/5063334/what-is-the-difference-between-the-ox-and-o2-compiler-options
+    target_compile_options(compile-options INTERFACE $<$<CONFIG:Release>:/Ox>)
+  else()
+    message(WARNING "Unknown compiler: ${CMAKE_CXX_COMPILER_ID}")
+  endif()
+endif()
+
+message("C++ compiler flags: ${CMAKE_CXX_FLAGS}")
+message("C++ compiler flags release: ${CMAKE_CXX_FLAGS_RELEASE}")
+
+add_subdirectory(deglib)
+
+if (ENABLE_BENCHMARKS)
+	add_subdirectory(external/fmt)
+	add_subdirectory(benchmark)
+endif()

deglib-0.1.0/lib/cmake_modules/DetectCPUFeatures.cmake

@@ -0,0 +1,52 @@
+include(CheckCXXSourceCompiles)
+
+set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+
+set(SSE4PROG "
+
+#include<smmintrin.h>
+int main(){
+__m128 x=_mm_set1_ps(0.5);
+x=_mm_dp_ps(x,x,0x77);
+return _mm_movemask_ps(x);
+}")
+
+set(AVXPROG "
+
+#include<immintrin.h>
+int main(){
+__m128 x=_mm_set1_ps(0.5);
+x=_mm_permute_ps(x,1);
+return _mm_movemask_ps(x);
+}")
+
+set(AVX2PROG "
+
+#include<immintrin.h>
+int main(){
+__m256i x=_mm256_set1_epi32(5);
+x=_mm256_add_epi32(x,x);
+return _mm256_movemask_epi8(x);
+}")
+
+if(MSVC)
+	set(CMAKE_REQUIRED_FLAGS "/EHsc /arch:SSE2")
+	check_cxx_source_compiles("${SSE4PROG}" SUPPORT_SSE42)
+	message(STATUS "SUPPORT_SSE42 ${SUPPORT_SSE42}")
+	set(CMAKE_REQUIRED_FLAGS "/EHsc /arch:AVX")
+	check_cxx_source_compiles("${AVXPROG}" SUPPORT_AVX)
+	message(STATUS "SUPPORT_AVX ${SUPPORT_AVX}")
+	set(CMAKE_REQUIRED_FLAGS "/EHsc /arch:AVX2")
+	check_cxx_source_compiles("${AVX2PROG}" SUPPORT_AVX2)
+	message(STATUS "SUPPORT_AVX2 ${SUPPORT_AVX2}")
+else()
+	set(CMAKE_REQUIRED_FLAGS "-march=native -msse4.2")
+	check_cxx_source_compiles("${SSE4PROG}" SUPPORT_SSE42)
+	set(CMAKE_REQUIRED_FLAGS "-march=native -mavx")
+	check_cxx_source_compiles("${AVXPROG}" SUPPORT_AVX)
+	set(CMAKE_REQUIRED_FLAGS "-march=native -mavx2")
+	check_cxx_source_compiles("${AVX2PROG}" SUPPORT_AVX2)
+endif()	
+
+set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+	

deglib-0.1.0/lib/deglib/CMakeLists.txt

@@ -0,0 +1,10 @@
+cmake_minimum_required (VERSION 3.19)
+
+project(deglib)
+
+# setup the deg lib
+# http://mariobadr.com/creating-a-header-only-library-with-cmake.html
+add_library(DEG_LIB INTERFACE)
+target_include_directories(DEG_LIB INTERFACE ${PROJECT_SOURCE_DIR}/include)
+target_include_directories(DEG_LIB INTERFACE ${PROJECT_SOURCE_DIR}/include/graph)
+target_compile_features(DEG_LIB INTERFACE cxx_std_20)