vextor 0.1.0__tar.gz

vextor-0.1.0/.clang-format

@@ -0,0 +1,5 @@
+BasedOnStyle: Google
+ColumnLimit: 100
+IndentWidth: 4
+PointerAlignment: Left
+AllowShortFunctionsOnASingleLine: Inline

vextor-0.1.0/.clangd

@@ -0,0 +1,3 @@
+CompileFlags:
+  CompilationDatabase: build
+  Add: [-Isrc, -std=c++20]

vextor-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,101 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  format-check:
+    name: clang-format check
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run clang-format check
+        uses: jidicula/clang-format-action@v4.14.0
+        with:
+          clang-format-version: '18'
+          check-path: 'src'
+
+      - name: Check test formatting
+        uses: jidicula/clang-format-action@v4.14.0
+        with:
+          clang-format-version: '18'
+          check-path: 'tests'
+
+      - name: Check benchmark formatting
+        uses: jidicula/clang-format-action@v4.14.0
+        with:
+          clang-format-version: '18'
+          check-path: 'benchmarks'
+
+  build-and-test:
+    name: ${{ matrix.compiler }} (${{ matrix.build_type }})
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - compiler: gcc-14
+            cc: gcc-14
+            cxx: g++-14
+            build_type: Release
+            sanitizers: false
+
+          - compiler: gcc-14
+            cc: gcc-14
+            cxx: g++-14
+            build_type: Debug
+            sanitizers: true
+
+          - compiler: clang-18
+            cc: clang-18
+            cxx: clang++-18
+            build_type: Release
+            sanitizers: false
+
+          - compiler: clang-18
+            cc: clang-18
+            cxx: clang++-18
+            build_type: Debug
+            sanitizers: true
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache FetchContent dependencies
+        uses: actions/cache@v4
+        with:
+          path: build/_deps
+          key: fetchcontent-${{ matrix.compiler }}-${{ hashFiles('CMakeLists.txt', 'tests/CMakeLists.txt', 'benchmarks/CMakeLists.txt') }}
+
+      - name: Configure
+        env:
+          CC: ${{ matrix.cc }}
+          CXX: ${{ matrix.cxx }}
+        # -O1 for Debug: sanitizers don't need -O0, and the HNSW construction
+        # tests are unusably slow unoptimized (ASan docs recommend -O1).
+        # Asserts stay active (still a Debug build, NDEBUG undefined).
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g" \
+            -DVEXTOR_ENABLE_ASAN=${{ matrix.sanitizers && 'ON' || 'OFF' }} \
+            -DVEXTOR_ENABLE_UBSAN=${{ matrix.sanitizers && 'ON' || 'OFF' }}
+
+      - name: Build
+        run: cmake --build build -j$(nproc)
+
+      - name: Test
+        run: ctest --test-dir build --output-on-failure --parallel $(nproc)

vextor-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,49 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build-sdist:
+    name: Build and verify sdist
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Verify sdist installs and passes smoke tests
+        env:
+          CC: gcc-14
+          CXX: g++-14
+        run: |
+          python3 -m pip install dist/*.tar.gz
+          python3 python/test_vextor.py
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: Publish to PyPI (trusted publishing)
+    needs: build-sdist
+    runs-on: ubuntu-24.04
+    environment:
+      name: pypi
+      url: https://pypi.org/p/vextor
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

vextor-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
+CMakeUserPresets.json
+build*/
+ai-notes/
+benchmarks/sift1m/data/
+.claude/

vextor-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v18.1.8
+    hooks:
+      - id: clang-format
+        types_or: [c, c++]

vextor-0.1.0/CMakeLists.txt

@@ -0,0 +1,75 @@
+cmake_minimum_required(VERSION 3.20)
+project(vextor VERSION 0.1.0 LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# --- Options ---
+option(VEXTOR_BUILD_TESTS "Build unit tests" ON)
+option(VEXTOR_BUILD_BENCHMARKS "Build benchmarks" ON)
+option(VEXTOR_ENABLE_ASAN "Enable AddressSanitizer" OFF)
+option(VEXTOR_ENABLE_UBSAN "Enable UndefinedBehaviorSanitizer" OFF)
+
+# --- Compiler warnings ---
+add_compile_options(-Wall -Wextra -Wpedantic)
+
+# --- Sanitizers ---
+if(VEXTOR_ENABLE_ASAN)
+    add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
+    add_link_options(-fsanitize=address)
+endif()
+
+if(VEXTOR_ENABLE_UBSAN)
+    add_compile_options(-fsanitize=undefined)
+    add_link_options(-fsanitize=undefined)
+endif()
+
+# --- AVX2 detection ---
+include(CheckCXXCompilerFlag)
+check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
+check_cxx_compiler_flag("-mfma" COMPILER_SUPPORTS_FMA)
+
+# --- Library target ---
+add_library(vextor STATIC
+    src/core/distance.cpp
+    src/core/sq8.cpp
+    src/store/in_memory_store.cpp
+    src/store/mmap_store.cpp
+    src/segment/id_mapping.cpp
+    src/segment/active_segment.cpp
+    src/segment/sealed_segment.cpp
+    src/segment/segment_manager.cpp
+    src/persistence/serializer.cpp
+    src/persistence/loader.cpp
+)
+
+target_include_directories(vextor PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+target_compile_features(vextor PUBLIC cxx_std_20)
+set_target_properties(vextor PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+if(COMPILER_SUPPORTS_AVX2 AND COMPILER_SUPPORTS_FMA)
+    target_compile_options(vextor PRIVATE -mavx2 -mfma)
+    target_compile_definitions(vextor PUBLIC VEXTOR_AVX2=1)
+endif()
+
+# --- Tests ---
+if(VEXTOR_BUILD_TESTS)
+    enable_testing()
+    add_subdirectory(tests)
+endif()
+
+# --- Benchmarks ---
+if(VEXTOR_BUILD_BENCHMARKS)
+    add_subdirectory(benchmarks)
+endif()
+
+# --- Python bindings ---
+option(VEXTOR_BUILD_PYTHON "Build Python bindings" OFF)
+if(VEXTOR_BUILD_PYTHON)
+    add_subdirectory(python)
+endif()

vextor-0.1.0/CMakePresets.json

@@ -0,0 +1,52 @@
+{
+    "version": 6,
+    "cmakeMinimumRequired": {
+        "major": 3,
+        "minor": 20,
+        "patch": 0
+    },
+    "configurePresets": [
+        {
+            "name": "dev",
+            "displayName": "Debug + Sanitizers",
+            "generator": "Ninja",
+            "binaryDir": "${sourceDir}/build-dev",
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Debug",
+                "VEXTOR_BUILD_TESTS": "ON",
+                "VEXTOR_BUILD_BENCHMARKS": "ON",
+                "VEXTOR_ENABLE_ASAN": "ON",
+                "VEXTOR_ENABLE_UBSAN": "ON",
+                "VEXTOR_BUILD_PYTHON": "OFF"
+            }
+        },
+        {
+            "name": "release",
+            "displayName": "Release",
+            "generator": "Ninja",
+            "binaryDir": "${sourceDir}/build-release",
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Release",
+                "VEXTOR_BUILD_TESTS": "ON",
+                "VEXTOR_BUILD_BENCHMARKS": "ON",
+                "VEXTOR_ENABLE_ASAN": "OFF",
+                "VEXTOR_ENABLE_UBSAN": "OFF",
+                "VEXTOR_BUILD_PYTHON": "OFF"
+            }
+        },
+        {
+            "name": "release-python",
+            "displayName": "Release + Python bindings",
+            "generator": "Ninja",
+            "binaryDir": "${sourceDir}/build-release-python",
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Release",
+                "VEXTOR_BUILD_TESTS": "ON",
+                "VEXTOR_BUILD_BENCHMARKS": "ON",
+                "VEXTOR_ENABLE_ASAN": "OFF",
+                "VEXTOR_ENABLE_UBSAN": "OFF",
+                "VEXTOR_BUILD_PYTHON": "ON"
+            }
+        }
+    ]
+}

vextor-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 mariorch22
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vextor-0.1.0/PKG-INFO

@@ -0,0 +1,207 @@
+Metadata-Version: 2.1
+Name: vextor
+Version: 0.1.0
+Summary: Segmented vector database for approximate nearest neighbor search (HNSW, AVX2, mmap)
+Author: Mario Raach
+License: MIT License
+         
+         Copyright (c) 2026 mariorch22
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+         
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Project-URL: Repository, https://github.com/mariorch22/vextor
+Requires-Python: >=3.8
+Requires-Dist: numpy
+Description-Content-Type: text/markdown
+
+# vextor
+
+A segmented vector database for Approximate Nearest Neighbor search, written in C++20. Uses AVX2 SIMD distance kernels, HNSW graph indexing, and memory-mapped storage.
+
+Vectors are written to an active in-memory segment, sealed to disk when full, and served as read-only mmap-backed segments. Search fans out across all segments and merges results.
+
+## Architecture
+
+```mermaid
+graph LR
+    A[core] --> B[store] --> C[index] --> D[segment] --> E[persistence]
+```
+
+| Layer | Contents |
+|---|---|
+| **core** | types, L2 distance (AVX2 + scalar, compile-time dispatch), SQ8 quantization |
+| **store** | VectorStore concept, InMemoryStore, MmapStore |
+| **index** | HnswIndex\<Store\>, FlatIndex\<Store\> |
+| **segment** | ActiveSegment, SealedSegment, SegmentManager |
+| **persistence** | Serializer, Loader, VEX0/HNSW/IDS binary formats |
+
+Templates live in `store/` and `index/`. Everything from `segment/` up exposes only concrete types.
+
+See [docs/PRD.md](docs/PRD.md) for the full design rationale.
+
+## Code style
+
+Code follows STL/snake_case naming convention: types in `PascalCase`, functions and variables in `snake_case`, namespace `vextor` in lowercase.
+
+## Build
+
+Requires CMake 3.20+, Ninja, and a C++20 compiler (GCC 14+ or Clang 18+). Three presets are available:
+
+| Preset | Description |
+|---|---|
+| `dev` | Debug build with ASan + UBSan |
+| `release` | Optimized build |
+| `release-python` | Optimized build + Python bindings |
+
+```bash
+cmake --preset release
+cmake --build build-release
+```
+
+Run tests and benchmarks:
+
+```bash
+ctest --test-dir build-release --output-on-failure
+./build-release/benchmarks/vextor_bench
+```
+
+### SIFT1M benchmark (optional)
+
+Requires the SIFT1M dataset (~160 MB download).
+
+```bash
+./benchmarks/sift1m/download.sh
+cmake --preset release -DVEXTOR_BUILD_SIFT1M=ON
+cmake --build build-release
+./build-release/benchmarks/sift1m/vextor_sift1m
+```
+
+Results are written to `benchmarks/sift1m/results.md`.
+
+### Python bindings (optional)
+
+Requires Python 3.8+ and NumPy.
+
+Via pip (builds a wheel using scikit-build-core):
+
+```bash
+pip install .
+python3 -c "import vextor; print('ok')"
+```
+
+On CPython ≥ 3.12 this produces an abi3 wheel that works across Python versions. Note: the wheel is built with the host compiler's AVX2 support — a wheel built on an AVX2 machine requires AVX2 at runtime.
+
+Alternatively, as part of a CMake build:
+
+```bash
+cmake --preset release-python
+cmake --build build-release-python
+PYTHONPATH=build-release-python/python python3 -c "import vextor; print('ok')"
+```
+
+## Usage
+
+### C++
+
+```cpp
+#include <vector>
+#include <vextor/vextor.h>
+
+// In-memory only
+vextor::Database db(/*dim=*/768, /*segment_capacity=*/1000000);
+
+// Insert
+std::vector<float> vec(768, 0.0f);
+db.insert(/*user_id=*/42, vec);
+
+// Search
+std::vector<float> query(768, 1.0f);
+auto results = db.search(query, /*k=*/10);
+for (const auto& r : results) {
+    // r.user_id, r.distance
+}
+
+// With persistence
+vextor::Database db2(768, 1000000, "path/to/db");
+db2.insert(42, vec);
+db2.save();
+auto loaded = vextor::Database::load("path/to/db");
+```
+
+### Python
+
+```python
+import numpy as np
+import vextor
+
+db = vextor.Database(dimensions=768, segment_capacity=1_000_000, path="path/to/db")
+
+db.insert(user_id=42, vector=np.random.randn(768).astype(np.float32))
+
+results = db.search(query=np.random.randn(768).astype(np.float32), k=10)
+for user_id, distance in results:
+    print(f"  {user_id}: {distance:.4f}")
+
+db.save()
+db2 = vextor.Database.load("path/to/db")
+```
+
+## Benchmarks
+
+Release build, single-threaded. Selected results from local runs:
+
+| | Time |
+|---|---|
+| L2 distance (scalar, 128d) | 43 ns |
+| L2 distance (AVX2, 128d) | 9 ns |
+| L2 distance (AVX2, 768d) | 80 ns |
+| FlatIndex search (10K, 128d) | 163 μs |
+| HNSW search (10K, 128d) | 39 μs |
+| HNSW search (100K, 128d) | 145 μs |
+
+HNSW is 4.2x faster than brute-force at 10K vectors. At 100K, HNSW search time grows sub-linearly (39 μs → 145 μs for 10x more vectors).
+
+### SIFT1M results
+
+1M vectors, 128d float32, single-threaded, via `SegmentManager` (capacity 1.1M, no seal during build).
+
+**Machine:** 12th Gen Intel(R) Core(TM) i7-1260P | 12 GB RAM | Linux 5.15.153.1-microsoft-standard-WSL2
+
+| M | ef_construction | ef_search | Recall@1 | Recall@10 | Recall@100 | QPS | Build (s) |
+|---|---|---|---|---|---|---|---|
+| 16 | 200 | 64 | 0.9902 | 0.9903 | 0.9478 | 3503 | 603.7 |
+| 16 | 200 | 128 | 0.9919 | 0.9941 | 0.9664 | 2810 | 603.7 |
+| 16 | 200 | 256 | 0.9939 | 0.9986 | 0.9923 | 1523 | 603.7 |
+| 32 | 400 | 128 | 0.9937 | 0.9985 | 0.9911 | 1605 | 1868.3 |
+| 32 | 400 | 256 | 0.9940 | 0.9993 | 0.9986 | 944 | 1868.3 |
+
+v0.1 gate (Recall@10 > 0.90): **PASSED** — alle 5 Configs erfüllen das Kriterium.
+
+## Project status
+
+v0.1 — MVP. Single-node, single-threaded. See [milestones](https://github.com/mariorch22/vextor/milestones) for the roadmap.
+
+## License
+
+MIT