vector-vault-db 0.1.1__tar.gz

vector_vault_db-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cpp:
+    name: C++ core + tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install toolchain
+        run: sudo apt-get update && sudo apt-get install -y g++ cmake ninja-build
+
+      - name: Configure
+        run: >
+          cmake -S . -B build -G Ninja
+          -DCMAKE_BUILD_TYPE=Release
+          -DVECTORVAULT_BUILD_TESTS=ON
+          -DVECTORVAULT_BUILD_PYTHON=OFF
+
+      - name: Build
+        run: cmake --build build
+
+      - name: Test
+        run: ctest --test-dir build --output-on-failure
+
+  python:
+    name: Python binding + tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install toolchain
+        run: sudo apt-get update && sudo apt-get install -y g++ cmake ninja-build
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and test dependencies
+        run: pip install -e ".[test]"
+
+      - name: Test
+        run: pytest -q

vector_vault_db-0.1.1/.github/workflows/wheels.yml

@@ -0,0 +1,64 @@
+name: Wheels
+
+on:
+  workflow_dispatch:
+  push:
+    tags: ["v*"]
+
+jobs:
+  wheels:
+    name: Wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.21
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  sdist:
+    name: Source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: Publish to PyPI
+    needs: [wheels, sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Download sdist
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1

vector_vault_db-0.1.1/.gitignore

@@ -0,0 +1,40 @@
+# Build output
+/build/
+/_skbuild/
+/dist/
+*.egg-info/
+/wheelhouse/
+
+# CMake / FetchContent caches
+CMakeCache.txt
+CMakeFiles/
+cmake_install.cmake
+CTestTestfile.cmake
+Testing/
+_deps/
+
+# Compiled artifacts
+*.o
+*.obj
+*.a
+*.lib
+*.so
+*.dylib
+*.pyd
+*.dll
+
+# Python
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.hypothesis/
+.venv/
+venv/
+
+# Editor/OS
+.vscode/
+.idea/
+.DS_Store
+
+# Tooling
+.kiro/

vector_vault_db-0.1.1/CMakeLists.txt

@@ -0,0 +1,79 @@
+cmake_minimum_required(VERSION 3.20)
+
+project(VectorVaultDB
+    VERSION 0.1.1
+    DESCRIPTION "High-performance vector database with a C++ core and Python bindings"
+    LANGUAGES CXX)
+
+# ---------------------------------------------------------------------------
+# Global settings
+# ---------------------------------------------------------------------------
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+endif()
+
+# Build options. The Python module is built by default (this is how
+# scikit-build-core drives the build). C++ tests are opt-in so that wheel
+# builds do not need to fetch the test dependencies.
+option(VECTORVAULT_BUILD_PYTHON "Build the pybind11 Python extension" ON)
+option(VECTORVAULT_BUILD_TESTS  "Build the C++ test harness (Catch2 + RapidCheck)" OFF)
+
+# ---------------------------------------------------------------------------
+# Core engine library (C++17)
+# ---------------------------------------------------------------------------
+add_library(vectorvault_core STATIC
+    src/core/version.cpp
+    src/core/error.cpp
+    src/core/memory_allocator.cpp
+    src/core/distance.cpp
+    src/core/collection.cpp
+    src/core/hnsw_index.cpp
+    src/core/ivf_index.cpp
+    src/core/crc64.cpp
+    src/core/persistence.cpp
+    src/core/mmap_region.cpp
+    src/core/engine.cpp)
+
+target_include_directories(vectorvault_core
+    PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
+
+set_target_properties(vectorvault_core PROPERTIES
+    POSITION_INDEPENDENT_CODE ON)
+
+# ---------------------------------------------------------------------------
+# Python extension (pybind11)
+# ---------------------------------------------------------------------------
+if(VECTORVAULT_BUILD_PYTHON)
+    find_package(pybind11 CONFIG REQUIRED)
+
+    pybind11_add_module(_vectorvault src/python/module.cpp)
+    target_link_libraries(_vectorvault PRIVATE vectorvault_core)
+
+    # On MinGW/MSYS2 the extension would otherwise depend on libstdc++,
+    # libgcc, and libwinpthread DLLs from the toolchain's bin directory.
+    # Link them statically so the .pyd is self-contained and importable by a
+    # stock CPython interpreter without the toolchain on PATH.
+    if(MINGW)
+        target_link_options(_vectorvault PRIVATE
+            -static-libgcc -static-libstdc++
+            -Wl,-Bstatic,--whole-archive -lwinpthread
+            -Wl,--no-whole-archive)
+    endif()
+
+    # Install the extension into the vectorvault package (scikit-build-core
+    # collects this together with the pure-Python package files).
+    install(TARGETS _vectorvault DESTINATION vectorvault)
+endif()
+
+# ---------------------------------------------------------------------------
+# C++ test harness
+# ---------------------------------------------------------------------------
+if(VECTORVAULT_BUILD_TESTS)
+    enable_testing()
+    add_subdirectory(tests/cpp)
+endif()

vector_vault_db-0.1.1/PKG-INFO

@@ -0,0 +1,167 @@
+Metadata-Version: 2.2
+Name: vector-vault-db
+Version: 0.1.1
+Summary: High-performance vector database with a C++ core and Python bindings
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.21
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: hypothesis>=6.0; extra == "test"
+Requires-Dist: numpy>=1.21; extra == "test"
+Description-Content-Type: text/markdown
+
+# Vector-Vault-DB
+
+A high-performance vector database written from scratch. The engine is C++17;
+the public interface is a Python extension built with pybind11. There is no
+dependency on an existing vector store — the index, distance kernels, allocator,
+and on-disk format are all implemented directly.
+
+## Features
+
+- **Approximate nearest-neighbor search** over float32 vectors using either an
+  HNSW graph or an IVF (inverted-file) index.
+- **SIMD-accelerated distance kernels** (AVX-512) for Euclidean, cosine, and dot
+  product, with a scalar fallback selected once at startup via CPU detection.
+- **Custom arena allocator** that hands out 64-byte aligned blocks for vector
+  storage and tracks per-collection usage.
+- **Memory-mapped persistence**: a versioned, CRC-checked binary snapshot format.
+  Saves are atomic (temp file + fsync + rename); loads map the vector region
+  instead of copying it onto the heap.
+- **Python bindings** with NumPy support and an exception hierarchy that mirrors
+  the engine's error categories.
+
+## Documentation
+
+Full documentation lives in [`docs/`](docs/index.md):
+
+- [Installation](docs/installation.md)
+- [Quickstart](docs/quickstart.md)
+- [Concepts](docs/concepts.md) — metrics, indexes, tuning, persistence
+- [Python API reference](docs/python-api.md)
+- [C++ API guide](docs/cpp-api.md)
+
+## Architecture
+
+```
+Python (vectorvault)            pybind11 extension, NumPy interop, exceptions
+        │
+        ▼
+Engine                          collection registry, shared allocator + persistence
+        │
+        ▼
+Collection                      record store, validation, query orchestration
+   ├── Index            HNSW / IVF — graph + inverted-file ANN
+   ├── DistanceCalculator   AVX-512 kernels with scalar fallback
+   ├── MemoryAllocator      64-byte aligned arena, per-collection accounting
+   └── PersistenceManager   atomic save, mmap-backed load, CRC validation
+```
+
+A collection is guarded by a single readers-writer lock: reads (get, query) take
+a shared lock, mutations (insert, delete, build, save) take an exclusive lock, so
+index membership stays consistent with the record store.
+
+## Requirements
+
+- A C++17 compiler (GCC, Clang, or MSVC)
+- CMake >= 3.20
+- Python >= 3.9 with NumPy (for the bindings)
+- pybind11 (resolved by the build backend)
+
+Catch2, RapidCheck, and Hypothesis are fetched automatically for the test builds.
+
+## Install
+
+```bash
+pip install -e ".[test]"
+```
+
+This builds the native extension through scikit-build-core and installs the
+`vectorvault` package in editable mode.
+
+## Quickstart
+
+```python
+import vectorvault as vv
+
+engine = vv.Engine()
+coll = engine.create_collection("documents", dim=128, metric="cosine")
+
+coll.insert("doc-1", embedding_a, metadata={"title": "intro"})
+coll.insert("doc-2", embedding_b)
+
+coll.build_index("hnsw", m=16, ef_construction=200)
+
+results = coll.query(query_vector, k=10, ef_search=64)
+for record_id, distance in results:
+    print(record_id, distance)
+
+engine.save("documents", "documents.vv")
+restored = vv.Engine().load("documents.vv")
+```
+
+Metrics: `"euclidean"` (`"l2"`), `"cosine"`, `"dot"` (`"dot_product"`).
+Index types: `"hnsw"`, `"ivf"`.
+
+Errors surface as typed exceptions under `vectorvault.VectorVaultError`; several
+also derive from a builtin (for example `NotFoundError` is a `KeyError` and
+`SnapshotNotFoundError` is a `FileNotFoundError`).
+
+## On-disk format
+
+A snapshot is a single file: a fixed header (magic, version, dimensionality,
+metric, index type, region offsets, CRC-64 of the content), the collection name,
+a record directory sorted by id with optional metadata, a 64-byte aligned vector
+region, and the serialized index. Records, metadata keys, and index nodes are
+written in a canonical order, so saving a collection, loading it, and saving
+again produces a byte-identical file. Loads validate existence, magic, version,
+and checksum before mapping the vector region.
+
+## Testing
+
+C++ (Catch2 + RapidCheck):
+
+```bash
+cmake -S . -B build -DVECTORVAULT_BUILD_TESTS=ON -DVECTORVAULT_BUILD_PYTHON=OFF
+cmake --build build
+ctest --test-dir build --output-on-failure
+```
+
+Python (pytest + Hypothesis):
+
+```bash
+pytest
+```
+
+Correctness-critical behavior — distance accuracy against a reference, allocator
+accounting, query ordering, index membership, and the snapshot round-trip — is
+covered by property-based tests. A recall benchmark builds an index over 10,000
+vectors and asserts mean recall@10 against an exact baseline.
+
+## Layout
+
+```
+include/vectorvault/   Public C++ headers
+src/core/              Core engine implementation
+src/python/            pybind11 module + vectorvault package
+tests/cpp/             C++ tests (Catch2 + RapidCheck)
+tests/python/          Python tests (pytest + Hypothesis)
+```
+
+## Notes and limitations
+
+- The HNSW index selects neighbours with the diversity heuristic from Malkov &
+  Yashunin (SELECT-NEIGHBORS-HEURISTIC), applied both when linking a new node and
+  when pruning an existing node whose adjacency list overflows. On uniform random
+  data this measures mean recall@10 of ~0.96 at dim 32 and ~0.85 at dim 64
+  (m=16, ef_construction=200, ef_search=50); recall still tapers on harder,
+  higher-dimensional distributions.
+- The snapshot reader assumes a little-endian host (it reads float components
+  directly from the mapping).
+- AVX-512 is detected at runtime; on hosts without it the scalar kernels are
+  used, which are also the accuracy reference.
+
+## License
+
+MIT

vector_vault_db-0.1.1/README.md

@@ -0,0 +1,154 @@
+# Vector-Vault-DB
+
+A high-performance vector database written from scratch. The engine is C++17;
+the public interface is a Python extension built with pybind11. There is no
+dependency on an existing vector store — the index, distance kernels, allocator,
+and on-disk format are all implemented directly.
+
+## Features
+
+- **Approximate nearest-neighbor search** over float32 vectors using either an
+  HNSW graph or an IVF (inverted-file) index.
+- **SIMD-accelerated distance kernels** (AVX-512) for Euclidean, cosine, and dot
+  product, with a scalar fallback selected once at startup via CPU detection.
+- **Custom arena allocator** that hands out 64-byte aligned blocks for vector
+  storage and tracks per-collection usage.
+- **Memory-mapped persistence**: a versioned, CRC-checked binary snapshot format.
+  Saves are atomic (temp file + fsync + rename); loads map the vector region
+  instead of copying it onto the heap.
+- **Python bindings** with NumPy support and an exception hierarchy that mirrors
+  the engine's error categories.
+
+## Documentation
+
+Full documentation lives in [`docs/`](docs/index.md):
+
+- [Installation](docs/installation.md)
+- [Quickstart](docs/quickstart.md)
+- [Concepts](docs/concepts.md) — metrics, indexes, tuning, persistence
+- [Python API reference](docs/python-api.md)
+- [C++ API guide](docs/cpp-api.md)
+
+## Architecture
+
+```
+Python (vectorvault)            pybind11 extension, NumPy interop, exceptions
+        │
+        ▼
+Engine                          collection registry, shared allocator + persistence
+        │
+        ▼
+Collection                      record store, validation, query orchestration
+   ├── Index            HNSW / IVF — graph + inverted-file ANN
+   ├── DistanceCalculator   AVX-512 kernels with scalar fallback
+   ├── MemoryAllocator      64-byte aligned arena, per-collection accounting
+   └── PersistenceManager   atomic save, mmap-backed load, CRC validation
+```
+
+A collection is guarded by a single readers-writer lock: reads (get, query) take
+a shared lock, mutations (insert, delete, build, save) take an exclusive lock, so
+index membership stays consistent with the record store.
+
+## Requirements
+
+- A C++17 compiler (GCC, Clang, or MSVC)
+- CMake >= 3.20
+- Python >= 3.9 with NumPy (for the bindings)
+- pybind11 (resolved by the build backend)
+
+Catch2, RapidCheck, and Hypothesis are fetched automatically for the test builds.
+
+## Install
+
+```bash
+pip install -e ".[test]"
+```
+
+This builds the native extension through scikit-build-core and installs the
+`vectorvault` package in editable mode.
+
+## Quickstart
+
+```python
+import vectorvault as vv
+
+engine = vv.Engine()
+coll = engine.create_collection("documents", dim=128, metric="cosine")
+
+coll.insert("doc-1", embedding_a, metadata={"title": "intro"})
+coll.insert("doc-2", embedding_b)
+
+coll.build_index("hnsw", m=16, ef_construction=200)
+
+results = coll.query(query_vector, k=10, ef_search=64)
+for record_id, distance in results:
+    print(record_id, distance)
+
+engine.save("documents", "documents.vv")
+restored = vv.Engine().load("documents.vv")
+```
+
+Metrics: `"euclidean"` (`"l2"`), `"cosine"`, `"dot"` (`"dot_product"`).
+Index types: `"hnsw"`, `"ivf"`.
+
+Errors surface as typed exceptions under `vectorvault.VectorVaultError`; several
+also derive from a builtin (for example `NotFoundError` is a `KeyError` and
+`SnapshotNotFoundError` is a `FileNotFoundError`).
+
+## On-disk format
+
+A snapshot is a single file: a fixed header (magic, version, dimensionality,
+metric, index type, region offsets, CRC-64 of the content), the collection name,
+a record directory sorted by id with optional metadata, a 64-byte aligned vector
+region, and the serialized index. Records, metadata keys, and index nodes are
+written in a canonical order, so saving a collection, loading it, and saving
+again produces a byte-identical file. Loads validate existence, magic, version,
+and checksum before mapping the vector region.
+
+## Testing
+
+C++ (Catch2 + RapidCheck):
+
+```bash
+cmake -S . -B build -DVECTORVAULT_BUILD_TESTS=ON -DVECTORVAULT_BUILD_PYTHON=OFF
+cmake --build build
+ctest --test-dir build --output-on-failure
+```
+
+Python (pytest + Hypothesis):
+
+```bash
+pytest
+```
+
+Correctness-critical behavior — distance accuracy against a reference, allocator
+accounting, query ordering, index membership, and the snapshot round-trip — is
+covered by property-based tests. A recall benchmark builds an index over 10,000
+vectors and asserts mean recall@10 against an exact baseline.
+
+## Layout
+
+```
+include/vectorvault/   Public C++ headers
+src/core/              Core engine implementation
+src/python/            pybind11 module + vectorvault package
+tests/cpp/             C++ tests (Catch2 + RapidCheck)
+tests/python/          Python tests (pytest + Hypothesis)
+```
+
+## Notes and limitations
+
+- The HNSW index selects neighbours with the diversity heuristic from Malkov &
+  Yashunin (SELECT-NEIGHBORS-HEURISTIC), applied both when linking a new node and
+  when pruning an existing node whose adjacency list overflows. On uniform random
+  data this measures mean recall@10 of ~0.96 at dim 32 and ~0.85 at dim 64
+  (m=16, ef_construction=200, ef_search=50); recall still tapers on harder,
+  higher-dimensional distributions.
+- The snapshot reader assumes a little-endian host (it reads float components
+  directly from the mapping).
+- AVX-512 is detected at runtime; on hosts without it the scalar kernels are
+  used, which are also the accuracy reference.
+
+## License
+
+MIT

vector_vault_db-0.1.1/docs/concepts.md

@@ -0,0 +1,115 @@
+# Concepts
+
+## Collections
+
+A collection is a named container for vectors that share a fixed
+**dimensionality** and a single **distance metric**. Both are set at creation
+and immutable thereafter. Each record is a unique string id, a float32 vector,
+and optional metadata.
+
+Constraints enforced at the boundary:
+
+- Collection name: 1–255 characters, unique within an engine.
+- Dimensionality: an integer in `[1, 65536]`.
+- Record id: non-empty string.
+- Vector: length equals the collection dimensionality; all components finite
+  (no NaN or infinity).
+- Metadata: a map of string keys to str / int / float / bool values.
+
+## Distance metrics
+
+| Metric | String | Definition | Ordering |
+|---|---|---|---|
+| Euclidean (L2) | `"euclidean"`, `"l2"` | √Σ(aᵢ−bᵢ)² | nearest = smallest |
+| Cosine | `"cosine"` | 1 − (a·b)/(‖a‖‖b‖), in [0, 2] | nearest = smallest |
+| Dot product | `"dot"`, `"dot_product"` | Σ aᵢbᵢ | see note below |
+
+Query results are always ordered by **ascending metric value**, with ties
+broken by ascending id for determinism.
+
+- For **Euclidean** and **Cosine**, a smaller value means more similar, so
+  ascending order returns the nearest neighbors first — the expected behavior.
+- For **Dot product**, the raw dot product is treated as the distance and sorted
+  ascending, so the largest dot products come *last*. If you want
+  maximum-inner-product semantics (largest dot first), use cosine on normalized
+  vectors, or reverse the result list yourself.
+
+Cosine distance is undefined for a zero-magnitude vector; computing it raises
+`UndefinedDistanceError`.
+
+## Indexes
+
+An index makes nearest-neighbor search sublinear. With no index, queries fall
+back to an exact brute-force scan, which is correct but O(n) per query.
+
+### HNSW (Hierarchical Navigable Small World)
+
+A layered proximity graph. Search descends through sparse upper layers to a good
+entry point, then explores the base layer. Neighbor selection uses a diversity
+heuristic so the graph stays navigable.
+
+Build parameters:
+
+- `m` (default 16): max neighbors per node on upper layers (base layer keeps
+  `2*m`). Higher `m` improves recall and memory use. Must be ≥ 2.
+- `ef_construction` (default 200): build-time search breadth. Higher values build
+  a better graph more slowly. Must be ≥ 1.
+
+Query parameter:
+
+- `ef_search` (default 50): search breadth. Higher values trade speed for recall.
+  Must be ≥ 1.
+
+### IVF (Inverted File)
+
+Partitions vectors into `nlist` cells via k-means. A query scans only the
+`nprobe` cells nearest to it.
+
+Build parameter:
+
+- `nlist` (default 0 = auto, ≈ √n): number of partitions. Must be in
+  `[1, record_count]`.
+
+Query parameter:
+
+- `nprobe` (default 1): partitions scanned per query. Higher values trade speed
+  for recall. Must be in `[1, nlist]`.
+
+### Choosing an index
+
+- **HNSW** generally gives higher recall at a given speed and is a good default.
+- **IVF** has lower memory overhead and a smaller build cost; recall is tuned
+  primarily through `nprobe`.
+
+Deletions are handled by tombstoning: a removed record is excluded from results
+immediately and the surrounding structure is preserved.
+
+## Persistence
+
+`engine.save(name, path)` writes a single binary snapshot containing the
+collection metadata, all records (with metadata), and the index. The format is:
+
+- A fixed header: magic, format version, dimensionality, metric, index type,
+  region offsets, and a CRC-64 of the content.
+- The collection name.
+- A record directory sorted by id, each with optional metadata.
+- A 64-byte aligned vector region.
+- The serialized index (if one was built).
+
+Records, metadata keys, and index nodes are written in a canonical order, so
+saving a collection, loading it, and saving again produces a **byte-identical**
+file.
+
+`engine.load(path)` validates the file in order — existence and size, magic and
+header, format version, then content checksum — before mapping the vector region
+into memory. Vector data is read through the mapping rather than copied onto the
+heap. Failures raise `SnapshotNotFoundError`, `UnsupportedVersionError`, or
+`CorruptionError`.
+
+## Performance notes
+
+- Distance kernels use AVX-512 when the CPU supports it and fall back to scalar
+  code otherwise; both accumulate in double precision for accuracy.
+- Vector storage is 64-byte aligned to suit SIMD access.
+- A collection is guarded by a readers-writer lock: concurrent reads are allowed;
+  mutations are serialized.