vegamdb 0.1.0__tar.gz

vegamdb-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+/garbage/
+/.vscode/
+build/
+__pycache__/
+.cache/
+*.so
+.venv
+.clangd
+artifacts/
+docs_internal/
+
+# Ignore binary data files
+*.bin
+
+# Ignore dist
+dist/

vegamdb-0.1.0/CMakeLists.txt

@@ -0,0 +1,57 @@
+# 1. Check the CMake version
+cmake_minimum_required(VERSION 3.15)
+
+# 2. Name the project
+project(VectorDBProject)
+
+# 3. Set the C++ Standard
+# We are forcing C++17. Why? 
+# Because older C++ versions are painful. C++17 gives us modern features 
+# (like better filesystem handling) that make life easier.
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+include(FetchContent)
+
+FetchContent_Declare(
+    pybind11
+    GIT_REPOSITORY https://github.com/pybind/pybind11
+    GIT_TAG        v2.11.1
+)
+
+FetchContent_MakeAvailable(pybind11)
+
+# 4. Tell CMake where to look for "Menu" files (Headers)
+# This is equivalent to the "-I./include" flag we typed manually.
+include_directories(include)
+
+# 5. Create the Executable
+# "add_executable" tells CMake:
+# "I want to build a program named 'my_db_app'.
+#  Here are the ingredients (source files) you need to cook it."
+# add_executable(my_db_app 
+#     src/main.cpp 
+#     src/VectorDB.cpp
+# )
+
+# 1. Turn on Maximum Optimization (-O3)
+# 2. Tune for the architecture of YOUR specific computer (-march=native)
+if(MSVC)
+    add_compile_options(/O2 /arch:AVX2)
+else()
+    add_compile_options(-O3 -march=native -ffast-math)
+endif()
+
+pybind11_add_module(_vegamdb
+    src/VegamDB.cpp
+    src/bindings.cpp
+    src/indexes/FlatIndex.cpp
+    src/indexes/IVFIndex.cpp
+    src/indexes/AnnoyIndex.cpp
+    src/indexes/KMeans.cpp
+    src/storage/VectorStore.cpp
+    src/utils/Math.cpp
+)
+
+install(TARGETS _vegamdb DESTINATION vegamdb)

vegamdb-0.1.0/PKG-INFO

@@ -0,0 +1,244 @@
+Metadata-Version: 2.2
+Name: vegamdb
+Version: 0.1.0
+Summary: A high-performance vector database written in C++ with Python bindings
+Author: Naredla Ajay Kumar Reddy
+License: MIT
+Requires-Python: >=3.8
+Requires-Dist: numpy
+Description-Content-Type: text/markdown
+
+# VegamDB
+
+A high-performance vector database written in C++ with Python bindings. VegamDB provides fast nearest neighbor search with pluggable index types, zero-copy NumPy integration, and built-in persistence.
+
+## Features
+
+- **Multiple Index Types** -- Flat (exact brute-force), IVF (inverted file with K-Means), and Annoy (random projection trees)
+- **C++ Core** -- All indexing and search logic runs in optimized C++17 with `-O3` and `-march=native`
+- **Zero-Copy NumPy** -- Vectors pass directly from NumPy arrays to C++ via pointer, with no intermediate copies
+- **Persistence** -- Save and load the entire database (vectors + index) to a single binary file
+- **Pluggable Architecture** -- Switch index types at runtime without changing application code
+- **Type-Safe Python API** -- Full type stubs (`.pyi`) for IDE autocomplete and static analysis
+
+## Installation
+
+### From Source
+
+```bash
+git clone https://github.com/LuciAkirami/vegamdb.git
+cd vegamdb
+pip install .
+```
+
+### Requirements
+
+- Python >= 3.8
+- CMake >= 3.15
+- A C++17 compatible compiler (GCC 7+, Clang 5+, MSVC 2017+)
+- NumPy
+
+### Development Install
+
+For development, use an editable install so changes to Python files take effect immediately:
+
+```bash
+pip install scikit-build-core pybind11 numpy
+pip install -e . --no-build-isolation
+```
+
+## Quick Start
+
+```python
+import numpy as np
+from vegamdb import VegamDB
+
+# Create a database
+db = VegamDB()
+
+# Add vectors
+data = np.random.random((10000, 128)).astype(np.float32)
+for vec in data:
+    db.add_vector_numpy(vec)
+
+# Search (defaults to exact flat search)
+query = np.random.random(128).astype(np.float32)
+results = db.search(query, k=5)
+
+print(results.ids)        # [4823, 1092, 7744, 331, 5619]
+print(results.distances)  # [4.12, 4.15, 4.18, 4.21, 4.23]
+```
+
+## Index Types
+
+VegamDB supports three index types, each offering a different trade-off between speed and accuracy.
+
+### Flat Index (Default)
+
+Exact brute-force search. Computes the Euclidean distance between the query and every stored vector. Always returns the true nearest neighbors.
+
+```python
+db.use_flat_index()
+results = db.search(query, k=10)
+```
+
+| Metric       | Value        |
+| ------------ | ------------ |
+| Accuracy     | 100%         |
+| Build Time   | None         |
+| Best For     | Small datasets (< 50K vectors), ground truth validation |
+
+### IVF Index (Inverted File)
+
+Partitions vectors into clusters using K-Means. At query time, only the closest clusters are searched, trading some accuracy for a large speedup.
+
+```python
+db.use_ivf_index(n_clusters=100, max_iters=20, n_probe=1)
+db.build_index()
+
+# Search with custom probe count
+from vegamdb import IVFSearchParams
+params = IVFSearchParams()
+params.n_probe = 10  # Search 10 of 100 clusters
+results = db.search(query, k=10, params=params)
+```
+
+| Parameter     | Description                                      | Default |
+| ------------- | ------------------------------------------------ | ------- |
+| `n_clusters`  | Number of Voronoi cells (partitions)             | --      |
+| `max_iters`   | Maximum K-Means training iterations              | 50      |
+| `n_probe`     | Clusters to search at query time                 | 1       |
+
+### Annoy Index (Approximate Nearest Neighbors)
+
+Builds a forest of random projection trees. Each tree recursively splits the vector space with random hyperplanes. At query time, multiple trees are traversed to collect candidate neighbors.
+
+```python
+db.use_annoy_index(num_trees=10, k_leaf=50)
+db.build_index()
+
+results = db.search(query, k=10)
+```
+
+| Parameter        | Description                                    | Default |
+| ---------------- | ---------------------------------------------- | ------- |
+| `num_trees`      | Number of random projection trees              | --      |
+| `k_leaf`         | Maximum points per leaf node                   | --      |
+
+### Choosing an Index
+
+| Use Case                     | Recommended Index | Why                                   |
+| ---------------------------- | ----------------- | ------------------------------------- |
+| Small dataset (< 50K)        | Flat              | Exact results, no training overhead   |
+| Medium dataset (50K - 1M)    | IVF               | Good speed/accuracy with tunable probe|
+| Large dataset (1M+)          | Annoy             | Fast tree traversal, low memory       |
+| Ground truth / benchmarking  | Flat              | Guaranteed correct results            |
+
+## Persistence
+
+Save and load the entire database state, including vectors and the trained index:
+
+```python
+# Save
+db.save("my_database.bin")
+
+# Load into a fresh instance
+db2 = VegamDB()
+db2.load("my_database.bin")
+
+assert db2.size() == db.size()
+```
+
+The index type and its trained state are serialized automatically. After loading, the index is ready to search without rebuilding.
+
+## API Reference
+
+### VegamDB
+
+| Method                 | Description                                                       |
+| ---------------------- | ----------------------------------------------------------------- |
+| `VegamDB()`            | Create a new empty database instance                              |
+| `add_vector(vec)`      | Add a vector from a Python list of floats                         |
+| `add_vector_numpy(arr)`| Add a vector from a 1D NumPy float32 array (zero-copy)            |
+| `size()`               | Return the number of stored vectors                               |
+| `dimension()`          | Return the dimensionality of stored vectors (0 if empty)          |
+| `use_flat_index()`     | Set index to brute-force flat search                              |
+| `use_ivf_index(...)`   | Set index to IVF with specified cluster configuration             |
+| `use_annoy_index(...)` | Set index to Annoy with specified tree configuration              |
+| `build_index()`        | Explicitly build/train the current index                          |
+| `search(query, k, params=None)` | Search for k nearest neighbors, returns `SearchResults` |
+| `save(filename)`       | Save database and index to a binary file                          |
+| `load(filename)`       | Load database and index from a binary file                        |
+
+### SearchResults
+
+| Attribute    | Type          | Description                                      |
+| ------------ | ------------- | ------------------------------------------------ |
+| `ids`        | `list[int]`   | Indices of nearest neighbors (insertion order)   |
+| `distances`  | `list[float]` | Euclidean distances to the query vector           |
+
+### Search Parameters
+
+**IVFSearchParams** -- Override the default probe count for IVF search:
+- `n_probe` (int): Number of clusters to search. Higher values improve recall at the cost of latency.
+
+## Architecture
+
+```
+                        VegamDB (Orchestrator)
+                       /                      \
+               VectorStore                  IndexBase
+            (raw float vectors)           (search strategy)
+                                         /      |       \
+                                     Flat      IVF     Annoy
+                                   (exact)  (K-Means)  (trees)
+```
+
+- **VegamDB** -- Main entry point. Manages the vector store and delegates search to the active index.
+- **VectorStore** -- Stores raw vectors in a `vector<vector<float>>`. Handles serialization.
+- **IndexBase** -- Abstract interface that all index types implement (`build`, `search`, `save`, `load`).
+- **FlatIndex** -- Iterates over all vectors, computing Euclidean distance. O(n) per query.
+- **IVFIndex** -- Trains K-Means centroids, assigns vectors to clusters, searches only nearby clusters.
+- **AnnoyIndex** -- Builds a forest of binary trees using random hyperplane splits for fast traversal.
+
+## Project Structure
+
+```
+vegamdb/
+├── include/                  # C++ headers
+│   ├── VegamDB.hpp
+│   ├── indexes/              # IndexBase, FlatIndex, IVFIndex, AnnoyIndex, KMeans
+│   ├── storage/              # VectorStore
+│   └── utils/                # Math utilities (Euclidean distance, dot product)
+├── src/                      # C++ implementation
+│   ├── VegamDB.cpp
+│   ├── bindings.cpp          # pybind11 Python bindings
+│   ├── indexes/
+│   ├── storage/
+│   └── utils/
+├── vegamdb/                  # Python package
+│   ├── __init__.py           # Public API re-exports
+│   └── _vegamdb.pyi          # Type stubs for IDE support
+├── benchmarks/               # Performance benchmarks
+├── CMakeLists.txt            # C++ build configuration
+└── pyproject.toml            # Python packaging (scikit-build-core)
+```
+
+## Benchmarks
+
+Run the included benchmarks to evaluate performance on your hardware:
+
+```bash
+# Stress test (Flat index, varying dataset sizes)
+python benchmarks/stress_test.py
+
+# IVF benchmark (accuracy vs speed trade-off across probe counts)
+python benchmarks/ivf_benchmarks.py
+
+# Annoy benchmark (accuracy vs speed trade-off across tree counts)
+python benchmarks/annoy_benchmark.py
+```
+
+## License
+
+MIT

vegamdb-0.1.0/README.md

@@ -0,0 +1,234 @@
+# VegamDB
+
+A high-performance vector database written in C++ with Python bindings. VegamDB provides fast nearest neighbor search with pluggable index types, zero-copy NumPy integration, and built-in persistence.
+
+## Features
+
+- **Multiple Index Types** -- Flat (exact brute-force), IVF (inverted file with K-Means), and Annoy (random projection trees)
+- **C++ Core** -- All indexing and search logic runs in optimized C++17 with `-O3` and `-march=native`
+- **Zero-Copy NumPy** -- Vectors pass directly from NumPy arrays to C++ via pointer, with no intermediate copies
+- **Persistence** -- Save and load the entire database (vectors + index) to a single binary file
+- **Pluggable Architecture** -- Switch index types at runtime without changing application code
+- **Type-Safe Python API** -- Full type stubs (`.pyi`) for IDE autocomplete and static analysis
+
+## Installation
+
+### From Source
+
+```bash
+git clone https://github.com/LuciAkirami/vegamdb.git
+cd vegamdb
+pip install .
+```
+
+### Requirements
+
+- Python >= 3.8
+- CMake >= 3.15
+- A C++17 compatible compiler (GCC 7+, Clang 5+, MSVC 2017+)
+- NumPy
+
+### Development Install
+
+For development, use an editable install so changes to Python files take effect immediately:
+
+```bash
+pip install scikit-build-core pybind11 numpy
+pip install -e . --no-build-isolation
+```
+
+## Quick Start
+
+```python
+import numpy as np
+from vegamdb import VegamDB
+
+# Create a database
+db = VegamDB()
+
+# Add vectors
+data = np.random.random((10000, 128)).astype(np.float32)
+for vec in data:
+    db.add_vector_numpy(vec)
+
+# Search (defaults to exact flat search)
+query = np.random.random(128).astype(np.float32)
+results = db.search(query, k=5)
+
+print(results.ids)        # [4823, 1092, 7744, 331, 5619]
+print(results.distances)  # [4.12, 4.15, 4.18, 4.21, 4.23]
+```
+
+## Index Types
+
+VegamDB supports three index types, each offering a different trade-off between speed and accuracy.
+
+### Flat Index (Default)
+
+Exact brute-force search. Computes the Euclidean distance between the query and every stored vector. Always returns the true nearest neighbors.
+
+```python
+db.use_flat_index()
+results = db.search(query, k=10)
+```
+
+| Metric       | Value        |
+| ------------ | ------------ |
+| Accuracy     | 100%         |
+| Build Time   | None         |
+| Best For     | Small datasets (< 50K vectors), ground truth validation |
+
+### IVF Index (Inverted File)
+
+Partitions vectors into clusters using K-Means. At query time, only the closest clusters are searched, trading some accuracy for a large speedup.
+
+```python
+db.use_ivf_index(n_clusters=100, max_iters=20, n_probe=1)
+db.build_index()
+
+# Search with custom probe count
+from vegamdb import IVFSearchParams
+params = IVFSearchParams()
+params.n_probe = 10  # Search 10 of 100 clusters
+results = db.search(query, k=10, params=params)
+```
+
+| Parameter     | Description                                      | Default |
+| ------------- | ------------------------------------------------ | ------- |
+| `n_clusters`  | Number of Voronoi cells (partitions)             | --      |
+| `max_iters`   | Maximum K-Means training iterations              | 50      |
+| `n_probe`     | Clusters to search at query time                 | 1       |
+
+### Annoy Index (Approximate Nearest Neighbors)
+
+Builds a forest of random projection trees. Each tree recursively splits the vector space with random hyperplanes. At query time, multiple trees are traversed to collect candidate neighbors.
+
+```python
+db.use_annoy_index(num_trees=10, k_leaf=50)
+db.build_index()
+
+results = db.search(query, k=10)
+```
+
+| Parameter        | Description                                    | Default |
+| ---------------- | ---------------------------------------------- | ------- |
+| `num_trees`      | Number of random projection trees              | --      |
+| `k_leaf`         | Maximum points per leaf node                   | --      |
+
+### Choosing an Index
+
+| Use Case                     | Recommended Index | Why                                   |
+| ---------------------------- | ----------------- | ------------------------------------- |
+| Small dataset (< 50K)        | Flat              | Exact results, no training overhead   |
+| Medium dataset (50K - 1M)    | IVF               | Good speed/accuracy with tunable probe|
+| Large dataset (1M+)          | Annoy             | Fast tree traversal, low memory       |
+| Ground truth / benchmarking  | Flat              | Guaranteed correct results            |
+
+## Persistence
+
+Save and load the entire database state, including vectors and the trained index:
+
+```python
+# Save
+db.save("my_database.bin")
+
+# Load into a fresh instance
+db2 = VegamDB()
+db2.load("my_database.bin")
+
+assert db2.size() == db.size()
+```
+
+The index type and its trained state are serialized automatically. After loading, the index is ready to search without rebuilding.
+
+## API Reference
+
+### VegamDB
+
+| Method                 | Description                                                       |
+| ---------------------- | ----------------------------------------------------------------- |
+| `VegamDB()`            | Create a new empty database instance                              |
+| `add_vector(vec)`      | Add a vector from a Python list of floats                         |
+| `add_vector_numpy(arr)`| Add a vector from a 1D NumPy float32 array (zero-copy)            |
+| `size()`               | Return the number of stored vectors                               |
+| `dimension()`          | Return the dimensionality of stored vectors (0 if empty)          |
+| `use_flat_index()`     | Set index to brute-force flat search                              |
+| `use_ivf_index(...)`   | Set index to IVF with specified cluster configuration             |
+| `use_annoy_index(...)` | Set index to Annoy with specified tree configuration              |
+| `build_index()`        | Explicitly build/train the current index                          |
+| `search(query, k, params=None)` | Search for k nearest neighbors, returns `SearchResults` |
+| `save(filename)`       | Save database and index to a binary file                          |
+| `load(filename)`       | Load database and index from a binary file                        |
+
+### SearchResults
+
+| Attribute    | Type          | Description                                      |
+| ------------ | ------------- | ------------------------------------------------ |
+| `ids`        | `list[int]`   | Indices of nearest neighbors (insertion order)   |
+| `distances`  | `list[float]` | Euclidean distances to the query vector           |
+
+### Search Parameters
+
+**IVFSearchParams** -- Override the default probe count for IVF search:
+- `n_probe` (int): Number of clusters to search. Higher values improve recall at the cost of latency.
+
+## Architecture
+
+```
+                        VegamDB (Orchestrator)
+                       /                      \
+               VectorStore                  IndexBase
+            (raw float vectors)           (search strategy)
+                                         /      |       \
+                                     Flat      IVF     Annoy
+                                   (exact)  (K-Means)  (trees)
+```
+
+- **VegamDB** -- Main entry point. Manages the vector store and delegates search to the active index.
+- **VectorStore** -- Stores raw vectors in a `vector<vector<float>>`. Handles serialization.
+- **IndexBase** -- Abstract interface that all index types implement (`build`, `search`, `save`, `load`).
+- **FlatIndex** -- Iterates over all vectors, computing Euclidean distance. O(n) per query.
+- **IVFIndex** -- Trains K-Means centroids, assigns vectors to clusters, searches only nearby clusters.
+- **AnnoyIndex** -- Builds a forest of binary trees using random hyperplane splits for fast traversal.
+
+## Project Structure
+
+```
+vegamdb/
+├── include/                  # C++ headers
+│   ├── VegamDB.hpp
+│   ├── indexes/              # IndexBase, FlatIndex, IVFIndex, AnnoyIndex, KMeans
+│   ├── storage/              # VectorStore
+│   └── utils/                # Math utilities (Euclidean distance, dot product)
+├── src/                      # C++ implementation
+│   ├── VegamDB.cpp
+│   ├── bindings.cpp          # pybind11 Python bindings
+│   ├── indexes/
+│   ├── storage/
+│   └── utils/
+├── vegamdb/                  # Python package
+│   ├── __init__.py           # Public API re-exports
+│   └── _vegamdb.pyi          # Type stubs for IDE support
+├── benchmarks/               # Performance benchmarks
+├── CMakeLists.txt            # C++ build configuration
+└── pyproject.toml            # Python packaging (scikit-build-core)
+```
+
+## Benchmarks
+
+Run the included benchmarks to evaluate performance on your hardware:
+
+```bash
+# Stress test (Flat index, varying dataset sizes)
+python benchmarks/stress_test.py
+
+# IVF benchmark (accuracy vs speed trade-off across probe counts)
+python benchmarks/ivf_benchmarks.py
+
+# Annoy benchmark (accuracy vs speed trade-off across tree counts)
+python benchmarks/annoy_benchmark.py
+```
+
+## License
+
+MIT

vegamdb-0.1.0/benchmarks/annoy_benchmark.py

@@ -0,0 +1,121 @@
+import numpy as np
+import vegamdb
+import time
+
+
+def calculate_recall(ground_truth, predicted, k):
+    """
+    Calculates the intersection between Ground Truth and Predicted results.
+    Recall = (Number of Overlapping Indices) / K
+    """
+    total_recall = 0
+
+    for gt, pred in zip(ground_truth, predicted):
+        gt_set = set(gt)
+        pred_set = set(pred)
+        intersection_count = len(gt_set.intersection(pred_set))
+        total_recall += intersection_count / k
+
+    return (total_recall / len(ground_truth)) * 100
+
+
+def generate_clustered_data(n_vectors, dim, n_clusters_real=100):
+    print(f"Generating {n_vectors} CLUSTERED vectors...")
+    topic_centers = np.random.random((n_clusters_real, dim)).astype(np.float32)
+    data = []
+    vectors_per_cluster = n_vectors // n_clusters_real
+
+    for i in range(n_clusters_real):
+        center = topic_centers[i]
+        noise = np.random.normal(scale=0.1, size=(vectors_per_cluster, dim))
+        cluster_points = center + noise
+        data.append(cluster_points)
+
+    data = np.vstack(data).astype(np.float32)
+    np.random.shuffle(data)
+    return data
+
+
+def benchmark():
+    # ---------------------------------------------------------
+    # CONFIGURATION
+    # ---------------------------------------------------------
+    N_VECTORS = 100000
+    DIM = 128
+    K = 10
+    N_QUERIES = 100
+
+    # Annoy Param: Max items in a leaf before splitting
+    K_LEAF = 50
+
+    # We will test different Forest Sizes (Number of Trees)
+    # More Trees = Better Chance of finding neighbors = Slower Search
+    TREE_COUNTS = [1, 2, 5, 10, 50, 100, 200, 500]
+
+    # ---------------------------------------------------------
+    # 1. DATA GENERATION
+    # ---------------------------------------------------------
+    print(f"Generating {N_VECTORS} vectors (Dim: {DIM})...")
+    data = generate_clustered_data(N_VECTORS, DIM, n_clusters_real=500)
+    queries = generate_clustered_data(N_QUERIES, DIM, n_clusters_real=10)
+
+    # Use VegamDB to hold data and calculate ground truth
+    db = vegamdb.VegamDB()
+    print("Ingesting data into VegamDB...")
+    for i in range(N_VECTORS):
+        db.add_vector_numpy(data[i])
+
+    # ---------------------------------------------------------
+    # 2. ESTABLISH GROUND TRUTH (Brute Force)
+    # ---------------------------------------------------------
+    print(f"Calculating Ground Truth for {N_QUERIES} queries...")
+    start_flat = time.time()
+    ground_truth_results = []
+    for i in range(N_QUERIES):
+        res = db.search(queries[i], K)
+        ground_truth_results.append(res.ids)
+
+    time_flat = time.time() - start_flat
+    avg_flat_ms = (time_flat / N_QUERIES) * 1000
+    print(f"Avg Flat Search Latency: {avg_flat_ms:.4f} ms")
+
+    # ---------------------------------------------------------
+    # 3. ANNOY BENCHMARK LOOP
+    # ---------------------------------------------------------
+    print("\n--- ANNOY PERFORMANCE (Varying Tree Count) ---")
+    header = f"{'TREES':<6} | {'BUILD TIME':<10} | {'AVG LATENCY (ms)':<18} | {'SPEEDUP':<10} | {'ACCURACY (%)':<12}"
+    print("-" * len(header))
+    print(header)
+    print("-" * len(header))
+
+    # To test different tree counts, we must rebuild the index each time.
+    for n_trees in TREE_COUNTS:
+
+        # A. Build Index — use VegamDB's factory method
+        t0 = time.time()
+        db.use_annoy_index(num_trees=n_trees, k_leaf=K_LEAF)
+        db.build_index()
+        build_time = time.time() - t0
+
+        # B. Run Search
+        start_probe = time.time()
+        annoy_results = []
+        for i in range(N_QUERIES):
+            res = db.search(queries[i], K)
+            annoy_results.append(res.ids)
+
+        # C. Metrics
+        total_time = time.time() - start_probe
+        avg_time_ms = (total_time / N_QUERIES) * 1000
+        speedup = avg_flat_ms / avg_time_ms
+        accuracy = calculate_recall(ground_truth_results, annoy_results, K)
+
+        print(
+            f"{n_trees:<6} | {build_time:<10.2f} | {avg_time_ms:<18.4f} | {speedup:<10.1f} | {accuracy:<12.1f}"
+        )
+
+    print("-" * len(header))
+
+
+if __name__ == "__main__":
+    benchmark()