pylattice-db 0.1.2__tar.gz

pylattice_db-0.1.2/.github/workflows/ci.yml

@@ -0,0 +1,143 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-and-test:
+    name: Build and test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake build-essential
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install pybind11
+        run: pip install pybind11
+
+      - name: Configure
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLATTICE_BUILD_TESTS=ON \
+            -Dpybind11_DIR=$(python -m pybind11 --cmakedir)
+
+      - name: Build
+        run: cmake --build build -j$(nproc)
+
+      - name: Run tests
+        working-directory: build
+        run: ctest --output-on-failure
+
+  benchmark:
+    name: Benchmark regression gate
+    runs-on: ubuntu-latest
+    needs: build-and-test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake build-essential
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Python dependencies
+        run: pip install pybind11 numpy
+
+      - name: Configure and build
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -Dpybind11_DIR=$(python -m pybind11 --cmakedir)
+          cmake --build build -j$(nproc)
+
+      - name: Cache SIFT small dataset
+        id: cache-sift
+        uses: actions/cache@v4
+        with:
+          path: bench/data/siftsmall
+          key: siftsmall-v1
+
+      - name: Download SIFT small
+        if: steps.cache-sift.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p bench/data
+          cd bench/data
+          curl -O ftp://ftp.irisa.fr/local/texmex/corpus/siftsmall.tar.gz
+          tar -xzf siftsmall.tar.gz
+
+      - name: Run benchmark
+        working-directory: bench
+        env:
+          LATTICE_BUILD_DIR: build
+        run: python run.py --skip qdrant,chroma --json current.json
+
+      - name: Check for regression
+        working-directory: bench
+        run: python check_regression.py --current current.json
+
+  coverage:
+    name: Test coverage
+    runs-on: ubuntu-latest
+    needs: build-and-test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake build-essential lcov
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install pybind11
+        run: pip install pybind11
+
+      - name: Configure with coverage
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DLATTICE_BUILD_TESTS=ON \
+            -DLATTICE_COVERAGE=ON \
+            -Dpybind11_DIR=$(python -m pybind11 --cmakedir)
+
+      - name: Build and test
+        run: |
+          cmake --build build -j$(nproc)
+          cd build && ctest --output-on-failure
+
+      - name: Generate coverage report
+        run: |
+          lcov --capture --directory build --output-file coverage.info \
+            --ignore-errors mismatch,gcov
+          lcov --remove coverage.info '/usr/*' '*/_deps/*' '*/tests/*' \
+            --output-file coverage.info
+          lcov --list coverage.info
+
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: coverage.info
+          fail_ci_if_error: false

pylattice_db-0.1.2/.gitignore

@@ -0,0 +1,38 @@
+# build output
+# build output
+build/
+build-tsan/
+build-test/
+build-cov/
+*.o
+*.a
+*.so
+*.dylib
+
+# editor / OS
+.vscode/settings.json
+.vscode/ipch/
+.DS_Store
+
+# python
+__pycache__/
+*.pyc
+.venv/
+venv/
+.venv-tools/
+bench/.venv/
+
+
+# secrets
+.env
+
+# test scratch files
+*.wal
+
+notes.md
+
+# benchmark datasets - large, downloadable, not ours to redistribute
+bench/data/
+
+app/.venv/
+app/data/

pylattice_db-0.1.2/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.20)
+project(lattice CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Debug)
+endif()
+
+enable_testing()
+
+add_subdirectory(core)
+add_subdirectory(cli)
+add_subdirectory(bindings)

pylattice_db-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Aman Karki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pylattice_db-0.1.2/PKG-INFO

@@ -0,0 +1,281 @@
+Metadata-Version: 2.2
+Name: pylattice-db
+Version: 0.1.2
+Summary: An embedded vector database with a from-scratch HNSW index
+Keywords: vector-database,hnsw,similarity-search,embeddings
+Author: Aman Karki
+License: MIT License
+         
+         Copyright (c) 2026 Aman Karki
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database :: Database Engines/Servers
+Project-URL: Homepage, https://github.com/amankarki151/lattice
+Project-URL: Repository, https://github.com/amankarki151/lattice
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# Lattice
+
+[![CI](https://github.com/amankarki151/lattice/actions/workflows/ci.yml/badge.svg)](https://github.com/amankarki151/lattice/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/lattice-db.svg)](https://pypi.org/project/lattice-db/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+An embedded vector database, written from scratch in C++ — a
+hand-implemented HNSW index, a disk-backed storage engine with
+write-ahead logging, scalar quantization, and a concurrent query path.
+
+Most vector databases are services you run. Lattice is a library you
+link against — closer to SQLite than to Postgres. Single node, no
+cluster, no network hop unless you want one.
+
+On SIFT (10,000 vectors, 128 dimensions, k=10), Lattice answers
+queries in **583µs at 95.4% recall** — about 3.5x faster than Qdrant's
+in-memory mode on the same data. Full numbers, including where it
+loses, are below.
+
+## Contents
+
+- [Installing](#installing)
+- [Usage](#usage)
+- [Benchmarks](#benchmarks)
+- [Document assistant](#document-assistant)
+- [Building from source](#building-from-source)
+- [Repository layout](#repository-layout)
+- [Status](#status)
+- [Known limitations](#known-limitations)
+- [Writing](#writing)
+- [License](#license)
+
+## Installing
+
+```bash
+pip install lattice-db
+```
+
+Or [build from source](#building-from-source) if you want the CLI,
+the test suite, or you're on a platform without a prebuilt wheel.
+
+## Usage
+
+### Python
+
+```python
+import lattice
+
+db = lattice.Database("/path/to/db")
+db.insert(lattice.Vector(1, [1.0, 0.0, 0.0]))
+db.insert(lattice.Vector(2, [0.0, 1.0, 0.0]))
+
+hits = db.search([1.0, 0.0, 0.0], k=5)
+for h in hits:
+    print(h.id, h.distance)
+```
+
+### Command line (built from source)
+
+```bash
+lattice /path/to/db insert 1 1.0,0.0,0.0
+lattice /path/to/db query 1.0,0.0,0.0 5
+lattice /path/to/db checkpoint
+lattice /path/to/db stats
+```
+
+### HTTP
+
+```bash
+cd server
+python3 -m venv .venv && source .venv/bin/activate
+pip install -r requirements.txt
+uvicorn main:app --port 8000
+```
+
+Interactive API docs at `http://localhost:8000/docs`.
+
+## Benchmarks
+
+SIFT (10,000 base vectors, 128 dimensions, k=10), against Qdrant
+(in-memory mode) and Chroma. Ground truth is SIFT's own precomputed
+neighbour list, not derived from any of the systems being compared.
+
+| system | ef | build (s) | recall | p50 (µs) | p99 (µs) |
+|---|---|---|---|---|---|
+| Lattice | 10 | 18.5 | 0.8980 | 237 | 407 |
+| Lattice | 50 | 18.5 | 0.9540 | 583 | 799 |
+| Lattice | 100 | 19.0 | 0.9590 | 934 | 1234 |
+| Qdrant (in-memory) | — | 0.6 | 1.0000 | 2094 | 2621 |
+| Chroma | — | 0.4 | 0.9970 | 311 | 361 |
+
+At every ef tested, Lattice beats Qdrant's query latency by 2x or
+more while closing most of the recall gap. Chroma is faster on raw
+query latency in this run — that's a real result, not omitted.
+
+Where Lattice loses: build time, by a wide margin (18–19s vs under a
+second for both competitors). The gap is partly architectural —
+Lattice inserts one vector at a time through an exclusive lock, with
+no batch-insert path yet — and partly a benchmark-harness limitation,
+since Qdrant and Chroma are both inserted here via their bulk APIs.
+
+### SIFT1M (1,000,000 base vectors, 128 dimensions, k=10, ef=50)
+
+| system | build (s) | recall | p50 (µs) | p99 (µs) |
+|---|---|---|---|---|
+| Lattice | 3954.1 | 0.8831 | 1325 | 2045 |
+| Qdrant (in-memory)* | 333.7 | 0.9993 | 232946 | 1728312 |
+| Chroma | 105.1 | 0.9775 | 471 | 577 |
+
+\* Qdrant's own client warned before this run started: *"Local mode
+is not recommended for collections with more than 20,000 points."*
+A p99 of 1.7 seconds confirms it — this reflects local mode's storage
+backend at 50x its recommended limit, not Qdrant's real capability.
+Measuring Qdrant fairly at this scale would require Docker or Cloud
+mode. This row is included for completeness, not as a fair
+comparison.
+
+Chroma's numbers here are a legitimate comparison point and it holds
+up well at this scale. Lattice's recall dropped from 0.9540 (at 10k
+vectors) to 0.8831 (at 1M vectors) at the same ef — consistent with
+more vectors meaning more true neighbours competing for the same
+top-k slots; a higher ef would recover recall at this size. The
+build-time gap seen at 10k also holds at 1M scale — Lattice takes
+3954s against Chroma's 105s — confirming this is a systemic
+architectural cost, not an artifact of the smaller dataset.
+
+This is not a like-for-like comparison. Qdrant and Chroma are full
+services with networking, persistence policies, filtering, and
+metadata support. Lattice is an embedded library. Some of their
+latency is buying features Lattice doesn't have. Qdrant runs in
+`:memory:` mode here specifically to remove the network hop, as the
+closest available approximation to comparing against a library.
+
+Full methodology and raw output in [bench/results.md](bench/results.md).
+
+## Document assistant
+
+A local-first assistant built on top of the database: ingest notes or
+PDFs, ask questions in plain language, get answers with citations.
+Everything — embedding, retrieval, re-ranking, and answer generation
+— runs on-device. No API keys, no network calls.
+
+```bash
+cd app
+python3.12 -m venv .venv && source .venv/bin/activate
+pip install -r requirements.txt
+
+python cli.py ingest /path/to/your/notes
+python cli.py ask "what did I write about X"
+
+python gui.py   # then open http://127.0.0.1:8080
+```
+
+![The assistant answering a question with citations](docs/screenshots/gui-correct-answer.png)
+
+## Building from source
+
+Requires CMake 3.20+, a C++20 compiler, and Python 3.10+.
+
+```bash
+git clone https://github.com/amankarki151/lattice.git
+cd lattice
+
+pip install pybind11
+cmake -B build -Dpybind11_DIR=$(python -m pybind11 --cmakedir)
+cmake --build build
+```
+
+To build and run the test suite:
+
+```bash
+cmake -B build-test -DLATTICE_BUILD_TESTS=ON \
+  -Dpybind11_DIR=$(python -m pybind11 --cmakedir)
+cmake --build build-test
+cd build-test && ctest --output-on-failure
+```
+
+## Repository layout
+
+```
+core/        the database itself - storage, index, search
+core/tests/  30 tests covering storage, search, indexing, quantization
+cli/         command-line tool
+bindings/    pybind11 Python module
+server/      FastAPI HTTP wrapper
+bench/       benchmark harness and results
+app/         local-first document assistant built on the database
+docs/        design decisions and known limitations
+```
+
+## Status
+
+Storage, search, indexing, quantization, concurrency, interfaces,
+CI, and a working document assistant:
+
+- Append-only write-ahead log with replay
+- Memory-mapped segment files for settled data
+- Recovery on open: load the segment, replay the WAL on top
+- Checkpointing folds the WAL into a new segment via atomic rename
+- Brute-force k-nearest-neighbour search over squared L2 distance
+- HNSW index: layered graph construction, neighbour pruning, and
+  coarse-to-fine search, with recall measured against the exact path
+- Scalar quantization: float32 to uint8, 4x smaller, with the recall
+  and speed cost measured rather than assumed
+- Concurrent query path: many readers alongside a single writer,
+  verified clean under the thread sanitizer
+- Python bindings via pybind11, with the GIL released around calls,
+  published on [PyPI](https://pypi.org/project/lattice-db/)
+- A FastAPI HTTP server wrapping the bindings
+- GoogleTest suite: 30 tests covering storage, search, indexing, and
+  quantization
+- CI that runs the tests and fails the build on a benchmark regression
+- A local-first document assistant: retrieve, re-rank with a
+  cross-encoder, synthesize an answer, and cite every source —
+  entirely offline
+
+## Known limitations
+
+- **Build time is slow.** 66 minutes for a million vectors, against
+  Chroma's 105 seconds. Partly architectural (single-writer, no batch
+  insert path), partly a benchmark artifact (competitors are fed
+  through bulk APIs, Lattice isn't).
+- **Single node only.** No sharding, no replication. This is a
+  library, not a cluster.
+- **The assistant produces nonsense for out-of-scope questions.**
+  Asked something the documents don't cover, the local synthesis
+  model returns malformed text rather than refusing. Real example and
+  screenshot in the design doc.
+- **Citation precision is imperfect.** Of four cited sources,
+  typically one or two are directly relevant and the rest are
+  topically adjacent.
+- **No per-record checksums.** Torn writes are caught; silent bit
+  corruption inside an otherwise-valid record isn't.
+
+Full detail and reasoning in [docs/DESIGN.md](docs/DESIGN.md).
+
+## Writing
+
+- [Building an HNSW index from scratch](https://amankarki.hashnode.dev/building-an-hnsw-index-from-scratch)
+
+## License
+
+MIT — see [LICENSE](LICENSE).