epzero 0.3.0__tar.gz

epzero-0.3.0/CMakeLists.txt

@@ -0,0 +1,219 @@
+cmake_minimum_required(VERSION 3.20)
+project(epzero VERSION 0.1.0 LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+  set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+endif()
+
+option(EPZERO_BUILD_TESTS "Build unit tests" ON)
+option(EPZERO_BUILD_BENCH "Build benchmarks" ON)
+option(EPZERO_BUILD_PYTHON "Build the Python module (normally via pip install ./engine)" OFF)
+set(EPZERO_SANITIZE "" CACHE STRING "Sanitizer to enable: address, thread, or undefined")
+
+if(EPZERO_SANITIZE)
+  add_compile_options(-fsanitize=${EPZERO_SANITIZE} -fno-omit-frame-pointer -g)
+  add_link_options(-fsanitize=${EPZERO_SANITIZE})
+endif()
+
+# Fuzzing (Clang + libFuzzer): every parser of untrusted bytes, under AddressSanitizer and UBSan.
+# The engine is instrumented for coverage; only the fuzz targets link libFuzzer's main().
+option(EPZERO_BUILD_FUZZ "Build the libFuzzer targets (Clang only)" OFF)
+if(EPZERO_BUILD_FUZZ)
+  if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    message(FATAL_ERROR "EPZERO_BUILD_FUZZ needs Clang (libFuzzer)")
+  endif()
+  add_compile_options(-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined
+                      -fno-omit-frame-pointer -g)
+  add_link_options(-fsanitize=address,undefined)
+endif()
+
+# WebAssembly: fixed-width SIMD is supported by every browser the demo targets, so the whole
+# build uses it (there is no runtime feature detection in WASM). Threads map to Web Workers
+# sharing one SharedArrayBuffer, which requires a cross-origin isolated page.
+option(EPZERO_WASM_THREADS "WebAssembly build uses threads (needs a cross-origin isolated page)" ON)
+if(EMSCRIPTEN)
+  add_compile_options(-msimd128 -fwasm-exceptions)
+  add_link_options(-fwasm-exceptions -sALLOW_MEMORY_GROWTH=1 -sMAXIMUM_MEMORY=4GB)
+  if(EPZERO_WASM_THREADS)
+    add_compile_options(-pthread)
+    add_link_options(-pthread -Wno-pthreads-mem-growth)
+  endif()
+endif()
+
+find_package(Threads REQUIRED)
+
+add_library(epzero_engine
+  src/distance.cpp
+  src/distance_scalar.cpp
+  src/hnsw.cpp
+  src/recall.cpp
+  src/database.cpp
+  src/text.cpp
+  src/brute_force.cpp
+  src/dataset.cpp
+)
+target_include_directories(epzero_engine PUBLIC include PRIVATE src)
+target_link_libraries(epzero_engine PUBLIC Threads::Threads)
+target_compile_options(epzero_engine PRIVATE -Wall -Wextra -Wpedantic)
+set_target_properties(epzero_engine PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+# SIMD kernels get ISA flags per file; the rest of the library stays baseline so the binary
+# runs on any CPU of the architecture and picks the best kernel at runtime.
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|amd64")
+  target_sources(epzero_engine PRIVATE src/distance_avx2.cpp src/distance_avx512.cpp)
+  set_source_files_properties(src/distance_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mpopcnt")
+  set_source_files_properties(src/distance_avx512.cpp PROPERTIES COMPILE_OPTIONS "-mavx512f")
+  # VNNI integer kernels, where the toolchain can build them: the compiler must know the
+  # instructions (GCC 11+, Clang 12+ for AVX-VNNI) and so must the assembler it hands them to
+  # (binutils 2.36+ for AVX-VNNI; RHEL 9's 2.35 doesn't), so each check compiles and links a use.
+  include(CheckCXXSourceCompiles)
+  set(CMAKE_REQUIRED_FLAGS "-mavx2 -mavxvnni")
+  check_cxx_source_compiles([[
+    #include <immintrin.h>
+    int main(int argc, char**) {
+      __m256i a = _mm256_set1_epi8(static_cast<char>(argc));
+      a = _mm256_dpbusd_avx_epi32(a, a, a);
+      return _mm256_extract_epi32(a, 0);
+    }]] EPZERO_TOOLCHAIN_AVXVNNI)
+  set(CMAKE_REQUIRED_FLAGS "-mavx512f -mavx512bw -mavx512vnni")
+  check_cxx_source_compiles([[
+    #include <immintrin.h>
+    int main(int argc, char**) {
+      __m512i a = _mm512_set1_epi8(static_cast<char>(argc));
+      a = _mm512_dpbusd_epi32(a, a, a);
+      return _mm512_reduce_add_epi32(a);
+    }]] EPZERO_TOOLCHAIN_AVX512VNNI)
+  unset(CMAKE_REQUIRED_FLAGS)
+  if(EPZERO_TOOLCHAIN_AVXVNNI)
+    target_sources(epzero_engine PRIVATE src/distance_avxvnni.cpp)
+    set_source_files_properties(src/distance_avxvnni.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mavxvnni")
+    set_property(SOURCE src/distance.cpp APPEND PROPERTY COMPILE_DEFINITIONS EPZERO_HAVE_AVXVNNI)
+  endif()
+  if(EPZERO_TOOLCHAIN_AVX512VNNI)
+    target_sources(epzero_engine PRIVATE src/distance_avx512vnni.cpp)
+    set_source_files_properties(src/distance_avx512vnni.cpp PROPERTIES COMPILE_OPTIONS "-mavx512f;-mavx512bw;-mavx512vnni")
+    set_property(SOURCE src/distance.cpp APPEND PROPERTY COMPILE_DEFINITIONS EPZERO_HAVE_AVX512VNNI)
+  endif()
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+  target_sources(epzero_engine PRIVATE src/distance_neon.cpp)
+elseif(EMSCRIPTEN)
+  target_sources(epzero_engine PRIVATE src/distance_wasm.cpp)
+endif()
+
+if(EMSCRIPTEN AND NOT EPZERO_WASM_THREADS)
+  set(EPZERO_BUILD_TESTS OFF)  # the concurrency tests need threads
+endif()
+
+if(EPZERO_BUILD_TESTS)
+  include(FetchContent)
+  FetchContent_Declare(googletest
+    URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.tar.gz
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
+  set(INSTALL_GTEST OFF CACHE BOOL "" FORCE)
+  FetchContent_MakeAvailable(googletest)
+
+  enable_testing()
+  add_executable(epzero_tests tests/distance_test.cpp tests/hnsw_test.cpp tests/recall_test.cpp tests/dataset_test.cpp tests/database_test.cpp tests/text_test.cpp)
+  target_link_libraries(epzero_tests PRIVATE epzero_engine GTest::gtest_main)
+  if(EMSCRIPTEN)
+    # Runs under Node (CMAKE_CROSSCOMPILING_EMULATOR); workers are pre-spawned for the
+    # concurrency tests and the real filesystem backs the save/load tests.
+    target_link_options(epzero_tests PRIVATE -sPTHREAD_POOL_SIZE=16 -sNODERAWFS=1 -sENVIRONMENT=node)
+  endif()
+  include(GoogleTest)
+  gtest_discover_tests(epzero_tests DISCOVERY_TIMEOUT 60)
+endif()
+
+if(EPZERO_BUILD_BENCH)
+  add_executable(bench_distance bench/bench_distance.cpp)
+  target_link_libraries(bench_distance PRIVATE epzero_engine)
+  add_executable(bench_recall bench/bench_recall.cpp)
+  target_link_libraries(bench_recall PRIVATE epzero_engine)
+  add_executable(bench_filter bench/bench_filter.cpp)
+  target_link_libraries(bench_filter PRIVATE epzero_engine)
+  add_executable(bench_churn bench/bench_churn.cpp)
+  target_link_libraries(bench_churn PRIVATE epzero_engine)
+  add_executable(bench_guard bench/bench_guard.cpp)
+  target_link_libraries(bench_guard PRIVATE epzero_engine)
+  add_executable(bench_adaptive bench/bench_adaptive.cpp)
+  target_link_libraries(bench_adaptive PRIVATE epzero_engine)
+endif()
+
+if(EPZERO_BUILD_FUZZ)
+  foreach(target snapshot graph wal text)
+    add_executable(fuzz_${target} fuzz/fuzz_${target}.cpp)
+    target_link_libraries(fuzz_${target} PRIVATE epzero_engine)
+    target_link_options(fuzz_${target} PRIVATE -fsanitize=fuzzer)
+  endforeach()
+  add_executable(fuzz_seeds fuzz/make_seeds.cpp)
+  target_link_libraries(fuzz_seeds PRIVATE epzero_engine)
+endif()
+
+# Writes the web demo's prebuilt graph (see tools/export_graph.cpp and web/scripts/build-graph.sh).
+if(NOT EMSCRIPTEN)
+  add_executable(epzero_export_graph tools/export_graph.cpp)
+  set_target_properties(epzero_export_graph PROPERTIES OUTPUT_NAME epzero-export-graph)
+  target_link_libraries(epzero_export_graph PRIVATE epzero_engine)
+endif()
+
+option(EPZERO_BUILD_NODE "Build the cluster data node (needs gRPC C++ and protobuf)" ON)
+if(EPZERO_BUILD_NODE AND NOT EMSCRIPTEN)
+  find_package(PkgConfig)
+  if(PkgConfig_FOUND)
+    pkg_check_modules(GRPCPP IMPORTED_TARGET grpc++)
+    pkg_check_modules(PROTOBUF IMPORTED_TARGET protobuf)
+  endif()
+  find_program(PROTOC protoc)
+  find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin)
+  if(GRPCPP_FOUND AND PROTOBUF_FOUND AND PROTOC AND GRPC_CPP_PLUGIN)
+    set(PROTO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../proto)
+    set(PROTO_OUT ${CMAKE_CURRENT_BINARY_DIR}/proto)
+    file(MAKE_DIRECTORY ${PROTO_OUT})
+    set(PROTO_SRCS ${PROTO_OUT}/epzero/v1/epzero.pb.cc ${PROTO_OUT}/epzero/v1/epzero.grpc.pb.cc)
+    add_custom_command(
+      OUTPUT ${PROTO_SRCS}
+      COMMAND ${PROTOC} -I ${PROTO_ROOT} --cpp_out=${PROTO_OUT} --grpc_out=${PROTO_OUT}
+              --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} epzero/v1/epzero.proto
+      DEPENDS ${PROTO_ROOT}/epzero/v1/epzero.proto
+      COMMENT "Generating gRPC code for epzero.proto")
+    add_executable(epzero_node server/node_main.cpp ${PROTO_SRCS})
+    set_target_properties(epzero_node PROPERTIES OUTPUT_NAME epzero-node)
+    target_include_directories(epzero_node PRIVATE ${PROTO_OUT})
+    target_link_libraries(epzero_node PRIVATE epzero_engine PkgConfig::GRPCPP PkgConfig::PROTOBUF)
+    install(TARGETS epzero_node RUNTIME DESTINATION bin)
+  else()
+    message(STATUS "gRPC C++ not found: skipping epzero-node")
+  endif()
+endif()
+
+if(EMSCRIPTEN)
+  # ES module for the browser demo: `import createEpzero from "./epzero.js"`.
+  add_executable(epzero_web wasm/epzero_wasm.cpp)
+  target_link_libraries(epzero_web PRIVATE epzero_engine)
+  set_target_properties(epzero_web PROPERTIES OUTPUT_NAME epzero SUFFIX .js)
+  target_link_options(epzero_web PRIVATE
+    -lembind -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=createEpzero -sENVIRONMENT=web,worker -sSTACK_SIZE=1MB
+    # No eval / new Function in the generated bindings, so the site's Content Security Policy can
+    # forbid it (only WebAssembly compilation is allowed).
+    -sDYNAMIC_EXECUTION=0)
+  if(EPZERO_WASM_THREADS)
+    # Pre-spawn the workers: the module's own thread blocks while a build runs, so it cannot
+    # create new ones on demand.
+    target_link_options(epzero_web PRIVATE "-sPTHREAD_POOL_SIZE=Math.min(navigator.hardwareConcurrency,16)")
+  endif()
+endif()
+
+if(EPZERO_BUILD_PYTHON)
+  find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+  find_package(pybind11 CONFIG REQUIRED)
+  pybind11_add_module(epzero_python python/epzero_py.cpp)
+  # The compiled core of the Python package epzero (engine/python/epzero), installed inside it.
+  set_target_properties(epzero_python PROPERTIES OUTPUT_NAME _core)
+  target_link_libraries(epzero_python PRIVATE epzero_engine)
+  install(TARGETS epzero_python DESTINATION epzero)
+endif()

epzero-0.3.0/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.2
+Name: epzero
+Version: 0.3.0
+Summary: An HNSW vector database built from scratch in C++20: SIMD search, filtered search, recall monitoring, crash-safe storage
+Keywords: vector database,hnsw,ann,nearest neighbor,embeddings,rag,langchain
+Author: Lajat
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Project-URL: Homepage, https://epzero.dev/
+Project-URL: Source, https://github.com/Lazeeez/Epzero
+Requires-Python: >=3.10
+Requires-Dist: numpy
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3; extra == "langchain"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
+Requires-Dist: langchain-core>=0.3; extra == "test"
+Requires-Dist: langchain-tests>=0.3; extra == "test"
+Description-Content-Type: text/markdown
+
+# epzero
+
+An HNSW vector database written from scratch in C++20, with Python bindings.
+
+- **Fast search:** hand-written SIMD kernels (AVX2, AVX-512, NEON), chosen at runtime, and
+  lock-free reads. On SIFT1M it serves 1.13× hnswlib's and 1.17× FAISS's queries per second at
+  equal recall, on the same machine.
+- **Filtered search that picks its own strategy:** an exact scan, a filtered graph walk, or a
+  two-hop walk (the idea behind ACORN), chosen per query by a cost model.
+- **Recall you can see:** `RecallGuard` measures recall on a sample of live queries with a
+  confidence interval and moves the search effort to hold a target; `explain()` says why any given
+  item was not returned. With `per_query=True` each query also stops as soon as its answer settles,
+  so easy queries cost less and hard ones search further.
+- **Hybrid search:** `TextIndex` (BM25 keyword search) and `hybrid_search()`, which merges it
+  with vector search by reciprocal rank fusion.
+- **Durable:** `Database` adds a write-ahead log with crash recovery and checkpoints.
+- **8-bit quantization** with exact re-ranking.
+- **A LangChain vector store** in `epzero.langchain`.
+
+```bash
+pip install epzero                 # the engine
+pip install "epzero[langchain]"    # plus the LangChain vector store
+```
+
+## Quick start
+
+```python
+import numpy as np
+import epzero
+
+vectors = np.random.rand(100_000, 128).astype(np.float32)
+index = epzero.Index(dim=128, metric="l2", max_elements=100_000, M=16, ef_construction=200)
+index.add(vectors)                                   # labels 0..n-1 (or pass labels=)
+
+labels, distances = index.search(vectors[:5], k=10, ef=64)
+
+# Filtered search: compile the allowed labels once, reuse across queries.
+allowed = index.filter(np.arange(0, 100_000, 50))
+labels, distances = index.search(vectors[:5], k=10, filter=allowed)
+
+# Hold recall at 0.95 as the workload drifts, checking 1% of queries; per_query=True lets each
+# query stop once its answer settles (cheaper for the same recall).
+guard = epzero.RecallGuard(index, target=0.95, sample_rate=0.01, reference_ef=1024, per_query=True)
+labels, distances = guard.search(vectors[:5], k=10)
+print(guard.report())
+
+# Optional: measure the best memory-prefetch setting for this machine and data (not saved).
+index.tune_prefetch(vectors[:500])
+
+index.save("vectors.epzero")
+index = epzero.Index.load("vectors.epzero")
+
+# Hybrid: keyword search on the same labels, merged with the vector search.
+text = epzero.TextIndex()
+text.add(None, documents)                            # labels 0..n-1, matching the index
+labels, scores = epzero.hybrid_search(index, text, query_vector, "error E-1042 at checkout", k=10)
+```
+
+## LangChain
+
+```python
+from langchain_openai import OpenAIEmbeddings
+from epzero.langchain import EpzeroVectorStore
+
+store = EpzeroVectorStore.from_texts(texts, OpenAIEmbeddings(), metadatas=metadatas)
+store.similarity_search("how do I reset my password?", k=4, filter={"source": "faq"})
+store.max_marginal_relevance_search("pricing", k=4, fetch_k=20)
+store.hybrid_search("error E-1042 at checkout", k=4)   # keywords + vectors
+retriever = store.as_retriever(search_kwargs={"k": 8})
+
+store.save_local("./store")
+store = EpzeroVectorStore.load_local("./store", OpenAIEmbeddings())
+```
+
+Metadata filters are a dict of required values (a list means "any of"), or a callable on the
+metadata. They are compiled into an Epzero filter, so the engine chooses how to search the allowed
+documents instead of over-fetching and discarding. The store passes LangChain's standard
+vector-store test suite.
+
+## Platforms
+
+Wheels for Linux (x86-64 and ARM64, glibc 2.28+) and macOS (Apple silicon and Intel), Python 3.10
+to 3.13. Elsewhere, `pip install epzero` builds from source (CMake 3.20+, a C++20 compiler).
+
+Live demo, benchmarks and design notes: https://epzero.dev/ · MIT license.

epzero-0.3.0/bench/bench_adaptive.cpp

@@ -0,0 +1,171 @@
+// Per-query effort: a fixed beam width for every query against patience, which stops each query
+// once its best k have stopped changing (SearchOptions::patience, with ef as a cap).
+//
+// A fixed ef spends the same effort on every query: too much on easy ones, too little on the
+// hardest. For each setting this reports mean recall@k and mean distance computations (the cost
+// curve), and the tail: the share of queries answered badly (recall@k below 0.8 and 0.5), the
+// 99th-percentile cost, and single-thread throughput.
+//
+//   bench_adaptive --base sift_base.fvecs --query sift_query.fvecs --truth sift_groundtruth.ivecs \
+//                  --index sift.idx --json adaptive.json
+//   (--metric cosine for angular datasets such as GloVe; ground truth as ivecs, top-k first)
+
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "epzero/dataset.h"
+#include "epzero/hnsw.h"
+#include "epzero/parallel.h"
+
+using namespace epzero;
+
+namespace {
+
+struct Options {
+  std::string base, query, truth, index, json, name = "dataset";
+  Metric metric = Metric::L2;
+  std::size_t k = 10, M = 16, ef_construction = 200, threads = 0, queries = 0, rounds = 3;
+  std::vector<std::size_t> efs = {10, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024};
+  std::vector<std::size_t> patiences = {2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128};
+  std::vector<std::size_t> caps = {256, 1024};
+};
+
+std::vector<std::size_t> list(const std::string& s) {
+  std::vector<std::size_t> out;
+  std::size_t start = 0;
+  while (start < s.size()) {
+    const std::size_t comma = s.find(',', start);
+    out.push_back(std::stoul(s.substr(start, comma - start)));
+    if (comma == std::string::npos) break;
+    start = comma + 1;
+  }
+  return out;
+}
+
+Options parse(int argc, char** argv) {
+  Options o;
+  for (int i = 1; i < argc; ++i) {
+    const std::string arg = argv[i];
+    auto value = [&]() -> std::string {
+      if (i + 1 >= argc) throw std::invalid_argument(arg + " needs a value");
+      return argv[++i];
+    };
+    if (arg == "--base") o.base = value();
+    else if (arg == "--query") o.query = value();
+    else if (arg == "--truth") o.truth = value();
+    else if (arg == "--index") o.index = value();
+    else if (arg == "--json") o.json = value();
+    else if (arg == "--name") o.name = value();
+    else if (arg == "--metric") o.metric = value() == "cosine" ? Metric::Cosine : Metric::L2;
+    else if (arg == "--k") o.k = std::stoul(value());
+    else if (arg == "--M") o.M = std::stoul(value());
+    else if (arg == "--efc") o.ef_construction = std::stoul(value());
+    else if (arg == "--threads") o.threads = std::stoul(value());
+    else if (arg == "--queries") o.queries = std::stoul(value());
+    else if (arg == "--rounds") o.rounds = std::stoul(value());
+    else if (arg == "--efs") o.efs = list(value());
+    else if (arg == "--patience") o.patiences = list(value());
+    else if (arg == "--caps") o.caps = list(value());
+    else throw std::invalid_argument("unknown argument " + arg);
+  }
+  if (o.base.empty() || o.query.empty() || o.truth.empty()) throw std::invalid_argument("pass --base, --query and --truth");
+  if (o.threads == 0) o.threads = std::max(1u, std::thread::hardware_concurrency());
+  return o;
+}
+
+struct Row {
+  std::string policy;  // "fixed" or "patience"
+  std::size_t ef, patience;
+  double recall, distances, p99_distances, below_08, below_05, qps;
+};
+
+}  // namespace
+
+int main(int argc, char** argv) try {
+  const Options opt = parse(argc, argv);
+  const Dataset queries = read_fvecs(opt.query, opt.queries);
+  const auto truth = read_ivecs(opt.truth, queries.n);
+  const std::size_t nq = queries.n, k = opt.k;
+
+  std::unique_ptr<HnswIndex> index;
+  if (!opt.index.empty() && std::filesystem::exists(opt.index)) {
+    index = HnswIndex::load(opt.index);
+    std::printf("loaded %s (%zu vectors)\n", opt.index.c_str(), index->size());
+  } else {
+    const Dataset base = read_fvecs(opt.base);
+    index = std::make_unique<HnswIndex>(HnswParams{.dim = base.dim, .metric = opt.metric, .max_elements = base.n,
+                                                   .M = opt.M, .ef_construction = opt.ef_construction});
+    const auto start = std::chrono::steady_clock::now();
+    parallel_for(0, base.n, opt.threads, [&](std::size_t i) { index->add(i, base.row(i)); });
+    std::printf("built %zu x %zu in %.1f s\n", base.n, base.dim,
+                std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
+    if (!opt.index.empty()) index->save(opt.index);
+  }
+
+  // One setting: per-query recall and cost, then throughput as the best of `rounds` timed passes.
+  const auto run = [&](const std::string& policy, std::size_t ef, std::size_t patience) {
+    std::vector<double> recall(nq), cost(nq);
+    for (std::size_t q = 0; q < nq; ++q) {
+      SearchStats stats;
+      const auto got = index->search(queries.row(q), k, {.ef = ef, .patience = patience, .stats = &stats});
+      std::size_t hits = 0;
+      for (std::size_t t = 0; t < k && t < truth[q].size(); ++t) {
+        hits += std::any_of(got.begin(), got.end(), [&](const SearchResult& r) { return r.label == truth[q][t]; });
+      }
+      recall[q] = static_cast<double>(hits) / static_cast<double>(k);
+      cost[q] = static_cast<double>(stats.distance_computations);
+    }
+    double best = 0;
+    for (std::size_t r = 0; r < opt.rounds; ++r) {
+      const auto start = std::chrono::steady_clock::now();
+      for (std::size_t q = 0; q < nq; ++q) index->search(queries.row(q), k, {.ef = ef, .patience = patience});
+      best = std::max(best, static_cast<double>(nq) / std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
+    }
+    Row row{policy, ef, patience, 0, 0, 0, 0, 0, best};
+    for (std::size_t q = 0; q < nq; ++q) {
+      row.recall += recall[q] / static_cast<double>(nq);
+      row.distances += cost[q] / static_cast<double>(nq);
+      row.below_08 += (recall[q] < 0.8) / static_cast<double>(nq);
+      row.below_05 += (recall[q] < 0.5) / static_cast<double>(nq);
+    }
+    std::sort(cost.begin(), cost.end());
+    row.p99_distances = cost[std::min(nq - 1, nq * 99 / 100)];
+    std::printf("%-8s ef %5zu  patience %4zu  recall %.4f  dist %7.0f  p99 dist %7.0f  <0.8 %5.2f%%  <0.5 %5.2f%%  qps %8.0f\n",
+                policy.c_str(), ef, patience, row.recall, row.distances, row.p99_distances, 100 * row.below_08,
+                100 * row.below_05, row.qps);
+    return row;
+  };
+
+  std::vector<Row> rows;
+  for (const auto ef : opt.efs) rows.push_back(run("fixed", ef, 0));
+  for (const auto cap : opt.caps) {
+    for (const auto p : opt.patiences) rows.push_back(run("patience", cap, p));
+  }
+
+  if (!opt.json.empty()) {
+    std::ofstream out(opt.json);
+    out << "{\n  \"dataset\": \"" << opt.name << "\", \"n\": " << index->size() << ", \"dim\": " << index->params().dim
+        << ", \"queries\": " << nq << ", \"k\": " << k << ", \"M\": " << index->params().M
+        << ", \"ef_construction\": " << index->params().ef_construction << ",\n  \"rows\": [\n";
+    for (std::size_t i = 0; i < rows.size(); ++i) {
+      const Row& r = rows[i];
+      out << "    {\"policy\": \"" << r.policy << "\", \"ef\": " << r.ef << ", \"patience\": " << r.patience
+          << ", \"recall\": " << r.recall << ", \"distances\": " << r.distances << ", \"p99_distances\": " << r.p99_distances
+          << ", \"below_0.8\": " << r.below_08 << ", \"below_0.5\": " << r.below_05 << ", \"qps\": " << r.qps << "}"
+          << (i + 1 < rows.size() ? "," : "") << "\n";
+    }
+    out << "  ]\n}\n";
+    std::printf("wrote %s\n", opt.json.c_str());
+  }
+  return 0;
+} catch (const std::exception& e) {
+  std::fprintf(stderr, "error: %s\n", e.what());
+  return 1;
+}