nextok 0.1.0__tar.gz

nextok-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+build/
+.venv/
+__pycache__/
+*.egg-info/
+*.so
+*.pyd
+dist/
+.pytest_cache/

nextok-0.1.0/CMakeLists.txt

@@ -0,0 +1,121 @@
+cmake_minimum_required(VERSION 3.18...3.30)
+# The version lives in include/nextok.h (single source of truth).
+file(STRINGS include/nextok.h _nextok_version_lines
+     REGEX "^#define NEXTOK_VERSION_(MAJOR|MINOR|PATCH) [0-9]+$")
+foreach(_part MAJOR MINOR PATCH)
+  string(REGEX MATCH "NEXTOK_VERSION_${_part} ([0-9]+)" _ "${_nextok_version_lines}")
+  set(_nextok_version_${_part} ${CMAKE_MATCH_1})
+endforeach()
+project(nextok
+  VERSION ${_nextok_version_MAJOR}.${_nextok_version_MINOR}.${_nextok_version_PATCH}
+  LANGUAGES C CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+  set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
+endif()
+
+if(DEFINED SKBUILD)
+  set(_nextok_python_default ON)
+else()
+  set(_nextok_python_default OFF)
+endif()
+option(NEXTOK_BUILD_PYTHON "Build the Python extension" ${_nextok_python_default})
+option(NEXTOK_BUILD_TESTS "Build native tests" OFF)
+option(NEXTOK_BUILD_BENCH "Build native benchmarks" OFF)
+option(NEXTOK_SIMD "Build the Highway SIMD backends (runtime dispatch)" ON)
+option(NEXTOK_USE_SYSTEM_HWY "Use an installed Highway instead of fetching it" OFF)
+
+if(NEXTOK_SIMD)
+  if(NEXTOK_USE_SYSTEM_HWY)
+    find_package(hwy 1.1 CONFIG REQUIRED)
+  else()
+    # Pinned release. Set FETCHCONTENT_SOURCE_DIR_HIGHWAY to build offline.
+    include(FetchContent)
+    set(HWY_ENABLE_TESTS OFF CACHE BOOL "" FORCE)
+    set(HWY_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE)
+    set(HWY_ENABLE_CONTRIB OFF CACHE BOOL "" FORCE)
+    set(HWY_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
+    set(HWY_FORCE_STATIC_LIBS ON CACHE BOOL "" FORCE)
+    FetchContent_Declare(highway
+      URL https://github.com/google/highway/archive/refs/tags/1.3.0.tar.gz
+      URL_HASH SHA256=07b3c1ba2c1096878a85a31a5b9b3757427af963b1141ca904db2f9f4afe0bc2
+      DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
+    set(_nextok_pic ${CMAKE_POSITION_INDEPENDENT_CODE})
+    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+    FetchContent_MakeAvailable(highway)
+    set(CMAKE_POSITION_INDEPENDENT_CODE ${_nextok_pic})
+    if(NOT TARGET hwy::hwy)
+      add_library(hwy::hwy ALIAS hwy)
+    endif()
+  endif()
+endif()
+
+add_library(nextok_core STATIC
+  src/core/fast.cpp
+  src/core/sampler.cpp
+  src/core/thread_pool.cpp
+  src/kernels/dispatch.cpp
+  src/kernels/scalar.cpp
+  src/reference/reference.cpp
+)
+if(NEXTOK_SIMD)
+  target_sources(nextok_core PRIVATE src/kernels/simd.cpp)
+  target_compile_definitions(nextok_core PRIVATE NEXTOK_HAVE_SIMD)
+  target_link_libraries(nextok_core PRIVATE hwy::hwy)
+endif()
+target_include_directories(nextok_core
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+find_package(Threads REQUIRED)
+target_link_libraries(nextok_core PUBLIC Threads::Threads)
+set_target_properties(nextok_core PROPERTIES POSITION_INDEPENDENT_CODE ON)
+if(MSVC)
+  target_compile_options(nextok_core PRIVATE /W4 /fp:precise)
+else()
+  # No FMA contraction: keeps results bit-identical across x86 and ARM.
+  target_compile_options(nextok_core PRIVATE -Wall -Wextra -ffp-contract=off)
+endif()
+
+if(NEXTOK_BUILD_PYTHON)
+  find_package(Python 3.10 REQUIRED
+    COMPONENTS Interpreter Development.Module
+    OPTIONAL_COMPONENTS Development.SABIModule)
+  execute_process(
+    COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
+    OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT)
+  find_package(nanobind CONFIG REQUIRED)
+
+  nanobind_add_module(_core STABLE_ABI NB_STATIC python/src/bindings.cpp)
+  target_link_libraries(_core PRIVATE nextok_core)
+  target_include_directories(_core PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+  install(TARGETS _core LIBRARY DESTINATION nextok)
+
+  nanobind_add_stub(_core_stub
+    MODULE _core
+    OUTPUT _core.pyi
+    PYTHON_PATH $<TARGET_FILE_DIR:_core>
+    MARKER_FILE py.typed
+    DEPENDS _core)
+  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/_core.pyi
+                ${CMAKE_CURRENT_BINARY_DIR}/py.typed
+          DESTINATION nextok)
+endif()
+
+if(NEXTOK_BUILD_TESTS)
+  enable_testing()
+  add_executable(test_capi tests/c/test_capi.c)
+  target_link_libraries(test_capi PRIVATE nextok_core)
+  set_target_properties(test_capi PROPERTIES LINKER_LANGUAGE CXX)
+  add_test(NAME test_capi COMMAND test_capi)
+endif()
+
+if(NEXTOK_BUILD_BENCH)
+  add_executable(bench_sampler bench/bench_sampler.cpp)
+  target_link_libraries(bench_sampler PRIVATE nextok_core)
+  target_include_directories(bench_sampler PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+endif()

nextok-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 sjjeong94
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nextok-0.1.0/PKG-INFO

@@ -0,0 +1,108 @@
+Metadata-Version: 2.1
+Name: nextok
+Version: 0.1.0
+Summary: Fast cross-platform LLM token sampler for CPUs (temperature, top-k/p, min-p, penalties, logit bias)
+Keywords: llm,sampling,top-p,top-k,inference,simd
+Author: sjjeong94
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.21
+Description-Content-Type: text/markdown
+
+# nextok
+
+Fast, cross-platform LLM token sampler for CPUs, with a NumPy-friendly Python API.
+
+Supported options: temperature, top-k, top-p, min-p, repetition / frequency / presence
+penalty, logit_bias, seed. Exact semantics: [docs/semantics.md](docs/semantics.md).
+Benchmarks: [docs/performance.md](docs/performance.md). For example, at vocab 128k a
+top-p=0.95 draw takes ~50 µs on one core, ~190× faster than a sort-based sampler.
+Roadmap: [plan.md](plan.md).
+
+```bash
+pip install nextok   # Python >= 3.10; wheels for Linux, macOS and Windows (x86-64 / arm64)
+```
+
+```python
+import numpy as np
+import nextok
+
+sampler = nextok.Sampler(vocab_size=32000)
+seq = sampler.new_sequence(prompt_ids)
+token = sampler.sample(logits, temperature=0.8, top_p=0.95, seed=42, seq=seq)
+```
+
+## CPU backends
+
+SIMD kernels are built with [Highway](https://github.com/google/highway) for every
+supported instruction set (SSE2 through AVX-512/AVX10 on x86, NEON/SVE on ARM), and
+the best one the CPU supports is picked at runtime:
+
+```python
+>>> nextok.cpu_info()
+{'backend': 'AVX2', 'available': ['AVX2', 'SSE4', 'SSSE3', 'SSE2', 'scalar']}
+```
+
+Set `NEXTOK_BACKEND=<name>` (for example `scalar`) to override the choice.
+
+## Threads
+
+`Sampler(vocab_size, num_threads=0)` samples the rows of a batch in parallel
+(`num_threads=0` means up to min(physical cores, 8) threads). A single row stays on
+the calling thread unless `num_threads > 1` is given explicitly. In that case a large
+row (vocab >= 64k) is also split across threads, which helps when calls come back to
+back. Waking idle workers can cost more than the split saves when calls are spaced
+out. Results never depend on the thread count.
+
+## Development
+
+Requires [uv](https://docs.astral.sh/uv/), CMake >= 3.18, and a C++17 compiler.
+
+```bash
+uv sync              # create .venv, build the extension (editable), install dev deps
+uv run pytest        # run tests; edits to C++ sources trigger a rebuild automatically
+uv build             # build sdist + wheel into dist/
+
+# Native C ABI test
+cmake -S . -B build/native -DNEXTOK_BUILD_TESTS=ON && cmake --build build/native && ctest --test-dir build/native
+
+# Benchmarks
+cmake -S . -B build/bench -DNEXTOK_BUILD_BENCH=ON && cmake --build build/bench
+./build/bench/bench_sampler --reference   # C++ latency per preset, vs. scalar reference
+./build/bench/bench_sampler --backend SSE4  # force a kernel backend
+./build/bench/bench_sampler --threads 4 --batch 16  # parallel batch
+uv run python bench/bench.py              # Python latency vs. a NumPy sampler
+```
+
+Highway is downloaded at configure time (pinned release). To build offline, point
+`FETCHCONTENT_SOURCE_DIR_HIGHWAY` at a local checkout, or use an installed copy with
+`-DNEXTOK_USE_SYSTEM_HWY=ON`. `-DNEXTOK_SIMD=OFF` builds the scalar backend only.
+
+### Releasing
+
+`.github/workflows/wheels.yml` builds the sdist and wheels with cibuildwheel. That means
+abi3 wheels for CPython >= 3.12, per-version wheels for 3.10–3.11, manylinux_2_28 and
+musllinux for x86_64 and aarch64, macOS arm64 and x86_64, and Windows AMD64. Every wheel
+runs the test suite before it is uploaded. Pushing a `v*` tag publishes to PyPI through
+Trusted Publishing. Before the first release:
+
+1. Create a GitHub environment named `pypi`. Adding required reviewers is recommended.
+2. Register this repository as a trusted publisher of project `nextok` on pypi.org
+   (workflow `wheels.yml`, environment `pypi`).
+3. Bump `NEXTOK_VERSION_MAJOR/MINOR/PATCH` in `include/nextok.h`, the single source of the
+   version (CMake and `pyproject.toml` read it from there). Then tag:
+   `git tag v0.1.0 && git push --tags`.
+
+The Python version is pinned in `.python-version`; uv prefers its managed interpreters,
+which include the headers needed to build the extension.

nextok-0.1.0/README.md

@@ -0,0 +1,86 @@
+# nextok
+
+Fast, cross-platform LLM token sampler for CPUs, with a NumPy-friendly Python API.
+
+Supported options: temperature, top-k, top-p, min-p, repetition / frequency / presence
+penalty, logit_bias, seed. Exact semantics: [docs/semantics.md](docs/semantics.md).
+Benchmarks: [docs/performance.md](docs/performance.md). For example, at vocab 128k a
+top-p=0.95 draw takes ~50 µs on one core, ~190× faster than a sort-based sampler.
+Roadmap: [plan.md](plan.md).
+
+```bash
+pip install nextok   # Python >= 3.10; wheels for Linux, macOS and Windows (x86-64 / arm64)
+```
+
+```python
+import numpy as np
+import nextok
+
+sampler = nextok.Sampler(vocab_size=32000)
+seq = sampler.new_sequence(prompt_ids)
+token = sampler.sample(logits, temperature=0.8, top_p=0.95, seed=42, seq=seq)
+```
+
+## CPU backends
+
+SIMD kernels are built with [Highway](https://github.com/google/highway) for every
+supported instruction set (SSE2 through AVX-512/AVX10 on x86, NEON/SVE on ARM), and
+the best one the CPU supports is picked at runtime:
+
+```python
+>>> nextok.cpu_info()
+{'backend': 'AVX2', 'available': ['AVX2', 'SSE4', 'SSSE3', 'SSE2', 'scalar']}
+```
+
+Set `NEXTOK_BACKEND=<name>` (for example `scalar`) to override the choice.
+
+## Threads
+
+`Sampler(vocab_size, num_threads=0)` samples the rows of a batch in parallel
+(`num_threads=0` means up to min(physical cores, 8) threads). A single row stays on
+the calling thread unless `num_threads > 1` is given explicitly. In that case a large
+row (vocab >= 64k) is also split across threads, which helps when calls come back to
+back. Waking idle workers can cost more than the split saves when calls are spaced
+out. Results never depend on the thread count.
+
+## Development
+
+Requires [uv](https://docs.astral.sh/uv/), CMake >= 3.18, and a C++17 compiler.
+
+```bash
+uv sync              # create .venv, build the extension (editable), install dev deps
+uv run pytest        # run tests; edits to C++ sources trigger a rebuild automatically
+uv build             # build sdist + wheel into dist/
+
+# Native C ABI test
+cmake -S . -B build/native -DNEXTOK_BUILD_TESTS=ON && cmake --build build/native && ctest --test-dir build/native
+
+# Benchmarks
+cmake -S . -B build/bench -DNEXTOK_BUILD_BENCH=ON && cmake --build build/bench
+./build/bench/bench_sampler --reference   # C++ latency per preset, vs. scalar reference
+./build/bench/bench_sampler --backend SSE4  # force a kernel backend
+./build/bench/bench_sampler --threads 4 --batch 16  # parallel batch
+uv run python bench/bench.py              # Python latency vs. a NumPy sampler
+```
+
+Highway is downloaded at configure time (pinned release). To build offline, point
+`FETCHCONTENT_SOURCE_DIR_HIGHWAY` at a local checkout, or use an installed copy with
+`-DNEXTOK_USE_SYSTEM_HWY=ON`. `-DNEXTOK_SIMD=OFF` builds the scalar backend only.
+
+### Releasing
+
+`.github/workflows/wheels.yml` builds the sdist and wheels with cibuildwheel. That means
+abi3 wheels for CPython >= 3.12, per-version wheels for 3.10–3.11, manylinux_2_28 and
+musllinux for x86_64 and aarch64, macOS arm64 and x86_64, and Windows AMD64. Every wheel
+runs the test suite before it is uploaded. Pushing a `v*` tag publishes to PyPI through
+Trusted Publishing. Before the first release:
+
+1. Create a GitHub environment named `pypi`. Adding required reviewers is recommended.
+2. Register this repository as a trusted publisher of project `nextok` on pypi.org
+   (workflow `wheels.yml`, environment `pypi`).
+3. Bump `NEXTOK_VERSION_MAJOR/MINOR/PATCH` in `include/nextok.h`, the single source of the
+   version (CMake and `pyproject.toml` read it from there). Then tag:
+   `git tag v0.1.0 && git push --tags`.
+
+The Python version is pinned in `.python-version`; uv prefers its managed interpreters,
+which include the headers needed to build the extension.

nextok-0.1.0/docs/performance.md

@@ -0,0 +1,246 @@
+# Performance
+
+How fast nextok samples one token, and what it is compared against. All numbers
+below were measured on a single machine (see [Setup](#setup)); rerun the
+commands in [Reproducing](#reproducing) on your own hardware.
+
+## Summary
+
+Latency per call with vocabulary 128,256 (Llama 3), batch 1, AVX2, one thread:
+
+| preset | nextok | scalar reference | NumPy sampler |
+|---|---:|---:|---:|
+| greedy | **8.6 µs** | 158 µs | 13 µs (`np.argmax`) |
+| temperature 0.8 | **49 µs** | 16,342 µs | 13,146 µs |
+| top_k=40 | **12 µs** | 9,548 µs | 931 µs |
+| top_p=0.95 | **51 µs** | 9,554 µs | 11,659 µs |
+| min_p=0.05 | **10 µs** | 10,273 µs | 11,474 µs |
+| top_k + top_p + min_p | **13 µs** | 9,863 µs | 882 µs |
+| top_p + penalties + bias | **66 µs** | 9,388 µs | — |
+
+- **140–1000× faster than a sort-based sampler written in C++.** The
+  reference is nextok's own float64 implementation of the spec. It sorts the
+  whole vocabulary, which is the textbook approach.
+- **65–1150× faster than a typical NumPy sampler** in the Python benchmark.
+  Greedy is 1.4× faster than `np.argmax`.
+- **Greedy, top-k and min-p run at 40–60 GB/s.** That is close to the speed of
+  simply reading the logits from L2 cache once. Temperature and top-p must
+  evaluate `exp` for every token and run at ~10 GB/s.
+- **Python call overhead is 0.6 µs** (argument parsing, array validation,
+  releasing the GIL).
+
+## Setup
+
+| | |
+|---|---|
+| CPU | Intel Core i5-14400F (6 P-cores + 4 E-cores; AVX2, no AVX-512) |
+| OS | Ubuntu 24.04 under WSL2, Linux 6.18 |
+| Compiler | GCC 13.3, `-O3`, no `-march=native` (runtime dispatch picks AVX2) |
+| Kernels | Highway 1.3.0 |
+| Python | CPython 3.12, NumPy 2.x |
+| Date | 2026-10-31, commit after `62d3df0` |
+
+**Input.** Each row is roughly LLM-shaped: a Gaussian floor (σ = 2) plus 8
+strongly boosted tokens (+8 to +14). The same logits are reused for every call,
+so they stay in L2/L3 cache. That models a sampler that runs right after the
+model wrote its logits. The `top_p+penalties` preset adds a 1,024-token prompt,
+1,024 generated tokens, repetition/frequency/presence penalties and 16
+`logit_bias` entries.
+
+**Method.** `bench/bench_sampler.cpp` calls `nt_sample_batch` back to back and
+reports the median of 15 repetitions of ~10 ms each. "GB/s" means
+`vocab × 4 bytes × batch / time`, i.e. how fast the input is consumed.
+
+## nextok vs. a sort-based sampler
+
+The scalar reference (`src/reference/`) implements the spec directly: apply
+penalties, sort all tokens, then truncate and draw, in float64. It is what a
+straightforward C++ sampler looks like, and nextok is tested against it for
+correctness.
+
+Speedup of nextok (AVX2, 1 thread) over the reference, batch 1:
+
+| preset | V=32,000 | V=128,256 | V=151,936 | V=256,000 |
+|---|---:|---:|---:|---:|
+| greedy | 19× | 18× | 18× | 20× |
+| temperature | 303× | 335× | 307× | 347× |
+| top_k=40 | 442× | 801× | 824× | 924× |
+| top_p=0.95 | 154× | 189× | 190× | 40× |
+| min_p=0.05 | 929× | 1001× | 1063× | 1248× |
+| top_k + top_p + min_p | 417× | 784× | 795× | 913× |
+| top_p + penalties | 76× | 141× | 151× | 31× |
+
+Absolute latency of nextok in µs:
+
+| preset | V=32,000 | V=128,256 | V=151,936 | V=256,000 |
+|---|---:|---:|---:|---:|
+| greedy | 2.4 | 8.6 | 11.2 | 16.8 |
+| temperature | 11.8 | 48.8 | 63.9 | 94.6 |
+| top_k=40 | 4.6 | 11.9 | 13.0 | 22.5 |
+| top_p=0.95 | 12.9 | 50.6 | 61.9 | 578 |
+| min_p=0.05 | 2.3 | 10.3 | 10.2 | 18.3 |
+| top_k + top_p + min_p | 4.4 | 12.6 | 12.8 | 23.0 |
+| top_p + penalties | 26.3 | 66.4 | 74.7 | 693 |
+
+Where the speed comes from:
+
+- **No sorting.** Each parameter combination takes its own path. top-k uses a
+  streaming heap seeded with a safe threshold, min-p uses a threshold
+  compaction, and top-p first guesses a small candidate set. Only the few
+  tokens that can matter are ever ordered.
+- **No full copy.** Penalties and `logit_bias` become a sparse patch list that
+  is applied block by block on the fly.
+- **SIMD kernels** for max, `exp`, compaction and search, chosen at runtime
+  for the CPU (see below).
+
+The top-p numbers at V=256,000 are a deliberately bad case; see
+[Known slow case](#known-slow-case).
+
+## nextok vs. NumPy (Python)
+
+`bench/bench.py` compares `Sampler.sample` with a common NumPy implementation:
+`argpartition` for top-k, `argsort` plus `cumsum` for top-p, and
+`rng.choice` for the draw. Microseconds per call:
+
+| preset | V=32,000 | | V=128,256 | | V=256,000 | |
+|---|---:|---:|---:|---:|---:|---:|
+| | nextok | NumPy | nextok | NumPy | nextok | NumPy |
+| greedy | 2.9 | 4.3 | 9.2 | 13.1 | 19.3 | 25.5 |
+| temperature | 14.1 | 2,405 | 59.2 | 13,146 | 103 | 25,145 |
+| top_k=40 | 4.8 | 225 | 13.2 | 931 | 22.5 | 1,538 |
+| top_p=0.95 | 13.1 | 2,429 | 53.5 | 11,659 | 102 | 25,437 |
+| min_p=0.05 | 2.9 | 2,437 | 10.0 | 11,474 | 18.7 | 23,849 |
+| top_k + top_p + min_p | 6.4 | 234 | 13.3 | 882 | 24.3 | 1,564 |
+
+The Python numbers include the 0.6 µs binding overhead. Otherwise they match
+the C++ numbers above.
+
+## SIMD backends
+
+The same binary contains kernels for SSE2 through AVX-512/AVX10 (x86) and
+NEON/SVE (ARM), plus a portable scalar fallback. The best one is picked at
+runtime, and `NEXTOK_BACKEND` forces one. µs per call at V=128,256, batch 1:
+
+| preset | scalar | SSE4 | AVX2 |
+|---|---:|---:|---:|
+| greedy | 106 | 10.4 | 8.6 |
+| temperature | 597 | 135 | 48.8 |
+| top_k=40 | 135 | 16.3 | 11.9 |
+| top_p=0.95 | 634 | 141 | 50.6 |
+| min_p=0.05 | 123 | 11.8 | 10.3 |
+| top_p + penalties | 569 | 150 | 66.4 |
+
+- **Greedy, top-k and min-p** are mostly comparisons and are already limited by
+  memory speed with SSE4.
+- **Temperature and top-p** are dominated by `exp`. They gain 2.8× from SSE4
+  to AVX2: twice the vector width, plus fused multiply-add.
+- **AVX-512 and NEON** kernels are compiled in but have not been measured yet.
+  This CPU has no AVX-512.
+
+## Threads
+
+### One row split across threads
+
+With an explicit `num_threads > 1`, a row with vocab ≥ 64k is split into parts
+that run in parallel. µs per call, batch 1, back-to-back calls:
+
+| preset | V=128,256, 1 thread | V=128,256, 4 threads | V=256,000, 1 thread | V=256,000, 4 threads |
+|---|---:|---:|---:|---:|
+| greedy | 8.6 | 4.0 | 16.8 | 5.7 |
+| temperature | 48.8 | 23.1 | 94.6 | 29.5 |
+| top_k=40 | 11.9 | 7.4 | 22.5 | 8.2 |
+| top_p=0.95 | 50.6 | 22.7 | 578 | 260 |
+| min_p=0.05 | 10.3 | 5.1 | 18.3 | 6.3 |
+| top_p + penalties | 66.4 | 45.3 | 693 | 300 |
+
+At V=32,000 a row is not split (each part needs ≥ 32k tokens), so the numbers
+match one thread. Results are bit-identical for every thread count.
+
+### Batches
+
+Rows of a batch are spread across threads. That is what the default
+`num_threads=0` does. µs per call for the whole batch, back-to-back calls:
+
+| preset (V=128,256) | batch 16, 1 thread | batch 16, 6 threads | batch 64, 1 thread | batch 64, 6 threads |
+|---|---:|---:|---:|---:|
+| greedy | 136 | 29 | 586 | 108 |
+| temperature | 762 | 161 | 3,187 | 591 |
+| top_k=40 | 177 | 41 | 804 | 141 |
+| top_p=0.95 | 857 | 169 | 3,557 | 646 |
+| min_p=0.05 | 135 | 30 | 550 | 109 |
+| top_p + penalties | 1,145 | 238 | 4,864 | 1,000 |
+
+With 6 threads the batch is 4.7–5.7× faster. Throughput reaches 280–300 GB/s for
+greedy and min-p, which is past single-core cache bandwidth.
+
+### Calls with idle time in between
+
+The numbers above are for back-to-back calls. In a real decode loop the
+sampler waits ~10–50 ms for the model between calls. Caches cool down, cores
+clock down, and idle worker threads go to sleep. With `--gap-us 20000` (20 ms
+of sleep before each call), V=128,256, median µs per call:
+
+| preset | batch 1, 1 thread | batch 1, 4 threads | batch 16, 1 thread | batch 16, 6 threads |
+|---|---:|---:|---:|---:|
+| greedy | 53 | 130 | 195 | 366 |
+| temperature | 159 | 245 | 916 | 598 |
+| top_k=40 | 73 | 260 | 308 | 360 |
+| top_p=0.95 | 154 | 244 | 1,034 | 731 |
+| top_p + penalties | 266 | 261 | 1,974 | 1,124 |
+
+- **Every call is 3–6× slower after an idle gap, even with one thread.** This
+  comes from the machine (cold caches, frequency scaling), not from the
+  sampler. Absolute numbers here are noisy, especially under WSL2.
+- **Splitting a single row does not pay off with idle gaps.** Waking sleeping
+  workers costs more than the parallel speedup. That is why the default
+  (`num_threads=0`) keeps single rows on the calling thread.
+- **Batches still gain** for exp-heavy presets (temperature, top-p), but not
+  for the cheap ones (greedy, top-k).
+
+## Known slow case
+
+top-p at V=256,000 takes 578 µs, ten times the V=151,936 number. The synthetic
+input makes the cause visible. With 256k Gaussian tokens, the tail holds over
+5% of the probability mass, so `top_p=0.95` has to keep tens of thousands of
+tail tokens. nextok's quick path (guessing a small candidate set) gives up
+above 8,192 candidates and falls back to a weighted histogram over the whole
+row. That costs a second `exp` pass plus a scattered histogram update per
+token.
+
+Real LLM distributions are usually far more peaked, so they stay on the quick
+path. Even this case is 40× faster than the sort-based reference, and 4
+threads bring it to 260 µs.
+
+## Not measured yet
+
+- **llama.cpp's sampler chain, PyTorch/vLLM samplers.** These are the most
+  relevant baselines, but they could not be built in this environment.
+  `bench/bench_sampler.cpp` shows the input data and presets to mirror.
+  - llama.cpp applies temperature *after* top-k/top-p, whereas nextok follows
+    vLLM's order (see [semantics.md](semantics.md)). The work is comparable,
+    but outputs are not identical.
+- **AVX-512, ARM NEON/SVE, macOS and Windows.** These will be covered by the CI
+  matrix (P4/P5).
+- **Real model logits** instead of synthetic rows.
+
+## Reproducing
+
+```bash
+# C++: build the benchmark
+cmake -S . -B build/bench -DNEXTOK_BUILD_BENCH=ON && cmake --build build/bench
+B=./build/bench/bench_sampler
+
+$B --reference                    # vs. the sort-based reference (all vocab sizes)
+$B --backend scalar               # force a backend (scalar, SSE4, AVX2, AVX3, NEON, ...)
+$B --threads 4                    # split single rows across 4 threads
+$B --batch 16 --threads 6         # batch parallelism
+$B --gap-us 20000                 # 20 ms idle before each call (decode-loop conditions)
+$B --quick ...                    # shorter runs, for any of the above
+
+# Python: nextok vs. NumPy, plus call overhead
+uv run python bench/bench.py --vocab 128256
+```
+
+Expect different absolute numbers on other machines. On a quiet bare-metal
+Linux system, the idle-gap and threading numbers in particular should look
+better than under WSL2.