simd-ml-kernels 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/build-and-test.yml +1 -1
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/ci.yml +4 -2
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/wheels.yml +1 -1
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.gitignore +14 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/AGENTS.md +25 -7
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CHANGELOG.md +108 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CITATION.cff +12 -8
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CMakeLists.txt +38 -9
- simd_ml_kernels-0.4.0/PKG-INFO +813 -0
- simd_ml_kernels-0.4.0/README.md +565 -0
- simd_ml_kernels-0.4.0/benchmarks/make_charts.py +599 -0
- simd_ml_kernels-0.4.0/benchmarks/make_tables.py +808 -0
- simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-19_290b91baad26.json +3973 -0
- simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-20_38cd710929a3_threads.json +10895 -0
- simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-20_42f64c7d76b3.json +4856 -0
- simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-20_da63d7762dfc_threads.json +10919 -0
- simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-21_f4a501dd2db4_llama.json +907 -0
- simd_ml_kernels-0.4.0/docs/BENCHMARKS.md +904 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/DESIGN.md +536 -31
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/README.md +3 -1
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/ROADMAP.md +65 -25
- simd_ml_kernels-0.4.0/docs/VALIDATION.md +80 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/api/python_api.md +91 -1
- simd_ml_kernels-0.4.0/docs/img/dispatch.svg +74 -0
- simd_ml_kernels-0.4.0/docs/img/gemv_roof.svg +117 -0
- simd_ml_kernels-0.4.0/docs/img/llama_vs_llamacpp.svg +132 -0
- simd_ml_kernels-0.4.0/docs/img/ratio_vs_baseline.svg +214 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/setup.md +5 -3
- simd_ml_kernels-0.4.0/examples/llama/bench.py +260 -0
- simd_ml_kernels-0.4.0/examples/llama/data/alice_ch1-2.txt +420 -0
- simd_ml_kernels-0.4.0/examples/llama/generate.py +63 -0
- simd_ml_kernels-0.4.0/examples/llama/gguf_reader.py +129 -0
- simd_ml_kernels-0.4.0/examples/llama/llama.py +564 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/pyproject.toml +10 -7
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/__init__.py +12 -3
- simd_ml_kernels-0.4.0/python/simd_ml_kernels/_cli.py +1501 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/_core.pyi +65 -4
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/bindings/pybind_entry.cpp +346 -15
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/dispatch/dispatch.cpp +36 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/dispatch/kernels.hpp +10 -2
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx2_gemm_packed.cpp +60 -3
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx2_gemm_packed.hpp +23 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx512_gemm_packed.cpp +27 -2
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx512vnni_gemm.cpp +26 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avxvnni_gemm.cpp +25 -0
- simd_ml_kernels-0.4.0/src/kernels/gemm/int8_rows.hpp +118 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/kernels_avx2.cpp +3 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/kernels_avx512.cpp +31 -1
- simd_ml_kernels-0.4.0/src/kernels/quant/q4_0.hpp +83 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_avx2.cpp +52 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_avx512vnni.cpp +40 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_avxvnni.cpp +39 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_rows.hpp +171 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_scalar.cpp +108 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q8_0.hpp +112 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_avx2.cpp +49 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_avx512vnni.cpp +45 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_avxvnni.cpp +39 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_rows.hpp +104 -0
- simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_scalar.cpp +209 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/scalar_kernels.cpp +4 -1
- simd_ml_kernels-0.4.0/src/kernels/stream_read.hpp +22 -0
- simd_ml_kernels-0.4.0/src/kernels/stream_read_avx2.cpp +60 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/CMakeLists.txt +2 -0
- simd_ml_kernels-0.4.0/tests/data/q4_0_ggml_v0.7.0.npz +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_bindings_edge_cases.py +121 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_cli.py +11 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gemm_int8.cpp +18 -0
- simd_ml_kernels-0.4.0/tests/test_llama.py +208 -0
- simd_ml_kernels-0.4.0/tests/test_llama_gguf.py +82 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_precision.py +421 -0
- simd_ml_kernels-0.4.0/tests/test_q4_0.cpp +237 -0
- simd_ml_kernels-0.4.0/tests/test_q8_0.cpp +240 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_runner.cpp +4 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_threads.cpp +154 -1
- simd_ml_kernels-0.3.0/PKG-INFO +0 -685
- simd_ml_kernels-0.3.0/README.md +0 -437
- simd_ml_kernels-0.3.0/benchmarks/make_tables.py +0 -330
- simd_ml_kernels-0.3.0/docs/BENCHMARKS.md +0 -435
- simd_ml_kernels-0.3.0/python/simd_ml_kernels/_cli.py +0 -763
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/bench.yml +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/guardrail.yml +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/publish.yml +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CLAUDE.md +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CONTRIBUTING.md +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/Dockerfile +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/LICENSE +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/README.md +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/cloud-xeon-emr-2vcpu_2026-10-18_255f27755c31.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/cloud-xeon-emr-2vcpu_2026-10-18_9029f1566ddb.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-17_7a698867be89.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_045918e51839.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_19aeaf83cd36.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_44194476d395.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_b193633ecec1.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_ecc219baa1f2.json +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/cmake/simd_ml_kernels-config.cmake.in +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/security.md +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/examples/bench_to_json.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/examples/linear_trans_b.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/examples/mlp_block.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_kernels/__init__.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/__main__.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/py.typed +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/bench_stat.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/dispatch/cpuid.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/activations.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/layer_norm_avx2.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/relu_avx2.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/silu_avx2.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/softmax_avx2.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/cache_alloc.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx512bf16_gemm.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/bf16_gemm.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/f16_gemm.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/int8_gemm.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/int8_pack.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/naive_gemm.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/intrinsic_gelu.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/intrinsic_gelu.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/simd_math.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/main_bench.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/check_isa_units.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/conftest.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/kernel_tables.hpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/stubtest_allowlist.txt +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_activations.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_alignment.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_dispatch.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_fp_env.py +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gelu.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gemm_lowp.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gemm_packed.cpp +0 -0
- {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_stubs.py +0 -0
|
@@ -145,7 +145,7 @@ jobs:
|
|
|
145
145
|
# ─── macOS: Apple Clang, scalar fallback path ───────────────────────────────
|
|
146
146
|
build-macos:
|
|
147
147
|
name: macOS / Apple Clang / Scalar fallback
|
|
148
|
-
runs-on: macos-
|
|
148
|
+
runs-on: macos-15 # Apple Silicon (ARM) — no AVX2; tests scalar fallback
|
|
149
149
|
steps:
|
|
150
150
|
- uses: actions/checkout@v4
|
|
151
151
|
- name: Install CMake and pybind11
|
|
@@ -187,8 +187,10 @@ jobs:
|
|
|
187
187
|
python3 examples/linear_trans_b.py
|
|
188
188
|
python3 examples/bench_to_json.py /tmp/example.json --sizes 128 --reps 3
|
|
189
189
|
|
|
190
|
-
- name: Benchmark tables match committed JSON
|
|
191
|
-
run:
|
|
190
|
+
- name: Benchmark tables and charts match committed JSON
|
|
191
|
+
run: |
|
|
192
|
+
python3 benchmarks/make_tables.py --check
|
|
193
|
+
python3 benchmarks/make_charts.py --check
|
|
192
194
|
|
|
193
195
|
# ─── ASan / UBSan (Clang-15, Ubuntu 22.04) ───────────────────────────────────
|
|
194
196
|
sanitizers:
|
|
@@ -76,6 +76,20 @@ gmon.out
|
|
|
76
76
|
# ─── Jupyter notebooks (if any) ───────────────────────────────────────────────
|
|
77
77
|
.ipynb_checkpoints/
|
|
78
78
|
|
|
79
|
+
# ─── Model weights and credentials (examples/llama) ───────────────────────────
|
|
80
|
+
# Never commit checkpoints, GGUFs or Hugging Face tokens; models live outside
|
|
81
|
+
# the repo (SIMD_ML_LLAMA_DIR).
|
|
82
|
+
*.safetensors
|
|
83
|
+
*.safetensors.index.json
|
|
84
|
+
*.gguf
|
|
85
|
+
*.pt
|
|
86
|
+
*.pth
|
|
87
|
+
*.ckpt
|
|
88
|
+
/models/
|
|
89
|
+
.cache/huggingface/
|
|
90
|
+
**/huggingface/token
|
|
91
|
+
hf_token*
|
|
92
|
+
|
|
79
93
|
# ─── Environment files ────────────────────────────────────────────────────────
|
|
80
94
|
.env
|
|
81
95
|
.envrc
|
|
@@ -23,6 +23,10 @@ export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1
|
|
|
23
23
|
taskset -c 2 python -m simd_ml_kernels bench --json benchmarks/results/<label>_<date>_<commit>.json \
|
|
24
24
|
--label <label> --core "<which core>"
|
|
25
25
|
python benchmarks/make_tables.py # regenerate README/BENCHMARKS tables from the JSON
|
|
26
|
+
python benchmarks/make_charts.py # redraw the README charts (docs/img/*.svg) from the same JSON
|
|
27
|
+
# Threaded decode GEMV (OpenMP build: SKBUILD_CMAKE_DEFINE=SIMD_ML_OPENMP=ON pip install .), its own run
|
|
28
|
+
taskset -c 0-3 python -m simd_ml_kernels bench --cases threads --threads 1,2,4 \
|
|
29
|
+
--json benchmarks/results/<label>_<date>_<commit>_threads.json --label <label> --core "<which cores>"
|
|
26
30
|
```
|
|
27
31
|
|
|
28
32
|
Pitfalls:
|
|
@@ -35,6 +39,8 @@ Pitfalls:
|
|
|
35
39
|
benchmarks only), `SIMD_ML_ENABLE_X86=OFF` (scalar-only build).
|
|
36
40
|
- `SIMD_ML_BF16_DOT=1` opts `sgemm_bf16` into the `vdpbf16ps` path (AVX512_BF16
|
|
37
41
|
CPUs; off by default, slower where measured).
|
|
42
|
+
- `SIMD_ML_GGML_TEST_BACKEND_OPS=<llama.cpp>/build/bin/test-backend-ops` adds ggml's
|
|
43
|
+
Q8_0 and Q4_0 MUL_MAT to `bench` (and the ggml byte-comparison tests); unset: "not measured".
|
|
38
44
|
- `SIMD_ML_ISA=scalar|avx2` caps the kernel table, e.g. to benchmark AVX2 on
|
|
39
45
|
an AVX-512 machine. AVX-512 without AVX-512 hardware: run under Intel SDE,
|
|
40
46
|
`sde64 -skx -- python -m simd_ml_kernels selftest` (or `-nhm` for no AVX).
|
|
@@ -49,7 +55,12 @@ src/kernels/kernels_{avx2,avx512}.cpp the AVX2 / AVX-512 tables
|
|
|
49
55
|
src/kernels/gemm/avx2_gemm_packed.{hpp,cpp} FP32/FP16/BF16 GEMM, AVX2: 6x16 kernel, direct (small) and 5-loop packed paths
|
|
50
56
|
src/kernels/gemm/avx512_gemm_packed.cpp the same on AVX-512 (12x32 kernel); avx512bf16_gemm.cpp: vdpbf16ps, CPUID-gated
|
|
51
57
|
src/kernels/gemm/{f16,bf16}_gemm.hpp FP16 / BF16-input GEMM declarations (inputs converted to FP32 while packing)
|
|
52
|
-
src/kernels/gemm/int8_gemm.hpp, *vnni_gemm.cpp INT8 GEMM (uint8 x int8, per-column scales): AVX-VNNI / AVX512_VNNI units
|
|
58
|
+
src/kernels/gemm/int8_gemm.hpp, *vnni_gemm.cpp INT8 GEMM (uint8 x int8, per-column scales): AVX-VNNI / AVX512_VNNI units;
|
|
59
|
+
int8_rows.hpp: its M = 1, trans_b row-streaming path
|
|
60
|
+
src/kernels/quant/q8_0* ggml Q8_0 quantization (scalar unit, no fast-math) and gemv_q8_0
|
|
61
|
+
(q8_0_rows.hpp: the YMM loop; avx2 / avxvnni / avx512vnni units)
|
|
62
|
+
src/kernels/quant/q4_0* ggml Q4_0 quantization and gemv_q4_0 (q4_0_rows.hpp: two rows per
|
|
63
|
+
YMM; x through the Q8_0 driver gemv_q8_0_run)
|
|
53
64
|
src/kernels/gemm/naive_gemm.hpp scalar reference GEMM (tests)
|
|
54
65
|
src/kernels/intrinsic_gelu.* GeLU exact (erf) and tanh (AVX2)
|
|
55
66
|
src/kernels/activations/ relu, silu, softmax, layer_norm (AVX2; + activations.hpp)
|
|
@@ -62,8 +73,11 @@ python/simd_kernels/ deprecated import alias (removed in
|
|
|
62
73
|
tests/test_*.cpp C++ tests: one binary, simd_tests, each suite run on every usable table
|
|
63
74
|
tests/check_isa_units.py ctest isa_units: no out-of-ISA instruction or shared inline code in a unit
|
|
64
75
|
tests/test_*.py float64 references, edge cases, CLI, stubs
|
|
76
|
+
tests/data/ fixtures from other projects (q4_0: llama.cpp's own blocks)
|
|
65
77
|
benchmarks/results/*.json committed benchmark results; make_tables.py renders them
|
|
66
78
|
examples/ short runnable scripts, run in CI
|
|
79
|
+
examples/llama/ Llama-3.x decode from our GEMVs + NumPy (not API; needs SIMD_ML_LLAMA_DIR,
|
|
80
|
+
not run in CI); tests/test_llama.py checks it against transformers
|
|
67
81
|
```
|
|
68
82
|
|
|
69
83
|
## Rules
|
|
@@ -143,16 +157,20 @@ schema, command, python, timestamp_utc
|
|
|
143
157
|
machine label, cpu_model, core, pinned_cpus, caches[] (CPUID leaf 4), logical_cpus, os, note,
|
|
144
158
|
stream_{read,triad}_gbs, stream_{read,triad}_allcore_gbs (null: no OpenMP or one CPU)
|
|
145
159
|
build version, git_commit, compiler, kernel_flags, isa_compiled, openmp,
|
|
146
|
-
sgemm_tile, sgemm_blocking {kc, mc, nc}, u8s8_path
|
|
147
|
-
baselines numpy, blas {name, version}, torch
|
|
160
|
+
sgemm_tile, sgemm_blocking {kc, mc, nc}, u8s8_path, q8_0_path
|
|
161
|
+
baselines numpy, blas {name, version}, torch, ggml {tag, commit, cmake, system_info, ...} | "not measured (...)"
|
|
148
162
|
env OPENBLAS_NUM_THREADS, OMP_NUM_THREADS, MKL_NUM_THREADS, SIMD_ML_ISA, SIMD_ML_BF16_DOT
|
|
149
|
-
protocol warmup, reps, statistic, interleaved, clock, min_ghz, malloc, stream, gemv
|
|
163
|
+
protocol warmup, reps, statistic, interleaved, clock, min_ghz, malloc, stream, gemv, gemv_roof,
|
|
164
|
+
threads {counts, method, llama_3_2_1b} (threaded runs)
|
|
150
165
|
results[] kernel, impl, shape, median_s, p25_s, p75_s, min_s, reps,
|
|
151
166
|
core_ghz {median, min, max, samples, attempts, below_min_ghz},
|
|
152
167
|
gflops + flop_per_cycle (GEMM; GEMV also bytes, gb_per_s, pct_stream_{read,triad},
|
|
153
|
-
weight_copies
|
|
168
|
+
weight_copies, roof_read_gbs, roof_unstable; Q8_0/Q4_0: err_vs_fp32; ggml rows: time_s,
|
|
169
|
+
runs, threads_max/timed, command; threaded runs: threads, llama_3_2_1b) | gelem_per_s + elem_per_cycle (elementwise) | gb_per_s (stream_*)
|
|
154
170
|
```
|
|
155
171
|
|
|
156
|
-
Files are named `benchmarks/results/<label>_<YYYY-MM-DD>_<commit>.json
|
|
172
|
+
Files are named `benchmarks/results/<label>_<YYYY-MM-DD>_<commit>.json` (threaded
|
|
173
|
+
runs: `..._threads.json`, rendered in their own blocks), one
|
|
157
174
|
machine per file. `benchmarks/make_tables.py` renders every file into the
|
|
158
|
-
generated blocks of README.md and docs/BENCHMARKS.md
|
|
175
|
+
generated blocks of README.md and docs/BENCHMARKS.md, and `benchmarks/make_charts.py`
|
|
176
|
+
draws the README charts in docs/img/ from the same runs; CI fails if either differs.
|
|
@@ -4,6 +4,114 @@ All notable user-visible changes. The project follows semantic versioning;
|
|
|
4
4
|
before 1.0, minor versions may contain breaking changes, which are listed
|
|
5
5
|
under **Changed** with a migration note.
|
|
6
6
|
|
|
7
|
+
## Unreleased
|
|
8
|
+
|
|
9
|
+
## 0.4.0 — 2026-11-21
|
|
10
|
+
|
|
11
|
+
Changes on main since 0.3.0. Not tagged and not published.
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
- Docs: the README opens with three charts drawn from the committed benchmark
|
|
15
|
+
JSON (`benchmarks/make_charts.py`, SVG in `docs/img/`; CI checks they are up
|
|
16
|
+
to date) and a dispatch diagram; the long tables are collapsed, and the
|
|
17
|
+
terms and the summary by run moved to `docs/BENCHMARKS.md`.
|
|
18
|
+
`docs/VALIDATION.md` records which kernel paths ran on real hardware before
|
|
19
|
+
0.4.0 and which did not.
|
|
20
|
+
- Docs: the README kernel × ISA matrix is the coverage claim. The
|
|
21
|
+
Llama-3.2-1B check states its prompt, token, weight and tolerance scope.
|
|
22
|
+
The Zenodo preprint is labeled as the earlier AVX2 kernels. `CITATION.cff`
|
|
23
|
+
matches this version.
|
|
24
|
+
- Threaded `gemv_q8_0` / `gemv_q4_0`: x is quantized by the thread team
|
|
25
|
+
(one run of blocks per thread) instead of the calling thread alone; same
|
|
26
|
+
quants, results bitwise unchanged. Rerun (`lnl-226v-wsl_2026-10-20_38cd710929a3_threads`): at 4 threads Q4_0
|
|
27
|
+
1.22–2.22× ggml on every shape (512×2048 was 0.97×), Llama-3.2-1B GEMV time
|
|
28
|
+
per token 10.4 ms (Q4_0) and 17.0 ms (Q8_0) against ggml's 14.5 and 21.6.
|
|
29
|
+
- `bench`: rows that reuse their weights on every call (ggml, our "same
|
|
30
|
+
weights" row) no longer count towards `roof_unstable`; the Llama-3.2-1B
|
|
31
|
+
shapes come from its config.json when `SIMD_ML_LLAMA_CONFIG` points at it.
|
|
32
|
+
|
|
33
|
+
### Added
|
|
34
|
+
- `examples/llama/`: Llama-3.x greedy decode built from the decode GEMV,
|
|
35
|
+
`silu`, `softmax` and NumPy (`generate.py --prompt ... --tokens 64
|
|
36
|
+
--threads N --weights fp32`); an example, not library API. With
|
|
37
|
+
`SIMD_ML_LLAMA_DIR` pointing at meta-llama/Llama-3.2-1B,
|
|
38
|
+
`tests/test_llama.py` checks it against transformers FP32: logits of the
|
|
39
|
+
first 8 positions within 1e-4 + 1e-5·|ref| and 64 greedy tokens identical
|
|
40
|
+
on 5 prompts (skipped without the variable). docs/DESIGN.md §9.
|
|
41
|
+
- `examples/llama/` on llama.cpp's own GGUF files (F32, Q8_0, Q4_0; blocks
|
|
42
|
+
read as is by `gemv_q8_0` / `gemv_q4_0`), `generate.py --weights
|
|
43
|
+
q8_0|q4_0`, `bench.py` (tokens/s against `llama-bench`, per-token
|
|
44
|
+
breakdown, perplexity against `llama-perplexity`) and README/BENCHMARKS
|
|
45
|
+
tables from its JSON. Laptop, 4 P-cores
|
|
46
|
+
(`lnl-226v-wsl_2026-10-21_f4a501dd2db4_llama`): 1.02–1.69× llama.cpp's
|
|
47
|
+
tokens/s at 1/2/4 threads (Q4_0 73.8 against 72.1 tokens/s at 4 threads);
|
|
48
|
+
perplexity within 0.018 of llama.cpp's.
|
|
49
|
+
- **Threaded decode GEMV** in OpenMP builds (`-DSIMD_ML_OPENMP=ON`): `sgemm` /
|
|
50
|
+
`sgemm_f16` / `sgemm_bf16` with `trans_b=True` and at most 8 rows of x, and
|
|
51
|
+
`gemv_q8_0` / `gemv_q4_0`, split W's rows into one contiguous slice per
|
|
52
|
+
thread (whole 8-row groups), up to `get_num_threads()` threads and at
|
|
53
|
+
least 64 KiB of weights per thread; results are bitwise the same for any
|
|
54
|
+
thread count. No new API. docs/DESIGN.md §2.10. Measured on the reference
|
|
55
|
+
laptop's 4 P-cores (`lnl-226v-wsl_2026-10-20_da63d7762dfc_threads`): at 4 threads Q8_0 1.54–1.67× and Q4_0
|
|
56
|
+
1.52–2.19× ggml's MUL_MAT at the same thread count on the same weights,
|
|
57
|
+
except Q4_0 512×2048 (0.97×); Llama-3.2-1B Q4_0 per-token GEMV time 11.6 ms
|
|
58
|
+
against ggml's 15.1 ms at 4 threads.
|
|
59
|
+
- `python -m simd_ml_kernels bench --cases threads [--threads 1,2,4]`: the
|
|
60
|
+
threaded GEMV against the read roof at the same thread count, PyTorch at
|
|
61
|
+
`torch.set_num_threads(N)` and ggml with `OMP_THREAD_LIMIT=N`, including
|
|
62
|
+
the Llama-3.2-1B decode shapes; written to its own `..._threads.json`.
|
|
63
|
+
|
|
64
|
+
### Fixed
|
|
65
|
+
- `build_info()["git_commit"]` is the bare 12-digit commit hash again (plus
|
|
66
|
+
`-dirty` for local changes). Since the v0.3.0 tag, CMake's `git describe`
|
|
67
|
+
had returned `v0.3.0-<n>-g<hash>`; benchmark JSONs recorded that form (one
|
|
68
|
+
file, `lnl-226v-wsl_2026-10-19_290b91baad26.json`, keeps it as recorded).
|
|
69
|
+
|
|
70
|
+
### Added
|
|
71
|
+
- **`quantize_q4_0(W)`** and **`gemv_q4_0(x, Wq, N, K)`**: decode GEMV on
|
|
72
|
+
weights in ggml's Q4_0 format (`block_q4_0`, byte-identical: float16
|
|
73
|
+
scale + 16 bytes of 4-bit values; quantization as `quantize_row_q4_0_ref`,
|
|
74
|
+
checked against blocks written by llama.cpp v0.7.0). x is quantized to Q8
|
|
75
|
+
blocks once per call, as for `gemv_q8_0`; exact integer block dot
|
|
76
|
+
products, two weight rows per register; same paths as
|
|
77
|
+
`build_info()['q8_0_path']`. K % 32 != 0 raises `ArgumentError`.
|
|
78
|
+
docs/DESIGN.md §2.9. Measured on the reference laptop's P-core
|
|
79
|
+
(`lnl-226v-wsl_2026-10-20_42f64c7d76b3`): 68–83% of the read roof at 1×4096×4096, 1×4096×11008 and
|
|
80
|
+
1×2048×8192; 1.38–1.67× our Q8_0 GEMV, 5.12–6.05× our FP32 GEMV, and
|
|
81
|
+
1.58–2.19× ggml's Q4_0 MUL_MAT (llama.cpp v0.7.0, one thread).
|
|
82
|
+
- `python -m simd_ml_kernels bench`: Q4_0 GEMV cases (`--cases q4_0`) against
|
|
83
|
+
our Q8_0 and FP32 GEMV, PyTorch's FP32 `F.linear` on the dequantized
|
|
84
|
+
weights and ggml's Q4_0 MUL_MAT on one thread.
|
|
85
|
+
- **`quantize_q8_0(W)`** and **`gemv_q8_0(x, Wq, N, K)`**: decode GEMV on
|
|
86
|
+
weights in ggml's Q8_0 format (`block_q8_0`, byte-identical: float16 scale
|
|
87
|
+
+ 32 int8 per block; quantization as `quantize_row_q8_0_ref`). x is
|
|
88
|
+
quantized to Q8 blocks once per call (float32 scales); exact integer
|
|
89
|
+
block dot products with `vpdpbusd` (AVX-VNNI, AVX512_VNNI) or
|
|
90
|
+
`vpmaddubsw` (AVX2), FP32 sums; `build_info()['q8_0_path']`. K % 32 != 0
|
|
91
|
+
raises `ArgumentError`. docs/DESIGN.md §2.8. Measured on the reference laptop's P-core
|
|
92
|
+
(`lnl-226v-wsl_2026-10-19_290b91baad26`): 92–95% of the read roof at 1×4096×4096,
|
|
93
|
+
1×4096×11008 and 1×2048×8192; 3.49–3.61× our FP32 GEMV, 1.77–2.05× our
|
|
94
|
+
BF16 GEMV, and 1.46–1.82× ggml's Q8_0 MUL_MAT (llama.cpp v0.7.0, one
|
|
95
|
+
thread, `test-backend-ops perf`).
|
|
96
|
+
- `python -m simd_ml_kernels bench`: Q8_0 GEMV cases against our FP32/BF16
|
|
97
|
+
GEMV, PyTorch's FP32 `F.linear` on the dequantized weights and, when
|
|
98
|
+
`SIMD_ML_GGML_TEST_BACKEND_OPS` names llama.cpp's `test-backend-ops`,
|
|
99
|
+
ggml's Q8_0 MUL_MAT on one thread; `--cases` to run a subset (development).
|
|
100
|
+
|
|
101
|
+
### Changed
|
|
102
|
+
- `sgemm_u8s8` with one row of A and `trans_b=True` (x·Wᵀ for one token)
|
|
103
|
+
reads B in place instead of packing it, on every x86 path (exact integer
|
|
104
|
+
sums; same results within the documented bound). At 1×4096×4096 on the
|
|
105
|
+
laptop: 1.36× PyTorch's quantized linear, was 0.46× (`lnl-226v-wsl_2026-10-19_290b91baad26`).
|
|
106
|
+
- `bench`: each GEMV case measures its own read roof (the read stream timed
|
|
107
|
+
interleaved with the case); `pct_stream_read` is relative to it
|
|
108
|
+
(`roof_read_gbs`). A case with an implementation above 105% of its own roof
|
|
109
|
+
is marked `roof_unstable` and not shown in the tables. The triad stays once
|
|
110
|
+
per run.
|
|
111
|
+
- docs/ROADMAP.md: milestones renamed from "v0.2 … v1.0" to "Milestone 1 … 9".
|
|
112
|
+
They were work stages, not package versions, and their names collided with
|
|
113
|
+
real versions (the old "v0.3" milestone is not package 0.3.0).
|
|
114
|
+
|
|
7
115
|
## 0.3.0 — 2026-10-19
|
|
8
116
|
|
|
9
117
|
First release on PyPI (`pip install simd-ml-kernels`); 0.3.0rc1 was on
|
|
@@ -3,27 +3,31 @@ title: "simd-ml-kernels: hand-vectorized SIMD kernels for machine learning workl
|
|
|
3
3
|
authors:
|
|
4
4
|
- family-names: "Myet"
|
|
5
5
|
given-names: "Min Htet"
|
|
6
|
-
version: "0.
|
|
7
|
-
date-released: "2026-
|
|
6
|
+
version: "0.4.0"
|
|
7
|
+
date-released: "2026-11-21"
|
|
8
8
|
license: "Apache-2.0"
|
|
9
9
|
repository-code: "https://github.com/Mattral/simd-ml-kernels"
|
|
10
10
|
url: "https://zenodo.org/records/20616502"
|
|
11
11
|
abstract: >
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
C++17 library of hand-vectorized x86 kernels for GEMM (FP32, FP16, BF16
|
|
13
|
+
and INT8), decode GEMV, ggml Q8_0 and Q4_0 GEMV, and GeLU, ReLU, SiLU,
|
|
14
|
+
softmax and LayerNorm, with Python bindings. Every kernel has a portable
|
|
15
|
+
scalar path. AVX2 covers every kernel. AVX-512 covers the FP32, FP16 and
|
|
16
|
+
BF16 GEMMs and the decode GEMV. AVX-VNNI and AVX512-VNNI cover INT8 GEMM
|
|
17
|
+
and the Q8_0 and Q4_0 GEMVs. Opt-in AVX512-BF16 dot covers packed BF16
|
|
18
|
+
GEMM. The Zenodo preprint describes the earlier AVX2 kernels; current
|
|
19
|
+
scope and results are in the repository README.
|
|
17
20
|
keywords:
|
|
18
21
|
- simd
|
|
19
22
|
- avx2
|
|
20
23
|
- avx-512
|
|
24
|
+
- avx-vnni
|
|
21
25
|
- gemm
|
|
26
|
+
- gemv
|
|
22
27
|
- machine-learning
|
|
23
28
|
- microkernels
|
|
24
29
|
- gelu
|
|
25
30
|
- layer-normalization
|
|
26
|
-
- high-performance-computing
|
|
27
31
|
- cpp
|
|
28
32
|
preferred-citation:
|
|
29
33
|
type: unpublished
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
cmake_minimum_required(VERSION 3.22)
|
|
2
|
-
project(simd_ml_kernels VERSION 0.
|
|
2
|
+
project(simd_ml_kernels VERSION 0.4.0 LANGUAGES CXX)
|
|
3
3
|
# CMake's VERSION is numeric only; the PEP 440 pre-release suffix of
|
|
4
4
|
# pyproject.toml's version goes here ("" for a release). build_info()["version"]
|
|
5
5
|
# is "${PROJECT_VERSION}${SIMD_ML_PRERELEASE}" and must equal the package version.
|
|
@@ -105,6 +105,8 @@ endif()
|
|
|
105
105
|
set(SIMD_ML_SCALAR_SRCS
|
|
106
106
|
src/dispatch/dispatch.cpp
|
|
107
107
|
src/kernels/scalar_kernels.cpp
|
|
108
|
+
src/kernels/quant/q8_0_scalar.cpp
|
|
109
|
+
src/kernels/quant/q4_0_scalar.cpp
|
|
108
110
|
)
|
|
109
111
|
set(SIMD_ML_AVX2_SRCS
|
|
110
112
|
src/kernels/kernels_avx2.cpp
|
|
@@ -114,6 +116,9 @@ set(SIMD_ML_AVX2_SRCS
|
|
|
114
116
|
src/kernels/activations/silu_avx2.cpp
|
|
115
117
|
src/kernels/activations/softmax_avx2.cpp
|
|
116
118
|
src/kernels/activations/layer_norm_avx2.cpp
|
|
119
|
+
src/kernels/quant/q8_0_avx2.cpp
|
|
120
|
+
src/kernels/quant/q4_0_avx2.cpp
|
|
121
|
+
src/kernels/stream_read_avx2.cpp
|
|
117
122
|
)
|
|
118
123
|
set(SIMD_ML_AVX512_SRCS
|
|
119
124
|
src/kernels/kernels_avx512.cpp
|
|
@@ -123,12 +128,14 @@ set(SIMD_ML_AVX512_SRCS
|
|
|
123
128
|
# Extension units (no kernel table of their own), each called from a table
|
|
124
129
|
# unit after a CPUID check, and built only if the compiler accepts its flag:
|
|
125
130
|
# avx512bf16 vdpbf16ps BF16 GEMM, from the AVX-512 table (GCC 10+, Clang 9+)
|
|
126
|
-
# avxvnni AVX-VNNI INT8 GEMM, from the AVX2 table (GCC 11+, Clang 12+)
|
|
127
|
-
# avx512vnni AVX512_VNNI INT8 GEMM, from the AVX-512 table (GCC 8+, Clang 6+)
|
|
131
|
+
# avxvnni AVX-VNNI INT8 GEMM and Q8_0/Q4_0 GEMV, from the AVX2 table (GCC 11+, Clang 12+)
|
|
132
|
+
# avx512vnni AVX512_VNNI INT8 GEMM and Q8_0/Q4_0 GEMV, from the AVX-512 table (GCC 8+, Clang 6+)
|
|
128
133
|
# The calling unit gets SIMD_ML_HAVE_<UNIT>.
|
|
129
134
|
set(SIMD_ML_AVX512BF16_SRCS src/kernels/gemm/avx512bf16_gemm.cpp)
|
|
130
|
-
set(SIMD_ML_AVXVNNI_SRCS src/kernels/gemm/avxvnni_gemm.cpp
|
|
131
|
-
|
|
135
|
+
set(SIMD_ML_AVXVNNI_SRCS src/kernels/gemm/avxvnni_gemm.cpp src/kernels/quant/q8_0_avxvnni.cpp
|
|
136
|
+
src/kernels/quant/q4_0_avxvnni.cpp)
|
|
137
|
+
set(SIMD_ML_AVX512VNNI_SRCS src/kernels/gemm/avx512vnni_gemm.cpp src/kernels/quant/q8_0_avx512vnni.cpp
|
|
138
|
+
src/kernels/quant/q4_0_avx512vnni.cpp)
|
|
132
139
|
set(_ext_units avx512bf16 avxvnni avx512vnni)
|
|
133
140
|
set(_ext_flags -mavx512bf16 -mavxvnni -mavx512vnni)
|
|
134
141
|
set(_ext_parent avx512 avx2 avx512)
|
|
@@ -173,6 +180,14 @@ foreach(isa IN LISTS SIMD_ML_ISAS)
|
|
|
173
180
|
list(APPEND SIMD_ML_ISA_OBJECTS $<TARGET_OBJECTS:simd_ml_isa_${isa}>)
|
|
174
181
|
message(STATUS "ISA unit ${isa}: ${_flags_str}")
|
|
175
182
|
endforeach()
|
|
183
|
+
# Q4_0 quantization rounds w·id, then w·id + 8.5, in two binary32 steps, as
|
|
184
|
+
# ggml's reference built for baseline x86-64. GCC contracts the pair into
|
|
185
|
+
# one FMA by default (-ffp-contract=fast) wherever FMA exists (aarch64,
|
|
186
|
+
# SIMD_ML_NATIVE), and the fused result truncates differently next to an
|
|
187
|
+
# integer, so the blocks would no longer match ggml's byte for byte.
|
|
188
|
+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang")
|
|
189
|
+
set_source_files_properties(src/kernels/quant/q4_0_scalar.cpp PROPERTIES COMPILE_OPTIONS -ffp-contract=off)
|
|
190
|
+
endif()
|
|
176
191
|
foreach(ext parent IN ZIP_LISTS _ext_units _ext_parent)
|
|
177
192
|
if(ext IN_LIST SIMD_ML_ISAS)
|
|
178
193
|
string(TOUPPER ${ext} EXT)
|
|
@@ -217,13 +232,27 @@ if(pybind11_FOUND)
|
|
|
217
232
|
# Build metadata reported by build_info() and `python -m simd_ml_kernels
|
|
218
233
|
# info`, and recorded in benchmark JSON. The commit is taken when CMake
|
|
219
234
|
# configures, so a build directory reused across commits reports the
|
|
220
|
-
# commit it was configured at
|
|
235
|
+
# commit it was configured at: the first 12 hex digits of HEAD (exactly
|
|
236
|
+
# 12; `git describe` gave "v0.3.0-4-g<hash>" once a tag existed), plus
|
|
237
|
+
# "-dirty" if tracked files differ from HEAD (`git status` refreshes the
|
|
238
|
+
# index first, so a touched but unchanged file does not count, as with
|
|
239
|
+
# `describe --dirty`). Without git (an sdist build): "unknown".
|
|
221
240
|
execute_process(
|
|
222
|
-
COMMAND git
|
|
241
|
+
COMMAND git rev-parse HEAD
|
|
223
242
|
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
|
|
224
|
-
OUTPUT_VARIABLE
|
|
243
|
+
OUTPUT_VARIABLE _simd_ml_head OUTPUT_STRIP_TRAILING_WHITESPACE
|
|
225
244
|
RESULT_VARIABLE _simd_ml_git_rc ERROR_QUIET)
|
|
226
|
-
if(
|
|
245
|
+
if(_simd_ml_git_rc EQUAL 0 AND _simd_ml_head MATCHES "^[0-9a-f]+$")
|
|
246
|
+
string(SUBSTRING "${_simd_ml_head}" 0 12 _simd_ml_git)
|
|
247
|
+
execute_process(
|
|
248
|
+
COMMAND git status --porcelain --untracked-files=no
|
|
249
|
+
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
|
|
250
|
+
OUTPUT_VARIABLE _simd_ml_git_status OUTPUT_STRIP_TRAILING_WHITESPACE
|
|
251
|
+
RESULT_VARIABLE _simd_ml_status_rc ERROR_QUIET)
|
|
252
|
+
if(NOT _simd_ml_status_rc EQUAL 0 OR NOT _simd_ml_git_status STREQUAL "")
|
|
253
|
+
string(APPEND _simd_ml_git "-dirty")
|
|
254
|
+
endif()
|
|
255
|
+
else()
|
|
227
256
|
set(_simd_ml_git "unknown")
|
|
228
257
|
endif()
|
|
229
258
|
target_compile_definitions(simd_ml_kernels_core PRIVATE
|