simd-ml-kernels 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/build-and-test.yml +1 -1
  2. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/ci.yml +4 -2
  3. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/wheels.yml +1 -1
  4. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.gitignore +14 -0
  5. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/AGENTS.md +25 -7
  6. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CHANGELOG.md +108 -0
  7. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CITATION.cff +12 -8
  8. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CMakeLists.txt +38 -9
  9. simd_ml_kernels-0.4.0/PKG-INFO +813 -0
  10. simd_ml_kernels-0.4.0/README.md +565 -0
  11. simd_ml_kernels-0.4.0/benchmarks/make_charts.py +599 -0
  12. simd_ml_kernels-0.4.0/benchmarks/make_tables.py +808 -0
  13. simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-19_290b91baad26.json +3973 -0
  14. simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-20_38cd710929a3_threads.json +10895 -0
  15. simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-20_42f64c7d76b3.json +4856 -0
  16. simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-20_da63d7762dfc_threads.json +10919 -0
  17. simd_ml_kernels-0.4.0/benchmarks/results/lnl-226v-wsl_2026-10-21_f4a501dd2db4_llama.json +907 -0
  18. simd_ml_kernels-0.4.0/docs/BENCHMARKS.md +904 -0
  19. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/DESIGN.md +536 -31
  20. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/README.md +3 -1
  21. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/ROADMAP.md +65 -25
  22. simd_ml_kernels-0.4.0/docs/VALIDATION.md +80 -0
  23. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/api/python_api.md +91 -1
  24. simd_ml_kernels-0.4.0/docs/img/dispatch.svg +74 -0
  25. simd_ml_kernels-0.4.0/docs/img/gemv_roof.svg +117 -0
  26. simd_ml_kernels-0.4.0/docs/img/llama_vs_llamacpp.svg +132 -0
  27. simd_ml_kernels-0.4.0/docs/img/ratio_vs_baseline.svg +214 -0
  28. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/setup.md +5 -3
  29. simd_ml_kernels-0.4.0/examples/llama/bench.py +260 -0
  30. simd_ml_kernels-0.4.0/examples/llama/data/alice_ch1-2.txt +420 -0
  31. simd_ml_kernels-0.4.0/examples/llama/generate.py +63 -0
  32. simd_ml_kernels-0.4.0/examples/llama/gguf_reader.py +129 -0
  33. simd_ml_kernels-0.4.0/examples/llama/llama.py +564 -0
  34. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/pyproject.toml +10 -7
  35. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/__init__.py +12 -3
  36. simd_ml_kernels-0.4.0/python/simd_ml_kernels/_cli.py +1501 -0
  37. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/_core.pyi +65 -4
  38. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/bindings/pybind_entry.cpp +346 -15
  39. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/dispatch/dispatch.cpp +36 -0
  40. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/dispatch/kernels.hpp +10 -2
  41. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx2_gemm_packed.cpp +60 -3
  42. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx2_gemm_packed.hpp +23 -0
  43. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx512_gemm_packed.cpp +27 -2
  44. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx512vnni_gemm.cpp +26 -0
  45. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avxvnni_gemm.cpp +25 -0
  46. simd_ml_kernels-0.4.0/src/kernels/gemm/int8_rows.hpp +118 -0
  47. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/kernels_avx2.cpp +3 -0
  48. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/kernels_avx512.cpp +31 -1
  49. simd_ml_kernels-0.4.0/src/kernels/quant/q4_0.hpp +83 -0
  50. simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_avx2.cpp +52 -0
  51. simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_avx512vnni.cpp +40 -0
  52. simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_avxvnni.cpp +39 -0
  53. simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_rows.hpp +171 -0
  54. simd_ml_kernels-0.4.0/src/kernels/quant/q4_0_scalar.cpp +108 -0
  55. simd_ml_kernels-0.4.0/src/kernels/quant/q8_0.hpp +112 -0
  56. simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_avx2.cpp +49 -0
  57. simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_avx512vnni.cpp +45 -0
  58. simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_avxvnni.cpp +39 -0
  59. simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_rows.hpp +104 -0
  60. simd_ml_kernels-0.4.0/src/kernels/quant/q8_0_scalar.cpp +209 -0
  61. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/scalar_kernels.cpp +4 -1
  62. simd_ml_kernels-0.4.0/src/kernels/stream_read.hpp +22 -0
  63. simd_ml_kernels-0.4.0/src/kernels/stream_read_avx2.cpp +60 -0
  64. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/CMakeLists.txt +2 -0
  65. simd_ml_kernels-0.4.0/tests/data/q4_0_ggml_v0.7.0.npz +0 -0
  66. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_bindings_edge_cases.py +121 -0
  67. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_cli.py +11 -0
  68. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gemm_int8.cpp +18 -0
  69. simd_ml_kernels-0.4.0/tests/test_llama.py +208 -0
  70. simd_ml_kernels-0.4.0/tests/test_llama_gguf.py +82 -0
  71. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_precision.py +421 -0
  72. simd_ml_kernels-0.4.0/tests/test_q4_0.cpp +237 -0
  73. simd_ml_kernels-0.4.0/tests/test_q8_0.cpp +240 -0
  74. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_runner.cpp +4 -0
  75. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_threads.cpp +154 -1
  76. simd_ml_kernels-0.3.0/PKG-INFO +0 -685
  77. simd_ml_kernels-0.3.0/README.md +0 -437
  78. simd_ml_kernels-0.3.0/benchmarks/make_tables.py +0 -330
  79. simd_ml_kernels-0.3.0/docs/BENCHMARKS.md +0 -435
  80. simd_ml_kernels-0.3.0/python/simd_ml_kernels/_cli.py +0 -763
  81. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/bench.yml +0 -0
  82. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/guardrail.yml +0 -0
  83. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/.github/workflows/publish.yml +0 -0
  84. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CLAUDE.md +0 -0
  85. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/CONTRIBUTING.md +0 -0
  86. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/Dockerfile +0 -0
  87. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/LICENSE +0 -0
  88. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/README.md +0 -0
  89. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/cloud-xeon-emr-2vcpu_2026-10-18_255f27755c31.json +0 -0
  90. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/cloud-xeon-emr-2vcpu_2026-10-18_9029f1566ddb.json +0 -0
  91. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-17_7a698867be89.json +0 -0
  92. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_045918e51839.json +0 -0
  93. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_19aeaf83cd36.json +0 -0
  94. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_44194476d395.json +0 -0
  95. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_b193633ecec1.json +0 -0
  96. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/benchmarks/results/lnl-226v-wsl_2026-10-18_ecc219baa1f2.json +0 -0
  97. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/cmake/simd_ml_kernels-config.cmake.in +0 -0
  98. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/docs/security.md +0 -0
  99. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/examples/bench_to_json.py +0 -0
  100. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/examples/linear_trans_b.py +0 -0
  101. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/examples/mlp_block.py +0 -0
  102. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_kernels/__init__.py +0 -0
  103. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/__main__.py +0 -0
  104. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/python/simd_ml_kernels/py.typed +0 -0
  105. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/bench_stat.cpp +0 -0
  106. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/dispatch/cpuid.hpp +0 -0
  107. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/activations.hpp +0 -0
  108. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/layer_norm_avx2.cpp +0 -0
  109. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/relu_avx2.cpp +0 -0
  110. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/silu_avx2.cpp +0 -0
  111. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/activations/softmax_avx2.cpp +0 -0
  112. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/cache_alloc.hpp +0 -0
  113. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/avx512bf16_gemm.cpp +0 -0
  114. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/bf16_gemm.hpp +0 -0
  115. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/f16_gemm.hpp +0 -0
  116. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/int8_gemm.hpp +0 -0
  117. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/int8_pack.hpp +0 -0
  118. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/gemm/naive_gemm.hpp +0 -0
  119. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/intrinsic_gelu.cpp +0 -0
  120. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/intrinsic_gelu.hpp +0 -0
  121. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/kernels/simd_math.hpp +0 -0
  122. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/src/main_bench.cpp +0 -0
  123. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/check_isa_units.py +0 -0
  124. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/conftest.py +0 -0
  125. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/kernel_tables.hpp +0 -0
  126. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/stubtest_allowlist.txt +0 -0
  127. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_activations.cpp +0 -0
  128. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_alignment.cpp +0 -0
  129. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_dispatch.cpp +0 -0
  130. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_fp_env.py +0 -0
  131. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gelu.cpp +0 -0
  132. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gemm_lowp.cpp +0 -0
  133. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_gemm_packed.cpp +0 -0
  134. {simd_ml_kernels-0.3.0 → simd_ml_kernels-0.4.0}/tests/test_stubs.py +0 -0
@@ -145,7 +145,7 @@ jobs:
145
145
  # ─── macOS: Apple Clang, scalar fallback path ───────────────────────────────
146
146
  build-macos:
147
147
  name: macOS / Apple Clang / Scalar fallback
148
- runs-on: macos-14 # Apple Silicon (ARM) — no AVX2; tests scalar fallback
148
+ runs-on: macos-15 # Apple Silicon (ARM) — no AVX2; tests scalar fallback
149
149
  steps:
150
150
  - uses: actions/checkout@v4
151
151
  - name: Install CMake and pybind11
@@ -187,8 +187,10 @@ jobs:
187
187
  python3 examples/linear_trans_b.py
188
188
  python3 examples/bench_to_json.py /tmp/example.json --sizes 128 --reps 3
189
189
 
190
- - name: Benchmark tables match committed JSON
191
- run: python3 benchmarks/make_tables.py --check
190
+ - name: Benchmark tables and charts match committed JSON
191
+ run: |
192
+ python3 benchmarks/make_tables.py --check
193
+ python3 benchmarks/make_charts.py --check
192
194
 
193
195
  # ─── ASan / UBSan (Clang-15, Ubuntu 22.04) ───────────────────────────────────
194
196
  sanitizers:
@@ -26,7 +26,7 @@ jobs:
26
26
  strategy:
27
27
  fail-fast: false
28
28
  matrix:
29
- os: [ubuntu-24.04, macos-14]
29
+ os: [ubuntu-24.04, macos-15]
30
30
  steps:
31
31
  - uses: actions/checkout@v4
32
32
  - name: Build and test wheels
@@ -76,6 +76,20 @@ gmon.out
76
76
  # ─── Jupyter notebooks (if any) ───────────────────────────────────────────────
77
77
  .ipynb_checkpoints/
78
78
 
79
+ # ─── Model weights and credentials (examples/llama) ───────────────────────────
80
+ # Never commit checkpoints, GGUFs or Hugging Face tokens; models live outside
81
+ # the repo (SIMD_ML_LLAMA_DIR).
82
+ *.safetensors
83
+ *.safetensors.index.json
84
+ *.gguf
85
+ *.pt
86
+ *.pth
87
+ *.ckpt
88
+ /models/
89
+ .cache/huggingface/
90
+ **/huggingface/token
91
+ hf_token*
92
+
79
93
  # ─── Environment files ────────────────────────────────────────────────────────
80
94
  .env
81
95
  .envrc
@@ -23,6 +23,10 @@ export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1
23
23
  taskset -c 2 python -m simd_ml_kernels bench --json benchmarks/results/<label>_<date>_<commit>.json \
24
24
  --label <label> --core "<which core>"
25
25
  python benchmarks/make_tables.py # regenerate README/BENCHMARKS tables from the JSON
26
+ python benchmarks/make_charts.py # redraw the README charts (docs/img/*.svg) from the same JSON
27
+ # Threaded decode GEMV (OpenMP build: SKBUILD_CMAKE_DEFINE=SIMD_ML_OPENMP=ON pip install .), its own run
28
+ taskset -c 0-3 python -m simd_ml_kernels bench --cases threads --threads 1,2,4 \
29
+ --json benchmarks/results/<label>_<date>_<commit>_threads.json --label <label> --core "<which cores>"
26
30
  ```
27
31
 
28
32
  Pitfalls:
@@ -35,6 +39,8 @@ Pitfalls:
35
39
  benchmarks only), `SIMD_ML_ENABLE_X86=OFF` (scalar-only build).
36
40
  - `SIMD_ML_BF16_DOT=1` opts `sgemm_bf16` into the `vdpbf16ps` path (AVX512_BF16
37
41
  CPUs; off by default, slower where measured).
42
+ - `SIMD_ML_GGML_TEST_BACKEND_OPS=<llama.cpp>/build/bin/test-backend-ops` adds ggml's
43
+ Q8_0 and Q4_0 MUL_MAT to `bench` (and the ggml byte-comparison tests); unset: "not measured".
38
44
  - `SIMD_ML_ISA=scalar|avx2` caps the kernel table, e.g. to benchmark AVX2 on
39
45
  an AVX-512 machine. AVX-512 without AVX-512 hardware: run under Intel SDE,
40
46
  `sde64 -skx -- python -m simd_ml_kernels selftest` (or `-nhm` for no AVX).
@@ -49,7 +55,12 @@ src/kernels/kernels_{avx2,avx512}.cpp the AVX2 / AVX-512 tables
49
55
  src/kernels/gemm/avx2_gemm_packed.{hpp,cpp} FP32/FP16/BF16 GEMM, AVX2: 6x16 kernel, direct (small) and 5-loop packed paths
50
56
  src/kernels/gemm/avx512_gemm_packed.cpp the same on AVX-512 (12x32 kernel); avx512bf16_gemm.cpp: vdpbf16ps, CPUID-gated
51
57
  src/kernels/gemm/{f16,bf16}_gemm.hpp FP16 / BF16-input GEMM declarations (inputs converted to FP32 while packing)
52
- src/kernels/gemm/int8_gemm.hpp, *vnni_gemm.cpp INT8 GEMM (uint8 x int8, per-column scales): AVX-VNNI / AVX512_VNNI units
58
+ src/kernels/gemm/int8_gemm.hpp, *vnni_gemm.cpp INT8 GEMM (uint8 x int8, per-column scales): AVX-VNNI / AVX512_VNNI units;
59
+ int8_rows.hpp: its M = 1, trans_b row-streaming path
60
+ src/kernels/quant/q8_0* ggml Q8_0 quantization (scalar unit, no fast-math) and gemv_q8_0
61
+ (q8_0_rows.hpp: the YMM loop; avx2 / avxvnni / avx512vnni units)
62
+ src/kernels/quant/q4_0* ggml Q4_0 quantization and gemv_q4_0 (q4_0_rows.hpp: two rows per
63
+ YMM; x through the Q8_0 driver gemv_q8_0_run)
53
64
  src/kernels/gemm/naive_gemm.hpp scalar reference GEMM (tests)
54
65
  src/kernels/intrinsic_gelu.* GeLU exact (erf) and tanh (AVX2)
55
66
  src/kernels/activations/ relu, silu, softmax, layer_norm (AVX2; + activations.hpp)
@@ -62,8 +73,11 @@ python/simd_kernels/ deprecated import alias (removed in
62
73
  tests/test_*.cpp C++ tests: one binary, simd_tests, each suite run on every usable table
63
74
  tests/check_isa_units.py ctest isa_units: no out-of-ISA instruction or shared inline code in a unit
64
75
  tests/test_*.py float64 references, edge cases, CLI, stubs
76
+ tests/data/ fixtures from other projects (q4_0: llama.cpp's own blocks)
65
77
  benchmarks/results/*.json committed benchmark results; make_tables.py renders them
66
78
  examples/ short runnable scripts, run in CI
79
+ examples/llama/ Llama-3.x decode from our GEMVs + NumPy (not API; needs SIMD_ML_LLAMA_DIR,
80
+ not run in CI); tests/test_llama.py checks it against transformers
67
81
  ```
68
82
 
69
83
  ## Rules
@@ -143,16 +157,20 @@ schema, command, python, timestamp_utc
143
157
  machine label, cpu_model, core, pinned_cpus, caches[] (CPUID leaf 4), logical_cpus, os, note,
144
158
  stream_{read,triad}_gbs, stream_{read,triad}_allcore_gbs (null: no OpenMP or one CPU)
145
159
  build version, git_commit, compiler, kernel_flags, isa_compiled, openmp,
146
- sgemm_tile, sgemm_blocking {kc, mc, nc}, u8s8_path
147
- baselines numpy, blas {name, version}, torch
160
+ sgemm_tile, sgemm_blocking {kc, mc, nc}, u8s8_path, q8_0_path
161
+ baselines numpy, blas {name, version}, torch, ggml {tag, commit, cmake, system_info, ...} | "not measured (...)"
148
162
  env OPENBLAS_NUM_THREADS, OMP_NUM_THREADS, MKL_NUM_THREADS, SIMD_ML_ISA, SIMD_ML_BF16_DOT
149
- protocol warmup, reps, statistic, interleaved, clock, min_ghz, malloc, stream, gemv
163
+ protocol warmup, reps, statistic, interleaved, clock, min_ghz, malloc, stream, gemv, gemv_roof,
164
+ threads {counts, method, llama_3_2_1b} (threaded runs)
150
165
  results[] kernel, impl, shape, median_s, p25_s, p75_s, min_s, reps,
151
166
  core_ghz {median, min, max, samples, attempts, below_min_ghz},
152
167
  gflops + flop_per_cycle (GEMM; GEMV also bytes, gb_per_s, pct_stream_{read,triad},
153
- weight_copies) | gelem_per_s + elem_per_cycle (elementwise) | gb_per_s (stream_*)
168
+ weight_copies, roof_read_gbs, roof_unstable; Q8_0/Q4_0: err_vs_fp32; ggml rows: time_s,
169
+ runs, threads_max/timed, command; threaded runs: threads, llama_3_2_1b) | gelem_per_s + elem_per_cycle (elementwise) | gb_per_s (stream_*)
154
170
  ```
155
171
 
156
- Files are named `benchmarks/results/<label>_<YYYY-MM-DD>_<commit>.json`, one
172
+ Files are named `benchmarks/results/<label>_<YYYY-MM-DD>_<commit>.json` (threaded
173
+ runs: `..._threads.json`, rendered in their own blocks), one
157
174
  machine per file. `benchmarks/make_tables.py` renders every file into the
158
- generated blocks of README.md and docs/BENCHMARKS.md; CI fails if they differ.
175
+ generated blocks of README.md and docs/BENCHMARKS.md, and `benchmarks/make_charts.py`
176
+ draws the README charts in docs/img/ from the same runs; CI fails if either differs.
@@ -4,6 +4,114 @@ All notable user-visible changes. The project follows semantic versioning;
4
4
  before 1.0, minor versions may contain breaking changes, which are listed
5
5
  under **Changed** with a migration note.
6
6
 
7
+ ## Unreleased
8
+
9
+ ## 0.4.0 — 2026-11-21
10
+
11
+ Changes on main since 0.3.0. Not tagged and not published.
12
+
13
+ ### Changed
14
+ - Docs: the README opens with three charts drawn from the committed benchmark
15
+ JSON (`benchmarks/make_charts.py`, SVG in `docs/img/`; CI checks they are up
16
+ to date) and a dispatch diagram; the long tables are collapsed, and the
17
+ terms and the summary by run moved to `docs/BENCHMARKS.md`.
18
+ `docs/VALIDATION.md` records which kernel paths ran on real hardware before
19
+ 0.4.0 and which did not.
20
+ - Docs: the README kernel × ISA matrix is the coverage claim. The
21
+ Llama-3.2-1B check states its prompt, token, weight and tolerance scope.
22
+ The Zenodo preprint is labeled as the earlier AVX2 kernels. `CITATION.cff`
23
+ matches this version.
24
+ - Threaded `gemv_q8_0` / `gemv_q4_0`: x is quantized by the thread team
25
+ (one run of blocks per thread) instead of the calling thread alone; same
26
+ quants, results bitwise unchanged. Rerun (`lnl-226v-wsl_2026-10-20_38cd710929a3_threads`): at 4 threads Q4_0
27
+ 1.22–2.22× ggml on every shape (512×2048 was 0.97×), Llama-3.2-1B GEMV time
28
+ per token 10.4 ms (Q4_0) and 17.0 ms (Q8_0) against ggml's 14.5 and 21.6.
29
+ - `bench`: rows that reuse their weights on every call (ggml, our "same
30
+ weights" row) no longer count towards `roof_unstable`; the Llama-3.2-1B
31
+ shapes come from its config.json when `SIMD_ML_LLAMA_CONFIG` points at it.
32
+
33
+ ### Added
34
+ - `examples/llama/`: Llama-3.x greedy decode built from the decode GEMV,
35
+ `silu`, `softmax` and NumPy (`generate.py --prompt ... --tokens 64
36
+ --threads N --weights fp32`); an example, not library API. With
37
+ `SIMD_ML_LLAMA_DIR` pointing at meta-llama/Llama-3.2-1B,
38
+ `tests/test_llama.py` checks it against transformers FP32: logits of the
39
+ first 8 positions within 1e-4 + 1e-5·|ref| and 64 greedy tokens identical
40
+ on 5 prompts (skipped without the variable). docs/DESIGN.md §9.
41
+ - `examples/llama/` on llama.cpp's own GGUF files (F32, Q8_0, Q4_0; blocks
42
+ read as is by `gemv_q8_0` / `gemv_q4_0`), `generate.py --weights
43
+ q8_0|q4_0`, `bench.py` (tokens/s against `llama-bench`, per-token
44
+ breakdown, perplexity against `llama-perplexity`) and README/BENCHMARKS
45
+ tables from its JSON. Laptop, 4 P-cores
46
+ (`lnl-226v-wsl_2026-10-21_f4a501dd2db4_llama`): 1.02–1.69× llama.cpp's
47
+ tokens/s at 1/2/4 threads (Q4_0 73.8 against 72.1 tokens/s at 4 threads);
48
+ perplexity within 0.018 of llama.cpp's.
49
+ - **Threaded decode GEMV** in OpenMP builds (`-DSIMD_ML_OPENMP=ON`): `sgemm` /
50
+ `sgemm_f16` / `sgemm_bf16` with `trans_b=True` and at most 8 rows of x, and
51
+ `gemv_q8_0` / `gemv_q4_0`, split W's rows into one contiguous slice per
52
+ thread (whole 8-row groups), up to `get_num_threads()` threads and at
53
+ least 64 KiB of weights per thread; results are bitwise the same for any
54
+ thread count. No new API. docs/DESIGN.md §2.10. Measured on the reference
55
+ laptop's 4 P-cores (`lnl-226v-wsl_2026-10-20_da63d7762dfc_threads`): at 4 threads Q8_0 1.54–1.67× and Q4_0
56
+ 1.52–2.19× ggml's MUL_MAT at the same thread count on the same weights,
57
+ except Q4_0 512×2048 (0.97×); Llama-3.2-1B Q4_0 per-token GEMV time 11.6 ms
58
+ against ggml's 15.1 ms at 4 threads.
59
+ - `python -m simd_ml_kernels bench --cases threads [--threads 1,2,4]`: the
60
+ threaded GEMV against the read roof at the same thread count, PyTorch at
61
+ `torch.set_num_threads(N)` and ggml with `OMP_THREAD_LIMIT=N`, including
62
+ the Llama-3.2-1B decode shapes; written to its own `..._threads.json`.
63
+
64
+ ### Fixed
65
+ - `build_info()["git_commit"]` is the bare 12-digit commit hash again (plus
66
+ `-dirty` for local changes). Since the v0.3.0 tag, CMake's `git describe`
67
+ had returned `v0.3.0-<n>-g<hash>`; benchmark JSONs recorded that form (one
68
+ file, `lnl-226v-wsl_2026-10-19_290b91baad26.json`, keeps it as recorded).
69
+
70
+ ### Added
71
+ - **`quantize_q4_0(W)`** and **`gemv_q4_0(x, Wq, N, K)`**: decode GEMV on
72
+ weights in ggml's Q4_0 format (`block_q4_0`, byte-identical: float16
73
+ scale + 16 bytes of 4-bit values; quantization as `quantize_row_q4_0_ref`,
74
+ checked against blocks written by llama.cpp v0.7.0). x is quantized to Q8
75
+ blocks once per call, as for `gemv_q8_0`; exact integer block dot
76
+ products, two weight rows per register; same paths as
77
+ `build_info()['q8_0_path']`. K % 32 != 0 raises `ArgumentError`.
78
+ docs/DESIGN.md §2.9. Measured on the reference laptop's P-core
79
+ (`lnl-226v-wsl_2026-10-20_42f64c7d76b3`): 68–83% of the read roof at 1×4096×4096, 1×4096×11008 and
80
+ 1×2048×8192; 1.38–1.67× our Q8_0 GEMV, 5.12–6.05× our FP32 GEMV, and
81
+ 1.58–2.19× ggml's Q4_0 MUL_MAT (llama.cpp v0.7.0, one thread).
82
+ - `python -m simd_ml_kernels bench`: Q4_0 GEMV cases (`--cases q4_0`) against
83
+ our Q8_0 and FP32 GEMV, PyTorch's FP32 `F.linear` on the dequantized
84
+ weights and ggml's Q4_0 MUL_MAT on one thread.
85
+ - **`quantize_q8_0(W)`** and **`gemv_q8_0(x, Wq, N, K)`**: decode GEMV on
86
+ weights in ggml's Q8_0 format (`block_q8_0`, byte-identical: float16 scale
87
+ + 32 int8 per block; quantization as `quantize_row_q8_0_ref`). x is
88
+ quantized to Q8 blocks once per call (float32 scales); exact integer
89
+ block dot products with `vpdpbusd` (AVX-VNNI, AVX512_VNNI) or
90
+ `vpmaddubsw` (AVX2), FP32 sums; `build_info()['q8_0_path']`. K % 32 != 0
91
+ raises `ArgumentError`. docs/DESIGN.md §2.8. Measured on the reference laptop's P-core
92
+ (`lnl-226v-wsl_2026-10-19_290b91baad26`): 92–95% of the read roof at 1×4096×4096,
93
+ 1×4096×11008 and 1×2048×8192; 3.49–3.61× our FP32 GEMV, 1.77–2.05× our
94
+ BF16 GEMV, and 1.46–1.82× ggml's Q8_0 MUL_MAT (llama.cpp v0.7.0, one
95
+ thread, `test-backend-ops perf`).
96
+ - `python -m simd_ml_kernels bench`: Q8_0 GEMV cases against our FP32/BF16
97
+ GEMV, PyTorch's FP32 `F.linear` on the dequantized weights and, when
98
+ `SIMD_ML_GGML_TEST_BACKEND_OPS` names llama.cpp's `test-backend-ops`,
99
+ ggml's Q8_0 MUL_MAT on one thread; `--cases` to run a subset (development).
100
+
101
+ ### Changed
102
+ - `sgemm_u8s8` with one row of A and `trans_b=True` (x·Wᵀ for one token)
103
+ reads B in place instead of packing it, on every x86 path (exact integer
104
+ sums; same results within the documented bound). At 1×4096×4096 on the
105
+ laptop: 1.36× PyTorch's quantized linear, was 0.46× (`lnl-226v-wsl_2026-10-19_290b91baad26`).
106
+ - `bench`: each GEMV case measures its own read roof (the read stream timed
107
+ interleaved with the case); `pct_stream_read` is relative to it
108
+ (`roof_read_gbs`). A case with an implementation above 105% of its own roof
109
+ is marked `roof_unstable` and not shown in the tables. The triad stays once
110
+ per run.
111
+ - docs/ROADMAP.md: milestones renamed from "v0.2 … v1.0" to "Milestone 1 … 9".
112
+ They were work stages, not package versions, and their names collided with
113
+ real versions (the old "v0.3" milestone is not package 0.3.0).
114
+
7
115
  ## 0.3.0 — 2026-10-19
8
116
 
9
117
  First release on PyPI (`pip install simd-ml-kernels`); 0.3.0rc1 was on
@@ -3,27 +3,31 @@ title: "simd-ml-kernels: hand-vectorized SIMD kernels for machine learning workl
3
3
  authors:
4
4
  - family-names: "Myet"
5
5
  given-names: "Min Htet"
6
- version: "0.2.0"
7
- date-released: "2026-06-09"
6
+ version: "0.4.0"
7
+ date-released: "2026-11-21"
8
8
  license: "Apache-2.0"
9
9
  repository-code: "https://github.com/Mattral/simd-ml-kernels"
10
10
  url: "https://zenodo.org/records/20616502"
11
11
  abstract: >
12
- An open-source C++17 library of clean, well-documented hand-vectorized
13
- microkernels for GEMM and ML activation functions (GeLU, ReLU, SiLU,
14
- Softmax, LayerNorm) targeting x86 processors with AVX2/FMA support.
15
- Includes Python/NumPy bindings via pybind11, a statistical benchmark
16
- harness, and a CI pipeline with performance regression gating.
12
+ C++17 library of hand-vectorized x86 kernels for GEMM (FP32, FP16, BF16
13
+ and INT8), decode GEMV, ggml Q8_0 and Q4_0 GEMV, and GeLU, ReLU, SiLU,
14
+ softmax and LayerNorm, with Python bindings. Every kernel has a portable
15
+ scalar path. AVX2 covers every kernel. AVX-512 covers the FP32, FP16 and
16
+ BF16 GEMMs and the decode GEMV. AVX-VNNI and AVX512-VNNI cover INT8 GEMM
17
+ and the Q8_0 and Q4_0 GEMVs. Opt-in AVX512-BF16 dot covers packed BF16
18
+ GEMM. The Zenodo preprint describes the earlier AVX2 kernels; current
19
+ scope and results are in the repository README.
17
20
  keywords:
18
21
  - simd
19
22
  - avx2
20
23
  - avx-512
24
+ - avx-vnni
21
25
  - gemm
26
+ - gemv
22
27
  - machine-learning
23
28
  - microkernels
24
29
  - gelu
25
30
  - layer-normalization
26
- - high-performance-computing
27
31
  - cpp
28
32
  preferred-citation:
29
33
  type: unpublished
@@ -1,5 +1,5 @@
1
1
  cmake_minimum_required(VERSION 3.22)
2
- project(simd_ml_kernels VERSION 0.3.0 LANGUAGES CXX)
2
+ project(simd_ml_kernels VERSION 0.4.0 LANGUAGES CXX)
3
3
  # CMake's VERSION is numeric only; the PEP 440 pre-release suffix of
4
4
  # pyproject.toml's version goes here ("" for a release). build_info()["version"]
5
5
  # is "${PROJECT_VERSION}${SIMD_ML_PRERELEASE}" and must equal the package version.
@@ -105,6 +105,8 @@ endif()
105
105
  set(SIMD_ML_SCALAR_SRCS
106
106
  src/dispatch/dispatch.cpp
107
107
  src/kernels/scalar_kernels.cpp
108
+ src/kernels/quant/q8_0_scalar.cpp
109
+ src/kernels/quant/q4_0_scalar.cpp
108
110
  )
109
111
  set(SIMD_ML_AVX2_SRCS
110
112
  src/kernels/kernels_avx2.cpp
@@ -114,6 +116,9 @@ set(SIMD_ML_AVX2_SRCS
114
116
  src/kernels/activations/silu_avx2.cpp
115
117
  src/kernels/activations/softmax_avx2.cpp
116
118
  src/kernels/activations/layer_norm_avx2.cpp
119
+ src/kernels/quant/q8_0_avx2.cpp
120
+ src/kernels/quant/q4_0_avx2.cpp
121
+ src/kernels/stream_read_avx2.cpp
117
122
  )
118
123
  set(SIMD_ML_AVX512_SRCS
119
124
  src/kernels/kernels_avx512.cpp
@@ -123,12 +128,14 @@ set(SIMD_ML_AVX512_SRCS
123
128
  # Extension units (no kernel table of their own), each called from a table
124
129
  # unit after a CPUID check, and built only if the compiler accepts its flag:
125
130
  # avx512bf16 vdpbf16ps BF16 GEMM, from the AVX-512 table (GCC 10+, Clang 9+)
126
- # avxvnni AVX-VNNI INT8 GEMM, from the AVX2 table (GCC 11+, Clang 12+)
127
- # avx512vnni AVX512_VNNI INT8 GEMM, from the AVX-512 table (GCC 8+, Clang 6+)
131
+ # avxvnni AVX-VNNI INT8 GEMM and Q8_0/Q4_0 GEMV, from the AVX2 table (GCC 11+, Clang 12+)
132
+ # avx512vnni AVX512_VNNI INT8 GEMM and Q8_0/Q4_0 GEMV, from the AVX-512 table (GCC 8+, Clang 6+)
128
133
  # The calling unit gets SIMD_ML_HAVE_<UNIT>.
129
134
  set(SIMD_ML_AVX512BF16_SRCS src/kernels/gemm/avx512bf16_gemm.cpp)
130
- set(SIMD_ML_AVXVNNI_SRCS src/kernels/gemm/avxvnni_gemm.cpp)
131
- set(SIMD_ML_AVX512VNNI_SRCS src/kernels/gemm/avx512vnni_gemm.cpp)
135
+ set(SIMD_ML_AVXVNNI_SRCS src/kernels/gemm/avxvnni_gemm.cpp src/kernels/quant/q8_0_avxvnni.cpp
136
+ src/kernels/quant/q4_0_avxvnni.cpp)
137
+ set(SIMD_ML_AVX512VNNI_SRCS src/kernels/gemm/avx512vnni_gemm.cpp src/kernels/quant/q8_0_avx512vnni.cpp
138
+ src/kernels/quant/q4_0_avx512vnni.cpp)
132
139
  set(_ext_units avx512bf16 avxvnni avx512vnni)
133
140
  set(_ext_flags -mavx512bf16 -mavxvnni -mavx512vnni)
134
141
  set(_ext_parent avx512 avx2 avx512)
@@ -173,6 +180,14 @@ foreach(isa IN LISTS SIMD_ML_ISAS)
173
180
  list(APPEND SIMD_ML_ISA_OBJECTS $<TARGET_OBJECTS:simd_ml_isa_${isa}>)
174
181
  message(STATUS "ISA unit ${isa}: ${_flags_str}")
175
182
  endforeach()
183
+ # Q4_0 quantization rounds w·id, then w·id + 8.5, in two binary32 steps, as
184
+ # ggml's reference built for baseline x86-64. GCC contracts the pair into
185
+ # one FMA by default (-ffp-contract=fast) wherever FMA exists (aarch64,
186
+ # SIMD_ML_NATIVE), and the fused result truncates differently next to an
187
+ # integer, so the blocks would no longer match ggml's byte for byte.
188
+ if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang")
189
+ set_source_files_properties(src/kernels/quant/q4_0_scalar.cpp PROPERTIES COMPILE_OPTIONS -ffp-contract=off)
190
+ endif()
176
191
  foreach(ext parent IN ZIP_LISTS _ext_units _ext_parent)
177
192
  if(ext IN_LIST SIMD_ML_ISAS)
178
193
  string(TOUPPER ${ext} EXT)
@@ -217,13 +232,27 @@ if(pybind11_FOUND)
217
232
  # Build metadata reported by build_info() and `python -m simd_ml_kernels
218
233
  # info`, and recorded in benchmark JSON. The commit is taken when CMake
219
234
  # configures, so a build directory reused across commits reports the
220
- # commit it was configured at; "-dirty" marks uncommitted changes.
235
+ # commit it was configured at: the first 12 hex digits of HEAD (exactly
236
+ # 12; `git describe` gave "v0.3.0-4-g<hash>" once a tag existed), plus
237
+ # "-dirty" if tracked files differ from HEAD (`git status` refreshes the
238
+ # index first, so a touched but unchanged file does not count, as with
239
+ # `describe --dirty`). Without git (an sdist build): "unknown".
221
240
  execute_process(
222
- COMMAND git describe --always --dirty --abbrev=12
241
+ COMMAND git rev-parse HEAD
223
242
  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
224
- OUTPUT_VARIABLE _simd_ml_git OUTPUT_STRIP_TRAILING_WHITESPACE
243
+ OUTPUT_VARIABLE _simd_ml_head OUTPUT_STRIP_TRAILING_WHITESPACE
225
244
  RESULT_VARIABLE _simd_ml_git_rc ERROR_QUIET)
226
- if(NOT _simd_ml_git_rc EQUAL 0 OR NOT _simd_ml_git)
245
+ if(_simd_ml_git_rc EQUAL 0 AND _simd_ml_head MATCHES "^[0-9a-f]+$")
246
+ string(SUBSTRING "${_simd_ml_head}" 0 12 _simd_ml_git)
247
+ execute_process(
248
+ COMMAND git status --porcelain --untracked-files=no
249
+ WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
250
+ OUTPUT_VARIABLE _simd_ml_git_status OUTPUT_STRIP_TRAILING_WHITESPACE
251
+ RESULT_VARIABLE _simd_ml_status_rc ERROR_QUIET)
252
+ if(NOT _simd_ml_status_rc EQUAL 0 OR NOT _simd_ml_git_status STREQUAL "")
253
+ string(APPEND _simd_ml_git "-dirty")
254
+ endif()
255
+ else()
227
256
  set(_simd_ml_git "unknown")
228
257
  endif()
229
258
  target_compile_definitions(simd_ml_kernels_core PRIVATE