kernel-lens 1.2.0__tar.gz → 1.2.1__tar.gz

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: kernel-lens
-Version: 1.2.0
-Summary: A production-grade, multi-backend compiler for Triton kernels (ONNX & TensorRT)
-Author: Compiler Engineer
+Version: 1.2.1
+Summary: Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins
+Author-email: Justin Duc <justin.duc@galiad-research.com>
 License: MIT License
         
         Copyright (c) 2026 Duc Justin
@@ -25,11 +25,23 @@ License: MIT License
         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
         SOFTWARE.
         
+Project-URL: Homepage, https://just1truc.github.io/kernel_lens/
+Project-URL: Documentation, https://just1truc.github.io/kernel_lens/
+Project-URL: Changelog, https://github.com/Just1truc/kernel_lens/blob/main/CHANGELOG.md
 Project-URL: Source Code, https://github.com/Just1truc/kernel_lens
 Project-URL: Bug Tracker, https://github.com/Just1truc/kernel_lens/issues
+Project-URL: Paper, https://doi.org/10.5281/zenodo.23194523
+Keywords: triton,tensorrt,onnxruntime,compiler,gpu,cuda,kernel,inference,deep-learning
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: POSIX :: Linux
+Classifier: Environment :: GPU :: NVIDIA CUDA
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -46,6 +58,8 @@ Provides-Extra: all
 Requires-Dist: onnxruntime-gpu>=1.17.0; extra == "all"
 Requires-Dist: tensorrt<11.0.0,>=8.6.0; extra == "all"
 Requires-Dist: tensorrt-cu12-libs<11.0.0; extra == "all"
+Provides-Extra: bench
+Requires-Dist: transformers>=4.40.0; extra == "bench"
 Dynamic: license-file
 
 # KernelLens
@@ -113,11 +127,16 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
   dynamic scalars, strided and dynamic inputs, loops, conditionals, nested
   helpers, in-place mutation, 3-D grids, autotune, and fp16/bf16.
 
-> **Honest scope.** KernelLens is research-grade. The documentation clearly
-> separates what is **implemented and measured** from what is **planned**. Two
-> TensorRT paths (in-place aliasing, INT8 dequantisation) are known to be
-> incorrect and are reported rather than hidden. See the
-> [verification page](https://just1truc.github.io/kernel_lens/verification/).
+> **Honest scope.** KernelLens is research-grade, and the results are reported
+> per configuration rather than blended. The composition is bit-exact against
+> native Triton (`0.00e+00`) across the research kernels, the fourteen
+> conformance patterns and the quantized GEMMs, on both backends; two
+> multi-operator fusions land just above zero and are reported as such. Keeping
+> the kernel pays when the baseline leaves the graph unfused and breaks even when
+> it already fuses. The TensorRT-LLM backend is design-only (see
+> [`docs/assets/data/a100/trtllm_backend_design.md`](docs/assets/data/a100/trtllm_backend_design.md)),
+> and the quantized GEMMs are validated on ONNX Runtime only. Full methodology
+> and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
 
 ## Installation
 
@@ -129,7 +148,16 @@ pip install kernel-lens[all]     # both
 ```
 
 **Requirements:** Linux x86-64, Python ≥ 3.10, CUDA ≥ 12.0, PyTorch ≥ 2.0,
-Triton ≥ 2.1, and `nvcc` + `g++` on `PATH`.
+Triton ≥ 2.1, and `nvcc` + `g++` on `PATH`. TensorRT must be `< 11` (the
+generator emits `IPluginV2DynamicExt`).
+
+**Environment:** `KERNEL_LENS_CACHE_DIR` (put it on a persistent volume so the
+`.so` / `.engine` artifacts survive `kl.load`), `KERNEL_LENS_CCBIN` (host
+compiler for `nvcc` — CUDA toolkits before 12.9 reject GCC 13's exception
+specifiers, so set it to `g++-12` there), `KERNEL_LENS_DEBUG` (or `KL_VERBOSE`),
+`TENSORRT_INCLUDE_DIR`, and `KERNEL_LENS_STREAM_READER=1` to deserialize engines
+above 2 GB from disk rather than from a memory buffer (opt-in; see the
+[changelog](CHANGELOG.md)).
 
 ## Quickstart
 
@@ -172,22 +200,52 @@ out_trt = compiled.run((x,), backend="tensorrt")
 
 ## Results at a glance
 
-Measured on an **RTX 5060 Ti (Blackwell, `sm_120a`)**, CUDA 12.8, PyTorch 2.14,
-Triton 3.8, ONNX Runtime 1.30, TensorRT 10.16. Full tables and methodology are on
-the [results page](https://just1truc.github.io/kernel_lens/#results).
-
-- **Fold + keep, and it pays**: on a two-layer block whose attention is a custom
-  squared-ReLU flash kernel, KernelLens runs it in **7.63 ms**, versus **15.15 ms**
-  for the folded-only `torch.compile` formulation and **28.04 ms** for eager
-  folding — **1.99×** and **3.68×** faster — and **1.08×** faster than the
-  unfused Triton-kernel path. The engine folds the standard layers; the kernel
-  keeps the operations folding cannot express.
-- **Exact parity with native Triton** (`MaxDiff = 0.00e+00`) for LLaMA 3 RMSNorm,
-  PaLM SwiGLU, Qwen 2.5 RoPE, Liger fused cross-entropy, in-place mutation and
-  the quantized GEMMs, on both backends.
-- **Lowest VRAM** in a 4-layer LLaMA decoder: **464 MB** peak vs 478 MB (eager)
-  and 473 MB (`torch.compile`), with the fastest time-to-first-token (4.14 ms vs
-  7.85/7.92 ms).
+Two architectures: an **RTX 5060 Ti** (Blackwell, `sm_120a`) for the folding,
+single-operator, end-to-end and quantized results, and an **A100-SXM4-80GB**
+(Ampere, `sm_80`) for the conformance, parity and real-model results. Results are
+reported per configuration and never blended. Full methodology, complete tables
+and every raw JSON are on the
+[results page](https://just1truc.github.io/kernel_lens/#results) and under
+[`docs/assets/data/`](docs/assets/data/).
+
+**The code you don't write.** The deployment is **nine lines of Python**; the C++
+it replaces is **1,137 lines across six files** for one fused RMSNorm kernel on
+both backends, measured with `tests/measure_codegen.py`.
+
+**Real pretrained models.** Every decoder `LayerNorm` of a real OPT checkpoint
+(125M → 6.7B) replaced by the Triton RMSNorm kernel, graded against
+`torch.compile` — the baseline that *captures the same Triton kernels* rather
+than reimplementing them. The result splits by precision:
+
+- **fp32** — `torch.compile` barely improves on eager (1.02–1.12×) because it
+  leaves the operators around the kernel unfused. KernelLens fuses them:
+  **3.18–5.45× faster** under TensorRT and **2.27–4.62×** under ONNX Runtime,
+  and the margin grows with model size.
+- **fp16** — `torch.compile` optimises well there, so we only match it: TensorRT
+  edges it by **1.06–1.22×**, while ONNX Runtime is consistently **slower
+  (0.67–0.84×)** — per-node dispatch dominates when the graph already fuses.
+- **Accuracy tracks precision, not speed.** fp32 holds the same greedy token on
+  **≥ 98.8%** of positions; fp16 under TensorRT falls to 93.4–97.9%. OPT-350M in
+  fp16 under ONNX Runtime is bit-exact.
+
+**Fold + keep, and it pays.** On a two-layer block whose attention is a custom
+squared-ReLU flash kernel, KernelLens runs it in **7.63 ms**, versus **15.15 ms**
+for the folded-only `torch.compile` formulation and **28.04 ms** for eager
+folding — **1.99×** and **3.68×** faster. Against the unfused kernel path it is
+**1.08×** faster than eager Triton (8.24 ms) and level with `torch.compile`
+(7.55 ms). The engine folds the standard layers; the kernel keeps the operations
+folding cannot express.
+
+**Exact parity with native Triton** (`MaxDiff = 0.00e+00`) for LLaMA 3 RMSNorm,
+PaLM SwiGLU, Qwen 2.5 RoPE, Liger fused cross-entropy, in-place mutation and the
+quantized GEMMs, on both backends — and across all **fourteen** module patterns
+of `tests/test_conformance.py`. Two multi-operator fusions are near-exact rather
+than exact (`≤ 9.5e-07` residual+RMSNorm, `4.8e-07` Mamba's scan) and are
+reported as such.
+
+**Lowest VRAM** in a 4-layer LLaMA decoder: **464 MB** peak vs 478 MB (eager)
+and 473 MB (`torch.compile`), with the fastest time-to-first-token (4.14 ms vs
+7.85/7.92 ms).
 
 ## Documentation
 
@@ -195,9 +253,11 @@ the [results page](https://just1truc.github.io/kernel_lens/#results).
 | :--- | :--- |
 | [Problem](https://just1truc.github.io/kernel_lens/#problem) | The five gaps a naive export hits |
 | [Pipeline](https://just1truc.github.io/kernel_lens/#pipeline) | What you build by hand vs. the compile/run pipeline |
-| [Results](https://just1truc.github.io/kernel_lens/#results) | Folding + kernels, parity, latency, end-to-end |
+| [Results](https://just1truc.github.io/kernel_lens/#results) | Folding + kernels, parity, conformance, single-operator and end-to-end latency, **real pretrained models** |
+| [Why an engine](https://just1truc.github.io/kernel_lens/#why) | `torch.compile` vs TensorRT, both directions |
 | [Install & API](https://just1truc.github.io/kernel_lens/#install) | Installation and API reference |
 | [Status](https://just1truc.github.io/kernel_lens/#status) | Roadmap |
+| [FAQ](https://just1truc.github.io/kernel_lens/#faq) | Fair objections |
 | [Paper](https://just1truc.github.io/kernel_lens/#paper) | Abstract and citation |
 
 ## Reproduce the results
@@ -212,14 +272,30 @@ python tests/test_research_kernels.py          # numerical parity
 python tests/benchmark_research_kernels.py     # single-operator latency
 python tests/benchmark_e2e_llama.py            # end-to-end decoder
 python tests/test_fp8_gemm.py                  # FP8 Tensor Core path
+python tests/measure_codegen.py                # generated C++ volume, per file
+python tests/benchmark_real_model.py --model facebook/opt-1.3b   # real OPT, both precisions
 ```
 
+The real-model matrix (OPT-125M → 6.7B, fp32 and fp16) runs unattended on a fresh
+CUDA host: `scripts/remote_setup.sh` provisions nvcc, TensorRT, `onnxruntime-gpu`
+and `transformers` and installs the package; `scripts/remote_bench_all.sh` then
+sweeps the matrix into `results/` and writes `summary.json`, recording failures
+instead of aborting. See
+[`docs/assets/data/a100/runpod_runbook.md`](docs/assets/data/a100/runpod_runbook.md) for the
+exact invocation and the environment traps (CUDA 12.8's `nvcc` does not build
+against a modern libstdc++; CUDA 13.3's does).
+
 Raw measurement JSON is committed under [`docs/assets/data/`](docs/assets/data/).
 
 ## Technical report
 
-The LaTeX source is [`architecture_report.tex`](architecture_report.tex); the TMLR
-variant is generated from it with `python3 scripts/make_tmlr.py`.
+The paper source is [`kernel_lens_fresh.tex`](kernel_lens_fresh.tex). Two derived
+variants are generated from it and must not be edited by hand:
+`scripts/make_mlsys.py` produces the 2-column MLSys form
+(`kernel_lens_mlsys.tex`) and `scripts/make_tmlr_fresh.py` produces the TMLR
+forms — `kernel_lens_tmlr_fresh.tex` (named preprint) and
+`kernel_lens_tmlr_submission.tex` (anonymized; **do not publish it alongside the
+named preprint**).
 
 ```bibtex
 @article{duc2026kernellens,
@@ -227,7 +303,7 @@ variant is generated from it with `python3 scripts/make_tmlr.py`.
   author      = {Duc, Justin},
   institution = {Galiad Research},
   year        = {2026},
-  version     = {1.1.9},
+  version     = {1.2.1},
   doi         = {10.5281/zenodo.23194523},
   url         = {https://doi.org/10.5281/zenodo.23194523}
 }

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/README.md

@@ -63,11 +63,16 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
   dynamic scalars, strided and dynamic inputs, loops, conditionals, nested
   helpers, in-place mutation, 3-D grids, autotune, and fp16/bf16.
 
-> **Honest scope.** KernelLens is research-grade. The documentation clearly
-> separates what is **implemented and measured** from what is **planned**. Two
-> TensorRT paths (in-place aliasing, INT8 dequantisation) are known to be
-> incorrect and are reported rather than hidden. See the
-> [verification page](https://just1truc.github.io/kernel_lens/verification/).
+> **Honest scope.** KernelLens is research-grade, and the results are reported
+> per configuration rather than blended. The composition is bit-exact against
+> native Triton (`0.00e+00`) across the research kernels, the fourteen
+> conformance patterns and the quantized GEMMs, on both backends; two
+> multi-operator fusions land just above zero and are reported as such. Keeping
+> the kernel pays when the baseline leaves the graph unfused and breaks even when
+> it already fuses. The TensorRT-LLM backend is design-only (see
+> [`docs/assets/data/a100/trtllm_backend_design.md`](docs/assets/data/a100/trtllm_backend_design.md)),
+> and the quantized GEMMs are validated on ONNX Runtime only. Full methodology
+> and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
 
 ## Installation
 
@@ -79,7 +84,16 @@ pip install kernel-lens[all]     # both
 ```
 
 **Requirements:** Linux x86-64, Python ≥ 3.10, CUDA ≥ 12.0, PyTorch ≥ 2.0,
-Triton ≥ 2.1, and `nvcc` + `g++` on `PATH`.
+Triton ≥ 2.1, and `nvcc` + `g++` on `PATH`. TensorRT must be `< 11` (the
+generator emits `IPluginV2DynamicExt`).
+
+**Environment:** `KERNEL_LENS_CACHE_DIR` (put it on a persistent volume so the
+`.so` / `.engine` artifacts survive `kl.load`), `KERNEL_LENS_CCBIN` (host
+compiler for `nvcc` — CUDA toolkits before 12.9 reject GCC 13's exception
+specifiers, so set it to `g++-12` there), `KERNEL_LENS_DEBUG` (or `KL_VERBOSE`),
+`TENSORRT_INCLUDE_DIR`, and `KERNEL_LENS_STREAM_READER=1` to deserialize engines
+above 2 GB from disk rather than from a memory buffer (opt-in; see the
+[changelog](CHANGELOG.md)).
 
 ## Quickstart
 
@@ -122,22 +136,52 @@ out_trt = compiled.run((x,), backend="tensorrt")
 
 ## Results at a glance
 
-Measured on an **RTX 5060 Ti (Blackwell, `sm_120a`)**, CUDA 12.8, PyTorch 2.14,
-Triton 3.8, ONNX Runtime 1.30, TensorRT 10.16. Full tables and methodology are on
-the [results page](https://just1truc.github.io/kernel_lens/#results).
-
-- **Fold + keep, and it pays**: on a two-layer block whose attention is a custom
-  squared-ReLU flash kernel, KernelLens runs it in **7.63 ms**, versus **15.15 ms**
-  for the folded-only `torch.compile` formulation and **28.04 ms** for eager
-  folding — **1.99×** and **3.68×** faster — and **1.08×** faster than the
-  unfused Triton-kernel path. The engine folds the standard layers; the kernel
-  keeps the operations folding cannot express.
-- **Exact parity with native Triton** (`MaxDiff = 0.00e+00`) for LLaMA 3 RMSNorm,
-  PaLM SwiGLU, Qwen 2.5 RoPE, Liger fused cross-entropy, in-place mutation and
-  the quantized GEMMs, on both backends.
-- **Lowest VRAM** in a 4-layer LLaMA decoder: **464 MB** peak vs 478 MB (eager)
-  and 473 MB (`torch.compile`), with the fastest time-to-first-token (4.14 ms vs
-  7.85/7.92 ms).
+Two architectures: an **RTX 5060 Ti** (Blackwell, `sm_120a`) for the folding,
+single-operator, end-to-end and quantized results, and an **A100-SXM4-80GB**
+(Ampere, `sm_80`) for the conformance, parity and real-model results. Results are
+reported per configuration and never blended. Full methodology, complete tables
+and every raw JSON are on the
+[results page](https://just1truc.github.io/kernel_lens/#results) and under
+[`docs/assets/data/`](docs/assets/data/).
+
+**The code you don't write.** The deployment is **nine lines of Python**; the C++
+it replaces is **1,137 lines across six files** for one fused RMSNorm kernel on
+both backends, measured with `tests/measure_codegen.py`.
+
+**Real pretrained models.** Every decoder `LayerNorm` of a real OPT checkpoint
+(125M → 6.7B) replaced by the Triton RMSNorm kernel, graded against
+`torch.compile` — the baseline that *captures the same Triton kernels* rather
+than reimplementing them. The result splits by precision:
+
+- **fp32** — `torch.compile` barely improves on eager (1.02–1.12×) because it
+  leaves the operators around the kernel unfused. KernelLens fuses them:
+  **3.18–5.45× faster** under TensorRT and **2.27–4.62×** under ONNX Runtime,
+  and the margin grows with model size.
+- **fp16** — `torch.compile` optimises well there, so we only match it: TensorRT
+  edges it by **1.06–1.22×**, while ONNX Runtime is consistently **slower
+  (0.67–0.84×)** — per-node dispatch dominates when the graph already fuses.
+- **Accuracy tracks precision, not speed.** fp32 holds the same greedy token on
+  **≥ 98.8%** of positions; fp16 under TensorRT falls to 93.4–97.9%. OPT-350M in
+  fp16 under ONNX Runtime is bit-exact.
+
+**Fold + keep, and it pays.** On a two-layer block whose attention is a custom
+squared-ReLU flash kernel, KernelLens runs it in **7.63 ms**, versus **15.15 ms**
+for the folded-only `torch.compile` formulation and **28.04 ms** for eager
+folding — **1.99×** and **3.68×** faster. Against the unfused kernel path it is
+**1.08×** faster than eager Triton (8.24 ms) and level with `torch.compile`
+(7.55 ms). The engine folds the standard layers; the kernel keeps the operations
+folding cannot express.
+
+**Exact parity with native Triton** (`MaxDiff = 0.00e+00`) for LLaMA 3 RMSNorm,
+PaLM SwiGLU, Qwen 2.5 RoPE, Liger fused cross-entropy, in-place mutation and the
+quantized GEMMs, on both backends — and across all **fourteen** module patterns
+of `tests/test_conformance.py`. Two multi-operator fusions are near-exact rather
+than exact (`≤ 9.5e-07` residual+RMSNorm, `4.8e-07` Mamba's scan) and are
+reported as such.
+
+**Lowest VRAM** in a 4-layer LLaMA decoder: **464 MB** peak vs 478 MB (eager)
+and 473 MB (`torch.compile`), with the fastest time-to-first-token (4.14 ms vs
+7.85/7.92 ms).
 
 ## Documentation
 
@@ -145,9 +189,11 @@ the [results page](https://just1truc.github.io/kernel_lens/#results).
 | :--- | :--- |
 | [Problem](https://just1truc.github.io/kernel_lens/#problem) | The five gaps a naive export hits |
 | [Pipeline](https://just1truc.github.io/kernel_lens/#pipeline) | What you build by hand vs. the compile/run pipeline |
-| [Results](https://just1truc.github.io/kernel_lens/#results) | Folding + kernels, parity, latency, end-to-end |
+| [Results](https://just1truc.github.io/kernel_lens/#results) | Folding + kernels, parity, conformance, single-operator and end-to-end latency, **real pretrained models** |
+| [Why an engine](https://just1truc.github.io/kernel_lens/#why) | `torch.compile` vs TensorRT, both directions |
 | [Install & API](https://just1truc.github.io/kernel_lens/#install) | Installation and API reference |
 | [Status](https://just1truc.github.io/kernel_lens/#status) | Roadmap |
+| [FAQ](https://just1truc.github.io/kernel_lens/#faq) | Fair objections |
 | [Paper](https://just1truc.github.io/kernel_lens/#paper) | Abstract and citation |
 
 ## Reproduce the results
@@ -162,14 +208,30 @@ python tests/test_research_kernels.py          # numerical parity
 python tests/benchmark_research_kernels.py     # single-operator latency
 python tests/benchmark_e2e_llama.py            # end-to-end decoder
 python tests/test_fp8_gemm.py                  # FP8 Tensor Core path
+python tests/measure_codegen.py                # generated C++ volume, per file
+python tests/benchmark_real_model.py --model facebook/opt-1.3b   # real OPT, both precisions
 ```
 
+The real-model matrix (OPT-125M → 6.7B, fp32 and fp16) runs unattended on a fresh
+CUDA host: `scripts/remote_setup.sh` provisions nvcc, TensorRT, `onnxruntime-gpu`
+and `transformers` and installs the package; `scripts/remote_bench_all.sh` then
+sweeps the matrix into `results/` and writes `summary.json`, recording failures
+instead of aborting. See
+[`docs/assets/data/a100/runpod_runbook.md`](docs/assets/data/a100/runpod_runbook.md) for the
+exact invocation and the environment traps (CUDA 12.8's `nvcc` does not build
+against a modern libstdc++; CUDA 13.3's does).
+
 Raw measurement JSON is committed under [`docs/assets/data/`](docs/assets/data/).
 
 ## Technical report
 
-The LaTeX source is [`architecture_report.tex`](architecture_report.tex); the TMLR
-variant is generated from it with `python3 scripts/make_tmlr.py`.
+The paper source is [`kernel_lens_fresh.tex`](kernel_lens_fresh.tex). Two derived
+variants are generated from it and must not be edited by hand:
+`scripts/make_mlsys.py` produces the 2-column MLSys form
+(`kernel_lens_mlsys.tex`) and `scripts/make_tmlr_fresh.py` produces the TMLR
+forms — `kernel_lens_tmlr_fresh.tex` (named preprint) and
+`kernel_lens_tmlr_submission.tex` (anonymized; **do not publish it alongside the
+named preprint**).
 
 ```bibtex
 @article{duc2026kernellens,
@@ -177,7 +239,7 @@ variant is generated from it with `python3 scripts/make_tmlr.py`.
   author      = {Duc, Justin},
   institution = {Galiad Research},
   year        = {2026},
-  version     = {1.1.9},
+  version     = {1.2.1},
   doi         = {10.5281/zenodo.23194523},
   url         = {https://doi.org/10.5281/zenodo.23194523}
 }

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/kernel_lens/__init__.py

@@ -7,7 +7,7 @@ from .config import set_verbose, is_verbose
 
 
 
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 
 __all__ = [
     "compile",

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/kernel_lens/backends/builder.py

@@ -6,6 +6,20 @@ import shutil
 
 from ..utils.env_check import find_nvcc
 
+
+def _host_compiler():
+    """Host compiler for nvcc.
+
+    CUDA toolkits before 12.9 declare rsqrt/sinpi/... without noexcept, which
+    GCC 13 rejects ("has a different exception specifier"), so the right choice
+    is environment-specific. Override with KERNEL_LENS_CCBIN.
+    """
+    override = os.environ.get("KERNEL_LENS_CCBIN")
+    if override:
+        return override
+    return "g++-13" if os.path.exists("/usr/bin/g++-13") else "g++"
+
+
 def get_cuda_home() -> str:
     nvcc_path = find_nvcc()
     if nvcc_path:
@@ -152,7 +166,7 @@ def build_ort_plugin(ort_plugins_dir: str, cache_dir: str):
             arch_flag = "-gencode=arch=compute_75,code=sm_75"
 
         user_trt_inc = os.path.expanduser("~/tensorrt_headers")
-        cpp_compiler = "g++-13" if os.path.exists("/usr/bin/g++-13") else "g++"
+        cpp_compiler = _host_compiler()
         ccbin_flag = ["-ccbin", cpp_compiler] if os.path.exists(f"/usr/bin/{cpp_compiler}") else []
 
         cmd = [
@@ -318,7 +332,7 @@ def build_trt_plugin(trt_plugins_dir: str, cache_dir: str):
             and not os.path.exists(os.path.join(d, "crt", "math_functions.h"))
         ]
 
-        cpp_compiler = "g++-13" if os.path.exists("/usr/bin/g++-13") else "g++"
+        cpp_compiler = _host_compiler()
         ccbin_flag = ["-ccbin", cpp_compiler] if os.path.exists(f"/usr/bin/{cpp_compiler}") else []
 
         cmd = [
@@ -383,7 +397,7 @@ def build_trt_plugin(trt_plugins_dir: str, cache_dir: str):
                     nvinfer_link_flag = [os.path.join(d, f)]
                     break
 
-    cpp_compiler = "g++-13" if os.path.exists("/usr/bin/g++-13") else "g++"
+    cpp_compiler = _host_compiler()
     cmd = [
         cpp_compiler, "-shared", "-o", so_path
     ] + obj_files + [

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/kernel_lens/runtime/engine.py

@@ -31,6 +31,63 @@ def _graph_has_int8_custom_inputs(onnx_path: str) -> bool:
         return False
     return False
 
+def _make_stream_reader(trt, path):
+    """File-backed ``IStreamReaderV2``.
+
+    Engines above roughly 2 GB fail to deserialize from a memory buffer; the
+    stream-reader path reads them straight from disk instead. TensorRT fills a
+    caller-provided capsule, so the bytes have to be memmove'd into it.
+    """
+    import ctypes
+
+    _get_ptr = ctypes.pythonapi.PyCapsule_GetPointer
+    _get_ptr.restype = ctypes.c_void_p
+    _get_ptr.argtypes = [ctypes.py_object, ctypes.c_char_p]
+
+    _whence = {trt.SeekPosition.SET: os.SEEK_SET,
+               trt.SeekPosition.CUR: os.SEEK_CUR,
+               trt.SeekPosition.END: os.SEEK_END}
+
+    class _FileStreamReader(trt.IStreamReaderV2):
+        def __init__(self):
+            super().__init__()
+            self._f = open(path, "rb")
+
+        def read(self, num_bytes, destination):
+            # The binding dispatches (num_bytes, destination_pointer) — the
+            # reverse of the order in the published docstring, which is itself
+            # swapped with seek's. Verified empirically against TensorRT 10.16.
+            data = self._f.read(int(num_bytes))
+            if not data:
+                return 0
+            ctypes.memmove(int(destination), data, len(data))
+            return len(data)
+
+        def seek(self, offset, where):
+            self._f.seek(int(offset), _whence.get(where, os.SEEK_SET))
+            return True
+
+    return _FileStreamReader()
+
+
+def _deserialize_engine(trt, runtime, engine_path):
+    # The stream-reader path is experimental: TensorRT's Python binding
+    # dispatches read(num_bytes, destination) — the reverse of its documented
+    # order — and writing through that pointer is not yet reliable. Engines
+    # below 2 GB take the memory path regardless; above it, opt in explicitly.
+    if (os.environ.get("KERNEL_LENS_STREAM_READER") == "1"
+            and os.path.getsize(engine_path) > (1 << 31)
+            and hasattr(trt, "IStreamReaderV2")):
+        try:
+            eng = runtime.deserialize_cuda_engine(_make_stream_reader(trt, engine_path))
+            if eng is not None:
+                return eng
+        except Exception:
+            pass
+    with open(engine_path, "rb") as f:
+        return runtime.deserialize_cuda_engine(f.read())
+
+
 class CompiledModel:
     def __init__(self, cache_dir: str, model_name: str, backends: list[str], output_shapes: list = None, output_strides: list = None, output_dtypes: list = None, saved_args: list = None):
         """
@@ -306,16 +363,46 @@ class CompiledModel:
                 else:
                     network = builder.create_network()
                 parser = trt.OnnxParser(network, TRT_LOGGER)
-                
-                with open(onnx_path, 'rb') as model:
-                    if not parser.parse(model.read()):
-                        for error in range(parser.num_errors):
-                            print(parser.get_error(error))
-                        raise RuntimeError("Failed to parse ONNX graph for TensorRT.")
+
+                # Parse from the file path rather than a bytes buffer. Graphs
+                # above the 2 GB protobuf limit spill their weights to external
+                # files beside the model, and the parser can only resolve those
+                # relative locations when it knows which directory that is.
+                if hasattr(parser, "parse_from_file"):
+                    ok = parser.parse_from_file(onnx_path)
+                else:
+                    with open(onnx_path, 'rb') as model:
+                        ok = parser.parse(model.read())
+                if not ok:
+                    for error in range(parser.num_errors):
+                        print(parser.get_error(error))
+                    raise RuntimeError("Failed to parse ONNX graph for TensorRT.")
                         
                 config = builder.create_builder_config()
                 if hasattr(config, "set_memory_pool_limit"):
                     config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 30)
+
+                # TensorRT keeps plugin layers in FP32 unless the matching
+                # precision flag is enabled. Without it the builder inserts a
+                # cast on the plugin's inputs, and the plugin's dtype guard
+                # (which mirrors the kernel's signature) then rejects them, so
+                # the engine build fails with "could not find any supported
+                # formats consistent with input/output data types".
+                try:
+                    import onnx as _onnx
+                    _g = _onnx.load(onnx_path, load_external_data=False).graph
+                    # TensorProto: 10 = FLOAT16, 16 = BFLOAT16. An LLM's graph
+                    # *inputs* are token ids (int64), so the precision has to be
+                    # read from the initializers and the internal tensors too.
+                    _elem = {t.type.tensor_type.elem_type
+                             for t in list(_g.input) + list(_g.output) + list(_g.value_info)}
+                    _elem |= {t.data_type for t in _g.initializer}
+                    if 10 in _elem and hasattr(trt.BuilderFlag, "FP16"):
+                        config.set_flag(trt.BuilderFlag.FP16)
+                    if 16 in _elem and hasattr(trt.BuilderFlag, "BF16"):
+                        config.set_flag(trt.BuilderFlag.BF16)
+                except Exception:
+                    pass
                 
                 profile = builder.create_optimization_profile()
                 has_dynamic = False
@@ -358,8 +445,8 @@ class CompiledModel:
             # ---------------------------------------------------
             
             # On lit le SSD et on désérialise UNE SEULE FOIS
-            with open(engine_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
-                self._trt_engine = runtime.deserialize_cuda_engine(f.read())
+            with trt.Runtime(TRT_LOGGER) as runtime:
+                self._trt_engine = _deserialize_engine(trt, runtime, engine_path)
                 
             self._trt_context = self._trt_engine.create_execution_context()
             self._output_tensors = {}   # (name, shape, dtype) -> cached CUDA tensor

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/kernel_lens.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: kernel-lens
-Version: 1.2.0
-Summary: A production-grade, multi-backend compiler for Triton kernels (ONNX & TensorRT)
-Author: Compiler Engineer
+Version: 1.2.1
+Summary: Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins
+Author-email: Justin Duc <justin.duc@galiad-research.com>
 License: MIT License
         
         Copyright (c) 2026 Duc Justin
@@ -25,11 +25,23 @@ License: MIT License
         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
         SOFTWARE.
         
+Project-URL: Homepage, https://just1truc.github.io/kernel_lens/
+Project-URL: Documentation, https://just1truc.github.io/kernel_lens/
+Project-URL: Changelog, https://github.com/Just1truc/kernel_lens/blob/main/CHANGELOG.md
 Project-URL: Source Code, https://github.com/Just1truc/kernel_lens
 Project-URL: Bug Tracker, https://github.com/Just1truc/kernel_lens/issues
+Project-URL: Paper, https://doi.org/10.5281/zenodo.23194523
+Keywords: triton,tensorrt,onnxruntime,compiler,gpu,cuda,kernel,inference,deep-learning
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: POSIX :: Linux
+Classifier: Environment :: GPU :: NVIDIA CUDA
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -46,6 +58,8 @@ Provides-Extra: all
 Requires-Dist: onnxruntime-gpu>=1.17.0; extra == "all"
 Requires-Dist: tensorrt<11.0.0,>=8.6.0; extra == "all"
 Requires-Dist: tensorrt-cu12-libs<11.0.0; extra == "all"
+Provides-Extra: bench
+Requires-Dist: transformers>=4.40.0; extra == "bench"
 Dynamic: license-file
 
 # KernelLens
@@ -113,11 +127,16 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
   dynamic scalars, strided and dynamic inputs, loops, conditionals, nested
   helpers, in-place mutation, 3-D grids, autotune, and fp16/bf16.
 
-> **Honest scope.** KernelLens is research-grade. The documentation clearly
-> separates what is **implemented and measured** from what is **planned**. Two
-> TensorRT paths (in-place aliasing, INT8 dequantisation) are known to be
-> incorrect and are reported rather than hidden. See the
-> [verification page](https://just1truc.github.io/kernel_lens/verification/).
+> **Honest scope.** KernelLens is research-grade, and the results are reported
+> per configuration rather than blended. The composition is bit-exact against
+> native Triton (`0.00e+00`) across the research kernels, the fourteen
+> conformance patterns and the quantized GEMMs, on both backends; two
+> multi-operator fusions land just above zero and are reported as such. Keeping
+> the kernel pays when the baseline leaves the graph unfused and breaks even when
+> it already fuses. The TensorRT-LLM backend is design-only (see
+> [`docs/assets/data/a100/trtllm_backend_design.md`](docs/assets/data/a100/trtllm_backend_design.md)),
+> and the quantized GEMMs are validated on ONNX Runtime only. Full methodology
+> and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
 
 ## Installation
 
@@ -129,7 +148,16 @@ pip install kernel-lens[all]     # both
 ```
 
 **Requirements:** Linux x86-64, Python ≥ 3.10, CUDA ≥ 12.0, PyTorch ≥ 2.0,
-Triton ≥ 2.1, and `nvcc` + `g++` on `PATH`.
+Triton ≥ 2.1, and `nvcc` + `g++` on `PATH`. TensorRT must be `< 11` (the
+generator emits `IPluginV2DynamicExt`).
+
+**Environment:** `KERNEL_LENS_CACHE_DIR` (put it on a persistent volume so the
+`.so` / `.engine` artifacts survive `kl.load`), `KERNEL_LENS_CCBIN` (host
+compiler for `nvcc` — CUDA toolkits before 12.9 reject GCC 13's exception
+specifiers, so set it to `g++-12` there), `KERNEL_LENS_DEBUG` (or `KL_VERBOSE`),
+`TENSORRT_INCLUDE_DIR`, and `KERNEL_LENS_STREAM_READER=1` to deserialize engines
+above 2 GB from disk rather than from a memory buffer (opt-in; see the
+[changelog](CHANGELOG.md)).
 
 ## Quickstart
 
@@ -172,22 +200,52 @@ out_trt = compiled.run((x,), backend="tensorrt")
 
 ## Results at a glance
 
-Measured on an **RTX 5060 Ti (Blackwell, `sm_120a`)**, CUDA 12.8, PyTorch 2.14,
-Triton 3.8, ONNX Runtime 1.30, TensorRT 10.16. Full tables and methodology are on
-the [results page](https://just1truc.github.io/kernel_lens/#results).
-
-- **Fold + keep, and it pays**: on a two-layer block whose attention is a custom
-  squared-ReLU flash kernel, KernelLens runs it in **7.63 ms**, versus **15.15 ms**
-  for the folded-only `torch.compile` formulation and **28.04 ms** for eager
-  folding — **1.99×** and **3.68×** faster — and **1.08×** faster than the
-  unfused Triton-kernel path. The engine folds the standard layers; the kernel
-  keeps the operations folding cannot express.
-- **Exact parity with native Triton** (`MaxDiff = 0.00e+00`) for LLaMA 3 RMSNorm,
-  PaLM SwiGLU, Qwen 2.5 RoPE, Liger fused cross-entropy, in-place mutation and
-  the quantized GEMMs, on both backends.
-- **Lowest VRAM** in a 4-layer LLaMA decoder: **464 MB** peak vs 478 MB (eager)
-  and 473 MB (`torch.compile`), with the fastest time-to-first-token (4.14 ms vs
-  7.85/7.92 ms).
+Two architectures: an **RTX 5060 Ti** (Blackwell, `sm_120a`) for the folding,
+single-operator, end-to-end and quantized results, and an **A100-SXM4-80GB**
+(Ampere, `sm_80`) for the conformance, parity and real-model results. Results are
+reported per configuration and never blended. Full methodology, complete tables
+and every raw JSON are on the
+[results page](https://just1truc.github.io/kernel_lens/#results) and under
+[`docs/assets/data/`](docs/assets/data/).
+
+**The code you don't write.** The deployment is **nine lines of Python**; the C++
+it replaces is **1,137 lines across six files** for one fused RMSNorm kernel on
+both backends, measured with `tests/measure_codegen.py`.
+
+**Real pretrained models.** Every decoder `LayerNorm` of a real OPT checkpoint
+(125M → 6.7B) replaced by the Triton RMSNorm kernel, graded against
+`torch.compile` — the baseline that *captures the same Triton kernels* rather
+than reimplementing them. The result splits by precision:
+
+- **fp32** — `torch.compile` barely improves on eager (1.02–1.12×) because it
+  leaves the operators around the kernel unfused. KernelLens fuses them:
+  **3.18–5.45× faster** under TensorRT and **2.27–4.62×** under ONNX Runtime,
+  and the margin grows with model size.
+- **fp16** — `torch.compile` optimises well there, so we only match it: TensorRT
+  edges it by **1.06–1.22×**, while ONNX Runtime is consistently **slower
+  (0.67–0.84×)** — per-node dispatch dominates when the graph already fuses.
+- **Accuracy tracks precision, not speed.** fp32 holds the same greedy token on
+  **≥ 98.8%** of positions; fp16 under TensorRT falls to 93.4–97.9%. OPT-350M in
+  fp16 under ONNX Runtime is bit-exact.
+
+**Fold + keep, and it pays.** On a two-layer block whose attention is a custom
+squared-ReLU flash kernel, KernelLens runs it in **7.63 ms**, versus **15.15 ms**
+for the folded-only `torch.compile` formulation and **28.04 ms** for eager
+folding — **1.99×** and **3.68×** faster. Against the unfused kernel path it is
+**1.08×** faster than eager Triton (8.24 ms) and level with `torch.compile`
+(7.55 ms). The engine folds the standard layers; the kernel keeps the operations
+folding cannot express.
+
+**Exact parity with native Triton** (`MaxDiff = 0.00e+00`) for LLaMA 3 RMSNorm,
+PaLM SwiGLU, Qwen 2.5 RoPE, Liger fused cross-entropy, in-place mutation and the
+quantized GEMMs, on both backends — and across all **fourteen** module patterns
+of `tests/test_conformance.py`. Two multi-operator fusions are near-exact rather
+than exact (`≤ 9.5e-07` residual+RMSNorm, `4.8e-07` Mamba's scan) and are
+reported as such.
+
+**Lowest VRAM** in a 4-layer LLaMA decoder: **464 MB** peak vs 478 MB (eager)
+and 473 MB (`torch.compile`), with the fastest time-to-first-token (4.14 ms vs
+7.85/7.92 ms).
 
 ## Documentation
 
@@ -195,9 +253,11 @@ the [results page](https://just1truc.github.io/kernel_lens/#results).
 | :--- | :--- |
 | [Problem](https://just1truc.github.io/kernel_lens/#problem) | The five gaps a naive export hits |
 | [Pipeline](https://just1truc.github.io/kernel_lens/#pipeline) | What you build by hand vs. the compile/run pipeline |
-| [Results](https://just1truc.github.io/kernel_lens/#results) | Folding + kernels, parity, latency, end-to-end |
+| [Results](https://just1truc.github.io/kernel_lens/#results) | Folding + kernels, parity, conformance, single-operator and end-to-end latency, **real pretrained models** |
+| [Why an engine](https://just1truc.github.io/kernel_lens/#why) | `torch.compile` vs TensorRT, both directions |
 | [Install & API](https://just1truc.github.io/kernel_lens/#install) | Installation and API reference |
 | [Status](https://just1truc.github.io/kernel_lens/#status) | Roadmap |
+| [FAQ](https://just1truc.github.io/kernel_lens/#faq) | Fair objections |
 | [Paper](https://just1truc.github.io/kernel_lens/#paper) | Abstract and citation |
 
 ## Reproduce the results
@@ -212,14 +272,30 @@ python tests/test_research_kernels.py          # numerical parity
 python tests/benchmark_research_kernels.py     # single-operator latency
 python tests/benchmark_e2e_llama.py            # end-to-end decoder
 python tests/test_fp8_gemm.py                  # FP8 Tensor Core path
+python tests/measure_codegen.py                # generated C++ volume, per file
+python tests/benchmark_real_model.py --model facebook/opt-1.3b   # real OPT, both precisions
 ```
 
+The real-model matrix (OPT-125M → 6.7B, fp32 and fp16) runs unattended on a fresh
+CUDA host: `scripts/remote_setup.sh` provisions nvcc, TensorRT, `onnxruntime-gpu`
+and `transformers` and installs the package; `scripts/remote_bench_all.sh` then
+sweeps the matrix into `results/` and writes `summary.json`, recording failures
+instead of aborting. See
+[`docs/assets/data/a100/runpod_runbook.md`](docs/assets/data/a100/runpod_runbook.md) for the
+exact invocation and the environment traps (CUDA 12.8's `nvcc` does not build
+against a modern libstdc++; CUDA 13.3's does).
+
 Raw measurement JSON is committed under [`docs/assets/data/`](docs/assets/data/).
 
 ## Technical report
 
-The LaTeX source is [`architecture_report.tex`](architecture_report.tex); the TMLR
-variant is generated from it with `python3 scripts/make_tmlr.py`.
+The paper source is [`kernel_lens_fresh.tex`](kernel_lens_fresh.tex). Two derived
+variants are generated from it and must not be edited by hand:
+`scripts/make_mlsys.py` produces the 2-column MLSys form
+(`kernel_lens_mlsys.tex`) and `scripts/make_tmlr_fresh.py` produces the TMLR
+forms — `kernel_lens_tmlr_fresh.tex` (named preprint) and
+`kernel_lens_tmlr_submission.tex` (anonymized; **do not publish it alongside the
+named preprint**).
 
 ```bibtex
 @article{duc2026kernellens,
@@ -227,7 +303,7 @@ variant is generated from it with `python3 scripts/make_tmlr.py`.
   author      = {Duc, Justin},
   institution = {Galiad Research},
   year        = {2026},
-  version     = {1.1.9},
+  version     = {1.2.1},
   doi         = {10.5281/zenodo.23194523},
   url         = {https://doi.org/10.5281/zenodo.23194523}
 }

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/kernel_lens.egg-info/requires.txt

@@ -7,6 +7,9 @@ onnxruntime-gpu>=1.17.0
 tensorrt<11.0.0,>=8.6.0
 tensorrt-cu12-libs<11.0.0
 
+[bench]
+transformers>=4.40.0
+
 [ort]
 onnxruntime-gpu>=1.17.0
 

{kernel_lens-1.2.0 → kernel_lens-1.2.1}/pyproject.toml

@@ -4,18 +4,36 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kernel-lens"
-version = "1.2.0"
-description = "A production-grade, multi-backend compiler for Triton kernels (ONNX & TensorRT)"
+version = "1.2.1"
+description = "Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins"
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.10"
 authors = [
-    { name = "Compiler Engineer" }
+    { name = "Justin Duc", email = "justin.duc@galiad-research.com" }
+]
+keywords = [
+    "triton",
+    "tensorrt",
+    "onnxruntime",
+    "compiler",
+    "gpu",
+    "cuda",
+    "kernel",
+    "inference",
+    "deep-learning",
 ]
 classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "License :: OSI Approved :: MIT License",
     "Operating System :: POSIX :: Linux",
+    "Environment :: GPU :: NVIDIA CUDA",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 
@@ -31,10 +49,16 @@ dependencies = [
 ort = ["onnxruntime-gpu>=1.17.0"]
 trt = ["tensorrt>=8.6.0,<11.0.0", "tensorrt-cu12-libs<11.0.0"]
 all = ["onnxruntime-gpu>=1.17.0", "tensorrt>=8.6.0,<11.0.0", "tensorrt-cu12-libs<11.0.0"]
+# Needed only by tests/benchmark_real_model.py, which serves real checkpoints.
+bench = ["transformers>=4.40.0"]
 
 [project.urls]
+Homepage = "https://just1truc.github.io/kernel_lens/"
+Documentation = "https://just1truc.github.io/kernel_lens/"
+Changelog = "https://github.com/Just1truc/kernel_lens/blob/main/CHANGELOG.md"
 "Source Code" = "https://github.com/Just1truc/kernel_lens"
 "Bug Tracker" = "https://github.com/Just1truc/kernel_lens/issues"
+Paper = "https://doi.org/10.5281/zenodo.23194523"
 
 [tool.setuptools.packages.find]
 include = ["kernel_lens*"]