kernel-lens 2.0.0__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/PKG-INFO +9 -5
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/README.md +8 -4
- kernel_lens-2.0.1/kernel_lens/compiler/ast_analyzer.py +300 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/PKG-INFO +9 -5
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/pyproject.toml +1 -1
- kernel_lens-2.0.0/kernel_lens/compiler/ast_analyzer.py +0 -179
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/LICENSE +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/__init__.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/__init__.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/builder.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/ort_gen.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/trt_gen.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/trt_v3_gen.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/__init__.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/core.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/interaction.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/manifest.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/onnx_exporter.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/tracer.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/config.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInfer.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferImpl.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferLegacyDims.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferPlugin.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferPluginBase.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferPluginUtils.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntime.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntimeBase.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntimeCommon.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntimePlugin.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferVersion.h +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/runtime/__init__.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/runtime/engine.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/__init__.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/deployment.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/env_check.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/trt_headers.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/SOURCES.txt +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/dependency_links.txt +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/requires.txt +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/top_level.txt +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/setup.cfg +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_alignment_guard.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_architectural_fixes.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_conformance.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_edge_cases.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_fp8_gemm.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_fused_cross_entropy.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_fused_residual_rms.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_inplace.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_inplace_dtypes.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_int4_dequant_gemm.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_mamba_selective_scan.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_research_kernels.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_standalone.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_standalone_launch.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_standalone_ptx.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_tricky_triton_kernels.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_triton_vs_torch_compile.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_trt.py +0 -0
- {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_unit_cpu.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kernel-lens
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins
|
|
5
5
|
Author-email: Justin Duc <justin.duc@galiad-research.com>
|
|
6
6
|
License: MIT License
|
|
@@ -86,7 +86,8 @@ Dynamic: license-file
|
|
|
86
86
|
|
|
87
87
|
**KernelLens** compiles a PyTorch model that contains custom `@triton.jit` kernels
|
|
88
88
|
into native C++ artifacts for **ONNX Runtime** (`OrtCustomOp`) and **NVIDIA
|
|
89
|
-
TensorRT
|
|
89
|
+
TensorRT 8.6 to 11** (`nvinfer1::IPluginV2DynamicExt`, and `nvinfer1::IPluginV3`
|
|
90
|
+
on TensorRT 10 and later). You write Triton in Python; it
|
|
90
91
|
emits the C++ plugin, builds the `.so`, and runs it with **zero-copy VRAM
|
|
91
92
|
binding** — no manual C++ and no Python in the serving path.
|
|
92
93
|
|
|
@@ -145,9 +146,12 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
|
|
|
145
146
|
> conformance patterns and the quantized GEMMs, on both backends; two
|
|
146
147
|
> multi-operator fusions land just above zero and are reported as such. Keeping
|
|
147
148
|
> the kernel pays when the baseline leaves the graph unfused and breaks even when
|
|
148
|
-
> it already fuses.
|
|
149
|
-
> [
|
|
150
|
-
>
|
|
149
|
+
> it already fuses. A TensorRT-LLM backend is out of scope; the
|
|
150
|
+
> [design note](docs/assets/data/a100/trtllm_backend_design.md) records what
|
|
151
|
+
> integrating one would take. The kernels themselves are usable there — they are
|
|
152
|
+
> ordinary `@triton.jit` functions, and an A100 measurement confirmed a KernelLens
|
|
153
|
+
> RMSNorm is bit-identical to the flashinfer kernel TensorRT-LLM already uses
|
|
154
|
+
> (`0.00e+00`). The quantized GEMMs are validated on ONNX Runtime only. Full methodology
|
|
151
155
|
> and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
|
|
152
156
|
|
|
153
157
|
## Installation
|
|
@@ -22,7 +22,8 @@
|
|
|
22
22
|
|
|
23
23
|
**KernelLens** compiles a PyTorch model that contains custom `@triton.jit` kernels
|
|
24
24
|
into native C++ artifacts for **ONNX Runtime** (`OrtCustomOp`) and **NVIDIA
|
|
25
|
-
TensorRT
|
|
25
|
+
TensorRT 8.6 to 11** (`nvinfer1::IPluginV2DynamicExt`, and `nvinfer1::IPluginV3`
|
|
26
|
+
on TensorRT 10 and later). You write Triton in Python; it
|
|
26
27
|
emits the C++ plugin, builds the `.so`, and runs it with **zero-copy VRAM
|
|
27
28
|
binding** — no manual C++ and no Python in the serving path.
|
|
28
29
|
|
|
@@ -81,9 +82,12 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
|
|
|
81
82
|
> conformance patterns and the quantized GEMMs, on both backends; two
|
|
82
83
|
> multi-operator fusions land just above zero and are reported as such. Keeping
|
|
83
84
|
> the kernel pays when the baseline leaves the graph unfused and breaks even when
|
|
84
|
-
> it already fuses.
|
|
85
|
-
> [
|
|
86
|
-
>
|
|
85
|
+
> it already fuses. A TensorRT-LLM backend is out of scope; the
|
|
86
|
+
> [design note](docs/assets/data/a100/trtllm_backend_design.md) records what
|
|
87
|
+
> integrating one would take. The kernels themselves are usable there — they are
|
|
88
|
+
> ordinary `@triton.jit` functions, and an A100 measurement confirmed a KernelLens
|
|
89
|
+
> RMSNorm is bit-identical to the flashinfer kernel TensorRT-LLM already uses
|
|
90
|
+
> (`0.00e+00`). The quantized GEMMs are validated on ONNX Runtime only. Full methodology
|
|
87
91
|
> and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
|
|
88
92
|
|
|
89
93
|
## Installation
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import inspect
|
|
3
|
+
import re
|
|
4
|
+
import sys
|
|
5
|
+
import textwrap
|
|
6
|
+
from typing import Any, Dict, List, Optional, Set
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
|
|
10
|
+
from ..config import debug_print
|
|
11
|
+
|
|
12
|
+
# REMOVED the circular import: from .manifest import KernelManifest
|
|
13
|
+
# REMOVED the circular import: from .interaction import InteractionHandler
|
|
14
|
+
|
|
15
|
+
def raw_sym_expr(sym_int: Any) -> Any:
|
|
16
|
+
"""The *unsubstituted* symbolic expression behind a SymInt.
|
|
17
|
+
|
|
18
|
+
``SymNode.expr`` runs ``shape_env.replace()`` with the values the env has
|
|
19
|
+
recorded, so once tracing has bound s75 -> 8 and s96 -> 512 it folds
|
|
20
|
+
``(s75*s96 + 255)//256`` to the constant 16. That is the right answer for a
|
|
21
|
+
shape-specialised engine and exactly the wrong one for a dynamic engine.
|
|
22
|
+
``SymNode._expr`` still holds the symbol, so read that first.
|
|
23
|
+
"""
|
|
24
|
+
node = getattr(sym_int, "node", None)
|
|
25
|
+
if node is None:
|
|
26
|
+
return sym_int
|
|
27
|
+
raw = getattr(node, "_expr", None)
|
|
28
|
+
if raw is not None and not isinstance(raw, int):
|
|
29
|
+
return raw
|
|
30
|
+
return getattr(node, "expr", raw)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _cxx_pow(expr_str: str) -> str:
|
|
34
|
+
"""sympy renders powers as ``a**b``; C++ has no ``**``.
|
|
35
|
+
|
|
36
|
+
Small integer powers are expanded by hand -- ``std::pow`` would drag in
|
|
37
|
+
<cmath> and go through a double, and a launch grid is exact integer maths.
|
|
38
|
+
"""
|
|
39
|
+
for _ in range(4):
|
|
40
|
+
m = re.search(r'\b(\w+)\*\*(\d+)\b', expr_str)
|
|
41
|
+
if not m:
|
|
42
|
+
break
|
|
43
|
+
base, exp = m.group(1), int(m.group(2))
|
|
44
|
+
expr_str = expr_str[:m.start()] + "(" + "*".join([base] * exp) + ")" + expr_str[m.end():]
|
|
45
|
+
if "**" in expr_str:
|
|
46
|
+
# Only reachable for a non-integer or large exponent; keep it valid C++.
|
|
47
|
+
expr_str = re.sub(r'\b(\w+)\*\*(\w+)\b', r'((int64_t)std::pow((double)\1, (double)\2))', expr_str)
|
|
48
|
+
return expr_str
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def translate_symint_to_cxx(sym_int: Any, sym_map: Dict[str, str]) -> str:
|
|
52
|
+
"""
|
|
53
|
+
Converts a PyTorch SymInt into a TensorRT/ORT C++ string.
|
|
54
|
+
"""
|
|
55
|
+
if not isinstance(sym_int, torch.SymInt):
|
|
56
|
+
return str(sym_int)
|
|
57
|
+
|
|
58
|
+
raw = raw_sym_expr(sym_int)
|
|
59
|
+
# Expand sympy powers BEFORE substituting: at this point the base is a bare
|
|
60
|
+
# symbol (s76**2), which is easy to rewrite. Afterwards the base is a
|
|
61
|
+
# parenthesised C++ expression and the rewrite gets ambiguous.
|
|
62
|
+
expr_str = _cxx_pow(str(raw))
|
|
63
|
+
|
|
64
|
+
sorted_map = sorted(sym_map.items(), key=lambda x: len(x[0]), reverse=True)
|
|
65
|
+
|
|
66
|
+
for sym_var, cxx_code in sorted_map:
|
|
67
|
+
if sym_var.isalnum():
|
|
68
|
+
expr_str = re.sub(rf'\b{re.escape(sym_var)}\b', lambda _m, c=cxx_code: c, expr_str)
|
|
69
|
+
else:
|
|
70
|
+
expr_str = expr_str.replace(sym_var, f"({cxx_code})")
|
|
71
|
+
|
|
72
|
+
if re.search(r'\bs\d+\b', expr_str):
|
|
73
|
+
# A symbol survived: the map did not cover it. Emitting it would be a
|
|
74
|
+
# C++ identifier error, so fall back to the value the shape env
|
|
75
|
+
# resolved -- the old behaviour. Static, but valid.
|
|
76
|
+
expr_str = str(sym_int)
|
|
77
|
+
|
|
78
|
+
expr_str = _cxx_pow(expr_str)
|
|
79
|
+
expr_str = expr_str.replace("//", "/")
|
|
80
|
+
return expr_str
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------------------
|
|
83
|
+
# Argument taxonomy
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
def warn(msg: str) -> None:
|
|
87
|
+
"""Taxonomy diagnostics go to stderr so stdout stays parseable."""
|
|
88
|
+
sys.stderr.write(f"[kernel_lens] {msg}\n")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _source_ast(fn_obj: Any) -> Optional[ast.FunctionDef]:
|
|
92
|
+
"""The FunctionDef behind a kernel or helper, or None if it is unavailable."""
|
|
93
|
+
actual = getattr(fn_obj, "fn", fn_obj)
|
|
94
|
+
if not callable(actual):
|
|
95
|
+
return None
|
|
96
|
+
try:
|
|
97
|
+
source = inspect.getsource(actual)
|
|
98
|
+
except Exception:
|
|
99
|
+
return None
|
|
100
|
+
# A nested function's source comes back indented; ast.parse needs it flush.
|
|
101
|
+
for candidate in (source, textwrap.dedent(source)):
|
|
102
|
+
try:
|
|
103
|
+
tree = ast.parse(candidate)
|
|
104
|
+
except SyntaxError:
|
|
105
|
+
continue
|
|
106
|
+
for node in ast.walk(tree):
|
|
107
|
+
if isinstance(node, ast.FunctionDef):
|
|
108
|
+
return node
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _definitions(fn_node: ast.FunctionDef) -> Dict[str, List[Any]]:
|
|
113
|
+
"""Local name -> the expressions assigned to it.
|
|
114
|
+
|
|
115
|
+
``p += off`` is treated as ``p = p + off`` (self-referential; the resolver's
|
|
116
|
+
``seen`` set terminates it), and a ``for`` target takes its iterable.
|
|
117
|
+
"""
|
|
118
|
+
defs: Dict[str, List[Any]] = {}
|
|
119
|
+
for node in ast.walk(fn_node):
|
|
120
|
+
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
|
|
121
|
+
defs.setdefault(node.targets[0].id, []).append(node.value)
|
|
122
|
+
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None:
|
|
123
|
+
defs.setdefault(node.target.id, []).append(node.value)
|
|
124
|
+
elif isinstance(node, ast.AugAssign) and isinstance(node.target, ast.Name):
|
|
125
|
+
defs.setdefault(node.target.id, []).append(
|
|
126
|
+
ast.BinOp(left=node.target, op=node.op, right=node.value))
|
|
127
|
+
elif isinstance(node, ast.For) and isinstance(node.target, ast.Name):
|
|
128
|
+
defs.setdefault(node.target.id, []).append(node.iter)
|
|
129
|
+
return defs
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class TargetResolver:
|
|
133
|
+
"""Resolves load/store targets back to the arguments they derive from.
|
|
134
|
+
|
|
135
|
+
This replaces a string test -- "does this argument's name occur in the
|
|
136
|
+
unparsed target?" -- which misses every pointer renamed through an
|
|
137
|
+
intermediate (``q_ptrs = Q_ptr + offs`` names neither ``Q`` nor ``Q_ptr``)
|
|
138
|
+
and over-matches short names (``o`` is a substring of both ``row_x`` and
|
|
139
|
+
``row_out``, which turns a plain output into an in-place argument).
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
_MAX_DEPTH = 12
|
|
143
|
+
|
|
144
|
+
def __init__(self, fn_node, env, globals_map, visited):
|
|
145
|
+
self.fn = fn_node
|
|
146
|
+
self.env = env # this scope's parameter -> kernel arguments
|
|
147
|
+
self.globals = globals_map
|
|
148
|
+
self.visited = visited # (qualname, bound env) recursion guard
|
|
149
|
+
self.defs = _definitions(fn_node)
|
|
150
|
+
self.loads: Set[str] = set()
|
|
151
|
+
self.stores: Set[str] = set()
|
|
152
|
+
|
|
153
|
+
def resolve(self, node, seen=frozenset(), depth: int = 0) -> Set[str]:
|
|
154
|
+
"""Kernel arguments reachable by walking ``node`` back through assignments."""
|
|
155
|
+
out: Set[str] = set()
|
|
156
|
+
if depth > self._MAX_DEPTH:
|
|
157
|
+
return out
|
|
158
|
+
for child in ast.walk(node):
|
|
159
|
+
if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load):
|
|
160
|
+
if child.id in self.env:
|
|
161
|
+
out |= self.env[child.id]
|
|
162
|
+
elif child.id in self.defs and child.id not in seen:
|
|
163
|
+
for rhs in self.defs[child.id]:
|
|
164
|
+
out |= self.resolve(rhs, seen | {child.id}, depth + 1)
|
|
165
|
+
return out
|
|
166
|
+
|
|
167
|
+
def scan(self):
|
|
168
|
+
for node in ast.walk(self.fn):
|
|
169
|
+
if not isinstance(node, ast.Call):
|
|
170
|
+
continue
|
|
171
|
+
callee = node.func.attr if isinstance(node.func, ast.Attribute) else getattr(node.func, "id", "")
|
|
172
|
+
if callee in ("load", "store") and node.args:
|
|
173
|
+
reached = self.resolve(node.args[0])
|
|
174
|
+
(self.loads if callee == "load" else self.stores).update(reached)
|
|
175
|
+
elif callee:
|
|
176
|
+
self._descend(callee, node)
|
|
177
|
+
return self.loads, self.stores
|
|
178
|
+
|
|
179
|
+
def _descend(self, callee_name: str, call_node: ast.Call) -> None:
|
|
180
|
+
"""Follow a call into a helper, binding its parameters to the call site.
|
|
181
|
+
|
|
182
|
+
The binding is what makes nested helpers work. Attributing the helper's
|
|
183
|
+
own parameter names, as a bare recursion does, only lines up with the
|
|
184
|
+
kernel's arguments when the two happen to share a name.
|
|
185
|
+
"""
|
|
186
|
+
target = self.globals.get(callee_name)
|
|
187
|
+
if target is None or not callable(target):
|
|
188
|
+
return
|
|
189
|
+
helper = _source_ast(target)
|
|
190
|
+
if helper is None:
|
|
191
|
+
return
|
|
192
|
+
actual = getattr(target, "fn", target)
|
|
193
|
+
try:
|
|
194
|
+
bound = inspect.signature(actual).bind_partial(
|
|
195
|
+
*call_node.args,
|
|
196
|
+
**{kw.arg: kw.value for kw in call_node.keywords if kw.arg})
|
|
197
|
+
except Exception:
|
|
198
|
+
warn(f"could not bind the arguments of '{callee_name}'; "
|
|
199
|
+
"its loads and stores are not attributed")
|
|
200
|
+
return
|
|
201
|
+
env = {}
|
|
202
|
+
for pname, pval in bound.arguments.items():
|
|
203
|
+
env[pname] = self.resolve(pval) if isinstance(pval, ast.AST) else set()
|
|
204
|
+
key = (getattr(actual, "__qualname__", callee_name),
|
|
205
|
+
frozenset((k, frozenset(v)) for k, v in env.items()))
|
|
206
|
+
if key in self.visited:
|
|
207
|
+
return
|
|
208
|
+
self.visited.add(key)
|
|
209
|
+
sub = TargetResolver(helper, env, getattr(actual, "__globals__", self.globals), self.visited)
|
|
210
|
+
loads, stores = sub.scan()
|
|
211
|
+
self.loads |= loads
|
|
212
|
+
self.stores |= stores
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# A tensor argument the resolver could not reach is guessed from its name.
|
|
216
|
+
# These hints only ever decide arguments the resolution left undetermined.
|
|
217
|
+
_OUTPUT_HINTS = ['out', 'dst', 'res', 'output', 'result', 'y']
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def analyze_grid_asts(
|
|
221
|
+
manifests: List[Any],
|
|
222
|
+
handler: Optional[Any] = None
|
|
223
|
+
) -> List[Any]:
|
|
224
|
+
"""Classify each tensor argument as input, output or in-place.
|
|
225
|
+
|
|
226
|
+
Every ``tl.load`` / ``tl.store`` target is resolved backwards through the
|
|
227
|
+
function's own assignments to the arguments it derives from. Only arguments
|
|
228
|
+
that resolution does not reach are guessed, and each guess is reported on
|
|
229
|
+
stderr so a silent misclassification cannot pass unnoticed.
|
|
230
|
+
"""
|
|
231
|
+
for manifest in manifests:
|
|
232
|
+
debug_print(f"\n{'='*50}\nConfiguring I/O for: {manifest.kernel_name}\n{'='*50}")
|
|
233
|
+
|
|
234
|
+
load_reached: Set[str] = set()
|
|
235
|
+
store_reached: Set[str] = set()
|
|
236
|
+
|
|
237
|
+
if hasattr(manifest, 'fn') and manifest.fn is not None:
|
|
238
|
+
fn_node = _source_ast(manifest.fn)
|
|
239
|
+
if fn_node is not None:
|
|
240
|
+
actual = getattr(manifest.fn, "fn", manifest.fn)
|
|
241
|
+
params = [a.arg for a in fn_node.args.args] + [a.arg for a in fn_node.args.kwonlyargs]
|
|
242
|
+
resolver = TargetResolver(
|
|
243
|
+
fn_node,
|
|
244
|
+
{p: {p} for p in params},
|
|
245
|
+
getattr(actual, "__globals__", {}),
|
|
246
|
+
set(),
|
|
247
|
+
)
|
|
248
|
+
load_reached, store_reached = resolver.scan()
|
|
249
|
+
else:
|
|
250
|
+
warn(f"source unavailable for '{manifest.kernel_name}'; "
|
|
251
|
+
"argument roles fall back to name heuristics")
|
|
252
|
+
|
|
253
|
+
tensor_args = [a for a in manifest.arguments if a.shape]
|
|
254
|
+
tensor_names = {a.name for a in tensor_args}
|
|
255
|
+
input_arg_names = load_reached & tensor_names
|
|
256
|
+
output_arg_names = store_reached & tensor_names
|
|
257
|
+
|
|
258
|
+
undetermined = tensor_names - input_arg_names - output_arg_names
|
|
259
|
+
if undetermined and tensor_args:
|
|
260
|
+
warn(f"'{manifest.kernel_name}': no load/store target resolved to "
|
|
261
|
+
f"{sorted(undetermined)}; guessing from names")
|
|
262
|
+
for name in sorted(undetermined):
|
|
263
|
+
lname = name.lower()
|
|
264
|
+
if any(kw in lname for kw in _OUTPUT_HINTS) and not lname.startswith('in'):
|
|
265
|
+
output_arg_names.add(name)
|
|
266
|
+
if not output_arg_names and len(tensor_args) > 1:
|
|
267
|
+
last = tensor_args[-1].name
|
|
268
|
+
output_arg_names.add(last)
|
|
269
|
+
warn(f"'{manifest.kernel_name}': defaulting the last tensor "
|
|
270
|
+
f"argument '{last}' to output")
|
|
271
|
+
|
|
272
|
+
for arg in manifest.arguments:
|
|
273
|
+
if arg.shape:
|
|
274
|
+
if arg.name in output_arg_names and arg.name in input_arg_names:
|
|
275
|
+
arg.kind = 'inplace'
|
|
276
|
+
debug_print(f"[AST-Analysis] '{arg.name}' resolved to INPLACE (loaded and stored).")
|
|
277
|
+
elif arg.name in output_arg_names:
|
|
278
|
+
arg.kind = 'output'
|
|
279
|
+
debug_print(f"[AST-Analysis] '{arg.name}' resolved to OUTPUT.")
|
|
280
|
+
elif arg.name in input_arg_names:
|
|
281
|
+
arg.kind = 'input'
|
|
282
|
+
debug_print(f"[AST-Analysis] '{arg.name}' resolved to INPUT.")
|
|
283
|
+
elif handler is not None:
|
|
284
|
+
arg.kind = handler.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
|
|
285
|
+
else:
|
|
286
|
+
from .interaction import AutoInteractionHandler
|
|
287
|
+
h = AutoInteractionHandler()
|
|
288
|
+
arg.kind = h.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
|
|
289
|
+
else:
|
|
290
|
+
arg.kind = 'scalar'
|
|
291
|
+
debug_print(f"[Auto] Mapped scalar constant: {arg.name} = {arg.value}")
|
|
292
|
+
|
|
293
|
+
# A kernel with no stride arguments addresses a flat buffer, so it is
|
|
294
|
+
# agnostic to the physical layout and can accept channel-last formats.
|
|
295
|
+
manifest.layout_agnostic = not any(
|
|
296
|
+
'stride' in a.name.lower() for a in manifest.arguments
|
|
297
|
+
)
|
|
298
|
+
debug_print(f"[Layout] '{manifest.kernel_name}' layout_agnostic={manifest.layout_agnostic}")
|
|
299
|
+
|
|
300
|
+
return manifests
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kernel-lens
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins
|
|
5
5
|
Author-email: Justin Duc <justin.duc@galiad-research.com>
|
|
6
6
|
License: MIT License
|
|
@@ -86,7 +86,8 @@ Dynamic: license-file
|
|
|
86
86
|
|
|
87
87
|
**KernelLens** compiles a PyTorch model that contains custom `@triton.jit` kernels
|
|
88
88
|
into native C++ artifacts for **ONNX Runtime** (`OrtCustomOp`) and **NVIDIA
|
|
89
|
-
TensorRT
|
|
89
|
+
TensorRT 8.6 to 11** (`nvinfer1::IPluginV2DynamicExt`, and `nvinfer1::IPluginV3`
|
|
90
|
+
on TensorRT 10 and later). You write Triton in Python; it
|
|
90
91
|
emits the C++ plugin, builds the `.so`, and runs it with **zero-copy VRAM
|
|
91
92
|
binding** — no manual C++ and no Python in the serving path.
|
|
92
93
|
|
|
@@ -145,9 +146,12 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
|
|
|
145
146
|
> conformance patterns and the quantized GEMMs, on both backends; two
|
|
146
147
|
> multi-operator fusions land just above zero and are reported as such. Keeping
|
|
147
148
|
> the kernel pays when the baseline leaves the graph unfused and breaks even when
|
|
148
|
-
> it already fuses.
|
|
149
|
-
> [
|
|
150
|
-
>
|
|
149
|
+
> it already fuses. A TensorRT-LLM backend is out of scope; the
|
|
150
|
+
> [design note](docs/assets/data/a100/trtllm_backend_design.md) records what
|
|
151
|
+
> integrating one would take. The kernels themselves are usable there — they are
|
|
152
|
+
> ordinary `@triton.jit` functions, and an A100 measurement confirmed a KernelLens
|
|
153
|
+
> RMSNorm is bit-identical to the flashinfer kernel TensorRT-LLM already uses
|
|
154
|
+
> (`0.00e+00`). The quantized GEMMs are validated on ONNX Runtime only. Full methodology
|
|
151
155
|
> and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
|
|
152
156
|
|
|
153
157
|
## Installation
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "kernel-lens"
|
|
7
|
-
version = "2.0.
|
|
7
|
+
version = "2.0.1"
|
|
8
8
|
description = "Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
import re
|
|
2
|
-
import torch
|
|
3
|
-
from typing import List, Optional, Dict, Any
|
|
4
|
-
from ..config import debug_print
|
|
5
|
-
|
|
6
|
-
# REMOVED the circular import: from .manifest import KernelManifest
|
|
7
|
-
# REMOVED the circular import: from .interaction import InteractionHandler
|
|
8
|
-
|
|
9
|
-
def raw_sym_expr(sym_int: Any) -> Any:
|
|
10
|
-
"""The *unsubstituted* symbolic expression behind a SymInt.
|
|
11
|
-
|
|
12
|
-
``SymNode.expr`` runs ``shape_env.replace()`` with the values the env has
|
|
13
|
-
recorded, so once tracing has bound s75 -> 8 and s96 -> 512 it folds
|
|
14
|
-
``(s75*s96 + 255)//256`` to the constant 16. That is the right answer for a
|
|
15
|
-
shape-specialised engine and exactly the wrong one for a dynamic engine.
|
|
16
|
-
``SymNode._expr`` still holds the symbol, so read that first.
|
|
17
|
-
"""
|
|
18
|
-
node = getattr(sym_int, "node", None)
|
|
19
|
-
if node is None:
|
|
20
|
-
return sym_int
|
|
21
|
-
raw = getattr(node, "_expr", None)
|
|
22
|
-
if raw is not None and not isinstance(raw, int):
|
|
23
|
-
return raw
|
|
24
|
-
return getattr(node, "expr", raw)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def _cxx_pow(expr_str: str) -> str:
|
|
28
|
-
"""sympy renders powers as ``a**b``; C++ has no ``**``.
|
|
29
|
-
|
|
30
|
-
Small integer powers are expanded by hand -- ``std::pow`` would drag in
|
|
31
|
-
<cmath> and go through a double, and a launch grid is exact integer maths.
|
|
32
|
-
"""
|
|
33
|
-
for _ in range(4):
|
|
34
|
-
m = re.search(r'\b(\w+)\*\*(\d+)\b', expr_str)
|
|
35
|
-
if not m:
|
|
36
|
-
break
|
|
37
|
-
base, exp = m.group(1), int(m.group(2))
|
|
38
|
-
expr_str = expr_str[:m.start()] + "(" + "*".join([base] * exp) + ")" + expr_str[m.end():]
|
|
39
|
-
if "**" in expr_str:
|
|
40
|
-
# Only reachable for a non-integer or large exponent; keep it valid C++.
|
|
41
|
-
expr_str = re.sub(r'\b(\w+)\*\*(\w+)\b', r'((int64_t)std::pow((double)\1, (double)\2))', expr_str)
|
|
42
|
-
return expr_str
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def translate_symint_to_cxx(sym_int: Any, sym_map: Dict[str, str]) -> str:
|
|
46
|
-
"""
|
|
47
|
-
Converts a PyTorch SymInt into a TensorRT/ORT C++ string.
|
|
48
|
-
"""
|
|
49
|
-
if not isinstance(sym_int, torch.SymInt):
|
|
50
|
-
return str(sym_int)
|
|
51
|
-
|
|
52
|
-
raw = raw_sym_expr(sym_int)
|
|
53
|
-
# Expand sympy powers BEFORE substituting: at this point the base is a bare
|
|
54
|
-
# symbol (s76**2), which is easy to rewrite. Afterwards the base is a
|
|
55
|
-
# parenthesised C++ expression and the rewrite gets ambiguous.
|
|
56
|
-
expr_str = _cxx_pow(str(raw))
|
|
57
|
-
|
|
58
|
-
sorted_map = sorted(sym_map.items(), key=lambda x: len(x[0]), reverse=True)
|
|
59
|
-
|
|
60
|
-
for sym_var, cxx_code in sorted_map:
|
|
61
|
-
if sym_var.isalnum():
|
|
62
|
-
expr_str = re.sub(rf'\b{re.escape(sym_var)}\b', lambda _m, c=cxx_code: c, expr_str)
|
|
63
|
-
else:
|
|
64
|
-
expr_str = expr_str.replace(sym_var, f"({cxx_code})")
|
|
65
|
-
|
|
66
|
-
if re.search(r'\bs\d+\b', expr_str):
|
|
67
|
-
# A symbol survived: the map did not cover it. Emitting it would be a
|
|
68
|
-
# C++ identifier error, so fall back to the value the shape env
|
|
69
|
-
# resolved -- the old behaviour. Static, but valid.
|
|
70
|
-
expr_str = str(sym_int)
|
|
71
|
-
|
|
72
|
-
expr_str = _cxx_pow(expr_str)
|
|
73
|
-
expr_str = expr_str.replace("//", "/")
|
|
74
|
-
return expr_str
|
|
75
|
-
|
|
76
|
-
def analyze_grid_asts(
|
|
77
|
-
manifests: List[Any],
|
|
78
|
-
handler: Optional[Any] = None
|
|
79
|
-
) -> List[Any]:
|
|
80
|
-
"""
|
|
81
|
-
Analyzes symbolic ASTs and classifies tensor arguments deterministically via tl.store inspection,
|
|
82
|
-
recursively traversing called helper functions if necessary.
|
|
83
|
-
"""
|
|
84
|
-
import inspect, ast
|
|
85
|
-
|
|
86
|
-
for manifest in manifests:
|
|
87
|
-
debug_print(f"\n{'='*50}\nConfiguring I/O for: {manifest.kernel_name}\n{'='*50}")
|
|
88
|
-
|
|
89
|
-
output_arg_names = set()
|
|
90
|
-
input_arg_names = set()
|
|
91
|
-
|
|
92
|
-
if hasattr(manifest, 'fn') and manifest.fn is not None:
|
|
93
|
-
try:
|
|
94
|
-
visited_fns = set()
|
|
95
|
-
|
|
96
|
-
def inspect_fn_ast(fn_obj):
|
|
97
|
-
if fn_obj in visited_fns:
|
|
98
|
-
return
|
|
99
|
-
visited_fns.add(fn_obj)
|
|
100
|
-
|
|
101
|
-
actual_fn = fn_obj.fn if hasattr(fn_obj, 'fn') else fn_obj
|
|
102
|
-
if not callable(actual_fn):
|
|
103
|
-
return
|
|
104
|
-
|
|
105
|
-
try:
|
|
106
|
-
source = inspect.getsource(actual_fn)
|
|
107
|
-
tree = ast.parse(source)
|
|
108
|
-
except Exception:
|
|
109
|
-
return
|
|
110
|
-
|
|
111
|
-
for node in ast.walk(tree):
|
|
112
|
-
if isinstance(node, ast.Call):
|
|
113
|
-
func_name = ""
|
|
114
|
-
if isinstance(node.func, ast.Attribute):
|
|
115
|
-
func_name = node.func.attr
|
|
116
|
-
elif isinstance(node.func, ast.Name):
|
|
117
|
-
func_name = node.func.id
|
|
118
|
-
|
|
119
|
-
if func_name == 'store' and node.args:
|
|
120
|
-
target_str = ast.unparse(node.args[0])
|
|
121
|
-
for arg in manifest.arguments:
|
|
122
|
-
if arg.shape and (arg.name in target_str or target_str.startswith(arg.name) or re.search(rf'\b{arg.name}\b', target_str)):
|
|
123
|
-
output_arg_names.add(arg.name)
|
|
124
|
-
elif func_name == 'load' and node.args:
|
|
125
|
-
target_str = ast.unparse(node.args[0])
|
|
126
|
-
for arg in manifest.arguments:
|
|
127
|
-
if arg.shape and (arg.name in target_str or target_str.startswith(arg.name) or re.search(rf'\b{arg.name}\b', target_str)):
|
|
128
|
-
input_arg_names.add(arg.name)
|
|
129
|
-
else:
|
|
130
|
-
# Check if calling another triton jit or helper function
|
|
131
|
-
try:
|
|
132
|
-
glob = getattr(actual_fn, '__globals__', {})
|
|
133
|
-
if func_name in glob and callable(glob[func_name]):
|
|
134
|
-
inspect_fn_ast(glob[func_name])
|
|
135
|
-
except Exception:
|
|
136
|
-
pass
|
|
137
|
-
|
|
138
|
-
fn_obj = manifest.fn
|
|
139
|
-
inspect_fn_ast(fn_obj)
|
|
140
|
-
except Exception as e:
|
|
141
|
-
debug_print(f"Warning during AST analysis: {e}")
|
|
142
|
-
|
|
143
|
-
# Heuristic fallback if output_arg_names is empty but tensor args exist
|
|
144
|
-
tensor_args = [a for a in manifest.arguments if a.shape]
|
|
145
|
-
if not output_arg_names and tensor_args:
|
|
146
|
-
for arg in tensor_args:
|
|
147
|
-
lname = arg.name.lower()
|
|
148
|
-
if any(kw in lname for kw in ['out', 'dst', 'res', 'output', 'result', 'y']) and not lname.startswith('in'):
|
|
149
|
-
output_arg_names.add(arg.name)
|
|
150
|
-
if not output_arg_names and len(tensor_args) > 1:
|
|
151
|
-
# Default the last tensor argument as output if no output detected
|
|
152
|
-
output_arg_names.add(tensor_args[-1].name)
|
|
153
|
-
|
|
154
|
-
for arg in manifest.arguments:
|
|
155
|
-
if arg.shape:
|
|
156
|
-
if arg.name in output_arg_names and arg.name in input_arg_names:
|
|
157
|
-
arg.kind = 'inplace'
|
|
158
|
-
debug_print(f"[AST-Analysis] '{arg.name}' statically resolved to INPLACE via tl.load and tl.store analysis.")
|
|
159
|
-
elif arg.name in output_arg_names:
|
|
160
|
-
arg.kind = 'output'
|
|
161
|
-
debug_print(f"[AST-Analysis] '{arg.name}' statically resolved to OUTPUT via tl.store analysis.")
|
|
162
|
-
elif handler is not None:
|
|
163
|
-
arg.kind = handler.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
|
|
164
|
-
else:
|
|
165
|
-
from .interaction import AutoInteractionHandler
|
|
166
|
-
h = AutoInteractionHandler()
|
|
167
|
-
arg.kind = h.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
|
|
168
|
-
else:
|
|
169
|
-
arg.kind = 'scalar'
|
|
170
|
-
debug_print(f"[Auto] Mapped scalar constant: {arg.name} = {arg.value}")
|
|
171
|
-
|
|
172
|
-
# A kernel with no stride arguments addresses a flat buffer, so it is
|
|
173
|
-
# agnostic to the physical layout and can accept channel-last formats.
|
|
174
|
-
manifest.layout_agnostic = not any(
|
|
175
|
-
'stride' in a.name.lower() for a in manifest.arguments
|
|
176
|
-
)
|
|
177
|
-
debug_print(f"[Layout] '{manifest.kernel_name}' layout_agnostic={manifest.layout_agnostic}")
|
|
178
|
-
|
|
179
|
-
return manifests
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|