kernel-lens 2.0.0__tar.gz → 2.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/PKG-INFO +9 -5
  2. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/README.md +8 -4
  3. kernel_lens-2.0.1/kernel_lens/compiler/ast_analyzer.py +300 -0
  4. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/PKG-INFO +9 -5
  5. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/pyproject.toml +1 -1
  6. kernel_lens-2.0.0/kernel_lens/compiler/ast_analyzer.py +0 -179
  7. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/LICENSE +0 -0
  8. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/__init__.py +0 -0
  9. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/__init__.py +0 -0
  10. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/builder.py +0 -0
  11. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/ort_gen.py +0 -0
  12. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/trt_gen.py +0 -0
  13. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/backends/trt_v3_gen.py +0 -0
  14. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/__init__.py +0 -0
  15. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/core.py +0 -0
  16. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/interaction.py +0 -0
  17. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/manifest.py +0 -0
  18. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/onnx_exporter.py +0 -0
  19. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/compiler/tracer.py +0 -0
  20. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/config.py +0 -0
  21. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInfer.h +0 -0
  22. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferImpl.h +0 -0
  23. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferLegacyDims.h +0 -0
  24. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferPlugin.h +0 -0
  25. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferPluginBase.h +0 -0
  26. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferPluginUtils.h +0 -0
  27. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntime.h +0 -0
  28. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntimeBase.h +0 -0
  29. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntimeCommon.h +0 -0
  30. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferRuntimePlugin.h +0 -0
  31. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/include/tensorrt/NvInferVersion.h +0 -0
  32. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/runtime/__init__.py +0 -0
  33. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/runtime/engine.py +0 -0
  34. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/__init__.py +0 -0
  35. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/deployment.py +0 -0
  36. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/env_check.py +0 -0
  37. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens/utils/trt_headers.py +0 -0
  38. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/SOURCES.txt +0 -0
  39. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/dependency_links.txt +0 -0
  40. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/requires.txt +0 -0
  41. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/kernel_lens.egg-info/top_level.txt +0 -0
  42. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/setup.cfg +0 -0
  43. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_alignment_guard.py +0 -0
  44. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_architectural_fixes.py +0 -0
  45. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_conformance.py +0 -0
  46. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_edge_cases.py +0 -0
  47. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_fp8_gemm.py +0 -0
  48. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_fused_cross_entropy.py +0 -0
  49. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_fused_residual_rms.py +0 -0
  50. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_inplace.py +0 -0
  51. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_inplace_dtypes.py +0 -0
  52. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_int4_dequant_gemm.py +0 -0
  53. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_mamba_selective_scan.py +0 -0
  54. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_research_kernels.py +0 -0
  55. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_standalone.py +0 -0
  56. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_standalone_launch.py +0 -0
  57. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_standalone_ptx.py +0 -0
  58. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_tricky_triton_kernels.py +0 -0
  59. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_triton_vs_torch_compile.py +0 -0
  60. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_trt.py +0 -0
  61. {kernel_lens-2.0.0 → kernel_lens-2.0.1}/tests/test_unit_cpu.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kernel-lens
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins
5
5
  Author-email: Justin Duc <justin.duc@galiad-research.com>
6
6
  License: MIT License
@@ -86,7 +86,8 @@ Dynamic: license-file
86
86
 
87
87
  **KernelLens** compiles a PyTorch model that contains custom `@triton.jit` kernels
88
88
  into native C++ artifacts for **ONNX Runtime** (`OrtCustomOp`) and **NVIDIA
89
- TensorRT 10.x** (`nvinfer1::IPluginV2DynamicExt`). You write Triton in Python; it
89
+ TensorRT 8.6 to 11** (`nvinfer1::IPluginV2DynamicExt`, and `nvinfer1::IPluginV3`
90
+ on TensorRT 10 and later). You write Triton in Python; it
90
91
  emits the C++ plugin, builds the `.so`, and runs it with **zero-copy VRAM
91
92
  binding** — no manual C++ and no Python in the serving path.
92
93
 
@@ -145,9 +146,12 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
145
146
  > conformance patterns and the quantized GEMMs, on both backends; two
146
147
  > multi-operator fusions land just above zero and are reported as such. Keeping
147
148
  > the kernel pays when the baseline leaves the graph unfused and breaks even when
148
- > it already fuses. The TensorRT-LLM backend is design-only (see
149
- > [`docs/assets/data/a100/trtllm_backend_design.md`](docs/assets/data/a100/trtllm_backend_design.md)),
150
- > and the quantized GEMMs are validated on ONNX Runtime only. Full methodology
149
+ > it already fuses. A TensorRT-LLM backend is out of scope; the
150
+ > [design note](docs/assets/data/a100/trtllm_backend_design.md) records what
151
+ > integrating one would take. The kernels themselves are usable there — they are
152
+ > ordinary `@triton.jit` functions, and an A100 measurement confirmed a KernelLens
153
+ > RMSNorm is bit-identical to the flashinfer kernel TensorRT-LLM already uses
154
+ > (`0.00e+00`). The quantized GEMMs are validated on ONNX Runtime only. Full methodology
151
155
  > and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
152
156
 
153
157
  ## Installation
@@ -22,7 +22,8 @@
22
22
 
23
23
  **KernelLens** compiles a PyTorch model that contains custom `@triton.jit` kernels
24
24
  into native C++ artifacts for **ONNX Runtime** (`OrtCustomOp`) and **NVIDIA
25
- TensorRT 10.x** (`nvinfer1::IPluginV2DynamicExt`). You write Triton in Python; it
25
+ TensorRT 8.6 to 11** (`nvinfer1::IPluginV2DynamicExt`, and `nvinfer1::IPluginV3`
26
+ on TensorRT 10 and later). You write Triton in Python; it
26
27
  emits the C++ plugin, builds the `.so`, and runs it with **zero-copy VRAM
27
28
  binding** — no manual C++ and no Python in the serving path.
28
29
 
@@ -81,9 +82,12 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
81
82
  > conformance patterns and the quantized GEMMs, on both backends; two
82
83
  > multi-operator fusions land just above zero and are reported as such. Keeping
83
84
  > the kernel pays when the baseline leaves the graph unfused and breaks even when
84
- > it already fuses. The TensorRT-LLM backend is design-only (see
85
- > [`docs/assets/data/a100/trtllm_backend_design.md`](docs/assets/data/a100/trtllm_backend_design.md)),
86
- > and the quantized GEMMs are validated on ONNX Runtime only. Full methodology
85
+ > it already fuses. A TensorRT-LLM backend is out of scope; the
86
+ > [design note](docs/assets/data/a100/trtllm_backend_design.md) records what
87
+ > integrating one would take. The kernels themselves are usable there — they are
88
+ > ordinary `@triton.jit` functions, and an A100 measurement confirmed a KernelLens
89
+ > RMSNorm is bit-identical to the flashinfer kernel TensorRT-LLM already uses
90
+ > (`0.00e+00`). The quantized GEMMs are validated on ONNX Runtime only. Full methodology
87
91
  > and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
88
92
 
89
93
  ## Installation
@@ -0,0 +1,300 @@
1
+ import ast
2
+ import inspect
3
+ import re
4
+ import sys
5
+ import textwrap
6
+ from typing import Any, Dict, List, Optional, Set
7
+
8
+ import torch
9
+
10
+ from ..config import debug_print
11
+
12
+ # REMOVED the circular import: from .manifest import KernelManifest
13
+ # REMOVED the circular import: from .interaction import InteractionHandler
14
+
15
+ def raw_sym_expr(sym_int: Any) -> Any:
16
+ """The *unsubstituted* symbolic expression behind a SymInt.
17
+
18
+ ``SymNode.expr`` runs ``shape_env.replace()`` with the values the env has
19
+ recorded, so once tracing has bound s75 -> 8 and s96 -> 512 it folds
20
+ ``(s75*s96 + 255)//256`` to the constant 16. That is the right answer for a
21
+ shape-specialised engine and exactly the wrong one for a dynamic engine.
22
+ ``SymNode._expr`` still holds the symbol, so read that first.
23
+ """
24
+ node = getattr(sym_int, "node", None)
25
+ if node is None:
26
+ return sym_int
27
+ raw = getattr(node, "_expr", None)
28
+ if raw is not None and not isinstance(raw, int):
29
+ return raw
30
+ return getattr(node, "expr", raw)
31
+
32
+
33
+ def _cxx_pow(expr_str: str) -> str:
34
+ """sympy renders powers as ``a**b``; C++ has no ``**``.
35
+
36
+ Small integer powers are expanded by hand -- ``std::pow`` would drag in
37
+ <cmath> and go through a double, and a launch grid is exact integer maths.
38
+ """
39
+ for _ in range(4):
40
+ m = re.search(r'\b(\w+)\*\*(\d+)\b', expr_str)
41
+ if not m:
42
+ break
43
+ base, exp = m.group(1), int(m.group(2))
44
+ expr_str = expr_str[:m.start()] + "(" + "*".join([base] * exp) + ")" + expr_str[m.end():]
45
+ if "**" in expr_str:
46
+ # Only reachable for a non-integer or large exponent; keep it valid C++.
47
+ expr_str = re.sub(r'\b(\w+)\*\*(\w+)\b', r'((int64_t)std::pow((double)\1, (double)\2))', expr_str)
48
+ return expr_str
49
+
50
+
51
+ def translate_symint_to_cxx(sym_int: Any, sym_map: Dict[str, str]) -> str:
52
+ """
53
+ Converts a PyTorch SymInt into a TensorRT/ORT C++ string.
54
+ """
55
+ if not isinstance(sym_int, torch.SymInt):
56
+ return str(sym_int)
57
+
58
+ raw = raw_sym_expr(sym_int)
59
+ # Expand sympy powers BEFORE substituting: at this point the base is a bare
60
+ # symbol (s76**2), which is easy to rewrite. Afterwards the base is a
61
+ # parenthesised C++ expression and the rewrite gets ambiguous.
62
+ expr_str = _cxx_pow(str(raw))
63
+
64
+ sorted_map = sorted(sym_map.items(), key=lambda x: len(x[0]), reverse=True)
65
+
66
+ for sym_var, cxx_code in sorted_map:
67
+ if sym_var.isalnum():
68
+ expr_str = re.sub(rf'\b{re.escape(sym_var)}\b', lambda _m, c=cxx_code: c, expr_str)
69
+ else:
70
+ expr_str = expr_str.replace(sym_var, f"({cxx_code})")
71
+
72
+ if re.search(r'\bs\d+\b', expr_str):
73
+ # A symbol survived: the map did not cover it. Emitting it would be a
74
+ # C++ identifier error, so fall back to the value the shape env
75
+ # resolved -- the old behaviour. Static, but valid.
76
+ expr_str = str(sym_int)
77
+
78
+ expr_str = _cxx_pow(expr_str)
79
+ expr_str = expr_str.replace("//", "/")
80
+ return expr_str
81
+
82
+ # ---------------------------------------------------------------------------
83
+ # Argument taxonomy
84
+ # ---------------------------------------------------------------------------
85
+
86
+ def warn(msg: str) -> None:
87
+ """Taxonomy diagnostics go to stderr so stdout stays parseable."""
88
+ sys.stderr.write(f"[kernel_lens] {msg}\n")
89
+
90
+
91
+ def _source_ast(fn_obj: Any) -> Optional[ast.FunctionDef]:
92
+ """The FunctionDef behind a kernel or helper, or None if it is unavailable."""
93
+ actual = getattr(fn_obj, "fn", fn_obj)
94
+ if not callable(actual):
95
+ return None
96
+ try:
97
+ source = inspect.getsource(actual)
98
+ except Exception:
99
+ return None
100
+ # A nested function's source comes back indented; ast.parse needs it flush.
101
+ for candidate in (source, textwrap.dedent(source)):
102
+ try:
103
+ tree = ast.parse(candidate)
104
+ except SyntaxError:
105
+ continue
106
+ for node in ast.walk(tree):
107
+ if isinstance(node, ast.FunctionDef):
108
+ return node
109
+ return None
110
+
111
+
112
+ def _definitions(fn_node: ast.FunctionDef) -> Dict[str, List[Any]]:
113
+ """Local name -> the expressions assigned to it.
114
+
115
+ ``p += off`` is treated as ``p = p + off`` (self-referential; the resolver's
116
+ ``seen`` set terminates it), and a ``for`` target takes its iterable.
117
+ """
118
+ defs: Dict[str, List[Any]] = {}
119
+ for node in ast.walk(fn_node):
120
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
121
+ defs.setdefault(node.targets[0].id, []).append(node.value)
122
+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.value is not None:
123
+ defs.setdefault(node.target.id, []).append(node.value)
124
+ elif isinstance(node, ast.AugAssign) and isinstance(node.target, ast.Name):
125
+ defs.setdefault(node.target.id, []).append(
126
+ ast.BinOp(left=node.target, op=node.op, right=node.value))
127
+ elif isinstance(node, ast.For) and isinstance(node.target, ast.Name):
128
+ defs.setdefault(node.target.id, []).append(node.iter)
129
+ return defs
130
+
131
+
132
+ class TargetResolver:
133
+ """Resolves load/store targets back to the arguments they derive from.
134
+
135
+ This replaces a string test -- "does this argument's name occur in the
136
+ unparsed target?" -- which misses every pointer renamed through an
137
+ intermediate (``q_ptrs = Q_ptr + offs`` names neither ``Q`` nor ``Q_ptr``)
138
+ and over-matches short names (``o`` is a substring of both ``row_x`` and
139
+ ``row_out``, which turns a plain output into an in-place argument).
140
+ """
141
+
142
+ _MAX_DEPTH = 12
143
+
144
+ def __init__(self, fn_node, env, globals_map, visited):
145
+ self.fn = fn_node
146
+ self.env = env # this scope's parameter -> kernel arguments
147
+ self.globals = globals_map
148
+ self.visited = visited # (qualname, bound env) recursion guard
149
+ self.defs = _definitions(fn_node)
150
+ self.loads: Set[str] = set()
151
+ self.stores: Set[str] = set()
152
+
153
+ def resolve(self, node, seen=frozenset(), depth: int = 0) -> Set[str]:
154
+ """Kernel arguments reachable by walking ``node`` back through assignments."""
155
+ out: Set[str] = set()
156
+ if depth > self._MAX_DEPTH:
157
+ return out
158
+ for child in ast.walk(node):
159
+ if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load):
160
+ if child.id in self.env:
161
+ out |= self.env[child.id]
162
+ elif child.id in self.defs and child.id not in seen:
163
+ for rhs in self.defs[child.id]:
164
+ out |= self.resolve(rhs, seen | {child.id}, depth + 1)
165
+ return out
166
+
167
+ def scan(self):
168
+ for node in ast.walk(self.fn):
169
+ if not isinstance(node, ast.Call):
170
+ continue
171
+ callee = node.func.attr if isinstance(node.func, ast.Attribute) else getattr(node.func, "id", "")
172
+ if callee in ("load", "store") and node.args:
173
+ reached = self.resolve(node.args[0])
174
+ (self.loads if callee == "load" else self.stores).update(reached)
175
+ elif callee:
176
+ self._descend(callee, node)
177
+ return self.loads, self.stores
178
+
179
+ def _descend(self, callee_name: str, call_node: ast.Call) -> None:
180
+ """Follow a call into a helper, binding its parameters to the call site.
181
+
182
+ The binding is what makes nested helpers work. Attributing the helper's
183
+ own parameter names, as a bare recursion does, only lines up with the
184
+ kernel's arguments when the two happen to share a name.
185
+ """
186
+ target = self.globals.get(callee_name)
187
+ if target is None or not callable(target):
188
+ return
189
+ helper = _source_ast(target)
190
+ if helper is None:
191
+ return
192
+ actual = getattr(target, "fn", target)
193
+ try:
194
+ bound = inspect.signature(actual).bind_partial(
195
+ *call_node.args,
196
+ **{kw.arg: kw.value for kw in call_node.keywords if kw.arg})
197
+ except Exception:
198
+ warn(f"could not bind the arguments of '{callee_name}'; "
199
+ "its loads and stores are not attributed")
200
+ return
201
+ env = {}
202
+ for pname, pval in bound.arguments.items():
203
+ env[pname] = self.resolve(pval) if isinstance(pval, ast.AST) else set()
204
+ key = (getattr(actual, "__qualname__", callee_name),
205
+ frozenset((k, frozenset(v)) for k, v in env.items()))
206
+ if key in self.visited:
207
+ return
208
+ self.visited.add(key)
209
+ sub = TargetResolver(helper, env, getattr(actual, "__globals__", self.globals), self.visited)
210
+ loads, stores = sub.scan()
211
+ self.loads |= loads
212
+ self.stores |= stores
213
+
214
+
215
+ # A tensor argument the resolver could not reach is guessed from its name.
216
+ # These hints only ever decide arguments the resolution left undetermined.
217
+ _OUTPUT_HINTS = ['out', 'dst', 'res', 'output', 'result', 'y']
218
+
219
+
220
+ def analyze_grid_asts(
221
+ manifests: List[Any],
222
+ handler: Optional[Any] = None
223
+ ) -> List[Any]:
224
+ """Classify each tensor argument as input, output or in-place.
225
+
226
+ Every ``tl.load`` / ``tl.store`` target is resolved backwards through the
227
+ function's own assignments to the arguments it derives from. Only arguments
228
+ that resolution does not reach are guessed, and each guess is reported on
229
+ stderr so a silent misclassification cannot pass unnoticed.
230
+ """
231
+ for manifest in manifests:
232
+ debug_print(f"\n{'='*50}\nConfiguring I/O for: {manifest.kernel_name}\n{'='*50}")
233
+
234
+ load_reached: Set[str] = set()
235
+ store_reached: Set[str] = set()
236
+
237
+ if hasattr(manifest, 'fn') and manifest.fn is not None:
238
+ fn_node = _source_ast(manifest.fn)
239
+ if fn_node is not None:
240
+ actual = getattr(manifest.fn, "fn", manifest.fn)
241
+ params = [a.arg for a in fn_node.args.args] + [a.arg for a in fn_node.args.kwonlyargs]
242
+ resolver = TargetResolver(
243
+ fn_node,
244
+ {p: {p} for p in params},
245
+ getattr(actual, "__globals__", {}),
246
+ set(),
247
+ )
248
+ load_reached, store_reached = resolver.scan()
249
+ else:
250
+ warn(f"source unavailable for '{manifest.kernel_name}'; "
251
+ "argument roles fall back to name heuristics")
252
+
253
+ tensor_args = [a for a in manifest.arguments if a.shape]
254
+ tensor_names = {a.name for a in tensor_args}
255
+ input_arg_names = load_reached & tensor_names
256
+ output_arg_names = store_reached & tensor_names
257
+
258
+ undetermined = tensor_names - input_arg_names - output_arg_names
259
+ if undetermined and tensor_args:
260
+ warn(f"'{manifest.kernel_name}': no load/store target resolved to "
261
+ f"{sorted(undetermined)}; guessing from names")
262
+ for name in sorted(undetermined):
263
+ lname = name.lower()
264
+ if any(kw in lname for kw in _OUTPUT_HINTS) and not lname.startswith('in'):
265
+ output_arg_names.add(name)
266
+ if not output_arg_names and len(tensor_args) > 1:
267
+ last = tensor_args[-1].name
268
+ output_arg_names.add(last)
269
+ warn(f"'{manifest.kernel_name}': defaulting the last tensor "
270
+ f"argument '{last}' to output")
271
+
272
+ for arg in manifest.arguments:
273
+ if arg.shape:
274
+ if arg.name in output_arg_names and arg.name in input_arg_names:
275
+ arg.kind = 'inplace'
276
+ debug_print(f"[AST-Analysis] '{arg.name}' resolved to INPLACE (loaded and stored).")
277
+ elif arg.name in output_arg_names:
278
+ arg.kind = 'output'
279
+ debug_print(f"[AST-Analysis] '{arg.name}' resolved to OUTPUT.")
280
+ elif arg.name in input_arg_names:
281
+ arg.kind = 'input'
282
+ debug_print(f"[AST-Analysis] '{arg.name}' resolved to INPUT.")
283
+ elif handler is not None:
284
+ arg.kind = handler.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
285
+ else:
286
+ from .interaction import AutoInteractionHandler
287
+ h = AutoInteractionHandler()
288
+ arg.kind = h.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
289
+ else:
290
+ arg.kind = 'scalar'
291
+ debug_print(f"[Auto] Mapped scalar constant: {arg.name} = {arg.value}")
292
+
293
+ # A kernel with no stride arguments addresses a flat buffer, so it is
294
+ # agnostic to the physical layout and can accept channel-last formats.
295
+ manifest.layout_agnostic = not any(
296
+ 'stride' in a.name.lower() for a in manifest.arguments
297
+ )
298
+ debug_print(f"[Layout] '{manifest.kernel_name}' layout_agnostic={manifest.layout_agnostic}")
299
+
300
+ return manifests
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kernel-lens
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins
5
5
  Author-email: Justin Duc <justin.duc@galiad-research.com>
6
6
  License: MIT License
@@ -86,7 +86,8 @@ Dynamic: license-file
86
86
 
87
87
  **KernelLens** compiles a PyTorch model that contains custom `@triton.jit` kernels
88
88
  into native C++ artifacts for **ONNX Runtime** (`OrtCustomOp`) and **NVIDIA
89
- TensorRT 10.x** (`nvinfer1::IPluginV2DynamicExt`). You write Triton in Python; it
89
+ TensorRT 8.6 to 11** (`nvinfer1::IPluginV2DynamicExt`, and `nvinfer1::IPluginV3`
90
+ on TensorRT 10 and later). You write Triton in Python; it
90
91
  emits the C++ plugin, builds the `.so`, and runs it with **zero-copy VRAM
91
92
  binding** — no manual C++ and no Python in the serving path.
92
93
 
@@ -145,9 +146,12 @@ compiled_model.run(inputs, backend="onnx"|"tensorrt")
145
146
  > conformance patterns and the quantized GEMMs, on both backends; two
146
147
  > multi-operator fusions land just above zero and are reported as such. Keeping
147
148
  > the kernel pays when the baseline leaves the graph unfused and breaks even when
148
- > it already fuses. The TensorRT-LLM backend is design-only (see
149
- > [`docs/assets/data/a100/trtllm_backend_design.md`](docs/assets/data/a100/trtllm_backend_design.md)),
150
- > and the quantized GEMMs are validated on ONNX Runtime only. Full methodology
149
+ > it already fuses. A TensorRT-LLM backend is out of scope; the
150
+ > [design note](docs/assets/data/a100/trtllm_backend_design.md) records what
151
+ > integrating one would take. The kernels themselves are usable there — they are
152
+ > ordinary `@triton.jit` functions, and an A100 measurement confirmed a KernelLens
153
+ > RMSNorm is bit-identical to the flashinfer kernel TensorRT-LLM already uses
154
+ > (`0.00e+00`). The quantized GEMMs are validated on ONNX Runtime only. Full methodology
151
155
  > and every raw JSON are under [`docs/assets/data/`](docs/assets/data/).
152
156
 
153
157
  ## Installation
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "kernel-lens"
7
- version = "2.0.0"
7
+ version = "2.0.1"
8
8
  description = "Compile PyTorch models with custom @triton.jit kernels into ONNX Runtime and TensorRT C++ plugins"
9
9
  readme = "README.md"
10
10
  license = { file = "LICENSE" }
@@ -1,179 +0,0 @@
1
- import re
2
- import torch
3
- from typing import List, Optional, Dict, Any
4
- from ..config import debug_print
5
-
6
- # REMOVED the circular import: from .manifest import KernelManifest
7
- # REMOVED the circular import: from .interaction import InteractionHandler
8
-
9
- def raw_sym_expr(sym_int: Any) -> Any:
10
- """The *unsubstituted* symbolic expression behind a SymInt.
11
-
12
- ``SymNode.expr`` runs ``shape_env.replace()`` with the values the env has
13
- recorded, so once tracing has bound s75 -> 8 and s96 -> 512 it folds
14
- ``(s75*s96 + 255)//256`` to the constant 16. That is the right answer for a
15
- shape-specialised engine and exactly the wrong one for a dynamic engine.
16
- ``SymNode._expr`` still holds the symbol, so read that first.
17
- """
18
- node = getattr(sym_int, "node", None)
19
- if node is None:
20
- return sym_int
21
- raw = getattr(node, "_expr", None)
22
- if raw is not None and not isinstance(raw, int):
23
- return raw
24
- return getattr(node, "expr", raw)
25
-
26
-
27
- def _cxx_pow(expr_str: str) -> str:
28
- """sympy renders powers as ``a**b``; C++ has no ``**``.
29
-
30
- Small integer powers are expanded by hand -- ``std::pow`` would drag in
31
- <cmath> and go through a double, and a launch grid is exact integer maths.
32
- """
33
- for _ in range(4):
34
- m = re.search(r'\b(\w+)\*\*(\d+)\b', expr_str)
35
- if not m:
36
- break
37
- base, exp = m.group(1), int(m.group(2))
38
- expr_str = expr_str[:m.start()] + "(" + "*".join([base] * exp) + ")" + expr_str[m.end():]
39
- if "**" in expr_str:
40
- # Only reachable for a non-integer or large exponent; keep it valid C++.
41
- expr_str = re.sub(r'\b(\w+)\*\*(\w+)\b', r'((int64_t)std::pow((double)\1, (double)\2))', expr_str)
42
- return expr_str
43
-
44
-
45
- def translate_symint_to_cxx(sym_int: Any, sym_map: Dict[str, str]) -> str:
46
- """
47
- Converts a PyTorch SymInt into a TensorRT/ORT C++ string.
48
- """
49
- if not isinstance(sym_int, torch.SymInt):
50
- return str(sym_int)
51
-
52
- raw = raw_sym_expr(sym_int)
53
- # Expand sympy powers BEFORE substituting: at this point the base is a bare
54
- # symbol (s76**2), which is easy to rewrite. Afterwards the base is a
55
- # parenthesised C++ expression and the rewrite gets ambiguous.
56
- expr_str = _cxx_pow(str(raw))
57
-
58
- sorted_map = sorted(sym_map.items(), key=lambda x: len(x[0]), reverse=True)
59
-
60
- for sym_var, cxx_code in sorted_map:
61
- if sym_var.isalnum():
62
- expr_str = re.sub(rf'\b{re.escape(sym_var)}\b', lambda _m, c=cxx_code: c, expr_str)
63
- else:
64
- expr_str = expr_str.replace(sym_var, f"({cxx_code})")
65
-
66
- if re.search(r'\bs\d+\b', expr_str):
67
- # A symbol survived: the map did not cover it. Emitting it would be a
68
- # C++ identifier error, so fall back to the value the shape env
69
- # resolved -- the old behaviour. Static, but valid.
70
- expr_str = str(sym_int)
71
-
72
- expr_str = _cxx_pow(expr_str)
73
- expr_str = expr_str.replace("//", "/")
74
- return expr_str
75
-
76
- def analyze_grid_asts(
77
- manifests: List[Any],
78
- handler: Optional[Any] = None
79
- ) -> List[Any]:
80
- """
81
- Analyzes symbolic ASTs and classifies tensor arguments deterministically via tl.store inspection,
82
- recursively traversing called helper functions if necessary.
83
- """
84
- import inspect, ast
85
-
86
- for manifest in manifests:
87
- debug_print(f"\n{'='*50}\nConfiguring I/O for: {manifest.kernel_name}\n{'='*50}")
88
-
89
- output_arg_names = set()
90
- input_arg_names = set()
91
-
92
- if hasattr(manifest, 'fn') and manifest.fn is not None:
93
- try:
94
- visited_fns = set()
95
-
96
- def inspect_fn_ast(fn_obj):
97
- if fn_obj in visited_fns:
98
- return
99
- visited_fns.add(fn_obj)
100
-
101
- actual_fn = fn_obj.fn if hasattr(fn_obj, 'fn') else fn_obj
102
- if not callable(actual_fn):
103
- return
104
-
105
- try:
106
- source = inspect.getsource(actual_fn)
107
- tree = ast.parse(source)
108
- except Exception:
109
- return
110
-
111
- for node in ast.walk(tree):
112
- if isinstance(node, ast.Call):
113
- func_name = ""
114
- if isinstance(node.func, ast.Attribute):
115
- func_name = node.func.attr
116
- elif isinstance(node.func, ast.Name):
117
- func_name = node.func.id
118
-
119
- if func_name == 'store' and node.args:
120
- target_str = ast.unparse(node.args[0])
121
- for arg in manifest.arguments:
122
- if arg.shape and (arg.name in target_str or target_str.startswith(arg.name) or re.search(rf'\b{arg.name}\b', target_str)):
123
- output_arg_names.add(arg.name)
124
- elif func_name == 'load' and node.args:
125
- target_str = ast.unparse(node.args[0])
126
- for arg in manifest.arguments:
127
- if arg.shape and (arg.name in target_str or target_str.startswith(arg.name) or re.search(rf'\b{arg.name}\b', target_str)):
128
- input_arg_names.add(arg.name)
129
- else:
130
- # Check if calling another triton jit or helper function
131
- try:
132
- glob = getattr(actual_fn, '__globals__', {})
133
- if func_name in glob and callable(glob[func_name]):
134
- inspect_fn_ast(glob[func_name])
135
- except Exception:
136
- pass
137
-
138
- fn_obj = manifest.fn
139
- inspect_fn_ast(fn_obj)
140
- except Exception as e:
141
- debug_print(f"Warning during AST analysis: {e}")
142
-
143
- # Heuristic fallback if output_arg_names is empty but tensor args exist
144
- tensor_args = [a for a in manifest.arguments if a.shape]
145
- if not output_arg_names and tensor_args:
146
- for arg in tensor_args:
147
- lname = arg.name.lower()
148
- if any(kw in lname for kw in ['out', 'dst', 'res', 'output', 'result', 'y']) and not lname.startswith('in'):
149
- output_arg_names.add(arg.name)
150
- if not output_arg_names and len(tensor_args) > 1:
151
- # Default the last tensor argument as output if no output detected
152
- output_arg_names.add(tensor_args[-1].name)
153
-
154
- for arg in manifest.arguments:
155
- if arg.shape:
156
- if arg.name in output_arg_names and arg.name in input_arg_names:
157
- arg.kind = 'inplace'
158
- debug_print(f"[AST-Analysis] '{arg.name}' statically resolved to INPLACE via tl.load and tl.store analysis.")
159
- elif arg.name in output_arg_names:
160
- arg.kind = 'output'
161
- debug_print(f"[AST-Analysis] '{arg.name}' statically resolved to OUTPUT via tl.store analysis.")
162
- elif handler is not None:
163
- arg.kind = handler.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
164
- else:
165
- from .interaction import AutoInteractionHandler
166
- h = AutoInteractionHandler()
167
- arg.kind = h.ask_tensor_kind(manifest.kernel_name, arg.name, arg.shape)
168
- else:
169
- arg.kind = 'scalar'
170
- debug_print(f"[Auto] Mapped scalar constant: {arg.name} = {arg.value}")
171
-
172
- # A kernel with no stride arguments addresses a flat buffer, so it is
173
- # agnostic to the physical layout and can accept channel-last formats.
174
- manifest.layout_agnostic = not any(
175
- 'stride' in a.name.lower() for a in manifest.arguments
176
- )
177
- debug_print(f"[Layout] '{manifest.kernel_name}' layout_agnostic={manifest.layout_agnostic}")
178
-
179
- return manifests
File without changes
File without changes