kernelport 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. kernelport/__init__.py +3 -0
  2. kernelport/__main__.py +5 -0
  3. kernelport/_skills/kernelport/SKILL.md +200 -0
  4. kernelport/_skills/kernelport/backends/cpu.md +138 -0
  5. kernelport/_skills/kernelport/backends/hip.md +53 -0
  6. kernelport/_skills/kernelport/backends/metal.md +76 -0
  7. kernelport/_skills/kernelport/backends/triton.md +117 -0
  8. kernelport/_skills/kernelport/backends/webgpu.md +146 -0
  9. kernelport/_skills/kernelport/common_bugs.md +40 -0
  10. kernelport/agent.py +178 -0
  11. kernelport/analyzer/__init__.py +0 -0
  12. kernelport/analyzer/cuda_parser.py +480 -0
  13. kernelport/analyzer/models.py +80 -0
  14. kernelport/analyzer/suggest.py +120 -0
  15. kernelport/backends/__init__.py +5 -0
  16. kernelport/backends/base.py +138 -0
  17. kernelport/backends/cpu/__init__.py +1 -0
  18. kernelport/backends/cpu/capability.py +25 -0
  19. kernelport/backends/cpu/config.py +36 -0
  20. kernelport/backends/cpu/runner.py +67 -0
  21. kernelport/backends/hip/__init__.py +1 -0
  22. kernelport/backends/hip/capability.py +34 -0
  23. kernelport/backends/hip/config.py +28 -0
  24. kernelport/backends/hip/runner.py +48 -0
  25. kernelport/backends/metal/__init__.py +1 -0
  26. kernelport/backends/metal/capability.py +32 -0
  27. kernelport/backends/metal/config.py +35 -0
  28. kernelport/backends/metal/runner.py +142 -0
  29. kernelport/backends/triton/__init__.py +1 -0
  30. kernelport/backends/triton/capability.py +29 -0
  31. kernelport/backends/triton/config.py +41 -0
  32. kernelport/backends/triton/runner.py +44 -0
  33. kernelport/backends/webgpu/__init__.py +1 -0
  34. kernelport/backends/webgpu/capability.py +34 -0
  35. kernelport/backends/webgpu/config.py +38 -0
  36. kernelport/backends/webgpu/runner.py +285 -0
  37. kernelport/bench/__init__.py +0 -0
  38. kernelport/bench/timer.py +76 -0
  39. kernelport/cli.py +501 -0
  40. kernelport/doctor.py +285 -0
  41. kernelport/hardware.py +60 -0
  42. kernelport/harness/__init__.py +0 -0
  43. kernelport/harness/compare.py +152 -0
  44. kernelport/harness/device_source.py +273 -0
  45. kernelport/harness/expr.py +80 -0
  46. kernelport/harness/inputs.py +194 -0
  47. kernelport/harness/reference.py +76 -0
  48. kernelport/harness/results.py +118 -0
  49. kernelport/harness/run_cuda.py +126 -0
  50. kernelport/harness/spec.py +199 -0
  51. kernelport/harness/tolerances.py +48 -0
  52. kernelport/harness/verify.py +131 -0
  53. kernelport/hashing.py +28 -0
  54. kernelport/mcp_server.py +95 -0
  55. kernelport/report/__init__.py +1 -0
  56. kernelport/report/data.py +147 -0
  57. kernelport/report/html.py +174 -0
  58. kernelport/report/markdown.py +75 -0
  59. kernelport/setup_agents/__init__.py +1 -0
  60. kernelport/setup_agents/backup.py +59 -0
  61. kernelport/setup_agents/detect.py +78 -0
  62. kernelport/setup_agents/register.py +302 -0
  63. kernelport/setup_agents/skills_install.py +31 -0
  64. kernelport/tools.py +455 -0
  65. kernelport/tune.py +99 -0
  66. kernelport/workspace.py +400 -0
  67. kernelport/zoo.py +176 -0
  68. kernelport-0.1.0.dist-info/METADATA +228 -0
  69. kernelport-0.1.0.dist-info/RECORD +72 -0
  70. kernelport-0.1.0.dist-info/WHEEL +4 -0
  71. kernelport-0.1.0.dist-info/entry_points.txt +2 -0
  72. kernelport-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,118 @@
1
+ """result.json schema: the record of one verification run.
2
+
3
+ Status values:
4
+ - PASS every case this hardware can run ran on it and matched the frozen reference
5
+ - FAIL at least one case mismatched or crashed
6
+ - UNVERIFIED the port exists but could not be run here (backend unavailable)
7
+ - SKIPPED not attempted (e.g. no port written, or no case runnable here)
8
+
9
+ Cases a device cannot run at all (a dtype it lacks, a buffer larger than its binding
10
+ limit) are not tested, and each one is named in ``reason``; they never count as passed.
11
+ Only PASS means "proven". Nothing else may be displayed as PASS.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from dataclasses import asdict, dataclass, field
18
+ from datetime import datetime, timezone
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ SCHEMA_VERSION = 1
23
+ STATUSES = ("PASS", "FAIL", "UNVERIFIED", "SKIPPED")
24
+
25
+
26
+ class CaseNotRunnable(Exception):
27
+ """A runner cannot run this case on this device at all (a device limit, not a port bug).
28
+
29
+ The message says which limit, e.g. "table needs a 500 MiB storage binding; this adapter
30
+ allows 128 MiB". Raised before anything is launched.
31
+ """
32
+
33
+
34
+ @dataclass
35
+ class CaseResult:
36
+ """Outcome of one (case, dtype)."""
37
+
38
+ case: str
39
+ dtype: str
40
+ passed: bool
41
+ input_hash: str
42
+ outputs: list[dict[str, Any]] = field(default_factory=list) # TensorCompare dicts
43
+ error: str = "" # crash / compile error text, truncated
44
+
45
+
46
+ @dataclass
47
+ class RunResult:
48
+ """Outcome of verifying one kernel on one backend on one machine."""
49
+
50
+ kernel: str
51
+ backend: str
52
+ status: str
53
+ hardware: str
54
+ reason: str = ""
55
+ code_hash: str = ""
56
+ reference_hash: str = ""
57
+ spec_hash: str = ""
58
+ tolerances: dict[str, dict] = field(default_factory=dict)
59
+ cases: list[CaseResult] = field(default_factory=list)
60
+ timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat(timespec="seconds"))
61
+ schema_version: int = SCHEMA_VERSION
62
+
63
+ def __post_init__(self) -> None:
64
+ if self.status not in STATUSES:
65
+ raise ValueError(f"status must be one of {STATUSES}, got {self.status!r}")
66
+
67
+ @property
68
+ def num_passed(self) -> int:
69
+ return sum(c.passed for c in self.cases)
70
+
71
+ def to_dict(self) -> dict:
72
+ """JSON-serialisable dict."""
73
+ return asdict(self)
74
+
75
+ def save(self, path: Path) -> None:
76
+ """Write as JSON, creating parent dirs."""
77
+ path = Path(path)
78
+ path.parent.mkdir(parents=True, exist_ok=True)
79
+ path.write_bytes(json.dumps(self.to_dict(), indent=2).encode("utf-8")) # LF on every OS
80
+
81
+ @classmethod
82
+ def from_dict(cls, d: dict) -> "RunResult":
83
+ """Inverse of to_dict, with validation."""
84
+ d = dict(d)
85
+ if d.get("schema_version") != SCHEMA_VERSION:
86
+ raise ValueError(f"unsupported result schema {d.get('schema_version')}")
87
+ d["cases"] = [CaseResult(**c) for c in d.get("cases", [])]
88
+ return cls(**d)
89
+
90
+ @classmethod
91
+ def load(cls, path: Path) -> "RunResult":
92
+ """Read a result.json file."""
93
+ return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
94
+
95
+ def failure_report(self, max_cases: int = 3) -> str:
96
+ """Compact text for an agent: what failed, where, and by how much."""
97
+ if self.status != "FAIL":
98
+ return f"{self.kernel}/{self.backend}: {self.status} {self.reason}".strip()
99
+ failed = [c for c in self.cases if not c.passed]
100
+ lines = [f"{self.kernel}/{self.backend} on {self.hardware}: FAIL "
101
+ f"({self.num_passed}/{len(self.cases)} cases passed)"]
102
+ if self.reason:
103
+ lines.append(self.reason)
104
+ for c in failed[:max_cases]:
105
+ lines.append(f"- case {c.case} dtype {c.dtype}:")
106
+ if c.error:
107
+ lines.append(f" error: {c.error}")
108
+ for o in c.outputs:
109
+ if o["passed"]:
110
+ continue
111
+ lines.append(f" {o['name']} shape {o['shape']}: {o['reason']}")
112
+ if o["expected_slice"]:
113
+ lines.append(f" flat[{o['slice_start']}:] expected {o['expected_slice']}")
114
+ lines.append(f" flat[{o['slice_start']}:] actual {o['actual_slice']}")
115
+ if len(failed) > max_cases:
116
+ lines.append(f"... and {len(failed) - max_cases} more failing cases: "
117
+ + ", ".join(f"{c.case}/{c.dtype}" for c in failed[max_cases:]))
118
+ return "\n".join(lines)
@@ -0,0 +1,126 @@
1
+ """Run an original CUDA kernel through CuPy (NVRTC). No nvcc or Visual Studio needed.
2
+
3
+ The kernel source is compiled once per (source, entry list) and launched with
4
+ the grid/block from the launch spec. Inputs go in as CPU tensors and outputs
5
+ come back as CPU tensors.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from functools import lru_cache
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ import numpy as np
15
+ import torch
16
+
17
+ from kernelport.harness.inputs import CaseData
18
+ from kernelport.harness.spec import LaunchSpec
19
+ from kernelport.hashing import sha256_file
20
+
21
+ NVRTC_OPTIONS = ("-std=c++17",)
22
+
23
+ # nvcc gets these from host math headers; NVRTC does not. Defined only if missing.
24
+ NVRTC_PRELUDE = """
25
+ #ifndef INFINITY
26
+ #define INFINITY __int_as_float(0x7f800000)
27
+ #endif
28
+ #ifndef NAN
29
+ #define NAN __int_as_float(0x7fffffff)
30
+ #endif
31
+ #line 1
32
+ """
33
+
34
+ _SCALAR_NP = {
35
+ "float32": np.float32, "float64": np.float64, "float16": np.float16,
36
+ "int8": np.int8, "uint8": np.uint8, "int16": np.int16, "int32": np.int32, "int64": np.int64,
37
+ }
38
+
39
+
40
+ def is_available() -> tuple[bool, str]:
41
+ """Can original CUDA kernels run here?"""
42
+ try:
43
+ import cupy # noqa: F401
44
+ except Exception as exc:
45
+ return False, f"cupy not importable ({exc}); pip install kernelport[cuda]"
46
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
47
+ return False, "no NVIDIA CUDA GPU visible to torch"
48
+ return True, "ok"
49
+
50
+
51
+ @lru_cache(maxsize=64)
52
+ def _compile(source: str, entries: tuple[str, ...], include_dir: str,
53
+ prelude: str = NVRTC_PRELUDE, options: tuple[str, ...] = NVRTC_OPTIONS):
54
+ import cupy as cp
55
+
56
+ opts = options + (f"-I{include_dir}",) # so #include "local.cuh" works
57
+ mod = cp.RawModule(code=prelude + source, options=opts, name_expressions=list(entries))
58
+ return {e: mod.get_function(e) for e in entries}
59
+
60
+
61
+ def get_kernel(source_path: Path, spec: LaunchSpec, dtype: str, *, type_names: dict[str, str] | None = None,
62
+ prelude: str = NVRTC_PRELUDE, options: tuple[str, ...] = NVRTC_OPTIONS,
63
+ device_only: bool = True):
64
+ """Compiled CuPy function for this spec's entry at ``dtype``.
65
+
66
+ ``type_names`` maps dtypes to the C++ names substituted for ``{T}`` in the entry
67
+ (CUDA names by default; the hip backend passes HIP names). CuPy compiles with
68
+ NVRTC on NVIDIA and hipRTC on ROCm. With ``device_only`` (the default, for original
69
+ CUDA files) host code is sliced away first, see ``device_source``.
70
+ """
71
+ from kernelport.harness.device_source import device_source
72
+
73
+ def entry(t: str) -> str:
74
+ return spec.entry.replace("{T}", type_names[t]) if type_names else spec.entry_for(t)
75
+
76
+ if device_only:
77
+ source = device_source(Path(source_path))
78
+ else:
79
+ source = Path(source_path).read_text(encoding="utf-8")
80
+ entries = tuple(sorted({entry(t) for t in spec.dtypes}))
81
+ funcs = _compile(source, entries, str(Path(source_path).resolve().parent), prelude, options)
82
+ return funcs[entry(dtype)]
83
+
84
+
85
+ def _as_cupy(t: torch.Tensor):
86
+ import cupy as cp
87
+
88
+ if t.dtype == torch.bfloat16: # cupy has no bf16; same bits as int16
89
+ t = t.view(torch.int16)
90
+ return cp.asarray(t)
91
+
92
+
93
+ def kernel_args(spec: LaunchSpec, data: CaseData, dev: dict[str, torch.Tensor]) -> list[Any]:
94
+ args: list[Any] = []
95
+ for p in spec.params:
96
+ if p.is_tensor:
97
+ args.append(_as_cupy(dev[p.name]))
98
+ else:
99
+ args.append(_SCALAR_NP[p.resolved_dtype(data.dtype)](data.scalars[p.name]))
100
+ return args
101
+
102
+
103
+ def prepare(source_path: Path, spec: LaunchSpec, data: CaseData):
104
+ """Return (launch_fn, device_tensors). launch_fn() runs the kernel once, async."""
105
+ kern = get_kernel(source_path, spec, data.dtype)
106
+ dev = {k: v.to("cuda") for k, v in data.tensors.items()}
107
+ grid, block, shmem = spec.grid_block(data.dims)
108
+ args = tuple(kernel_args(spec, data, dev))
109
+
110
+ def launch() -> None:
111
+ kern(grid, block, args, shared_mem=shmem)
112
+
113
+ return launch, dev
114
+
115
+
116
+ def run_original(source_path: Path, spec: LaunchSpec, data: CaseData) -> dict[str, torch.Tensor]:
117
+ """Run once and return output tensors on the CPU."""
118
+ launch, dev = prepare(source_path, spec, data)
119
+ launch()
120
+ torch.cuda.synchronize()
121
+ return {p.name: dev[p.name].cpu() for p in spec.params if p.is_output}
122
+
123
+
124
+ def source_hash(path: Path) -> str:
125
+ """sha256 of the kernel source."""
126
+ return sha256_file(path)
@@ -0,0 +1,199 @@
1
+ """Launch spec: how to call a kernel and which input cases to test.
2
+
3
+ A spec is plain JSON (``launch.json``) so any machine can regenerate the exact
4
+ same inputs. It does not import torch, so the core package can load it.
5
+
6
+ Example::
7
+
8
+ {
9
+ "name": "vector_add",
10
+ "source": "kernel.cu",
11
+ "entry": "vector_add<{T}>",
12
+ "dtypes": ["float32", "float16"],
13
+ "params": [
14
+ {"name": "a", "kind": "input", "dtype": "T", "shape": ["n"]},
15
+ {"name": "out", "kind": "output", "dtype": "T", "shape": ["n"]},
16
+ {"name": "n", "kind": "scalar", "dtype": "int32", "value": "n"}
17
+ ],
18
+ "launch": {"grid": ["cdiv(n, 256)"], "block": [256], "shared_mem": 0},
19
+ "cases": [{"name": "tiny", "dims": {"n": 5}}],
20
+ "init": {"a": {"dist": "normal", "edges": true}}
21
+ }
22
+
23
+ ``dtype: "T"`` is replaced by each entry of ``dtypes`` in turn.
24
+ ``kind`` is one of input, output, inout, scalar.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import hashlib
30
+ import json
31
+ from dataclasses import asdict, dataclass, field
32
+ from pathlib import Path
33
+ from typing import Any
34
+
35
+ from kernelport.harness.expr import evaluate, evaluate_int
36
+
37
+ FLOAT_DTYPES = ("float32", "float16", "bfloat16", "float64")
38
+ INT_DTYPES = ("int8", "uint8", "int16", "int32", "int64")
39
+ ALL_DTYPES = FLOAT_DTYPES + INT_DTYPES
40
+
41
+ # CUDA C type names used to fill template entries like "kernel<{T}>".
42
+ CUDA_TYPE_NAMES = {
43
+ "float32": "float",
44
+ "float16": "__half",
45
+ "bfloat16": "__nv_bfloat16",
46
+ "float64": "double",
47
+ "int8": "signed char",
48
+ "uint8": "unsigned char",
49
+ "int16": "short",
50
+ "int32": "int",
51
+ "int64": "long long",
52
+ }
53
+
54
+ PARAM_KINDS = ("input", "output", "inout", "scalar")
55
+
56
+
57
+ @dataclass
58
+ class ParamSpec:
59
+ """One kernel parameter, in signature order."""
60
+
61
+ name: str
62
+ kind: str
63
+ dtype: str
64
+ shape: list[Any] = field(default_factory=list) # tensors only
65
+ value: Any = None # scalars only: expression over dims
66
+ cuda_name: str | None = None # original CUDA name if renamed (e.g. "in" -> "in_")
67
+
68
+ def resolved_dtype(self, t: str) -> str:
69
+ """Dtype with the template placeholder ``T`` replaced."""
70
+ return t if self.dtype == "T" else self.dtype
71
+
72
+ def resolved_shape(self, dims: dict[str, Any]) -> tuple[int, ...]:
73
+ """Concrete shape for a case."""
74
+ return tuple(evaluate_int(s, dims) for s in self.shape)
75
+
76
+ def resolved_value(self, dims: dict[str, Any]) -> Any:
77
+ """Concrete scalar value for a case."""
78
+ return evaluate(self.value, dims)
79
+
80
+ @property
81
+ def is_tensor(self) -> bool:
82
+ return self.kind != "scalar"
83
+
84
+ @property
85
+ def is_output(self) -> bool:
86
+ return self.kind in ("output", "inout")
87
+
88
+
89
+ @dataclass
90
+ class CaseSpec:
91
+ """A named input case: concrete dims plus optional per-case overrides."""
92
+
93
+ name: str
94
+ dims: dict[str, Any]
95
+ dtypes: list[str] | None = None # restrict this case to some dtypes
96
+ init: dict[str, dict] = field(default_factory=dict) # per-param init overrides
97
+
98
+
99
+ @dataclass
100
+ class LaunchSpec:
101
+ """Everything needed to run the original kernel and regenerate its inputs."""
102
+
103
+ name: str
104
+ entry: str
105
+ params: list[ParamSpec]
106
+ cases: list[CaseSpec]
107
+ dtypes: list[str] = field(default_factory=lambda: ["float32"])
108
+ source: str = "kernel.cu"
109
+ launch: dict[str, Any] = field(default_factory=dict)
110
+ init: dict[str, dict] = field(default_factory=dict)
111
+ seed: int = 0
112
+ bench_case: str | None = None
113
+
114
+ # ------------------------------------------------------------- loading
115
+ @classmethod
116
+ def from_dict(cls, d: dict) -> "LaunchSpec":
117
+ """Build and validate a spec from parsed JSON."""
118
+ d = {k: v for k, v in d.items() if not k.startswith("_")} # "_notes", "_comment"
119
+ d["params"] = [ParamSpec(**p) for p in d["params"]]
120
+ d["cases"] = [CaseSpec(**c) for c in d["cases"]]
121
+ spec = cls(**d)
122
+ spec.validate()
123
+ return spec
124
+
125
+ @classmethod
126
+ def load(cls, path: Path) -> "LaunchSpec":
127
+ """Load a spec from a launch.json file."""
128
+ return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
129
+
130
+ def to_dict(self) -> dict:
131
+ """JSON-serialisable dict."""
132
+ return asdict(self)
133
+
134
+ def hash(self) -> str:
135
+ """Stable hash of the spec contents."""
136
+ blob = json.dumps(self.to_dict(), sort_keys=True).encode()
137
+ return hashlib.sha256(blob).hexdigest()
138
+
139
+ # ----------------------------------------------------------- validation
140
+ def validate(self) -> None:
141
+ """Raise ValueError on any inconsistency."""
142
+ names = [p.name for p in self.params]
143
+ if len(set(names)) != len(names):
144
+ raise ValueError(f"duplicate parameter names in {names}")
145
+ for t in self.dtypes:
146
+ if t not in ALL_DTYPES:
147
+ raise ValueError(f"unknown dtype {t!r}")
148
+ for p in self.params:
149
+ if p.kind not in PARAM_KINDS:
150
+ raise ValueError(f"param {p.name}: kind must be one of {PARAM_KINDS}")
151
+ if p.dtype != "T" and p.dtype not in ALL_DTYPES:
152
+ raise ValueError(f"param {p.name}: unknown dtype {p.dtype!r}")
153
+ if p.kind == "scalar" and p.value is None:
154
+ raise ValueError(f"scalar param {p.name} needs a value expression")
155
+ if not any(p.is_output for p in self.params):
156
+ raise ValueError("spec has no output or inout parameter")
157
+ for name in self.init:
158
+ if name not in names:
159
+ raise ValueError(f"init given for unknown param {name!r}")
160
+ if not self.cases:
161
+ raise ValueError("spec has no cases")
162
+ for c in self.cases:
163
+ for p in self.params:
164
+ if p.is_tensor:
165
+ p.resolved_shape(c.dims)
166
+ else:
167
+ p.resolved_value(c.dims)
168
+ if self.bench_case and self.bench_case not in [c.name for c in self.cases]:
169
+ raise ValueError(f"bench_case {self.bench_case!r} is not a case name")
170
+
171
+ # ------------------------------------------------------------- helpers
172
+ def entry_for(self, t: str) -> str:
173
+ """Kernel entry name (template filled) for dtype ``t``."""
174
+ return self.entry.replace("{T}", CUDA_TYPE_NAMES[t])
175
+
176
+ def case_dtypes(self, case: CaseSpec) -> list[str]:
177
+ """Dtypes to run for one case."""
178
+ return [t for t in self.dtypes if case.dtypes is None or t in case.dtypes]
179
+
180
+ def iter_cases(self):
181
+ """Yield (case, dtype) pairs in a fixed order."""
182
+ for case in self.cases:
183
+ for t in self.case_dtypes(case):
184
+ yield case, t
185
+
186
+ def get_case(self, name: str) -> CaseSpec:
187
+ """Find a case by name."""
188
+ for c in self.cases:
189
+ if c.name == name:
190
+ return c
191
+ raise KeyError(name)
192
+
193
+ def grid_block(self, dims: dict[str, Any]) -> tuple[tuple[int, ...], tuple[int, ...], int]:
194
+ """Concrete (grid, block, shared_mem) for the original CUDA launch."""
195
+ grid = tuple(evaluate_int(g, dims) for g in self.launch.get("grid", [1]))
196
+ block = tuple(evaluate_int(b, dims) for b in self.launch.get("block", [1]))
197
+ shmem = evaluate_int(self.launch.get("shared_mem", 0), dims)
198
+ pad = lambda t: t + (1,) * (3 - len(t)) # noqa: E731
199
+ return pad(grid), pad(block), shmem
@@ -0,0 +1,48 @@
1
+ """Comparison tolerances. These live in kernelport, never come from the agent.
2
+
3
+ A value passes when ``|actual - expected| <= atol + rtol * |expected|`` where
4
+ ``expected`` is the float64 reference rounded to the output dtype. Integer
5
+ outputs must match exactly.
6
+
7
+ Defaults are torch.testing.assert_close's per-dtype defaults, a widely used
8
+ standard:
9
+ - float32 rtol 1.3e-6, atol 1e-5: ~11 ulp relative slack for reordered
10
+ arithmetic; atol covers values that cancel to ~0.
11
+ - float16 rtol 1e-3, atol 1e-5: one fp16 ulp is ~9.8e-4 relative, so this
12
+ allows about one ulp of disagreement after rounding.
13
+ - bfloat16 rtol 1.6e-2, atol 1e-5: one bf16 ulp is ~7.8e-3 relative, so this
14
+ allows about two ulps.
15
+ - float64 rtol 1e-7, atol 1e-7: as torch.testing.
16
+
17
+ Backends may override a dtype with a documented reason (backends/*/config.py).
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from dataclasses import dataclass
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class Tolerance:
27
+ """Allowed error for one dtype."""
28
+
29
+ rtol: float
30
+ atol: float
31
+ reason: str = ""
32
+
33
+
34
+ DEFAULT_TOLERANCES: dict[str, Tolerance] = {
35
+ "float32": Tolerance(1.3e-6, 1e-5, "torch.testing default for float32"),
36
+ "float16": Tolerance(1e-3, 1e-5, "torch.testing default for float16 (~1 ulp)"),
37
+ "bfloat16": Tolerance(1.6e-2, 1e-5, "torch.testing default for bfloat16 (~2 ulp)"),
38
+ "float64": Tolerance(1e-7, 1e-7, "torch.testing default for float64"),
39
+ }
40
+
41
+ EXACT = Tolerance(0.0, 0.0, "integer outputs must match exactly")
42
+
43
+
44
+ def tolerance_for(dtype: str, overrides: dict[str, Tolerance] | None = None) -> Tolerance:
45
+ """Tolerance for an output dtype, honouring backend overrides."""
46
+ if overrides and dtype in overrides:
47
+ return overrides[dtype]
48
+ return DEFAULT_TOLERANCES.get(dtype, EXACT)
@@ -0,0 +1,131 @@
1
+ """Run something (the original kernel or a port) on every case and compare to the reference.
2
+
3
+ This is the single code path behind both reference validation and port
4
+ verification, so both are held to the same checks.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import traceback
10
+ from pathlib import Path
11
+ from typing import Callable
12
+
13
+ import torch
14
+
15
+ from kernelport.harness.compare import compare_tensor
16
+ from kernelport.harness.inputs import TORCH_DTYPES, CaseData, make_case
17
+ from kernelport.harness.reference import file_hash, run_reference
18
+ from kernelport.harness.results import CaseNotRunnable, CaseResult, RunResult
19
+ from kernelport.harness.spec import LaunchSpec
20
+ from kernelport.harness.tolerances import Tolerance, tolerance_for
21
+
22
+ Runner = Callable[[CaseData], dict[str, torch.Tensor]]
23
+ MAX_ERROR_CHARS = 1200
24
+
25
+
26
+ def _user_frames(exc: BaseException) -> str:
27
+ """Traceback lines from the port/reference file only (not kernelport, not libraries)."""
28
+ pkg_dir = Path(__file__).resolve().parent.parent
29
+ keep = []
30
+ for fr in traceback.extract_tb(exc.__traceback__):
31
+ f = Path(fr.filename)
32
+ if "site-packages" in f.parts or fr.filename.startswith("<") or pkg_dir in f.resolve().parents:
33
+ continue
34
+ keep.append(f" {f.name}, line {fr.lineno}, in {fr.name}: {(fr.line or '').strip()}")
35
+ return "\n".join(keep[-3:])
36
+
37
+
38
+ # Shader compilers (naga, Metal) draw source excerpts with box characters; Windows
39
+ # consoles in legacy code pages cannot print them.
40
+ _BOX = str.maketrans({"┌": "+", "─": "-", "│": "|", "└": "+", "├": "+"})
41
+
42
+
43
+ def _short_error(exc: BaseException) -> str:
44
+ """Exception text plus user-code frames, capped so it fits an agent's context."""
45
+ msg = str(exc).split("\nTraceback (most recent call last)")[0].strip().translate(_BOX)
46
+ text = f"{type(exc).__name__}: {msg}"
47
+ frames = _user_frames(exc)
48
+ full = f"{text}\n{frames}" if frames else text
49
+ return full if len(full) <= MAX_ERROR_CHARS else full[:MAX_ERROR_CHARS] + " ...[truncated]"
50
+
51
+
52
+ def run_cases(*, kernel: str, backend: str, hardware: str, spec: LaunchSpec,
53
+ reference_path: Path, runner: Runner, code_hash: str,
54
+ tol_overrides: dict[str, Tolerance] | None = None,
55
+ stop_after_failures: int | None = None,
56
+ supported_dtypes: tuple[str, ...] | None = None) -> RunResult:
57
+ """Run ``runner`` on every (case, dtype) and compare against the reference.
58
+
59
+ Cases whose dtype is not in ``supported_dtypes`` are skipped and named in
60
+ the result's reason. Status is PASS only if every remaining case ran and
61
+ every output matched; if nothing remains, status is SKIPPED.
62
+ """
63
+ cases: list[CaseResult] = []
64
+ not_runnable: list[str] = []
65
+ used_tols: dict[str, dict] = {}
66
+ failures = 0
67
+ todo = [(c, t) for c, t in spec.iter_cases() if supported_dtypes is None or t in supported_dtypes]
68
+ skipped = sorted({t for _, t in spec.iter_cases()} - {t for _, t in todo})
69
+ for case, dtype in todo:
70
+ data = make_case(spec, case, dtype)
71
+ cr = CaseResult(case.name, dtype, False, data.input_hash)
72
+ try:
73
+ expected = run_reference(reference_path, spec, data)
74
+ except Exception as exc:
75
+ cr.error = "reference failed: " + _short_error(exc)
76
+ cases.append(cr)
77
+ failures += 1
78
+ if stop_after_failures and failures >= stop_after_failures:
79
+ break
80
+ continue
81
+ try:
82
+ actual = runner(data)
83
+ except CaseNotRunnable as exc:
84
+ not_runnable.append(f"{case.name}/{dtype} ({exc})")
85
+ continue
86
+ except Exception as exc:
87
+ err = _short_error(exc)
88
+ prev = next((c for c in cases if c.error == err or c.error.startswith(err)), None)
89
+ cr.error = f"same error as {prev.case}/{prev.dtype}" if prev else err
90
+ actual = None
91
+ if actual is not None:
92
+ ok = True
93
+ for p in spec.params:
94
+ if not p.is_output:
95
+ continue
96
+ out_dtype = p.resolved_dtype(dtype)
97
+ tol = tolerance_for(out_dtype, tol_overrides)
98
+ used_tols[out_dtype] = {"rtol": tol.rtol, "atol": tol.atol, "reason": tol.reason}
99
+ if p.name not in actual:
100
+ cr.outputs.append({"name": p.name, "passed": False, "reason": "output missing",
101
+ "shape": [], "expected_slice": [], "actual_slice": [],
102
+ "slice_start": 0})
103
+ ok = False
104
+ continue
105
+ tc = compare_tensor(p.name, actual[p.name], expected[p.name],
106
+ TORCH_DTYPES[out_dtype], tol)
107
+ cr.outputs.append(tc.to_dict())
108
+ ok = ok and tc.passed
109
+ cr.passed = ok
110
+ cases.append(cr)
111
+ if not cr.passed:
112
+ failures += 1
113
+ if stop_after_failures and failures >= stop_after_failures:
114
+ break
115
+
116
+ all_ran = len(cases) + len(not_runnable) == len(todo)
117
+ if not cases:
118
+ status = "SKIPPED"
119
+ else:
120
+ status = "PASS" if all_ran and all(c.passed for c in cases) else "FAIL"
121
+ notes = []
122
+ if not all_ran:
123
+ notes.append(f"stopped early after {failures} failing cases")
124
+ if skipped:
125
+ notes.append(f"dtypes not supported by this backend, not tested: {', '.join(skipped)}")
126
+ if not_runnable:
127
+ notes.append(f"cases this device cannot run, not tested: {'; '.join(not_runnable)}")
128
+ reason = "; ".join(notes)
129
+ return RunResult(kernel=kernel, backend=backend, status=status, hardware=hardware,
130
+ reason=reason, code_hash=code_hash, reference_hash=file_hash(reference_path),
131
+ spec_hash=spec.hash(), tolerances=used_tols, cases=cases)
kernelport/hashing.py ADDED
@@ -0,0 +1,28 @@
1
+ """Content hashes for kernels, references, specs and ports.
2
+
3
+ Hashes ignore line-ending style: CRLF is folded to LF first. A ``.kernelport/``
4
+ workspace frozen on Linux or Colab and checked out on Windows (git
5
+ ``core.autocrlf=true``) must still match its manifest, and a port verified on a
6
+ Mac must keep its PASS when the repo is pulled elsewhere. Any other byte change
7
+ still changes the hash.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ from pathlib import Path
14
+
15
+
16
+ def sha256_bytes(data: bytes) -> str:
17
+ """sha256 of ``data`` with CRLF folded to LF."""
18
+ return hashlib.sha256(data.replace(b"\r\n", b"\n")).hexdigest()
19
+
20
+
21
+ def sha256_text(text: str) -> str:
22
+ """sha256 of UTF-8 text (line-ending insensitive)."""
23
+ return sha256_bytes(text.encode("utf-8"))
24
+
25
+
26
+ def sha256_file(path: Path) -> str:
27
+ """sha256 of a file's contents (line-ending insensitive)."""
28
+ return sha256_bytes(Path(path).read_bytes())