compilersutra-perf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compilersutra_perf-0.1.0.dist-info/METADATA +633 -0
- compilersutra_perf-0.1.0.dist-info/RECORD +106 -0
- compilersutra_perf-0.1.0.dist-info/WHEEL +5 -0
- compilersutra_perf-0.1.0.dist-info/entry_points.txt +2 -0
- compilersutra_perf-0.1.0.dist-info/licenses/LICENSE +176 -0
- compilersutra_perf-0.1.0.dist-info/licenses/NOTICE +6 -0
- compilersutra_perf-0.1.0.dist-info/top_level.txt +1 -0
- csperf/__init__.py +8 -0
- csperf/backends/__init__.py +6 -0
- csperf/backends/base.py +16 -0
- csperf/backends/cpu.py +16 -0
- csperf/backends/gpu.py +47 -0
- csperf/backends/registry.py +36 -0
- csperf/cli.py +575 -0
- csperf/config.py +67 -0
- csperf/dashboard_app.py +118 -0
- csperf/detector.py +32 -0
- csperf/execution/__init__.py +57 -0
- csperf/execution/base.py +23 -0
- csperf/execution/build.py +110 -0
- csperf/execution/cpu.py +237 -0
- csperf/execution/gpu.py +66 -0
- csperf/execution/hip.py +246 -0
- csperf/execution/opencl.py +182 -0
- csperf/execution/utils.py +172 -0
- csperf/execution/vulkan.py +144 -0
- csperf/experiments/__init__.py +6 -0
- csperf/experiments/registry.py +47 -0
- csperf/hardware.py +223 -0
- csperf/models.py +50 -0
- csperf/native_project/CMakeLists.txt +13 -0
- csperf/native_project/native/runtime/CMakeLists.txt +143 -0
- csperf/native_project/native/runtime/include/csperf/native/NativeApplication.hpp +16 -0
- csperf/native_project/native/runtime/include/csperf/native/NativeArgumentParser.hpp +16 -0
- csperf/native_project/native/runtime/include/csperf/native/NativeOptions.hpp +16 -0
- csperf/native_project/native/runtime/include/csperf/native/NativeRunner.hpp +17 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLApplication.hpp +16 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLArgumentParser.hpp +20 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLDeviceCatalog.hpp +35 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLDevicePolicy.hpp +40 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLKernelExecutor.hpp +22 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLSupport.hpp +26 -0
- csperf/native_project/native/runtime/include/csperf/opencl/OpenCLTypes.hpp +78 -0
- csperf/native_project/native/runtime/include/csperf/runtime/Application.hpp +15 -0
- csperf/native_project/native/runtime/include/csperf/runtime/Process.hpp +27 -0
- csperf/native_project/native/runtime/include/csperf/support/Debug.hpp +13 -0
- csperf/native_project/native/runtime/include/csperf/support/JSON.hpp +14 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanApplication.hpp +16 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanArgumentParser.hpp +21 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanDeviceCatalog.hpp +36 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanDevicePolicy.hpp +36 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanLibraryPolicy.hpp +25 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanLoader.hpp +29 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanOptions.hpp +19 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanQueueFamilyPolicy.hpp +29 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanShaderValidator.hpp +36 -0
- csperf/native_project/native/runtime/include/csperf/vulkan/VulkanSupport.hpp +158 -0
- csperf/native_project/native/runtime/include/runtime.h +7 -0
- csperf/native_project/native/runtime/lib/Native/NativeApplication.cpp +38 -0
- csperf/native_project/native/runtime/lib/Native/NativeArgumentParser.cpp +30 -0
- csperf/native_project/native/runtime/lib/Native/NativeRunner.cpp +16 -0
- csperf/native_project/native/runtime/lib/OpenCL/OpenCLApplication.cpp +57 -0
- csperf/native_project/native/runtime/lib/OpenCL/OpenCLArgumentParser.cpp +159 -0
- csperf/native_project/native/runtime/lib/OpenCL/OpenCLDeviceCatalog.cpp +86 -0
- csperf/native_project/native/runtime/lib/OpenCL/OpenCLDevicePolicy.cpp +75 -0
- csperf/native_project/native/runtime/lib/OpenCL/OpenCLKernelExecutor.cpp +232 -0
- csperf/native_project/native/runtime/lib/OpenCL/OpenCLSupport.cpp +116 -0
- csperf/native_project/native/runtime/lib/Runtime/Application.cpp +5 -0
- csperf/native_project/native/runtime/lib/Runtime/Process.cpp +125 -0
- csperf/native_project/native/runtime/lib/Runtime/RuntimeInfo.cpp +16 -0
- csperf/native_project/native/runtime/lib/Support/JSON.cpp +46 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanApplication.cpp +63 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanArgumentParser.cpp +48 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanDeviceCatalog.cpp +82 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanDevicePolicy.cpp +57 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanLibraryPolicy.cpp +19 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanLoader.cpp +88 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanQueueFamilyPolicy.cpp +25 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanShaderValidator.cpp +120 -0
- csperf/native_project/native/runtime/lib/Vulkan/VulkanSupport.cpp +69 -0
- csperf/native_project/native/runtime/tools/csperf-native-runner/main.cpp +9 -0
- csperf/native_project/native/runtime/tools/csperf-opencl-runner/main.cpp +9 -0
- csperf/native_project/native/runtime/tools/csperf-vulkan-runner/main.cpp +9 -0
- csperf/pipelines/__init__.py +6 -0
- csperf/pipelines/base.py +50 -0
- csperf/pipelines/factory.py +64 -0
- csperf/profile_summary.py +589 -0
- csperf/profiler/__init__.py +5 -0
- csperf/profiler/base.py +71 -0
- csperf/profiler/cpu/__init__.py +123 -0
- csperf/profiler/cpu/base.py +64 -0
- csperf/profiler/cpu/collectors/__init__.py +13 -0
- csperf/profiler/cpu/collectors/base.py +43 -0
- csperf/profiler/cpu/collectors/linux.py +91 -0
- csperf/profiler/cpu/collectors/macos.py +155 -0
- csperf/profiler/cpu/detectors/__init__.py +13 -0
- csperf/profiler/cpu/detectors/architecture.py +26 -0
- csperf/profiler/cpu/detectors/platform.py +24 -0
- csperf/profiler/cpu/detectors/vendor.py +71 -0
- csperf/profiler/cpu/factory.py +34 -0
- csperf/profiler/cpu/metrics/__init__.py +20 -0
- csperf/profiler/cpu/metrics/definitions.py +106 -0
- csperf/profiler/cpu/metrics/mapping.py +33 -0
- csperf/profiler/cpu/metrics/registry.py +44 -0
- csperf/results.py +49 -0
- csperf/visualize.py +208 -0
|
@@ -0,0 +1,633 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: compilersutra-perf
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Modular performance experimentation framework for native and GPU workloads
|
|
5
|
+
Author-email: Abhinav <contact@compilersutra.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://compilersutra.com
|
|
8
|
+
Project-URL: Documentation, https://compilersutra.com
|
|
9
|
+
Project-URL: Author, https://www.linkedin.com/in/abhinavcompilerllvm/
|
|
10
|
+
Project-URL: Changelog, https://compilersutra.com
|
|
11
|
+
Keywords: benchmarking,compiler,gpu,hip,llvm,opencl,performance,profiling,systems,vulkan
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering
|
|
21
|
+
Classifier: Topic :: Software Development :: Compilers
|
|
22
|
+
Classifier: Topic :: System :: Benchmark
|
|
23
|
+
Classifier: Topic :: System :: Hardware
|
|
24
|
+
Classifier: Topic :: System :: Systems Administration
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
License-File: NOTICE
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
31
|
+
Provides-Extra: visualize
|
|
32
|
+
Requires-Dist: streamlit>=1.33; extra == "visualize"
|
|
33
|
+
Requires-Dist: plotly>=5.20; extra == "visualize"
|
|
34
|
+
Provides-Extra: track
|
|
35
|
+
Requires-Dist: mlflow>=2.11; extra == "track"
|
|
36
|
+
Provides-Extra: tune
|
|
37
|
+
Requires-Dist: optuna>=3.5; extra == "tune"
|
|
38
|
+
Requires-Dist: nevergrad>=1.0.5; extra == "tune"
|
|
39
|
+
Provides-Extra: distributed
|
|
40
|
+
Requires-Dist: ray>=2.10; extra == "distributed"
|
|
41
|
+
Requires-Dist: dask>=2024.3.1; extra == "distributed"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
<!-- SPDX-License-Identifier: Apache-2.0 -->
|
|
45
|
+
<!-- Copyright (c) 2026 CompilerSutraPerfTool -->
|
|
46
|
+
<!-- Author: Abhinav | https://compilersutra.com | https://www.linkedin.com/in/abhinavcompilerllvm/ -->
|
|
47
|
+
|
|
48
|
+
# CompilerSutraPerfTool
|
|
49
|
+
|
|
50
|
+
CompilerSutraPerfTool is a modular performance experimentation framework for native programs, GPU kernels, and shader workloads. It is designed for compiler engineers, systems developers, GPU developers, and performance researchers who need reproducible experiments across CPU and GPU backends.
|
|
51
|
+
|
|
52
|
+
## Installation
|
|
53
|
+
|
|
54
|
+
Install locally in editable mode:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
cd CompilerSutraPerfTool
|
|
58
|
+
python3 -m venv .venv
|
|
59
|
+
source .venv/bin/activate
|
|
60
|
+
pip install -e .
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Install optional extras:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install -e '.[dev]'
|
|
67
|
+
pip install -e '.[visualize]'
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
For a published package, the intended distribution name is:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install compilersutra-perf
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
For a fuller design explanation, see [GETTING_STARTED.md](docs/GETTING_STARTED.md), [ARCHITECTURE.md](docs/ARCHITECTURE.md), [USAGE.md](docs/USAGE.md), [TESTING.md](docs/TESTING.md), [RELEASING.md](RELEASING.md), [SECURITY.md](SECURITY.md), [CONTRIBUTING.md](CONTRIBUTING.md), and [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md).
|
|
77
|
+
|
|
78
|
+
## Project Metadata
|
|
79
|
+
|
|
80
|
+
- Website: `https://compilersutra.com`
|
|
81
|
+
- Author: `Abhinav`
|
|
82
|
+
- LinkedIn: `https://www.linkedin.com/in/abhinavcompilerllvm/`
|
|
83
|
+
- License: `Apache-2.0`, see [LICENSE](LICENSE)
|
|
84
|
+
|
|
85
|
+
## Repository Hygiene
|
|
86
|
+
|
|
87
|
+
- Contribution guide: [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
88
|
+
- Security policy: [SECURITY.md](SECURITY.md)
|
|
89
|
+
- Release procedure: [RELEASING.md](RELEASING.md)
|
|
90
|
+
- CI workflow: [.github/workflows/ci.yml](.github/workflows/ci.yml)
|
|
91
|
+
|
|
92
|
+
This repository is structured as a production-style MVP:
|
|
93
|
+
|
|
94
|
+
- Stable CPU-oriented workflow with real compile-and-run support
|
|
95
|
+
- Native OpenCL execution, native HIP execution, and native Vulkan shader validation
|
|
96
|
+
- Structured result generation for JSON/CSV consumers
|
|
97
|
+
- Lightweight HTML visualization and Streamlit dashboard support with derived metrics, comparison tables, and comparison charts
|
|
98
|
+
- Dedicated CPU-vs-GPU comparison summaries with normalized runtime and GPU speedup reporting
|
|
99
|
+
- A native C++ runtime runner used by CPU execution, ready for LLVM/MLIR-backed expansion
|
|
100
|
+
|
|
101
|
+
## Current Capabilities
|
|
102
|
+
|
|
103
|
+
| Backend | Current State | Notes |
|
|
104
|
+
| --- | --- | --- |
|
|
105
|
+
| CPU | Execute + profile | Native execution through C++ runner, LLVM IR emission, curated `perf` counters, warmup/repeat support, and optional CPU affinity |
|
|
106
|
+
| GPU | Auto-route | Generic `--backend gpu` alias resolves to HIP for `.hip`, OpenCL for `.cl`, and Vulkan for shader inputs, with vendor-aware device inventory |
|
|
107
|
+
| OpenCL | Execute | Native kernel build and launch with generalized buffer/scalar/local argument binding, warmup/repeat support, vendor-aware device listing, and device selection |
|
|
108
|
+
| Vulkan | Compile + validate | GLSL to SPIR-V, native device selection, vendor-aware device listing, shader-module validation |
|
|
109
|
+
| CUDA | Planned | No native runner yet |
|
|
110
|
+
| HIP | Execute | Native `hipcc` compile-and-run path for `.hip` inputs, ROCm device discovery, warmup/repeat support, device selection, and `rocprof`-backed profiling artifacts |
|
|
111
|
+
| Metal | Planned | Required for full macOS GPU support |
|
|
112
|
+
|
|
113
|
+
## Supported On This Machine
|
|
114
|
+
|
|
115
|
+
The current repository has been exercised on this Linux host:
|
|
116
|
+
|
|
117
|
+
- CPU: AMD Ryzen 7 9700X
|
|
118
|
+
- OpenCL GPU devices detected: AMD Radeon RX 9060 XT and AMD Radeon Graphics
|
|
119
|
+
- HIP/ROCm GPU devices detected: AMD Radeon RX 9060 XT and AMD Radeon Graphics
|
|
120
|
+
- Vulkan runtime detected and native shader validation verified
|
|
121
|
+
- Tooling detected: `perf`, `clang`, `clang++`, `hipcc`, `hipconfig`, `glslangValidator`, `spirv-opt`, `clinfo`, `vulkaninfo`
|
|
122
|
+
|
|
123
|
+
This means the following paths have been verified locally:
|
|
124
|
+
|
|
125
|
+
- CPU compile + native execution
|
|
126
|
+
- CPU `perf`-based profiling with curated counters
|
|
127
|
+
- HIP native compile + execution
|
|
128
|
+
- OpenCL native kernel execution
|
|
129
|
+
- Vulkan SPIR-V compilation + native shader-module validation
|
|
130
|
+
|
|
131
|
+
## Example Commands
|
|
132
|
+
|
|
133
|
+
### CPU
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
csperf run --input examples/cpp/matrix_traversal.cpp --backend cpu --warmup-runs 1 --repeat-runs 3 --output results/cpu-perf.json
|
|
137
|
+
csperf profile results/cpu-perf.json
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### CPU + HIP + OpenCL Flow
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
csperf list-devices --backend gpu
|
|
144
|
+
csperf list-devices --backend hip
|
|
145
|
+
csperf run --input examples/cpp/matrix_traversal.cpp --backend cpu --warmup-runs 1 --repeat-runs 3 --output results/cpu-perf.json
|
|
146
|
+
csperf run --input examples/hip/vector_add.hip --backend hip --device-index 0 --warmup-runs 1 --repeat-runs 2 --output results/hip-profiled.json
|
|
147
|
+
csperf run --input examples/opencl/saxpy.cl --backend gpu --device-index 0 --warmup-runs 1 --repeat-runs 3 --output results/gpu-opencl.json
|
|
148
|
+
csperf visualize results/cpu-perf.json results/hip-profiled.json --output results/cpu-vs-hip.html
|
|
149
|
+
csperf visualize results/cpu-perf.json results/gpu-opencl.json --output results/cpu-vs-gpu.html
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### OpenCL
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
csperf run --input examples/opencl/saxpy.cl --backend gpu --device-index 0 --warmup-runs 1 --repeat-runs 3 --output results/gpu-opencl.json
|
|
156
|
+
csperf profile results/gpu-opencl.json
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### HIP
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
csperf list-devices --backend hip
|
|
163
|
+
csperf run --input examples/hip/vector_add.hip --backend hip --device-index 0 --warmup-runs 1 --repeat-runs 2 --output results/hip.json
|
|
164
|
+
csperf run --input examples/hip/vector_add.hip --backend hip --device-index 0 --warmup-runs 1 --repeat-runs 2 --no-perf --output results/hip-no-prof.json
|
|
165
|
+
csperf run --input examples/hip/vector_add.hip --backend gpu --device-index 0 --warmup-runs 1 --repeat-runs 2 --output results/gpu-hip.json
|
|
166
|
+
csperf profile results/hip.json
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Generic GPU Alias
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
csperf list-devices --backend gpu
|
|
173
|
+
csperf run --input examples/opencl/saxpy.cl --backend gpu --device-index 0 --warmup-runs 1 --repeat-runs 3 --output results/gpu-opencl.json
|
|
174
|
+
csperf profile results/gpu-opencl.json
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### Vulkan
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
csperf run --input examples/shaders/vector_add.comp --backend vulkan --device-index 0 --output results/vulkan-native.json
|
|
181
|
+
csperf profile results/vulkan-native.json
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
### Comparison Report
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
csperf visualize results/cpu-perf.json results/gpu-opencl.json --output results/compare-charts.html
|
|
188
|
+
csperf dashboard results/cpu-perf.json results/gpu-opencl.json
|
|
189
|
+
csperf list-devices --backend gpu
|
|
190
|
+
csperf list-devices --backend opencl
|
|
191
|
+
csperf list-devices --backend vulkan
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### Result Diff
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
csperf diff results/gcc.json results/clang.json
|
|
198
|
+
csperf diff results/gcc.json results/clang.json --csv results/gcc-vs-clang.csv
|
|
199
|
+
csperf diff results/gcc.json results/clang.json --output results/gcc-vs-clang.json --csv results/gcc-vs-clang.csv
|
|
200
|
+
csperf diff results/gcc.json results/clang.json --derived-config configs/derived_metrics.sample.json --output results/gcc-vs-clang-derived.json --csv results/gcc-vs-clang-derived.csv
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
The diff command compares two stored result files and can export:
|
|
204
|
+
|
|
205
|
+
- raw metric differences
|
|
206
|
+
- percentage differences
|
|
207
|
+
- the exact compiler used for each result from `CC` / `CXX`
|
|
208
|
+
- config-driven derived metrics from [derived_metrics.sample.json](configs/derived_metrics.sample.json)
|
|
209
|
+
- bottleneck analysis from config thresholds and rules
|
|
210
|
+
|
|
211
|
+
### Compiler Diff Batch
|
|
212
|
+
|
|
213
|
+
For CPU-only compiler comparison across a folder of C/C++ files, use:
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
python3 scripts/compiler_diff_batch.py examples/cpp \
|
|
217
|
+
--config1 configs/compiler_gcc.sample.json \
|
|
218
|
+
--config2 configs/compiler_clang.sample.json
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Recursive folder scan:
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
python3 scripts/compiler_diff_batch.py examples \
|
|
225
|
+
--config1 configs/compiler_gcc.sample.json \
|
|
226
|
+
--config2 configs/compiler_clang.sample.json \
|
|
227
|
+
--recursive
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
What it does:
|
|
231
|
+
|
|
232
|
+
- scans the folder for `.c`, `.cc`, `.cpp`, and `.cxx` files
|
|
233
|
+
- runs every file with compiler config 1 and compiler config 2
|
|
234
|
+
- generates per-file diff CSV files
|
|
235
|
+
- generates derived diff CSV files unless `--skip-derived` is used
|
|
236
|
+
- writes a combined summary CSV
|
|
237
|
+
- writes a summary XLSX workbook containing the CSV data as spreadsheet sheets
|
|
238
|
+
|
|
239
|
+
Inputs:
|
|
240
|
+
|
|
241
|
+
- `--config1` and `--config2` must point to compiler config JSON files such as [compiler_gcc.sample.json](configs/compiler_gcc.sample.json) and [compiler_clang.sample.json](configs/compiler_clang.sample.json)
|
|
242
|
+
- the script is CPU-only and uses `csperf run --backend cpu` internally
|
|
243
|
+
- use `--results-dir` to choose the output directory and `--skip-derived` to omit derived-metric sheets
|
|
244
|
+
|
|
245
|
+
Outputs:
|
|
246
|
+
|
|
247
|
+
- `results/compiler-diff/summary.csv`
|
|
248
|
+
- `results/compiler-diff/summary.xlsx`
|
|
249
|
+
- per-file result JSON artifacts
|
|
250
|
+
- per-file diff CSV and JSON artifacts
|
|
251
|
+
- per-file derived diff CSV and JSON artifacts when derived diff is enabled
|
|
252
|
+
|
|
253
|
+
### CPU vs GPU Comparison
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
csperf visualize results/cpu-perf.json results/gpu-opencl.json --output results/cpu-vs-gpu.html
|
|
257
|
+
csperf dashboard results/cpu-perf.json results/gpu-opencl.json
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
The current comparison mode adds:
|
|
261
|
+
|
|
262
|
+
- normalized comparable runtime in milliseconds
|
|
263
|
+
- CPU-vs-GPU summary table
|
|
264
|
+
- GPU speedup versus CPU
|
|
265
|
+
- backend-aware throughput and cache charts
|
|
266
|
+
|
|
267
|
+
## Goals
|
|
268
|
+
|
|
269
|
+
- Accept a source file and infer the correct compilation and execution pipeline
|
|
270
|
+
- Run workload experiments with explicit flags such as memory layout or tiling
|
|
271
|
+
- Collect hardware context and profiling-ready metadata
|
|
272
|
+
- Store reproducible results for later comparison and visualization
|
|
273
|
+
- Keep compiler, runtime, profiling, and visualization layers modular
|
|
274
|
+
|
|
275
|
+
## Repository Layout
|
|
276
|
+
|
|
277
|
+
```text
|
|
278
|
+
CompilerSutraPerfTool/
|
|
279
|
+
├── README.md
|
|
280
|
+
├── CMakeLists.txt
|
|
281
|
+
├── pyproject.toml
|
|
282
|
+
├── configs/
|
|
283
|
+
├── examples/
|
|
284
|
+
├── native/
|
|
285
|
+
│ └── runtime/
|
|
286
|
+
├── src/
|
|
287
|
+
│ └── csperf/
|
|
288
|
+
└── tests/
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
## Key Modules
|
|
292
|
+
|
|
293
|
+
- `src/csperf/cli.py`: CLI entry point with `run`, `profile`, `diff`, `visualize`, `list-backends`, and `list-experiments`
|
|
294
|
+
- `src/csperf/execution.py`: Python orchestration layer that builds and invokes native runtime components, benchmark controls, and trial summaries
|
|
295
|
+
- `src/csperf/hardware.py`: GPU tooling detection and vendor classification helpers
|
|
296
|
+
- `src/csperf/detector.py`: Source type detection based on extension
|
|
297
|
+
- `src/csperf/pipelines/`: Compilation and execution pipeline planners
|
|
298
|
+
- `src/csperf/backends/`: Backend registry and backend capability metadata
|
|
299
|
+
- `src/csperf/profiler/`: Profiling abstractions and CPU-oriented metric definitions
|
|
300
|
+
- `src/csperf/profile_summary.py`: Shared derived-metric, unit-aware, and CPU-vs-GPU comparison summaries
|
|
301
|
+
- `src/csperf/results.py`: Result schema and JSON/CSV export
|
|
302
|
+
- `src/csperf/visualize.py`: HTML report generator with multi-result comparison tables, CPU-vs-GPU summaries, and bar-style charts
|
|
303
|
+
- `native/runtime/`: C++ runtime split into reusable libraries and thin native tools for low-overhead execution
|
|
304
|
+
|
|
305
|
+
## Supported Workload Types
|
|
306
|
+
|
|
307
|
+
- C programs
|
|
308
|
+
- C++ programs
|
|
309
|
+
- OpenCL kernels
|
|
310
|
+
- GPU compute kernels
|
|
311
|
+
- Vulkan compute shaders
|
|
312
|
+
- Compute workloads and shader validation
|
|
313
|
+
- Memory access benchmarks
|
|
314
|
+
|
|
315
|
+
## Current Stability Model
|
|
316
|
+
|
|
317
|
+
- **Stable**
|
|
318
|
+
- CLI orchestration
|
|
319
|
+
- Workload detection
|
|
320
|
+
- CPU compile and native execution
|
|
321
|
+
- CPU perf collection with curated default counters
|
|
322
|
+
- Experiment configuration
|
|
323
|
+
- Structured result export
|
|
324
|
+
- Unit-aware profiling summaries
|
|
325
|
+
- **Experimental**
|
|
326
|
+
- HIP native execution with `hipcc` compile-and-run
|
|
327
|
+
- OpenCL native execution with generalized buffer/scalar/local binding
|
|
328
|
+
- Vulkan native shader validation
|
|
329
|
+
- Chart-heavy visualization
|
|
330
|
+
- **Research**
|
|
331
|
+
- MLIR-based transformations
|
|
332
|
+
- Architecture-aware autotuning
|
|
333
|
+
- Cross-backend optimization studies
|
|
334
|
+
|
|
335
|
+
## Quick Start
|
|
336
|
+
|
|
337
|
+
Detailed usage instructions are in [USAGE.md](docs/USAGE.md).
|
|
338
|
+
|
|
339
|
+
### 1. Create a virtual environment
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
cd CompilerSutraPerfTool
|
|
343
|
+
python3 -m venv .venv
|
|
344
|
+
source .venv/bin/activate
|
|
345
|
+
pip install -e .
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
### 2. List available capabilities
|
|
349
|
+
|
|
350
|
+
```bash
|
|
351
|
+
csperf list-backends
|
|
352
|
+
csperf list-experiments
|
|
353
|
+
csperf cpuinfo
|
|
354
|
+
csperf gpuinfo
|
|
355
|
+
csperf deviceinfo
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
### 3. Run an example workload
|
|
359
|
+
|
|
360
|
+
```bash
|
|
361
|
+
csperf run --input examples/cpp/matrix_traversal.cpp --experiment row-major,column-major
|
|
362
|
+
csperf run --input examples/cpp/tiled_matmul.cpp --experiment tiled --tile-size 32
|
|
363
|
+
csperf run --input examples/shaders/vector_add.comp --backend gpu
|
|
364
|
+
csperf run --input examples/opencl/saxpy.cl --backend gpu
|
|
365
|
+
csperf run --input examples/hip/vector_add.hip --backend gpu
|
|
366
|
+
csperf run --input examples/opencl/saxpy.cl --backend opencl --vendor amd
|
|
367
|
+
csperf run --input examples/shaders/vector_add.comp --backend vulkan --vendor amd
|
|
368
|
+
csperf run --input examples/opencl/saxpy.cl --backend opencl --policy-config configs/policy_config.sample.json
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
For CPU workloads, `csperf run` compiles and executes the program by default when `clang` or `clang++` is installed. For HIP workloads, the tool compiles `.hip` sources with `hipcc` and executes them against the ROCm runtime. For Vulkan shaders, the tool compiles GLSL to SPIR-V and then validates the shader natively against the Vulkan runtime. For OpenCL kernels, the tool builds and launches kernels natively through the C++ OpenCL runner.
|
|
372
|
+
|
|
373
|
+
### 4. Inspect or visualize results
|
|
374
|
+
|
|
375
|
+
```bash
|
|
376
|
+
csperf profile results/latest.json
|
|
377
|
+
csperf diff results/gcc.json results/clang.json --csv results/gcc-vs-clang.csv
|
|
378
|
+
csperf diff results/gcc.json results/clang.json --derived-config configs/derived_metrics.sample.json --output results/gcc-vs-clang-derived.json --csv results/gcc-vs-clang-derived.csv
|
|
379
|
+
csperf visualize results/latest.json --output results/report.html
|
|
380
|
+
csperf dashboard results/latest.json
|
|
381
|
+
csperf visualize results/cpu-perf.json results/gpu-opencl.json --output results/compare-charts.html
|
|
382
|
+
csperf dashboard results/cpu-perf.json results/gpu-opencl.json
|
|
383
|
+
csperf visualize results/cpu-perf.json results/gpu-opencl.json --output results/cpu-vs-gpu.html
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
## CLI Overview
|
|
387
|
+
|
|
388
|
+
### Run a workload
|
|
389
|
+
|
|
390
|
+
```bash
|
|
391
|
+
csperf run --input program.c
|
|
392
|
+
csperf run --input matmul.cpp --backend cpu
|
|
393
|
+
csperf run --input shader.comp --backend gpu
|
|
394
|
+
csperf run --input kernel.cl --backend gpu
|
|
395
|
+
csperf run --input kernel.hip --backend gpu
|
|
396
|
+
```
|
|
397
|
+
|
|
398
|
+
Useful execution flags:
|
|
399
|
+
|
|
400
|
+
```bash
|
|
401
|
+
csperf run --input program.cpp --plan-only
|
|
402
|
+
csperf run --input program.cpp --no-perf
|
|
403
|
+
csperf run --input program.cpp --build-dir build/debug
|
|
404
|
+
csperf run --input program.cpp --warmup-runs 1 --repeat-runs 5
|
|
405
|
+
csperf run --input examples/opencl/saxpy.cl --backend gpu --device-index 0
|
|
406
|
+
csperf run --input examples/opencl/saxpy.cl --backend opencl --vendor amd --device-index 0
|
|
407
|
+
csperf run --input examples/opencl/saxpy.cl --backend opencl --policy-config configs/policy_config.sample.json --device-index 0
|
|
408
|
+
csperf run --input examples/opencl/saxpy.cl --backend gpu --kernel-name saxpy --kernel-arg buffer:float:read:4096:1.0 --kernel-arg buffer:float:read:4096:2.0 --kernel-arg buffer:float:write:4096:0.0 --kernel-arg scalar:uint32:4096 --readback-arg 2
|
|
409
|
+
csperf run --input examples/hip/vector_add.hip --backend hip --device-index 0
|
|
410
|
+
csperf run --input examples/shaders/vector_add.comp --backend vulkan --vendor amd --device-index 0
|
|
411
|
+
csperf run --input examples/shaders/vector_add.comp --backend vulkan --policy-config configs/policy_config.sample.json --device-index 0
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
### Memory layout experiments
|
|
415
|
+
|
|
416
|
+
```bash
|
|
417
|
+
csperf run --input matrix.cpp --experiment row-major
|
|
418
|
+
csperf run --input matrix.cpp --experiment column-major
|
|
419
|
+
csperf run --input matrix.cpp --experiment tiled --tile-size 32
|
|
420
|
+
csperf run --input matrix.cpp --experiment row-major,column-major,tiled --tile-size 32
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
### Custom optimization experiment
|
|
424
|
+
|
|
425
|
+
```bash
|
|
426
|
+
csperf run \
|
|
427
|
+
--input examples/cpp/tiled_matmul.cpp \
|
|
428
|
+
--experiment custom \
|
|
429
|
+
--opt-config configs/sample_tuning.json
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
## Example Workloads
|
|
433
|
+
|
|
434
|
+
### CPU examples
|
|
435
|
+
|
|
436
|
+
- `examples/cpp/matrix_traversal.cpp`: row-major vs column-major traversal
|
|
437
|
+
- `examples/cpp/tiled_matmul.cpp`: tiled matrix multiplication
|
|
438
|
+
- `examples/c/memory_stride.c`: cache locality and memory stride behavior
|
|
439
|
+
|
|
440
|
+
### GPU / shader examples
|
|
441
|
+
|
|
442
|
+
- `examples/shaders/vector_add.comp`: Vulkan compute shader
|
|
443
|
+
- `examples/opencl/vector_add.cl`: OpenCL kernel
|
|
444
|
+
- `examples/opencl/saxpy.cl`: OpenCL kernel with scalar argument binding
|
|
445
|
+
- `examples/hip/vector_add.hip`: HIP vector-add example with JSON metric output
|
|
446
|
+
|
|
447
|
+
Each example is accompanied by simple commands and is intended as a starting point for backend experimentation.
|
|
448
|
+
|
|
449
|
+
## GPU Status
|
|
450
|
+
|
|
451
|
+
- Vulkan: shader compilation to SPIR-V plus native device and shader-module validation is implemented
|
|
452
|
+
- HIP: native compile-and-run is implemented with ROCm device discovery
|
|
453
|
+
- HIP profiling: `rocprof --stats --hip-trace` integration is implemented and parsed into result metrics plus artifacts
|
|
454
|
+
- OpenCL: native kernel build and launch is implemented with configurable kernel argument binding
|
|
455
|
+
- CUDA / Metal: planning only
|
|
456
|
+
|
|
457
|
+
The next implementation step is workload matching plus baseline/regression analysis, then full Vulkan compute dispatch and broader backend coverage.
|
|
458
|
+
|
|
459
|
+
## Native Runtime
|
|
460
|
+
|
|
461
|
+
The `native/runtime` directory now follows a more LLVM-like split:
|
|
462
|
+
|
|
463
|
+
- `include/csperf/support/`: shared support headers such as debug assertions and JSON helpers
|
|
464
|
+
- `include/csperf/runtime/`: backend-independent runtime interfaces such as process helpers
|
|
465
|
+
- `include/csperf/native/`: CPU-native application interfaces
|
|
466
|
+
- `include/csperf/opencl/`: OpenCL application and support interfaces
|
|
467
|
+
- `include/csperf/vulkan/`: Vulkan application and support interfaces
|
|
468
|
+
- `lib/Support/`, `lib/Runtime/`, `lib/Native/`, `lib/OpenCL/`, `lib/Vulkan/`: reusable implementation libraries
|
|
469
|
+
- `tools/csperf-native-runner/`, `tools/csperf-opencl-runner/`, `tools/csperf-vulkan-runner/`: thin executable entrypoints
|
|
470
|
+
|
|
471
|
+
The backend libraries are also split by responsibility:
|
|
472
|
+
|
|
473
|
+
- Native CPU: argument parser, process runner, and application entry
|
|
474
|
+
- OpenCL: argument parser, device catalog, kernel executor, and application entry
|
|
475
|
+
- Vulkan: argument parser, runtime loader, device catalog, shader validator, and application entry
|
|
476
|
+
|
|
477
|
+
Hardware-specific decisions are now intended to live behind policy objects instead of preprocessor branches:
|
|
478
|
+
|
|
479
|
+
- OpenCL uses device-filter policies
|
|
480
|
+
- Vulkan uses library-loading and queue-family-selection policies
|
|
481
|
+
|
|
482
|
+
This keeps vendor or hardware-specific behavior injectable and reusable without spreading `#ifdef` logic through backend code.
|
|
483
|
+
|
|
484
|
+
The native code is organized around namespaces that mirror the layout:
|
|
485
|
+
|
|
486
|
+
- `csperf::support`
|
|
487
|
+
- `csperf::runtime`
|
|
488
|
+
- `csperf::native`
|
|
489
|
+
- `csperf::opencl`
|
|
490
|
+
- `csperf::vulkan`
|
|
491
|
+
|
|
492
|
+
The public CLI behavior is unchanged, but the internal runtime is now split into libraries first and executables second instead of monolithic backend source files.
|
|
493
|
+
|
|
494
|
+
The intent is to grow this further into:
|
|
495
|
+
|
|
496
|
+
- low-overhead launcher APIs
|
|
497
|
+
- pinned-memory and buffer abstractions
|
|
498
|
+
- platform/backend adapters
|
|
499
|
+
- future LLVM JIT or ahead-of-time execution support
|
|
500
|
+
|
|
501
|
+
## Visualization
|
|
502
|
+
|
|
503
|
+
- `csperf profile` prints unit-aware metric summaries and derived metrics such as IPC and miss-rate ratios.
|
|
504
|
+
- `csperf diff` compares two stored result files, can export CSV/JSON output, and supports config-driven derived metrics and bottleneck rules.
|
|
505
|
+
- `csperf visualize` generates an HTML report from one or more JSON result artifacts.
|
|
506
|
+
- `csperf visualize` shows derived metrics, units, comparison tables, CPU-vs-GPU summaries, and bar-style comparison charts.
|
|
507
|
+
- `csperf dashboard` launches a Streamlit dashboard against one or more result files and supports comparison tables, CPU-vs-GPU summaries, and charts.
|
|
508
|
+
|
|
509
|
+
HIP runs can also emit profiler artifacts such as:
|
|
510
|
+
|
|
511
|
+
- kernel stats CSV
|
|
512
|
+
- HIP API stats CSV
|
|
513
|
+
- copy stats CSV
|
|
514
|
+
- JSON trace
|
|
515
|
+
|
|
516
|
+
Install dashboard dependencies with:
|
|
517
|
+
|
|
518
|
+
```bash
|
|
519
|
+
pip install -e '.[visualize]'
|
|
520
|
+
```
|
|
521
|
+
|
|
522
|
+
## Testing
|
|
523
|
+
|
|
524
|
+
Detailed validation steps are in [TESTING.md](docs/TESTING.md).
|
|
525
|
+
|
|
526
|
+
```bash
|
|
527
|
+
python3 -m compileall src
|
|
528
|
+
python -m pytest
|
|
529
|
+
```
|
|
530
|
+
|
|
531
|
+
The included tests validate:
|
|
532
|
+
|
|
533
|
+
- source type detection
|
|
534
|
+
- experiment registry behavior
|
|
535
|
+
- CLI smoke behavior
|
|
536
|
+
- native runner build and backend verification are covered in the manual testing guide
|
|
537
|
+
|
|
538
|
+
## CPU Execution Notes
|
|
539
|
+
|
|
540
|
+
- CPU runs use `clang` for C and `clang++` for C++ when available.
|
|
541
|
+
- LLVM IR is emitted into the selected build directory alongside the native binary.
|
|
542
|
+
- CPU binary execution is handled by the native C++ runner built through CMake.
|
|
543
|
+
- `perf` collection is attempted by default on CPU runs using a curated event set from the local PMU list.
|
|
544
|
+
- CPU runs support warmup and repeated timing trials.
|
|
545
|
+
- CPU runs support optional CPU affinity through `--cpu-affinity`.
|
|
546
|
+
- CPU profile output includes units, derived metrics, and timing summaries across repeated runs.
|
|
547
|
+
- On systems with restrictive `perf_event_paranoid` settings, execution still succeeds and the result artifact records the profiling error in the `execution.perf_error` field.
|
|
548
|
+
|
|
549
|
+
## HIP Execution Notes
|
|
550
|
+
|
|
551
|
+
- HIP runs use `hipcc` when available.
|
|
552
|
+
- HIP device discovery uses `rocminfo`.
|
|
553
|
+
- HIP profiling uses `rocprof --stats --hip-trace` when `--no-perf` is not set.
|
|
554
|
+
- HIP result artifacts can include parsed profiler metrics and generated profiler CSV/JSON files.
|
|
555
|
+
- On this ROCm stack, `rocprof` prints deprecation/support warnings but still produces usable profiling outputs.
|
|
556
|
+
|
|
557
|
+
## Benchmark Controls
|
|
558
|
+
|
|
559
|
+
The current implementation supports:
|
|
560
|
+
|
|
561
|
+
- `--warmup-runs N`: warmup iterations before measurement
|
|
562
|
+
- `--repeat-runs N`: repeated measured trials with summary statistics
|
|
563
|
+
- `--device-index N`: explicit device selection for HIP execution, OpenCL execution, and Vulkan validation
|
|
564
|
+
- `--cpu-affinity 0,1`: pin CPU execution to specific logical cores through the native runner
|
|
565
|
+
- `--backend gpu`: generic GPU alias for currently supported GPU workload types
|
|
566
|
+
- `--compiler-flag FLAG`: append a compiler option; repeat the flag to pass multiple options
|
|
567
|
+
|
|
568
|
+
Example:
|
|
569
|
+
|
|
570
|
+
```bash
|
|
571
|
+
csperf run --input examples/cpp/matrix_traversal.cpp --backend cpu \
|
|
572
|
+
--compiler-flag=-march=native --compiler-flag=-funroll-loops
|
|
573
|
+
```
|
|
574
|
+
|
|
575
|
+
Device inventory commands:
|
|
576
|
+
|
|
577
|
+
- `csperf list-devices --backend cpu`
|
|
578
|
+
- `csperf list-devices --backend gpu`
|
|
579
|
+
- `csperf list-devices --backend hip`
|
|
580
|
+
- `csperf list-devices --backend opencl`
|
|
581
|
+
- `csperf list-devices --backend vulkan`
|
|
582
|
+
|
|
583
|
+
## Recommended Next Steps
|
|
584
|
+
|
|
585
|
+
If you are using the tool now:
|
|
586
|
+
|
|
587
|
+
1. Run matched CPU and GPU workloads and save the result JSON files.
|
|
588
|
+
2. Generate `cpu-vs-hip.html` and `cpu-vs-gpu.html` reports.
|
|
589
|
+
3. Inspect HIP profiler metrics and artifacts in the result JSON.
|
|
590
|
+
4. Use the dashboard for side-by-side comparison across CPU, HIP, and OpenCL runs.
|
|
591
|
+
|
|
592
|
+
If you are extending the tool next:
|
|
593
|
+
|
|
594
|
+
1. Add workload matching and baseline/regression support.
|
|
595
|
+
2. Add richer HIP metric extraction from newer ROCm profiler paths.
|
|
596
|
+
3. Add full Vulkan compute dispatch.
|
|
597
|
+
4. Add CUDA and Metal backends.
|
|
598
|
+
|
|
599
|
+
## Roadmap
|
|
600
|
+
|
|
601
|
+
### Phase 1
|
|
602
|
+
|
|
603
|
+
- Complete CPU compile-and-run path using Clang/LLVM
|
|
604
|
+
- Add `perf` integration for hardware counters
|
|
605
|
+
- Improve result schema and metric normalization
|
|
606
|
+
|
|
607
|
+
### Phase 2
|
|
608
|
+
|
|
609
|
+
- Add workload matching and baseline/regression support
|
|
610
|
+
- Add Vulkan compute execution and profiling hooks
|
|
611
|
+
- Add CUDA and Metal adapters and deeper HIP profiling
|
|
612
|
+
- Introduce MLflow-backed experiment tracking
|
|
613
|
+
|
|
614
|
+
### Phase 3
|
|
615
|
+
|
|
616
|
+
- Add MLIR-based transformation passes
|
|
617
|
+
- Add Optuna-based autotuning workflows
|
|
618
|
+
- Add distributed experiment scheduling with Ray
|
|
619
|
+
|
|
620
|
+
## Technology Stack
|
|
621
|
+
|
|
622
|
+
See [CompilerSutraPerfTool-Technology-Stack.md](CompilerSutraPerfTool-Technology-Stack.md) for the full stack recommendation that accompanies this repository.
|
|
623
|
+
|
|
624
|
+
## License
|
|
625
|
+
|
|
626
|
+
CompilerSutraPerfTool is licensed under the Apache License 2.0. See [LICENSE](LICENSE).
|
|
627
|
+
|
|
628
|
+
## Author
|
|
629
|
+
|
|
630
|
+
- Abhinav
|
|
631
|
+
- CompilerSutra: `https://compilersutra.com`
|
|
632
|
+
- LinkedIn: `https://www.linkedin.com/in/abhinavcompilerllvm/`
|
|
633
|
+
- Maintainer details: [AUTHORS.md](AUTHORS.md)
|