scope-profiler 0.3.6__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scope_profiler-0.4.0/LICENSE.txt +21 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/PKG-INFO +5 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/pyproject.toml +23 -2
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/__init__.py +6 -2
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/__main__.py +9 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/benchmark.py +2 -2
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/diff.py +21 -4
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/h5reader.py +348 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/h5writer.py +93 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/html_report.py +33 -4
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/inspection.py +34 -5
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/mcp_server/server.py +3 -2
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/mcp_server/tools.py +22 -6
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/metadata.py +5 -4
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/mpi_region.py +59 -13
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/__init__.py +125 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/_utils.py +452 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/callgraph.py +431 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/duration_timeseries.py +265 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/durations.py +688 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/flame.py +532 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/gantt.py +403 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/heatmap.py +149 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/histogram.py +191 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/imbalance.py +218 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/likwid.py +244 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/scaling.py +540 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/statistics.py +273 -0
- scope_profiler-0.4.0/src/scope_profiler/plotting_scripts/timeline.py +150 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/post_processing.py +46 -12
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/profile_config.py +13 -5
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/profile_manager.py +8 -7
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/region.py +58 -14
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/results.py +36 -4
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/summary.py +210 -27
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/check_parallel_hdf5.py +25 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_aggregation.py +2 -3
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_app.py +1 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_diff.py +32 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_html_report.py +48 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_label.py +5 -3
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_metadata.py +3 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_overhead.py +5 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_post_processing.py +36 -2
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_results_api.py +67 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_tui.py +24 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_h5writer.py +180 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tui.py +35 -16
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler.egg-info/PKG-INFO +5 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler.egg-info/SOURCES.txt +15 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler.egg-info/requires.txt +1 -0
- scope_profiler-0.3.6/src/scope_profiler/plotting_scripts.py +0 -4011
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/README.md +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/setup.cfg +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/c/Makefile +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/c/example.c +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/c/scope_profiler.c +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/c/scope_profiler.h +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/call_stack.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/fortran/Makefile +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/fortran/example.f90 +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/fortran/scope_profiler.f90 +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/gpu_timing.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/h5schema.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/likwid_data.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/line_profile_cli.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/mcp_server/__init__.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/mcp_server/__main__.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/mpi_launch.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/native_trace.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/prof_export.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/region_profiler.py +1 -1
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/speedscope_export.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/__init__.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/check_mpi_launch.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/examples.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/examples_pylikwid.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/pylikwid_readme.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_benchmark.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_buffer_growth.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_c_api.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_fortran_api.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_in_memory_results.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_inspection.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_likwid_data.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_line_profile_cli.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_main_cli.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_mcp_server.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_mcp_tools.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_mixed_language.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_mpi.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_prof_export.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_profile_config_toml.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_profiling_options.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_readme.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_region_source.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_repeated_finalize.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_speedscope_export.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/test_summary_likwid.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/__init__.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_call_stack.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_lazy_config.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_likwid_child_env.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_merge_results.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_mpi_launch.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler/tests/unit/test_payload_collection.py +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler.egg-info/dependency_links.txt +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler.egg-info/entry_points.txt +0 -0
- {scope_profiler-0.3.6 → scope_profiler-0.4.0}/src/scope_profiler.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Max
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scope-profiler
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Profile code regions in python, optionally with LIKWID markers.
|
|
5
5
|
Author: Max
|
|
6
|
+
License-Expression: MIT
|
|
6
7
|
Project-URL: Source, https://github.com/max-models/scope-profiler
|
|
7
8
|
Keywords: python
|
|
8
9
|
Classifier: Development Status :: 3 - Alpha
|
|
@@ -14,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
14
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
15
16
|
Requires-Python: >=3.10
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE.txt
|
|
17
19
|
Requires-Dist: h5py
|
|
18
20
|
Requires-Dist: numpy
|
|
19
21
|
Requires-Dist: tabulate
|
|
@@ -48,6 +50,7 @@ Requires-Dist: pyvis>=0.3.2; extra == "graph"
|
|
|
48
50
|
Provides-Extra: dev
|
|
49
51
|
Requires-Dist: black[jupyter]; extra == "dev"
|
|
50
52
|
Requires-Dist: isort>=9.0.0; extra == "dev"
|
|
53
|
+
Requires-Dist: mypy>=1.11.0; extra == "dev"
|
|
51
54
|
Requires-Dist: ruff; extra == "dev"
|
|
52
55
|
Requires-Dist: scope-profiler[docs,fortran,line-profiler,mcp,mpi,nvtx,pproc,test,tui]; extra == "dev"
|
|
53
56
|
Provides-Extra: docs
|
|
@@ -65,6 +68,7 @@ Requires-Dist: pytest; extra == "test"
|
|
|
65
68
|
Provides-Extra: tui
|
|
66
69
|
Requires-Dist: tabulate; extra == "tui"
|
|
67
70
|
Requires-Dist: textual>=0.86; extra == "tui"
|
|
71
|
+
Dynamic: license-file
|
|
68
72
|
|
|
69
73
|
|
|
70
74
|
|
|
@@ -5,11 +5,12 @@ requires = [ "setuptools", "wheel" ]
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "scope-profiler"
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.4.0"
|
|
9
9
|
description = "Profile code regions in python, optionally with LIKWID markers."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
keywords = [ "python" ]
|
|
12
|
-
license =
|
|
12
|
+
license = "MIT"
|
|
13
|
+
license-files = [ "LICENSE.txt" ]
|
|
13
14
|
authors = [ { name = "Max" } ]
|
|
14
15
|
requires-python = ">=3.10"
|
|
15
16
|
classifiers = [
|
|
@@ -78,6 +79,7 @@ optional-dependencies.graph = [
|
|
|
78
79
|
optional-dependencies.dev = [
|
|
79
80
|
"black[jupyter]",
|
|
80
81
|
"isort >= 9.0.0",
|
|
82
|
+
"mypy >= 1.11.0",
|
|
81
83
|
"ruff",
|
|
82
84
|
"scope-profiler[docs,fortran,line-profiler,mcp,mpi,nvtx,pproc,test,tui]",
|
|
83
85
|
]
|
|
@@ -123,3 +125,22 @@ scope_profiler = [
|
|
|
123
125
|
|
|
124
126
|
[tool.isort]
|
|
125
127
|
profile = "black"
|
|
128
|
+
|
|
129
|
+
[tool.mypy]
|
|
130
|
+
# Incremental typing starts with the core public result/configuration objects.
|
|
131
|
+
# Optional integrations and older dynamically typed code can be migrated
|
|
132
|
+
# independently without weakening this check.
|
|
133
|
+
files = [
|
|
134
|
+
"src/scope_profiler/profile_config.py",
|
|
135
|
+
"src/scope_profiler/region.py",
|
|
136
|
+
"src/scope_profiler/mpi_region.py",
|
|
137
|
+
]
|
|
138
|
+
# Run the checker on the lowest CI-supported modern typing environment. The
|
|
139
|
+
# source remains constrained by requires-python >= 3.10.
|
|
140
|
+
python_version = "3.12"
|
|
141
|
+
check_untyped_defs = true
|
|
142
|
+
disallow_untyped_defs = false
|
|
143
|
+
ignore_missing_imports = true
|
|
144
|
+
follow_imports = "skip"
|
|
145
|
+
no_implicit_optional = true
|
|
146
|
+
warn_unused_ignores = true
|
|
@@ -10,12 +10,12 @@ from scope_profiler.call_stack import (
|
|
|
10
10
|
call_stack_children,
|
|
11
11
|
call_stack_roots,
|
|
12
12
|
)
|
|
13
|
-
from scope_profiler.h5reader import read_h5
|
|
13
|
+
from scope_profiler.h5reader import read_h5, read_h5_summary
|
|
14
14
|
from scope_profiler.likwid_data import LikwidRegionResult
|
|
15
15
|
from scope_profiler.mpi_region import MPIRegion
|
|
16
16
|
from scope_profiler.profile_config import ProfilingOptions
|
|
17
17
|
from scope_profiler.profile_manager import ProfileManager
|
|
18
|
-
from scope_profiler.region import Region
|
|
18
|
+
from scope_profiler.region import EventDataUnavailableError, Region
|
|
19
19
|
from scope_profiler.results import ProfilingResults, merge_results
|
|
20
20
|
|
|
21
21
|
try:
|
|
@@ -32,6 +32,8 @@ _LAZY_ATTRS = {
|
|
|
32
32
|
"plot_duration_timeseries": "plotting_scripts",
|
|
33
33
|
"plot_durations": "plotting_scripts",
|
|
34
34
|
"plot_flame": "plotting_scripts",
|
|
35
|
+
"plot_flame_chart": "plotting_scripts",
|
|
36
|
+
"plot_flame_graph": "plotting_scripts",
|
|
35
37
|
"plot_gantt": "plotting_scripts",
|
|
36
38
|
"plot_speedup": "plotting_scripts",
|
|
37
39
|
"plot_weak_scaling": "plotting_scripts",
|
|
@@ -47,6 +49,7 @@ _LAZY_ATTRS = {
|
|
|
47
49
|
|
|
48
50
|
__all__ = [
|
|
49
51
|
"CallArrays",
|
|
52
|
+
"EventDataUnavailableError",
|
|
50
53
|
"LikwidRegionResult",
|
|
51
54
|
"MPIRegion",
|
|
52
55
|
"NestingError",
|
|
@@ -60,6 +63,7 @@ __all__ = [
|
|
|
60
63
|
"call_stack_roots",
|
|
61
64
|
"merge_results",
|
|
62
65
|
"read_h5",
|
|
66
|
+
"read_h5_summary",
|
|
63
67
|
*sorted(_LAZY_ATTRS),
|
|
64
68
|
]
|
|
65
69
|
|
|
@@ -198,6 +198,11 @@ def _report(argv):
|
|
|
198
198
|
action="store_true",
|
|
199
199
|
help="Omit embedded interactive charts",
|
|
200
200
|
)
|
|
201
|
+
parser.add_argument(
|
|
202
|
+
"--show",
|
|
203
|
+
action="store_true",
|
|
204
|
+
help="Open the generated report in the default browser",
|
|
205
|
+
)
|
|
201
206
|
args = parser.parse_args(argv)
|
|
202
207
|
ranks = None
|
|
203
208
|
if args.ranks:
|
|
@@ -213,6 +218,10 @@ def _report(argv):
|
|
|
213
218
|
include_charts=not args.no_charts,
|
|
214
219
|
)
|
|
215
220
|
print(f"Report written to: {output}")
|
|
221
|
+
if args.show:
|
|
222
|
+
import webbrowser
|
|
223
|
+
|
|
224
|
+
webbrowser.open(output.resolve().as_uri())
|
|
216
225
|
return 0
|
|
217
226
|
|
|
218
227
|
|
|
@@ -21,7 +21,7 @@ try: # Python 3.11+
|
|
|
21
21
|
except ModuleNotFoundError: # Python 3.10
|
|
22
22
|
import tomli as tomllib
|
|
23
23
|
|
|
24
|
-
from scope_profiler.h5reader import
|
|
24
|
+
from scope_profiler.h5reader import read_h5_summary
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class BenchmarkError(Exception):
|
|
@@ -119,7 +119,7 @@ def _profile_once(config: BenchmarkConfig, path: Path) -> dict:
|
|
|
119
119
|
)
|
|
120
120
|
if not path.exists():
|
|
121
121
|
raise BenchmarkError(f"Benchmark run produced no profile: {path}")
|
|
122
|
-
results =
|
|
122
|
+
results = read_h5_summary(path, include_likwid=False, include_line_profile=False)
|
|
123
123
|
total = results.total_time
|
|
124
124
|
if total is None or not math.isfinite(total):
|
|
125
125
|
raise BenchmarkError(f"Profile has no finite total time: {path}")
|
|
@@ -12,11 +12,12 @@ import sys
|
|
|
12
12
|
|
|
13
13
|
from tabulate import tabulate
|
|
14
14
|
|
|
15
|
-
from scope_profiler.h5reader import read_h5
|
|
15
|
+
from scope_profiler.h5reader import read_h5, read_h5_summary
|
|
16
16
|
from scope_profiler.results import ProfilingResults
|
|
17
17
|
from scope_profiler.summary import region_rows
|
|
18
18
|
|
|
19
19
|
METRICS = ("total", "avg", "min", "max", "p50", "p95", "p99", "imbalance", "calls")
|
|
20
|
+
_SUMMARY_METRICS = {"total", "avg", "min", "max", "imbalance", "calls"}
|
|
20
21
|
SORT_KEYS = ("delta", "pct", "name")
|
|
21
22
|
|
|
22
23
|
_METRIC_LABELS = {
|
|
@@ -206,8 +207,16 @@ def diff_files(
|
|
|
206
207
|
Where to write (default: stdout).
|
|
207
208
|
"""
|
|
208
209
|
stream = sys.stdout if stream is None else stream
|
|
209
|
-
|
|
210
|
-
|
|
210
|
+
if metric in _SUMMARY_METRICS:
|
|
211
|
+
results_a = read_h5_summary(
|
|
212
|
+
file_a, include_likwid=False, include_line_profile=False
|
|
213
|
+
)
|
|
214
|
+
results_b = read_h5_summary(
|
|
215
|
+
file_b, include_likwid=False, include_line_profile=False
|
|
216
|
+
)
|
|
217
|
+
else:
|
|
218
|
+
results_a = read_h5(file_a)
|
|
219
|
+
results_b = read_h5(file_b)
|
|
211
220
|
|
|
212
221
|
print("=" * 78, file=stream)
|
|
213
222
|
print(f"a: {results_a.default_title()}", file=stream)
|
|
@@ -274,7 +283,15 @@ def check_files(
|
|
|
274
283
|
) -> int:
|
|
275
284
|
"""Print a CI regression report and return 0 for pass, 1 for failure."""
|
|
276
285
|
stream = sys.stdout if stream is None else stream
|
|
277
|
-
|
|
286
|
+
if metric in _SUMMARY_METRICS:
|
|
287
|
+
results_a = read_h5_summary(
|
|
288
|
+
file_a, include_likwid=False, include_line_profile=False
|
|
289
|
+
)
|
|
290
|
+
results_b = read_h5_summary(
|
|
291
|
+
file_b, include_likwid=False, include_line_profile=False
|
|
292
|
+
)
|
|
293
|
+
else:
|
|
294
|
+
results_a, results_b = read_h5(file_a), read_h5(file_b)
|
|
278
295
|
failures = check_rows(
|
|
279
296
|
results_a,
|
|
280
297
|
results_b,
|
|
@@ -9,18 +9,94 @@ arguments.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
from copy import copy
|
|
12
13
|
from pathlib import Path
|
|
13
14
|
|
|
14
15
|
import h5py
|
|
15
16
|
import numpy as np
|
|
16
17
|
|
|
17
|
-
from scope_profiler.h5schema import migrate_schema, read_schema_version
|
|
18
|
+
from scope_profiler.h5schema import HDF5SchemaError, migrate_schema, read_schema_version
|
|
18
19
|
from scope_profiler.likwid_data import LIKWID_GROUP, LikwidRegionResult
|
|
19
20
|
from scope_profiler.mpi_region import MPIRegion
|
|
20
21
|
from scope_profiler.region import Region
|
|
21
22
|
from scope_profiler.results import ProfilingResults
|
|
22
23
|
|
|
23
24
|
|
|
25
|
+
class SummaryDataUnavailable(ValueError):
|
|
26
|
+
"""Raised when a file predates fixed-size schema-2 summary columns."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
_SUMMARY_DATASET = "summary_statistics"
|
|
30
|
+
_SUMMARY_FIELDS = frozenset(
|
|
31
|
+
{
|
|
32
|
+
"total",
|
|
33
|
+
"minimum",
|
|
34
|
+
"maximum",
|
|
35
|
+
"first",
|
|
36
|
+
"last",
|
|
37
|
+
"start_minimum",
|
|
38
|
+
"end_maximum",
|
|
39
|
+
"gpu_count",
|
|
40
|
+
"gpu_total",
|
|
41
|
+
"mean",
|
|
42
|
+
"m2",
|
|
43
|
+
}
|
|
44
|
+
)
|
|
45
|
+
_SUMMARY_ROW_DATASETS = (
|
|
46
|
+
"region_ids",
|
|
47
|
+
"ranks",
|
|
48
|
+
"event_counts",
|
|
49
|
+
"source_lines",
|
|
50
|
+
"source_files",
|
|
51
|
+
"source_texts",
|
|
52
|
+
"tags",
|
|
53
|
+
_SUMMARY_DATASET,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _validate_summary_index(index, num_region_names: int) -> int:
|
|
58
|
+
"""Validate fixed-size row columns before constructing summary objects."""
|
|
59
|
+
missing = [name for name in _SUMMARY_ROW_DATASETS if name not in index]
|
|
60
|
+
if missing:
|
|
61
|
+
raise HDF5SchemaError(
|
|
62
|
+
"schema-2 summary index is missing dataset(s): " + ", ".join(missing)
|
|
63
|
+
)
|
|
64
|
+
row_count = len(index["region_ids"])
|
|
65
|
+
for name in _SUMMARY_ROW_DATASETS:
|
|
66
|
+
dataset = index[name]
|
|
67
|
+
if dataset.ndim != 1 or len(dataset) != row_count:
|
|
68
|
+
raise HDF5SchemaError(
|
|
69
|
+
f"rank_region_index/{name} must be one-dimensional with "
|
|
70
|
+
f"{row_count} rows, got shape {dataset.shape}"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
fields = index[_SUMMARY_DATASET].dtype.names
|
|
74
|
+
missing_fields = sorted(_SUMMARY_FIELDS - set(fields or ()))
|
|
75
|
+
if missing_fields:
|
|
76
|
+
raise HDF5SchemaError(
|
|
77
|
+
"summary_statistics is missing field(s): " + ", ".join(missing_fields)
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
region_ids = index["region_ids"][()]
|
|
81
|
+
if len(region_ids) and (
|
|
82
|
+
np.any(region_ids < 0) or np.any(region_ids >= num_region_names)
|
|
83
|
+
):
|
|
84
|
+
raise HDF5SchemaError("summary index contains an out-of-range region id")
|
|
85
|
+
ranks = index["ranks"][()]
|
|
86
|
+
pairs = list(zip(region_ids.tolist(), ranks.tolist()))
|
|
87
|
+
if len(set(pairs)) != len(pairs):
|
|
88
|
+
raise HDF5SchemaError("summary index contains a duplicate rank/region row")
|
|
89
|
+
return row_count
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _selection_set(values, *, coerce):
|
|
93
|
+
if values is None:
|
|
94
|
+
return None
|
|
95
|
+
if isinstance(values, (str, int)):
|
|
96
|
+
values = [values]
|
|
97
|
+
return {coerce(value) for value in values}
|
|
98
|
+
|
|
99
|
+
|
|
24
100
|
def _decode_attribute(value):
|
|
25
101
|
"""Convert an HDF5 attribute into a plain Python value.
|
|
26
102
|
|
|
@@ -349,6 +425,202 @@ def load_h5(file_path: str | Path, verbose: bool = False) -> dict:
|
|
|
349
425
|
}
|
|
350
426
|
|
|
351
427
|
|
|
428
|
+
def load_h5_summary(
|
|
429
|
+
file_path: str | Path,
|
|
430
|
+
verbose: bool = False,
|
|
431
|
+
*,
|
|
432
|
+
include_likwid: bool = True,
|
|
433
|
+
include_line_profile: bool = True,
|
|
434
|
+
regions: str | list[str] | tuple[str, ...] | None = None,
|
|
435
|
+
ranks: int | list[int] | tuple[int, ...] | None = None,
|
|
436
|
+
) -> dict:
|
|
437
|
+
"""Parse fixed-size profile statistics without reading event datasets.
|
|
438
|
+
|
|
439
|
+
Summary columns were added compatibly within schema 2. Files that predate
|
|
440
|
+
them raise :class:`SummaryDataUnavailable`; callers that require broad
|
|
441
|
+
compatibility should use :func:`read_h5_summary`, which falls back to the
|
|
442
|
+
normal eager reader by default.
|
|
443
|
+
"""
|
|
444
|
+
file_path = Path(file_path)
|
|
445
|
+
selected_names = _selection_set(regions, coerce=str)
|
|
446
|
+
selected_ranks = _selection_set(ranks, coerce=int)
|
|
447
|
+
if not file_path.exists():
|
|
448
|
+
raise FileNotFoundError(f"HDF5 file not found: {file_path}")
|
|
449
|
+
|
|
450
|
+
metadata = {}
|
|
451
|
+
likwid = {}
|
|
452
|
+
line_profile = {}
|
|
453
|
+
exclusive_totals = {}
|
|
454
|
+
with h5py.File(file_path, "r") as h5file:
|
|
455
|
+
schema_version = read_schema_version(h5file)
|
|
456
|
+
migrate_schema(h5file, schema_version)
|
|
457
|
+
if schema_version != 2:
|
|
458
|
+
raise SummaryDataUnavailable(
|
|
459
|
+
"summary-only reads require a schema-2 profiling file"
|
|
460
|
+
)
|
|
461
|
+
if "metadata" in h5file:
|
|
462
|
+
metadata = {
|
|
463
|
+
key: _decode_attribute(value)
|
|
464
|
+
for key, value in h5file["metadata"].attrs.items()
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
layout = h5file.attrs.get("storage_layout", "columnar")
|
|
468
|
+
if layout == "aggregate":
|
|
469
|
+
per_region, region_names, exclusive_totals = _read_aggregate_regions(h5file)
|
|
470
|
+
region_names = [
|
|
471
|
+
name
|
|
472
|
+
for name in region_names
|
|
473
|
+
if selected_names is None or name in selected_names
|
|
474
|
+
]
|
|
475
|
+
per_region = {
|
|
476
|
+
name: {
|
|
477
|
+
rank: region
|
|
478
|
+
for rank, region in per_region[name].items()
|
|
479
|
+
if selected_ranks is None or rank in selected_ranks
|
|
480
|
+
}
|
|
481
|
+
for name in region_names
|
|
482
|
+
}
|
|
483
|
+
region_names = [name for name in region_names if per_region[name]]
|
|
484
|
+
exclusive_totals = {
|
|
485
|
+
name: {
|
|
486
|
+
rank: total
|
|
487
|
+
for rank, total in rank_totals.items()
|
|
488
|
+
if selected_ranks is None or rank in selected_ranks
|
|
489
|
+
}
|
|
490
|
+
for name, rank_totals in exclusive_totals.items()
|
|
491
|
+
if name in per_region
|
|
492
|
+
}
|
|
493
|
+
else:
|
|
494
|
+
index = h5file["rank_region_index"]
|
|
495
|
+
if _SUMMARY_DATASET not in index:
|
|
496
|
+
raise SummaryDataUnavailable(
|
|
497
|
+
"profiling file has no fixed-size summary statistics"
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
all_region_names = [
|
|
501
|
+
_decode_attribute(value) for value in h5file["region_table/names"][()]
|
|
502
|
+
]
|
|
503
|
+
row_count = _validate_summary_index(index, len(all_region_names))
|
|
504
|
+
all_region_ids = index["region_ids"][()]
|
|
505
|
+
all_ranks = index["ranks"][()]
|
|
506
|
+
keep = np.ones(row_count, dtype=bool)
|
|
507
|
+
if selected_names is not None:
|
|
508
|
+
keep &= np.fromiter(
|
|
509
|
+
(
|
|
510
|
+
all_region_names[int(region_id)] in selected_names
|
|
511
|
+
for region_id in all_region_ids
|
|
512
|
+
),
|
|
513
|
+
dtype=bool,
|
|
514
|
+
count=row_count,
|
|
515
|
+
)
|
|
516
|
+
if selected_ranks is not None:
|
|
517
|
+
keep &= np.isin(all_ranks, list(selected_ranks))
|
|
518
|
+
selected_rows = np.flatnonzero(keep)
|
|
519
|
+
|
|
520
|
+
def read_selected(name):
|
|
521
|
+
dataset = index[name]
|
|
522
|
+
if len(selected_rows):
|
|
523
|
+
return dataset[selected_rows]
|
|
524
|
+
return np.empty(0, dtype=dataset.dtype)
|
|
525
|
+
|
|
526
|
+
region_ids = all_region_ids[selected_rows]
|
|
527
|
+
row_ranks = all_ranks[selected_rows]
|
|
528
|
+
counts = read_selected("event_counts")
|
|
529
|
+
source_lines = read_selected("source_lines")
|
|
530
|
+
source_files = read_selected("source_files")
|
|
531
|
+
source_texts = read_selected("source_texts")
|
|
532
|
+
tag_blobs = read_selected("tags")
|
|
533
|
+
summary_statistics = read_selected(_SUMMARY_DATASET)
|
|
534
|
+
exclusive_column = (
|
|
535
|
+
read_selected("exclusive_totals")
|
|
536
|
+
if "exclusive_totals" in index
|
|
537
|
+
else None
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
used_region_ids = {int(region_id) for region_id in region_ids}
|
|
541
|
+
region_names = [
|
|
542
|
+
name
|
|
543
|
+
for region_id, name in enumerate(all_region_names)
|
|
544
|
+
if region_id in used_region_ids
|
|
545
|
+
]
|
|
546
|
+
per_region = {name: {} for name in region_names}
|
|
547
|
+
for row, (region_id, rank, count) in enumerate(
|
|
548
|
+
zip(region_ids, row_ranks, counts)
|
|
549
|
+
):
|
|
550
|
+
name = all_region_names[int(region_id)]
|
|
551
|
+
rank = int(rank)
|
|
552
|
+
aggregate = {
|
|
553
|
+
"count": int(count),
|
|
554
|
+
**{
|
|
555
|
+
field: (
|
|
556
|
+
float(summary_statistics[field][row])
|
|
557
|
+
if field in {"mean", "m2"}
|
|
558
|
+
else int(summary_statistics[field][row])
|
|
559
|
+
)
|
|
560
|
+
for field in summary_statistics.dtype.names
|
|
561
|
+
},
|
|
562
|
+
}
|
|
563
|
+
exclusive = (
|
|
564
|
+
int(exclusive_column[row])
|
|
565
|
+
if exclusive_column is not None and exclusive_column[row] >= 0
|
|
566
|
+
else aggregate["total"]
|
|
567
|
+
)
|
|
568
|
+
aggregate["exclusive"] = exclusive
|
|
569
|
+
if exclusive_column is not None and exclusive_column[row] >= 0:
|
|
570
|
+
exclusive_totals.setdefault(name, {})[rank] = exclusive
|
|
571
|
+
source_line = int(source_lines[row])
|
|
572
|
+
per_region[name][rank] = Region(
|
|
573
|
+
np.empty(0, dtype=np.int64),
|
|
574
|
+
np.empty(0, dtype=np.int64),
|
|
575
|
+
aggregate=aggregate,
|
|
576
|
+
source_file=_decode_attribute(source_files[row]) or None,
|
|
577
|
+
source_lineno=source_line if source_line >= 0 else None,
|
|
578
|
+
source_text=_decode_attribute(source_texts[row]) or None,
|
|
579
|
+
tags=tuple(json.loads(_decode_attribute(tag_blobs[row]) or "[]")),
|
|
580
|
+
event_data_available=False,
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
rank_group_names = sorted(
|
|
584
|
+
(name for name in h5file if name.startswith("rank") and name[4:].isdigit()),
|
|
585
|
+
key=lambda name: int(name[4:]),
|
|
586
|
+
)
|
|
587
|
+
for rank_group_name in rank_group_names:
|
|
588
|
+
rank_group = h5file[rank_group_name]
|
|
589
|
+
rank = int(rank_group_name[4:])
|
|
590
|
+
if selected_ranks is not None and rank not in selected_ranks:
|
|
591
|
+
continue
|
|
592
|
+
if verbose:
|
|
593
|
+
print(f"rank_group_name = {rank_group_name!r}")
|
|
594
|
+
if include_likwid and LIKWID_GROUP in rank_group:
|
|
595
|
+
likwid[rank] = _read_likwid_group(rank_group[LIKWID_GROUP])
|
|
596
|
+
if include_line_profile and "line_profile" in rank_group:
|
|
597
|
+
line_profile[rank] = _read_line_profile_group(
|
|
598
|
+
rank_group["line_profile"]
|
|
599
|
+
)
|
|
600
|
+
|
|
601
|
+
recorded_ranks = h5file["rank_region_index/ranks"][()]
|
|
602
|
+
inferred_ranks = int(np.max(recorded_ranks)) + 1 if len(recorded_ranks) else 0
|
|
603
|
+
num_ranks = (
|
|
604
|
+
max(int(metadata.get("mpi_size", 1)), inferred_ranks)
|
|
605
|
+
if len(recorded_ranks)
|
|
606
|
+
else 0
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
regions = {
|
|
610
|
+
name: MPIRegion(name=name, regions=per_region[name]) for name in region_names
|
|
611
|
+
}
|
|
612
|
+
return {
|
|
613
|
+
"regions": regions,
|
|
614
|
+
"metadata": metadata,
|
|
615
|
+
"num_ranks": num_ranks,
|
|
616
|
+
"likwid": likwid,
|
|
617
|
+
"line_profile": line_profile,
|
|
618
|
+
"file_path": file_path,
|
|
619
|
+
"exclusive_totals": exclusive_totals,
|
|
620
|
+
"event_data_available": False,
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
|
|
352
624
|
def read_h5(file_path: str | Path, verbose: bool = False) -> ProfilingResults:
|
|
353
625
|
"""
|
|
354
626
|
Load a merged profiling file for post-processing.
|
|
@@ -384,3 +656,78 @@ def read_h5(file_path: str | Path, verbose: bool = False) -> ProfilingResults:
|
|
|
384
656
|
If the specified HDF5 file does not exist.
|
|
385
657
|
"""
|
|
386
658
|
return ProfilingResults.from_h5(file_path, verbose=verbose)
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def read_h5_summary(
|
|
662
|
+
file_path: str | Path,
|
|
663
|
+
verbose: bool = False,
|
|
664
|
+
*,
|
|
665
|
+
fallback: bool = True,
|
|
666
|
+
include_likwid: bool = True,
|
|
667
|
+
include_line_profile: bool = True,
|
|
668
|
+
regions: str | list[str] | tuple[str, ...] | None = None,
|
|
669
|
+
ranks: int | list[int] | tuple[int, ...] | None = None,
|
|
670
|
+
) -> ProfilingResults:
|
|
671
|
+
"""Load fixed-size statistics, optionally falling back for older files.
|
|
672
|
+
|
|
673
|
+
The returned object supports metadata and scalar region statistics. Event,
|
|
674
|
+
timeline, call-stack and percentile APIs intentionally have no per-call
|
|
675
|
+
data; use :func:`read_h5` when those are required.
|
|
676
|
+
"""
|
|
677
|
+
try:
|
|
678
|
+
return ProfilingResults(
|
|
679
|
+
**load_h5_summary(
|
|
680
|
+
file_path,
|
|
681
|
+
verbose=verbose,
|
|
682
|
+
include_likwid=include_likwid,
|
|
683
|
+
include_line_profile=include_line_profile,
|
|
684
|
+
regions=regions,
|
|
685
|
+
ranks=ranks,
|
|
686
|
+
)
|
|
687
|
+
)
|
|
688
|
+
except SummaryDataUnavailable:
|
|
689
|
+
if not fallback:
|
|
690
|
+
raise
|
|
691
|
+
results = read_h5(file_path, verbose=verbose)
|
|
692
|
+
selected_names = _selection_set(regions, coerce=str)
|
|
693
|
+
selected_ranks = _selection_set(ranks, coerce=int)
|
|
694
|
+
if selected_names is not None or selected_ranks is not None:
|
|
695
|
+
results = copy(results)
|
|
696
|
+
results._region_dict = {
|
|
697
|
+
name: MPIRegion(
|
|
698
|
+
name=name,
|
|
699
|
+
regions={
|
|
700
|
+
rank: region
|
|
701
|
+
for rank, region in mpi_region.regions.items()
|
|
702
|
+
if selected_ranks is None or rank in selected_ranks
|
|
703
|
+
},
|
|
704
|
+
)
|
|
705
|
+
for name, mpi_region in results._region_dict.items()
|
|
706
|
+
if selected_names is None or name in selected_names
|
|
707
|
+
}
|
|
708
|
+
results._region_dict = {
|
|
709
|
+
name: region
|
|
710
|
+
for name, region in results._region_dict.items()
|
|
711
|
+
if region.regions
|
|
712
|
+
}
|
|
713
|
+
if not include_likwid or selected_ranks is not None:
|
|
714
|
+
results._likwid = (
|
|
715
|
+
{
|
|
716
|
+
rank: data
|
|
717
|
+
for rank, data in results._likwid.items()
|
|
718
|
+
if selected_ranks is None or rank in selected_ranks
|
|
719
|
+
}
|
|
720
|
+
if include_likwid
|
|
721
|
+
else {}
|
|
722
|
+
)
|
|
723
|
+
if not include_line_profile or selected_ranks is not None:
|
|
724
|
+
results._line_profile = (
|
|
725
|
+
{
|
|
726
|
+
rank: data
|
|
727
|
+
for rank, data in results._line_profile.items()
|
|
728
|
+
if selected_ranks is None or rank in selected_ranks
|
|
729
|
+
}
|
|
730
|
+
if include_line_profile
|
|
731
|
+
else {}
|
|
732
|
+
)
|
|
733
|
+
return results
|