dftracer-analyzer 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dftracer_analyzer-0.0.2/.devcontainer.json +36 -0
- dftracer_analyzer-0.0.2/.gitattributes +8 -0
- dftracer_analyzer-0.0.2/.gitignore +199 -0
- dftracer_analyzer-0.0.2/.readthedocs.yaml +34 -0
- dftracer_analyzer-0.0.2/CITATION.cff +56 -0
- dftracer_analyzer-0.0.2/CONTRIBUTING.md +93 -0
- dftracer_analyzer-0.0.2/Dockerfile +38 -0
- dftracer_analyzer-0.0.2/LICENSE +21 -0
- dftracer_analyzer-0.0.2/PKG-INFO +163 -0
- dftracer_analyzer-0.0.2/README.md +125 -0
- dftracer_analyzer-0.0.2/docs/Makefile +20 -0
- dftracer_analyzer-0.0.2/docs/conf.py +65 -0
- dftracer_analyzer-0.0.2/docs/configuration.rst +313 -0
- dftracer_analyzer-0.0.2/docs/getting-started.rst +106 -0
- dftracer_analyzer-0.0.2/docs/index.rst +26 -0
- dftracer_analyzer-0.0.2/docs/interactive-analysis.rst +142 -0
- dftracer_analyzer-0.0.2/docs/overview.rst +102 -0
- dftracer_analyzer-0.0.2/docs/requirements.txt +31 -0
- dftracer_analyzer-0.0.2/docs/tools.rst +119 -0
- dftracer_analyzer-0.0.2/meson.build +58 -0
- dftracer_analyzer-0.0.2/meson_options.txt +3 -0
- dftracer_analyzer-0.0.2/pyproject.toml +80 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/__init__.py +125 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/__main__.py +73 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/analysis.py +17 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/analysis_utils.py +203 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/analyzer.py +1053 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/cluster.py +71 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/config.py +404 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/constants.py +306 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/darshan.py +335 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/dftracer.py +704 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/meson.build +22 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/metrics.py +138 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/output.py +282 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/recorder.py +92 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/types.py +234 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/__init__.py +0 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/collection_utils.py +44 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/common_utils.py +41 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/dask_agg.py +116 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/dask_utils.py +5 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/distributed_plugins.py +68 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/env_utils.py +12 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/expr_utils.py +50 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/file_utils.py +17 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/json_encoders.py +15 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/log_utils.py +106 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/meson.build +18 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/notebook_utils.py +14 -0
- dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/warning_utils.py +20 -0
- dftracer_analyzer-0.0.2/scripts/versioning/_version.py.in +31 -0
- dftracer_analyzer-0.0.2/scripts/versioning/scm_version.py +80 -0
- dftracer_analyzer-0.0.2/streamlit_app.py +240 -0
- dftracer_analyzer-0.0.2/subprojects/backward-cpp.wrap +9 -0
- dftracer_analyzer-0.0.2/subprojects/cpp-logger.wrap +10 -0
- dftracer_analyzer-0.0.2/subprojects/packagefiles/cpp-logger.patch +13 -0
- dftracer_analyzer-0.0.2/subprojects/packagefiles/recorder.patch +15 -0
- dftracer_analyzer-0.0.2/subprojects/recorder.wrap +10 -0
- dftracer_analyzer-0.0.2/tools/.gitignore +2 -0
- dftracer_analyzer-0.0.2/tools/meson.build +107 -0
- dftracer_analyzer-0.0.2/tools/recorder2parquet.cpp +1262 -0
- dftracer_analyzer-0.0.2/tools/spack.yaml +11 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "dfanalyzer-devcontainer",
|
|
3
|
+
"build": {
|
|
4
|
+
"dockerfile": "Dockerfile"
|
|
5
|
+
},
|
|
6
|
+
"customizations": {
|
|
7
|
+
"vscode": {
|
|
8
|
+
"extensions": [
|
|
9
|
+
"GitHub.copilot",
|
|
10
|
+
"GitHub.copilot-chat",
|
|
11
|
+
"Tyriar.sort-lines",
|
|
12
|
+
"WakaTime.vscode-wakatime",
|
|
13
|
+
"charliermarsh.ruff",
|
|
14
|
+
"foxundermoon.shell-format",
|
|
15
|
+
"mesonbuild.mesonbuild",
|
|
16
|
+
"ms-azuretools.vscode-containers",
|
|
17
|
+
"ms-python.debugpy",
|
|
18
|
+
"ms-python.python",
|
|
19
|
+
"ms-toolsai.jupyter",
|
|
20
|
+
"ms-toolsai.jupyter-renderers",
|
|
21
|
+
"ms-toolsai.vscode-jupyter-cell-tags",
|
|
22
|
+
"ms-vscode.cpptools-extension-pack",
|
|
23
|
+
"ms-vscode-remote.remote-containers",
|
|
24
|
+
"redhat.vscode-yaml",
|
|
25
|
+
"tamasfe.even-better-toml",
|
|
26
|
+
"vscode-icons-team.vscode-icons"
|
|
27
|
+
]
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"postCreateCommand": "ulimit -c unlimited && git config --global core.autocrlf input",
|
|
31
|
+
"remoteEnv": {
|
|
32
|
+
"OMPI_ALLOW_RUN_AS_ROOT": "1",
|
|
33
|
+
"OMPI_ALLOW_RUN_AS_ROOT_CONFIRM": "1",
|
|
34
|
+
"PYTHONPATH": "${containerWorkspaceFolder}"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py,cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
#uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
#poetry.lock
|
|
109
|
+
|
|
110
|
+
# pdm
|
|
111
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
112
|
+
#pdm.lock
|
|
113
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
114
|
+
# in version control.
|
|
115
|
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
|
116
|
+
.pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
121
|
+
__pypackages__/
|
|
122
|
+
|
|
123
|
+
# Celery stuff
|
|
124
|
+
celerybeat-schedule
|
|
125
|
+
celerybeat.pid
|
|
126
|
+
|
|
127
|
+
# SageMath parsed files
|
|
128
|
+
*.sage.py
|
|
129
|
+
|
|
130
|
+
# Environments
|
|
131
|
+
.env
|
|
132
|
+
.venv*
|
|
133
|
+
env/
|
|
134
|
+
venv/
|
|
135
|
+
ENV/
|
|
136
|
+
env.bak/
|
|
137
|
+
venv.bak/
|
|
138
|
+
|
|
139
|
+
# Spyder project settings
|
|
140
|
+
.spyderproject
|
|
141
|
+
.spyproject
|
|
142
|
+
|
|
143
|
+
# Rope project settings
|
|
144
|
+
.ropeproject
|
|
145
|
+
|
|
146
|
+
# mkdocs documentation
|
|
147
|
+
/site
|
|
148
|
+
|
|
149
|
+
# mypy
|
|
150
|
+
.mypy_cache/
|
|
151
|
+
.dmypy.json
|
|
152
|
+
dmypy.json
|
|
153
|
+
|
|
154
|
+
# Pyre type checker
|
|
155
|
+
.pyre/
|
|
156
|
+
|
|
157
|
+
# pytype static type analyzer
|
|
158
|
+
.pytype/
|
|
159
|
+
|
|
160
|
+
# Cython debug symbols
|
|
161
|
+
cython_debug/
|
|
162
|
+
|
|
163
|
+
# PyCharm
|
|
164
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
165
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
166
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
167
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
168
|
+
#.idea/
|
|
169
|
+
|
|
170
|
+
# Abstra
|
|
171
|
+
# Abstra is an AI-powered process automation framework.
|
|
172
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
173
|
+
# Learn more at https://abstra.io/docs
|
|
174
|
+
.abstra/
|
|
175
|
+
|
|
176
|
+
# Visual Studio Code
|
|
177
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
178
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
179
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
180
|
+
# you could uncomment the following to ignore the enitre vscode folder
|
|
181
|
+
# .vscode/
|
|
182
|
+
|
|
183
|
+
# Ruff stuff:
|
|
184
|
+
.ruff_cache/
|
|
185
|
+
|
|
186
|
+
# PyPI configuration file
|
|
187
|
+
.pypirc
|
|
188
|
+
|
|
189
|
+
# Cursor
|
|
190
|
+
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
|
191
|
+
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
|
192
|
+
# refer to https://docs.cursor.com/context/ignore-files
|
|
193
|
+
.cursorignore
|
|
194
|
+
.cursorindexingignore
|
|
195
|
+
|
|
196
|
+
# DFAnalyzer
|
|
197
|
+
.dfanalyzer
|
|
198
|
+
outputs/
|
|
199
|
+
tmp/
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Read the Docs configuration file for Sphinx projects
|
|
2
|
+
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
|
|
3
|
+
|
|
4
|
+
# Required
|
|
5
|
+
version: 2
|
|
6
|
+
|
|
7
|
+
# Set the OS, Python version and other tools you might need
|
|
8
|
+
build:
|
|
9
|
+
os: ubuntu-22.04
|
|
10
|
+
tools:
|
|
11
|
+
python: "3.12"
|
|
12
|
+
# You can also specify other tool versions:
|
|
13
|
+
# nodejs: "20"
|
|
14
|
+
# rust: "1.70"
|
|
15
|
+
# golang: "1.20"
|
|
16
|
+
|
|
17
|
+
# Build documentation in the "docs/" directory with Sphinx
|
|
18
|
+
sphinx:
|
|
19
|
+
configuration: docs/conf.py
|
|
20
|
+
# You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
|
|
21
|
+
# builder: "dirhtml"
|
|
22
|
+
# Fail on all warnings to avoid broken references
|
|
23
|
+
# fail_on_warning: true
|
|
24
|
+
# Optionally build your docs in additional formats such as PDF and ePub
|
|
25
|
+
# formats:
|
|
26
|
+
# - pdf
|
|
27
|
+
# - epub
|
|
28
|
+
|
|
29
|
+
# Optional but recommended, declare the Python requirements required
|
|
30
|
+
# to build your documentation
|
|
31
|
+
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
|
|
32
|
+
python:
|
|
33
|
+
install:
|
|
34
|
+
- requirements: docs/requirements.txt
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite the software and the paper."
|
|
3
|
+
title: "WisIO: Automated I/O Bottleneck Detection with Multi-Perspective Views for HPC Workflows"
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
abstract: "Analyze, visualize, and understand I/O performance issues in HPC workflows."
|
|
6
|
+
license: MIT
|
|
7
|
+
url: https://github.com/grc-iit/wisio
|
|
8
|
+
repository-code: https://github.com/grc-iit/wisio
|
|
9
|
+
contact:
|
|
10
|
+
- name: Izzet Yildirim
|
|
11
|
+
email: izzetcyildirim@gmail.com
|
|
12
|
+
authors:
|
|
13
|
+
- family-names: Yildirim
|
|
14
|
+
given-names: Izzet
|
|
15
|
+
orcid: https://orcid.org/0000-0003-3513-0764
|
|
16
|
+
- family-names: Devarajan
|
|
17
|
+
given-names: Hariharan
|
|
18
|
+
orcid: https://orcid.org/0000-0001-5625-3494
|
|
19
|
+
- family-names: Kougkas
|
|
20
|
+
given-names: Anthony
|
|
21
|
+
orcid: https://orcid.org/0000-0003-3943-663X
|
|
22
|
+
- family-names: Sun
|
|
23
|
+
given-names: Xian-He
|
|
24
|
+
orcid: https://orcid.org/0000-0002-1093-0792
|
|
25
|
+
- family-names: Mohror
|
|
26
|
+
given-names: Kathryn
|
|
27
|
+
orcid: https://orcid.org/0000-0002-1366-1655
|
|
28
|
+
preferred-citation:
|
|
29
|
+
type: conference-paper
|
|
30
|
+
title: "WisIO: Automated I/O Bottleneck Detection with Multi-Perspective Views for HPC Workflows"
|
|
31
|
+
year: 2025
|
|
32
|
+
authors:
|
|
33
|
+
- family-names: Yildirim
|
|
34
|
+
given-names: Izzet
|
|
35
|
+
orcid: https://orcid.org/0000-0003-3513-0764
|
|
36
|
+
- family-names: Devarajan
|
|
37
|
+
given-names: Hariharan
|
|
38
|
+
orcid: https://orcid.org/0000-0001-5625-3494
|
|
39
|
+
- family-names: Kougkas
|
|
40
|
+
given-names: Anthony
|
|
41
|
+
orcid: https://orcid.org/0000-0003-3943-663X
|
|
42
|
+
- family-names: Sun
|
|
43
|
+
given-names: Xian-He
|
|
44
|
+
orcid: https://orcid.org/0000-0002-1093-0792
|
|
45
|
+
- family-names: Mohror
|
|
46
|
+
given-names: Kathryn
|
|
47
|
+
orcid: https://orcid.org/0000-0002-1366-1655
|
|
48
|
+
conference:
|
|
49
|
+
name: "ICS'25: 2025 International Conference on Supercomputing"
|
|
50
|
+
city: "Salt Lake City"
|
|
51
|
+
region: UT
|
|
52
|
+
country: USA
|
|
53
|
+
date-start: 2025-06-08
|
|
54
|
+
date-end: 2025-06-11
|
|
55
|
+
doi: 10.1145/3721145.3725742
|
|
56
|
+
url: https://doi.org/10.1145/3721145.3725742
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Contributing to DFAnalyzer
|
|
2
|
+
|
|
3
|
+
First off, thank you for considering contributing to DFAnalyzer! We welcome any help to make this tool better.
|
|
4
|
+
|
|
5
|
+
## How Can I Contribute?
|
|
6
|
+
|
|
7
|
+
### Reporting Bugs
|
|
8
|
+
|
|
9
|
+
If you encounter a bug, please report it by opening an issue on our [GitHub Issue Tracker](https://github.com/LLNL/dfanalyzer/issues).
|
|
10
|
+
|
|
11
|
+
When reporting a bug, please include:
|
|
12
|
+
|
|
13
|
+
- A clear and descriptive title.
|
|
14
|
+
- Steps to reproduce the bug.
|
|
15
|
+
- What you expected to happen.
|
|
16
|
+
- What actually happened.
|
|
17
|
+
- Your environment details (e.g., OS, Python version, DFAnalyzer version, relevant libraries).
|
|
18
|
+
|
|
19
|
+
### Suggesting Enhancements
|
|
20
|
+
|
|
21
|
+
If you have an idea for a new feature or an improvement to an existing one, please open an issue on our [GitHub Issue Tracker](https://github.com/LLNL/dfanalyzer/issues) to discuss it.
|
|
22
|
+
|
|
23
|
+
Please include:
|
|
24
|
+
|
|
25
|
+
- A clear and descriptive title.
|
|
26
|
+
- A detailed description of the proposed enhancement.
|
|
27
|
+
- The use case or problem this enhancement would solve.
|
|
28
|
+
- Any potential benefits or drawbacks.
|
|
29
|
+
|
|
30
|
+
### Submitting Pull Requests
|
|
31
|
+
|
|
32
|
+
We love pull requests! If you'd like to contribute code:
|
|
33
|
+
|
|
34
|
+
1. **Fork the Repository:** Start by forking the [main DFAnalyzer repository](https://github.com/LLNL/dfanalyzer).
|
|
35
|
+
2. **Create a Branch:** Create a new branch in your fork for your changes (e.g., `git checkout -b feature/my-new-feature` or `bugfix/issue-123`).
|
|
36
|
+
3. **Make Your Changes:** Implement your feature or bug fix.
|
|
37
|
+
4. **Test Your Changes:** Ensure your changes pass all tests. You can run tests locally (see "Setting Up a Development Environment" below and refer to the test execution steps in `.github/workflows/ci.yml`).
|
|
38
|
+
5. **Format and Lint:** Ensure your code adheres to our coding standards (see below).
|
|
39
|
+
6. **Commit Your Changes:** Use clear and descriptive commit messages.
|
|
40
|
+
7. **Push to Your Fork:** Push your changes to your forked repository.
|
|
41
|
+
8. **Submit a Pull Request:** Open a pull request from your branch to the `main` branch of the DFAnalyzer repository. Provide a clear description of your changes in the PR.
|
|
42
|
+
|
|
43
|
+
## Coding Standards
|
|
44
|
+
|
|
45
|
+
We use [Ruff](https://github.com/astral-sh/ruff) for linting and formatting. Please ensure your contributions adhere to these standards. Key configurations (from `pyproject.toml`):
|
|
46
|
+
|
|
47
|
+
- Line length: 88 characters
|
|
48
|
+
- Quote style: Preserve existing quotes (`quote-style = "preserve"`)
|
|
49
|
+
|
|
50
|
+
You can run Ruff locally to check and format your code:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
ruff check .
|
|
54
|
+
ruff format .
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Setting Up a Development Environment
|
|
58
|
+
|
|
59
|
+
1. **Clone the Repository:**
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
git clone https://github.com/LLNL/dfanalyzer.git
|
|
63
|
+
cd dfanalyzer
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
2. **Install Python Dependencies:**
|
|
67
|
+
Install DFAnalyzer in editable mode along with its optional dependencies (like Darshan support) and dependencies required for running tests.
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
python -m pip install --upgrade pip
|
|
71
|
+
# Install DFAnalyzer in editable mode with optional extras
|
|
72
|
+
pip install -e .[darshan]
|
|
73
|
+
# Install dependencies for tests
|
|
74
|
+
pip install -r tests/requirements.txt
|
|
75
|
+
# Ensure build tools are available
|
|
76
|
+
# pip install meson-python setuptools wheel
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
3. **Install System Dependencies (for C++ components):** DFAnalyzer includes C++ components. To develop these, you'll need system dependencies such as a C++ compiler, CMake, Meson, Ninja, and various development libraries (e.g., `libarrow-dev`, `libhdf5-dev`).
|
|
80
|
+
A comprehensive list and example installation commands for Ubuntu can be found in the "Install system dependencies" step within the `.github/workflows/ci.yml` file.
|
|
81
|
+
|
|
82
|
+
4. **Build C++ Components (if making changes to them):**
|
|
83
|
+
When you install DFAnalyzer in editable mode (`pip install -e .`), the C++ components are typically built. If you make changes to the C++ source files, you may need to recompile them.
|
|
84
|
+
```bash
|
|
85
|
+
# Navigate to the build directory (usually created by the editable install)
|
|
86
|
+
# meson compile -C build
|
|
87
|
+
# Or, if you need to reconfigure and then build:
|
|
88
|
+
# meson setup build --reconfigure -Denable_tests=true -Denable_tools=true
|
|
89
|
+
# meson compile -C build
|
|
90
|
+
```
|
|
91
|
+
The editable pip install should handle most build aspects, but manual recompilation with Meson might be needed for iterative C++ development.
|
|
92
|
+
|
|
93
|
+
We look forward to your contributions!
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
FROM ubuntu:22.04
|
|
2
|
+
|
|
3
|
+
RUN apt-get update && \
|
|
4
|
+
apt-get install -y \
|
|
5
|
+
ca-certificates \
|
|
6
|
+
lsb-release \
|
|
7
|
+
wget && \
|
|
8
|
+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
|
|
9
|
+
apt-get install -y ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
|
|
10
|
+
apt-get update && \
|
|
11
|
+
apt-get install -y \
|
|
12
|
+
build-essential \
|
|
13
|
+
cmake \
|
|
14
|
+
git \
|
|
15
|
+
libarrow-dev \
|
|
16
|
+
libhdf5-dev \
|
|
17
|
+
libncurses-dev \
|
|
18
|
+
libopenmpi-dev \
|
|
19
|
+
libparquet-dev \
|
|
20
|
+
libreadline-dev \
|
|
21
|
+
meson \
|
|
22
|
+
ninja-build \
|
|
23
|
+
nlohmann-json3-dev \
|
|
24
|
+
openmpi-bin \
|
|
25
|
+
openmpi-common \
|
|
26
|
+
python3.10 \
|
|
27
|
+
python3-pip && \
|
|
28
|
+
rm -rf /var/lib/apt/lists/*
|
|
29
|
+
|
|
30
|
+
WORKDIR /dfanalyzer
|
|
31
|
+
|
|
32
|
+
COPY . .
|
|
33
|
+
|
|
34
|
+
RUN pip install --upgrade pip && \
|
|
35
|
+
pip install build meson-python setuptools streamlit wheel && \
|
|
36
|
+
pip install .[darshan] -Csetup-args="-Denable_tools=true"
|
|
37
|
+
|
|
38
|
+
ENTRYPOINT ["dfanalyzer"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Lawrence Livermore National Laboratory
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: dftracer-analyzer
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Analyze, visualize, and understand I/O performance issues in HPC workflows
|
|
5
|
+
Author-Email: Izzet Yildirim <izzetcyildirim@gmail.com>
|
|
6
|
+
Maintainer-Email: Izzet Yildirim <izzetcyildirim@gmail.com>
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Intended Audience :: Science/Research
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering
|
|
17
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Project-URL: documentation, https://dfanalyzer.readthedocs.io/en/latest/
|
|
20
|
+
Project-URL: source, https://github.com/LLNL/dfanalyzer
|
|
21
|
+
Project-URL: tracker, https://github.com/LLNL/dfanalyzer/issues
|
|
22
|
+
Requires-Python: <3.12,>3.8
|
|
23
|
+
Requires-Dist: dask[bag,dataframe,distributed]<2025,>=2024.1.0
|
|
24
|
+
Requires-Dist: dask_jobqueue~=0.8.0
|
|
25
|
+
Requires-Dist: hydra-core~=1.3.0
|
|
26
|
+
Requires-Dist: inflect==7.0
|
|
27
|
+
Requires-Dist: numpy==1.24.3
|
|
28
|
+
Requires-Dist: pandas>=2.0
|
|
29
|
+
Requires-Dist: portion>=2.4.0
|
|
30
|
+
Requires-Dist: pyarrow>=13
|
|
31
|
+
Requires-Dist: rich~=14.1.0
|
|
32
|
+
Requires-Dist: strenum>=0.4
|
|
33
|
+
Requires-Dist: structlog>=25
|
|
34
|
+
Requires-Dist: dftracer-utils>=0.0.5
|
|
35
|
+
Provides-Extra: darshan
|
|
36
|
+
Requires-Dist: darshan>=3.4; extra == "darshan"
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# Data Flow Analyzer
|
|
40
|
+
|
|
41
|
+
<!-- 
|
|
42
|
+

|
|
43
|
+

|
|
44
|
+
 -->
|
|
45
|
+
|
|
46
|
+
## Overview
|
|
47
|
+
|
|
48
|
+
DFAnalyzer is an open-source tool for analyzing performance data from large-scale workflows on distributed systems. It presents a hierarchical, layer-by-layer summary of an application's execution, from high-level application events down to low-level POSIX calls. For each layer, DFAnalyzer quantifies time, operation counts, and data volume, and calculates key performance metrics like bandwidth and operations per second. It also visualizes the overlap between different layers, helping to characterize and understand complex I/O and compute patterns.
|
|
49
|
+
|
|
50
|
+
## Installation
|
|
51
|
+
|
|
52
|
+
To install DFAnalyzer through `pip` (recommended for most users):
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
# Ensure runtime dependencies for optional features (e.g., Darshan, Recorder) are installed.
|
|
56
|
+
# This might involve using your system's package manager or a tool like Spack.
|
|
57
|
+
# Example using Spack to prepare the environment:
|
|
58
|
+
# spack -e tools install
|
|
59
|
+
pip install dfanalyzer
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
To install DFAnalyzer from source (for developers or custom builds):
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
# 1. Install system dependencies:
|
|
66
|
+
# Refer to the "Install system dependencies" step in .github/workflows/ci.yml
|
|
67
|
+
# (e.g., build-essential, cmake, libarrow-dev, libhdf5-dev, ninja-build, etc.).
|
|
68
|
+
# Alternatively, tools like Spack can help manage these:
|
|
69
|
+
# # spack -e tools install
|
|
70
|
+
module load ninja
|
|
71
|
+
|
|
72
|
+
# 2. Install Python build dependencies:
|
|
73
|
+
python -m pip install --upgrade pip meson-python setuptools wheel
|
|
74
|
+
|
|
75
|
+
# 3. Install DFAnalyzer from the root of this repository:
|
|
76
|
+
# The following command includes optional C++ components (tests and tools).
|
|
77
|
+
# The --prefix argument is optional and specifies the installation location.
|
|
78
|
+
pip install -e . \
|
|
79
|
+
-Csetup-args="--prefix=$HOME/.local" \
|
|
80
|
+
-Csetup-args="-Denable_tests=true" \
|
|
81
|
+
-Csetup-args="-Denable_tools=true"
|
|
82
|
+
|
|
83
|
+
# (Optional) Install dependencies for running tests if you plan to contribute or run local tests:
|
|
84
|
+
# pip install -r tests/requirements.txt
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Usage
|
|
88
|
+
|
|
89
|
+
Here's an example of how to run DFAnalyzer using sample data included in the repository:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
# Before running, ensure the sample data is extracted.
|
|
93
|
+
# For example, to extract the 'dftracer-dlio' sample used below:
|
|
94
|
+
# mkdir -p tests/data/extracted
|
|
95
|
+
# tar -xzf tests/data/dftracer-dlio.tar.gz -C tests/data/extracted
|
|
96
|
+
dfanalyzer analyzer/preset=dlio trace_path=tests/data/extracted/dftracer-dlio view_types=[time_range]
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
This command analyzes the traces and prints a high-level summary of the application's execution. Below is a sample of the "Time Period Summary" output:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
Time Period Summary
|
|
103
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
|
|
104
|
+
┃ Metric ┃ Unit ┃ Value ┃
|
|
105
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
|
|
106
|
+
│ Job Time │ seconds │ 56.695 │
|
|
107
|
+
│ Total Count │ count │ 15,901 │
|
|
108
|
+
│ Total Files │ count │ 87 │
|
|
109
|
+
│ Total Nodes │ count │ 0 │
|
|
110
|
+
│ Total Processes │ count │ 23 │
|
|
111
|
+
│ App Count │ count │ 8 │
|
|
112
|
+
│ Training Count │ count │ 40 │
|
|
113
|
+
│ Compute Count │ count │ 200 │
|
|
114
|
+
│ Fetch Data Count │ count │ 160 │
|
|
115
|
+
│ Data Loader Count │ count │ 808 │
|
|
116
|
+
│ Data Loader Fork Count │ count │ 96 │
|
|
117
|
+
│ Reader Count │ count │ 4,008 │
|
|
118
|
+
│ Reader POSIX (Lustre) Count │ count │ 10,432 │
|
|
119
|
+
│ Reader POSIX (Lustre) Size │ MB │ 111833.161 │
|
|
120
|
+
│ Reader POSIX (Lustre) Bandwidth │ MB/s │ 874.982 │
|
|
121
|
+
│ Reader POSIX (Lustre) Avg Transfer Size │ MB │ 10.720 │
|
|
122
|
+
│ Checkpoint Count │ count │ 8 │
|
|
123
|
+
│ Checkpoint POSIX (Lustre) Count │ count │ 45 │
|
|
124
|
+
│ Checkpoint POSIX (Lustre) Size │ MB │ 0.011 │
|
|
125
|
+
│ Checkpoint POSIX (Lustre) Bandwidth │ MB/s │ 0.791 │
|
|
126
|
+
│ Checkpoint POSIX (Lustre) Avg Transfer Size │ MB │ 0.000 │
|
|
127
|
+
│ Other POSIX Count │ count │ 96 │
|
|
128
|
+
└───────────────────────────────────────────────────────────────────────────────┴────────────────┴────────────────────┘
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
DFAnalyzer also provides a detailed breakdown of performance metrics for each layer of the application. Here is a snippet of the "Layer Breakdown" section from the same run, which includes the percentage of time each layer overlaps with its parent layer:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
Layer Breakdown (w/ overlap %)
|
|
135
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓
|
|
136
|
+
┃ Layer ┃ Time (s) ┃ Ops ┃ Ops/sec ┃ Size (MB) ┃ Bandwidth (MB/s) ┃
|
|
137
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩
|
|
138
|
+
│ App │ 441.967 (----) │ 8 (----) │ 0.018 │ - │ - │
|
|
139
|
+
│ Training │ 439.442 (----) │ 40 (----) │ 0.091 │ - │ - │
|
|
140
|
+
│ Compute │ 272.356 (----) │ 200 (----) │ 0.734 │ - │ - │
|
|
141
|
+
│ Fetch Data │ 126.179 ( 16%) │ 160 ( 25%) │ 1.268 │ - │ - │
|
|
142
|
+
│ Data Loader │ 151.471 ( 45%) │ 808 ( 46%) │ 5.334 │ - │ - │
|
|
143
|
+
│ Data Loader Fork │ 2.392 ( 0%) │ 96 ( 0%) │ 40.135 │ - │ - │
|
|
144
|
+
│ Reader │ 299.992 ( 40%) │ 4,008 ( 51%) │ 13.360 │ - │ - │
|
|
145
|
+
│ Reader POSIX (Lustre) │ 127.812 ( 45%) │ 10,432 ( 48%) │ 81.620 │ 111833.161 ( 46%) │ 874.982 │
|
|
146
|
+
│ Checkpoint │ 0.014 ( 0%) │ 8 ( 0%) │ 571.551 │ - │ - │
|
|
147
|
+
│ Checkpoint POSIX (Lustre) │ 0.014 ( 0%) │ 45 ( 0%) │ 3268.686 │ 0.011 ( 0%) │ 0.791 │
|
|
148
|
+
│ Other POSIX │ 2.392 ( 0%) │ 96 ( 0%) │ 40.135 │ 0.000 (----) │ - │
|
|
149
|
+
└─────────────────────────────┴──────────────────┴────────────────┴───────────┴────────────────────┴──────────────────┘
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Further Information
|
|
153
|
+
|
|
154
|
+
For more details, to report issues, or to contribute to DFAnalyzer, please refer to the following resources:
|
|
155
|
+
|
|
156
|
+
- **[Official DFAnalyzer Documentation](https://dfanalyzer.readthedocs.io/)**: For detailed usage, configuration options, and information about analyzers.
|
|
157
|
+
- **[Issue Tracker](https://github.com/LLNL/dfanalyzer/issues)**: To report bugs or suggest new features.
|
|
158
|
+
- **[Contributing Guidelines](./CONTRIBUTING.md)**: For information on how to contribute to the project, including setting up a development environment and coding standards.
|
|
159
|
+
- **[Citation File](./CITATION.cff)**: If you use DFAnalyzer in your research, please cite it using the information in this file.
|
|
160
|
+
|
|
161
|
+
## Acknowledgments
|
|
162
|
+
|
|
163
|
+
This work was performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344. This material is based upon work supported by the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research under the DOE Early Career Research Program (LLNL-CONF-862440). Also, this research is supported in part by the National Science Foundation (NSF) under Grants OAC-2104013, OAC-2313154, and OAC-2411318.
|