dftracer-analyzer 0.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. dftracer_analyzer-0.0.2/.devcontainer.json +36 -0
  2. dftracer_analyzer-0.0.2/.gitattributes +8 -0
  3. dftracer_analyzer-0.0.2/.gitignore +199 -0
  4. dftracer_analyzer-0.0.2/.readthedocs.yaml +34 -0
  5. dftracer_analyzer-0.0.2/CITATION.cff +56 -0
  6. dftracer_analyzer-0.0.2/CONTRIBUTING.md +93 -0
  7. dftracer_analyzer-0.0.2/Dockerfile +38 -0
  8. dftracer_analyzer-0.0.2/LICENSE +21 -0
  9. dftracer_analyzer-0.0.2/PKG-INFO +163 -0
  10. dftracer_analyzer-0.0.2/README.md +125 -0
  11. dftracer_analyzer-0.0.2/docs/Makefile +20 -0
  12. dftracer_analyzer-0.0.2/docs/conf.py +65 -0
  13. dftracer_analyzer-0.0.2/docs/configuration.rst +313 -0
  14. dftracer_analyzer-0.0.2/docs/getting-started.rst +106 -0
  15. dftracer_analyzer-0.0.2/docs/index.rst +26 -0
  16. dftracer_analyzer-0.0.2/docs/interactive-analysis.rst +142 -0
  17. dftracer_analyzer-0.0.2/docs/overview.rst +102 -0
  18. dftracer_analyzer-0.0.2/docs/requirements.txt +31 -0
  19. dftracer_analyzer-0.0.2/docs/tools.rst +119 -0
  20. dftracer_analyzer-0.0.2/meson.build +58 -0
  21. dftracer_analyzer-0.0.2/meson_options.txt +3 -0
  22. dftracer_analyzer-0.0.2/pyproject.toml +80 -0
  23. dftracer_analyzer-0.0.2/python/dftracer/analyzer/__init__.py +125 -0
  24. dftracer_analyzer-0.0.2/python/dftracer/analyzer/__main__.py +73 -0
  25. dftracer_analyzer-0.0.2/python/dftracer/analyzer/analysis.py +17 -0
  26. dftracer_analyzer-0.0.2/python/dftracer/analyzer/analysis_utils.py +203 -0
  27. dftracer_analyzer-0.0.2/python/dftracer/analyzer/analyzer.py +1053 -0
  28. dftracer_analyzer-0.0.2/python/dftracer/analyzer/cluster.py +71 -0
  29. dftracer_analyzer-0.0.2/python/dftracer/analyzer/config.py +404 -0
  30. dftracer_analyzer-0.0.2/python/dftracer/analyzer/constants.py +306 -0
  31. dftracer_analyzer-0.0.2/python/dftracer/analyzer/darshan.py +335 -0
  32. dftracer_analyzer-0.0.2/python/dftracer/analyzer/dftracer.py +704 -0
  33. dftracer_analyzer-0.0.2/python/dftracer/analyzer/meson.build +22 -0
  34. dftracer_analyzer-0.0.2/python/dftracer/analyzer/metrics.py +138 -0
  35. dftracer_analyzer-0.0.2/python/dftracer/analyzer/output.py +282 -0
  36. dftracer_analyzer-0.0.2/python/dftracer/analyzer/recorder.py +92 -0
  37. dftracer_analyzer-0.0.2/python/dftracer/analyzer/types.py +234 -0
  38. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/__init__.py +0 -0
  39. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/collection_utils.py +44 -0
  40. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/common_utils.py +41 -0
  41. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/dask_agg.py +116 -0
  42. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/dask_utils.py +5 -0
  43. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/distributed_plugins.py +68 -0
  44. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/env_utils.py +12 -0
  45. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/expr_utils.py +50 -0
  46. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/file_utils.py +17 -0
  47. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/json_encoders.py +15 -0
  48. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/log_utils.py +106 -0
  49. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/meson.build +18 -0
  50. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/notebook_utils.py +14 -0
  51. dftracer_analyzer-0.0.2/python/dftracer/analyzer/utils/warning_utils.py +20 -0
  52. dftracer_analyzer-0.0.2/scripts/versioning/_version.py.in +31 -0
  53. dftracer_analyzer-0.0.2/scripts/versioning/scm_version.py +80 -0
  54. dftracer_analyzer-0.0.2/streamlit_app.py +240 -0
  55. dftracer_analyzer-0.0.2/subprojects/backward-cpp.wrap +9 -0
  56. dftracer_analyzer-0.0.2/subprojects/cpp-logger.wrap +10 -0
  57. dftracer_analyzer-0.0.2/subprojects/packagefiles/cpp-logger.patch +13 -0
  58. dftracer_analyzer-0.0.2/subprojects/packagefiles/recorder.patch +15 -0
  59. dftracer_analyzer-0.0.2/subprojects/recorder.wrap +10 -0
  60. dftracer_analyzer-0.0.2/tools/.gitignore +2 -0
  61. dftracer_analyzer-0.0.2/tools/meson.build +107 -0
  62. dftracer_analyzer-0.0.2/tools/recorder2parquet.cpp +1262 -0
  63. dftracer_analyzer-0.0.2/tools/spack.yaml +11 -0
@@ -0,0 +1,36 @@
1
+ {
2
+ "name": "dfanalyzer-devcontainer",
3
+ "build": {
4
+ "dockerfile": "Dockerfile"
5
+ },
6
+ "customizations": {
7
+ "vscode": {
8
+ "extensions": [
9
+ "GitHub.copilot",
10
+ "GitHub.copilot-chat",
11
+ "Tyriar.sort-lines",
12
+ "WakaTime.vscode-wakatime",
13
+ "charliermarsh.ruff",
14
+ "foxundermoon.shell-format",
15
+ "mesonbuild.mesonbuild",
16
+ "ms-azuretools.vscode-containers",
17
+ "ms-python.debugpy",
18
+ "ms-python.python",
19
+ "ms-toolsai.jupyter",
20
+ "ms-toolsai.jupyter-renderers",
21
+ "ms-toolsai.vscode-jupyter-cell-tags",
22
+ "ms-vscode.cpptools-extension-pack",
23
+ "ms-vscode-remote.remote-containers",
24
+ "redhat.vscode-yaml",
25
+ "tamasfe.even-better-toml",
26
+ "vscode-icons-team.vscode-icons"
27
+ ]
28
+ }
29
+ },
30
+ "postCreateCommand": "ulimit -c unlimited && git config --global core.autocrlf input",
31
+ "remoteEnv": {
32
+ "OMPI_ALLOW_RUN_AS_ROOT": "1",
33
+ "OMPI_ALLOW_RUN_AS_ROOT_CONFIRM": "1",
34
+ "PYTHONPATH": "${containerWorkspaceFolder}"
35
+ }
36
+ }
@@ -0,0 +1,8 @@
1
+ .git/* export-ignore
2
+ .github/* export-ignore
3
+ .vscode/* export-ignore
4
+
5
+ notebooks/* export-ignore
6
+ tests/* export-ignore
7
+
8
+ .wakatime-project export-ignore
@@ -0,0 +1,199 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ #uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+
110
+ # pdm
111
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112
+ #pdm.lock
113
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114
+ # in version control.
115
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
116
+ .pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
121
+ __pypackages__/
122
+
123
+ # Celery stuff
124
+ celerybeat-schedule
125
+ celerybeat.pid
126
+
127
+ # SageMath parsed files
128
+ *.sage.py
129
+
130
+ # Environments
131
+ .env
132
+ .venv*
133
+ env/
134
+ venv/
135
+ ENV/
136
+ env.bak/
137
+ venv.bak/
138
+
139
+ # Spyder project settings
140
+ .spyderproject
141
+ .spyproject
142
+
143
+ # Rope project settings
144
+ .ropeproject
145
+
146
+ # mkdocs documentation
147
+ /site
148
+
149
+ # mypy
150
+ .mypy_cache/
151
+ .dmypy.json
152
+ dmypy.json
153
+
154
+ # Pyre type checker
155
+ .pyre/
156
+
157
+ # pytype static type analyzer
158
+ .pytype/
159
+
160
+ # Cython debug symbols
161
+ cython_debug/
162
+
163
+ # PyCharm
164
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
165
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
166
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
167
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
168
+ #.idea/
169
+
170
+ # Abstra
171
+ # Abstra is an AI-powered process automation framework.
172
+ # Ignore directories containing user credentials, local state, and settings.
173
+ # Learn more at https://abstra.io/docs
174
+ .abstra/
175
+
176
+ # Visual Studio Code
177
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
178
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
179
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
180
+ # you could uncomment the following to ignore the enitre vscode folder
181
+ # .vscode/
182
+
183
+ # Ruff stuff:
184
+ .ruff_cache/
185
+
186
+ # PyPI configuration file
187
+ .pypirc
188
+
189
+ # Cursor
190
+ # Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
191
+ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
192
+ # refer to https://docs.cursor.com/context/ignore-files
193
+ .cursorignore
194
+ .cursorindexingignore
195
+
196
+ # DFAnalyzer
197
+ .dfanalyzer
198
+ outputs/
199
+ tmp/
@@ -0,0 +1,34 @@
1
+ # Read the Docs configuration file for Sphinx projects
2
+ # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
3
+
4
+ # Required
5
+ version: 2
6
+
7
+ # Set the OS, Python version and other tools you might need
8
+ build:
9
+ os: ubuntu-22.04
10
+ tools:
11
+ python: "3.12"
12
+ # You can also specify other tool versions:
13
+ # nodejs: "20"
14
+ # rust: "1.70"
15
+ # golang: "1.20"
16
+
17
+ # Build documentation in the "docs/" directory with Sphinx
18
+ sphinx:
19
+ configuration: docs/conf.py
20
+ # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
21
+ # builder: "dirhtml"
22
+ # Fail on all warnings to avoid broken references
23
+ # fail_on_warning: true
24
+ # Optionally build your docs in additional formats such as PDF and ePub
25
+ # formats:
26
+ # - pdf
27
+ # - epub
28
+
29
+ # Optional but recommended, declare the Python requirements required
30
+ # to build your documentation
31
+ # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
32
+ python:
33
+ install:
34
+ - requirements: docs/requirements.txt
@@ -0,0 +1,56 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite the software and the paper."
3
+ title: "WisIO: Automated I/O Bottleneck Detection with Multi-Perspective Views for HPC Workflows"
4
+ version: 0.1.0
5
+ abstract: "Analyze, visualize, and understand I/O performance issues in HPC workflows."
6
+ license: MIT
7
+ url: https://github.com/grc-iit/wisio
8
+ repository-code: https://github.com/grc-iit/wisio
9
+ contact:
10
+ - name: Izzet Yildirim
11
+ email: izzetcyildirim@gmail.com
12
+ authors:
13
+ - family-names: Yildirim
14
+ given-names: Izzet
15
+ orcid: https://orcid.org/0000-0003-3513-0764
16
+ - family-names: Devarajan
17
+ given-names: Hariharan
18
+ orcid: https://orcid.org/0000-0001-5625-3494
19
+ - family-names: Kougkas
20
+ given-names: Anthony
21
+ orcid: https://orcid.org/0000-0003-3943-663X
22
+ - family-names: Sun
23
+ given-names: Xian-He
24
+ orcid: https://orcid.org/0000-0002-1093-0792
25
+ - family-names: Mohror
26
+ given-names: Kathryn
27
+ orcid: https://orcid.org/0000-0002-1366-1655
28
+ preferred-citation:
29
+ type: conference-paper
30
+ title: "WisIO: Automated I/O Bottleneck Detection with Multi-Perspective Views for HPC Workflows"
31
+ year: 2025
32
+ authors:
33
+ - family-names: Yildirim
34
+ given-names: Izzet
35
+ orcid: https://orcid.org/0000-0003-3513-0764
36
+ - family-names: Devarajan
37
+ given-names: Hariharan
38
+ orcid: https://orcid.org/0000-0001-5625-3494
39
+ - family-names: Kougkas
40
+ given-names: Anthony
41
+ orcid: https://orcid.org/0000-0003-3943-663X
42
+ - family-names: Sun
43
+ given-names: Xian-He
44
+ orcid: https://orcid.org/0000-0002-1093-0792
45
+ - family-names: Mohror
46
+ given-names: Kathryn
47
+ orcid: https://orcid.org/0000-0002-1366-1655
48
+ conference:
49
+ name: "ICS'25: 2025 International Conference on Supercomputing"
50
+ city: "Salt Lake City"
51
+ region: UT
52
+ country: USA
53
+ date-start: 2025-06-08
54
+ date-end: 2025-06-11
55
+ doi: 10.1145/3721145.3725742
56
+ url: https://doi.org/10.1145/3721145.3725742
@@ -0,0 +1,93 @@
1
+ # Contributing to DFAnalyzer
2
+
3
+ First off, thank you for considering contributing to DFAnalyzer! We welcome any help to make this tool better.
4
+
5
+ ## How Can I Contribute?
6
+
7
+ ### Reporting Bugs
8
+
9
+ If you encounter a bug, please report it by opening an issue on our [GitHub Issue Tracker](https://github.com/LLNL/dfanalyzer/issues).
10
+
11
+ When reporting a bug, please include:
12
+
13
+ - A clear and descriptive title.
14
+ - Steps to reproduce the bug.
15
+ - What you expected to happen.
16
+ - What actually happened.
17
+ - Your environment details (e.g., OS, Python version, DFAnalyzer version, relevant libraries).
18
+
19
+ ### Suggesting Enhancements
20
+
21
+ If you have an idea for a new feature or an improvement to an existing one, please open an issue on our [GitHub Issue Tracker](https://github.com/LLNL/dfanalyzer/issues) to discuss it.
22
+
23
+ Please include:
24
+
25
+ - A clear and descriptive title.
26
+ - A detailed description of the proposed enhancement.
27
+ - The use case or problem this enhancement would solve.
28
+ - Any potential benefits or drawbacks.
29
+
30
+ ### Submitting Pull Requests
31
+
32
+ We love pull requests! If you'd like to contribute code:
33
+
34
+ 1. **Fork the Repository:** Start by forking the [main DFAnalyzer repository](https://github.com/LLNL/dfanalyzer).
35
+ 2. **Create a Branch:** Create a new branch in your fork for your changes (e.g., `git checkout -b feature/my-new-feature` or `bugfix/issue-123`).
36
+ 3. **Make Your Changes:** Implement your feature or bug fix.
37
+ 4. **Test Your Changes:** Ensure your changes pass all tests. You can run tests locally (see "Setting Up a Development Environment" below and refer to the test execution steps in `.github/workflows/ci.yml`).
38
+ 5. **Format and Lint:** Ensure your code adheres to our coding standards (see below).
39
+ 6. **Commit Your Changes:** Use clear and descriptive commit messages.
40
+ 7. **Push to Your Fork:** Push your changes to your forked repository.
41
+ 8. **Submit a Pull Request:** Open a pull request from your branch to the `main` branch of the DFAnalyzer repository. Provide a clear description of your changes in the PR.
42
+
43
+ ## Coding Standards
44
+
45
+ We use [Ruff](https://github.com/astral-sh/ruff) for linting and formatting. Please ensure your contributions adhere to these standards. Key configurations (from `pyproject.toml`):
46
+
47
+ - Line length: 88 characters
48
+ - Quote style: Preserve existing quotes (`quote-style = "preserve"`)
49
+
50
+ You can run Ruff locally to check and format your code:
51
+
52
+ ```bash
53
+ ruff check .
54
+ ruff format .
55
+ ```
56
+
57
+ ## Setting Up a Development Environment
58
+
59
+ 1. **Clone the Repository:**
60
+
61
+ ```bash
62
+ git clone https://github.com/LLNL/dfanalyzer.git
63
+ cd dfanalyzer
64
+ ```
65
+
66
+ 2. **Install Python Dependencies:**
67
+ Install DFAnalyzer in editable mode along with its optional dependencies (like Darshan support) and dependencies required for running tests.
68
+
69
+ ```bash
70
+ python -m pip install --upgrade pip
71
+ # Install DFAnalyzer in editable mode with optional extras
72
+ pip install -e .[darshan]
73
+ # Install dependencies for tests
74
+ pip install -r tests/requirements.txt
75
+ # Ensure build tools are available
76
+ # pip install meson-python setuptools wheel
77
+ ```
78
+
79
+ 3. **Install System Dependencies (for C++ components):** DFAnalyzer includes C++ components. To develop these, you'll need system dependencies such as a C++ compiler, CMake, Meson, Ninja, and various development libraries (e.g., `libarrow-dev`, `libhdf5-dev`).
80
+ A comprehensive list and example installation commands for Ubuntu can be found in the "Install system dependencies" step within the `.github/workflows/ci.yml` file.
81
+
82
+ 4. **Build C++ Components (if making changes to them):**
83
+ When you install DFAnalyzer in editable mode (`pip install -e .`), the C++ components are typically built. If you make changes to the C++ source files, you may need to recompile them.
84
+ ```bash
85
+ # Navigate to the build directory (usually created by the editable install)
86
+ # meson compile -C build
87
+ # Or, if you need to reconfigure and then build:
88
+ # meson setup build --reconfigure -Denable_tests=true -Denable_tools=true
89
+ # meson compile -C build
90
+ ```
91
+ The editable pip install should handle most build aspects, but manual recompilation with Meson might be needed for iterative C++ development.
92
+
93
+ We look forward to your contributions!
@@ -0,0 +1,38 @@
1
+ FROM ubuntu:22.04
2
+
3
+ RUN apt-get update && \
4
+ apt-get install -y \
5
+ ca-certificates \
6
+ lsb-release \
7
+ wget && \
8
+ wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
9
+ apt-get install -y ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
10
+ apt-get update && \
11
+ apt-get install -y \
12
+ build-essential \
13
+ cmake \
14
+ git \
15
+ libarrow-dev \
16
+ libhdf5-dev \
17
+ libncurses-dev \
18
+ libopenmpi-dev \
19
+ libparquet-dev \
20
+ libreadline-dev \
21
+ meson \
22
+ ninja-build \
23
+ nlohmann-json3-dev \
24
+ openmpi-bin \
25
+ openmpi-common \
26
+ python3.10 \
27
+ python3-pip && \
28
+ rm -rf /var/lib/apt/lists/*
29
+
30
+ WORKDIR /dfanalyzer
31
+
32
+ COPY . .
33
+
34
+ RUN pip install --upgrade pip && \
35
+ pip install build meson-python setuptools streamlit wheel && \
36
+ pip install .[darshan] -Csetup-args="-Denable_tools=true"
37
+
38
+ ENTRYPOINT ["dfanalyzer"]
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Lawrence Livermore National Laboratory
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,163 @@
1
+ Metadata-Version: 2.1
2
+ Name: dftracer-analyzer
3
+ Version: 0.0.2
4
+ Summary: Analyze, visualize, and understand I/O performance issues in HPC workflows
5
+ Author-Email: Izzet Yildirim <izzetcyildirim@gmail.com>
6
+ Maintainer-Email: Izzet Yildirim <izzetcyildirim@gmail.com>
7
+ Classifier: Development Status :: 3 - Alpha
8
+ Classifier: Intended Audience :: Science/Research
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Programming Language :: Python
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Topic :: Scientific/Engineering
17
+ Classifier: Topic :: Software Development :: Build Tools
18
+ Classifier: Typing :: Typed
19
+ Project-URL: documentation, https://dfanalyzer.readthedocs.io/en/latest/
20
+ Project-URL: source, https://github.com/LLNL/dfanalyzer
21
+ Project-URL: tracker, https://github.com/LLNL/dfanalyzer/issues
22
+ Requires-Python: <3.12,>3.8
23
+ Requires-Dist: dask[bag,dataframe,distributed]<2025,>=2024.1.0
24
+ Requires-Dist: dask_jobqueue~=0.8.0
25
+ Requires-Dist: hydra-core~=1.3.0
26
+ Requires-Dist: inflect==7.0
27
+ Requires-Dist: numpy==1.24.3
28
+ Requires-Dist: pandas>=2.0
29
+ Requires-Dist: portion>=2.4.0
30
+ Requires-Dist: pyarrow>=13
31
+ Requires-Dist: rich~=14.1.0
32
+ Requires-Dist: strenum>=0.4
33
+ Requires-Dist: structlog>=25
34
+ Requires-Dist: dftracer-utils>=0.0.5
35
+ Provides-Extra: darshan
36
+ Requires-Dist: darshan>=3.4; extra == "darshan"
37
+ Description-Content-Type: text/markdown
38
+
39
+ # Data Flow Analyzer
40
+
41
+ <!-- ![Build and Test](https://github.com/LLNL/dfanalyzer/actions/workflows/ci.yml/badge.svg)
42
+ ![PyPI - Version](https://img.shields.io/pypi/v/dfanalyzer?label=PyPI)
43
+ ![PyPI - Wheel](https://img.shields.io/pypi/wheel/dfanalyzer?label=Wheel)
44
+ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dfanalyzer?label=Python) -->
45
+
46
+ ## Overview
47
+
48
+ DFAnalyzer is an open-source tool for analyzing performance data from large-scale workflows on distributed systems. It presents a hierarchical, layer-by-layer summary of an application's execution, from high-level application events down to low-level POSIX calls. For each layer, DFAnalyzer quantifies time, operation counts, and data volume, and calculates key performance metrics like bandwidth and operations per second. It also visualizes the overlap between different layers, helping to characterize and understand complex I/O and compute patterns.
49
+
50
+ ## Installation
51
+
52
+ To install DFAnalyzer through `pip` (recommended for most users):
53
+
54
+ ```bash
55
+ # Ensure runtime dependencies for optional features (e.g., Darshan, Recorder) are installed.
56
+ # This might involve using your system's package manager or a tool like Spack.
57
+ # Example using Spack to prepare the environment:
58
+ # spack -e tools install
59
+ pip install dfanalyzer
60
+ ```
61
+
62
+ To install DFAnalyzer from source (for developers or custom builds):
63
+
64
+ ```bash
65
+ # 1. Install system dependencies:
66
+ # Refer to the "Install system dependencies" step in .github/workflows/ci.yml
67
+ # (e.g., build-essential, cmake, libarrow-dev, libhdf5-dev, ninja-build, etc.).
68
+ # Alternatively, tools like Spack can help manage these:
69
+ # # spack -e tools install
70
+ module load ninja
71
+
72
+ # 2. Install Python build dependencies:
73
+ python -m pip install --upgrade pip meson-python setuptools wheel
74
+
75
+ # 3. Install DFAnalyzer from the root of this repository:
76
+ # The following command includes optional C++ components (tests and tools).
77
+ # The --prefix argument is optional and specifies the installation location.
78
+ pip install -e . \
79
+ -Csetup-args="--prefix=$HOME/.local" \
80
+ -Csetup-args="-Denable_tests=true" \
81
+ -Csetup-args="-Denable_tools=true"
82
+
83
+ # (Optional) Install dependencies for running tests if you plan to contribute or run local tests:
84
+ # pip install -r tests/requirements.txt
85
+ ```
86
+
87
+ ## Usage
88
+
89
+ Here's an example of how to run DFAnalyzer using sample data included in the repository:
90
+
91
+ ```bash
92
+ # Before running, ensure the sample data is extracted.
93
+ # For example, to extract the 'dftracer-dlio' sample used below:
94
+ # mkdir -p tests/data/extracted
95
+ # tar -xzf tests/data/dftracer-dlio.tar.gz -C tests/data/extracted
96
+ dfanalyzer analyzer/preset=dlio trace_path=tests/data/extracted/dftracer-dlio view_types=[time_range]
97
+ ```
98
+
99
+ This command analyzes the traces and prints a high-level summary of the application's execution. Below is a sample of the "Time Period Summary" output:
100
+
101
+ ```bash
102
+ Time Period Summary
103
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
104
+ ┃ Metric ┃ Unit ┃ Value ┃
105
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
106
+ │ Job Time │ seconds │ 56.695 │
107
+ │ Total Count │ count │ 15,901 │
108
+ │ Total Files │ count │ 87 │
109
+ │ Total Nodes │ count │ 0 │
110
+ │ Total Processes │ count │ 23 │
111
+ │ App Count │ count │ 8 │
112
+ │ Training Count │ count │ 40 │
113
+ │ Compute Count │ count │ 200 │
114
+ │ Fetch Data Count │ count │ 160 │
115
+ │ Data Loader Count │ count │ 808 │
116
+ │ Data Loader Fork Count │ count │ 96 │
117
+ │ Reader Count │ count │ 4,008 │
118
+ │ Reader POSIX (Lustre) Count │ count │ 10,432 │
119
+ │ Reader POSIX (Lustre) Size │ MB │ 111833.161 │
120
+ │ Reader POSIX (Lustre) Bandwidth │ MB/s │ 874.982 │
121
+ │ Reader POSIX (Lustre) Avg Transfer Size │ MB │ 10.720 │
122
+ │ Checkpoint Count │ count │ 8 │
123
+ │ Checkpoint POSIX (Lustre) Count │ count │ 45 │
124
+ │ Checkpoint POSIX (Lustre) Size │ MB │ 0.011 │
125
+ │ Checkpoint POSIX (Lustre) Bandwidth │ MB/s │ 0.791 │
126
+ │ Checkpoint POSIX (Lustre) Avg Transfer Size │ MB │ 0.000 │
127
+ │ Other POSIX Count │ count │ 96 │
128
+ └───────────────────────────────────────────────────────────────────────────────┴────────────────┴────────────────────┘
129
+ ```
130
+
131
+ DFAnalyzer also provides a detailed breakdown of performance metrics for each layer of the application. Here is a snippet of the "Layer Breakdown" section from the same run, which includes the percentage of time each layer overlaps with its parent layer:
132
+
133
+ ```bash
134
+ Layer Breakdown (w/ overlap %)
135
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓
136
+ ┃ Layer ┃ Time (s) ┃ Ops ┃ Ops/sec ┃ Size (MB) ┃ Bandwidth (MB/s) ┃
137
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩
138
+ │ App │ 441.967 (----) │ 8 (----) │ 0.018 │ - │ - │
139
+ │ Training │ 439.442 (----) │ 40 (----) │ 0.091 │ - │ - │
140
+ │ Compute │ 272.356 (----) │ 200 (----) │ 0.734 │ - │ - │
141
+ │ Fetch Data │ 126.179 ( 16%) │ 160 ( 25%) │ 1.268 │ - │ - │
142
+ │ Data Loader │ 151.471 ( 45%) │ 808 ( 46%) │ 5.334 │ - │ - │
143
+ │ Data Loader Fork │ 2.392 ( 0%) │ 96 ( 0%) │ 40.135 │ - │ - │
144
+ │ Reader │ 299.992 ( 40%) │ 4,008 ( 51%) │ 13.360 │ - │ - │
145
+ │ Reader POSIX (Lustre) │ 127.812 ( 45%) │ 10,432 ( 48%) │ 81.620 │ 111833.161 ( 46%) │ 874.982 │
146
+ │ Checkpoint │ 0.014 ( 0%) │ 8 ( 0%) │ 571.551 │ - │ - │
147
+ │ Checkpoint POSIX (Lustre) │ 0.014 ( 0%) │ 45 ( 0%) │ 3268.686 │ 0.011 ( 0%) │ 0.791 │
148
+ │ Other POSIX │ 2.392 ( 0%) │ 96 ( 0%) │ 40.135 │ 0.000 (----) │ - │
149
+ └─────────────────────────────┴──────────────────┴────────────────┴───────────┴────────────────────┴──────────────────┘
150
+ ```
151
+
152
+ ## Further Information
153
+
154
+ For more details, to report issues, or to contribute to DFAnalyzer, please refer to the following resources:
155
+
156
+ - **[Official DFAnalyzer Documentation](https://dfanalyzer.readthedocs.io/)**: For detailed usage, configuration options, and information about analyzers.
157
+ - **[Issue Tracker](https://github.com/LLNL/dfanalyzer/issues)**: To report bugs or suggest new features.
158
+ - **[Contributing Guidelines](./CONTRIBUTING.md)**: For information on how to contribute to the project, including setting up a development environment and coding standards.
159
+ - **[Citation File](./CITATION.cff)**: If you use DFAnalyzer in your research, please cite it using the information in this file.
160
+
161
+ ## Acknowledgments
162
+
163
+ This work was performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344. This material is based upon work supported by the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research under the DOE Early Career Research Program (LLNL-CONF-862440). Also, this research is supported in part by the National Science Foundation (NSF) under Grants OAC-2104013, OAC-2313154, and OAC-2411318.