pydftracer 0.0.1.dev0__tar.gz

pydftracer-0.0.1.dev0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Lawrence Livermore National Laboratory
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pydftracer-0.0.1.dev0/MANIFEST.in

@@ -0,0 +1,20 @@
+exclude .git_archival.txt
+exclude .gitattributes
+exclude .gitignore
+exclude Makefile
+exclude coverage.xml
+exclude .coverage
+exclude dft.log
+prune tests
+prune .github
+prune htmlcov
+prune .mypy_cache
+prune __pycache__
+prune .pytest_cache
+prune .hypothesis
+prune build
+prune dist
+global-exclude *.pyc
+global-exclude *.pyo
+global-exclude *.orig
+global-exclude *.core

pydftracer-0.0.1.dev0/PKG-INFO

@@ -0,0 +1,157 @@
+Metadata-Version: 2.4
+Name: pydftracer
+Version: 0.0.1.dev0
+Summary: Python libs for DFTracer
+Author-email: "Hariharan Devarajan (Hari)" <hariharandev1@llnl.gov>, Ray Andrew Sinurat <raydreww@gmail.com>
+Maintainer-email: "Hariharan Devarajan (Hari)" <hariharandev1@llnl.gov>, Ray Andrew Sinurat <raydreww@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/LLNL/pydftracer
+Project-URL: Repository, https://github.com/LLNL/pydftracer
+Project-URL: Bug Tracker, https://github.com/LLNL/pydftracer/issues
+Project-URL: Documentation, https://dftracer.readthedocs.io/en/latest/
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-xdist; extra == "dev"
+Requires-Dist: h5py; extra == "dev"
+Requires-Dist: numpy; extra == "dev"
+Requires-Dist: pillow; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: dynamo
+Requires-Dist: torch>=2.5.1; extra == "dynamo"
+Dynamic: license-file
+
+# pydftracer
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+A no operation (**No-Op**) Python binding for [DFTracer](https://github.com/hariharan-devarajan/dftracer) that provides seamless API compatibility without requiring the full DFTracer installation. Perfect for testing, development, and environments where you want to use DFTracer's API without the overhead of actual tracing.
+
+## Purposes
+
+- Are prototyping applications that will eventually use full DFTracer
+- Maintain DFTracer API compatibility in environments where tracing is not needed (e.g. production apps)
+
+## Installation
+
+```bash
+pip install dftracer
+ # with libdftracer and dftracer_utils
+pip install dftracer[full]
+```
+
+## Development
+
+```bash
+# Install development dependencies
+pip install -e .[dev]
+
+# Using Make (recommended)
+make test-parallel       # Run all tests with parallel execution
+make test-subprocess     # Run only subprocess-based dftracer tests  
+make test-ci             # Run comprehensive tests matching CI configuration
+make test-ci-quick       # Run quick tests and checks (faster)
+make check-all           # Run all quality checks (lint, format, type-check, test)
+
+# Using pytest directly
+pytest tests/ -v -n 4    # All tests with parallel execution
+pytest tests/ --cov=dftracer --cov-report=term-missing -v -n 4  # Tests with coverage
+```
+
+## Documentation
+
+* Building DFTracer: [https://dftracer.readthedocs.io/en/latest/build.html](https://dftracer.readthedocs.io/en/latest/build.html)
+* Integrating DFTracer: [https://dftracer.readthedocs.io/en/latest/examples.html](https://dftracer.readthedocs.io/en/latest/examples.html)
+* Visualizing DFTracer Traces: [https://dftracer.readthedocs.io/en/latest/perfetto.html](https://dftracer.readthedocs.io/en/latest/perfetto.html)
+* Building DFAnalyzer: [https://dftracer.readthedocs.io/en/latest/dfanalyzer_build.html](https://dftracer.readthedocs.io/en/latest/dfanalyzer_build.html)
+
+## Development
+
+### Testing
+
+This project uses a comprehensive test suite with subprocess-based isolation for proper dftracer testing.
+
+#### Running Tests
+
+```bash
+# Install development dependencies
+pip install -e .[dev]
+
+# Using Make (recommended)
+make test-parallel    # Run all tests with parallel execution
+make test-subprocess  # Run only subprocess-based dftracer tests  
+make test-ci          # Run tests matching CI configuration
+make check-all        # Run all quality checks (lint, format, type-check, test)
+
+# Using pytest directly
+pytest tests/ -v -n 2                                    # All tests with parallel execution
+pytest tests/ --cov=dftracer --cov-report=term-missing -v -n 2  # Tests with coverage
+pytest tests/ -m subprocess -v -n 2                      # Only subprocess tests
+
+# Use the provided test script (matches CI)
+./scripts/test.sh
+```
+
+#### Test Structure
+
+- **Unit Tests**: General functionality tests in `tests/test_general.py`
+- **Integration Tests**: Subprocess-based dftracer tests in `tests/test_dftracer.py`
+- **Parallel Execution**: Tests run in parallel using `pytest-xdist` for faster execution
+- **Process Isolation**: dftracer tests run in separate subprocesses to handle the per-process nature of dftracer
+
+#### CI/CD
+
+The project uses GitHub Actions for continuous integration with:
+- Multi-Python version testing (3.9, 3.10, 3.11, 3.12)
+- Parallel test execution with coverage reporting
+- Code linting with `ruff`
+- Type checking with `mypy`
+- Package building and installation testing
+
+## Citation and Reference
+
+The original SC'24 paper describes the design and implementation of the DFTracer code. Please cite this paper and the code if you use DFTracer in your research. 
+
+```
+@inproceedings{devarajan_dftracer_2024,
+    address = {Atlanta, GA},
+    title = {{DFTracer}: {An} {Analysis}-{Friendly} {Data} {Flow} {Tracer} for {AI}-{Driven} {Workflows}},
+    shorttitle = {{DFTracer}},
+    urldate = {2024-07-31},
+    booktitle = {{SC24}: {International} {Conference} for {High} {Performance} {Computing}, {Networking}, {Storage} and {Analysis}},
+    publisher = {IEEE},
+    author = {Devarajan, Hariharan and Pottier, Loic and Velusamy, Kaushik and Zheng, Huihuo and Yildirim, Izzet and Kogiou, Olga and Yu, Weikuan and Kougkas, Anthony and Sun, Xian-He and Yeom, Jae Seung and Mohror, Kathryn},
+    month = nov,
+    year = {2024},
+}
+
+@misc{devarajan_dftracer_code_2024,
+    type = {Github},
+    title = {Github {DFTracer}},
+    shorttitle = {{DFTracer}},
+    url = {https://github.com/LLNL/dftracer.git},
+    urldate = {2024-07-31},
+    journal = {DFTracer: A multi-level dataflow tracer for capture I/O calls from worklows.},
+    author = {Devarajan, Hariharan and Pottier, Loic and Velusamy, Kaushik and Zheng, Huihuo and Yildirim, Izzet and Kogiou, Olga and Yu, Weikuan and Kougkas, Anthony and Sun, Xian-He and Yeom, Jae Seung and Mohror, Kathryn},
+    month = jun,
+    year = {2024},
+}
+```
+
+## Acknowledgments
+
+This work was performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344; and under the auspices of the National Cancer Institute (NCI) by Frederick National Laboratory for Cancer Research (FNLCR) under Contract 75N91019D00024. This research used resources of the Argonne Leadership Computing Facility, a U.S. Department of Energy (DOE) Office of Science user facility at Argonne National Laboratory and is based on research supported by the U.S. DOE Office of Science-Advanced Scientific Computing Research Program, under Contract No. DE-AC02-06CH11357. Office of Advanced Scientific Computing Research under the DOE Early Career Research Program. Also, This material is based upon work partially supported by LLNL LDRD 23-ERD-045 and 24-SI-005. LLNL-CONF-857447.

pydftracer-0.0.1.dev0/README.md

@@ -0,0 +1,121 @@
+# pydftracer
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+A no operation (**No-Op**) Python binding for [DFTracer](https://github.com/hariharan-devarajan/dftracer) that provides seamless API compatibility without requiring the full DFTracer installation. Perfect for testing, development, and environments where you want to use DFTracer's API without the overhead of actual tracing.
+
+## Purposes
+
+- Are prototyping applications that will eventually use full DFTracer
+- Maintain DFTracer API compatibility in environments where tracing is not needed (e.g. production apps)
+
+## Installation
+
+```bash
+pip install dftracer
+ # with libdftracer and dftracer_utils
+pip install dftracer[full]
+```
+
+## Development
+
+```bash
+# Install development dependencies
+pip install -e .[dev]
+
+# Using Make (recommended)
+make test-parallel       # Run all tests with parallel execution
+make test-subprocess     # Run only subprocess-based dftracer tests  
+make test-ci             # Run comprehensive tests matching CI configuration
+make test-ci-quick       # Run quick tests and checks (faster)
+make check-all           # Run all quality checks (lint, format, type-check, test)
+
+# Using pytest directly
+pytest tests/ -v -n 4    # All tests with parallel execution
+pytest tests/ --cov=dftracer --cov-report=term-missing -v -n 4  # Tests with coverage
+```
+
+## Documentation
+
+* Building DFTracer: [https://dftracer.readthedocs.io/en/latest/build.html](https://dftracer.readthedocs.io/en/latest/build.html)
+* Integrating DFTracer: [https://dftracer.readthedocs.io/en/latest/examples.html](https://dftracer.readthedocs.io/en/latest/examples.html)
+* Visualizing DFTracer Traces: [https://dftracer.readthedocs.io/en/latest/perfetto.html](https://dftracer.readthedocs.io/en/latest/perfetto.html)
+* Building DFAnalyzer: [https://dftracer.readthedocs.io/en/latest/dfanalyzer_build.html](https://dftracer.readthedocs.io/en/latest/dfanalyzer_build.html)
+
+## Development
+
+### Testing
+
+This project uses a comprehensive test suite with subprocess-based isolation for proper dftracer testing.
+
+#### Running Tests
+
+```bash
+# Install development dependencies
+pip install -e .[dev]
+
+# Using Make (recommended)
+make test-parallel    # Run all tests with parallel execution
+make test-subprocess  # Run only subprocess-based dftracer tests  
+make test-ci          # Run tests matching CI configuration
+make check-all        # Run all quality checks (lint, format, type-check, test)
+
+# Using pytest directly
+pytest tests/ -v -n 2                                    # All tests with parallel execution
+pytest tests/ --cov=dftracer --cov-report=term-missing -v -n 2  # Tests with coverage
+pytest tests/ -m subprocess -v -n 2                      # Only subprocess tests
+
+# Use the provided test script (matches CI)
+./scripts/test.sh
+```
+
+#### Test Structure
+
+- **Unit Tests**: General functionality tests in `tests/test_general.py`
+- **Integration Tests**: Subprocess-based dftracer tests in `tests/test_dftracer.py`
+- **Parallel Execution**: Tests run in parallel using `pytest-xdist` for faster execution
+- **Process Isolation**: dftracer tests run in separate subprocesses to handle the per-process nature of dftracer
+
+#### CI/CD
+
+The project uses GitHub Actions for continuous integration with:
+- Multi-Python version testing (3.9, 3.10, 3.11, 3.12)
+- Parallel test execution with coverage reporting
+- Code linting with `ruff`
+- Type checking with `mypy`
+- Package building and installation testing
+
+## Citation and Reference
+
+The original SC'24 paper describes the design and implementation of the DFTracer code. Please cite this paper and the code if you use DFTracer in your research. 
+
+```
+@inproceedings{devarajan_dftracer_2024,
+    address = {Atlanta, GA},
+    title = {{DFTracer}: {An} {Analysis}-{Friendly} {Data} {Flow} {Tracer} for {AI}-{Driven} {Workflows}},
+    shorttitle = {{DFTracer}},
+    urldate = {2024-07-31},
+    booktitle = {{SC24}: {International} {Conference} for {High} {Performance} {Computing}, {Networking}, {Storage} and {Analysis}},
+    publisher = {IEEE},
+    author = {Devarajan, Hariharan and Pottier, Loic and Velusamy, Kaushik and Zheng, Huihuo and Yildirim, Izzet and Kogiou, Olga and Yu, Weikuan and Kougkas, Anthony and Sun, Xian-He and Yeom, Jae Seung and Mohror, Kathryn},
+    month = nov,
+    year = {2024},
+}
+
+@misc{devarajan_dftracer_code_2024,
+    type = {Github},
+    title = {Github {DFTracer}},
+    shorttitle = {{DFTracer}},
+    url = {https://github.com/LLNL/dftracer.git},
+    urldate = {2024-07-31},
+    journal = {DFTracer: A multi-level dataflow tracer for capture I/O calls from worklows.},
+    author = {Devarajan, Hariharan and Pottier, Loic and Velusamy, Kaushik and Zheng, Huihuo and Yildirim, Izzet and Kogiou, Olga and Yu, Weikuan and Kougkas, Anthony and Sun, Xian-He and Yeom, Jae Seung and Mohror, Kathryn},
+    month = jun,
+    year = {2024},
+}
+```
+
+## Acknowledgments
+
+This work was performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344; and under the auspices of the National Cancer Institute (NCI) by Frederick National Laboratory for Cancer Research (FNLCR) under Contract 75N91019D00024. This research used resources of the Argonne Leadership Computing Facility, a U.S. Department of Energy (DOE) Office of Science user facility at Argonne National Laboratory and is based on research supported by the U.S. DOE Office of Science-Advanced Scientific Computing Research Program, under Contract No. DE-AC02-06CH11357. Office of Advanced Scientific Computing Research under the DOE Early Career Research Program. Also, This material is based upon work partially supported by LLNL LDRD 23-ERD-045 and 24-SI-005. LLNL-CONF-857447.

pydftracer-0.0.1.dev0/pyproject.toml

@@ -0,0 +1,146 @@
+[build-system]
+requires = ["setuptools>=80", "wheel", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pydftracer"
+dynamic = ["version"]
+description = "Python libs for DFTracer"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    { name = "Hariharan Devarajan (Hari)", email = "hariharandev1@llnl.gov" },
+    { name = "Ray Andrew Sinurat", email = "raydreww@gmail.com" }
+]
+maintainers = [
+    { name = "Hariharan Devarajan (Hari)", email = "hariharandev1@llnl.gov" },
+    { name = "Ray Andrew Sinurat", email = "raydreww@gmail.com" }
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "Topic :: Software Development :: Build Tools",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3 :: Only",
+]
+requires-python = ">=3.9"
+
+[project.urls]
+Homepage = "https://github.com/LLNL/pydftracer"
+Repository = "https://github.com/LLNL/pydftracer"
+"Bug Tracker" = "https://github.com/LLNL/pydftracer/issues"
+Documentation = "https://dftracer.readthedocs.io/en/latest/"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=6.0",
+    "pytest-cov",
+    "pytest-xdist",
+    "h5py",
+    "numpy",
+    "pillow",
+    "ruff",
+    "mypy",
+]
+dynamo = [
+    "torch>=2.5.1"
+]
+
+[tool.setuptools.package-dir]
+"" = "python"
+
+[tool.setuptools.packages.find]
+where = ["python"]
+include = ["dftracer*"]
+namespaces = true
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests"
+]
+python_files = "test_*.py"
+python_classes = "Test*"
+python_functions = "test_*"
+addopts = "-v --strict-markers --tb=short"
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "subprocess: marks tests that run in subprocess",
+]
+
+[tool.coverage.run]
+source = ["python/dftracer"]
+omit = [
+    "tests/*",
+    "demo_*.py",
+    "test_*.py",
+    "python/dftracer/dbg/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py39"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "B", "I", "UP"]
+ignore = [
+    "E501",   # line too long, handled by formatter
+    "B006",   # Do not use mutable data structures for argument defaults - acceptable in some cases
+    "B008",   # do not perform function calls in argument defaults
+    "C901",   # too complex
+    "F403",   # Star imports - used for __all__ exports and acceptable in __init__ files
+    "UP006",  # Use `list` instead of `List` for type annotation - keep for older Python compatibility
+    "UP028",  # Use `X | Y` for union types - keep for older Python compatibility
+    "UP035",  # typing.Dict/List is deprecated - keeping for older Python compatibility
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["B011", "B023"]
+"python/dftracer/python/logger.py" = ["E402", "F405"]
+"python/dftracer/python/dbg/logger.py" = ["E402", "F405"]
+"python/dftracer/python/dbg/ai.py" = ["E402"]
+"python/dftracer/python/ai.py" = ["E402"]
+"python/dftracer/python/ai_common.py" = ["F405"]
+"python/dftracer/python/ai_init.py" = ["F405"]
+"python/dftracer/python/common.py" = ["E701", "F405"]
+"python/dftracer/python/dynamo.py" = ["B904"]
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+
+[[tool.mypy.overrides]]
+module = [
+    "torch.*",
+    "functorch.*",
+]
+ignore_missing_imports = true
+follow_imports = "skip"
+
+[[tool.mypy.overrides]]
+module = "dftracer.python.dynamo"
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+
+[tool.setuptools_scm]
+local_scheme = "no-local-version"
+version_file = "python/dftracer/python/_version.py"

pydftracer-0.0.1.dev0/python/dftracer/python/__init__.py

@@ -0,0 +1,11 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from dftracer.python.ai import *
+from dftracer.python.dynamo import *
+from dftracer.python.logger import *
+
+try:
+    __version__ = version("pydftracer")
+except PackageNotFoundError:
+    # package is not installed
+    pass

pydftracer-0.0.1.dev0/python/dftracer/python/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.0.1.dev0'
+__version_tuple__ = version_tuple = (0, 0, 1, 'dev0')
+
+__commit_id__ = commit_id = 'gb666f039d'

pydftracer-0.0.1.dev0/python/dftracer/python/ai.py

@@ -0,0 +1,9 @@
+import dftracer.python.ai_common as ai_common
+from dftracer.python.logger import dft_fn as dft_fn_orig
+from dftracer.python.logger import dftracer as dftracer_orig
+
+ai_common.dft_fn = dft_fn_orig  # type: ignore[misc]
+ai_common.dftracer = dftracer_orig  # type: ignore[misc]
+
+from dftracer.python.ai_common import *
+from dftracer.python.ai_init import *