dftracer 2.2.0.post1.dev0__tar.gz → 2.2.0.post10.dev0__tar.gz

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/ci.yml

@@ -9,6 +9,10 @@ on:
       - ".readthedocs.yaml"
       - "LICENSE"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   build-and-test:
     strategy:
@@ -430,3 +434,65 @@ jobs:
               --args "$EXE" "${CMD_ARGS[@]}" || true
             echo "::endgroup::"
           done
+
+  glibc:
+    # manylinux_2_28 is the baseline of the wheels. 2_34 is the first glibc
+    # without the __xstat family and with stat as a real export, so together
+    # they cover both ways an application can call stat.
+    name: glibc ${{ matrix.glibc }}
+    runs-on: ubuntu-24.04
+    container:
+      image: quay.io/pypa/manylinux_${{ matrix.tag }}_x86_64
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { tag: "2_28", glibc: "2.28" }
+          - { tag: "2_34", glibc: "2.34" }
+    env:
+      CMAKE_BUILD_PARALLEL_LEVEL: 4
+      DFTRACER_BIND_SIGNALS: 1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Check glibc version
+        run: ldd --version | head -1 | grep -q " ${{ matrix.glibc }}$"
+
+      - name: Install test tools
+        run: |
+          dnf install -y jq
+          echo /opt/python/cp311-cp311/bin >> "$GITHUB_PATH"
+
+      # pydftracer 2.0.5 imports typing_extensions without declaring it.
+      - name: Install Python test dependencies
+        run: python3 -m pip install numpy pillow typing_extensions "pydftracer>=2.0.5"
+
+      - name: Build dependencies
+        run: |
+          cmake -S . -B deps \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX=/opt/dftracer \
+            -DDFTRACER_INSTALL_DEPENDENCIES=ON
+          cmake --build deps
+
+      - name: Build
+        run: |
+          cmake -S . -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX=/opt/dftracer \
+            -DCMAKE_PREFIX_PATH=/opt/dftracer \
+            -Dyaml-cpp_DIR=/opt/dftracer \
+            -DDFTRACER_INSTALL_DEPENDENCIES=OFF \
+            -DDFTRACER_ENABLE_TESTS=ON
+          cmake --build build
+
+      # pydftracer imports the extension as dftracer.dftracer, and the Python
+      # tests look for it under venv/lib64.
+      - name: Install Python bindings for the tests
+        run: |
+          mkdir -p venv/lib64/dftracer
+          cp build/lib64/dftracer*.so venv/lib64/dftracer/
+
+      - name: Test
+        run: ctest --test-dir build --output-on-failure

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/dlio-benchmark.yml

@@ -17,6 +17,10 @@ on:
       - 'LICENSE'
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   dlio-benchmark:
     name: DLIO ${{ matrix.workload }}

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/format-check.yml

@@ -7,6 +7,10 @@ on:
     branches: [ develop, stage-in ]
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   cpp-format-check:
     runs-on: ubuntu-24.04

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/hdf5-mpi-trace-ci.yml

@@ -9,6 +9,10 @@ on:
       - '.readthedocs.yaml'
       - 'LICENSE'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   hdf5-mpi-trace:
     name: ${{ matrix.mpi }}-${{ matrix.mpi_version }}-hdf5-${{ matrix.hdf5_version }}

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/prerelease.yml

@@ -90,13 +90,8 @@ jobs:
         with:
           python-version: "3.12"
 
-      # MANIFEST.in grafts dependency/, so fetching first makes the sdist
-      # self-contained: a source build needs no dependency downloads. This is
-      # what lets an MPI or HDF5 build run off a prerelease:
+      # The sdist is what lets an MPI or HDF5 build run off a prerelease:
       #   DFTRACER_ENABLE_MPI=ON pip install --pre --no-binary dftracer dftracer
-      - name: Fetch the public dependency archives
-        run: scripts/wheel/fetch_deps.sh --verify
-
       - name: Build sdist
         # setuptools-scm would derive <tag>.dev<distance> from setup.py's scheme;
         # pin the version the wheels were built with so the two agree.

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/python-publish.yml

@@ -28,11 +28,6 @@ jobs:
         with:
           python-version: "3.12"
 
-      # MANIFEST.in grafts dependency/, so fetching first makes the sdist
-      # self-contained: a source build needs no dependency downloads.
-      - name: Fetch the public dependency archives
-        run: scripts/wheel/fetch_deps.sh --verify
-
       - name: Build sdist
         run: |
           python -m pip install --upgrade pip build

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/valgrind-ci.yml

@@ -16,6 +16,10 @@ name: Valgrind Gates
       - '.readthedocs.yaml'
       - 'LICENSE'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   valgrind-gates:
     name: Valgrind Gates (Python-3.11 GCC-11)

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.github/workflows/wheels.yml

@@ -45,6 +45,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   plan:
     name: Plan
@@ -97,10 +101,7 @@ jobs:
           python-version: "3.12"
 
       - name: Check the manifest matches the CMake pins
-        run: python3 scripts/wheel/manifest.py --check
-
-      - name: Fetch the public dependency archives
-        run: scripts/wheel/fetch_deps.sh --verify
+        run: python3 scripts/wheel/manifest.py
 
       - name: Build and test wheel
         env:
@@ -110,7 +111,6 @@ jobs:
           scripts/wheel/build_wheels.sh \
             --python "${{ matrix.python }}" \
             --glibc "${{ needs.plan.outputs.glibc }}" \
-            --no-fetch \
             --output wheelhouse
 
       - uses: actions/upload-artifact@v4

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/.gitlab/flux-ci/wheels.sh

@@ -4,12 +4,11 @@
 set -eo pipefail
 cd "$CI_PROJECT_DIR"
 source /etc/profile.d/z00_lmod.sh 2>/dev/null || true
-python3 scripts/wheel/manifest.py --check
-scripts/wheel/fetch_deps.sh --verify
+python3 scripts/wheel/manifest.py
 # postdev: .postN outranks the tag, .dev0 keeps it a pre-release. A bare
 # .postN is a final release and would be installed without --pre.
 scripts/wheel/build_wheels.sh --python "${PYTHON}" --glibc "${GLIBC}" \
-  --version-scheme postdev --jobs "${WHEEL_JOBS:-6}" --no-fetch --output wheelhouse
+  --version-scheme postdev --jobs "${WHEEL_JOBS:-6}" --output wheelhouse
 
 # The wheels carry no MPI or HDF5, so `pip install --no-binary dftracer` needs
 # an sdist beside them. Versioned from git describe to match the wheels.

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dftracer
-Version: 2.2.0.post1.dev0
+Version: 2.2.0.post10.dev0
 Summary: I/O profiler for deep learning python apps. Specifically for dlio_benchmark.
 Author-email: "Hariharan Devarajan (Hari)" <hariharandev1@llnl.gov>
 License-Expression: MIT
@@ -171,14 +171,9 @@ provides `std::filesystem` (GCC 9 or newer; note that a system `libstdc++` older
 than the compiler on the `PATH` can shadow it and fail the link), CMake 3.24 or
 newer, and the development packages of whatever you enable.
 
-The C/C++ dependencies (cpp-logger, GOTCHA, brahma, yaml-cpp, libuv) are built
-automatically. Their source archives ship inside the source distribution, so a
-source install needs no access to their repositories. When building from a git
-clone, fetch the ones that are not committed first:
-
-```bash
-scripts/wheel/fetch_deps.sh
-```
+The C/C++ dependencies (cpp-logger, GOTCHA, brahma, yaml-cpp, libuv) are cloned
+and built automatically, so a source install needs access to their GitHub
+repositories.
 
 To confirm the build traces what you enabled, run your application with
 `DFTRACER_ENABLE=1` and check that the trace contains the matching categories

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/README.md

@@ -125,14 +125,9 @@ provides `std::filesystem` (GCC 9 or newer; note that a system `libstdc++` older
 than the compiler on the `PATH` can shadow it and fail the link), CMake 3.24 or
 newer, and the development packages of whatever you enable.
 
-The C/C++ dependencies (cpp-logger, GOTCHA, brahma, yaml-cpp, libuv) are built
-automatically. Their source archives ship inside the source distribution, so a
-source install needs no access to their repositories. When building from a git
-clone, fetch the ones that are not committed first:
-
-```bash
-scripts/wheel/fetch_deps.sh
-```
+The C/C++ dependencies (cpp-logger, GOTCHA, brahma, yaml-cpp, libuv) are cloned
+and built automatically, so a source install needs access to their GitHub
+repositories.
 
 To confirm the build traces what you enabled, run your application with
 `DFTRACER_ENABLE=1` and check that the trace contains the matching categories

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/cmake/modules/dftracer-utils.cmake

@@ -133,68 +133,6 @@ endfunction()
 # ###############################################################
 include(ExternalProject)
 
-# Directory holding pre-downloaded dependency source archives. Dependencies whose
-# repository is private (cpp-logger) are committed there; the public ones are
-# fetched by scripts/wheel/fetch_deps.sh when needed. An archive found here is
-# used instead of cloning, so a build needs no access to the dependency remotes.
-set(DFTRACER_DEPENDENCY_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../dependency/source"
-  CACHE PATH "Directory of pre-downloaded dependency source archives")
-
-# Case-insensitive: GOTCHA ships as GOTCHA-1.0.10.tar.gz, package name gotcha.
-function(dftracer_find_dependency_archive name out_var)
-  set(${out_var} "" PARENT_SCOPE)
-
-  if(NOT IS_DIRECTORY "${DFTRACER_DEPENDENCY_SOURCE_DIR}")
-    return()
-  endif()
-
-  string(TOLOWER "${name}" _wanted)
-  file(GLOB _archives
-    "${DFTRACER_DEPENDENCY_SOURCE_DIR}/*.tar.gz"
-    "${DFTRACER_DEPENDENCY_SOURCE_DIR}/*.tar.bz2"
-    "${DFTRACER_DEPENDENCY_SOURCE_DIR}/*.tar.xz"
-    "${DFTRACER_DEPENDENCY_SOURCE_DIR}/*.zip")
-  list(SORT _archives)
-
-  foreach(_archive IN LISTS _archives)
-    get_filename_component(_file "${_archive}" NAME)
-    string(TOLOWER "${_file}" _file)
-    if(_file MATCHES "^${_wanted}-[0-9]")
-      set(${out_var} "${_archive}" PARENT_SCOPE)
-      return()
-    endif()
-  endforeach()
-endfunction()
-
-# Extract a staged archive once and return its source directory, for
-# dependencies that fetch their own copies (brahma does, for cpp-logger).
-function(dftracer_stage_dependency_source name out_var)
-  set(${out_var} "" PARENT_SCOPE)
-
-  dftracer_find_dependency_archive(${name} _archive)
-  if(NOT _archive)
-    return()
-  endif()
-
-  set(_staged "${CMAKE_BINARY_DIR}/dependency-src/${name}")
-  if(NOT EXISTS "${_staged}/CMakeLists.txt")
-    set(_extract "${CMAKE_BINARY_DIR}/dependency-src/.extract-${name}")
-    file(REMOVE_RECURSE "${_extract}" "${_staged}")
-    file(MAKE_DIRECTORY "${_extract}")
-    file(ARCHIVE_EXTRACT INPUT "${_archive}" DESTINATION "${_extract}")
-    file(GLOB _top LIST_DIRECTORIES true "${_extract}/*")
-    list(LENGTH _top _top_count)
-    if(NOT _top_count EQUAL 1)
-      message(WARNING "[${PROJECT_NAME}] unexpected layout in ${_archive}")
-      return()
-    endif()
-    file(RENAME "${_top}" "${_staged}")
-    file(REMOVE_RECURSE "${_extract}")
-  endif()
-
-  set(${out_var} "${_staged}" PARENT_SCOPE)
-endfunction()
-
 function(dftracer_install_external_project name version var_name url tag install_prefix configure_args)
   find_package(${name} ${version} QUIET)
   set(found_var ${name}_FOUND)
@@ -206,19 +144,11 @@ function(dftracer_install_external_project name version var_name url tag install
     link_directories(${library_var})
     message(STATUS "[${PROJECT_NAME}] found dependency already installed ${name} with include ${include_var} and library ${library_var}")
   else()
-    dftracer_find_dependency_archive(${name} archive)
-    if(archive)
-      message(STATUS "[${PROJECT_NAME}] ${name}: using staged archive ${archive}")
-      set(download_args URL "${archive}" DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
-    else()
-      message(STATUS "[${PROJECT_NAME}] ${name}: no staged archive, cloning ${url}@${tag}")
-      set(download_args GIT_REPOSITORY ${url} GIT_TAG ${tag})
-    endif()
-
     ExternalProject_Add(
       ${name}
       PREFIX ${CMAKE_BINARY_DIR}
-      ${download_args}
+      GIT_REPOSITORY ${url}
+      GIT_TAG ${tag}
       TIMEOUT 10
       CMAKE_ARGS
       "-DCMAKE_INSTALL_PREFIX=${install_prefix}"

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/dependency/CMakeLists.txt

@@ -173,20 +173,6 @@ if(_dftracer_dep_hdf5_version_str)
     "-DBRAHMA_HDF5_VERSION_STRING:STRING=${_dftracer_dep_hdf5_version_str}")
 endif()
 
-# brahma fetches cpp-logger and gotcha itself. Its find_package() should resolve
-# both from the install prefix we populate first, but cpp-logger's repository is
-# private, so point brahma at the staged sources as a fallback instead of letting
-# it try a clone that cannot succeed.
-foreach(_staged_dep cpp-logger gotcha)
-  dftracer_stage_dependency_source(${_staged_dep} _staged_dir)
-  if(_staged_dir)
-    string(TOUPPER "${_staged_dep}" _staged_upper)
-    string(APPEND BRAHMA_CONFIGURE_ARGS
-      ";-DFETCHCONTENT_SOURCE_DIR_${_staged_upper}=${_staged_dir}")
-    message(STATUS "[${PROJECT_NAME}] brahma: ${_staged_dep} source staged at ${_staged_dir}")
-  endif()
-endforeach()
-
 find_package(brahma 5.1.0 QUIET)
 if(brahma_FOUND)
   message(STATUS "[${PROJECT_NAME}] found brahma already installed")
@@ -195,7 +181,9 @@ else()
     brahma
     PREFIX ${CMAKE_BINARY_DIR}
     GIT_REPOSITORY https://github.com/llnl-asr/brahma.git
-    GIT_TAG v1.1.0
+    # TODO(rayandrew): replace this commit with the brahma release tag before the
+    # dftracer release, and update dependency/manifest.txt to match.
+    GIT_TAG c86a1338697265d8c29d74b17718290e30de7fbf
     TIMEOUT 10
     PATCH_COMMAND ${CMAKE_COMMAND}
       -DSOURCE_DIR=<SOURCE_DIR>

dftracer-2.2.0.post10.dev0/dependency/manifest.txt

@@ -0,0 +1,11 @@
+# C/C++ dependencies the wheel build compiles into its prefix, in build order.
+# Columns: name, git tag or commit, repository. dependency/CMakeLists.txt pins the
+# same refs for the CMake source build; scripts/wheel/manifest.py compares them.
+
+gotcha       v1.0.10          https://github.com/LLNL/GOTCHA.git
+cpp-logger   v0.0.8           https://github.com/llnl-asr/cpp-logger.git
+yaml-cpp     yaml-cpp-0.6.3   https://github.com/jbeder/yaml-cpp.git
+libuv        v1.52.1          https://github.com/libuv/libuv.git
+# TODO(rayandrew): replace the brahma commit with its release tag before the
+# dftracer release, and update dependency/CMakeLists.txt to match.
+brahma       c86a1338697265d8c29d74b17718290e30de7fbf  https://github.com/llnl-asr/brahma.git

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/docs/api.rst

@@ -42,7 +42,7 @@ YAML configuration supported. WE need to set DFTRACER_CONFIGURATION ENV variable
     gotcha:
       priority: 1       # PRIORITY of DFTracer in GOTCHA (default: True).
     features:
-      metadata: True    # Include metadata (default False)
+      metadata: True    # Include metadata (default True)
       core_affinity: True # Include core affinity (default True). metadata needs to be enabled.
       io:
         enable: True    # Enable automatic binding of all I/O calls (default True).
@@ -52,6 +52,9 @@ YAML configuration supported. WE need to set DFTRACER_CONFIGURATION ENV variable
 
 ENV Variables supported
 
+Variables of type INT that act as switches accept ``1``, ``true``, ``on`` or ``yes`` (any case) to turn a feature on.
+Any other value turns it off, where a default of 1 applies.
+
 .. table:: section - main configuration settings using env variables
    :widths: auto
 
@@ -69,7 +72,7 @@ ENV Variables supported
                                             ``/local/scratch/data`` are in the list, the order matters—
                                             the last one will override the first. As a result, the first path won’t be traced.
                                             To avoid this, only use ``/local/scratch``.
-   DFTRACER_INC_METADATA            INT     Include or exclude metadata (default 0)
+   DFTRACER_INC_METADATA            INT     Include or exclude metadata (default 1)
    DFTRACER_SET_CORE_AFFINITY       INT     Include or exclude core affinity (default 0).
                                             ``DFTRACER_INC_METADATA`` needs to be enabled.
    DFTRACER_GOTCHA_PRIORITY         INT     PRIORITY of DFTracer in GOTCHA (default: 1).
@@ -89,16 +92,17 @@ ENV Variables supported
    DFTRACER_DISABLE_IO              INT     Disable automatic binding of all I/O calls (default: 0).
    DFTRACER_DISABLE_POSIX           INT     Disable automatic binding of POSIX I/O calls (default: 0).
    DFTRACER_DISABLE_STDIO           INT     Disable automatic binding of STDIO I/O calls (default: 0).
-   DFTRACER_TRACE_COMPRESSION       INT     Enable trace compression (default 0).
+   DFTRACER_TRACE_COMPRESSION       INT     Enable gzip trace compression (default 1).
    DFTRACER_DISABLE_TIDS            INT     Disable tracing of thread ids (default 0).
    DFTRACER_WRITE_BUFFER_SIZE       INT     Setup the buffering size for write optimization (default 0). Note: Disabled as
                                             this won't work for AI workloads which uses ``fork`` and ``spawn`` without a clear ``exit``.
                                             Also, it does not work for workloads which uses ``exec`` and rewrite process buffer state.
-   DFTRACER_TRACE_INTERVAL_MS       INT     Setup the interval for trace collection (default 1000).
+   DFTRACER_TRACE_INTERVAL_MS       INT     Setup the interval for trace collection (default 10).
    DFTRACER_LIBUV_THREADS           INT     Number of libuv worker threads for ``dftracer_service`` (default 1).
-   DFTRACER_ENABLE_AGGREGATION      INT     Enable aggregation of events (default 0).
-   DFTRACER_AGGREGATION_TYPE        STRING  set aggregation type FULL or SELECTIVE (default: FULL).
-                                            If SELECTIVE need to set DFTRACER_AGGREGATION_FILE else all are aggregated.
+   DFTRACER_ENABLE_AGGREGATION      INT     Enable aggregation of events (default 1).
+   DFTRACER_AGGREGATION_TYPE        STRING  set aggregation type FULL or SELECTIVE (default: SELECTIVE).
+                                            Setting ``DFTRACER_ENABLE_AGGREGATION=1`` without a type selects FULL.
+                                            If SELECTIVE has no DFTRACER_AGGREGATION_FILE, the rule ``dur < 100`` is used.
    DFTRACER_AGGREGATION_FILE        STRING  PATH to the aggregation rules file (default: empty).
    ================================ ======  ===========================================================================
 

{dftracer-2.2.0.post1.dev0 → dftracer-2.2.0.post10.dev0}/docs/build.rst

@@ -183,14 +183,9 @@ matching interception:
 Dependencies
 *******************************
 
-The C/C++ dependencies (cpp-logger, GOTCHA, brahma, yaml-cpp, libuv) are built
-automatically as part of the build. Their source archives ship inside the source
-distribution, so a source install needs no access to their repositories. When
-building from a git clone, fetch the archives that are not committed first:
-
-.. code-block:: Bash
-
-    scripts/wheel/fetch_deps.sh
+The C/C++ dependencies (cpp-logger, GOTCHA, brahma, yaml-cpp, libuv) are cloned
+and built automatically as part of the build, so a source install needs access to
+their GitHub repositories.
 
 Verifying
 *******************************