scope-profiler 0.2.2__tar.gz → 0.2.4__tar.gz

{scope_profiler-0.2.2 → scope_profiler-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scope-profiler
-Version: 0.2.2
+Version: 0.2.4
 Summary: Profile code regions in python, optionally with LIKWID markers.
 Author: Max
 Project-URL: Source, https://github.com/max-models/scope-profiler
@@ -16,7 +16,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: h5py
 Requires-Dist: numpy
-Requires-Dist: line-profiler
+Provides-Extra: line-profiler
+Requires-Dist: line-profiler; extra == "line-profiler"
 Provides-Extra: mpi
 Requires-Dist: mpi4py; extra == "mpi"
 Provides-Extra: pproc
@@ -25,11 +26,12 @@ Requires-Dist: jupyterlab; extra == "pproc"
 Requires-Dist: matplotlib; extra == "pproc"
 Requires-Dist: maxplotlibx>=0.1.5; extra == "pproc"
 Requires-Dist: pandas; extra == "pproc"
+Requires-Dist: snakeviz; extra == "pproc"
 Provides-Extra: dev
 Requires-Dist: black[jupyter]; extra == "dev"
 Requires-Dist: isort; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
-Requires-Dist: scope-profiler[docs,mpi,pproc,test]; extra == "dev"
+Requires-Dist: scope-profiler[docs,line-profiler,mpi,pproc,test]; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: myst-parser; extra == "docs"
 Requires-Dist: nbconvert; extra == "docs"
@@ -38,7 +40,7 @@ Requires-Dist: pre-commit; extra == "docs"
 Requires-Dist: pyproject-fmt; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-book-theme; extra == "docs"
-Requires-Dist: scope-profiler[pproc]; extra == "docs"
+Requires-Dist: scope-profiler[line-profiler,pproc]; extra == "docs"
 Provides-Extra: test
 Requires-Dist: coverage; extra == "test"
 Requires-Dist: pytest; extra == "test"
@@ -427,3 +429,34 @@ straight from the JSON:
 scope-profiler pproc profiling_data.h5 -o figures \
   --export-data --export-data-format json --skip-plot-images
 ```
+
+### Viewing a run in snakeviz
+
+`--export-prof` writes the profile in the `.prof` format of the standard
+library's `cProfile`, so a run can be explored with
+[snakeviz](https://jiffyclub.github.io/snakeviz/) or `python -m pstats`:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-prof --skip-plot-images
+snakeviz figures/profile_rank0.prof
+```
+
+Regions become "functions": `cumtime` is a region's total wall time and
+`tottime` is that minus the time spent in its nested regions. One file is
+written per exported rank, since `.prof` has no notion of ranks — see
+[the CLI docs](docs/source/cli.md) for the caveats of the reconstruction.
+
+### Viewing a run in speedscope
+
+`--export-speedscope` writes the run as a
+[speedscope](https://www.speedscope.app) JSON file. Unlike `.prof`, it keeps
+every individual call, so the timeline shows the run as it happened:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-speedscope --skip-plot-images
+npx speedscope figures/profile.speedscope.json  # or drop the file on speedscope.app
+```
+
+One file is written per input, holding one profile per exported rank, all
+sharing a time origin so ranks stay aligned. See
+[the CLI docs](docs/source/cli.md) for details.

{scope_profiler-0.2.2 → scope_profiler-0.2.4}/README.md

@@ -382,3 +382,34 @@ straight from the JSON:
 scope-profiler pproc profiling_data.h5 -o figures \
   --export-data --export-data-format json --skip-plot-images
 ```
+
+### Viewing a run in snakeviz
+
+`--export-prof` writes the profile in the `.prof` format of the standard
+library's `cProfile`, so a run can be explored with
+[snakeviz](https://jiffyclub.github.io/snakeviz/) or `python -m pstats`:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-prof --skip-plot-images
+snakeviz figures/profile_rank0.prof
+```
+
+Regions become "functions": `cumtime` is a region's total wall time and
+`tottime` is that minus the time spent in its nested regions. One file is
+written per exported rank, since `.prof` has no notion of ranks — see
+[the CLI docs](docs/source/cli.md) for the caveats of the reconstruction.
+
+### Viewing a run in speedscope
+
+`--export-speedscope` writes the run as a
+[speedscope](https://www.speedscope.app) JSON file. Unlike `.prof`, it keeps
+every individual call, so the timeline shows the run as it happened:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-speedscope --skip-plot-images
+npx speedscope figures/profile.speedscope.json  # or drop the file on speedscope.app
+```
+
+One file is written per input, holding one profile per exported rank, all
+sharing a time origin so ranks stay aligned. See
+[the CLI docs](docs/source/cli.md) for details.

{scope_profiler-0.2.2 → scope_profiler-0.2.4}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [ "setuptools", "wheel" ]
 
 [project]
 name = "scope-profiler"
-version = "0.2.2"
+version = "0.2.4"
 description = "Profile code regions in python, optionally with LIKWID markers."
 readme = "README.md"
 keywords = [ "python" ]
@@ -24,6 +24,9 @@ classifiers = [
 dependencies = [
   "h5py",
   "numpy",
+]
+
+optional-dependencies.line-profiler = [
   "line-profiler",
 ]
 
@@ -37,13 +40,14 @@ optional-dependencies.pproc = [
   "matplotlib",
   "maxplotlibx >= 0.1.5",
   "pandas",
+  "snakeviz",
 ]
 
 optional-dependencies.dev = [
   "black[jupyter]",
   "isort",
   "ruff",
-  "scope-profiler[docs,test,mpi,pproc]",
+  "scope-profiler[docs,line-profiler,mpi,pproc,test]",
 ]
 # https://medium.com/@pratikdomadiya123/build-project-documentation-quickly-with-the-sphinx-python-2a9732b66594
 optional-dependencies.docs = [
@@ -54,7 +58,7 @@ optional-dependencies.docs = [
   "pyproject-fmt",
   "sphinx",
   "sphinx-book-theme",
-  "scope-profiler[pproc]",
+  "scope-profiler[line-profiler,pproc]",
 ]
 optional-dependencies.test = [
   "coverage",

scope_profiler-0.2.4/src/scope_profiler/mpi_launch.py

@@ -0,0 +1,119 @@
+"""Detection of whether the process was launched by an MPI launcher.
+
+Importing ``mpi4py.MPI`` calls ``MPI_Init``, and any collective (``bcast``,
+``Barrier``, ...) issued afterwards costs something even on a single process.
+A plain ``python script.py`` run should therefore never touch MPI at all, even
+when mpi4py happens to be installed. This module answers the only question
+that decides it: was this process started by ``mpirun``/``mpiexec``/``srun``
+(or an equivalent launcher)?
+
+The answer is read from the environment the launcher sets up, so it is
+available before mpi4py is imported.
+"""
+
+import os
+import sys
+
+# Per-process variables exported by the process managers behind the common
+# launchers. Each is set only for processes started *by* the launcher, so the
+# presence of any one of them means "this rank belongs to an MPI job".
+# SLURM_PROCID is deliberately absent: it is also set for the script of a
+# plain `sbatch` job, which is not an MPI launch. `srun` is covered by the
+# PMI/PMIX variables its MPI plugin exports.
+_LAUNCHER_ENV_VARS = (
+    "OMPI_COMM_WORLD_RANK",  # Open MPI (and derivatives: Spectrum, ...)
+    "PMI_RANK",  # MPICH, Intel MPI, MS-MPI, Cray, srun (pmi2)
+    "PMIX_RANK",  # PMIx, used by srun --mpi=pmix and Open MPI 5
+    "MV2_COMM_WORLD_RANK",  # MVAPICH2
+    "MPI_LOCALRANKID",  # Hydra (mpiexec.hydra)
+    "ALPS_APP_PE",  # Cray ALPS aprun
+    "PALS_RANKID",  # Cray PALS palsrun
+)
+
+# Escape hatch: force the decision either way without touching code, e.g. for
+# a launcher whose variables are not listed above.
+_OVERRIDE_ENV_VAR = "SCOPE_PROFILER_MPI"
+
+_TRUE_VALUES = ("1", "true", "yes", "on")
+_FALSE_VALUES = ("0", "false", "no", "off")
+
+
+def _override() -> bool | None:
+    """Value of ``SCOPE_PROFILER_MPI``, or None if unset/unrecognized."""
+    value = os.environ.get(_OVERRIDE_ENV_VAR)
+    if value is None:
+        return None
+    value = value.strip().lower()
+    if value in _TRUE_VALUES:
+        return True
+    if value in _FALSE_VALUES:
+        return False
+    return None
+
+
+def launched_under_mpi() -> bool:
+    """Whether this process was started by an MPI launcher.
+
+    Returns
+    -------
+    bool
+        True if a launcher's per-rank environment variable is present, or if
+        the application itself already initialized MPI (in which case using
+        the communicator is free). ``SCOPE_PROFILER_MPI=0``/``1`` overrides
+        the detection.
+    """
+    override = _override()
+    if override is not None:
+        return override
+
+    if any(var in os.environ for var in _LAUNCHER_ENV_VARS):
+        return True
+
+    # The application may have initialized MPI itself (embedded interpreter,
+    # or an explicit `from mpi4py import MPI`). Only inspect mpi4py if it is
+    # already imported: importing it here is exactly what must be avoided.
+    mpi_module = sys.modules.get("mpi4py.MPI")
+    if mpi_module is not None:
+        try:
+            return bool(mpi_module.Is_initialized())
+        except AttributeError:
+            return False
+
+    return False
+
+
+def get_comm(use_mpi: bool | None = None):
+    """Return ``MPI.COMM_WORLD``, or None when MPI must not be used.
+
+    Parameters
+    ----------
+    use_mpi : bool or None, optional
+        None (default) decides via :func:`launched_under_mpi`. True forces the
+        communicator (and fails loudly if mpi4py is missing); False disables
+        MPI unconditionally.
+
+    Returns
+    -------
+    mpi4py.MPI.Intracomm or None
+        The world communicator, or None if this is not an MPI run or mpi4py
+        is unavailable.
+    """
+    if use_mpi is False:
+        return None
+
+    if use_mpi is None and not launched_under_mpi():
+        return None
+
+    try:
+        from mpi4py import MPI
+    except ImportError:
+        if use_mpi:
+            raise ImportError(
+                "MPI profiling was requested (use_mpi=True) but mpi4py is not "
+                "installed."
+            )
+        # Launched by mpirun without mpi4py available: fall back to treating
+        # this rank as a standalone process rather than failing the run.
+        return None
+
+    return MPI.COMM_WORLD

{scope_profiler-0.2.2 → scope_profiler-0.2.4}/src/scope_profiler/post_processing.py

@@ -13,6 +13,8 @@ from scope_profiler.plotting_scripts import (
     plot_speedup,
     write_region_statistics_json,
 )
+from scope_profiler.prof_export import export_prof
+from scope_profiler.speedscope_export import export_speedscope
 
 
 def parse_ranks(spec: str, verbose: bool = False) -> list[int]:
@@ -155,14 +157,39 @@ def build_parser() -> argparse.ArgumentParser:
             "consistent colors."
         ),
     )
+    parser.add_argument(
+        "--export-prof",
+        action="store_true",
+        help=(
+            "Also write one profile_rank<N>.prof file per exported rank in the "
+            "cProfile/pstats format, so the run can be browsed with external "
+            "tools (`snakeviz profile_rank0.prof`, `python -m pstats ...`). "
+            "The call graph is reconstructed from region nesting; only ranks "
+            "selected with --ranks are exported (default: rank 0). Requires "
+            "-o/--output."
+        ),
+    )
+    parser.add_argument(
+        "--export-speedscope",
+        action="store_true",
+        help=(
+            "Also write a profile.speedscope.json file holding one profile per "
+            "exported rank, viewable at https://www.speedscope.app (or with "
+            "`npx speedscope profile.speedscope.json`). Unlike --export-prof "
+            "this keeps every individual call, so the timeline shows the run "
+            "as it happened. The call graph is reconstructed from region "
+            "nesting; only ranks selected with --ranks are exported (default: "
+            "rank 0). Requires -o/--output."
+        ),
+    )
     parser.add_argument(
         "--skip-plot-images",
         action="store_true",
         help=(
             "Do not render/save the PNG plot images, only the outputs from "
-            "--export-data. Useful when charts are rendered entirely client-"
-            "side (e.g. with Plotly) from the exported data. Requires "
-            "--export-data."
+            "--export-data/--export-prof/--export-speedscope. Useful when "
+            "charts are rendered entirely client-side (e.g. with Plotly) from "
+            "the exported data. Requires one of those export options."
         ),
     )
     return parser
@@ -203,8 +230,18 @@ def main(argv: list[str] | None = None):
     if args.export_data and not args.output:
         parser.error("--export-data requires -o/--output.")
 
-    if args.skip_plot_images and not args.export_data:
-        parser.error("--skip-plot-images requires --export-data.")
+    if args.export_prof and not args.output:
+        parser.error("--export-prof requires -o/--output.")
+
+    if args.export_speedscope and not args.output:
+        parser.error("--export-speedscope requires -o/--output.")
+
+    exports_requested = args.export_data or args.export_prof or args.export_speedscope
+    if args.skip_plot_images and not exports_requested:
+        parser.error(
+            "--skip-plot-images requires --export-data, --export-prof or "
+            "--export-speedscope."
+        )
 
     if args.ranks:
         ranks = []
@@ -243,6 +280,11 @@ def main(argv: list[str] | None = None):
     flame_data_path = None
     durations_data_path = None
     speedup_data_path = None
+    prof_path = None
+    prof_paths: list = []
+    speedscope_path = None
+    speedscope_paths: list = []
+    durations_paths: list = []
     if args.output:
         os.makedirs(args.output, exist_ok=True)
         if not args.skip_plot_images:
@@ -266,62 +308,92 @@ def main(argv: list[str] | None = None):
                 speedup_data_path = os.path.join(
                     args.output, f"speedup_data.{data_ext}"
                 )
+        if args.export_prof:
+            prof_path = os.path.join(args.output, "profile.prof")
+        if args.export_speedscope:
+            speedscope_path = os.path.join(args.output, "profile.speedscope.json")
 
-    plot_gantt(
-        profiling_data=readers,
-        filepath=gantt_path,
-        show=args.show,
-        include=args.include,
-        exclude=args.exclude,
-        ranks=args.ranks,
-        cmap=args.cmap,
-        data_filepath=gantt_data_path,
-        data_format=args.export_data_format,
-        backend=args.backend,
-    )
+    # --skip-plot-images still needs the plotting functions to produce the
+    # --export-data files, but an export-only run should not touch the
+    # plotting backend at all.
+    render_plots = not args.skip_plot_images or args.export_data
 
-    plot_flame(
-        profiling_data=readers,
-        filepath=flame_path,
-        show=args.show,
-        include=args.include,
-        exclude=args.exclude,
-        ranks=args.ranks,
-        cmap=args.cmap,
-        data_filepath=flame_data_path,
-        data_format=args.export_data_format,
-        backend=args.backend,
-    )
+    if args.export_prof:
+        prof_paths = export_prof(
+            profiling_data=readers,
+            filepath=prof_path,
+            ranks=args.ranks,
+            include=args.include,
+            exclude=args.exclude,
+            verbose=False,
+        )
 
-    durations_paths = plot_durations(
-        profiling_data=readers,
-        filepath=durations_path,
-        show=args.show,
-        include=args.include,
-        exclude=args.exclude,
-        ranks=args.ranks,
-        metrics=args.metrics,
-        cmap=args.cmap,
-        data_filepath=durations_data_path,
-        data_format=args.export_data_format,
-        backend=args.backend,
-    )
+    if args.export_speedscope:
+        speedscope_paths = export_speedscope(
+            profiling_data=readers,
+            filepath=speedscope_path,
+            ranks=args.ranks,
+            include=args.include,
+            exclude=args.exclude,
+            verbose=False,
+        )
+
+    if render_plots:
+        plot_gantt(
+            profiling_data=readers,
+            filepath=gantt_path,
+            show=args.show,
+            include=args.include,
+            exclude=args.exclude,
+            ranks=args.ranks,
+            cmap=args.cmap,
+            data_filepath=gantt_data_path,
+            data_format=args.export_data_format,
+            backend=args.backend,
+        )
 
-    if len(readers) > 1:
-        plot_speedup(
+        plot_flame(
             profiling_data=readers,
-            x_field=args.x_field,
+            filepath=flame_path,
+            show=args.show,
+            include=args.include,
+            exclude=args.exclude,
             ranks=args.ranks,
-            filepath=speedup_path,
+            cmap=args.cmap,
+            data_filepath=flame_data_path,
+            data_format=args.export_data_format,
+            backend=args.backend,
+        )
+
+        durations_paths = plot_durations(
+            profiling_data=readers,
+            filepath=durations_path,
             show=args.show,
             include=args.include,
             exclude=args.exclude,
+            ranks=args.ranks,
+            metrics=args.metrics,
             cmap=args.cmap,
-            data_filepath=speedup_data_path,
+            data_filepath=durations_data_path,
             data_format=args.export_data_format,
             backend=args.backend,
         )
 
+        if len(readers) > 1:
+            plot_speedup(
+                profiling_data=readers,
+                x_field=args.x_field,
+                ranks=args.ranks,
+                filepath=speedup_path,
+                show=args.show,
+                include=args.include,
+                exclude=args.exclude,
+                cmap=args.cmap,
+                data_filepath=speedup_data_path,
+                data_format=args.export_data_format,
+                backend=args.backend,
+            )
+
     if statistics_path:
         write_region_statistics_json(
             profiling_data=readers,
@@ -333,7 +405,7 @@ def main(argv: list[str] | None = None):
 
     if args.output and not args.show:
         saved = [
-            path
+            str(path)
             for path in (
                 gantt_path,
                 flame_path,
@@ -344,10 +416,19 @@ def main(argv: list[str] | None = None):
                 flame_data_path,
                 durations_data_path,
                 speedup_data_path,
+                *prof_paths,
+                *speedscope_paths,
             )
             if path
         ]
         print("Outputs saved to:\n  " + "\n  ".join(saved))
+        if prof_paths:
+            print(f"\nView a .prof file with: snakeviz {prof_paths[0]}")
+        if speedscope_paths:
+            print(
+                f"\nView {speedscope_paths[0]} at https://www.speedscope.app "
+                "(or: npx speedscope <file>)"
+            )
 
 
 if __name__ == "__main__":