scope-profiler 0.2.2__tar.gz → 0.2.3__tar.gz

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scope-profiler
-Version: 0.2.2
+Version: 0.2.3
 Summary: Profile code regions in python, optionally with LIKWID markers.
 Author: Max
 Project-URL: Source, https://github.com/max-models/scope-profiler
@@ -16,7 +16,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: h5py
 Requires-Dist: numpy
-Requires-Dist: line-profiler
+Provides-Extra: line-profiler
+Requires-Dist: line-profiler; extra == "line-profiler"
 Provides-Extra: mpi
 Requires-Dist: mpi4py; extra == "mpi"
 Provides-Extra: pproc
@@ -25,11 +26,12 @@ Requires-Dist: jupyterlab; extra == "pproc"
 Requires-Dist: matplotlib; extra == "pproc"
 Requires-Dist: maxplotlibx>=0.1.5; extra == "pproc"
 Requires-Dist: pandas; extra == "pproc"
+Requires-Dist: snakeviz; extra == "pproc"
 Provides-Extra: dev
 Requires-Dist: black[jupyter]; extra == "dev"
 Requires-Dist: isort; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
-Requires-Dist: scope-profiler[docs,mpi,pproc,test]; extra == "dev"
+Requires-Dist: scope-profiler[docs,line-profiler,mpi,pproc,test]; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: myst-parser; extra == "docs"
 Requires-Dist: nbconvert; extra == "docs"
@@ -38,7 +40,7 @@ Requires-Dist: pre-commit; extra == "docs"
 Requires-Dist: pyproject-fmt; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-book-theme; extra == "docs"
-Requires-Dist: scope-profiler[pproc]; extra == "docs"
+Requires-Dist: scope-profiler[line-profiler,pproc]; extra == "docs"
 Provides-Extra: test
 Requires-Dist: coverage; extra == "test"
 Requires-Dist: pytest; extra == "test"
@@ -427,3 +429,19 @@ straight from the JSON:
 scope-profiler pproc profiling_data.h5 -o figures \
   --export-data --export-data-format json --skip-plot-images
 ```
+
+### Viewing a run in snakeviz
+
+`--export-prof` writes the profile in the `.prof` format of the standard
+library's `cProfile`, so a run can be explored with
+[snakeviz](https://jiffyclub.github.io/snakeviz/) or `python -m pstats`:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-prof --skip-plot-images
+snakeviz figures/profile_rank0.prof
+```
+
+Regions become "functions": `cumtime` is a region's total wall time and
+`tottime` is that minus the time spent in its nested regions. One file is
+written per exported rank, since `.prof` has no notion of ranks — see
+[the CLI docs](docs/source/cli.md) for the caveats of the reconstruction.

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/README.md

@@ -382,3 +382,19 @@ straight from the JSON:
 scope-profiler pproc profiling_data.h5 -o figures \
   --export-data --export-data-format json --skip-plot-images
 ```
+
+### Viewing a run in snakeviz
+
+`--export-prof` writes the profile in the `.prof` format of the standard
+library's `cProfile`, so a run can be explored with
+[snakeviz](https://jiffyclub.github.io/snakeviz/) or `python -m pstats`:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-prof --skip-plot-images
+snakeviz figures/profile_rank0.prof
+```
+
+Regions become "functions": `cumtime` is a region's total wall time and
+`tottime` is that minus the time spent in its nested regions. One file is
+written per exported rank, since `.prof` has no notion of ranks — see
+[the CLI docs](docs/source/cli.md) for the caveats of the reconstruction.

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [ "setuptools", "wheel" ]
 
 [project]
 name = "scope-profiler"
-version = "0.2.2"
+version = "0.2.3"
 description = "Profile code regions in python, optionally with LIKWID markers."
 readme = "README.md"
 keywords = [ "python" ]
@@ -24,6 +24,9 @@ classifiers = [
 dependencies = [
   "h5py",
   "numpy",
+]
+
+optional-dependencies.line-profiler = [
   "line-profiler",
 ]
 
@@ -37,13 +40,14 @@ optional-dependencies.pproc = [
   "matplotlib",
   "maxplotlibx >= 0.1.5",
   "pandas",
+  "snakeviz",
 ]
 
 optional-dependencies.dev = [
   "black[jupyter]",
   "isort",
   "ruff",
-  "scope-profiler[docs,test,mpi,pproc]",
+  "scope-profiler[docs,line-profiler,mpi,pproc,test]",
 ]
 # https://medium.com/@pratikdomadiya123/build-project-documentation-quickly-with-the-sphinx-python-2a9732b66594
 optional-dependencies.docs = [
@@ -54,7 +58,7 @@ optional-dependencies.docs = [
   "pyproject-fmt",
   "sphinx",
   "sphinx-book-theme",
-  "scope-profiler[pproc]",
+  "scope-profiler[line-profiler,pproc]",
 ]
 optional-dependencies.test = [
   "coverage",

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler/post_processing.py

@@ -13,6 +13,7 @@ from scope_profiler.plotting_scripts import (
     plot_speedup,
     write_region_statistics_json,
 )
+from scope_profiler.prof_export import export_prof
 
 
 def parse_ranks(spec: str, verbose: bool = False) -> list[int]:
@@ -155,14 +156,26 @@ def build_parser() -> argparse.ArgumentParser:
             "consistent colors."
         ),
     )
+    parser.add_argument(
+        "--export-prof",
+        action="store_true",
+        help=(
+            "Also write one profile_rank<N>.prof file per exported rank in the "
+            "cProfile/pstats format, so the run can be browsed with external "
+            "tools (`snakeviz profile_rank0.prof`, `python -m pstats ...`). "
+            "The call graph is reconstructed from region nesting; only ranks "
+            "selected with --ranks are exported (default: rank 0). Requires "
+            "-o/--output."
+        ),
+    )
     parser.add_argument(
         "--skip-plot-images",
         action="store_true",
         help=(
             "Do not render/save the PNG plot images, only the outputs from "
-            "--export-data. Useful when charts are rendered entirely client-"
-            "side (e.g. with Plotly) from the exported data. Requires "
-            "--export-data."
+            "--export-data/--export-prof. Useful when charts are rendered "
+            "entirely client-side (e.g. with Plotly) from the exported data. "
+            "Requires --export-data or --export-prof."
         ),
     )
     return parser
@@ -203,8 +216,11 @@ def main(argv: list[str] | None = None):
     if args.export_data and not args.output:
         parser.error("--export-data requires -o/--output.")
 
-    if args.skip_plot_images and not args.export_data:
-        parser.error("--skip-plot-images requires --export-data.")
+    if args.export_prof and not args.output:
+        parser.error("--export-prof requires -o/--output.")
+
+    if args.skip_plot_images and not (args.export_data or args.export_prof):
+        parser.error("--skip-plot-images requires --export-data or --export-prof.")
 
     if args.ranks:
         ranks = []
@@ -243,6 +259,9 @@ def main(argv: list[str] | None = None):
     flame_data_path = None
     durations_data_path = None
     speedup_data_path = None
+    prof_path = None
+    prof_paths: list = []
+    durations_paths: list = []
     if args.output:
         os.makedirs(args.output, exist_ok=True)
         if not args.skip_plot_images:
@@ -266,62 +285,80 @@ def main(argv: list[str] | None = None):
                 speedup_data_path = os.path.join(
                     args.output, f"speedup_data.{data_ext}"
                 )
+        if args.export_prof:
+            prof_path = os.path.join(args.output, "profile.prof")
 
-    plot_gantt(
-        profiling_data=readers,
-        filepath=gantt_path,
-        show=args.show,
-        include=args.include,
-        exclude=args.exclude,
-        ranks=args.ranks,
-        cmap=args.cmap,
-        data_filepath=gantt_data_path,
-        data_format=args.export_data_format,
-        backend=args.backend,
-    )
+    # --skip-plot-images still needs the plotting functions to produce the
+    # --export-data files, but a prof-only run should not touch the plotting
+    # backend at all.
+    render_plots = not args.skip_plot_images or args.export_data
 
-    plot_flame(
-        profiling_data=readers,
-        filepath=flame_path,
-        show=args.show,
-        include=args.include,
-        exclude=args.exclude,
-        ranks=args.ranks,
-        cmap=args.cmap,
-        data_filepath=flame_data_path,
-        data_format=args.export_data_format,
-        backend=args.backend,
-    )
+    if args.export_prof:
+        prof_paths = export_prof(
+            profiling_data=readers,
+            filepath=prof_path,
+            ranks=args.ranks,
+            include=args.include,
+            exclude=args.exclude,
+            verbose=False,
+        )
 
-    durations_paths = plot_durations(
-        profiling_data=readers,
-        filepath=durations_path,
-        show=args.show,
-        include=args.include,
-        exclude=args.exclude,
-        ranks=args.ranks,
-        metrics=args.metrics,
-        cmap=args.cmap,
-        data_filepath=durations_data_path,
-        data_format=args.export_data_format,
-        backend=args.backend,
-    )
+    if render_plots:
+        plot_gantt(
+            profiling_data=readers,
+            filepath=gantt_path,
+            show=args.show,
+            include=args.include,
+            exclude=args.exclude,
+            ranks=args.ranks,
+            cmap=args.cmap,
+            data_filepath=gantt_data_path,
+            data_format=args.export_data_format,
+            backend=args.backend,
+        )
 
-    if len(readers) > 1:
-        plot_speedup(
+        plot_flame(
             profiling_data=readers,
-            x_field=args.x_field,
+            filepath=flame_path,
+            show=args.show,
+            include=args.include,
+            exclude=args.exclude,
             ranks=args.ranks,
-            filepath=speedup_path,
+            cmap=args.cmap,
+            data_filepath=flame_data_path,
+            data_format=args.export_data_format,
+            backend=args.backend,
+        )
+
+        durations_paths = plot_durations(
+            profiling_data=readers,
+            filepath=durations_path,
             show=args.show,
             include=args.include,
             exclude=args.exclude,
+            ranks=args.ranks,
+            metrics=args.metrics,
             cmap=args.cmap,
-            data_filepath=speedup_data_path,
+            data_filepath=durations_data_path,
             data_format=args.export_data_format,
             backend=args.backend,
         )
 
+        if len(readers) > 1:
+            plot_speedup(
+                profiling_data=readers,
+                x_field=args.x_field,
+                ranks=args.ranks,
+                filepath=speedup_path,
+                show=args.show,
+                include=args.include,
+                exclude=args.exclude,
+                cmap=args.cmap,
+                data_filepath=speedup_data_path,
+                data_format=args.export_data_format,
+                backend=args.backend,
+            )
+
     if statistics_path:
         write_region_statistics_json(
             profiling_data=readers,
@@ -333,7 +370,7 @@ def main(argv: list[str] | None = None):
 
     if args.output and not args.show:
         saved = [
-            path
+            str(path)
             for path in (
                 gantt_path,
                 flame_path,
@@ -344,10 +381,13 @@ def main(argv: list[str] | None = None):
                 flame_data_path,
                 durations_data_path,
                 speedup_data_path,
+                *prof_paths,
             )
             if path
         ]
         print("Outputs saved to:\n  " + "\n  ".join(saved))
+        if prof_paths:
+            print(f"\nView a .prof file with: snakeviz {prof_paths[0]}")
 
 
 if __name__ == "__main__":

scope_profiler-0.2.3/src/scope_profiler/prof_export.py

@@ -0,0 +1,208 @@
+"""Export merged HDF5 profiling data to the cProfile ``.prof`` (pstats) format.
+
+A ``.prof`` file is nothing more than a :mod:`marshal` dump of the dict that
+``cProfile.Profile.dump_stats`` writes: it maps a ``(filename, lineno,
+funcname)`` key to ``(cc, nc, tt, ct, callers)``, where ``callers`` maps a
+caller key to its own ``(cc, nc, tt, ct)`` sub-tuple. Writing that dict is
+enough for :mod:`pstats`, ``snakeviz`` and friends to read the data.
+
+Regions carry no call graph of their own, so the caller/callee relations are
+reconstructed from timestamp containment - the same reconstruction the flame
+chart uses (:func:`~scope_profiler.plotting_scripts._build_call_stack_intervals`).
+"""
+
+from __future__ import annotations
+
+import marshal
+from collections import defaultdict
+from collections.abc import Sequence
+from pathlib import Path
+
+from scope_profiler.h5reader import ProfilingH5Reader
+from scope_profiler.plotting_scripts import (
+    _as_readers,
+    _build_call_stack_intervals,
+    _normalize_ranks,
+    _unique_labels,
+)
+
+# pstats keys are (filename, lineno, funcname) triples, and
+# ``pstats.func_std_string`` renders a key starting with ("~", 0) as the bare
+# function name - the convention cProfile uses for builtins. Regions have no
+# source location, so borrowing it keeps them labelled "solve" rather than
+# "profiling_data.h5:0(solve)" in snakeviz.
+PSEUDO_FILENAME = "~"
+PSEUDO_LINENO = 0
+
+
+def _key(name: str) -> tuple[str, int, str]:
+    """Build the pstats key identifying a region by name."""
+    return (PSEUDO_FILENAME, PSEUDO_LINENO, name)
+
+
+def _ancestor_names(calls: list[dict], index: int) -> set[str]:
+    """Names of all calls enclosing ``calls[index]``."""
+    names = set()
+    parent = calls[index]["parent"]
+    while parent is not None:
+        names.add(calls[parent]["name"])
+        parent = calls[parent]["parent"]
+    return names
+
+
+def build_pstats_dict(
+    calls: list[dict], root_name: str | None = None
+) -> dict[tuple[str, int, str], tuple]:
+    """Turn reconstructed calls into a pstats-format statistics dict.
+
+    Parameters
+    ----------
+    calls : list[dict]
+        Calls as returned by
+        :func:`~scope_profiler.plotting_scripts._build_call_stack_intervals`:
+        each entry has ``name``, ``start`` and ``end`` in seconds, and
+        ``parent`` (an index into this list, or ``None`` for a top-level call).
+    root_name : str, optional
+        When given, a synthetic frame of this name is added as the caller of
+        every top-level call, so viewers that draw a single tree (snakeviz)
+        show the whole run instead of only its largest region.
+
+    Returns
+    -------
+    dict
+        Maps ``(filename, lineno, funcname)`` to
+        ``(cc, nc, tt, ct, callers)`` with times in seconds.
+    """
+    child_time: dict[int, float] = defaultdict(float)
+    for call in calls:
+        if call["parent"] is not None:
+            child_time[call["parent"]] += call["end"] - call["start"]
+
+    root_key = _key(root_name) if root_name is not None else None
+    # [primitive_calls, total_calls, self_time, cumulative_time, callers]
+    stats: dict[tuple[str, int, str], list] = {}
+    root_cumulative = 0.0
+
+    for index, call in enumerate(calls):
+        duration = call["end"] - call["start"]
+        # Regions that only partially overlap their enclosing region are
+        # reconstructed as children of it, which can push the parent's self
+        # time below zero; pstats has no meaning for a negative tt.
+        self_time = max(duration - child_time[index], 0.0)
+        # pstats counts cumulative time for the outermost call of a recursive
+        # chain only, otherwise the same seconds are counted at every level.
+        primitive = call["name"] not in _ancestor_names(calls, index)
+        cumulative = duration if primitive else 0.0
+
+        entry = stats.setdefault(_key(call["name"]), [0, 0, 0.0, 0.0, {}])
+        entry[0] += int(primitive)
+        entry[1] += 1
+        entry[2] += self_time
+        entry[3] += cumulative
+
+        if call["parent"] is not None:
+            caller_key = _key(calls[call["parent"]]["name"])
+        else:
+            caller_key = root_key
+            root_cumulative += duration
+        if caller_key is not None:
+            caller = entry[4].setdefault(caller_key, [0, 0, 0.0, 0.0])
+            caller[0] += int(primitive)
+            caller[1] += 1
+            caller[2] += self_time
+            caller[3] += cumulative
+
+    if root_key is not None and calls:
+        stats[root_key] = [1, 1, 0.0, root_cumulative, {}]
+
+    return {
+        key: (
+            value[0],
+            value[1],
+            value[2],
+            value[3],
+            {caller: tuple(times) for caller, times in value[4].items()},
+        )
+        for key, value in stats.items()
+    }
+
+
+def write_prof_file(
+    filepath: str | Path, stats: dict[tuple[str, int, str], tuple]
+) -> Path:
+    """Marshal a pstats dict to ``filepath``, as ``cProfile`` would."""
+    output_path = Path(filepath)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "wb") as f:
+        marshal.dump(stats, f)
+    return output_path
+
+
+def export_prof(
+    profiling_data: ProfilingH5Reader | Sequence[ProfilingH5Reader],
+    filepath: str | Path,
+    ranks: list[int] | int | None = None,
+    include: list[str] | str | None = None,
+    exclude: list[str] | str | None = None,
+    verbose: bool = True,
+) -> list[Path]:
+    """Write per-rank ``.prof`` files readable by ``pstats``/``snakeviz``.
+
+    Parameters
+    ----------
+    profiling_data : ProfilingH5Reader | Sequence[ProfilingH5Reader]
+        Reader(s) for the merged HDF5 file(s) to export.
+    filepath : str | Path
+        Base output path, e.g. ``figures/profile.prof``. A ``_rank<N>`` suffix
+        is appended per rank (and the input file's stem too, when more than one
+        file is exported), since ``.prof`` has no notion of ranks or runs.
+    ranks : list[int] | int, optional
+        Ranks to export (default: rank 0 only).
+    include, exclude : list[str] | str, optional
+        Region name filters, as for the plotting functions.
+
+    Returns
+    -------
+    list[Path]
+        The files written, in the order they were written.
+    """
+    readers = _as_readers(profiling_data)
+    if not readers:
+        raise ValueError("No profiling data provided.")
+
+    normalized_ranks = _normalize_ranks(ranks) if ranks is not None else [0]
+
+    labels = _unique_labels([reader.file_path.stem for reader in readers])
+
+    prepared = []
+    for label, reader in zip(labels, readers):
+        regions = reader.get_regions(include=include, exclude=exclude)
+        if not regions:
+            raise ValueError("No regions matched the selected filters.")
+        for rank in normalized_ranks:
+            if rank < 0 or rank >= reader.num_ranks:
+                raise ValueError(f"Invalid rank requested: {rank}")
+            calls = _build_call_stack_intervals(regions, rank)
+            if calls:
+                prepared.append((label, rank, calls))
+
+    if not prepared:
+        raise ValueError("No calls recorded for the requested ranks.")
+
+    base_path = Path(filepath)
+    suffix = base_path.suffix or ".prof"
+    multiple_files = len(readers) > 1
+
+    written = []
+    for label, rank, calls in prepared:
+        parts = [base_path.stem]
+        if multiple_files:
+            parts.append(label)
+        parts.append(f"rank{rank}")
+        out_path = base_path.with_name("_".join(parts) + suffix)
+        stats = build_pstats_dict(calls, root_name=f"<{label} rank {rank}>")
+        written.append(write_prof_file(out_path, stats))
+        if verbose:
+            print(f"Wrote {out_path} (view with: snakeviz {out_path})")
+
+    return written

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler/profile_manager.py

@@ -376,13 +376,26 @@ class ProfileManager:
         rank = config._rank
         size = config._size
 
+        # 0. Discard any per-rank file left by an earlier finalize() on this
+        # config. Nothing else writes it, so its presence means a previous run
+        # in this process already finalized, and its regions would otherwise be
+        # merged into this run's output alongside the current ones.
+        stale_file = config._local_file_path
+        if os.path.exists(stale_file):
+            os.remove(stale_file)
+
         # 1. Write every region's buffered timestamps to its per-rank file.
         # Regions that record no timestamps write only their call count, which
         # is cheap and has nothing to do with timing buffers, so they write
         # even when flush_to_disk is off. Otherwise their counts would be lost.
+        # Once written, a region is marked so finalize() acts as a run
+        # boundary: a second run in the same process (e.g. a restart) writes
+        # only its own events. Regions that were not written keep their
+        # buffers, because with flush_to_disk off those are the only copy.
         for region in cls.get_all_regions().values():
             if config.flush_to_disk or not region._records_time:
                 region.write_to_disk()
+                region.mark_written()
 
         # 2. Barrier to ensure all ranks finished writing
         if comm is not None:

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler/region_profiler.py

@@ -27,10 +27,17 @@ def _import_pylikwid():
 def _import_line_profiler():
     """Import and return the LineProfiler class from line_profiler.
 
-    This function exists to defer the import of line_profiler until needed,
-    preventing unnecessary overhead when line profiling is disabled.
+    Imported lazily: line_profiler is an optional dependency, needed only when
+    ``use_line_profiler=True``.
     """
-    from line_profiler import LineProfiler
+    try:
+        from line_profiler import LineProfiler
+    except ImportError as exc:
+        raise ImportError(
+            "Line-by-line profiling requested but line_profiler is not "
+            "installed. Install scope-profiler[line-profiler], or "
+            "line_profiler directly."
+        ) from exc
 
     return LineProfiler
 
@@ -57,6 +64,7 @@ class BaseProfileRegion:
         "start_times",
         "end_times",
         "num_calls",
+        "_num_calls_written",
         "ptr",
         "buffer_limit",
         "capacity",
@@ -83,6 +91,8 @@ class BaseProfileRegion:
         self.region_name = region_name
         self.config = config
         self.num_calls = 0
+        # Calls already persisted by an earlier finalize(); see mark_written.
+        self._num_calls_written = 0
 
         # Preallocate buffers (skipped entirely when no timing is recorded).
         # `buffer_limit` is the *initial* capacity: `_grow` doubles it as
@@ -163,6 +173,12 @@ class BaseProfileRegion:
 
         with h5py.File(self.config._local_file_path, "a") as f:
             grp = f.require_group(self.group_path)
+            # Never fail on a dataset that is already there: writing twice into
+            # the same per-rank file (a second write_to_disk() outside the
+            # finalize() path) should replace the data, not raise from h5py.
+            for name in ("start_times", "end_times"):
+                if name in grp:
+                    del grp[name]
             grp.create_dataset("start_times", data=self.start_times[: self.ptr])
             grp.create_dataset("end_times", data=self.end_times[: self.ptr])
 
@@ -171,15 +187,38 @@ class BaseProfileRegion:
 
         Used by regions that record no timestamps, so their call counts survive
         into the merged output file instead of being lost with the process.
+        Only the calls made since the last finalize() are written, matching the
+        per-run slice of timestamps the timing regions write.
         """
         with h5py.File(self.config._local_file_path, "a") as f:
             grp = f.require_group(self.group_path)
-            grp.attrs["num_calls"] = self.num_calls
+            grp.attrs["num_calls"] = self.num_calls - self._num_calls_written
 
     def get_durations_numpy(self) -> np.ndarray:
         """Return durations (end - start) for buffered entries as a NumPy array."""
         return self.end_times[: self.ptr] - self.start_times[: self.ptr]
 
+    def mark_written(self) -> None:
+        """Record that everything buffered so far has reached the disk.
+
+        Called by ``finalize()`` once the data is safely written, so that a
+        second run in the same process reports only its own events instead of
+        re-reporting the first run's. The timestamp buffer rewinds (the arrays
+        are reused; anything past ``ptr`` is unread scratch), while
+        ``num_calls`` keeps counting for the lifetime of the process — it is
+        the in-memory view of the region, which callers inspect after
+        ``finalize()`` — so the watermark below is what makes the *written*
+        count per-run.
+
+        A region that is currently open has a slot reserved in the buffer and
+        an index waiting to be popped on exit, so rewinding under it would let
+        the next call overwrite a live slot. Such a region is left untouched.
+        """
+        if self._scope_ptr_stack:
+            return
+        self.ptr = 0
+        self._num_calls_written = self.num_calls
+
     def get_end_times_numpy(self) -> np.ndarray:
         """Return end times offset by config creation time."""
         return self.end_times[: self.ptr] - self.config.config_creation_time
@@ -249,7 +288,7 @@ class NCallsOnlyProfileRegion(BaseProfileRegion):
 
     def write_to_disk(self):
         """Persist the call count — the only thing this region records."""
-        if self.num_calls:
+        if self.num_calls > self._num_calls_written:
             self._write_num_calls()
 
     def get_durations_numpy(self):
@@ -330,7 +369,7 @@ class LikwidOnlyProfileRegion(BaseProfileRegion):
 
     def write_to_disk(self):
         """Persist the call count; LIKWID counters go to LIKWID's own output."""
-        if self.num_calls:
+        if self.num_calls > self._num_calls_written:
             self._write_num_calls()
 
     def wrap(self, func):

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler/tests/test_app.py

@@ -155,6 +155,7 @@ def test_all_region_types():
 
 
 def test_line_profiler_decorator():
+    pytest.importorskip("line_profiler")
     ProfileManager.setup(
         use_line_profiler=True,
         time_trace=True,
@@ -187,6 +188,7 @@ def test_line_profiler_decorator():
 
 
 def test_line_profiler_context_manager():
+    pytest.importorskip("line_profiler")
     ProfileManager.setup(
         use_line_profiler=True,
         time_trace=True,

scope_profiler-0.2.3/src/scope_profiler/tests/test_prof_export.py

@@ -0,0 +1,225 @@
+import marshal
+import pstats
+
+import pytest
+
+from scope_profiler.h5reader import ProfilingH5Reader
+from scope_profiler.post_processing import main
+from scope_profiler.prof_export import build_pstats_dict, export_prof
+from scope_profiler.tests.test_post_processing import _write_sample_h5
+
+MS = 1_000_000  # nanoseconds per millisecond, the unit stored in the HDF5 files
+
+
+def _nested_file_data():
+    """One rank whose regions nest: main > (setup, solve > assemble)."""
+    return {
+        0: {
+            "main": ([0], [100 * MS]),
+            "setup": ([0], [20 * MS]),
+            "solve": ([20 * MS], [90 * MS]),
+            "assemble": ([30 * MS], [60 * MS]),
+        }
+    }
+
+
+def _calls(*specs):
+    """Build the call list ``build_pstats_dict`` expects, in seconds."""
+    return [
+        {"name": name, "start": start, "end": end, "parent": parent}
+        for name, start, end, parent in specs
+    ]
+
+
+def _stats_of(path):
+    with open(path, "rb") as f:
+        return marshal.load(f)
+
+
+def test_build_pstats_dict_nesting_and_self_time():
+    calls = _calls(
+        ("main", 0.0, 1.0, None),
+        ("child", 0.1, 0.4, 0),
+        ("child", 0.5, 0.7, 0),
+    )
+
+    stats = build_pstats_dict(calls)
+
+    main = stats[("~", 0, "main")]
+    child = stats[("~", 0, "child")]
+
+    assert main[:2] == (1, 1)
+    assert main[2] == pytest.approx(0.5)  # 1.0 total minus 0.3 + 0.2 of children
+    assert main[3] == pytest.approx(1.0)
+    assert main[4] == {}
+
+    assert child[:2] == (2, 2)
+    assert child[2] == pytest.approx(0.5)
+    assert child[3] == pytest.approx(0.5)
+    assert child[4] == {
+        ("~", 0, "main"): (2, 2, pytest.approx(0.5), pytest.approx(0.5))
+    }
+
+
+def test_build_pstats_dict_counts_recursion_like_cprofile():
+    calls = _calls(
+        ("recurse", 0.0, 1.0, None),
+        ("recurse", 0.2, 0.6, 0),
+    )
+
+    entry = build_pstats_dict(calls)[("~", 0, "recurse")]
+
+    # One primitive call, two total, and the shared seconds counted once.
+    assert entry[0] == 1
+    assert entry[1] == 2
+    assert entry[2] == pytest.approx(1.0)
+    assert entry[3] == pytest.approx(1.0)
+
+
+def test_build_pstats_dict_clamps_partial_overlap():
+    # "long" is reconstructed as a child of "short" because it starts inside
+    # it, even though it runs past its end - self time must not go negative.
+    calls = _calls(
+        ("short", 0.0, 0.5, None),
+        ("long", 0.1, 2.0, 0),
+    )
+
+    stats = build_pstats_dict(calls)
+
+    assert stats[("~", 0, "short")][2] == 0.0
+    assert all(entry[2] >= 0.0 for entry in stats.values())
+
+
+def test_build_pstats_dict_synthetic_root():
+    calls = _calls(("a", 0.0, 1.0, None), ("b", 2.0, 2.5, None))
+
+    stats = build_pstats_dict(calls, root_name="<run>")
+
+    root_key = ("~", 0, "<run>")
+    assert stats[root_key][:4] == (1, 1, 0.0, pytest.approx(1.5))
+    for name in ("a", "b"):
+        assert root_key in stats[("~", 0, name)][4]
+
+
+def test_export_prof_readable_by_pstats(tmp_path):
+    h5_file = tmp_path / "profiling_data.h5"
+    _write_sample_h5(h5_file, _nested_file_data())
+
+    written = export_prof(
+        ProfilingH5Reader(h5_file), tmp_path / "profile.prof", verbose=False
+    )
+
+    assert written == [tmp_path / "profile_rank0.prof"]
+
+    stats = pstats.Stats(str(written[0]))
+    names = {key[2] for key in stats.stats}
+    assert {"main", "setup", "solve", "assemble"} <= names
+
+    # main: 0.1s wall, minus setup (0.02) and solve (0.07); solve minus assemble.
+    assert stats.stats[("~", 0, "main")][2] == pytest.approx(0.01)
+    assert stats.stats[("~", 0, "solve")][2] == pytest.approx(0.04)
+    assert stats.stats[("~", 0, "assemble")][3] == pytest.approx(0.03)
+    assert stats.stats[("~", 0, "assemble")][4].keys() == {("~", 0, "solve")}
+    assert stats.total_calls == 5  # four regions plus the synthetic root
+
+
+def test_export_prof_per_rank_and_per_file(tmp_path):
+    file_one = tmp_path / "run_one.h5"
+    file_two = tmp_path / "run_two.h5"
+    _write_sample_h5(file_one, {rank: _nested_file_data()[0] for rank in (0, 1)})
+    _write_sample_h5(file_two, {rank: _nested_file_data()[0] for rank in (0, 1)})
+
+    readers = [ProfilingH5Reader(file_one), ProfilingH5Reader(file_two)]
+    written = export_prof(
+        readers, tmp_path / "profile.prof", ranks=[0, 1], verbose=False
+    )
+
+    assert [path.name for path in written] == [
+        "profile_run_one_rank0.prof",
+        "profile_run_one_rank1.prof",
+        "profile_run_two_rank0.prof",
+        "profile_run_two_rank1.prof",
+    ]
+    for path in written:
+        assert pstats.Stats(str(path)).total_calls == 5
+
+
+def test_export_prof_rejects_unknown_rank(tmp_path):
+    h5_file = tmp_path / "profiling_data.h5"
+    _write_sample_h5(h5_file, _nested_file_data())
+
+    with pytest.raises(ValueError, match="Invalid rank"):
+        export_prof(
+            ProfilingH5Reader(h5_file),
+            tmp_path / "profile.prof",
+            ranks=[3],
+            verbose=False,
+        )
+
+
+def test_cli_export_prof_without_plots(tmp_path, capsys):
+    h5_file = tmp_path / "profiling_data.h5"
+    _write_sample_h5(h5_file, _nested_file_data())
+    out_dir = tmp_path / "figures"
+
+    main(
+        [
+            str(h5_file),
+            "-o",
+            str(out_dir),
+            "--export-prof",
+            "--skip-plot-images",
+        ]
+    )
+
+    prof_file = out_dir / "profile_rank0.prof"
+    assert prof_file.exists()
+    assert not list(out_dir.glob("*.png"))
+    assert "snakeviz" in capsys.readouterr().out
+
+    stats = _stats_of(prof_file)
+    assert {key[2] for key in stats} >= {"main", "setup", "solve", "assemble"}
+
+
+def test_cli_export_prof_alongside_plots(tmp_path):
+    h5_file = tmp_path / "profiling_data.h5"
+    _write_sample_h5(h5_file, _nested_file_data())
+    out_dir = tmp_path / "figures"
+
+    main([str(h5_file), "-o", str(out_dir), "--export-prof", "--ranks", "0"])
+
+    assert (out_dir / "profile_rank0.prof").exists()
+    assert (out_dir / "flame_plot.png").exists()
+
+
+def test_cli_export_prof_requires_output(tmp_path):
+    h5_file = tmp_path / "profiling_data.h5"
+    _write_sample_h5(h5_file, _nested_file_data())
+
+    with pytest.raises(SystemExit):
+        main([str(h5_file), "--export-prof"])
+
+
+def test_exported_prof_loads_in_snakeviz(tmp_path):
+    snakeviz_stats = pytest.importorskip("snakeviz.stats")
+
+    h5_file = tmp_path / "profiling_data.h5"
+    _write_sample_h5(h5_file, _nested_file_data())
+    written = export_prof(
+        ProfilingH5Reader(h5_file), tmp_path / "profile.prof", verbose=False
+    )
+
+    stats = pstats.Stats(str(written[0]))
+    assert len(snakeviz_stats.table_rows(stats)) == 5
+
+    # snakeviz drops entries that neither call nor are called; every region has
+    # to survive that, otherwise it cannot be drawn.
+    tree = snakeviz_stats.json_stats(stats)
+    assert set(tree) == {
+        "~:0(<profiling_data rank 0>)",
+        "~:0(main)",
+        "~:0(setup)",
+        "~:0(solve)",
+        "~:0(assemble)",
+    }
+    assert set(tree["~:0(solve)"]["children"]) == {"~:0(assemble)"}

scope_profiler-0.2.3/src/scope_profiler/tests/test_repeated_finalize.py

@@ -0,0 +1,99 @@
+"""Calling finalize() more than once in a process, as a restarted run does."""
+
+from time import sleep
+
+from scope_profiler import ProfileManager
+from scope_profiler.h5reader import ProfilingH5Reader
+
+
+def _run(label: str, num_calls: int) -> None:
+    for _ in range(num_calls):
+        with ProfileManager.profile_region(label):
+            sleep(0.0001)
+
+
+def test_second_finalize_without_setup(tmp_path):
+    """A second run must finalize cleanly and report only its own events."""
+    out = tmp_path / "profiling_data.h5"
+    ProfileManager.setup(file_path=str(out), flush_to_disk=True)
+
+    _run("step", 3)
+    ProfileManager.finalize(verbose=False)
+
+    region = ProfilingH5Reader(str(out)).get_region("step")
+    assert region.num_calls == 3
+
+    # Second run, reusing the config and the region objects of the first.
+    _run("step", 5)
+    ProfileManager.finalize(verbose=False)
+
+    region = ProfilingH5Reader(str(out)).get_region("step")
+    assert region.num_calls == 5, "second run inherited the first run's events"
+    assert len(region.durations) == 5
+
+
+def test_stale_regions_do_not_leak_into_the_second_run(tmp_path):
+    """A region used only in the first run must not reappear in the second."""
+    out = tmp_path / "profiling_data.h5"
+    ProfileManager.setup(file_path=str(out), flush_to_disk=True)
+
+    _run("first_only", 2)
+    ProfileManager.finalize(verbose=False)
+    assert "first_only" in ProfilingH5Reader(str(out)).region_names
+
+    _run("second_only", 2)
+    ProfileManager.finalize(verbose=False)
+
+    names = ProfilingH5Reader(str(out)).region_names
+    assert "second_only" in names
+    assert "first_only" not in names
+
+
+def test_call_counts_are_per_run_without_timing(tmp_path):
+    """Call-count-only regions also report per-run numbers, not running totals."""
+    out = tmp_path / "profiling_data.h5"
+    ProfileManager.setup(file_path=str(out), time_trace=False)
+
+    _run("step", 3)
+    ProfileManager.finalize(verbose=False)
+    assert ProfilingH5Reader(str(out)).get_region("step").num_calls == 3
+
+    _run("step", 5)
+    ProfileManager.finalize(verbose=False)
+    assert ProfilingH5Reader(str(out)).get_region("step").num_calls == 5
+
+    # In memory the counter keeps running for the lifetime of the process.
+    assert ProfileManager.get_region("step").num_calls == 8
+
+
+def test_finalize_keeps_buffers_when_not_flushing(tmp_path):
+    """With flush_to_disk off the buffers are the only copy, so they survive."""
+    ProfileManager.setup(
+        file_path=str(tmp_path / "profiling_data.h5"),
+        flush_to_disk=False,
+    )
+
+    _run("step", 4)
+    ProfileManager.finalize(verbose=False)
+
+    region = ProfileManager.get_region("step")
+    assert region.num_calls == 4
+    assert region.ptr == 4
+
+
+def test_finalize_inside_an_open_region(tmp_path):
+    """A region still open at finalize() keeps its reserved slot."""
+    ProfileManager.setup(
+        file_path=str(tmp_path / "profiling_data.h5"),
+        flush_to_disk=True,
+    )
+
+    with ProfileManager.profile_region("outer"):
+        ProfileManager.finalize(verbose=False)
+
+    region = ProfileManager.get_region("outer")
+    # Rewinding under the open scope would have let the exit write into a slot
+    # the next call could reuse; the region is left alone instead.
+    assert region.num_calls == 1
+    assert region.ptr == 1
+    assert region.get_durations_numpy()[0] > 0

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scope-profiler
-Version: 0.2.2
+Version: 0.2.3
 Summary: Profile code regions in python, optionally with LIKWID markers.
 Author: Max
 Project-URL: Source, https://github.com/max-models/scope-profiler
@@ -16,7 +16,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: h5py
 Requires-Dist: numpy
-Requires-Dist: line-profiler
+Provides-Extra: line-profiler
+Requires-Dist: line-profiler; extra == "line-profiler"
 Provides-Extra: mpi
 Requires-Dist: mpi4py; extra == "mpi"
 Provides-Extra: pproc
@@ -25,11 +26,12 @@ Requires-Dist: jupyterlab; extra == "pproc"
 Requires-Dist: matplotlib; extra == "pproc"
 Requires-Dist: maxplotlibx>=0.1.5; extra == "pproc"
 Requires-Dist: pandas; extra == "pproc"
+Requires-Dist: snakeviz; extra == "pproc"
 Provides-Extra: dev
 Requires-Dist: black[jupyter]; extra == "dev"
 Requires-Dist: isort; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
-Requires-Dist: scope-profiler[docs,mpi,pproc,test]; extra == "dev"
+Requires-Dist: scope-profiler[docs,line-profiler,mpi,pproc,test]; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: myst-parser; extra == "docs"
 Requires-Dist: nbconvert; extra == "docs"
@@ -38,7 +40,7 @@ Requires-Dist: pre-commit; extra == "docs"
 Requires-Dist: pyproject-fmt; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-book-theme; extra == "docs"
-Requires-Dist: scope-profiler[pproc]; extra == "docs"
+Requires-Dist: scope-profiler[line-profiler,pproc]; extra == "docs"
 Provides-Extra: test
 Requires-Dist: coverage; extra == "test"
 Requires-Dist: pytest; extra == "test"
@@ -427,3 +429,19 @@ straight from the JSON:
 scope-profiler pproc profiling_data.h5 -o figures \
   --export-data --export-data-format json --skip-plot-images
 ```
+
+### Viewing a run in snakeviz
+
+`--export-prof` writes the profile in the `.prof` format of the standard
+library's `cProfile`, so a run can be explored with
+[snakeviz](https://jiffyclub.github.io/snakeviz/) or `python -m pstats`:
+
+```bash
+scope-profiler pproc profiling_data.h5 -o figures --export-prof --skip-plot-images
+snakeviz figures/profile_rank0.prof
+```
+
+Regions become "functions": `cumtime` is a region's total wall time and
+`tottime` is that minus the time spent in its nested regions. One file is
+written per exported rank, since `.prof` has no notion of ranks — see
+[the CLI docs](docs/source/cli.md) for the caveats of the reconstruction.

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ src/scope_profiler/metadata.py
 src/scope_profiler/mpi_region.py
 src/scope_profiler/plotting_scripts.py
 src/scope_profiler/post_processing.py
+src/scope_profiler/prof_export.py
 src/scope_profiler/profile_config.py
 src/scope_profiler/profile_manager.py
 src/scope_profiler/region.py
@@ -30,5 +31,7 @@ src/scope_profiler/tests/test_metadata.py
 src/scope_profiler/tests/test_mpi.py
 src/scope_profiler/tests/test_overhead.py
 src/scope_profiler/tests/test_post_processing.py
+src/scope_profiler/tests/test_prof_export.py
 src/scope_profiler/tests/test_reader_api.py
-src/scope_profiler/tests/test_readme.py
+src/scope_profiler/tests/test_readme.py
+src/scope_profiler/tests/test_repeated_finalize.py

{scope_profiler-0.2.2 → scope_profiler-0.2.3}/src/scope_profiler.egg-info/requires.txt

@@ -1,12 +1,11 @@
 h5py
 numpy
-line-profiler
 
 [dev]
 black[jupyter]
 isort
 ruff
-scope-profiler[docs,mpi,pproc,test]
+scope-profiler[docs,line-profiler,mpi,pproc,test]
 
 [docs]
 myst-parser
@@ -16,7 +15,10 @@ pre-commit
 pyproject-fmt
 sphinx
 sphinx-book-theme
-scope-profiler[pproc]
+scope-profiler[line-profiler,pproc]
+
+[line-profiler]
+line-profiler
 
 [mpi]
 mpi4py
@@ -27,6 +29,7 @@ jupyterlab
 matplotlib
 maxplotlibx>=0.1.5
 pandas
+snakeviz
 
 [test]
 coverage