scope-profiler 0.3.2__tar.gz → 0.3.3__tar.gz

scope_profiler-0.3.3/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: scope-profiler
+Version: 0.3.3
+Summary: Profile code regions in python, optionally with LIKWID markers.
+Author: Max
+Project-URL: Source, https://github.com/max-models/scope-profiler
+Keywords: python
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: h5py
+Requires-Dist: numpy
+Requires-Dist: tabulate
+Requires-Dist: tomli>=2.0; python_version < "3.11"
+Provides-Extra: fortran
+Requires-Dist: meson; extra == "fortran"
+Requires-Dist: ninja; extra == "fortran"
+Provides-Extra: likwid
+Requires-Dist: pylikwid; extra == "likwid"
+Provides-Extra: line-profiler
+Requires-Dist: line-profiler; extra == "line-profiler"
+Provides-Extra: nvtx
+Requires-Dist: nvtx; extra == "nvtx"
+Provides-Extra: mcp
+Requires-Dist: mcp<2.0.0,>=1.6.0; extra == "mcp"
+Provides-Extra: mpi
+Requires-Dist: mpi4py; extra == "mpi"
+Provides-Extra: pproc
+Requires-Dist: ipykernel; extra == "pproc"
+Requires-Dist: jupyterlab; extra == "pproc"
+Requires-Dist: matplotlib; extra == "pproc"
+Requires-Dist: maxplotlibx>=0.1.6; extra == "pproc"
+Requires-Dist: pandas; extra == "pproc"
+Requires-Dist: snakeviz; extra == "pproc"
+Requires-Dist: tabulate; extra == "pproc"
+Requires-Dist: textual>=0.86; extra == "pproc"
+Provides-Extra: dev
+Requires-Dist: black[jupyter]; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: scope-profiler[docs,fortran,line-profiler,mcp,mpi,nvtx,pproc,test,tui]; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: nbconvert; extra == "docs"
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: pre-commit; extra == "docs"
+Requires-Dist: pyproject-fmt; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-book-theme; extra == "docs"
+Requires-Dist: scope-profiler[line-profiler,pproc,tui]; extra == "docs"
+Provides-Extra: test
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: pytest; extra == "test"
+Provides-Extra: tui
+Requires-Dist: tabulate; extra == "tui"
+Requires-Dist: textual>=0.86; extra == "tui"
+
+
+
+<!-- Generated README.md is rendered from this file by docs/render_markdown.py. -->
+
+# scope-profiler
+
+Profile Python code regions—and optionally C, Fortran, MPI, NVTX, and
+[LIKWID](https://github.com/RRZE-HPC/likwid)—with one consistent API and
+HDF5 output format.
+
+``` bash
+pip install scope-profiler
+```
+
+## Quick start
+
+``` python
+from scope_profiler import ProfileManager
+
+with ProfileManager.session():
+    @ProfileManager.profile("main")
+    def main():
+        with ProfileManager.profile_region("work"):
+            sum(range(100))  # replace with the code you want to measure
+
+    main()
+# writes profiling_data.h5 and prints a summary
+```
+
+You can also profile a script without changing its source:
+
+``` bash
+scope-profiler run my_script.py
+scope-profiler inspect profiling_data.h5
+scope-profiler plot default profiling_data.h5 -o figures
+```
+
+## Example output
+
+The plotting tools include duration summaries and timelines for finding
+expensive regions:
+
+![Duration
+summary](https://raw.githubusercontent.com/max-models/scope-profiler/refs/heads/devel/figures/durations_plot.png)
+
+![Gantt
+chart](https://raw.githubusercontent.com/max-models/scope-profiler/refs/heads/devel/figures/gantt_plot.png)
+
+The overhead benchmark measures the cost of each instrumentation mode:
+
+``` bash
+python examples/benchmark_overhead.py
+```
+
+![Profiling overhead by region
+type](https://raw.githubusercontent.com/max-models/scope-profiler/refs/heads/devel/figures/benchmark_overhead.png)
+
+## Documentation
+
+- [Installation](https://max-models.github.io/scope-profiler/installation.html)
+- [Quick
+  start](https://max-models.github.io/scope-profiler/quickstart.html)
+- [Python API and
+  post-processing](https://max-models.github.io/scope-profiler/guide/hdf5_and_python_api.html)
+- [CLI reference](https://max-models.github.io/scope-profiler/cli.html)
+- [Configuration and profiling
+  regions](https://max-models.github.io/scope-profiler/guide/configuration.html)
+- [MPI](https://max-models.github.io/scope-profiler/guide/mpi.html),
+  [C](https://max-models.github.io/scope-profiler/guide/c.html), and
+  [Fortran](https://max-models.github.io/scope-profiler/guide/fortran.html)
+- [LIKWID](https://max-models.github.io/scope-profiler/guide/likwid.html),
+  [line
+  profiling](https://max-models.github.io/scope-profiler/guide/line_profiler.html),
+  and [MCP](https://max-models.github.io/scope-profiler/guide/mcp.html)
+- [Tutorial
+  notebooks](https://max-models.github.io/scope-profiler/tutorials.html)
+- [Examples](https://github.com/max-models/scope-profiler/tree/devel/examples)
+
+## Development
+
+``` bash
+pip install -e '.[dev]'
+pytest
+```
+
+See
+[AGENTS.md](https://github.com/max-models/scope-profiler/blob/devel/AGENTS.md)
+for the measured benchmark workflow used when optimizing this project.

scope_profiler-0.3.3/README.md

@@ -0,0 +1,88 @@
+
+
+<!-- Generated README.md is rendered from this file by docs/render_markdown.py. -->
+
+# scope-profiler
+
+Profile Python code regions—and optionally C, Fortran, MPI, NVTX, and
+[LIKWID](https://github.com/RRZE-HPC/likwid)—with one consistent API and
+HDF5 output format.
+
+``` bash
+pip install scope-profiler
+```
+
+## Quick start
+
+``` python
+from scope_profiler import ProfileManager
+
+with ProfileManager.session():
+    @ProfileManager.profile("main")
+    def main():
+        with ProfileManager.profile_region("work"):
+            sum(range(100))  # replace with the code you want to measure
+
+    main()
+# writes profiling_data.h5 and prints a summary
+```
+
+You can also profile a script without changing its source:
+
+``` bash
+scope-profiler run my_script.py
+scope-profiler inspect profiling_data.h5
+scope-profiler plot default profiling_data.h5 -o figures
+```
+
+## Example output
+
+The plotting tools include duration summaries and timelines for finding
+expensive regions:
+
+![Duration
+summary](https://raw.githubusercontent.com/max-models/scope-profiler/refs/heads/devel/figures/durations_plot.png)
+
+![Gantt
+chart](https://raw.githubusercontent.com/max-models/scope-profiler/refs/heads/devel/figures/gantt_plot.png)
+
+The overhead benchmark measures the cost of each instrumentation mode:
+
+``` bash
+python examples/benchmark_overhead.py
+```
+
+![Profiling overhead by region
+type](https://raw.githubusercontent.com/max-models/scope-profiler/refs/heads/devel/figures/benchmark_overhead.png)
+
+## Documentation
+
+- [Installation](https://max-models.github.io/scope-profiler/installation.html)
+- [Quick
+  start](https://max-models.github.io/scope-profiler/quickstart.html)
+- [Python API and
+  post-processing](https://max-models.github.io/scope-profiler/guide/hdf5_and_python_api.html)
+- [CLI reference](https://max-models.github.io/scope-profiler/cli.html)
+- [Configuration and profiling
+  regions](https://max-models.github.io/scope-profiler/guide/configuration.html)
+- [MPI](https://max-models.github.io/scope-profiler/guide/mpi.html),
+  [C](https://max-models.github.io/scope-profiler/guide/c.html), and
+  [Fortran](https://max-models.github.io/scope-profiler/guide/fortran.html)
+- [LIKWID](https://max-models.github.io/scope-profiler/guide/likwid.html),
+  [line
+  profiling](https://max-models.github.io/scope-profiler/guide/line_profiler.html),
+  and [MCP](https://max-models.github.io/scope-profiler/guide/mcp.html)
+- [Tutorial
+  notebooks](https://max-models.github.io/scope-profiler/tutorials.html)
+- [Examples](https://github.com/max-models/scope-profiler/tree/devel/examples)
+
+## Development
+
+``` bash
+pip install -e '.[dev]'
+pytest
+```
+
+See
+[AGENTS.md](https://github.com/max-models/scope-profiler/blob/devel/AGENTS.md)
+for the measured benchmark workflow used when optimizing this project.

{scope_profiler-0.3.2 → scope_profiler-0.3.3}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [ "setuptools", "wheel" ]
 
 [project]
 name = "scope-profiler"
-version = "0.3.2"
+version = "0.3.3"
 description = "Profile code regions in python, optionally with LIKWID markers."
 readme = "README.md"
 keywords = [ "python" ]
@@ -24,6 +24,8 @@ classifiers = [
 dependencies = [
   "h5py",
   "numpy",
+  "tabulate",
+  "tomli>=2.0; python_version < '3.11'",
 ]
 
 optional-dependencies.fortran = [

{scope_profiler-0.3.2 → scope_profiler-0.3.3}/src/scope_profiler/__main__.py

@@ -29,6 +29,10 @@ Six subcommands:
   ``scope_profiler.diff``.
 - ``scope-profiler check a.h5 b.h5`` -- applies a regression budget and
   returns a CI-friendly exit code.
+- ``scope-profiler benchmark run config.toml`` -- runs a repeatable benchmark
+  with a correctness gate and writes a JSON manifest.
+- ``scope-profiler benchmark compare baseline.json candidate.json`` -- makes a
+  median-based keep/reject decision for an AI agent or CI.
 - ``scope-profiler import-native traces/ -o out.h5`` -- converts the trace
   files written by the Fortran region API
   (``scope_profiler/fortran/scope_profiler.f90``)
@@ -52,9 +56,14 @@ def _parse_run_args(argv):
     parser.add_argument(
         "-o",
         "--outfile",
-        default="profiling_data.h5",
+        default=None,
         help="Path to the merged HDF5 output file (default: profiling_data.h5)",
     )
+    parser.add_argument(
+        "--config",
+        metavar="FILE",
+        help="TOML file containing profiling settings ([profiling] table)",
+    )
     parser.add_argument(
         "-q",
         "--quiet",
@@ -70,13 +79,14 @@ def _parse_run_args(argv):
     parser.add_argument(
         "--line-profile",
         action="store_true",
+        default=None,
         help="Also collect line-by-line timings via line_profiler "
         "(requires scope-profiler[line-profiler])",
     )
     parser.add_argument(
         "--buffer-limit",
         type=int,
-        default=1024,
+        default=None,
         help="Initial buffer capacity per region; grows as needed (default: 1024)",
     )
     parser.add_argument("script", help="Script to run and profile")
@@ -100,11 +110,14 @@ def _run(argv):
         raise SystemExit(1)
 
     ProfileManager.setup(
-        recursive_profile=True,
-        use_likwid=False,
+        # ``run`` historically enables recursive profiling.  A TOML file may
+        # override it, while the no-config path keeps that default.
+        recursive_profile=True if args.config is None else None,
+        use_likwid=None,
         use_line_profiler=args.line_profile,
         buffer_limit=args.buffer_limit,
         file_path=args.outfile,
+        config_path=args.config,
     )
 
     try:
@@ -166,6 +179,48 @@ def _check(argv):
     return check_main(argv)
 
 
+def _benchmark(argv):
+    """Run or compare declarative, repeated benchmark manifests."""
+    from scope_profiler.benchmark import (
+        BenchmarkError,
+        compare_benchmarks,
+        load_config,
+        run_benchmark,
+    )
+
+    parser = argparse.ArgumentParser(
+        prog="scope-profiler benchmark",
+        description="Run or compare repeatable AI/CI benchmark workflows.",
+    )
+    subparsers = parser.add_subparsers(dest="action", required=True)
+    run_parser = subparsers.add_parser("run", help="run a benchmark config")
+    run_parser.add_argument("config", help="benchmark TOML configuration")
+    run_parser.add_argument(
+        "--label", default="candidate", help="baseline or candidate label"
+    )
+    run_parser.add_argument("--json", action="store_true", help="print only JSON")
+    compare_parser = subparsers.add_parser(
+        "compare", help="compare two benchmark manifests"
+    )
+    compare_parser.add_argument("baseline")
+    compare_parser.add_argument("candidate")
+    compare_parser.add_argument("--json", action="store_true", help="print only JSON")
+    args = parser.parse_args(argv)
+    try:
+        if args.action == "run":
+            result = run_benchmark(load_config(args.config), label=args.label)
+            exit_code = 0 if result["correctness"]["passed"] else 1
+        else:
+            result = compare_benchmarks(args.baseline, args.candidate)
+            exit_code = 0 if result["decision"] == "keep" else 1
+    except BenchmarkError as exc:
+        parser.error(str(exc))
+    print(__import__("json").dumps(result, indent=None if args.json else 2))
+    if exit_code:
+        raise SystemExit(exit_code)
+    return 0
+
+
 def _import_fortran(argv):
     """Handle ``scope-profiler import-native``: Fortran traces -> HDF5."""
     from scope_profiler.native_trace import TRACE_SUFFIX, convert_traces
@@ -242,6 +297,7 @@ _COMMANDS = {
     "line-profile": _line_profile,
     "diff": _diff,
     "check": _check,
+    "benchmark": _benchmark,
     "import-native": _import_fortran,
 }
 
@@ -306,6 +362,11 @@ def main(argv=None):
         add_help=False,
         help="Fail on profiling regressions (see `scope-profiler check --help`)",
     )
+    subparsers.add_parser(
+        "benchmark",
+        add_help=False,
+        help="Run or compare repeatable benchmarks (see `scope-profiler benchmark --help`)",
+    )
     subparsers.add_parser(
         "import-native",
         add_help=False,

scope_profiler-0.3.3/src/scope_profiler/benchmark.py

@@ -0,0 +1,234 @@
+"""Repeatable benchmark workflows for coding agents and CI.
+
+The benchmark format is TOML so it works with Python 3.10+ without adding a
+configuration dependency.  A benchmark run produces a JSON manifest containing
+the individual profile paths, robust summary statistics, and correctness status.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import statistics
+import subprocess
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+try:  # Python 3.11+
+    import tomllib
+except ModuleNotFoundError:  # Python 3.10
+    import tomli as tomllib
+
+from scope_profiler.h5reader import read_h5
+
+
+class BenchmarkError(Exception):
+    """A user-facing benchmark configuration or execution error."""
+
+
+@dataclass(frozen=True)
+class BenchmarkConfig:
+    name: str
+    script: str
+    args: tuple[str, ...] = ()
+    runs: int = 5
+    warmups: int = 1
+    timeout_seconds: float = 300.0
+    only_user_code: bool = True
+    output_dir: str = ".scope-profiler"
+    correctness_command: tuple[str, ...] = ()
+    correctness_timeout_seconds: float = 300.0
+    threshold_pct: float = 2.0
+
+
+def load_config(path: str | os.PathLike[str]) -> BenchmarkConfig:
+    """Load and validate a benchmark TOML file."""
+    config_path = Path(path).resolve()
+    try:
+        with config_path.open("rb") as stream:
+            raw = tomllib.load(stream)
+    except OSError as exc:
+        raise BenchmarkError(
+            f"Could not read benchmark config {path!r}: {exc}"
+        ) from exc
+    except tomllib.TOMLDecodeError as exc:
+        raise BenchmarkError(f"Invalid benchmark TOML {path!r}: {exc}") from exc
+
+    bench = raw.get("benchmark", raw)
+    script = bench.get("script")
+    if not script:
+        raise BenchmarkError("Benchmark config must define benchmark.script")
+    script_path = (config_path.parent / script).resolve()
+    if not script_path.is_file():
+        raise BenchmarkError(f"Benchmark script does not exist: {script_path}")
+
+    correctness = raw.get("correctness", {})
+    command = correctness.get("command", bench.get("correctness_command", []))
+    if isinstance(command, str):
+        command = [command]
+    config = BenchmarkConfig(
+        name=str(bench.get("name", config_path.stem)),
+        script=str(script_path),
+        args=tuple(str(x) for x in bench.get("args", [])),
+        runs=int(bench.get("runs", 5)),
+        warmups=int(bench.get("warmups", 1)),
+        timeout_seconds=float(bench.get("timeout_seconds", 300.0)),
+        only_user_code=bool(bench.get("only_user_code", True)),
+        output_dir=str(
+            (config_path.parent / bench.get("output_dir", ".scope-profiler")).resolve()
+        ),
+        correctness_command=tuple(str(x) for x in command),
+        correctness_timeout_seconds=float(correctness.get("timeout_seconds", 300.0)),
+        threshold_pct=float(bench.get("threshold_pct", 2.0)),
+    )
+    if config.runs < 2:
+        raise BenchmarkError("benchmark.runs must be at least 2")
+    if config.warmups < 0 or config.timeout_seconds <= 0:
+        raise BenchmarkError(
+            "warmups must be non-negative and timeout_seconds positive"
+        )
+    if not 0 <= config.threshold_pct:
+        raise BenchmarkError("benchmark.threshold_pct must be non-negative")
+    return config
+
+
+def _profile_once(config: BenchmarkConfig, path: Path) -> dict:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    command = [sys.executable, "-m", "scope_profiler", "run", "-q", "-o", str(path)]
+    if not config.only_user_code:
+        command.append("--all")
+    command.extend([config.script, *config.args])
+    try:
+        completed = subprocess.run(
+            command,
+            capture_output=True,
+            text=True,
+            timeout=config.timeout_seconds,
+            check=False,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise BenchmarkError(
+            f"Benchmark run exceeded {config.timeout_seconds}s"
+        ) from exc
+    if completed.returncode != 0:
+        raise BenchmarkError(
+            f"Benchmark run failed with exit code {completed.returncode}: "
+            f"{completed.stderr[-2000:]}"
+        )
+    if not path.exists():
+        raise BenchmarkError(f"Benchmark run produced no profile: {path}")
+    results = read_h5(path)
+    total = results.total_time
+    if total is None or not math.isfinite(total):
+        raise BenchmarkError(f"Profile has no finite total time: {path}")
+    regions = {
+        region.name: float(region.total_duration) for region in results.get_regions()
+    }
+    return {"path": str(path), "total_time_seconds": float(total), "regions": regions}
+
+
+def _run_correctness(config: BenchmarkConfig) -> dict:
+    if not config.correctness_command:
+        return {
+            "configured": False,
+            "passed": True,
+            "command": [],
+            "stdout_tail": "",
+            "stderr_tail": "",
+        }
+    command = list(config.correctness_command)
+    if command[0] == "python":
+        command[0] = sys.executable
+    try:
+        completed = subprocess.run(
+            command,
+            cwd=Path(config.script).parent,
+            capture_output=True,
+            text=True,
+            timeout=config.correctness_timeout_seconds,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return {
+            "configured": True,
+            "passed": False,
+            "command": command,
+            "error": "timeout",
+        }
+    return {
+        "configured": True,
+        "passed": completed.returncode == 0,
+        "returncode": completed.returncode,
+        "command": command,
+        "stdout_tail": completed.stdout[-2000:],
+        "stderr_tail": completed.stderr[-2000:],
+    }
+
+
+def _stats(values: list[float]) -> dict:
+    return {
+        "count": len(values),
+        "min": min(values),
+        "max": max(values),
+        "mean": statistics.fmean(values),
+        "median": statistics.median(values),
+        "stdev": statistics.stdev(values) if len(values) > 1 else 0.0,
+    }
+
+
+def run_benchmark(config: BenchmarkConfig, label: str = "candidate") -> dict:
+    """Run warmups, correctness, and repeated measured profiles."""
+    output_dir = Path(config.output_dir) / config.name / label
+    for index in range(config.warmups):
+        _profile_once(config, output_dir / f"warmup-{index}.h5")
+    correctness = _run_correctness(config)
+    profiles = [
+        _profile_once(config, output_dir / f"run-{index}.h5")
+        for index in range(config.runs)
+    ]
+    totals = [item["total_time_seconds"] for item in profiles]
+    manifest = {
+        "format": 1,
+        "name": config.name,
+        "label": label,
+        "config": asdict(config),
+        "correctness": correctness,
+        "profiles": profiles,
+        "total_time_seconds": _stats(totals),
+    }
+    manifest_path = output_dir / "benchmark.json"
+    manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    manifest["manifest_path"] = str(manifest_path)
+    return manifest
+
+
+def compare_benchmarks(baseline: dict | str, candidate: dict | str) -> dict:
+    """Compare benchmark manifests using medians and report an agent decision."""
+
+    def load(value):
+        if isinstance(value, dict):
+            return value
+        return json.loads(Path(value).read_text(encoding="utf-8"))
+
+    base, cand = load(baseline), load(candidate)
+    base_median = base["total_time_seconds"]["median"]
+    cand_median = cand["total_time_seconds"]["median"]
+    change_pct = (
+        ((cand_median - base_median) / base_median * 100) if base_median else None
+    )
+    threshold = float(cand.get("config", {}).get("threshold_pct", 2.0))
+    faster = bool(cand_median < base_median and change_pct <= -threshold)
+    correctness_passed = bool(cand.get("correctness", {}).get("passed", False))
+    decision = "keep" if faster and correctness_passed else "reject"
+    return {
+        "baseline": {"label": base.get("label"), "stats": base["total_time_seconds"]},
+        "candidate": {"label": cand.get("label"), "stats": cand["total_time_seconds"]},
+        "relative_change_pct": change_pct,
+        "speedup": base_median / cand_median if cand_median else None,
+        "faster_beyond_threshold": faster,
+        "correctness_passed": correctness_passed,
+        "decision": decision,
+        "threshold_pct": threshold,
+    }

{scope_profiler-0.3.2 → scope_profiler-0.3.3}/src/scope_profiler/diff.py

@@ -10,6 +10,8 @@ configs, two job sizes) shows up as a single table instead of two separate
 import argparse
 import sys
 
+from tabulate import tabulate
+
 from scope_profiler.h5reader import read_h5
 from scope_profiler.results import ProfilingResults
 from scope_profiler.summary import region_rows
@@ -163,24 +165,14 @@ def print_diff_table(rows, metric: str = "total", title=None, stream=None) -> No
         for row in rows
     ]
 
-    widths = {
-        key: max(len(header), max(len(row[key]) for row in formatted))
-        for key, header in columns
-    }
-
-    def render(row):
-        cells = [f"{row['name']:<{widths['name']}}"]
-        cells += [f"{row[key]:>{widths[key]}}" for key, _ in columns[1:]]
-        return "  ".join(cells).rstrip()
-
-    header_line = render({key: header for key, header in columns})
-    rule = "-" * len(header_line)
-
-    print(f"  {header_line}", file=stream)
-    print(f"  {rule}", file=stream)
-    for row in formatted:
-        print(f"  {render(row)}", file=stream)
-    print(f"  {rule}", file=stream)
+    table_rows = [[row[key] for key, _ in columns] for row in formatted]
+    for line in tabulate(
+        table_rows,
+        headers=[header for _, header in columns],
+        tablefmt="rounded_outline",
+        disable_numparse=True,
+    ).splitlines():
+        print(f"  {line}", file=stream)
 
     only_a = [row["name"] for row in rows if row["b"] is None]
     only_b = [row["name"] for row in rows if row["a"] is None]