pyperf 2.7.0__tar.gz → 2.8.0__tar.gz

{pyperf-2.7.0/pyperf.egg-info → pyperf-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyperf
-Version: 2.7.0
+Version: 2.8.0
 Summary: Python module to run and analyze benchmarks
 Author-email: Victor Stinner <vstinner@redhat.com>
 Maintainer-email: Dong-hee Na <donghee.na@python.org>

{pyperf-2.7.0 → pyperf-2.8.0}/doc/changelog.rst

@@ -1,6 +1,22 @@
 Changelog
 =========
 
+* Benchmarks now have a timeout.
+  Patch by Diego Russo.
+* The example code in bench_time_func.py uses the more modern `time.perf_counter()`.
+  Patch by Hajime Senuma.
+* pystats are now correctly collected in subprocesses.
+  Patch by Michael Droettboom.
+* pyperf now supports "hooks" to add custom functionality around each benchmarking call.
+  pystats are no longer collected automatically, and you must pass `--hook pystats`.
+  Patch by Michael Droettboom.
+* Memory units are now correctly stated in KiB and MiB.
+  Patch by Ungve Mardal Moe.
+* max_mem_rss is now correctly measured on macOS.
+  Patch by Michael Droettboom.
+* `pyperf system tune` now works correctly on aarch64.
+  Patch by Diego Russo.
+
 Version 2.7.0 (2024-05-18)
 ---------------------------
 

{pyperf-2.7.0 → pyperf-2.8.0}/doc/cli.rst

@@ -229,8 +229,8 @@ Example::
 Values:
 
 * `Median <https://en.wikipedia.org/wiki/Median>`_
-* "std dev": `Standard deviation (standard error)
-  <https://en.wikipedia.org/wiki/Standard_error>`_
+* "std dev": `Standard deviation
+  <https://en.wikipedia.org/wiki/Standard_deviation>`_
 
 See also `Outlier (Wikipedia) <https://en.wikipedia.org/wiki/Outlier>`_.
 

{pyperf-2.7.0 → pyperf-2.8.0}/doc/conf.py

@@ -50,7 +50,7 @@ author = 'Victor Stinner'
 # built documents.
 #
 # The short X.Y version.
-version = release = '2.7.0'
+version = release = '2.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

{pyperf-2.7.0 → pyperf-2.8.0}/doc/examples/bench_time_func.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
+import time
 import pyperf
 
 
 def bench_dict(loops, mydict):
     range_it = range(loops)
-    t0 = pyperf.perf_counter()
+    t0 = time.perf_counter()
 
     for _ in range_it:
         mydict['0']
@@ -18,7 +19,7 @@ def bench_dict(loops, mydict):
         mydict['800']
         mydict['900']
 
-    return pyperf.perf_counter() - t0
+    return time.perf_counter() - t0
 
 
 runner = pyperf.Runner()

{pyperf-2.7.0 → pyperf-2.8.0}/doc/run_benchmark.rst

@@ -192,12 +192,14 @@ the benchmark less reliable since two runs may use a different number of warmup
 values.
 
 Specializer statistics (``pystats``)
-==================================
+====================================
 
 ``pyperf`` has built-in support for `specializer statistics (``pystats``) <https://docs.python.org/dev/using/configure.html#cmdoption-enable-pystats>`_.
-If running benchmarks on a CPython built with the ``--enable-pystats`` flag, pyperf will automatically collect ``pystats`` on the benchmark code by calling ``sys._stats_on`` immediately before the benchmark and calling ``sys._stats_off`` immediately after.
+If running benchmarks on a CPython built with the ``--enable-pystats`` flag, when you pass ``--hook pystats``, pyperf will collect ``pystats`` on the benchmark code by calling ``sys._stats_on`` immediately before the benchmark and calling ``sys._stats_off`` immediately after.
 Stats are not collected when running ``pyperf``'s own code or when warming up or calibrating the benchmarks.
 
+**New in 2.8.0:** The ``--hook pystats`` flag must be given to collect pystats.
+
 Due to the overhead of collecting the statistics, the timing results will be meaningless.
 
 The `Tools/scripts/summarize_stats.py <https://github.com/python/cpython/blob/main/Tools/scripts/summarize_stats.py>`_ script can be used to summarize the statistics in a human-readable form.

{pyperf-2.7.0 → pyperf-2.8.0}/doc/runner.rst

@@ -98,6 +98,7 @@ Option::
     --inherit-environ=VARS
     --copy-env
     --no-locale
+    --timeout TIMEOUT
     --track-memory
     --tracemalloc
 
@@ -140,6 +141,9 @@ Option::
   - ``LC_TELEPHONE``
   - ``LC_TIME``
 
+* ``--timeout``: set a timeout in seconds for an execution of the benchmark.
+  If the benchmark execution times out, pyperf exits with error code 124.
+  There is no time out by default.
 * ``--tracemalloc``: Use the ``tracemalloc`` module to track Python memory
   allocation and get the peak of memory usage in metadata
   (``tracemalloc_peak``). The module is only available on Python 3.4 and newer.

{pyperf-2.7.0 → pyperf-2.8.0}/doc/system.rst

@@ -23,7 +23,7 @@ be checked in benchmark metadata: it is enabled if the ``cpu_affinity``
 
 :func:`os.sched_setaffinity` is used to pin processes.
 
-Even if no CPU is isolated, CPU pining makes benchmarks more stable: use the
+Even if no CPU is isolated, CPU pinning makes benchmarks more stable: use the
 ``--affinity`` command line option.
 
 Check the CPU topology for HyperThreading and NUMA for best performances.

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/__init__.py

@@ -1,6 +1,6 @@
 from time import perf_counter
 
-VERSION = (2, 7, 0)
+VERSION = (2, 8, 0)
 __version__ = '.'.join(map(str, VERSION))
 
 # Export pyperf.perf_counter for backward compatibility with pyperf 1.7

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/_collect_metadata.py

@@ -125,10 +125,6 @@ def collect_python_metadata(metadata):
         if not gc.isenabled():
             metadata['python_gc'] = 'disabled'
 
-    # pystats enabled?
-    if hasattr(sys, "_stats_clear"):
-        metadata['pystats'] = 'enabled'
-
 
 def read_proc(path):
     path = proc_path(path)
@@ -208,7 +204,7 @@ def collect_system_metadata(metadata):
 
 def collect_memory_metadata(metadata):
     if resource is not None:
-        metadata["mem_max_rss"] = get_max_rss()
+        metadata["mem_max_rss"] = get_max_rss(children=False)
 
     # Note: Don't collect VmPeak of /proc/self/status on Linux because it is
     # not accurate. See pyperf._linux_memory for more accurate memory metrics.

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/_command.py

@@ -1,4 +1,5 @@
 import functools
+import json
 import os.path
 import subprocess
 import sys
@@ -7,6 +8,26 @@ from pyperf._utils import shell_quote, popen_communicate
 from pyperf._worker import WorkerTask
 
 
+def parse_subprocess_data(output):
+    # Parse the data send from the subprocess.
+    # It is three lines containing:
+    #    - The runtime (in seconds)
+    #    - max_rss (or -1, if not able to compute)
+    #    - The metadata to add to the benchmark entry, as a JSON dictionary
+
+    rss = None
+    metadata = {}
+    try:
+        lines = output.splitlines()
+        timing = float(lines[0])
+        rss = int(lines[1])
+        metadata = json.loads(lines[2])
+    except ValueError:
+        raise ValueError("failed to parse worker output: %r" % output)
+
+    return timing, rss, metadata
+
+
 def bench_command(command, task, loops):
     path = os.path.dirname(__file__)
     script = os.path.join(path, '_process_time.py')
@@ -22,19 +43,15 @@ def bench_command(command, task, loops):
         raise Exception("Command failed with exit code %s"
                         % proc.returncode)
 
-    rss = None
-    try:
-        lines = output.splitlines()
-        timing = float(lines[0])
-        if len(lines) >= 2:
-            rss = int(lines[1])
-    except ValueError:
-        raise ValueError("failed to parse script output: %r" % output)
+    timing, rss, metadata = parse_subprocess_data(output)
 
-    if rss:
+    if rss and rss > 0:
         # store the maximum
         max_rss = task.metadata.get('command_max_rss', 0)
         task.metadata['command_max_rss'] = max(max_rss, rss)
+
+    task.metadata.update(metadata)
+
     return timing
 
 

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/_formatter.py

@@ -30,9 +30,9 @@ def format_filesize(size):
             return '%.0f byte' % size
 
     if size > 10 * 1024 * 1024:
-        return '%.1f MB' % (size / (1024.0 * 1024.0))
+        return '%.1f MiB' % (size / (1024.0 * 1024.0))
 
-    return '%.1f kB' % (size / 1024.0)
+    return '%.1f KiB' % (size / 1024.0)
 
 
 def format_filesizes(sizes):

pyperf-2.8.0/pyperf/_hooks.py

@@ -0,0 +1,110 @@
+# Hooks are installable context managers defined as entry points so that
+# arbitrary code can by run right before and after the actual internal
+# benchmarking code is run.
+
+
+import abc
+import importlib.metadata
+import sys
+
+
+def get_hooks():
+    hook_prefix = "pyperf.hook"
+    entry_points = importlib.metadata.entry_points()
+    if sys.version_info[:2] < (3, 10):
+        group = entry_points[hook_prefix]
+    else:
+        group = entry_points.select(group=hook_prefix)
+    return group
+
+
+def get_hook_names():
+    return (x.name for x in get_hooks())
+
+
+def get_selected_hooks(hook_names):
+    if hook_names is None:
+        return
+
+    hook_mapping = {hook.name: hook for hook in get_hooks()}
+    for hook_name in hook_names:
+        yield hook_mapping[hook_name]
+
+
+def instantiate_selected_hooks(hook_names):
+    hook_managers = {}
+    for hook in get_selected_hooks(hook_names):
+        try:
+            hook_managers[hook.name] = hook.load()()
+        except HookError as e:
+            print(f"ERROR setting up hook '{hook.name}':", file=sys.stderr)
+            print(str(e), file=sys.stderr)
+            sys.exit(1)
+
+    return hook_managers
+
+
+class HookError(Exception):
+    pass
+
+
+class HookBase(abc.ABC):
+    def __init__(self):
+        """
+        Create a new instance of the hook.
+        """
+        pass
+
+    def teardown(self, _metadata):
+        """
+        Called when the hook is completed for a process. May add any information
+        collected to the passed-in `metadata` dictionary.
+        """
+        pass
+
+    def __enter__(self):
+        """
+        Called immediately before running benchmark code.
+
+        May be called multiple times per instance.
+        """
+        pass
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        """
+        Called immediately after running benchmark code.
+        """
+        pass
+
+
+class _test_hook(HookBase):
+    def __init__(self):
+        self._count = 0
+
+    def teardown(self, metadata):
+        metadata["_test_hook"] = self._count
+
+    def __enter__(self):
+        self._count += 1
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        pass
+
+
+class pystats(HookBase):
+    def __init__(self):
+        if not hasattr(sys, "_stats_on"):
+            raise HookError(
+                "Can not collect pystats because python was not built with --enable-pystats"
+            )
+        sys._stats_off()
+        sys._stats_clear()
+
+    def teardown(self, metadata):
+        metadata["pystats"] = "enabled"
+
+    def __enter__(self):
+        sys._stats_on()
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        sys._stats_off()

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/_manager.py

@@ -7,6 +7,8 @@ from pyperf._formatter import format_number
 from pyperf._utils import MS_WINDOWS, create_environ, create_pipe, popen_killer
 
 
+EXIT_TIMEOUT = 60
+
 # Limit to 5 calibration processes
 # (10 if calibration is needed for loops and warmups)
 MAX_CALIBRATION = 5
@@ -69,6 +71,13 @@ class Manager:
         if args.profile:
             cmd.extend(['--profile', args.profile])
 
+        if args.timeout:
+            cmd.extend(['--timeout', str(args.timeout)])
+
+        if args.hook:
+            for hook in args.hook:
+                cmd.extend(['--hook', hook])
+
         if self.runner._add_cmdline_args:
             self.runner._add_cmdline_args(cmd, args)
 
@@ -98,10 +107,12 @@ class Manager:
                 proc = subprocess.Popen(cmd, env=env, **kw)
 
             with popen_killer(proc):
-                with rpipe.open_text() as rfile:
-                    bench_json = rfile.read()
-
-                exitcode = proc.wait()
+                try:
+                    bench_json = rpipe.read_text(timeout=self.args.timeout)
+                    exitcode = proc.wait(timeout=EXIT_TIMEOUT)
+                except TimeoutError as exc:
+                    print(exc)
+                    sys.exit(124)
 
         if exitcode:
             raise RuntimeError("%s failed with exit code %s"

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/_process_time.py

@@ -14,6 +14,8 @@ Measure wall-time, not CPU time.
 If resource.getrusage() is available: compute the maximum RSS memory in bytes
 per process and writes it into stdout as a second line.
 """
+import contextlib
+import json
 import os
 import subprocess
 import sys
@@ -26,9 +28,13 @@ except ImportError:
     resource = None
 
 
-def get_max_rss():
+def get_max_rss(*, children):
     if resource is not None:
-        usage = resource.getrusage(resource.RUSAGE_CHILDREN)
+        if children:
+            resource_type = resource.RUSAGE_CHILDREN
+        else:
+            resource_type = resource.RUSAGE_SELF
+        usage = resource.getrusage(resource_type)
         if sys.platform == 'darwin':
             return usage.ru_maxrss
         return usage.ru_maxrss * 1024
@@ -61,7 +67,7 @@ def bench_process(loops, args, kw, profile_filename=None):
         args = [args[0], "-m", "cProfile", "-o", temp_profile_filename] + args[1:]
 
     for _ in range_it:
-        start_rss = get_max_rss()
+        start_rss = get_max_rss(children=True)
 
         proc = subprocess.Popen(args, **kw)
         with proc:
@@ -75,7 +81,7 @@ def bench_process(loops, args, kw, profile_filename=None):
                 os.unlink(temp_profile_filename)
             sys.exit(exitcode)
 
-        rss = get_max_rss() - start_rss
+        rss = get_max_rss(children=True) - start_rss
         max_rss = max(max_rss, rss)
 
         if profile_filename:
@@ -87,6 +93,39 @@ def bench_process(loops, args, kw, profile_filename=None):
     return (dt, max_rss)
 
 
+def load_hooks(metadata):
+    hook_names = []
+    while "--hook" in sys.argv:
+        hook_idx = sys.argv.index("--hook")
+        hook_name = sys.argv[hook_idx + 1]
+        hook_names.append(hook_name)
+        del sys.argv[hook_idx]
+        del sys.argv[hook_idx]
+
+    if len(hook_names):
+        # Only import pyperf if we know we have hooks
+        import pyperf._hooks
+
+        hook_managers = pyperf._hooks.instantiate_selected_hooks(hook_names)
+        metadata["hooks"] = ", ".join(hook_managers.values())
+    else:
+        hook_managers = {}
+
+    return hook_managers
+
+
+def write_data(dt, max_rss, metadata, out=sys.stdout):
+    # Write the data that is communicated back to the main orchestration process.
+    # It is three lines containing:
+    #    - The runtime (in seconds)
+    #    - max_rss (or -1, if not able to compute)
+    #    - The metadata to add to the benchmark entry, as a JSON dictionary
+    print(dt, file=out)
+    print(max_rss or -1, file=out)
+    json.dump(metadata, fp=out)
+    print(file=out)
+
+
 def main():
     # Make sure that the pyperf module wasn't imported
     if 'pyperf' in sys.modules:
@@ -107,6 +146,9 @@ def main():
     else:
         profile_filename = None
 
+    metadata = {}
+    hook_managers = load_hooks(metadata)
+
     loops = int(sys.argv[1])
     args = sys.argv[2:]
 
@@ -121,15 +163,18 @@ def main():
         kw['stdout'] = devnull
     kw['stderr'] = subprocess.STDOUT
 
-    dt, max_rss = bench_process(loops, args, kw, profile_filename)
+    with contextlib.ExitStack() as stack:
+        for hook in hook_managers.values():
+            stack.enter_context(hook)
+        dt, max_rss = bench_process(loops, args, kw, profile_filename)
 
     if devnull is not None:
         devnull.close()
 
-    # Write timing in seconds into stdout
-    print(dt)
-    if max_rss:
-        print(max_rss)
+    for hook in hook_managers.values():
+        hook.teardown(metadata)
+
+    write_data(dt, max_rss, metadata)
 
 
 if __name__ == "__main__":

{pyperf-2.7.0 → pyperf-2.8.0}/pyperf/_runner.py

@@ -11,6 +11,7 @@ from pyperf._cpu_utils import (format_cpu_list, parse_cpu_list,
                                get_isolated_cpus, set_cpu_affinity,
                                set_highest_priority)
 from pyperf._formatter import format_timedelta
+from pyperf._hooks import get_hook_names
 from pyperf._utils import (MS_WINDOWS, MAC_OS, abs_executable,
                            WritePipe, get_python_names,
                            merge_profile_stats)
@@ -75,14 +76,7 @@ class Runner:
                  loops=0, min_time=0.1, metadata=None,
                  show_name=True,
                  program_args=None, add_cmdline_args=None,
-                 _argparser=None):
-
-        # Reset the stats collection if running a --enable-pystats build
-        try:
-            sys._stats_off()
-            sys._stats_clear()
-        except AttributeError:
-            pass
+                 _argparser=None, warmups=1):
 
         # Watchdog: ensure that only once instance of Runner (or a Runner
         # subclass) is created per process to prevent bad surprises
@@ -165,7 +159,7 @@ class Runner:
                             help='number of values per process (default: %s)'
                                  % values)
         parser.add_argument('-w', '--warmups',
-                            type=positive_or_nul,
+                            type=positive_or_nul, default=warmups,
                             help='number of skipped values per run used '
                                  'to warmup the benchmark')
         parser.add_argument('-l', '--loops',
@@ -189,6 +183,10 @@ class Runner:
                                  'value, used to calibrate the number of '
                                  'loops (default: %s)'
                             % format_timedelta(min_time))
+        parser.add_argument('--timeout',
+                            help='Specify a timeout in seconds for a single '
+                                 'benchmark execution (default: disabled)',
+                            type=strictly_positive)
         parser.add_argument('--worker', action='store_true',
                             help='Worker process, run the benchmark.')
         parser.add_argument('--worker-task', type=positive_or_nul, metavar='TASK_ID',
@@ -248,6 +246,13 @@ class Runner:
                             help='Collect profile data using cProfile '
                                  'and output to the given file.')
 
+        hook_names = list(get_hook_names())
+        parser.add_argument(
+            '--hook', action="append", choices=hook_names,
+            metavar=f"{', '.join(x for x in hook_names if not x.startswith('_'))}",
+            help='Use the given pyperf hooks'
+        )
+
         memory = parser.add_mutually_exclusive_group()
         memory.add_argument('--tracemalloc', action="store_true",
                             help='Trace memory allocations using tracemalloc')
@@ -732,6 +737,10 @@ class Runner:
         if self.args.profile:
             command.extend(["--profile", self.args.profile])
 
+        if self.args.hook:
+            for hook in self.args.hook:
+                command.extend(["--hook", hook])
+
         # Use lazy import to limit imports on 'import pyperf'
         from pyperf._command import BenchCommandTask
         task = BenchCommandTask(self, name, command)