pyperf 2.6.3__tar.gz → 2.8.0__tar.gz

{pyperf-2.6.3/pyperf.egg-info → pyperf-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyperf
-Version: 2.6.3
+Version: 2.8.0
 Summary: Python module to run and analyze benchmarks
 Author-email: Victor Stinner <vstinner@redhat.com>
 Maintainer-email: Dong-hee Na <donghee.na@python.org>

{pyperf-2.6.3 → pyperf-2.8.0}/doc/analyze.rst

@@ -223,9 +223,15 @@ two-sample, two-tailed t-test
 
 Render a table using ``--table`` option::
 
-    $ python3 -m pyperf compare_to py36.json py38.json --table
-    +-----------+---------+------------------------------+
-    | Benchmark | py36    | py38                         |
-    +===========+=========+==============================+
-    | timeit    | 4.70 us | 4.22 us: 1.11x faster (-10%) |
-    +-----------+---------+------------------------------+
+    $ python3 -m pyperf compare_to mult_list_py36.json mult_list_py38.json --table
+    +----------------+----------------+-----------------------+
+    | Benchmark      | mult_list_py36 | mult_list_py38        |
+    +================+================+=======================+
+    | [1,2]*1000     | 3.70 us        | 3.18 us: 1.16x faster |
+    +----------------+----------------+-----------------------+
+    | [1,2,3]*1000   | 4.61 us        | 4.17 us: 1.11x faster |
+    +----------------+----------------+-----------------------+
+    | Geometric mean | (ref)          | 1.09x faster          |
+    +----------------+----------------+-----------------------+
+
+    Benchmark hidden because not significant (1): [1]*1000

{pyperf-2.6.3 → pyperf-2.8.0}/doc/api.rst

@@ -679,7 +679,7 @@ Benchmark:
 * ``inner_loops`` (``int >= 1``): number of inner-loops of the benchmark (``int``)
 * ``timer``: Implementation of ``time.perf_counter()``, and also resolution if
   available
-* ``tags``: (list of str, optional): A list of tags associated with the benchmark. If provided, the results output will be aggreggated by each tag.
+* ``tags``: (list of str, optional): A list of tags associated with the benchmark. If provided, the results output will be aggregated by each tag.
 
 Python metadata:
 

{pyperf-2.6.3 → pyperf-2.8.0}/doc/changelog.rst

@@ -1,6 +1,34 @@
 Changelog
 =========
 
+* Benchmarks now have a timeout.
+  Patch by Diego Russo.
+* The example code in bench_time_func.py uses the more modern `time.perf_counter()`.
+  Patch by Hajime Senuma.
+* pystats are now correctly collected in subprocesses.
+  Patch by Michael Droettboom.
+* pyperf now supports "hooks" to add custom functionality around each benchmarking call.
+  pystats are no longer collected automatically, and you must pass `--hook pystats`.
+  Patch by Michael Droettboom.
+* Memory units are now correctly stated in KiB and MiB.
+  Patch by Ungve Mardal Moe.
+* max_mem_rss is now correctly measured on macOS.
+  Patch by Michael Droettboom.
+* `pyperf system tune` now works correctly on aarch64.
+  Patch by Diego Russo.
+
+Version 2.7.0 (2024-05-18)
+---------------------------
+
+* Fix logic in MSR availability test.
+  Patch by Mike Droettboom.
+* Disable tuning TurboBoost setting on non-Intel device.
+  Patch by Mike Droettboom.
+* Include CONFIG_ARGS in Metadata.
+  Patch by Jeff Glass.
+* Support environment variables PYTHON_GIL / PYTHON_CPU_COUNT.
+  Patch by Donghee Na.
+
 Version 2.6.3 (2024-03-05)
 ---------------------------
 
@@ -88,7 +116,7 @@ Version 2.1.0 (2021-01-14)
 * The :ref:`compare_to command <compare_to_cmd>` no longer displays
   percentages: display less numbers to make the output easier to read. Also,
   percentage were almost the same values than the xxx faster or xxx slower
-  values, but rounded differenly which introduced confusion.
+  values, but rounded differently which introduced confusion.
 * Project moved to https://github.com/psf/pyperf/
 * system command now only emits a warning rather than failing with a hard error
   if it fails to get or set the frequency of a CPU.

{pyperf-2.6.3 → pyperf-2.8.0}/doc/cli.rst

@@ -229,8 +229,8 @@ Example::
 Values:
 
 * `Median <https://en.wikipedia.org/wiki/Median>`_
-* "std dev": `Standard deviation (standard error)
-  <https://en.wikipedia.org/wiki/Standard_error>`_
+* "std dev": `Standard deviation
+  <https://en.wikipedia.org/wiki/Standard_deviation>`_
 
 See also `Outlier (Wikipedia) <https://en.wikipedia.org/wiki/Outlier>`_.
 

{pyperf-2.6.3 → pyperf-2.8.0}/doc/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # pyperf documentation build configuration file, created by
 # sphinx-quickstart on Wed Jun  1 15:28:03 2016.
@@ -42,16 +41,16 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'pyperf'
-copyright = u'2016, Victor Stinner'
-author = u'Victor Stinner'
+project = 'pyperf'
+copyright = '2016, Victor Stinner'
+author = 'Victor Stinner'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = release = '2.6.3'
+version = release = '2.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -216,8 +215,8 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  (master_doc, 'pyperf.tex', u'pyperf Documentation',
-   u'Victor Stinner', 'manual'),
+  (master_doc, 'pyperf.tex', 'pyperf Documentation',
+   'Victor Stinner', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -246,7 +245,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'pyperf', u'pyperf Documentation',
+    (master_doc, 'pyperf', 'pyperf Documentation',
      [author], 1)
 ]
 
@@ -260,7 +259,7 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  (master_doc, 'pyperf', u'pyperf Documentation',
+  (master_doc, 'pyperf', 'pyperf Documentation',
    author, 'pyperf', 'One line description of project.',
    'Miscellaneous'),
 ]

{pyperf-2.6.3 → pyperf-2.8.0}/doc/examples/bench_time_func.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
+import time
 import pyperf
 
 
 def bench_dict(loops, mydict):
     range_it = range(loops)
-    t0 = pyperf.perf_counter()
+    t0 = time.perf_counter()
 
     for _ in range_it:
         mydict['0']
@@ -18,7 +19,7 @@ def bench_dict(loops, mydict):
         mydict['800']
         mydict['900']
 
-    return pyperf.perf_counter() - t0
+    return time.perf_counter() - t0
 
 
 runner = pyperf.Runner()

{pyperf-2.6.3 → pyperf-2.8.0}/doc/examples.rst

@@ -31,8 +31,9 @@ bench_async_func() method
 timeit() method
 ---------------
 
-Benchmark using the :meth:`Runner.timeit` method to performance of sorting a
-sorted list of 1000 numbers using a ``key`` function (which does nothing):
+Benchmark using the :meth:`Runner.timeit` method to measure the performance of
+sorting a sorted list of 1000 numbers using a ``key`` function (which does
+nothing):
 
 .. literalinclude:: examples/bench_timeit.py
 

{pyperf-2.6.3 → pyperf-2.8.0}/doc/index.rst

@@ -4,7 +4,7 @@ Python pyperf module
 
 The Python ``pyperf`` module is a toolkit to write, run and analyze benchmarks.
 
-Documenation:
+Documentation:
 
 .. toctree::
    :maxdepth: 2

{pyperf-2.6.3 → pyperf-2.8.0}/doc/run_benchmark.rst

@@ -192,12 +192,14 @@ the benchmark less reliable since two runs may use a different number of warmup
 values.
 
 Specializer statistics (``pystats``)
-==================================
+====================================
 
 ``pyperf`` has built-in support for `specializer statistics (``pystats``) <https://docs.python.org/dev/using/configure.html#cmdoption-enable-pystats>`_.
-If running benchmarks on a CPython built with the ``--enable-pystats`` flag, pyperf will automatically collect ``pystats`` on the benchmark code by calling ``sys._stats_on`` immediately before the benchmark and calling ``sys._stats_off`` immediately after.
+If running benchmarks on a CPython built with the ``--enable-pystats`` flag, when you pass ``--hook pystats``, pyperf will collect ``pystats`` on the benchmark code by calling ``sys._stats_on`` immediately before the benchmark and calling ``sys._stats_off`` immediately after.
 Stats are not collected when running ``pyperf``'s own code or when warming up or calibrating the benchmarks.
 
+**New in 2.8.0:** The ``--hook pystats`` flag must be given to collect pystats.
+
 Due to the overhead of collecting the statistics, the timing results will be meaningless.
 
 The `Tools/scripts/summarize_stats.py <https://github.com/python/cpython/blob/main/Tools/scripts/summarize_stats.py>`_ script can be used to summarize the statistics in a human-readable form.

{pyperf-2.6.3 → pyperf-2.8.0}/doc/runner.rst

@@ -98,6 +98,7 @@ Option::
     --inherit-environ=VARS
     --copy-env
     --no-locale
+    --timeout TIMEOUT
     --track-memory
     --tracemalloc
 
@@ -140,6 +141,9 @@ Option::
   - ``LC_TELEPHONE``
   - ``LC_TIME``
 
+* ``--timeout``: set a timeout in seconds for an execution of the benchmark.
+  If the benchmark execution times out, pyperf exits with error code 124.
+  There is no time out by default.
 * ``--tracemalloc``: Use the ``tracemalloc`` module to track Python memory
   allocation and get the peak of memory usage in metadata
   (``tracemalloc_peak``). The module is only available on Python 3.4 and newer.

{pyperf-2.6.3 → pyperf-2.8.0}/doc/system.rst

@@ -23,7 +23,7 @@ be checked in benchmark metadata: it is enabled if the ``cpu_affinity``
 
 :func:`os.sched_setaffinity` is used to pin processes.
 
-Even if no CPU is isolated, CPU pining makes benchmarks more stable: use the
+Even if no CPU is isolated, CPU pinning makes benchmarks more stable: use the
 ``--affinity`` command line option.
 
 Check the CPU topology for HyperThreading and NUMA for best performances.

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/__init__.py

@@ -1,6 +1,6 @@
 from time import perf_counter
 
-VERSION = (2, 6, 3)
+VERSION = (2, 8, 0)
 __version__ = '.'.join(map(str, VERSION))
 
 # Export pyperf.perf_counter for backward compatibility with pyperf 1.7

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_bench.py

@@ -86,7 +86,7 @@ def _cached_attr(func):
     return method
 
 
-class Run(object):
+class Run:
     # Run is immutable, so it can be shared/exchanged between two benchmarks
 
     __slots__ = ('_warmups', '_values', '_metadata')
@@ -319,7 +319,7 @@ class Run(object):
         return self._replace(metadata=metadata2)
 
 
-class Benchmark(object):
+class Benchmark:
     def __init__(self, runs):
         self._runs = []   # list of Run objects
         self._clear_runs_cache()
@@ -627,7 +627,7 @@ class Benchmark(object):
         self._replace_runs(new_runs)
 
 
-class BenchmarkSuite(object):
+class BenchmarkSuite:
     def __init__(self, benchmarks, filename=None):
         if not benchmarks:
             raise ValueError("benchmarks must be a non-empty "
@@ -727,7 +727,7 @@ class BenchmarkSuite(object):
         if isinstance(filename, bytes):
             suffix = b'.gz'
         else:
-            suffix = u'.gz'
+            suffix = '.gz'
 
         if filename.endswith(suffix):
             # Use lazy import to limit imports on 'import pyperf'
@@ -770,7 +770,7 @@ class BenchmarkSuite(object):
         if isinstance(filename, bytes):
             suffix = b'.gz'
         else:
-            suffix = u'.gz'
+            suffix = '.gz'
 
         if not replace and os.path.exists(filename):
             raise OSError(errno.EEXIST, "File already exists")

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_cli.py

@@ -604,7 +604,7 @@ def catch_broken_pipe_error(file=None):
         # was closed by the consumer
         for file in files:
             file.flush()
-    except IOError as exc:
+    except OSError as exc:
         if exc.errno != errno.EPIPE:
             raise
         # got a broken pipe error: ignore it
@@ -615,5 +615,5 @@ def catch_broken_pipe_error(file=None):
         for file in files:
             try:
                 file.close()
-            except IOError:
+            except OSError:
                 pass

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_collect_metadata.py

@@ -111,6 +111,11 @@ def collect_python_metadata(metadata):
             cflags = normalize_text(cflags)
             metadata['python_cflags'] = cflags
 
+        config_args = sysconfig.get_config_var('CONFIG_ARGS')
+        if config_args:
+            config_args = normalize_text(config_args)
+            metadata['python_config_args'] = config_args
+
     # GC disabled?
     try:
         import gc
@@ -120,10 +125,6 @@ def collect_python_metadata(metadata):
         if not gc.isenabled():
             metadata['python_gc'] = 'disabled'
 
-    # pystats enabled?
-    if hasattr(sys, "_stats_clear"):
-        metadata['pystats'] = 'enabled'
-
 
 def read_proc(path):
     path = proc_path(path)
@@ -131,7 +132,7 @@ def read_proc(path):
         with open_text(path) as fp:
             for line in fp:
                 yield line.rstrip()
-    except (OSError, IOError):
+    except OSError:
         return
 
 
@@ -203,7 +204,7 @@ def collect_system_metadata(metadata):
 
 def collect_memory_metadata(metadata):
     if resource is not None:
-        metadata["mem_max_rss"] = get_max_rss()
+        metadata["mem_max_rss"] = get_max_rss(children=False)
 
     # Note: Don't collect VmPeak of /proc/self/status on Linux because it is
     # not accurate. See pyperf._linux_memory for more accurate memory metrics.
@@ -327,7 +328,7 @@ def get_cpu_temperature(path, cpu_temp):
 
         try:
             temp_label = read_first_line(template % 'label', error=True)
-        except IOError:
+        except OSError:
             break
 
         temp_input = read_first_line(template % 'input', error=True)

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_command.py

@@ -1,4 +1,5 @@
 import functools
+import json
 import os.path
 import subprocess
 import sys
@@ -7,6 +8,26 @@ from pyperf._utils import shell_quote, popen_communicate
 from pyperf._worker import WorkerTask
 
 
+def parse_subprocess_data(output):
+    # Parse the data send from the subprocess.
+    # It is three lines containing:
+    #    - The runtime (in seconds)
+    #    - max_rss (or -1, if not able to compute)
+    #    - The metadata to add to the benchmark entry, as a JSON dictionary
+
+    rss = None
+    metadata = {}
+    try:
+        lines = output.splitlines()
+        timing = float(lines[0])
+        rss = int(lines[1])
+        metadata = json.loads(lines[2])
+    except ValueError:
+        raise ValueError("failed to parse worker output: %r" % output)
+
+    return timing, rss, metadata
+
+
 def bench_command(command, task, loops):
     path = os.path.dirname(__file__)
     script = os.path.join(path, '_process_time.py')
@@ -22,19 +43,15 @@ def bench_command(command, task, loops):
         raise Exception("Command failed with exit code %s"
                         % proc.returncode)
 
-    rss = None
-    try:
-        lines = output.splitlines()
-        timing = float(lines[0])
-        if len(lines) >= 2:
-            rss = int(lines[1])
-    except ValueError:
-        raise ValueError("failed to parse script output: %r" % output)
+    timing, rss, metadata = parse_subprocess_data(output)
 
-    if rss:
+    if rss and rss > 0:
         # store the maximum
         max_rss = task.metadata.get('command_max_rss', 0)
         task.metadata['command_max_rss'] = max(max_rss, rss)
+
+    task.metadata.update(metadata)
+
     return timing
 
 

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_compare.py

@@ -53,7 +53,7 @@ def get_tags_for_result(result):
     return result.ref.benchmark.get_metadata().get("tags", [])
 
 
-class CompareResult(object):
+class CompareResult:
     def __init__(self, ref, changed, min_speed=None):
         # CompareData object
         self.ref = ref

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_formatter.py

@@ -30,9 +30,9 @@ def format_filesize(size):
             return '%.0f byte' % size
 
     if size > 10 * 1024 * 1024:
-        return '%.1f MB' % (size / (1024.0 * 1024.0))
+        return '%.1f MiB' % (size / (1024.0 * 1024.0))
 
-    return '%.1f kB' % (size / 1024.0)
+    return '%.1f KiB' % (size / 1024.0)
 
 
 def format_filesizes(sizes):

pyperf-2.8.0/pyperf/_hooks.py

@@ -0,0 +1,110 @@
+# Hooks are installable context managers defined as entry points so that
+# arbitrary code can by run right before and after the actual internal
+# benchmarking code is run.
+
+
+import abc
+import importlib.metadata
+import sys
+
+
+def get_hooks():
+    hook_prefix = "pyperf.hook"
+    entry_points = importlib.metadata.entry_points()
+    if sys.version_info[:2] < (3, 10):
+        group = entry_points[hook_prefix]
+    else:
+        group = entry_points.select(group=hook_prefix)
+    return group
+
+
+def get_hook_names():
+    return (x.name for x in get_hooks())
+
+
+def get_selected_hooks(hook_names):
+    if hook_names is None:
+        return
+
+    hook_mapping = {hook.name: hook for hook in get_hooks()}
+    for hook_name in hook_names:
+        yield hook_mapping[hook_name]
+
+
+def instantiate_selected_hooks(hook_names):
+    hook_managers = {}
+    for hook in get_selected_hooks(hook_names):
+        try:
+            hook_managers[hook.name] = hook.load()()
+        except HookError as e:
+            print(f"ERROR setting up hook '{hook.name}':", file=sys.stderr)
+            print(str(e), file=sys.stderr)
+            sys.exit(1)
+
+    return hook_managers
+
+
+class HookError(Exception):
+    pass
+
+
+class HookBase(abc.ABC):
+    def __init__(self):
+        """
+        Create a new instance of the hook.
+        """
+        pass
+
+    def teardown(self, _metadata):
+        """
+        Called when the hook is completed for a process. May add any information
+        collected to the passed-in `metadata` dictionary.
+        """
+        pass
+
+    def __enter__(self):
+        """
+        Called immediately before running benchmark code.
+
+        May be called multiple times per instance.
+        """
+        pass
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        """
+        Called immediately after running benchmark code.
+        """
+        pass
+
+
+class _test_hook(HookBase):
+    def __init__(self):
+        self._count = 0
+
+    def teardown(self, metadata):
+        metadata["_test_hook"] = self._count
+
+    def __enter__(self):
+        self._count += 1
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        pass
+
+
+class pystats(HookBase):
+    def __init__(self):
+        if not hasattr(sys, "_stats_on"):
+            raise HookError(
+                "Can not collect pystats because python was not built with --enable-pystats"
+            )
+        sys._stats_off()
+        sys._stats_clear()
+
+    def teardown(self, metadata):
+        metadata["pystats"] = "enabled"
+
+    def __enter__(self):
+        sys._stats_on()
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        sys._stats_off()

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_linux_memory.py

@@ -54,7 +54,7 @@ def check_tracking_memory():
     mem_thread = PeakMemoryUsageThread()
     try:
         mem_thread.get()
-    except IOError as exc:
+    except OSError as exc:
         path = proc_path("self/smaps")
         return "unable to read %s: %s" % (path, exc)
 

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_manager.py

@@ -7,12 +7,14 @@ from pyperf._formatter import format_number
 from pyperf._utils import MS_WINDOWS, create_environ, create_pipe, popen_killer
 
 
+EXIT_TIMEOUT = 60
+
 # Limit to 5 calibration processes
 # (10 if calibration is needed for loops and warmups)
 MAX_CALIBRATION = 5
 
 
-class Manager(object):
+class Manager:
     """
     Manager process which spawns worker processes to:
     - calibrate warmups
@@ -69,6 +71,13 @@ class Manager(object):
         if args.profile:
             cmd.extend(['--profile', args.profile])
 
+        if args.timeout:
+            cmd.extend(['--timeout', str(args.timeout)])
+
+        if args.hook:
+            for hook in args.hook:
+                cmd.extend(['--hook', hook])
+
         if self.runner._add_cmdline_args:
             self.runner._add_cmdline_args(cmd, args)
 
@@ -98,10 +107,12 @@ class Manager(object):
                 proc = subprocess.Popen(cmd, env=env, **kw)
 
             with popen_killer(proc):
-                with rpipe.open_text() as rfile:
-                    bench_json = rfile.read()
-
-                exitcode = proc.wait()
+                try:
+                    bench_json = rpipe.read_text(timeout=self.args.timeout)
+                    exitcode = proc.wait(timeout=EXIT_TIMEOUT)
+                except TimeoutError as exc:
+                    print(exc)
+                    sys.exit(124)
 
         if exitcode:
             raise RuntimeError("%s failed with exit code %s"

{pyperf-2.6.3 → pyperf-2.8.0}/pyperf/_metadata.py

@@ -137,7 +137,7 @@ def format_metadata(name, value):
     return info.formatter(value)
 
 
-class Metadata(object):
+class Metadata:
     def __init__(self, name, value):
         self._name = name
         self._value = value