pyperf 2.8.1__tar.gz → 2.9.0__tar.gz

{pyperf-2.8.1/pyperf.egg-info → pyperf-2.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pyperf
-Version: 2.8.1
+Version: 2.9.0
 Summary: Python module to run and analyze benchmarks
 Author-email: Victor Stinner <vstinner@redhat.com>
 Maintainer-email: Dong-hee Na <donghee.na@python.org>

{pyperf-2.8.1 → pyperf-2.9.0}/doc/api.rst

@@ -345,6 +345,19 @@ Benchmark class
 
       Raise an exception if the benchmark has no values.
 
+   .. method:: required_nprocesses()
+
+      Determines the number of separate process runs that would be required
+      achieve stable results. Specifically, the target is to have 95% certainty
+      that there is a variance of less than 1%. If the result is greater than
+      the number of processes recorded in the input data, the value is
+      meaningless and only means "more samples are required".
+
+      The method used is described in this Wikipedia article about estimating
+      the sampling of a mean:
+
+      https://en.wikipedia.org/wiki/Sample_size_determination#Estimation_of_a_mean
+
    .. method:: update_metadata(metadata: dict)
 
       Update metadata of all runs of the benchmark.

{pyperf-2.8.1 → pyperf-2.9.0}/doc/changelog.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+Version 2.9.0 (2025-03-03)
+--------------------------
+
+* Feature: A new hook, ``perf_record``, is available to turn Linux perf profiling on and off around the benchmark code.
+  To use it, pass ``--hook perf_record``.
+  Patch by Matt Page.
+* Feature: pyperf will now warn if you are collecting too few or too many samples for stable results.
+  Patch by Michael Droettboom
+
 Version 2.8.1 (2024-11-13)
 --------------------------
 

{pyperf-2.8.1 → pyperf-2.9.0}/doc/conf.py

@@ -50,7 +50,7 @@ author = 'Victor Stinner'
 # built documents.
 #
 # The short X.Y version.
-version = release = '2.8.1'
+version = release = '2.9.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

{pyperf-2.8.1 → pyperf-2.9.0}/doc/run_benchmark.rst

@@ -206,3 +206,17 @@ The `Tools/scripts/summarize_stats.py <https://github.com/python/cpython/blob/ma
 
 Statistics are not cleared between runs.
 If you need to delete statistics from a previous run, remove the files in ``/tmp/py_stats`` (Unix) or ``C:\temp\py_stats`` (Windows).
+
+Profiling benchmarks using ``perf record``
+==========================================
+``pyperf`` supports profiling benchmark execution using ``perf
+record``. ``perf`` is only enabled while the benchmark is running to avoid
+profiling unrelated parts of ``pyperf`` itself.
+
+One profile data file is generated for each benchmark run.  These files have
+the basename of ``perf.data.<uuid>`` and are written to the current directory
+by default. The directory can be overridden by setting the
+``PYPERF_PERF_RECORD_DATA_DIR`` environment variable.
+
+The value of the ``PYPERF_PERF_RECORD_EXTRA_OPTS`` environment variable is
+appended to the command line of ``perf record`` if it is provided.

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/__init__.py

@@ -1,6 +1,6 @@
 from time import perf_counter
 
-VERSION = (2, 8, 1)
+VERSION = (2, 9, 0)
 __version__ = '.'.join(map(str, VERSION))
 
 # Export pyperf.perf_counter for backward compatibility with pyperf 1.7

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/__main__.py

@@ -455,7 +455,8 @@ def display_benchmarks(args, show_metadata=False, hist=False, stats=False,
                                            dump=dump,
                                            checks=checks,
                                            result=result,
-                                           display_runs_args=display_runs_args)
+                                           display_runs_args=display_runs_args,
+                                           only_checks=only_checks)
 
             if bench_lines:
                 empty_line(lines)
@@ -491,10 +492,13 @@ def display_benchmarks(args, show_metadata=False, hist=False, stats=False,
                 empty_line(output)
                 output.extend(lines)
 
+        contains_warning = False
         for line in output:
+            if line.startswith("WARNING:"):
+                contains_warning = True
             print(line)
 
-        if not output and only_checks:
+        if not contains_warning and only_checks:
             if len(data) == 1:
                 print("The benchmark seems to be stable")
             else:

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/_bench.py

@@ -424,6 +424,47 @@ class Benchmark:
             raise ValueError("MAD must be >= 0")
         return value
 
+    def required_nprocesses(self):
+        """
+        Determines the number of separate process runs that would be required
+        achieve stable results. Specifically, the target is to have 95%
+        certainty that there is a variance of less than 1%. If the result is
+        greater than the number of processes recorded in the input data, the
+        value is meaningless and only means "more samples are required".
+
+        The method used is described in this Wikipedia article about estimating
+        the sampling of a mean:
+
+        https://en.wikipedia.org/wiki/Sample_size_determination#Estimation_of_a_mean
+        """
+        # Get the means of the values per process. The values within the process
+        # often vary considerably (e.g. due to cache effects), but the variances
+        # between processes should be fairly consistent. Additionally, this
+        # value is intended to be advice for the number of processes to run.
+        values = []
+        for run in self._runs:
+            if len(run.values):
+                values.append(statistics.mean(run.values))
+
+        if len(values) < 2:
+            return None
+
+        total = math.fsum(values)
+        mean = total / len(values)
+        stddev = statistics.stdev(values)
+
+        # Normalize the stddev so we can target "percentage changed" rather than
+        # absolute time
+        sigma = stddev / mean
+
+        # 95% certainty
+        Z = 1.96
+        # 1% variation
+        W = 0.01
+
+        # (4Z²σ²)/(W²)
+        return math.ceil((4 * Z ** 2 * sigma ** 2) / (W ** 2))
+
     def percentile(self, p):
         if not (0 <= p <= 100):
             raise ValueError("p must be in the range [0; 100]")

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/_cli.py

@@ -400,7 +400,7 @@ def format_histogram(benchmarks, bins=20, extend=False, lines=None,
     return lines
 
 
-def format_checks(bench, lines=None):
+def format_checks(bench, lines=None, check_too_many_processes=False):
     if lines is None:
         lines = []
 
@@ -412,6 +412,7 @@ def format_checks(bench, lines=None):
     mean = bench.mean()
     warnings = []
     warn = warnings.append
+    required_nprocesses = None
 
     # Display a warning if the standard deviation is greater than 10%
     # of the mean
@@ -421,6 +422,14 @@ def format_checks(bench, lines=None):
         if percent >= 10.0:
             warn("the standard deviation (%s) is %.0f%% of the mean (%s)"
                  % (bench.format_value(stdev), percent, bench.format_value(mean)))
+        else:
+            # display a warning if the number of samples isn't enough to get a stable result
+            required_nprocesses = bench.required_nprocesses()
+            if (
+                required_nprocesses is not None and
+                required_nprocesses > len(bench._runs)
+            ):
+                warn("Not enough samples to get a stable result (95% certainly of less than 1% variation)")
 
     # Minimum and maximum, detect obvious outliers
     for minimum, value in (
@@ -457,6 +466,19 @@ def format_checks(bench, lines=None):
         lines.append("Use pyperf stats, pyperf dump and pyperf hist to analyze results.")
         lines.append("Use --quiet option to hide these warnings.")
 
+    if check_too_many_processes:
+        if required_nprocesses is None:
+            required_nprocesses = bench.required_nprocesses()
+        if (
+            required_nprocesses is not None and
+            required_nprocesses < len(bench._runs) * 0.75
+        ):
+            lines.append("Benchmark was run more times than necessary to get a stable result.")
+            lines.append(
+                "Consider passing processes=%d to the Runner constructor to save time." %
+                required_nprocesses
+            )
+
     # Warn if nohz_full+intel_pstate combo if found in cpu_config metadata
     for run in bench._runs:
         cpu_config = run._metadata.get('cpu_config')
@@ -549,7 +571,7 @@ def format_result(bench):
 
 def format_benchmark(bench, checks=True, metadata=False,
                      dump=False, stats=False, hist=False, show_name=False,
-                     result=True, display_runs_args=None):
+                     result=True, display_runs_args=None, only_checks=False):
     lines = []
 
     if metadata:
@@ -568,7 +590,7 @@ def format_benchmark(bench, checks=True, metadata=False,
         format_stats(bench, lines=lines)
 
     if checks:
-        format_checks(bench, lines=lines)
+        format_checks(bench, lines=lines, check_too_many_processes=only_checks)
 
     if result:
         empty_line(lines)

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/_hooks.py

@@ -5,7 +5,12 @@
 
 import abc
 import importlib.metadata
+import os.path
+import shlex
+import subprocess
 import sys
+import tempfile
+import uuid
 
 
 def get_hooks():
@@ -108,3 +113,58 @@ class pystats(HookBase):
 
     def __exit__(self, _exc_type, _exc_value, _traceback):
         sys._stats_off()
+
+
+class perf_record(HookBase):
+    """Profile the benchmark using perf-record.
+
+    Profile data is written to the current directory directory by default, or
+    to the value of the `PYPERF_PERF_RECORD_DATA_DIR` environment variable, if
+    it is provided.
+
+    Profile data files have a basename of the form `perf.data.<uuid>`
+
+    The value of the `PYPERF_PERF_RECORD_EXTRA_OPTS` environment variable is
+    appended to the command line of perf-record, if provided.
+    """
+
+    def __init__(self):
+        self.tempdir = tempfile.TemporaryDirectory()
+        self.ctl_fifo = self.mkfifo(self.tempdir.name, "ctl_fifo")
+        self.ack_fifo = self.mkfifo(self.tempdir.name, "ack_fifo")
+        perf_data_dir = os.environ.get("PYPERF_PERF_RECORD_DATA_DIR", "")
+        perf_data_basename = f"perf.data.{uuid.uuid4()}"
+        cmd = ["perf", "record",
+               "--pid", str(os.getpid()),
+               "--output", os.path.join(perf_data_dir, perf_data_basename),
+               "--control", f"fifo:{self.ctl_fifo},{self.ack_fifo}"]
+        extra_opts = os.environ.get("PYPERF_PERF_RECORD_EXTRA_OPTS", "")
+        cmd += shlex.split(extra_opts)
+        self.perf = subprocess.Popen(
+            cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        self.ctl_fd = open(self.ctl_fifo, "w")
+        self.ack_fd = open(self.ack_fifo, "r")
+
+    def __enter__(self):
+        self.exec_perf_cmd("enable")
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        self.exec_perf_cmd("disable")
+
+    def teardown(self, metadata):
+        try:
+            self.exec_perf_cmd("stop")
+            self.perf.wait(timeout=120)
+        finally:
+            self.ctl_fd.close()
+            self.ack_fd.close()
+
+    def mkfifo(self, tmpdir, basename):
+        path = os.path.join(tmpdir, basename)
+        os.mkfifo(path)
+        return path
+
+    def exec_perf_cmd(self, cmd):
+        self.ctl_fd.write(f"{cmd}\n")
+        self.ctl_fd.flush()
+        self.ack_fd.readline()

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/_utils.py

@@ -270,7 +270,10 @@ def create_environ(inherit_environ, locale, copy_all):
     env = {}
     copy_env = ["PATH", "HOME", "TEMP", "COMSPEC", "SystemRoot", "SystemDrive",
                 # Python specific variables
-                "PYTHONPATH", "PYTHON_CPU_COUNT", "PYTHON_GIL"]
+                "PYTHONPATH", "PYTHON_CPU_COUNT", "PYTHON_GIL",
+                # Pyperf specific variables
+                "PYPERF_PERF_RECORD_DATA_DIR", "PYPERF_PERF_RECORD_EXTRA_OPTS",
+                ]
     if locale:
         copy_env.extend(('LANG', 'LC_ADDRESS', 'LC_ALL', 'LC_COLLATE',
                          'LC_CTYPE', 'LC_IDENTIFICATION', 'LC_MEASUREMENT',

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf/tests/test_perf_cli.py

@@ -628,8 +628,18 @@ class TestPerfCLI(BaseTestCase, unittest.TestCase):
 
     def test_check_stable(self):
         stdout = self.run_command('check', TELCO)
-        self.assertEqual(stdout.rstrip(),
-                         'The benchmark seems to be stable')
+        self.assertIn(
+            textwrap.dedent(
+                """
+                Benchmark was run more times than necessary to get a stable result.
+                Consider passing processes=7 to the Runner constructor to save time.
+                """
+            ).strip(), stdout.rstrip()
+        )
+        self.assertIn(
+            'The benchmark seems to be stable',
+            stdout.rstrip()
+        )
 
     def test_command(self):
         command = [sys.executable, '-c', 'pass']
@@ -689,7 +699,7 @@ class TestPerfCLI(BaseTestCase, unittest.TestCase):
                              '[1,2]*1000',
                              '-o', tmp_name)
             bench = pyperf.Benchmark.load(tmp_name)
-        
+
         self._check_track_memory_bench(bench, loops=5)
 
     def test_track_memory(self):

{pyperf-2.8.1 → pyperf-2.9.0/pyperf.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pyperf
-Version: 2.8.1
+Version: 2.9.0
 Summary: Python module to run and analyze benchmarks
 Author-email: Victor Stinner <vstinner@redhat.com>
 Maintainer-email: Dong-hee Na <donghee.na@python.org>

{pyperf-2.8.1 → pyperf-2.9.0}/pyperf.egg-info/entry_points.txt

@@ -3,4 +3,5 @@ pyperf = pyperf.__main__:main
 
 [pyperf.hook]
 _test_hook = pyperf._hooks:_test_hook
+perf_record = pyperf._hooks:perf_record
 pystats = pyperf._hooks:pystats

{pyperf-2.8.1 → pyperf-2.9.0}/pyproject.toml

@@ -56,6 +56,7 @@ dev = [
 pyperf = "pyperf.__main__:main"
 
 [project.entry-points."pyperf.hook"]
+perf_record = "pyperf._hooks:perf_record"
 pystats = "pyperf._hooks:pystats"
 _test_hook = "pyperf._hooks:_test_hook"