pytest-codspeed 2.2.1__tar.gz → 3.0.0b0__tar.gz

{pytest_codspeed-2.2.1 → pytest_codspeed-3.0.0b0}/.gitignore

@@ -162,3 +162,5 @@ cython_debug/
 .venvubuntu
 .python-version
 *.o
+
+.codspeed

{pytest_codspeed-2.2.1 → pytest_codspeed-3.0.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pytest-codspeed
-Version: 2.2.1
+Version: 3.0.0b0
 Summary: Pytest plugin to create CodSpeed benchmarks
 Project-URL: Homepage, https://codspeed.io/
 Project-URL: Documentation, https://docs.codspeed.io/
@@ -35,27 +35,29 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: System :: Benchmark
 Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
-Requires-Python: >=3.7
-Requires-Dist: cffi>=1.15.1
+Requires-Python: >=3.8
+Requires-Dist: cffi>=1.17.1
 Requires-Dist: filelock>=3.12.2
+Requires-Dist: importlib-metadata>=8.5.0; python_version < '3.10'
 Requires-Dist: pytest>=3.8
+Requires-Dist: rich>=13.8.1
 Requires-Dist: setuptools; python_full_version >= '3.12.0'
 Provides-Extra: compat
 Requires-Dist: pytest-benchmark~=4.0.0; extra == 'compat'
-Requires-Dist: pytest-xdist~=2.0.0; extra == 'compat'
+Requires-Dist: pytest-xdist~=3.6.1; extra == 'compat'
 Provides-Extra: lint
-Requires-Dist: mypy~=1.3.0; extra == 'lint'
-Requires-Dist: ruff~=0.3.3; extra == 'lint'
+Requires-Dist: mypy~=1.11.2; extra == 'lint'
+Requires-Dist: ruff~=0.6.5; extra == 'lint'
 Provides-Extra: test
 Requires-Dist: pytest-cov~=4.0.0; extra == 'test'
 Requires-Dist: pytest~=7.0; extra == 'test'
@@ -66,7 +68,7 @@ Description-Content-Type: text/markdown
 
 [![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
 [![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
-![Python Version](https://img.shields.io/badge/python-3.7%20|%203.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12-informational.svg)
+![Python Version](https://img.shields.io/badge/python-3.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12%20|%203.13-informational.svg)
 [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
 [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
 
@@ -76,7 +78,7 @@ Pytest plugin to create CodSpeed benchmarks
 
 ## Requirements
 
-**Python**: 3.7 and later
+**Python**: 3.8 and later
 
 **pytest**: any recent version
 
@@ -163,14 +165,14 @@ jobs:
   benchmarks:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.12"
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Run benchmarks
-        uses: CodSpeedHQ/action@v2
+        uses: CodSpeedHQ/action@v3
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: pytest tests/ --codspeed

{pytest_codspeed-2.2.1 → pytest_codspeed-3.0.0b0}/README.md

@@ -3,7 +3,7 @@
 
 [![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
 [![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
-![Python Version](https://img.shields.io/badge/python-3.7%20|%203.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12-informational.svg)
+![Python Version](https://img.shields.io/badge/python-3.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12%20|%203.13-informational.svg)
 [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
 [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
 
@@ -13,7 +13,7 @@ Pytest plugin to create CodSpeed benchmarks
 
 ## Requirements
 
-**Python**: 3.7 and later
+**Python**: 3.8 and later
 
 **pytest**: any recent version
 
@@ -100,14 +100,14 @@ jobs:
   benchmarks:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.12"
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Run benchmarks
-        uses: CodSpeedHQ/action@v2
+        uses: CodSpeedHQ/action@v3
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: pytest tests/ --codspeed

{pytest_codspeed-2.2.1 → pytest_codspeed-3.0.0b0}/pyproject.toml

@@ -13,7 +13,7 @@ dynamic = ["version"]
 description = "Pytest plugin to create CodSpeed benchmarks"
 readme = "README.md"
 license = { file = "LICENSE" }
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 authors = [{ name = "Arthur Pastel", email = "arthur@codspeed.io" }]
 keywords = ["codspeed", "benchmark", "performance", "pytest"]
 classifiers = [
@@ -23,31 +23,36 @@ classifiers = [
     "Intended Audience :: Information Technology",
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Software Development :: Testing",
     "Topic :: System :: Benchmark",
     "Topic :: Utilities",
     "Typing :: Typed",
 ]
 dependencies = [
-    "cffi >= 1.15.1",
+    "cffi >= 1.17.1",
     # cffi doesn't automatically install setuptools with python 3.12+
     # cf https://github.com/python-cffi/cffi/releases/tag/v1.16.0
     "setuptools; python_full_version >= '3.12.0'",
     "pytest>=3.8",
     "filelock >= 3.12.2",
+    "rich>=13.8.1",
+    "importlib-metadata>=8.5.0; python_version < '3.10'",
 ]
 
 [project.optional-dependencies]
-lint = ["mypy ~= 1.3.0", "ruff ~= 0.3.3"]
-compat = ["pytest-benchmark ~= 4.0.0", "pytest-xdist ~= 2.0.0"]
+lint = ["mypy ~= 1.11.2", "ruff ~= 0.6.5"]
+compat = [
+    "pytest-benchmark ~= 4.0.0",
+    "pytest-xdist ~= 3.6.1",
+    # "pytest-speed>=0.3.5",
+]
 test = ["pytest ~= 7.0", "pytest-cov ~= 4.0.0"]
-
 [project.entry-points]
 pytest11 = { codspeed = "pytest_codspeed.plugin" }
 
@@ -59,7 +64,7 @@ features = ["lint", "test", "compat"]
 features = ["test"]
 
 [[tool.hatch.envs.test.matrix]]
-python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 features = ["compat", "test"]
 
 [tool.hatch.version]
@@ -72,9 +77,11 @@ include = ["/src"]
 python_version = "3.12"
 
 [tool.ruff]
-line-length = 88
+target-version = "py37"
+
 [tool.ruff.lint]
-select = ["E", "F", "I", "C"]
+select = ["E", "F", "I", "C", "TCH", "FA", "UP"]
+flake8-type-checking = { exempt-modules = [], strict = true }
 
 [tool.isort]
 line_length = 88
@@ -85,8 +92,9 @@ force_grid_wrap = 0
 float_to_top = true
 
 [tool.pytest.ini_options]
-addopts = "--ignore=tests/benchmarks --ignore=tests/examples"
+addopts = "--ignore=tests/benchmarks --ignore=tests/examples --ignore=tests/benchmarks/TheAlgorithms"
 filterwarnings = ["ignore::DeprecationWarning:pytest_benchmark.utils.*:"]
+pythonpath = ["tests/benchmarks/TheAlgorithms"]
 
 [tool.coverage.run]
 branch = true

{pytest_codspeed-2.2.1 → pytest_codspeed-3.0.0b0}/src/pytest_codspeed/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.2.1"
+__version__ = "3.0.0b0"
 
 from .plugin import BenchmarkFixture
 

pytest_codspeed-3.0.0b0/src/pytest_codspeed/instruments/__init__.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from enum import Enum
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any, Callable, ClassVar, ParamSpec, TypeVar
+
+    import pytest
+
+    from pytest_codspeed.plugin import CodSpeedConfig
+
+    T = TypeVar("T")
+    P = ParamSpec("P")
+
+
+class Instrument(metaclass=ABCMeta):
+    instrument: ClassVar[str]
+
+    @abstractmethod
+    def __init__(self, config: CodSpeedConfig): ...
+
+    @abstractmethod
+    def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
+
+    @abstractmethod
+    def measure(
+        self,
+        name: str,
+        uri: str,
+        fn: Callable[P, T],
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> T: ...
+
+    @abstractmethod
+    def report(self, session: pytest.Session) -> None: ...
+
+    @abstractmethod
+    def get_result_dict(
+        self,
+    ) -> dict[str, Any]: ...
+
+
+class MeasurementMode(str, Enum):
+    Instrumentation = "instrumentation"
+    WallTime = "walltime"
+
+
+def get_instrument_from_mode(mode: MeasurementMode) -> type[Instrument]:
+    from pytest_codspeed.instruments.valgrind import (
+        ValgrindInstrument,
+    )
+    from pytest_codspeed.instruments.walltime import WallTimeInstrument
+
+    if mode == MeasurementMode.Instrumentation:
+        return ValgrindInstrument
+    else:
+        return WallTimeInstrument

pytest_codspeed-3.0.0b0/src/pytest_codspeed/instruments/valgrind/__init__.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import os
+import sys
+from typing import TYPE_CHECKING
+
+from pytest_codspeed import __version__
+from pytest_codspeed.instruments import Instrument
+from pytest_codspeed.instruments.valgrind._wrapper import get_lib
+
+if TYPE_CHECKING:
+    from typing import Any, Callable
+
+    from pytest import Session
+
+    from pytest_codspeed.instruments import P, T
+    from pytest_codspeed.instruments.valgrind._wrapper import LibType
+    from pytest_codspeed.plugin import CodSpeedConfig
+
+SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
+
+
+class ValgrindInstrument(Instrument):
+    instrument = "valgrind"
+    lib: LibType | None
+
+    def __init__(self, config: CodSpeedConfig) -> None:
+        self.benchmark_count = 0
+        self.should_measure = os.environ.get("CODSPEED_ENV") is not None
+        if self.should_measure:
+            self.lib = get_lib()
+            self.lib.dump_stats_at(
+                f"Metadata: pytest-codspeed {__version__}".encode("ascii")
+            )
+            if SUPPORTS_PERF_TRAMPOLINE:
+                sys.activate_stack_trampoline("perf")  # type: ignore
+        else:
+            self.lib = None
+
+    def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
+        config = (
+            f"mode: instrumentation, "
+            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE  else 'not supported'}"
+        )
+        warnings = []
+        if not self.should_measure:
+            warnings.append(
+                "\033[1m"
+                "NOTICE: codspeed is enabled, but no performance measurement"
+                " will be made since it's running in an unknown environment."
+                "\033[0m"
+            )
+        return config, warnings
+
+    def measure(
+        self,
+        name: str,
+        uri: str,
+        fn: Callable[P, T],
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> T:
+        self.benchmark_count += 1
+        if self.lib is None:  # Thus should_measure is False
+            return fn(*args, **kwargs)
+
+        def __codspeed_root_frame__() -> T:
+            return fn(*args, **kwargs)
+
+        if SUPPORTS_PERF_TRAMPOLINE:
+            # Warmup CPython performance map cache
+            __codspeed_root_frame__()
+
+        self.lib.zero_stats()
+        self.lib.start_instrumentation()
+        try:
+            return __codspeed_root_frame__()
+        finally:
+            # Ensure instrumentation is stopped even if the test failed
+            self.lib.stop_instrumentation()
+            self.lib.dump_stats_at(uri.encode("ascii"))
+
+    def report(self, session: Session) -> None:
+        reporter = session.config.pluginmanager.get_plugin("terminalreporter")
+        count_suffix = "benchmarked" if self.should_measure else "benchmark tested"
+        reporter.write_sep(
+            "=",
+            f"{self.benchmark_count} {count_suffix}",
+        )
+
+    def get_result_dict(self) -> dict[str, Any]:
+        return {
+            "instrument": {"type": self.instrument},
+            # bench results will be dumped by valgrind
+        }

{pytest_codspeed-2.2.1/src/pytest_codspeed → pytest_codspeed-3.0.0b0/src/pytest_codspeed/instruments/valgrind}/_wrapper/.gitignore

@@ -1 +1,2 @@
 dist_callgrind_wrapper.*
+build.lock

{pytest_codspeed-2.2.1/src/pytest_codspeed → pytest_codspeed-3.0.0b0/src/pytest_codspeed/instruments/valgrind}/_wrapper/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 from typing import TYPE_CHECKING
 
@@ -22,7 +24,7 @@ def _get_ffi():
     return ffi
 
 
-def get_lib() -> "LibType":
+def get_lib() -> LibType:
     try:
         ffi = _get_ffi()
         build_lock = FileLock(f"{_wrapper_dir}/build.lock")

pytest_codspeed-3.0.0b0/src/pytest_codspeed/instruments/walltime.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from math import ceil
+from statistics import mean, median, stdev
+from time import get_clock_info, perf_counter_ns
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from pytest_codspeed.instruments import Instrument
+
+if TYPE_CHECKING:
+    from typing import Any, Callable
+
+    from pytest import Session
+
+    from pytest_codspeed.instruments import P, T
+    from pytest_codspeed.plugin import CodSpeedConfig
+
+DEFAULT_WARMUP_TIME_NS = 1_000_000_000
+DEFAULT_MAX_TIME_NS = 3_000_000_000
+TIMER_RESOLUTION_NS = get_clock_info("perf_counter").resolution * 1e9
+DEFAULT_MIN_ROUND_TIME_NS = TIMER_RESOLUTION_NS * 100
+
+
+@dataclass
+class BenchmarkConfig:
+    warmup_time_ns: int
+    min_round_time_ns: float
+    max_time_ns: int
+    max_rounds: int | None
+
+    @classmethod
+    def from_codspeed_config(cls, config: CodSpeedConfig) -> BenchmarkConfig:
+        return cls(
+            warmup_time_ns=config.warmup_time_ns
+            if config.warmup_time_ns is not None
+            else DEFAULT_WARMUP_TIME_NS,
+            min_round_time_ns=DEFAULT_MIN_ROUND_TIME_NS,
+            max_time_ns=DEFAULT_MAX_TIME_NS,
+            max_rounds=config.max_rounds or None,
+        )
+
+
+@dataclass
+class BenchmarkStats:
+    min_ns: float
+    max_ns: float
+    stdev_ns: float
+    mean_ns: float
+    median_ns: float
+
+    rounds: int
+    total_time: float
+    outlier_rounds: int
+    iter_per_round: int
+    warmup_iters: int
+
+    @classmethod
+    def from_list(
+        cls,
+        times_ns: list[float],
+        *,
+        rounds: int,
+        iter_per_round: int,
+        warmup_iters: int,
+        total_time: float,
+    ) -> BenchmarkStats:
+        stdev_ns = stdev(times_ns) if len(times_ns) > 1 else 0
+        mean_ns = mean(times_ns)
+        return cls(
+            min_ns=min(times_ns),
+            max_ns=max(times_ns),
+            stdev_ns=stdev_ns,
+            mean_ns=mean_ns,
+            median_ns=median(times_ns),
+            rounds=rounds,
+            total_time=total_time,
+            outlier_rounds=sum(1 for t in times_ns if abs(t - mean_ns) > stdev_ns),
+            iter_per_round=iter_per_round,
+            warmup_iters=warmup_iters,
+        )
+
+
+@dataclass
+class Benchmark:
+    name: str
+    uri: str
+
+    config: BenchmarkConfig
+    stats: BenchmarkStats
+
+
+def run_benchmark(
+    name: str, uri: str, fn: Callable[P, T], args, kwargs, config: BenchmarkConfig
+) -> tuple[Benchmark, T]:
+    # Compute the actual result of the function
+    out = fn(*args, **kwargs)
+
+    # Warmup
+    times_ns: list[float] = []
+    warmup_start = start = perf_counter_ns()
+    while True:
+        start = perf_counter_ns()
+        fn(*args, **kwargs)
+        end = perf_counter_ns()
+        times_ns.append(end - start)
+        if end - warmup_start > config.warmup_time_ns:
+            break
+
+    # Round sizing
+    warmup_mean_ns = mean(times_ns)
+    warmup_iters = len(times_ns)
+    times_ns.clear()
+    iter_per_round = (
+        int(ceil(config.min_round_time_ns / warmup_mean_ns))
+        if warmup_mean_ns <= config.min_round_time_ns
+        else 1
+    )
+    if config.max_rounds is None:
+        round_time_ns = warmup_mean_ns * iter_per_round
+        rounds = int(config.max_time_ns / round_time_ns)
+    else:
+        rounds = config.max_rounds
+
+    # Benchmark
+    iter_range = range(iter_per_round)
+    run_start = perf_counter_ns()
+    for _ in range(rounds):
+        start = perf_counter_ns()
+        for _ in iter_range:
+            fn(*args, **kwargs)
+        end = perf_counter_ns()
+        times_ns.append(end - start)
+
+        if end - run_start > config.max_time_ns:
+            # TODO: log something
+            break
+    benchmark_end = perf_counter_ns()
+    total_time = (benchmark_end - run_start) / 1e9
+
+    stats = BenchmarkStats.from_list(
+        times_ns,
+        rounds=rounds,
+        total_time=total_time,
+        iter_per_round=iter_per_round,
+        warmup_iters=warmup_iters,
+    )
+
+    return Benchmark(name=name, uri=uri, config=config, stats=stats), out
+
+
+class WallTimeInstrument(Instrument):
+    instrument = "walltime"
+
+    def __init__(self, config: CodSpeedConfig) -> None:
+        self.config = config
+        self.benchmarks: list[Benchmark] = []
+
+    def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
+        return f"mode: walltime, timer_resolution: {TIMER_RESOLUTION_NS:.1f}ns", []
+
+    def measure(
+        self,
+        name: str,
+        uri: str,
+        fn: Callable[P, T],
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> T:
+        bench, out = run_benchmark(
+            name=name,
+            uri=uri,
+            fn=fn,
+            args=args,
+            kwargs=kwargs,
+            config=BenchmarkConfig.from_codspeed_config(self.config),
+        )
+        self.benchmarks.append(bench)
+        return out
+
+    def report(self, session: Session) -> None:
+        reporter = session.config.pluginmanager.get_plugin("terminalreporter")
+
+        if len(self.benchmarks) == 0:
+            reporter.write_sep(
+                "=",
+                f"{len(self.benchmarks)} benchmarked",
+            )
+            return
+        self._print_benchmark_table()
+        reporter.write_sep(
+            "=",
+            f"{len(self.benchmarks)} benchmarked",
+        )
+
+    def _print_benchmark_table(self) -> None:
+        table = Table(title="Benchmark Results")
+
+        table.add_column("Benchmark", justify="right", style="cyan", no_wrap=True)
+        table.add_column("Time (best)", justify="right", style="green bold")
+        table.add_column(
+            "Rel. StdDev",
+            justify="right",
+        )
+        table.add_column("Run time", justify="right")
+        table.add_column("Iters", justify="right")
+
+        for bench in self.benchmarks:
+            rsd = bench.stats.stdev_ns / bench.stats.mean_ns
+            rsd_text = Text(f"{rsd*100:.1f}%")
+            if rsd > 0.1:
+                rsd_text.stylize("red bold")
+            table.add_row(
+                bench.name,
+                f"{bench.stats.min_ns/bench.stats.iter_per_round:,.0f}ns",
+                rsd_text,
+                f"{bench.stats.total_time:,.2f}s",
+                f"{bench.stats.iter_per_round * bench.stats.rounds:,}",
+            )
+
+        console = Console()
+        print("\n")
+        console.print(table)
+
+    def get_result_dict(self) -> dict[str, Any]:
+        return {
+            "instrument": {
+                "type": self.instrument,
+                "clock_info": get_clock_info("perf_counter").__dict__,
+            },
+            "benchmarks": [asdict(bench) for bench in self.benchmarks],
+        }

pytest_codspeed-3.0.0b0/src/pytest_codspeed/plugin.py

@@ -0,0 +1,338 @@
+from __future__ import annotations
+
+import functools
+import gc
+import importlib.util
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING
+
+import pytest
+from _pytest.fixtures import FixtureManager
+
+from pytest_codspeed.instruments import (
+    MeasurementMode,
+    get_instrument_from_mode,
+)
+from pytest_codspeed.utils import (
+    get_environment_metadata,
+    get_git_relative_uri_and_name,
+)
+
+from . import __version__
+
+if TYPE_CHECKING:
+    from typing import Callable, ParamSpec, TypeVar
+
+    from pytest_codspeed.instruments import Instrument
+
+    T = TypeVar("T")
+    P = ParamSpec("P")
+
+IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None
+IS_PYTEST_SPEED_INSTALLED = importlib.util.find_spec("pytest_speed") is not None
+BEFORE_PYTEST_8_1_1 = pytest.version_tuple < (8, 1, 1)
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_addoption(parser: pytest.Parser):
+    group = parser.getgroup("CodSpeed benchmarking")
+    group.addoption(
+        "--codspeed",
+        action="store_true",
+        default=False,
+        help="Enable codspeed (not required when using the CodSpeed action)",
+    )
+    group.addoption(
+        "--codspeed-mode",
+        action="store",
+        choices=[mode.value for mode in MeasurementMode],
+        help="The measurement tool to use for measuring performance",
+    )
+    group.addoption(
+        "--codspeed-warmup-time",
+        action="store",
+        type=float,
+        help=(
+            "The time to warm up the benchmark for (in seconds), "
+            "only for walltime mode"
+        ),
+    )
+    group.addoption(
+        "--codspeed-max-rounds",
+        action="store",
+        type=int,
+        help=(
+            "The maximum number of rounds to run a benchmark for"
+            ", only for walltime mode"
+        ),
+    )
+
+
+@dataclass(frozen=True)
+class CodSpeedConfig:
+    warmup_time_ns: int | None = None
+    max_rounds: int | None = None
+
+    @classmethod
+    def from_pytest_config(cls, config: pytest.Config) -> CodSpeedConfig:
+        warmup_time = config.getoption("--codspeed-warmup-time", None)
+        warmup_time_ns = (
+            int(warmup_time * 1_000_000_000) if warmup_time is not None else None
+        )
+        return cls(
+            warmup_time_ns=warmup_time_ns,
+            max_rounds=config.getoption("--codspeed-max-rounds", None),
+        )
+
+
+@dataclass(unsafe_hash=True)
+class CodSpeedPlugin:
+    is_codspeed_enabled: bool
+    mode: MeasurementMode
+    instrument: Instrument
+    config: CodSpeedConfig
+    disabled_plugins: tuple[str, ...]
+    profile_folder: Path | None
+    benchmark_count: int = field(default=0, hash=False, compare=False)
+
+
+PLUGIN_NAME = "codspeed_plugin"
+
+
+def get_plugin(config: pytest.Config) -> CodSpeedPlugin:
+    return config.pluginmanager.get_plugin(PLUGIN_NAME)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_configure(config: pytest.Config):
+    config.addinivalue_line(
+        "markers", "codspeed_benchmark: mark an entire test for codspeed benchmarking"
+    )
+    config.addinivalue_line(
+        "markers", "benchmark: mark an entire test for codspeed benchmarking"
+    )
+    is_codspeed_enabled = (
+        config.getoption("--codspeed") or os.environ.get("CODSPEED_ENV") is not None
+    )
+
+    if os.environ.get("CODSPEED_ENV") is not None:
+        if os.environ.get("CODSPEED_RUNNER_MODE") == "walltime":
+            default_mode = MeasurementMode.WallTime.value
+        else:
+            default_mode = MeasurementMode.Instrumentation.value
+    else:
+        default_mode = MeasurementMode.WallTime.value
+
+    mode = MeasurementMode(config.getoption("--codspeed-mode", None) or default_mode)
+    instrument = get_instrument_from_mode(mode)
+    disabled_plugins: list[str] = []
+    if is_codspeed_enabled:
+        if IS_PYTEST_BENCHMARK_INSTALLED:
+            # Disable pytest-benchmark
+            object.__setattr__(config.option, "benchmark_disable", True)
+            config.pluginmanager.set_blocked("pytest_benchmark")
+            config.pluginmanager.set_blocked("pytest-benchmark")
+            disabled_plugins.append("pytest-benchmark")
+        if IS_PYTEST_SPEED_INSTALLED:
+            # Disable pytest-speed
+            config.pluginmanager.set_blocked("speed")
+            disabled_plugins.append("pytest-speed")
+
+    profile_folder = os.environ.get("CODSPEED_PROFILE_FOLDER")
+
+    codspeedconfig = CodSpeedConfig.from_pytest_config(config)
+
+    plugin = CodSpeedPlugin(
+        disabled_plugins=tuple(disabled_plugins),
+        is_codspeed_enabled=is_codspeed_enabled,
+        mode=mode,
+        instrument=instrument(codspeedconfig),
+        config=codspeedconfig,
+        profile_folder=Path(profile_folder) if profile_folder else None,
+    )
+    config.pluginmanager.register(plugin, PLUGIN_NAME)
+
+
+@pytest.hookimpl()
+def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager):
+    """
+    Patch the benchmark fixture to use the codspeed one if codspeed is enabled and an
+    alternative benchmark fixture is available
+    """
+    if (IS_PYTEST_BENCHMARK_INSTALLED or IS_PYTEST_SPEED_INSTALLED) and isinstance(
+        plugin, FixtureManager
+    ):
+        fixture_manager = plugin
+        codspeed_plugin: CodSpeedPlugin = manager.get_plugin(PLUGIN_NAME)
+        if codspeed_plugin.is_codspeed_enabled:
+            codspeed_benchmark_fixtures = plugin.getfixturedefs(
+                "codspeed_benchmark",
+                fixture_manager.session.nodeid
+                if BEFORE_PYTEST_8_1_1
+                else fixture_manager.session,
+            )
+            assert codspeed_benchmark_fixtures is not None
+            # Archive the alternative benchmark fixture
+            fixture_manager._arg2fixturedefs["__benchmark"] = (
+                fixture_manager._arg2fixturedefs["benchmark"]
+            )
+            # Replace the alternative fixture with the codspeed one
+            fixture_manager._arg2fixturedefs["benchmark"] = codspeed_benchmark_fixtures
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_report_header(config: pytest.Config):
+    plugin = get_plugin(config)
+    config_str, warns = plugin.instrument.get_instrument_config_str_and_warns()
+    out = [
+        (
+            f"codspeed: {__version__} ("
+            f"{'enabled' if plugin.is_codspeed_enabled else 'disabled'}, {config_str}"
+            ")"
+        ),
+        *warns,
+    ]
+    if len(plugin.disabled_plugins) > 0:
+        out.append(
+            "\033[93mCodSpeed had to disable the following plugins: "
+            f"{', '.join(plugin.disabled_plugins)}\033[0m"
+        )
+    return "\n".join(out)
+
+
+def has_benchmark_fixture(item: pytest.Item) -> bool:
+    item_fixtures = getattr(item, "fixturenames", [])
+    return "benchmark" in item_fixtures or "codspeed_benchmark" in item_fixtures
+
+
+def has_benchmark_marker(item: pytest.Item) -> bool:
+    return (
+        item.get_closest_marker("codspeed_benchmark") is not None
+        or item.get_closest_marker("benchmark") is not None
+    )
+
+
+def should_benchmark_item(item: pytest.Item) -> bool:
+    return has_benchmark_fixture(item) or has_benchmark_marker(item)
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_collection_modifyitems(
+    session: pytest.Session, config: pytest.Config, items: list[pytest.Item]
+):
+    """Filter out items that should not be benchmarked when codspeed is enabled"""
+    plugin = get_plugin(config)
+    if plugin.is_codspeed_enabled:
+        deselected = []
+        selected = []
+        for item in items:
+            if should_benchmark_item(item):
+                selected.append(item)
+            else:
+                deselected.append(item)
+        config.hook.pytest_deselected(items=deselected)
+        items[:] = selected
+
+
+def _measure(
+    plugin: CodSpeedPlugin,
+    nodeid: str,
+    config: pytest.Config,
+    fn: Callable[P, T],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> T:
+    is_gc_enabled = gc.isenabled()
+    if is_gc_enabled:
+        gc.collect()
+        gc.disable()
+    try:
+        uri, name = get_git_relative_uri_and_name(nodeid, config.rootpath)
+        return plugin.instrument.measure(name, uri, fn, *args, **kwargs)
+    finally:
+        # Ensure GC is re-enabled even if the test failed
+        if is_gc_enabled:
+            gc.enable()
+
+
+def wrap_runtest(
+    plugin: CodSpeedPlugin,
+    nodeid: str,
+    config: pytest.Config,
+    fn: Callable[P, T],
+) -> Callable[P, T]:
+    @functools.wraps(fn)
+    def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
+        return _measure(plugin, nodeid, config, fn, *args, **kwargs)
+
+    return wrapped
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None):
+    plugin = get_plugin(item.config)
+    if not plugin.is_codspeed_enabled or not should_benchmark_item(item):
+        # Defer to the default test protocol since no benchmarking is needed
+        return None
+
+    if has_benchmark_fixture(item):
+        # Instrumentation is handled by the fixture
+        return None
+
+    # Wrap runtest and defer to default protocol
+    item.runtest = wrap_runtest(plugin, item.nodeid, item.config, item.runtest)
+    return None
+
+
+@pytest.hookimpl()
+def pytest_sessionfinish(session: pytest.Session, exitstatus):
+    plugin = get_plugin(session.config)
+    if plugin.is_codspeed_enabled:
+        plugin.instrument.report(session)
+        if plugin.profile_folder:
+            result_path = plugin.profile_folder / "results" / f"{os.getpid()}.json"
+        else:
+            result_path = (
+                session.config.rootpath / f".codspeed/results_{time() * 1000:.0f}.json"
+            )
+        data = {**get_environment_metadata(), **plugin.instrument.get_result_dict()}
+        result_path.parent.mkdir(parents=True, exist_ok=True)
+        result_path.write_text(json.dumps(data, indent=2))
+
+
+class BenchmarkFixture:
+    """The fixture that can be used to benchmark a function."""
+
+    def __init__(self, request: pytest.FixtureRequest):
+        self.extra_info: dict = {}
+
+        self._request = request
+
+    def __call__(self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
+        config = self._request.config
+        plugin = get_plugin(config)
+        if plugin.is_codspeed_enabled:
+            return _measure(
+                plugin, self._request.node.nodeid, config, func, *args, **kwargs
+            )
+        else:
+            return func(*args, **kwargs)
+
+
+@pytest.fixture(scope="function")
+def codspeed_benchmark(request: pytest.FixtureRequest) -> Callable:
+    return BenchmarkFixture(request)
+
+
+if not IS_PYTEST_BENCHMARK_INSTALLED:
+
+    @pytest.fixture(scope="function")
+    def benchmark(codspeed_benchmark, request: pytest.FixtureRequest):
+        """
+        Compatibility with pytest-benchmark
+        """
+        return codspeed_benchmark

{pytest_codspeed-2.2.1 → pytest_codspeed-3.0.0b0}/src/pytest_codspeed/utils.py

@@ -1,5 +1,17 @@
+from __future__ import annotations
+
+import os
+import sys
+import sysconfig
 from pathlib import Path
 
+from pytest_codspeed import __version__
+
+if sys.version_info < (3, 10):
+    import importlib_metadata as importlib_metadata
+else:
+    import importlib.metadata as importlib_metadata
+
 
 def get_git_relative_path(abs_path: Path) -> Path:
     """Get the path relative to the git root directory. If the path is not
@@ -15,11 +27,11 @@ def get_git_relative_path(abs_path: Path) -> Path:
     return abs_path
 
 
-def get_git_relative_uri(uri: str, pytest_rootdir: Path) -> str:
-    """Get the benchmark uri relative to the git root dir.
+def get_git_relative_uri_and_name(nodeid: str, pytest_rootdir: Path) -> tuple[str, str]:
+    """Get the benchmark uri relative to the git root dir and the benchmark name.
 
     Args:
-        uri (str): the benchmark uri, for example:
+        nodeid (str): the pytest nodeid, for example:
           testing/test_excinfo.py::TestFormattedExcinfo::test_repr_source
         pytest_rootdir (str): the pytest root dir, for example:
           /home/user/gitrepo/folder
@@ -29,7 +41,23 @@ def get_git_relative_uri(uri: str, pytest_rootdir: Path) -> str:
           folder/testing/test_excinfo.py::TestFormattedExcinfo::test_repr_source
 
     """
-    file_path, function_path = uri.split("::", 1)
+    file_path, bench_name = nodeid.split("::", 1)
     absolute_file_path = pytest_rootdir / Path(file_path)
     relative_git_path = get_git_relative_path(absolute_file_path)
-    return f"{str(relative_git_path)}::{function_path}"
+    return (f"{str(relative_git_path)}::{bench_name}", bench_name)
+
+
+def get_environment_metadata() -> dict[str, dict]:
+    return {
+        "creator": {
+            "name": "pytest-codspeed",
+            "version": __version__,
+            "pid": os.getpid(),
+        },
+        "python": {
+            "sysconfig": sysconfig.get_config_vars(),
+            "dependencies": {
+                d.name: d.version for d in importlib_metadata.distributions()
+            },
+        },
+    }

pytest_codspeed-2.2.1/src/pytest_codspeed/plugin.py

@@ -1,309 +0,0 @@
-import gc
-import os
-import pkgutil
-import sys
-from dataclasses import dataclass, field
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Tuple,
-    TypeVar,
-    Union,
-)
-
-import pytest
-from _pytest.fixtures import FixtureManager
-
-from pytest_codspeed.utils import get_git_relative_uri
-
-from . import __version__
-from ._wrapper import get_lib
-
-if TYPE_CHECKING:
-    from ._wrapper import LibType
-
-IS_PYTEST_BENCHMARK_INSTALLED = pkgutil.find_loader("pytest_benchmark") is not None
-SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
-BEFORE_PYTEST_8_1_1 = pytest.version_tuple < (8, 1, 1)
-
-
-@pytest.hookimpl(trylast=True)
-def pytest_addoption(parser: "pytest.Parser"):
-    group = parser.getgroup("CodSpeed benchmarking")
-    group.addoption(
-        "--codspeed",
-        action="store_true",
-        default=False,
-        help="Enable codspeed (not required when using the CodSpeed action)",
-    )
-
-
-@dataclass(unsafe_hash=True)
-class CodSpeedPlugin:
-    is_codspeed_enabled: bool
-    should_measure: bool
-    lib: Optional["LibType"]
-    disabled_plugins: Tuple[str, ...]
-    benchmark_count: int = field(default=0, hash=False, compare=False)
-
-
-PLUGIN_NAME = "codspeed_plugin"
-
-
-def get_plugin(config: "pytest.Config") -> "CodSpeedPlugin":
-    return config.pluginmanager.get_plugin(PLUGIN_NAME)
-
-
-@pytest.hookimpl(tryfirst=True)
-def pytest_configure(config: "pytest.Config"):
-    config.addinivalue_line(
-        "markers", "codspeed_benchmark: mark an entire test for codspeed benchmarking"
-    )
-    config.addinivalue_line(
-        "markers", "benchmark: mark an entire test for codspeed benchmarking"
-    )
-    is_codspeed_enabled = (
-        config.getoption("--codspeed") or os.environ.get("CODSPEED_ENV") is not None
-    )
-    should_measure = os.environ.get("CODSPEED_ENV") is not None
-    lib = get_lib() if should_measure else None
-    if lib is not None:
-        lib.dump_stats_at(f"Metadata: pytest-codspeed {__version__}".encode("ascii"))
-    disabled_plugins: List[str] = []
-    # Disable pytest-benchmark if codspeed is enabled
-    if is_codspeed_enabled and IS_PYTEST_BENCHMARK_INSTALLED:
-        object.__setattr__(config.option, "benchmark_disable", True)
-        config.pluginmanager.set_blocked("pytest-benchmark")
-        disabled_plugins.append("pytest-benchmark")
-
-    plugin = CodSpeedPlugin(
-        is_codspeed_enabled=is_codspeed_enabled,
-        should_measure=should_measure,
-        lib=lib,
-        disabled_plugins=tuple(disabled_plugins),
-    )
-    config.pluginmanager.register(plugin, PLUGIN_NAME)
-
-
-def pytest_plugin_registered(plugin, manager: "pytest.PytestPluginManager"):
-    """Patch the benchmark fixture to use the codspeed one if codspeed is enabled"""
-    if IS_PYTEST_BENCHMARK_INSTALLED and isinstance(plugin, FixtureManager):
-        fixture_manager = plugin
-        codspeed_plugin: CodSpeedPlugin = manager.get_plugin(PLUGIN_NAME)
-        if codspeed_plugin.is_codspeed_enabled:
-            codspeed_benchmark_fixtures = plugin.getfixturedefs(
-                "codspeed_benchmark",
-                fixture_manager.session.nodeid
-                if BEFORE_PYTEST_8_1_1
-                else fixture_manager.session,
-            )
-            assert codspeed_benchmark_fixtures is not None
-            # Archive the pytest-benchmark fixture
-            fixture_manager._arg2fixturedefs["__benchmark"] = (
-                fixture_manager._arg2fixturedefs["benchmark"]
-            )
-            # Replace the pytest-benchmark fixture with the codspeed one
-            fixture_manager._arg2fixturedefs["benchmark"] = codspeed_benchmark_fixtures
-
-
-@pytest.hookimpl(trylast=True)
-def pytest_report_header(config: "pytest.Config"):
-    out = [
-        f"codspeed: {__version__} "
-        f"(callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE  else 'not supported'})"
-    ]
-    plugin = get_plugin(config)
-    if plugin.is_codspeed_enabled and not plugin.should_measure:
-        out.append(
-            "\033[1m"
-            "NOTICE: codspeed is enabled, but no performance measurement"
-            " will be made since it's running in an unknown environment."
-            "\033[0m"
-        )
-    if len(plugin.disabled_plugins) > 0:
-        out.append(
-            "\033[93mCodSpeed had to disable the following plugins: "
-            f"{', '.join(plugin.disabled_plugins)}\033[0m"
-        )
-    return "\n".join(out)
-
-
-def has_benchmark_fixture(item: "pytest.Item") -> bool:
-    item_fixtures = getattr(item, "fixturenames", [])
-    return "benchmark" in item_fixtures or "codspeed_benchmark" in item_fixtures
-
-
-def has_benchmark_marker(item: "pytest.Item") -> bool:
-    return (
-        item.get_closest_marker("codspeed_benchmark") is not None
-        or item.get_closest_marker("benchmark") is not None
-    )
-
-
-def should_benchmark_item(item: "pytest.Item") -> bool:
-    return has_benchmark_fixture(item) or has_benchmark_marker(item)
-
-
-@pytest.hookimpl()
-def pytest_sessionstart(session: "pytest.Session"):
-    plugin = get_plugin(session.config)
-    if plugin.is_codspeed_enabled:
-        plugin.benchmark_count = 0
-        if plugin.should_measure and SUPPORTS_PERF_TRAMPOLINE:
-            sys.activate_stack_trampoline("perf")  # type: ignore
-
-
-@pytest.hookimpl(trylast=True)
-def pytest_collection_modifyitems(
-    session: "pytest.Session", config: "pytest.Config", items: "List[pytest.Item]"
-):
-    plugin = get_plugin(config)
-    if plugin.is_codspeed_enabled:
-        deselected = []
-        selected = []
-        for item in items:
-            if should_benchmark_item(item):
-                selected.append(item)
-            else:
-                deselected.append(item)
-        config.hook.pytest_deselected(items=deselected)
-        items[:] = selected
-
-
-def _run_with_instrumentation(
-    lib: "LibType",
-    nodeId: str,
-    config: "pytest.Config",
-    fn: Callable[..., Any],
-    *args,
-    **kwargs,
-):
-    is_gc_enabled = gc.isenabled()
-    if is_gc_enabled:
-        gc.collect()
-        gc.disable()
-
-    result = None
-
-    def __codspeed_root_frame__():
-        nonlocal result
-        result = fn(*args, **kwargs)
-
-    if SUPPORTS_PERF_TRAMPOLINE:
-        # Warmup CPython performance map cache
-        __codspeed_root_frame__()
-    lib.zero_stats()
-    lib.start_instrumentation()
-    __codspeed_root_frame__()
-    lib.stop_instrumentation()
-    uri = get_git_relative_uri(nodeId, config.rootpath)
-    lib.dump_stats_at(uri.encode("ascii"))
-    if is_gc_enabled:
-        gc.enable()
-
-    return result
-
-
-@pytest.hookimpl(tryfirst=True)
-def pytest_runtest_protocol(item: "pytest.Item", nextitem: Union["pytest.Item", None]):
-    plugin = get_plugin(item.config)
-    if not plugin.is_codspeed_enabled or not should_benchmark_item(item):
-        return (
-            None  # Defer to the default test protocol since no benchmarking is needed
-        )
-
-    if has_benchmark_fixture(item):
-        return None  # Instrumentation is handled by the fixture
-
-    plugin.benchmark_count += 1
-    if not plugin.should_measure:
-        return None  # Benchmark counted but will be run in the default protocol
-
-    # Setup phase
-    reports = []
-    ihook = item.ihook
-    ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location)
-    setup_call = pytest.CallInfo.from_call(
-        lambda: ihook.pytest_runtest_setup(item=item, nextitem=nextitem), "setup"
-    )
-    setup_report = ihook.pytest_runtest_makereport(item=item, call=setup_call)
-    ihook.pytest_runtest_logreport(report=setup_report)
-    reports.append(setup_report)
-    # Run phase
-    if setup_report.passed and not item.config.getoption("setuponly"):
-        assert plugin.lib is not None
-        runtest_call = pytest.CallInfo.from_call(
-            lambda: _run_with_instrumentation(
-                plugin.lib, item.nodeid, item.config, item.runtest
-            ),
-            "call",
-        )
-        runtest_report = ihook.pytest_runtest_makereport(item=item, call=runtest_call)
-        ihook.pytest_runtest_logreport(report=runtest_report)
-        reports.append(runtest_report)
-
-    # Teardown phase
-    teardown_call = pytest.CallInfo.from_call(
-        lambda: ihook.pytest_runtest_teardown(item=item, nextitem=nextitem), "teardown"
-    )
-    teardown_report = ihook.pytest_runtest_makereport(item=item, call=teardown_call)
-    ihook.pytest_runtest_logreport(report=teardown_report)
-    reports.append(teardown_report)
-    ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location)
-
-    return reports  # Deny further protocol hooks execution
-
-
-T = TypeVar("T")
-
-
-class BenchmarkFixture:
-    """The fixture that can be used to benchmark a function."""
-
-    def __init__(self, request: "pytest.FixtureRequest"):
-        self.extra_info: Dict = {}
-
-        self._request = request
-
-    def __call__(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
-        config = self._request.config
-        plugin = get_plugin(config)
-        plugin.benchmark_count += 1
-        if plugin.is_codspeed_enabled and plugin.should_measure:
-            assert plugin.lib is not None
-            return _run_with_instrumentation(
-                plugin.lib, self._request.node.nodeid, config, func, *args, **kwargs
-            )
-        else:
-            return func(*args, **kwargs)
-
-
-@pytest.fixture(scope="function")
-def codspeed_benchmark(request: "pytest.FixtureRequest") -> Callable:
-    return BenchmarkFixture(request)
-
-
-if not IS_PYTEST_BENCHMARK_INSTALLED:
-
-    @pytest.fixture(scope="function")
-    def benchmark(codspeed_benchmark, request: "pytest.FixtureRequest"):
-        """
-        Compatibility with pytest-benchmark
-        """
-        return codspeed_benchmark
-
-
-@pytest.hookimpl()
-def pytest_sessionfinish(session: "pytest.Session", exitstatus):
-    plugin = get_plugin(session.config)
-    if plugin.is_codspeed_enabled:
-        reporter = session.config.pluginmanager.get_plugin("terminalreporter")
-        count_suffix = "benchmarked" if plugin.should_measure else "benchmark tested"
-        reporter.write_sep(
-            "=",
-            f"{plugin.benchmark_count} {count_suffix}",
-        )