pytest-codspeed 3.3.0b0__tar.gz → 4.0.0b0__tar.gz

{pytest_codspeed-3.3.0b0/src/pytest_codspeed.egg-info → pytest_codspeed-4.0.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytest-codspeed
-Version: 3.3.0b0
+Version: 4.0.0b0
 Summary: Pytest plugin to create CodSpeed benchmarks
 Author-email: Arthur Pastel <arthur@codspeed.io>
 License: The MIT License (MIT)
@@ -52,7 +52,7 @@ Requires-Dist: rich>=13.8.1
 Requires-Dist: importlib-metadata>=8.5.0; python_version < "3.10"
 Provides-Extra: lint
 Requires-Dist: mypy~=1.11.2; extra == "lint"
-Requires-Dist: ruff~=0.6.5; extra == "lint"
+Requires-Dist: ruff~=0.11.12; extra == "lint"
 Provides-Extra: compat
 Requires-Dist: pytest-benchmark~=5.0.0; extra == "compat"
 Requires-Dist: pytest-xdist~=3.6.1; extra == "compat"

{pytest_codspeed-3.3.0b0 → pytest_codspeed-4.0.0b0}/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-lint = ["mypy ~= 1.11.2", "ruff ~= 0.6.5"]
+lint = ["mypy ~= 1.11.2", "ruff ~= 0.11.12"]
 compat = [
     "pytest-benchmark ~= 5.0.0",
     "pytest-xdist ~= 3.6.1",
@@ -66,7 +66,7 @@ version = { attr = "pytest_codspeed.__version__" }
 
 
 [tool.bumpver]
-current_version = "3.3.0-beta"
+current_version = "4.0.0-beta"
 version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]"
 commit_message = "Release v{new_version} 🚀"
 tag_message = "Release v{new_version} 🚀"

{pytest_codspeed-3.3.0b0 → pytest_codspeed-4.0.0b0}/src/pytest_codspeed/__init__.py

@@ -1,6 +1,6 @@
-__version__ = "3.3.0b0"
+__version__ = "4.0.0b0"
 # We also have the semver version since __version__ is not semver compliant
-__semver_version__ = "3.3.0-beta"
+__semver_version__ = "4.0.0-beta"
 
 from .plugin import BenchmarkFixture
 

pytest_codspeed-4.0.0b0/src/pytest_codspeed/config.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Generic, TypeVar
+
+T = TypeVar("T")
+
+if TYPE_CHECKING:
+    from typing import Any, Callable
+
+    import pytest
+
+
+@dataclass(frozen=True)
+class CodSpeedConfig:
+    """
+    The configuration for the codspeed plugin.
+    Usually created from the command line arguments.
+    """
+
+    warmup_time_ns: int | None = None
+    max_time_ns: int | None = None
+    max_rounds: int | None = None
+
+    @classmethod
+    def from_pytest_config(cls, config: pytest.Config) -> CodSpeedConfig:
+        warmup_time = config.getoption("--codspeed-warmup-time", None)
+        warmup_time_ns = (
+            int(warmup_time * 1_000_000_000) if warmup_time is not None else None
+        )
+        max_time = config.getoption("--codspeed-max-time", None)
+        max_time_ns = int(max_time * 1_000_000_000) if max_time is not None else None
+        return cls(
+            warmup_time_ns=warmup_time_ns,
+            max_rounds=config.getoption("--codspeed-max-rounds", None),
+            max_time_ns=max_time_ns,
+        )
+
+
+@dataclass(frozen=True)
+class BenchmarkMarkerOptions:
+    group: str | None = None
+    """The group name to use for the benchmark."""
+    min_time: int | None = None
+    """
+    The minimum time of a round (in seconds).
+    Only available in walltime mode.
+    """
+    max_time: int | None = None
+    """
+    The maximum time to run the benchmark for (in seconds).
+    Only available in walltime mode.
+    """
+    max_rounds: int | None = None
+    """
+    The maximum number of rounds to run the benchmark for.
+    Takes precedence over max_time. Only available in walltime mode.
+    """
+
+    @classmethod
+    def from_pytest_item(cls, item: pytest.Item) -> BenchmarkMarkerOptions:
+        marker = item.get_closest_marker(
+            "codspeed_benchmark"
+        ) or item.get_closest_marker("benchmark")
+        if marker is None:
+            return cls()
+        if len(marker.args) > 0:
+            raise ValueError(
+                "Positional arguments are not allowed in the benchmark marker"
+            )
+        kwargs = marker.kwargs
+
+        unknown_kwargs = set(kwargs.keys()) - {
+            field.name for field in dataclasses.fields(cls)
+        }
+        if unknown_kwargs:
+            raise ValueError(
+                "Unknown kwargs passed to benchmark marker: "
+                + ", ".join(sorted(unknown_kwargs))
+            )
+
+        return cls(**kwargs)
+
+
+@dataclass(frozen=True)
+class PedanticOptions(Generic[T]):
+    """Parameters for running a benchmark using the pedantic fixture API."""
+
+    target: Callable[..., T]
+    setup: Callable[[], Any | None] | None
+    teardown: Callable[..., Any | None] | None
+    rounds: int
+    warmup_rounds: int
+    iterations: int
+    args: tuple[Any, ...] = field(default_factory=tuple)
+    kwargs: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.rounds < 0:
+            raise ValueError("rounds must be positive")
+        if self.warmup_rounds < 0:
+            raise ValueError("warmup_rounds must be non-negative")
+        if self.iterations <= 0:
+            raise ValueError("iterations must be positive")
+        if self.iterations > 1 and self.setup is not None:
+            raise ValueError(
+                "setup cannot be used with multiple iterations, use multiple rounds"
+            )
+
+    def setup_and_get_args_kwargs(self) -> tuple[tuple[Any, ...], dict[str, Any]]:
+        if self.setup is None:
+            return self.args, self.kwargs
+        maybe_result = self.setup(*self.args, **self.kwargs)
+        if maybe_result is not None:
+            if len(self.args) > 0 or len(self.kwargs) > 0:
+                raise ValueError("setup cannot return a value when args are provided")
+            return maybe_result
+        return self.args, self.kwargs

{pytest_codspeed-3.3.0b0 → pytest_codspeed-4.0.0b0}/src/pytest_codspeed/instruments/__init__.py

@@ -5,14 +5,14 @@ from enum import Enum
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Any, Callable, ClassVar, ParamSpec, TypeVar
+    from typing import Any, Callable, ClassVar, TypeVar
 
     import pytest
 
+    from pytest_codspeed.config import BenchmarkMarkerOptions, PedanticOptions
     from pytest_codspeed.plugin import CodSpeedConfig
 
     T = TypeVar("T")
-    P = ParamSpec("P")
 
 
 class Instrument(metaclass=ABCMeta):
@@ -27,11 +27,21 @@ class Instrument(metaclass=ABCMeta):
     @abstractmethod
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
+        name: str,
+        uri: str,
+        fn: Callable[..., T],
+        *args: tuple,
+        **kwargs: dict[str, Any],
+    ) -> T: ...
+
+    @abstractmethod
+    def measure_pedantic(
+        self,
+        marker_options: BenchmarkMarkerOptions,
+        pedantic_options: PedanticOptions[T],
         name: str,
         uri: str,
-        fn: Callable[P, T],
-        *args: P.args,
-        **kwargs: P.kwargs,
     ) -> T: ...
 
     @abstractmethod

{pytest_codspeed-3.3.0b0 → pytest_codspeed-4.0.0b0}/src/pytest_codspeed/instruments/hooks/__init__.py

@@ -2,10 +2,11 @@ from __future__ import annotations
 
 import os
 import sys
+import warnings
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from .dist_instrument_hooks import lib as LibType
+    from .dist_instrument_hooks import InstrumentHooksPointer, LibType
 
 SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
 
@@ -14,7 +15,7 @@ class InstrumentHooks:
     """Zig library wrapper class providing benchmark measurement functionality."""
 
     lib: LibType
-    instance: int
+    instance: InstrumentHooksPointer
 
     def __init__(self) -> None:
         if os.environ.get("CODSPEED_ENV") is None:
@@ -27,17 +28,15 @@ class InstrumentHooks:
             from .dist_instrument_hooks import lib  # type: ignore
         except ImportError as e:
             raise RuntimeError(f"Failed to load instrument hooks library: {e}") from e
+        self.lib = lib
 
-        instance = lib.instrument_hooks_init()
-        if instance == 0:
+        self.instance = self.lib.instrument_hooks_init()
+        if self.instance == 0:
             raise RuntimeError("Failed to initialize CodSpeed instrumentation library.")
 
         if SUPPORTS_PERF_TRAMPOLINE:
             sys.activate_stack_trampoline("perf")  # type: ignore
 
-        self.lib = lib
-        self.instance = instance
-
     def __del__(self):
         if hasattr(self, "lib") and hasattr(self, "instance"):
             self.lib.instrument_hooks_deinit(self.instance)
@@ -46,13 +45,13 @@ class InstrumentHooks:
         """Start a new benchmark measurement."""
         ret = self.lib.instrument_hooks_start_benchmark(self.instance)
         if ret != 0:
-            raise RuntimeError("Failed to start benchmark measurement")
+            warnings.warn("Failed to start benchmark measurement", RuntimeWarning)
 
     def stop_benchmark(self) -> None:
         """Stop the current benchmark measurement."""
         ret = self.lib.instrument_hooks_stop_benchmark(self.instance)
         if ret != 0:
-            raise RuntimeError("Failed to stop benchmark measurement")
+            warnings.warn("Failed to stop benchmark measurement", RuntimeWarning)
 
     def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None:
         """Set the executed benchmark URI and process ID.
@@ -68,7 +67,7 @@ class InstrumentHooks:
             self.instance, pid, uri.encode("ascii")
         )
         if ret != 0:
-            raise RuntimeError("Failed to set executed benchmark")
+            warnings.warn("Failed to set executed benchmark", RuntimeWarning)
 
     def set_integration(self, name: str, version: str) -> None:
         """Set the integration name and version."""
@@ -76,7 +75,7 @@ class InstrumentHooks:
             self.instance, name.encode("ascii"), version.encode("ascii")
         )
         if ret != 0:
-            raise RuntimeError("Failed to set integration name and version")
+            warnings.warn("Failed to set integration name and version", RuntimeWarning)
 
     def is_instrumented(self) -> bool:
         """Check if instrumentation is active."""

pytest_codspeed-4.0.0b0/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi

@@ -0,0 +1,27 @@
+InstrumentHooksPointer = object
+
+class lib:
+    @staticmethod
+    def instrument_hooks_init() -> InstrumentHooksPointer: ...
+    @staticmethod
+    def instrument_hooks_deinit(hooks: InstrumentHooksPointer) -> None: ...
+    @staticmethod
+    def instrument_hooks_is_instrumented(hooks: InstrumentHooksPointer) -> bool: ...
+    @staticmethod
+    def instrument_hooks_start_benchmark(hooks: InstrumentHooksPointer) -> int: ...
+    @staticmethod
+    def instrument_hooks_stop_benchmark(hooks: InstrumentHooksPointer) -> int: ...
+    @staticmethod
+    def instrument_hooks_executed_benchmark(
+        hooks: InstrumentHooksPointer, pid: int, uri: bytes
+    ) -> int: ...
+    @staticmethod
+    def instrument_hooks_set_integration(
+        hooks: InstrumentHooksPointer, name: bytes, version: bytes
+    ) -> int: ...
+    @staticmethod
+    def callgrind_start_instrumentation() -> int: ...
+    @staticmethod
+    def callgrind_stop_instrumentation() -> int: ...
+
+LibType = type[lib]

{pytest_codspeed-3.3.0b0 → pytest_codspeed-4.0.0b0}/src/pytest_codspeed/instruments/valgrind.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import warnings
 from typing import TYPE_CHECKING
 
 from pytest_codspeed import __semver_version__
@@ -12,8 +13,9 @@ if TYPE_CHECKING:
 
     from pytest import Session
 
-    from pytest_codspeed.instruments import P, T
-    from pytest_codspeed.plugin import CodSpeedConfig
+    from pytest_codspeed.config import PedanticOptions
+    from pytest_codspeed.instruments import T
+    from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
 
@@ -35,7 +37,7 @@ class ValgrindInstrument(Instrument):
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
         config = (
             f"mode: instrumentation, "
-            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE  else 'not supported'}"
+            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
         )
         warnings = []
         if not self.should_measure:
@@ -49,11 +51,12 @@ class ValgrindInstrument(Instrument):
 
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
-        fn: Callable[P, T],
-        *args: P.args,
-        **kwargs: P.kwargs,
+        fn: Callable[..., T],
+        *args: tuple,
+        **kwargs: dict[str, Any],
     ) -> T:
         self.benchmark_count += 1
 
@@ -77,8 +80,54 @@ class ValgrindInstrument(Instrument):
             self.instrument_hooks.lib.callgrind_stop_instrumentation()
             self.instrument_hooks.set_executed_benchmark(uri)
 
+    def measure_pedantic(
+        self,
+        marker_options: BenchmarkMarkerOptions,
+        pedantic_options: PedanticOptions[T],
+        name: str,
+        uri: str,
+    ) -> T:
+        if pedantic_options.rounds != 1 or pedantic_options.iterations != 1:
+            warnings.warn(
+                "Valgrind instrument ignores rounds and iterations settings "
+                "in pedantic mode"
+            )
+        if not self.instrument_hooks:
+            args, kwargs = pedantic_options.setup_and_get_args_kwargs()
+            out = pedantic_options.target(*args, **kwargs)
+            if pedantic_options.teardown is not None:
+                pedantic_options.teardown(*args, **kwargs)
+            return out
+
+        def __codspeed_root_frame__(*args, **kwargs) -> T:
+            return pedantic_options.target(*args, **kwargs)
+
+        # Warmup
+        warmup_rounds = max(
+            pedantic_options.warmup_rounds, 1 if SUPPORTS_PERF_TRAMPOLINE else 0
+        )
+        for _ in range(warmup_rounds):
+            args, kwargs = pedantic_options.setup_and_get_args_kwargs()
+            __codspeed_root_frame__(*args, **kwargs)
+            if pedantic_options.teardown is not None:
+                pedantic_options.teardown(*args, **kwargs)
+
+        # Compute the actual result of the function
+        args, kwargs = pedantic_options.setup_and_get_args_kwargs()
+        self.instrument_hooks.lib.callgrind_start_instrumentation()
+        try:
+            out = __codspeed_root_frame__(*args, **kwargs)
+        finally:
+            self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.set_executed_benchmark(uri)
+            if pedantic_options.teardown is not None:
+                pedantic_options.teardown(*args, **kwargs)
+
+        return out
+
     def report(self, session: Session) -> None:
         reporter = session.config.pluginmanager.get_plugin("terminalreporter")
+        assert reporter is not None, "terminalreporter not found"
         count_suffix = "benchmarked" if self.should_measure else "benchmark tested"
         reporter.write_sep(
             "=",