pytest-codspeed 4.2.0b0__tar.gz → 4.4.0__tar.gz

{pytest_codspeed-4.2.0b0/src/pytest_codspeed.egg-info → pytest_codspeed-4.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytest-codspeed
-Version: 4.2.0b0
+Version: 4.4.0
 Summary: Pytest plugin to create CodSpeed benchmarks
 Author-email: Arthur Pastel <arthur@codspeed.io>
 License: The MIT License (MIT)
@@ -40,6 +40,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: System :: Benchmark
 Classifier: Topic :: Utilities
@@ -59,7 +60,7 @@ Requires-Dist: pytest-xdist~=3.6.1; extra == "compat"
 
 [![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
 [![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
-![Python Version](https://img.shields.io/badge/python-3.9%20|%203.10%20|%203.11%20|%203.12%20|%203.13-informational.svg)
+![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-informational.svg)
 [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
 [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
 
@@ -117,10 +118,10 @@ If you want to run the benchmarks tests locally, you can use the `--codspeed` py
 ```sh
 $ pytest tests/ --codspeed
 ============================= test session starts ====================
-platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
-codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
+platform darwin -- Python 3.14.0, pytest-7.4.4, pluggy-1.5.0
+codspeed: 4.2.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
 rootdir: /home/user/codspeed-test, configfile: pytest.ini
-plugins: codspeed-3.0.0
+plugins: codspeed-4.2.0
 collected 1 items
 
 tests/test_sum_squares.py .                                    [ 100%]
@@ -158,10 +159,10 @@ jobs:
     name: Run benchmarks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
 
       - name: Install dependencies
         run: pip install -r requirements.txt
@@ -169,7 +170,7 @@ jobs:
       - name: Run benchmarks
         uses: CodSpeedHQ/action@v4
         with:
-          mode: instrumentation   # or `walltime`
+          mode: simulation # or `walltime`
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: pytest tests/ --codspeed
 ```

{pytest_codspeed-4.2.0b0 → pytest_codspeed-4.4.0}/README.md

@@ -3,7 +3,7 @@
 
 [![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
 [![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
-![Python Version](https://img.shields.io/badge/python-3.9%20|%203.10%20|%203.11%20|%203.12%20|%203.13-informational.svg)
+![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-informational.svg)
 [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
 [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
 
@@ -61,10 +61,10 @@ If you want to run the benchmarks tests locally, you can use the `--codspeed` py
 ```sh
 $ pytest tests/ --codspeed
 ============================= test session starts ====================
-platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
-codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
+platform darwin -- Python 3.14.0, pytest-7.4.4, pluggy-1.5.0
+codspeed: 4.2.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
 rootdir: /home/user/codspeed-test, configfile: pytest.ini
-plugins: codspeed-3.0.0
+plugins: codspeed-4.2.0
 collected 1 items
 
 tests/test_sum_squares.py .                                    [ 100%]
@@ -102,10 +102,10 @@ jobs:
     name: Run benchmarks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
 
       - name: Install dependencies
         run: pip install -r requirements.txt
@@ -113,7 +113,7 @@ jobs:
       - name: Run benchmarks
         uses: CodSpeedHQ/action@v4
         with:
-          mode: instrumentation   # or `walltime`
+          mode: simulation # or `walltime`
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: pytest tests/ --codspeed
 ```

{pytest_codspeed-4.2.0b0 → pytest_codspeed-4.4.0}/pyproject.toml

@@ -24,6 +24,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Testing",
     "Topic :: System :: Benchmark",
     "Topic :: Utilities",
@@ -43,13 +44,18 @@ compat = [
     # "pytest-speed>=0.3.5",
 ]
 
+[tool.uv]
+# Python builds change with uv versions, and we are quite susceptible to that.
+# We pin uv to to make sure reproducibility is maintained for any contributor.
+required-version = "0.9.5"
+
 [tool.uv.sources]
 pytest-codspeed = { workspace = true }
 
 [dependency-groups]
 dev = [
     "pytest-codspeed",
-    "mypy ~= 1.11.2",
+    "mypy ~= 1.18.2",
     "ruff ~= 0.11.12",
     "pytest ~= 7.0",
     "pytest-cov ~= 4.0.0",
@@ -71,7 +77,7 @@ version = { attr = "pytest_codspeed.__version__" }
 
 
 [tool.bumpver]
-current_version = "4.2.0-beta"
+current_version = "4.4.0"
 version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]"
 commit_message = "Release v{new_version} 🚀"
 tag_message = "Release v{new_version} 🚀"

{pytest_codspeed-4.2.0b0 → pytest_codspeed-4.4.0}/src/pytest_codspeed/__init__.py

@@ -1,6 +1,6 @@
-__version__ = "4.2.0b0"
+__version__ = "4.4.0"
 # We also have the semver version since __version__ is not semver compliant
-__semver_version__ = "4.2.0-beta"
+__semver_version__ = "4.4.0"
 
 from .plugin import BenchmarkFixture
 

{pytest_codspeed-4.2.0b0 → pytest_codspeed-4.4.0}/src/pytest_codspeed/instruments/__init__.py

@@ -21,7 +21,7 @@ class Instrument(metaclass=ABCMeta):
     instrument: ClassVar[str]
 
     @abstractmethod
-    def __init__(self, config: CodSpeedConfig): ...
+    def __init__(self, config: CodSpeedConfig, mode: MeasurementMode): ...
 
     @abstractmethod
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
@@ -56,17 +56,25 @@ class Instrument(metaclass=ABCMeta):
 
 
 class MeasurementMode(str, Enum):
-    Instrumentation = "instrumentation"
+    Simulation = "simulation"
+    Memory = "memory"
     WallTime = "walltime"
 
+    @classmethod
+    def _missing_(cls, value: object):
+        # Accept "instrumentation" as deprecated alias for "simulation"
+        if value == "instrumentation":
+            return cls.Simulation
+        return None
+
 
 def get_instrument_from_mode(mode: MeasurementMode) -> type[Instrument]:
-    from pytest_codspeed.instruments.valgrind import (
-        ValgrindInstrument,
+    from pytest_codspeed.instruments.analysis import (
+        AnalysisInstrument,
     )
     from pytest_codspeed.instruments.walltime import WallTimeInstrument
 
-    if mode == MeasurementMode.Instrumentation:
-        return ValgrindInstrument
+    if mode in (MeasurementMode.Simulation, MeasurementMode.Memory):
+        return AnalysisInstrument
     else:
         return WallTimeInstrument

pytest_codspeed-4.2.0b0/src/pytest_codspeed/instruments/valgrind.py → pytest_codspeed-4.4.0/src/pytest_codspeed/instruments/analysis.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
+import os
 import warnings
 from typing import TYPE_CHECKING
 
 from pytest_codspeed import __semver_version__
 from pytest_codspeed.instruments import Instrument
-from pytest_codspeed.instruments.hooks import InstrumentHooks
+from pytest_codspeed.instruments.hooks import (
+    FEATURE_DISABLE_CALLGRIND_MARKERS,
+    InstrumentHooks,
+)
 from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE
 
 if TYPE_CHECKING:
@@ -14,27 +18,34 @@ if TYPE_CHECKING:
     from pytest import Session
 
     from pytest_codspeed.config import PedanticOptions
-    from pytest_codspeed.instruments import P, T
+    from pytest_codspeed.instruments import MeasurementMode, P, T
     from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 
-class ValgrindInstrument(Instrument):
-    instrument = "valgrind"
+class AnalysisInstrument(Instrument):
+    instrument = "analysis"
     instrument_hooks: InstrumentHooks | None
+    mode: MeasurementMode
 
-    def __init__(self, config: CodSpeedConfig) -> None:
+    def __init__(self, config: CodSpeedConfig, mode: MeasurementMode) -> None:
+        self.mode = mode
         self.benchmark_count = 0
         try:
             self.instrument_hooks = InstrumentHooks()
             self.instrument_hooks.set_integration("pytest-codspeed", __semver_version__)
-        except RuntimeError:
+            self.instrument_hooks.collect_and_write_python_environment()
+        except RuntimeError as e:
+            if os.environ.get("CODSPEED_ENV") is not None:
+                raise Exception(
+                    f"Failed to initialize {self.mode.value} instrument hooks"
+                ) from e
             self.instrument_hooks = None
 
         self.should_measure = self.instrument_hooks is not None
 
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
         config = (
-            f"mode: instrumentation, "
+            f"mode: {self.mode.value}, "
             f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
         )
         warnings = []
@@ -68,6 +79,9 @@ class ValgrindInstrument(Instrument):
             # Warmup CPython performance map cache
             __codspeed_root_frame__()
 
+        self.instrument_hooks.set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, True)
+        self.instrument_hooks.start_benchmark()
+
         # Manually call the library function to avoid an extra stack frame. Also
         # call the callgrind markers directly to avoid extra overhead.
         self.instrument_hooks.lib.callgrind_start_instrumentation()
@@ -76,6 +90,7 @@ class ValgrindInstrument(Instrument):
         finally:
             # Ensure instrumentation is stopped even if the test failed
             self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
 
     def measure_pedantic(
@@ -87,8 +102,8 @@ class ValgrindInstrument(Instrument):
     ) -> T:
         if pedantic_options.rounds != 1 or pedantic_options.iterations != 1:
             warnings.warn(
-                "Valgrind instrument ignores rounds and iterations settings "
-                "in pedantic mode"
+                f"{self.mode.value.capitalize()} instrument ignores rounds and "
+                "iterations settings in pedantic mode"
             )
         if not self.instrument_hooks:
             args, kwargs = pedantic_options.setup_and_get_args_kwargs()
@@ -112,11 +127,18 @@ class ValgrindInstrument(Instrument):
 
         # Compute the actual result of the function
         args, kwargs = pedantic_options.setup_and_get_args_kwargs()
+
+        self.instrument_hooks.set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, True)
+        self.instrument_hooks.start_benchmark()
+
+        # Manually call the library function to avoid an extra stack frame. Also
+        # call the callgrind markers directly to avoid extra overhead.
         self.instrument_hooks.lib.callgrind_start_instrumentation()
         try:
             out = __codspeed_root_frame__(*args, **kwargs)
         finally:
             self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
             if pedantic_options.teardown is not None:
                 pedantic_options.teardown(*args, **kwargs)
@@ -135,5 +157,5 @@ class ValgrindInstrument(Instrument):
     def get_result_dict(self) -> dict[str, Any]:
         return {
             "instrument": {"type": self.instrument},
-            # bench results will be dumped by valgrind
+            # bench results will be dumped by the runner
         }

pytest_codspeed-4.4.0/src/pytest_codspeed/instruments/hooks/__init__.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import os
+import platform
+import shlex
+import sys
+import sysconfig
+import warnings
+from typing import TYPE_CHECKING
+
+from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE
+
+if TYPE_CHECKING:
+    from cffi import FFI
+
+    from .dist_instrument_hooks import InstrumentHooksPointer, LibType
+
+# Feature flags for instrument hooks
+FEATURE_DISABLE_CALLGRIND_MARKERS = 0
+
+
+class InstrumentHooks:
+    """Zig library wrapper class providing benchmark measurement functionality."""
+
+    lib: LibType
+    ffi: FFI
+    instance: InstrumentHooksPointer
+
+    def __init__(self) -> None:
+        if os.environ.get("CODSPEED_ENV") is None:
+            raise RuntimeError(
+                "Can't run benchmarks outside of CodSpeed environment."
+                "Please set the CODSPEED_ENV environment variable."
+            )
+
+        try:
+            from .dist_instrument_hooks import ffi, lib  # type: ignore
+        except ImportError as e:
+            raise RuntimeError(f"Failed to load instrument hooks library: {e}") from e
+        self.lib = lib
+        self.ffi = ffi
+
+        self.instance = self.lib.instrument_hooks_init()
+        if self.instance == 0:
+            raise RuntimeError("Failed to initialize CodSpeed instrumentation library.")
+
+        if SUPPORTS_PERF_TRAMPOLINE and not sys.is_stack_trampoline_active():
+            sys.activate_stack_trampoline("perf")  # type: ignore
+
+    def __del__(self):
+        if hasattr(self, "lib") and hasattr(self, "instance"):
+            self.lib.instrument_hooks_deinit(self.instance)
+
+    def start_benchmark(self) -> None:
+        """Start a new benchmark measurement."""
+        ret = self.lib.instrument_hooks_start_benchmark(self.instance)
+        if ret != 0:
+            warnings.warn("Failed to start benchmark measurement", RuntimeWarning)
+
+    def stop_benchmark(self) -> None:
+        """Stop the current benchmark measurement."""
+        ret = self.lib.instrument_hooks_stop_benchmark(self.instance)
+        if ret != 0:
+            warnings.warn("Failed to stop benchmark measurement", RuntimeWarning)
+
+    def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None:
+        """Set the executed benchmark URI and process ID.
+
+        Args:
+            uri: The benchmark URI string identifier
+            pid: Optional process ID (defaults to current process)
+        """
+        if pid is None:
+            pid = os.getpid()
+
+        ret = self.lib.instrument_hooks_set_executed_benchmark(
+            self.instance, pid, uri.encode("ascii")
+        )
+        if ret != 0:
+            warnings.warn("Failed to set executed benchmark", RuntimeWarning)
+
+    def set_integration(self, name: str, version: str) -> None:
+        """Set the integration name and version."""
+        ret = self.lib.instrument_hooks_set_integration(
+            self.instance, name.encode("ascii"), version.encode("ascii")
+        )
+        if ret != 0:
+            warnings.warn("Failed to set integration name and version", RuntimeWarning)
+
+    def is_instrumented(self) -> bool:
+        """Check if simulation is active."""
+        return self.lib.instrument_hooks_is_instrumented(self.instance)
+
+    def set_feature(self, feature: int, enabled: bool) -> None:
+        """Set a feature flag in the instrument hooks library.
+
+        Args:
+            feature: The feature flag to set
+            enabled: Whether to enable or disable the feature
+        """
+        self.lib.instrument_hooks_set_feature(feature, enabled)
+
+    def set_environment(self, section_name: str, key: str, value: str) -> None:
+        """Register a key-value pair under a named section for environment collection.
+
+        Args:
+            section_name: The section name (e.g. "Python")
+            key: The key (e.g. "version")
+            value: The value (e.g. "3.13.12")
+        """
+        ret = self.lib.instrument_hooks_set_environment(
+            self.instance,
+            section_name.encode("utf-8"),
+            key.encode("utf-8"),
+            value.encode("utf-8"),
+        )
+        if ret != 0:
+            warnings.warn("Failed to set environment data", RuntimeWarning)
+
+    def set_environment_list(
+        self, section_name: str, key: str, values: list[str]
+    ) -> None:
+        """Register a list of values under a named section for environment collection.
+
+        Args:
+            section_name: The section name (e.g. "python")
+            key: The key (e.g. "build_args")
+            values: The list of string values
+        """
+        encoded = [self.ffi.new("char[]", v.encode("utf-8")) for v in values]
+        c_values = self.ffi.new("char*[]", encoded)
+        ret = self.lib.instrument_hooks_set_environment_list(
+            self.instance,
+            section_name.encode("utf-8"),
+            key.encode("utf-8"),
+            c_values,
+            len(encoded),
+        )
+        if ret != 0:
+            warnings.warn("Failed to set environment list data", RuntimeWarning)
+
+    def write_environment(self, pid: int | None = None) -> None:
+        """Flush all registered environment sections to disk.
+
+        Writes to $CODSPEED_PROFILE_FOLDER/environment-<pid>.json.
+
+        Args:
+            pid: Optional process ID (defaults to current process)
+        """
+        if pid is None:
+            pid = os.getpid()
+        ret = self.lib.instrument_hooks_write_environment(self.instance, pid)
+        if ret != 0:
+            warnings.warn("Failed to write environment data", RuntimeWarning)
+
+    def collect_and_write_python_environment(self) -> None:
+        """Collect Python toolchain information and write it to disk."""
+        section = "python"
+        set_env = self.set_environment
+
+        # Core identity
+        set_env(section, "version", sys.version.strip())
+        set_env(section, "implementation", sys.implementation.name.strip())
+        set_env(section, "compiler", platform.python_compiler().strip())
+
+        config_vars = sysconfig.get_config_vars()
+
+        # Build arguments as a list
+        config_args = config_vars.get("CONFIG_ARGS", "")
+        if config_args:
+            build_args = [arg.strip() for arg in shlex.split(config_args)]
+            self.set_environment_list(section, "build_args", build_args)
+
+        # Performance-relevant build configuration as "KEY=value" list
+        _SYSCONFIG_KEYS = (
+            "abiflags",
+            "PY_ENABLE_SHARED",
+            "Py_GIL_DISABLED",
+            "Py_DEBUG",
+            "WITH_PYMALLOC",
+            "WITH_MIMALLOC",
+            "WITH_FREELISTS",
+            "HAVE_COMPUTED_GOTOS",
+            "Py_STATS",
+            "Py_TRACE_REFS",
+            "WITH_VALGRIND",
+            "WITH_DTRACE",
+        )
+        config_items = []
+        for key in _SYSCONFIG_KEYS:
+            value = config_vars.get(key)
+            if value is not None:
+                config_items.append(f"{key}={str(value).strip()}")
+        config_items.append(f"perf_trampoline={SUPPORTS_PERF_TRAMPOLINE}")
+        self.set_environment_list(section, "config", config_items)
+
+        self.write_environment()

{pytest_codspeed-4.2.0b0 → pytest_codspeed-4.4.0}/src/pytest_codspeed/instruments/hooks/build.py

@@ -34,6 +34,17 @@ uint64_t instrument_hooks_current_timestamp(void);
 
 void callgrind_start_instrumentation();
 void callgrind_stop_instrumentation();
+
+void instrument_hooks_set_feature(uint64_t feature, bool enabled);
+
+uint8_t instrument_hooks_set_environment(InstrumentHooks *, const char *section_name,
+                                         const char *key, const char *value);
+uint8_t instrument_hooks_set_environment_list(InstrumentHooks *,
+                                              const char *section_name,
+                                              const char *key,
+                                              const char *const *values,
+                                              uint32_t count);
+uint8_t instrument_hooks_write_environment(InstrumentHooks *, uint32_t pid);
 """)
 
 ffibuilder.set_source(
@@ -45,6 +56,8 @@ ffibuilder.set_source(
         "src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c",
     ],
     include_dirs=[str(includes_dir)],
+    # IMPORTANT: Keep in sync with instrument-hooks/ci.yml (COMMON_CFLAGS)
+    extra_compile_args=["-Wno-format-security"],
 )
 
 if __name__ == "__main__":

{pytest_codspeed-4.2.0b0 → pytest_codspeed-4.4.0}/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi

@@ -1,3 +1,5 @@
+from cffi import FFI
+
 InstrumentHooksPointer = object
 
 class lib:
@@ -29,5 +31,23 @@ class lib:
     def callgrind_start_instrumentation() -> int: ...
     @staticmethod
     def callgrind_stop_instrumentation() -> int: ...
+    @staticmethod
+    def instrument_hooks_set_feature(feature: int, enabled: bool) -> None: ...
+    @staticmethod
+    def instrument_hooks_set_environment(
+        hooks: InstrumentHooksPointer, section_name: bytes, key: bytes, value: bytes
+    ) -> int: ...
+    @staticmethod
+    def instrument_hooks_set_environment_list(
+        hooks: InstrumentHooksPointer,
+        section_name: bytes,
+        key: bytes,
+        values: FFI.CData,
+        count: int,
+    ) -> int: ...
+    @staticmethod
+    def instrument_hooks_write_environment(
+        hooks: InstrumentHooksPointer, pid: int
+    ) -> int: ...
 
 LibType = type[lib]