pytest-codspeed 4.0.0b1__tar.gz → 4.2.0b0__tar.gz

{pytest_codspeed-4.0.0b1/src/pytest_codspeed.egg-info → pytest_codspeed-4.2.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytest-codspeed
-Version: 4.0.0b1
+Version: 4.2.0b0
 Summary: Pytest plugin to create CodSpeed benchmarks
 Author-email: Arthur Pastel <arthur@codspeed.io>
 License: The MIT License (MIT)
@@ -26,7 +26,7 @@ License: The MIT License (MIT)
         THE SOFTWARE.
         
 Project-URL: Homepage, https://codspeed.io/
-Project-URL: Documentation, https://docs.codspeed.io/
+Project-URL: Documentation, https://codspeed.io/docs/reference/pytest-codspeed
 Project-URL: Source, https://github.com/CodSpeedHQ/pytest-codspeed
 Keywords: codspeed,benchmark,performance,pytest
 Classifier: Development Status :: 5 - Production/Stable
@@ -50,15 +50,9 @@ Requires-Dist: cffi>=1.17.1
 Requires-Dist: pytest>=3.8
 Requires-Dist: rich>=13.8.1
 Requires-Dist: importlib-metadata>=8.5.0; python_version < "3.10"
-Provides-Extra: lint
-Requires-Dist: mypy~=1.11.2; extra == "lint"
-Requires-Dist: ruff~=0.11.12; extra == "lint"
 Provides-Extra: compat
 Requires-Dist: pytest-benchmark~=5.0.0; extra == "compat"
 Requires-Dist: pytest-xdist~=3.6.1; extra == "compat"
-Provides-Extra: test
-Requires-Dist: pytest~=7.0; extra == "test"
-Requires-Dist: pytest-cov~=4.0.0; extra == "test"
 
 <div align="center">
 <h1>pytest-codspeed</h1>
@@ -73,11 +67,11 @@ Pytest plugin to create CodSpeed benchmarks
 
 </div>
 
-## Requirements
+---
 
-**Python**: 3.9 and later
+**Documentation**: https://codspeed.io/docs/reference/pytest-codspeed
 
-**pytest**: any recent version
+---
 
 ## Installation
 
@@ -89,9 +83,9 @@ pip install pytest-codspeed
 
 ### Creating benchmarks
 
-Creating benchmarks with `pytest-codspeed` is compatible with the standard `pytest-benchmark` API. So if you already have benchmarks written with it, you can start using `pytest-codspeed` right away.
+In a nutshell, `pytest-codspeed` offers two approaches to create performance benchmarks that integrate seamlessly with your existing test suite.
 
-#### Marking a whole test function as a benchmark with `pytest.mark.benchmark`
+Use `@pytest.mark.benchmark` to measure entire test functions automatically:
 
 ```python
 import pytest
@@ -99,55 +93,53 @@ from statistics import median
 
 @pytest.mark.benchmark
 def test_median_performance():
-    return median([1, 2, 3, 4, 5])
+    input = [1, 2, 3, 4, 5]
+    output = sum(i**2 for i in input)
+    assert output == 55
 ```
 
-#### Benchmarking selected lines of a test function with the `benchmark` fixture
+Since this measure the entire function, you might want to use the `benchmark` fixture for precise control over what code gets measured:
 
 ```python
-import pytest
-from statistics import mean
-
 def test_mean_performance(benchmark):
-    # Precompute some data useful for the benchmark but that should not be
-    # included in the benchmark time
     data = [1, 2, 3, 4, 5]
+    # Only the function call is measured
+    result = benchmark(lambda: sum(i**2 for i in data))
+    assert result == 55
+```
 
-    # Benchmark the execution of the function
-    benchmark(lambda: mean(data))
-
+Check out the [full documentation](https://codspeed.io/docs/reference/pytest-codspeed) for more details.
 
-def test_mean_and_median_performance(benchmark):
-    # Precompute some data useful for the benchmark but that should not be
-    # included in the benchmark time
-    data = [1, 2, 3, 4, 5]
+### Testing the benchmarks locally
 
-    # Benchmark the execution of the function:
-    # The `@benchmark` decorator will automatically call the function and
-    # measure its execution
-    @benchmark
-    def bench():
-        mean(data)
-        median(data)
-```
+If you want to run the benchmarks tests locally, you can use the `--codspeed` pytest flag:
 
-### Running benchmarks
+```sh
+$ pytest tests/ --codspeed
+============================= test session starts ====================
+platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
+codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
+rootdir: /home/user/codspeed-test, configfile: pytest.ini
+plugins: codspeed-3.0.0
+collected 1 items
 
-#### Testing the benchmarks locally
+tests/test_sum_squares.py .                                    [ 100%]
 
-If you want to run only the benchmarks tests locally, you can use the `--codspeed` pytest flag:
-
-```shell
-pytest tests/ --codspeed
+                         Benchmark Results
+┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
+┃     Benchmark  ┃ Time (best) ┃ Rel. StdDev ┃ Run time ┃ Iters  ┃
+┣━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━╋━━━━━━━━┫
+┃test_sum_squares┃     1,873ns ┃        4.8% ┃    3.00s ┃ 66,930 ┃
+┗━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━┻━━━━━━━━┛
+=============================== 1 benchmarked ========================
+=============================== 1 passed in 4.12s ====================
 ```
 
-> **Note:** Running `pytest-codspeed` locally will not produce any performance reporting. It's only useful for making sure that your benchmarks are working as expected. If you want to get performance reporting, you should run the benchmarks in your CI.
-
-#### In your CI
+### Running the benchmarks in your CI
 
 You can use the [CodSpeedHQ/action](https://github.com/CodSpeedHQ/action) to run the benchmarks in Github Actions and upload the results to CodSpeed.
 
-Example workflow:
+Here is an example of a GitHub Actions workflow that runs the benchmarks and reports the results to CodSpeed on every push to the `main` branch and every pull request:
 
 ```yaml
 name: CodSpeed
@@ -156,7 +148,10 @@ on:
   push:
     branches:
       - "main" # or "master"
-  pull_request:
+  pull_request: # required to have reports on PRs
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
 
 jobs:
   benchmarks:
@@ -166,12 +161,15 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: "3.13"
+
       - name: Install dependencies
         run: pip install -r requirements.txt
+
       - name: Run benchmarks
-        uses: CodSpeedHQ/action@v3
+        uses: CodSpeedHQ/action@v4
         with:
+          mode: instrumentation   # or `walltime`
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: pytest tests/ --codspeed
 ```

pytest_codspeed-4.2.0b0/README.md

@@ -0,0 +1,119 @@
+<div align="center">
+<h1>pytest-codspeed</h1>
+
+[![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
+[![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
+![Python Version](https://img.shields.io/badge/python-3.9%20|%203.10%20|%203.11%20|%203.12%20|%203.13-informational.svg)
+[![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
+[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
+
+Pytest plugin to create CodSpeed benchmarks
+
+</div>
+
+---
+
+**Documentation**: https://codspeed.io/docs/reference/pytest-codspeed
+
+---
+
+## Installation
+
+```shell
+pip install pytest-codspeed
+```
+
+## Usage
+
+### Creating benchmarks
+
+In a nutshell, `pytest-codspeed` offers two approaches to create performance benchmarks that integrate seamlessly with your existing test suite.
+
+Use `@pytest.mark.benchmark` to measure entire test functions automatically:
+
+```python
+import pytest
+from statistics import median
+
+@pytest.mark.benchmark
+def test_median_performance():
+    input = [1, 2, 3, 4, 5]
+    output = sum(i**2 for i in input)
+    assert output == 55
+```
+
+Since this measure the entire function, you might want to use the `benchmark` fixture for precise control over what code gets measured:
+
+```python
+def test_mean_performance(benchmark):
+    data = [1, 2, 3, 4, 5]
+    # Only the function call is measured
+    result = benchmark(lambda: sum(i**2 for i in data))
+    assert result == 55
+```
+
+Check out the [full documentation](https://codspeed.io/docs/reference/pytest-codspeed) for more details.
+
+### Testing the benchmarks locally
+
+If you want to run the benchmarks tests locally, you can use the `--codspeed` pytest flag:
+
+```sh
+$ pytest tests/ --codspeed
+============================= test session starts ====================
+platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
+codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
+rootdir: /home/user/codspeed-test, configfile: pytest.ini
+plugins: codspeed-3.0.0
+collected 1 items
+
+tests/test_sum_squares.py .                                    [ 100%]
+
+                         Benchmark Results
+┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
+┃     Benchmark  ┃ Time (best) ┃ Rel. StdDev ┃ Run time ┃ Iters  ┃
+┣━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━╋━━━━━━━━┫
+┃test_sum_squares┃     1,873ns ┃        4.8% ┃    3.00s ┃ 66,930 ┃
+┗━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━┻━━━━━━━━┛
+=============================== 1 benchmarked ========================
+=============================== 1 passed in 4.12s ====================
+```
+
+### Running the benchmarks in your CI
+
+You can use the [CodSpeedHQ/action](https://github.com/CodSpeedHQ/action) to run the benchmarks in Github Actions and upload the results to CodSpeed.
+
+Here is an example of a GitHub Actions workflow that runs the benchmarks and reports the results to CodSpeed on every push to the `main` branch and every pull request:
+
+```yaml
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - "main" # or "master"
+  pull_request: # required to have reports on PRs
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: instrumentation   # or `walltime`
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: pytest tests/ --codspeed
+```

{pytest_codspeed-4.0.0b1 → pytest_codspeed-4.2.0b0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project.urls]
 Homepage = "https://codspeed.io/"
-Documentation = "https://docs.codspeed.io/"
+Documentation = "https://codspeed.io/docs/reference/pytest-codspeed"
 Source = "https://github.com/CodSpeedHQ/pytest-codspeed"
 
 [project]
@@ -37,19 +37,24 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-lint = ["mypy ~= 1.11.2", "ruff ~= 0.11.12"]
 compat = [
     "pytest-benchmark ~= 5.0.0",
     "pytest-xdist ~= 3.6.1",
     # "pytest-speed>=0.3.5",
 ]
-test = ["pytest ~= 7.0", "pytest-cov ~= 4.0.0"]
 
 [tool.uv.sources]
 pytest-codspeed = { workspace = true }
 
 [dependency-groups]
-dev = ["pytest-codspeed"]
+dev = [
+    "pytest-codspeed",
+    "mypy ~= 1.11.2",
+    "ruff ~= 0.11.12",
+    "pytest ~= 7.0",
+    "pytest-cov ~= 4.0.0",
+    "pytest-test-groups>=1.1.0",
+]
 
 [project.entry-points]
 pytest11 = { codspeed = "pytest_codspeed.plugin" }
@@ -66,7 +71,7 @@ version = { attr = "pytest_codspeed.__version__" }
 
 
 [tool.bumpver]
-current_version = "4.0.0-beta1"
+current_version = "4.2.0-beta"
 version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]"
 commit_message = "Release v{new_version} 🚀"
 tag_message = "Release v{new_version} 🚀"

{pytest_codspeed-4.0.0b1 → pytest_codspeed-4.2.0b0}/src/pytest_codspeed/__init__.py

@@ -1,6 +1,6 @@
-__version__ = "4.0.0b1"
+__version__ = "4.2.0b0"
 # We also have the semver version since __version__ is not semver compliant
-__semver_version__ = "4.0.0-beta1"
+__semver_version__ = "4.2.0-beta"
 
 from .plugin import BenchmarkFixture
 

{pytest_codspeed-4.0.0b1 → pytest_codspeed-4.2.0b0}/src/pytest_codspeed/instruments/__init__.py

@@ -8,11 +8,13 @@ if TYPE_CHECKING:
     from typing import Any, Callable, ClassVar, TypeVar
 
     import pytest
+    from typing_extensions import ParamSpec
 
     from pytest_codspeed.config import BenchmarkMarkerOptions, PedanticOptions
     from pytest_codspeed.plugin import CodSpeedConfig
 
     T = TypeVar("T")
+    P = ParamSpec("P")
 
 
 class Instrument(metaclass=ABCMeta):
@@ -30,9 +32,9 @@ class Instrument(metaclass=ABCMeta):
         marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
-        fn: Callable[..., T],
-        *args: tuple,
-        **kwargs: dict[str, Any],
+        fn: Callable[P, T],
+        *args: P.args,
+        **kwargs: P.kwargs,
     ) -> T: ...
 
     @abstractmethod

{pytest_codspeed-4.0.0b1 → pytest_codspeed-4.2.0b0}/src/pytest_codspeed/instruments/hooks/__init__.py

@@ -5,11 +5,11 @@ import sys
 import warnings
 from typing import TYPE_CHECKING
 
+from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE
+
 if TYPE_CHECKING:
     from .dist_instrument_hooks import InstrumentHooksPointer, LibType
 
-SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12)
-
 
 class InstrumentHooks:
     """Zig library wrapper class providing benchmark measurement functionality."""
@@ -63,7 +63,7 @@ class InstrumentHooks:
         if pid is None:
             pid = os.getpid()
 
-        ret = self.lib.instrument_hooks_executed_benchmark(
+        ret = self.lib.instrument_hooks_set_executed_benchmark(
             self.instance, pid, uri.encode("ascii")
         )
         if ret != 0:

pytest_codspeed-4.2.0b0/src/pytest_codspeed/instruments/hooks/build.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+
+from cffi import FFI  # type: ignore
+
+ffibuilder = FFI()
+
+includes_dir = Path(__file__).parent.joinpath("instrument-hooks/includes")
+header_text = (includes_dir / "core.h").read_text()
+
+
+# Manually copied from `instrument-hooks/includes/core.h` to avoid parsing issues
+ffibuilder.cdef("""
+typedef uint64_t *InstrumentHooks;
+
+InstrumentHooks *instrument_hooks_init(void);
+void instrument_hooks_deinit(InstrumentHooks *);
+
+bool instrument_hooks_is_instrumented(InstrumentHooks *);
+uint8_t instrument_hooks_start_benchmark(InstrumentHooks *);
+uint8_t instrument_hooks_stop_benchmark(InstrumentHooks *);
+uint8_t instrument_hooks_set_executed_benchmark(InstrumentHooks *, int32_t pid,
+                                               const char *uri);
+uint8_t instrument_hooks_set_integration(InstrumentHooks *, const char *name,
+                                        const char *version);
+
+#define MARKER_TYPE_SAMPLE_START 0
+#define MARKER_TYPE_SAMPLE_END 1
+#define MARKER_TYPE_BENCHMARK_START 2
+#define MARKER_TYPE_BENCHMARK_END 3
+
+uint8_t instrument_hooks_add_marker(InstrumentHooks *, uint32_t pid,
+                                   uint8_t marker_type, uint64_t timestamp);
+uint64_t instrument_hooks_current_timestamp(void);
+
+void callgrind_start_instrumentation();
+void callgrind_stop_instrumentation();
+""")
+
+ffibuilder.set_source(
+    "pytest_codspeed.instruments.hooks.dist_instrument_hooks",
+    """
+    #include "core.h"
+    """,
+    sources=[
+        "src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c",
+    ],
+    include_dirs=[str(includes_dir)],
+)
+
+if __name__ == "__main__":
+    ffibuilder.compile(verbose=True)

{pytest_codspeed-4.0.0b1 → pytest_codspeed-4.2.0b0}/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi

@@ -12,7 +12,7 @@ class lib:
     @staticmethod
     def instrument_hooks_stop_benchmark(hooks: InstrumentHooksPointer) -> int: ...
     @staticmethod
-    def instrument_hooks_executed_benchmark(
+    def instrument_hooks_set_executed_benchmark(
         hooks: InstrumentHooksPointer, pid: int, uri: bytes
     ) -> int: ...
     @staticmethod
@@ -20,6 +20,12 @@ class lib:
         hooks: InstrumentHooksPointer, name: bytes, version: bytes
     ) -> int: ...
     @staticmethod
+    def instrument_hooks_add_marker(
+        hooks: InstrumentHooksPointer, pid: int, marker_type: int, timestamp: int
+    ) -> int: ...
+    @staticmethod
+    def instrument_hooks_current_timestamp() -> int: ...
+    @staticmethod
     def callgrind_start_instrumentation() -> int: ...
     @staticmethod
     def callgrind_stop_instrumentation() -> int: ...