pytest-codspeed 3.2.0__tar.gz → 4.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {pytest_codspeed-3.2.0/src/pytest_codspeed.egg-info → pytest_codspeed-4.0.0}/PKG-INFO +45 -42
  2. pytest_codspeed-4.0.0/README.md +118 -0
  3. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/pyproject.toml +3 -3
  4. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/setup.py +3 -5
  5. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/__init__.py +2 -2
  6. pytest_codspeed-4.0.0/src/pytest_codspeed/config.py +119 -0
  7. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/instruments/__init__.py +15 -5
  8. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/__init__.py +82 -0
  9. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/build.py +26 -0
  10. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi +27 -0
  11. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c +7486 -0
  12. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/callgrind.h +124 -0
  13. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/core.h +24 -0
  14. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/valgrind.h +7168 -0
  15. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/zig.h +4209 -0
  16. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/valgrind.py +139 -0
  17. pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/walltime.py +379 -0
  18. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/plugin.py +94 -55
  19. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/utils.py +9 -0
  20. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0/src/pytest_codspeed.egg-info}/PKG-INFO +45 -42
  21. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/SOURCES.txt +10 -6
  22. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/requires.txt +1 -1
  23. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/tests/test_pytest_plugin.py +140 -0
  24. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/tests/test_pytest_plugin_cpu_instrumentation.py +87 -6
  25. pytest_codspeed-4.0.0/tests/test_pytest_plugin_walltime.py +88 -0
  26. pytest_codspeed-3.2.0/README.md +0 -115
  27. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/__init__.py +0 -95
  28. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/__init__.py +0 -15
  29. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/build.py +0 -19
  30. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.c +0 -25
  31. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.h +0 -6
  32. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.pyi +0 -13
  33. pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/walltime.py +0 -273
  34. pytest_codspeed-3.2.0/tests/test_pytest_plugin_walltime.py +0 -37
  35. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/LICENSE +0 -0
  36. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/setup.cfg +0 -0
  37. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/py.typed +0 -0
  38. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/dependency_links.txt +0 -0
  39. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/entry_points.txt +0 -0
  40. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/top_level.txt +0 -0
  41. {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: pytest-codspeed
3
- Version: 3.2.0
3
+ Version: 4.0.0
4
4
  Summary: Pytest plugin to create CodSpeed benchmarks
5
5
  Author-email: Arthur Pastel <arthur@codspeed.io>
6
6
  License: The MIT License (MIT)
@@ -26,7 +26,7 @@ License: The MIT License (MIT)
26
26
  THE SOFTWARE.
27
27
 
28
28
  Project-URL: Homepage, https://codspeed.io/
29
- Project-URL: Documentation, https://docs.codspeed.io/
29
+ Project-URL: Documentation, https://codspeed.io/docs/reference/pytest-codspeed
30
30
  Project-URL: Source, https://github.com/CodSpeedHQ/pytest-codspeed
31
31
  Keywords: codspeed,benchmark,performance,pytest
32
32
  Classifier: Development Status :: 5 - Production/Stable
@@ -52,7 +52,7 @@ Requires-Dist: rich>=13.8.1
52
52
  Requires-Dist: importlib-metadata>=8.5.0; python_version < "3.10"
53
53
  Provides-Extra: lint
54
54
  Requires-Dist: mypy~=1.11.2; extra == "lint"
55
- Requires-Dist: ruff~=0.6.5; extra == "lint"
55
+ Requires-Dist: ruff~=0.11.12; extra == "lint"
56
56
  Provides-Extra: compat
57
57
  Requires-Dist: pytest-benchmark~=5.0.0; extra == "compat"
58
58
  Requires-Dist: pytest-xdist~=3.6.1; extra == "compat"
@@ -73,11 +73,11 @@ Pytest plugin to create CodSpeed benchmarks
73
73
 
74
74
  </div>
75
75
 
76
- ## Requirements
76
+ ---
77
77
 
78
- **Python**: 3.9 and later
78
+ **Documentation**: https://codspeed.io/docs/reference/pytest-codspeed
79
79
 
80
- **pytest**: any recent version
80
+ ---
81
81
 
82
82
  ## Installation
83
83
 
@@ -89,9 +89,9 @@ pip install pytest-codspeed
89
89
 
90
90
  ### Creating benchmarks
91
91
 
92
- Creating benchmarks with `pytest-codspeed` is compatible with the standard `pytest-benchmark` API. So if you already have benchmarks written with it, you can start using `pytest-codspeed` right away.
92
+ In a nutshell, `pytest-codspeed` offers two approaches to create performance benchmarks that integrate seamlessly with your existing test suite.
93
93
 
94
- #### Marking a whole test function as a benchmark with `pytest.mark.benchmark`
94
+ Use `@pytest.mark.benchmark` to measure entire test functions automatically:
95
95
 
96
96
  ```python
97
97
  import pytest
@@ -99,55 +99,53 @@ from statistics import median
99
99
 
100
100
  @pytest.mark.benchmark
101
101
  def test_median_performance():
102
- return median([1, 2, 3, 4, 5])
102
+ input = [1, 2, 3, 4, 5]
103
+ output = sum(i**2 for i in input)
104
+ assert output == 55
103
105
  ```
104
106
 
105
- #### Benchmarking selected lines of a test function with the `benchmark` fixture
107
+ Since this measure the entire function, you might want to use the `benchmark` fixture for precise control over what code gets measured:
106
108
 
107
109
  ```python
108
- import pytest
109
- from statistics import mean
110
-
111
110
  def test_mean_performance(benchmark):
112
- # Precompute some data useful for the benchmark but that should not be
113
- # included in the benchmark time
114
111
  data = [1, 2, 3, 4, 5]
112
+ # Only the function call is measured
113
+ result = benchmark(lambda: sum(i**2 for i in data))
114
+ assert result == 55
115
+ ```
115
116
 
116
- # Benchmark the execution of the function
117
- benchmark(lambda: mean(data))
118
-
117
+ Check out the [full documentation](https://codspeed.io/docs/reference/pytest-codspeed) for more details.
119
118
 
120
- def test_mean_and_median_performance(benchmark):
121
- # Precompute some data useful for the benchmark but that should not be
122
- # included in the benchmark time
123
- data = [1, 2, 3, 4, 5]
119
+ ### Testing the benchmarks locally
124
120
 
125
- # Benchmark the execution of the function:
126
- # The `@benchmark` decorator will automatically call the function and
127
- # measure its execution
128
- @benchmark
129
- def bench():
130
- mean(data)
131
- median(data)
132
- ```
121
+ If you want to run the benchmarks tests locally, you can use the `--codspeed` pytest flag:
133
122
 
134
- ### Running benchmarks
123
+ ```sh
124
+ $ pytest tests/ --codspeed
125
+ ============================= test session starts ====================
126
+ platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
127
+ codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
128
+ rootdir: /home/user/codspeed-test, configfile: pytest.ini
129
+ plugins: codspeed-3.0.0
130
+ collected 1 items
135
131
 
136
- #### Testing the benchmarks locally
132
+ tests/test_sum_squares.py . [ 100%]
137
133
 
138
- If you want to run only the benchmarks tests locally, you can use the `--codspeed` pytest flag:
139
-
140
- ```shell
141
- pytest tests/ --codspeed
134
+ Benchmark Results
135
+ ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
136
+ ┃ Benchmark ┃ Time (best) ┃ Rel. StdDev ┃ Run time ┃ Iters ┃
137
+ ┣━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━╋━━━━━━━━┫
138
+ ┃test_sum_squares┃ 1,873ns ┃ 4.8% ┃ 3.00s ┃ 66,930 ┃
139
+ ┗━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━┻━━━━━━━━┛
140
+ =============================== 1 benchmarked ========================
141
+ =============================== 1 passed in 4.12s ====================
142
142
  ```
143
143
 
144
- > **Note:** Running `pytest-codspeed` locally will not produce any performance reporting. It's only useful for making sure that your benchmarks are working as expected. If you want to get performance reporting, you should run the benchmarks in your CI.
145
-
146
- #### In your CI
144
+ ### Running the benchmarks in your CI
147
145
 
148
146
  You can use the [CodSpeedHQ/action](https://github.com/CodSpeedHQ/action) to run the benchmarks in Github Actions and upload the results to CodSpeed.
149
147
 
150
- Example workflow:
148
+ Here is an example of a GitHub Actions workflow that runs the benchmarks and reports the results to CodSpeed on every push to the `main` branch and every pull request:
151
149
 
152
150
  ```yaml
153
151
  name: CodSpeed
@@ -157,6 +155,9 @@ on:
157
155
  branches:
158
156
  - "main" # or "master"
159
157
  pull_request:
158
+ # `workflow_dispatch` allows CodSpeed to trigger backtest
159
+ # performance analysis in order to generate initial data.
160
+ workflow_dispatch:
160
161
 
161
162
  jobs:
162
163
  benchmarks:
@@ -166,9 +167,11 @@ jobs:
166
167
  - uses: actions/checkout@v4
167
168
  - uses: actions/setup-python@v5
168
169
  with:
169
- python-version: "3.12"
170
+ python-version: "3.13"
171
+
170
172
  - name: Install dependencies
171
173
  run: pip install -r requirements.txt
174
+
172
175
  - name: Run benchmarks
173
176
  uses: CodSpeedHQ/action@v3
174
177
  with:
@@ -0,0 +1,118 @@
1
+ <div align="center">
2
+ <h1>pytest-codspeed</h1>
3
+
4
+ [![CI](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
5
+ [![PyPi Version](https://img.shields.io/pypi/v/pytest-codspeed?color=%2334D058&label=pypi)](https://pypi.org/project/pytest-codspeed)
6
+ ![Python Version](https://img.shields.io/badge/python-3.9%20|%203.10%20|%203.11%20|%203.12%20|%203.13-informational.svg)
7
+ [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
8
+ [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
9
+
10
+ Pytest plugin to create CodSpeed benchmarks
11
+
12
+ </div>
13
+
14
+ ---
15
+
16
+ **Documentation**: https://codspeed.io/docs/reference/pytest-codspeed
17
+
18
+ ---
19
+
20
+ ## Installation
21
+
22
+ ```shell
23
+ pip install pytest-codspeed
24
+ ```
25
+
26
+ ## Usage
27
+
28
+ ### Creating benchmarks
29
+
30
+ In a nutshell, `pytest-codspeed` offers two approaches to create performance benchmarks that integrate seamlessly with your existing test suite.
31
+
32
+ Use `@pytest.mark.benchmark` to measure entire test functions automatically:
33
+
34
+ ```python
35
+ import pytest
36
+ from statistics import median
37
+
38
+ @pytest.mark.benchmark
39
+ def test_median_performance():
40
+ input = [1, 2, 3, 4, 5]
41
+ output = sum(i**2 for i in input)
42
+ assert output == 55
43
+ ```
44
+
45
+ Since this measure the entire function, you might want to use the `benchmark` fixture for precise control over what code gets measured:
46
+
47
+ ```python
48
+ def test_mean_performance(benchmark):
49
+ data = [1, 2, 3, 4, 5]
50
+ # Only the function call is measured
51
+ result = benchmark(lambda: sum(i**2 for i in data))
52
+ assert result == 55
53
+ ```
54
+
55
+ Check out the [full documentation](https://codspeed.io/docs/reference/pytest-codspeed) for more details.
56
+
57
+ ### Testing the benchmarks locally
58
+
59
+ If you want to run the benchmarks tests locally, you can use the `--codspeed` pytest flag:
60
+
61
+ ```sh
62
+ $ pytest tests/ --codspeed
63
+ ============================= test session starts ====================
64
+ platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
65
+ codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
66
+ rootdir: /home/user/codspeed-test, configfile: pytest.ini
67
+ plugins: codspeed-3.0.0
68
+ collected 1 items
69
+
70
+ tests/test_sum_squares.py . [ 100%]
71
+
72
+ Benchmark Results
73
+ ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
74
+ ┃ Benchmark ┃ Time (best) ┃ Rel. StdDev ┃ Run time ┃ Iters ┃
75
+ ┣━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━╋━━━━━━━━┫
76
+ ┃test_sum_squares┃ 1,873ns ┃ 4.8% ┃ 3.00s ┃ 66,930 ┃
77
+ ┗━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━┻━━━━━━━━┛
78
+ =============================== 1 benchmarked ========================
79
+ =============================== 1 passed in 4.12s ====================
80
+ ```
81
+
82
+ ### Running the benchmarks in your CI
83
+
84
+ You can use the [CodSpeedHQ/action](https://github.com/CodSpeedHQ/action) to run the benchmarks in Github Actions and upload the results to CodSpeed.
85
+
86
+ Here is an example of a GitHub Actions workflow that runs the benchmarks and reports the results to CodSpeed on every push to the `main` branch and every pull request:
87
+
88
+ ```yaml
89
+ name: CodSpeed
90
+
91
+ on:
92
+ push:
93
+ branches:
94
+ - "main" # or "master"
95
+ pull_request:
96
+ # `workflow_dispatch` allows CodSpeed to trigger backtest
97
+ # performance analysis in order to generate initial data.
98
+ workflow_dispatch:
99
+
100
+ jobs:
101
+ benchmarks:
102
+ name: Run benchmarks
103
+ runs-on: ubuntu-latest
104
+ steps:
105
+ - uses: actions/checkout@v4
106
+ - uses: actions/setup-python@v5
107
+ with:
108
+ python-version: "3.13"
109
+
110
+ - name: Install dependencies
111
+ run: pip install -r requirements.txt
112
+
113
+ - name: Run benchmarks
114
+ uses: CodSpeedHQ/action@v3
115
+ with:
116
+ token: ${{ secrets.CODSPEED_TOKEN }}
117
+ run: pytest tests/ --codspeed
118
+ ```
@@ -1,6 +1,6 @@
1
1
  [project.urls]
2
2
  Homepage = "https://codspeed.io/"
3
- Documentation = "https://docs.codspeed.io/"
3
+ Documentation = "https://codspeed.io/docs/reference/pytest-codspeed"
4
4
  Source = "https://github.com/CodSpeedHQ/pytest-codspeed"
5
5
 
6
6
  [project]
@@ -37,7 +37,7 @@ dependencies = [
37
37
  ]
38
38
 
39
39
  [project.optional-dependencies]
40
- lint = ["mypy ~= 1.11.2", "ruff ~= 0.6.5"]
40
+ lint = ["mypy ~= 1.11.2", "ruff ~= 0.11.12"]
41
41
  compat = [
42
42
  "pytest-benchmark ~= 5.0.0",
43
43
  "pytest-xdist ~= 3.6.1",
@@ -66,7 +66,7 @@ version = { attr = "pytest_codspeed.__version__" }
66
66
 
67
67
 
68
68
  [tool.bumpver]
69
- current_version = "3.2.0"
69
+ current_version = "4.0.0"
70
70
  version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]"
71
71
  commit_message = "Release v{new_version} 🚀"
72
72
  tag_message = "Release v{new_version} 🚀"
@@ -5,9 +5,7 @@ from pathlib import Path
5
5
 
6
6
  from setuptools import setup
7
7
 
8
- build_path = (
9
- Path(__file__).parent / "src/pytest_codspeed/instruments/valgrind/_wrapper/build.py"
10
- )
8
+ build_path = Path(__file__).parent / "src/pytest_codspeed/instruments/hooks/build.py"
11
9
 
12
10
  spec = importlib.util.spec_from_file_location("build", build_path)
13
11
  assert spec is not None, "The spec should be initialized"
@@ -52,8 +50,8 @@ print(
52
50
  setup(
53
51
  package_data={
54
52
  "pytest_codspeed": [
55
- "instruments/valgrind/_wrapper/*.h",
56
- "instruments/valgrind/_wrapper/*.c",
53
+ "instruments/hooks/instrument-hooks/includes/*.h",
54
+ "instruments/hooks/instrument-hooks/dist/*.c",
57
55
  ]
58
56
  },
59
57
  ext_modules=(
@@ -1,6 +1,6 @@
1
- __version__ = "3.2.0"
1
+ __version__ = "4.0.0"
2
2
  # We also have the semver version since __version__ is not semver compliant
3
- __semver_version__ = "3.2.0"
3
+ __semver_version__ = "4.0.0"
4
4
 
5
5
  from .plugin import BenchmarkFixture
6
6
 
@@ -0,0 +1,119 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses
4
+ from dataclasses import dataclass, field
5
+ from typing import TYPE_CHECKING, Generic, TypeVar
6
+
7
+ T = TypeVar("T")
8
+
9
+ if TYPE_CHECKING:
10
+ from typing import Any, Callable
11
+
12
+ import pytest
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class CodSpeedConfig:
17
+ """
18
+ The configuration for the codspeed plugin.
19
+ Usually created from the command line arguments.
20
+ """
21
+
22
+ warmup_time_ns: int | None = None
23
+ max_time_ns: int | None = None
24
+ max_rounds: int | None = None
25
+
26
+ @classmethod
27
+ def from_pytest_config(cls, config: pytest.Config) -> CodSpeedConfig:
28
+ warmup_time = config.getoption("--codspeed-warmup-time", None)
29
+ warmup_time_ns = (
30
+ int(warmup_time * 1_000_000_000) if warmup_time is not None else None
31
+ )
32
+ max_time = config.getoption("--codspeed-max-time", None)
33
+ max_time_ns = int(max_time * 1_000_000_000) if max_time is not None else None
34
+ return cls(
35
+ warmup_time_ns=warmup_time_ns,
36
+ max_rounds=config.getoption("--codspeed-max-rounds", None),
37
+ max_time_ns=max_time_ns,
38
+ )
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class BenchmarkMarkerOptions:
43
+ group: str | None = None
44
+ """The group name to use for the benchmark."""
45
+ min_time: int | None = None
46
+ """
47
+ The minimum time of a round (in seconds).
48
+ Only available in walltime mode.
49
+ """
50
+ max_time: int | None = None
51
+ """
52
+ The maximum time to run the benchmark for (in seconds).
53
+ Only available in walltime mode.
54
+ """
55
+ max_rounds: int | None = None
56
+ """
57
+ The maximum number of rounds to run the benchmark for.
58
+ Takes precedence over max_time. Only available in walltime mode.
59
+ """
60
+
61
+ @classmethod
62
+ def from_pytest_item(cls, item: pytest.Item) -> BenchmarkMarkerOptions:
63
+ marker = item.get_closest_marker(
64
+ "codspeed_benchmark"
65
+ ) or item.get_closest_marker("benchmark")
66
+ if marker is None:
67
+ return cls()
68
+ if len(marker.args) > 0:
69
+ raise ValueError(
70
+ "Positional arguments are not allowed in the benchmark marker"
71
+ )
72
+ kwargs = marker.kwargs
73
+
74
+ unknown_kwargs = set(kwargs.keys()) - {
75
+ field.name for field in dataclasses.fields(cls)
76
+ }
77
+ if unknown_kwargs:
78
+ raise ValueError(
79
+ "Unknown kwargs passed to benchmark marker: "
80
+ + ", ".join(sorted(unknown_kwargs))
81
+ )
82
+
83
+ return cls(**kwargs)
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class PedanticOptions(Generic[T]):
88
+ """Parameters for running a benchmark using the pedantic fixture API."""
89
+
90
+ target: Callable[..., T]
91
+ setup: Callable[[], Any | None] | None
92
+ teardown: Callable[..., Any | None] | None
93
+ rounds: int
94
+ warmup_rounds: int
95
+ iterations: int
96
+ args: tuple[Any, ...] = field(default_factory=tuple)
97
+ kwargs: dict[str, Any] = field(default_factory=dict)
98
+
99
+ def __post_init__(self) -> None:
100
+ if self.rounds < 0:
101
+ raise ValueError("rounds must be positive")
102
+ if self.warmup_rounds < 0:
103
+ raise ValueError("warmup_rounds must be non-negative")
104
+ if self.iterations <= 0:
105
+ raise ValueError("iterations must be positive")
106
+ if self.iterations > 1 and self.setup is not None:
107
+ raise ValueError(
108
+ "setup cannot be used with multiple iterations, use multiple rounds"
109
+ )
110
+
111
+ def setup_and_get_args_kwargs(self) -> tuple[tuple[Any, ...], dict[str, Any]]:
112
+ if self.setup is None:
113
+ return self.args, self.kwargs
114
+ maybe_result = self.setup(*self.args, **self.kwargs)
115
+ if maybe_result is not None:
116
+ if len(self.args) > 0 or len(self.kwargs) > 0:
117
+ raise ValueError("setup cannot return a value when args are provided")
118
+ return maybe_result
119
+ return self.args, self.kwargs
@@ -5,14 +5,14 @@ from enum import Enum
5
5
  from typing import TYPE_CHECKING
6
6
 
7
7
  if TYPE_CHECKING:
8
- from typing import Any, Callable, ClassVar, ParamSpec, TypeVar
8
+ from typing import Any, Callable, ClassVar, TypeVar
9
9
 
10
10
  import pytest
11
11
 
12
+ from pytest_codspeed.config import BenchmarkMarkerOptions, PedanticOptions
12
13
  from pytest_codspeed.plugin import CodSpeedConfig
13
14
 
14
15
  T = TypeVar("T")
15
- P = ParamSpec("P")
16
16
 
17
17
 
18
18
  class Instrument(metaclass=ABCMeta):
@@ -27,11 +27,21 @@ class Instrument(metaclass=ABCMeta):
27
27
  @abstractmethod
28
28
  def measure(
29
29
  self,
30
+ marker_options: BenchmarkMarkerOptions,
31
+ name: str,
32
+ uri: str,
33
+ fn: Callable[..., T],
34
+ *args: tuple,
35
+ **kwargs: dict[str, Any],
36
+ ) -> T: ...
37
+
38
+ @abstractmethod
39
+ def measure_pedantic(
40
+ self,
41
+ marker_options: BenchmarkMarkerOptions,
42
+ pedantic_options: PedanticOptions[T],
30
43
  name: str,
31
44
  uri: str,
32
- fn: Callable[P, T],
33
- *args: P.args,
34
- **kwargs: P.kwargs,
35
45
  ) -> T: ...
36
46
 
37
47
  @abstractmethod
@@ -0,0 +1,82 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ import sys
5
+ import warnings
6
+ from typing import TYPE_CHECKING
7
+
8
+ from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE
9
+
10
+ if TYPE_CHECKING:
11
+ from .dist_instrument_hooks import InstrumentHooksPointer, LibType
12
+
13
+
14
+ class InstrumentHooks:
15
+ """Zig library wrapper class providing benchmark measurement functionality."""
16
+
17
+ lib: LibType
18
+ instance: InstrumentHooksPointer
19
+
20
+ def __init__(self) -> None:
21
+ if os.environ.get("CODSPEED_ENV") is None:
22
+ raise RuntimeError(
23
+ "Can't run benchmarks outside of CodSpeed environment."
24
+ "Please set the CODSPEED_ENV environment variable."
25
+ )
26
+
27
+ try:
28
+ from .dist_instrument_hooks import lib # type: ignore
29
+ except ImportError as e:
30
+ raise RuntimeError(f"Failed to load instrument hooks library: {e}") from e
31
+ self.lib = lib
32
+
33
+ self.instance = self.lib.instrument_hooks_init()
34
+ if self.instance == 0:
35
+ raise RuntimeError("Failed to initialize CodSpeed instrumentation library.")
36
+
37
+ if SUPPORTS_PERF_TRAMPOLINE:
38
+ sys.activate_stack_trampoline("perf") # type: ignore
39
+
40
+ def __del__(self):
41
+ if hasattr(self, "lib") and hasattr(self, "instance"):
42
+ self.lib.instrument_hooks_deinit(self.instance)
43
+
44
+ def start_benchmark(self) -> None:
45
+ """Start a new benchmark measurement."""
46
+ ret = self.lib.instrument_hooks_start_benchmark(self.instance)
47
+ if ret != 0:
48
+ warnings.warn("Failed to start benchmark measurement", RuntimeWarning)
49
+
50
+ def stop_benchmark(self) -> None:
51
+ """Stop the current benchmark measurement."""
52
+ ret = self.lib.instrument_hooks_stop_benchmark(self.instance)
53
+ if ret != 0:
54
+ warnings.warn("Failed to stop benchmark measurement", RuntimeWarning)
55
+
56
+ def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None:
57
+ """Set the executed benchmark URI and process ID.
58
+
59
+ Args:
60
+ uri: The benchmark URI string identifier
61
+ pid: Optional process ID (defaults to current process)
62
+ """
63
+ if pid is None:
64
+ pid = os.getpid()
65
+
66
+ ret = self.lib.instrument_hooks_executed_benchmark(
67
+ self.instance, pid, uri.encode("ascii")
68
+ )
69
+ if ret != 0:
70
+ warnings.warn("Failed to set executed benchmark", RuntimeWarning)
71
+
72
+ def set_integration(self, name: str, version: str) -> None:
73
+ """Set the integration name and version."""
74
+ ret = self.lib.instrument_hooks_set_integration(
75
+ self.instance, name.encode("ascii"), version.encode("ascii")
76
+ )
77
+ if ret != 0:
78
+ warnings.warn("Failed to set integration name and version", RuntimeWarning)
79
+
80
+ def is_instrumented(self) -> bool:
81
+ """Check if instrumentation is active."""
82
+ return self.lib.instrument_hooks_is_instrumented(self.instance)
@@ -0,0 +1,26 @@
1
+ from pathlib import Path
2
+
3
+ from cffi import FFI # type: ignore
4
+
5
+ ffibuilder = FFI()
6
+
7
+ includes_dir = Path(__file__).parent.joinpath("instrument-hooks/includes")
8
+ header_text = (includes_dir / "core.h").read_text()
9
+ filtered_header = "\n".join(
10
+ line for line in header_text.splitlines() if not line.strip().startswith("#")
11
+ )
12
+ ffibuilder.cdef(filtered_header)
13
+
14
+ ffibuilder.set_source(
15
+ "pytest_codspeed.instruments.hooks.dist_instrument_hooks",
16
+ """
17
+ #include "core.h"
18
+ """,
19
+ sources=[
20
+ "src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c",
21
+ ],
22
+ include_dirs=[str(includes_dir)],
23
+ )
24
+
25
+ if __name__ == "__main__":
26
+ ffibuilder.compile(verbose=True)
@@ -0,0 +1,27 @@
1
+ InstrumentHooksPointer = object
2
+
3
+ class lib:
4
+ @staticmethod
5
+ def instrument_hooks_init() -> InstrumentHooksPointer: ...
6
+ @staticmethod
7
+ def instrument_hooks_deinit(hooks: InstrumentHooksPointer) -> None: ...
8
+ @staticmethod
9
+ def instrument_hooks_is_instrumented(hooks: InstrumentHooksPointer) -> bool: ...
10
+ @staticmethod
11
+ def instrument_hooks_start_benchmark(hooks: InstrumentHooksPointer) -> int: ...
12
+ @staticmethod
13
+ def instrument_hooks_stop_benchmark(hooks: InstrumentHooksPointer) -> int: ...
14
+ @staticmethod
15
+ def instrument_hooks_executed_benchmark(
16
+ hooks: InstrumentHooksPointer, pid: int, uri: bytes
17
+ ) -> int: ...
18
+ @staticmethod
19
+ def instrument_hooks_set_integration(
20
+ hooks: InstrumentHooksPointer, name: bytes, version: bytes
21
+ ) -> int: ...
22
+ @staticmethod
23
+ def callgrind_start_instrumentation() -> int: ...
24
+ @staticmethod
25
+ def callgrind_stop_instrumentation() -> int: ...
26
+
27
+ LibType = type[lib]