pytest-codspeed 3.2.0__tar.gz → 4.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytest_codspeed-3.2.0/src/pytest_codspeed.egg-info → pytest_codspeed-4.0.0}/PKG-INFO +45 -42
- pytest_codspeed-4.0.0/README.md +118 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/pyproject.toml +3 -3
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/setup.py +3 -5
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/__init__.py +2 -2
- pytest_codspeed-4.0.0/src/pytest_codspeed/config.py +119 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/instruments/__init__.py +15 -5
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/__init__.py +82 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/build.py +26 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi +27 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c +7486 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/callgrind.h +124 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/core.h +24 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/valgrind.h +7168 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/hooks/instrument-hooks/includes/zig.h +4209 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/valgrind.py +139 -0
- pytest_codspeed-4.0.0/src/pytest_codspeed/instruments/walltime.py +379 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/plugin.py +94 -55
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/utils.py +9 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0/src/pytest_codspeed.egg-info}/PKG-INFO +45 -42
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/SOURCES.txt +10 -6
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/requires.txt +1 -1
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/tests/test_pytest_plugin.py +140 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/tests/test_pytest_plugin_cpu_instrumentation.py +87 -6
- pytest_codspeed-4.0.0/tests/test_pytest_plugin_walltime.py +88 -0
- pytest_codspeed-3.2.0/README.md +0 -115
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/__init__.py +0 -95
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/__init__.py +0 -15
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/build.py +0 -19
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.c +0 -25
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.h +0 -6
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.pyi +0 -13
- pytest_codspeed-3.2.0/src/pytest_codspeed/instruments/walltime.py +0 -273
- pytest_codspeed-3.2.0/tests/test_pytest_plugin_walltime.py +0 -37
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/LICENSE +0 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/setup.cfg +0 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed/py.typed +0 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/dependency_links.txt +0 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/entry_points.txt +0 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/src/pytest_codspeed.egg-info/top_level.txt +0 -0
- {pytest_codspeed-3.2.0 → pytest_codspeed-4.0.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pytest-codspeed
|
|
3
|
-
Version:
|
|
3
|
+
Version: 4.0.0
|
|
4
4
|
Summary: Pytest plugin to create CodSpeed benchmarks
|
|
5
5
|
Author-email: Arthur Pastel <arthur@codspeed.io>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -26,7 +26,7 @@ License: The MIT License (MIT)
|
|
|
26
26
|
THE SOFTWARE.
|
|
27
27
|
|
|
28
28
|
Project-URL: Homepage, https://codspeed.io/
|
|
29
|
-
Project-URL: Documentation, https://
|
|
29
|
+
Project-URL: Documentation, https://codspeed.io/docs/reference/pytest-codspeed
|
|
30
30
|
Project-URL: Source, https://github.com/CodSpeedHQ/pytest-codspeed
|
|
31
31
|
Keywords: codspeed,benchmark,performance,pytest
|
|
32
32
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -52,7 +52,7 @@ Requires-Dist: rich>=13.8.1
|
|
|
52
52
|
Requires-Dist: importlib-metadata>=8.5.0; python_version < "3.10"
|
|
53
53
|
Provides-Extra: lint
|
|
54
54
|
Requires-Dist: mypy~=1.11.2; extra == "lint"
|
|
55
|
-
Requires-Dist: ruff~=0.
|
|
55
|
+
Requires-Dist: ruff~=0.11.12; extra == "lint"
|
|
56
56
|
Provides-Extra: compat
|
|
57
57
|
Requires-Dist: pytest-benchmark~=5.0.0; extra == "compat"
|
|
58
58
|
Requires-Dist: pytest-xdist~=3.6.1; extra == "compat"
|
|
@@ -73,11 +73,11 @@ Pytest plugin to create CodSpeed benchmarks
|
|
|
73
73
|
|
|
74
74
|
</div>
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
---
|
|
77
77
|
|
|
78
|
-
**
|
|
78
|
+
**Documentation**: https://codspeed.io/docs/reference/pytest-codspeed
|
|
79
79
|
|
|
80
|
-
|
|
80
|
+
---
|
|
81
81
|
|
|
82
82
|
## Installation
|
|
83
83
|
|
|
@@ -89,9 +89,9 @@ pip install pytest-codspeed
|
|
|
89
89
|
|
|
90
90
|
### Creating benchmarks
|
|
91
91
|
|
|
92
|
-
|
|
92
|
+
In a nutshell, `pytest-codspeed` offers two approaches to create performance benchmarks that integrate seamlessly with your existing test suite.
|
|
93
93
|
|
|
94
|
-
|
|
94
|
+
Use `@pytest.mark.benchmark` to measure entire test functions automatically:
|
|
95
95
|
|
|
96
96
|
```python
|
|
97
97
|
import pytest
|
|
@@ -99,55 +99,53 @@ from statistics import median
|
|
|
99
99
|
|
|
100
100
|
@pytest.mark.benchmark
|
|
101
101
|
def test_median_performance():
|
|
102
|
-
|
|
102
|
+
input = [1, 2, 3, 4, 5]
|
|
103
|
+
output = sum(i**2 for i in input)
|
|
104
|
+
assert output == 55
|
|
103
105
|
```
|
|
104
106
|
|
|
105
|
-
|
|
107
|
+
Since this measure the entire function, you might want to use the `benchmark` fixture for precise control over what code gets measured:
|
|
106
108
|
|
|
107
109
|
```python
|
|
108
|
-
import pytest
|
|
109
|
-
from statistics import mean
|
|
110
|
-
|
|
111
110
|
def test_mean_performance(benchmark):
|
|
112
|
-
# Precompute some data useful for the benchmark but that should not be
|
|
113
|
-
# included in the benchmark time
|
|
114
111
|
data = [1, 2, 3, 4, 5]
|
|
112
|
+
# Only the function call is measured
|
|
113
|
+
result = benchmark(lambda: sum(i**2 for i in data))
|
|
114
|
+
assert result == 55
|
|
115
|
+
```
|
|
115
116
|
|
|
116
|
-
|
|
117
|
-
benchmark(lambda: mean(data))
|
|
118
|
-
|
|
117
|
+
Check out the [full documentation](https://codspeed.io/docs/reference/pytest-codspeed) for more details.
|
|
119
118
|
|
|
120
|
-
|
|
121
|
-
# Precompute some data useful for the benchmark but that should not be
|
|
122
|
-
# included in the benchmark time
|
|
123
|
-
data = [1, 2, 3, 4, 5]
|
|
119
|
+
### Testing the benchmarks locally
|
|
124
120
|
|
|
125
|
-
|
|
126
|
-
# The `@benchmark` decorator will automatically call the function and
|
|
127
|
-
# measure its execution
|
|
128
|
-
@benchmark
|
|
129
|
-
def bench():
|
|
130
|
-
mean(data)
|
|
131
|
-
median(data)
|
|
132
|
-
```
|
|
121
|
+
If you want to run the benchmarks tests locally, you can use the `--codspeed` pytest flag:
|
|
133
122
|
|
|
134
|
-
|
|
123
|
+
```sh
|
|
124
|
+
$ pytest tests/ --codspeed
|
|
125
|
+
============================= test session starts ====================
|
|
126
|
+
platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
|
|
127
|
+
codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
|
|
128
|
+
rootdir: /home/user/codspeed-test, configfile: pytest.ini
|
|
129
|
+
plugins: codspeed-3.0.0
|
|
130
|
+
collected 1 items
|
|
135
131
|
|
|
136
|
-
|
|
132
|
+
tests/test_sum_squares.py . [ 100%]
|
|
137
133
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
134
|
+
Benchmark Results
|
|
135
|
+
┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
|
|
136
|
+
┃ Benchmark ┃ Time (best) ┃ Rel. StdDev ┃ Run time ┃ Iters ┃
|
|
137
|
+
┣━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━╋━━━━━━━━┫
|
|
138
|
+
┃test_sum_squares┃ 1,873ns ┃ 4.8% ┃ 3.00s ┃ 66,930 ┃
|
|
139
|
+
┗━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━┻━━━━━━━━┛
|
|
140
|
+
=============================== 1 benchmarked ========================
|
|
141
|
+
=============================== 1 passed in 4.12s ====================
|
|
142
142
|
```
|
|
143
143
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
#### In your CI
|
|
144
|
+
### Running the benchmarks in your CI
|
|
147
145
|
|
|
148
146
|
You can use the [CodSpeedHQ/action](https://github.com/CodSpeedHQ/action) to run the benchmarks in Github Actions and upload the results to CodSpeed.
|
|
149
147
|
|
|
150
|
-
|
|
148
|
+
Here is an example of a GitHub Actions workflow that runs the benchmarks and reports the results to CodSpeed on every push to the `main` branch and every pull request:
|
|
151
149
|
|
|
152
150
|
```yaml
|
|
153
151
|
name: CodSpeed
|
|
@@ -157,6 +155,9 @@ on:
|
|
|
157
155
|
branches:
|
|
158
156
|
- "main" # or "master"
|
|
159
157
|
pull_request:
|
|
158
|
+
# `workflow_dispatch` allows CodSpeed to trigger backtest
|
|
159
|
+
# performance analysis in order to generate initial data.
|
|
160
|
+
workflow_dispatch:
|
|
160
161
|
|
|
161
162
|
jobs:
|
|
162
163
|
benchmarks:
|
|
@@ -166,9 +167,11 @@ jobs:
|
|
|
166
167
|
- uses: actions/checkout@v4
|
|
167
168
|
- uses: actions/setup-python@v5
|
|
168
169
|
with:
|
|
169
|
-
python-version: "3.
|
|
170
|
+
python-version: "3.13"
|
|
171
|
+
|
|
170
172
|
- name: Install dependencies
|
|
171
173
|
run: pip install -r requirements.txt
|
|
174
|
+
|
|
172
175
|
- name: Run benchmarks
|
|
173
176
|
uses: CodSpeedHQ/action@v3
|
|
174
177
|
with:
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<h1>pytest-codspeed</h1>
|
|
3
|
+
|
|
4
|
+
[](https://github.com/CodSpeedHQ/pytest-codspeed/actions/workflows/ci.yml)
|
|
5
|
+
[](https://pypi.org/project/pytest-codspeed)
|
|
6
|
+

|
|
7
|
+
[](https://discord.com/invite/MxpaCfKSqF)
|
|
8
|
+
[](https://codspeed.io/CodSpeedHQ/pytest-codspeed)
|
|
9
|
+
|
|
10
|
+
Pytest plugin to create CodSpeed benchmarks
|
|
11
|
+
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
**Documentation**: https://codspeed.io/docs/reference/pytest-codspeed
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
```shell
|
|
23
|
+
pip install pytest-codspeed
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
### Creating benchmarks
|
|
29
|
+
|
|
30
|
+
In a nutshell, `pytest-codspeed` offers two approaches to create performance benchmarks that integrate seamlessly with your existing test suite.
|
|
31
|
+
|
|
32
|
+
Use `@pytest.mark.benchmark` to measure entire test functions automatically:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
import pytest
|
|
36
|
+
from statistics import median
|
|
37
|
+
|
|
38
|
+
@pytest.mark.benchmark
|
|
39
|
+
def test_median_performance():
|
|
40
|
+
input = [1, 2, 3, 4, 5]
|
|
41
|
+
output = sum(i**2 for i in input)
|
|
42
|
+
assert output == 55
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Since this measure the entire function, you might want to use the `benchmark` fixture for precise control over what code gets measured:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
def test_mean_performance(benchmark):
|
|
49
|
+
data = [1, 2, 3, 4, 5]
|
|
50
|
+
# Only the function call is measured
|
|
51
|
+
result = benchmark(lambda: sum(i**2 for i in data))
|
|
52
|
+
assert result == 55
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Check out the [full documentation](https://codspeed.io/docs/reference/pytest-codspeed) for more details.
|
|
56
|
+
|
|
57
|
+
### Testing the benchmarks locally
|
|
58
|
+
|
|
59
|
+
If you want to run the benchmarks tests locally, you can use the `--codspeed` pytest flag:
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
$ pytest tests/ --codspeed
|
|
63
|
+
============================= test session starts ====================
|
|
64
|
+
platform darwin -- Python 3.13.0, pytest-7.4.4, pluggy-1.5.0
|
|
65
|
+
codspeed: 3.0.0 (enabled, mode: walltime, timer_resolution: 41.7ns)
|
|
66
|
+
rootdir: /home/user/codspeed-test, configfile: pytest.ini
|
|
67
|
+
plugins: codspeed-3.0.0
|
|
68
|
+
collected 1 items
|
|
69
|
+
|
|
70
|
+
tests/test_sum_squares.py . [ 100%]
|
|
71
|
+
|
|
72
|
+
Benchmark Results
|
|
73
|
+
┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
|
|
74
|
+
┃ Benchmark ┃ Time (best) ┃ Rel. StdDev ┃ Run time ┃ Iters ┃
|
|
75
|
+
┣━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━╋━━━━━━━━┫
|
|
76
|
+
┃test_sum_squares┃ 1,873ns ┃ 4.8% ┃ 3.00s ┃ 66,930 ┃
|
|
77
|
+
┗━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━┻━━━━━━━━┛
|
|
78
|
+
=============================== 1 benchmarked ========================
|
|
79
|
+
=============================== 1 passed in 4.12s ====================
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Running the benchmarks in your CI
|
|
83
|
+
|
|
84
|
+
You can use the [CodSpeedHQ/action](https://github.com/CodSpeedHQ/action) to run the benchmarks in Github Actions and upload the results to CodSpeed.
|
|
85
|
+
|
|
86
|
+
Here is an example of a GitHub Actions workflow that runs the benchmarks and reports the results to CodSpeed on every push to the `main` branch and every pull request:
|
|
87
|
+
|
|
88
|
+
```yaml
|
|
89
|
+
name: CodSpeed
|
|
90
|
+
|
|
91
|
+
on:
|
|
92
|
+
push:
|
|
93
|
+
branches:
|
|
94
|
+
- "main" # or "master"
|
|
95
|
+
pull_request:
|
|
96
|
+
# `workflow_dispatch` allows CodSpeed to trigger backtest
|
|
97
|
+
# performance analysis in order to generate initial data.
|
|
98
|
+
workflow_dispatch:
|
|
99
|
+
|
|
100
|
+
jobs:
|
|
101
|
+
benchmarks:
|
|
102
|
+
name: Run benchmarks
|
|
103
|
+
runs-on: ubuntu-latest
|
|
104
|
+
steps:
|
|
105
|
+
- uses: actions/checkout@v4
|
|
106
|
+
- uses: actions/setup-python@v5
|
|
107
|
+
with:
|
|
108
|
+
python-version: "3.13"
|
|
109
|
+
|
|
110
|
+
- name: Install dependencies
|
|
111
|
+
run: pip install -r requirements.txt
|
|
112
|
+
|
|
113
|
+
- name: Run benchmarks
|
|
114
|
+
uses: CodSpeedHQ/action@v3
|
|
115
|
+
with:
|
|
116
|
+
token: ${{ secrets.CODSPEED_TOKEN }}
|
|
117
|
+
run: pytest tests/ --codspeed
|
|
118
|
+
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project.urls]
|
|
2
2
|
Homepage = "https://codspeed.io/"
|
|
3
|
-
Documentation = "https://
|
|
3
|
+
Documentation = "https://codspeed.io/docs/reference/pytest-codspeed"
|
|
4
4
|
Source = "https://github.com/CodSpeedHQ/pytest-codspeed"
|
|
5
5
|
|
|
6
6
|
[project]
|
|
@@ -37,7 +37,7 @@ dependencies = [
|
|
|
37
37
|
]
|
|
38
38
|
|
|
39
39
|
[project.optional-dependencies]
|
|
40
|
-
lint = ["mypy ~= 1.11.2", "ruff ~= 0.
|
|
40
|
+
lint = ["mypy ~= 1.11.2", "ruff ~= 0.11.12"]
|
|
41
41
|
compat = [
|
|
42
42
|
"pytest-benchmark ~= 5.0.0",
|
|
43
43
|
"pytest-xdist ~= 3.6.1",
|
|
@@ -66,7 +66,7 @@ version = { attr = "pytest_codspeed.__version__" }
|
|
|
66
66
|
|
|
67
67
|
|
|
68
68
|
[tool.bumpver]
|
|
69
|
-
current_version = "
|
|
69
|
+
current_version = "4.0.0"
|
|
70
70
|
version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]"
|
|
71
71
|
commit_message = "Release v{new_version} 🚀"
|
|
72
72
|
tag_message = "Release v{new_version} 🚀"
|
|
@@ -5,9 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from setuptools import setup
|
|
7
7
|
|
|
8
|
-
build_path = (
|
|
9
|
-
Path(__file__).parent / "src/pytest_codspeed/instruments/valgrind/_wrapper/build.py"
|
|
10
|
-
)
|
|
8
|
+
build_path = Path(__file__).parent / "src/pytest_codspeed/instruments/hooks/build.py"
|
|
11
9
|
|
|
12
10
|
spec = importlib.util.spec_from_file_location("build", build_path)
|
|
13
11
|
assert spec is not None, "The spec should be initialized"
|
|
@@ -52,8 +50,8 @@ print(
|
|
|
52
50
|
setup(
|
|
53
51
|
package_data={
|
|
54
52
|
"pytest_codspeed": [
|
|
55
|
-
"instruments/
|
|
56
|
-
"instruments/
|
|
53
|
+
"instruments/hooks/instrument-hooks/includes/*.h",
|
|
54
|
+
"instruments/hooks/instrument-hooks/dist/*.c",
|
|
57
55
|
]
|
|
58
56
|
},
|
|
59
57
|
ext_modules=(
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import TYPE_CHECKING, Generic, TypeVar
|
|
6
|
+
|
|
7
|
+
T = TypeVar("T")
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from typing import Any, Callable
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class CodSpeedConfig:
|
|
17
|
+
"""
|
|
18
|
+
The configuration for the codspeed plugin.
|
|
19
|
+
Usually created from the command line arguments.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
warmup_time_ns: int | None = None
|
|
23
|
+
max_time_ns: int | None = None
|
|
24
|
+
max_rounds: int | None = None
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def from_pytest_config(cls, config: pytest.Config) -> CodSpeedConfig:
|
|
28
|
+
warmup_time = config.getoption("--codspeed-warmup-time", None)
|
|
29
|
+
warmup_time_ns = (
|
|
30
|
+
int(warmup_time * 1_000_000_000) if warmup_time is not None else None
|
|
31
|
+
)
|
|
32
|
+
max_time = config.getoption("--codspeed-max-time", None)
|
|
33
|
+
max_time_ns = int(max_time * 1_000_000_000) if max_time is not None else None
|
|
34
|
+
return cls(
|
|
35
|
+
warmup_time_ns=warmup_time_ns,
|
|
36
|
+
max_rounds=config.getoption("--codspeed-max-rounds", None),
|
|
37
|
+
max_time_ns=max_time_ns,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class BenchmarkMarkerOptions:
|
|
43
|
+
group: str | None = None
|
|
44
|
+
"""The group name to use for the benchmark."""
|
|
45
|
+
min_time: int | None = None
|
|
46
|
+
"""
|
|
47
|
+
The minimum time of a round (in seconds).
|
|
48
|
+
Only available in walltime mode.
|
|
49
|
+
"""
|
|
50
|
+
max_time: int | None = None
|
|
51
|
+
"""
|
|
52
|
+
The maximum time to run the benchmark for (in seconds).
|
|
53
|
+
Only available in walltime mode.
|
|
54
|
+
"""
|
|
55
|
+
max_rounds: int | None = None
|
|
56
|
+
"""
|
|
57
|
+
The maximum number of rounds to run the benchmark for.
|
|
58
|
+
Takes precedence over max_time. Only available in walltime mode.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def from_pytest_item(cls, item: pytest.Item) -> BenchmarkMarkerOptions:
|
|
63
|
+
marker = item.get_closest_marker(
|
|
64
|
+
"codspeed_benchmark"
|
|
65
|
+
) or item.get_closest_marker("benchmark")
|
|
66
|
+
if marker is None:
|
|
67
|
+
return cls()
|
|
68
|
+
if len(marker.args) > 0:
|
|
69
|
+
raise ValueError(
|
|
70
|
+
"Positional arguments are not allowed in the benchmark marker"
|
|
71
|
+
)
|
|
72
|
+
kwargs = marker.kwargs
|
|
73
|
+
|
|
74
|
+
unknown_kwargs = set(kwargs.keys()) - {
|
|
75
|
+
field.name for field in dataclasses.fields(cls)
|
|
76
|
+
}
|
|
77
|
+
if unknown_kwargs:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
"Unknown kwargs passed to benchmark marker: "
|
|
80
|
+
+ ", ".join(sorted(unknown_kwargs))
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
return cls(**kwargs)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class PedanticOptions(Generic[T]):
|
|
88
|
+
"""Parameters for running a benchmark using the pedantic fixture API."""
|
|
89
|
+
|
|
90
|
+
target: Callable[..., T]
|
|
91
|
+
setup: Callable[[], Any | None] | None
|
|
92
|
+
teardown: Callable[..., Any | None] | None
|
|
93
|
+
rounds: int
|
|
94
|
+
warmup_rounds: int
|
|
95
|
+
iterations: int
|
|
96
|
+
args: tuple[Any, ...] = field(default_factory=tuple)
|
|
97
|
+
kwargs: dict[str, Any] = field(default_factory=dict)
|
|
98
|
+
|
|
99
|
+
def __post_init__(self) -> None:
|
|
100
|
+
if self.rounds < 0:
|
|
101
|
+
raise ValueError("rounds must be positive")
|
|
102
|
+
if self.warmup_rounds < 0:
|
|
103
|
+
raise ValueError("warmup_rounds must be non-negative")
|
|
104
|
+
if self.iterations <= 0:
|
|
105
|
+
raise ValueError("iterations must be positive")
|
|
106
|
+
if self.iterations > 1 and self.setup is not None:
|
|
107
|
+
raise ValueError(
|
|
108
|
+
"setup cannot be used with multiple iterations, use multiple rounds"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
def setup_and_get_args_kwargs(self) -> tuple[tuple[Any, ...], dict[str, Any]]:
|
|
112
|
+
if self.setup is None:
|
|
113
|
+
return self.args, self.kwargs
|
|
114
|
+
maybe_result = self.setup(*self.args, **self.kwargs)
|
|
115
|
+
if maybe_result is not None:
|
|
116
|
+
if len(self.args) > 0 or len(self.kwargs) > 0:
|
|
117
|
+
raise ValueError("setup cannot return a value when args are provided")
|
|
118
|
+
return maybe_result
|
|
119
|
+
return self.args, self.kwargs
|
|
@@ -5,14 +5,14 @@ from enum import Enum
|
|
|
5
5
|
from typing import TYPE_CHECKING
|
|
6
6
|
|
|
7
7
|
if TYPE_CHECKING:
|
|
8
|
-
from typing import Any, Callable, ClassVar,
|
|
8
|
+
from typing import Any, Callable, ClassVar, TypeVar
|
|
9
9
|
|
|
10
10
|
import pytest
|
|
11
11
|
|
|
12
|
+
from pytest_codspeed.config import BenchmarkMarkerOptions, PedanticOptions
|
|
12
13
|
from pytest_codspeed.plugin import CodSpeedConfig
|
|
13
14
|
|
|
14
15
|
T = TypeVar("T")
|
|
15
|
-
P = ParamSpec("P")
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class Instrument(metaclass=ABCMeta):
|
|
@@ -27,11 +27,21 @@ class Instrument(metaclass=ABCMeta):
|
|
|
27
27
|
@abstractmethod
|
|
28
28
|
def measure(
|
|
29
29
|
self,
|
|
30
|
+
marker_options: BenchmarkMarkerOptions,
|
|
31
|
+
name: str,
|
|
32
|
+
uri: str,
|
|
33
|
+
fn: Callable[..., T],
|
|
34
|
+
*args: tuple,
|
|
35
|
+
**kwargs: dict[str, Any],
|
|
36
|
+
) -> T: ...
|
|
37
|
+
|
|
38
|
+
@abstractmethod
|
|
39
|
+
def measure_pedantic(
|
|
40
|
+
self,
|
|
41
|
+
marker_options: BenchmarkMarkerOptions,
|
|
42
|
+
pedantic_options: PedanticOptions[T],
|
|
30
43
|
name: str,
|
|
31
44
|
uri: str,
|
|
32
|
-
fn: Callable[P, T],
|
|
33
|
-
*args: P.args,
|
|
34
|
-
**kwargs: P.kwargs,
|
|
35
45
|
) -> T: ...
|
|
36
46
|
|
|
37
47
|
@abstractmethod
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import warnings
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from .dist_instrument_hooks import InstrumentHooksPointer, LibType
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class InstrumentHooks:
|
|
15
|
+
"""Zig library wrapper class providing benchmark measurement functionality."""
|
|
16
|
+
|
|
17
|
+
lib: LibType
|
|
18
|
+
instance: InstrumentHooksPointer
|
|
19
|
+
|
|
20
|
+
def __init__(self) -> None:
|
|
21
|
+
if os.environ.get("CODSPEED_ENV") is None:
|
|
22
|
+
raise RuntimeError(
|
|
23
|
+
"Can't run benchmarks outside of CodSpeed environment."
|
|
24
|
+
"Please set the CODSPEED_ENV environment variable."
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
from .dist_instrument_hooks import lib # type: ignore
|
|
29
|
+
except ImportError as e:
|
|
30
|
+
raise RuntimeError(f"Failed to load instrument hooks library: {e}") from e
|
|
31
|
+
self.lib = lib
|
|
32
|
+
|
|
33
|
+
self.instance = self.lib.instrument_hooks_init()
|
|
34
|
+
if self.instance == 0:
|
|
35
|
+
raise RuntimeError("Failed to initialize CodSpeed instrumentation library.")
|
|
36
|
+
|
|
37
|
+
if SUPPORTS_PERF_TRAMPOLINE:
|
|
38
|
+
sys.activate_stack_trampoline("perf") # type: ignore
|
|
39
|
+
|
|
40
|
+
def __del__(self):
|
|
41
|
+
if hasattr(self, "lib") and hasattr(self, "instance"):
|
|
42
|
+
self.lib.instrument_hooks_deinit(self.instance)
|
|
43
|
+
|
|
44
|
+
def start_benchmark(self) -> None:
|
|
45
|
+
"""Start a new benchmark measurement."""
|
|
46
|
+
ret = self.lib.instrument_hooks_start_benchmark(self.instance)
|
|
47
|
+
if ret != 0:
|
|
48
|
+
warnings.warn("Failed to start benchmark measurement", RuntimeWarning)
|
|
49
|
+
|
|
50
|
+
def stop_benchmark(self) -> None:
|
|
51
|
+
"""Stop the current benchmark measurement."""
|
|
52
|
+
ret = self.lib.instrument_hooks_stop_benchmark(self.instance)
|
|
53
|
+
if ret != 0:
|
|
54
|
+
warnings.warn("Failed to stop benchmark measurement", RuntimeWarning)
|
|
55
|
+
|
|
56
|
+
def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None:
|
|
57
|
+
"""Set the executed benchmark URI and process ID.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
uri: The benchmark URI string identifier
|
|
61
|
+
pid: Optional process ID (defaults to current process)
|
|
62
|
+
"""
|
|
63
|
+
if pid is None:
|
|
64
|
+
pid = os.getpid()
|
|
65
|
+
|
|
66
|
+
ret = self.lib.instrument_hooks_executed_benchmark(
|
|
67
|
+
self.instance, pid, uri.encode("ascii")
|
|
68
|
+
)
|
|
69
|
+
if ret != 0:
|
|
70
|
+
warnings.warn("Failed to set executed benchmark", RuntimeWarning)
|
|
71
|
+
|
|
72
|
+
def set_integration(self, name: str, version: str) -> None:
|
|
73
|
+
"""Set the integration name and version."""
|
|
74
|
+
ret = self.lib.instrument_hooks_set_integration(
|
|
75
|
+
self.instance, name.encode("ascii"), version.encode("ascii")
|
|
76
|
+
)
|
|
77
|
+
if ret != 0:
|
|
78
|
+
warnings.warn("Failed to set integration name and version", RuntimeWarning)
|
|
79
|
+
|
|
80
|
+
def is_instrumented(self) -> bool:
|
|
81
|
+
"""Check if instrumentation is active."""
|
|
82
|
+
return self.lib.instrument_hooks_is_instrumented(self.instance)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from cffi import FFI # type: ignore
|
|
4
|
+
|
|
5
|
+
ffibuilder = FFI()
|
|
6
|
+
|
|
7
|
+
includes_dir = Path(__file__).parent.joinpath("instrument-hooks/includes")
|
|
8
|
+
header_text = (includes_dir / "core.h").read_text()
|
|
9
|
+
filtered_header = "\n".join(
|
|
10
|
+
line for line in header_text.splitlines() if not line.strip().startswith("#")
|
|
11
|
+
)
|
|
12
|
+
ffibuilder.cdef(filtered_header)
|
|
13
|
+
|
|
14
|
+
ffibuilder.set_source(
|
|
15
|
+
"pytest_codspeed.instruments.hooks.dist_instrument_hooks",
|
|
16
|
+
"""
|
|
17
|
+
#include "core.h"
|
|
18
|
+
""",
|
|
19
|
+
sources=[
|
|
20
|
+
"src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c",
|
|
21
|
+
],
|
|
22
|
+
include_dirs=[str(includes_dir)],
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
if __name__ == "__main__":
|
|
26
|
+
ffibuilder.compile(verbose=True)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
InstrumentHooksPointer = object
|
|
2
|
+
|
|
3
|
+
class lib:
|
|
4
|
+
@staticmethod
|
|
5
|
+
def instrument_hooks_init() -> InstrumentHooksPointer: ...
|
|
6
|
+
@staticmethod
|
|
7
|
+
def instrument_hooks_deinit(hooks: InstrumentHooksPointer) -> None: ...
|
|
8
|
+
@staticmethod
|
|
9
|
+
def instrument_hooks_is_instrumented(hooks: InstrumentHooksPointer) -> bool: ...
|
|
10
|
+
@staticmethod
|
|
11
|
+
def instrument_hooks_start_benchmark(hooks: InstrumentHooksPointer) -> int: ...
|
|
12
|
+
@staticmethod
|
|
13
|
+
def instrument_hooks_stop_benchmark(hooks: InstrumentHooksPointer) -> int: ...
|
|
14
|
+
@staticmethod
|
|
15
|
+
def instrument_hooks_executed_benchmark(
|
|
16
|
+
hooks: InstrumentHooksPointer, pid: int, uri: bytes
|
|
17
|
+
) -> int: ...
|
|
18
|
+
@staticmethod
|
|
19
|
+
def instrument_hooks_set_integration(
|
|
20
|
+
hooks: InstrumentHooksPointer, name: bytes, version: bytes
|
|
21
|
+
) -> int: ...
|
|
22
|
+
@staticmethod
|
|
23
|
+
def callgrind_start_instrumentation() -> int: ...
|
|
24
|
+
@staticmethod
|
|
25
|
+
def callgrind_stop_instrumentation() -> int: ...
|
|
26
|
+
|
|
27
|
+
LibType = type[lib]
|