pytest-benchmem 0.4.2__tar.gz → 0.4.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/dependabot.yml +3 -3
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/ci.yaml +7 -7
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/docs.yaml +2 -2
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/nightly.yaml +1 -1
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/release.yaml +1 -1
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.gitignore +0 -3
- pytest_benchmem-0.4.4/.release-please-manifest.json +3 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/CHANGELOG.md +14 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/PKG-INFO +1 -1
- pytest_benchmem-0.4.4/docs/changelog.md +9 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/compare-plot.md +68 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/mkdocs.yml +5 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/pyproject.toml +2 -1
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/cli.py +72 -12
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/compare.py +80 -24
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/plotting.py +34 -12
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/snapshot.py +87 -3
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_cli.py +84 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_compare.py +140 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_plotting.py +89 -0
- pytest_benchmem-0.4.4/uv.lock +2441 -0
- pytest_benchmem-0.4.2/.release-please-manifest.json +0 -3
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/dependabot-auto-merge.yaml +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/pr-title.yaml +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.pre-commit-config.yaml +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.readthedocs.yaml +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.release-please-config.json +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/LICENSE +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/README.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/dims.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/getting-started.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/index.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/javascripts/notebook-prompts.js +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/metrics.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/reference.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/stylesheets/extra.css +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/sweeps.md +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs_macros.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/__init__.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/combined.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/format.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/memray.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/py.typed +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/pytest_plugin.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/sweep.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/tables.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/__init__.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/conftest.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_combined.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_distribution.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_inline_compare.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_memray.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_memray_platform.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_plugin.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_snapshot.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_sweep.py +0 -0
- {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_tables.py +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
version: 2
|
|
2
2
|
|
|
3
3
|
updates:
|
|
4
|
-
- package-ecosystem: "
|
|
4
|
+
- package-ecosystem: "uv"
|
|
5
5
|
directory: "/"
|
|
6
6
|
schedule:
|
|
7
|
-
interval: "
|
|
7
|
+
interval: "monthly"
|
|
8
8
|
labels:
|
|
9
9
|
- "dependencies"
|
|
10
10
|
commit-message:
|
|
@@ -27,7 +27,7 @@ updates:
|
|
|
27
27
|
- package-ecosystem: "github-actions"
|
|
28
28
|
directory: "/"
|
|
29
29
|
schedule:
|
|
30
|
-
interval: "
|
|
30
|
+
interval: "monthly"
|
|
31
31
|
labels:
|
|
32
32
|
- "dependencies"
|
|
33
33
|
commit-message:
|
|
@@ -16,11 +16,11 @@ jobs:
|
|
|
16
16
|
runs-on: ubuntu-24.04
|
|
17
17
|
timeout-minutes: 5
|
|
18
18
|
steps:
|
|
19
|
-
- uses: actions/checkout@
|
|
19
|
+
- uses: actions/checkout@v7
|
|
20
20
|
- uses: astral-sh/setup-uv@v7
|
|
21
21
|
with:
|
|
22
22
|
enable-cache: true
|
|
23
|
-
- run: uv sync --group dev
|
|
23
|
+
- run: uv sync --group dev --frozen
|
|
24
24
|
- run: uv run ruff check --output-format=github
|
|
25
25
|
- run: uv run ruff format --diff
|
|
26
26
|
|
|
@@ -29,11 +29,11 @@ jobs:
|
|
|
29
29
|
runs-on: ubuntu-24.04
|
|
30
30
|
timeout-minutes: 5
|
|
31
31
|
steps:
|
|
32
|
-
- uses: actions/checkout@
|
|
32
|
+
- uses: actions/checkout@v7
|
|
33
33
|
- uses: astral-sh/setup-uv@v7
|
|
34
34
|
with:
|
|
35
35
|
enable-cache: true
|
|
36
|
-
- run: uv sync --group dev
|
|
36
|
+
- run: uv sync --group dev --frozen
|
|
37
37
|
- run: uv run mypy src/pytest_benchmem tests
|
|
38
38
|
|
|
39
39
|
test:
|
|
@@ -49,14 +49,14 @@ jobs:
|
|
|
49
49
|
- os: macos-latest
|
|
50
50
|
python-version: "3.13"
|
|
51
51
|
steps:
|
|
52
|
-
- uses: actions/checkout@
|
|
52
|
+
- uses: actions/checkout@v7
|
|
53
53
|
- uses: astral-sh/setup-uv@v7
|
|
54
54
|
with:
|
|
55
55
|
enable-cache: true
|
|
56
56
|
- uses: actions/setup-python@v6
|
|
57
57
|
with:
|
|
58
58
|
python-version: ${{ matrix.python-version }}
|
|
59
|
-
- run: uv sync --group dev
|
|
59
|
+
- run: uv sync --group dev --frozen
|
|
60
60
|
- run: uv run pytest -q
|
|
61
61
|
|
|
62
62
|
smoke-install:
|
|
@@ -64,7 +64,7 @@ jobs:
|
|
|
64
64
|
runs-on: ubuntu-24.04
|
|
65
65
|
timeout-minutes: 10
|
|
66
66
|
steps:
|
|
67
|
-
- uses: actions/checkout@
|
|
67
|
+
- uses: actions/checkout@v7
|
|
68
68
|
- uses: astral-sh/setup-uv@v7
|
|
69
69
|
with:
|
|
70
70
|
enable-cache: true
|
|
@@ -18,11 +18,11 @@ jobs:
|
|
|
18
18
|
runs-on: ubuntu-24.04
|
|
19
19
|
timeout-minutes: 10
|
|
20
20
|
steps:
|
|
21
|
-
- uses: actions/checkout@
|
|
21
|
+
- uses: actions/checkout@v7
|
|
22
22
|
- uses: astral-sh/setup-uv@v7
|
|
23
23
|
with:
|
|
24
24
|
enable-cache: true
|
|
25
25
|
# Build the notebooks from their jupytext sources, then build+execute the site.
|
|
26
|
-
- run: uv sync --group docs
|
|
26
|
+
- run: uv sync --group docs --frozen
|
|
27
27
|
- run: uv run jupytext --to ipynb docs/compare-plot.md
|
|
28
28
|
- run: uv run mkdocs gh-deploy --force
|
|
@@ -49,7 +49,7 @@ jobs:
|
|
|
49
49
|
name: pypi
|
|
50
50
|
url: https://pypi.org/project/pytest-benchmem
|
|
51
51
|
steps:
|
|
52
|
-
- uses: actions/checkout@
|
|
52
|
+
- uses: actions/checkout@v7
|
|
53
53
|
with:
|
|
54
54
|
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || needs.release-please.outputs.tag_name }}
|
|
55
55
|
fetch-depth: 0
|
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.4.4](https://github.com/fluxopt/pytest-benchmem/compare/v0.4.3...v0.4.4) (2026-06-29)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* `--pivot` comparison axis for compare/plot (fold one run along a config dim) ([#137](https://github.com/fluxopt/pytest-benchmem/issues/137)) ([d0672f7](https://github.com/fluxopt/pytest-benchmem/commit/d0672f74d7040b4f777df342c346052fc9f77678))
|
|
9
|
+
|
|
10
|
+
## [0.4.3](https://github.com/fluxopt/pytest-benchmem/compare/v0.4.2...v0.4.3) (2026-06-29)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Features
|
|
14
|
+
|
|
15
|
+
* benchmem compare accepts a single run (plain table) ([#123](https://github.com/fluxopt/pytest-benchmem/issues/123)) ([96275d9](https://github.com/fluxopt/pytest-benchmem/commit/96275d93628323ffd4f3cc74decdebdcb132004e))
|
|
16
|
+
|
|
3
17
|
## [0.4.2](https://github.com/fluxopt/pytest-benchmem/compare/v0.4.1...v0.4.2) (2026-06-18)
|
|
4
18
|
|
|
5
19
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pytest-benchmem
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.4
|
|
4
4
|
Summary: The memory companion to pytest-benchmark: a memray peak-memory pass on the same test, plus dims-aware plots and cross-version sweeps.
|
|
5
5
|
Project-URL: Homepage, https://github.com/fluxopt/pytest-benchmem
|
|
6
6
|
Project-URL: Documentation, https://fluxopt.github.io/pytest-benchmem/
|
|
@@ -99,6 +99,72 @@ Pick the metrics with `--columns` (a comma list of `time` / `peak` / `allocated`
|
|
|
99
99
|
`module` | `class` | `param:NAME`, comma-composable); pass it to cluster param-variants
|
|
100
100
|
(`--group-by func`) or collapse everything into one table.
|
|
101
101
|
|
|
102
|
+
## One run, two configs — `--pivot`
|
|
103
|
+
|
|
104
|
+
Everything above compares **run-files**: `compare a.json b.json` lays one file against
|
|
105
|
+
another. But the file is just the *series axis* — the thing whose values sit side by side and
|
|
106
|
+
get the `(N.NN)` multiplier. **The series axis is a [dim](dims.md)**, and the run-file is only
|
|
107
|
+
the default one.
|
|
108
|
+
|
|
109
|
+
`--pivot DIM` re-points it at a data dim, so a **single combined run** is A/B'd along that dim —
|
|
110
|
+
the role a file-pair plays today, without splitting into N runs. Say `semantics` is a param
|
|
111
|
+
(so it's in the node id, `test_build[legacy-100]`, *and* a dim):
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
@pytest.mark.parametrize("semantics", ["legacy", "v1"])
|
|
115
|
+
@pytest.mark.parametrize("n", [100, 200])
|
|
116
|
+
def test_build(benchmark, semantics, n):
|
|
117
|
+
benchmark(build, semantics=semantics, n=n)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
One `pytest` invocation produces one `build.json`; `--pivot param:semantics` folds it so rows
|
|
121
|
+
that differ *only* in `semantics` pair up (`legacy` ↔ `v1`) per `n`:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
pytest benchmarks/ --benchmark-only --benchmark-memory --benchmark-json=build.json
|
|
125
|
+
benchmem compare build.json --pivot param:semantics --columns time,peak # the A/B table
|
|
126
|
+
benchmem plot build.json --pivot param:semantics --columns peak --view compare # the A/B bars
|
|
127
|
+
benchmem plot build.json --x n --facet semantics --columns peak # --pivot optional here
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Don't confuse it with `--group-by`, which touches the *other* axis: `--group-by` **partitions
|
|
131
|
+
rows** into sub-tables along a dim while the compared series stays the run-files (`legacy` and
|
|
132
|
+
`v1` would sit in separate sections, never set against each other); `--pivot` makes the dim
|
|
133
|
+
*itself* the compared series, folding rows that differ only in it into one ranked A/B row. They
|
|
134
|
+
compose on orthogonal axes — `--pivot param:semantics --group-by node.func` folds `legacy` ↔ `v1`
|
|
135
|
+
*and* clusters the folded rows into per-function sub-tables. In short: `--group-by` says which
|
|
136
|
+
rows belong together; `--pivot` says what you're comparing.
|
|
137
|
+
|
|
138
|
+
`--pivot` accepts the same dims as `--group-by`/`--facet` — `param:NAME` or a bare `extra_info`
|
|
139
|
+
name — and composes with `--columns`, `--stat`, `--facet`, `--where`, and `--sort` unchanged.
|
|
140
|
+
It's mutually exclusive with multiple runs (an A/B view has *one* series axis: comparing files
|
|
141
|
+
*and* folding a dim would be a 2-D matrix), and only the paired views consume it — on `plot`
|
|
142
|
+
it applies to `--view compare`/`scatter`; for `scaling`/`sweep` the dim is a normal `--x` /
|
|
143
|
+
`--facet` axis instead. One combined run then drives the A/B table, the A/B plot, the scaling
|
|
144
|
+
plot, *and* an external per-id gate (e.g. CodSpeed, which wants the config value in the node id
|
|
145
|
+
of one run) — no file-splitting, no per-config reruns.
|
|
146
|
+
|
|
147
|
+
`--fail-on` follows the same axis: it normally gates the first run vs the last, but with
|
|
148
|
+
`--pivot` it gates the **first dim value vs the last** (e.g. `legacy` → `v1`) out of the one
|
|
149
|
+
run — so the combined run also gates itself in CI, no second file required:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
# fail if v1's peak grew >10% over legacy, for any benchmark, from one run:
|
|
153
|
+
pytest benchmarks/ --benchmark-only --benchmark-memory --benchmark-json=build.json
|
|
154
|
+
benchmem compare build.json --pivot param:semantics --fail-on peak:10%
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The **base** (the `--fail-on` reference, and the first row) is the dim's *first* value in
|
|
158
|
+
parametrize / collection order — the `--pivot` analogue of file argument order. So
|
|
159
|
+
`parametrize("semantics", ["legacy", "v1"])` gates `v1` against `legacy`; reorder the list to
|
|
160
|
+
flip it. (The table's `(1.0)` annotation is separate — it marks each column's *best* value,
|
|
161
|
+
green, exactly as the run-file table does, not the base.)
|
|
162
|
+
|
|
163
|
+
> A `--pivot` param is lifted out of the id when rows fold (`test_build[legacy-100]` → `test_build[100]`),
|
|
164
|
+
> matched by value. A param given a custom `pytest.param(id=…)` whose label differs from its
|
|
165
|
+
> value won't fold — use a plain parametrize value, or an `extra_info` dim (which isn't in the
|
|
166
|
+
> id at all), for the comparison axis.
|
|
167
|
+
|
|
102
168
|
## Gate CI on a regression
|
|
103
169
|
|
|
104
170
|
`--fail-on` exits non-zero past a threshold — drop it into CI after the run. Thresholds are
|
|
@@ -289,6 +355,8 @@ A minimal GitHub Actions job using the two-file approach, caching the baseline a
|
|
|
289
355
|
| 2 | `compare` (`--view compare`) | ranked — what moved *most*, in native units? |
|
|
290
356
|
| 3+ | `sweep` | fold-change across versions, one cell per (id, run) |
|
|
291
357
|
|
|
358
|
+
`--pivot` re-points the series axis of the paired views (`compare`/`scatter`) from the run-file
|
|
359
|
+
onto a dim, folding a single run along it (see [One run, two configs](#one-run-two-configs-pivot)).
|
|
292
360
|
`--facet` splits any view into small multiples by a [dim](dims.md) (including `node.*`),
|
|
293
361
|
`--where` keeps only rows matching a `dim=value` filter (repeatable, AND-combined),
|
|
294
362
|
`--free-axes` unmatches a faceted axis from the shared default, and `--label`/`-l` names
|
|
@@ -39,6 +39,7 @@ nav:
|
|
|
39
39
|
- Grouping by dims: dims.md
|
|
40
40
|
- Cross-version sweeps: sweeps.md
|
|
41
41
|
- Reference: reference.md
|
|
42
|
+
- Release notes: changelog.md
|
|
42
43
|
|
|
43
44
|
plugins:
|
|
44
45
|
- search
|
|
@@ -93,6 +94,10 @@ markdown_extensions:
|
|
|
93
94
|
anchor_linenums: true
|
|
94
95
|
use_pygments: true
|
|
95
96
|
- pymdownx.inlinehilite
|
|
97
|
+
- pymdownx.snippets:
|
|
98
|
+
# Pull docs/changelog.md's `--8<-- "CHANGELOG.md"` from the repo root.
|
|
99
|
+
base_path: ["."]
|
|
100
|
+
check_paths: true
|
|
96
101
|
- pymdownx.superfences
|
|
97
102
|
- pymdownx.details
|
|
98
103
|
- pymdownx.tabbed:
|
|
@@ -48,7 +48,8 @@ build-backend = "hatchling.build"
|
|
|
48
48
|
source = "vcs"
|
|
49
49
|
|
|
50
50
|
[dependency-groups]
|
|
51
|
-
|
|
51
|
+
# mypy <2.1: 2.1.0 errors on numpy's PEP 695 stub under python_version=3.11. Lift once fixed upstream.
|
|
52
|
+
dev = ["ruff", "mypy<2.2", "pre-commit", "pytest-benchmem[plot]", { include-group = "docs" }]
|
|
52
53
|
docs = [
|
|
53
54
|
"jupytext",
|
|
54
55
|
"ipykernel",
|
|
@@ -100,6 +100,16 @@ def plot(
|
|
|
100
100
|
typer.Option(help="compare | scatter | sweep | scaling (default: by count)."),
|
|
101
101
|
] = None,
|
|
102
102
|
facet: Annotated[str | None, typer.Option(help="Dim to facet by.")] = None,
|
|
103
|
+
pivot: Annotated[
|
|
104
|
+
str | None,
|
|
105
|
+
typer.Option(
|
|
106
|
+
"--pivot",
|
|
107
|
+
help="Comparison axis for --view compare/scatter: fold a single run along this dim "
|
|
108
|
+
"instead of across run-files (param:NAME or a bare extra_info name); its values "
|
|
109
|
+
"become the compared series. Like --group-by but it sets what's *compared*, not how "
|
|
110
|
+
"rows cluster. Mutually exclusive with multiple runs.",
|
|
111
|
+
),
|
|
112
|
+
] = None,
|
|
103
113
|
x: Annotated[str | None, typer.Option(help="scaling: dim for the x-axis.")] = None,
|
|
104
114
|
clip: Annotated[float | None, typer.Option(help="Clamp the colour scale.")] = None,
|
|
105
115
|
where: Annotated[
|
|
@@ -129,7 +139,21 @@ def plot(
|
|
|
129
139
|
"""Render an interactive plotly view from one or more pytest-benchmark runs."""
|
|
130
140
|
_require_runs_exist(runs, suggest=True)
|
|
131
141
|
|
|
132
|
-
|
|
142
|
+
# --pivot's natural home is the A/B compare view, so when --view is unset it defaults there
|
|
143
|
+
# (otherwise one run would default to scaling, which has no series axis to pivot). An explicit
|
|
144
|
+
# --view still wins in both cases; the guard below rejects a non-paired view under --pivot.
|
|
145
|
+
if pivot is not None:
|
|
146
|
+
chosen = view or "compare"
|
|
147
|
+
else:
|
|
148
|
+
chosen = view or ("scaling" if len(runs) == 1 else "scatter" if len(runs) == 2 else "sweep")
|
|
149
|
+
# --pivot re-points the series axis at a dim; only the paired views (compare/scatter) have a
|
|
150
|
+
# series axis to re-point. For scaling/sweep the dim is a normal x/colour/facet axis.
|
|
151
|
+
if pivot is not None and chosen not in ("compare", "scatter"):
|
|
152
|
+
raise _fail(
|
|
153
|
+
f"--pivot re-points the comparison series and applies to --view compare or scatter, "
|
|
154
|
+
f"not {chosen!r}; for scaling/sweep use --x / --facet to place the dim.",
|
|
155
|
+
2,
|
|
156
|
+
)
|
|
133
157
|
_need_plotly()
|
|
134
158
|
from pytest_benchmem import plotting
|
|
135
159
|
|
|
@@ -145,6 +169,7 @@ def plot(
|
|
|
145
169
|
where=filters,
|
|
146
170
|
free_axes=free_axes,
|
|
147
171
|
labels=labels,
|
|
172
|
+
pivot=pivot,
|
|
148
173
|
)
|
|
149
174
|
elif chosen == "scatter":
|
|
150
175
|
fig, n = plotting.plot_scatter(
|
|
@@ -155,6 +180,7 @@ def plot(
|
|
|
155
180
|
where=filters,
|
|
156
181
|
free_axes=free_axes,
|
|
157
182
|
labels=labels,
|
|
183
|
+
pivot=pivot,
|
|
158
184
|
)
|
|
159
185
|
elif chosen == "sweep":
|
|
160
186
|
fig, n = plotting.plot_sweep(
|
|
@@ -188,7 +214,10 @@ def plot(
|
|
|
188
214
|
def compare(
|
|
189
215
|
runs: Annotated[
|
|
190
216
|
list[Path],
|
|
191
|
-
typer.Argument(
|
|
217
|
+
typer.Argument(
|
|
218
|
+
help="One or more pytest-benchmark runs, oldest → newest. One prints a plain "
|
|
219
|
+
"table; two or more compare (a sweep is N)."
|
|
220
|
+
),
|
|
192
221
|
],
|
|
193
222
|
columns: Annotated[
|
|
194
223
|
str | None,
|
|
@@ -219,6 +248,16 @@ def compare(
|
|
|
219
248
|
str,
|
|
220
249
|
typer.Option(help="Row order: name (id) | value (largest in the last run) | change."),
|
|
221
250
|
] = "name",
|
|
251
|
+
pivot: Annotated[
|
|
252
|
+
str | None,
|
|
253
|
+
typer.Option(
|
|
254
|
+
"--pivot",
|
|
255
|
+
help="Comparison axis: fold a single run along this dim instead of across run-files "
|
|
256
|
+
"— param:NAME or a bare extra_info name. Rows differing only in it pair up and its "
|
|
257
|
+
"values become the compared series. Like --group-by but it sets what's *compared*, "
|
|
258
|
+
"not how rows cluster. Mutually exclusive with multiple runs.",
|
|
259
|
+
),
|
|
260
|
+
] = None,
|
|
222
261
|
csv: Annotated[
|
|
223
262
|
Path | None,
|
|
224
263
|
typer.Option(help="Also write the raw (unscaled) comparison to this CSV file."),
|
|
@@ -227,29 +266,50 @@ def compare(
|
|
|
227
266
|
list[str] | None,
|
|
228
267
|
typer.Option(
|
|
229
268
|
"--fail-on",
|
|
230
|
-
help="Exit non-zero on a regression of the first run vs the last
|
|
231
|
-
"FIELD:THRESHOLD, repeatable — e.g. --fail-on
|
|
232
|
-
"--fail-on rss:10% (rss gates only isolated runs).",
|
|
269
|
+
help="Exit non-zero on a regression of the first run vs the last (or, with --pivot, "
|
|
270
|
+
"the first dim value vs the last). FIELD:THRESHOLD, repeatable — e.g. --fail-on "
|
|
271
|
+
"peak:10% --fail-on peak:5MiB --fail-on rss:10% (rss gates only isolated runs).",
|
|
233
272
|
),
|
|
234
273
|
] = None,
|
|
235
274
|
) -> None:
|
|
236
|
-
"""Print a per-id
|
|
237
|
-
if
|
|
238
|
-
raise _fail("compare needs at least
|
|
275
|
+
"""Print a per-id table for one run, or compare two or more (and optionally gate CI)."""
|
|
276
|
+
if not runs:
|
|
277
|
+
raise _fail("compare needs at least one run", 2)
|
|
278
|
+
# A growth gate needs a base and a head: two runs, or one run folded along --pivot (which
|
|
279
|
+
# supplies the two sides from one file). Without either, refuse rather than silently pass.
|
|
280
|
+
if fail_on and len(runs) < 2 and pivot is None: # noqa: PLR2004
|
|
281
|
+
raise _fail(
|
|
282
|
+
"compare --fail-on needs at least two runs — it gates growth of the first run vs "
|
|
283
|
+
"the last. To gate one combined run, add --pivot DIM (first dim value vs last); for "
|
|
284
|
+
"an absolute ceiling, use the @pytest.mark.benchmem(max_peak=...) marker instead.",
|
|
285
|
+
2,
|
|
286
|
+
)
|
|
239
287
|
_require_runs_exist(runs, suggest=False)
|
|
240
|
-
from pytest_benchmem.compare import
|
|
288
|
+
from pytest_benchmem.compare import (
|
|
289
|
+
compare_runs,
|
|
290
|
+
find_pivot_regressions,
|
|
291
|
+
find_regressions,
|
|
292
|
+
parse_threshold,
|
|
293
|
+
)
|
|
241
294
|
|
|
242
295
|
with _exit_on_value_error():
|
|
243
|
-
compare_runs(
|
|
296
|
+
compare_runs(
|
|
297
|
+
runs, columns=columns, group_by=group_by, stat=stat, sort=sort, pivot=pivot, csv=csv
|
|
298
|
+
)
|
|
244
299
|
|
|
245
300
|
if not fail_on:
|
|
246
301
|
return
|
|
247
302
|
with _exit_on_value_error(code=2):
|
|
248
303
|
thresholds = [parse_threshold(expr) for expr in fail_on]
|
|
249
304
|
|
|
250
|
-
# Gate the first run
|
|
305
|
+
# Gate base vs head: the first run vs the last (oldest vs newest in a sweep), or — with
|
|
306
|
+
# --pivot — the first value of the dim vs the last, folded out of the one run.
|
|
251
307
|
with _exit_on_value_error(code=2): # e.g. an rss gate against non-isolated runs
|
|
252
|
-
regressions =
|
|
308
|
+
regressions = (
|
|
309
|
+
find_pivot_regressions(runs[0], pivot, thresholds)
|
|
310
|
+
if pivot is not None
|
|
311
|
+
else find_regressions(runs[0], runs[-1], thresholds)
|
|
312
|
+
)
|
|
253
313
|
if regressions:
|
|
254
314
|
typer.secho(f"{len(regressions)} regression(s) over threshold:", fg=typer.colors.RED)
|
|
255
315
|
for reg in regressions:
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
"""Text
|
|
1
|
+
"""Text table for one or more pytest-benchmark runs — keyed on the benchmark id.
|
|
2
2
|
|
|
3
|
-
:func:`compare_runs` prints the
|
|
3
|
+
:func:`compare_runs` prints the table, modelled on pytest-benchmark's own:
|
|
4
4
|
rows are one per ``(benchmark × run)``, columns are ``metric × stat`` (``--columns``
|
|
5
5
|
picks the metrics, default ``time,peak``; ``--stat`` picks the stat, default ``all`` for
|
|
6
|
-
the full min/max/mean/median/stddev spread),
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
the full min/max/mean/median/stddev spread), and ``group_by`` splits the rows into
|
|
7
|
+
sub-tables. With **two or more** runs every cell carries a relative ``(N.NN)`` multiplier
|
|
8
|
+
vs its column's best (best green, worst red); a **single** run is a plain readout — there's
|
|
9
|
+
no other run to rank against, so the multiplier and colour are dropped.
|
|
9
10
|
|
|
10
11
|
The rest is the regression gate behind ``benchmem compare --fail-on``: parse a threshold
|
|
11
12
|
like ``peak:10%`` / ``peak:5MiB`` / ``allocations:5%`` / ``time:5%``, find the ids whose
|
|
@@ -141,14 +142,18 @@ def _short(test_id: str) -> str:
|
|
|
141
142
|
|
|
142
143
|
|
|
143
144
|
def _load_columns(
|
|
144
|
-
runs: Sequence[str | Path],
|
|
145
|
+
runs: Sequence[str | Path],
|
|
146
|
+
metrics: Sequence[Metric],
|
|
147
|
+
stats: Sequence[str],
|
|
148
|
+
pivot: str | None = None,
|
|
145
149
|
) -> tuple[list[str], dict[tuple[str, str, str], float], dict[str, str], dict[str, dict[str, Any]]]:
|
|
146
150
|
"""Load each ``metric × stat`` column into ``(labels, values, units, dims)``.
|
|
147
151
|
|
|
148
|
-
Columns are keyed by a ``"metric:stat"`` id; ``values`` maps ``(col_id, id,
|
|
149
|
-
value``, ``units`` maps ``metric → unit``.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
+
Columns are keyed by a ``"metric:stat"`` id; ``values`` maps ``(col_id, id, series) →
|
|
153
|
+
value``, ``units`` maps ``metric → unit``. The series labels keep their natural order —
|
|
154
|
+
file order (oldest → newest) for run-files, or dim-value order when ``pivot`` folds a single
|
|
155
|
+
run along a dim. Each stat reads pytest-benchmark's own ``stats`` for ``time`` and reduces
|
|
156
|
+
the per-repeat series for the memory metrics.
|
|
152
157
|
"""
|
|
153
158
|
paths = [Path(r) for r in runs]
|
|
154
159
|
labels: list[str] = []
|
|
@@ -157,7 +162,7 @@ def _load_columns(
|
|
|
157
162
|
dims: dict[str, dict[str, Any]] = {}
|
|
158
163
|
for metric in metrics:
|
|
159
164
|
for stat in stats:
|
|
160
|
-
df, unit = load_long_df(paths, metric=metric, stat=stat)
|
|
165
|
+
df, unit = load_long_df(paths, metric=metric, stat=stat, pivot=pivot)
|
|
161
166
|
units[metric] = unit
|
|
162
167
|
if df.empty: # metric absent from every run (e.g. memory on timing-only files)
|
|
163
168
|
continue
|
|
@@ -224,25 +229,36 @@ def compare_runs(
|
|
|
224
229
|
stat: str | None = None,
|
|
225
230
|
group_by: str | None = "fullname",
|
|
226
231
|
sort: str = "name",
|
|
232
|
+
pivot: str | None = None,
|
|
227
233
|
csv: Path | None = None,
|
|
228
234
|
out: TextIO | None = None,
|
|
229
235
|
) -> None:
|
|
230
|
-
"""Print a per-(benchmark,
|
|
231
|
-
|
|
232
|
-
Mirrors pytest-benchmark's table model: rows are one per ``(benchmark ×
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
236
|
+
"""Print a per-(benchmark, series) table for one or more runs.
|
|
237
|
+
|
|
238
|
+
Mirrors pytest-benchmark's table model: rows are one per ``(benchmark × series)``, where
|
|
239
|
+
the **series axis** is the run-file by default (one column-group per run). With two or more
|
|
240
|
+
series each cell carries a relative ``(N.NN)`` multiplier vs the group's best (best green,
|
|
241
|
+
worst red); a single series is a plain readout with neither. ``columns`` selects which metric
|
|
242
|
+
columns appear (``time`` / ``peak`` / ``allocated`` / ``allocations``). ``group_by`` splits
|
|
243
|
+
the rows into sub-tables (``fullname`` / ``name`` / ``func`` / ``group`` / ``module`` /
|
|
244
|
+
``class`` / ``param:NAME``, comma-composable); with no ``group_by`` the whole comparison is
|
|
245
|
+
one table.
|
|
246
|
+
|
|
247
|
+
``pivot`` re-points the series axis from the run-file onto a data dim (``param:NAME`` or a
|
|
248
|
+
bare ``extra_info`` name), folding a *single* combined run along that dim: rows that differ
|
|
249
|
+
only in it pair up and its values become the compared series — the A/B a run-file pair gives
|
|
250
|
+
you today, without splitting the data into N files. Distinct from ``group_by`` (which only
|
|
251
|
+
clusters rows into sub-tables, leaving the run-files as the compared series): ``pivot`` sets
|
|
252
|
+
*what* is compared, ``group_by`` sets how rows cluster. It composes with ``columns`` /
|
|
253
|
+
``stat`` / ``sort`` / ``group_by`` unchanged and is mutually exclusive with multiple runs
|
|
254
|
+
(one series axis per table).
|
|
239
255
|
|
|
240
256
|
Every column is a ``metric × stat`` pair. ``columns`` is a comma list of metric names
|
|
241
257
|
(default ``time,peak``); ``stat`` is one of ``min`` / ``max`` / ``mean`` / ``median`` /
|
|
242
258
|
``stddev`` or ``all`` (the default), which expands each metric into its full stat
|
|
243
259
|
spread — so no single statistic is privileged. A metric absent from every run is
|
|
244
260
|
dropped rather than shown all dashes (so timing-only runs collapse to just ``time``).
|
|
245
|
-
``sort`` orders rows within a group: ``name``, ``value`` (largest in the last
|
|
261
|
+
``sort`` orders rows within a group: ``name``, ``value`` (largest in the last series), or
|
|
246
262
|
``change`` (biggest growth first). ``csv`` also writes the raw (unscaled) comparison.
|
|
247
263
|
"""
|
|
248
264
|
from rich import box
|
|
@@ -253,12 +269,17 @@ def compare_runs(
|
|
|
253
269
|
if sort not in _SORTS:
|
|
254
270
|
raise ValueError(f"unknown --sort {sort!r}; use one of {', '.join(_SORTS)}")
|
|
255
271
|
metrics, stats = _resolve_columns(columns), _resolve_stats(stat)
|
|
256
|
-
labels, values, units, dims = _load_columns(runs, metrics, stats)
|
|
257
|
-
if
|
|
272
|
+
labels, values, units, dims = _load_columns(runs, metrics, stats, pivot)
|
|
273
|
+
if not labels:
|
|
274
|
+
raise ValueError("no benchmarks found in the given run(s)")
|
|
275
|
+
if len(labels) < 2 and len(runs) > 1: # noqa: PLR2004 — distinct paths collapsed to one series
|
|
258
276
|
raise ValueError(
|
|
259
277
|
f"compare needs at least two distinct runs; got {labels!r} "
|
|
260
278
|
f"(the same file more than once?). Pass distinct files."
|
|
261
279
|
)
|
|
280
|
+
# One run is a plain readout: with no other run to rank against, the (N.NN) multiplier and
|
|
281
|
+
# the best/worst colour are meaningless (best == worst == the value), so they're dropped.
|
|
282
|
+
single = len(labels) == 1
|
|
262
283
|
with_data = {col_id for col_id, _i, _lab in values}
|
|
263
284
|
all_cols = [(m, s) for m in metrics for s in stats]
|
|
264
285
|
cols = [(m, s) for m, s in all_cols if f"{m}:{s}" in with_data] or all_cols # drop empties
|
|
@@ -338,7 +359,9 @@ def compare_runs(
|
|
|
338
359
|
if unit == ""
|
|
339
360
|
else f"{v:.4g}"
|
|
340
361
|
)
|
|
341
|
-
|
|
362
|
+
mult = "" if single else _mult(v, best)
|
|
363
|
+
style = "" if single else (rank_style(v, best, worst) or "")
|
|
364
|
+
cells.append(Text(body + mult, style=style))
|
|
342
365
|
table.add_row(*cells)
|
|
343
366
|
console.print(table)
|
|
344
367
|
console.print() # one blank line between sub-tables
|
|
@@ -465,6 +488,39 @@ def find_regressions(a: str | Path, b: str | Path, thresholds: list[Threshold])
|
|
|
465
488
|
return regressions
|
|
466
489
|
|
|
467
490
|
|
|
491
|
+
def find_pivot_regressions(
|
|
492
|
+
run: str | Path, pivot: str, thresholds: list[Threshold]
|
|
493
|
+
) -> list[Regression]:
|
|
494
|
+
"""Regressions along a ``--pivot`` axis within *one* run — the gate generalized the same way
|
|
495
|
+
the table is: with run-files the base is ``runs[0]`` and the head ``runs[-1]``; here the base
|
|
496
|
+
is the first value of the ``pivot`` dim and the head the last (mirroring oldest→newest). Each
|
|
497
|
+
fold-collapsed id is paired across them and flagged only on growth, reusing
|
|
498
|
+
:func:`_regressions_for`.
|
|
499
|
+
"""
|
|
500
|
+
regressions: list[Regression] = []
|
|
501
|
+
for th in thresholds:
|
|
502
|
+
# ``pivot`` folds the run so ``snapshot`` carries the dim's values and ``id`` is the
|
|
503
|
+
# collapsed pairing key; the fail-on field doubles as the metric to read (headline scalar).
|
|
504
|
+
df, _unit = load_long_df(run, metric=cast("Metric", th.field), pivot=pivot)
|
|
505
|
+
labels = list(dict.fromkeys(df["snapshot"]))
|
|
506
|
+
if len(labels) < 2: # noqa: PLR2004 — a growth gate needs a base value and a head value
|
|
507
|
+
raise ValueError(
|
|
508
|
+
f"--fail-on with --pivot {pivot!r} needs two distinct values to gate; got "
|
|
509
|
+
f"{labels}. Does the run vary {pivot!r}?"
|
|
510
|
+
)
|
|
511
|
+
base_label, head_label = labels[0], labels[-1]
|
|
512
|
+
rows = zip(df["id"], df["value"], df["snapshot"], strict=True)
|
|
513
|
+
side: dict[str, dict[str, float]] = {base_label: {}, head_label: {}}
|
|
514
|
+
for test_id, value, label in rows:
|
|
515
|
+
if label in side:
|
|
516
|
+
side[label][test_id] = float(value)
|
|
517
|
+
base, head = side[base_label], side[head_label]
|
|
518
|
+
if th.field == "rss" and not (base.keys() & head.keys()):
|
|
519
|
+
raise ValueError(_RSS_GATE_EMPTY)
|
|
520
|
+
regressions.extend(_regressions_for(base, head, th))
|
|
521
|
+
return regressions
|
|
522
|
+
|
|
523
|
+
|
|
468
524
|
#: Memory-blob key per fail-on field — the fields ``--benchmark-memory-compare-fail`` gates on.
|
|
469
525
|
_BLOB_FIELD = {
|
|
470
526
|
"peak": "peak_bytes",
|