pytest-benchmem 0.4.2__tar.gz → 0.4.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/dependabot.yml +3 -3
  2. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/ci.yaml +7 -7
  3. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/docs.yaml +2 -2
  4. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/nightly.yaml +1 -1
  5. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/release.yaml +1 -1
  6. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.gitignore +0 -3
  7. pytest_benchmem-0.4.4/.release-please-manifest.json +3 -0
  8. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/CHANGELOG.md +14 -0
  9. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/PKG-INFO +1 -1
  10. pytest_benchmem-0.4.4/docs/changelog.md +9 -0
  11. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/compare-plot.md +68 -0
  12. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/mkdocs.yml +5 -0
  13. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/pyproject.toml +2 -1
  14. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/cli.py +72 -12
  15. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/compare.py +80 -24
  16. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/plotting.py +34 -12
  17. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/snapshot.py +87 -3
  18. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_cli.py +84 -0
  19. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_compare.py +140 -0
  20. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_plotting.py +89 -0
  21. pytest_benchmem-0.4.4/uv.lock +2441 -0
  22. pytest_benchmem-0.4.2/.release-please-manifest.json +0 -3
  23. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/dependabot-auto-merge.yaml +0 -0
  24. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.github/workflows/pr-title.yaml +0 -0
  25. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.pre-commit-config.yaml +0 -0
  26. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.readthedocs.yaml +0 -0
  27. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/.release-please-config.json +0 -0
  28. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/LICENSE +0 -0
  29. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/README.md +0 -0
  30. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/dims.md +0 -0
  31. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/getting-started.md +0 -0
  32. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/index.md +0 -0
  33. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/javascripts/notebook-prompts.js +0 -0
  34. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/metrics.md +0 -0
  35. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/reference.md +0 -0
  36. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/stylesheets/extra.css +0 -0
  37. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs/sweeps.md +0 -0
  38. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/docs_macros.py +0 -0
  39. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/__init__.py +0 -0
  40. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/combined.py +0 -0
  41. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/format.py +0 -0
  42. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/memray.py +0 -0
  43. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/py.typed +0 -0
  44. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/pytest_plugin.py +0 -0
  45. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/sweep.py +0 -0
  46. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/src/pytest_benchmem/tables.py +0 -0
  47. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/__init__.py +0 -0
  48. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/conftest.py +0 -0
  49. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_combined.py +0 -0
  50. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_distribution.py +0 -0
  51. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_inline_compare.py +0 -0
  52. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_memray.py +0 -0
  53. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_memray_platform.py +0 -0
  54. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_plugin.py +0 -0
  55. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_snapshot.py +0 -0
  56. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_sweep.py +0 -0
  57. {pytest_benchmem-0.4.2 → pytest_benchmem-0.4.4}/tests/test_tables.py +0 -0
@@ -1,10 +1,10 @@
1
1
  version: 2
2
2
 
3
3
  updates:
4
- - package-ecosystem: "pip"
4
+ - package-ecosystem: "uv"
5
5
  directory: "/"
6
6
  schedule:
7
- interval: "weekly"
7
+ interval: "monthly"
8
8
  labels:
9
9
  - "dependencies"
10
10
  commit-message:
@@ -27,7 +27,7 @@ updates:
27
27
  - package-ecosystem: "github-actions"
28
28
  directory: "/"
29
29
  schedule:
30
- interval: "weekly"
30
+ interval: "monthly"
31
31
  labels:
32
32
  - "dependencies"
33
33
  commit-message:
@@ -16,11 +16,11 @@ jobs:
16
16
  runs-on: ubuntu-24.04
17
17
  timeout-minutes: 5
18
18
  steps:
19
- - uses: actions/checkout@v6
19
+ - uses: actions/checkout@v7
20
20
  - uses: astral-sh/setup-uv@v7
21
21
  with:
22
22
  enable-cache: true
23
- - run: uv sync --group dev
23
+ - run: uv sync --group dev --frozen
24
24
  - run: uv run ruff check --output-format=github
25
25
  - run: uv run ruff format --diff
26
26
 
@@ -29,11 +29,11 @@ jobs:
29
29
  runs-on: ubuntu-24.04
30
30
  timeout-minutes: 5
31
31
  steps:
32
- - uses: actions/checkout@v6
32
+ - uses: actions/checkout@v7
33
33
  - uses: astral-sh/setup-uv@v7
34
34
  with:
35
35
  enable-cache: true
36
- - run: uv sync --group dev
36
+ - run: uv sync --group dev --frozen
37
37
  - run: uv run mypy src/pytest_benchmem tests
38
38
 
39
39
  test:
@@ -49,14 +49,14 @@ jobs:
49
49
  - os: macos-latest
50
50
  python-version: "3.13"
51
51
  steps:
52
- - uses: actions/checkout@v6
52
+ - uses: actions/checkout@v7
53
53
  - uses: astral-sh/setup-uv@v7
54
54
  with:
55
55
  enable-cache: true
56
56
  - uses: actions/setup-python@v6
57
57
  with:
58
58
  python-version: ${{ matrix.python-version }}
59
- - run: uv sync --group dev
59
+ - run: uv sync --group dev --frozen
60
60
  - run: uv run pytest -q
61
61
 
62
62
  smoke-install:
@@ -64,7 +64,7 @@ jobs:
64
64
  runs-on: ubuntu-24.04
65
65
  timeout-minutes: 10
66
66
  steps:
67
- - uses: actions/checkout@v6
67
+ - uses: actions/checkout@v7
68
68
  - uses: astral-sh/setup-uv@v7
69
69
  with:
70
70
  enable-cache: true
@@ -18,11 +18,11 @@ jobs:
18
18
  runs-on: ubuntu-24.04
19
19
  timeout-minutes: 10
20
20
  steps:
21
- - uses: actions/checkout@v6
21
+ - uses: actions/checkout@v7
22
22
  - uses: astral-sh/setup-uv@v7
23
23
  with:
24
24
  enable-cache: true
25
25
  # Build the notebooks from their jupytext sources, then build+execute the site.
26
- - run: uv sync --group docs
26
+ - run: uv sync --group docs --frozen
27
27
  - run: uv run jupytext --to ipynb docs/compare-plot.md
28
28
  - run: uv run mkdocs gh-deploy --force
@@ -31,7 +31,7 @@ jobs:
31
31
  python: "3.13"
32
32
  resolution: "--upgrade"
33
33
  steps:
34
- - uses: actions/checkout@v6
34
+ - uses: actions/checkout@v7
35
35
  - uses: astral-sh/setup-uv@v7
36
36
  - uses: actions/setup-python@v6
37
37
  with:
@@ -49,7 +49,7 @@ jobs:
49
49
  name: pypi
50
50
  url: https://pypi.org/project/pytest-benchmem
51
51
  steps:
52
- - uses: actions/checkout@v6
52
+ - uses: actions/checkout@v7
53
53
  with:
54
54
  ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || needs.release-please.outputs.tag_name }}
55
55
  fetch-depth: 0
@@ -10,9 +10,6 @@ build/
10
10
  .pytest_cache/
11
11
  *.bin
12
12
 
13
- # lockfile not tracked for a library — resolve fresh against current deps
14
- uv.lock
15
-
16
13
  # jupytext-built notebooks (source is the .md) + mkdocs build
17
14
  docs/*.ipynb
18
15
  site/
@@ -0,0 +1,3 @@
1
+ {
2
+ ".": "0.4.4"
3
+ }
@@ -1,5 +1,19 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.4.4](https://github.com/fluxopt/pytest-benchmem/compare/v0.4.3...v0.4.4) (2026-06-29)
4
+
5
+
6
+ ### Features
7
+
8
+ * `--pivot` comparison axis for compare/plot (fold one run along a config dim) ([#137](https://github.com/fluxopt/pytest-benchmem/issues/137)) ([d0672f7](https://github.com/fluxopt/pytest-benchmem/commit/d0672f74d7040b4f777df342c346052fc9f77678))
9
+
10
+ ## [0.4.3](https://github.com/fluxopt/pytest-benchmem/compare/v0.4.2...v0.4.3) (2026-06-29)
11
+
12
+
13
+ ### Features
14
+
15
+ * benchmem compare accepts a single run (plain table) ([#123](https://github.com/fluxopt/pytest-benchmem/issues/123)) ([96275d9](https://github.com/fluxopt/pytest-benchmem/commit/96275d93628323ffd4f3cc74decdebdcb132004e))
16
+
3
17
  ## [0.4.2](https://github.com/fluxopt/pytest-benchmem/compare/v0.4.1...v0.4.2) (2026-06-18)
4
18
 
5
19
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pytest-benchmem
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: The memory companion to pytest-benchmark: a memray peak-memory pass on the same test, plus dims-aware plots and cross-version sweeps.
5
5
  Project-URL: Homepage, https://github.com/fluxopt/pytest-benchmem
6
6
  Project-URL: Documentation, https://fluxopt.github.io/pytest-benchmem/
@@ -0,0 +1,9 @@
1
+ ---
2
+ hide:
3
+ - navigation
4
+ ---
5
+
6
+ <!-- Rendered from the repo-root CHANGELOG.md (release-please source of truth) so the
7
+ published release notes can't drift from the tagged releases. -->
8
+
9
+ --8<-- "CHANGELOG.md"
@@ -99,6 +99,72 @@ Pick the metrics with `--columns` (a comma list of `time` / `peak` / `allocated`
99
99
  `module` | `class` | `param:NAME`, comma-composable); pass it to cluster param-variants
100
100
  (`--group-by func`) or collapse everything into one table.
101
101
 
102
+ ## One run, two configs — `--pivot`
103
+
104
+ Everything above compares **run-files**: `compare a.json b.json` lays one file against
105
+ another. But the file is just the *series axis* — the thing whose values sit side by side and
106
+ get the `(N.NN)` multiplier. **The series axis is a [dim](dims.md)**, and the run-file is only
107
+ the default one.
108
+
109
+ `--pivot DIM` re-points it at a data dim, so a **single combined run** is A/B'd along that dim —
110
+ the role a file-pair plays today, without splitting into N runs. Say `semantics` is a param
111
+ (so it's in the node id, `test_build[legacy-100]`, *and* a dim):
112
+
113
+ ```python
114
+ @pytest.mark.parametrize("semantics", ["legacy", "v1"])
115
+ @pytest.mark.parametrize("n", [100, 200])
116
+ def test_build(benchmark, semantics, n):
117
+ benchmark(build, semantics=semantics, n=n)
118
+ ```
119
+
120
+ One `pytest` invocation produces one `build.json`; `--pivot param:semantics` folds it so rows
121
+ that differ *only* in `semantics` pair up (`legacy` ↔ `v1`) per `n`:
122
+
123
+ ```bash
124
+ pytest benchmarks/ --benchmark-only --benchmark-memory --benchmark-json=build.json
125
+ benchmem compare build.json --pivot param:semantics --columns time,peak # the A/B table
126
+ benchmem plot build.json --pivot param:semantics --columns peak --view compare # the A/B bars
127
+ benchmem plot build.json --x n --facet semantics --columns peak # --pivot optional here
128
+ ```
129
+
130
+ Don't confuse it with `--group-by`, which touches the *other* axis: `--group-by` **partitions
131
+ rows** into sub-tables along a dim while the compared series stays the run-files (`legacy` and
132
+ `v1` would sit in separate sections, never set against each other); `--pivot` makes the dim
133
+ *itself* the compared series, folding rows that differ only in it into one ranked A/B row. They
134
+ compose on orthogonal axes — `--pivot param:semantics --group-by node.func` folds `legacy` ↔ `v1`
135
+ *and* clusters the folded rows into per-function sub-tables. In short: `--group-by` says which
136
+ rows belong together; `--pivot` says what you're comparing.
137
+
138
+ `--pivot` accepts the same dims as `--group-by`/`--facet` — `param:NAME` or a bare `extra_info`
139
+ name — and composes with `--columns`, `--stat`, `--facet`, `--where`, and `--sort` unchanged.
140
+ It's mutually exclusive with multiple runs (an A/B view has *one* series axis: comparing files
141
+ *and* folding a dim would be a 2-D matrix), and only the paired views consume it — on `plot`
142
+ it applies to `--view compare`/`scatter`; for `scaling`/`sweep` the dim is a normal `--x` /
143
+ `--facet` axis instead. One combined run then drives the A/B table, the A/B plot, the scaling
144
+ plot, *and* an external per-id gate (e.g. CodSpeed, which wants the config value in the node id
145
+ of one run) — no file-splitting, no per-config reruns.
146
+
147
+ `--fail-on` follows the same axis: it normally gates the first run vs the last, but with
148
+ `--pivot` it gates the **first dim value vs the last** (e.g. `legacy` → `v1`) out of the one
149
+ run — so the combined run also gates itself in CI, no second file required:
150
+
151
+ ```bash
152
+ # fail if v1's peak grew >10% over legacy, for any benchmark, from one run:
153
+ pytest benchmarks/ --benchmark-only --benchmark-memory --benchmark-json=build.json
154
+ benchmem compare build.json --pivot param:semantics --fail-on peak:10%
155
+ ```
156
+
157
+ The **base** (the `--fail-on` reference, and the first row) is the dim's *first* value in
158
+ parametrize / collection order — the `--pivot` analogue of file argument order. So
159
+ `parametrize("semantics", ["legacy", "v1"])` gates `v1` against `legacy`; reorder the list to
160
+ flip it. (The table's `(1.0)` annotation is separate — it marks each column's *best* value,
161
+ green, exactly as the run-file table does, not the base.)
162
+
163
+ > A `--pivot` param is lifted out of the id when rows fold (`test_build[legacy-100]` → `test_build[100]`),
164
+ > matched by value. A param given a custom `pytest.param(id=…)` whose label differs from its
165
+ > value won't fold — use a plain parametrize value, or an `extra_info` dim (which isn't in the
166
+ > id at all), for the comparison axis.
167
+
102
168
  ## Gate CI on a regression
103
169
 
104
170
  `--fail-on` exits non-zero past a threshold — drop it into CI after the run. Thresholds are
@@ -289,6 +355,8 @@ A minimal GitHub Actions job using the two-file approach, caching the baseline a
289
355
  | 2 | `compare` (`--view compare`) | ranked — what moved *most*, in native units? |
290
356
  | 3+ | `sweep` | fold-change across versions, one cell per (id, run) |
291
357
 
358
+ `--pivot` re-points the series axis of the paired views (`compare`/`scatter`) from the run-file
359
+ onto a dim, folding a single run along it (see [One run, two configs](#one-run-two-configs-pivot)).
292
360
  `--facet` splits any view into small multiples by a [dim](dims.md) (including `node.*`),
293
361
  `--where` keeps only rows matching a `dim=value` filter (repeatable, AND-combined),
294
362
  `--free-axes` unmatches a faceted axis from the shared default, and `--label`/`-l` names
@@ -39,6 +39,7 @@ nav:
39
39
  - Grouping by dims: dims.md
40
40
  - Cross-version sweeps: sweeps.md
41
41
  - Reference: reference.md
42
+ - Release notes: changelog.md
42
43
 
43
44
  plugins:
44
45
  - search
@@ -93,6 +94,10 @@ markdown_extensions:
93
94
  anchor_linenums: true
94
95
  use_pygments: true
95
96
  - pymdownx.inlinehilite
97
+ - pymdownx.snippets:
98
+ # Pull docs/changelog.md's `--8<-- "CHANGELOG.md"` from the repo root.
99
+ base_path: ["."]
100
+ check_paths: true
96
101
  - pymdownx.superfences
97
102
  - pymdownx.details
98
103
  - pymdownx.tabbed:
@@ -48,7 +48,8 @@ build-backend = "hatchling.build"
48
48
  source = "vcs"
49
49
 
50
50
  [dependency-groups]
51
- dev = ["ruff", "mypy", "pre-commit", "pytest-benchmem[plot]", { include-group = "docs" }]
51
+ # mypy <2.1: 2.1.0 errors on numpy's PEP 695 stub under python_version=3.11. Lift once fixed upstream.
52
+ dev = ["ruff", "mypy<2.2", "pre-commit", "pytest-benchmem[plot]", { include-group = "docs" }]
52
53
  docs = [
53
54
  "jupytext",
54
55
  "ipykernel",
@@ -100,6 +100,16 @@ def plot(
100
100
  typer.Option(help="compare | scatter | sweep | scaling (default: by count)."),
101
101
  ] = None,
102
102
  facet: Annotated[str | None, typer.Option(help="Dim to facet by.")] = None,
103
+ pivot: Annotated[
104
+ str | None,
105
+ typer.Option(
106
+ "--pivot",
107
+ help="Comparison axis for --view compare/scatter: fold a single run along this dim "
108
+ "instead of across run-files (param:NAME or a bare extra_info name); its values "
109
+ "become the compared series. Like --group-by but it sets what's *compared*, not how "
110
+ "rows cluster. Mutually exclusive with multiple runs.",
111
+ ),
112
+ ] = None,
103
113
  x: Annotated[str | None, typer.Option(help="scaling: dim for the x-axis.")] = None,
104
114
  clip: Annotated[float | None, typer.Option(help="Clamp the colour scale.")] = None,
105
115
  where: Annotated[
@@ -129,7 +139,21 @@ def plot(
129
139
  """Render an interactive plotly view from one or more pytest-benchmark runs."""
130
140
  _require_runs_exist(runs, suggest=True)
131
141
 
132
- chosen = view or ("scaling" if len(runs) == 1 else "scatter" if len(runs) == 2 else "sweep")
142
+ # --pivot's natural home is the A/B compare view, so when --view is unset it defaults there
143
+ # (otherwise one run would default to scaling, which has no series axis to pivot). An explicit
144
+ # --view still wins in both cases; the guard below rejects a non-paired view under --pivot.
145
+ if pivot is not None:
146
+ chosen = view or "compare"
147
+ else:
148
+ chosen = view or ("scaling" if len(runs) == 1 else "scatter" if len(runs) == 2 else "sweep")
149
+ # --pivot re-points the series axis at a dim; only the paired views (compare/scatter) have a
150
+ # series axis to re-point. For scaling/sweep the dim is a normal x/colour/facet axis.
151
+ if pivot is not None and chosen not in ("compare", "scatter"):
152
+ raise _fail(
153
+ f"--pivot re-points the comparison series and applies to --view compare or scatter, "
154
+ f"not {chosen!r}; for scaling/sweep use --x / --facet to place the dim.",
155
+ 2,
156
+ )
133
157
  _need_plotly()
134
158
  from pytest_benchmem import plotting
135
159
 
@@ -145,6 +169,7 @@ def plot(
145
169
  where=filters,
146
170
  free_axes=free_axes,
147
171
  labels=labels,
172
+ pivot=pivot,
148
173
  )
149
174
  elif chosen == "scatter":
150
175
  fig, n = plotting.plot_scatter(
@@ -155,6 +180,7 @@ def plot(
155
180
  where=filters,
156
181
  free_axes=free_axes,
157
182
  labels=labels,
183
+ pivot=pivot,
158
184
  )
159
185
  elif chosen == "sweep":
160
186
  fig, n = plotting.plot_sweep(
@@ -188,7 +214,10 @@ def plot(
188
214
  def compare(
189
215
  runs: Annotated[
190
216
  list[Path],
191
- typer.Argument(help="Two or more pytest-benchmark runs, oldest → newest (a sweep is N)."),
217
+ typer.Argument(
218
+ help="One or more pytest-benchmark runs, oldest → newest. One prints a plain "
219
+ "table; two or more compare (a sweep is N)."
220
+ ),
192
221
  ],
193
222
  columns: Annotated[
194
223
  str | None,
@@ -219,6 +248,16 @@ def compare(
219
248
  str,
220
249
  typer.Option(help="Row order: name (id) | value (largest in the last run) | change."),
221
250
  ] = "name",
251
+ pivot: Annotated[
252
+ str | None,
253
+ typer.Option(
254
+ "--pivot",
255
+ help="Comparison axis: fold a single run along this dim instead of across run-files "
256
+ "— param:NAME or a bare extra_info name. Rows differing only in it pair up and its "
257
+ "values become the compared series. Like --group-by but it sets what's *compared*, "
258
+ "not how rows cluster. Mutually exclusive with multiple runs.",
259
+ ),
260
+ ] = None,
222
261
  csv: Annotated[
223
262
  Path | None,
224
263
  typer.Option(help="Also write the raw (unscaled) comparison to this CSV file."),
@@ -227,29 +266,50 @@ def compare(
227
266
  list[str] | None,
228
267
  typer.Option(
229
268
  "--fail-on",
230
- help="Exit non-zero on a regression of the first run vs the last. "
231
- "FIELD:THRESHOLD, repeatable — e.g. --fail-on peak:10% --fail-on peak:5MiB "
232
- "--fail-on rss:10% (rss gates only isolated runs).",
269
+ help="Exit non-zero on a regression of the first run vs the last (or, with --pivot, "
270
+ "the first dim value vs the last). FIELD:THRESHOLD, repeatable — e.g. --fail-on "
271
+ "peak:10% --fail-on peak:5MiB --fail-on rss:10% (rss gates only isolated runs).",
233
272
  ),
234
273
  ] = None,
235
274
  ) -> None:
236
- """Print a per-id comparison table across two or more runs (and optionally gate CI)."""
237
- if len(runs) < 2: # noqa: PLR2004 — a comparison needs two sides
238
- raise _fail("compare needs at least two runs", 2)
275
+ """Print a per-id table for one run, or compare two or more (and optionally gate CI)."""
276
+ if not runs:
277
+ raise _fail("compare needs at least one run", 2)
278
+ # A growth gate needs a base and a head: two runs, or one run folded along --pivot (which
279
+ # supplies the two sides from one file). Without either, refuse rather than silently pass.
280
+ if fail_on and len(runs) < 2 and pivot is None: # noqa: PLR2004
281
+ raise _fail(
282
+ "compare --fail-on needs at least two runs — it gates growth of the first run vs "
283
+ "the last. To gate one combined run, add --pivot DIM (first dim value vs last); for "
284
+ "an absolute ceiling, use the @pytest.mark.benchmem(max_peak=...) marker instead.",
285
+ 2,
286
+ )
239
287
  _require_runs_exist(runs, suggest=False)
240
- from pytest_benchmem.compare import compare_runs, find_regressions, parse_threshold
288
+ from pytest_benchmem.compare import (
289
+ compare_runs,
290
+ find_pivot_regressions,
291
+ find_regressions,
292
+ parse_threshold,
293
+ )
241
294
 
242
295
  with _exit_on_value_error():
243
- compare_runs(runs, columns=columns, group_by=group_by, stat=stat, sort=sort, csv=csv)
296
+ compare_runs(
297
+ runs, columns=columns, group_by=group_by, stat=stat, sort=sort, pivot=pivot, csv=csv
298
+ )
244
299
 
245
300
  if not fail_on:
246
301
  return
247
302
  with _exit_on_value_error(code=2):
248
303
  thresholds = [parse_threshold(expr) for expr in fail_on]
249
304
 
250
- # Gate the first run (base) against the last (head) — oldest vs newest in a sweep.
305
+ # Gate base vs head: the first run vs the last (oldest vs newest in a sweep), or — with
306
+ # --pivot — the first value of the dim vs the last, folded out of the one run.
251
307
  with _exit_on_value_error(code=2): # e.g. an rss gate against non-isolated runs
252
- regressions = find_regressions(runs[0], runs[-1], thresholds)
308
+ regressions = (
309
+ find_pivot_regressions(runs[0], pivot, thresholds)
310
+ if pivot is not None
311
+ else find_regressions(runs[0], runs[-1], thresholds)
312
+ )
253
313
  if regressions:
254
314
  typer.secho(f"{len(regressions)} regression(s) over threshold:", fg=typer.colors.RED)
255
315
  for reg in regressions:
@@ -1,11 +1,12 @@
1
- """Text comparison of two or more pytest-benchmark runs — keyed on the benchmark id.
1
+ """Text table for one or more pytest-benchmark runs — keyed on the benchmark id.
2
2
 
3
- :func:`compare_runs` prints the comparison table, modelled on pytest-benchmark's own:
3
+ :func:`compare_runs` prints the table, modelled on pytest-benchmark's own:
4
4
  rows are one per ``(benchmark × run)``, columns are ``metric × stat`` (``--columns``
5
5
  picks the metrics, default ``time,peak``; ``--stat`` picks the stat, default ``all`` for
6
- the full min/max/mean/median/stddev spread), every cell carries a relative ``(N.NN)``
7
- multiplier vs its column's best (best green, worst red), and ``group_by`` splits the rows
8
- into sub-tables.
6
+ the full min/max/mean/median/stddev spread), and ``group_by`` splits the rows into
7
+ sub-tables. With **two or more** runs every cell carries a relative ``(N.NN)`` multiplier
8
+ vs its column's best (best green, worst red); a **single** run is a plain readout — there's
9
+ no other run to rank against, so the multiplier and colour are dropped.
9
10
 
10
11
  The rest is the regression gate behind ``benchmem compare --fail-on``: parse a threshold
11
12
  like ``peak:10%`` / ``peak:5MiB`` / ``allocations:5%`` / ``time:5%``, find the ids whose
@@ -141,14 +142,18 @@ def _short(test_id: str) -> str:
141
142
 
142
143
 
143
144
  def _load_columns(
144
- runs: Sequence[str | Path], metrics: Sequence[Metric], stats: Sequence[str]
145
+ runs: Sequence[str | Path],
146
+ metrics: Sequence[Metric],
147
+ stats: Sequence[str],
148
+ pivot: str | None = None,
145
149
  ) -> tuple[list[str], dict[tuple[str, str, str], float], dict[str, str], dict[str, dict[str, Any]]]:
146
150
  """Load each ``metric × stat`` column into ``(labels, values, units, dims)``.
147
151
 
148
- Columns are keyed by a ``"metric:stat"`` id; ``values`` maps ``(col_id, id, run) →
149
- value``, ``units`` maps ``metric → unit``. Runs keep file order (oldest → newest).
150
- Each stat reads pytest-benchmark's own ``stats`` for ``time`` and reduces the
151
- per-repeat series for the memory metrics.
152
+ Columns are keyed by a ``"metric:stat"`` id; ``values`` maps ``(col_id, id, series) →
153
+ value``, ``units`` maps ``metric → unit``. The series labels keep their natural order
154
+ file order (oldest → newest) for run-files, or dim-value order when ``pivot`` folds a single
155
+ run along a dim. Each stat reads pytest-benchmark's own ``stats`` for ``time`` and reduces
156
+ the per-repeat series for the memory metrics.
152
157
  """
153
158
  paths = [Path(r) for r in runs]
154
159
  labels: list[str] = []
@@ -157,7 +162,7 @@ def _load_columns(
157
162
  dims: dict[str, dict[str, Any]] = {}
158
163
  for metric in metrics:
159
164
  for stat in stats:
160
- df, unit = load_long_df(paths, metric=metric, stat=stat)
165
+ df, unit = load_long_df(paths, metric=metric, stat=stat, pivot=pivot)
161
166
  units[metric] = unit
162
167
  if df.empty: # metric absent from every run (e.g. memory on timing-only files)
163
168
  continue
@@ -224,25 +229,36 @@ def compare_runs(
224
229
  stat: str | None = None,
225
230
  group_by: str | None = "fullname",
226
231
  sort: str = "name",
232
+ pivot: str | None = None,
227
233
  csv: Path | None = None,
228
234
  out: TextIO | None = None,
229
235
  ) -> None:
230
- """Print a per-(benchmark, run) comparison table across two or more runs.
231
-
232
- Mirrors pytest-benchmark's table model: rows are one per ``(benchmark × run)``,
233
- ``columns`` selects which metric columns appear (``time`` / ``peak`` / ``allocated``
234
- / ``allocations``), each cell carries the value plus a relative ``(N.NN)`` multiplier
235
- vs the group's best (best green, worst red), and ``group_by`` splits the rows into
236
- sub-tables (``fullname`` / ``name`` / ``func`` / ``group`` / ``module`` / ``class`` /
237
- ``param:NAME``, comma-composable). With no ``group_by`` the whole comparison is one
238
- table.
236
+ """Print a per-(benchmark, series) table for one or more runs.
237
+
238
+ Mirrors pytest-benchmark's table model: rows are one per ``(benchmark × series)``, where
239
+ the **series axis** is the run-file by default (one column-group per run). With two or more
240
+ series each cell carries a relative ``(N.NN)`` multiplier vs the group's best (best green,
241
+ worst red); a single series is a plain readout with neither. ``columns`` selects which metric
242
+ columns appear (``time`` / ``peak`` / ``allocated`` / ``allocations``). ``group_by`` splits
243
+ the rows into sub-tables (``fullname`` / ``name`` / ``func`` / ``group`` / ``module`` /
244
+ ``class`` / ``param:NAME``, comma-composable); with no ``group_by`` the whole comparison is
245
+ one table.
246
+
247
+ ``pivot`` re-points the series axis from the run-file onto a data dim (``param:NAME`` or a
248
+ bare ``extra_info`` name), folding a *single* combined run along that dim: rows that differ
249
+ only in it pair up and its values become the compared series — the A/B a run-file pair gives
250
+ you today, without splitting the data into N files. Distinct from ``group_by`` (which only
251
+ clusters rows into sub-tables, leaving the run-files as the compared series): ``pivot`` sets
252
+ *what* is compared, ``group_by`` sets how rows cluster. It composes with ``columns`` /
253
+ ``stat`` / ``sort`` / ``group_by`` unchanged and is mutually exclusive with multiple runs
254
+ (one series axis per table).
239
255
 
240
256
  Every column is a ``metric × stat`` pair. ``columns`` is a comma list of metric names
241
257
  (default ``time,peak``); ``stat`` is one of ``min`` / ``max`` / ``mean`` / ``median`` /
242
258
  ``stddev`` or ``all`` (the default), which expands each metric into its full stat
243
259
  spread — so no single statistic is privileged. A metric absent from every run is
244
260
  dropped rather than shown all dashes (so timing-only runs collapse to just ``time``).
245
- ``sort`` orders rows within a group: ``name``, ``value`` (largest in the last run), or
261
+ ``sort`` orders rows within a group: ``name``, ``value`` (largest in the last series), or
246
262
  ``change`` (biggest growth first). ``csv`` also writes the raw (unscaled) comparison.
247
263
  """
248
264
  from rich import box
@@ -253,12 +269,17 @@ def compare_runs(
253
269
  if sort not in _SORTS:
254
270
  raise ValueError(f"unknown --sort {sort!r}; use one of {', '.join(_SORTS)}")
255
271
  metrics, stats = _resolve_columns(columns), _resolve_stats(stat)
256
- labels, values, units, dims = _load_columns(runs, metrics, stats)
257
- if len(labels) < 2: # noqa: PLR2004 — a comparison needs two sides
272
+ labels, values, units, dims = _load_columns(runs, metrics, stats, pivot)
273
+ if not labels:
274
+ raise ValueError("no benchmarks found in the given run(s)")
275
+ if len(labels) < 2 and len(runs) > 1: # noqa: PLR2004 — distinct paths collapsed to one series
258
276
  raise ValueError(
259
277
  f"compare needs at least two distinct runs; got {labels!r} "
260
278
  f"(the same file more than once?). Pass distinct files."
261
279
  )
280
+ # One run is a plain readout: with no other run to rank against, the (N.NN) multiplier and
281
+ # the best/worst colour are meaningless (best == worst == the value), so they're dropped.
282
+ single = len(labels) == 1
262
283
  with_data = {col_id for col_id, _i, _lab in values}
263
284
  all_cols = [(m, s) for m in metrics for s in stats]
264
285
  cols = [(m, s) for m, s in all_cols if f"{m}:{s}" in with_data] or all_cols # drop empties
@@ -338,7 +359,9 @@ def compare_runs(
338
359
  if unit == ""
339
360
  else f"{v:.4g}"
340
361
  )
341
- cells.append(Text(body + _mult(v, best), style=rank_style(v, best, worst) or ""))
362
+ mult = "" if single else _mult(v, best)
363
+ style = "" if single else (rank_style(v, best, worst) or "")
364
+ cells.append(Text(body + mult, style=style))
342
365
  table.add_row(*cells)
343
366
  console.print(table)
344
367
  console.print() # one blank line between sub-tables
@@ -465,6 +488,39 @@ def find_regressions(a: str | Path, b: str | Path, thresholds: list[Threshold])
465
488
  return regressions
466
489
 
467
490
 
491
+ def find_pivot_regressions(
492
+ run: str | Path, pivot: str, thresholds: list[Threshold]
493
+ ) -> list[Regression]:
494
+ """Regressions along a ``--pivot`` axis within *one* run — the gate generalized the same way
495
+ the table is: with run-files the base is ``runs[0]`` and the head ``runs[-1]``; here the base
496
+ is the first value of the ``pivot`` dim and the head the last (mirroring oldest→newest). Each
497
+ fold-collapsed id is paired across them and flagged only on growth, reusing
498
+ :func:`_regressions_for`.
499
+ """
500
+ regressions: list[Regression] = []
501
+ for th in thresholds:
502
+ # ``pivot`` folds the run so ``snapshot`` carries the dim's values and ``id`` is the
503
+ # collapsed pairing key; the fail-on field doubles as the metric to read (headline scalar).
504
+ df, _unit = load_long_df(run, metric=cast("Metric", th.field), pivot=pivot)
505
+ labels = list(dict.fromkeys(df["snapshot"]))
506
+ if len(labels) < 2: # noqa: PLR2004 — a growth gate needs a base value and a head value
507
+ raise ValueError(
508
+ f"--fail-on with --pivot {pivot!r} needs two distinct values to gate; got "
509
+ f"{labels}. Does the run vary {pivot!r}?"
510
+ )
511
+ base_label, head_label = labels[0], labels[-1]
512
+ rows = zip(df["id"], df["value"], df["snapshot"], strict=True)
513
+ side: dict[str, dict[str, float]] = {base_label: {}, head_label: {}}
514
+ for test_id, value, label in rows:
515
+ if label in side:
516
+ side[label][test_id] = float(value)
517
+ base, head = side[base_label], side[head_label]
518
+ if th.field == "rss" and not (base.keys() & head.keys()):
519
+ raise ValueError(_RSS_GATE_EMPTY)
520
+ regressions.extend(_regressions_for(base, head, th))
521
+ return regressions
522
+
523
+
468
524
  #: Memory-blob key per fail-on field — the fields ``--benchmark-memory-compare-fail`` gates on.
469
525
  _BLOB_FIELD = {
470
526
  "peak": "peak_bytes",