pytest-benchmem 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. pytest_benchmem-0.1.0/.github/dependabot.yml +34 -0
  2. pytest_benchmem-0.1.0/.github/workflows/ci.yaml +53 -0
  3. pytest_benchmem-0.1.0/.github/workflows/dependabot-auto-merge.yaml +27 -0
  4. pytest_benchmem-0.1.0/.github/workflows/docs.yaml +28 -0
  5. pytest_benchmem-0.1.0/.github/workflows/pr-title.yaml +32 -0
  6. pytest_benchmem-0.1.0/.github/workflows/release.yaml +70 -0
  7. pytest_benchmem-0.1.0/.gitignore +23 -0
  8. pytest_benchmem-0.1.0/.pre-commit-config.yaml +27 -0
  9. pytest_benchmem-0.1.0/.release-please-config.json +15 -0
  10. pytest_benchmem-0.1.0/.release-please-manifest.json +3 -0
  11. pytest_benchmem-0.1.0/CHANGELOG.md +54 -0
  12. pytest_benchmem-0.1.0/LICENSE +21 -0
  13. pytest_benchmem-0.1.0/PKG-INFO +152 -0
  14. pytest_benchmem-0.1.0/README.md +126 -0
  15. pytest_benchmem-0.1.0/docs/index.md +51 -0
  16. pytest_benchmem-0.1.0/docs/walkthrough.md +225 -0
  17. pytest_benchmem-0.1.0/mkdocs.yml +41 -0
  18. pytest_benchmem-0.1.0/pyproject.toml +77 -0
  19. pytest_benchmem-0.1.0/src/pytest_benchmem/__init__.py +38 -0
  20. pytest_benchmem-0.1.0/src/pytest_benchmem/cli.py +104 -0
  21. pytest_benchmem-0.1.0/src/pytest_benchmem/compare.py +38 -0
  22. pytest_benchmem-0.1.0/src/pytest_benchmem/memray.py +59 -0
  23. pytest_benchmem-0.1.0/src/pytest_benchmem/plotting.py +352 -0
  24. pytest_benchmem-0.1.0/src/pytest_benchmem/py.typed +0 -0
  25. pytest_benchmem-0.1.0/src/pytest_benchmem/pytest_plugin.py +263 -0
  26. pytest_benchmem-0.1.0/src/pytest_benchmem/snapshot.py +141 -0
  27. pytest_benchmem-0.1.0/src/pytest_benchmem/sweep.py +147 -0
  28. pytest_benchmem-0.1.0/tests/conftest.py +1 -0
  29. pytest_benchmem-0.1.0/tests/test_memray.py +24 -0
  30. pytest_benchmem-0.1.0/tests/test_plugin.py +106 -0
  31. pytest_benchmem-0.1.0/tests/test_snapshot.py +102 -0
@@ -0,0 +1,34 @@
1
+ version: 2
2
+
3
+ updates:
4
+ - package-ecosystem: "pip"
5
+ directory: "/"
6
+ schedule:
7
+ interval: "weekly"
8
+ labels:
9
+ - "dependencies"
10
+ commit-message:
11
+ prefix: "chore"
12
+ groups:
13
+ linting:
14
+ patterns:
15
+ - "ruff"
16
+ - "mypy"
17
+ - "pre-commit"
18
+ testing:
19
+ patterns:
20
+ - "pytest*"
21
+ documentation:
22
+ patterns:
23
+ - "mkdocs*"
24
+ - "jupytext"
25
+ - "ipykernel"
26
+
27
+ - package-ecosystem: "github-actions"
28
+ directory: "/"
29
+ schedule:
30
+ interval: "weekly"
31
+ labels:
32
+ - "dependencies"
33
+ commit-message:
34
+ prefix: "ci"
@@ -0,0 +1,53 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main, master]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ concurrency:
10
+ group: ${{ github.workflow }}-${{ github.ref }}
11
+ cancel-in-progress: true
12
+
13
+ jobs:
14
+ lint:
15
+ name: Lint
16
+ runs-on: ubuntu-24.04
17
+ timeout-minutes: 5
18
+ steps:
19
+ - uses: actions/checkout@v6
20
+ - uses: astral-sh/ruff-action@v3
21
+ - run: ruff check --output-format=github
22
+ - run: ruff format --diff
23
+
24
+ typecheck:
25
+ name: Type check
26
+ runs-on: ubuntu-24.04
27
+ timeout-minutes: 5
28
+ steps:
29
+ - uses: actions/checkout@v6
30
+ - uses: astral-sh/setup-uv@v7
31
+ with:
32
+ enable-cache: true
33
+ - run: uv sync --group dev
34
+ - run: uv run mypy src/pytest_benchmem
35
+
36
+ test:
37
+ name: Test (Python ${{ matrix.python-version }})
38
+ runs-on: ubuntu-24.04
39
+ timeout-minutes: 10
40
+ strategy:
41
+ fail-fast: false
42
+ matrix:
43
+ python-version: ["3.11", "3.12", "3.13"]
44
+ steps:
45
+ - uses: actions/checkout@v6
46
+ - uses: astral-sh/setup-uv@v7
47
+ with:
48
+ enable-cache: true
49
+ - uses: actions/setup-python@v6
50
+ with:
51
+ python-version: ${{ matrix.python-version }}
52
+ - run: uv sync --group dev
53
+ - run: uv run pytest -q
@@ -0,0 +1,27 @@
1
+ name: Dependabot auto-merge
2
+
3
+ on:
4
+ pull_request:
5
+
6
+ permissions:
7
+ contents: write
8
+ pull-requests: write
9
+
10
+ jobs:
11
+ auto-merge:
12
+ name: Auto-merge patch
13
+ if: github.actor == 'dependabot[bot]'
14
+ runs-on: ubuntu-24.04
15
+ steps:
16
+ - uses: dependabot/fetch-metadata@v3
17
+ id: metadata
18
+
19
+ # No GitHub App / secrets: just enable auto-merge with the default token.
20
+ # The PR merges once required checks pass (no separate approval step,
21
+ # which the default GITHUB_TOKEN can't give a bot PR anyway).
22
+ - name: Enable auto-merge for patch updates
23
+ if: steps.metadata.outputs.update-type == 'version-update:semver-patch'
24
+ run: gh pr merge "$PR" --auto --squash
25
+ env:
26
+ PR: ${{ github.event.pull_request.html_url }}
27
+ GH_TOKEN: ${{ github.token }}
@@ -0,0 +1,28 @@
1
+ name: Docs
2
+
3
+ on:
4
+ push:
5
+ branches: [main, master]
6
+ workflow_dispatch:
7
+
8
+ permissions:
9
+ contents: write
10
+
11
+ concurrency:
12
+ group: docs-${{ github.ref }}
13
+ cancel-in-progress: true
14
+
15
+ jobs:
16
+ deploy:
17
+ name: Build & deploy
18
+ runs-on: ubuntu-24.04
19
+ timeout-minutes: 10
20
+ steps:
21
+ - uses: actions/checkout@v6
22
+ - uses: astral-sh/setup-uv@v7
23
+ with:
24
+ enable-cache: true
25
+ # Build the notebook from its jupytext source, then build+execute the site.
26
+ - run: uv sync --group docs
27
+ - run: uv run jupytext --to ipynb docs/walkthrough.md
28
+ - run: uv run mkdocs gh-deploy --force
@@ -0,0 +1,32 @@
1
+ name: PR Title
2
+
3
+ on:
4
+ push:
5
+ branches: ["release-please--**"]
6
+ pull_request:
7
+ types: [opened, edited, synchronize, reopened]
8
+
9
+ jobs:
10
+ validate:
11
+ name: Validate conventional commit format
12
+ if: github.event_name == 'pull_request'
13
+ runs-on: ubuntu-24.04
14
+ permissions:
15
+ pull-requests: read
16
+ steps:
17
+ - uses: amannn/action-semantic-pull-request@v6
18
+ with:
19
+ types: |
20
+ feat
21
+ fix
22
+ refactor
23
+ perf
24
+ docs
25
+ test
26
+ build
27
+ ci
28
+ chore
29
+ revert
30
+ style
31
+ env:
32
+ GITHUB_TOKEN: ${{ github.token }}
@@ -0,0 +1,70 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ workflow_dispatch:
7
+ inputs:
8
+ tag:
9
+ description: "Existing tag to (re)publish to PyPI, e.g. v0.0.1"
10
+ required: true
11
+
12
+ permissions:
13
+ contents: write
14
+ pull-requests: write
15
+
16
+ jobs:
17
+ release-please:
18
+ name: Release Please
19
+ if: github.event_name == 'push'
20
+ runs-on: ubuntu-24.04
21
+ outputs:
22
+ release_created: ${{ steps.release.outputs.release_created }}
23
+ tag_name: ${{ steps.release.outputs.tag_name }}
24
+ steps:
25
+ # Default GITHUB_TOKEN — no GitHub App / secrets. Publishing is a
26
+ # downstream job in this same run, so the "GITHUB_TOKEN won't trigger
27
+ # another workflow" limitation never applies.
28
+ - uses: googleapis/release-please-action@v5
29
+ id: release
30
+ with:
31
+ config-file: .release-please-config.json
32
+ manifest-file: .release-please-manifest.json
33
+
34
+ publish:
35
+ name: Build & publish to PyPI
36
+ needs: release-please
37
+ # Auto: when release-please cuts a release. Manual: workflow_dispatch with a
38
+ # tag (always() lets this run even though release-please is skipped on dispatch).
39
+ if: |
40
+ always() &&
41
+ (needs.release-please.outputs.release_created == 'true' || github.event_name == 'workflow_dispatch')
42
+ runs-on: ubuntu-24.04
43
+ timeout-minutes: 10
44
+ permissions:
45
+ contents: read
46
+ id-token: write
47
+ attestations: write
48
+ environment:
49
+ name: pypi
50
+ url: https://pypi.org/project/pytest-benchmem
51
+ steps:
52
+ - uses: actions/checkout@v6
53
+ with:
54
+ ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || needs.release-please.outputs.tag_name }}
55
+ fetch-depth: 0
56
+
57
+ - uses: astral-sh/setup-uv@v7
58
+ with:
59
+ enable-cache: true
60
+
61
+ - uses: actions/setup-python@v6
62
+ with:
63
+ python-version: "3.12"
64
+
65
+ - name: Build
66
+ run: uv build
67
+
68
+ # OIDC trusted publishing — no API token. The PyPI publisher must point at
69
+ # this workflow file (release.yaml) and the 'pypi' environment.
70
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,23 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .benchmarks/
8
+ .mypy_cache/
9
+ .ruff_cache/
10
+ .pytest_cache/
11
+ *.bin
12
+
13
+ # lockfile not tracked for a library — resolve fresh against current deps
14
+ uv.lock
15
+
16
+ # jupytext-built notebooks (source is the .md) + mkdocs build
17
+ docs/*.ipynb
18
+ site/
19
+ .cache/
20
+
21
+ # editors
22
+ .idea/
23
+ .vscode/
@@ -0,0 +1,27 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v6.0.0
4
+ hooks:
5
+ - id: trailing-whitespace
6
+ - id: end-of-file-fixer
7
+ - id: check-yaml
8
+ - id: check-added-large-files
9
+ - id: check-merge-conflict
10
+ - id: debug-statements
11
+
12
+ - repo: https://github.com/astral-sh/ruff-pre-commit
13
+ rev: v0.15.9
14
+ hooks:
15
+ - id: ruff
16
+ args: [--fix]
17
+ - id: ruff-format
18
+
19
+ - repo: local
20
+ hooks:
21
+ - id: mypy
22
+ name: mypy
23
+ entry: uv run mypy src/pytest_benchmem
24
+ language: system
25
+ pass_filenames: false
26
+ files: ^src/
27
+ types: [python]
@@ -0,0 +1,15 @@
1
+ {
2
+ "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json",
3
+ "bootstrap-sha": "384d876391b7d0c496ee01f34f25731f4c8487f9",
4
+ "include-component-in-tag": false,
5
+ "packages": {
6
+ ".": {
7
+ "release-type": "simple",
8
+ "package-name": "pytest-benchmem",
9
+ "bump-minor-for-major-pre-major": true,
10
+ "bump-patch-for-minor-pre-major": true,
11
+ "changelog-path": "CHANGELOG.md",
12
+ "initial-version": "0.0.1"
13
+ }
14
+ }
15
+ }
@@ -0,0 +1,3 @@
1
+ {
2
+ ".": "0.1.0"
3
+ }
@@ -0,0 +1,54 @@
1
+ # Changelog
2
+
3
+ ## [0.1.0](https://github.com/fluxopt/pytest-benchmem/compare/v0.1.0...v0.1.0) (2026-06-13)
4
+
5
+
6
+ ### ⚠ BREAKING CHANGES
7
+
8
+ * rename package peakbench → pytest-benchmem ([#10](https://github.com/fluxopt/pytest-benchmem/issues/10))
9
+ * pivot to the memory companion to pytest-benchmark ([#7](https://github.com/fluxopt/pytest-benchmem/issues/7))
10
+
11
+ ### Features
12
+
13
+ * --benchmark-memory flag to augment existing benchmark() calls ([#16](https://github.com/fluxopt/pytest-benchmem/issues/16)) ([aff17c7](https://github.com/fluxopt/pytest-benchmem/commit/aff17c76ad742cedc963e7914bd575846ddc6035))
14
+ * pivot to the memory companion to pytest-benchmark ([#7](https://github.com/fluxopt/pytest-benchmem/issues/7)) ([52f2b6b](https://github.com/fluxopt/pytest-benchmem/commit/52f2b6b56af1ee61f84fcd0ab9dc0947004fd31d))
15
+ * resolve [#2](https://github.com/fluxopt/pytest-benchmem/issues/2) — drop select=, bless filter-before-convert ([89fbfcd](https://github.com/fluxopt/pytest-benchmem/commit/89fbfcdde439eb6f1535df8e65d2295b2ea69add))
16
+
17
+
18
+ ### Miscellaneous Chores
19
+
20
+ * release pytest-benchmem as 0.1.0 ([#13](https://github.com/fluxopt/pytest-benchmem/issues/13)) ([8e51760](https://github.com/fluxopt/pytest-benchmem/commit/8e51760d7bdfe4e44ea77856e8a0c4e158de0099))
21
+
22
+
23
+ ### Code Refactoring
24
+
25
+ * rename package peakbench → pytest-benchmem ([#10](https://github.com/fluxopt/pytest-benchmem/issues/10)) ([1e48259](https://github.com/fluxopt/pytest-benchmem/commit/1e482590bef2767df1619bcae15a6d132f4cc610))
26
+
27
+ ## [0.1.0](https://github.com/fluxopt/pytest-benchmem/compare/v0.0.1...v0.1.0) (2026-06-13)
28
+
29
+
30
+ ### ⚠ BREAKING CHANGES
31
+
32
+ * rename package peakbench → pytest-benchmem ([#10](https://github.com/fluxopt/pytest-benchmem/issues/10))
33
+ * pivot to the memory companion to pytest-benchmark ([#7](https://github.com/fluxopt/pytest-benchmem/issues/7))
34
+
35
+ ### Features
36
+
37
+ * pivot to the memory companion to pytest-benchmark ([#7](https://github.com/fluxopt/pytest-benchmem/issues/7)) ([52f2b6b](https://github.com/fluxopt/pytest-benchmem/commit/52f2b6b56af1ee61f84fcd0ab9dc0947004fd31d))
38
+
39
+
40
+ ### Miscellaneous Chores
41
+
42
+ * release pytest-benchmem as 0.1.0 ([#13](https://github.com/fluxopt/pytest-benchmem/issues/13)) ([8e51760](https://github.com/fluxopt/pytest-benchmem/commit/8e51760d7bdfe4e44ea77856e8a0c4e158de0099))
43
+
44
+
45
+ ### Code Refactoring
46
+
47
+ * rename package peakbench → pytest-benchmem ([#10](https://github.com/fluxopt/pytest-benchmem/issues/10)) ([1e48259](https://github.com/fluxopt/pytest-benchmem/commit/1e482590bef2767df1619bcae15a6d132f4cc610))
48
+
49
+ ## 0.0.1 (2026-06-13)
50
+
51
+
52
+ ### Features
53
+
54
+ * resolve [#2](https://github.com/fluxopt/pytest-benchmem/issues/2) — drop select=, bless filter-before-convert ([89fbfcd](https://github.com/fluxopt/pytest-benchmem/commit/89fbfcdde439eb6f1535df8e65d2295b2ea69add))
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Felix Bumann
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,152 @@
1
+ Metadata-Version: 2.4
2
+ Name: pytest-benchmem
3
+ Version: 0.1.0
4
+ Summary: The memory companion to pytest-benchmark: a memray peak-memory pass on the same test, plus dims-aware plots and cross-version sweeps.
5
+ Project-URL: Homepage, https://github.com/fluxopt/pytest-benchmem
6
+ Project-URL: Documentation, https://fluxopt.github.io/pytest-benchmem/
7
+ Project-URL: Repository, https://github.com/fluxopt/pytest-benchmem
8
+ Project-URL: Issues, https://github.com/fluxopt/pytest-benchmem/issues
9
+ Author: Felix Bumann
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: benchmark,memory,memray,performance,pytest,pytest-benchmark
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Framework :: Pytest
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Topic :: Software Development :: Testing
17
+ Classifier: Typing :: Typed
18
+ Requires-Python: >=3.11
19
+ Requires-Dist: memray>=1.11; platform_system == 'Linux' or platform_system == 'Darwin'
20
+ Requires-Dist: pytest-benchmark<6,>=4
21
+ Provides-Extra: plot
22
+ Requires-Dist: pandas; extra == 'plot'
23
+ Requires-Dist: plotly>=5; extra == 'plot'
24
+ Requires-Dist: typer>=0.12; extra == 'plot'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # pytest-benchmem
28
+
29
+ [![CI](https://github.com/fluxopt/pytest-benchmem/actions/workflows/ci.yaml/badge.svg)](https://github.com/fluxopt/pytest-benchmem/actions/workflows/ci.yaml)
30
+ ![Python](https://img.shields.io/badge/python-3.11%2B-blue)
31
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
32
+
33
+ **The memory companion to [pytest-benchmark].** It times your code; pytest-benchmem
34
+ adds a memray **peak-memory** pass to the *same test, in the same run* — one node
35
+ id, one JSON file, both metrics. Plus dims-aware plots and cross-version sweeps.
36
+
37
+ ## Quickstart
38
+
39
+ Write a normal pytest-benchmark test; swap `benchmark` for `benchmark_memory`:
40
+
41
+ ```python
42
+ import pytest
43
+
44
+
45
+ @pytest.mark.parametrize("n", [10_000, 100_000, 1_000_000])
46
+ def test_sort(benchmark_memory, n):
47
+ data = list(range(n, 0, -1))
48
+ benchmark_memory(sorted, data)
49
+ ```
50
+
51
+ ```bash
52
+ pytest --benchmark-only --benchmark-json=run.json
53
+ ```
54
+
55
+ One run, one `run.json`, for each benchmark id — both metrics, one node id:
56
+
57
+ - `stats: {min, mean, median, …}` — **timing**, from pytest-benchmark.
58
+ - `extra_info: {"peak_mib": 3.81}` — **peak memory**, from pytest-benchmem.
59
+
60
+ The two passes never overlap: pytest-benchmark times the action untracked, then
61
+ memray measures peak on a *separate, untimed* call — so the allocator hooks cost
62
+ the timing nothing. The parametrize `params` become the analysis dims the plots
63
+ scale by.
64
+
65
+ ## Already have a pytest-benchmark suite?
66
+
67
+ Don't rewrite a thing — add `--benchmark-memory` and every `benchmark(...)` call
68
+ also records peak memory:
69
+
70
+ ```python
71
+ def test_sort(benchmark): # unchanged
72
+ benchmark(sorted, list(range(1_000_000, 0, -1)))
73
+ ```
74
+
75
+ ```bash
76
+ pytest --benchmark-only --benchmark-memory # timing + peak_mib for the whole suite
77
+ ```
78
+
79
+ It's opt-in at the run level: without the flag, plain `benchmark` tests are
80
+ untouched. (Reach for the `benchmark_memory` fixture when you want memory on
81
+ specific tests only, or `pedantic` control.)
82
+
83
+ ## Reading it back
84
+
85
+ Timing rides pytest-benchmark's own tooling (`pytest-benchmark compare`,
86
+ `--benchmark-histogram`) — pytest-benchmem doesn't reimplement it. For **memory**,
87
+ and dims-aware views over either metric:
88
+
89
+ ```bash
90
+ benchmem compare base.json head.json --metric memory # per-id delta table
91
+ benchmem plot base.json head.json --metric memory # interactive plotly view
92
+ ```
93
+
94
+ ```
95
+ id base.json head.json change (MiB)
96
+ --------------------------------------------------------------------
97
+ test_sort[10000] 0.076 0.078 +2.6%
98
+ test_sort[100000] 0.76 0.74 -2.6%
99
+ test_sort[1000000] 7.63 9.155 +20.0%
100
+ ```
101
+
102
+ Or pull the numbers into your own analysis:
103
+
104
+ ```python
105
+ from pytest_benchmem import from_pytest_benchmark, memory_from_pytest_benchmark
106
+
107
+ _, timing, _ = from_pytest_benchmark("run.json") # seconds, from stats
108
+ _, memory, _ = memory_from_pytest_benchmark("run.json") # MiB, from extra_info
109
+ ```
110
+
111
+ Outside pytest, `measure_peak(lambda: build_model(1000))` is the bare memray
112
+ engine — a one-liner peak number for a REPL or notebook.
113
+
114
+ ## Where it sits
115
+
116
+ Its reason to exist is the gap nothing else fills cleanly: **memray-precision
117
+ memory benchmarking** of your own code, right where you already benchmark. ASV's
118
+ `peakmem` is coarse RSS sampling that misses numpy/C-allocation detail; CodSpeed
119
+ covers CI timing.
120
+
121
+ | Need | Reach for | pytest-benchmem |
122
+ |---|---|---|
123
+ | CI regression, per-PR dashboard | **CodSpeed** | — (don't rebuild it) |
124
+ | Local timing + A/B compare | **pytest-benchmark** | rides it (timing is its job) |
125
+ | Rigorous perf history across commits | **ASV** | — (heavier, RSS memory) |
126
+ | **Precise local peak memory (numpy/C allocs)** | **memray** | ⭐ the core |
127
+ | Memory *in your pytest-benchmark tests* | — | ⭐ fixture **or** `--benchmark-memory` |
128
+ | Same runs across installed versions | — | ⭐ `sweep` |
129
+
130
+ > **Not** a CI dashboard (use [CodSpeed]) and **not** a rigorous perf-history
131
+ > system (use [ASV]). If your core need is *precise local memory* over the
132
+ > benchmarks you already write — timing/sweeps/plots in one vocabulary — that's
133
+ > pytest-benchmem.
134
+
135
+ ## Install
136
+
137
+ ```bash
138
+ uv add pytest-benchmem # the fixture + flag + memray engine
139
+ uv add "pytest-benchmem[plot]" # + the plot/compare CLI (pandas, plotly, typer)
140
+ ```
141
+
142
+ pytest-benchmark and memray are core deps; memray is Linux/macOS only, so Windows
143
+ installs cleanly with timing-only (the memory pass raises a clear error there).
144
+
145
+ ## Status
146
+
147
+ Early. Extracted from the linopy internal benchmark suite, where it's the local
148
+ memory-profiling layer. API may move before 1.0.
149
+
150
+ [pytest-benchmark]: https://pytest-benchmark.readthedocs.io
151
+ [CodSpeed]: https://codspeed.io
152
+ [ASV]: https://asv.readthedocs.io
@@ -0,0 +1,126 @@
1
+ # pytest-benchmem
2
+
3
+ [![CI](https://github.com/fluxopt/pytest-benchmem/actions/workflows/ci.yaml/badge.svg)](https://github.com/fluxopt/pytest-benchmem/actions/workflows/ci.yaml)
4
+ ![Python](https://img.shields.io/badge/python-3.11%2B-blue)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
6
+
7
+ **The memory companion to [pytest-benchmark].** It times your code; pytest-benchmem
8
+ adds a memray **peak-memory** pass to the *same test, in the same run* — one node
9
+ id, one JSON file, both metrics. Plus dims-aware plots and cross-version sweeps.
10
+
11
+ ## Quickstart
12
+
13
+ Write a normal pytest-benchmark test; swap `benchmark` for `benchmark_memory`:
14
+
15
+ ```python
16
+ import pytest
17
+
18
+
19
+ @pytest.mark.parametrize("n", [10_000, 100_000, 1_000_000])
20
+ def test_sort(benchmark_memory, n):
21
+ data = list(range(n, 0, -1))
22
+ benchmark_memory(sorted, data)
23
+ ```
24
+
25
+ ```bash
26
+ pytest --benchmark-only --benchmark-json=run.json
27
+ ```
28
+
29
+ One run, one `run.json`, for each benchmark id — both metrics, one node id:
30
+
31
+ - `stats: {min, mean, median, …}` — **timing**, from pytest-benchmark.
32
+ - `extra_info: {"peak_mib": 3.81}` — **peak memory**, from pytest-benchmem.
33
+
34
+ The two passes never overlap: pytest-benchmark times the action untracked, then
35
+ memray measures peak on a *separate, untimed* call — so the allocator hooks cost
36
+ the timing nothing. The parametrize `params` become the analysis dims the plots
37
+ scale by.
38
+
39
+ ## Already have a pytest-benchmark suite?
40
+
41
+ Don't rewrite a thing — add `--benchmark-memory` and every `benchmark(...)` call
42
+ also records peak memory:
43
+
44
+ ```python
45
+ def test_sort(benchmark): # unchanged
46
+ benchmark(sorted, list(range(1_000_000, 0, -1)))
47
+ ```
48
+
49
+ ```bash
50
+ pytest --benchmark-only --benchmark-memory # timing + peak_mib for the whole suite
51
+ ```
52
+
53
+ It's opt-in at the run level: without the flag, plain `benchmark` tests are
54
+ untouched. (Reach for the `benchmark_memory` fixture when you want memory on
55
+ specific tests only, or `pedantic` control.)
56
+
57
+ ## Reading it back
58
+
59
+ Timing rides pytest-benchmark's own tooling (`pytest-benchmark compare`,
60
+ `--benchmark-histogram`) — pytest-benchmem doesn't reimplement it. For **memory**,
61
+ and dims-aware views over either metric:
62
+
63
+ ```bash
64
+ benchmem compare base.json head.json --metric memory # per-id delta table
65
+ benchmem plot base.json head.json --metric memory # interactive plotly view
66
+ ```
67
+
68
+ ```
69
+ id base.json head.json change (MiB)
70
+ --------------------------------------------------------------------
71
+ test_sort[10000] 0.076 0.078 +2.6%
72
+ test_sort[100000] 0.76 0.74 -2.6%
73
+ test_sort[1000000] 7.63 9.155 +20.0%
74
+ ```
75
+
76
+ Or pull the numbers into your own analysis:
77
+
78
+ ```python
79
+ from pytest_benchmem import from_pytest_benchmark, memory_from_pytest_benchmark
80
+
81
+ _, timing, _ = from_pytest_benchmark("run.json") # seconds, from stats
82
+ _, memory, _ = memory_from_pytest_benchmark("run.json") # MiB, from extra_info
83
+ ```
84
+
85
+ Outside pytest, `measure_peak(lambda: build_model(1000))` is the bare memray
86
+ engine — a one-liner peak number for a REPL or notebook.
87
+
88
+ ## Where it sits
89
+
90
+ Its reason to exist is the gap nothing else fills cleanly: **memray-precision
91
+ memory benchmarking** of your own code, right where you already benchmark. ASV's
92
+ `peakmem` is coarse RSS sampling that misses numpy/C-allocation detail; CodSpeed
93
+ covers CI timing.
94
+
95
+ | Need | Reach for | pytest-benchmem |
96
+ |---|---|---|
97
+ | CI regression, per-PR dashboard | **CodSpeed** | — (don't rebuild it) |
98
+ | Local timing + A/B compare | **pytest-benchmark** | rides it (timing is its job) |
99
+ | Rigorous perf history across commits | **ASV** | — (heavier, RSS memory) |
100
+ | **Precise local peak memory (numpy/C allocs)** | **memray** | ⭐ the core |
101
+ | Memory *in your pytest-benchmark tests* | — | ⭐ fixture **or** `--benchmark-memory` |
102
+ | Same runs across installed versions | — | ⭐ `sweep` |
103
+
104
+ > **Not** a CI dashboard (use [CodSpeed]) and **not** a rigorous perf-history
105
+ > system (use [ASV]). If your core need is *precise local memory* over the
106
+ > benchmarks you already write — timing/sweeps/plots in one vocabulary — that's
107
+ > pytest-benchmem.
108
+
109
+ ## Install
110
+
111
+ ```bash
112
+ uv add pytest-benchmem # the fixture + flag + memray engine
113
+ uv add "pytest-benchmem[plot]" # + the plot/compare CLI (pandas, plotly, typer)
114
+ ```
115
+
116
+ pytest-benchmark and memray are core deps; memray is Linux/macOS only, so Windows
117
+ installs cleanly with timing-only (the memory pass raises a clear error there).
118
+
119
+ ## Status
120
+
121
+ Early. Extracted from the linopy internal benchmark suite, where it's the local
122
+ memory-profiling layer. API may move before 1.0.
123
+
124
+ [pytest-benchmark]: https://pytest-benchmark.readthedocs.io
125
+ [CodSpeed]: https://codspeed.io
126
+ [ASV]: https://asv.readthedocs.io