pytest-benchmem 0.1.0__py3-none-any.whl

pytest_benchmem/__init__.py

@@ -0,0 +1,38 @@
+"""pytest-benchmem — the memory companion to pytest-benchmark.
+
+pytest-benchmark times your code; pytest-benchmem adds a memray peak-memory pass to the
+same test via the ``benchmark_memory`` fixture, stored in the same run, plus
+dims-aware plots and cross-version sweeps it has no answer for.
+
+Light to import: this re-exports only the engine (``measure_peak``) and the
+readers/loader that turn pytest-benchmark JSON into a tidy frame. ``pytest_benchmem.plotting``
+pulls numpy/plotly and ``pytest_benchmem.sweep`` shells out to ``uv`` — import those
+submodules directly when needed. The ``benchmark_memory`` fixture is delivered by
+the pytest plugin (entry point), not by import.
+"""
+
+from __future__ import annotations
+
+from pytest_benchmem.memray import measure_peak
+from pytest_benchmem.snapshot import (
+    DimValue,
+    Metric,
+    Sample,
+    discover_runs,
+    from_pytest_benchmark,
+    load_long_df,
+    load_samples,
+    memory_from_pytest_benchmark,
+)
+
+__all__ = [
+    "DimValue",
+    "Metric",
+    "Sample",
+    "discover_runs",
+    "from_pytest_benchmark",
+    "load_long_df",
+    "load_samples",
+    "measure_peak",
+    "memory_from_pytest_benchmark",
+]

pytest_benchmem/cli.py

@@ -0,0 +1,104 @@
+"""pytest-benchmem CLI — ``plot`` and ``compare`` over pytest-benchmark JSON runs.
+
+Both commands read the JSON pytest-benchmark writes (``.benchmarks/…``) and pick
+a ``--metric`` (``time`` from ``stats``, ``memory`` from ``extra_info.peak_mib``).
+Timing comparison/histograms are pytest-benchmark's own job; these commands are
+the memory-aware, dims-aware views on top.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from pytest_benchmem.snapshot import Metric, discover_runs
+
+app = typer.Typer(help="pytest-benchmem — plot and compare benchmark runs.", no_args_is_help=True)
+
+MetricOpt = Annotated[Metric, typer.Option(help="Which metric to read: time | memory.")]
+
+
+def _need_plotly() -> None:
+    if importlib.util.find_spec("plotly") is None:
+        typer.secho(
+            "plotting needs extras: pip install 'pytest-benchmem[plot]'",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(code=2)
+
+
+@app.command()
+def plot(
+    runs: Annotated[list[Path], typer.Argument(help="pytest-benchmark JSON file(s).")],
+    metric: MetricOpt = "time",
+    view: Annotated[
+        str | None,
+        typer.Option(help="compare | scatter | sweep | scaling (default: by count)."),
+    ] = None,
+    facet: Annotated[str | None, typer.Option(help="Dim to facet by.")] = None,
+    x: Annotated[str | None, typer.Option(help="scaling: dim for the x-axis.")] = None,
+    clip: Annotated[float | None, typer.Option(help="Clamp the colour scale.")] = None,
+    output: Annotated[Path | None, typer.Option("--output", "-o", help="HTML out.")] = None,
+    open_browser: Annotated[bool, typer.Option("--open/--no-open")] = False,
+) -> None:
+    """Render an interactive plotly view from one or more pytest-benchmark runs."""
+    missing = [p for p in runs if not p.exists()]
+    if missing:
+        typer.secho(f"missing: {[str(p) for p in missing]}", fg=typer.colors.RED, err=True)
+        found = discover_runs()
+        if found:
+            typer.echo("available:\n  " + "\n  ".join(str(p) for p in found), err=True)
+        raise typer.Exit(code=2)
+
+    chosen = view or ("scaling" if len(runs) == 1 else "scatter" if len(runs) == 2 else "sweep")
+    _need_plotly()
+    from pytest_benchmem import plotting
+
+    try:
+        if chosen == "compare":
+            fig, n = plotting.plot_compare(runs, metric=metric, facet=facet, clip=clip)
+        elif chosen == "scatter":
+            fig, n = plotting.plot_scatter(runs, metric=metric, facet=facet, clip=clip)
+        elif chosen == "sweep":
+            fig, n = plotting.plot_sweep(runs, metric=metric, clip=clip)
+        elif chosen == "scaling":
+            fig, n = plotting.plot_scaling(runs, metric=metric, x=x, facet=facet)
+        else:
+            typer.secho(f"unknown view {chosen!r}", fg=typer.colors.RED, err=True)
+            raise typer.Exit(code=2)
+    except ValueError as exc:
+        typer.secho(str(exc), fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1) from exc
+
+    output = output or Path(".benchmarks") / "plots" / f"{chosen}-{metric}.html"
+    output.parent.mkdir(parents=True, exist_ok=True)
+    fig.write_html(output)
+    typer.secho(f"{chosen} ({metric}): {n} ids → {output}", fg=typer.colors.GREEN)
+    if open_browser:
+        import webbrowser
+
+        webbrowser.open(output.resolve().as_uri())
+
+
+@app.command()
+def compare(
+    a: Annotated[Path, typer.Argument(help="Baseline run.")],
+    b: Annotated[Path, typer.Argument(help="Candidate run.")],
+    metric: MetricOpt = "time",
+) -> None:
+    """Print a per-id delta table for two pytest-benchmark runs."""
+    for p in (a, b):
+        if not p.exists():
+            typer.secho(f"missing: {p}", fg=typer.colors.RED, err=True)
+            raise typer.Exit(code=2)
+    from pytest_benchmem.compare import compare_runs
+
+    try:
+        compare_runs(a, b, metric=metric)
+    except ValueError as exc:
+        typer.secho(str(exc), fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1) from exc

pytest_benchmem/compare.py

@@ -0,0 +1,38 @@
+"""Text comparison of two pytest-benchmark runs — keyed on the benchmark id."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import TextIO
+
+from pytest_benchmem.snapshot import Metric, load_long_df
+
+
+def compare_runs(
+    a: str | Path, b: str | Path, *, metric: Metric = "time", out: TextIO | None = None
+) -> None:
+    """Print a per-id table of ``a`` vs ``b`` (for ``metric``) with percent change.
+
+    Reads the chosen metric (timing or memory) from two pytest-benchmark files.
+    Ids present in only one run show ``—``.
+    """
+    out = out or sys.stdout
+    df, unit = load_long_df([Path(a), Path(b)], metric=metric)
+    labels = df["snapshot"].drop_duplicates().tolist()
+    la, lb = labels[0], labels[1]
+    wide = df.pivot(index="id", columns="snapshot", values="value")
+
+    print(f"\n{'id':<70} {la:>12} {lb:>12} {'change':>10}  ({unit})", file=out)
+    print("-" * 108, file=out)
+    for test_id in sorted(wide.index):
+        va = wide.at[test_id, la] if la in wide.columns else None
+        vb = wide.at[test_id, lb] if lb in wide.columns else None
+        va = None if va is None or va != va else va  # noqa: PLR0124 — NaN check
+        vb = None if vb is None or vb != vb else vb
+        a_str = f"{va:.4g}" if va is not None else "—"
+        b_str = f"{vb:.4g}" if vb is not None else "—"
+        change = f"{(vb - va) / va * 100:+.1f}%" if va and vb and va > 0 else "—"
+        short = test_id.split("::")[-1]
+        print(f"{short:<70} {a_str:>12} {b_str:>12} {change:>10}", file=out)
+    print(file=out)

pytest_benchmem/memray.py

@@ -0,0 +1,59 @@
+"""Peak-memory measurement via ``memray`` — the core pytest-benchmem engine.
+
+:func:`measure_peak` runs one action under a ``memray.Tracker`` and returns the
+peak in MiB. It's what the ``benchmark_memory`` pytest fixture calls for its
+memory pass, and it doubles as a standalone one-liner for a REPL or notebook::
+
+    from pytest_benchmem import measure_peak
+
+    peak = measure_peak(lambda: build_model(1000))   # MiB
+
+Why memray and not RSS sampling: peak RSS (what ASV's ``peakmem`` and naive
+samplers use) misses the numpy/C-allocation detail that's usually the point of
+profiling a numeric library. memray tracks the allocator directly.
+"""
+
+from __future__ import annotations
+
+import gc
+import platform
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+#: A zero-arg callable doing the work that gets memory-tracked.
+Action = Callable[[], object]
+
+
+def _require_memray() -> None:
+    """Raise if memory measurement isn't supported — memray runs only on Linux/macOS."""
+    if platform.system() not in ("Linux", "Darwin"):
+        raise RuntimeError(
+            "memory measurement requires memray, which supports only Linux and macOS "
+            f"(this is {platform.system()}). Timing still works; run memory benchmarks "
+            "on Linux or macOS."
+        )
+
+
+def measure_peak(action: Action, repeats: int = 1) -> float:
+    """Run ``action()`` under ``memray.Tracker`` and return peak MiB.
+
+    ``repeats > 1`` runs it that many times in fresh trackers and returns the
+    *minimum* peak — peak memory is noisier than expected (GC timing, lazy
+    imports, page cache), so min-of-N is the cleanest "floor this can hit".
+    """
+    _require_memray()
+
+    import memray
+
+    peaks: list[float] = []
+    for _ in range(max(1, repeats)):
+        with tempfile.TemporaryDirectory(prefix="pytest-benchmem-") as tmp:
+            out = Path(tmp) / "track.bin"  # memray must create the file itself
+            with memray.Tracker(out):
+                action()
+            peak_bytes = memray.FileReader(out).metadata.peak_memory
+            peaks.append(round(peak_bytes / (1024**2), 3))
+        gc.collect()
+
+    return min(peaks)

pytest_benchmem/plotting.py

@@ -0,0 +1,352 @@
+"""Interactive plotly views over pytest-benchmark runs — dims-driven.
+
+Each view reads one ``metric`` (``"time"`` or ``"memory"``) out of the
+pytest-benchmark JSON files and returns ``(figure, n_rendered)``:
+
+- :func:`plot_compare` (2 runs) — bar chart of per-id delta.
+- :func:`plot_scatter` (2+) — baseline cost vs ratio; top-right = the regressed one.
+- :func:`plot_sweep` (3+) — heatmap of per-id fold-change (log2 ratio) vs the first.
+- :func:`plot_scaling` (1) — cost vs a numeric dim, faceted/coloured by others.
+
+The sweep/scatter/compare views key on the opaque ``id``; only ``scaling`` reads
+``dims`` — and which dim is x / colour / facet is auto-inferred from dtype
+(numeric → x) but overridable. plotly is imported lazily.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal
+
+import numpy as np
+
+from pytest_benchmem.snapshot import Metric, load_long_df
+
+if TYPE_CHECKING:
+    import numpy.typing as npt
+    import pandas as pd
+    from plotly.graph_objects import Figure
+
+SortMode = Literal["absolute", "relative"]
+
+
+def _value_label(unit: str) -> str:
+    return {"s": "time", "MiB": "peak"}.get(unit, "value")
+
+
+def _diverging_kwargs(midpoint: float = 0.0) -> dict[str, object]:
+    """green→white→red continuous colour scale centred on ``midpoint``."""
+    return {
+        "color_continuous_scale": ["green", "white", "red"],
+        "color_continuous_midpoint": midpoint,
+    }
+
+
+def _symmetric_clip(
+    magnitudes: npt.NDArray[np.float64], override: float | None, pct: float = 95.0
+) -> float:
+    """Symmetric colour bound: ``override`` if given, else the ``pct`` percentile
+    of ``|magnitudes|`` so outliers don't wash the rest to the midpoint.
+    """
+    if override is not None:
+        return float(override)
+    mags = np.abs(np.asarray(magnitudes, dtype=float))
+    mags = mags[np.isfinite(mags)]
+    if mags.size == 0:
+        return 1.0
+    bound = float(np.percentile(mags, pct))
+    return bound if bound > 0 else (float(mags.max()) or 1e-9)
+
+
+def _fold_label(fold: float) -> str:
+    """Format a fold-change for a colourbar tick: ``2×``, ``1/4×``, ``1.5×``."""
+    if abs(fold - 1.0) < 1e-9:
+        return "1×"
+    return f"{fold:.3g}×" if fold > 1.0 else f"1/{1.0 / fold:.3g}×"
+
+
+def _fold_ticks(bound: float) -> tuple[list[float], list[str]]:
+    """Colourbar ticks (log2 positions, fold labels) spanning ±bound — clean
+    powers of two for a wide range, round sub-2× folds for a tight one.
+    """
+    if bound >= 1.0:
+        pos = [2.0**t for t in range(1, int(bound) + 1)]
+    else:
+        pos = [f for f in (1.1, 1.25, 1.5) if float(np.log2(f)) <= bound + 1e-9]
+        if len(pos) < 2:
+            pos = [2.0 ** (bound / 2), 2.0**bound]
+    vals, text = [0.0], ["1×"]
+    for f in sorted(pos):
+        lv = float(np.log2(f))
+        vals = [-lv, *vals, lv]
+        text = [_fold_label(1.0 / f), *text, _fold_label(f)]
+    return vals, text
+
+
+def _axis_kwargs(unit: str) -> dict[str, object]:
+    if unit == "s":
+        return {"tickformat": ".2s", "ticksuffix": "s"}
+    return {"ticksuffix": f" {unit}"}
+
+
+def _dim_columns(df: pd.DataFrame) -> list[str]:
+    """The dim columns (everything but the fixed snapshot/id/value)."""
+    return [c for c in df.columns if c not in ("snapshot", "id", "value")]
+
+
+def plot_compare(
+    snapshots: list[Path],
+    *,
+    metric: Metric = "time",
+    sort: SortMode = "absolute",
+    facet: str | None = None,
+    clip: float | None = None,
+) -> tuple[Figure, int]:
+    """Bar chart of per-id delta, sorted by the chosen Δ (biggest regressions on top).
+
+    ``sort`` picks the bar dimension: ``absolute`` plots ``b - a`` in the native
+    unit, ``relative`` plots percent change. ``facet`` splits into subplots by any
+    dim. ``clip`` clamps the colour scale (default symmetric p95).
+    """
+    import sys
+
+    import plotly.express as px
+
+    df_long, unit = load_long_df(snapshots[:2], metric=metric)
+    vlabel = _value_label(unit)
+    labels = df_long["snapshot"].drop_duplicates().tolist()
+    a_label, b_label = labels[0], labels[1]
+
+    dims = _dim_columns(df_long)
+    wide = (
+        df_long.pivot(index=["id", *dims], columns="snapshot", values="value")
+        .reset_index()
+        .rename_axis(columns=None)
+    )
+    only_a = wide[wide[a_label].notna() & wide[b_label].isna()]
+    only_b = wide[wide[a_label].isna() & wide[b_label].notna()]
+    df = wide.dropna(subset=[a_label, b_label]).copy()
+    if df.empty:
+        raise ValueError("no ids in common between the two snapshots")
+    if len(only_a) or len(only_b):
+        print(
+            f"compare: {len(only_a)} only in {a_label}, {len(only_b)} only in "
+            f"{b_label} (intersection: {len(df)}).",
+            file=sys.stderr,
+        )
+
+    df["delta_abs"] = df[b_label] - df[a_label]
+    df["delta_pct"] = (df["delta_abs"] / df[a_label]) * 100.0
+    x_col = "delta_abs" if sort == "absolute" else "delta_pct"
+    df = df.sort_values(x_col).reset_index(drop=True)
+
+    x_label = f"{vlabel} delta ({unit})" if sort == "absolute" else f"{vlabel} delta %"
+    text_fmt = (".2s" if unit == "s" else ".2f") if sort == "absolute" else ".1f"
+    direction = "slower" if unit == "s" else "more memory"
+    title = f"{vlabel} delta ({sort}): {a_label} → {b_label} (positive = {direction})"
+
+    facet_kwargs: dict[str, object] = {}
+    if facet is not None and facet in df.columns:
+        facet_kwargs = {"facet_col": facet, "facet_col_wrap": 3}
+
+    color_clip = _symmetric_clip(df[x_col].to_numpy(), clip)
+    fig = px.bar(
+        df,
+        x=x_col,
+        y="id",
+        orientation="h",
+        color=x_col,
+        **_diverging_kwargs(),
+        range_color=[-color_clip, color_clip],
+        title=title,
+        labels={x_col: x_label, "id": ""},
+        text_auto=text_fmt,
+        **facet_kwargs,
+    )
+    if sort == "absolute":
+        fig.update_xaxes(**_axis_kwargs(unit))
+    fig.update_traces(textposition="outside", cliponaxis=False)
+    fig.update_layout(height=max(500, len(df) * 22), showlegend=False)
+    return fig, len(df)
+
+
+def plot_scatter(
+    snapshots: list[Path],
+    *,
+    metric: Metric = "time",
+    facet: str | None = None,
+    clip: float | None = None,
+) -> tuple[Figure, int]:
+    """Baseline cost (log-x) vs candidate/baseline ratio (log-y).
+
+    Top-right = slow *and* slower (the regressed corner). The first snapshot is
+    the baseline; with 3+, the rest animate. Colour encodes absolute Δ; ``clip``
+    clamps it (default p95). ``facet`` splits by any dim.
+    """
+    import plotly.express as px
+
+    if len(snapshots) < 2:
+        raise ValueError("scatter needs at least 2 snapshots (baseline + 1)")
+
+    df_long, unit = load_long_df(snapshots, metric=metric)
+    vlabel = _value_label(unit)
+    labels = df_long["snapshot"].drop_duplicates().tolist()
+    baseline_label = labels[0]
+
+    base = df_long.loc[df_long["snapshot"] == baseline_label, ["id", "value"]].rename(
+        columns={"value": "baseline"}
+    )
+    df = df_long.merge(base, on="id", how="inner")
+    df = df[df["baseline"] > 0].copy()
+    if df.empty:
+        raise ValueError(f"no ids shared with baseline ({baseline_label})")
+
+    df = df.rename(columns={"snapshot": "version", "value": "candidate"})
+    df["ratio"] = df["candidate"] / df["baseline"]
+    df["delta_abs"] = df["candidate"] - df["baseline"]
+    df = df[df["ratio"] > 0]
+    y_lo, y_hi = df["ratio"].min(), df["ratio"].max()
+    bound = max(y_hi, 1.0 / y_lo, 1.1) ** 1.05
+    color_clip = _symmetric_clip(df["delta_abs"].to_numpy(), clip)
+
+    extra: dict[str, object] = {}
+    if len(snapshots) >= 3:
+        extra["animation_frame"] = "version"
+        extra["category_orders"] = {"version": labels}
+    if facet is not None and facet in df.columns:
+        extra["facet_col"] = facet
+        extra["facet_col_wrap"] = 3
+
+    fig = px.scatter(
+        df,
+        x="baseline",
+        y="ratio",
+        color="delta_abs",
+        **_diverging_kwargs(),
+        range_color=[-color_clip, color_clip],
+        log_x=True,
+        log_y=True,
+        range_y=[1.0 / bound, bound],
+        hover_name="id",
+        title=f"{vlabel} scatter vs baseline ({baseline_label}) — top-right = regressed",
+        labels={
+            "baseline": f"baseline {vlabel} ({unit}, log)",
+            "ratio": "ratio (candidate / baseline, log)",
+        },
+        **extra,
+    )
+    fig.add_hline(y=1.0, line_dash="dash", line_color="grey")
+    fig.update_traces(marker={"size": 8, "line": {"width": 0.5, "color": "DarkSlateGrey"}})
+    fig.update_layout(height=600)
+    return fig, int(df["id"].nunique())
+
+
+def plot_sweep(
+    snapshots: list[Path], *, metric: Metric = "time", clip: float | None = None
+) -> tuple[Figure, int]:
+    """Heatmap of per-id fold-change (log2 ratio) vs the first snapshot."""
+    import plotly.express as px
+
+    df_long, unit = load_long_df(snapshots, metric=metric)
+    vlabel = _value_label(unit)
+    versions = df_long["snapshot"].drop_duplicates().tolist()
+    baseline = versions[0]
+
+    abs_df = df_long.pivot(index="id", columns="snapshot", values="value").reindex(columns=versions)
+    abs_df = abs_df.dropna(subset=[baseline])
+    if abs_df.empty:
+        raise ValueError(f"no overlap with baseline snapshot {baseline}")
+    ratio = abs_df.div(abs_df[baseline], axis=0)
+    logr = np.log2(ratio.where(ratio > 0))
+    abs_df.index.name = ratio.index.name = logr.index.name = "id"
+
+    bound = _symmetric_clip(logr.values, float(np.log2(clip)) if clip else None)
+    fig = px.imshow(
+        logr,
+        color_continuous_scale=["green", "white", "red"],
+        color_continuous_midpoint=0.0,
+        zmin=-bound,
+        zmax=bound,
+        aspect="auto",
+        title=f"{vlabel} fold-change vs baseline ({baseline})",
+        labels={"x": "snapshot", "y": "id", "color": "fold"},
+    )
+    tickvals, ticktext = _fold_ticks(bound)
+    fig.update_coloraxes(colorbar={"tickvals": tickvals, "ticktext": ticktext, "title": "fold"})
+    fig.update_traces(
+        text=ratio.round(2).values,
+        texttemplate="%{text}×",
+        customdata=abs_df.values,
+        hovertemplate=(
+            "id: %{y}<br>snapshot: %{x}<br>fold: %{text}×<br>"
+            f"{vlabel}: %{{customdata:.4g}}{unit}<extra></extra>"
+        ),
+    )
+    fig.update_layout(height=max(500, len(logr) * 22))
+    return fig, len(logr)
+
+
+def _infer_roles(
+    df: pd.DataFrame, x: str | None, color: str | None, facet: str | None
+) -> tuple[str, str | None, str | None]:
+    """Pick (x, color, facet) dim columns, auto-filling unset ones.
+
+    x defaults to the lone *numeric* dim; color/facet to leftover categoricals.
+    """
+    dims = _dim_columns(df)
+    numeric = [d for d in dims if df[d].dropna().map(lambda v: isinstance(v, (int, float))).all()]
+    if x is None:
+        if len(numeric) != 1:
+            raise ValueError(
+                f"scaling needs exactly one numeric dim for x (found {numeric}); pass x= explicitly"
+            )
+        x = numeric[0]
+    leftover = [d for d in dims if d != x]
+    if color is None:
+        color = leftover[0] if leftover else None
+    if facet is None:
+        facet = next((d for d in leftover if d != color), None)
+    return x, color, facet
+
+
+def plot_scaling(
+    snapshots: list[Path],
+    *,
+    metric: Metric = "time",
+    x: str | None = None,
+    color: str | None = None,
+    facet: str | None = None,
+    log: bool | Literal["auto"] = "auto",
+) -> tuple[Figure, int]:
+    """Cost vs a numeric dim, coloured/faceted by other dims.
+
+    ``x``/``color``/``facet`` default to inference from the dims (the lone numeric
+    dim → x); pass them to override. ``log="auto"`` log-scales when x is numeric
+    and strictly positive.
+    """
+    import plotly.express as px
+
+    df_long, unit = load_long_df(snapshots[:1], metric=metric)
+    vlabel = _value_label(unit)
+    x, color, facet = _infer_roles(df_long, x, color, facet)
+    df = df_long.dropna(subset=[x]).sort_values([c for c in (facet, color, x) if c])
+    if df.empty:
+        raise ValueError("no rows with the x dim set")
+
+    use_log = bool((df[x] > 0).all()) if log == "auto" else bool(log)
+    fig = px.line(
+        df,
+        x=x,
+        y="value",
+        color=color,
+        facet_col=facet,
+        facet_col_wrap=3 if facet else None,
+        log_x=use_log,
+        log_y=use_log,
+        markers=True,
+        labels={"value": f"{vlabel} ({unit})", x: x},
+        title=f"Scaling: {vlabel} ({unit}) vs {x} ({snapshots[0].stem})",
+    )
+    n_facets = df[facet].nunique() if facet else 1
+    fig.update_layout(height=max(400, ((n_facets + 2) // 3) * 350))
+    return fig, len(df)

pytest_benchmem/pytest_plugin.py

@@ -0,0 +1,263 @@
+"""pytest plugin — peak-memory alongside pytest-benchmark timing.
+
+pytest-benchmark times your code but can't measure memory. This plugin adds a
+memray peak-memory pass *to the same test, in the same run*, stored in
+pytest-benchmark's own JSON (``extra_info.peak_mib``) so timing and memory share
+one node id. Two ways in:
+
+- The ``benchmark_memory`` fixture — explicit, per-test::
+
+      def test_build(benchmark_memory):
+          benchmark_memory(build_model, 1000)
+
+- The ``--benchmark-memory`` flag — augments the stock ``benchmark`` fixture, so
+  an *existing* pytest-benchmark suite records memory with no code change::
+
+      pytest --benchmark-only --benchmark-memory
+
+Either way the two passes never overlap: pytest-benchmark times the action with
+no tracker installed, then memray measures peak on a separate, untimed call — so
+the allocator hooks cost the timing numbers nothing. Memory comes back out via
+:func:`pytest_benchmem.memory_from_pytest_benchmark` into the unit-aware
+``plot`` / ``compare``.
+
+Registered via the ``pytest11`` entry point — installing pytest-benchmem is enough.
+"""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable, Mapping
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from pytest_benchmem.memray import measure_peak
+
+if TYPE_CHECKING:
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+#: Key under which the peak (MiB) is written into pytest-benchmark ``extra_info``.
+PEAK_KEY = "peak_mib"
+
+
+def _record_peak(
+    benchmark: BenchmarkFixture, action: Callable[[], Any], *, repeats: int = 1
+) -> float:
+    """Memory-profile ``action`` into ``benchmark.extra_info[PEAK_KEY]``, once.
+
+    Idempotent: if a peak is already recorded for this benchmark (e.g. the
+    ``--benchmark-memory`` patch and the explicit fixture both fire), reuse it
+    rather than measuring twice.
+    """
+    existing = benchmark.extra_info.get(PEAK_KEY)
+    if existing is not None:
+        return float(existing)
+    peak = measure_peak(action, repeats=repeats)
+    benchmark.extra_info[PEAK_KEY] = peak
+    return peak
+
+
+class MemoryBenchmark:
+    """Callable wrapper that times via pytest-benchmark, then memory-profiles.
+
+    Mirrors the ``benchmark`` fixture's surface (``__call__`` and ``pedantic``)
+    so it's a drop-in for tests that want memory too. After a call, the peak is
+    on :attr:`peak_mib` and in the benchmark's ``extra_info``.
+    """
+
+    def __init__(self, benchmark: BenchmarkFixture, *, repeats: int = 1) -> None:
+        self._benchmark = benchmark
+        self._repeats = repeats
+        self.peak_mib: float | None = None
+
+    @property
+    def extra_info(self) -> dict[str, Any]:
+        """pytest-benchmark's per-benchmark metadata dict.
+
+        Set scalars here to attach analysis dims — ``benchmark_memory.extra_info["op"]
+        = "sort"`` — and pytest-benchmem's readers pick them up as dims alongside the
+        parametrize params. The peak is stored here too, under ``peak_mib``.
+        """
+        return self._benchmark.extra_info
+
+    def __call__(self, function_to_benchmark: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+        """Time ``function_to_benchmark(*args, **kwargs)``, then record its peak memory."""
+        result = self._benchmark(function_to_benchmark, *args, **kwargs)
+        self.peak_mib = _record_peak(
+            self._benchmark, lambda: function_to_benchmark(*args, **kwargs), repeats=self._repeats
+        )
+        return result
+
+    def pedantic(
+        self,
+        target: Callable[..., Any],
+        args: tuple[Any, ...] = (),
+        kwargs: Mapping[str, Any] | None = None,
+        setup: Callable[[], Any] | None = None,
+        rounds: int = 1,
+        warmup_rounds: int = 0,
+        iterations: int = 1,
+    ) -> Any:
+        """Like :meth:`pytest_benchmark.fixture.BenchmarkFixture.pedantic`, plus a memory pass.
+
+        ``setup`` runs untracked before the measured call (and, if it returns
+        ``(args, kwargs)``, supplies them) — matching pytest-benchmark, and
+        giving back the untimed-setup/measured-action split cleanly.
+        """
+        kwargs = dict(kwargs or {})
+        result = self._benchmark.pedantic(  # type: ignore[no-untyped-call]
+            target,
+            args=args,
+            kwargs=kwargs,
+            setup=setup,
+            rounds=rounds,
+            warmup_rounds=warmup_rounds,
+            iterations=iterations,
+        )
+        self.peak_mib = _record_peak(
+            self._benchmark, _pedantic_action(target, args, kwargs, setup), repeats=self._repeats
+        )
+        return result
+
+
+def _pedantic_action(
+    target: Callable[..., Any],
+    args: tuple[Any, ...],
+    kwargs: Mapping[str, Any],
+    setup: Callable[[], Any] | None,
+) -> Callable[[], Any]:
+    """Build the zero-arg measured action for a pedantic call, honoring ``setup``."""
+
+    def action() -> Any:
+        call_args, call_kwargs = args, dict(kwargs)
+        if setup is not None:
+            produced = setup()
+            if produced is not None:
+                call_args, call_kwargs = produced
+        return target(*call_args, **call_kwargs)
+
+    return action
+
+
+@pytest.fixture
+def benchmark_memory(benchmark: BenchmarkFixture) -> MemoryBenchmark:
+    """Time *and* peak-memory-profile a callable in one pytest-benchmark test.
+
+    Depends on the ``benchmark`` fixture, so timing rides pytest-benchmark fully;
+    the memray memory pass is added on top and stored in the same entry. For an
+    existing suite, prefer the ``--benchmark-memory`` flag over rewriting tests.
+    """
+    return MemoryBenchmark(benchmark)
+
+
+# --- --benchmark-memory: augment the stock `benchmark` fixture --------------------
+
+_PATCHED: tuple[Any, Any] | None = None
+
+
+#: pedantic kwargs the patch passes through — guard against a version that drops one.
+_PEDANTIC_PARAMS = frozenset({"args", "kwargs", "setup", "rounds", "warmup_rounds", "iterations"})
+
+
+def _patch_compatible(fixture_cls: Any) -> bool:
+    """True if ``BenchmarkFixture.pedantic`` still takes the kwargs the patch passes.
+
+    (``__call__`` is invoked positionally, so it needs no signature guard.)
+    """
+    try:
+        pedantic_params = set(inspect.signature(fixture_cls.pedantic).parameters)
+    except (TypeError, ValueError, AttributeError):
+        return False
+    return pedantic_params >= _PEDANTIC_PARAMS
+
+
+def _install_auto_memory() -> None:
+    """Wrap ``BenchmarkFixture.__call__``/``.pedantic`` to also record peak memory."""
+    global _PATCHED
+    if _PATCHED is not None:
+        return
+    import pytest_benchmark
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+    if not _patch_compatible(BenchmarkFixture):
+        raise pytest.UsageError(
+            f"--benchmark-memory: pytest-benchmark {pytest_benchmark.__version__} has a "
+            f"BenchmarkFixture.pedantic() signature pytest-benchmem doesn't recognize, so "
+            f"the auto-memory patch can't apply safely. Either:\n"
+            f"  - install a pytest-benchmark version pytest-benchmem supports, or\n"
+            f"  - drop --benchmark-memory (timing still runs normally), or\n"
+            f"  - report it so we add support: "
+            f"https://github.com/fluxopt/pytest-benchmem/issues"
+        )
+
+    orig_call = BenchmarkFixture.__call__
+    orig_pedantic = BenchmarkFixture.pedantic
+
+    def call(
+        self: BenchmarkFixture, function_to_benchmark: Callable[..., Any], *a: Any, **k: Any
+    ) -> Any:
+        result = orig_call(self, function_to_benchmark, *a, **k)  # type: ignore[no-untyped-call]
+        _record_peak(self, lambda: function_to_benchmark(*a, **k))
+        return result
+
+    def pedantic(
+        self: BenchmarkFixture,
+        target: Callable[..., Any],
+        args: tuple[Any, ...] = (),
+        kwargs: Mapping[str, Any] | None = None,
+        setup: Callable[[], Any] | None = None,
+        rounds: int = 1,
+        warmup_rounds: int = 0,
+        iterations: int = 1,
+    ) -> Any:
+        kwargs = dict(kwargs or {})
+        result = orig_pedantic(  # type: ignore[no-untyped-call]
+            self,
+            target,
+            args=args,
+            kwargs=kwargs,
+            setup=setup,
+            rounds=rounds,
+            warmup_rounds=warmup_rounds,
+            iterations=iterations,
+        )
+        _record_peak(self, _pedantic_action(target, args, kwargs, setup))
+        return result
+
+    BenchmarkFixture.__call__ = call  # type: ignore[method-assign]
+    BenchmarkFixture.pedantic = pedantic  # type: ignore[method-assign,assignment]
+    _PATCHED = (orig_call, orig_pedantic)
+
+
+def _remove_auto_memory() -> None:
+    global _PATCHED
+    if _PATCHED is None:
+        return
+    from pytest_benchmark.fixture import BenchmarkFixture
+
+    BenchmarkFixture.__call__, BenchmarkFixture.pedantic = _PATCHED  # type: ignore[method-assign]
+    _PATCHED = None
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Register the ``--benchmark-memory`` flag."""
+    group = parser.getgroup("pytest-benchmem", "peak-memory benchmarking")
+    group.addoption(
+        "--benchmark-memory",
+        action="store_true",
+        default=False,
+        help="Record peak memory (memray) for every benchmark() call, "
+        "not only benchmark_memory — no test changes needed.",
+    )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Patch the stock ``benchmark`` fixture when ``--benchmark-memory`` is set."""
+    if config.getoption("--benchmark-memory"):
+        _install_auto_memory()
+
+
+def pytest_unconfigure(config: pytest.Config) -> None:
+    """Restore the stock ``benchmark`` fixture after the session."""
+    _remove_auto_memory()

pytest_benchmem/snapshot.py

@@ -0,0 +1,141 @@
+"""The reading spine — normalize pytest-benchmark JSON into a tidy frame for plots.
+
+pytest-benchmem defines no on-disk format of its own. The source of truth is the JSON
+pytest-benchmark writes under ``.benchmarks/<machine>/NNNN_*.json``; a single
+file carries *both* metrics for each benchmark id:
+
+- ``stats`` — timing (min/mean/median/…), in seconds.
+- ``extra_info.peak_mib`` — peak memory in MiB, written by the
+  ``benchmark_memory`` fixture.
+
+So reads are *per metric*: :func:`from_pytest_benchmark` pulls timing,
+:func:`memory_from_pytest_benchmark` pulls memory, each yielding a list of
+:class:`Sample` — an opaque ``id``, a ``value``, and analysis ``dims``. Dims come
+from the benchmark's ``params`` (``@pytest.mark.parametrize``) plus any scalar
+``extra_info`` you set in the test. :func:`load_long_df` stacks several files
+(e.g. one per version from a sweep) into the single frame every plot pivots.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple
+
+#: A value of a dim axis — categorical (str) or numeric (int/float).
+DimValue = str | int | float
+
+#: Which metric to read out of a pytest-benchmark file.
+Metric = Literal["time", "memory"]
+
+#: Default key under which ``benchmark_memory`` stores the peak (MiB).
+PEAK_KEY = "peak_mib"
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+class Sample(NamedTuple):
+    """One measured result: an opaque ``id``, a ``value``, and analysis ``dims``."""
+
+    id: str
+    value: float
+    dims: Mapping[str, DimValue]
+
+
+def _read_benchmarks(path: str | Path) -> tuple[Path, list[dict[str, Any]]]:
+    """Read a pytest-benchmark file, returning ``(path, benchmarks)`` or raising."""
+    p = Path(path)
+    try:
+        data = json.loads(p.read_text())
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"{path}: not a readable JSON file ({exc})") from exc
+    if "benchmarks" not in data:
+        raise ValueError(f"{path}: not a pytest-benchmark file (no 'benchmarks' key)")
+    return p, data["benchmarks"]
+
+
+def _dims(bm: Mapping[str, Any]) -> dict[str, DimValue]:
+    """Analysis dims for one benchmark — its parametrize ``params`` plus any
+    scalar ``extra_info`` (minus the reserved ``peak_mib``). No id parsing.
+    """
+    dims: dict[str, DimValue] = {}
+    params = bm.get("params")
+    if isinstance(params, Mapping):
+        dims.update(params)
+    extra = bm.get("extra_info")
+    if isinstance(extra, Mapping):
+        dims.update({k: v for k, v in extra.items() if k != PEAK_KEY})
+    return dims
+
+
+def from_pytest_benchmark(
+    path: str | Path, *, metric: str = "min"
+) -> tuple[str, list[Sample], str]:
+    """Read timing out of a pytest-benchmark file → ``(label, samples, "s")``.
+
+    ``metric`` picks the stat (``min`` / ``median`` / …). Dims come from each
+    benchmark's parametrize ``params`` and ``extra_info``.
+    """
+    p, benchmarks = _read_benchmarks(path)
+    samples = [
+        Sample(id=bm["fullname"], value=bm["stats"][metric], dims=_dims(bm)) for bm in benchmarks
+    ]
+    return p.stem, samples, "s"
+
+
+def memory_from_pytest_benchmark(
+    path: str | Path, *, key: str = PEAK_KEY
+) -> tuple[str, list[Sample], str]:
+    """Read peak memory out of a pytest-benchmark file → ``(label, samples, "MiB")``.
+
+    The ``benchmark_memory`` fixture stores each run's peak under
+    ``extra_info[key]``, keyed by the same benchmark id pytest-benchmark uses.
+    Benchmarks lacking the key (timing-only tests) are skipped. Dims come from
+    parametrize ``params`` and ``extra_info``.
+    """
+    p, benchmarks = _read_benchmarks(path)
+    samples = [
+        Sample(id=bm["fullname"], value=float(bm["extra_info"][key]), dims=_dims(bm))
+        for bm in benchmarks
+        if isinstance(bm.get("extra_info"), Mapping) and bm["extra_info"].get(key) is not None
+    ]
+    return p.stem, samples, "MiB"
+
+
+def load_samples(
+    path: str | Path, *, metric: Metric = "time", stat: str = "min"
+) -> tuple[str, list[Sample], str]:
+    """Read one pytest-benchmark file for the chosen ``metric`` → ``(label, samples, unit)``."""
+    if metric == "time":
+        return from_pytest_benchmark(path, metric=stat)
+    if metric == "memory":
+        return memory_from_pytest_benchmark(path)
+    raise ValueError(f"metric must be 'time' or 'memory', got {metric!r}")
+
+
+def discover_runs(root: str | Path = ".benchmarks") -> list[Path]:
+    """Return pytest-benchmark JSON files under ``root`` (for CLI suggestions)."""
+    base = Path(root)
+    return sorted(base.rglob("*.json")) if base.exists() else []
+
+
+def load_long_df(
+    runs: Sequence[str | Path], *, metric: Metric = "time", stat: str = "min"
+) -> tuple[pd.DataFrame, str]:
+    """Stack pytest-benchmark files into one long frame → ``(df, unit)``.
+
+    One row per ``(run, id)`` for the chosen ``metric``. Columns: ``snapshot``
+    (the file stem / version label), ``id``, ``value``, then one column per dim
+    key seen (missing dims are ``NaN``). Every plot view pivots this frame.
+    """
+    import pandas as pd
+
+    rows: list[dict[str, object]] = []
+    unit = ""
+    for path in runs:
+        label, samples, unit = load_samples(path, metric=metric, stat=stat)
+        for s in samples:
+            rows.append({"snapshot": label, "id": s.id, "value": s.value, **s.dims})
+    return pd.DataFrame(rows), unit

pytest_benchmem/sweep.py

@@ -0,0 +1,147 @@
+"""Cross-version sweep — run benchmarks against several installed versions of a
+package in fresh, isolated ``uv`` venvs.
+
+Generic: the package install spec, the extra pins, the directory to copy for
+import isolation, and *what to run* in each venv are all injected. A consumer
+(e.g. linopy) wires ``install_spec=lambda v: f"linopy=={v}"`` and a ``run``
+callback that invokes its pytest / memory command.
+
+Isolation: if the repo root contains a dev copy of the package under test, it
+would shadow the venv's installed version on ``sys.path``. So each venv runs
+with ``cwd`` set to a fresh tempdir holding only a *copy* of ``copy_dir`` — the
+benchmark code imports from there, while the package-under-test resolves to the
+venv. A preflight asserts that resolution held.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections.abc import Callable, Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+_PLAIN_VERSION_RE = re.compile(r"^\d+(\.\d+)*([a-z]+\d*)?$")
+
+
+def version_label(version: str) -> str:
+    """Filesystem-safe label from a version/pip-spec (part after the last ``@``)."""
+    label = version.rsplit("@", 1)[-1] if "@" in version else version
+    return re.sub(r"[^0-9A-Za-z._-]+", "-", label).strip("-._") or "spec"
+
+
+def _venv_python(venv: Path) -> Path:
+    return venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+
+
+@dataclass(frozen=True)
+class Venv:
+    """One provisioned per-version venv (or a failure)."""
+
+    version: str
+    python: Path | None
+    env: dict[str, str] | None
+    cwd: Path | None
+    failed_at: str | None  # "venv" | "install" | "isolation" | None
+
+
+def provision(
+    versions: Sequence[str],
+    *,
+    install_spec: Callable[[str], str] = lambda v: v,
+    pins: Sequence[str] = (),
+    copy_dir: str | Path | None = None,
+    import_check: str | None = None,
+    as_of: str | None = None,
+    tmp_prefix: str = "pytest-benchmem-",
+) -> Iterator[Venv]:
+    """Yield one fresh ``uv`` venv per version.
+
+    ``install_spec(v)`` → the pip spec for the package under test (default:
+    ``v`` verbatim, so ``"pkg==1.2"`` works). ``pins`` are extra specs installed
+    alongside. ``copy_dir`` is copied into each venv's cwd for import isolation.
+    ``import_check`` is a module name asserted to resolve to the venv (preflight).
+    ``as_of`` (``YYYY-MM-DD``) freezes the whole resolution via ``--exclude-newer``.
+    """
+    if shutil.which("uv") is None:
+        raise RuntimeError("uv not found on PATH — https://docs.astral.sh/uv/")
+
+    for version in versions:
+        with tempfile.TemporaryDirectory(prefix=tmp_prefix) as tmp:
+            venv = Path(tmp) / "venv"
+            r = subprocess.run(["uv", "venv", "--python", sys.executable, str(venv)], check=False)
+            if r.returncode != 0:
+                yield Venv(version, None, None, None, "venv")
+                continue
+
+            vpy = _venv_python(venv)
+            spec = install_spec(version) if _PLAIN_VERSION_RE.match(version) else version
+            install = [
+                "uv",
+                "pip",
+                "install",
+                "--python",
+                str(vpy),
+                *(["--exclude-newer", as_of] if as_of else []),
+                *pins,
+                spec,
+            ]
+            if subprocess.run(install, check=False).returncode != 0:
+                yield Venv(version, None, None, None, "install")
+                continue
+
+            cwd = Path(tmp) / "iso"
+            cwd.mkdir()
+            if copy_dir is not None:
+                src = Path(copy_dir)
+                shutil.copytree(
+                    src,
+                    cwd / src.name,
+                    ignore=shutil.ignore_patterns("__pycache__", "*.ipynb", ".DS_Store"),
+                )
+            env = os.environ.copy()
+            env.pop("PYTHONPATH", None)
+
+            if import_check is not None:
+                pre = subprocess.run(
+                    [
+                        str(vpy),
+                        "-c",
+                        f"import {import_check} as _m; "
+                        f"assert {str(venv)!r} in _m.__file__, _m.__file__",
+                    ],
+                    cwd=str(cwd),
+                    env=env,
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                )
+                if pre.returncode != 0:
+                    yield Venv(version, None, None, None, "isolation")
+                    continue
+
+            yield Venv(version, vpy, env, cwd, None)
+
+
+def sweep(
+    versions: Sequence[str],
+    run: Callable[[Venv], None],
+    **provision_kwargs: object,
+) -> list[str]:
+    """Provision a venv per version and call ``run(venv)`` in each.
+
+    ``run`` does whatever the consumer needs (invoke pytest / a memory command
+    with ``venv.python`` and ``cwd=venv.cwd``). Returns the list of versions
+    that failed to provision.
+    """
+    failed: list[str] = []
+    for prov in provision(versions, **provision_kwargs):  # type: ignore[arg-type]
+        if prov.failed_at:
+            failed.append(prov.version)
+            continue
+        run(prov)
+    return failed

pytest_benchmem-0.1.0.dist-info/METADATA

@@ -0,0 +1,152 @@
+Metadata-Version: 2.4
+Name: pytest-benchmem
+Version: 0.1.0
+Summary: The memory companion to pytest-benchmark: a memray peak-memory pass on the same test, plus dims-aware plots and cross-version sweeps.
+Project-URL: Homepage, https://github.com/fluxopt/pytest-benchmem
+Project-URL: Documentation, https://fluxopt.github.io/pytest-benchmem/
+Project-URL: Repository, https://github.com/fluxopt/pytest-benchmem
+Project-URL: Issues, https://github.com/fluxopt/pytest-benchmem/issues
+Author: Felix Bumann
+License-Expression: MIT
+License-File: LICENSE
+Keywords: benchmark,memory,memray,performance,pytest,pytest-benchmark
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: memray>=1.11; platform_system == 'Linux' or platform_system == 'Darwin'
+Requires-Dist: pytest-benchmark<6,>=4
+Provides-Extra: plot
+Requires-Dist: pandas; extra == 'plot'
+Requires-Dist: plotly>=5; extra == 'plot'
+Requires-Dist: typer>=0.12; extra == 'plot'
+Description-Content-Type: text/markdown
+
+# pytest-benchmem
+
+[![CI](https://github.com/fluxopt/pytest-benchmem/actions/workflows/ci.yaml/badge.svg)](https://github.com/fluxopt/pytest-benchmem/actions/workflows/ci.yaml)
+![Python](https://img.shields.io/badge/python-3.11%2B-blue)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+**The memory companion to [pytest-benchmark].** It times your code; pytest-benchmem
+adds a memray **peak-memory** pass to the *same test, in the same run* — one node
+id, one JSON file, both metrics. Plus dims-aware plots and cross-version sweeps.
+
+## Quickstart
+
+Write a normal pytest-benchmark test; swap `benchmark` for `benchmark_memory`:
+
+```python
+import pytest
+
+
+@pytest.mark.parametrize("n", [10_000, 100_000, 1_000_000])
+def test_sort(benchmark_memory, n):
+    data = list(range(n, 0, -1))
+    benchmark_memory(sorted, data)
+```
+
+```bash
+pytest --benchmark-only --benchmark-json=run.json
+```
+
+One run, one `run.json`, for each benchmark id — both metrics, one node id:
+
+- `stats: {min, mean, median, …}` — **timing**, from pytest-benchmark.
+- `extra_info: {"peak_mib": 3.81}` — **peak memory**, from pytest-benchmem.
+
+The two passes never overlap: pytest-benchmark times the action untracked, then
+memray measures peak on a *separate, untimed* call — so the allocator hooks cost
+the timing nothing. The parametrize `params` become the analysis dims the plots
+scale by.
+
+## Already have a pytest-benchmark suite?
+
+Don't rewrite a thing — add `--benchmark-memory` and every `benchmark(...)` call
+also records peak memory:
+
+```python
+def test_sort(benchmark):            # unchanged
+    benchmark(sorted, list(range(1_000_000, 0, -1)))
+```
+
+```bash
+pytest --benchmark-only --benchmark-memory   # timing + peak_mib for the whole suite
+```
+
+It's opt-in at the run level: without the flag, plain `benchmark` tests are
+untouched. (Reach for the `benchmark_memory` fixture when you want memory on
+specific tests only, or `pedantic` control.)
+
+## Reading it back
+
+Timing rides pytest-benchmark's own tooling (`pytest-benchmark compare`,
+`--benchmark-histogram`) — pytest-benchmem doesn't reimplement it. For **memory**,
+and dims-aware views over either metric:
+
+```bash
+benchmem compare base.json head.json --metric memory   # per-id delta table
+benchmem plot    base.json head.json --metric memory   # interactive plotly view
+```
+
+```
+id                          base.json    head.json     change  (MiB)
+--------------------------------------------------------------------
+test_sort[10000]                0.076        0.078       +2.6%
+test_sort[100000]               0.76         0.74        -2.6%
+test_sort[1000000]              7.63         9.155       +20.0%
+```
+
+Or pull the numbers into your own analysis:
+
+```python
+from pytest_benchmem import from_pytest_benchmark, memory_from_pytest_benchmark
+
+_, timing, _ = from_pytest_benchmark("run.json")         # seconds, from stats
+_, memory, _ = memory_from_pytest_benchmark("run.json")  # MiB, from extra_info
+```
+
+Outside pytest, `measure_peak(lambda: build_model(1000))` is the bare memray
+engine — a one-liner peak number for a REPL or notebook.
+
+## Where it sits
+
+Its reason to exist is the gap nothing else fills cleanly: **memray-precision
+memory benchmarking** of your own code, right where you already benchmark. ASV's
+`peakmem` is coarse RSS sampling that misses numpy/C-allocation detail; CodSpeed
+covers CI timing.
+
+| Need | Reach for | pytest-benchmem |
+|---|---|---|
+| CI regression, per-PR dashboard | **CodSpeed** | — (don't rebuild it) |
+| Local timing + A/B compare | **pytest-benchmark** | rides it (timing is its job) |
+| Rigorous perf history across commits | **ASV** | — (heavier, RSS memory) |
+| **Precise local peak memory (numpy/C allocs)** | **memray** | ⭐ the core |
+| Memory *in your pytest-benchmark tests* | — | ⭐ fixture **or** `--benchmark-memory` |
+| Same runs across installed versions | — | ⭐ `sweep` |
+
+> **Not** a CI dashboard (use [CodSpeed]) and **not** a rigorous perf-history
+> system (use [ASV]). If your core need is *precise local memory* over the
+> benchmarks you already write — timing/sweeps/plots in one vocabulary — that's
+> pytest-benchmem.
+
+## Install
+
+```bash
+uv add pytest-benchmem            # the fixture + flag + memray engine
+uv add "pytest-benchmem[plot]"    # + the plot/compare CLI (pandas, plotly, typer)
+```
+
+pytest-benchmark and memray are core deps; memray is Linux/macOS only, so Windows
+installs cleanly with timing-only (the memory pass raises a clear error there).
+
+## Status
+
+Early. Extracted from the linopy internal benchmark suite, where it's the local
+memory-profiling layer. API may move before 1.0.
+
+[pytest-benchmark]: https://pytest-benchmark.readthedocs.io
+[CodSpeed]: https://codspeed.io
+[ASV]: https://asv.readthedocs.io

pytest_benchmem-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+pytest_benchmem/__init__.py,sha256=yUI5fVb4Neo6KACY3EsU5fauu0uolIsNbvmq_16Q6Cc,1173
+pytest_benchmem/cli.py,sha256=nElgZ7wK8V9dFZEvvhd79PnUh6xv5KXA9LlW_YalS-Q,4076
+pytest_benchmem/compare.py,sha256=yzTeOJFEUMxQQ2l60XF9piJsupbZcsgbBwDNkrQU9eE,1579
+pytest_benchmem/memray.py,sha256=2dW1jNZRqctsnjUcS07FEo8aWfQ0ffBh4SAJqrV_Ow8,2180
+pytest_benchmem/plotting.py,sha256=Lg9cLbpvegdKxngxn4a2QJ5lKZqu3xwR5tSsRzVrgSw,12662
+pytest_benchmem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytest_benchmem/pytest_plugin.py,sha256=--m9UB8zgCav3M-VM1exVGtJvjjoSrFY5GSTCQ9ndz0,9681
+pytest_benchmem/snapshot.py,sha256=TB3SeuGYt0ePYp0vuqlkSMvcFO9ZX7DTkVE3UhnmyME,5429
+pytest_benchmem/sweep.py,sha256=VAFB64rmyMbjdRlI7R3A7vsiQKcqQzS9u5Kaz5NSlnY,5260
+pytest_benchmem-0.1.0.dist-info/METADATA,sha256=DfiMmjdVOPKU0FG2pogVF7YDSoqkHwfnaNmNqZh8bsg,5966
+pytest_benchmem-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+pytest_benchmem-0.1.0.dist-info/entry_points.txt,sha256=v8fZRWsQTl2Tk6opcZ3BDKbqRhJzOBWsco5Isc8FN3M,106
+pytest_benchmem-0.1.0.dist-info/licenses/LICENSE,sha256=jRwycONJR6wDwPOtHRIFKIU7X-EqCXRyOnktVz2UaYs,1069
+pytest_benchmem-0.1.0.dist-info/RECORD,,

pytest_benchmem-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

pytest_benchmem-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+benchmem = pytest_benchmem.cli:app
+
+[pytest11]
+benchmem = pytest_benchmem.pytest_plugin

pytest_benchmem-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Felix Bumann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.