flopscope 0.2.0__py3-none-any.whl

benchmarks/_stats.py

@@ -0,0 +1,137 @@
+"""Benchmark scipy.stats-compatible distribution methods (pdf/cdf/ppf).
+
+These ops use raw numpy internally (no ufunc layer), so measurement_mode
+is ``custom`` (no overhead subtraction). The analytical cost is
+``numel(input)`` for all methods — the weight captures the per-element
+FP cost.
+"""
+
+from __future__ import annotations
+
+import statistics
+
+from benchmarks._perf import measure_flops
+
+# 8 distributions × 3 methods = 24 ops
+DISTRIBUTIONS = [
+    "norm",
+    "uniform",
+    "expon",
+    "cauchy",
+    "logistic",
+    "laplace",
+    "lognorm",
+    "truncnorm",
+]
+METHODS = ["pdf", "cdf", "ppf"]
+
+_FORMULA_STRINGS: dict[str, str] = {
+    f"stats.{dist}.{method}": "numel(input)"
+    for dist in DISTRIBUTIONS
+    for method in METHODS
+}
+
+
+def _setup_code(dist: str, method: str, n: int, di: int) -> str:
+    """Build setup code for a stats benchmark.
+
+    The import of the stats distribution object is done HERE in setup
+    (not in bench_code) so the import cost is not measured — only the
+    actual computation is in the hot loop.
+    """
+    seeds = [42, 123, 7]
+    seed = seeds[di % len(seeds)]
+
+    # Import the distribution in setup so it's available in the bench loop
+    import_line = f"from flopscope.stats import {dist} as _dist"
+
+    if method == "ppf":
+        return (
+            f"import numpy as np; {import_line}; "
+            f"rng = np.random.default_rng({seed}); x = rng.uniform(0.01, 0.99, {n})"
+        )
+    else:
+        setups = [
+            f"import numpy as np; {import_line}; x = np.random.default_rng({seed}).standard_normal({n})",
+            f"import numpy as np; {import_line}; x = np.random.default_rng({seed}).uniform(-5, 5, {n})",
+            f"import numpy as np; {import_line}; x = np.random.default_rng({seed}).uniform(0.01, 10, {n})",
+        ]
+        return setups[di % len(setups)]
+
+
+def _bench_code(dist: str, method: str) -> str:
+    """Build benchmark code — just the computation, no imports.
+
+    The distribution object is imported as ``_dist`` in setup_code.
+    We call _compute_* to avoid budget.deduct overhead in measurement.
+    """
+    if dist == "lognorm":
+        return f"_dist._compute_{method}(x, s=0.5)"
+    elif dist == "truncnorm":
+        return f"_dist._compute_{method}(x, a=-2, b=2)"
+    else:
+        return f"_dist._compute_{method}(x)"
+
+
+def benchmark_stats(
+    n: int = 10_000_000,
+    dtype: str = "float64",
+    repeats: int = 10,
+    **kwargs,
+) -> tuple[dict[str, float], dict[str, dict]]:
+    """Benchmark all stats distribution methods.
+
+    Parameters
+    ----------
+    n : int
+        Array size for benchmarks.
+    dtype : str
+        Ignored (stats always use float64).
+    repeats : int
+        Repetitions per distribution.
+
+    Returns
+    -------
+    tuple[dict[str, float], dict[str, dict]]
+        (alphas, details) — alphas maps op name to median alpha,
+        details maps op name to benchmark metadata.
+    """
+    distributions = 3
+    results: dict[str, float] = {}
+    details: dict[str, dict] = {}
+
+    for dist in DISTRIBUTIONS:
+        for method in METHODS:
+            op_name = f"stats.{dist}.{method}"
+            dist_values: list[float] = []
+            dist_raw_totals: list[int] = []
+
+            for di in range(distributions):
+                setup = _setup_code(dist, method, n, di)
+                bench = _bench_code(dist, method)
+
+                try:
+                    result = measure_flops(setup, bench, repeats=repeats)
+                except RuntimeError:
+                    continue
+
+                # Analytical cost = numel(input) = n
+                alpha = result.total_flops / (n * repeats)
+                dist_values.append(alpha)
+                dist_raw_totals.append(result.total_flops)
+
+            if dist_values:
+                results[op_name] = statistics.median(dist_values)
+                details[op_name] = {
+                    "category": "counted_custom",
+                    "measurement_mode": "custom",
+                    "analytical_formula": "numel(input)",
+                    "analytical_flops": n,
+                    "benchmark_size": f"x: ({n},)",
+                    "bench_code": _bench_code(dist, method),
+                    "repeats": repeats,
+                    "perf_instructions_total": dist_raw_totals,
+                    "distribution_alphas": dist_values,
+                }
+
+    return results, details

benchmarks/_window.py

@@ -0,0 +1,92 @@
+"""Benchmark window function operations."""
+
+from __future__ import annotations
+
+import statistics
+
+from benchmarks._perf import measure_flops
+
+WINDOW_OPS: list[str] = ["bartlett", "blackman", "hamming", "hanning", "kaiser"]
+
+_ANALYTICAL_COST: dict[str, int] = {
+    "bartlett": 1,  # multiplied by n
+    "blackman": 3,  # multiplied by n
+    "hamming": 1,  # multiplied by n
+    "hanning": 1,  # multiplied by n
+    "kaiser": 3,  # multiplied by n
+}
+
+_FORMULA_STRINGS: dict[str, str] = {
+    "bartlett": "n",
+    "blackman": "3*n",
+    "hamming": "n",
+    "hanning": "n",
+    "kaiser": "3*n",
+}
+
+
+def benchmark_window(
+    n: int = 10_000_000,
+    dtype: str = "float64",
+    repeats: int = 10,
+) -> tuple[dict[str, float], dict[str, dict]]:
+    """Benchmark window ops, returning alpha(op) = measured/analytical.
+
+    Parameters
+    ----------
+    n : int
+        Window size.
+    dtype : str
+        Accepted for interface consistency; window functions always produce
+        float64.
+    repeats : int
+        Number of repetitions per measurement.
+
+    Returns
+    -------
+    tuple[dict[str, float], dict[str, dict]]
+        A pair of (alphas, details). ``alphas`` maps op name to median
+        alpha(op). ``details`` maps op name to a dict of raw benchmark
+        metadata.
+    """
+    results: dict[str, float] = {}
+    details: dict[str, dict] = {}
+
+    for op in WINDOW_OPS:
+        dist_values: list[float] = []
+        dist_raw_totals: list[int] = []
+        analytical = _ANALYTICAL_COST[op] * n
+        bench = ""
+
+        # Window functions are deterministic (no input distribution),
+        # but we still take 3 measurements with different seeds for
+        # consistency (they'll be nearly identical).
+        for _seed in [42, 123, 999]:
+            setup = "import numpy as np"
+            if op == "kaiser":
+                bench = f"np.kaiser({n}, 14.0)"
+            else:
+                bench = f"np.{op}({n})"
+
+            try:
+                result = measure_flops(setup, bench, repeats=repeats)
+            except RuntimeError:
+                continue
+            dist_values.append(result.total_flops / (analytical * repeats))
+            dist_raw_totals.append(result.total_flops)
+
+        if dist_values:
+            results[op] = statistics.median(dist_values)
+            details[op] = {
+                "category": "counted_custom",
+                "measurement_mode": "custom",
+                "analytical_formula": _FORMULA_STRINGS[op],
+                "analytical_flops": analytical,
+                "benchmark_size": f"output: ({n},)",
+                "bench_code": bench,
+                "repeats": repeats,
+                "perf_instructions_total": dist_raw_totals,
+                "distribution_alphas": dist_values,
+            }
+
+    return results, details

benchmarks/accumulation/bench_cost_compute.py

@@ -0,0 +1,138 @@
+"""Cold and warm-call latency benchmarks for einsum_accumulation_cost.
+
+Run: `uv run python benchmarks/accumulation/bench_cost_compute.py`
+
+Failure threshold: cold-call latency for the whole suite must stay below
+COLD_CALL_BUDGET_SECONDS. This is the R7 risk gate from the spec — protects
+against a regime implementation that's accidentally O(n²) on partition counts.
+"""
+
+from __future__ import annotations
+
+import time
+
+import numpy as np
+
+import flopscope as fps
+
+COLD_CALL_BUDGET_SECONDS = 0.5  # whole suite, no cache hits
+
+
+CASES = [
+    {
+        "name": "matrix_chain_n3",
+        "subscripts": "ij,jk",
+        "output": "ik",
+        "per_op_sym": (None, None),
+        "sizes": {"i": 3, "j": 3, "k": 3},
+    },
+    {
+        "name": "symmetric_matvec",
+        "subscripts": "ij,j",
+        "output": "i",
+        "per_op_sym": ("symmetric", None),
+        "sizes": {"i": 4, "j": 4},
+    },
+    {
+        "name": "fully_symmetric_self_contract",
+        "subscripts": "ijk,ijk",
+        "output": "",
+        "per_op_sym": ("symmetric", "symmetric"),
+        "sizes": {"i": 4, "j": 4, "k": 4},
+    },
+    {
+        "name": "triple_outer",
+        "subscripts": "ia,ib,ic",
+        "output": "abc",
+        "per_op_sym": (None, None, None),
+        "sizes": {"i": 4, "a": 4, "b": 4, "c": 4},
+    },
+    {
+        "name": "cyclic_t_to_ab",
+        "subscripts": "abc",
+        "output": "ab",
+        "per_op_sym": ({"type": "cyclic", "axes": [0, 1, 2]},),
+        "sizes": {"a": 3, "b": 3, "c": 3},
+    },
+]
+
+
+def _build_operands(case_def):
+    parts = case_def["subscripts"].split(",")
+    operands = []
+    for op_idx, part in enumerate(parts):
+        shape = tuple(case_def["sizes"][lbl] for lbl in part)
+        op = np.zeros(shape) if shape else np.zeros(1)
+        sym_decl = case_def["per_op_sym"][op_idx]
+        if sym_decl == "symmetric":
+            axes = tuple(range(len(part)))
+            op = fps.as_symmetric(op, symmetry=axes)
+        elif isinstance(sym_decl, dict) and sym_decl.get("type") == "cyclic":
+            from flopscope._perm_group import SymmetryGroup
+
+            axes = tuple(sym_decl.get("axes", range(len(part))))
+            group = SymmetryGroup.cyclic(axes=axes)
+            op = fps.as_symmetric(op, symmetry=group)
+        operands.append(op)
+    return operands
+
+
+def bench_cold_call() -> dict[str, float]:
+    """Time each case with a fresh cache."""
+    timings = {}
+    for case in CASES:
+        from flopscope._einsum import _accumulation_cache, _path_cache
+
+        _accumulation_cache.cache_clear()
+        _path_cache.cache_clear()
+
+        operands = _build_operands(case)
+        subscripts = case["subscripts"] + "->" + case["output"]
+
+        start = time.perf_counter()
+        fps.einsum_accumulation_cost(subscripts, *operands)
+        elapsed = time.perf_counter() - start
+        timings[case["name"]] = elapsed
+    return timings
+
+
+def bench_warm_call() -> dict[str, float]:
+    """Time each case after a warm-up call (cache hit)."""
+    timings = {}
+    for case in CASES:
+        operands = _build_operands(case)
+        subscripts = case["subscripts"] + "->" + case["output"]
+        fps.einsum_accumulation_cost(subscripts, *operands)
+        start = time.perf_counter()
+        for _ in range(100):
+            fps.einsum_accumulation_cost(subscripts, *operands)
+        elapsed = (time.perf_counter() - start) / 100
+        timings[case["name"]] = elapsed
+    return timings
+
+
+def main():
+    cold = bench_cold_call()
+    warm = bench_warm_call()
+    print("Cold-call latency (seconds):")
+    for name, t in cold.items():
+        print(f"  {name:40s} {t * 1000:8.2f} ms")
+    cold_total = sum(cold.values())
+    print(f"  {'TOTAL':40s} {cold_total * 1000:8.2f} ms")
+
+    print("\nWarm-call latency (averaged over 100 iterations):")
+    for name, t in warm.items():
+        print(f"  {name:40s} {t * 1e6:8.2f} µs")
+
+    if cold_total > COLD_CALL_BUDGET_SECONDS:
+        print(
+            f"\nFAIL: cold-call total {cold_total:.3f}s exceeds budget {COLD_CALL_BUDGET_SECONDS}s"
+        )
+        raise SystemExit(1)
+    print(
+        f"\nOK: cold-call total {cold_total:.3f}s within budget {COLD_CALL_BUDGET_SECONDS}s"
+    )
+
+
+if __name__ == "__main__":
+    main()

benchmarks/dashboard.py

@@ -0,0 +1,312 @@
+"""Benchmark dashboard with rich terminal and HTML output."""
+
+from __future__ import annotations
+
+import html as _html
+from io import StringIO
+from typing import Any
+
+_REDUCTION_NAMES = frozenset(
+    {
+        "sum",
+        "mean",
+        "std",
+        "var",
+        "max",
+        "min",
+        "prod",
+        "any",
+        "all",
+        "cumsum",
+        "cumprod",
+        "nansum",
+        "nanmean",
+        "nanstd",
+        "nanvar",
+        "nanmax",
+        "nanmin",
+        "nanprod",
+        "median",
+        "nanmedian",
+        "percentile",
+        "nanpercentile",
+        "quantile",
+        "nanquantile",
+        "count_nonzero",
+        "average",
+        "argmax",
+        "argmin",
+        "nancumprod",
+        "nancumsum",
+    }
+)
+_SORTING_NAMES = frozenset(
+    {
+        "sort",
+        "argsort",
+        "partition",
+        "argpartition",
+        "searchsorted",
+        "unique",
+        "lexsort",
+    }
+)
+_POLY_NAMES = frozenset(
+    {
+        "polyval",
+        "polyfit",
+        "polyder",
+        "polyint",
+        "polyadd",
+        "polysub",
+        "polymul",
+        "polydiv",
+        "poly",
+        "roots",
+    }
+)
+
+
+def _categorize_weights(weights: dict[str, float]) -> dict[str, dict[str, float]]:
+    """Group weights by category based on op_name prefix."""
+    categories: dict[str, dict[str, float]] = {}
+    for op, w in sorted(weights.items()):
+        if op.startswith("linalg."):
+            cat = "Linalg"
+        elif op.startswith("fft."):
+            cat = "FFT"
+        elif op.startswith("random."):
+            cat = "Random"
+        elif op in _REDUCTION_NAMES:
+            cat = "Reductions"
+        elif op in _SORTING_NAMES:
+            cat = "Sorting"
+        elif op in _POLY_NAMES:
+            cat = "Polynomial"
+        else:
+            cat = "Pointwise"
+        categories.setdefault(cat, {})[op] = w
+    return {k: v for k, v in categories.items() if v}
+
+
+def _bar_text(value: float, max_value: float, width: int = 20) -> str:
+    """Create a unicode bar string scaled to max_value."""
+    if max_value <= 0:
+        return ""
+    ratio = min(value / max_value, 1.0)
+    full = int(ratio * width)
+    return "\u2588" * max(full, 0)
+
+
+def render_terminal(
+    meta: dict[str, Any],
+    weights: dict[str, float],
+    baseline_fpe: float,
+    total_ops: int,
+    duration_seconds: float,
+) -> str:
+    """Render benchmark results to a terminal string.
+
+    Uses ``rich`` for styled output, falling back to plain text.
+    """
+    try:
+        return _render_terminal_rich(
+            meta, weights, baseline_fpe, total_ops, duration_seconds
+        )
+    except Exception:
+        return _render_terminal_plain(
+            meta, weights, baseline_fpe, total_ops, duration_seconds
+        )
+
+
+def _render_terminal_rich(
+    meta: dict[str, Any],
+    weights: dict[str, float],
+    baseline_fpe: float,
+    total_ops: int,
+    duration_seconds: float,
+) -> str:
+    from rich.console import Console
+    from rich.panel import Panel
+    from rich.table import Table
+
+    buf = StringIO()
+    console = Console(file=buf, force_terminal=True, width=100)
+
+    hw = meta.get("hardware", {})
+    sw = meta.get("software", {})
+    bc = meta.get("benchmark_config", {})
+    mode = bc.get("measurement_mode", "unknown")
+
+    header_lines = [
+        f"Timestamp : {meta.get('timestamp', 'N/A')[:19]}",
+        f"CPU       : {hw.get('cpu_model', 'N/A')} ({hw.get('cpu_cores', '?')} cores, {hw.get('arch', '?')})",
+        f"dtype     : {bc.get('dtype', 'N/A')}",
+        f"NumPy     : {sw.get('numpy', 'N/A')} ({sw.get('blas', 'N/A')})",
+        f"Mode      : {mode}"
+        + (" (wall-clock proxy)" if mode == "timing" else " (hardware counters)"),
+    ]
+    console.print(
+        Panel("\n".join(header_lines), title="flopscope FLOP Weight Benchmark")
+    )
+
+    categories = _categorize_weights(weights)
+
+    for cat_name, ops in categories.items():
+        # Scale bars per-category so each category uses its full width
+        cat_max = max(ops.values()) if ops else 1.0
+
+        table = Table(title=cat_name, show_lines=False)
+        table.add_column("Operation", style="cyan", min_width=25)
+        table.add_column("Weight", justify="right", style="green", min_width=10)
+        table.add_column("", min_width=22)  # bar column
+
+        for op, w in sorted(ops.items(), key=lambda x: -x[1]):
+            bar = _bar_text(w, cat_max, width=20)
+            table.add_row(op, f"{w:.2f}", f"[yellow]{bar}[/yellow]")
+        console.print(table)
+        console.print()
+
+    mins = int(duration_seconds // 60)
+    secs = int(duration_seconds % 60)
+    footer_lines = [
+        f"Baseline: np.add = 1.0 ({baseline_fpe:.4f} raw units/elem)",
+        f"Total operations benchmarked: {total_ops}",
+        f"Duration: {mins}m {secs:02d}s",
+    ]
+    console.print(Panel("\n".join(footer_lines), title="Summary"))
+
+    return buf.getvalue()
+
+
+def _render_terminal_plain(
+    meta: dict[str, Any],
+    weights: dict[str, float],
+    baseline_fpe: float,
+    total_ops: int,
+    duration_seconds: float,
+) -> str:
+    lines: list[str] = []
+    hw = meta.get("hardware", {})
+    sw = meta.get("software", {})
+    bc = meta.get("benchmark_config", {})
+
+    lines.append("=" * 70)
+    lines.append("  flopscope FLOP Weight Benchmark")
+    lines.append("=" * 70)
+    lines.append(f"  Timestamp : {meta.get('timestamp', 'N/A')[:19]}")
+    lines.append(f"  CPU       : {hw.get('cpu_model', 'N/A')}")
+    lines.append(f"  dtype     : {bc.get('dtype', 'N/A')}")
+    lines.append(f"  NumPy     : {sw.get('numpy', 'N/A')}")
+    lines.append("")
+
+    categories = _categorize_weights(weights)
+
+    for cat_name, ops in categories.items():
+        cat_max = max(ops.values()) if ops else 1.0
+        lines.append(f"  --- {cat_name} ---")
+        for op, w in sorted(ops.items(), key=lambda x: -x[1]):
+            bar = _bar_text(w, cat_max, width=20)
+            lines.append(f"    {op:<28s} {w:>10.2f}  {bar}")
+        lines.append("")
+
+    lines.append("-" * 70)
+    lines.append(f"  Baseline: np.add = {baseline_fpe:.4f} raw units/elem")
+    lines.append(f"  Total operations: {total_ops}")
+    lines.append(f"  Duration: {duration_seconds:.1f}s")
+    lines.append("=" * 70)
+
+    return "\n".join(lines)
+
+
+def render_html(
+    meta: dict[str, Any],
+    weights: dict[str, float],
+    baseline_fpe: float,
+    total_ops: int,
+    duration_seconds: float,
+) -> str:
+    """Render benchmark results as a standalone HTML document."""
+    hw = meta.get("hardware", {})
+    sw = meta.get("software", {})
+    bc = meta.get("benchmark_config", {})
+    categories = _categorize_weights(weights)
+
+    esc = _html.escape
+
+    parts: list[str] = []
+    parts.append("""\
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>flopscope FLOP Weight Benchmark</title>
+<style>
+  body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+         max-width: 960px; margin: 2rem auto; padding: 0 1rem; color: #333; }
+  h1 { border-bottom: 2px solid #2563eb; padding-bottom: .5rem; }
+  .meta { background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px;
+          padding: 1rem 1.5rem; margin-bottom: 1.5rem; line-height: 1.8; }
+  .meta b { min-width: 80px; display: inline-block; }
+  h2 { color: #1e40af; margin-top: 2rem; }
+  table { width: 100%; border-collapse: collapse; margin-bottom: 1.5rem; }
+  th { text-align: left; border-bottom: 2px solid #cbd5e1; padding: .5rem; }
+  td { padding: .4rem .5rem; border-bottom: 1px solid #e2e8f0; }
+  .bar-cell { width: 40%; }
+  .bar { height: 1.2rem; background: linear-gradient(90deg, #22c55e, #16a34a);
+         border-radius: 3px; min-width: 2px; }
+  .summary { background: #f0fdf4; border: 1px solid #bbf7d0; border-radius: 8px;
+             padding: 1rem 1.5rem; margin-top: 2rem; }
+</style>
+</head>
+<body>
+<h1>flopscope FLOP Weight Benchmark</h1>
+""")
+
+    parts.append('<div class="meta">')
+    parts.append(f"<b>Timestamp:</b> {esc(str(meta.get('timestamp', 'N/A'))[:19])}<br>")
+    parts.append(
+        f"<b>CPU:</b> {esc(str(hw.get('cpu_model', 'N/A')))} "
+        f"({hw.get('cpu_cores', '?')} cores, {esc(str(hw.get('arch', '?')))})<br>"
+    )
+    parts.append(f"<b>dtype:</b> {esc(str(bc.get('dtype', 'N/A')))} | ")
+    parts.append(
+        f"<b>NumPy:</b> {esc(str(sw.get('numpy', 'N/A')))} ({esc(str(sw.get('blas', 'N/A')))})<br>"
+    )
+    mode = bc.get("measurement_mode", "unknown")
+    mode_label = (
+        "hardware counters (perf)" if mode == "perf" else "wall-clock time (proxy)"
+    )
+    parts.append(f"<b>Mode:</b> {esc(mode_label)}")
+    parts.append("</div>")
+
+    for cat_name, ops in categories.items():
+        cat_max = max(ops.values()) if ops else 1.0
+        parts.append(f"<h2>{esc(cat_name)}</h2>")
+        parts.append(
+            "<table><tr><th>Operation</th><th>Weight</th>"
+            "<th class='bar-cell'></th></tr>"
+        )
+        for op, w in sorted(ops.items(), key=lambda x: -x[1]):
+            pct = (w / cat_max * 100) if cat_max > 0 else 0
+            parts.append(
+                f"<tr><td><code>{esc(op)}</code></td>"
+                f"<td style='text-align:right'>{w:.2f}</td>"
+                f'<td class="bar-cell">'
+                f'<div class="bar" style="width:{pct:.1f}%"></div>'
+                f"</td></tr>"
+            )
+        parts.append("</table>")
+
+    mins = int(duration_seconds // 60)
+    secs = int(duration_seconds % 60)
+    parts.append('<div class="summary">')
+    parts.append(
+        f"<b>Baseline:</b> np.add = 1.0 ({baseline_fpe:.4f} raw units/elem)<br>"
+    )
+    parts.append(f"<b>Total operations:</b> {total_ops}<br>")
+    parts.append(f"<b>Duration:</b> {mins}m {secs:02d}s")
+    parts.append("</div>")
+    parts.append("</body></html>")
+
+    return "\n".join(parts)