flopscope 0.2.0__py3-none-any.whl

benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark suite for measuring empirical FLOP weights."""

benchmarks/__main__.py

@@ -0,0 +1,6 @@
+# benchmarks/__main__.py
+"""Allow running benchmarks as: python -m benchmarks.runner"""
+
+from benchmarks.runner import main
+
+main()

benchmarks/_baseline.py

@@ -0,0 +1,171 @@
+"""Baseline measurements for overhead-subtracted weight normalization.
+
+Measures three baselines:
+
+1. **alpha(add)** — raw FP instructions per element for ``np.add`` (includes
+   ufunc overhead). Used to derive the binary ufunc overhead.
+2. **alpha(abs)** — raw FP instructions per element for ``np.abs``. Since abs
+   on float64 is a bitwise sign-bit clear (NOT an FP instruction), all
+   measured FP instructions are pure **unary ufunc overhead**.
+3. **Binary ufunc overhead** = ``alpha(add) - 1.0`` (since one add = exactly
+   one FP instruction; the rest is overhead).
+
+The runner subtracts the appropriate overhead from each counted operation's
+raw alpha before storing it as the weight::
+
+    weight(op) = max(alpha_raw(op) - overhead_for_category, 0.0)
+
+Known analytical zero-FLOP operations are stored separately with
+``weight(op) = 0.0`` so the published artifacts surface them as free rather
+than as unit-cost operations.
+
+This replaces the old ``weight(op) = alpha(op) / alpha(add)`` formula which
+penalized BLAS ops (that bypass the ufunc layer) with ufunc overhead they
+don't have.
+"""
+
+from __future__ import annotations
+
+import statistics
+from dataclasses import dataclass
+
+from benchmarks._perf import measure_flops
+
+
+@dataclass(frozen=True)
+class BaselineResult:
+    """All baseline measurements needed for overhead-subtracted normalization."""
+
+    alpha_add: float
+    """Raw alpha for np.add (FP instructions per element, including overhead)."""
+
+    alpha_abs: float
+    """Raw alpha for np.abs (pure unary ufunc overhead — abs is bitwise)."""
+
+    @property
+    def overhead_ufunc_unary(self) -> float:
+        """Unary ufunc overhead per element (from abs measurement)."""
+        return self.alpha_abs
+
+    @property
+    def overhead_ufunc_binary(self) -> float:
+        """Binary ufunc overhead per element.
+
+        Derived as alpha(add) - 1.0, since one add = exactly 1 FP instruction.
+        """
+        return max(self.alpha_add - 1.0, 0.0)
+
+    @property
+    def overhead_ufunc_reduction(self) -> float:
+        """Reduction ufunc overhead (same iterator structure as unary)."""
+        return self.alpha_abs
+
+    def overhead_for_mode(self, mode: str) -> float:
+        """Return the overhead to subtract for a given measurement mode."""
+        return {
+            "ufunc_unary": self.overhead_ufunc_unary,
+            "ufunc_binary": self.overhead_ufunc_binary,
+            "ufunc_reduction": self.overhead_ufunc_reduction,
+            "blas": 0.0,
+            "linalg": 0.0,
+            "custom": 0.0,
+            "instructions": 0.0,
+        }.get(mode, 0.0)
+
+    def to_dict(self) -> dict:
+        """Serialize for weights.json metadata."""
+        return {
+            "alpha_add_raw": self.alpha_add,
+            "alpha_abs_raw": self.alpha_abs,
+            "overhead_ufunc_unary": self.overhead_ufunc_unary,
+            "overhead_ufunc_binary": self.overhead_ufunc_binary,
+            "overhead_ufunc_reduction": self.overhead_ufunc_reduction,
+            "normalization": (
+                "subtract per-category ufunc overhead; known zero-FLOP ops use "
+                "weight 0.0"
+            ),
+        }
+
+
+def _measure_alpha(setups: list[str], bench: str, n: int, repeats: int) -> float:
+    """Measure median alpha across distributions."""
+    dist_alphas = []
+    for setup in setups:
+        result = measure_flops(setup, bench, repeats=repeats)
+        dist_alphas.append(result.total_flops / (n * repeats))
+    return statistics.median(dist_alphas)
+
+
+def _unary_setups(n: int, dtype: str) -> list[str]:
+    return [
+        f"x = np.random.default_rng(42).standard_normal({n}).astype(np.{dtype})",
+        (
+            f"rng = np.random.default_rng(42); "
+            f"x = rng.uniform(0.01, 100, size={n}).astype(np.{dtype})"
+        ),
+        (
+            f"rng = np.random.default_rng(42); "
+            f"x = rng.uniform(-1000, 1000, size={n}).astype(np.{dtype})"
+        ),
+    ]
+
+
+def _binary_setups(n: int, dtype: str) -> list[str]:
+    return [
+        (
+            f"x = np.random.default_rng(42).standard_normal({n}).astype(np.{dtype}); "
+            f"y = np.random.default_rng(43).standard_normal({n}).astype(np.{dtype}); "
+            f"_out = np.empty({n}, dtype=np.{dtype})"
+        ),
+        (
+            f"rng = np.random.default_rng(42); "
+            f"x = rng.uniform(0.01, 100, size={n}).astype(np.{dtype}); "
+            f"y = rng.uniform(0.01, 100, size={n}).astype(np.{dtype}); "
+            f"_out = np.empty({n}, dtype=np.{dtype})"
+        ),
+        (
+            f"rng = np.random.default_rng(42); "
+            f"x = rng.uniform(-1000, 1000, size={n}).astype(np.{dtype}); "
+            f"y = rng.uniform(-1000, 1000, size={n}).astype(np.{dtype}); "
+            f"_out = np.empty({n}, dtype=np.{dtype})"
+        ),
+    ]
+
+
+def measure_baseline(
+    n: int = 10_000_000, dtype: str = "float64", repeats: int = 10
+) -> float:
+    """Return alpha(add) for backwards compatibility.
+
+    Prefer :func:`measure_baselines` which returns the full
+    :class:`BaselineResult` with overhead measurements.
+    """
+    return _measure_alpha(
+        _binary_setups(n, dtype), "np.add(x, y, out=_out)", n, repeats
+    )
+
+
+def measure_baselines(
+    n: int = 10_000_000, dtype: str = "float64", repeats: int = 10
+) -> BaselineResult:
+    """Measure all baselines needed for overhead-subtracted normalization.
+
+    Returns
+    -------
+    BaselineResult
+        Contains alpha(add), alpha(abs), and derived overhead values.
+    """
+    alpha_add = _measure_alpha(
+        _binary_setups(n, dtype), "np.add(x, y, out=_out)", n, repeats
+    )
+    alpha_abs = _measure_alpha(_unary_setups(n, dtype), "np.abs(x)", n, repeats)
+
+    result = BaselineResult(alpha_add=alpha_add, alpha_abs=alpha_abs)
+    print(f"  alpha(add) = {alpha_add:.4f}")
+    print(f"  alpha(abs) = {alpha_abs:.4f} (pure unary ufunc overhead)")
+    print("  Derived overheads:")
+    print(f"    ufunc_unary:     {result.overhead_ufunc_unary:.4f}")
+    print(f"    ufunc_binary:    {result.overhead_ufunc_binary:.4f}")
+    print(f"    ufunc_reduction: {result.overhead_ufunc_reduction:.4f}")
+    print("    blas/linalg:     0.0000")
+    return result

benchmarks/_bitwise.py

@@ -0,0 +1,231 @@
+"""Benchmark bitwise and integer operations via ``instructions`` counter.
+
+These ops operate on integers, so ``fp_arith_inst_retired`` perf counters
+read 0. We use ``perf stat -e instructions`` (total retired instructions)
+as the hardware-counter fallback — more stable and deterministic than
+wall-clock timing. Falls back to timing if ``perf`` is unavailable.
+
+Also includes ``isnat`` which operates on datetime64 arrays.
+"""
+
+from __future__ import annotations
+
+import statistics
+
+from benchmarks._perf import measure_instructions
+
+# --- Operation lists -------------------------------------------------------
+
+UNARY_OPS: list[str] = [
+    "bitwise_not",
+    "bitwise_invert",
+    "bitwise_count",
+    "invert",
+]
+
+BINARY_OPS: list[str] = [
+    "bitwise_and",
+    "bitwise_or",
+    "bitwise_xor",
+    "gcd",
+    "lcm",
+]
+
+SHIFT_OPS: list[str] = [
+    "bitwise_left_shift",
+    "bitwise_right_shift",
+    "left_shift",
+    "right_shift",
+]
+
+SPECIAL_OPS: list[str] = [
+    "isnat",
+]
+
+BITWISE_OPS: list[str] = UNARY_OPS + BINARY_OPS + SHIFT_OPS + SPECIAL_OPS
+
+# --- Analytical formula strings (all cost = n) ----------------------------
+
+_FORMULA_STRINGS: dict[str, str] = dict.fromkeys(BITWISE_OPS, "n")
+
+
+def _analytical_cost(op: str, n: int) -> int:  # noqa: ARG001
+    """Return analytical FLOP cost for *op* on arrays of length *n*."""
+    return n
+
+
+# --- Setup helpers ----------------------------------------------------------
+
+
+def _unary_setup(n: int, dist_idx: int) -> str:
+    """Build setup code for a unary integer op."""
+    seeds = [42, 123, 7]
+    seed = seeds[dist_idx]
+    return (
+        f"import numpy as np; "
+        f"x = np.random.default_rng({seed}).integers(-1_000_000, 1_000_000, "
+        f"size={n}, dtype=np.int64)"
+    )
+
+
+def _binary_setup(n: int, dist_idx: int) -> str:
+    """Build setup code for a binary integer op."""
+    seeds = [42, 123, 7]
+    seed = seeds[dist_idx]
+    return (
+        f"import numpy as np; "
+        f"rng = np.random.default_rng({seed}); "
+        f"a = rng.integers(-1_000_000, 1_000_000, size={n}, dtype=np.int64); "
+        f"b = rng.integers(-1_000_000, 1_000_000, size={n}, dtype=np.int64)"
+    )
+
+
+def _gcd_lcm_setup(n: int, dist_idx: int) -> str:
+    """Build setup code for gcd/lcm (positive integers)."""
+    seeds = [42, 123, 7]
+    seed = seeds[dist_idx]
+    return (
+        f"import numpy as np; "
+        f"rng = np.random.default_rng({seed}); "
+        f"a = rng.integers(1, 1_000_000, size={n}, dtype=np.int64); "
+        f"b = rng.integers(1, 1_000_000, size={n}, dtype=np.int64)"
+    )
+
+
+def _shift_setup(n: int, dist_idx: int) -> str:
+    """Build setup code for shift ops (second operand 0-10)."""
+    seeds = [42, 123, 7]
+    seed = seeds[dist_idx]
+    return (
+        f"import numpy as np; "
+        f"rng = np.random.default_rng({seed}); "
+        f"a = rng.integers(-1_000_000, 1_000_000, size={n}, dtype=np.int64); "
+        f"b = rng.integers(0, 11, size={n}, dtype=np.int64)"
+    )
+
+
+def _isnat_setup(n: int, dist_idx: int) -> str:
+    """Build setup code for isnat (datetime64 input with some NaTs)."""
+    seeds = [42, 123, 7]
+    seeds[dist_idx]
+    # Create datetime64 array with ~1/3 NaT values
+    return (
+        f"import numpy as np; "
+        f"x = np.array(['2020-01-01', 'NaT', '2020-06-15'] * ({n} // 3), "
+        f"dtype='datetime64')"
+    )
+
+
+# --- Main benchmark function -----------------------------------------------
+
+
+def benchmark_bitwise(
+    n: int = 10_000_000,
+    dtype: str = "int64",
+    repeats: int = 10,
+) -> tuple[dict[str, float], dict[str, dict]]:
+    """Benchmark bitwise/integer ops using timing mode only.
+
+    Parameters
+    ----------
+    n : int
+        Array size.
+    dtype : str
+        Ignored (always uses int64 for bitwise ops). Kept for interface
+        consistency with other benchmark modules.
+    repeats : int
+        Number of repetitions per measurement.
+
+    Returns
+    -------
+    tuple[dict[str, float], dict[str, dict]]
+        A pair of (alphas, details). ``alphas`` maps op name to median
+        timing per element (nanoseconds). ``details`` maps op name to a
+        dict of raw benchmark metadata.
+    """
+    distributions = 3
+    results: dict[str, float] = {}
+    details: dict[str, dict] = {}
+
+    def _bench_op(
+        op: str, setup_fn, bench_code: str, category: str, size_desc: str
+    ) -> None:
+        """Benchmark a single op across distributions using instructions counter."""
+        dist_values: list[float] = []
+        dist_raw_totals: list[int] = []
+        for di in range(distributions):
+            setup = setup_fn(n, di)
+            try:
+                result = measure_instructions(setup, bench_code, repeats=repeats)
+            except RuntimeError:
+                continue
+            dist_values.append(result.total_flops / (n * repeats))
+            dist_raw_totals.append(result.total_flops)
+        if dist_values:
+            results[op] = statistics.median(dist_values)
+            details[op] = {
+                "category": category,
+                "measurement_mode": "instructions",
+                "analytical_formula": _FORMULA_STRINGS[op],
+                "analytical_flops": n,
+                "benchmark_size": size_desc,
+                "bench_code": bench_code,
+                "repeats": repeats,
+                "perf_instructions_total": dist_raw_totals,
+                "distribution_alphas": dist_values,
+            }
+
+    # --- Unary ops ---
+    for op in UNARY_OPS:
+        _bench_op(op, _unary_setup, f"np.{op}(x)", "instructions_unary", f"x: ({n},)")
+
+    # --- Binary ops ---
+    for op in BINARY_OPS:
+        setup_fn = _gcd_lcm_setup if op in ("gcd", "lcm") else _binary_setup
+        _bench_op(
+            op,
+            setup_fn,
+            f"np.{op}(a, b)",
+            "instructions_binary",
+            f"a: ({n},), b: ({n},)",
+        )
+
+    # --- Shift ops ---
+    for op in SHIFT_OPS:
+        _bench_op(
+            op,
+            _shift_setup,
+            f"np.{op}(a, b)",
+            "instructions_shift",
+            f"a: ({n},), b: ({n},) (values 0-10)",
+        )
+
+    # --- Special ops ---
+    # isnat: operates on datetime64 arrays
+    op = "isnat"
+    dist_values: list[float] = []
+    dist_raw_totals: list[int] = []
+    bench = "np.isnat(x)"
+    for di in range(distributions):
+        setup = _isnat_setup(n, di)
+        try:
+            result = measure_instructions(setup, bench, repeats=repeats)
+        except RuntimeError:
+            continue
+        dist_values.append(result.total_flops / (n * repeats))
+        dist_raw_totals.append(result.total_flops)
+    if dist_values:
+        results[op] = statistics.median(dist_values)
+        details[op] = {
+            "category": "instructions_special",
+            "measurement_mode": "instructions",
+            "analytical_formula": _FORMULA_STRINGS[op],
+            "analytical_flops": n,
+            "benchmark_size": f"x: ({n},) datetime64 with NaTs",
+            "bench_code": bench,
+            "repeats": repeats,
+            "perf_instructions_total": dist_raw_totals,
+            "distribution_alphas": dist_values,
+        }
+
+    return results, details

benchmarks/_complex.py

@@ -0,0 +1,176 @@
+"""Benchmark complex-number operations.
+
+Most ops use perf mode with complex128 input (they DO retire FP instructions
+on complex data).  Two type-check ops (``iscomplexobj``, ``isrealobj``) use
+the ``instructions`` counter because they inspect the dtype, not the array
+elements (so ``fp_arith_inst_retired`` reads 0).
+"""
+
+from __future__ import annotations
+
+import statistics
+
+from benchmarks._perf import measure_flops, measure_instructions
+
+COMPLEX_OPS: list[str] = [
+    "angle",
+    "conj",
+    "conjugate",
+    "imag",
+    "real",
+    "real_if_close",
+    "iscomplex",
+    "isreal",
+    "sort_complex",
+    "iscomplexobj",
+    "isrealobj",
+]
+
+# Ops that use the ``instructions`` counter instead of ``fp_arith_inst_retired``
+# because they inspect the dtype, not the array elements.
+_INSTRUCTIONS_OPS: frozenset[str] = frozenset({"iscomplexobj", "isrealobj"})
+
+_FORMULA_STRINGS: dict[str, str] = {
+    "angle": "numel(output)",
+    "conj": "numel(output)",
+    "conjugate": "numel(output)",
+    "imag": "numel(output)",
+    "real": "numel(output)",
+    "real_if_close": "numel(output)",
+    "iscomplex": "numel(output)",
+    "isreal": "numel(output)",
+    "sort_complex": "numel(output)",
+    "iscomplexobj": "numel(output)",
+    "isrealobj": "numel(output)",
+}
+
+# Seeds for the 3 input distributions.
+_DIST_SEEDS: list[tuple[int, int]] = [
+    (42, 43),
+    (100, 101),
+    (200, 201),
+]
+
+
+def _complex_setup(n: int, seed_real: int, seed_imag: int) -> str:
+    """Build setup code that creates a complex128 array from two RNGs."""
+    return (
+        f"import numpy as np; "
+        f"x = np.random.default_rng({seed_real}).standard_normal({n}).astype(np.float64) "
+        f"+ 1j * np.random.default_rng({seed_imag}).standard_normal({n}).astype(np.float64)"
+    )
+
+
+def _real_if_close_setup(n: int, dist_idx: int) -> str:
+    """Build setup for ``real_if_close``.
+
+    Distribution 0 has negligible imaginary parts (tests the "close to real"
+    path).  Distributions 1 and 2 have substantial imaginary parts.
+    """
+    if dist_idx == 0:
+        # Negligible imaginary part — real_if_close may strip it.
+        return (
+            f"import numpy as np; "
+            f"x = np.random.default_rng(42).standard_normal({n}).astype(np.float64) "
+            f"+ 1j * np.random.default_rng(43).standard_normal({n}).astype(np.float64) * 1e-15"
+        )
+    seed_r, seed_i = _DIST_SEEDS[dist_idx]
+    return _complex_setup(n, seed_r, seed_i)
+
+
+def _timing_setup(n: int) -> str:
+    """Build setup for type-check ops (``iscomplexobj`` / ``isrealobj``)."""
+    return (
+        f"import numpy as np; "
+        f"x = np.random.default_rng(42).standard_normal({n}) "
+        f"+ 1j * np.random.default_rng(43).standard_normal({n})"
+    )
+
+
+def _bench_code(op: str) -> str:
+    """Return the benchmark statement for *op*."""
+    return f"np.{op}(x)"
+
+
+def benchmark_complex(
+    n: int = 10_000_000,
+    dtype: str = "complex128",
+    repeats: int = 10,
+    distributions: int = 3,
+) -> tuple[dict[str, float], dict[str, dict]]:
+    """Benchmark complex-number ops, returning raw measurement per element.
+
+    Parameters
+    ----------
+    n : int
+        Array size (element count).
+    dtype : str
+        NumPy dtype string (unused — always complex128, kept for API parity).
+    repeats : int
+        Number of repetitions per measurement.
+    distributions : int
+        Number of input distributions to measure (median is taken).
+
+    Returns
+    -------
+    tuple[dict[str, float], dict[str, dict]]
+        ``(alphas, details)`` — *alphas* maps op name to median measurement
+        per analytical FLOP; *details* maps op name to benchmark metadata.
+    """
+    results: dict[str, float] = {}
+    details: dict[str, dict] = {}
+
+    for op in COMPLEX_OPS:
+        # --- Determine n for this op ---
+        op_n = 1_000_000 if op == "sort_complex" else n
+
+        # --- Choose measurement function ---
+        use_instructions = op in _INSTRUCTIONS_OPS
+        measure_fn = measure_instructions if use_instructions else measure_flops
+
+        dist_values: list[float] = []
+        dist_raw_totals: list[int] = []
+        bench = _bench_code(op)
+
+        for di in range(distributions):
+            # --- Build setup code ---
+            if use_instructions:
+                setup = _timing_setup(op_n)
+            elif op == "real_if_close":
+                setup = _real_if_close_setup(op_n, di)
+            else:
+                seed_r, seed_i = _DIST_SEEDS[di]
+                setup = _complex_setup(op_n, seed_r, seed_i)
+
+            try:
+                result = measure_fn(setup, bench, repeats=repeats)
+            except RuntimeError:
+                continue
+
+            # Analytical cost = numel(output) = op_n for all complex ops.
+            analytical = op_n
+            dist_values.append(result.total_flops / (analytical * repeats))
+            dist_raw_totals.append(result.total_flops)
+
+        if dist_values:
+            results[op] = statistics.median(dist_values)
+
+            if use_instructions:
+                bm_size = f"x: ({op_n},) complex128 (instructions counter)"
+            else:
+                bm_size = f"x: ({op_n},) complex128"
+
+            mm = "instructions" if op in _INSTRUCTIONS_OPS else "ufunc_unary"
+            details[op] = {
+                "category": "counted_complex",
+                "measurement_mode": mm,
+                "analytical_formula": _FORMULA_STRINGS[op],
+                "analytical_flops": op_n,
+                "benchmark_size": bm_size,
+                "bench_code": bench,
+                "repeats": repeats,
+                "perf_instructions_total": dist_raw_totals,
+                "distribution_alphas": dist_values,
+            }
+
+    return results, details