flopscope 0.2.0__py3-none-any.whl

benchmarks/_perf.py

@@ -0,0 +1,321 @@
+"""Measure floating-point work for benchmark operations.
+
+Primary method: Linux ``perf stat`` hardware counters (exact FP op counts).
+Fallback: wall-clock time measurement (relative proxy, works everywhere).
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+PERF_EVENTS = [
+    "fp_arith_inst_retired.scalar_double",
+    "fp_arith_inst_retired.128b_packed_double",
+    "fp_arith_inst_retired.256b_packed_double",
+    "fp_arith_inst_retired.512b_packed_double",
+]
+
+# Mapping from event name suffix to SIMD width multiplier.
+_WIDTH = {
+    "scalar_double": 1,
+    "128b_packed_double": 2,
+    "256b_packed_double": 4,
+    "512b_packed_double": 8,
+}
+
+
+@dataclass(frozen=True)
+class PerfResult:
+    """Counts of retired floating-point arithmetic instructions."""
+
+    scalar_double: int
+    packed_128_double: int
+    packed_256_double: int
+    packed_512_double: int
+
+    @property
+    def total_flops(self) -> int:
+        """Total double-precision FLOPs, weighted by SIMD width."""
+        return (
+            self.scalar_double * 1
+            + self.packed_128_double * 2
+            + self.packed_256_double * 4
+            + self.packed_512_double * 8
+        )
+
+
+@dataclass(frozen=True)
+class TimingResult:
+    """Wall-clock timing result used as fallback when perf is unavailable.
+
+    Stores elapsed nanoseconds. Consumers use ``total_flops`` which returns
+    the raw nanosecond value — the normalization step (op_time / add_time)
+    in the runner cancels units, producing valid relative weights.
+    """
+
+    elapsed_ns: int
+
+    @property
+    def total_flops(self) -> int:
+        """Return elapsed nanoseconds as a proxy for FP work.
+
+        This is intentionally named ``total_flops`` so that all benchmark
+        modules can use the same interface regardless of measurement mode.
+        The values are only meaningful as ratios (normalized against the
+        baseline ``np.add`` measurement).
+        """
+        return self.elapsed_ns
+
+
+@dataclass(frozen=True)
+class InstructionsResult:
+    """Total retired instructions measured via ``perf stat -e instructions``.
+
+    Used as a hardware-counter fallback for integer/bitwise operations
+    where ``fp_arith_inst_retired`` reads 0. More stable than wall-clock
+    timing because it is deterministic and independent of system load.
+
+    Like ``TimingResult``, the ``total_flops`` property returns the raw
+    instruction count — normalization against the baseline (``np.add``)
+    in the runner cancels units, producing valid relative weights.
+    """
+
+    instructions: int
+
+    @property
+    def total_flops(self) -> int:
+        """Return total retired instructions as a proxy for work."""
+        return self.instructions
+
+
+# Union type for all measurement modes
+MeasureResult = PerfResult | TimingResult | InstructionsResult
+
+
+def has_perf() -> bool:
+    """Return True if the ``perf`` binary is on PATH.
+
+    The check can be overridden by setting the environment variable
+    ``FLOPSCOPE_FORCE_TIMING=1``, which forces timing mode regardless
+    of whether ``perf`` is available.  This is used by the dual-mode
+    validation loop to re-run benchmarks in timing mode.
+    """
+    if os.environ.get("FLOPSCOPE_FORCE_TIMING") == "1":
+        return False
+    return shutil.which("perf") is not None
+
+
+def measurement_mode() -> str:
+    """Return the active measurement mode: ``'perf'`` or ``'timing'``."""
+    return "perf" if has_perf() else "timing"
+
+
+def _parse_perf_csv(output: str) -> PerfResult:
+    """Parse the CSV output produced by ``perf stat -x ,``."""
+    counts: dict[str, int] = {}
+    for line in output.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split(",")
+        if len(parts) < 3:
+            continue
+        raw_count = parts[0].strip()
+        event_name = parts[2].strip()
+        # Match against known events.
+        for evt in PERF_EVENTS:
+            if evt == event_name:
+                try:
+                    counts[evt] = int(raw_count)
+                except (ValueError, TypeError):
+                    # <not supported> or similar
+                    counts[evt] = 0
+                break
+
+    return PerfResult(
+        scalar_double=counts.get(PERF_EVENTS[0], 0),
+        packed_128_double=counts.get(PERF_EVENTS[1], 0),
+        packed_256_double=counts.get(PERF_EVENTS[2], 0),
+        packed_512_double=counts.get(PERF_EVENTS[3], 0),
+    )
+
+
+def _build_script(setup_code: str, bench_code: str, repeats: int) -> str:
+    """Build the benchmark Python script content."""
+    return (
+        "import numpy as np\n"
+        f"{setup_code}\n"
+        f"for _i in range({repeats}):\n"
+        f"    {bench_code}\n"
+    )
+
+
+def _measure_perf(setup_code: str, bench_code: str, repeats: int) -> PerfResult:
+    """Measure using Linux perf stat hardware counters."""
+    script = _build_script(setup_code, bench_code, repeats)
+
+    tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
+    try:
+        tmp.write(script)
+        tmp.close()
+
+        events_arg = ",".join(PERF_EVENTS)
+        proc = subprocess.run(
+            [
+                "perf",
+                "stat",
+                "-e",
+                events_arg,
+                "-x",
+                ",",
+                sys.executable,
+                tmp.name,
+            ],
+            capture_output=True,
+            text=True,
+        )
+        return _parse_perf_csv(proc.stderr)
+    finally:
+        Path(tmp.name).unlink(missing_ok=True)
+
+
+def _measure_timing(setup_code: str, bench_code: str, repeats: int) -> TimingResult:
+    """Measure using wall-clock time in a subprocess."""
+    script = (
+        "import time\n"
+        "import numpy as np\n"
+        f"{setup_code}\n"
+        "# Warmup\n"
+        f"for _i in range(2):\n"
+        f"    {bench_code}\n"
+        "# Timed run\n"
+        "_t0 = time.perf_counter_ns()\n"
+        f"for _i in range({repeats}):\n"
+        f"    {bench_code}\n"
+        "_t1 = time.perf_counter_ns()\n"
+        "print(_t1 - _t0)\n"
+    )
+
+    tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
+    try:
+        tmp.write(script)
+        tmp.close()
+
+        proc = subprocess.run(
+            [sys.executable, tmp.name],
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"Benchmark subprocess failed (exit {proc.returncode}):\n"
+                f"stderr: {proc.stderr}"
+            )
+        elapsed_ns = int(proc.stdout.strip())
+        return TimingResult(elapsed_ns=elapsed_ns)
+    finally:
+        Path(tmp.name).unlink(missing_ok=True)
+
+
+def _measure_instructions(
+    setup_code: str, bench_code: str, repeats: int
+) -> InstructionsResult:
+    """Measure total retired instructions via ``perf stat -e instructions``.
+
+    This is a hardware counter that counts all retired instructions (integer,
+    FP, branch, load/store). It is deterministic and independent of system
+    load, making it a better fallback than wall-clock timing for integer
+    and bitwise operations where ``fp_arith_inst_retired`` reads 0.
+    """
+    script = _build_script(setup_code, bench_code, repeats)
+
+    tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False)
+    try:
+        tmp.write(script)
+        tmp.close()
+
+        proc = subprocess.run(
+            [
+                "perf",
+                "stat",
+                "-e",
+                "instructions",
+                "-x",
+                ",",
+                sys.executable,
+                tmp.name,
+            ],
+            capture_output=True,
+            text=True,
+        )
+        # Parse CSV: first field is count, third is event name
+        instructions = 0
+        for line in proc.stderr.splitlines():
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split(",")
+            if len(parts) >= 3 and "instructions" in parts[2]:
+                try:
+                    instructions = int(parts[0].strip())
+                except (ValueError, TypeError):
+                    pass
+                break
+        return InstructionsResult(instructions=instructions)
+    finally:
+        Path(tmp.name).unlink(missing_ok=True)
+
+
+def measure_flops(
+    setup_code: str,
+    bench_code: str,
+    repeats: int = 10,
+) -> MeasureResult:
+    """Measure FP work for a benchmark operation.
+
+    Uses ``perf stat`` hardware counters when available (Linux). Falls back
+    to wall-clock time measurement on other platforms. Both return an object
+    with a ``total_flops`` property — for perf mode this is actual FP ops,
+    for timing mode it is elapsed nanoseconds (valid as a relative proxy
+    when normalized against the baseline).
+
+    Parameters
+    ----------
+    setup_code:
+        Python code executed once before the hot loop (numpy is
+        already imported as ``np``).
+    bench_code:
+        Python code executed *repeats* times inside the hot loop.
+    repeats:
+        Number of iterations of the hot loop.
+    """
+    if has_perf():
+        return _measure_perf(setup_code, bench_code, repeats)
+    return _measure_timing(setup_code, bench_code, repeats)
+
+
+def measure_instructions(
+    setup_code: str,
+    bench_code: str,
+    repeats: int = 10,
+) -> InstructionsResult:
+    """Measure total retired instructions for a benchmark operation.
+
+    Uses ``perf stat -e instructions`` hardware counter. This is the
+    preferred fallback for integer/bitwise operations where
+    ``fp_arith_inst_retired`` reads 0. Requires ``perf`` to be available.
+
+    Falls back to timing if ``perf`` is not available.
+    """
+    if shutil.which("perf") is not None:
+        return _measure_instructions(setup_code, bench_code, repeats)
+    # If perf isn't available at all, fall back to timing
+    timing = _measure_timing(setup_code, bench_code, repeats)
+    return InstructionsResult(instructions=timing.elapsed_ns)

benchmarks/_perm_group_calibration.py

@@ -0,0 +1,175 @@
+"""Calibration script for the dimino_budget setting.
+
+Measures cold ``_dimino`` enumeration time plus ``burnside_unique_count``
+time across representative permutation groups, then prints a
+recommendation for ``dimino_budget``.
+
+Run with::
+
+    python -m benchmarks._perm_group_calibration
+    python -m benchmarks._perm_group_calibration --budget-ms 50 \\
+        --output benchmarks/_perm_group_calibration.json
+
+Output is informational. Users on faster/slower machines run this and
+``flops.configure(dimino_budget=<recommended>)``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import platform
+import time
+from dataclasses import dataclass
+
+from flopscope._perm_group import SymmetryGroup, _dimino
+
+
+@dataclass(frozen=True)
+class Measurement:
+    label: str
+    group_order: int
+    degree: int
+    dimino_ms: float
+    burnside_ms: float
+
+    @property
+    def total_ms(self) -> float:
+        return self.dimino_ms + self.burnside_ms
+
+
+def _measure_one(group: SymmetryGroup, label: str) -> Measurement:
+    """Time cold ``_dimino`` + Burnside enumeration on one group."""
+    # Cold dimino: bypass the cache by calling _dimino directly.
+    t = time.perf_counter()
+    _dimino(group._generators)
+    dimino_ms = (time.perf_counter() - t) * 1000.0
+
+    # Burnside: realistic size_dict (uniform 4-dim across all axes).
+    size_dict = dict.fromkeys(range(group.degree), 4)
+    t = time.perf_counter()
+    group.burnside_unique_count(size_dict)
+    burnside_ms = (time.perf_counter() - t) * 1000.0
+
+    return Measurement(
+        label=label,
+        group_order=group.order(),
+        degree=group.degree,
+        dimino_ms=dimino_ms,
+        burnside_ms=burnside_ms,
+    )
+
+
+def _sample_groups() -> list[tuple[str, SymmetryGroup]]:
+    """Return the representative group sample for calibration."""
+    samples: list[tuple[str, SymmetryGroup]] = []
+    for n in (3, 4, 5, 6, 7, 8, 9):
+        samples.append((f"S_{n}", SymmetryGroup.symmetric(axes=tuple(range(n)))))
+    for n in (4, 8, 16, 32, 64):
+        samples.append((f"C_{n}", SymmetryGroup.cyclic(axes=tuple(range(n)))))
+    for n in (4, 8, 16, 32, 64):
+        samples.append((f"D_{n}", SymmetryGroup.dihedral(axes=tuple(range(n)))))
+    samples.append(
+        (
+            "S_3 x S_3",
+            SymmetryGroup.direct_product(
+                SymmetryGroup.symmetric(axes=(0, 1, 2)),
+                SymmetryGroup.symmetric(axes=(3, 4, 5)),
+            ),
+        )
+    )
+    samples.append(
+        (
+            "S_4 x S_4",
+            SymmetryGroup.direct_product(
+                SymmetryGroup.symmetric(axes=(0, 1, 2, 3)),
+                SymmetryGroup.symmetric(axes=(4, 5, 6, 7)),
+            ),
+        )
+    )
+    samples.append(
+        (
+            "C_5 x C_5 x C_5",
+            SymmetryGroup.direct_product(
+                SymmetryGroup.cyclic(axes=(0, 1, 2, 3, 4)),
+                SymmetryGroup.cyclic(axes=(5, 6, 7, 8, 9)),
+                SymmetryGroup.cyclic(axes=(10, 11, 12, 13, 14)),
+            ),
+        )
+    )
+    return samples
+
+
+def _print_table(measurements: list[Measurement]) -> None:
+    header = f"{'group':<22}{'|G|':>12}{'dimino':>12}{'burnside':>12}{'total':>12}"
+    print(header)
+    print("-" * len(header))
+    for m in measurements:
+        print(
+            f"{m.label:<22}"
+            f"{m.group_order:>12}"
+            f"{m.dimino_ms:>10.2f}ms"
+            f"{m.burnside_ms:>10.2f}ms"
+            f"{m.total_ms:>10.2f}ms"
+        )
+
+
+def _recommend_budget(measurements: list[Measurement], budget_ms: float) -> int:
+    """Largest |G| whose total cold cost stays under ``budget_ms``."""
+    under_budget = [m for m in measurements if m.total_ms <= budget_ms]
+    if not under_budget:
+        return 1  # nothing fits; degenerate machine
+    return max(m.group_order for m in under_budget)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--budget-ms",
+        type=float,
+        default=100.0,
+        help="Wall-clock budget per group construction (default: 100ms)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Optional JSON output path",
+    )
+    args = parser.parse_args()
+
+    samples = _sample_groups()
+    measurements = [_measure_one(g, label) for label, g in samples]
+
+    _print_table(measurements)
+    recommended = _recommend_budget(measurements, args.budget_ms)
+    print()
+    print(
+        f"Recommended dimino_budget: {recommended}   "
+        f"(group_order_budget_ms={args.budget_ms}, machine: {platform.platform()})"
+    )
+
+    if args.output:
+        payload = {
+            "machine": platform.platform(),
+            "budget_ms": args.budget_ms,
+            "recommended_dimino_budget": recommended,
+            "measurements": [
+                {
+                    "label": m.label,
+                    "group_order": m.group_order,
+                    "degree": m.degree,
+                    "dimino_ms": m.dimino_ms,
+                    "burnside_ms": m.burnside_ms,
+                    "total_ms": m.total_ms,
+                }
+                for m in measurements
+            ],
+        }
+        with open(args.output, "w") as f:
+            json.dump(payload, f, indent=2)
+        print(f"Wrote calibration data to {args.output}")
+
+
+if __name__ == "__main__":
+    main()