codegreen 0.3.2__cp313-cp313-win_amd64.whl

benchmark/__init__.py

@@ -0,0 +1,13 @@
+"""CodeGreen Benchmarking Module - Energy measurement benchmarking harness."""
+from benchmark.config import BenchmarkConfig, Problem, RunResult
+from benchmark.harness import BenchmarkHarness
+from benchmark.profilers import CodeGreenProfiler, PerfProfiler
+from benchmark.results import ResultCollector, StatisticalAnalysis, ComparisonReport
+from benchmark.suites import get_suite, SUITES
+
+__all__ = [
+    "BenchmarkConfig", "Problem", "RunResult",
+    "BenchmarkHarness", "CodeGreenProfiler", "PerfProfiler",
+    "ResultCollector", "StatisticalAnalysis", "ComparisonReport",
+    "get_suite", "SUITES",
+]

benchmark/__main__.py

@@ -0,0 +1,53 @@
+"""Entry point for `python3 -m benchmark`."""
+import argparse
+import sys
+from pathlib import Path
+from benchmark.config import BenchmarkConfig
+from benchmark.harness import BenchmarkHarness
+
+def main():
+    parser = argparse.ArgumentParser(description="CodeGreen Energy Benchmark Suite")
+    parser.add_argument("--problems", nargs="+", help="Problems to run (default: all)")
+    parser.add_argument("--languages", nargs="+", default=["python"], help="Languages (default: python)")
+    parser.add_argument("--sizes", nargs="+", help="Input sizes (default: from config)")
+    parser.add_argument("--profilers", nargs="+", default=["codegreen"], help="Profilers (default: codegreen)")
+    parser.add_argument("--repetitions", type=int, default=5, help="Repetitions per run (default: 5)")
+    parser.add_argument("--output", type=Path, help="Output JSON file path")
+    parser.add_argument("--csv", type=Path, help="Output CSV file path")
+    parser.add_argument("--config", type=Path, help="YAML config file")
+    args = parser.parse_args()
+
+    config = BenchmarkConfig.from_yaml(args.config) if args.config else BenchmarkConfig.default()
+    harness = BenchmarkHarness(config)
+
+    try:
+        collector = harness.run_suite(
+            problems=args.problems,
+            languages=args.languages,
+            sizes=args.sizes,
+            profilers=args.profilers,
+            repetitions=args.repetitions,
+        )
+        summaries = collector.summarize_all()
+        print(f"\n{'='*60}")
+        print(f"Results: {len(collector.results)} runs")
+        print(f"{'='*60}")
+        for key, data in sorted(summaries.items()):
+            e = data.get("energy")
+            t = data.get("time")
+            valid = data.get("valid_runs", 0)
+            total = data.get("total_runs", 0)
+            energy_str = f"{e.mean:.4f} +/- {e.std:.4f} J" if e else "N/A"
+            print(f"  {key}: energy={energy_str}  time={t.mean:.3f}s  valid={valid}/{total}")
+
+        if args.output:
+            collector.to_json(args.output)
+            print(f"\nJSON saved: {args.output}")
+        if args.csv:
+            collector.to_csv(args.csv)
+            print(f"CSV saved: {args.csv}")
+    finally:
+        harness.cleanup()
+
+if __name__ == "__main__":
+    main()

benchmark/compilers.py

@@ -0,0 +1,64 @@
+"""Compiler management for C/C++/Java benchmarks."""
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import List, Optional
+from benchmark.config import LanguageEnv
+
+class CompilerManager:
+    def __init__(self, build_dir: Optional[Path] = None):
+        self.build_dir = build_dir or Path(tempfile.mkdtemp(prefix="codegreen_bench_"))
+        self.build_dir.mkdir(parents=True, exist_ok=True)
+
+    def compile(self, source: Path, env: LanguageEnv) -> Path:
+        if not env.compiler:
+            return source
+        lang = self._detect_language(source, env)
+        if lang == "java":
+            return self._compile_java(source, env)
+        return self._compile_native(source, env, lang)
+
+    def _detect_language(self, source: Path, env: LanguageEnv) -> str:
+        if env.compiler == "javac":
+            return "java"
+        if env.compiler in ("g++", "clang++"):
+            return "cpp"
+        return "c"
+
+    def _compile_native(self, source: Path, env: LanguageEnv, lang: str) -> Path:
+        binary = self.build_dir / source.stem
+        extra_flags = []
+        if source.suffix in (".gpp", ".gcc") or ".gpp" in source.name or ".gcc" in source.name:
+            extra_flags = ["-x", "c++" if lang == "cpp" else "c"]
+        # Separate linker flags (like -lm) from compiler flags - linker flags go at end
+        compiler_flags = [f for f in env.flags if not f.startswith('-l')]
+        linker_flags = [f for f in env.flags if f.startswith('-l')]
+        cmd = [env.compiler] + extra_flags + compiler_flags + [str(source), "-o", str(binary)] + linker_flags
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+        if result.returncode != 0:
+            raise RuntimeError(f"Compilation failed: {result.stderr}")
+        return binary
+
+    def _compile_java(self, source: Path, env: LanguageEnv) -> Path:
+        cmd = [env.compiler] + env.flags + ["-d", str(self.build_dir), str(source)]
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+        if result.returncode != 0:
+            raise RuntimeError(f"Java compilation failed: {result.stderr}")
+        class_name = source.stem
+        return self.build_dir / f"{class_name}.class"
+
+    def get_run_command(self, source: Path, binary: Path, env: LanguageEnv, args: List[str]) -> List[str]:
+        if not env.compiler:
+            cmd_str = env.run_cmd.format(source=str(source))
+            return cmd_str.split() + args
+        if env.compiler == "javac":
+            class_name = source.stem
+            cmd_str = env.run_cmd.format(build_dir=str(self.build_dir), class_name=class_name)
+            return cmd_str.split() + args
+        cmd_str = env.run_cmd.format(binary=str(binary))
+        return cmd_str.split() + args
+
+    def cleanup(self):
+        import shutil
+        if self.build_dir.exists():
+            shutil.rmtree(self.build_dir, ignore_errors=True)

benchmark/config.py

@@ -0,0 +1,131 @@
+"""Benchmark configuration dataclasses and YAML loader."""
+import platform
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+@dataclass
+class SystemState:
+    cpu_model: str = ""
+    cpu_governor: str = ""
+    kernel_version: str = ""
+    rapl_domains: List[str] = field(default_factory=list)
+
+    @classmethod
+    def capture(cls) -> "SystemState":
+        state = cls()
+        state.kernel_version = platform.release()
+        try:
+            with open("/proc/cpuinfo") as f:
+                for line in f:
+                    if line.startswith("model name"):
+                        state.cpu_model = line.split(":", 1)[1].strip()
+                        break
+        except (FileNotFoundError, OSError):
+            pass
+        try:
+            with open("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") as f:
+                state.cpu_governor = f.read().strip()
+        except (FileNotFoundError, OSError):
+            pass
+        import os
+        for i in range(10):
+            path = f"/sys/class/powercap/intel-rapl:0:{i}/name"
+            if os.path.exists(path):
+                try:
+                    with open(path) as f:
+                        state.rapl_domains.append(f.read().strip())
+                except OSError:
+                    pass
+            else:
+                break
+        return state
+
+@dataclass
+class Problem:
+    name: str
+    sizes: List[str]
+    validation_output: Optional[str] = None
+
+@dataclass
+class LanguageEnv:
+    extension: str
+    run_cmd: str
+    compiler: Optional[str] = None
+    flags: List[str] = field(default_factory=list)
+
+@dataclass
+class RunResult:
+    problem: str
+    language: str
+    size: str
+    profiler: str
+    energy_joules: float
+    time_seconds: float
+    output_valid: bool
+    repetition: int = 0
+    variant: str = "default"
+    suite: str = "benchmarksgame"
+    timestamp: datetime = field(default_factory=datetime.now)
+    checkpoints: List[Dict[str, Any]] = field(default_factory=list)
+
+@dataclass
+class BenchmarkConfig:
+    problems: List[Problem]
+    languages: Dict[str, LanguageEnv]
+    repetitions: int = 30
+    warmup_runs: int = 1
+    timeout_seconds: int = 60
+    clear_cache: bool = True
+    cpu_governor: str = "performance"
+    min_runtime_seconds: float = 1.0
+    mode: str = "local"
+    benchmarks_dir: Optional[str] = None
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> "BenchmarkConfig":
+        if yaml is None:
+            raise ImportError("PyYAML required for YAML config: pip install pyyaml")
+        with open(path) as f:
+            data = yaml.safe_load(f)
+        cfg = data.get("benchmark", data)
+        problems = [Problem(**p) for p in cfg.get("problems", [])]
+        languages = {k: LanguageEnv(**v) for k, v in cfg.get("languages", {}).items()}
+        return cls(
+            problems=problems,
+            languages=languages,
+            repetitions=cfg.get("repetitions", 30),
+            warmup_runs=cfg.get("warmup_runs", 3),
+            timeout_seconds=cfg.get("timeout_seconds", 300),
+            clear_cache=cfg.get("best_practices", {}).get("clear_cache", True),
+            cpu_governor=cfg.get("best_practices", {}).get("cpu_governor", "performance"),
+            min_runtime_seconds=cfg.get("min_runtime_seconds", 1.0),
+            mode=cfg.get("mode", "local"),
+            benchmarks_dir=cfg.get("benchmarks_dir"),
+        )
+
+    @classmethod
+    def default(cls) -> "BenchmarkConfig":
+        return cls(
+            problems=[
+                Problem("nbody", ["1000", "5000", "50000"], "1000_out"),
+                Problem("spectralnorm", ["100", "500", "1000"], "100_out"),
+                Problem("binarytrees", ["10", "14", "18"], "10_out"),
+                Problem("fannkuchredux", ["7", "10", "11"], "7_out"),
+            ],
+            languages={
+                "python": LanguageEnv(".python3", "python3 {source}"),
+                "c": LanguageEnv(".gcc", "{binary}", compiler="gcc", flags=["-O3", "-march=native", "-lm", "-lpthread"]),
+                "cpp": LanguageEnv(".gpp", "{binary}", compiler="g++", flags=["-O3", "-march=native", "-lpthread"]),
+                "java": LanguageEnv(".java", "java -cp {build_dir} {class_name}", compiler="javac", flags=[]),
+            },
+            repetitions=5,
+            warmup_runs=1,
+            timeout_seconds=180,
+        )

benchmark/harness.py

@@ -0,0 +1,149 @@
+"""Benchmark harness - core execution engine using Suite protocol."""
+import math
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Callable, List, Optional
+
+from benchmark.config import RunResult
+from benchmark.profilers import ProfilerInterface, PerfProfiler, get_profiler
+from benchmark.results import ResultCollector
+from benchmark.suites.base import Suite, Task
+
+SLEEP_BETWEEN_RUNS = 2.0
+
+
+def _check_cpu_governor(expected: str, callback=None):
+    try:
+        with open("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") as f:
+            actual = f.read().strip()
+        if actual != expected:
+            msg = (f"WARNING: CPU governor is '{actual}', expected '{expected}'. "
+                   f"Results may have higher variance. "
+                   f"Set with: sudo cpupower frequency-set -g {expected}")
+            if callback:
+                callback(msg)
+            else:
+                print(msg, file=sys.stderr)
+    except FileNotFoundError:
+        pass
+
+
+class BenchmarkHarness:
+    def __init__(self, suite: Optional[Suite] = None,
+                 repetitions: int = 5, warmup_runs: int = 1,
+                 timeout_seconds: int = 300, min_runtime_seconds: float = 1.0,
+                 progress_callback: Optional[Callable] = None):
+        self.suite = suite
+        self.repetitions = repetitions
+        self.warmup_runs = warmup_runs
+        self.timeout = timeout_seconds or (suite.default_timeout if suite else 300)
+        self.min_runtime = min_runtime_seconds
+        self.collector = ResultCollector()
+        self.progress_callback = progress_callback or (lambda msg: print(msg, file=sys.stderr))
+        _check_cpu_governor("performance", self.progress_callback)
+
+    def run_task(self, task: Task, profiler_name: str = "perf",
+                 repetitions: Optional[int] = None) -> List[RunResult]:
+        """Run a single task with a single profiler."""
+        reps = repetitions or self.repetitions
+        profiler = get_profiler(profiler_name)
+        if hasattr(profiler, 'set_source') and task.source_file:
+            profiler.set_source(task.source_file, task.language)
+
+        # Build the task (compile, construct run command)
+        task = self.suite.build(task)
+        if not task.run_command:
+            self.progress_callback(f"  No run command for {task.name}, skipping")
+            return []
+
+        results = []
+
+        # Warmup
+        for i in range(self.warmup_runs):
+            self.progress_callback(f"  Warmup {i+1}/{self.warmup_runs}")
+            try:
+                profiler.run(task.run_command, timeout=self.timeout)
+            except Exception:
+                pass
+
+        # Estimate runtime for auto-repeat
+        estimated = self._estimate_runtime(task.run_command)
+        self._configure_auto_repeat(profiler, estimated)
+        self._clear_cache()
+
+        # Measurement runs
+        for i in range(reps):
+            if i > 0:
+                time.sleep(SLEEP_BETWEEN_RUNS)
+            self.progress_callback(f"  Run {i+1}/{reps}")
+            try:
+                profile_result = profiler.run(task.run_command, timeout=self.timeout)
+                repeat_count = getattr(profiler, 'repeat_count', 1)
+                valid = (True if repeat_count > 1
+                         else self.suite.validate_output(profile_result.output, task))
+
+                result = RunResult(
+                    problem=task.name,
+                    language=task.language,
+                    size=task.metadata.get("size", "default"),
+                    profiler=profiler_name,
+                    energy_joules=profile_result.energy_joules,
+                    time_seconds=profile_result.time_seconds,
+                    output_valid=valid,
+                    repetition=i + 1,
+                    variant=task.variant,
+                    suite=self.suite.name,
+                    checkpoints=profile_result.checkpoints,
+                )
+                results.append(result)
+                self.collector.add(result)
+            except subprocess.TimeoutExpired:
+                self.progress_callback(f"  Run {i+1} timed out")
+            except Exception as e:
+                self.progress_callback(f"  Run {i+1} failed: {e}")
+        return results
+
+    def run_suite(self, profilers: Optional[List[str]] = None,
+                  repetitions: Optional[int] = None,
+                  filters: Optional[dict] = None) -> ResultCollector:
+        """Discover all tasks and run them with specified profilers."""
+        profs = profilers or ["perf"]
+        tasks = self.suite.discover(filters)
+        self.progress_callback(f"Discovered {len(tasks)} tasks in {self.suite.name}")
+
+        for task in tasks:
+            for prof in profs:
+                self.progress_callback(f"\n{task.name} [{task.variant}] profiler={prof}")
+                self.run_task(task, prof, repetitions)
+        return self.collector
+
+    def _estimate_runtime(self, cmd: List[str]) -> float:
+        try:
+            start = time.perf_counter()
+            subprocess.run(cmd, capture_output=True, text=True, timeout=self.timeout)
+            return time.perf_counter() - start
+        except Exception:
+            return 0.0
+
+    def _configure_auto_repeat(self, profiler: ProfilerInterface, estimated_time: float):
+        if not isinstance(profiler, PerfProfiler):
+            return
+        if estimated_time > 0 and estimated_time < self.min_runtime:
+            repeat = math.ceil(self.min_runtime / estimated_time)
+            profiler.repeat_count = repeat
+            self.progress_callback(f"  Short workload ({estimated_time:.2f}s < {self.min_runtime}s), "
+                                   f"auto-repeat x{repeat}")
+
+    def _clear_cache(self):
+        try:
+            subprocess.run(["sync"], check=False)
+            subprocess.run(["sudo", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"],
+                           capture_output=True, check=False)
+        except Exception:
+            pass
+
+    def cleanup(self):
+        if self.suite:
+            self.suite.cleanup()