macsmart 0.1.0__py3-none-any.whl

macsmart/__init__.py

@@ -0,0 +1,3 @@
+"""MacSmart LLM — Memory-intelligent local LLM orchestration for 16GB Apple Silicon Macs."""
+
+__version__ = "0.1.0"

macsmart/benchmark/__init__.py

@@ -0,0 +1 @@
+"""Benchmark runner for measuring LLM inference performance."""

macsmart/benchmark/energy.py

@@ -0,0 +1,140 @@
+"""Energy measurement using powermetrics wrapper."""
+
+from __future__ import annotations
+
+import platform
+import re
+import subprocess
+from dataclasses import dataclass
+
+
+@dataclass
+class EnergyMeasurement:
+    """Energy consumption during a benchmark run."""
+
+    cpu_power_watts: float | None
+    gpu_power_watts: float | None
+    ane_power_watts: float | None  # Apple Neural Engine
+    total_power_watts: float | None
+    duration_sec: float
+    total_energy_joules: float | None
+
+
+def _parse_powermetrics_output(output: str) -> dict[str, float]:
+    """Parse power values from powermetrics output.
+
+    Looks for lines like:
+        CPU Power: 1234 mW
+        GPU Power: 567 mW
+        ANE Power: 89 mW
+        Combined Power (CPU + GPU + ANE): 1890 mW
+
+    Args:
+        output: Raw stdout from powermetrics.
+
+    Returns:
+        Dict mapping component names to power in watts.
+    """
+    powers: dict[str, float] = {}
+    for line in output.splitlines():
+        # Match lines like "CPU Power: 1234 mW"
+        match = re.match(r"\s*([\w\s\+\(\)]+?)\s*Power[^:]*:\s*([\d.]+)\s*mW", line, re.IGNORECASE)
+        if match:
+            name = match.group(1).strip().lower()
+            mw = float(match.group(2))
+            powers[name] = mw / 1000.0  # Convert mW to W
+    return powers
+
+
+def measure_energy(duration_sec: float = 10.0) -> EnergyMeasurement:
+    """Measure energy consumption using powermetrics.
+
+    Requires sudo access for powermetrics. Samples for the given
+    duration and returns averaged power readings.
+
+    Args:
+        duration_sec: How long to measure in seconds.
+
+    Returns:
+        EnergyMeasurement with power data. Fields are None if
+        powermetrics is unavailable or the data cannot be parsed.
+    """
+    if not is_powermetrics_available():
+        return EnergyMeasurement(
+            cpu_power_watts=None,
+            gpu_power_watts=None,
+            ane_power_watts=None,
+            total_power_watts=None,
+            duration_sec=duration_sec,
+            total_energy_joules=None,
+        )
+
+    interval_ms = int(duration_sec * 1000)
+    try:
+        result = subprocess.run(
+            [
+                "sudo", "-n", "powermetrics",
+                "--samplers", "cpu_power",
+                "-i", str(interval_ms),
+                "-n", "1",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=duration_sec + 10,
+        )
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        return EnergyMeasurement(
+            cpu_power_watts=None,
+            gpu_power_watts=None,
+            ane_power_watts=None,
+            total_power_watts=None,
+            duration_sec=duration_sec,
+            total_energy_joules=None,
+        )
+
+    powers = _parse_powermetrics_output(result.stdout)
+
+    cpu_w = powers.get("cpu")
+    gpu_w = powers.get("gpu")
+    ane_w = powers.get("ane")
+
+    # Look for combined/total power
+    total_w: float | None = None
+    for key, val in powers.items():
+        if "combined" in key or "total" in key:
+            total_w = val
+            break
+
+    if total_w is None and any(v is not None for v in [cpu_w, gpu_w, ane_w]):
+        total_w = sum(v for v in [cpu_w, gpu_w, ane_w] if v is not None)
+
+    energy_j = total_w * duration_sec if total_w is not None else None
+
+    return EnergyMeasurement(
+        cpu_power_watts=cpu_w,
+        gpu_power_watts=gpu_w,
+        ane_power_watts=ane_w,
+        total_power_watts=total_w,
+        duration_sec=duration_sec,
+        total_energy_joules=round(energy_j, 2) if energy_j is not None else None,
+    )
+
+
+def is_powermetrics_available() -> bool:
+    """Check if powermetrics is available and we have passwordless sudo.
+
+    Returns:
+        True if energy measurement is possible.
+    """
+    if platform.system() != "Darwin":
+        return False
+
+    try:
+        result = subprocess.run(
+            ["sudo", "-n", "powermetrics", "--samplers", "cpu_power", "-i", "1", "-n", "1"],
+            capture_output=True,
+            timeout=5,
+        )
+        return result.returncode == 0
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        return False

macsmart/benchmark/energy_compare.py

@@ -0,0 +1,182 @@
+"""Energy benchmark comparison between battery and AC power."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from macsmart.benchmark.energy import EnergyMeasurement, measure_energy
+from macsmart.benchmark.runner import BenchmarkResult, run_benchmark
+
+_DEFAULT_RESULTS_DIR = Path(__file__).resolve().parent.parent.parent / "benchmarks"
+
+
+@dataclass
+class EnergyBenchmarkResult:
+    """Benchmark result combined with energy measurement and power source."""
+
+    benchmark: BenchmarkResult
+    energy: EnergyMeasurement
+    power_source: str  # "battery" or "ac"
+
+
+@dataclass
+class EnergyComparison:
+    """Side-by-side comparison of battery vs AC energy benchmarks."""
+
+    battery: EnergyBenchmarkResult
+    ac: EnergyBenchmarkResult
+    speed_ratio: float | None  # ac.tokens_per_sec / battery.tokens_per_sec
+    ttft_delta_ms: float  # ac.ttft - battery.ttft
+    energy_ratio: float | None  # ac.total_energy / battery.total_energy
+    efficiency_battery: float | None  # tokens / joule on battery
+    efficiency_ac: float | None  # tokens / joule on AC
+
+
+def run_energy_benchmark(
+    model_repo: str,
+    power_source: str,
+    prompt: str | None = None,
+    max_tokens: int = 256,
+    energy_duration_sec: float = 10.0,
+) -> EnergyBenchmarkResult:
+    """Run a benchmark with energy measurement.
+
+    Args:
+        model_repo: HuggingFace repo ID.
+        power_source: "battery" or "ac".
+        prompt: Custom prompt.
+        max_tokens: Maximum tokens to generate.
+        energy_duration_sec: Duration for energy sampling.
+
+    Returns:
+        Combined benchmark + energy result.
+    """
+    benchmark = run_benchmark(model_repo, prompt=prompt, max_tokens=max_tokens)
+    energy = measure_energy(duration_sec=energy_duration_sec)
+
+    return EnergyBenchmarkResult(
+        benchmark=benchmark,
+        energy=energy,
+        power_source=power_source,
+    )
+
+
+def compare_energy(
+    battery: EnergyBenchmarkResult,
+    ac: EnergyBenchmarkResult,
+) -> EnergyComparison:
+    """Compare battery vs AC energy benchmark results.
+
+    Args:
+        battery: Benchmark result on battery power.
+        ac: Benchmark result on AC power.
+
+    Returns:
+        EnergyComparison with computed ratios and deltas.
+    """
+    batt_tps = battery.benchmark.tokens_per_sec
+    ac_tps = ac.benchmark.tokens_per_sec
+
+    speed_ratio = round(ac_tps / batt_tps, 3) if batt_tps > 0 else None
+    ttft_delta = round(ac.benchmark.ttft_ms - battery.benchmark.ttft_ms, 2)
+
+    batt_energy = battery.energy.total_energy_joules
+    ac_energy = ac.energy.total_energy_joules
+
+    energy_ratio: float | None = None
+    if batt_energy and batt_energy > 0 and ac_energy is not None:
+        energy_ratio = round(ac_energy / batt_energy, 3)
+
+    eff_battery: float | None = None
+    if batt_energy and batt_energy > 0:
+        eff_battery = round(battery.benchmark.generation_tokens / batt_energy, 3)
+
+    eff_ac: float | None = None
+    if ac_energy and ac_energy > 0:
+        eff_ac = round(ac.benchmark.generation_tokens / ac_energy, 3)
+
+    return EnergyComparison(
+        battery=battery,
+        ac=ac,
+        speed_ratio=speed_ratio,
+        ttft_delta_ms=ttft_delta,
+        energy_ratio=energy_ratio,
+        efficiency_battery=eff_battery,
+        efficiency_ac=eff_ac,
+    )
+
+
+def save_energy_result(
+    result: EnergyBenchmarkResult,
+    output_dir: Path | None = None,
+) -> Path:
+    """Save an energy benchmark result to JSON.
+
+    Args:
+        result: The energy benchmark result.
+        output_dir: Output directory. Defaults to benchmarks/.
+
+    Returns:
+        Path to the saved JSON file.
+    """
+    out = output_dir or _DEFAULT_RESULTS_DIR
+    out.mkdir(parents=True, exist_ok=True)
+
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    safe_name = result.benchmark.model_name.replace("/", "_").replace(" ", "_")
+    filename = f"energy_{result.power_source}_{safe_name}_{timestamp}.json"
+
+    data = {
+        "benchmark": asdict(result.benchmark),
+        "energy": asdict(result.energy),
+        "power_source": result.power_source,
+        "timestamp": timestamp,
+    }
+
+    path = out / filename
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+
+    return path
+
+
+def load_energy_results(
+    results_dir: Path | None = None,
+) -> list[EnergyBenchmarkResult]:
+    """Load all energy benchmark results from disk.
+
+    Args:
+        results_dir: Directory to load from. Defaults to benchmarks/.
+
+    Returns:
+        List of EnergyBenchmarkResult objects.
+    """
+    directory = results_dir or _DEFAULT_RESULTS_DIR
+    if not directory.exists():
+        return []
+
+    results: list[EnergyBenchmarkResult] = []
+    for path in sorted(directory.glob("energy_*.json"), reverse=True):
+        try:
+            with open(path) as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+        try:
+            data.pop("timestamp", None)
+            benchmark = BenchmarkResult(**data["benchmark"])
+            energy = EnergyMeasurement(**data["energy"])
+            result = EnergyBenchmarkResult(
+                benchmark=benchmark,
+                energy=energy,
+                power_source=data["power_source"],
+            )
+            results.append(result)
+        except (TypeError, KeyError):
+            continue
+
+    return results

macsmart/benchmark/report.py

@@ -0,0 +1,253 @@
+"""Generate benchmark reports in JSON and markdown formats."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+from macsmart.benchmark.runner import BenchmarkResult
+
+# Default output directory for benchmark results.
+_DEFAULT_RESULTS_DIR = Path(__file__).resolve().parent.parent.parent / "benchmarks"
+
+
+def save_result(result: BenchmarkResult, output_dir: Path | None = None) -> Path:
+    """Save a benchmark result to the benchmarks/ directory as JSON.
+
+    Files are named with model name and timestamp for uniqueness.
+
+    Args:
+        result: The benchmark result to save.
+        output_dir: Optional output directory. Defaults to benchmarks/.
+
+    Returns:
+        Path to the saved JSON file.
+    """
+    out = output_dir or _DEFAULT_RESULTS_DIR
+    out.mkdir(parents=True, exist_ok=True)
+
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    safe_name = result.model_name.replace("/", "_").replace(" ", "_")
+    filename = f"{safe_name}_{result.quantization}_{timestamp}.json"
+
+    data = asdict(result)
+    data["timestamp"] = timestamp
+
+    path = out / filename
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+
+    return path
+
+
+def generate_markdown_report(results: list[BenchmarkResult]) -> str:
+    """Generate a markdown comparison table from benchmark results.
+
+    Args:
+        results: List of benchmark results to compare.
+
+    Returns:
+        Markdown-formatted report string.
+    """
+    if not results:
+        return "No benchmark results to report."
+
+    lines: list[str] = []
+    lines.append("# Benchmark Report")
+    lines.append("")
+    lines.append(f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
+    lines.append("")
+    lines.append(
+        "| Model | Quant | TTFT (ms) | Tokens/s | Peak Mem (GB) | "
+        "Swap (GB) | Gen Tokens | Duration (s) |"
+    )
+    lines.append(
+        "|-------|-------|----------:|----------:|--------------:|"
+        "----------:|-----------:|-------------:|"
+    )
+
+    for r in results:
+        lines.append(
+            f"| {r.model_name} | {r.quantization} | "
+            f"{r.ttft_ms:.1f} | {r.tokens_per_sec:.1f} | "
+            f"{r.peak_memory_gb:.2f} | {r.swap_used_gb:.2f} | "
+            f"{r.generation_tokens} | {r.duration_sec:.1f} |"
+        )
+
+    lines.append("")
+    return "\n".join(lines)
+
+
+def generate_energy_comparison_report(comparison: object) -> str:
+    """Generate a markdown report comparing battery vs AC energy usage.
+
+    Args:
+        comparison: EnergyComparison object.
+
+    Returns:
+        Markdown-formatted report string.
+    """
+    lines: list[str] = []
+    lines.append("# Energy Comparison Report")
+    lines.append("")
+    lines.append(f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
+    lines.append("")
+    lines.append(f"**Model:** {comparison.battery.benchmark.model_name}")
+    lines.append("")
+
+    lines.append("| Metric | Battery | AC | Delta |")
+    lines.append("|--------|--------:|---:|------:|")
+
+    b = comparison.battery.benchmark
+    a = comparison.ac.benchmark
+
+    lines.append(
+        f"| Tokens/s | {b.tokens_per_sec:.1f} | {a.tokens_per_sec:.1f} | "
+        f"{a.tokens_per_sec - b.tokens_per_sec:+.1f} |"
+    )
+    lines.append(
+        f"| TTFT (ms) | {b.ttft_ms:.1f} | {a.ttft_ms:.1f} | "
+        f"{a.ttft_ms - b.ttft_ms:+.1f} |"
+    )
+    lines.append(
+        f"| Peak Memory (GB) | {b.peak_memory_gb:.2f} | {a.peak_memory_gb:.2f} | "
+        f"{a.peak_memory_gb - b.peak_memory_gb:+.2f} |"
+    )
+
+    be = comparison.battery.energy
+    ae = comparison.ac.energy
+    bw = be.total_power_watts if be.total_power_watts is not None else 0.0
+    aw = ae.total_power_watts if ae.total_power_watts is not None else 0.0
+    lines.append(
+        f"| Power (W) | {bw:.1f} | {aw:.1f} | {aw - bw:+.1f} |"
+    )
+
+    bj = be.total_energy_joules if be.total_energy_joules is not None else 0.0
+    aj = ae.total_energy_joules if ae.total_energy_joules is not None else 0.0
+    lines.append(
+        f"| Energy (J) | {bj:.1f} | {aj:.1f} | {aj - bj:+.1f} |"
+    )
+
+    lines.append("")
+
+    if comparison.efficiency_battery is not None:
+        lines.append(f"**Battery efficiency:** {comparison.efficiency_battery:.3f} tokens/J")
+    if comparison.efficiency_ac is not None:
+        lines.append(f"**AC efficiency:** {comparison.efficiency_ac:.3f} tokens/J")
+    if comparison.speed_ratio is not None:
+        lines.append(f"**Speed ratio (AC/Battery):** {comparison.speed_ratio:.3f}x")
+
+    lines.append("")
+    return "\n".join(lines)
+
+
+def load_results(results_dir: Path | None = None) -> list[BenchmarkResult]:
+    """Load all stored benchmark results from the benchmarks/ directory.
+
+    Args:
+        results_dir: Optional directory to load from. Defaults to benchmarks/.
+
+    Returns:
+        List of previously saved BenchmarkResult objects, sorted by
+        timestamp (newest first).
+    """
+    directory = results_dir or _DEFAULT_RESULTS_DIR
+    if not directory.exists():
+        return []
+
+    results: list[tuple[str, BenchmarkResult]] = []
+    for path in directory.glob("*.json"):
+        try:
+            with open(path) as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+        timestamp = data.pop("timestamp", "")
+        try:
+            result = BenchmarkResult(**data)
+            results.append((timestamp, result))
+        except TypeError:
+            continue
+
+    # Sort newest first
+    results.sort(key=lambda x: x[0], reverse=True)
+    return [r for _, r in results]
+
+
+def load_result_from_file(path: Path) -> BenchmarkResult:
+    """Load a single benchmark result from a JSON file.
+
+    Args:
+        path: Path to the JSON result file.
+
+    Returns:
+        BenchmarkResult loaded from the file.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        ValueError: If the file is not valid benchmark JSON.
+    """
+    if not path.exists():
+        raise FileNotFoundError(f"Result file not found: {path}")
+
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON in {path}: {e}") from e
+
+    data.pop("timestamp", None)
+    try:
+        return BenchmarkResult(**data)
+    except TypeError as e:
+        raise ValueError(f"Invalid benchmark result in {path}: {e}") from e
+
+
+def get_latest_result_paths(
+    n: int = 2,
+    results_dir: Path | None = None,
+) -> list[Path]:
+    """Return the N most recent non-energy benchmark result file paths.
+
+    Args:
+        n: Number of result paths to return.
+        results_dir: Directory to search. Defaults to benchmarks/.
+
+    Returns:
+        List of Path objects sorted newest-first.
+
+    Raises:
+        ValueError: If fewer than N result files exist.
+    """
+    directory = results_dir or _DEFAULT_RESULTS_DIR
+    if not directory.exists():
+        raise ValueError(f"Results directory does not exist: {directory}")
+
+    paths = sorted(
+        [p for p in directory.glob("*.json") if "energy" not in p.name.lower()],
+        key=lambda p: p.stat().st_mtime,
+        reverse=True,
+    )
+
+    if len(paths) < n:
+        raise ValueError(
+            f"Need at least {n} result files, found {len(paths)} in {directory}"
+        )
+
+    return paths[:n]
+
+
+def results_to_json_api(results_dir: Path | None = None) -> list[dict]:
+    """Serialize benchmark results for the dashboard JSON API.
+
+    Args:
+        results_dir: Directory to load results from.
+
+    Returns:
+        List of result dicts ready for JSON serialization.
+    """
+    results = load_results(results_dir=results_dir)
+    return [asdict(r) for r in results]