genarena 0.0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

genarena/experiments.py

@@ -0,0 +1,121 @@
+# Copyright 2026 Ruihang Li.
+# Licensed under the Apache License, Version 2.0.
+# See LICENSE file in the project root for details.
+
+"""Experiment name utilities for GenArena.
+
+This module centralizes experiment naming rules and milestone detection.
+
+Key rules:
+- exp_name must end with a date suffix: `_yyyymmdd` (8 digits)
+- milestone experiments are identified by a fixed prefix constant
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+from datetime import date, datetime
+from typing import List, Optional, Tuple
+
+# Fixed milestone prefix (must be constant across the codebase)
+MILESTONE_PREFIX: str = "GenArena_"
+
+# exp_name must end with "_yyyymmdd"
+_EXP_DATE_SUFFIX_RE = re.compile(r"^(?P<prefix>.+)_(?P<date>\d{8})$")
+
+
+def parse_exp_date_suffix(exp_name: str) -> Optional[date]:
+    """Parse the `_yyyymmdd` suffix from exp_name.
+
+    Returns None if exp_name does not match the required format or if the date is invalid.
+    """
+    m = _EXP_DATE_SUFFIX_RE.match(exp_name or "")
+    if not m:
+        return None
+    datestr = m.group("date")
+    try:
+        return datetime.strptime(datestr, "%Y%m%d").date()
+    except ValueError:
+        return None
+
+
+def is_valid_exp_name(exp_name: str) -> bool:
+    """Return True iff exp_name ends with a valid `_yyyymmdd` date suffix."""
+    return parse_exp_date_suffix(exp_name) is not None
+
+
+def validate_exp_name(exp_name: str) -> Tuple[bool, str]:
+    """Validate exp_name. Returns (ok, error_message)."""
+    if not exp_name:
+        return False, "exp_name is empty"
+    if not _EXP_DATE_SUFFIX_RE.match(exp_name):
+        return False, "exp_name must end with `_yyyymmdd` (e.g., `MyExp_20260128`)"
+    if parse_exp_date_suffix(exp_name) is None:
+        return False, "exp_name has an invalid date suffix; expected a real calendar date in `_yyyymmdd`"
+    return True, ""
+
+
+def require_valid_exp_name(exp_name: str) -> None:
+    """Raise ValueError if exp_name is invalid."""
+    ok, msg = validate_exp_name(exp_name)
+    if not ok:
+        raise ValueError(msg)
+
+
+def is_milestone_exp(exp_name: str) -> bool:
+    """Return True if exp_name is a milestone experiment (fixed prefix)."""
+    return (exp_name or "").startswith(MILESTONE_PREFIX)
+
+
+@dataclass(frozen=True)
+class ExperimentInfo:
+    """Parsed experiment directory info."""
+
+    name: str
+    date: date
+
+
+def discover_experiments(models_root: str) -> List[ExperimentInfo]:
+    """Discover valid experiment directories under a models root directory.
+
+    Args:
+        models_root: `arena_dir/<subset>/models`
+
+    Returns:
+        List of ExperimentInfo for directories whose name matches `_yyyymmdd`.
+    """
+    infos: List[ExperimentInfo] = []
+    if not os.path.isdir(models_root):
+        return infos
+
+    for name in os.listdir(models_root):
+        if name.startswith("."):
+            continue
+        path = os.path.join(models_root, name)
+        if not os.path.isdir(path):
+            continue
+        d = parse_exp_date_suffix(name)
+        if d is None:
+            continue
+        infos.append(ExperimentInfo(name=name, date=d))
+
+    # Sort by (date, name) for deterministic ordering
+    infos.sort(key=lambda x: (x.date, x.name))
+    return infos
+
+
+def pick_latest_experiment_name(models_root: str) -> str:
+    """Pick the latest experiment name under models_root by `_yyyymmdd` suffix.
+
+    Raises ValueError if no valid experiment directories exist.
+    """
+    infos = discover_experiments(models_root)
+    if not infos:
+        raise ValueError(
+            f"No valid experiments found under models dir: {models_root}. "
+            f"Expected subdirectories named like `<something>_yyyymmdd`."
+        )
+    return infos[-1].name
+

genarena/leaderboard.py

@@ -0,0 +1,270 @@
+# Copyright 2026 Ruihang Li.
+# Licensed under the Apache License, Version 2.0.
+# See LICENSE file in the project root for details.
+
+"""Leaderboard generation module."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+from genarena.state import ArenaState
+
+
+@dataclass
+class LeaderboardEntry:
+    """Entry in the leaderboard."""
+
+    rank: int
+    model: str
+    elo: float
+    wins: int
+    losses: int
+    ties: int
+    total_battles: int
+    win_rate: float
+    ci_lower: Optional[float] = None
+    ci_upper: Optional[float] = None
+
+    @property
+    def ci_width(self) -> Optional[float]:
+        """95% CI width, or None if CI not computed."""
+        if self.ci_lower is None or self.ci_upper is None:
+            return None
+        return self.ci_upper - self.ci_lower
+
+    @property
+    def ci_str(self) -> str:
+        """Format CI as string (e.g., '±7.5' or 'N/A')."""
+        width = self.ci_width
+        if width is None:
+            return "N/A"
+        return f"±{width / 2:.1f}"
+
+
+def generate_leaderboard(
+    state: ArenaState,
+    title: Optional[str] = None,
+    show_ci: bool = True,
+) -> str:
+    """
+    Generate a Markdown leaderboard from arena state.
+
+    Args:
+        state: ArenaState with model statistics
+        title: Optional title for the leaderboard
+        show_ci: Whether to show CI column (default: True)
+
+    Returns:
+        Markdown formatted leaderboard string
+    """
+    # Build list of entries sorted by ELO
+    entries: list[LeaderboardEntry] = []
+
+    for model, stats in state.models.items():
+        entries.append(LeaderboardEntry(
+            rank=0,  # Will be set after sorting
+            model=model,
+            elo=stats.elo,
+            wins=stats.wins,
+            losses=stats.losses,
+            ties=stats.ties,
+            total_battles=stats.total_battles,
+            win_rate=stats.win_rate,
+            ci_lower=stats.ci_lower,
+            ci_upper=stats.ci_upper,
+        ))
+
+    # Sort by ELO descending
+    entries.sort(key=lambda e: e.elo, reverse=True)
+
+    # Assign ranks
+    for i, entry in enumerate(entries):
+        entry.rank = i + 1
+
+    # Check if any entry has CI information
+    has_ci = any(e.ci_lower is not None for e in entries)
+    show_ci = show_ci and has_ci
+
+    # Generate Markdown
+    lines = []
+
+    # Title
+    if title:
+        lines.append(f"# {title}")
+    else:
+        lines.append("# ELO Leaderboard")
+    lines.append("")
+
+    # Summary
+    lines.append(f"**Total Models:** {len(entries)}")
+    lines.append(f"**Total Battles:** {state.total_battles}")
+    if state.last_updated:
+        lines.append(f"**Last Updated:** {state.last_updated}")
+    lines.append("")
+
+    # Table header
+    if show_ci:
+        lines.append("| Rank | Model | ELO | 95% CI | Win Rate | W/L/T | Battles |")
+        lines.append("|------|-------|-----|--------|----------|-------|---------|")
+    else:
+        lines.append("| Rank | Model | ELO | Win Rate | W/L/T | Battles |")
+        lines.append("|------|-------|-----|----------|-------|---------|")
+
+    # Table rows
+    for entry in entries:
+        win_rate_pct = f"{entry.win_rate * 100:.1f}%"
+        wlt = f"{entry.wins}/{entry.losses}/{entry.ties}"
+        elo_str = f"{entry.elo:.0f}"
+
+        if show_ci:
+            lines.append(
+                f"| {entry.rank} | {entry.model} | {elo_str} | {entry.ci_str} | "
+                f"{win_rate_pct} | {wlt} | {entry.total_battles} |"
+            )
+        else:
+            lines.append(
+                f"| {entry.rank} | {entry.model} | {elo_str} | {win_rate_pct} | "
+                f"{wlt} | {entry.total_battles} |"
+            )
+
+    lines.append("")
+
+    # Footer
+    lines.append("---")
+    lines.append(f"*Generated by GenArena Arena Evaluation*")
+
+    return "\n".join(lines)
+
+
+def save_leaderboard(
+    state: ArenaState,
+    path: str,
+    title: Optional[str] = None
+) -> None:
+    """
+    Generate and save leaderboard to a file.
+
+    Args:
+        state: ArenaState with model statistics
+        path: Path to save the Markdown file (typically README.md)
+        title: Optional title for the leaderboard
+    """
+    content = generate_leaderboard(state, title)
+
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(content)
+
+
+def generate_experiment_readme(
+    exp_name: str,
+    elo: dict[str, float],
+    model_count: int,
+    battle_count: int,
+    ci_lower: Optional[dict[str, float]] = None,
+    ci_upper: Optional[dict[str, float]] = None,
+) -> str:
+    """
+    Generate README.md content for an experiment directory.
+
+    Shows the cumulative ELO leaderboard up to (and including) this experiment.
+
+    Args:
+        exp_name: Experiment name (e.g., "MyExp_20260128")
+        elo: Dict mapping model name to ELO rating
+        model_count: Number of models
+        battle_count: Total battles up to this experiment
+        ci_lower: Optional dict of CI lower bounds
+        ci_upper: Optional dict of CI upper bounds
+
+    Returns:
+        Markdown formatted README content
+    """
+    lines = []
+
+    # Title
+    lines.append(f"# {exp_name}")
+    lines.append("")
+
+    # Summary
+    lines.append(f"**Models:** {model_count}")
+    lines.append(f"**Cumulative Battles:** {battle_count}")
+    lines.append("")
+
+    # Build sorted entries
+    entries = sorted(elo.items(), key=lambda x: x[1], reverse=True)
+
+    # Check if CI info is available
+    has_ci = ci_lower is not None and ci_upper is not None and len(ci_lower) > 0
+
+    # Table header
+    if has_ci:
+        lines.append("| Rank | Model | ELO | 95% CI |")
+        lines.append("|------|-------|-----|--------|")
+    else:
+        lines.append("| Rank | Model | ELO |")
+        lines.append("|------|-------|-----|")
+
+    # Table rows
+    for rank, (model, elo_val) in enumerate(entries, start=1):
+        elo_str = f"{elo_val:.0f}"
+        if has_ci and model in ci_lower and model in ci_upper:
+            width = ci_upper[model] - ci_lower[model]
+            ci_str = f"±{width / 2:.1f}"
+            lines.append(f"| {rank} | {model} | {elo_str} | {ci_str} |")
+        elif has_ci:
+            lines.append(f"| {rank} | {model} | {elo_str} | N/A |")
+        else:
+            lines.append(f"| {rank} | {model} | {elo_str} |")
+
+    lines.append("")
+    lines.append("---")
+    lines.append("*Generated by GenArena*")
+
+    return "\n".join(lines)
+
+
+def print_leaderboard(state: ArenaState, title: Optional[str] = None) -> None:
+    """
+    Print leaderboard to stdout.
+
+    Args:
+        state: ArenaState with model statistics
+        title: Optional title for the leaderboard
+    """
+    content = generate_leaderboard(state, title)
+    print(content)
+
+
+def get_leaderboard_entries(state: ArenaState) -> list[LeaderboardEntry]:
+    """
+    Get leaderboard entries as a list.
+
+    Args:
+        state: ArenaState with model statistics
+
+    Returns:
+        List of LeaderboardEntry objects sorted by ELO
+    """
+    entries: list[LeaderboardEntry] = []
+
+    for model, stats in state.models.items():
+        entries.append(LeaderboardEntry(
+            rank=0,
+            model=model,
+            elo=stats.elo,
+            wins=stats.wins,
+            losses=stats.losses,
+            ties=stats.ties,
+            total_battles=stats.total_battles,
+            win_rate=stats.win_rate,
+            ci_lower=stats.ci_lower,
+            ci_upper=stats.ci_upper,
+        ))
+
+    entries.sort(key=lambda e: e.elo, reverse=True)
+
+    for i, entry in enumerate(entries):
+        entry.rank = i + 1
+
+    return entries