themis-eval 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

themis/cli/commands/leaderboard.py

@@ -0,0 +1,362 @@
+"""Leaderboard generation for benchmarks."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.experiment.comparison import compare_experiments
+
+
+def leaderboard_command(
+    *,
+    run_ids: Annotated[list[str], Parameter(help="Run IDs to include in leaderboard")],
+    storage: Annotated[Path, Parameter(help="Storage directory")] = Path(".cache/runs"),
+    metric: Annotated[str, Parameter(help="Primary metric for ranking")] = "accuracy",
+    format: Annotated[
+        str, Parameter(help="Output format: markdown, latex, csv")
+    ] = "markdown",
+    output: Annotated[
+        Path | None, Parameter(help="Output file path (optional)")
+    ] = None,
+    title: Annotated[str, Parameter(help="Leaderboard title")] = "Leaderboard",
+    ascending: Annotated[
+        bool, Parameter(help="Rank in ascending order (lower is better)")
+    ] = False,
+    include_cost: Annotated[bool, Parameter(help="Include cost column")] = True,
+    include_metadata: Annotated[
+        list[str] | None,
+        Parameter(help="Metadata fields to include (e.g., model, temperature)"),
+    ] = None,
+) -> int:
+    """Generate benchmark leaderboard from experiment runs.
+
+    Creates a ranked table of experiments based on a primary metric.
+    Perfect for README files, documentation, and benchmark tracking.
+
+    Examples:
+        # Basic leaderboard
+        uv run python -m themis.cli leaderboard \\
+          --run-ids run-1 run-2 run-3 \\
+          --metric accuracy \\
+          --output LEADERBOARD.md
+
+        # With custom metadata
+        uv run python -m themis.cli leaderboard \\
+          --run-ids run-gpt4 run-claude run-gemini \\
+          --metric accuracy \\
+          --include-metadata model \\
+          --include-metadata temperature \\
+          --output results.md
+
+        # LaTeX for papers
+        uv run python -m themis.cli leaderboard \\
+          --run-ids run-1 run-2 run-3 \\
+          --metric accuracy \\
+          --format latex \\
+          --title "Math500 Benchmark Results" \\
+          --output leaderboard.tex
+
+        # Cost-optimized ranking (lower is better)
+        uv run python -m themis.cli leaderboard \\
+          --run-ids run-1 run-2 run-3 \\
+          --metric cost \\
+          --ascending true \\
+          --output cost_leaderboard.md
+    """
+    try:
+        # Load experiments
+        print(f"Loading experiments from {storage}...")
+        comparison = compare_experiments(
+            run_ids=run_ids,
+            storage_dir=storage,
+            include_metadata=True,
+        )
+
+        print(f"✓ Loaded {len(comparison.experiments)} experiments")
+
+        # Rank by metric
+        ranked = comparison.rank_by_metric(metric, ascending=ascending)
+
+        print(f"✓ Ranked by {metric} ({'ascending' if ascending else 'descending'})")
+
+        # Generate leaderboard
+        if format == "markdown":
+            content = _generate_markdown_leaderboard(
+                ranked=ranked,
+                metric=metric,
+                title=title,
+                include_cost=include_cost,
+                include_metadata=include_metadata,
+                comparison=comparison,
+            )
+        elif format == "latex":
+            content = _generate_latex_leaderboard(
+                ranked=ranked,
+                metric=metric,
+                title=title,
+                include_cost=include_cost,
+                include_metadata=include_metadata,
+                comparison=comparison,
+            )
+        elif format == "csv":
+            content = _generate_csv_leaderboard(
+                ranked=ranked,
+                metric=metric,
+                include_cost=include_cost,
+                include_metadata=include_metadata,
+                comparison=comparison,
+            )
+        else:
+            print(f"Error: Unknown format '{format}'")
+            print("Available formats: markdown, latex, csv")
+            return 1
+
+        # Output
+        if output:
+            output = Path(output)
+            output.write_text(content, encoding="utf-8")
+            print(f"\n✓ Leaderboard saved to {output}")
+        else:
+            print("\n" + "=" * 80)
+            print(content)
+            print("=" * 80)
+
+        # Show top 3
+        print(f"\n🏆 Top 3 by {metric}:")
+        for i, exp in enumerate(ranked[:3], 1):
+            value = exp.get_metric(metric)
+            emoji = ["🥇", "🥈", "🥉"][i - 1]
+            print(f"  {emoji} {i}. {exp.run_id}: {value:.4f}")
+
+        return 0
+
+    except Exception as e:
+        print(f"Error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+def _generate_markdown_leaderboard(
+    ranked,
+    metric: str,
+    title: str,
+    include_cost: bool,
+    include_metadata: list[str] | None,
+    comparison,
+) -> str:
+    """Generate markdown leaderboard."""
+    lines = [f"# {title}\n"]
+
+    # Date
+    from datetime import datetime
+
+    lines.append(f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*\n")
+
+    # Build header
+    headers = ["Rank", "Run ID", metric.capitalize()]
+
+    # Add metadata columns
+    if include_metadata:
+        headers.extend(include_metadata)
+
+    # Add cost if requested
+    has_cost = include_cost and any(exp.get_cost() is not None for exp in ranked)
+    if has_cost:
+        headers.append("Cost ($)")
+
+    # Add other metrics
+    other_metrics = [m for m in comparison.metrics if m != metric]
+    headers.extend(other_metrics)
+
+    headers.extend(["Samples", "Failures"])
+
+    lines.append("| " + " | ".join(headers) + " |")
+    lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+    # Build rows
+    for rank, exp in enumerate(ranked, 1):
+        values = [str(rank), exp.run_id]
+
+        # Primary metric
+        val = exp.get_metric(metric)
+        values.append(f"**{val:.4f}**" if val is not None else "N/A")
+
+        # Metadata
+        if include_metadata:
+            for field in include_metadata:
+                val = exp.metadata.get(field, "—")
+                values.append(str(val))
+
+        # Cost
+        if has_cost:
+            cost = exp.get_cost()
+            values.append(f"{cost:.4f}" if cost is not None else "—")
+
+        # Other metrics
+        for m in other_metrics:
+            val = exp.get_metric(m)
+            values.append(f"{val:.4f}" if val is not None else "N/A")
+
+        values.append(str(exp.sample_count))
+        values.append(str(exp.failure_count))
+
+        lines.append("| " + " | ".join(values) + " |")
+
+    return "\n".join(lines)
+
+
+def _generate_latex_leaderboard(
+    ranked,
+    metric: str,
+    title: str,
+    include_cost: bool,
+    include_metadata: list[str] | None,
+    comparison,
+) -> str:
+    """Generate LaTeX leaderboard."""
+    lines = []
+
+    # Calculate columns
+    n_cols = 2  # Rank + Run ID
+    n_cols += 1  # Primary metric
+
+    if include_metadata:
+        n_cols += len(include_metadata)
+
+    has_cost = include_cost and any(exp.get_cost() is not None for exp in ranked)
+    if has_cost:
+        n_cols += 1
+
+    other_metrics = [m for m in comparison.metrics if m != metric]
+    n_cols += len(other_metrics)
+    n_cols += 2  # Samples + Failures
+
+    # Table preamble
+    lines.append("\\begin{table}[htbp]")
+    lines.append("\\centering")
+    lines.append(f"\\caption{{{title}}}")
+    lines.append("\\label{tab:leaderboard}")
+
+    col_spec = "c" + "l" + "r" * (n_cols - 2)  # Center rank, left run_id, right numbers
+    lines.append(f"\\begin{{tabular}}{{{col_spec}}}")
+    lines.append("\\toprule")
+
+    # Header
+    headers = ["Rank", "Run ID", f"\\textbf{{{metric}}}"]
+
+    if include_metadata:
+        headers.extend(include_metadata)
+
+    if has_cost:
+        headers.append("Cost (\\$)")
+
+    headers.extend(other_metrics)
+    headers.extend(["Samples", "Failures"])
+
+    lines.append(" & ".join(headers) + " \\\\")
+    lines.append("\\midrule")
+
+    # Rows
+    for rank, exp in enumerate(ranked, 1):
+        values = [str(rank), exp.run_id.replace("_", "\\_")]
+
+        # Primary metric (bold)
+        val = exp.get_metric(metric)
+        values.append(f"\\textbf{{{val:.4f}}}" if val is not None else "---")
+
+        # Metadata
+        if include_metadata:
+            for field in include_metadata:
+                val = exp.metadata.get(field, "---")
+                val_str = str(val).replace("_", "\\_")
+                values.append(val_str)
+
+        # Cost
+        if has_cost:
+            cost = exp.get_cost()
+            values.append(f"{cost:.4f}" if cost is not None else "---")
+
+        # Other metrics
+        for m in other_metrics:
+            val = exp.get_metric(m)
+            values.append(f"{val:.4f}" if val is not None else "---")
+
+        values.append(str(exp.sample_count))
+        values.append(str(exp.failure_count))
+
+        lines.append(" & ".join(values) + " \\\\")
+
+    lines.append("\\bottomrule")
+    lines.append("\\end{tabular}")
+    lines.append("\\end{table}")
+
+    return "\n".join(lines)
+
+
+def _generate_csv_leaderboard(
+    ranked,
+    metric: str,
+    include_cost: bool,
+    include_metadata: list[str] | None,
+    comparison,
+) -> str:
+    """Generate CSV leaderboard."""
+    import csv
+    import io
+
+    output = io.StringIO()
+
+    # Build header
+    headers = ["rank", "run_id", metric]
+
+    if include_metadata:
+        headers.extend(include_metadata)
+
+    has_cost = include_cost and any(exp.get_cost() is not None for exp in ranked)
+    if has_cost:
+        headers.append("cost")
+
+    other_metrics = [m for m in comparison.metrics if m != metric]
+    headers.extend(other_metrics)
+    headers.extend(["sample_count", "failure_count"])
+
+    writer = csv.DictWriter(output, fieldnames=headers)
+    writer.writeheader()
+
+    # Write rows
+    for rank, exp in enumerate(ranked, 1):
+        row = {"rank": rank, "run_id": exp.run_id}
+
+        # Primary metric
+        val = exp.get_metric(metric)
+        row[metric] = val if val is not None else ""
+
+        # Metadata
+        if include_metadata:
+            for field in include_metadata:
+                row[field] = exp.metadata.get(field, "")
+
+        # Cost
+        if has_cost:
+            cost = exp.get_cost()
+            row["cost"] = cost if cost is not None else ""
+
+        # Other metrics
+        for m in other_metrics:
+            val = exp.get_metric(m)
+            row[m] = val if val is not None else ""
+
+        row["sample_count"] = exp.sample_count
+        row["failure_count"] = exp.failure_count
+
+        writer.writerow(row)
+
+    return output.getvalue()
+
+
+__all__ = ["leaderboard_command"]

themis/cli/commands/math_benchmarks.py

@@ -0,0 +1,318 @@
+"""Math benchmark command implementations."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Callable, Literal, Sequence
+
+from cyclopts import Parameter
+
+from themis.cli.utils import effective_total, export_outputs
+from themis.datasets import (
+    competition_math as competition_math_dataset,
+)
+from themis.datasets import (
+    math500 as math500_dataset,
+)
+from themis.experiment import math as math_experiment
+from themis.experiment import storage as experiment_storage
+from themis.utils.logging_utils import configure_logging
+from themis.utils.progress import ProgressReporter
+
+
+def load_math_dataset(
+    *,
+    source: Literal["huggingface", "local"],
+    data_dir: Path | None,
+    limit: int | None,
+    subjects: Sequence[str] | None,
+    split: str = "test",
+):
+    """Load MATH-500 dataset.
+
+    Args:
+        source: Dataset source (huggingface or local)
+        data_dir: Directory containing local dataset
+        limit: Max rows to load
+        subjects: Subjects to filter
+        split: Dataset split
+
+    Returns:
+        List of generation examples
+    """
+    if source == "local":
+        if data_dir is None:
+            raise ValueError(
+                "The --data-dir option is required when --source=local so Themis "
+                "knows where to read the dataset."
+            )
+        samples = math500_dataset.load_math500(
+            source="local",
+            data_dir=data_dir,
+            split=split,
+            limit=limit,
+            subjects=subjects,
+        )
+    else:
+        samples = math500_dataset.load_math500(
+            source="huggingface",
+            split=split,
+            limit=limit,
+            subjects=subjects,
+        )
+    return [sample.to_generation_example() for sample in samples]
+
+
+def load_competition_math_dataset(
+    *,
+    dataset: str,
+    subset: str | None,
+    source: Literal["huggingface", "local"],
+    data_dir: Path | None,
+    split: str,
+    limit: int | None,
+    subjects: Sequence[str] | None,
+):
+    """Load competition math dataset.
+
+    Args:
+        dataset: Dataset name
+        subset: Dataset subset
+        source: Dataset source
+        data_dir: Directory containing local dataset
+        split: Dataset split
+        limit: Max rows to load
+        subjects: Subjects to filter
+
+    Returns:
+        List of generation examples
+    """
+    if source == "local" and data_dir is None:
+        raise ValueError(
+            "The --data-dir option is required when --source=local so Themis "
+            "knows where to read the dataset."
+        )
+    samples = competition_math_dataset.load_competition_math(
+        dataset=dataset,
+        subset=subset,
+        source=source,
+        data_dir=data_dir,
+        split=split,
+        limit=limit,
+        subjects=subjects,
+    )
+    return [sample.to_generation_example() for sample in samples]
+
+
+def run_math_benchmark(
+    rows: Sequence[dict[str, object]],
+    *,
+    max_samples: int | None,
+    storage: Path | None,
+    run_id: str | None,
+    resume: bool,
+    temperature: float,
+    csv_output: Path | None,
+    html_output: Path | None,
+    json_output: Path | None,
+    title: str,
+    task_name: str,
+) -> int:
+    """Run math benchmark experiment.
+
+    Args:
+        rows: Dataset rows
+        max_samples: Maximum samples to run
+        storage: Cache directory
+        run_id: Run identifier
+        resume: Whether to resume from cache
+        temperature: Sampling temperature
+        csv_output: CSV export path
+        html_output: HTML export path
+        json_output: JSON export path
+        title: Experiment title
+        task_name: Task name
+
+    Returns:
+        Exit code
+    """
+    storage_impl = experiment_storage.ExperimentStorage(storage) if storage else None
+    experiment = math_experiment.build_math500_zero_shot_experiment(
+        temperature=temperature,
+        storage=storage_impl,
+        task_name=task_name,
+    )
+
+    total = effective_total(len(rows), max_samples)
+    with ProgressReporter(total=total, description="Generating") as progress:
+        report = experiment.run(
+            rows,
+            max_samples=max_samples,
+            run_id=run_id,
+            resume=resume,
+            on_result=progress.on_result,
+        )
+    print(math_experiment.summarize_report(report))
+    export_outputs(
+        report,
+        csv_output=csv_output,
+        html_output=html_output,
+        json_output=json_output,
+        title=f"{title} experiment",
+    )
+    return 0
+
+
+def math500_command(
+    *,
+    source: Annotated[
+        Literal["huggingface", "local"], Parameter(help="Dataset source")
+    ] = "huggingface",
+    data_dir: Annotated[
+        Path | None, Parameter(help="Directory containing local dataset")
+    ] = None,
+    limit: Annotated[int | None, Parameter(help="Max rows to load")] = None,
+    subjects: Annotated[tuple[str, ...], Parameter(help="Subjects to filter")] = (),
+    max_samples: Annotated[int | None, Parameter(help="Maximum samples to run")] = None,
+    storage: Annotated[
+        Path | None, Parameter(help="Cache directory for datasets/results")
+    ] = None,
+    run_id: Annotated[str | None, Parameter(help="Identifier for cached run")] = None,
+    resume: Annotated[
+        bool, Parameter(help="Reuse cached generations when storage is set")
+    ] = True,
+    temperature: Annotated[float, Parameter(help="Sampling temperature")] = 0.0,
+    log_level: Annotated[
+        str, Parameter(help="Logging level (critical/error/warning/info/debug/trace)")
+    ] = "info",
+    csv_output: Annotated[
+        Path | None, Parameter(help="Write CSV export to this path")
+    ] = None,
+    html_output: Annotated[
+        Path | None, Parameter(help="Write HTML summary to this path")
+    ] = None,
+    json_output: Annotated[
+        Path | None, Parameter(help="Write JSON export to this path")
+    ] = None,
+) -> int:
+    """Run the zero-shot MATH-500 evaluation."""
+    configure_logging(log_level)
+    subject_filter = list(subjects) if subjects else None
+    rows = load_math_dataset(
+        source=source,
+        data_dir=data_dir,
+        limit=limit,
+        subjects=subject_filter,
+    )
+    return run_math_benchmark(
+        rows,
+        max_samples=max_samples,
+        storage=storage,
+        run_id=run_id,
+        resume=resume,
+        temperature=temperature,
+        csv_output=csv_output,
+        html_output=html_output,
+        json_output=json_output,
+        title="math500",
+        task_name="math500",
+    )
+
+
+def _create_competition_math_command(
+    dataset_name: str,
+    dataset_id: str,
+    subset: str | None = None,
+) -> Callable:
+    """Create a competition math command function.
+
+    Args:
+        dataset_name: Display name for the dataset
+        dataset_id: HuggingFace dataset ID
+        subset: Optional dataset subset
+
+    Returns:
+        Command function
+    """
+
+    def command(
+        *,
+        source: Annotated[
+            Literal["huggingface", "local"], Parameter(help="Dataset source")
+        ] = "huggingface",
+        split: Annotated[str, Parameter(help="Dataset split to load")] = "test",
+        data_dir: Annotated[
+            Path | None, Parameter(help="Directory containing local dataset")
+        ] = None,
+        limit: Annotated[int | None, Parameter(help="Max rows to load")] = None,
+        subjects: Annotated[
+            tuple[str, ...], Parameter(help="Optional subject filters")
+        ] = (),
+        max_samples: Annotated[
+            int | None, Parameter(help="Maximum samples to run")
+        ] = None,
+        storage: Annotated[
+            Path | None, Parameter(help="Cache directory for datasets/results")
+        ] = None,
+        run_id: Annotated[
+            str | None, Parameter(help="Identifier for cached run")
+        ] = None,
+        resume: Annotated[
+            bool, Parameter(help="Reuse cached generations when storage is set")
+        ] = True,
+        temperature: Annotated[float, Parameter(help="Sampling temperature")] = 0.0,
+        log_level: Annotated[
+            str,
+            Parameter(help="Logging level (critical/error/warning/info/debug/trace)"),
+        ] = "info",
+        csv_output: Annotated[
+            Path | None, Parameter(help="Write CSV export to this path")
+        ] = None,
+        html_output: Annotated[
+            Path | None, Parameter(help="Write HTML summary to this path")
+        ] = None,
+        json_output: Annotated[
+            Path | None, Parameter(help="Write JSON export to this path")
+        ] = None,
+    ) -> int:
+        f"""Run the {dataset_name} benchmark."""
+        configure_logging(log_level)
+        subject_filter = list(subjects) if subjects else None
+        rows = load_competition_math_dataset(
+            dataset=dataset_id,
+            subset=subset,
+            source=source,
+            data_dir=data_dir,
+            split=split,
+            limit=limit,
+            subjects=subject_filter,
+        )
+
+        return run_math_benchmark(
+            rows,
+            max_samples=max_samples,
+            storage=storage,
+            run_id=run_id,
+            resume=resume,
+            temperature=temperature,
+            csv_output=csv_output,
+            html_output=html_output,
+            json_output=json_output,
+            title=dataset_name,
+            task_name=dataset_name,
+        )
+
+    command.__doc__ = f"Run the {dataset_name} benchmark."
+    return command
+
+
+# Create specific competition math commands
+aime24_command = _create_competition_math_command("aime24", "math-ai/aime24")
+aime25_command = _create_competition_math_command("aime25", "math-ai/aime25")
+amc23_command = _create_competition_math_command("amc23", "math-ai/amc23")
+olympiadbench_command = _create_competition_math_command(
+    "olympiadbench", "math-ai/olympiadbench"
+)
+beyond_aime_command = _create_competition_math_command(
+    "beyondaime", "ByteDance-Seed/BeyondAIME"
+)