themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/cli/commands/mcq_benchmarks.py

@@ -0,0 +1,207 @@
+"""Multiple-choice question benchmark commands."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Callable, Literal, Sequence
+
+from cyclopts import Parameter
+
+from themis.cli.utils import effective_total, export_outputs
+from themis.datasets import (
+    mmlu_pro as mmlu_pro_dataset,
+)
+from themis.datasets import (
+    super_gpqa as super_gpqa_dataset,
+)
+from themis.experiment import mcq as mcq_experiment
+from themis.experiment import storage as experiment_storage
+from themis.utils.logging_utils import configure_logging
+from themis.utils.progress import ProgressReporter
+
+
+def load_multiple_choice_dataset(
+    *,
+    loader: Callable[..., Sequence],
+    source: Literal["huggingface", "local"],
+    data_dir: Path | None,
+    split: str,
+    limit: int | None,
+    subjects: Sequence[str] | None,
+):
+    """Load multiple choice dataset.
+
+    Args:
+        loader: Dataset loader function
+        source: Dataset source
+        data_dir: Directory containing local dataset
+        split: Dataset split
+        limit: Max rows to load
+        subjects: Subjects to filter
+
+    Returns:
+        List of generation examples
+    """
+    if source == "local" and data_dir is None:
+        raise ValueError(
+            "The --data-dir option is required when --source=local so Themis "
+            "knows where to read the dataset."
+        )
+    samples = loader(
+        source=source,
+        data_dir=data_dir,
+        split=split,
+        limit=limit,
+        subjects=subjects,
+    )
+    return [sample.to_generation_example() for sample in samples]
+
+
+def supergpqa_command(
+    *,
+    source: Annotated[
+        Literal["huggingface", "local"], Parameter(help="Dataset source")
+    ] = "huggingface",
+    split: Annotated[str, Parameter(help="Dataset split to load")] = "test",
+    data_dir: Annotated[
+        Path | None, Parameter(help="Directory containing local dataset")
+    ] = None,
+    limit: Annotated[int | None, Parameter(help="Max rows to load")] = None,
+    subjects: Annotated[
+        tuple[str, ...], Parameter(help="Subjects or categories to filter")
+    ] = (),
+    max_samples: Annotated[int | None, Parameter(help="Maximum samples to run")] = None,
+    storage: Annotated[
+        Path | None, Parameter(help="Cache directory for datasets/results")
+    ] = None,
+    run_id: Annotated[str | None, Parameter(help="Identifier for cached run")] = None,
+    resume: Annotated[
+        bool, Parameter(help="Reuse cached generations when storage is set")
+    ] = True,
+    temperature: Annotated[float, Parameter(help="Sampling temperature")] = 0.0,
+    log_level: Annotated[
+        str, Parameter(help="Logging level (critical/error/warning/info/debug/trace)")
+    ] = "info",
+    csv_output: Annotated[
+        Path | None, Parameter(help="Write CSV export to this path")
+    ] = None,
+    html_output: Annotated[
+        Path | None, Parameter(help="Write HTML summary to this path")
+    ] = None,
+    json_output: Annotated[
+        Path | None, Parameter(help="Write JSON export to this path")
+    ] = None,
+) -> int:
+    """Run the SuperGPQA multiple-choice evaluation."""
+    configure_logging(log_level)
+    subject_filter = list(subjects) if subjects else None
+    rows = load_multiple_choice_dataset(
+        loader=super_gpqa_dataset.load_super_gpqa,
+        source=source,
+        data_dir=data_dir,
+        split=split,
+        limit=limit,
+        subjects=subject_filter,
+    )
+
+    storage_impl = experiment_storage.ExperimentStorage(storage) if storage else None
+    experiment = mcq_experiment.build_multiple_choice_json_experiment(
+        dataset_name="supergpqa",
+        task_id="supergpqa",
+        temperature=temperature,
+        storage=storage_impl,
+    )
+
+    total = effective_total(len(rows), max_samples)
+    with ProgressReporter(total=total, description="Generating") as progress:
+        report = experiment.run(
+            rows,
+            max_samples=max_samples,
+            run_id=run_id,
+            resume=resume,
+            on_result=progress.on_result,
+        )
+    print(mcq_experiment.summarize_report(report))
+    export_outputs(
+        report,
+        csv_output=csv_output,
+        html_output=html_output,
+        json_output=json_output,
+        title="supergpqa experiment",
+    )
+    return 0
+
+
+def mmlu_pro_command(
+    *,
+    source: Annotated[
+        Literal["huggingface", "local"], Parameter(help="Dataset source")
+    ] = "huggingface",
+    split: Annotated[str, Parameter(help="Dataset split to load")] = "test",
+    data_dir: Annotated[
+        Path | None, Parameter(help="Directory containing local dataset")
+    ] = None,
+    limit: Annotated[int | None, Parameter(help="Max rows to load")] = None,
+    subjects: Annotated[
+        tuple[str, ...], Parameter(help="Subjects or categories to filter")
+    ] = (),
+    max_samples: Annotated[int | None, Parameter(help="Maximum samples to run")] = None,
+    storage: Annotated[
+        Path | None, Parameter(help="Cache directory for datasets/results")
+    ] = None,
+    run_id: Annotated[str | None, Parameter(help="Identifier for cached run")] = None,
+    resume: Annotated[
+        bool, Parameter(help="Reuse cached generations when storage is set")
+    ] = True,
+    temperature: Annotated[float, Parameter(help="Sampling temperature")] = 0.0,
+    log_level: Annotated[
+        str, Parameter(help="Logging level (critical/error/warning/info/debug/trace)")
+    ] = "info",
+    csv_output: Annotated[
+        Path | None, Parameter(help="Write CSV export to this path")
+    ] = None,
+    html_output: Annotated[
+        Path | None, Parameter(help="Write HTML summary to this path")
+    ] = None,
+    json_output: Annotated[
+        Path | None, Parameter(help="Write JSON export to this path")
+    ] = None,
+) -> int:
+    """Run the MMLU-Pro multiple-choice evaluation."""
+    configure_logging(log_level)
+    subject_filter = list(subjects) if subjects else None
+    rows = load_multiple_choice_dataset(
+        loader=mmlu_pro_dataset.load_mmlu_pro,
+        source=source,
+        data_dir=data_dir,
+        split=split,
+        limit=limit,
+        subjects=subject_filter,
+    )
+
+    storage_impl = experiment_storage.ExperimentStorage(storage) if storage else None
+    experiment = mcq_experiment.build_multiple_choice_json_experiment(
+        dataset_name="mmlu-pro",
+        task_id="mmlu_pro",
+        temperature=temperature,
+        storage=storage_impl,
+    )
+
+    total = effective_total(len(rows), max_samples)
+    with ProgressReporter(total=total, description="Generating") as progress:
+        report = experiment.run(
+            rows,
+            max_samples=max_samples,
+            run_id=run_id,
+            resume=resume,
+            on_result=progress.on_result,
+        )
+    print(mcq_experiment.summarize_report(report))
+    export_outputs(
+        report,
+        csv_output=csv_output,
+        html_output=html_output,
+        json_output=json_output,
+        title="mmlu_pro experiment",
+    )
+    return 0

themis/cli/commands/results.py

@@ -0,0 +1,252 @@
+"""Quick results viewing commands for experiment summaries."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+from cyclopts import Parameter
+
+
+def summary_command(
+    *,
+    run_id: Annotated[
+        str,
+        Parameter(
+            help="Run ID to view summary for",
+        ),
+    ],
+    storage: Annotated[
+        Path,
+        Parameter(
+            help="Storage directory containing experiment results",
+        ),
+    ] = Path(".cache/runs"),
+) -> int:
+    """View quick summary of a single experiment run.
+
+    This command reads the lightweight summary.json file (~1KB) instead of
+    the full report.json (~1.6MB), making it much faster for quick checks.
+
+    Examples:
+        # View summary for a specific run
+        uv run python -m themis.cli results summary \\
+          --run-id run-20260118-032014 \\
+          --storage outputs/evaluation
+
+        # Quick check of latest run
+        uv run python -m themis.cli results summary \\
+          --run-id $(ls -t outputs/evaluation | head -1)
+    """
+    try:
+        # Try to find summary.json
+        run_dir = storage / run_id
+        summary_path = run_dir / "summary.json"
+
+        if not summary_path.exists():
+            print(f"Error: Summary file not found at {summary_path}")
+            print("\nNote: summary.json is only available for runs created with")
+            print("the updated export functionality. For older runs, use the")
+            print("'compare' command which reads full report.json files.")
+            return 1
+
+        # Load summary
+        with summary_path.open("r", encoding="utf-8") as f:
+            summary = json.load(f)
+
+        # Display summary
+        print("=" * 80)
+        print(f"Experiment Summary: {run_id}")
+        print("=" * 80)
+
+        # Basic info
+        print(f"\nRun ID: {summary.get('run_id', 'N/A')}")
+        print(f"Total Samples: {summary.get('total_samples', 0)}")
+
+        # Metadata
+        metadata = summary.get("metadata", {})
+        if metadata:
+            print("\nConfiguration:")
+            print(f"  Model: {metadata.get('model', 'N/A')}")
+            print(f"  Prompt: {metadata.get('prompt_template', 'N/A')}")
+            sampling = metadata.get("sampling", {})
+            if sampling:
+                print(f"  Temperature: {sampling.get('temperature', 'N/A')}")
+                print(f"  Max Tokens: {sampling.get('max_tokens', 'N/A')}")
+
+        # Metrics
+        metrics = summary.get("metrics", {})
+        if metrics:
+            print("\nMetrics:")
+            for name, data in metrics.items():
+                mean = data.get("mean", 0)
+                count = data.get("count", 0)
+                print(f"  {name}: {mean:.4f} (n={count})")
+
+        # Cost
+        cost = summary.get("cost_usd")
+        if cost is not None:
+            print(f"\nCost: ${cost:.4f}")
+
+        # Failures
+        failures = summary.get("failures", 0)
+        failure_rate = summary.get("failure_rate", 0)
+        if failures > 0:
+            print(f"\nFailures: {failures} ({failure_rate:.2%})")
+
+        print("\n" + "=" * 80)
+        return 0
+
+    except FileNotFoundError:
+        print(f"Error: Run directory not found: {run_dir}")
+        return 1
+    except json.JSONDecodeError as e:
+        print(f"Error: Invalid JSON in summary file: {e}")
+        return 1
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+def list_command(
+    *,
+    storage: Annotated[
+        Path,
+        Parameter(
+            help="Storage directory containing experiment results",
+        ),
+    ] = Path(".cache/runs"),
+    limit: Annotated[
+        int | None,
+        Parameter(
+            help="Maximum number of runs to display",
+        ),
+    ] = None,
+    sort_by: Annotated[
+        str,
+        Parameter(
+            help="Sort runs by: time (newest first) or metric name",
+        ),
+    ] = "time",
+) -> int:
+    """List all experiment runs with quick summaries.
+
+    This command scans for summary.json files and displays a table of all runs.
+    Much faster than loading full report.json files.
+
+    Examples:
+        # List all runs
+        uv run python -m themis.cli results list
+
+        # List 10 most recent runs
+        uv run python -m themis.cli results list --limit 10
+
+        # List runs sorted by accuracy
+        uv run python -m themis.cli results list --sort-by accuracy
+    """
+    try:
+        if not storage.exists():
+            print(f"Error: Storage directory not found: {storage}")
+            return 1
+
+        # Find all summary.json files
+        summaries = []
+        for run_dir in storage.iterdir():
+            if not run_dir.is_dir():
+                continue
+            summary_path = run_dir / "summary.json"
+            if summary_path.exists():
+                try:
+                    with summary_path.open("r", encoding="utf-8") as f:
+                        summary = json.load(f)
+                    summary["_run_dir"] = run_dir.name
+                    summary["_mtime"] = summary_path.stat().st_mtime
+                    summaries.append(summary)
+                except Exception:
+                    continue
+
+        if not summaries:
+            print(f"No experiment runs found in {storage}")
+            print("\nNote: Only runs with summary.json files are shown.")
+            return 0
+
+        # Sort summaries
+        if sort_by == "time":
+            summaries.sort(key=lambda s: s.get("_mtime", 0), reverse=True)
+        else:
+            # Sort by metric value
+            summaries.sort(
+                key=lambda s: s.get("metrics", {}).get(sort_by, {}).get("mean", 0),
+                reverse=True,
+            )
+
+        # Apply limit
+        if limit:
+            summaries = summaries[:limit]
+
+        # Display table
+        print("=" * 120)
+        print(f"Found {len(summaries)} experiment run(s)")
+        print("=" * 120)
+
+        # Collect all metric names
+        all_metrics = set()
+        for s in summaries:
+            all_metrics.update(s.get("metrics", {}).keys())
+        metric_names = sorted(all_metrics)
+
+        # Header
+        header_cols = ["Run ID", "Model", "Samples"] + metric_names + ["Cost ($)"]
+        col_widths = [25, 30, 8] + [12] * len(metric_names) + [10]
+
+        header = " | ".join(
+            col.ljust(width)[:width] for col, width in zip(header_cols, col_widths)
+        )
+        print(header)
+        print("-" * len(header))
+
+        # Rows
+        for summary in summaries:
+            run_id = summary.get("_run_dir", "N/A")[:25]
+            model = summary.get("metadata", {}).get("model", "N/A")[:30]
+            samples = str(summary.get("total_samples", 0))
+            cost = summary.get("cost_usd")
+
+            row_values = [run_id, model, samples]
+
+            # Add metric values
+            for metric_name in metric_names:
+                metric_data = summary.get("metrics", {}).get(metric_name, {})
+                mean = metric_data.get("mean")
+                if mean is not None:
+                    row_values.append(f"{mean:.4f}")
+                else:
+                    row_values.append("N/A")
+
+            # Add cost
+            if cost is not None:
+                row_values.append(f"{cost:.4f}")
+            else:
+                row_values.append("N/A")
+
+            row = " | ".join(
+                val.ljust(width)[:width] for val, width in zip(row_values, col_widths)
+            )
+            print(row)
+
+        print("=" * 120)
+        return 0
+
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+__all__ = ["summary_command", "list_command"]

themis/cli/commands/sample_run.py

@@ -0,0 +1,244 @@
+"""Sample run command for quick testing before full experiments."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.cli.commands.config_commands import run_configured_experiment
+
+
+def sample_run_command(
+    *,
+    config: Annotated[Path, Parameter(help="Path to experiment configuration file")],
+    n: Annotated[int, Parameter(help="Number of samples to test")] = 5,
+    verbose: Annotated[bool, Parameter(help="Show detailed output")] = False,
+    show_outputs: Annotated[
+        bool, Parameter(help="Display sample outputs and predictions")
+    ] = False,
+    estimate_cost: Annotated[
+        bool, Parameter(help="Estimate full run cost based on sample")
+    ] = True,
+) -> int:
+    """Quick test run on N samples before running full experiment.
+
+    This command helps you:
+    - Test your configuration works correctly
+    - Preview sample outputs before full run
+    - Estimate total cost based on actual token usage
+    - Catch configuration errors early
+    - Iterate on prompts quickly
+
+    Examples:
+        # Basic quick test
+        uv run python -m themis.cli sample-run \\
+          --config my_config.yaml \\
+          --n 5
+
+        # Test with verbose output
+        uv run python -m themis.cli sample-run \\
+          --config my_config.yaml \\
+          --n 3 \\
+          --verbose \\
+          --show-outputs
+
+        # Test and estimate full run cost
+        uv run python -m themis.cli sample-run \\
+          --config my_config.yaml \\
+          --n 10 \\
+          --estimate-cost
+    """
+    try:
+        import json
+        import tempfile
+
+        from hydra import compose, initialize_config_dir
+
+        # Load config
+        config_path = Path(config).resolve()
+        if not config_path.exists():
+            print(f"Error: Config file not found: {config_path}")
+            return 1
+
+        config_dir = str(config_path.parent)
+        config_name = config_path.stem
+
+        print("=" * 80)
+        print(f"🧪 Sample Run: Testing {n} samples")
+        print("=" * 80)
+        print(f"Config: {config_path}")
+        print(f"Samples: {n}")
+        print()
+
+        # Initialize Hydra
+        with initialize_config_dir(config_dir=config_dir, version_base=None):
+            cfg = compose(config_name=config_name)
+
+            # Override dataset limit
+            original_limit = cfg.dataset.get("limit")
+            cfg.dataset.limit = n
+
+            # Use temporary storage
+            with tempfile.TemporaryDirectory() as temp_dir:
+                cfg.storage.path = temp_dir
+
+                # Generate temporary run_id
+                cfg.run_id = "sample-run-temp"
+                cfg.resume = False
+
+                print("📋 Configuration:")
+                print(f"  Model: {cfg.generation.model_identifier}")
+                print(f"  Provider: {cfg.generation.provider.name}")
+                print(f"  Temperature: {cfg.generation.sampling.temperature}")
+                print(f"  Max tokens: {cfg.generation.sampling.max_tokens}")
+                if hasattr(cfg.dataset, "source"):
+                    print(f"  Dataset: {cfg.dataset.source}")
+                print()
+
+                # Run experiment on sample
+                print("🚀 Running sample experiment...")
+                print()
+
+                # Redirect to capture run
+                result = run_configured_experiment(
+                    config_path=config_path,
+                    overrides=[
+                        f"dataset.limit={n}",
+                        f"storage.path={temp_dir}",
+                        "run_id=sample-run-temp",
+                        "resume=false",
+                    ],
+                )
+
+                if result != 0:
+                    print("\n❌ Sample run failed")
+                    return result
+
+                # Load results
+                report_path = Path(temp_dir) / "sample-run-temp" / "report.json"
+                if not report_path.exists():
+                    print("\n⚠️  No report generated")
+                    return 1
+
+                with report_path.open("r") as f:
+                    report_data = json.load(f)
+
+                # Display results
+                print("\n" + "=" * 80)
+                print("✅ Sample Run Complete")
+                print("=" * 80)
+
+                # Metrics
+                metrics = report_data.get("metrics", [])
+                if metrics:
+                    print("\n📊 Metrics:")
+                    for metric in metrics:
+                        name = metric["name"]
+                        mean = metric["mean"]
+                        count = metric["count"]
+                        print(f"  {name}: {mean:.4f} (n={count})")
+
+                # Cost analysis
+                cost_data = report_data.get("summary", {}).get("cost")
+                if cost_data:
+                    total_cost = cost_data.get("total_cost", 0)
+                    token_counts = cost_data.get("token_counts", {})
+                    prompt_tokens = token_counts.get("prompt_tokens", 0)
+                    completion_tokens = token_counts.get("completion_tokens", 0)
+
+                    print("\n💰 Cost (sample run):")
+                    print(f"  Total: ${total_cost:.4f}")
+                    print(f"  Per sample: ${total_cost / n:.6f}")
+                    print(
+                        f"  Prompt tokens: {prompt_tokens} ({prompt_tokens / n:.0f} avg)"
+                    )
+                    print(
+                        f"  Completion tokens: {completion_tokens} ({completion_tokens / n:.0f} avg)"
+                    )
+
+                    # Estimate full run cost
+                    if estimate_cost and original_limit:
+                        full_cost = (total_cost / n) * original_limit
+                        print("\n📈 Estimated full run cost:")
+                        print(f"  Dataset size: {original_limit} samples")
+                        print(f"  Estimated cost: ${full_cost:.2f}")
+                        print(
+                            f"  95% CI: ${full_cost * 0.8:.2f} - ${full_cost * 1.2:.2f}"
+                        )
+
+                        if full_cost > 10.0:
+                            print(f"\n⚠️  Warning: Estimated cost is ${full_cost:.2f}")
+                            print("  Consider using --limit for initial testing")
+
+                # Failures
+                failures = report_data.get("run_failures", [])
+                eval_failures = report_data.get("evaluation_failures", [])
+                total_failures = len(failures) + len(eval_failures)
+
+                if total_failures > 0:
+                    print(f"\n⚠️  Failures: {total_failures}")
+                    if failures:
+                        print(f"  Generation failures: {len(failures)}")
+                        if verbose:
+                            for failure in failures[:3]:
+                                print(
+                                    f"    - {failure.get('sample_id')}: {failure.get('message')}"
+                                )
+                    if eval_failures:
+                        print(f"  Evaluation failures: {len(eval_failures)}")
+
+                # Show sample outputs
+                if show_outputs:
+                    samples = report_data.get("samples", [])
+                    print("\n📝 Sample Outputs (showing up to 3):")
+                    for i, sample in enumerate(samples[:3], 1):
+                        sample_id = sample.get("sample_id", f"sample-{i}")
+                        scores = sample.get("scores", [])
+
+                        print(f"\n  Sample {i}: {sample_id}")
+                        if scores:
+                            for score in scores:
+                                metric_name = score.get("metric")
+                                value = score.get("value")
+                                print(f"    {metric_name}: {value:.4f}")
+
+                # Summary
+                print("\n" + "=" * 80)
+                print("✨ Next Steps:")
+                print("=" * 80)
+
+                if total_failures == 0 and metrics:
+                    avg_metric = metrics[0]["mean"]
+                    if avg_metric > 0.1:  # Reasonable performance
+                        print("  ✅ Configuration looks good!")
+                        print("  Run full experiment with:")
+                        print(
+                            f"     uv run python -m themis.cli run-config --config {config_path}"
+                        )
+                    else:
+                        print("  ⚠️  Low performance on sample - consider:")
+                        print("     - Adjusting prompt template")
+                        print("     - Tuning temperature/max_tokens")
+                        print("     - Testing different model")
+                else:
+                    print("  ⚠️  Issues detected:")
+                    if total_failures > 0:
+                        print("     - Fix failures before full run")
+                    if not metrics:
+                        print("     - Check evaluation metrics")
+                    print("     - Review configuration")
+
+                return 0
+
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+
+        if verbose:
+            traceback.print_exc()
+        return 1
+
+
+__all__ = ["sample_run_command"]