themis-eval 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

themis/cli/commands/visualize.py

@@ -0,0 +1,299 @@
+"""Visualization commands for interactive charts."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+from cyclopts import Parameter
+
+from themis.experiment.comparison import compare_experiments
+from themis.experiment.visualization import (
+    PLOTLY_AVAILABLE,
+    InteractiveVisualizer,
+    export_interactive_html,
+)
+
+
+def visualize_comparison_command(
+    *,
+    run_ids: Annotated[list[str], Parameter(help="Run IDs to visualize")],
+    storage: Annotated[Path, Parameter(help="Storage directory")] = Path(".cache/runs"),
+    metric: Annotated[str | None, Parameter(help="Metric to visualize")] = None,
+    output: Annotated[Path, Parameter(help="Output HTML file path")] = Path(
+        "visualization.html"
+    ),
+    chart_type: Annotated[
+        str,
+        Parameter(help="Chart type: comparison, evolution, dashboard, pareto"),
+    ] = "comparison",
+) -> int:
+    """Generate interactive visualization for experiments.
+
+    Examples:
+        # Bar chart comparing accuracy across runs
+        uv run python -m themis.cli visualize \\
+          --run-ids run-1 run-2 run-3 \\
+          --metric accuracy \\
+          --output accuracy_comparison.html
+
+        # Evolution chart showing metric over time
+        uv run python -m themis.cli visualize \\
+          --run-ids run-1 run-2 run-3 run-4 \\
+          --metric accuracy \\
+          --chart-type evolution \\
+          --output accuracy_evolution.html
+
+        # Dashboard with multiple metrics
+        uv run python -m themis.cli visualize \\
+          --run-ids run-1 run-2 run-3 \\
+          --chart-type dashboard \\
+          --output dashboard.html
+
+        # Pareto frontier (requires --pareto-metrics and --maximize)
+        uv run python -m themis.cli visualize-pareto \\
+          --run-ids run-1 run-2 run-3 \\
+          --metric1 accuracy \\
+          --metric2 cost \\
+          --output pareto.html
+    """
+    if not PLOTLY_AVAILABLE:
+        print("Error: Plotly is not installed.")
+        print("Install with: pip install plotly")
+        return 1
+
+    try:
+        # Load experiments
+        print(f"Loading experiments from {storage}...")
+        comparison = compare_experiments(
+            run_ids=run_ids,
+            storage_dir=storage,
+            include_metadata=True,
+        )
+
+        print(f"✓ Loaded {len(comparison.experiments)} experiments")
+
+        # Create visualizer
+        visualizer = InteractiveVisualizer()
+
+        # Generate chart based on type
+        if chart_type == "comparison":
+            if not metric:
+                metric = comparison.metrics[0] if comparison.metrics else "accuracy"
+                print(f"Using default metric: {metric}")
+
+            print(f"Creating comparison chart for '{metric}'...")
+            fig = visualizer.plot_metric_comparison(comparison, metric)
+
+        elif chart_type == "evolution":
+            if not metric:
+                metric = comparison.metrics[0] if comparison.metrics else "accuracy"
+                print(f"Using default metric: {metric}")
+
+            print(f"Creating evolution chart for '{metric}'...")
+            fig = visualizer.plot_metric_evolution(comparison, metric)
+
+        elif chart_type == "dashboard":
+            print("Creating dashboard with multiple metrics...")
+            fig = visualizer.create_dashboard(comparison)
+
+        else:
+            print(f"Error: Unknown chart type '{chart_type}'")
+            print("Available: comparison, evolution, dashboard")
+            return 1
+
+        # Export to HTML
+        export_interactive_html(fig, output)
+        print(f"\n✓ Visualization saved to {output}")
+        print("  Open in browser to interact with chart")
+
+        return 0
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+
+def visualize_pareto_command(
+    *,
+    run_ids: Annotated[list[str], Parameter(help="Run IDs to visualize")],
+    storage: Annotated[Path, Parameter(help="Storage directory")] = Path(".cache/runs"),
+    metric1: Annotated[str, Parameter(help="First metric (x-axis)")],
+    metric2: Annotated[str, Parameter(help="Second metric (y-axis)")],
+    maximize1: Annotated[bool, Parameter(help="Maximize metric1")] = True,
+    maximize2: Annotated[bool, Parameter(help="Maximize metric2")] = True,
+    output: Annotated[Path, Parameter(help="Output HTML file path")] = Path(
+        "pareto.html"
+    ),
+) -> int:
+    """Generate Pareto frontier visualization.
+
+    Examples:
+        # Maximize accuracy, minimize cost
+        uv run python -m themis.cli visualize-pareto \\
+          --run-ids run-1 run-2 run-3 run-4 \\
+          --metric1 accuracy \\
+          --metric2 cost \\
+          --maximize1 true \\
+          --maximize2 false \\
+          --output pareto.html
+    """
+    if not PLOTLY_AVAILABLE:
+        print("Error: Plotly is not installed.")
+        print("Install with: pip install plotly")
+        return 1
+
+    try:
+        # Load experiments
+        print(f"Loading experiments from {storage}...")
+        comparison = compare_experiments(
+            run_ids=run_ids,
+            storage_dir=storage,
+            include_metadata=True,
+        )
+
+        print(f"✓ Loaded {len(comparison.experiments)} experiments")
+
+        # Compute Pareto frontier
+        print(f"Computing Pareto frontier for {metric1} and {metric2}...")
+        pareto_ids = comparison.pareto_frontier(
+            objectives=[metric1, metric2],
+            maximize=[maximize1, maximize2],
+        )
+
+        print(f"✓ Found {len(pareto_ids)} Pareto-optimal experiments:")
+        for run_id in pareto_ids:
+            print(f"  - {run_id}")
+
+        # Create visualization
+        visualizer = InteractiveVisualizer()
+        fig = visualizer.plot_pareto_frontier(
+            comparison, metric1, metric2, pareto_ids, maximize1, maximize2
+        )
+
+        # Export to HTML
+        export_interactive_html(fig, output)
+        print(f"\n✓ Visualization saved to {output}")
+        print("  Red points are Pareto-optimal")
+        print("  Blue points are dominated")
+
+        return 0
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+
+def visualize_distribution_command(
+    *,
+    run_id: Annotated[str, Parameter(help="Run ID to visualize")],
+    storage: Annotated[Path, Parameter(help="Storage directory")] = Path(".cache/runs"),
+    metric: Annotated[str, Parameter(help="Metric to visualize")],
+    plot_type: Annotated[
+        str, Parameter(help="Plot type: histogram, box, violin")
+    ] = "histogram",
+    output: Annotated[Path, Parameter(help="Output HTML file path")] = Path(
+        "distribution.html"
+    ),
+) -> int:
+    """Generate metric distribution visualization.
+
+    Shows the distribution of a metric across all samples in an experiment.
+
+    Examples:
+        # Histogram of accuracy scores
+        uv run python -m themis.cli visualize-distribution \\
+          --run-id my-run \\
+          --metric accuracy \\
+          --output accuracy_dist.html
+
+        # Violin plot
+        uv run python -m themis.cli visualize-distribution \\
+          --run-id my-run \\
+          --metric accuracy \\
+          --plot-type violin \\
+          --output accuracy_violin.html
+    """
+    if not PLOTLY_AVAILABLE:
+        print("Error: Plotly is not installed.")
+        print("Install with: pip install plotly")
+        return 1
+
+    try:
+        import json
+
+        # Load report
+        print(f"Loading report from {storage / run_id}...")
+        report_path = storage / run_id / "report.json"
+
+        if not report_path.exists():
+            print(f"Error: Report not found at {report_path}")
+            return 1
+
+        with report_path.open("r", encoding="utf-8") as f:
+            report_data = json.load(f)
+
+        # Extract evaluation report
+        # Note: This is simplified - in production you'd deserialize properly
+        from themis.core.entities import EvaluationRecord, MetricScore
+        from themis.evaluation.reports import EvaluationReport, MetricAggregate
+
+        # Build evaluation report from JSON
+        records = []
+        for sample_data in report_data.get("samples", []):
+            scores = [
+                MetricScore(
+                    metric_name=score["metric"],
+                    value=score["value"],
+                    details=score.get("details"),
+                    metadata=score.get("metadata", {}),
+                )
+                for score in sample_data["scores"]
+            ]
+            records.append(
+                EvaluationRecord(
+                    sample_id=sample_data["sample_id"],
+                    scores=scores,
+                    failures=[],
+                )
+            )
+
+        # Build metric aggregates
+        metrics = {}
+        for metric_data in report_data.get("metrics", []):
+            metrics[metric_data["name"]] = MetricAggregate(
+                count=metric_data["count"],
+                mean=metric_data["mean"],
+            )
+
+        eval_report = EvaluationReport(
+            records=records,
+            metrics=metrics,
+            failures=[],
+        )
+
+        print(f"✓ Loaded report with {len(records)} samples")
+
+        # Create visualization
+        visualizer = InteractiveVisualizer()
+        fig = visualizer.plot_metric_distribution(eval_report, metric, plot_type)
+
+        # Export to HTML
+        export_interactive_html(fig, output)
+        print(f"\n✓ Visualization saved to {output}")
+
+        return 0
+
+    except Exception as e:
+        print(f"Error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+__all__ = [
+    "visualize_comparison_command",
+    "visualize_pareto_command",
+    "visualize_distribution_command",
+]

themis/cli/main.py

@@ -0,0 +1,93 @@
+"""Cyclopts-powered CLI entrypoints for Themis."""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+from cyclopts import App
+
+# Import command modules
+from themis.cli.commands import (
+    benchmarks,
+    comparison,
+    config_commands,
+    cost,
+    demo,
+    info,
+    leaderboard,
+    sample_run,
+    visualize,
+)
+from themis.cli.commands import math_benchmarks as math_cmds
+from themis.cli.commands import mcq_benchmarks as mcq_cmds
+
+# Import provider modules to ensure they register themselves
+try:
+    from themis.generation import clients  # noqa: F401 - registers fake provider
+    from themis.generation.providers import (
+        litellm_provider,  # noqa: F401
+        vllm_provider,  # noqa: F401
+    )
+except ImportError:
+    pass  # Some providers may not be available
+
+app = App(help="Run Themis experiments from the command line")
+
+# Register demo command
+app.command(name="demo")(demo.demo_command)
+
+# Register math benchmark commands
+app.command(name="math500")(math_cmds.math500_command)
+app.command(name="aime24")(math_cmds.aime24_command)
+app.command(name="aime25")(math_cmds.aime25_command)
+app.command(name="amc23")(math_cmds.amc23_command)
+app.command(name="olympiadbench")(math_cmds.olympiadbench_command)
+app.command(name="beyondaime")(math_cmds.beyond_aime_command)
+
+# Register MCQ benchmark commands
+app.command(name="supergpqa")(mcq_cmds.supergpqa_command)
+app.command(name="mmlu-pro")(mcq_cmds.mmlu_pro_command)
+
+# Register config commands
+app.command(name="run-config")(config_commands.run_configured_experiment)
+app.command(name="validate-config")(config_commands.validate_config)
+app.command(name="init")(config_commands.init_config)
+
+# Register info and listing commands
+app.command(name="list-providers")(benchmarks.list_providers)
+app.command(name="list-benchmarks")(benchmarks.list_benchmarks)
+app.command(name="info")(info.show_info)
+app.command(name="new-project")(info.new_project)
+
+# Register comparison commands
+app.command(name="compare")(comparison.compare_command)
+app.command(name="diff")(comparison.diff_command)
+app.command(name="pareto")(comparison.pareto_command)
+
+# Register cost commands
+app.command(name="estimate-cost")(cost.estimate_cost_command)
+app.command(name="show-pricing")(cost.show_pricing_command)
+
+# Register visualization commands
+app.command(name="visualize")(visualize.visualize_comparison_command)
+app.command(name="visualize-pareto")(visualize.visualize_pareto_command)
+app.command(name="visualize-distribution")(visualize.visualize_distribution_command)
+
+# Register leaderboard command
+app.command(name="leaderboard")(leaderboard.leaderboard_command)
+
+# Register sample-run command
+app.command(name="sample-run")(sample_run.sample_run_command)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parsed_argv = list(argv) if argv is not None else None
+    try:
+        result = app(parsed_argv)
+    except SystemExit as exc:  # pragma: no cover - CLI integration path
+        return int(exc.code or 0)
+    return int(result) if isinstance(result, int) else 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

themis/cli/new_project.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def create_project(project_name: str, project_path: Path) -> None:
+    if (project_path / project_name).exists():
+        raise FileExistsError(
+            f"Project '{project_name}' already exists in {project_path}"
+        )
+
+    project_dir = project_path / project_name
+    project_dir.mkdir()
+
+    templates_dir = Path(__file__).parent / "templates"
+
+    # Create config.sample.json
+    with open(templates_dir / "config.sample.json.tpl", "r") as f:
+        config_template = f.read()
+    with open(project_dir / "config.sample.json", "w") as f:
+        f.write(config_template.replace("{{project_name}}", project_name))
+
+    # Create cli.py
+    with open(templates_dir / "cli.py.tpl", "r") as f:
+        cli_template = f.read()
+    with open(project_dir / "cli.py", "w") as f:
+        f.write(cli_template)
+
+    # Create README.md
+    with open(templates_dir / "README.md.tpl", "r") as f:
+        readme_template = f.read()
+    with open(project_dir / "README.md", "w") as f:
+        f.write(readme_template.replace("{{project_name}}", project_name))

themis/cli/utils.py

@@ -0,0 +1,51 @@
+"""CLI utility functions."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from themis.experiment import export as experiment_export
+from themis.experiment import orchestrator
+
+
+def export_outputs(
+    report: orchestrator.ExperimentReport,
+    *,
+    csv_output: Path | None,
+    html_output: Path | None,
+    json_output: Path | None,
+    title: str,
+) -> None:
+    """Export experiment report to various formats.
+
+    Args:
+        report: Experiment report to export
+        csv_output: Optional path for CSV export
+        html_output: Optional path for HTML export
+        json_output: Optional path for JSON export
+        title: Title for the report
+    """
+    outputs = experiment_export.export_report_bundle(
+        report,
+        csv_path=csv_output,
+        html_path=html_output,
+        json_path=json_output,
+        title=title,
+    )
+    for kind, output_path in outputs.items():
+        print(f"Exported {kind.upper()} to {output_path}")
+
+
+def effective_total(total: int, limit: int | None) -> int:
+    """Calculate effective total based on limit.
+
+    Args:
+        total: Total number of items
+        limit: Optional limit
+
+    Returns:
+        Effective total (min of total and limit)
+    """
+    if limit is None:
+        return total
+    return min(total, limit)

themis/config/__init__.py

@@ -0,0 +1,19 @@
+"""Hydra-backed configuration helpers for assembling experiments."""
+
+from __future__ import annotations
+
+from .loader import load_experiment_config
+from .runtime import (
+    load_dataset_from_config,
+    run_experiment_from_config,
+    summarize_report_for_config,
+)
+from .schema import ExperimentConfig
+
+__all__ = [
+    "ExperimentConfig",
+    "load_dataset_from_config",
+    "load_experiment_config",
+    "run_experiment_from_config",
+    "summarize_report_for_config",
+]

themis/config/loader.py

@@ -0,0 +1,27 @@
+"""Utilities for loading experiment configs via Hydra/OmegaConf."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable
+
+from omegaconf import OmegaConf
+
+from . import schema
+
+
+def load_experiment_config(
+    config_path: Path,
+    overrides: Iterable[str] | None = None,
+) -> schema.ExperimentConfig:
+    """Load and validate an experiment config file with optional overrides."""
+
+    base = OmegaConf.structured(schema.ExperimentConfig)
+    file_conf = OmegaConf.load(config_path)
+    merged = OmegaConf.merge(base, file_conf)
+
+    if overrides:
+        override_conf = OmegaConf.from_dotlist(list(overrides))
+        merged = OmegaConf.merge(merged, override_conf)
+
+    return OmegaConf.to_object(merged)

themis/config/registry.py

@@ -0,0 +1,34 @@
+"""Registry for experiment builders."""
+
+from __future__ import annotations
+
+from typing import Callable
+
+from themis.config import schema
+from themis.experiment import orchestrator
+
+ExperimentBuilder = Callable[
+    [schema.ExperimentConfig], orchestrator.ExperimentOrchestrator
+]
+
+_EXPERIMENT_BUILDERS: dict[str, ExperimentBuilder] = {}
+
+
+def register_experiment_builder(task: str) -> Callable[[ExperimentBuilder], ExperimentBuilder]:
+    """Decorator to register an experiment builder for a specific task."""
+
+    def decorator(builder: ExperimentBuilder) -> ExperimentBuilder:
+        _EXPERIMENT_BUILDERS[task] = builder
+        return builder
+
+    return decorator
+
+
+def get_experiment_builder(task: str) -> ExperimentBuilder:
+    """Get the experiment builder for a specific task."""
+    if task not in _EXPERIMENT_BUILDERS:
+        raise ValueError(
+            f"No experiment builder registered for task '{task}'. "
+            f"Available tasks: {', '.join(sorted(_EXPERIMENT_BUILDERS.keys()))}"
+        )
+    return _EXPERIMENT_BUILDERS[task]