sandboxy 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

sandboxy/cli/main.py

@@ -1,6 +1,5 @@
 """CLI entrypoint for Sandboxy."""
 
-import csv
 import json
 import os
 import sys
@@ -35,6 +34,59 @@ def main() -> None:
     pass
 
 
+@main.command()
+@click.argument("shell", type=click.Choice(["bash", "zsh", "fish"]), default="bash")
+def completion(shell: str) -> None:
+    """Generate shell completion and show setup instructions.
+
+    Writes completion script to ~/.sandboxy-completion.<shell>
+    and shows the line to add to your shell config.
+
+    Examples:
+        sandboxy completion         # Generate bash completion
+        sandboxy completion zsh     # Generate zsh completion
+    """
+    import subprocess
+
+    home = Path.home()
+    ext = shell if shell != "bash" else "bash"
+    completion_file = home / f".sandboxy-completion.{ext}"
+
+    # Generate completion script using Click's built-in mechanism
+    env = os.environ.copy()
+    env["_SANDBOXY_COMPLETE"] = f"{shell}_source"
+
+    result = subprocess.run(  # noqa: S603
+        [sys.executable, "-m", "sandboxy.cli.main"],
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    # Write to file
+    completion_file.write_text(result.stdout)
+    click.echo(f"Generated: {completion_file}")
+    click.echo("")
+    click.echo("Add this line to your shell config:")
+    click.echo("")
+
+    if shell == "bash":
+        click.echo("# Sandboxy completion")
+        click.echo(f'. "{completion_file}"')
+        click.echo("")
+        click.echo("(Add to ~/.bashrc)")
+    elif shell == "zsh":
+        click.echo("# Sandboxy completion")
+        click.echo(f'. "{completion_file}"')
+        click.echo("")
+        click.echo("(Add to ~/.zshrc)")
+    elif shell == "fish":
+        click.echo("# Sandboxy completion")
+        click.echo(f'source "{completion_file}"')
+        click.echo("")
+        click.echo("(Add to ~/.config/fish/config.fish)")
+
+
 def _load_variables_from_env() -> dict:
     """Load variables from SANDBOXY_VARIABLES environment variable."""
     env_vars = os.environ.get("SANDBOXY_VARIABLES", "")
@@ -46,6 +98,322 @@ def _load_variables_from_env() -> dict:
         return {}
 
 
+def _export_to_mlflow(
+    result: Any,
+    spec: Any,
+    scenario_path: Path,
+    mlflow_export: bool,
+    no_mlflow: bool,
+    mlflow_tracking_uri: str | None,
+    mlflow_experiment: str | None,
+    agent_name: str = "default",
+) -> None:
+    """Export scenario result to MLflow if enabled.
+
+    Args:
+        result: ScenarioResult from runner
+        spec: ScenarioSpec
+        scenario_path: Path to scenario file
+        mlflow_export: --mlflow-export flag
+        no_mlflow: --no-mlflow flag
+        mlflow_tracking_uri: --mlflow-tracking-uri value
+        mlflow_experiment: --mlflow-experiment value
+        agent_name: Agent configuration name
+    """
+    from sandboxy.mlflow.config import MLflowConfig
+
+    # Get YAML config from spec
+    yaml_config = None
+    if spec.mlflow:
+        yaml_config = {
+            "enabled": spec.mlflow.enabled,
+            "experiment": spec.mlflow.experiment,
+            "tracking_uri": spec.mlflow.tracking_uri,
+            "tags": spec.mlflow.tags,
+        }
+
+    # Resolve config with precedence
+    config = MLflowConfig.resolve(
+        cli_export=mlflow_export,
+        cli_no_mlflow=no_mlflow,
+        cli_tracking_uri=mlflow_tracking_uri,
+        cli_experiment=mlflow_experiment,
+        yaml_config=yaml_config,
+        scenario_name=spec.name,
+    )
+
+    if not config.enabled:
+        return
+
+    # Import and use exporter
+    try:
+        from sandboxy.mlflow.exporter import MLflowExporter
+
+        exporter = MLflowExporter(config)
+
+        # Convert ScenarioResult to RunResult-like for exporter
+        # ScenarioResult has different structure, create adapter
+        run_id = exporter.export(
+            result=_adapt_scenario_result(result),
+            scenario_path=scenario_path,
+            scenario_name=spec.name,
+            scenario_id=spec.id,
+            agent_name=agent_name,
+        )
+
+        if run_id:
+            click.echo(f"\nExported to MLflow: run_id={run_id}")
+
+    except ImportError:
+        click.echo(
+            "\nMLflow not installed. Install with: pip install sandboxy[mlflow]",
+            err=True,
+        )
+    except Exception as e:
+        click.echo(f"\nWarning: MLflow export failed: {e}", err=True)
+
+
+def _adapt_scenario_result(result: Any) -> Any:
+    """Adapt ScenarioResult to RunResult-like interface for MLflowExporter.
+
+    The exporter expects RunResult fields, but ScenarioRunner returns ScenarioResult.
+    This creates an adapter object.
+    """
+    from dataclasses import dataclass, field
+
+    @dataclass
+    class GoalResultAdapter:
+        name: str
+        score: float
+        passed: bool = True
+
+    @dataclass
+    class EvaluationAdapter:
+        goals: list[GoalResultAdapter] = field(default_factory=list)
+        total_score: float = 0.0
+        max_score: float = 0.0
+        percentage: float = 0.0
+
+    @dataclass
+    class RunResultAdapter:
+        model: str = ""
+        error: str | None = None
+        latency_ms: int = 0
+        input_tokens: int = 0
+        output_tokens: int = 0
+        evaluation: EvaluationAdapter | None = None
+
+    # Extract data from ScenarioResult
+    adapter = RunResultAdapter(
+        model=getattr(result, "agent_id", "unknown"),
+        error=None,
+    )
+
+    # Build evaluation from goals
+    goals = []
+    total = 0.0
+    for goal_name in getattr(result, "goals_achieved", []):
+        goals.append(GoalResultAdapter(name=goal_name, score=1.0, passed=True))
+        total += 1.0
+
+    score = getattr(result, "score", 0.0)
+    max_score = max(score, len(goals)) if goals else score
+
+    adapter.evaluation = EvaluationAdapter(
+        goals=goals,
+        total_score=score,
+        max_score=max_score,
+        percentage=(score / max_score * 100) if max_score > 0 else 0.0,
+    )
+
+    return adapter
+
+
+@main.command()
+@click.option("--with-examples", is_flag=True, help="Include example scenarios and tools")
+@click.option(
+    "--dir",
+    "-d",
+    "directory",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Directory to initialize (default: current directory)",
+)
+def init(with_examples: bool, directory: Path | None) -> None:
+    """Initialize a new Sandboxy project.
+
+    Creates the standard folder structure for scenarios, tools, agents, and datasets.
+
+    Examples:
+        sandboxy init
+        sandboxy init --with-examples
+        sandboxy init --dir my-project
+    """
+    root = directory or Path.cwd()
+
+    # Create directory if specified and doesn't exist
+    if directory and not root.exists():
+        root.mkdir(parents=True)
+        click.echo(f"Created directory: {root}")
+
+    # Standard folders
+    folders = ["scenarios", "tools", "agents", "datasets", "runs"]
+    created = []
+
+    for folder in folders:
+        folder_path = root / folder
+        if not folder_path.exists():
+            folder_path.mkdir(parents=True)
+            created.append(folder)
+
+    if created:
+        click.echo(f"Created folders: {', '.join(created)}")
+    else:
+        click.echo("All folders already exist")
+
+    # Create .env.example if it doesn't exist
+    env_example = root / ".env.example"
+    if not env_example.exists():
+        env_example.write_text(
+            """# Sandboxy Environment Variables
+# Copy this to .env and fill in your API keys
+
+# OpenRouter API key (recommended - access to 400+ models)
+OPENROUTER_API_KEY=
+
+# Or use direct provider keys
+OPENAI_API_KEY=
+ANTHROPIC_API_KEY=
+"""
+        )
+        click.echo("Created .env.example")
+
+    # Create .gitignore if it doesn't exist
+    gitignore = root / ".gitignore"
+    if not gitignore.exists():
+        gitignore.write_text(
+            """.env
+runs/
+__pycache__/
+*.pyc
+"""
+        )
+        click.echo("Created .gitignore")
+
+    # Add examples if requested
+    if with_examples:
+        _create_example_files(root)
+
+    click.echo("")
+    click.echo("Project initialized! Next steps:")
+    click.echo("  1. Copy .env.example to .env and add your API key")
+    click.echo("  2. Create scenarios in scenarios/")
+    click.echo("  3. Run: sandboxy open")
+
+
+def _create_example_files(root: Path) -> None:
+    """Create example scenario and tool files."""
+    # Example scenario
+    example_scenario = root / "scenarios" / "hello-world.yml"
+    if not example_scenario.exists():
+        example_scenario.write_text(
+            """name: Hello World
+description: A simple greeting scenario to test your setup
+
+system_prompt: |
+  You are a friendly assistant. Greet the user warmly.
+
+prompt: |
+  Hello! Can you introduce yourself?
+
+evaluation:
+  goals:
+    - id: greeted
+      name: Greeted the user
+      description: The assistant should greet the user
+      outcome: true
+      check: "'hello' in response.lower() or 'hi' in response.lower()"
+"""
+        )
+        click.echo("Created scenarios/hello-world.yml")
+
+    # Example tool
+    example_tool = root / "tools" / "calculator.yml"
+    if not example_tool.exists():
+        example_tool.write_text(
+            """name: calculator
+description: A simple calculator tool
+
+tools:
+  calculator:
+    description: Perform basic math operations
+    actions:
+      add:
+        description: Add two numbers
+        parameters:
+          type: object
+          properties:
+            a:
+              type: number
+              description: First number
+            b:
+              type: number
+              description: Second number
+          required: [a, b]
+        returns:
+          result: "{{a}} + {{b}}"
+
+      multiply:
+        description: Multiply two numbers
+        parameters:
+          type: object
+          properties:
+            a:
+              type: number
+            b:
+              type: number
+          required: [a, b]
+        returns:
+          result: "{{a}} * {{b}}"
+"""
+        )
+        click.echo("Created tools/calculator.yml")
+
+    # Example scenario using the tool
+    tool_scenario = root / "scenarios" / "calculator-test.yml"
+    if not tool_scenario.exists():
+        tool_scenario.write_text(
+            """name: Calculator Test
+description: Test the calculator tool
+
+system_prompt: |
+  You are a helpful assistant with access to a calculator.
+  Use the calculator tool to perform math operations.
+
+tools_from:
+  - calculator
+
+prompt: |
+  What is 42 + 17?
+
+evaluation:
+  goals:
+    - id: used_calculator
+      name: Used calculator
+      description: The agent should use the calculator tool
+      outcome: true
+      check: "any(tc.tool == 'calculator' for tc in tool_calls)"
+
+    - id: correct_answer
+      name: Correct answer
+      description: The response should contain 59
+      outcome: true
+      check: "'59' in response"
+"""
+        )
+        click.echo("Created scenarios/calculator-test.yml")
+
+
 @main.command()
 @click.argument("module_path", type=click.Path(exists=True))
 @click.option("--agent-id", "-a", help="Agent ID to use", default=None)
@@ -135,148 +503,6 @@ def validate(module_path: str) -> None:
     click.echo("Module is valid.")
 
 
-@main.command()
-@click.argument("module_path", type=click.Path(exists=True))
-@click.option("--agents", required=True, help="Comma-separated agent IDs")
-@click.option("--runs-per-agent", type=int, default=1, help="Number of runs per agent")
-@click.option("--output", "-o", type=click.Path(), default=None, help="Output CSV file")
-@click.option("--var", "-v", multiple=True, help="Variable in name=value format")
-@click.option("--seed", type=int, default=None, help="Random seed for reproducibility")
-def bench(
-    module_path: str,
-    agents: str,
-    runs_per_agent: int,
-    output: str | None,
-    var: tuple[str, ...],
-    seed: int | None,
-) -> None:
-    """Benchmark a module against multiple agents.
-
-    MODULE_PATH is the path to an MDL YAML file.
-
-    Examples:
-        sandboxy bench modules/lemonade.yml --agents gpt4,claude --runs 5
-        sandboxy bench modules/lemonade.yml --agents gpt4 -v difficulty=8 -v starting_cash=100
-    """
-    import random
-
-    # Set random seed for reproducibility
-    if seed is not None:
-        random.seed(seed)
-
-    try:
-        module = load_module(Path(module_path))
-    except MDLParseError as e:
-        click.echo(f"Error loading module: {e}", err=True)
-        sys.exit(1)
-
-    # Load variables from environment and CLI
-    variables = _load_variables_from_env()
-    for v in var:
-        if "=" in v:
-            name, value = v.split("=", 1)
-            try:
-                variables[name] = json.loads(value)
-            except json.JSONDecodeError:
-                variables[name] = value
-
-    # Apply variables to module
-    if variables:
-        module = apply_variables(module, variables)
-        click.echo(f"Variables: {variables}")
-
-    loader = AgentLoader(DEFAULT_AGENT_DIRS)
-    agent_ids = [a.strip() for a in agents.split(",")]
-
-    results: list[dict[str, str | float | int]] = []
-
-    for agent_id in agent_ids:
-        try:
-            agent = loader.load(agent_id)
-        except ValueError as e:
-            click.echo(f"Warning: Skipping agent {agent_id}: {e}", err=True)
-            continue
-
-        # Apply module's agent_config overrides
-        if module.agent_config:
-            if "system_prompt" in module.agent_config:
-                agent.config.system_prompt = module.agent_config["system_prompt"]
-
-        click.echo(f"Benchmarking agent: {agent_id}")
-
-        for run_idx in range(runs_per_agent):
-            runner = Runner(module=module, agent=agent)
-            result = runner.run()
-
-            row: dict[str, str | float | int] = {
-                "agent_id": agent_id,
-                "run_idx": run_idx,
-                "score": result.evaluation.score,
-                "num_events": result.evaluation.num_events,
-                "status": result.evaluation.status,
-            }
-
-            # Add seed if used for reproducibility tracking
-            if seed is not None:
-                row["seed"] = seed
-
-            # Add env_state metrics if available
-            if "cash_balance" in runner.env_state:
-                row["final_cash"] = runner.env_state["cash_balance"]
-            if "starting_cash" in module.environment.initial_state:
-                initial = module.environment.initial_state["starting_cash"]
-                if "final_cash" in row:
-                    row["profit"] = float(row["final_cash"]) - float(initial)
-
-            # Add all evaluation check results
-            for check_name, check_result in result.evaluation.checks.items():
-                if isinstance(check_result, int | float | bool):
-                    row[f"check_{check_name}"] = check_result
-
-            results.append(row)
-            click.echo(f"  Run {run_idx + 1}: score={result.evaluation.score:.2f}")
-
-    if not results:
-        click.echo("No results to report.", err=True)
-        sys.exit(1)
-
-    # Output results
-    if output:
-        fieldnames = list(results[0].keys())
-        with open(output, "w", newline="") as f:
-            writer = csv.DictWriter(f, fieldnames=fieldnames)
-            writer.writeheader()
-            writer.writerows(results)
-        click.echo(f"\nResults saved to: {output}")
-    else:
-        # Print summary table
-        click.echo("\nBenchmark Results:")
-        click.echo("-" * 60)
-
-        # Group by agent
-        from collections import defaultdict
-
-        by_agent: dict[str, list[dict[str, str | float | int]]] = defaultdict(list)
-        for r in results:
-            by_agent[str(r["agent_id"])].append(r)
-
-        for agent_id, runs in by_agent.items():
-            scores = [r["score"] for r in runs if isinstance(r["score"], int | float)]
-            avg_score = sum(scores) / len(scores) if scores else 0
-            click.echo(f"{agent_id}:")
-            click.echo(f"  Runs: {len(runs)}")
-            click.echo(f"  Avg Score: {avg_score:.3f}")
-            if "final_cash" in runs[0]:
-                cash_values = [
-                    float(r["final_cash"])
-                    for r in runs
-                    if "final_cash" in r and isinstance(r["final_cash"], int | float)
-                ]
-                avg_cash = sum(cash_values) / len(cash_values) if cash_values else 0.0
-                click.echo(f"  Avg Final Cash: {avg_cash:.2f}")
-            click.echo("")
-
-
 @main.command()
 @click.option("--port", "-p", type=int, default=8000, help="Port to run server on")
 @click.option("--host", default="127.0.0.1", help="Host to bind to")
@@ -433,22 +659,54 @@ def info(module_path: str) -> None:
 @click.option(
     "--model",
     "-m",
-    help="Model to use (e.g., openai/gpt-4o, anthropic/claude-3.5-sonnet)",
-    default=None,
+    multiple=True,
+    help="Model(s) to use. Can specify multiple: -m gpt-4o -m claude-3.5-sonnet",
 )
 @click.option("--agent-id", "-a", help="Agent ID from config files", default=None)
 @click.option("--output", "-o", help="Output file for results JSON", default=None)
 @click.option("--pretty", "-p", is_flag=True, help="Pretty print output")
 @click.option("--max-turns", type=int, default=20, help="Maximum conversation turns")
 @click.option("--var", "-v", multiple=True, help="Variable in name=value format")
+@click.option(
+    "--mlflow-export",
+    is_flag=True,
+    help="Export run results to MLflow tracking server",
+)
+@click.option(
+    "--no-mlflow",
+    is_flag=True,
+    help="Disable MLflow export (overrides YAML config)",
+)
+@click.option(
+    "--mlflow-tracking-uri",
+    type=str,
+    default=None,
+    help="MLflow tracking server URI (overrides MLFLOW_TRACKING_URI env)",
+)
+@click.option(
+    "--mlflow-experiment",
+    type=str,
+    default=None,
+    help="MLflow experiment name (defaults to scenario name)",
+)
+@click.option(
+    "--mlflow-no-tracing",
+    is_flag=True,
+    help="Disable LLM call tracing (only log summary metrics)",
+)
 def scenario(
     scenario_path: str,
-    model: str | None,
+    model: tuple[str, ...],
     agent_id: str | None,
     output: str | None,
     pretty: bool,
     max_turns: int,
     var: tuple[str, ...],
+    mlflow_export: bool,
+    no_mlflow: bool,
+    mlflow_tracking_uri: str | None,
+    mlflow_experiment: str | None,
+    mlflow_no_tracing: bool,
 ) -> None:
     """Run a scenario with YAML-defined tools.
 
@@ -459,8 +717,10 @@ def scenario(
 
     Examples:
         sandboxy scenario scenarios/trolley.yml -m openai/gpt-4o
-        sandboxy scenario scenarios/trolley.yml -m anthropic/claude-3.5-sonnet -p
+        sandboxy scenario scenarios/trolley.yml -m gpt-4o -m claude-3.5-sonnet  # multiple models
         sandboxy scenario scenarios/surgeon.yml -v patient="John Smith" -v condition="critical"
+        sandboxy scenario scenarios/test.yml -m gpt-4o --mlflow-export
+        sandboxy scenario scenarios/test.yml -m gpt-4o -m gpt-4o-mini --mlflow-export  # compare models
     """
     from sandboxy.agents.base import AgentConfig
     from sandboxy.agents.llm_prompt import LlmPromptAgent
@@ -472,6 +732,26 @@ def scenario(
         click.echo(f"Error loading scenario: {e}", err=True)
         sys.exit(1)
 
+    # Build MLflow config if export requested
+    mlflow_config = None
+    if mlflow_export and not no_mlflow:
+        try:
+            from sandboxy.mlflow import MLflowConfig
+
+            mlflow_config = MLflowConfig.resolve(
+                cli_export=True,
+                cli_tracking_uri=mlflow_tracking_uri,
+                cli_experiment=mlflow_experiment,
+                cli_tracing=not mlflow_no_tracing,
+                yaml_config=spec.mlflow.model_dump() if spec.mlflow else None,
+                scenario_name=spec.name,
+            )
+            click.echo(f"MLflow enabled → experiment: {mlflow_config.experiment}")
+            if mlflow_config.tracing:
+                click.echo("  Tracing: ON (LLM calls will be captured)")
+        except ImportError:
+            pass  # MLflow not installed
+
     # Parse and apply variables
     variables: dict[str, Any] = {}
     for v in var:
@@ -487,27 +767,17 @@ def scenario(
         spec = apply_scenario_variables(spec, variables)
         click.echo(f"Variables: {variables}")
 
-    # Determine which agent to use
-    agent = None
+    # Build list of models to run
+    models_to_run: list[str] = []
 
     if model:
-        # Create ad-hoc agent from model string
-        config = AgentConfig(
-            id=model,
-            name=model.split("/")[-1] if "/" in model else model,
-            kind="llm-prompt",
-            model=model,
-            system_prompt="",
-            tools=[],
-            params={"temperature": 0.7, "max_tokens": 4096},
-            impl={},
-        )
-        agent = LlmPromptAgent(config)
+        models_to_run = list(model)
     elif agent_id:
         # Load from agent config files
         loader = AgentLoader(DEFAULT_AGENT_DIRS)
         try:
             agent = loader.load(agent_id)
+            models_to_run = [agent.config.model]
         except ValueError as e:
             click.echo(f"Error loading agent: {e}", err=True)
             sys.exit(1)
@@ -516,6 +786,7 @@ def scenario(
         loader = AgentLoader(DEFAULT_AGENT_DIRS)
         try:
             agent = loader.load_default()
+            models_to_run = [agent.config.model]
         except ValueError:
             click.echo("No model specified. Use -m to specify a model:", err=True)
             click.echo("", err=True)
@@ -528,25 +799,110 @@ def scenario(
             )
             sys.exit(1)
 
-    # Apply scenario's system prompt to agent
-    if spec.system_prompt:
-        agent.config.system_prompt = spec.system_prompt
-
     click.echo(f"Running scenario: {spec.name}")
-    click.echo(f"Using model: {agent.config.model}")
+    click.echo(f"Models: {', '.join(models_to_run)}")
     click.echo(f"Tools loaded: {len(spec.tools) + len(spec.tools_from)} source(s)")
+    if len(models_to_run) > 1:
+        click.echo("Running models in parallel...")
     click.echo("")
 
-    runner = ScenarioRunner(scenario=spec, agent=agent)
-    result = runner.run(max_turns=max_turns)
+    def run_single_model(model_id: str) -> dict[str, Any]:
+        """Run scenario with a single model, with MLflow tracing if enabled."""
+        agent_config = AgentConfig(
+            id=model_id,
+            name=model_id.split("/")[-1] if "/" in model_id else model_id,
+            kind="llm-prompt",
+            model=model_id,
+            system_prompt=spec.system_prompt or "",
+            tools=[],
+            params={"temperature": 0.7, "max_tokens": 4096},
+            impl={},
+        )
+        agent = LlmPromptAgent(agent_config)
+
+        # If MLflow enabled, wrap execution in run context so traces are connected
+        if mlflow_config and mlflow_config.enabled:
+            from sandboxy.mlflow import MLflowExporter, mlflow_run_context
+            from sandboxy.mlflow.tracing import enable_tracing
+
+            # Enable tracing before the run starts
+            if mlflow_config.tracing:
+                enable_tracing(
+                    tracking_uri=mlflow_config.tracking_uri,
+                    experiment_name=mlflow_config.experiment,
+                )
+
+            # Start run, execute scenario, then log metrics - all connected
+            with mlflow_run_context(mlflow_config, run_name=model_id) as run_id:
+                runner = ScenarioRunner(scenario=spec, agent=agent)
+                result = runner.run(max_turns=max_turns)
+
+                # Log metrics to the active run (traces are already attached)
+                if run_id:
+                    exporter = MLflowExporter(mlflow_config)
+                    exporter.log_to_active_run(
+                        result=result,
+                        scenario_path=Path(scenario_path),
+                        scenario_name=spec.name,
+                        scenario_id=spec.id,
+                        agent_name=agent.config.name,
+                    )
+
+            return {"model": model_id, "result": result, "agent_name": agent.config.name}
+
+        # No MLflow - just run scenario
+        runner = ScenarioRunner(scenario=spec, agent=agent)
+        result = runner.run(max_turns=max_turns)
+        return {"model": model_id, "result": result, "agent_name": agent.config.name}
+
+    # Run models in parallel if multiple, otherwise just run single
+    results: list[Any] = []
+    if len(models_to_run) == 1:
+        results = [run_single_model(models_to_run[0])]
+    else:
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
+        with ThreadPoolExecutor(max_workers=len(models_to_run)) as executor:
+            futures = {executor.submit(run_single_model, m): m for m in models_to_run}
+            for future in as_completed(futures):
+                model_id = futures[future]
+                try:
+                    result_data = future.result()
+                    results.append(result_data)
+                    click.echo(f"✓ Completed: {model_id}")
+                except Exception as e:
+                    click.echo(f"✗ Failed: {model_id} - {e}", err=True)
+        click.echo("")
 
-    if output:
-        Path(output).write_text(result.to_json(indent=2))
-        click.echo(f"\nResults saved to: {output}")
-    elif pretty:
-        click.echo(result.pretty())
+    # Output results
+    if len(results) == 1:
+        result = results[0]["result"]
+        if output:
+            Path(output).write_text(result.to_json(indent=2))
+            click.echo(f"\nResults saved to: {output}")
+        elif pretty:
+            click.echo(result.pretty())
+        else:
+            click.echo(result.to_json(indent=2))
     else:
-        click.echo(result.to_json(indent=2))
+        # Multiple models - show summary
+        # Get max_score from spec (scoring config or sum of goal points)
+        max_score = spec.scoring.get("max_score", 0) if spec.scoring else 0
+        if not max_score and spec.goals:
+            max_score = sum(g.points for g in spec.goals)
+
+        click.echo("=== Results Summary ===")
+        for r in results:
+            model_name = r["model"]
+            res = r["result"]
+            score = getattr(res, "score", 0) or 0
+            pct = (score / max_score * 100) if max_score > 0 else 0
+            click.echo(f"  {model_name}: {score:.1f}/{max_score:.1f} ({pct:.0f}%)")
+
+        if output:
+            all_results = [{"model": r["model"], "result": r["result"].to_dict()} for r in results]
+            Path(output).write_text(json.dumps(all_results, indent=2))
+            click.echo(f"\nResults saved to: {output}")
 
 
 @main.command()