evaldeck 0.1.0__py3-none-any.whl

evaldeck/__init__.py

@@ -0,0 +1,88 @@
+"""Evaldeck - The evaluation framework for AI agents.
+
+Evaldeck helps you answer one question: "Is my agent actually working?"
+
+Basic usage:
+    from evaldeck import Trace, Step, Evaluator, EvalCase
+
+    # Create a trace (or capture with LangChain adapter)
+    trace = Trace(
+        input="Book a flight to NYC",
+        steps=[
+            Step.tool_call("search_flights", {"to": "NYC"}),
+            Step.tool_call("book_flight", {"flight_id": "123"}),
+        ],
+        output="Booked flight 123 to NYC",
+    )
+
+    # Define test case
+    test_case = EvalCase(
+        name="book_flight",
+        input="Book a flight to NYC",
+        expected=ExpectedBehavior(
+            tools_called=["search_flights", "book_flight"],
+            output_contains=["booked"],
+        ),
+    )
+
+    # Evaluate
+    evaluator = Evaluator()
+    result = evaluator.evaluate(trace, test_case)
+    print(f"Passed: {result.passed}")
+"""
+
+from evaldeck.config import EvaldeckConfig
+from evaldeck.evaluator import EvaluationRunner, Evaluator
+from evaldeck.results import (
+    EvaluationResult,
+    GradeResult,
+    GradeStatus,
+    MetricResult,
+    RunResult,
+    SuiteResult,
+)
+from evaldeck.test_case import (
+    EvalCase,
+    EvalSuite,
+    ExpectedBehavior,
+    GraderConfig,
+)
+from evaldeck.trace import (
+    Step,
+    StepStatus,
+    StepType,
+    TokenUsage,
+    Trace,
+    TraceStatus,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Version
+    "__version__",
+    # Trace
+    "Trace",
+    "Step",
+    "StepType",
+    "StepStatus",
+    "TraceStatus",
+    "TokenUsage",
+    # Test Case
+    "EvalCase",
+    "EvalSuite",
+    "ExpectedBehavior",
+    "GraderConfig",
+    # Results
+    "GradeResult",
+    "GradeStatus",
+    "MetricResult",
+    "EvaluationResult",
+    "SuiteResult",
+    "RunResult",
+    # Evaluator
+    "Evaluator",
+    "EvaluationRunner",
+    # Config
+    "EvaldeckConfig",
+]

evaldeck/cli.py

@@ -0,0 +1,324 @@
+"""Command-line interface for evaldeck."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+from rich import box
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+from evaldeck.config import EvaldeckConfig, generate_default_config, generate_example_test
+from evaldeck.results import EvaluationResult, GradeStatus, RunResult
+
+console = Console()
+
+
+@click.group()
+@click.version_option()
+def main() -> None:
+    """Evaldeck - The evaluation framework for AI agents."""
+    pass
+
+
+@main.command()
+@click.option("--force", "-f", is_flag=True, help="Overwrite existing files")
+def init(force: bool) -> None:
+    """Initialize a new evaldeck project."""
+    config_path = Path("evaldeck.yaml")
+    test_dir = Path("tests/evals")
+    example_test = test_dir / "example.yaml"
+
+    # Check for existing files
+    if config_path.exists() and not force:
+        console.print(f"[yellow]Config file already exists: {config_path}[/yellow]")
+        console.print("Use --force to overwrite")
+        return
+
+    # Create config
+    config_path.write_text(generate_default_config())
+    console.print(f"[green]Created:[/green] {config_path}")
+
+    # Create test directory
+    test_dir.mkdir(parents=True, exist_ok=True)
+    console.print(f"[green]Created:[/green] {test_dir}/")
+
+    # Create example test
+    if not example_test.exists() or force:
+        example_test.write_text(generate_example_test())
+        console.print(f"[green]Created:[/green] {example_test}")
+
+    # Create output directory
+    output_dir = Path(".evaldeck")
+    output_dir.mkdir(exist_ok=True)
+
+    console.print()
+    console.print(
+        Panel(
+            "[bold]Project initialized![/bold]\n\n"
+            "Next steps:\n"
+            "1. Edit [cyan]evaldeck.yaml[/cyan] to configure your agent\n"
+            "2. Add test cases to [cyan]tests/evals/[/cyan]\n"
+            "3. Run [cyan]evaldeck run[/cyan] to evaluate",
+            title="Evaldeck",
+            border_style="green",
+        )
+    )
+
+
+@main.command()
+@click.option("--config", "-c", type=click.Path(exists=True), help="Config file path")
+@click.option("--suite", "-s", multiple=True, help="Run specific suite(s)")
+@click.option("--tag", "-t", multiple=True, help="Filter by tag(s)")
+@click.option("--output", "-o", type=click.Choice(["text", "json", "junit"]), default="text")
+@click.option("--output-file", type=click.Path(), help="Output file path")
+@click.option("--verbose", "-v", is_flag=True, help="Verbose output")
+@click.option(
+    "--workers",
+    "-w",
+    type=int,
+    default=None,
+    help="Max concurrent tests (0=unlimited, 1=sequential). Default: from config or 0",
+)
+def run(
+    config: str | None,
+    suite: tuple[str, ...],
+    tag: tuple[str, ...],
+    output: str,
+    output_file: str | None,
+    verbose: bool,
+    workers: int | None,
+) -> None:
+    """Run evaluations."""
+    try:
+        # Load config
+        cfg = EvaldeckConfig.load(config)
+    except FileNotFoundError:
+        console.print("[red]No evaldeck.yaml found. Run 'evaldeck init' first.[/red]")
+        sys.exit(1)
+
+    console.print("[bold]Evaldeck[/bold] - Running evaluations...\n")
+
+    # Discover test suites
+    from evaldeck.evaluator import EvaluationRunner
+
+    runner = EvaluationRunner(config=cfg)
+
+    try:
+        suites = runner._discover_suites()
+    except Exception as e:
+        console.print(f"[red]Error discovering test suites: {e}[/red]")
+        sys.exit(1)
+
+    if not suites:
+        console.print("[yellow]No test suites found.[/yellow]")
+        console.print(f"Add test cases to: {cfg.test_dir}/")
+        sys.exit(0)
+
+    # Filter suites if specified
+    if suite:
+        suites = [s for s in suites if s.name in suite]
+
+    # Count total tests
+    total_tests = sum(len(s.test_cases) for s in suites)
+    console.print(
+        f"Found [cyan]{total_tests}[/cyan] test(s) in [cyan]{len(suites)}[/cyan] suite(s)\n"
+    )
+
+    # Check if agent is configured
+    if not cfg.agent.module or not cfg.agent.function:
+        console.print("[yellow]No agent configured in evaldeck.yaml[/yellow]")
+        console.print("Running in dry-run mode (no agent execution)\n")
+
+        # Show what would be run
+        for s in suites:
+            console.print(f"Suite: [bold]{s.name}[/bold]")
+            for tc in s.test_cases:
+                console.print(f"  - {tc.name}")
+        sys.exit(0)
+
+    # Run evaluations
+    def on_result(result: EvaluationResult) -> None:
+        """Print result as it completes."""
+        if result.passed:
+            icon = "[green]✓[/green]"
+        elif result.status == GradeStatus.ERROR:
+            icon = "[red]![/red]"
+        else:
+            icon = "[red]✗[/red]"
+
+        duration = f"({result.duration_ms:.1f}ms)" if result.duration_ms else ""
+        console.print(f"  {icon} {result.test_case_name} {duration}")
+
+        if verbose and not result.passed:
+            for grade in result.failed_grades:
+                console.print(f"      [dim]└─ {grade.grader_name}: {grade.message}[/dim]")
+
+    # Show concurrency info
+    effective_workers = workers if workers is not None else cfg.execution.workers
+    if effective_workers == 0:
+        console.print("[dim]Running with unlimited concurrency[/dim]\n")
+    elif effective_workers == 1:
+        console.print("[dim]Running sequentially[/dim]\n")
+    else:
+        console.print(f"[dim]Running with max {effective_workers} concurrent tests[/dim]\n")
+
+    try:
+        run_result = runner.run(
+            suites=suites,
+            tags=list(tag) if tag else None,
+            on_result=on_result,
+            max_concurrent=workers,
+        )
+    except ValueError as e:
+        console.print(f"[red]Error: {e}[/red]")
+        sys.exit(1)
+    except Exception as e:
+        console.print(f"[red]Evaluation error: {e}[/red]")
+        if verbose:
+            import traceback
+
+            console.print(traceback.format_exc())
+        sys.exit(1)
+
+    # Print summary
+    console.print()
+    _print_summary(run_result)
+
+    # Output to file if requested
+    if output_file:
+        _write_output(run_result, output, output_file)
+
+    # Exit with appropriate code
+    if cfg.thresholds.min_pass_rate > 0:
+        if run_result.pass_rate < cfg.thresholds.min_pass_rate:
+            console.print(
+                f"\n[red]Pass rate {run_result.pass_rate:.1%} < "
+                f"threshold {cfg.thresholds.min_pass_rate:.1%}[/red]"
+            )
+            sys.exit(1)
+
+    sys.exit(0 if run_result.all_passed else 1)
+
+
+def _print_summary(result: RunResult) -> None:
+    """Print evaluation summary."""
+    table = Table(box=box.SIMPLE)
+    table.add_column("Metric", style="bold")
+    table.add_column("Value", justify="right")
+
+    table.add_row("Total", str(result.total))
+    table.add_row("Passed", f"[green]{result.passed}[/green]")
+    table.add_row("Failed", f"[red]{result.failed}[/red]" if result.failed else "0")
+    table.add_row("Pass Rate", f"{result.pass_rate:.1%}")
+
+    console.print(Panel(table, title="Results", border_style="blue"))
+
+
+def _write_output(result: RunResult, format: str, path: str) -> None:
+    """Write results to file."""
+    if format == "json":
+        import json
+
+        with open(path, "w") as f:
+            json.dump(result.model_dump(mode="json"), f, indent=2, default=str)
+    elif format == "junit":
+        _write_junit(result, path)
+
+    console.print(f"[dim]Output written to: {path}[/dim]")
+
+
+def _write_junit(result: RunResult, path: str) -> None:
+    """Write results in JUnit XML format."""
+    import xml.etree.ElementTree as ET
+
+    testsuites = ET.Element("testsuites")
+    testsuites.set("tests", str(result.total))
+    testsuites.set("failures", str(result.failed))
+
+    for suite_result in result.suites:
+        testsuite = ET.SubElement(testsuites, "testsuite")
+        testsuite.set("name", suite_result.suite_name)
+        testsuite.set("tests", str(suite_result.total))
+        testsuite.set("failures", str(suite_result.failed))
+        testsuite.set("errors", str(suite_result.errors))
+
+        for eval_result in suite_result.results:
+            testcase = ET.SubElement(testsuite, "testcase")
+            testcase.set("name", eval_result.test_case_name)
+            testcase.set("time", str((eval_result.duration_ms or 0) / 1000))
+
+            if eval_result.status == GradeStatus.FAIL:
+                failure = ET.SubElement(testcase, "failure")
+                messages = [g.message for g in eval_result.failed_grades if g.message]
+                failure.set("message", "; ".join(messages) or "Test failed")
+
+            elif eval_result.status == GradeStatus.ERROR:
+                error = ET.SubElement(testcase, "error")
+                error.set("message", eval_result.error or "Unknown error")
+
+    tree = ET.ElementTree(testsuites)
+    ET.indent(tree, space="  ")
+    tree.write(path, encoding="unicode", xml_declaration=True)
+
+
+@main.command()
+@click.argument("name")
+@click.option("--input", "-i", "input_text", required=True, help="Test input")
+@click.option("--output-contains", "-c", multiple=True, help="Expected output contains")
+@click.option("--tools", "-t", multiple=True, help="Expected tool calls")
+def create(
+    name: str,
+    input_text: str,
+    output_contains: tuple[str, ...],
+    tools: tuple[str, ...],
+) -> None:
+    """Create a new test case."""
+    from evaldeck.test_case import EvalCase, ExpectedBehavior
+
+    expected = ExpectedBehavior(
+        output_contains=list(output_contains) if output_contains else None,
+        tools_called=list(tools) if tools else None,
+    )
+
+    test_case = EvalCase(
+        name=name,
+        input=input_text,
+        expected=expected,
+    )
+
+    console.print(test_case.to_yaml())
+
+
+@main.command()
+def validate() -> None:
+    """Validate configuration and test cases."""
+    try:
+        cfg = EvaldeckConfig.load()
+        console.print("[green]✓[/green] Config file is valid")
+    except FileNotFoundError:
+        console.print("[red]✗[/red] No config file found")
+        sys.exit(1)
+    except Exception as e:
+        console.print(f"[red]✗[/red] Config error: {e}")
+        sys.exit(1)
+
+    # Validate test cases
+    from evaldeck.evaluator import EvaluationRunner
+
+    runner = EvaluationRunner(config=cfg)
+
+    try:
+        suites = runner._discover_suites()
+        total_tests = sum(len(s.test_cases) for s in suites)
+        console.print(f"[green]✓[/green] Found {total_tests} valid test case(s)")
+    except Exception as e:
+        console.print(f"[red]✗[/red] Test case error: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

evaldeck/config.py

@@ -0,0 +1,223 @@
+"""Configuration loading and management."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class AgentConfig(BaseModel):
+    """Configuration for the agent to test."""
+
+    module: str | None = None
+    function: str | None = None
+    class_name: str | None = None
+
+
+class GraderDefaults(BaseModel):
+    """Default configuration for graders."""
+
+    llm_model: str = "gpt-4o-mini"
+    llm_provider: str | None = None
+    timeout: float = 30.0
+
+
+class ThresholdConfig(BaseModel):
+    """Threshold configuration for pass/fail."""
+
+    min_pass_rate: float = 0.0
+    max_failures: int | None = None
+
+
+class SuiteConfig(BaseModel):
+    """Configuration for a test suite."""
+
+    name: str
+    path: str
+    tags: list[str] = Field(default_factory=list)
+
+
+class ExecutionConfig(BaseModel):
+    """Configuration for test execution."""
+
+    workers: int = Field(
+        default=0,
+        ge=0,
+        description="Number of concurrent workers. 0 = unlimited (default).",
+    )
+    timeout: float = Field(default=30.0, gt=0)
+    retries: int = Field(default=0, ge=0)
+
+
+class EvaldeckConfig(BaseModel):
+    """Main evaldeck configuration."""
+
+    version: int = 1
+
+    # Agent configuration
+    agent: AgentConfig = Field(default_factory=AgentConfig)
+
+    # Test configuration
+    test_dir: str = "tests/evals"
+    suites: list[SuiteConfig] = Field(default_factory=list)
+
+    # Execution configuration
+    execution: ExecutionConfig = Field(default_factory=ExecutionConfig)
+
+    # Legacy execution defaults (deprecated, use execution instead)
+    defaults: dict[str, Any] = Field(default_factory=lambda: {
+        "timeout": 30,
+        "retries": 0,
+    })
+
+    # Grader configuration
+    graders: GraderDefaults = Field(default_factory=GraderDefaults)
+
+    # Thresholds
+    thresholds: ThresholdConfig = Field(default_factory=ThresholdConfig)
+
+    # Output configuration
+    output_dir: str = ".evaldeck"
+
+    @classmethod
+    def load(cls, path: str | Path | None = None) -> EvaldeckConfig:
+        """Load configuration from file.
+
+        Searches for evaldeck.yaml, evaldeck.yml in order.
+        """
+        if path:
+            path = Path(path)
+            if not path.exists():
+                raise FileNotFoundError(f"Config file not found: {path}")
+            return cls._load_file(path)
+
+        # Search for config file
+        for name in ["evaldeck.yaml", "evaldeck.yml"]:
+            p = Path(name)
+            if p.exists():
+                return cls._load_file(p)
+
+        # Return default config
+        return cls()
+
+    @classmethod
+    def _load_file(cls, path: Path) -> EvaldeckConfig:
+        """Load configuration from a specific file."""
+        with open(path) as f:
+            data = yaml.safe_load(f) or {}
+
+        # Handle nested objects
+        if "agent" in data and isinstance(data["agent"], dict):
+            data["agent"] = AgentConfig(**data["agent"])
+
+        if "graders" in data and isinstance(data["graders"], dict):
+            # Handle 'llm' sub-key
+            if "llm" in data["graders"]:
+                llm_config = data["graders"]["llm"]
+                data["graders"] = GraderDefaults(
+                    llm_model=llm_config.get("model", "gpt-4o-mini"),
+                    llm_provider=llm_config.get("provider"),
+                )
+            else:
+                data["graders"] = GraderDefaults(**data["graders"])
+
+        if "thresholds" in data and isinstance(data["thresholds"], dict):
+            data["thresholds"] = ThresholdConfig(**data["thresholds"])
+
+        if "execution" in data and isinstance(data["execution"], dict):
+            data["execution"] = ExecutionConfig(**data["execution"])
+
+        if "suites" in data:
+            suites = []
+            for s in data["suites"]:
+                if isinstance(s, dict):
+                    suites.append(SuiteConfig(**s))
+                else:
+                    suites.append(s)
+            data["suites"] = suites
+
+        return cls(**data)
+
+    def save(self, path: str | Path) -> None:
+        """Save configuration to file."""
+        with open(path, "w") as f:
+            yaml.dump(self.model_dump(exclude_none=True), f, default_flow_style=False)
+
+
+def generate_default_config() -> str:
+    """Generate default configuration YAML."""
+    return """# Evaldeck Configuration
+version: 1
+
+# Agent configuration (optional - can also be specified per test)
+# agent:
+#   module: my_agent
+#   function: run_agent
+
+# Test directory
+test_dir: tests/evals
+
+# Test suites (optional - auto-discovers from test_dir if not specified)
+# suites:
+#   - name: core
+#     path: tests/evals/core
+#   - name: safety
+#     path: tests/evals/safety
+
+# Execution configuration
+execution:
+  workers: 0      # 0 = unlimited concurrent (default)
+  timeout: 30
+  retries: 0
+
+# Grader configuration
+graders:
+  llm:
+    model: gpt-4o-mini
+    # API key from OPENAI_API_KEY environment variable
+
+# Pass/fail thresholds
+thresholds:
+  min_pass_rate: 0.0
+  # max_failures: 5
+
+# Output directory for traces and results
+output_dir: .evaldeck
+"""
+
+
+def generate_example_test() -> str:
+    """Generate example test case YAML."""
+    return """# Example test case
+name: example_test
+description: An example test case to get you started
+
+# Input to send to the agent
+input: "Hello, can you help me with a simple task?"
+
+# Expected behavior
+expected:
+  # Tools that must be called (if any)
+  # tools_called:
+  #   - search
+  #   - calculate
+
+  # Output must contain these strings
+  output_contains:
+    - "help"
+
+  # Maximum steps allowed
+  max_steps: 10
+
+  # Task must complete successfully
+  task_completed: true
+
+# Optional: Custom graders
+# graders:
+#   - type: llm
+#     prompt: "Did the agent respond helpfully? Answer PASS or FAIL."
+#     model: gpt-4o-mini
+"""