agenteval-py 0.1.0__py3-none-any.whl

agenteval/cli.py

@@ -0,0 +1,93 @@
+"""CLI entry point for agenteval."""
+
+from __future__ import annotations
+
+import pathlib
+from typing import Annotated, Optional
+
+import typer
+from rich.console import Console
+
+from agenteval.registry import TestRegistry
+from agenteval.reporter import RichReporter
+from agenteval.suite import run_suite
+
+app = typer.Typer(
+    name="agenteval",
+    help="Evaluation toolkit for LLM agents.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+@app.command(name="run")
+def run_cmd(
+    paths: Annotated[
+        list[str],
+        typer.Argument(help="Test files or directories to discover (default: current dir)"),
+    ] = [".", ],  # noqa: B006
+    pattern: str = typer.Option("test_*.py", "--pattern", "-p", help="File glob pattern"),
+    tags: Optional[list[str]] = typer.Option(None, "--tag", "-t", help="Only run tests with this tag (repeatable)"),
+    n: Optional[int] = typer.Option(None, "--n", help="Override number of runs per test"),
+    threshold: Optional[float] = typer.Option(None, "--threshold", help="Override pass rate threshold (0.0–1.0)"),
+    concurrency: int = typer.Option(4, "--concurrency", "-c", help="Max concurrent runs"),
+    output: Optional[pathlib.Path] = typer.Option(None, "--output", "-o", help="Write JSON report to this file"),
+    no_color: bool = typer.Option(False, "--no-color", help="Disable color output"),
+    show_traces: bool = typer.Option(False, "--traces", help="Show per-trace details"),
+    show_failures: bool = typer.Option(True, "--failures/--no-failures", help="Show failure reasons"),
+) -> None:
+    """Discover and run agenteval tests."""
+    console = Console(no_color=no_color)
+    reporter = RichReporter(console=console, show_traces=show_traces, show_failures=show_failures)
+
+    # Reset registry so re-running the CLI in the same process doesn't double-count
+    TestRegistry.reset()
+
+    try:
+        suite = run_suite(
+            paths=paths,
+            pattern=pattern,
+            tags=tags or None,
+            fail_under=threshold,
+            n_override=n,
+            concurrency=concurrency,
+            reporter=reporter,
+        )
+    except Exception as e:
+        console.print(f"[bold red]Error:[/bold red] {e}")
+        raise typer.Exit(code=2) from e
+
+    if output is not None:
+        reporter.export_json(suite, output)
+
+    raise typer.Exit(code=0 if suite.all_passed else 1)
+
+
+@app.command(name="report")
+def report_cmd(
+    json_file: Annotated[pathlib.Path, typer.Argument(help="JSON report file from a previous run")],
+    show_traces: bool = typer.Option(False, "--traces", help="Show per-trace details"),
+    no_color: bool = typer.Option(False, "--no-color"),
+) -> None:
+    """Pretty-print a saved JSON report."""
+    import json as _json
+
+    from agenteval.models import SuiteResult
+
+    console = Console(no_color=no_color)
+
+    if not json_file.exists():
+        console.print(f"[bold red]File not found:[/bold red] {json_file}")
+        raise typer.Exit(code=2)
+
+    try:
+        data = _json.loads(json_file.read_text(encoding="utf-8"))
+        suite = SuiteResult.model_validate(data)
+    except Exception as e:
+        console.print(f"[bold red]Failed to load report:[/bold red] {e}")
+        raise typer.Exit(code=2) from e
+
+    reporter = RichReporter(console=console, show_traces=show_traces)
+    for result in suite.results:
+        reporter.render_result(result)
+    reporter.render_suite(suite)

agenteval/exceptions.py

@@ -0,0 +1,17 @@
+"""Custom exception hierarchy for agenteval."""
+
+
+class AgentEvalError(Exception):
+    """Base exception for all agenteval errors."""
+
+
+class AssertionFailure(AgentEvalError):
+    """Raised when one or more trace assertions fail."""
+
+
+class DiscoveryError(AgentEvalError):
+    """Raised when test file discovery or import fails."""
+
+
+class TracerError(AgentEvalError):
+    """Raised for invalid tracer usage (e.g., accessing trace before run completes)."""

agenteval/models.py

@@ -0,0 +1,123 @@
+"""Pydantic data models for agenteval traces and results."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict, computed_field, field_validator
+
+
+class ToolCall(BaseModel):
+    """A single tool/function call recorded during an agent run."""
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    arguments: dict[str, Any]
+    result: Any = None
+    timestamp: float
+    duration_seconds: float
+    error: Optional[str] = None
+
+
+class AgentTrace(BaseModel):
+    """Full trace of one agent run, including all tool calls and outcome."""
+
+    run_id: str
+    input: Any
+    output: Any = None
+    tool_calls: list[ToolCall] = []
+    total_steps: Optional[int] = None
+    duration_seconds: float = 0.0
+    token_usage: Optional[dict[str, int]] = None
+    error: Optional[str] = None
+    assertion_errors: list[str] = []
+    passed: bool = True
+    metadata: dict[str, Any] = {}
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def effective_steps(self) -> int:
+        """Number of steps: explicit total_steps if set, otherwise len(tool_calls)."""
+        return self.total_steps if self.total_steps is not None else len(self.tool_calls)
+
+
+class TestResult(BaseModel):
+    """Aggregated results from running a test function N times."""
+
+    test_name: str
+    n_runs: int
+    n_passed: int
+    pass_rate: float
+    threshold: float
+    traces: list[AgentTrace]
+    tags: list[str] = []
+
+    @field_validator("pass_rate")
+    @classmethod
+    def _validate_pass_rate(cls, v: float) -> float:
+        if not 0.0 <= v <= 1.0:
+            raise ValueError(f"pass_rate must be between 0 and 1, got {v}")
+        return v
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def passed_traces(self) -> list[AgentTrace]:
+        return [t for t in self.traces if t.passed]
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def failed_traces(self) -> list[AgentTrace]:
+        return [t for t in self.traces if not t.passed]
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def avg_duration(self) -> float:
+        if not self.traces:
+            return 0.0
+        return sum(t.duration_seconds for t in self.traces) / len(self.traces)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def avg_steps(self) -> float:
+        if not self.traces:
+            return 0.0
+        return sum(t.effective_steps for t in self.traces) / len(self.traces)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def met_threshold(self) -> bool:
+        return self.pass_rate >= self.threshold
+
+
+class SuiteResult(BaseModel):
+    """Aggregated results for an entire test suite."""
+
+    results: list[TestResult]
+    start_time: float
+    end_time: float
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_tests(self) -> int:
+        return len(self.results)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def passed_tests(self) -> int:
+        return sum(1 for r in self.results if r.met_threshold)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def failed_tests(self) -> int:
+        return self.total_tests - self.passed_tests
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def all_passed(self) -> bool:
+        return self.failed_tests == 0
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def duration_seconds(self) -> float:
+        return self.end_time - self.start_time

agenteval/registry.py

@@ -0,0 +1,99 @@
+"""Global test registry and @agenteval.test decorator."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+
+@dataclass
+class RegisteredTest:
+    """Metadata for a test function registered with @agenteval.test."""
+
+    fn: Callable[..., Any]
+    name: str
+    n: int
+    threshold: float
+    tags: list[str]
+    module: str
+
+
+class TestRegistry:
+    """Singleton registry that collects all @agenteval.test-decorated functions."""
+
+    _instance: Optional["TestRegistry"] = None
+
+    def __init__(self) -> None:
+        self._tests: list[RegisteredTest] = []
+
+    @classmethod
+    def global_registry(cls) -> "TestRegistry":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def reset(cls) -> None:
+        """Replace the global registry with a fresh one. Used in tests."""
+        cls._instance = cls()
+
+    def register(self, entry: RegisteredTest) -> None:
+        self._tests.append(entry)
+
+    def clear(self) -> None:
+        self._tests.clear()
+
+    def get_all(self, tags: Optional[list[str]] = None) -> list[RegisteredTest]:
+        """Return all registered tests, optionally filtered by tags."""
+        if not tags:
+            return list(self._tests)
+        return [t for t in self._tests if any(tag in t.tags for tag in tags)]
+
+    def snapshot(self) -> list[RegisteredTest]:
+        """Return a copy of the current registered tests."""
+        return list(self._tests)
+
+
+def test(
+    fn: Optional[Callable[..., Any]] = None,
+    *,
+    n: int = 20,
+    threshold: float = 0.8,
+    tags: Optional[list[str]] = None,
+) -> Any:
+    """Decorator that registers a test function with the global TestRegistry.
+
+    Supports both bare and parameterized forms::
+
+        @agenteval.test
+        async def test_basic(tracer: Tracer) -> None: ...
+
+        @agenteval.test(n=10, threshold=0.9, tags=["slow"])
+        async def test_complex(tracer: Tracer) -> None: ...
+
+    Args:
+        fn: The test function (when used as bare decorator).
+        n: Number of runs. Default: 20.
+        threshold: Required pass rate. Default: 0.8.
+        tags: Optional tags for filtering.
+    """
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        registry = TestRegistry.global_registry()
+        registry.register(
+            RegisteredTest(
+                fn=func,
+                name=func.__name__,
+                n=n,
+                threshold=threshold,
+                tags=tags or [],
+                module=func.__module__,
+            )
+        )
+        return func
+
+    if fn is not None:
+        # Called as @agenteval.test (no parentheses)
+        return decorator(fn)
+
+    # Called as @agenteval.test(...) (with parentheses)
+    return decorator

agenteval/reporter.py

@@ -0,0 +1,139 @@
+"""Rich terminal reporter and JSON exporter."""
+
+from __future__ import annotations
+
+import json
+import pathlib
+from typing import Protocol, runtime_checkable
+
+from rich import box
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from agenteval.models import SuiteResult, TestResult
+
+
+@runtime_checkable
+class Reporter(Protocol):
+    def render_result(self, result: TestResult) -> None: ...
+    def render_suite(self, suite: SuiteResult) -> None: ...
+    def export_json(self, suite: SuiteResult, path: pathlib.Path) -> None: ...
+
+
+def _pass_rate_display(result: TestResult) -> tuple[str, str]:
+    """Return (label, style) for a test result's pass rate."""
+    rate = result.pass_rate
+    threshold = result.threshold
+    fraction = f"{result.n_passed}/{result.n_runs}"
+
+    if rate >= threshold:
+        return f"✅ {fraction}  ({rate:.0%})", "green"
+    elif rate >= threshold * 0.5:
+        return f"⚠️  {fraction}  ({rate:.0%})", "yellow"
+    else:
+        return f"❌ {fraction}  ({rate:.0%})", "red"
+
+
+class RichReporter:
+    """Rich terminal reporter with color-coded pass rates and summary tables.
+
+    Output format::
+
+        test_basic_search       18/20  ✅ 90%   avg 1.2s   3.1 steps
+        test_complex_reasoning   8/20  ⚠️  40%   avg 4.7s   7.2 steps
+        test_hallucination       3/20  ❌ 15%   avg 2.1s   5.0 steps
+    """
+
+    def __init__(
+        self,
+        console: Console | None = None,
+        *,
+        show_failures: bool = True,
+        show_traces: bool = False,
+    ) -> None:
+        self.console = console or Console()
+        self.show_failures = show_failures
+        self.show_traces = show_traces
+
+    def render_result(self, result: TestResult) -> None:
+        """Print a single test result line. Called after each test completes."""
+        label, style = _pass_rate_display(result)
+        self.console.print(
+            f"  [bold]{result.test_name}[/bold]"
+            f"  [{style}]{label}[/{style}]"
+            f"  [dim]avg {result.avg_duration:.2f}s  {result.avg_steps:.1f} steps[/dim]"
+        )
+
+        if self.show_failures and result.failed_traces:
+            for i, trace in enumerate(result.failed_traces[:3], 1):
+                reasons: list[str] = []
+                if trace.error:
+                    reasons.append(f"error: {trace.error}")
+                if trace.assertion_errors:
+                    reasons.extend(trace.assertion_errors[:2])
+                if reasons:
+                    reason_text = " | ".join(r[:120] for r in reasons[:2])
+                    self.console.print(f"    [dim red]↳ failure {i}: {reason_text}[/dim red]")
+
+        if self.show_traces:
+            self._print_traces(result)
+
+    def render_suite(self, suite: SuiteResult) -> None:
+        """Print a summary table for the full suite."""
+        if not suite.results:
+            self.console.print("[dim]No tests found.[/dim]")
+            return
+
+        total_runs = sum(r.n_runs for r in suite.results)
+        self.console.print()
+
+        table = Table(
+            box=box.ROUNDED,
+            show_header=True,
+            header_style="bold",
+            title=f"agenteval results  —  {suite.total_tests} test(s)  ·  {total_runs} total runs  ·  {suite.duration_seconds:.1f}s",
+        )
+        table.add_column("Test", style="bold", no_wrap=True)
+        table.add_column("Runs", justify="center")
+        table.add_column("Pass Rate", justify="center")
+        table.add_column("Avg Duration", justify="right")
+        table.add_column("Avg Steps", justify="right")
+        table.add_column("Threshold", justify="center")
+
+        for result in suite.results:
+            label, style = _pass_rate_display(result)
+            table.add_row(
+                result.test_name,
+                f"{result.n_passed}/{result.n_runs}",
+                Text(label, style=style),
+                f"{result.avg_duration:.2f}s",
+                f"{result.avg_steps:.1f}",
+                f"{result.threshold:.0%}",
+            )
+
+        self.console.print(table)
+
+        # Footer summary
+        if suite.all_passed:
+            self.console.print(
+                f"\n  [bold green]✅ All {suite.total_tests} test(s) passed their threshold.[/bold green]"
+            )
+        else:
+            self.console.print(
+                f"\n  [bold red]❌ {suite.failed_tests}/{suite.total_tests} test(s) failed their threshold.[/bold red]"
+            )
+
+    def export_json(self, suite: SuiteResult, path: pathlib.Path) -> None:
+        """Write suite results to a JSON file."""
+        data = suite.model_dump(mode="json")
+        path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+        self.console.print(f"\n  [dim]JSON report saved to {path}[/dim]")
+
+    def _print_traces(self, result: TestResult) -> None:
+        for i, trace in enumerate(result.traces, 1):
+            status = "✅" if trace.passed else "❌"
+            self.console.print(
+                f"    [dim]{status} run {i}  {trace.duration_seconds:.2f}s  "
+                f"{trace.effective_steps} steps  output={str(trace.output)[:60]!r}[/dim]"
+            )

agenteval/runner.py

@@ -0,0 +1,119 @@
+"""Test runner: executes a test function N times with bounded async concurrency."""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import time
+from typing import Any, Callable, Optional
+
+import anyio
+
+from agenteval.models import AgentTrace, TestResult
+from agenteval.tracer import _ACTIVE_TRACER, Tracer
+
+
+async def _run_single(
+    test_fn: Callable[[Tracer], Any],
+    run_index: int,
+) -> AgentTrace:
+    """Execute one run of test_fn with a fresh Tracer. Returns the completed AgentTrace."""
+    tracer = Tracer()
+    token = _ACTIVE_TRACER.set(tracer)
+    try:
+        await test_fn(tracer)
+    except AssertionError as e:
+        # Assertion failures from tracer.assert_that().check()
+        tracer._assertion_errors = [line.strip(" •") for line in str(e).splitlines() if line.strip()]
+        tracer._assertion_errors = [str(e)]
+    except Exception as e:
+        if tracer._run_error is None:
+            tracer._run_error = f"{type(e).__name__}: {e}"
+    finally:
+        _ACTIVE_TRACER.reset(token)
+        # Close the run context if the test forgot to (e.g. exception before __aexit__)
+        if tracer._start_time is not None and tracer._end_time is None:
+            tracer._end_time = time.perf_counter()
+
+    return tracer.build_trace()
+
+
+async def _run_async(
+    test_fn: Callable[[Tracer], Any],
+    n: int,
+    concurrency: int,
+) -> list[AgentTrace]:
+    """Run test_fn N times with bounded concurrency using anyio."""
+    traces: list[AgentTrace] = []
+    lock = anyio.Lock()
+    semaphore = anyio.Semaphore(concurrency)
+
+    async def bounded_run(i: int) -> None:
+        async with semaphore:
+            trace = await _run_single(test_fn, i)
+            async with lock:
+                traces.append(trace)
+
+    async with anyio.create_task_group() as tg:
+        for i in range(n):
+            tg.start_soon(bounded_run, i)
+
+    return traces
+
+
+def run(
+    test_fn: Callable[[Tracer], Any],
+    *,
+    n: int = 20,
+    concurrency: int = 4,
+    name: Optional[str] = None,
+    threshold: float = 0.8,
+    tags: Optional[list[str]] = None,
+) -> TestResult:
+    """Run a test function N times and return aggregated results.
+
+    Accepts both sync and async test functions. Sync functions are run in a
+    thread pool via anyio.to_thread.run_sync so they don't block the event loop.
+
+    Args:
+        test_fn: The test function. Signature: (tracer: Tracer) -> None.
+        n: Number of times to run the test. Default: 20.
+        concurrency: Maximum number of concurrent runs. Default: 4.
+        name: Override the test name (defaults to test_fn.__name__).
+        threshold: Pass rate required to consider the test successful. Default: 0.8.
+        tags: Optional list of tags for filtering in run_suite().
+
+    Returns:
+        TestResult with pass rate, all traces, and statistics.
+
+    Example::
+
+        result = agenteval.run(test_my_agent, n=20, threshold=0.9)
+        reporter.render_result(result)
+    """
+    actual_name = name or getattr(test_fn, "__name__", "unnamed_test")
+
+    # Normalize: wrap sync test functions so the runner is purely async internally
+    if not inspect.iscoroutinefunction(test_fn):
+        original = test_fn
+
+        async def async_wrapper(tracer: Tracer) -> None:
+            await anyio.to_thread.run_sync(functools.partial(original, tracer))
+
+        async_wrapper.__name__ = actual_name
+        wrapped: Callable[[Tracer], Any] = async_wrapper
+    else:
+        wrapped = test_fn
+
+    traces = anyio.run(_run_async, wrapped, n, concurrency)
+
+    n_passed = sum(1 for t in traces if t.passed)
+    return TestResult(
+        test_name=actual_name,
+        n_runs=n,
+        n_passed=n_passed,
+        pass_rate=n_passed / n if n > 0 else 0.0,
+        threshold=threshold,
+        traces=traces,
+        tags=tags or [],
+    )