openagent-eval 0.1.0__py3-none-any.whl

openagent_eval/__init__.py

@@ -0,0 +1,4 @@
+"""OpenAgent Eval - Open-source CLI framework for evaluating RAG systems and AI Agents."""
+
+__version__ = "0.1.0"
+__author__ = "OpenAgent Eval Contributors"

openagent_eval/cli/__init__.py

@@ -0,0 +1,5 @@
+"""CLI module for OpenAgent Eval."""
+
+from openagent_eval.cli.main import app
+
+__all__ = ["app"]

openagent_eval/cli/commands/__init__.py

@@ -0,0 +1,17 @@
+"""CLI commands for OpenAgent Eval."""
+
+from openagent_eval.cli.commands.init import init_command
+from openagent_eval.cli.commands.run import run_command
+from openagent_eval.cli.commands.report import report_command
+from openagent_eval.cli.commands.compare import compare_command
+from openagent_eval.cli.commands.list_evaluations import list_command
+from openagent_eval.cli.commands.doctor import doctor_command
+
+__all__ = [
+    "init_command",
+    "run_command",
+    "report_command",
+    "compare_command",
+    "list_command",
+    "doctor_command",
+]

openagent_eval/cli/commands/compare.py

@@ -0,0 +1,75 @@
+"""Compare command for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from openagent_eval.exceptions.cli import CommandError
+from openagent_eval.reports.comparison import ComparisonReport
+from openagent_eval.reports.base import ExperimentComparison
+from openagent_eval.reports.manager import ReportManager
+from openagent_eval.cli.utils.constants import DEFAULT_OUTPUT_DIR
+from openagent_eval.cli.utils.helpers import resolve_report_id
+
+console = Console()
+
+
+def compare_command(
+    experiment_a: str = typer.Argument(
+        help="First experiment ID or path.",
+    ),
+    experiment_b: str = typer.Argument(
+        help="Second experiment ID or path.",
+    ),
+    metrics: list[str] = typer.Option(
+        None,
+        "--metrics",
+        "-m",
+        help="Specific metrics to compare (default: all).",
+    ),
+    output_dir: str = typer.Option(
+        None,
+        "--output-dir",
+        "-d",
+        help="Directory where reports are stored (default: ./reports).",
+    ),
+) -> None:
+    """Compare two evaluation experiments side by side."""
+    console.print("[bold blue]OpenAgent Eval[/bold blue] - Experiment Comparison")
+    console.print(f"[dim]Comparing: {experiment_a} vs {experiment_b}[/dim]\n")
+
+    manager = ReportManager()
+    reports_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT_DIR
+
+    try:
+        data_a = resolve_report_id(experiment_a, reports_dir, manager)
+    except CommandError as e:
+        console.print(f"[red]Error:[/red] {e.message}")
+        raise typer.Exit(code=e.exit_code) from None
+
+    try:
+        data_b = resolve_report_id(experiment_b, reports_dir, manager)
+    except CommandError as e:
+        console.print(f"[red]Error:[/red] {e.message}")
+        raise typer.Exit(code=e.exit_code) from None
+
+    # Reconstruct EvaluationReports and extract PipelineResults
+    report_a = manager.reconstruct(data_a)
+    report_b = manager.reconstruct(data_b)
+
+    # Create ExperimentComparison (needs PipelineResult objects)
+    comparison = ExperimentComparison(
+        baseline_name=experiment_a,
+        experiment_name=experiment_b,
+        baseline_results=report_a.result,
+        experiment_results=report_b.result,
+    )
+
+    # Generate comparison
+    generator = ComparisonReport()
+    comparison_output = generator.generate(comparison)
+
+    console.print(comparison_output)

openagent_eval/cli/commands/doctor.py

@@ -0,0 +1,100 @@
+"""Doctor command for OpenAgent Eval."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from openagent_eval import __version__
+
+console = Console()
+
+
+def doctor_command(
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Show detailed information.",
+    ),
+) -> None:
+    """Check environment and dependencies for OpenAgent Eval."""
+    console.print("[bold blue]OpenAgent Eval[/bold blue] - Environment Check\n")
+
+    table = Table(title="Environment Status")
+    table.add_column("Component", style="cyan")
+    table.add_column("Status", style="bold")
+    table.add_column("Details", style="dim")
+
+    python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    python_ok = sys.version_info >= (3, 11)
+    table.add_row(
+        "Python",
+        "[green]OK[/green]" if python_ok else "[red]MISSING[/red]",
+        f"v{python_version}" + ("" if python_ok else " (3.11+ required)"),
+    )
+
+    dependencies = [
+        ("typer", "CLI framework"),
+        ("rich", "Terminal UI"),
+        ("pydantic", "Data validation"),
+        ("yaml", "Configuration"),
+        ("loguru", "Logging"),
+        ("jinja2", "HTML templates"),
+    ]
+
+    for dep_name, dep_desc in dependencies:
+        try:
+            importlib.import_module(dep_name.replace("-", "_"))
+            table.add_row(dep_name, "[green]OK[/green]", dep_desc)
+        except ImportError:
+            table.add_row(dep_name, "[red]MISSING[/red]", f"{dep_desc} (not installed)")
+
+    console.print(table)
+
+    key_table = Table(title="API Key Availability")
+    key_table.add_column("Provider", style="cyan")
+    key_table.add_column("Environment Variable", style="yellow")
+    key_table.add_column("Status", style="bold")
+
+    api_keys = [
+        ("OpenAI", "OPENAI_API_KEY"),
+        ("Gemini", "GEMINI_API_KEY"),
+        ("Anthropic", "ANTHROPIC_API_KEY"),
+        ("Groq", "GROQ_API_KEY"),
+    ]
+
+    available_providers: list[str] = []
+    for provider_name, env_var in api_keys:
+        key_value = os.environ.get(env_var)
+        if key_value:
+            key_table.add_row(provider_name, env_var, "[green]Available[/green]")
+            available_providers.append(provider_name)
+        else:
+            key_table.add_row(provider_name, env_var, "[dim]Not set[/dim]")
+
+    console.print(key_table)
+
+    console.print("\n[bold]Summary:[/bold]")
+    if python_ok:
+        console.print("[green]OK[/green] Python version is compatible")
+    else:
+        console.print("[red]MISSING[/red] Python 3.11+ required")
+
+    if available_providers:
+        console.print(
+            f"[green]OK[/green] Available providers: {', '.join(available_providers)}"
+        )
+    else:
+        console.print("[yellow]WARNING[/yellow] No API keys configured")
+
+    if verbose:
+        console.print(f"\n[dim]Python: {sys.executable}[/dim]")
+        console.print(f"[dim]Version: {__version__}[/dim]")
+
+    console.print("\n[dim]Run 'pip install openagent-eval[all]' to install all dependencies.[/dim]")

openagent_eval/cli/commands/init.py

@@ -0,0 +1,51 @@
+"""Init command for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.prompt import Confirm
+
+from openagent_eval.exceptions import ConfigurationError
+from openagent_eval.cli.utils.constants import DEFAULT_CONFIG_CONTENT
+
+console = Console()
+
+
+def init_command(
+    config_path: str = typer.Option(
+        "config.yaml",
+        "--config",
+        "-c",
+        help="Path to create configuration file.",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="Overwrite existing configuration file.",
+    ),
+) -> None:
+    """Create a new evaluation configuration file."""
+    path = Path(config_path)
+
+    if path.exists() and not force and not Confirm.ask(
+        f"Configuration file '{config_path}' already exists. Overwrite?"
+    ):
+        console.print("[yellow]Aborted.[/yellow]")
+        raise typer.Exit()
+
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(DEFAULT_CONFIG_CONTENT, encoding="utf-8")
+        console.print(f"[green]OK[/green] Configuration created: {config_path}")
+        console.print("\n[yellow]Next steps:[/yellow]")
+        console.print("  1. Edit the configuration file")
+        console.print("  2. Run [bold]oaeval run[/bold] to start evaluation")
+    except OSError as e:
+        raise ConfigurationError(
+            message=f"Failed to create configuration: {e}",
+            config_path=config_path,
+        ) from e

openagent_eval/cli/commands/list_evaluations.py

@@ -0,0 +1,49 @@
+"""List command for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from openagent_eval.reports.manager import ReportManager
+from openagent_eval.cli.utils.constants import DEFAULT_OUTPUT_DIR
+from openagent_eval.cli.utils.display import display_report_list
+
+console = Console()
+
+
+def list_command(
+    limit: int = typer.Option(
+        10,
+        "--limit",
+        "-l",
+        help="Number of evaluations to show.",
+    ),
+    output: str = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Filter by output format.",
+    ),
+    output_dir: str = typer.Option(
+        None,
+        "--output-dir",
+        "-d",
+        help="Directory where reports are stored (default: ./reports).",
+    ),
+) -> None:
+    """List previous evaluation runs."""
+    console.print("[bold blue]OpenAgent Eval[/bold blue] - Evaluation History\n")
+
+    manager = ReportManager()
+    reports_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT_DIR
+
+    try:
+        reports = manager.list_reports(reports_dir)
+    except Exception as e:
+        console.print(f"[red]Error:[/red] Failed to list reports: {e}")
+        raise typer.Exit(code=1) from e
+
+    display_report_list(reports, limit, output, reports_dir, manager)

openagent_eval/cli/commands/report.py

@@ -0,0 +1,64 @@
+"""Report command for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from openagent_eval.exceptions.cli import CommandError
+from openagent_eval.reports.manager import ReportManager
+from openagent_eval.reports.terminal import TerminalReport
+from openagent_eval.cli.utils.constants import DEFAULT_OUTPUT_DIR
+from openagent_eval.cli.utils.helpers import get_report_generator, resolve_report_id
+
+console = Console()
+
+
+def report_command(
+    report_id: str = typer.Argument(
+        help="Report ID or 'latest' for the most recent report.",
+    ),
+    output: str = typer.Option(
+        "terminal",
+        "--output",
+        "-o",
+        help="Output format (terminal, markdown, html, json).",
+    ),
+    output_dir: str = typer.Option(
+        None,
+        "--output-dir",
+        "-d",
+        help="Directory where reports are stored (default: ./reports).",
+    ),
+) -> None:
+    """View evaluation reports."""
+    console.print("[bold blue]OpenAgent Eval[/bold blue] - Report Viewer")
+    console.print(f"[dim]Report: {report_id}[/dim]\n")
+
+    manager = ReportManager()
+    reports_dir = Path(output_dir) if output_dir else DEFAULT_OUTPUT_DIR
+
+    try:
+        data = resolve_report_id(report_id, reports_dir, manager)
+    except CommandError as e:
+        console.print(f"[red]Error:[/red] {e.message}")
+        raise typer.Exit(code=e.exit_code) from None
+
+    evaluation_report = manager.reconstruct(data)
+
+    try:
+        generator = get_report_generator(output)
+    except CommandError as e:
+        console.print(f"[red]Error:[/red] {e.message}")
+        raise typer.Exit(code=e.exit_code) from None
+
+    if output == "terminal":
+        gen = TerminalReport()
+        gen.print_report(evaluation_report)
+    else:
+        report_content = generator.generate(evaluation_report)
+        console.print(report_content)
+
+    console.print(f"\n[dim]Report ID: {data.get('report_id', 'unknown')}[/dim]")

openagent_eval/cli/commands/run.py

@@ -0,0 +1,77 @@
+"""Run command for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+from openagent_eval import __version__
+from openagent_eval.exceptions import ConfigurationError
+from openagent_eval.reports.manager import ReportManager
+from openagent_eval.cli.utils.helpers import (
+    apply_output_override,
+    execute_evaluation,
+    load_config_from_path,
+    load_dataset_for_run,
+)
+from openagent_eval.cli.utils.display import display_run_result
+
+console = Console()
+
+
+def run_command(
+    config_path: str = typer.Argument(
+        help="Path to configuration file.",
+    ),
+    output: str = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Override output format (terminal, markdown, html, json).",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Enable verbose output.",
+    ),
+) -> None:
+    """Run evaluation pipeline with the specified configuration."""
+    path = Path(config_path)
+
+    if not path.exists():
+        raise ConfigurationError(
+            message=f"Configuration file not found: {config_path}",
+            config_path=config_path,
+        )
+
+    console.print(f"[bold blue]OpenAgent Eval[/bold blue] v{__version__}")
+    console.print(f"[dim]Configuration: {config_path}[/dim]\n")
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console,
+    ) as progress:
+        task = progress.add_task("Loading configuration...", total=None)
+        config = load_config_from_path(config_path)
+        apply_output_override(config, output)
+
+        progress.update(task, description="Loading dataset...")
+        dataset_items = load_dataset_for_run(config)
+
+        progress.update(task, description="Running evaluation...")
+        report = execute_evaluation(config, dataset_items)
+
+        progress.update(task, description="Generating report...")
+        format_name = config.report.output.value
+        manager = ReportManager()
+        output_dir = Path(config.report.output_dir)
+        report_path = manager.save_report(report, output_dir)
+
+        progress.update(task, description="Complete!", completed=True)
+
+    display_run_result(report, format_name, report_path, output_dir, verbose)

openagent_eval/cli/main.py

@@ -0,0 +1,43 @@
+"""Main CLI entry point for OpenAgent Eval."""
+
+from __future__ import annotations
+
+import typer
+
+from openagent_eval.cli.utils.callbacks import version_callback
+from openagent_eval.cli.commands.init import init_command
+from openagent_eval.cli.commands.run import run_command
+from openagent_eval.cli.commands.report import report_command
+from openagent_eval.cli.commands.compare import compare_command
+from openagent_eval.cli.commands.list_evaluations import list_command
+from openagent_eval.cli.commands.doctor import doctor_command
+
+app = typer.Typer(
+    name="oaeval",
+    help="Open-source CLI framework for evaluating RAG systems and AI Agents.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        False,
+        "--version",
+        "-V",
+        help="Show version and exit.",
+        callback=version_callback,
+        is_eager=True,
+    ),
+) -> None:
+    """OpenAgent Eval - Evaluate RAG systems and AI Agents."""
+
+
+# Register commands with explicit names
+app.command(name="init")(init_command)
+app.command(name="run")(run_command)
+app.command(name="report")(report_command)
+app.command(name="compare")(compare_command)
+app.command(name="list")(list_command)
+app.command(name="doctor")(doctor_command)

openagent_eval/cli/utils/__init__.py

@@ -0,0 +1,6 @@
+"""CLI utility helpers for OpenAgent Eval."""
+
+from openagent_eval.cli.utils.callbacks import version_callback
+from openagent_eval.cli.utils.constants import DEFAULT_CONFIG_CONTENT
+
+__all__ = ["version_callback", "DEFAULT_CONFIG_CONTENT"]

openagent_eval/cli/utils/callbacks.py

@@ -0,0 +1,14 @@
+"""CLI callbacks for OpenAgent Eval."""
+
+from __future__ import annotations
+
+import typer
+
+
+def version_callback(value: bool) -> None:
+    """Print version and exit."""
+    if value:
+        from openagent_eval import __version__
+
+        typer.echo(f"OpenAgent Eval v{__version__}")
+        raise typer.Exit()

openagent_eval/cli/utils/constants.py

@@ -0,0 +1,47 @@
+"""CLI constants for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+DEFAULT_OUTPUT_DIR = Path("./reports")
+
+DEFAULT_CONFIG_CONTENT = """\
+# OpenAgent Eval Configuration
+# See documentation for options: https://github.com/OpenAgentHQ/openagent-eval
+#
+# For a fully offline dry-run (no API keys / vector store required), set:
+#   llm.provider: mock
+#   retriever.provider: mock
+
+dataset:
+  path: data/questions.json
+  # limit: 100
+
+llm:
+  provider: openai
+  model: gpt-4o-mini
+  temperature: 0.0
+
+retriever:
+  provider: chroma
+  settings:
+    collection_name: my_collection
+
+metrics:
+  retrieval:
+    - context_precision
+    - context_recall
+    - mrr
+  generation:
+    - faithfulness
+    - answer_relevancy
+  performance:
+    - latency
+  cost:
+    - token_count
+
+report:
+  output: terminal
+  output_dir: ./reports
+"""

openagent_eval/cli/utils/display.py

@@ -0,0 +1,88 @@
+"""CLI display helpers for OpenAgent Eval."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+def display_run_result(
+    report: Any,
+    format_name: str,
+    report_path: Path,
+    output_dir: Path,
+    verbose: bool,
+) -> None:
+    """Display the evaluation run result."""
+    console.print("\n[green]OK[/green] Evaluation complete!")
+
+    if hasattr(report, "summary"):
+        summary = report.summary
+        total = summary.get("total_items", summary.get("total", 0))
+        errors = summary.get("failed_evaluations", summary.get("errors", 0))
+        console.print(f"[dim]Items: {total} | Errors: {errors}[/dim]")
+
+    console.print(f"[dim]Report saved to: {report_path}[/dim]")
+
+
+def display_report_list(
+    reports: list[dict[str, str]],
+    limit: int,
+    output_filter: str | None,
+    output_dir: Path,
+    manager: Any,
+) -> None:
+    """Display a table of evaluation reports."""
+    from openagent_eval.reports.manager import ReportManager
+
+    if not reports:
+        console.print("[yellow]No evaluations found.[/yellow]")
+        return
+
+    if output_filter:
+        filtered: list[dict[str, str]] = []
+        for r in reports:
+            try:
+                data = manager.load_report(r["report_id"], output_dir)
+                report_format = (
+                    data.get("config", {}).get("report", {}).get("output", "terminal")
+                )
+                if report_format == output_filter:
+                    filtered.append(r)
+            except (FileNotFoundError, KeyError):
+                continue
+        reports = filtered
+
+    reports = reports[:limit]
+
+    table = Table(title="Recent Evaluations")
+    table.add_column("ID", style="cyan")
+    table.add_column("Date", style="green")
+    table.add_column("Config", style="yellow")
+    table.add_column("Status", style="bold")
+
+    for r in reports:
+        report_id = r["report_id"]
+        created_at = r.get("created_at", "unknown")
+        config_name = "unknown"
+        status = "OK"
+        try:
+            data = manager.load_report(report_id, output_dir)
+            config_name = data.get("config", {}).get("dataset", {}).get("path", "unknown")
+            errors = data.get("errors", [])
+            status = "[green]OK[/green]" if not errors else "[red]FAILED[/red]"
+        except (FileNotFoundError, KeyError):
+            status = "[yellow]UNKNOWN[/yellow]"
+
+        if created_at and "T" in created_at:
+            created_at = created_at.split("T")[0]
+
+        table.add_row(report_id, created_at, config_name, status)
+
+    console.print(table)
+    console.print(f"\n[dim]Showing {len(reports)} evaluations[/dim]")