evalgrid 0.1.0__py3-none-any.whl

evalgrid/__init__.py

@@ -0,0 +1,3 @@
+"""evalgrid — open-source AI agent evaluation framework."""
+
+__version__ = "0.1.0"

evalgrid/__main__.py

@@ -0,0 +1,4 @@
+from evalgrid.cli.main import cli
+
+if __name__ == "__main__":
+    cli()

evalgrid/api/app.py

@@ -0,0 +1,442 @@
+"""FastAPI application — eval runner + results API."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+import uuid
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from evalgrid.core import EvalConfig, ScenarioLoader
+from evalgrid.core.models import PassFail
+from evalgrid.core.runner import ScenarioRunner
+
+from .database import RunRecord, ScenarioResultRecord, get_db, init_db
+
+logger = logging.getLogger("evalgrid")
+
+SCENARIOS_BASE_DIR = Path.home() / ".evalgrid" / "scenarios"
+
+_API_KEY_RE = re.compile(
+    r"(?:sk-ant-|sk-|pypi-)[A-Za-z0-9\-_]{6,}"
+    r"|AIza[A-Za-z0-9\-_]{30,}"
+)
+
+
+def _sanitize_error(msg: str) -> str:
+    return _API_KEY_RE.sub("[REDACTED]", msg)
+
+
+def _iso(dt: datetime | None) -> str | None:
+    """Return ISO-8601 string with explicit UTC offset so browsers parse it correctly."""
+    if dt is None:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.isoformat()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):  # noqa: ARG001
+    await init_db()
+
+    # Mark any runs that were left running (server restart) as failed
+    from .database import SessionLocal
+    try:
+        async with SessionLocal() as db:
+            stale_rows = await db.execute(
+                select(RunRecord).where(RunRecord.finished_at.is_(None))
+            )
+            stale = stale_rows.scalars().all()
+            for record in stale:
+                record.finished_at = datetime.now(timezone.utc)
+                record.total = record.total or 0
+                record.passed = record.passed or 0
+                record.pass_rate = 0.0
+                record.results = [{"error": "Server restarted"}]
+                logger.warning("Marked stale run %s as failed (server restart)", record.run_id)
+            if stale:
+                await db.commit()
+    except Exception as exc:
+        logger.warning("Stale run cleanup skipped: %s", exc)
+
+    yield
+
+
+app = FastAPI(title="evalgrid API", version="0.1.0", lifespan=lifespan)
+
+
+@app.exception_handler(Exception)
+async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    logger.error("Unhandled exception on %s: %s", request.url.path, exc)
+    sanitized = _sanitize_error(str(exc))
+    return JSONResponse(status_code=500, content={"detail": f"Internal server error: {sanitized}"})
+
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=[
+        "http://localhost:5173",
+        "http://localhost:8000",
+        "http://127.0.0.1:5173",
+        "http://127.0.0.1:8000",
+    ],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# ── Schema ────────────────────────────────────────────────────────────────────
+
+class RunRequest(BaseModel):
+    scenarios_dir: str = ""
+    tags: list[str] = []
+    scenario_ids: list[str] = []
+    mock: bool = False
+    judge_model: str | None = None
+
+
+class RunSummary(BaseModel):
+    run_id: str
+    started_at: str
+    finished_at: str | None = None
+    pass_rate: float
+    passed: int
+    total: int
+    status: str
+    scenario_names: list[str] = []
+    skipped_count: int = 0
+    skipped_scenarios: list[dict[str, str]] = []
+
+
+class ScenarioCreate(BaseModel):
+    name: str = Field(..., max_length=200)
+    description: str = Field(default="", max_length=1000)
+    agent_type: str = "generic"
+    prompt: str = Field(..., max_length=16000)
+    test_input: str = Field(..., max_length=8000)
+    expected_actions: list[dict[str, Any]] = []
+    tags: list[str] = []
+    scenarios_dir: str = ""
+
+
+def _validate_path(scenarios_dir: str) -> Path:
+    """Resolve and validate that the path stays within SCENARIOS_BASE_DIR."""
+    _base = SCENARIOS_BASE_DIR.resolve()
+    # Belt-and-suspenders: reject obvious traversal attempts
+    if ".." in scenarios_dir:
+        raise HTTPException(status_code=400, detail="Path outside allowed scenarios directory")
+    resolved = (_base / scenarios_dir).resolve()
+    if not resolved.is_relative_to(_base):
+        raise HTTPException(status_code=400, detail="Path outside allowed scenarios directory")
+    return resolved
+
+
+# ── Routes ────────────────────────────────────────────────────────────────────
+
+@app.get("/health")
+async def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/api/config-status")
+async def config_status() -> dict[str, Any]:
+    anthropic_set = bool(os.environ.get("ANTHROPIC_API_KEY"))
+    openai_set = bool(os.environ.get("OPENAI_API_KEY"))
+    return {
+        "anthropic_key_set": anthropic_set,
+        "openai_key_set": openai_set,
+        "any_key_set": anthropic_set or openai_set,
+        "mock_available": True,
+    }
+
+
+@app.post("/api/runs", response_model=RunSummary)
+async def create_run(
+    req: RunRequest,
+    background: BackgroundTasks,
+    db: AsyncSession = Depends(get_db),
+) -> RunSummary:
+    _validate_path(req.scenarios_dir)
+
+    if not req.mock and not any(os.environ.get(k) for k in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY")):
+        raise HTTPException(status_code=400, detail="No provider API key found in environment (set ANTHROPIC_API_KEY or OPENAI_API_KEY)")
+
+    run_id = str(uuid.uuid4())
+    record = RunRecord(
+        run_id=run_id,
+        started_at=datetime.now(timezone.utc),
+        results=[],
+    )
+    db.add(record)
+    await db.commit()
+
+    background.add_task(_execute_run, run_id, req)
+
+    return RunSummary(
+        run_id=run_id,
+        started_at=_iso(record.started_at),
+        pass_rate=0.0,
+        passed=0,
+        total=0,
+        status="running",
+    )
+
+
+@app.get("/api/runs", response_model=list[RunSummary])
+async def list_runs(db: AsyncSession = Depends(get_db)) -> list[RunSummary]:
+    rows = await db.execute(select(RunRecord).order_by(RunRecord.started_at.desc()).limit(100))
+    records = rows.scalars().all()
+
+    run_ids = [r.run_id for r in records]
+    name_rows = await db.execute(
+        select(ScenarioResultRecord.run_id, ScenarioResultRecord.scenario_name)
+        .where(ScenarioResultRecord.run_id.in_(run_ids))
+    )
+    names_by_run: dict[str, list[str]] = {}
+    for run_id, scenario_name in name_rows:
+        names_by_run.setdefault(run_id, []).append(scenario_name)
+
+    return [
+        RunSummary(
+            run_id=r.run_id,
+            started_at=_iso(r.started_at),
+            finished_at=_iso(r.finished_at),
+            pass_rate=r.pass_rate or 0.0,
+            passed=r.passed or 0,
+            total=r.total or 0,
+            status="completed" if r.finished_at else "running",
+            scenario_names=names_by_run.get(r.run_id, []),
+            skipped_count=len(r.skipped_scenarios or []),
+            skipped_scenarios=r.skipped_scenarios or [],
+        )
+        for r in records
+    ]
+
+
+@app.get("/api/runs/{run_id}")
+async def get_run(run_id: str, db: AsyncSession = Depends(get_db)) -> dict[str, Any]:
+    row = await db.execute(select(RunRecord).where(RunRecord.run_id == run_id))
+    record = row.scalar_one_or_none()
+    if not record:
+        raise HTTPException(status_code=404, detail="Run not found")
+
+    result_rows = await db.execute(
+        select(ScenarioResultRecord).where(ScenarioResultRecord.run_id == run_id)
+    )
+    scenario_results = result_rows.scalars().all()
+
+    return {
+        "run_id": record.run_id,
+        "started_at": _iso(record.started_at),
+        "finished_at": _iso(record.finished_at),
+        "pass_rate": record.pass_rate,
+        "passed": record.passed,
+        "total": record.total,
+        "status": "completed" if record.finished_at else "running",
+        "scenario_results": [
+            {
+                "scenario_id": r.scenario_id,
+                "scenario_name": r.scenario_name,
+                "agent_type": r.agent_type,
+                "status": r.status,
+                "overall_score": r.overall_score,
+                "pass_rate": r.pass_rate,
+                "reasoning": r.reasoning,
+                "error": r.error,
+                "steps": r.steps,
+            }
+            for r in scenario_results
+        ],
+    }
+
+
+@app.post("/api/scenarios")
+async def create_scenario(
+    req: ScenarioCreate,
+) -> dict[str, Any]:
+    scenario_id = re.sub(r"[^a-z0-9]+", "-", req.name.lower()).strip("-")
+
+    actions = req.expected_actions or [
+        {"action": "complete_task", "required": True, "description": "Agent completes the task correctly"}
+    ]
+
+    scenario_data = {
+        "id": scenario_id,
+        "name": req.name,
+        "description": req.description,
+        "agent_type": req.agent_type,
+        "prompt": req.prompt,
+        "test_input": req.test_input,
+        "expected_actions": actions,
+        "tags": req.tags,
+    }
+
+    scen_dir = _validate_path(req.scenarios_dir)
+    scen_dir.mkdir(parents=True, exist_ok=True)
+
+    path = scen_dir / f"{scenario_id}.yml"
+    if path.exists():
+        raise HTTPException(status_code=409, detail=f"Scenario with id '{scenario_id}' already exists")
+    with open(path, "w") as f:
+        yaml.dump(scenario_data, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
+
+    return {
+        "id": scenario_id,
+        "name": req.name,
+        "description": req.description,
+        "agent_type": req.agent_type,
+        "tags": req.tags,
+        "step_count": len(actions),
+    }
+
+
+@app.get("/api/scenarios")
+async def list_scenarios(scenarios_dir: str = "") -> list[dict[str, Any]]:
+    scen_dir = _validate_path(scenarios_dir)
+    if not scen_dir.exists():
+        return []
+    loader = ScenarioLoader()
+    result = []
+    for path in sorted(scen_dir.rglob("*.yml")) + sorted(scen_dir.rglob("*.yaml")):
+        if path.name.startswith("_"):
+            continue
+        try:
+            s = loader.load_file(path)
+            rel = path.relative_to(scen_dir)
+            suite = rel.parts[0] if len(rel.parts) > 1 else "default"
+            result.append({
+                "id": s.id,
+                "name": s.name,
+                "description": s.description,
+                "agent_type": s.agent_type.value,
+                "tags": s.tags,
+                "step_count": len(s.expected_actions),
+                "suite": suite,
+            })
+        except Exception as e:
+            logger.warning("Failed to load scenario %s: %s", path, e)
+    return result
+
+
+# ── Background task ────────────────────────────────────────────────────────────
+
+async def _mark_run_failed(run_id: str, error_message: str) -> None:
+    from .database import SessionLocal
+    async with SessionLocal() as db:
+        row = await db.execute(select(RunRecord).where(RunRecord.run_id == run_id))
+        record = row.scalar_one_or_none()
+        if record:
+            record.finished_at = datetime.now(timezone.utc)
+            record.total = 0
+            record.passed = 0
+            record.pass_rate = 0.0
+            record.results = [{"error": error_message}]
+            await db.commit()
+
+
+async def _execute_run(run_id: str, req: RunRequest) -> None:
+    from .database import SessionLocal
+
+    config = EvalConfig()
+    if req.judge_model:
+        config.judge.model = req.judge_model
+    loader = ScenarioLoader()
+    scen_dir = (SCENARIOS_BASE_DIR.resolve() / req.scenarios_dir).resolve()
+
+    if not scen_dir.exists():
+        await _mark_run_failed(run_id, f"Scenarios directory not found: {scen_dir}")
+        return
+
+    async def _run_with_timeout() -> None:
+        scenarios = loader.load_dir(scen_dir)
+
+        if loader.skipped:
+            async with SessionLocal() as db:
+                row = await db.execute(select(RunRecord).where(RunRecord.run_id == run_id))
+                record = row.scalar_one_or_none()
+                if record:
+                    record.skipped_scenarios = loader.skipped
+                    await db.commit()
+
+        if req.tags:
+            scenarios = [s for s in scenarios if any(t in s.tags for t in req.tags)]
+        if req.scenario_ids:
+            scenarios = [s for s in scenarios if s.id in req.scenario_ids]
+
+        if req.mock:
+            from evalgrid.core.scorer import MockScorer
+            runner = ScenarioRunner(
+                config,
+                scorer=MockScorer(),
+                agent_fn=lambda s: ("[mock agent output]", 0),
+            )
+        else:
+            runner = ScenarioRunner(config)
+
+        loop = asyncio.get_running_loop()
+        results = []
+        for scenario in scenarios:
+            result = await loop.run_in_executor(None, runner.run_scenario, scenario)
+            results.append(result)
+
+            async with SessionLocal() as db:
+                sr = ScenarioResultRecord(
+                    run_id=run_id,
+                    scenario_id=result.scenario_id,
+                    scenario_name=result.scenario_name,
+                    agent_type=result.agent_type.value,
+                    status=result.status.value,
+                    overall_score=result.overall_score,
+                    pass_rate=result.pass_rate,
+                    reasoning=result.reasoning,
+                    error=result.error,
+                    steps=[s.model_dump(mode="json") for s in result.steps],
+                    started_at=result.started_at,
+                    finished_at=result.finished_at,
+                )
+                db.add(sr)
+                await db.commit()
+
+        passed = sum(1 for r in results if r.status == PassFail.PASS)
+        total = len(results)
+
+        async with SessionLocal() as db:
+            row = await db.execute(select(RunRecord).where(RunRecord.run_id == run_id))
+            record = row.scalar_one_or_none()
+            if record:
+                record.finished_at = datetime.now(timezone.utc)
+                record.passed = passed
+                record.total = total
+                record.pass_rate = passed / total if total else 0.0
+                await db.commit()
+
+    try:
+        await asyncio.wait_for(_run_with_timeout(), timeout=1800)
+    except TimeoutError:
+        logger.error("Run %s timed out after 1800s", run_id)
+        await _mark_run_failed(run_id, "Run timed out (30 minute limit exceeded)")
+    except Exception as exc:
+        await _mark_run_failed(run_id, _sanitize_error(str(exc)))
+
+
+# Serve React dashboard — evalgrid/static/ (wheel) or dashboard/dist/ (dev)
+_dashboard = Path(__file__).parent.parent / "static"
+if not _dashboard.exists():
+    _dashboard = Path(__file__).parent.parent.parent / "dashboard" / "dist"
+if _dashboard.exists():
+    app.mount("/", StaticFiles(directory=_dashboard, html=True), name="static")

evalgrid/api/database.py

@@ -0,0 +1,78 @@
+"""Database setup — SQLite (dev) / Postgres (prod)."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+logger = logging.getLogger("evalgrid")
+
+from sqlalchemy import JSON, Column, DateTime, Float, ForeignKey, Integer, String, Text, func, text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.orm import DeclarativeBase
+
+from evalgrid.core.paths import default_database_url
+
+DATABASE_URL = default_database_url()
+
+engine = create_async_engine(DATABASE_URL, echo=False)
+SessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class RunRecord(Base):
+    __tablename__ = "runs"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    run_id = Column(String, unique=True, index=True)
+    started_at = Column(DateTime, default=func.now())
+    finished_at = Column(DateTime, nullable=True)
+    pass_rate = Column(Float, default=0.0)
+    passed = Column(Integer, default=0)
+    total = Column(Integer, default=0)
+    results = Column(JSON, default=list)
+    skipped_scenarios = Column(JSON, nullable=True)
+
+
+class ScenarioResultRecord(Base):
+    __tablename__ = "scenario_results"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    run_id = Column(String, ForeignKey("runs.run_id"), index=True)
+    scenario_id = Column(String, index=True)
+    scenario_name = Column(String)
+    agent_type = Column(String)
+    status = Column(String)
+    overall_score = Column(Float)
+    pass_rate = Column(Float)
+    reasoning = Column(Text, default="")
+    error = Column(Text, nullable=True)
+    steps = Column(JSON, default=list)
+    started_at = Column(DateTime)
+    finished_at = Column(DateTime, nullable=True)
+
+
+async def init_db() -> None:
+    """Initialize the database, creating parent directories as needed."""
+    url_str = str(engine.url)
+    if url_str.startswith("sqlite"):
+        db_path_str = url_str.split(":///", 1)[-1]
+        if db_path_str and db_path_str != ":memory:":
+            db_path = Path(db_path_str).expanduser()
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+        try:
+            await conn.execute(text("ALTER TABLE runs ADD COLUMN skipped_scenarios JSON DEFAULT '[]'"))
+        except Exception as e:
+            if "duplicate column" not in str(e).lower():
+                logger.warning("Migration skipped_scenarios failed: %s", e)
+
+
+async def get_db():
+    async with SessionLocal() as session:
+        yield session

evalgrid/cli/__init__.py

@@ -0,0 +1,3 @@
+from evalgrid.cli.main import cli as main
+
+__all__ = ["main"]

evalgrid/cli/commands/init.py

@@ -0,0 +1,139 @@
+"""evalgrid init — scaffold .evalgrid/ in the current project."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.panel import Panel
+
+console = Console(legacy_windows=False)
+
+CONFIG_TEMPLATE = """\
+# evalgrid configuration
+model: claude-sonnet-4-6
+max_concurrent: 5
+timeout_seconds: 120
+retry_on_error: true
+max_retries: 2
+output_dir: .evalgrid/results
+scenarios_dir: .evalgrid/scenarios
+# Database path: ~/.evalgrid/results.db (override via DATABASE_URL env var)
+
+# Choose your judge model (used to score agent responses)
+judge:
+  # Anthropic (default)
+  provider: anthropic
+  model: claude-sonnet-4-6
+  api_key_env: ANTHROPIC_API_KEY
+
+  # OpenAI alternative:
+  # provider: openai
+  # model: gpt-4o
+  # api_key_env: OPENAI_API_KEY
+
+  # Google alternative:
+  # provider: google
+  # model: gemini/gemini-1.5-pro
+  # api_key_env: GEMINI_API_KEY
+
+  # Local Ollama (no API key needed):
+  # provider: ollama
+  # model: ollama/llama3.1:70b
+  # api_key_env: ""
+
+  # OpenRouter (200+ models, one key):
+  # provider: openrouter
+  # model: openrouter/anthropic/claude-sonnet-4
+  # api_key_env: OPENROUTER_API_KEY
+"""
+
+EXAMPLE_SCENARIO = """\
+id: example-hello-world
+name: Hello World
+description: Verify the agent responds politely to a greeting
+agent_type: generic
+prompt: |
+  You are a helpful AI assistant. Respond to user messages in a friendly,
+  concise way. Always greet the user back and offer to help.
+test_input: "Hello! Can you help me today?"
+expected_actions:
+  - action: greet_user
+    required: true
+    description: Agent greets the user back
+  - action: offer_assistance
+    required: true
+    description: Agent offers to help
+  - action: ask_followup
+    required: false
+    description: Agent asks what the user needs help with
+success_criteria: >
+  The agent should acknowledge the greeting warmly and clearly offer assistance.
+  The tone should be friendly and professional.
+tags:
+  - greeting
+  - basic
+timeout_seconds: 30
+"""
+
+GITIGNORE_ADDITION = """
+# evalgrid
+.evalgrid/results/
+.evalgrid/results.db
+"""
+
+
+@click.command()
+@click.option("--dir", "target_dir", default=".", help="Project directory to initialize")
+@click.option("--with-examples", is_flag=True, default=True, help="Copy example scenarios")
+def init(target_dir: str, with_examples: bool) -> None:
+    """Scaffold a .evalgrid/ folder in your project."""
+    root = Path(target_dir).resolve()
+    evalgrid_dir = root / ".evalgrid"
+
+    if evalgrid_dir.exists():
+        console.print("[yellow]! .evalgrid/ already exists.[/yellow]")
+        if not click.confirm("Reinitialize?", default=False):
+            return
+
+    # Create directory structure
+    (evalgrid_dir / "scenarios").mkdir(parents=True, exist_ok=True)
+    (evalgrid_dir / "results").mkdir(parents=True, exist_ok=True)
+
+    # Write config
+    config_path = evalgrid_dir / "config.yml"
+    config_path.write_text(CONFIG_TEMPLATE)
+
+    # Write example scenario
+    if with_examples:
+        example_path = evalgrid_dir / "scenarios" / "example-hello-world.yml"
+        example_path.write_text(EXAMPLE_SCENARIO)
+
+    # Append to .gitignore if it exists
+    gitignore = root / ".gitignore"
+    if gitignore.exists():
+        content = gitignore.read_text()
+        if ".evalgrid/results" not in content:
+            gitignore.write_text(content + GITIGNORE_ADDITION)
+
+    console.print(
+        Panel.fit(
+            "[bold green]evalgrid initialized![/bold green]\n\n"
+            f"[dim]Config:[/dim]    [cyan]{config_path.relative_to(root)}[/cyan]\n"
+            f"[dim]Scenarios:[/dim] [cyan]{(evalgrid_dir / 'scenarios').relative_to(root)}[/cyan]\n\n"
+            "[bold]Next steps:[/bold]\n\n"
+            "  1. Try it now (no API key required):\n"
+            "     [cyan]evalgrid run --mock[/cyan]\n\n"
+            "  2. To run with real AI scoring, set your API key:\n"
+            "     [cyan]Linux/macOS:[/cyan]  export ANTHROPIC_API_KEY=sk-ant-...\n"
+            "     [cyan]Windows:[/cyan]      $env:ANTHROPIC_API_KEY = \"sk-ant-...\"\n\n"
+            "  3. Review the example scenario:\n"
+            "     [cyan].evalgrid/scenarios/example-hello-world.yml[/cyan]\n\n"
+            "  4. Open the dashboard:\n"
+            "     [cyan]evalgrid server[/cyan]  ->  http://localhost:8000",
+            title="evalgrid",
+            border_style="green",
+        )
+    )