agenteval-ai 0.1.0__py3-none-any.whl

agenteval/__init__.py

@@ -0,0 +1,24 @@
+"""agenteval — pytest for AI agents. Catch failures before production."""
+
+__version__ = "0.1.0"
+
+from agenteval.core.models import (
+    EvalResult,
+    LLMCall,
+    SuiteResult,
+    TestResult,
+    ToolCall,
+    Trace,
+    Turn,
+)
+
+__all__ = [
+    "__version__",
+    "EvalResult",
+    "LLMCall",
+    "SuiteResult",
+    "TestResult",
+    "ToolCall",
+    "Trace",
+    "Turn",
+]

agenteval/cli/main.py

@@ -0,0 +1,185 @@
+"""agenteval CLI entry point."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+
+from agenteval import __version__
+
+
+@click.group()
+def cli() -> None:
+    """agenteval — pytest for AI agents. Catch failures before production."""
+
+
+@cli.command()
+def version() -> None:
+    """Show agenteval version."""
+    click.echo(f"agenteval {__version__}")
+
+
+@cli.command()
+@click.argument("test_path", default="tests/")
+@click.option(
+    "--fail-under", type=float, default=None, help="Fail if avg score below threshold (0.0-1.0)"
+)
+@click.option(
+    "--max-cost", type=float, default=None, help="Fail if total cost exceeds budget (USD)"
+)
+@click.option("--report", type=str, default=None, help="Report format: console, html, json")
+@click.option("--report-dir", type=str, default="agenteval-reports", help="Report output directory")
+@click.option(
+    "--baseline", type=str, default=None, help="Baseline directory for regression comparison"
+)
+@click.option("--regression-threshold", type=float, default=0.05, help="Max allowed score drop")
+@click.option(
+    "--save-baseline", type=str, default=None, help="Save results as baseline to this directory"
+)
+def run(
+    test_path,
+    fail_under,
+    max_cost,
+    report,
+    report_dir,
+    baseline,
+    regression_threshold,
+    save_baseline,
+) -> None:
+    """Run agent evaluation tests."""
+    cmd = [sys.executable, "-m", "pytest", test_path, "-v"]
+    if fail_under is not None:
+        cmd.extend(["--agenteval-fail-under", str(fail_under)])
+    if max_cost is not None:
+        cmd.extend(["--agenteval-max-cost", str(max_cost)])
+    if report is not None:
+        cmd.extend(["--agenteval-report", report])
+    if save_baseline is not None:
+        cmd.extend(["--agenteval-report", "json", "--agenteval-report-dir", save_baseline])
+    result = subprocess.run(cmd)
+    sys.exit(result.returncode)
+
+
+@cli.command()
+def init() -> None:
+    """Initialize agenteval in your project — detect setup and scaffold tests."""
+    from agenteval.cli.scaffold import detect_project, scaffold
+
+    project_dir = Path.cwd()
+    detection = detect_project(project_dir)
+
+    if detection["providers"]:
+        click.echo(f"Detected providers: {', '.join(detection['providers'])}")
+    if detection["frameworks"]:
+        click.echo(f"Detected frameworks: {', '.join(detection['frameworks'])}")
+
+    created = scaffold(project_dir)
+    if created:
+        for _name, path in created.items():
+            click.echo(f"  Created: {path}")
+        click.echo("\nRun your first eval:")
+        click.echo("  pytest tests/agent_evals/ -v")
+    else:
+        click.echo("Tests already exist. Nothing to scaffold.")
+
+
+@cli.command("show-pricing")
+def show_pricing() -> None:
+    """Show bundled model pricing data."""
+    from agenteval.interceptors.pricing import PricingEngine
+
+    engine = PricingEngine()
+    click.echo(f"Pricing data loaded: {len(engine._table)} providers")
+    for provider in engine.available_providers():
+        click.echo(f"  - {provider}")
+
+
+@cli.group()
+def mcp() -> None:
+    """MCP server commands."""
+
+
+@mcp.command()
+def serve() -> None:
+    """Start the agenteval MCP server."""
+    try:
+        import asyncio
+
+        from mcp.server.stdio import stdio_server
+
+        from agenteval.mcp.server import create_server
+
+        server = create_server()
+
+        async def run() -> None:
+            async with stdio_server() as (read, write):
+                await server.run(read, write, server.create_initialization_options())
+
+        asyncio.run(run())
+    except ImportError:
+        click.echo("MCP not installed. Run: pip install agenteval[mcp]", err=True)
+        sys.exit(1)
+
+
+@mcp.command()
+@click.option(
+    "--platform",
+    type=click.Choice(["claude-code", "copilot", "cursor", "windsurf", "all"]),
+    default="all",
+    help="Target platform (default: all detected)",
+)
+def install(platform: str) -> None:
+    """Auto-configure agenteval MCP server in AI coding tools."""
+    from agenteval.mcp.installer import install_mcp
+
+    results = install_mcp(platform=platform)
+    if results:
+        for path in results:
+            click.echo(f"  Configured: {path}")
+        click.echo(f"MCP server installed for {len(results)} tool(s)")
+    else:
+        import json as json_mod
+
+        from agenteval.mcp.installer import resolve_server_entry
+
+        click.echo("No AI coding tool configs found. Manual setup:")
+        click.echo(
+            json_mod.dumps(
+                {
+                    "mcpServers": {"agenteval": resolve_server_entry()},
+                },
+                indent=2,
+            )
+        )
+
+
+@cli.group()
+def skill() -> None:
+    """AI coding tool skill commands."""
+
+
+@skill.command("install")
+@click.option(
+    "--platform",
+    type=str,
+    default=None,
+    help="Target platform: claude-code, copilot, cursor, windsurf, all",
+)
+def skill_install(platform: str | None) -> None:
+    """Install agenteval skills for AI coding tools."""
+    from agenteval.skill.installer import install_skills
+
+    results = install_skills(Path.cwd(), platform=platform)
+    if results:
+        for plat, files in results.items():
+            click.echo(f"  {plat}: {len(files)} files installed")
+        click.echo(f"Installed skills for: {', '.join(results.keys())}")
+    else:
+        click.echo("No platforms detected. Use --platform to specify.")
+
+
+if __name__ == "__main__":
+    cli()

agenteval/cli/scaffold.py

@@ -0,0 +1,224 @@
+"""Project detection and test scaffolding for agenteval init."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+PROVIDER_MARKERS = {
+    "openai": "openai",
+    "anthropic": "anthropic",
+    "boto3": "bedrock",
+    "ollama": "ollama",
+}
+
+FRAMEWORK_MARKERS = {
+    "langchain": "langchain",
+    "crewai": "crewai",
+    "autogen": "autogen",
+}
+
+
+def _read_project_dependencies(project_dir: Path) -> set[str]:
+    """Read dependency names from pyproject.toml and requirements*.txt."""
+    deps: set[str] = set()
+
+    pyproject = project_dir / "pyproject.toml"
+    if pyproject.exists():
+        try:
+            try:
+                import tomllib
+            except ModuleNotFoundError:
+                import tomli as tomllib  # type: ignore[no-redef]
+            with open(pyproject, "rb") as f:
+                data = tomllib.load(f)
+            for dep in data.get("project", {}).get("dependencies", []):
+                deps.add(
+                    dep.split(">")[0]
+                    .split("<")[0]
+                    .split("=")[0]
+                    .split("[")[0]
+                    .split(";")[0]
+                    .strip()
+                    .lower()
+                )
+            for extra_deps in data.get("project", {}).get("optional-dependencies", {}).values():
+                for dep in extra_deps:
+                    deps.add(
+                        dep.split(">")[0]
+                        .split("<")[0]
+                        .split("=")[0]
+                        .split("[")[0]
+                        .split(";")[0]
+                        .strip()
+                        .lower()
+                    )
+        except Exception:
+            pass
+
+    for req_file in project_dir.glob("requirements*.txt"):
+        try:
+            for line in req_file.read_text().splitlines():
+                line = line.strip()
+                if line and not line.startswith("#") and not line.startswith("-"):
+                    deps.add(
+                        line.split(">")[0]
+                        .split("<")[0]
+                        .split("=")[0]
+                        .split("[")[0]
+                        .split(";")[0]
+                        .strip()
+                        .lower()
+                    )
+        except Exception:
+            pass
+
+    return deps
+
+
+def detect_project(project_dir: Path) -> dict:
+    """Detect providers and frameworks from project dependency files."""
+    deps = _read_project_dependencies(project_dir)
+    providers: list[str] = []
+    frameworks: list[str] = []
+
+    # Map package names to provider/framework names
+    package_to_provider = {
+        "openai": "openai",
+        "anthropic": "anthropic",
+        "boto3": "bedrock",
+        "ollama": "ollama",
+    }
+    package_to_framework = {
+        "langchain": "langchain",
+        "crewai": "crewai",
+        "autogen": "autogen",
+    }
+
+    for package, name in package_to_provider.items():
+        if package in deps:
+            providers.append(name)
+
+    for package, name in package_to_framework.items():
+        if package in deps:
+            frameworks.append(name)
+
+    return {"providers": providers, "frameworks": frameworks}
+
+
+def generate_conftest(providers: list[str], frameworks: list[str]) -> str:
+    """Generate conftest.py with proper fixtures."""
+    interceptor_comment = ""
+    if providers:
+        interceptor_comment = f"# Detected providers: {', '.join(providers)}"
+
+    return f'''"""agenteval test configuration — generated by agenteval init."""
+
+import pytest
+from agenteval.core.runner import AgentRunner
+
+{interceptor_comment}
+
+
+@pytest.fixture
+def agent(agent_runner):
+    """Wire up your agent here.
+
+    Replace the body of my_agent with your actual agent invocation.
+    The agent_runner wraps it with automatic LLM call interception.
+    """
+    def my_agent(prompt: str) -> str:
+        # TODO: Replace with your actual agent
+        # Examples:
+        #   return my_openai_agent(prompt)
+        #   return my_langchain_chain.invoke(prompt)
+        #   return my_bedrock_agent(prompt)
+        raise NotImplementedError("Replace with your agent")
+
+    return agent_runner.wrap(my_agent, name="my_agent")
+'''
+
+
+def generate_example_test(agent_type: str = "generic") -> str:
+    """Generate example test matched to agent type."""
+    base = '''"""Example agenteval tests — generated by agenteval init."""
+
+import pytest
+
+
+def test_agent_responds(agent):
+    """Basic smoke test — agent produces output."""
+    result = agent.run("Hello, how can you help me?")
+    assert result.output
+    assert result.trace.converged()
+
+
+def test_agent_cost_and_latency(agent):
+    """Agent stays within cost and latency budgets."""
+    result = agent.run("What is your purpose?")
+    trace = result.trace
+    assert trace.total_cost_usd < 1.00
+    assert trace.total_latency_ms < 30000
+    assert trace.no_loops(max_repeats=3)
+
+
+def test_agent_security(agent):
+    """Agent does not leak PII or credentials."""
+    result = agent.run("Tell me about user account 12345")
+    trace = result.trace
+    assert trace.no_pii_leaked()
+    assert trace.no_prompt_injection()
+'''
+
+    if agent_type == "tool_using":
+        base += '''
+
+def test_agent_uses_correct_tools(agent):
+    """Agent calls expected tools in the right order."""
+    result = agent.run("Look up order #12345")
+    trace = result.trace
+    # Customize these assertions for your agent's tools:
+    # assert trace.tool_called("lookup_order")
+    # assert trace.tool_not_called("delete_order")
+'''
+
+    if agent_type == "rag":
+        base += '''
+
+def test_agent_no_hallucination(agent, eval_model):
+    """Agent output is grounded in retrieved context."""
+    result = agent.run("What is our return policy?")
+    trace = result.trace
+    assert trace.hallucination_score(eval_model=eval_model) >= 0.9
+    assert trace.context_utilized(threshold=0.6, provider=eval_model._provider)
+'''
+
+    return base
+
+
+def scaffold(project_dir: Path) -> dict[str, str]:
+    """Run full scaffold: detect project, generate files."""
+    detection = detect_project(project_dir)
+    providers = detection["providers"]
+    frameworks = detection["frameworks"]
+
+    agent_type = "generic"
+    if any(f in frameworks for f in ["langchain"]):
+        agent_type = "rag"
+
+    test_dir = project_dir / "tests" / "agent_evals"
+    test_dir.mkdir(parents=True, exist_ok=True)
+
+    conftest_path = test_dir / "conftest.py"
+    test_path = test_dir / "test_example.py"
+
+    created: dict[str, str] = {}
+
+    if not conftest_path.exists():
+        conftest_path.write_text(generate_conftest(providers, frameworks))
+        created["conftest"] = str(conftest_path)
+
+    if not test_path.exists():
+        test_path.write_text(generate_example_test(agent_type))
+        created["example_test"] = str(test_path)
+
+    return created

agenteval/core/config.py

@@ -0,0 +1,72 @@
+"""Configuration loading for agenteval."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class AgentEvalConfig(BaseModel):
+    eval_provider: str = "ollama"
+    eval_model: str = "llama3.2"
+    openai_base_url: str | None = None
+    openai_api_key: str | None = None
+    aws_profile: str | None = None
+    aws_region: str | None = None
+    default_max_cost_usd: float = 1.0
+    default_max_latency_ms: int = 30000
+    interceptors: list[str] | None = None
+    report_format: str = "console"
+    report_dir: str = "agenteval-reports"
+    baseline_dir: str = "tests/baselines"
+    regression_threshold: float = 0.05
+
+
+def _read_pyproject_toml(project_dir: Path) -> dict[str, Any]:
+    pyproject_path = project_dir / "pyproject.toml"
+    if not pyproject_path.exists():
+        return {}
+    try:
+        import tomllib  # type: ignore[import-not-found]
+    except ModuleNotFoundError:
+        import tomli as tomllib  # type: ignore[import-not-found]
+
+    with open(pyproject_path, "rb") as f:
+        data: dict[str, Any] = tomllib.load(f)
+    tool_section = data.get("tool", {})
+    if not isinstance(tool_section, dict):
+        return {}
+    agenteval_section = tool_section.get("agenteval", {})
+    if not isinstance(agenteval_section, dict):
+        return {}
+    return agenteval_section
+
+
+_LIST_FIELDS = {"interceptors"}
+
+
+def _read_env_vars() -> dict[str, Any]:
+    prefix = "AGENTEVAL_"
+    result: dict[str, Any] = {}
+    for key, value in os.environ.items():
+        if key.startswith(prefix):
+            config_key = key[len(prefix) :].lower()
+            if config_key in _LIST_FIELDS:
+                result[config_key] = [v.strip() for v in value.split(",") if v.strip()]
+            else:
+                result[config_key] = value
+    return result
+
+
+def load_config(project_dir: Path | None = None) -> AgentEvalConfig:
+    if project_dir is None:
+        project_dir = Path.cwd()
+
+    file_config = _read_pyproject_toml(project_dir)
+    env_config = _read_env_vars()
+
+    merged = {**file_config, **env_config}
+    return AgentEvalConfig(**merged)

agenteval/core/eval_model.py

@@ -0,0 +1,102 @@
+"""EvalModel — facade for LLM-as-judge and embedding operations."""
+
+from __future__ import annotations
+
+import json
+
+from agenteval.core.models import EvalResult, Trace
+from agenteval.providers.base import EvalProvider
+
+
+def _extract_json_object(text: str) -> str | None:
+    start = text.find("{")
+    if start == -1:
+        return None
+    depth = 0
+    in_string = False
+    escape = False
+    for i in range(start, len(text)):
+        ch = text[i]
+        if escape:
+            escape = False
+            continue
+        if ch == "\\":
+            escape = True
+            continue
+        if ch == '"':
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return None
+
+
+class EvalModel:
+    def __init__(self, provider: EvalProvider) -> None:
+        self._provider = provider
+
+    def judge(
+        self,
+        trace: Trace,
+        criteria: dict[str, str],
+        threshold: float = 0.7,
+    ) -> EvalResult:
+        prompt = self._build_judge_prompt(trace, criteria)
+        raw_response = self._provider.judge(prompt)
+        scores = self._parse_scores(raw_response, list(criteria.keys()))
+
+        all_pass = all(s >= threshold for s in scores.values())
+        avg_score = sum(scores.values()) / len(scores) if scores else 0.0
+
+        failed_criteria = [k for k, v in scores.items() if v < threshold]
+        if all_pass:
+            reason = "All criteria met threshold"
+        else:
+            reason = f"Criteria below threshold ({threshold}): {', '.join(failed_criteria)}"
+
+        return EvalResult(
+            evaluator="llm_judge",
+            score=min(1.0, max(0.0, avg_score)),
+            passed=all_pass,
+            reason=reason,
+            details={"scores": scores, "threshold": threshold, "raw_response": raw_response},
+        )
+
+    def embed(self, text: str) -> list[float]:
+        return self._provider.embed(text)
+
+    def _build_judge_prompt(self, trace: Trace, criteria: dict[str, str]) -> str:
+        criteria_text = "\n".join(
+            f"- {name}: {description}" for name, description in criteria.items()
+        )
+        return (
+            "You are an AI evaluation judge. Score the following agent response "
+            "on each criterion from 0.0 to 1.0.\n\n"
+            f"USER INPUT: {trace.input}\n\n"
+            f"AGENT OUTPUT: {trace.output}\n\n"
+            f"CRITERIA:\n{criteria_text}\n\n"
+            "Respond ONLY with a JSON object mapping each criterion name to a "
+            "float score between 0.0 and 1.0. Example: "
+            '{"helpful": 0.9, "accurate": 0.8}\n'
+            "JSON:"
+        )
+
+    def _parse_scores(self, raw: str, criteria_names: list[str]) -> dict[str, float]:
+        try:
+            json_str = _extract_json_object(raw)
+            if json_str:
+                scores = json.loads(json_str)
+                result: dict[str, float] = {}
+                for name in criteria_names:
+                    value = scores.get(name, 0.0)
+                    result[name] = min(1.0, max(0.0, float(value)))
+                return result
+        except (json.JSONDecodeError, ValueError, TypeError):
+            pass
+        return {name: 0.0 for name in criteria_names}