aievaluator 1.0.1__py3-none-any.whl

aievaluator/__init__.py

@@ -0,0 +1,3 @@
+"""AI Evaluator CLI — evaluate your LLM agents from the command line."""
+
+__version__ = "1.0.0"

aievaluator/api/client.py

@@ -0,0 +1,178 @@
+"""HTTP client for AI Evaluator Engine API."""
+
+import json as json_mod
+from typing import Any, Optional
+
+import httpx
+
+
+class APIError(Exception):
+    """Error from the AI Evaluator API."""
+
+    def __init__(self, status_code: int, message: str, detail: Any = None):
+        self.status_code = status_code
+        self.message = message
+        self.detail = detail
+        super().__init__(message)
+
+
+class APIClient:
+    """Thin HTTP wrapper around the AI Evaluator Engine API."""
+
+    def __init__(self, engine_url: str, api_key: Optional[str] = None, timeout: int = 300):
+        self.engine_url = engine_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+
+    def _headers(self) -> dict:
+        h = {"Content-Type": "application/json"}
+        if self.api_key:
+            h["X-API-Key"] = self.api_key
+        return h
+
+    async def _request(
+        self,
+        method: str,
+        path: str,
+        json_data: Optional[dict] = None,
+        data: Optional[dict] = None,
+        files: Optional[dict] = None,
+    ) -> dict:
+        """Make an HTTP request to the engine. Raises APIError on failure."""
+        url = f"{self.engine_url}{path}"
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            try:
+                if files:
+                    resp = await client.request(
+                        method, url, data=data, files=files, headers={"X-API-Key": self.api_key} if self.api_key else {},
+                    )
+                else:
+                    resp = await client.request(
+                        method, url, json=json_data, headers=self._headers(),
+                    )
+            except httpx.ConnectError:
+                raise APIError(0, f"Cannot connect to {self.engine_url}")
+            except httpx.TimeoutException:
+                raise APIError(0, f"Request timed out after {self.timeout}s")
+
+        if resp.status_code >= 400:
+            detail = None
+            try:
+                detail = resp.json()
+            except Exception:
+                detail = resp.text
+            raise APIError(resp.status_code, f"Engine returned HTTP {resp.status_code}", detail)
+
+        return resp.json()
+
+    async def health(self) -> dict:
+        """GET /health"""
+        return await self._request("GET", "/health")
+
+    async def get_usage(self) -> dict:
+        """GET /api/v1/tenants/me/usage"""
+        return await self._request("GET", "/api/v1/tenants/me/usage")
+
+    async def evaluate_sync(
+        self,
+        rows: list[dict],
+        agent_url: str,
+        agent_format: str = "openai",
+        metrics: Optional[list[str]] = None,
+        judge_model: Optional[str] = None,
+        name: Optional[str] = None,
+        custom_evaluators: list[dict] | None = None,
+        thresholds: Optional[dict[str, float]] = None,
+    ) -> dict:
+        """POST /api/v1/evaluations/sync"""
+        agent_json = {"url": agent_url, "format": agent_format}
+        body = {
+            "rows": rows,
+            "agent": agent_json,
+            "metrics": metrics or ["faithfulness", "g_eval"],
+            "custom_evaluators": custom_evaluators or [],
+        }
+        if name:
+            body["name"] = name
+        if judge_model:
+            body["judge_model"] = judge_model
+        if thresholds:
+            body["thresholds"] = thresholds
+
+        return await self._request("POST", "/api/v1/evaluations/sync", json_data=body)
+
+    async def evaluate_upload(
+        self,
+        file_path: str,
+        agent_url: str,
+        agent_format: str = "openai",
+        metrics: Optional[str] = None,
+    ) -> dict:
+        """POST /api/v1/evaluations/sync/upload (multipart form upload)."""
+        import os
+
+        url = f"{self.engine_url}/api/v1/evaluations/sync/upload"
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            try:
+                with open(file_path, "rb") as f:
+                    files = {"file": (os.path.basename(file_path), f, "application/json")}
+                    data = {
+                        "agent_endpoint": agent_url,
+                        "agent_format": agent_format,
+                        "metrics": metrics or "faithfulness,g_eval",
+                    }
+                    resp = await client.post(
+                        url,
+                        data=data,
+                        files=files,
+                        headers={"X-API-Key": self.api_key} if self.api_key else {},
+                    )
+            except httpx.ConnectError:
+                raise APIError(0, f"Cannot connect to {self.engine_url}")
+            except httpx.TimeoutException:
+                raise APIError(0, f"Request timed out after {self.timeout}s")
+
+        if resp.status_code >= 400:
+            detail = None
+            try:
+                detail = resp.json()
+            except Exception:
+                detail = resp.text
+            raise APIError(resp.status_code, f"Engine returned HTTP {resp.status_code}", detail)
+
+        return resp.json()
+
+    async def playground_evaluate(
+        self,
+        queries: Optional[list[str]] = None,
+        rows: Optional[list[dict]] = None,
+        agent_endpoint: Optional[str] = None,
+        agent_config: Optional[dict] = None,
+        metrics: Optional[list] = None,
+        judge: Optional[str] = None,
+    ) -> dict:
+        """POST /api/v1/playground/evaluate (no auth required).
+
+        metrics accepts strings or dicts with thresholds:
+        ["g_eval"] or [{"name": "g_eval", "threshold": 0.9}]
+        """
+        body: dict = {"metrics": metrics or ["faithfulness", "g_eval"]}
+        if queries:
+            body["queries"] = queries
+        if rows:
+            body["rows"] = rows
+        if agent_config:
+            body["agent"] = agent_config
+        if agent_endpoint:
+            body["agent_endpoint"] = agent_endpoint
+        if judge:
+            body["judge"] = judge
+        return await self._request("POST", "/api/v1/playground/evaluate", json_data=body)
+
+    async def playground_status(self) -> dict:
+        """GET /api/v1/playground/status (no auth required)."""
+        async with httpx.AsyncClient(timeout=10) as client:
+            resp = await client.get(f"{self.engine_url}/api/v1/playground/status")
+        if resp.status_code >= 400:
+            return {"used": 0, "limit": 5, "remaining": 5, "resets_at": "midnight UTC"}
+        return resp.json()

aievaluator/cli.py

@@ -0,0 +1,532 @@
+"""AI Evaluator CLI — main entry point.
+
+Commands:
+    aievaluator login      Authenticate with AI Evaluator
+    aievaluator whoami     Show current tenant info
+    aievaluator quick      Quick eval via playground (no API key)
+    aievaluator eval       Full evaluation against an agent
+    aievaluator config     Manage CLI configuration
+"""
+
+import asyncio
+import json as json_mod
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from . import __version__
+from .api.client import APIClient, APIError
+from .config import (
+    resolve_api_key,
+    resolve_engine_url,
+    resolve_default_metrics,
+    resolve_default_min_score,
+    save_config,
+    load_config,
+    get_all_config,
+)
+from .formatters import format_table, format_json_output, format_junit
+
+
+def _parse_dataset_file(file_path: str) -> list[dict]:
+    """Parse a dataset file (JSON or JSONL) into a list of rows."""
+    with open(file_path, "r", encoding="utf-8") as f:
+        raw = f.read()
+
+    if file_path.endswith(".jsonl"):
+        rows = []
+        for line in raw.strip().split("\n"):
+            line = line.strip()
+            if line:
+                rows.append(json_mod.loads(line))
+        return rows
+    else:
+        data = json_mod.loads(raw)
+        return data if isinstance(data, list) else [data]
+
+
+def _run_async(coro):
+    """Helper to run async coroutines from Click commands."""
+    return asyncio.run(coro)
+
+
+def _parse_quick_metrics(metrics_str: str | None, default_threshold: float | None = None) -> list | None:
+    """Parse --metrics for quick command.
+
+    CU1: "faithfulness:0.90,g_eval:0.75" → [{"name":"faithfulness","threshold":0.9}, ...]
+    CU2: "faithfulness,g_eval" with default_threshold=0.8 → [{"name":"faithfulness","threshold":0.8}, ...]
+    Simple: "faithfulness,g_eval" → ["faithfulness", "g_eval"]
+    """
+    if not metrics_str:
+        return None
+    result = []
+    for item in metrics_str.split(","):
+        item = item.strip()
+        if ":" in item:
+            name, val = item.split(":", 1)
+            result.append({"name": name.strip(), "threshold": float(val.strip())})
+        elif default_threshold is not None:
+            result.append({"name": item, "threshold": default_threshold})
+        else:
+            result.append(item)
+    return result
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  CLI Group
+# ═══════════════════════════════════════════════════════════════════
+
+@click.group()
+@click.version_option(version=__version__, prog_name="AI Evaluator CLI")
+def main():
+    """AI Evaluator CLI — evaluate your LLM agents from the command line.
+
+    \b
+    Quick start:
+        aievaluator quick "What is 2+2?" --expected "4"
+        aievaluator login
+        aievaluator eval --agent https://my-agent.com/chat --dataset ./tests.json
+    """
+    pass
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  login
+# ═══════════════════════════════════════════════════════════════════
+
+@main.command()
+@click.option("--api-key", help="API key (non-interactive mode)", default=None)
+@click.option("--engine-url", help="Engine URL", default=None)
+def login(api_key: Optional[str], engine_url: Optional[str]):
+    """Authenticate with AI Evaluator.
+
+    Saves your API key to ~/.config/aievaluator/config.json.
+    Get your key at https://aievaluator.dev/settings
+    """
+    if not api_key:
+        click.echo()
+        click.echo("Enter your AI Evaluator API key:")
+        click.echo("(Get one at https://aievaluator.dev/settings)")
+        api_key = click.prompt("API key", hide_input=False).strip()
+
+    if not api_key:
+        click.echo("❌ API key cannot be empty.", err=True)
+        sys.exit(2)
+
+    resolved_url = resolve_engine_url(engine_url)
+    client = APIClient(resolved_url, api_key)
+
+    async def _login():
+        try:
+            usage = await client.get_usage()
+        except APIError as e:
+            click.echo(f"❌ Invalid API key or engine unreachable: {e.message}", err=True)
+            sys.exit(2)
+
+        # Save to global config
+        config = load_config()
+        config["api_key"] = api_key
+        config["engine_url"] = resolved_url
+        save_config(config)
+
+        tenant_name = usage.get("tenant_name", "Unknown")
+        tier = usage.get("tier", "unknown")
+        evals_used = usage.get("evaluations_this_cycle", 0)
+        evals_limit = usage.get("evaluations_limit", "∞")
+
+        click.echo()
+        click.echo(f"✅ Logged in as {tenant_name} ({tier})")
+        click.echo(f"   Evals: {evals_used}/{evals_limit} this cycle")
+        click.echo(f"   Config saved to ~/.config/aievaluator/config.json")
+
+    _run_async(_login())
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  whoami
+# ═══════════════════════════════════════════════════════════════════
+
+@main.command()
+@click.option("--api-key", help="API key (overrides config)", default=None)
+def whoami(api_key: Optional[str]):
+    """Show current tenant info and usage."""
+    key = resolve_api_key(api_key)
+    if not key:
+        click.echo("❌ Not logged in. Run: aievaluator login", err=True)
+        sys.exit(2)
+
+    engine_url = resolve_engine_url()
+    client = APIClient(engine_url, key)
+
+    async def _whoami():
+        try:
+            usage = await client.get_usage()
+        except APIError as e:
+            click.echo(f"❌ {e.message}", err=True)
+            sys.exit(2)
+
+        tenant_name = usage.get("tenant_name", "Unknown")
+        tier = usage.get("tier", "unknown")
+        evals_used = usage.get("evaluations_this_cycle", 0)
+        evals_limit = usage.get("evaluations_limit", "∞")
+        tokens_in = usage.get("input_tokens_this_cycle", 0)
+        tokens_out = usage.get("output_tokens_this_cycle", 0)
+
+        click.echo()
+        click.echo(f"Tenant:  {tenant_name}")
+        click.echo(f"Tier:    {tier}")
+        click.echo(f"Evals:   {evals_used}/{evals_limit} this cycle")
+        click.echo(f"Tokens:  ↓{tokens_in:,} · ↑{tokens_out:,} this cycle")
+
+    _run_async(_whoami())
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  quick
+# ═══════════════════════════════════════════════════════════════════
+
+@main.command()
+@click.argument("query", required=False)
+@click.option("--dataset", "dataset_file", help="JSON dataset file", type=click.Path(exists=True), default=None)
+@click.option("--agent", "agent_url", help="Agent endpoint URL (default: internal chat agent)", default="/chat")
+@click.option("--expected", help="Expected output for query", default=None)
+@click.option("--metrics", help="Metrics: faithfulness,g_eval or faithfulness:0.90,g_eval:0.75", default=None)
+@click.option("--min-score", help="Apply threshold to all metrics and enforce exit code", type=float, default=None)
+@click.option("--judge", help="LLM judge model", default=None)
+@click.option("--engine-url", help="Engine URL", default=None)
+def quick(query, dataset_file, agent_url, expected, metrics, min_score, judge, engine_url):
+    """Quick evaluation via playground (no API key required).
+
+    \b
+    Examples:
+        aievaluator quick "What is 2+2?" --expected "4"
+        aievaluator quick --dataset ./smoke-tests.json --agent https://my-agent.com/chat
+    """
+    if not query and not dataset_file:
+        click.echo("❌ Provide a query or --dataset", err=True)
+        sys.exit(2)
+    if query and dataset_file:
+        click.echo("❌ Use query OR --dataset, not both", err=True)
+        sys.exit(2)
+
+    resolved_url = resolve_engine_url(engine_url)
+    client = APIClient(resolved_url)
+
+    # Parse metrics: CU1 (metric:threshold), CU2 (--min-score applies to all)
+    metrics_list = _parse_quick_metrics(metrics, min_score)
+
+    async def _quick():
+        # Check playground status first
+        try:
+            status = await client.playground_status()
+        except Exception:
+            status = {"used": 0, "limit": 5, "remaining": 5, "resets_at": "midnight UTC"}
+
+        remaining = status.get("remaining", 5)
+        limit = status.get("limit", 5)
+        click.echo(f"⚠️  Playground mode — {remaining}/{limit} remaining (resets at {status.get('resets_at', 'midnight UTC')})")
+        click.echo()
+
+        if remaining <= 0:
+            click.echo("❌ Playground limit reached. Run `aievaluator login` for 100 free evals/month.")
+            sys.exit(2)
+
+        if query:
+            rows = [{"input": query}]
+            if expected:
+                rows[0]["expected_output"] = expected
+        else:
+            rows = _parse_dataset_file(dataset_file)
+
+        try:
+            result = await client.playground_evaluate(
+                rows=rows,
+                agent_endpoint=agent_url,
+                metrics=metrics_list,
+                judge=judge,
+            )
+        except APIError as e:
+            click.echo(f"❌ {e.message}", err=True)
+            if e.detail:
+                click.echo(json_mod.dumps(e.detail, indent=2), err=True)
+            sys.exit(2)
+
+        overall_passed = all(r.get("passed", True) for r in result.get("results", []))
+        format_table(result, min_score or 0.0, resolved_url)
+
+        # CU2: exit code based on --min-score
+        if min_score is not None:
+            sys.exit(0 if overall_passed else 1)
+
+    _run_async(_quick())
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  eval
+# ═══════════════════════════════════════════════════════════════════
+
+@main.command()
+@click.option("--agent", required=True, help="Agent endpoint URL")
+@click.option("--dataset", "dataset_file", help="JSON dataset file", type=click.Path(exists=True), default=None)
+@click.option("--rows", help="Inline JSON array of test cases", default=None)
+@click.option("--metrics", help="Metrics (comma-separated)", default=None)
+@click.option("--agent-format", help="Agent API format", type=click.Choice(["openai", "claude", "custom"]), default="openai")
+@click.option("--min-score", help="Minimum overall score threshold (0-1)", type=float, default=None)
+@click.option("--thresholds", "thresholds_str", help="Per-metric thresholds: faithfulness:0.90,g_eval:0.75", default=None)
+@click.option("--custom", "custom_str", help="Inline custom evaluator: {\"name\":\"polite\",\"prompt\":\"Check...\",\"threshold\":0.8}", default=None)
+@click.option("--format", "output_format", help="Output format", type=click.Choice(["table", "json", "junit"]), default="table")
+@click.option("--ci", is_flag=True, help="CI mode (no colors, no prompts)")
+@click.option("--timeout", help="Timeout in seconds", type=int, default=300)
+@click.option("--judge-model", help="LLM judge model", default=None)
+@click.option("--name", "eval_name", help="Human-readable name for this evaluation", default=None)
+@click.option("--api-key", help="API key (overrides config)", default=None)
+@click.option("--engine-url", help="Engine URL", default=None)
+def eval_cmd(agent, dataset_file, rows, metrics, agent_format, min_score, thresholds_str, custom_str, output_format, ci, timeout, judge_model, eval_name, api_key, engine_url):
+    """Evaluate an AI agent against a dataset.
+
+    \b
+    Examples:
+        aievaluator eval --agent https://my-agent.com/chat --dataset ./tests.json
+        aievaluator eval --agent https://my-agent.com/chat --rows '[{"input":"Hi","expected_output":"Hello"}]'
+        aievaluator eval --agent $AGENT_URL --dataset ./evals.json --ci --format junit
+    """
+    # Validate data source
+    if not dataset_file and not rows:
+        click.echo("❌ Provide --dataset or --rows", err=True)
+        sys.exit(2)
+    if dataset_file and rows:
+        click.echo("❌ Use --dataset OR --rows, not both", err=True)
+        sys.exit(2)
+
+    key = resolve_api_key(api_key)
+    if not key:
+        click.echo("❌ API key required. Run: aievaluator login", err=True)
+        sys.exit(2)
+
+    resolved_url = resolve_engine_url(engine_url)
+    client = APIClient(resolved_url, key, timeout=timeout)
+
+    # Resolve metrics
+    if metrics:
+        metrics_list = [m.strip() for m in metrics.split(",")]
+    else:
+        metrics_list = resolve_default_metrics().split(",")
+
+    # Resolve min_score
+    if min_score is None:
+        min_score = resolve_default_min_score()
+
+    # Parse per-metric thresholds: "faithfulness:0.90,g_eval:0.75" -> {"faithfulness": 0.90, "g_eval": 0.75}
+    thresholds_dict = {}
+    if thresholds_str:
+        for pair in thresholds_str.split(","):
+            pair = pair.strip()
+            if ":" in pair:
+                metric_name, val = pair.split(":", 1)
+                try:
+                    thresholds_dict[metric_name.strip()] = float(val.strip())
+                except ValueError:
+                    click.echo(f"❌ Invalid threshold value in: {pair}", err=True)
+                    sys.exit(2)
+
+    # CU3: parse inline custom evaluator
+    custom_evaluators = None
+    if custom_str:
+        try:
+            custom_evaluators = json_mod.loads(custom_str)
+            if isinstance(custom_evaluators, dict):
+                custom_evaluators = [custom_evaluators]
+        except json_mod.JSONDecodeError:
+            click.echo(f"❌ Invalid JSON in --custom", err=True)
+            sys.exit(2)
+
+    async def _eval():
+        if dataset_file:
+            try:
+                rows_data = _parse_dataset_file(dataset_file)
+            except (json_mod.JSONDecodeError, FileNotFoundError) as e:
+                click.echo(f"❌ Cannot read dataset: {e}", err=True)
+                sys.exit(2)
+        else:
+            try:
+                rows_data = json_mod.loads(rows)
+            except json_mod.JSONDecodeError as e:
+                click.echo(f"❌ Invalid JSON in --rows: {e}", err=True)
+                sys.exit(2)
+            if not isinstance(rows_data, list):
+                rows_data = [rows_data]
+
+        try:
+            result = await client.evaluate_sync(
+                rows=rows_data,
+                agent_url=agent,
+                agent_format=agent_format,
+                metrics=metrics_list,
+                judge_model=judge_model,
+                name=eval_name,
+                thresholds=thresholds_dict if thresholds_dict else None,
+                custom_evaluators=custom_evaluators,
+            )
+        except APIError as e:
+            _handle_api_error(e)
+
+        # Format output
+        if output_format == "json":
+            output = format_json_output(result, min_score)
+            click.echo(output)
+        elif output_format == "junit":
+            output = format_junit(result, min_score)
+            click.echo(output)
+        else:
+            format_table(result, min_score, resolved_url)
+
+        # Exit code
+        overall_score = result.get("overall_score", 0)
+        if overall_score < min_score:
+            sys.exit(1)
+
+    _run_async(_eval())
+
+
+def _handle_api_error(e: APIError):
+    """Print API error and exit."""
+    click.echo(f"❌ {e.message}", err=True)
+    if e.detail:
+        if isinstance(e.detail, dict):
+            click.echo(json_mod.dumps(e.detail, indent=2), err=True)
+        else:
+            click.echo(str(e.detail)[:500], err=True)
+    sys.exit(3 if e.status_code == 0 else 2)
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  config
+# ═══════════════════════════════════════════════════════════════════
+
+@main.group()
+def config():
+    """Manage CLI configuration."""
+    pass
+
+
+@config.command("show")
+def config_show():
+    """Show current configuration."""
+    cfg = get_all_config()
+    if cfg:
+        click.echo(json_mod.dumps(cfg, indent=2))
+    else:
+        click.echo("No configuration found. Run: aievaluator login")
+
+
+@config.command("set")
+@click.argument("key")
+@click.argument("value")
+def config_set(key: str, value: str):
+    """Set a configuration value.
+
+    \b
+    Keys: engine-url, default-metrics, default-min-score
+    """
+    valid_keys = {"engine-url", "default-metrics", "default-min-score"}
+    if key not in valid_keys:
+        click.echo(f"❌ Invalid key: {key}. Valid keys: {', '.join(valid_keys)}", err=True)
+        sys.exit(2)
+
+    cfg = load_config()
+    if key == "default-min-score":
+        try:
+            cfg[key] = float(value)
+        except ValueError:
+            click.echo(f"❌ default-min-score must be a number (0-1)", err=True)
+            sys.exit(2)
+    else:
+        cfg[key] = value
+    save_config(cfg)
+    click.echo(f"✅ {key} = {value}")
+
+
+@config.command("unset")
+@click.argument("key")
+def config_unset(key: str):
+    """Remove a configuration value."""
+    cfg = load_config()
+    if key in cfg:
+        del cfg[key]
+        save_config(cfg)
+        click.echo(f"✅ {key} removed")
+    else:
+        click.echo(f"{key} was not set")
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  init
+# ═══════════════════════════════════════════════════════════════════
+
+_SMOKE_TEST_DATASET = [
+    {"input": "What is 2+2?", "expected_output": "4"},
+    {"input": "What is the capital of France?", "expected_output": "Paris"},
+    {"input": "Say hello in Spanish", "expected_output": "Hola"},
+]
+
+
+@main.command()
+def init():
+    """Initialize a new AI Evaluator project in the current directory.
+
+    Creates:
+      - aievaluator.config.json (project-local config)
+      - evals/smoke-test.json (example dataset)
+      - Updates .gitignore
+    """
+    cwd = Path.cwd()
+
+    # 1. Create aievaluator.config.json
+    config_path = cwd / "aievaluator.config.json"
+    if config_path.exists():
+        click.echo(f"⏭️  aievaluator.config.json already exists, skipping")
+    else:
+        config_path.write_text(json_mod.dumps({
+            "engine_url": "https://api.aievaluator.dev",
+            "default_metrics": "faithfulness,g_eval",
+            "default_min_score": 0.80,
+        }, indent=2) + "\n")
+        click.echo(f"✅ Created aievaluator.config.json")
+
+    # 2. Create evals/ directory + smoke-test.json
+    evals_dir = cwd / "evals"
+    evals_dir.mkdir(exist_ok=True)
+    smoke_path = evals_dir / "smoke-test.json"
+    if smoke_path.exists():
+        click.echo(f"⏭️  evals/smoke-test.json already exists, skipping")
+    else:
+        smoke_path.write_text(json_mod.dumps(_SMOKE_TEST_DATASET, indent=2) + "\n")
+        click.echo(f"✅ Created evals/smoke-test.json (3 example queries)")
+
+    # 3. Update .gitignore
+    gitignore_path = cwd / ".gitignore"
+    gitignore_lines = gitignore_path.read_text().split("\n") if gitignore_path.exists() else []
+    entry = "aievaluator.config.json"
+    if entry not in gitignore_lines:
+        with open(gitignore_path, "a") as f:
+            if gitignore_lines and gitignore_lines[-1].strip() != "":
+                f.write("\n")
+            f.write(f"{entry}\n")
+        click.echo(f"✅ Added {entry} to .gitignore")
+
+    click.echo()
+    click.echo("Next steps:")
+    click.echo("  aievaluator quick --dataset ./evals/smoke-test.json")
+    click.echo("  aievaluator login    (for 100 free evals/month)")
+    click.echo()
+
+
+# ═══════════════════════════════════════════════════════════════════
+#  Entry point
+# ═══════════════════════════════════════════════════════════════════
+
+if __name__ == "__main__":
+    main()

aievaluator/config.py

@@ -0,0 +1,115 @@
+"""Config manager for AI Evaluator CLI.
+
+Handles API key resolution with priority:
+1. --api-key flag
+2. AIEVALUATOR_API_KEY env var
+3. ./aievaluator.config.json (project-local)
+4. ~/.config/aievaluator/config.json (global)
+"""
+
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+
+def _global_config_path() -> Path:
+    """Returns the global config path, platform-aware."""
+    if os.name == "nt":
+        base = Path(os.environ.get("APPDATA", Path.home() / "AppData" / "Roaming"))
+    elif os.environ.get("XDG_CONFIG_HOME"):
+        base = Path(os.environ["XDG_CONFIG_HOME"])
+    else:
+        base = Path.home() / ".config"
+    return base / "aievaluator" / "config.json"
+
+
+def _load_json(path: Path) -> dict:
+    """Load a JSON file, returning {} if not found or invalid."""
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def _save_json(path: Path, data: dict) -> None:
+    """Save data as JSON, creating parent dirs."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def resolve_api_key(flag_value: Optional[str] = None) -> Optional[str]:
+    """Resolve API key by priority order. Returns None if not found."""
+    if flag_value:
+        return flag_value
+    env_value = os.environ.get("AIEVALUATOR_API_KEY")
+    if env_value:
+        return env_value
+    local = _load_json(Path("aievaluator.config.json"))
+    if local.get("api_key"):
+        return local["api_key"]
+    global_cfg = _load_json(_global_config_path())
+    return global_cfg.get("api_key")
+
+
+def resolve_engine_url(flag_value: Optional[str] = None) -> str:
+    """Resolve engine URL by priority. Defaults to https://api.aievaluator.dev."""
+    default = "https://api.aievaluator.dev"
+
+    if flag_value:
+        return flag_value.rstrip("/")
+
+    env_value = os.environ.get("AIEVALUATOR_ENGINE_URL")
+    if env_value:
+        return env_value.rstrip("/")
+
+    local = _load_json(Path("aievaluator.config.json"))
+    if local.get("engine_url"):
+        return local["engine_url"].rstrip("/")
+
+    global_cfg = _load_json(_global_config_path())
+    if global_cfg.get("engine_url"):
+        return global_cfg["engine_url"].rstrip("/")
+
+    return default
+
+
+def resolve_default_metrics() -> str:
+    """Resolve default metrics from config. Defaults to faithfulness,g_eval."""
+    default = "faithfulness,g_eval"
+    local = _load_json(Path("aievaluator.config.json"))
+    if local.get("default_metrics"):
+        return local["default_metrics"]
+    global_cfg = _load_json(_global_config_path())
+    return global_cfg.get("default_metrics", default)
+
+
+def resolve_default_min_score() -> float:
+    """Resolve default min_score from config. Defaults to 0.0."""
+    local = _load_json(Path("aievaluator.config.json"))
+    if "default_min_score" in local:
+        return float(local["default_min_score"])
+    global_cfg = _load_json(_global_config_path())
+    return float(global_cfg.get("default_min_score", 0.0))
+
+
+def save_config(data: dict, global_: bool = True) -> None:
+    """Save config dict. If global_=False, saves to project-local."""
+    path = _global_config_path() if global_ else Path("aievaluator.config.json")
+    _save_json(path, data)
+
+
+def load_config(global_: bool = True) -> dict:
+    """Load config dict."""
+    path = _global_config_path() if global_ else Path("aievaluator.config.json")
+    return _load_json(path)
+
+
+def get_all_config() -> dict:
+    """Get merged config: global + project-local on top."""
+    global_cfg = _load_json(_global_config_path())
+    local = _load_json(Path("aievaluator.config.json"))
+    merged = {**global_cfg, **local}
+    return merged

aievaluator/formatters/__init__.py

@@ -0,0 +1,7 @@
+"""Formatters for CLI output."""
+
+from .table import format_table
+from .json import format_json_output
+from .junit import format_junit
+
+__all__ = ["format_table", "format_json_output", "format_junit"]

aievaluator/formatters/json.py

@@ -0,0 +1,28 @@
+"""JSON formatter."""
+
+import json as json_mod
+
+
+def format_json_output(data: dict, min_score: float = 0.0) -> str:
+    """Return evaluation results as a JSON string."""
+    results = data.get("results", [])
+    overall_score = data.get("overall_score", 0)
+    total_rows = data.get("total_rows", len(results))
+    failed = sum(1 for r in results if not r.get("passed", True))
+    input_tokens = data.get("input_tokens", 0)
+    output_tokens = data.get("output_tokens", 0)
+    eval_id = data.get("evaluation_id", "")
+
+    output = {
+        "evaluation_id": eval_id,
+        "overall_score": overall_score,
+        "passed": overall_score >= min_score,
+        "min_score": min_score,
+        "total_rows": total_rows,
+        "failed_queries": failed,
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "results": results,
+    }
+
+    return json_mod.dumps(output, indent=2)

aievaluator/formatters/junit.py

@@ -0,0 +1,46 @@
+"""JUnit XML formatter for CI/CD integration."""
+
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+
+
+def format_junit(data: dict, min_score: float = 0.0) -> str:
+    """Return evaluation results as a JUnit XML string."""
+    results = data.get("results", [])
+    total = len(results)
+    failures = sum(1 for r in results if not r.get("passed", True))
+
+    testsuite = ET.Element("testsuite", {
+        "name": "AI Evaluator",
+        "tests": str(total),
+        "failures": str(failures),
+        "errors": "0",
+        "time": "0",
+    })
+
+    for i, r in enumerate(results):
+        query = r.get("query", "")[:80]
+        testcase = ET.SubElement(testsuite, "testcase", {
+            "classname": "AI Evaluator",
+            "name": f"Query {i+1}: {query}",
+            "time": "0",
+        })
+
+        if not r.get("passed", True):
+            scores = r.get("scores", {})
+            scores_str = ", ".join(f"{k}: {v:.2f}" for k, v in scores.items())
+            expected = r.get("expected_output", "") or ""
+            got = r.get("agent_response", "") or ""
+
+            failure_text = (
+                f"Query: {query}\n"
+                f"Expected: {expected}\n"
+                f"Got: {got}\n"
+                f"Scores: {{{scores_str}}}"
+            )
+            ET.SubElement(testcase, "failure", {
+                "message": f"Score below threshold {min_score}",
+            }).text = failure_text
+
+    xml_str = ET.tostring(testsuite, encoding="unicode")
+    return minidom.parseString(xml_str).toprettyxml(indent="  ")

aievaluator/formatters/table.py

@@ -0,0 +1,53 @@
+"""Table formatter using Rich."""
+
+from rich.console import Console
+from rich.table import Table
+
+
+def format_table(data: dict, min_score: float, engine_url: str) -> None:
+    """Print evaluation results as a Rich table."""
+    console = Console()
+    results = data.get("results", [])
+    overall_score = data.get("overall_score", 0)
+    total_rows = data.get("total_rows", len(results))
+    failed = sum(1 for r in results if not r.get("passed", True))
+    input_tokens = data.get("input_tokens", 0)
+    output_tokens = data.get("output_tokens", 0)
+    eval_id = data.get("evaluation_id", "")
+
+    score_pct = overall_score * 100
+    passed = overall_score >= min_score
+    icon = "✅" if passed else "❌"
+
+    console.print()
+    console.print(f"  [bold]AI Evaluator — Results[/bold]")
+    console.print(f"  Overall Score:  [bold]{score_pct:.1f}%[/bold]  {icon} {'above' if passed else 'below'} threshold ({min_score*100:.0f}%)")
+    console.print(f"  Total rows:     {total_rows}")
+    console.print(f"  Failed:         {failed}")
+    console.print(f"  Tokens:         ↓{input_tokens:,} · ↑{output_tokens:,}")
+    if eval_id:
+        console.print(f"  Dashboard:      [link={engine_url}/evaluations/{eval_id}/report]{engine_url}/evaluations/{eval_id}/report[/link]")
+    console.print()
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("#", style="dim", width=4)
+    table.add_column("Query", max_width=50)
+    table.add_column("Score", justify="right", width=8)
+    table.add_column("Pass", justify="center", width=6)
+
+    for i, r in enumerate(results):
+        query = r.get("query", "")[:50]
+        scores = r.get("scores", {})
+        first_score = list(scores.values())[0] if scores else 0
+        score_str = f"{first_score * 100:.0f}%"
+        passed_icon = "✅" if r.get("passed", True) else "❌"
+        table.add_row(str(i + 1), query, score_str, passed_icon)
+
+    console.print(table)
+    console.print()
+
+    if passed:
+        console.print(f"[green]✅ Score {score_pct:.1f}% meets threshold {min_score}[/green]")
+    else:
+        console.print(f"[red]❌ Score {score_pct:.1f}% below threshold {min_score}[/red]")
+    console.print()

aievaluator-1.0.1.dist-info/METADATA

@@ -0,0 +1,366 @@
+Metadata-Version: 2.4
+Name: aievaluator
+Version: 1.0.1
+Summary: AI Evaluator CLI — evaluate your LLM agents from the command line
+Author-email: AI Evaluator <support@aievaluator.dev>
+License: MIT
+Project-URL: Homepage, https://aievaluator.dev
+Project-URL: Repository, https://github.com/aievaluator-dev/aievaluator-cli
+Project-URL: Issues, https://github.com/aievaluator-dev/aievaluator-cli/issues
+Keywords: ai,evaluation,llm,agent,testing,ci-cd
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
+Requires-Dist: pytest-httpx>=0.30; extra == "dev"
+
+# AI Evaluator CLI — Python
+
+[![PyPI](https://img.shields.io/pypi/v/aievaluator)](https://pypi.org/project/aievaluator/)
+[![Python](https://img.shields.io/pypi/pyversions/aievaluator)](https://pypi.org/project/aievaluator/)
+
+Evaluate your LLM agents from the terminal. No browser. No dashboard.
+
+```bash
+pip install aievaluator
+```
+
+---
+
+## 🧭 Tutorial — From Zero to CI/CD
+
+Every step builds on the previous one. Start wherever makes sense for you.
+
+---
+
+### Level 0 — Try it without installing anything
+
+```bash
+curl -s -X POST https://api.aievaluator.dev/api/v1/playground/evaluate \
+  -H "Content-Type: application/json" \
+  -d '{"queries":["What is 2+2?"],"metrics":["faithfulness"]}' | jq .
+```
+
+5 free per day. No key. No install. Good enough to decide if it's useful.
+
+---
+
+### Level 1 — Install and evaluate a single prompt
+
+```bash
+pip install aievaluator
+
+# Ask a question, tell it what you expect
+aievaluator quick "What is the capital of France?" --expected "Paris"
+```
+
+You'll see a table with the score. The `--expected` is optional — without it, the judge evaluates
+the response on its own merits.
+
+```
+⚠️  Playground mode — 4/5 remaining
+
+  AI Evaluator — Results
+  Overall Score:  95.0%  ✅ above threshold (0%)
+  Total rows:     1
+  Failed:         0
+
+┌────┬────────────────────────────────────┬──────────┬──────┐
+│  # │ Query                              │ Score    │ Pass │
+├────┼────────────────────────────────────┼──────────┼──────┤
+│  1 │ What is the capital of France?     │  95%     │ ✅   │
+└────┴────────────────────────────────────┴──────────┴──────┘
+```
+
+---
+
+### Level 2 — Sign up and scaffold a project
+
+Playground is great for trying, but you'll want more than 5 evals/day.
+
+```bash
+# Get your API key at https://aievaluator.dev/settings
+aievaluator login
+
+# Check your account
+aievaluator whoami
+```
+
+Now scaffold your project:
+
+```bash
+aievaluator init
+```
+
+This creates:
+- `aievaluator.config.json` — project-local config
+- `evals/smoke-test.json` — sample dataset with 3 queries
+- Updates `.gitignore`
+
+Open `evals/smoke-test.json` and replace the sample queries with your own:
+
+```json
+[
+  {"input": "What are your business hours?", "expected_output": "Mon-Fri 9am-6pm"},
+  {"input": "How do I cancel my order?", "expected_output": "Go to My Orders → Cancel"},
+  {"input": "Do you ship internationally?", "expected_output": "Yes, via DHL Express"}
+]
+```
+
+Test it against the built-in agent:
+
+```bash
+aievaluator quick --dataset ./evals/smoke-test.json
+```
+
+---
+
+### Level 3 — Evaluate your own agent
+
+Point the CLI at your agent's endpoint:
+
+```bash
+aievaluator eval \
+  --agent https://chatbot-staging.acme.com/api/chat \
+  --dataset ./evals/smoke-test.json \
+  --metrics faithfulness,g_eval
+```
+
+The CLI calls your agent with each query, then an LLM judge scores the responses.
+
+---
+
+### Level 4 — Add quality gates
+
+Not all metrics are equally important. Set different thresholds per metric:
+
+```bash
+aievaluator eval \
+  --agent https://chatbot-staging.acme.com/api/chat \
+  --dataset ./evals/smoke-test.json \
+  --thresholds faithfulness:0.90,g_eval:0.75
+```
+
+- `faithfulness` must be ≥ 90% (hallucination = instant fail)
+- `g_eval` must be ≥ 75% (general quality)
+
+If any metric fails to meet its threshold, that row is marked ❌.
+
+**Or set one bar for everything:**
+
+```bash
+aievaluator eval \
+  --agent https://chatbot-staging.acme.com/api/chat \
+  --dataset ./evals/smoke-test.json \
+  --min-score 0.80
+```
+
+This works on `quick` too:
+
+```bash
+aievaluator quick "test prompt" --min-score 0.80
+# Exit code 1 if any metric drops below 0.80
+```
+
+---
+
+### Level 5 — Create your own evaluation criteria
+
+Sometimes the built-in metrics aren't enough. Define a custom evaluator inline:
+
+```bash
+aievaluator eval \
+  --agent https://chatbot-staging.acme.com/api/chat \
+  --dataset ./evals/smoke-test.json \
+  --metrics politeness,g_eval \
+  --custom '{"name":"politeness","prompt":"Is the response polite and professional? Answer YES or NO and explain.","threshold":0.85}'
+```
+
+The custom evaluator `politeness` is defined in the request, referenced in `--metrics` by name,
+and evaluated alongside `g_eval`. No dashboard needed.
+
+**Custom evaluator with per-metric threshold override:**
+
+```bash
+aievaluator eval \
+  --agent $URL --dataset ./tests.json \
+  --metrics politeness,g_eval \
+  --custom '{"name":"politeness","prompt":"Is the tone friendly?","threshold":0.7}' \
+  --thresholds politeness:0.90,g_eval:0.80
+```
+
+The `--thresholds` flag overrides whatever was set in `--custom`. The engine uses the
+per-evaluation value.
+
+---
+
+### Level 6 — CI/CD pipeline
+
+Add this to your GitHub Actions, GitLab CI, or Jenkins:
+
+```bash
+aievaluator eval \
+  --agent $STAGING_AGENT \
+  --dataset ./evals/regression.json \
+  --thresholds faithfulness:0.90,g_eval:0.75 \
+  --min-score 0.80 \
+  --ci \
+  --format junit > report.xml
+```
+
+| Flag | What it does |
+|---|---|
+| `--ci` | No colors, no prompts — clean output for logs |
+| `--format junit` | JUnit XML that CI systems understand natively |
+| `--min-score 0.80` | Overall score must be ≥ 80% |
+| `--thresholds` | Per-metric quality bars |
+
+Exit code 1 = pipeline fails = deploy blocked.
+
+**Environment variables for CI:**
+
+```bash
+export AIEVALUATOR_API_KEY="sk-..."       # No hardcoded keys in YAML
+export AIEVALUATOR_ENGINE_URL="https://api.aievaluator.dev"
+```
+
+---
+
+## 📋 Complete Command Reference
+
+### `aievaluator login`
+
+```bash
+aievaluator login                        # Interactive prompt
+aievaluator login --api-key sk-xxx       # Non-interactive (CI)
+aievaluator login --engine-url https://custom.engine.com
+```
+
+### `aievaluator whoami`
+
+```bash
+aievaluator whoami
+# Tenant:  acme-corp
+# Tier:    pro
+# Evals:   42/5000 this cycle
+# Tokens:  ↓124,800 · ↑89,200 this cycle
+```
+
+### `aievaluator quick`
+
+```bash
+# Single query
+aievaluator quick "What is 2+2?" --expected "4"
+
+# Per-metric thresholds
+aievaluator quick "test" --metrics faithfulness:0.90,g_eval:0.75
+
+# General threshold
+aievaluator quick "test" --min-score 0.80
+
+# From dataset (JSON or JSONL)
+aievaluator quick --dataset ./tests.json
+aievaluator quick --dataset ./tests.jsonl
+
+# Custom judge model
+aievaluator quick "test" --judge deepseek
+```
+
+### `aievaluator eval`
+
+```bash
+# Basic
+aievaluator eval --agent $URL --dataset ./tests.json
+
+# With quality gates
+aievaluator eval --agent $URL --dataset ./tests.json \
+  --thresholds faithfulness:0.90,g_eval:0.75 --min-score 0.80
+
+# Inline rows
+aievaluator eval --agent $URL \
+  --rows '[{"input":"Hi","expected_output":"Hello"}]'
+
+# Custom evaluator inline
+aievaluator eval --agent $URL --dataset ./tests.json \
+  --metrics my-eval --custom '{"name":"my-eval","prompt":"...","threshold":0.8}'
+
+# CI mode
+aievaluator eval --agent $URL --dataset ./tests.json --ci --format junit
+
+# Different agent format
+aievaluator eval --agent $URL --dataset ./tests.json --agent-format claude
+```
+
+### `aievaluator config`
+
+```bash
+aievaluator config show
+aievaluator config set default-metrics "faithfulness,g_eval"
+aievaluator config set default-min-score 0.80
+aievaluator config unset default-min-score
+```
+
+### `aievaluator init`
+
+```bash
+aievaluator init
+# Creates aievaluator.config.json + evals/smoke-test.json + updates .gitignore
+```
+
+---
+
+## 📊 Output Formats
+
+### Table (default)
+
+Human-readable table with scores, pass/fail icons, and token counts.
+
+### JSON (`--format json`)
+
+```bash
+aievaluator eval ... --format json | jq '.overall_score'
+```
+
+Clean JSON on stdout. All logs/warnings go to stderr.
+
+### JUnit XML (`--format junit`)
+
+```bash
+aievaluator eval ... --format junit > report.xml
+```
+
+Native CI integration. `<testcase>` per query, `<failure>` for queries below threshold.
+
+---
+
+## 🤖 VS Code Extension
+
+Prefer staying in your editor? Install the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=aievaluator.aievaluator).
+
+- Select text → right-click → Evaluate
+- Per-metric threshold editor with preset buttons
+- Custom evaluator support via Command Palette
+- Sidebar with evaluation history
+- Dataset file evaluation (JSON + JSONL)
+
+[Full VS Code tutorial →](../vscode/README.md)
+
+---
+
+## Requirements
+
+- Python 3.10+

aievaluator-1.0.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+aievaluator/__init__.py,sha256=Qx3qP1jslumH1wgtyTm2pGCVIu0-RZRbeWzFn3GzSIU,98
+aievaluator/cli.py,sha256=o7gC1yfIuqUf0gYVJSrf2ap1oBE2UmbF8Xx3r5RRLUc,21339
+aievaluator/config.py,sha256=5hm0wwEebZhrsbzV-nM0YpsyqpgV9ILe5c3DlYAE_r0,3755
+aievaluator/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aievaluator/api/client.py,sha256=dG-uH_TiL-mSv5PKDmQN5kEscdnyn-f5ZZeiRjYhgAQ,6423
+aievaluator/formatters/__init__.py,sha256=mLZD18yR0ioyxPDwnkzVy8wsG9vULw-BqNsQUfJPxpA,201
+aievaluator/formatters/json.py,sha256=MQ681J8CVtRzldfmWw6MlwbU7jqIKrDIXJk2gHm8F2I,910
+aievaluator/formatters/junit.py,sha256=iwDbnYdDdr87UWtPGLUArQMYyBb6zfuQ_OM1pgccBB0,1552
+aievaluator/formatters/table.py,sha256=OXou3cuev2PRZSQcC4MTU0l8qllY2j-kTzOvsGKEpkY,2197
+aievaluator-1.0.1.dist-info/METADATA,sha256=OsVHIgncs6cxs3WT7iV3tXvX88YWxxSNEbd77tuPfsE,9885
+aievaluator-1.0.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+aievaluator-1.0.1.dist-info/entry_points.txt,sha256=9h3s8XUIa2RZ9m-2Bh3e78elTimvVglfU7LvA9gxY4k,53
+aievaluator-1.0.1.dist-info/top_level.txt,sha256=IVDZHg6brLn7wBoRdLH_mbjntqKoEppiF_dZ_VVLL2E,12
+aievaluator-1.0.1.dist-info/RECORD,,

aievaluator-1.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

aievaluator-1.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+aievaluator = aievaluator.cli:main

aievaluator-1.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+aievaluator