fastevals 0.1.0__py3-none-any.whl

fastevals/__init__.py

@@ -0,0 +1,26 @@
+"""Fast, provider-agnostic LLM evaluation toolkit."""
+
+from .config import SUPPORTED_PROVIDERS, ModelSpec, RunConfig
+from .exceptions import ConfigError, FastEvalError, ProviderError, StructuredOutputError
+from .models import ModelResponse, RunResult
+from .registry import load_registry
+from .report import save_report
+from .runner import run
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "SUPPORTED_PROVIDERS",
+    "ConfigError",
+    "FastEvalError",
+    "ModelResponse",
+    "ModelSpec",
+    "ProviderError",
+    "RunConfig",
+    "RunResult",
+    "StructuredOutputError",
+    "__version__",
+    "load_registry",
+    "run",
+    "save_report",
+]

fastevals/cli.py

@@ -0,0 +1,125 @@
+"""Command-line interface for fastevals."""
+
+import argparse
+import asyncio
+import json
+import os
+from pathlib import Path
+
+from .config import DEFAULT_MAX_CONCURRENCY, SUPPORTED_PROVIDERS, RunConfig
+from .exceptions import FastEvalError
+from .report import save_report
+from .runner import run
+from .structured import shorthand_to_schema
+
+ALL_PROVIDERS = "all"
+
+
+def _dotenv_candidates() -> list[Path]:
+    return [Path.cwd() / ".env", Path(__file__).resolve().parents[1] / ".env"]
+
+
+def _load_dotenv() -> None:
+    """Load simple KEY=VALUE entries from a project .env if present."""
+    for env_path in _dotenv_candidates():
+        if not env_path.exists():
+            continue
+        for raw_line in env_path.read_text().splitlines():
+            line = raw_line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            key, value = line.split("=", 1)
+            key = key.strip()
+            value = value.strip().strip('"').strip("'")
+            if key and value and key not in os.environ:
+                os.environ[key] = value
+
+
+def _parse_providers(raw: str) -> frozenset[str]:
+    providers = {item.strip().lower() for item in raw.split("|") if item.strip()}
+    unknown = sorted(providers - set(SUPPORTED_PROVIDERS) - {ALL_PROVIDERS})
+    if unknown:
+        supported = ", ".join((*SUPPORTED_PROVIDERS, ALL_PROVIDERS))
+        raise argparse.ArgumentTypeError(f"Unknown provider(s): {', '.join(unknown)}. Supported: {supported}")
+    return frozenset(providers)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fastevals",
+        description="Compare one task results across LLM providers and models.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""Examples:
+  fastevals --prompt \"Summarize this\" --providers \"openai|gemini\" --out runs
+  fastevals --image image.png --prompt \"Find widget bboxes\" \\
+    --structured-output \"x:int(X coord),y:int(Y coord),width:int(Width),height:int(Height)\" \\
+    --providers openai
+  fastevals --dataset cases.jsonl --nruns 3 --providers openai --out runs/dataset
+
+""",
+    )
+    parser.add_argument("-p", "--prompt", help="Task prompt (omit when --dataset provides the prompts)")
+    parser.add_argument("-s", "--structured-output", help="Structured output compact schema for the response")
+    parser.add_argument("-f", "--file", type=Path, help="Input document (sent to the model as an attachment)")
+    parser.add_argument("-i", "--image", type=Path, help="Input image")
+    parser.add_argument(
+        "-pr",
+        "--providers",
+        type=_parse_providers,
+        default=frozenset({ALL_PROVIDERS}),
+        help=f"Pipe-separated providers: {'|'.join(SUPPORTED_PROVIDERS)}|all (default: all)",
+    )
+    parser.add_argument(
+        "-r", "--registry", type=Path, help="Path to the model registry TOML (default: config/models.toml)"
+    )
+    parser.add_argument(
+        "-d",
+        "--dataset",
+        type=Path,
+        help="JSONL or CSV file with evaluation cases (columns: prompt, expected, evaluator, pattern)",
+    )
+    parser.add_argument(
+        "-n", "--nruns", type=int, default=1, help="Repeat every case this many times for consistency checks"
+    )
+    parser.add_argument(
+        "-c", "--concurrency", type=int, default=DEFAULT_MAX_CONCURRENCY, help="Max parallel model calls"
+    )
+    parser.add_argument("-o", "--out", type=Path, default=Path("runs"), help="Output directory")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    _load_dotenv()
+    args = build_parser().parse_args(argv)
+
+    try:
+        config = RunConfig(
+            prompt=args.prompt or "",
+            providers=args.providers,
+            file=str(args.file) if args.file else None,
+            image=str(args.image) if args.image else None,
+            structured_output=shorthand_to_schema(args.structured_output) if args.structured_output else None,
+            dataset=str(args.dataset) if args.dataset else None,
+            nruns=max(1, args.nruns),
+            registry=str(args.registry) if args.registry else None,
+            max_concurrency=max(1, args.concurrency),
+            out=str(args.out),
+        )
+        results = asyncio.run(run(config))
+    except (FastEvalError, ValueError) as exc:
+        print(json.dumps({"ok": False, "error": str(exc), "results": []}, ensure_ascii=False))
+        return 1
+
+    json_path, html_path = save_report(config, results, args.out)
+    payload = {
+        "ok": all(row.ok for row in results),
+        "json_path": str(json_path),
+        "html_path": str(html_path) if html_path else None,
+        "results": [row.as_dict() for row in results],
+    }
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+    return 0 if payload["ok"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fastevals/config.py

@@ -0,0 +1,130 @@
+"""Typed configuration objects for runs and model registry entries."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .exceptions import ConfigError
+
+__all__ = [
+    "ALL_PROVIDERS",
+    "DEFAULT_MAX_CONCURRENCY",
+    "DEFAULT_TIMEOUT_S",
+    "SUPPORTED_PROVIDERS",
+    "ModelSpec",
+    "RunConfig",
+]
+
+ALL_PROVIDERS = "all"
+SUPPORTED_PROVIDERS = ("openai", "gemini", "openrouter")
+DEFAULT_TIMEOUT_S = 120
+DEFAULT_MAX_CONCURRENCY = 4
+DEFAULT_OUT_DIR = "runs"
+MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024
+
+_KNOWN_SPEC_KEYS = frozenset(
+    {
+        "id",
+        "provider",
+        "model",
+        "api_key_env",
+        "reasoning_effort",
+        "reasoning_efforts",
+        "reasoning_parameter",
+        "input_cost_usd_per_mtok",
+        "cached_input_cost_usd_per_mtok",
+        "cached_cost_usd_per_mtok",
+        "cached_write_cost_usd_per_mtok",
+        "output_cost_usd_per_mtok",
+        "reasoning_cost_usd_per_mtok",
+        "timeout_s",
+    }
+)
+
+
+@dataclass(frozen=True)
+class RunConfig:
+    """Everything needed to execute one evaluation run."""
+
+    prompt: str
+    providers: frozenset[str] = frozenset({ALL_PROVIDERS})
+    file: str | None = None
+    image: str | None = None
+    structured_output: dict[str, Any] | None = None
+    dataset: str | None = None
+    nruns: int = 1
+    registry: str | None = None
+    max_concurrency: int = DEFAULT_MAX_CONCURRENCY
+    out: str = DEFAULT_OUT_DIR
+
+    def __post_init__(self) -> None:
+        if not self.prompt.strip() and not self.dataset:
+            raise ConfigError("Prompt must not be empty when no dataset is given")
+        if self.max_concurrency < 1:
+            raise ConfigError(f"max_concurrency must be >= 1, got {self.max_concurrency}")
+        if self.nruns < 1:
+            raise ConfigError(f"nruns must be >= 1, got {self.nruns}")
+        for label, path in (("file", self.file), ("image", self.image), ("dataset", self.dataset)):
+            if path and not Path(path).exists():
+                raise ConfigError(f"{label} not found: {path}")
+        unknown = self.requested_providers() - set(SUPPORTED_PROVIDERS) - {ALL_PROVIDERS}
+        if unknown:
+            supported = ", ".join((*SUPPORTED_PROVIDERS, ALL_PROVIDERS))
+            raise ConfigError(f"Unknown provider(s): {', '.join(sorted(unknown))}. Supported: {supported}")
+        if self.structured_output is not None:
+            schema = self.structured_output
+            if not isinstance(schema, dict) or "properties" not in schema:
+                raise ConfigError("structured_output must be a JSON Schema object with 'properties'")
+
+    def requested_providers(self) -> set[str]:
+        return {provider.lower() for provider in self.providers}
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    """One concrete provider/model/reasoning combination from the registry."""
+
+    id: str
+    provider: str
+    model: str
+    api_key_env: str | None = None
+    reasoning_effort: str = "off"
+    reasoning_parameter: str | None = None
+    input_cost_usd_per_mtok: float | None = None
+    cached_input_cost_usd_per_mtok: float | None = None
+    cached_cost_usd_per_mtok: float | None = None
+    cached_write_cost_usd_per_mtok: float | None = None
+    output_cost_usd_per_mtok: float | None = None
+    reasoning_cost_usd_per_mtok: float | None = None
+    timeout_s: int = DEFAULT_TIMEOUT_S
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any], spec_id: str) -> "ModelSpec":
+        unknown = set(raw) - _KNOWN_SPEC_KEYS - {"type"}
+        if unknown:
+            raise ConfigError(f"Unknown key(s) in registry entry '{spec_id}': {', '.join(sorted(unknown))}")
+        if not isinstance(raw.get("model"), str) or not raw["model"]:
+            raise ConfigError(f"Registry entry '{spec_id}' is missing a valid 'model'")
+        provider = raw.get("provider", spec_id.split(":", 1)[0])
+        if not provider:
+            raise ConfigError(f"Registry entry '{spec_id}' is missing a 'provider'")
+        fields: dict[str, Any] = {"id": spec_id, "provider": str(provider).lower(), "model": raw["model"]}
+        for key in (
+            "api_key_env",
+            "reasoning_effort",
+            "reasoning_parameter",
+            "input_cost_usd_per_mtok",
+            "cached_input_cost_usd_per_mtok",
+            "cached_cost_usd_per_mtok",
+            "cached_write_cost_usd_per_mtok",
+            "output_cost_usd_per_mtok",
+            "reasoning_cost_usd_per_mtok",
+        ):
+            if raw.get(key) is not None:
+                fields[key] = raw[key]
+        if raw.get("timeout_s") is not None:
+            try:
+                fields["timeout_s"] = max(1, int(raw["timeout_s"]))
+            except (TypeError, ValueError) as exc:
+                raise ConfigError(f"Registry entry '{spec_id}' has invalid timeout_s: {raw['timeout_s']!r}") from exc
+        return cls(**fields)

fastevals/dataset.py

@@ -0,0 +1,74 @@
+"""Dataset loading: JSONL and CSV evaluation cases."""
+
+import csv
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from .exceptions import ConfigError
+
+__all__ = ["Case", "load_dataset"]
+
+_REQUIRED_FIELDS = frozenset({"prompt"})
+
+
+@dataclass(frozen=True)
+class Case:
+    """One evaluation input with optional scoring instructions."""
+
+    id: str
+    prompt: str
+    expected: str | None = None
+    evaluator: str | None = None
+    pattern: str | None = None
+
+    def as_dict(self) -> dict[str, str | None]:
+        return {
+            "id": self.id,
+            "prompt": self.prompt,
+            "expected": self.expected,
+            "evaluator": self.evaluator,
+            "pattern": self.pattern,
+        }
+
+
+def _make_case(index: int, row: dict[str, str]) -> Case:
+    missing = _REQUIRED_FIELDS - {key for key, value in row.items() if value}
+    if missing:
+        raise ConfigError(f"Dataset row {index + 1} is missing required field(s): {', '.join(sorted(missing))}")
+    case_id = str(row.get("id") or f"case-{index + 1:03d}")
+    return Case(
+        id=case_id,
+        prompt=str(row["prompt"]).strip(),
+        expected=(str(row["expected"]) if row.get("expected") else None),
+        evaluator=(str(row["evaluator"]).strip() or None) if row.get("evaluator") else None,
+        pattern=row.get("pattern") or None,
+    )
+
+
+def load_dataset(path: str | Path) -> list[Case]:
+    """Load cases from a ``.jsonl`` or ``.csv`` file."""
+    path = Path(path)
+    suffix = path.suffix.lower()
+    rows: list[dict[str, str]]
+    if suffix == ".jsonl":
+        rows = []
+        for line_number, line in enumerate(path.read_text().splitlines(), start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                item = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ConfigError(f"Dataset line {line_number} is not valid JSON: {exc.msg}") from exc
+            if not isinstance(item, dict):
+                raise ConfigError(f"Dataset line {line_number} must be a JSON object")
+            rows.append({key: "" if value is None else str(value) for key, value in item.items()})
+    elif suffix == ".csv":
+        with path.open(newline="") as dataset_file:
+            rows = [{key: (value or "") for key, value in row.items()} for row in csv.DictReader(dataset_file)]
+    else:
+        raise ConfigError(f"Unsupported dataset format '{suffix}'. Use .jsonl or .csv")
+    if not rows:
+        raise ConfigError(f"Dataset is empty: {path}")
+    return [_make_case(index, row) for index, row in enumerate(rows)]

fastevals/evaluators.py

@@ -0,0 +1,54 @@
+"""Output evaluators: deterministic scoring for evaluation cases."""
+
+import json
+import re
+from typing import Any
+
+from .dataset import Case
+from .exceptions import ConfigError
+
+__all__ = ["evaluate_output"]
+
+EVALUATORS = ("exact_match", "contains", "json_valid", "regex")
+
+
+def _as_text(output: Any) -> str:
+    if isinstance(output, (dict, list)):
+        return json.dumps(output, ensure_ascii=False)
+    return str(output or "")
+
+
+def evaluate_output(case: Case, output: Any) -> dict[str, Any]:
+    """Score one output against the case instructions.
+
+    Returns ``{"evaluator", "passed", "detail"}``; ``passed`` is ``None``
+    when the case defines no evaluator.
+    """
+    if not case.evaluator:
+        return {"evaluator": None, "passed": None, "detail": None}
+
+    name = case.evaluator.strip().lower()
+    text = _as_text(output)
+    if name == "exact_match":
+        passed = text.strip() == (case.expected or "").strip()
+        detail = None if passed else f"expected {case.expected!r}, got {text.strip()[:200]!r}"
+    elif name == "contains":
+        needle = case.expected or ""
+        passed = bool(needle) and needle in text
+        detail = None if passed else f"{needle!r} not found in output"
+    elif name == "json_valid":
+        try:
+            json.loads(text)
+            passed, detail = True, None
+        except json.JSONDecodeError as exc:
+            passed, detail = False, f"output is not valid JSON: {exc.msg}"
+    elif name == "regex":
+        if not case.pattern:
+            raise ConfigError(f"Case '{case.id}' uses the regex evaluator but defines no pattern")
+        match = re.search(case.pattern, text)
+        passed, detail = match is not None, None if match else f"pattern {case.pattern!r} not found"
+    else:
+        raise ConfigError(
+            f"Unknown evaluator '{case.evaluator}' for case '{case.id}'. Supported: {', '.join(EVALUATORS)}"
+        )
+    return {"evaluator": name, "passed": passed, "detail": detail}

fastevals/exceptions.py

@@ -0,0 +1,19 @@
+"""Exception hierarchy for fastevals."""
+
+__all__ = ["ConfigError", "FastEvalError", "ProviderError", "StructuredOutputError"]
+
+
+class FastEvalError(Exception):
+    """Base class for all fastevals errors."""
+
+
+class ConfigError(FastEvalError):
+    """Invalid configuration, registry, or provider selection."""
+
+
+class ProviderError(FastEvalError):
+    """A model call failed: missing credentials, network, or provider error."""
+
+
+class StructuredOutputError(FastEvalError):
+    """Model output did not satisfy the requested JSON Schema."""

fastevals/mcp_server.py

@@ -0,0 +1,157 @@
+"""MCP server exposing fastevals to AI assistants.
+
+Run locally with ``fastevals-mcp`` (stdio transport) and register it from any
+MCP client, e.g. Claude Desktop or Claude Code.
+"""
+
+import asyncio
+from pathlib import Path
+from typing import Any
+
+from mcp.server.mcpserver.server import MCPServer
+
+from .config import ALL_PROVIDERS, SUPPORTED_PROVIDERS, RunConfig
+from .exceptions import FastEvalError
+from .registry import default_registry_path, load_registry
+from .report import save_report
+from .runner import run
+from .structured import shorthand_to_schema
+
+__all__ = ["build_server", "main"]
+
+mcp = MCPServer(
+    name="fastevals",
+    instructions=(
+        "fastevals runs one prompt across a matrix of LLM models and providers, "
+        "saves every response and returns a comparison summary with cost, "
+        "latency and token metrics."
+    ),
+)
+
+
+@mcp.tool()
+async def run_evaluation(
+    prompt: str = "",
+    providers: str = ALL_PROVIDERS,
+    structured_output: str | None = None,
+    dataset: str | None = None,
+    file: str | None = None,
+    image: str | None = None,
+    nruns: int = 1,
+    out: str = "runs",
+) -> dict[str, Any]:
+    """Run an evaluation matrix and save a JSON + HTML report.
+
+    Args:
+        prompt: The task prompt (omit when ``dataset`` supplies prompts).
+        providers: Pipe-separated provider list, e.g. ``openai|openrouter`` or ``all``.
+        structured_output: Optional compact schema like ``name:str,age:int``.
+        dataset: Optional JSONL/CSV path with cases (prompt, expected, evaluator, pattern).
+        file: Optional document attachment (image, PDF or text file).
+        image: Optional image attachment.
+        nruns: Repeat every case this many times for consistency checks.
+        out: Directory where reports are written.
+    """
+    try:
+        schema = shorthand_to_schema(structured_output) if structured_output else None
+        config = RunConfig(
+            prompt=prompt,
+            providers=frozenset(part.strip().lower() for part in providers.split("|") if part.strip()),
+            structured_output=schema,
+            dataset=dataset,
+            file=file,
+            image=image,
+            nruns=max(1, nruns),
+            out=out,
+        )
+        results = await run(config)
+        json_path, html_path = save_report(config, results, out)
+    except FastEvalError as exc:
+        return {"ok": False, "error": str(exc)}
+    return {
+        "ok": all(row.ok for row in results),
+        "json_path": str(json_path),
+        "html_path": str(html_path),
+        "total_cost_usd": sum(row.total_cost_usd or 0 for row in results),
+        "results": [
+            {
+                "case_id": row.case_id,
+                "provider": row.provider,
+                "model": row.model,
+                "reasoning_effort": row.reasoning_effort,
+                "latency_ms": row.latency_ms,
+                "total_cost_usd": row.total_cost_usd,
+                "output": row.output,
+                "error": row.error or None,
+                "evaluation": row.evaluation,
+            }
+            for row in results
+        ],
+    }
+
+
+@mcp.tool()
+def list_models(registry: str | None = None) -> dict[str, Any]:
+    """List models available in the fastevals registry.
+
+    Args:
+        registry: Optional path to an alternative TOML registry.
+    """
+    path = Path(registry) if registry else default_registry_path()
+    if not path or not Path(path).exists():
+        return {"models": [], "registry": None}
+    try:
+        entries = load_registry(path)
+    except FastEvalError as exc:
+        return {"ok": False, "error": str(exc), "models": []}
+    return {
+        "registry": str(path),
+        "supported_providers": list(SUPPORTED_PROVIDERS),
+        "models": [
+            {
+                "id": model_id,
+                "provider": entry.get("provider"),
+                "model": entry.get("model"),
+                "reasoning_efforts": entry.get("reasoning_efforts", entry.get("reasoning_effort", "off")),
+                "input_cost_usd_per_mtok": entry.get("input_cost_usd_per_mtok"),
+                "output_cost_usd_per_mtok": entry.get("output_cost_usd_per_mtok"),
+            }
+            for model_id, entry in entries.items()
+        ],
+    }
+
+
+@mcp.tool()
+def get_run(json_path: str) -> dict[str, Any]:
+    """Summarize a saved fastevals run from its ``run.json`` file."""
+    path = Path(json_path)
+    if not path.exists():
+        return {"ok": False, "error": f"Run file not found: {json_path}"}
+    import json
+
+    payload = json.loads(path.read_text())
+    results: list[dict[str, Any]] = payload.get("results", [])
+    scored = [row for row in results if (row.get("evaluation") or {}).get("passed") is not None]
+    return {
+        "ok": bool(results) and all(not row.get("error") for row in results),
+        "created_at": payload.get("created_at"),
+        "runs": len(results),
+        "errors": sum(1 for row in results if row.get("error")),
+        "pass_rate": (sum(1 for row in scored if row["evaluation"]["passed"]) / len(scored) if scored else None),
+        "total_cost_usd": sum(row.get("total_cost_usd") or 0 for row in results),
+        "html_report": str(path.parent / "report.html"),
+    }
+
+
+def build_server() -> MCPServer:
+    """Return the configured MCP server instance."""
+    return mcp
+
+
+def main() -> int:
+    asyncio.run(mcp.run_stdio_async())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fastevals/models.py

@@ -0,0 +1,80 @@
+"""Core result models shared across the package."""
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+__all__ = ["ModelResponse", "RunResult"]
+
+
+@dataclass
+class ModelResponse:
+    """Normalized single-model response returned by provider adapters."""
+
+    text: str
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    reasoning_tokens: int | None = None
+    cached_tokens: int | None = None
+    finish_reason: str | None = None
+    response_id: str | None = None
+
+
+@dataclass
+class RunResult:
+    """Outcome of one cell in the evaluation matrix.
+
+    Token buckets are disjoint: ``input_tokens`` excludes cached tokens and
+    ``output_tokens`` excludes reasoning tokens, so each bucket is billed at
+    most once.
+    """
+
+    provider: str
+    model: str
+    reasoning_effort: str
+
+    output: Any
+
+    case_id: str = "case-001"
+    attempt: int = 1
+    evaluation: dict[str, Any] | None = field(default=None)
+
+    time_to_first_token_ms: float | None = None
+    latency_ms: float | None = None
+
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    reasoning_tokens: int | None = None
+    cached_tokens: int | None = None
+
+    input_cost_usd: float | None = None
+    output_cost_usd: float | None = None
+    reasoning_cost_usd: float | None = None
+    cached_cost_usd: float | None = None
+
+    tokens_per_second: float | None = None
+
+    error: str | None = None
+    finish_reason: str | None = None
+    response_id: str | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        """Serialize including computed convenience fields."""
+        data: dict[str, Any] = asdict(self)
+        data["ok"] = self.ok
+        data["total_cost_usd"] = self.total_cost_usd
+        return data
+
+    @property
+    def ok(self) -> bool:
+        return not self.error
+
+    @property
+    def total_cost_usd(self) -> float | None:
+        costs = (
+            self.input_cost_usd,
+            self.output_cost_usd,
+            self.reasoning_cost_usd,
+            self.cached_cost_usd,
+        )
+        known_costs = [cost for cost in costs if cost is not None]
+        return sum(known_costs) if known_costs else None