agentevals-cli 0.5.2__py3-none-any.whl

agentevals/evaluator/resolver.py

@@ -0,0 +1,70 @@
+"""Resolve remote evaluator references to local cached files."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from .sources import EvaluatorSource, get_sources
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_CACHE_DIR = Path.home() / ".cache" / "agentevals" / "evaluators"
+
+
+class EvaluatorResolver:
+    """Downloads and caches remote evaluators, converting them to local paths."""
+
+    def __init__(self, cache_dir: Path | None = None):
+        self._cache_dir = cache_dir or _DEFAULT_CACHE_DIR
+        self._sources: dict[str, EvaluatorSource] = {}
+
+    def register_source(self, source: EvaluatorSource) -> None:
+        self._sources[source.source_name] = source
+
+    async def resolve(self, evaluator_def) -> "CodeEvaluatorDef":  # noqa: F821
+        """Download a remote evaluator and return a CodeEvaluatorDef pointing to the cached file."""
+        from ..config import CodeEvaluatorDef, RemoteEvaluatorDef
+
+        if not isinstance(evaluator_def, RemoteEvaluatorDef):
+            raise TypeError(f"Expected RemoteEvaluatorDef, got {type(evaluator_def).__name__}")
+
+        source = self._sources.get(evaluator_def.source)
+        if source is None:
+            raise ValueError(
+                f"Unknown evaluator source '{evaluator_def.source}'. Available: {sorted(self._sources.keys())}"
+            )
+
+        dest = self._cache_dir / evaluator_def.source / evaluator_def.ref
+        if not dest.exists():
+            logger.info(
+                "Downloading evaluator '%s' from %s (ref: %s)",
+                evaluator_def.name,
+                evaluator_def.source,
+                evaluator_def.ref,
+            )
+            await source.fetch_evaluator(evaluator_def.ref, dest)
+        else:
+            logger.debug("Using cached evaluator '%s' at %s", evaluator_def.name, dest)
+
+        return CodeEvaluatorDef(
+            name=evaluator_def.name,
+            path=str(dest),
+            threshold=evaluator_def.threshold,
+            timeout=evaluator_def.timeout,
+            config=evaluator_def.config,
+            executor=evaluator_def.executor,
+        )
+
+
+_default_resolver: EvaluatorResolver | None = None
+
+
+def get_default_resolver() -> EvaluatorResolver:
+    """Return a lazily-initialized resolver with all registered sources."""
+    global _default_resolver
+    if _default_resolver is None:
+        _default_resolver = EvaluatorResolver()
+        for source in get_sources():
+            _default_resolver.register_source(source)
+    return _default_resolver

agentevals/evaluator/sources.py

@@ -0,0 +1,293 @@
+"""Evaluator source backends: discover and fetch evaluators from various registries."""
+
+from __future__ import annotations
+
+import abc
+import json
+import logging
+import os
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_REPO = "agentevals-dev/evaluators"
+_DEFAULT_BRANCH = "main"
+_DEFAULT_INDEX = "index.yaml"
+
+
+@dataclass
+class EvaluatorInfo:
+    """Metadata for a single evaluator, regardless of where it comes from."""
+
+    name: str
+    description: str
+    source: str
+    language: str | None = None
+    ref: str | None = None
+    tags: list[str] = field(default_factory=list)
+    author: str | None = None
+    last_updated: str | None = None
+
+
+class EvaluatorSource(abc.ABC):
+    """Registry backend that can list and fetch evaluators."""
+
+    @property
+    @abc.abstractmethod
+    def source_name(self) -> str: ...
+
+    @abc.abstractmethod
+    async def list_evaluators(self) -> list[EvaluatorInfo]: ...
+
+    @abc.abstractmethod
+    async def fetch_evaluator(self, ref: str, dest: Path) -> Path:
+        """Download an evaluator identified by *ref* and write it to *dest*.
+
+        Returns the path to the downloaded file.
+        """
+
+
+_CACHE_TTL_SECONDS = 86400  # 24 hours
+
+
+def _cache_dir() -> Path:
+    base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+    d = base / "agentevals"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _read_cache(key: str, ttl: int = _CACHE_TTL_SECONDS) -> list[EvaluatorInfo] | None:
+    cache_file = _cache_dir() / f"{key}.json"
+    if not cache_file.exists():
+        return None
+    try:
+        data = json.loads(cache_file.read_text())
+        if time.time() - data.get("ts", 0) > ttl:
+            return None
+        return [EvaluatorInfo(**item) for item in data["evaluators"]]
+    except Exception:
+        return None
+
+
+def _write_cache(key: str, evaluators: list[EvaluatorInfo]) -> None:
+    cache_file = _cache_dir() / f"{key}.json"
+    try:
+        cache_file.write_text(
+            json.dumps(
+                {
+                    "ts": time.time(),
+                    "evaluators": [asdict(g) for g in evaluators],
+                }
+            )
+        )
+    except Exception:
+        pass
+
+
+class BuiltinEvaluatorSource(EvaluatorSource):
+    """Wraps ADK's built-in metric registry as an evaluator source."""
+
+    @property
+    def source_name(self) -> str:
+        return "builtin"
+
+    async def list_evaluators(self) -> list[EvaluatorInfo]:
+        import asyncio
+        import warnings
+
+        cached = _read_cache("builtin")
+        if cached is not None:
+            return cached
+
+        def _load() -> list[EvaluatorInfo]:
+            infos: list[EvaluatorInfo] = []
+            try:
+                with warnings.catch_warnings():
+                    warnings.simplefilter("ignore")
+                    from google.adk.evaluation.metric_evaluator_registry import (
+                        DEFAULT_METRIC_EVALUATOR_REGISTRY,
+                    )
+
+                for m in DEFAULT_METRIC_EVALUATOR_REGISTRY.get_registered_metrics():
+                    infos.append(
+                        EvaluatorInfo(
+                            name=m.metric_name,
+                            description=m.description or "No description",
+                            source=self.source_name,
+                            language=None,
+                            ref=None,
+                        )
+                    )
+            except ImportError:
+                from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+
+                for pm in PrebuiltMetrics:
+                    infos.append(
+                        EvaluatorInfo(
+                            name=pm.value,
+                            description="(install google-adk[eval] for full details)",
+                            source=self.source_name,
+                        )
+                    )
+            return infos
+
+        result = await asyncio.to_thread(_load)
+        _write_cache("builtin", result)
+        return result
+
+    async def fetch_evaluator(self, ref: str, dest: Path) -> Path:
+        raise NotImplementedError("Built-in evaluators are part of ADK and cannot be fetched as files.")
+
+
+class GitHubEvaluatorSource(EvaluatorSource):
+    """Fetches evaluators from a GitHub repository with a CI-generated index.yaml."""
+
+    def __init__(
+        self,
+        repo: str | None = None,
+        branch: str | None = None,
+        index_path: str | None = None,
+        token: str | None = None,
+    ):
+        self._repo = repo or os.environ.get("AGENTEVALS_EVALUATOR_REPO", _DEFAULT_REPO)
+        self._branch = branch or os.environ.get("AGENTEVALS_EVALUATOR_BRANCH", _DEFAULT_BRANCH)
+        self._index_path = index_path or _DEFAULT_INDEX
+        self._token = token or os.environ.get("AGENTEVALS_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN")
+
+    @property
+    def source_name(self) -> str:
+        return "github"
+
+    def _raw_url(self, path: str) -> str:
+        return f"https://raw.githubusercontent.com/{self._repo}/{self._branch}/{path}"
+
+    def _headers(self) -> dict[str, str]:
+        if self._token:
+            return {"Authorization": f"token {self._token}"}
+        return {}
+
+    async def list_evaluators(self) -> list[EvaluatorInfo]:
+        import httpx
+
+        url = self._raw_url(self._index_path)
+        logger.debug("Fetching evaluator index from %s", url)
+
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(url, headers=self._headers(), timeout=15)
+                resp.raise_for_status()
+        except httpx.HTTPError as exc:
+            logger.warning("Failed to fetch evaluator index from %s: %s", url, exc)
+            return []
+
+        data = yaml.safe_load(resp.text)
+        if not isinstance(data, dict):
+            logger.warning("Evaluator index at %s is not a valid YAML mapping", url)
+            return []
+
+        infos: list[EvaluatorInfo] = []
+        for entry in data.get("evaluators", []):
+            infos.append(
+                EvaluatorInfo(
+                    name=entry.get("name", "unknown"),
+                    description=entry.get("description", ""),
+                    source=self.source_name,
+                    language=entry.get("language"),
+                    ref=entry.get("path"),
+                    tags=entry.get("tags", []),
+                    author=entry.get("author"),
+                    last_updated=entry.get("lastUpdated"),
+                )
+            )
+        return infos
+
+    async def fetch_evaluator(self, ref: str, dest: Path) -> Path:
+        import httpx
+
+        url = self._raw_url(ref)
+        logger.info("Downloading evaluator from %s", url)
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url, headers=self._headers(), timeout=30)
+            resp.raise_for_status()
+
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(resp.text, encoding="utf-8")  # noqa: ASYNC240
+        return dest
+
+
+class FileEvaluatorSource(EvaluatorSource):
+    """Reads evaluators from a local index.yaml file (same schema as GitHubEvaluatorSource).
+
+    Useful for testing and local development.  Not registered in the default
+    source list — instantiate directly when needed.
+    """
+
+    def __init__(self, path: Path):
+        self._path = Path(path)
+        if not self._path.exists():
+            raise FileNotFoundError(f"Evaluator index file not found: {self._path}")
+
+    @property
+    def source_name(self) -> str:
+        return "file"
+
+    async def list_evaluators(self) -> list[EvaluatorInfo]:
+        data = yaml.safe_load(self._path.read_text(encoding="utf-8"))
+        if not isinstance(data, dict):
+            logger.warning("Evaluator index at %s is not a valid YAML mapping", self._path)
+            return []
+
+        infos: list[EvaluatorInfo] = []
+        for entry in data.get("evaluators", []):
+            infos.append(
+                EvaluatorInfo(
+                    name=entry.get("name", "unknown"),
+                    description=entry.get("description", ""),
+                    source=self.source_name,
+                    language=entry.get("language"),
+                    ref=entry.get("path"),
+                    tags=entry.get("tags", []),
+                    author=entry.get("author"),
+                    last_updated=entry.get("lastUpdated"),
+                )
+            )
+        return infos
+
+    async def fetch_evaluator(self, ref: str, dest: Path) -> Path:
+        src = (self._path.parent / ref).resolve()
+        if not src.exists():
+            raise FileNotFoundError(f"Evaluator file not found: {src} (ref: {ref}, index: {self._path})")
+        import shutil
+
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+        return dest
+
+
+# ---------------------------------------------------------------------------
+# Source registry
+# ---------------------------------------------------------------------------
+
+_SOURCES: list[EvaluatorSource] | None = None
+
+
+def get_sources() -> list[EvaluatorSource]:
+    """Return all registered evaluator sources (lazily initialized)."""
+    global _SOURCES
+    if _SOURCES is None:
+        _SOURCES = [
+            BuiltinEvaluatorSource(),
+            GitHubEvaluatorSource(),
+        ]
+    return _SOURCES
+
+
+def register_source(source: EvaluatorSource) -> None:
+    """Add a custom evaluator source to the registry."""
+    get_sources().append(source)

agentevals/evaluator/templates.py

@@ -0,0 +1,224 @@
+"""Evaluator scaffolding templates and the scaffold_evaluator function."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from string import Template
+
+import yaml
+
+PYTHON_TEMPLATE = Template('''\
+"""Custom evaluator: ${name}
+
+Usage in eval_config.yaml:
+
+    evaluators:
+      - name: ${name}
+        type: code
+        path: ./${name}/${name}.py
+        threshold: 0.5
+"""
+
+from agentevals_evaluator_sdk import evaluator, EvalInput, EvalResult
+
+
+@evaluator
+def ${name}(input: EvalInput) -> EvalResult:
+    scores: list[float] = []
+
+    for inv in input.invocations:
+        score = 1.0
+
+        if not inv.final_response:
+            score = 0.0
+            scores.append(score)
+            continue
+
+        # TODO: implement your scoring logic here
+
+        scores.append(max(0.0, score))
+
+    overall = sum(scores) / len(scores) if scores else 0.0
+    return EvalResult(
+        score=overall,
+        per_invocation_scores=scores,
+    )
+
+
+if __name__ == "__main__":
+    ${name}.run()
+''')
+
+
+JAVASCRIPT_TEMPLATE = Template("""\
+/**
+ * Custom evaluator: ${name}
+ *
+ * Usage in eval_config.yaml:
+ *
+ *     evaluators:
+ *       - name: ${name}
+ *         type: code
+ *         path: ./${name}/${name}.js
+ *         threshold: 0.5
+ */
+
+const input = JSON.parse(require("fs").readFileSync("/dev/stdin", "utf8"));
+
+const scores = [];
+
+for (const inv of input.invocations) {
+  let score = 1.0;
+
+  if (!inv.final_response) {
+    scores.push(0.0);
+    continue;
+  }
+
+  // TODO: implement your scoring logic here
+
+  scores.push(Math.max(0.0, score));
+}
+
+const overall = scores.length > 0
+  ? scores.reduce((a, b) => a + b, 0) / scores.length
+  : 0.0;
+
+console.log(JSON.stringify({
+  score: overall,
+  per_invocation_scores: scores,
+}));
+""")
+
+
+TYPESCRIPT_TEMPLATE = Template("""\
+/**
+ * Custom evaluator: ${name}
+ *
+ * Usage in eval_config.yaml:
+ *
+ *     evaluators:
+ *       - name: ${name}
+ *         type: code
+ *         path: ./${name}/${name}.ts
+ *         threshold: 0.5
+ */
+
+import * as fs from "fs";
+
+interface IntermediateSteps {
+  tool_calls: { name: string; args: Record<string, unknown> }[];
+  tool_responses: { name: string; output: string }[];
+}
+
+interface Invocation {
+  invocation_id: string;
+  user_content: string;
+  final_response: string | null;
+  intermediate_steps: IntermediateSteps;
+}
+
+interface EvalInput {
+  protocol_version: string;
+  metric_name: string;
+  threshold: number;
+  config: Record<string, unknown>;
+  invocations: Invocation[];
+  expected_invocations: Invocation[] | null;
+}
+
+const input: EvalInput = JSON.parse(fs.readFileSync("/dev/stdin", "utf8"));
+
+const scores: number[] = [];
+
+for (const inv of input.invocations) {
+  let score = 1.0;
+
+  if (!inv.final_response) {
+    scores.push(0.0);
+    continue;
+  }
+
+  // TODO: implement your scoring logic here
+
+  scores.push(Math.max(0.0, score));
+}
+
+const overall = scores.length > 0
+  ? scores.reduce((a, b) => a + b, 0) / scores.length
+  : 0.0;
+
+console.log(JSON.stringify({
+  score: overall,
+  per_invocation_scores: scores,
+}));
+""")
+
+
+_EXTENSION_TO_TEMPLATE: dict[str, Template] = {
+    ".py": PYTHON_TEMPLATE,
+    ".js": JAVASCRIPT_TEMPLATE,
+    ".ts": TYPESCRIPT_TEMPLATE,
+}
+
+_RUNTIME_ALIAS_TO_EXT: dict[str, str] = {
+    "py": ".py",
+    "python": ".py",
+    "js": ".js",
+    "javascript": ".js",
+    "ts": ".ts",
+    "typescript": ".ts",
+}
+
+_EXT_TO_LANGUAGE: dict[str, str] = {
+    ".py": "python",
+    ".js": "javascript",
+    ".ts": "typescript",
+}
+
+
+def scaffold_evaluator(
+    name: str,
+    output_dir: Path | None = None,
+    runtime: str | None = None,
+) -> Path:
+    """Create a new evaluator directory with code file and evaluator.yaml manifest.
+
+    Returns the path to the created directory.
+    """
+    output_dir = output_dir or Path.cwd()
+
+    raw_path = Path(name)
+    suffix = raw_path.suffix.lower()
+    evaluator_name = raw_path.stem
+
+    if suffix and suffix in _EXTENSION_TO_TEMPLATE:
+        ext = suffix
+    elif runtime:
+        ext = _RUNTIME_ALIAS_TO_EXT.get(runtime.lower())
+        if ext is None:
+            raise ValueError(f"Unknown runtime '{runtime}'. Supported: {sorted(_RUNTIME_ALIAS_TO_EXT.keys())}")
+    else:
+        ext = ".py"
+
+    template = _EXTENSION_TO_TEMPLATE[ext]
+    language = _EXT_TO_LANGUAGE[ext]
+
+    evaluator_dir = output_dir / evaluator_name
+    evaluator_dir.mkdir(parents=True, exist_ok=True)
+
+    code_file = evaluator_dir / f"{evaluator_name}{ext}"
+    code_file.write_text(template.substitute(name=evaluator_name), encoding="utf-8")
+
+    manifest = {
+        "name": evaluator_name,
+        "description": f"TODO: describe what {evaluator_name} evaluates",
+        "language": language,
+        "entrypoint": f"{evaluator_name}{ext}",
+        "tags": [],
+        "author": "",
+    }
+    manifest_file = evaluator_dir / "evaluator.yaml"
+    manifest_file.write_text(yaml.dump(manifest, sort_keys=False), encoding="utf-8")
+
+    return evaluator_dir