agentevals-cli 0.9.2__tar.gz → 0.9.4__tar.gz

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/.github/workflows/release.yml

@@ -121,6 +121,7 @@ jobs:
         run: |
           VERSION="${TAG#v}"
           make build-docker \
+            VERSION="$VERSION" \
             DOCKER_REGISTRY="ghcr.io/${{ github.repository_owner }}" \
             DOCKER_TAG="$VERSION"
         env:

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/Dockerfile

@@ -24,7 +24,14 @@ COPY src ./src
 
 COPY --from=ui /build/ui/dist ./src/agentevals/_static
 
-RUN uv sync --frozen --no-dev --extra live --extra postgres \
+# hatch-vcs reads the version from .git, which the docker build context omits.
+# Pass it in as a build arg. We use the generic SETUPTOOLS_SCM_PRETEND_VERSION
+# because hatch-vcs does not forward dist_name to setuptools-scm, so the
+# per-package SETUPTOOLS_SCM_PRETEND_VERSION_FOR_<DIST> form is never consulted.
+ARG VERSION
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}
+
+RUN uv sync --frozen --no-dev --extra live --extra postgres --extra kubernetes \
     && groupadd --gid 1000 app \
     && useradd --uid 1000 --gid app --home-dir /app --no-log-init app \
     && chown -R app:app /app

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/Makefile

@@ -28,7 +28,8 @@ build:
 	uv build
 
 build-docker:
-	docker buildx build --platform $(PLATFORMS) -t $(DOCKER_IMAGE_REF):$(DOCKER_TAG) --push .
+	@test -n "$(VERSION)" || { echo "ERROR: VERSION is empty. Pass VERSION=x.y.z explicitly, or install uv so hatch-vcs can resolve it."; exit 1; }
+	docker buildx build --platform $(PLATFORMS) --build-arg VERSION=$(VERSION) -t $(DOCKER_IMAGE_REF):$(DOCKER_TAG) --push .
 
 build-ui:
 	cd ui && npm ci && npm run build

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: agentevals-cli
-Version: 0.9.2
+Version: 0.9.4
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11
 Requires-Dist: click>=8.0
 Requires-Dist: fastapi>=0.115.0
-Requires-Dist: google-adk[eval]>=1.30.0
+Requires-Dist: google-adk[eval]<2.2,>=2.1.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: opentelemetry-proto>=1.36.0
 Requires-Dist: python-dotenv>=1.0.0
@@ -14,6 +14,8 @@ Requires-Dist: python-multipart>=0.0.12
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: uvicorn[standard]>=0.32.0
+Provides-Extra: kubernetes
+Requires-Dist: kubernetes>=36.0.0; extra == 'kubernetes'
 Provides-Extra: live
 Requires-Dist: httpx>=0.27.0; extra == 'live'
 Requires-Dist: mcp>=1.26.0; extra == 'live'

agentevals_cli-0.9.4/charts/agentevals/templates/rbac.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "agentevals.fullname" . }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+    {{- with .Values.rbac.secretNames }}
+    resourceNames:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "agentevals.fullname" . }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "agentevals.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "agentevals.serviceAccountName" . }}
+    namespace: {{ include "agentevals.namespace" . }}
+{{- end }}

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/charts/agentevals/values.yaml

@@ -57,6 +57,20 @@ serviceAccount:
   # -- ServiceAccount name override
   name: ""
 
+# ==============================================================================
+# RBAC
+# ==============================================================================
+
+# -- Namespaced Role + RoleBinding granting the pod's ServiceAccount read
+# access to Secrets. Enable this when the kubernetes secret resolver reads
+# provider credentials from Secrets via in-cluster config.
+rbac:
+  # -- Create the Role and RoleBinding
+  create: false
+  # -- Restrict the Role to these Secret names. Empty grants get on all
+  # Secrets in the release namespace.
+  secretNames: []
+
 # ==============================================================================
 # Pod
 # ==============================================================================

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/examples/zero-code-examples/adk/run.py

@@ -74,7 +74,7 @@ async def main():
 
         agent_response = ""
         async for event in runner.run_async(user_id=user_id, session_id=session.id, new_message=content):
-            if event.content.parts and event.content.parts[0].text:
+            if event.content and event.content.parts and event.content.parts[0].text:
                 agent_response = event.content.parts[0].text
 
         print(f"     Agent: {agent_response}")

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/pyproject.toml

@@ -9,7 +9,7 @@ description = "Standalone framework to evaluate agent correctness based on porta
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "google-adk[eval]>=1.30.0",
+    "google-adk[eval]>=2.1.0,<2.2",
     "click>=8.0",
     "tabulate>=0.9.0",
     "fastapi>=0.115.0",
@@ -36,10 +36,17 @@ openai = [
 postgres = [
     "asyncpg>=0.30.0",
 ]
+kubernetes = [
+    "kubernetes>=36.0.0",
+]
 
 [project.scripts]
 agentevals = "agentevals.cli:main"
 
+[project.entry-points."agentevals.secret_resolvers"]
+env = "agentevals.resolvers:create_env_resolver"
+kubernetes = "agentevals.resolvers.kubernetes:create_kubernetes_resolver"
+
 [tool.hatch.version]
 source = "vcs"
 

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/src/agentevals/builtin_metrics.py

@@ -27,6 +27,8 @@ from google.adk.evaluation.eval_metrics import (
 from google.adk.evaluation.eval_rubrics import Rubric, RubricContent
 from google.adk.evaluation.evaluator import EvaluationResult, Evaluator
 
+from .resolvers import get_resolved_credential
+
 logger = logging.getLogger(__name__)
 
 METRICS_NEEDING_EXPECTED = {
@@ -267,6 +269,67 @@ def get_evaluator(eval_metric: EvalMetric) -> Evaluator:
     return DEFAULT_METRIC_EVALUATOR_REGISTRY.get_evaluator(eval_metric)
 
 
+def _build_judge_model(model_id: str, api_key: str, base_url: str | None = None):
+    """Build a judge ``BaseLlm`` carrying *api_key* directly, instead of reading it from env.
+
+    LiteLlm-backed providers take ``api_key`` (and optional ``base_url``) as constructor
+    kwargs that forward into every ``litellm.acompletion`` call. The Gemini-native model
+    class takes no ``api_key``; its cached ``google.genai`` client is replaced with one
+    built from the resolved key.
+
+    Routing is by ADK's ``LLMRegistry`` class resolution, which is authoritative: the
+    evaluator already resolved this same *model_id* to a model class when ``_setup_auto_rater``
+    ran at construction, so this lookup cannot disagree or fail here.
+    """
+    from google.adk.models.lite_llm import LiteLlm
+    from google.adk.models.registry import LLMRegistry
+
+    if issubclass(LLMRegistry().resolve(model_id), LiteLlm):
+        kwargs: dict[str, Any] = {"api_key": api_key}
+        if base_url:
+            kwargs["base_url"] = base_url
+        return LiteLlm(model=model_id, **kwargs)
+
+    from google.adk.models.google_llm import Gemini
+    from google.genai import Client
+    from google.genai import types as genai_types
+
+    model = Gemini(model=model_id)
+    client_kwargs: dict[str, Any] = {"api_key": api_key}
+    if base_url:
+        client_kwargs["http_options"] = genai_types.HttpOptions(base_url=base_url)
+    # api_client is a functools.cached_property that memoizes into the instance __dict__;
+    # seeding that slot pre-empts the lazily-built client so the judge uses the resolved key.
+    model.__dict__["api_client"] = Client(**client_kwargs)
+    return model
+
+
+def _inject_judge_credential(evaluator: Evaluator, api_key: str, base_url: str | None = None) -> None:
+    """Replace a judge evaluator's auto-rater model with one built from *api_key*.
+
+    Keyed on the ADK private seam (``_judge_model_options`` / ``_judge_model``, set by
+    ``LlmAsJudge._setup_auto_rater``) rather than on a class, so this single path covers
+    ``FinalResponseMatchV2Evaluator``, the ``rubric_based_*_v1`` evaluators, and
+    ``HallucinationsV1Evaluator`` (which exposes the same attributes without subclassing
+    ``LlmAsJudge``). ``get_evaluator`` returns a fresh instance per evaluation, so mutating
+    it here carries no shared state and is safe across concurrent runs.
+
+    TODO(upstream): propose that ADK ``JudgeModelOptions`` carry a credential or a prebuilt
+    model instance, so judge auth no longer depends on this private seam or process env.
+    """
+    opts = getattr(evaluator, "_judge_model_options", None)
+    if opts is None or not hasattr(evaluator, "_judge_model"):
+        logger.warning("evaluator %s is not judge-backed; cannot inject credential", type(evaluator).__name__)
+        return
+    model_id = getattr(opts, "judge_model", None)
+    if not model_id:
+        logger.warning(
+            "evaluator %s has no resolved judge_model; skipping credential injection", type(evaluator).__name__
+        )
+        return
+    evaluator._judge_model = _build_judge_model(model_id, api_key, base_url)
+
+
 def extract_trajectory_details(eval_result: EvaluationResult) -> dict[str, Any]:
     """Extract expected vs actual tool call details from trajectory evaluation."""
     comparisons = []
@@ -305,6 +368,8 @@ async def evaluate_builtin_metric(
     judge_model: str | None,
     threshold: float | None,
     match_type: str | None = None,
+    credential_ref: str | None = None,
+    judge_base_url: str | None = None,
 ) -> dict[str, Any]:
     """Evaluate a single built-in ADK metric.
 
@@ -326,6 +391,18 @@ async def evaluate_builtin_metric(
         eval_metric = build_eval_metric(metric_name, judge_model, threshold, match_type=match_type)
         evaluator: Evaluator = get_evaluator(eval_metric)
 
+        if credential_ref:
+            api_key = get_resolved_credential(credential_ref)
+            if api_key is None:
+                return MetricResult(
+                    metric_name=metric_name,
+                    error=(
+                        f"Metric '{metric_name}' references credential '{credential_ref}', "
+                        f"which was not provided in the run's credentialRefs."
+                    ),
+                )
+            _inject_judge_credential(evaluator, api_key, judge_base_url)
+
         if metric_name in _METRICS_NEEDING_INVOCATION_EVENTS:
             actual_invocations = _enrich_app_details([_to_invocation_events(inv) for inv in actual_invocations])
             if expected_invocations is not None:

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/src/agentevals/config.py

@@ -27,6 +27,14 @@ class BuiltinMetricDef(BaseModel):
     threshold: float | None = Field(default=None, ge=0, le=1)
     judge_model: str | None = None
     trajectory_match_type: str | None = None
+    credential_ref: str | None = Field(
+        default=None,
+        description="Logical name of a RunSpec.credential_refs entry whose resolved value is the judge API key.",
+    )
+    judge_base_url: str | None = Field(
+        default=None,
+        description="Optional base URL for the judge endpoint (e.g. an OpenAI-compatible proxy).",
+    )
 
     @field_validator("trajectory_match_type")
     @classmethod

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/src/agentevals/custom_evaluators.py

@@ -453,6 +453,8 @@ async def evaluate_custom_evaluator(
             judge_model=evaluator_def.judge_model,
             threshold=evaluator_def.threshold,
             match_type=evaluator_def.trajectory_match_type,
+            credential_ref=evaluator_def.credential_ref,
+            judge_base_url=evaluator_def.judge_base_url,
         )
 
     if isinstance(evaluator_def, OpenAIEvalDef):

agentevals_cli-0.9.4/src/agentevals/resolvers/__init__.py

@@ -0,0 +1,167 @@
+"""Secret resolvers — a generic, pluggable layer for resolving secret references.
+
+A host attaches *secret references* to a run (``RunSpec.credential_refs``); each
+reference is a ``dict`` with a ``kind`` plus kind-specific locator fields. At run
+time the worker resolves every reference once to its secret value and stashes the
+``logical-name -> value`` map in a :class:`contextvars.ContextVar` scoped to that
+run's asyncio task. Consumers (e.g. judge construction) read the value they need
+with no ``os.environ`` mutation and no shared state across concurrently running
+evaluations.
+
+This layer is deliberately consumer-agnostic: a resolver turns a reference into a
+secret value and nothing more. How that value is used — which provider it
+authenticates, what base URL it pairs with — is the consumer's concern, configured
+where the consumer is built (for judges, on the evaluator definition).
+
+**Plugins:** third-party packages declare setuptools entry points in group
+``agentevals.secret_resolvers`` (entry **name** = ``kind`` string; **value** =
+``module:factory`` callable ``factory(spec: dict) -> SecretResolver``). The
+zero-dependency ``env`` resolver ships with agentevals through this same group so
+the discovery path is exercised in OSS. Hosts may replace any kind via
+:func:`register_resolver_factory` (highest precedence).
+
+Tests may call :func:`clear_resolver_plugin_registry` to drop programmatic
+registrations.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable
+from contextvars import ContextVar, Token
+from importlib.metadata import entry_points
+from typing import Any, Protocol, cast
+
+logger = logging.getLogger(__name__)
+
+SECRET_RESOLVER_ENTRY_POINT_GROUP = "agentevals.secret_resolvers"
+
+
+class SecretResolver(Protocol):
+    async def resolve(self, ref: dict[str, Any]) -> str: ...
+
+
+SecretResolverFactory = Callable[[dict[str, Any]], SecretResolver]
+
+_PLUGIN_FACTORIES: dict[str, SecretResolverFactory] = {}
+
+
+class EnvSecretResolver:
+    """Resolve ``{"kind": "env", "name": "OPENAI_API_KEY"}`` from ``os.environ``."""
+
+    async def resolve(self, ref: dict[str, Any]) -> str:
+        name = ref.get("name")
+        if not name:
+            raise ValueError("env secret reference requires a 'name' field")
+        value = os.environ.get(name)
+        if value is None:
+            raise ValueError(f"environment variable {name!r} is not set")
+        return value
+
+
+def create_env_resolver(spec: dict[str, Any]) -> EnvSecretResolver:
+    return EnvSecretResolver()
+
+
+def register_resolver_factory(kind: str, factory: SecretResolverFactory) -> None:
+    """Register or replace the factory for ``kind`` (overrides built-ins and entry points).
+
+    Call during process startup before run workers consume specs. The factory receives
+    the full reference dict (including ``kind``) and returns a :class:`SecretResolver`.
+    """
+    _PLUGIN_FACTORIES[kind] = factory
+
+
+def clear_resolver_plugin_registry() -> None:
+    """Drop all registrations from :func:`register_resolver_factory` (for tests)."""
+    _PLUGIN_FACTORIES.clear()
+
+
+def _builtin_factories() -> dict[str, SecretResolverFactory]:
+    """No hardcoded built-ins: ``env`` ships via the entry-point group."""
+    return {}
+
+
+def _merge_resolver_factories() -> dict[str, SecretResolverFactory]:
+    """Built-ins, then entry points (no built-in shadowing), then programmatic overrides."""
+    merged: dict[str, SecretResolverFactory] = dict(_builtin_factories())
+    eps = entry_points(group=SECRET_RESOLVER_ENTRY_POINT_GROUP)
+    for ep in eps:
+        if ep.name in merged:
+            logger.debug("skipping resolver entry point %r; built-in kind takes precedence", ep.name)
+            continue
+        try:
+            loaded = ep.load()
+            if not callable(loaded):
+                logger.warning("resolver entry point %r is not callable; skipping", ep.name)
+                continue
+            merged[ep.name] = cast(SecretResolverFactory, loaded)
+        except Exception:
+            logger.exception("failed to load resolver entry point %r", ep.name)
+    merged.update(_PLUGIN_FACTORIES)
+    return merged
+
+
+def registered_resolver_kinds() -> tuple[str, ...]:
+    """Sorted resolver ``kind`` strings that would resolve if :func:`build_resolver` ran now.
+
+    Includes built-ins, successfully loaded setuptools entry points for group
+    :data:`SECRET_RESOLVER_ENTRY_POINT_GROUP`, and registrations from
+    :func:`register_resolver_factory`.
+    """
+    return tuple(sorted(_merge_resolver_factories().keys()))
+
+
+def build_resolver(ref: dict[str, Any]) -> SecretResolver:
+    """Construct the :class:`SecretResolver` for a reference's ``kind``.
+
+    Factory lookup starts from built-ins, adds setuptools entry points (group
+    ``agentevals.secret_resolvers``) for ``kind`` names not already built-in, then
+    applies :func:`register_resolver_factory` registrations, which override any prior
+    factory for the same ``kind``.
+    """
+    kind = ref.get("kind")
+    if not kind:
+        raise ValueError("secret reference is missing a 'kind' field")
+    factories = _merge_resolver_factories()
+    factory = factories.get(kind)
+    if factory is None:
+        raise ValueError(
+            f"unknown secret resolver kind '{kind}'. Available: {', '.join(sorted(factories)) or '(none)'}"
+        )
+    return factory(ref)
+
+
+async def resolve_credential_refs(refs: dict[str, dict[str, Any]]) -> dict[str, str]:
+    """Resolve every ``logical-name -> reference`` entry to its secret value.
+
+    Each resolver reads only its own kind-specific locator fields. Any non-locator
+    fields a host puts on a reference are ignored here; consumer-specific config
+    belongs with the consumer (for judges, on the evaluator definition).
+    """
+    resolved: dict[str, str] = {}
+    for logical_name, ref in refs.items():
+        resolver = build_resolver(ref)
+        resolved[logical_name] = await resolver.resolve(ref)
+    return resolved
+
+
+_RESOLVED: ContextVar[dict[str, str] | None] = ContextVar("agentevals_resolved_credentials", default=None)
+
+
+def set_resolved_credentials(mapping: dict[str, str]) -> Token:
+    """Scope a ``logical-name -> secret value`` map to the current asyncio task. Returns a reset token."""
+    return _RESOLVED.set(mapping)
+
+
+def reset_resolved_credentials(token: Token) -> None:
+    _RESOLVED.reset(token)
+
+
+def get_resolved_credential(logical_name: str) -> str | None:
+    """Look up a secret value resolved for the current run, or ``None`` if absent."""
+    mapping = _RESOLVED.get()
+    if not mapping:
+        return None
+    return mapping.get(logical_name)

agentevals_cli-0.9.4/src/agentevals/resolvers/kubernetes.py

@@ -0,0 +1,62 @@
+"""Kubernetes Secret resolver — an optional :class:`SecretResolver` plugin.
+
+Resolves a reference of the form ``{"kind": "kubernetes", "namespace": ..., "name": ...,
+"key": ...}`` by reading the named Secret and base64-decoding the requested key. Ships
+behind the ``kubernetes`` extra and is wired through the ``agentevals.secret_resolvers``
+entry-point group; the ``kubernetes`` package is imported lazily inside the factory so
+installing agentevals without the extra never breaks import or plugin discovery.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesSecretResolver:
+    """Reads a key out of a Kubernetes Secret via a ``CoreV1Api`` client."""
+
+    def __init__(self, core_v1_api: Any) -> None:
+        self._core_v1 = core_v1_api
+
+    async def resolve(self, ref: dict[str, Any]) -> str:
+        namespace = ref.get("namespace")
+        name = ref.get("name")
+        key = ref.get("key")
+        missing = [f for f, v in (("namespace", namespace), ("name", name), ("key", key)) if not v]
+        if missing:
+            raise ValueError(f"kubernetes secret reference is missing required field(s): {', '.join(missing)}")
+
+        secret = await asyncio.to_thread(self._core_v1.read_namespaced_secret, name, namespace)
+        data = secret.data or {}
+        if key not in data:
+            available = ", ".join(sorted(data)) or "(none)"
+            raise ValueError(f"key '{key}' not found in Secret {namespace}/{name}; available keys: {available}")
+        return base64.b64decode(data[key]).decode("utf-8")
+
+
+def create_kubernetes_resolver(spec: dict[str, Any]) -> KubernetesSecretResolver:
+    """Build a :class:`KubernetesSecretResolver`, loading cluster config lazily.
+
+    Tries in-cluster config first (for pods with a mounted service account), then falls
+    back to the local kubeconfig for development. The ``kubernetes`` package is imported
+    here rather than at module load so the plugin can be discovered even when the extra
+    is not installed.
+    """
+    try:
+        from kubernetes import client, config
+    except ImportError as exc:
+        raise RuntimeError(
+            "the kubernetes secret resolver requires the 'kubernetes' extra; install agentevals-cli[kubernetes]"
+        ) from exc
+
+    try:
+        config.load_incluster_config()
+    except config.ConfigException:
+        config.load_kube_config()
+
+    return KubernetesSecretResolver(client.CoreV1Api())

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/src/agentevals/run/worker.py

@@ -21,6 +21,11 @@ from uuid import UUID
 from google.adk.evaluation.eval_set import EvalSet
 
 from ..config import EvalParams
+from ..resolvers import (
+    reset_resolved_credentials,
+    resolve_credential_refs,
+    set_resolved_credentials,
+)
 from ..runner import RunResult, TraceResult, run_evaluation_from_traces
 from ..storage.config import StorageSettings
 from ..storage.models import Run, RunStatus
@@ -107,7 +112,10 @@ class AsyncRunWorker:
         cancel_event = asyncio.Event()
         hb_task = asyncio.create_task(self._heartbeat(run.run_id, worker_id, cancel_event))
         sinks = build_sinks(run.spec.sinks or [])
+        cred_token = None
         try:
+            if run.spec.credential_refs:
+                cred_token = set_resolved_credentials(await resolve_credential_refs(run.spec.credential_refs))
             await self._run_evaluation(run, sinks, cancel_event)
         except asyncio.CancelledError:
             await self._runs.update_status(run.run_id, RunStatus.CANCELLED, error="worker cancelled")
@@ -126,6 +134,8 @@ class AsyncRunWorker:
             await self._runs.update_status(run.run_id, RunStatus.FAILED, error=str(exc))
             await sinks.emit_error(run.run_id, str(exc), run.attempt)
         finally:
+            if cred_token is not None:
+                reset_resolved_credentials(cred_token)
             hb_task.cancel()
             try:
                 await hb_task

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/src/agentevals/storage/models.py

@@ -84,6 +84,15 @@ class RunSpec(BaseModel):
     eval_config: dict[str, Any] = Field(default_factory=dict)
     sinks: list[dict[str, Any]] = Field(default_factory=list)
     context: dict[str, Any] = Field(default_factory=dict)
+    credential_refs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Map of logical credential name to a secret reference dict. Each reference has a "
+            "'kind' (the resolver to use) plus that kind's locator fields. Resolved per run to its "
+            "secret value; never written to the process environment. How a value is used (e.g. which "
+            "judge provider it authenticates) is configured on the consumer, not the reference."
+        ),
+    )
 
 
 class Run(BaseModel):

{agentevals_cli-0.9.2 → agentevals_cli-0.9.4}/tests/integration/test_live_agents.py

@@ -15,6 +15,7 @@ Tests are synchronous because:
 
 from __future__ import annotations
 
+import importlib.util
 import os
 import subprocess
 import sys
@@ -38,6 +39,11 @@ _skip_no_google = pytest.mark.skipif(
     reason="GOOGLE_API_KEY not set",
 )
 
+_skip_no_pydantic_ai = pytest.mark.skipif(
+    importlib.util.find_spec("pydantic_ai") is None,
+    reason="pydantic_ai SDK not installed",
+)
+
 
 def _run_agent(
     script: str,
@@ -305,6 +311,7 @@ class TestOpenAIAgentsZeroCode:
         assert session_name in session_ids
 
 
+@_skip_no_pydantic_ai
 @_skip_no_openai
 class TestPydanticAIZeroCode:
     """Run the Pydantic AI zero-code OTLP example and verify session grouping."""

agentevals_cli-0.9.4/tests/resolvers/test_kubernetes.py

@@ -0,0 +1,63 @@
+"""Kubernetes secret resolver tests.
+
+The kubernetes client is mocked, so these run whether or not the optional
+``kubernetes`` extra is installed.
+"""
+
+from __future__ import annotations
+
+import base64
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+
+from agentevals.resolvers.kubernetes import KubernetesSecretResolver, create_kubernetes_resolver
+
+
+def _client_returning(data: dict[str, str]) -> MagicMock:
+    client = MagicMock()
+    client.read_namespaced_secret.return_value = MagicMock(data=data)
+    return client
+
+
+def _b64(value: str) -> str:
+    return base64.b64encode(value.encode()).decode()
+
+
+class TestResolve:
+    async def test_reads_and_base64_decodes_value(self):
+        client = _client_returning({"api-key": _b64("sk-secret-value")})
+        resolver = KubernetesSecretResolver(client)
+
+        value = await resolver.resolve({"namespace": "ns", "name": "creds", "key": "api-key"})
+
+        assert value == "sk-secret-value"
+        client.read_namespaced_secret.assert_called_once_with("creds", "ns")
+
+    async def test_missing_fields_raise(self):
+        resolver = KubernetesSecretResolver(MagicMock())
+        with pytest.raises(ValueError, match="namespace, name, key"):
+            await resolver.resolve({"kind": "kubernetes"})
+
+    async def test_key_not_found_lists_names_not_values(self):
+        secret_value = _b64("sk-do-not-leak")
+        client = _client_returning({"api-key": secret_value, "tls.crt": _b64("cert")})
+        resolver = KubernetesSecretResolver(client)
+
+        with pytest.raises(ValueError) as exc:
+            await resolver.resolve({"namespace": "ns", "name": "creds", "key": "wrong"})
+
+        message = str(exc.value)
+        assert "api-key" in message and "tls.crt" in message
+        # The enumeration must never echo the base64-encoded secret values.
+        assert secret_value not in message
+
+
+class TestFactory:
+    def test_requires_kubernetes_extra(self, monkeypatch):
+        # Shadow the kubernetes package so the lazy import fails regardless of
+        # whether the extra is installed in the test venv.
+        monkeypatch.setitem(sys.modules, "kubernetes", None)
+        with pytest.raises(RuntimeError, match="kubernetes"):
+            create_kubernetes_resolver({"kind": "kubernetes"})