agentevals-cli 0.9.4__tar.gz → 0.9.5__tar.gz

{agentevals_cli-0.9.4 → agentevals_cli-0.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentevals-cli
-Version: 0.9.4
+Version: 0.9.5
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11

{agentevals_cli-0.9.4 → agentevals_cli-0.9.5}/src/agentevals/api/models.py

@@ -142,6 +142,15 @@ class EvaluateJsonRequest(CamelModel):
     traces: dict = Field(description="OTLP JSON export with resourceSpans structure.")
     config: EvalParams = Field(default_factory=EvalParams, description="Evaluation parameters.")
     eval_set: dict | None = Field(default=None, description="Optional ADK EvalSet JSON.")
+    credential_refs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Map of logical credential name to a secret reference dict. Each reference has a "
+            "'kind' (the resolver to use) plus that kind's locator fields. Resolved per call to its "
+            "secret value; never written to the process environment. How a value is used (e.g. which "
+            "judge provider it authenticates) is configured on the consumer, not the reference."
+        ),
+    )
 
 
 # ---------------------------------------------------------------------------

{agentevals_cli-0.9.4 → agentevals_cli-0.9.5}/src/agentevals/api/routes.py

@@ -9,6 +9,7 @@ import os
 import re
 import shutil
 import tempfile
+from contextlib import contextmanager
 from typing import Any
 
 from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
@@ -23,6 +24,11 @@ from ..converter import convert_traces
 from ..extraction import get_extractor
 from ..loader import load_traces
 from ..loader.otlp import OtlpJsonLoader
+from ..resolvers import (
+    reset_resolved_credentials,
+    resolve_credential_refs,
+    set_resolved_credentials,
+)
 from ..runner import (
     RunResult,
     load_eval_set,
@@ -53,6 +59,57 @@ from .models import (
 logger = logging.getLogger(__name__)
 
 
+@contextmanager
+def _scoped_credentials(resolved: dict[str, str] | None):
+    """Scope an already-resolved ``logical-name -> secret value`` map to the current task.
+
+    Mirrors the async worker's set/reset (``run/worker.py``) so the synchronous evaluate
+    paths populate the same credential ContextVar that judge graders read. A falsy map is a
+    no-op, keeping callers byte-for-byte backward compatible. For streaming endpoints, enter
+    this BEFORE ``asyncio.create_task`` so the eval task inherits the populated context (a
+    child task snapshots its parent's context at creation time). Resolution is done by the
+    caller so its failures surface as request errors rather than scoping concerns.
+    """
+    token = set_resolved_credentials(resolved) if resolved else None
+    try:
+        yield
+    finally:
+        if token is not None:
+            reset_resolved_credentials(token)
+
+
+async def _resolve_credentials(refs: dict[str, dict[str, Any]] | None) -> dict[str, str] | None:
+    """Resolve credentialRefs to secret values, mapping bad references to a 400.
+
+    Resolver ``ValueError``s (missing/unknown ``kind``, missing locator fields, an unset
+    env var) are request/input errors, so surface them as 400s instead of letting them
+    bubble up as 500s. Infrastructure failures from custom resolvers raise other exception
+    types and are left to propagate as 5xx.
+    """
+    if not refs:
+        return None
+    try:
+        return await resolve_credential_refs(refs)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=f"Could not resolve credentialRefs: {exc}") from exc
+
+
+def _parse_credential_refs_form(raw: str | None) -> dict[str, dict[str, Any]] | None:
+    """Parse and validate the multipart ``credential_refs`` form field (a JSON object string).
+
+    Empty/absent is treated as no credentials. Raises ``ValueError`` (which
+    ``json.JSONDecodeError`` subclasses) on malformed JSON or a non-object shape, so callers
+    map both to the same error they use for a bad ``config``. The JSON request endpoints get
+    this shape check for free from the ``EvaluateJsonRequest`` model.
+    """
+    if not raw:
+        return None
+    refs = json.loads(raw)
+    if not isinstance(refs, dict) or not all(isinstance(ref, dict) for ref in refs.values()):
+        raise ValueError("credentialRefs must be a JSON object mapping each logical name to a reference object")
+    return refs
+
+
 def _camel_keys(obj: Any) -> Any:
     """Recursively convert dict keys from snake_case to camelCase."""
     if isinstance(obj, dict):
@@ -462,6 +519,7 @@ async def evaluate_traces(
     trace_files: list[UploadFile] = File(...),
     config: str = Form(...),
     eval_set_file: UploadFile | None = File(None),
+    credential_refs: str | None = Form(None),
 ):
     """
     Evaluate agent traces using the provided evaluator configuration.
@@ -470,6 +528,8 @@ async def evaluate_traces(
         trace_files: List of Jaeger or OTLP JSON trace files
         config: JSON string with evaluation configuration
         eval_set_file: Optional golden eval set file
+        credential_refs: Optional JSON string mapping logical credential names to
+            secret references, resolved so LLM-as-Judge graders can authenticate
 
     Returns:
         RunResult with trace results and any errors
@@ -481,6 +541,11 @@ async def evaluate_traces(
         except json.JSONDecodeError as exc:
             raise HTTPException(status_code=400, detail=f"Invalid config JSON: {exc}") from exc
 
+        try:
+            cred_refs = _parse_credential_refs_form(credential_refs)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=f"Invalid credentialRefs: {exc}") from exc
+
         trace_paths = []
         for trace_file in trace_files:
             if not trace_file.filename:
@@ -548,7 +613,9 @@ async def evaluate_traces(
             len(trace_paths),
             [e.name for e in eval_config.evaluators],
         )
-        result = await run_evaluation(eval_config)
+        resolved_creds = await _resolve_credentials(cred_refs)
+        with _scoped_credentials(resolved_creds):
+            result = await run_evaluation(eval_config)
 
         run_id = await _maybe_persist_evaluate_run(
             request,
@@ -580,6 +647,7 @@ async def evaluate_traces_stream(
     trace_files: list[UploadFile] = File(...),
     config: str = Form(...),
     eval_set_file: UploadFile | None = File(None),
+    credential_refs: str | None = Form(None),
 ):
     """Evaluate traces with real-time progress via SSE."""
     temp_dir = tempfile.mkdtemp()
@@ -593,6 +661,12 @@ async def evaluate_traces_stream(
                 yield f"data: {SSEErrorEvent(error=f'Invalid config JSON: {exc}').model_dump_json(by_alias=True)}\n\n"
                 return
 
+            try:
+                cred_refs = _parse_credential_refs_form(credential_refs)
+            except ValueError as exc:
+                yield f"data: {SSEErrorEvent(error=f'Invalid credentialRefs: {exc}').model_dump_json(by_alias=True)}\n\n"
+                return
+
             trace_paths = []
             for trace_file in trace_files:
                 if not trace_file.filename:
@@ -674,47 +748,54 @@ async def evaluate_traces_stream(
                 result = await run_evaluation(eval_config, progress_callback, trace_progress_callback)
                 await queue.put(("done", result))
 
-            eval_task = asyncio.create_task(run_with_progress())
-
             try:
-                while True:
-                    msg = await queue.get()
-                    tag, payload = msg
-
-                    if tag == "done":
-                        run_id = await _maybe_persist_evaluate_run(
-                            request,
-                            params=eval_config,
-                            eval_set_dict=_load_eval_set_dict(eval_set_path),
-                            trace_format=eval_config.trace_format,
-                            upload_filenames=upload_filenames,
-                            run_result=payload,
-                        )
-                        if run_id:
-                            payload.run_id = run_id
-                        evt = SSEDoneEvent(
-                            result=_camel_keys(payload.model_dump(by_alias=True)),
-                        )
-                        yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
-                        break
-                    elif tag == "trace_progress":
-                        evt = SSETraceProgressEvent(
-                            trace_progress=SSETraceProgress(
-                                trace_id=payload.trace_id,
-                                partial_result=_camel_keys(payload.model_dump(by_alias=True)),
+                resolved_creds = await resolve_credential_refs(cred_refs) if cred_refs else None
+            except ValueError as exc:
+                yield f"data: {SSEErrorEvent(error=f'Could not resolve credentialRefs: {exc}').model_dump_json(by_alias=True)}\n\n"
+                return
+
+            with _scoped_credentials(resolved_creds):
+                eval_task = asyncio.create_task(run_with_progress())
+
+                try:
+                    while True:
+                        msg = await queue.get()
+                        tag, payload = msg
+
+                        if tag == "done":
+                            run_id = await _maybe_persist_evaluate_run(
+                                request,
+                                params=eval_config,
+                                eval_set_dict=_load_eval_set_dict(eval_set_path),
+                                trace_format=eval_config.trace_format,
+                                upload_filenames=upload_filenames,
+                                run_result=payload,
                             )
-                        )
-                        yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
-                    elif tag == "progress":
-                        evt = SSEProgressEvent(message=payload)
-                        yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
-            finally:
-                if not eval_task.done():
-                    eval_task.cancel()
-                    try:
-                        await eval_task
-                    except asyncio.CancelledError:
-                        pass
+                            if run_id:
+                                payload.run_id = run_id
+                            evt = SSEDoneEvent(
+                                result=_camel_keys(payload.model_dump(by_alias=True)),
+                            )
+                            yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
+                            break
+                        elif tag == "trace_progress":
+                            evt = SSETraceProgressEvent(
+                                trace_progress=SSETraceProgress(
+                                    trace_id=payload.trace_id,
+                                    partial_result=_camel_keys(payload.model_dump(by_alias=True)),
+                                )
+                            )
+                            yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
+                        elif tag == "progress":
+                            evt = SSEProgressEvent(message=payload)
+                            yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
+                finally:
+                    if not eval_task.done():
+                        eval_task.cancel()
+                        try:
+                            await eval_task
+                        except asyncio.CancelledError:
+                            pass
 
         except Exception as exc:
             logger.exception("Evaluation stream failed")
@@ -775,13 +856,15 @@ async def evaluate_traces_json(request: EvaluateJsonRequest, raw_request: Reques
     """Evaluate OTLP JSON traces passed in the request body."""
     _check_json_body_size(raw_request)
     traces, eval_set = _parse_json_request(request)
+    resolved_creds = await _resolve_credentials(request.credential_refs)
 
     try:
-        result = await run_evaluation_from_traces(
-            traces=traces,
-            config=request.config,
-            eval_set=eval_set,
-        )
+        with _scoped_credentials(resolved_creds):
+            result = await run_evaluation_from_traces(
+                traces=traces,
+                config=request.config,
+                eval_set=eval_set,
+            )
         run_id = await _maybe_persist_evaluate_run(
             raw_request,
             params=request.config,
@@ -793,6 +876,8 @@ async def evaluate_traces_json(request: EvaluateJsonRequest, raw_request: Reques
         if run_id:
             result.run_id = run_id
         return StandardResponse(data=_camel_keys(result.model_dump(by_alias=True)))
+    except HTTPException:
+        raise
     except Exception as exc:
         logger.exception("JSON evaluation failed")
         raise HTTPException(status_code=500, detail=f"Internal error: {exc!s}") from exc
@@ -843,47 +928,56 @@ async def evaluate_traces_json_stream(request: EvaluateJsonRequest, raw_request:
                 )
                 await queue.put(("done", result))
 
-            eval_task = asyncio.create_task(run_with_progress())
-
             try:
-                while True:
-                    msg = await queue.get()
-                    tag, payload = msg
-
-                    if tag == "done":
-                        run_id = await _maybe_persist_evaluate_run(
-                            raw_request,
-                            params=request.config,
-                            eval_set_dict=request.eval_set,
-                            trace_format=None,
-                            upload_filenames=None,
-                            run_result=payload,
-                        )
-                        if run_id:
-                            payload.run_id = run_id
-                        evt = SSEDoneEvent(
-                            result=_camel_keys(payload.model_dump(by_alias=True)),
-                        )
-                        yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
-                        break
-                    elif tag == "trace_progress":
-                        evt = SSETraceProgressEvent(
-                            trace_progress=SSETraceProgress(
-                                trace_id=payload.trace_id,
-                                partial_result=_camel_keys(payload.model_dump(by_alias=True)),
+                resolved_creds = (
+                    await resolve_credential_refs(request.credential_refs) if request.credential_refs else None
+                )
+            except ValueError as exc:
+                yield _sse_error(f"Could not resolve credentialRefs: {exc}")
+                return
+
+            with _scoped_credentials(resolved_creds):
+                eval_task = asyncio.create_task(run_with_progress())
+
+                try:
+                    while True:
+                        msg = await queue.get()
+                        tag, payload = msg
+
+                        if tag == "done":
+                            run_id = await _maybe_persist_evaluate_run(
+                                raw_request,
+                                params=request.config,
+                                eval_set_dict=request.eval_set,
+                                trace_format=None,
+                                upload_filenames=None,
+                                run_result=payload,
                             )
-                        )
-                        yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
-                    elif tag == "progress":
-                        evt = SSEProgressEvent(message=payload)
-                        yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
-            finally:
-                if not eval_task.done():
-                    eval_task.cancel()
-                    try:
-                        await eval_task
-                    except asyncio.CancelledError:
-                        pass
+                            if run_id:
+                                payload.run_id = run_id
+                            evt = SSEDoneEvent(
+                                result=_camel_keys(payload.model_dump(by_alias=True)),
+                            )
+                            yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
+                            break
+                        elif tag == "trace_progress":
+                            evt = SSETraceProgressEvent(
+                                trace_progress=SSETraceProgress(
+                                    trace_id=payload.trace_id,
+                                    partial_result=_camel_keys(payload.model_dump(by_alias=True)),
+                                )
+                            )
+                            yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
+                        elif tag == "progress":
+                            evt = SSEProgressEvent(message=payload)
+                            yield f"data: {evt.model_dump_json(by_alias=True)}\n\n"
+                finally:
+                    if not eval_task.done():
+                        eval_task.cancel()
+                        try:
+                            await eval_task
+                        except asyncio.CancelledError:
+                            pass
 
         except Exception as exc:
             logger.exception("JSON evaluation stream failed")

{agentevals_cli-0.9.4 → agentevals_cli-0.9.5}/tests/test_api.py

@@ -229,6 +229,35 @@ def _eval_config_json(**overrides) -> str:
     return json.dumps(cfg)
 
 
+def _judge_config(**overrides) -> dict:
+    cfg = {
+        "evaluators": [
+            {"name": "hallucinations_v1", "type": "builtin", "judgeModel": "openai/gpt-4o", "credentialRef": "k"}
+        ]
+    }
+    cfg.update(overrides)
+    return cfg
+
+
+def _capturing_run_eval(captured: dict):
+    """Build an AsyncMock side_effect that records, at evaluator-invocation time, the value the
+    judge would resolve for credential ``k``.
+
+    This is the correct boundary for the sync routes: their job is to populate the credential
+    ContextVar before the evaluator runs. The ContextVar -> judge injection step itself is
+    already covered by test_credential_injection.py, so recording ``get_resolved_credential``
+    here (rather than mocking it) is not a false positive -- it fails when the route omits the
+    set/reset, which is exactly the gap being closed.
+    """
+    from agentevals.resolvers import get_resolved_credential
+
+    def _side_effect(*args, **kwargs):
+        captured["judge_key"] = get_resolved_credential("k")
+        return _make_run_result()
+
+    return _side_effect
+
+
 # ---------------------------------------------------------------------------
 # Model Serialization
 # ---------------------------------------------------------------------------
@@ -528,6 +557,68 @@ class TestEvaluateTraces:
         )
         assert resp.status_code in (400, 422)
 
+    @patch("agentevals.api.routes.run_evaluation", new_callable=AsyncMock)
+    def test_evaluate_resolves_credential_refs(self, mock_eval, monkeypatch):
+        monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-multipart")
+        captured: dict = {}
+        mock_eval.side_effect = _capturing_run_eval(captured)
+        resp = self.client.post(
+            "/api/evaluate",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={
+                "config": json.dumps(_judge_config()),
+                "credential_refs": json.dumps({"k": {"kind": "env", "name": "AE_TEST_JUDGE_KEY"}}),
+            },
+        )
+        _assert_envelope(resp)
+        assert captured["judge_key"] == "sk-resolved-multipart"
+
+    @patch("agentevals.api.routes.run_evaluation", new_callable=AsyncMock)
+    def test_evaluate_without_credential_refs_is_noop(self, mock_eval):
+        captured: dict = {}
+        mock_eval.side_effect = _capturing_run_eval(captured)
+        resp = self.client.post(
+            "/api/evaluate",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={"config": _eval_config_json()},
+        )
+        _assert_envelope(resp)
+        assert captured["judge_key"] is None
+
+    def test_evaluate_bad_credential_refs_returns_400(self):
+        resp = self.client.post(
+            "/api/evaluate",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={"config": _eval_config_json(), "credential_refs": "{not json"},
+        )
+        assert resp.status_code == 400
+        assert "credentialRefs" in resp.json()["detail"]
+
+    def test_evaluate_credential_refs_wrong_shape_returns_400(self):
+        resp = self.client.post(
+            "/api/evaluate",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={"config": _eval_config_json(), "credential_refs": json.dumps(["not", "a", "map"])},
+        )
+        assert resp.status_code == 400
+        assert "credentialRefs" in resp.json()["detail"]
+
+    @patch("agentevals.api.routes.run_evaluation", new_callable=AsyncMock)
+    def test_evaluate_unresolvable_credential_returns_400(self, mock_eval, monkeypatch):
+        monkeypatch.delenv("AE_MISSING_KEY", raising=False)
+        mock_eval.return_value = _make_run_result()
+        resp = self.client.post(
+            "/api/evaluate",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={
+                "config": json.dumps(_judge_config()),
+                "credential_refs": json.dumps({"k": {"kind": "env", "name": "AE_MISSING_KEY"}}),
+            },
+        )
+        assert resp.status_code == 400
+        assert "Could not resolve credentialRefs" in resp.json()["detail"]
+        mock_eval.assert_not_called()
+
 
 # ---------------------------------------------------------------------------
 # POST /api/evaluate/stream (SSE)
@@ -591,6 +682,34 @@ class TestEvaluateStream:
         assert "result" in done
         assert "traceResults" in done["result"]
 
+    @patch("agentevals.api.routes.run_evaluation", new_callable=AsyncMock)
+    @patch("agentevals.api.routes.load_traces")
+    def test_stream_resolves_credential_refs(self, mock_load_traces, mock_eval, monkeypatch):
+        monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-stream")
+        mock_load_traces.return_value = []
+        captured: dict = {}
+        mock_eval.side_effect = _capturing_run_eval(captured)
+        resp = self.client.post(
+            "/api/evaluate/stream",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={
+                "config": json.dumps(_judge_config()),
+                "credential_refs": json.dumps({"k": {"kind": "env", "name": "AE_TEST_JUDGE_KEY"}}),
+            },
+        )
+        assert '"done"' in resp.text
+        assert captured["judge_key"] == "sk-resolved-stream"
+
+    def test_stream_bad_credential_refs(self):
+        resp = self.client.post(
+            "/api/evaluate/stream",
+            files={"trace_files": ("trace.json", io.BytesIO(_make_trace_json()))},
+            data={"config": _eval_config_json(), "credential_refs": "{not json"},
+        )
+        assert resp.status_code == 200
+        assert '"error"' in resp.text
+        assert "credentialRefs" in resp.text
+
 
 # ---------------------------------------------------------------------------
 # POST /api/evaluate/json
@@ -767,6 +886,56 @@ class TestEvaluateJson:
         body = _assert_envelope(resp)
         assert "traceResults" in body["data"]
 
+    @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock)
+    def test_evaluate_json_resolves_credential_refs(self, mock_eval, monkeypatch):
+        monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-json")
+        captured: dict = {}
+        mock_eval.side_effect = _capturing_run_eval(captured)
+        resp = self.client.post(
+            "/api/evaluate/json",
+            json={
+                "traces": _make_otlp_json_payload(),
+                "config": _judge_config(),
+                "credentialRefs": {"k": {"kind": "env", "name": "AE_TEST_JUDGE_KEY"}},
+            },
+        )
+        _assert_envelope(resp)
+        assert captured["judge_key"] == "sk-resolved-json"
+
+    @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock)
+    def test_evaluate_json_without_credential_refs_is_noop(self, mock_eval):
+        captured: dict = {}
+        mock_eval.side_effect = _capturing_run_eval(captured)
+        resp = self.client.post(
+            "/api/evaluate/json",
+            json={"traces": _make_otlp_json_payload(), "config": _judge_config()},
+        )
+        _assert_envelope(resp)
+        assert captured["judge_key"] is None
+
+    def test_evaluate_json_credential_refs_wrong_shape_returns_422(self):
+        resp = self.client.post(
+            "/api/evaluate/json",
+            json={"traces": _make_otlp_json_payload(), "credentialRefs": ["not", "a", "map"]},
+        )
+        assert resp.status_code == 422
+
+    @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock)
+    def test_evaluate_json_unresolvable_credential_returns_400(self, mock_eval, monkeypatch):
+        monkeypatch.delenv("AE_MISSING_KEY", raising=False)
+        mock_eval.return_value = _make_run_result()
+        resp = self.client.post(
+            "/api/evaluate/json",
+            json={
+                "traces": _make_otlp_json_payload(),
+                "config": _judge_config(),
+                "credentialRefs": {"k": {"kind": "env", "name": "AE_MISSING_KEY"}},
+            },
+        )
+        assert resp.status_code == 400
+        assert "Could not resolve credentialRefs" in resp.json()["detail"]
+        mock_eval.assert_not_called()
+
 
 # ---------------------------------------------------------------------------
 # POST /api/evaluate/json/stream (SSE)
@@ -827,6 +996,47 @@ class TestEvaluateJsonStream:
         assert '"error"' in body
         assert "No traces" in body
 
+    @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock)
+    @patch("agentevals.api.routes.OtlpJsonLoader")
+    def test_stream_resolves_credential_refs(self, mock_loader_cls, mock_eval, monkeypatch):
+        monkeypatch.setenv("AE_TEST_JUDGE_KEY", "sk-resolved-json-stream")
+        mock_trace = MagicMock()
+        mock_trace.trace_id = "abc123"
+        mock_loader_cls.return_value.load_from_dict.return_value = [mock_trace]
+        captured: dict = {}
+        mock_eval.side_effect = _capturing_run_eval(captured)
+        resp = self.client.post(
+            "/api/evaluate/json/stream",
+            json={
+                "traces": _make_otlp_json_payload(),
+                "config": _judge_config(),
+                "credentialRefs": {"k": {"kind": "env", "name": "AE_TEST_JUDGE_KEY"}},
+            },
+        )
+        assert '"done"' in resp.text
+        assert captured["judge_key"] == "sk-resolved-json-stream"
+
+    @patch("agentevals.api.routes.run_evaluation_from_traces", new_callable=AsyncMock)
+    @patch("agentevals.api.routes.OtlpJsonLoader")
+    def test_stream_unresolvable_credential_yields_error(self, mock_loader_cls, mock_eval, monkeypatch):
+        monkeypatch.delenv("AE_MISSING_KEY", raising=False)
+        mock_trace = MagicMock()
+        mock_trace.trace_id = "abc123"
+        mock_loader_cls.return_value.load_from_dict.return_value = [mock_trace]
+        mock_eval.return_value = _make_run_result()
+        resp = self.client.post(
+            "/api/evaluate/json/stream",
+            json={
+                "traces": _make_otlp_json_payload(),
+                "config": _judge_config(),
+                "credentialRefs": {"k": {"kind": "env", "name": "AE_MISSING_KEY"}},
+            },
+        )
+        assert '"error"' in resp.text
+        assert "Could not resolve credentialRefs" in resp.text
+        assert '"done"' not in resp.text
+        mock_eval.assert_not_called()
+
 
 # ---------------------------------------------------------------------------
 # GET /api/streaming/sessions