agentops-accelerator 0.5.1__tar.gz → 0.5.2__tar.gz

{agentops_accelerator-0.5.1 → agentops_accelerator-0.5.2}/CHANGELOG.md

@@ -5,7 +5,21 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres
 
 ## [Unreleased]
 
-## [0.5.1] - 2026-06-20
+## [0.5.2] - 2026-06-20
+
+### Added
+- **Per-evaluator input remapping via `evaluators[].input_mapping`.** Evaluator
+  overrides in `agentops.yaml` now accept an optional `input_mapping` map that is
+  merged onto the preset's default inputs, so you only list the keys you want to
+  change. This is what lets a grey-box HTTP/JSON target point a RAG evaluator at
+  the live retrieved context captured by `response_fields`, for example
+  `context: $response.context` on `GroundednessEvaluator` and
+  `RetrievalEvaluator`. The mapping applies to both explicitly listed overrides
+  and auto-selected presets. A bare evaluator name string (`- GroundednessEvaluator`)
+  is still accepted as shorthand for `{ name: GroundednessEvaluator }`, so
+  existing configs are unchanged.
+
+
 
 ### Added
 - **Rendered gate results in GitHub Actions job summaries.** When AgentOps runs

{agentops_accelerator-0.5.1/src/agentops_accelerator.egg-info → agentops_accelerator-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentops-accelerator
-Version: 0.5.1
+Version: 0.5.2
 Summary: Release readiness gates and evidence for Microsoft Foundry agents
 License: MIT License
         

{agentops_accelerator-0.5.1 → agentops_accelerator-0.5.2}/src/agentops/core/agentops_config.py

@@ -152,12 +152,51 @@ class EvaluatorOverride(BaseModel):
         evaluators:
           - GroundednessEvaluator
           - CoherenceEvaluator
+
+    Each entry may instead be a mapping to remap the evaluator inputs. This is
+    how an HTTP/JSON target scores the *live* retrieved context (grey-box):
+    capture the extra fields on the target with ``response_fields`` and point
+    the evaluator at them via ``$response.<name>`` tokens::
+
+        response_fields:
+          context: context
+          retrieved_documents: retrieved_documents
+        evaluators:
+          - name: GroundednessEvaluator
+            input_mapping:
+              query: $prompt
+              response: $prediction
+              context: $response.context
+          - name: RetrievalEvaluator
+            input_mapping:
+              query: $prompt
+              context: $response.context
+
+    ``input_mapping`` is merged onto the preset's default mapping, so you only
+    list the keys you want to change.
     """
 
     name: str
+    input_mapping: Optional[Dict[str, str]] = Field(
+        None,
+        description=(
+            "Optional per-evaluator input remap merged onto the preset "
+            "defaults. Values use the resolver tokens $prompt, $prediction, "
+            "$expected, $context, $row.<col>, and $response.<name> (the last "
+            "reads a field captured by the target's response_fields)."
+        ),
+    )
 
     model_config = ConfigDict(frozen=True)
 
+    @model_validator(mode="before")
+    @classmethod
+    def _coerce_bare_name(cls, data: Any) -> Any:
+        """Accept a bare evaluator name string as shorthand for ``{name: ...}``."""
+        if isinstance(data, str):
+            return {"name": data}
+        return data
+
     @field_validator("name")
     @classmethod
     def _name_non_empty(cls, value: str) -> str:
@@ -165,6 +204,22 @@ class EvaluatorOverride(BaseModel):
             raise ValueError("evaluator name must be non-empty")
         return value
 
+    @field_validator("input_mapping")
+    @classmethod
+    def _mapping_non_empty(
+        cls, value: Optional[Dict[str, str]]
+    ) -> Optional[Dict[str, str]]:
+        if value is None:
+            return None
+        for key, token in value.items():
+            if not str(key).strip():
+                raise ValueError("input_mapping keys must be non-empty")
+            if not str(token).strip():
+                raise ValueError(
+                    f"input_mapping value for {key!r} must be non-empty"
+                )
+        return value
+
 
 # ---------------------------------------------------------------------------
 # Dataset sync configuration

{agentops_accelerator-0.5.1 → agentops_accelerator-0.5.2}/src/agentops/core/evaluators.py

@@ -32,7 +32,7 @@ final word - no auto-detection runs.
 from __future__ import annotations
 
 import json
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from pathlib import Path
 from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
 
@@ -348,6 +348,7 @@ def select_evaluators(
     shape: DatasetShape,
     *,
     overrides: Optional[List[str]] = None,
+    override_mappings: Optional[Dict[str, Dict[str, str]]] = None,
     threshold_metrics: Optional[Iterable[str]] = None,
 ) -> List[EvaluatorPreset]:
     """Return the ordered list of evaluators to run.
@@ -356,6 +357,12 @@ def select_evaluators(
     bypassed. Each name must exist in :data:`CATALOG` or a ``ValueError`` is
     raised.
 
+    ``override_mappings`` maps an evaluator name to an ``input_mapping`` patch
+    that is merged onto that preset's default mapping. This lets a grey-box
+    HTTP/JSON target point an evaluator at a live response field captured via
+    ``response_fields`` (e.g. ``context: $response.context``). It applies to
+    both the explicit-override path and the auto-selected presets.
+
     Otherwise the rules are:
 
     * Always include the baseline quality evaluators.
@@ -369,6 +376,16 @@ def select_evaluators(
       evaluators.
     * Always append the runtime ``avg_latency_seconds`` evaluator.
     """
+
+    def _apply_mappings(preset: EvaluatorPreset) -> EvaluatorPreset:
+        if not override_mappings:
+            return preset
+        patch = override_mappings.get(preset.name)
+        if not patch:
+            return preset
+        merged = {**preset.input_mapping, **patch}
+        return replace(preset, input_mapping=merged)
+
     if overrides:
         resolved: List[EvaluatorPreset] = []
         for name in overrides:
@@ -379,7 +396,7 @@ def select_evaluators(
                     f"unknown evaluator override {name!r}. "
                     f"Known evaluators: {known}"
                 )
-            resolved.append(preset)
+            resolved.append(_apply_mappings(preset))
         return resolved
 
     selected: List[EvaluatorPreset] = list(_QUALITY_BASELINE)
@@ -411,7 +428,7 @@ def select_evaluators(
         )
 
     selected.append(_LATENCY)
-    return selected
+    return [_apply_mappings(preset) for preset in selected]
 
 
 def _is_agent_target(kind: TargetKind) -> bool:

{agentops_accelerator-0.5.1 → agentops_accelerator-0.5.2}/src/agentops/pipeline/orchestrator.py

@@ -132,10 +132,16 @@ def _run_evaluation_local(
     overrides = (
         [override.name for override in config.evaluators] if config.evaluators else None
     )
+    override_mappings = (
+        {o.name: dict(o.input_mapping) for o in config.evaluators if o.input_mapping}
+        if config.evaluators
+        else None
+    ) or None
     presets = select_evaluators(
         target,
         shape,
         overrides=overrides,
+        override_mappings=override_mappings,
         threshold_metrics=config.thresholds.keys(),
     )
     user_thresholds = [
@@ -277,10 +283,16 @@ def _run_evaluation_cloud(
     overrides = (
         [override.name for override in config.evaluators] if config.evaluators else None
     )
+    override_mappings = (
+        {o.name: dict(o.input_mapping) for o in config.evaluators if o.input_mapping}
+        if config.evaluators
+        else None
+    ) or None
     all_presets = select_evaluators(
         target,
         shape,
         overrides=overrides,
+        override_mappings=override_mappings,
         threshold_metrics=config.thresholds.keys(),
     )
 

{agentops_accelerator-0.5.1 → agentops_accelerator-0.5.2/src/agentops_accelerator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentops-accelerator
-Version: 0.5.1
+Version: 0.5.2
 Summary: Release readiness gates and evidence for Microsoft Foundry agents
 License: MIT License
         

{agentops_accelerator-0.5.1 → agentops_accelerator-0.5.2}/tests/unit/test_evaluators.py

@@ -201,6 +201,56 @@ class TestSelectEvaluators:
         with pytest.raises(ValueError, match="unknown evaluator"):
             select_evaluators(_PROMPT_AGENT, _shape(), overrides=["NotAnEvaluator"])
 
+    def test_override_mapping_patches_explicit_override(self) -> None:
+        # Grey-box: point GroundednessEvaluator at the live captured context.
+        result = select_evaluators(
+            _HTTP_AGENT,
+            _shape(),
+            overrides=["GroundednessEvaluator"],
+            override_mappings={
+                "GroundednessEvaluator": {"context": "$response.context"}
+            },
+        )
+        preset = [p for p in result if p.name == "GroundednessEvaluator"][0]
+        assert preset.input_mapping["context"] == "$response.context"
+        # Unpatched keys keep their preset defaults.
+        assert preset.input_mapping["response"] == "$prediction"
+
+    def test_override_mapping_patches_auto_selected_rag_preset(self) -> None:
+        # No explicit overrides: RAG presets are auto-selected from the
+        # context-bearing dataset, and the mapping still applies.
+        result = select_evaluators(
+            _HTTP_AGENT,
+            _shape(context=True),
+            override_mappings={
+                "RetrievalEvaluator": {"context": "$response.context"}
+            },
+        )
+        preset = [p for p in result if p.name == "RetrievalEvaluator"][0]
+        assert preset.input_mapping["context"] == "$response.context"
+
+    def test_override_mapping_does_not_mutate_catalog_preset(self) -> None:
+        select_evaluators(
+            _HTTP_AGENT,
+            _shape(),
+            overrides=["GroundednessEvaluator"],
+            override_mappings={
+                "GroundednessEvaluator": {"context": "$response.context"}
+            },
+        )
+        # The shared CATALOG preset must be untouched (dataclasses.replace
+        # returns a copy).
+        assert CATALOG["GroundednessEvaluator"].input_mapping["context"] == "$context"
+
+    def test_override_mapping_none_is_noop(self) -> None:
+        result = select_evaluators(
+            _HTTP_AGENT,
+            _shape(context=True),
+            override_mappings=None,
+        )
+        preset = [p for p in result if p.name == "GroundednessEvaluator"][0]
+        assert preset.input_mapping["context"] == "$context"
+
 
 # ---------------------------------------------------------------------------
 # merge_thresholds