agentengine-sdk-python 0.8.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

ksadk/evaluation/cloud_service.py

@@ -0,0 +1,199 @@
+"""Cloud EvalSet publication orchestration shared by CLI and Studio."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Protocol
+
+from pydantic import Field
+
+from .cloud_binding import CloudBinding, CloudBindingStore
+from .cloud_converter import (
+    CloudDatasetSnapshot,
+    evalset_from_dataset_snapshot,
+    evalset_to_dataset_snapshot,
+)
+from .contracts import CloudDatasetRef, DataPolicy, EvalSetVersion, EvaluationModel
+
+
+class CloudEvalSetPreviewError(ValueError):
+    """Raised before an EvalSet body is allowed to leave the local process."""
+
+
+class CloudEvalSetPublishResult(EvaluationModel):
+    """Provider acknowledgement for one immutable Dataset snapshot."""
+
+    dataset_id: str = Field(min_length=1)
+    dataset_version: int = Field(ge=1)
+    project_id: str | None = None
+    schema_hash: str = Field(min_length=64, max_length=64)
+    content_digest: str = Field(min_length=64, max_length=64)
+    row_count: int = Field(ge=0)
+
+
+class CloudEvalSetPullResult(EvaluationModel):
+    """One validated immutable cloud snapshot and its normalized local form."""
+
+    snapshot: CloudDatasetSnapshot
+    evalset: EvalSetVersion
+    cloud_dataset: CloudDatasetRef
+
+
+class CloudEvalSetCatalogItem(EvaluationModel):
+    """One immutable Dataset version exposed by the cloud catalog."""
+
+    dataset_id: str = Field(min_length=1)
+    name: str = Field(min_length=1)
+    project_id: str | None = None
+    version: int = Field(ge=1)
+    schema_hash: str = Field(min_length=64, max_length=64)
+    content_digest: str = Field(min_length=64, max_length=64)
+    row_count: int = Field(ge=0)
+
+
+class CloudDatasetClient(Protocol):
+    """Minimal provider contract required before a snapshot can be published."""
+
+    async def publish_snapshot(
+        self,
+        snapshot: CloudDatasetSnapshot,
+        *,
+        dataset_id: str | None,
+        base_version: int | None,
+        idempotency_key: str,
+    ) -> CloudEvalSetPublishResult: ...
+
+    async def read_snapshot(
+        self,
+        dataset_id: str,
+        version: int,
+        *,
+        project_id: str | None = None,
+    ) -> CloudDatasetSnapshot: ...
+
+    async def list_datasets(
+        self,
+        *,
+        project_id: str | None = None,
+    ) -> list[CloudEvalSetCatalogItem]: ...
+
+
+class CloudEvalSetService:
+    """Validate, publish, and bind EvalSets without changing runtime execution."""
+
+    def __init__(
+        self,
+        workspace_root: str | Path,
+        client: CloudDatasetClient,
+        *,
+        provider: str = "agent-eval/evalsmith",
+    ):
+        self.bindings = CloudBindingStore(workspace_root)
+        self.client = client
+        self.provider = provider
+
+    def preview(self, evalset: EvalSetVersion, *, data_policy: DataPolicy) -> CloudDatasetSnapshot:
+        """Return the exact outgoing snapshot or fail before any network request."""
+
+        if data_policy is DataPolicy.LOCAL_ONLY:
+            raise CloudEvalSetPreviewError("DataPolicy=local_only 禁止上传 EvalSet 正文")
+        if data_policy is DataPolicy.METADATA_ONLY:
+            raise CloudEvalSetPreviewError(
+                "DataPolicy=metadata_only 不能发布包含 Case 正文的 EvalSet"
+            )
+        if data_policy is not DataPolicy.FULL_TRACE:
+            raise CloudEvalSetPreviewError("当前端云评测集发布仅支持显式 DataPolicy=full_trace")
+        return evalset_to_dataset_snapshot(evalset)
+
+    async def publish(
+        self,
+        evalset: EvalSetVersion,
+        *,
+        evalset_path: str,
+        data_policy: DataPolicy,
+        dataset_id: str | None = None,
+        idempotency_key: str | None = None,
+    ) -> CloudEvalSetPublishResult:
+        """Publish a full snapshot to an existing Dataset and advance its binding."""
+
+        if idempotency_key is not None and not idempotency_key.strip():
+            raise ValueError("Idempotency-Key 不能为空")
+        snapshot = self.preview(evalset, data_policy=data_policy)
+        existing = self.bindings.read(evalset_path)
+        requested_dataset_id = str(dataset_id or "").strip() or None
+        target_dataset_id = requested_dataset_id or (existing.dataset_id if existing else None)
+        if target_dataset_id is None:
+            raise ValueError("datasetId is required for the first publish of an EvalSet")
+        resolved_idempotency_key = idempotency_key or self._idempotency_key(
+            target_dataset_id,
+            snapshot.content_digest,
+        )
+        result = await self.client.publish_snapshot(
+            snapshot,
+            dataset_id=target_dataset_id,
+            base_version=None,
+            idempotency_key=resolved_idempotency_key,
+        )
+        if result.dataset_id != target_dataset_id:
+            raise CloudEvalSetPreviewError("云端返回的 datasetId 与目标 Dataset 不一致")
+        if result.content_digest != snapshot.content_digest:
+            raise CloudEvalSetPreviewError("云端返回的 contentDigest 与本地预检结果不一致")
+        if result.schema_hash != snapshot.schema_hash:
+            raise CloudEvalSetPreviewError("云端返回的 schemaHash 与本地预检结果不一致")
+        if result.row_count != len(snapshot.rows):
+            raise CloudEvalSetPreviewError("云端返回的 RowCount 与本地预检结果不一致")
+        self.bindings.write(
+            CloudBinding(
+                evalset_path=evalset_path,
+                content_digest=snapshot.content_digest,
+                provider=self.provider,
+                project_id=result.project_id,
+                dataset_id=result.dataset_id,
+                dataset_version=result.dataset_version,
+                schema_hash=result.schema_hash,
+            )
+        )
+        return result
+
+    @staticmethod
+    def _idempotency_key(dataset_id: str, content_digest: str) -> str:
+        identity = f"{dataset_id}:{content_digest}".encode("utf-8")
+        return f"ksadk-evalset-{hashlib.sha256(identity).hexdigest()}"
+
+    async def pull(
+        self,
+        *,
+        dataset_id: str,
+        version: int,
+        project_id: str | None = None,
+    ) -> CloudEvalSetPullResult:
+        """Read and validate one immutable Dataset version before execution."""
+
+        if not dataset_id.strip() or version < 1:
+            raise ValueError("datasetId 和 version 必须有效")
+        snapshot = await self.client.read_snapshot(
+            dataset_id,
+            version,
+            project_id=project_id,
+        )
+        evalset = evalset_from_dataset_snapshot(snapshot)
+        if evalset.content_digest != snapshot.content_digest:
+            raise CloudEvalSetPreviewError("云端 snapshot 的 contentDigest 校验失败")
+        reference = CloudDatasetRef(
+            provider=self.provider,
+            project_id=project_id,
+            dataset_id=dataset_id,
+            version=version,
+            schema_hash=snapshot.schema_hash,
+            content_digest=snapshot.content_digest,
+            row_count=len(snapshot.rows),
+        )
+        return CloudEvalSetPullResult(
+            snapshot=snapshot,
+            evalset=evalset,
+            cloud_dataset=reference,
+        )
+
+    async def catalog(self, *, project_id: str | None = None) -> list[CloudEvalSetCatalogItem]:
+        return await self.client.list_datasets(project_id=project_id)

ksadk/evaluation/contracts.py

@@ -58,8 +58,11 @@ class AssertionType(str, Enum):
     RUNTIME_MAX_LATENCY_MS = "runtime.maxLatencyMs"
     RUNTIME_MAX_INPUT_TOKENS = "runtime.maxInputTokens"
     RUNTIME_MAX_OUTPUT_TOKENS = "runtime.maxOutputTokens"
+    RUNTIME_MAX_TOTAL_TOKENS = "runtime.maxTotalTokens"
     TOOL_CALLED = "tool.called"
     TOOL_NOT_CALLED = "tool.notCalled"
+    TOOL_SUCCEEDED = "tool.succeeded"
+    TOOL_SEQUENCE = "tool.sequence"
 
 
 class DataPolicy(str, Enum):
@@ -134,6 +137,13 @@ class AssertionSpec(EvaluationModel):
                 raise ValueError(f"{self.type} 的 value 必须是非负数字")
             if self.value < 0:
                 raise ValueError(f"{self.type} 的 value 必须是非负数字")
+        elif self.type is AssertionType.TOOL_SEQUENCE:
+            if (
+                not isinstance(self.value, list)
+                or not self.value
+                or any(not isinstance(item, str) or not item.strip() for item in self.value)
+            ):
+                raise ValueError("tool.sequence 的 value 必须是非空工具名称数组")
         elif not isinstance(self.value, str):
             raise ValueError(f"{self.type} 的 value 必须是字符串")
         return self
@@ -155,9 +165,21 @@ class EvalCase(EvaluationModel):
         data = dict(value)
         if "input" in data and "turns" not in data:
             turn = {"input": data.pop("input")}
-            for field in ("expected_output", "expectedOutput", "expected_tools", "expectedTools"):
+            for field in (
+                "expected_output",
+                "expectedOutput",
+                "reference_output",
+                "referenceOutput",
+                "expected_tools",
+                "expectedTools",
+            ):
                 if field in data:
-                    turn[field] = data.pop(field)
+                    normalized_field = (
+                        "expected_output"
+                        if field in {"reference_output", "referenceOutput"}
+                        else field
+                    )
+                    turn[normalized_field] = data.pop(field)
             data["turns"] = [turn]
         return data
 
@@ -200,6 +222,18 @@ class EvalSetVersion(EvaluationModel):
         return _content_digest(payload)
 
 
+class CloudDatasetRef(EvaluationModel):
+    """Immutable reference to the exact cloud Dataset snapshot used by a run."""
+
+    provider: str = Field(min_length=1, max_length=256)
+    project_id: str | None = Field(default=None, min_length=1, max_length=256)
+    dataset_id: str = Field(min_length=1, max_length=256)
+    version: int = Field(ge=1)
+    schema_hash: str = Field(min_length=64, max_length=64)
+    content_digest: str = Field(min_length=64, max_length=64)
+    row_count: int = Field(default=0, ge=0)
+
+
 # ---------------------------------------------------------------------------
 # Target identity & references
 # ---------------------------------------------------------------------------
@@ -262,6 +296,7 @@ class EvaluationRequest(EvaluationModel):
     target: TargetRef
     config: EvaluationConfig = Field(default_factory=EvaluationConfig)
     report_dir: str | None = Field(default=None, min_length=1, max_length=2048)
+    cloud_dataset: CloudDatasetRef | None = None
 
 
 class EvalRunSpec(EvaluationModel):
@@ -273,6 +308,7 @@ class EvalRunSpec(EvaluationModel):
     config: EvaluationConfig = Field(default_factory=EvaluationConfig)
     environment_digest: str = Field(default="", max_length=128)
     attempt: int = Field(default=1, ge=1)
+    cloud_dataset: CloudDatasetRef | None = None
 
 
 # ---------------------------------------------------------------------------
@@ -288,11 +324,25 @@ class TraceRef(EvaluationModel):
     run_id: str | None = None
     trace_id: str | None = None
     root_span_id: str | None = None
+    session_id: str | None = None
+    invocation_id: str | None = None
+    seq_start: int | None = Field(default=None, ge=1)
+    seq_end: int | None = Field(default=None, ge=1)
+    remote_task_id: str | None = None
     seq_id: int | None = Field(default=None, ge=1)
 
     @model_validator(mode="after")
     def require_reference(self) -> "TraceRef":
-        if not any((self.run_id, self.trace_id, self.seq_id)):
+        if not any(
+            (
+                self.run_id,
+                self.trace_id,
+                self.session_id,
+                self.invocation_id,
+                self.remote_task_id,
+                self.seq_id,
+            )
+        ):
             raise ValueError("TraceRef 至少需要一个可查询 ID")
         return self
 
@@ -306,6 +356,16 @@ class UsageSnapshot(EvaluationModel):
     reported: bool = False
 
 
+class ToolCallEvidence(EvaluationModel):
+    """Non-sensitive projection of one runtime tool invocation."""
+
+    call_id: str = Field(min_length=1, max_length=256)
+    name: str = Field(min_length=1, max_length=256)
+    status: Literal["SUCCEEDED", "ERROR", "INCOMPLETE"]
+    seq_start: int | None = Field(default=None, ge=1)
+    seq_end: int | None = Field(default=None, ge=1)
+
+
 class TargetRun(EvaluationModel):
     """Normalized result returned by a target adapter for one case."""
 
@@ -316,6 +376,8 @@ class TargetRun(EvaluationModel):
     error_code: str | None = None
     error_message: str | None = None
     trace_ref: TraceRef | None = None
+    trace_refs: list[TraceRef] = Field(default_factory=list)
+    tool_calls: list[ToolCallEvidence] = Field(default_factory=list)
     metadata: dict[str, Any] = Field(default_factory=dict)
 
 
@@ -377,7 +439,14 @@ class EvalRunReport(EvaluationModel):
         self.summary = self._summarize_cases()
         expected = self.compute_digest()
         if self.report_digest and self.report_digest != expected:
-            raise ValueError("reportDigest 与规范化报告内容不一致")
+            legacy_payload = self.model_dump(
+                mode="json", by_alias=False, exclude={"report_digest"}
+            )
+            legacy_payload["spec"].pop("cloud_dataset", None)
+            if self.spec.cloud_dataset is not None or self.report_digest != _content_digest(
+                legacy_payload
+            ):
+                raise ValueError("reportDigest 与规范化报告内容不一致")
         self.report_digest = expected
         return self
 

ksadk/evaluation/evaluators.py

@@ -49,11 +49,15 @@ class _EvaluatorDefinition:
 
 REFERENCE_MATCH_EVALUATOR = "reference_match@v1"
 LLM_JUDGE_EVALUATOR = "llm_judge@v1"
-DEFAULT_EVALUATORS = (
+BUSINESS_STANDARD_EVALUATOR = "business_standard@v1"
+LEGACY_ASSERTION_EVALUATORS = (
     "response_contract@v1",
     "runtime_budget@v1",
     "tool_trajectory@v1",
 )
+# Kept for external imports only. Empty evaluator selection uses the
+# data-derived automatic plan in _automatic_evaluator_names instead.
+DEFAULT_EVALUATORS = LEGACY_ASSERTION_EVALUATORS
 _RESPONSE_MATCH_THRESHOLD = 0.8
 _TOKEN_PATTERN = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]")
 
@@ -67,7 +71,7 @@ def evaluate_case(
     """Run selected evaluators in request order."""
 
     context = _EvaluationContext(case, target_run, config or EvaluationConfig())
-    evaluators = _resolve_evaluators(evaluator_names)
+    evaluators = _resolve_evaluators(evaluator_names, context)
     return _run_evaluators(context, evaluators)
 
 
@@ -82,14 +86,58 @@ async def evaluate_case_async(
     return await asyncio.to_thread(evaluate_case, case, target_run, evaluator_names, config)
 
 
-def _resolve_evaluators(evaluator_names: list[str]) -> list[_EvaluatorDefinition]:
-    selected_names = evaluator_names or DEFAULT_EVALUATORS
+def _resolve_evaluators(
+    evaluator_names: list[str], context: _EvaluationContext
+) -> list[_EvaluatorDefinition]:
+    selected_names = evaluator_names or _automatic_evaluator_names(context.case, context.config)
     unsupported_names = [name for name in selected_names if name not in _EVALUATOR_REGISTRY]
     if unsupported_names:
         raise ValueError(f"不支持的评估器: {', '.join(unsupported_names)}")
     return [_EVALUATOR_REGISTRY[name] for name in selected_names]
 
 
+def resolve_evaluator_plan(
+    cases: list[EvalCase],
+    evaluator_names: list[str],
+    config: EvaluationConfig,
+) -> list[str]:
+    """Return the explicit or data-derived evaluator plan for an EvalSet."""
+
+    if evaluator_names:
+        unsupported_names = [name for name in evaluator_names if name not in _EVALUATOR_REGISTRY]
+        if unsupported_names:
+            raise ValueError(f"不支持的评估器: {', '.join(unsupported_names)}")
+        return list(evaluator_names)
+
+    plan: list[str] = []
+    for case in cases:
+        for evaluator_name in _automatic_evaluator_names(case, config):
+            if evaluator_name not in plan:
+                plan.append(evaluator_name)
+    return plan
+
+
+def _automatic_evaluator_names(case: EvalCase, config: EvaluationConfig) -> list[str]:
+    """Select only the evaluators whose business standard is present in a Case."""
+
+    names: list[str] = []
+    if _response_assertions(case):
+        names.append("response_contract@v1")
+    if _runtime_assertions(case):
+        names.append("runtime_budget@v1")
+    if _tool_requirements(case):
+        names.append("tool_trajectory@v1")
+
+    if _final_expected_output(case):
+        if _judge_unavailable_reason(config) is None:
+            names.insert(0, LLM_JUDGE_EVALUATOR)
+        else:
+            names.insert(0, REFERENCE_MATCH_EVALUATOR)
+    elif not _response_assertions(case):
+        names.insert(0, BUSINESS_STANDARD_EVALUATOR)
+    return names
+
+
 def _run_evaluators(
     context: _EvaluationContext,
     evaluators: list[_EvaluatorDefinition],
@@ -159,6 +207,21 @@ def _evaluate_llm_judge(
     return [_judge_score_metric(score, context.config.judge_model)]
 
 
+def _evaluate_business_standard(context: _EvaluationContext) -> list[MetricResult]:
+    """Prevent execution-only Cases from being reported as business-quality passes."""
+
+    return [
+        MetricResult(
+            name="response_quality",
+            status=MetricStatus.UNAVAILABLE,
+            evidence={
+                "evaluator": BUSINESS_STANDARD_EVALUATOR,
+                "reason": "Case 未提供响应业务标准",
+            },
+        )
+    ]
+
+
 def _evaluate_response_contract(
     context: _EvaluationContext,
 ) -> list[MetricResult]:
@@ -184,24 +247,102 @@ def _evaluate_runtime_budget(
 def _evaluate_tool_trajectory(
     context: _EvaluationContext,
 ) -> list[MetricResult]:
-    """Report unavailable until A2A exposes normalized tool trajectories."""
+    """Evaluate tool requirements against normalized RuntimeEvent evidence."""
 
     requirements = _tool_requirements(context.case)
     if not requirements:
         return []
 
-    return [
-        MetricResult(
+    if context.target_run.trace_ref is None:
+        return [
+            MetricResult(
+                name="tool_trajectory",
+                status=MetricStatus.UNAVAILABLE,
+                required=required,
+                evidence={
+                    "assertion": assertion_type,
+                    "tool": expected,
+                    "reason": "Target 未提供可查询的标准化工具轨迹",
+                },
+            )
+            for assertion_type, expected, required in requirements
+        ]
+
+    results: list[MetricResult] = []
+    for assertion_type, expected, required in requirements:
+        results.append(
+            _tool_metric(
+                assertion_type,
+                expected,
+                required,
+                context.target_run.tool_calls,
+            )
+        )
+    return results
+
+
+def _tool_metric(
+    assertion_type: str,
+    expected: str | list[str],
+    required: bool,
+    tool_calls: list[Any],
+) -> MetricResult:
+    if assertion_type == AssertionType.TOOL_SEQUENCE.value:
+        expected_sequence = list(expected) if isinstance(expected, list) else []
+        ordered_calls = sorted(
+            (call for call in tool_calls if call.status == "SUCCEEDED"),
+            key=lambda call: call.seq_start if call.seq_start is not None else float("inf"),
+        )
+        actual_sequence = [call.name for call in ordered_calls]
+        matched_calls = _ordered_tool_subsequence(ordered_calls, expected_sequence)
+        passed = len(matched_calls) == len(expected_sequence)
+        return MetricResult(
             name="tool_trajectory",
-            status=MetricStatus.UNAVAILABLE,
+            status=MetricStatus.PASS if passed else MetricStatus.FAIL,
+            score=1.0 if passed else 0.0,
             required=required,
             evidence={
                 "assertion": assertion_type,
-                "reason": "A2A target 未提供标准化工具轨迹",
+                "expectedSequence": expected_sequence,
+                "actualSequence": actual_sequence,
+                "matchedCallIds": matched_calls,
             },
         )
-        for assertion_type, required in requirements
-    ]
+
+    tool_name = str(expected)
+    matched = [call for call in tool_calls if call.name == tool_name]
+    if assertion_type == AssertionType.TOOL_NOT_CALLED.value:
+        passed = not matched
+        matched_ids = [call.call_id for call in matched]
+    elif assertion_type == AssertionType.TOOL_SUCCEEDED.value:
+        matched_ids = [call.call_id for call in matched if call.status == "SUCCEEDED"]
+        passed = bool(matched_ids)
+    else:
+        matched_ids = [call.call_id for call in matched]
+        passed = bool(matched_ids)
+    return MetricResult(
+        name="tool_trajectory",
+        status=MetricStatus.PASS if passed else MetricStatus.FAIL,
+        score=1.0 if passed else 0.0,
+        required=required,
+        evidence={
+            "assertion": assertion_type,
+            "tool": tool_name,
+            "matchedCallIds": matched_ids,
+        },
+    )
+
+
+def _ordered_tool_subsequence(tool_calls: list[Any], expected_sequence: list[str]) -> list[str]:
+    matched_call_ids: list[str] = []
+    expected_index = 0
+    for call in tool_calls:
+        if expected_index == len(expected_sequence):
+            break
+        if call.name == expected_sequence[expected_index]:
+            matched_call_ids.append(call.call_id)
+            expected_index += 1
+    return matched_call_ids
 
 
 def _response_assertions(case: EvalCase) -> list[AssertionSpec]:
@@ -220,16 +361,37 @@ def _runtime_assertions(case: EvalCase) -> list[AssertionSpec]:
     ]
 
 
-def _tool_requirements(case: EvalCase) -> list[tuple[str, bool]]:
+def _tool_requirements(case: EvalCase) -> list[tuple[str, str | list[str], bool]]:
     requirements = [
-        (assertion.type.value, assertion.required)
+        (
+            assertion.type.value,
+            assertion.value,
+            assertion.required,
+        )
         for assertion in case.assertions
-        if assertion.type in {AssertionType.TOOL_CALLED, AssertionType.TOOL_NOT_CALLED}
+        if assertion.type
+        in {
+            AssertionType.TOOL_CALLED,
+            AssertionType.TOOL_NOT_CALLED,
+            AssertionType.TOOL_SUCCEEDED,
+            AssertionType.TOOL_SEQUENCE,
+        }
     ]
-    requirements.extend(("tool.expected", True) for turn in case.turns for _ in turn.expected_tools)
+    requirements.extend(
+        ("tool.expected", str(tool.get("name") or ""), True)
+        for turn in case.turns
+        for tool in turn.expected_tools
+        if str(tool.get("name") or "").strip()
+    )
     return requirements
 
 
+def evaluate_tool_trajectory(case: EvalCase, target_run: TargetRun) -> list[MetricResult]:
+    """Compatibility entry point for direct deterministic tool evaluation."""
+
+    return _evaluate_tool_trajectory(_EvaluationContext(case, target_run, EvaluationConfig()))
+
+
 def _response_metric(assertion: AssertionSpec, output: str) -> MetricResult:
     reason = ""
     if assertion.type is AssertionType.RESPONSE_EQUALS:
@@ -262,6 +424,8 @@ def _runtime_metric(assertion: AssertionSpec, target_run: TargetRun) -> MetricRe
         actual = None
     elif assertion.type is AssertionType.RUNTIME_MAX_INPUT_TOKENS:
         actual = target_run.usage.input_tokens
+    elif assertion.type is AssertionType.RUNTIME_MAX_TOTAL_TOKENS:
+        actual = target_run.usage.total_tokens
     else:
         actual = target_run.usage.output_tokens
 
@@ -429,8 +593,7 @@ def _run_llm_judge(
     metric = GEval(
         name="Response quality",
         criteria=(
-            "Determine whether the actual output is factually correct "
-            "based on the expected output."
+            "Determine whether the actual output is factually correct based on the expected output."
         ),
         evaluation_params=[
             LLMTestCaseParams.INPUT,
@@ -451,6 +614,9 @@ def _run_llm_judge(
 
 
 _EVALUATORS = (
+    _EvaluatorDefinition(
+        BUSINESS_STANDARD_EVALUATOR, "response_quality", _evaluate_business_standard
+    ),
     _EvaluatorDefinition("response_contract@v1", "response_contract", _evaluate_response_contract),
     _EvaluatorDefinition("runtime_budget@v1", "runtime_budget", _evaluate_runtime_budget),
     _EvaluatorDefinition("tool_trajectory@v1", "tool_trajectory", _evaluate_tool_trajectory),