agentengine-sdk-python 0.8.4__py3-none-any.whl → 0.8.6__py3-none-any.whl

ksadk/evaluation/evaluators.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import hashlib
 import json
 import os
 import re
@@ -54,6 +55,8 @@ LEGACY_ASSERTION_EVALUATORS = (
     "response_contract@v1",
     "runtime_budget@v1",
     "tool_trajectory@v1",
+    "artifact_contract@v1",
+    "session_contract@v1",
 )
 # Kept for external imports only. Empty evaluator selection uses the
 # data-derived automatic plan in _automatic_evaluator_names instead.
@@ -127,6 +130,10 @@ def _automatic_evaluator_names(case: EvalCase, config: EvaluationConfig) -> list
         names.append("runtime_budget@v1")
     if _tool_requirements(case):
         names.append("tool_trajectory@v1")
+    if _artifact_assertions(case):
+        names.append("artifact_contract@v1")
+    if _session_assertions(case):
+        names.append("session_contract@v1")
 
     if _final_expected_output(case):
         if _judge_unavailable_reason(config) is None:
@@ -281,6 +288,19 @@ def _evaluate_tool_trajectory(
     return results
 
 
+def _evaluate_artifact_contract(context: _EvaluationContext) -> list[MetricResult]:
+    """Evaluate declared file outcomes using content-free adapter evidence."""
+
+    return [
+        _artifact_metric(assertion, context.target_run)
+        for assertion in _artifact_assertions(context.case)
+    ]
+
+
+def _evaluate_session_contract(context: _EvaluationContext) -> list[MetricResult]:
+    return [_session_metric(assertion, context) for assertion in _session_assertions(context.case)]
+
+
 def _tool_metric(
     assertion_type: str,
     expected: str | list[str],
@@ -361,6 +381,27 @@ def _runtime_assertions(case: EvalCase) -> list[AssertionSpec]:
     ]
 
 
+def _artifact_assertions(case: EvalCase) -> list[AssertionSpec]:
+    return [
+        assertion
+        for assertion in case.assertions
+        if assertion.type
+        in {
+            AssertionType.ARTIFACT_FILE_EXISTS,
+            AssertionType.ARTIFACT_FILE_EQUALS,
+            AssertionType.ARTIFACT_FILE_CONTAINS,
+        }
+    ]
+
+
+def _session_assertions(case: EvalCase) -> list[AssertionSpec]:
+    return [
+        assertion
+        for assertion in case.assertions
+        if assertion.type is AssertionType.SESSION_SAME_ACROSS_TURNS
+    ]
+
+
 def _tool_requirements(case: EvalCase) -> list[tuple[str, str | list[str], bool]]:
     requirements = [
         (
@@ -418,8 +459,37 @@ def _response_metric(assertion: AssertionSpec, output: str) -> MetricResult:
 
 def _runtime_metric(assertion: AssertionSpec, target_run: TargetRun) -> MetricResult:
     actual: int | None
+    minimum = assertion.type.value.startswith("runtime.min")
     if assertion.type is AssertionType.RUNTIME_MAX_LATENCY_MS:
         actual = target_run.duration_ms
+    elif assertion.type is AssertionType.RUNTIME_MAX_TERMINAL_EVENTS_PER_SCOPE:
+        actual = _metadata_count(target_run, "maxTerminalEventsPerScope")
+    elif assertion.type is AssertionType.RUNTIME_MAX_RETRIES:
+        actual = _metadata_count(target_run, "retryCount")
+    elif assertion.type is AssertionType.RUNTIME_MAX_COMPACTIONS:
+        actual = _metadata_count(target_run, "compactionCount")
+    elif assertion.type is AssertionType.RUNTIME_MAX_SUBAGENTS:
+        actual = _metadata_count(target_run, "subagentCount")
+    elif assertion.type is AssertionType.RUNTIME_MAX_TOOL_CALLS:
+        actual = _metadata_count(target_run, "toolCallCount")
+    elif assertion.type is AssertionType.RUNTIME_MAX_SKILL_DISCLOSURES:
+        actual = _metadata_count(target_run, "skillDisclosureCount")
+    elif assertion.type is AssertionType.RUNTIME_MAX_MCP_DISCLOSURES:
+        actual = _metadata_count(target_run, "mcpDisclosureCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_RETRIES:
+        actual = _metadata_count(target_run, "retryCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_COMPACTIONS:
+        actual = _metadata_count(target_run, "compactionCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_SUBAGENTS:
+        actual = _metadata_count(target_run, "subagentCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_MEMORY_RECALLS:
+        actual = _metadata_count(target_run, "memoryRecallCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_MEMORY_WRITES:
+        actual = _metadata_count(target_run, "memoryWriteCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_SKILL_DISCLOSURES:
+        actual = _metadata_count(target_run, "skillDisclosureCount")
+    elif assertion.type is AssertionType.RUNTIME_MIN_MCP_DISCLOSURES:
+        actual = _metadata_count(target_run, "mcpDisclosureCount")
     elif not target_run.usage.reported:
         actual = None
     elif assertion.type is AssertionType.RUNTIME_MAX_INPUT_TOKENS:
@@ -442,11 +512,104 @@ def _runtime_metric(assertion: AssertionSpec, target_run: TargetRun) -> MetricRe
     return _metric(
         "runtime_budget",
         assertion,
-        passed=actual <= assertion.value,
+        passed=actual >= assertion.value if minimum else actual <= assertion.value,
         evidence={"actual": actual},
     )
 
 
+def _artifact_metric(assertion: AssertionSpec, target_run: TargetRun) -> MetricResult:
+    if isinstance(assertion.value, str):
+        path = assertion.value
+        expected_digest = None
+    else:
+        path = str(assertion.value["path"])
+        expected_digest = (
+            "sha256:" + hashlib.sha256(str(assertion.value["content"]).encode("utf-8")).hexdigest()
+        )
+    match = next(
+        (
+            check
+            for check in target_run.file_checks
+            if check.assertion_type == assertion.type.value
+            and check.path == path
+            and check.expected_digest == expected_digest
+        ),
+        None,
+    )
+    if match is None or match.status == "UNAVAILABLE":
+        return MetricResult(
+            name="artifact_contract",
+            status=MetricStatus.UNAVAILABLE,
+            required=assertion.required,
+            evidence={
+                "assertion": assertion.type.value,
+                "path": path,
+                "reason": "Target 未提供该工作区文件证据",
+            },
+        )
+    passed = match.status in {"PRESENT", "MATCH"}
+    evidence: dict[str, Any] = {
+        "assertion": assertion.type.value,
+        "path": path,
+        "status": match.status,
+    }
+    if match.size_bytes is not None:
+        evidence["sizeBytes"] = match.size_bytes
+    if match.actual_sha256 is not None:
+        evidence["actualSha256"] = match.actual_sha256
+    return MetricResult(
+        name="artifact_contract",
+        status=MetricStatus.PASS if passed else MetricStatus.FAIL,
+        score=1.0 if passed else 0.0,
+        required=assertion.required,
+        evidence=evidence,
+    )
+
+
+def _session_metric(
+    assertion: AssertionSpec,
+    context: _EvaluationContext,
+) -> MetricResult:
+    expected_turns = len(context.case.turns)
+    trace_refs = context.target_run.trace_refs
+    session_ids = [trace.session_id for trace in trace_refs if trace.session_id]
+    if (
+        expected_turns < 2
+        or len(trace_refs) != expected_turns
+        or len(session_ids) != expected_turns
+    ):
+        return MetricResult(
+            name="session_contract",
+            status=MetricStatus.UNAVAILABLE,
+            required=assertion.required,
+            evidence={
+                "assertion": assertion.type.value,
+                "expectedTurns": expected_turns,
+                "observedTurns": len(session_ids),
+                "reason": "Target 未提供完整的逐轮 Session 证据",
+            },
+        )
+    passed = len(set(session_ids)) == 1
+    return MetricResult(
+        name="session_contract",
+        status=MetricStatus.PASS if passed else MetricStatus.FAIL,
+        score=1.0 if passed else 0.0,
+        required=assertion.required,
+        evidence={
+            "assertion": assertion.type.value,
+            "turnCount": expected_turns,
+            "distinctSessionCount": len(set(session_ids)),
+        },
+    )
+
+
+def _metadata_count(target_run: TargetRun, key: str) -> int | None:
+    value = target_run.metadata.get(key)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        return None
+    return value
+
+
 def _metric(
     name: str,
     assertion: AssertionSpec,
@@ -620,6 +783,8 @@ _EVALUATORS = (
     _EvaluatorDefinition("response_contract@v1", "response_contract", _evaluate_response_contract),
     _EvaluatorDefinition("runtime_budget@v1", "runtime_budget", _evaluate_runtime_budget),
     _EvaluatorDefinition("tool_trajectory@v1", "tool_trajectory", _evaluate_tool_trajectory),
+    _EvaluatorDefinition("artifact_contract@v1", "artifact_contract", _evaluate_artifact_contract),
+    _EvaluatorDefinition("session_contract@v1", "session_contract", _evaluate_session_contract),
     _EvaluatorDefinition(REFERENCE_MATCH_EVALUATOR, "response_match", _evaluate_reference_match),
     _EvaluatorDefinition(LLM_JUDGE_EVALUATOR, "response_quality", _evaluate_llm_judge),
 )

ksadk/evaluation/evidence.py

@@ -10,16 +10,29 @@ from pathlib import Path
 from typing import Any
 
 from ksadk.events.canonical import (
+    ContextCompactionCompleted,
     ItemCompleted,
     ItemStarted,
+    ItemUpdated,
+    RunCanceled,
+    RunCompleted,
+    RunFailed,
     RuntimeEvent,
     dump_runtime_event,
 )
-from ksadk.events.content import ToolCallContent, ToolResultContent
+from ksadk.events.content import TextContent, ToolCallContent, ToolResultContent
 
-from .contracts import DataPolicy, ToolCallEvidence, TraceRef
+from .contracts import (
+    AssertionType,
+    DataPolicy,
+    EvalCase,
+    FileCheckEvidence,
+    ToolCallEvidence,
+    TraceRef,
+)
 
 _SAFE_EVIDENCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$")
+_MAX_VERIFICATION_FILE_BYTES = 8 * 1024 * 1024
 
 
 class EvidenceStoreError(RuntimeError):
@@ -108,16 +121,58 @@ class EvidenceStore:
 
 
 def project_tool_calls(events: Iterable[RuntimeEvent]) -> list[ToolCallEvidence]:
-    """Project tool lifecycle events without retaining arguments or results."""
+    """Project tool lifecycle events without retaining arguments or results.
+
+    Framework adapters normally emit canonical ``tool_call`` items.  Managed
+    Harness also publishes its stable public lifecycle as structured status
+    facts (``tool.call.begin`` / ``tool.call.end``).  Accept both projections
+    so evaluation measures the same execution fact instead of the adapter's
+    presentation shape.
+    """
 
     calls: dict[str, ToolCallEvidence] = {}
     order: list[str] = []
     for event in sorted(events, key=lambda item: item.seq):
+        status_payload = _status_payload(event)
+        status_event = str(status_payload.get("event") or "")
+        status_details = status_payload.get("details")
+        if status_event in {"tool.call.begin", "tool.call.end"} and isinstance(
+            status_details, dict
+        ):
+            call_id = str(
+                status_details.get("call_id") or status_details.get("callId") or ""
+            ).strip()
+            name = str(status_details.get("name") or "unknown").strip() or "unknown"
+            if call_id:
+                current = calls.get(call_id)
+                if current is None:
+                    order.append(call_id)
+                    current = ToolCallEvidence(
+                        call_id=call_id,
+                        name=name,
+                        status="INCOMPLETE",
+                        seq_start=event.seq,
+                    )
+                if status_event == "tool.call.end":
+                    terminal = str(status_details.get("status") or "").lower()
+                    current = current.model_copy(
+                        update={
+                            "name": name if current.name == "unknown" else current.name,
+                            "status": (
+                                "SUCCEEDED"
+                                if terminal in {"completed", "succeeded", "success"}
+                                else "ERROR"
+                            ),
+                            "seq_end": event.seq,
+                        }
+                    )
+                calls[call_id] = current
         parts = ()
         if isinstance(event, ItemStarted) and event.item_kind == "tool_call" and event.initial:
             parts = event.initial.parts
         elif isinstance(event, ItemCompleted) and event.item_kind == "tool_call":
             parts = event.snapshot.parts
+        completed_call_ids: list[str] = []
         for part in parts:
             if isinstance(part, ToolCallContent):
                 current = calls.get(part.call_id)
@@ -130,6 +185,8 @@ def project_tool_calls(events: Iterable[RuntimeEvent]) -> list[ToolCallEvidence]
                         seq_start=event.seq,
                     )
                 calls[part.call_id] = current
+                if isinstance(event, ItemCompleted):
+                    completed_call_ids.append(part.call_id)
             elif isinstance(part, ToolResultContent):
                 current = calls.get(part.call_id)
                 if current is None:
@@ -147,9 +204,213 @@ def project_tool_calls(events: Iterable[RuntimeEvent]) -> list[ToolCallEvidence]
                         "seq_end": event.seq,
                     }
                 )
+        # Some canonical framework projectors close the tool item without
+        # repeating its result payload.  The completed lifecycle is still a
+        # terminal success fact; an explicit ToolResultContent error above
+        # remains authoritative and is never overwritten here.
+        for call_id in completed_call_ids:
+            current = calls[call_id]
+            if current.status == "INCOMPLETE":
+                calls[call_id] = current.model_copy(
+                    update={"status": "SUCCEEDED", "seq_end": event.seq}
+                )
     return [calls[call_id] for call_id in order]
 
 
+def project_runtime_facts(events: Iterable[RuntimeEvent]) -> dict[str, int]:
+    """Derive non-sensitive, deterministic counters from canonical events.
+
+    Counts are deliberately based on event identity and lifecycle metadata,
+    never model reasoning, tool arguments or tool results.  This makes them
+    suitable for same-task Harness comparisons without leaking trace content.
+    """
+
+    terminal_by_scope: dict[tuple[str, str], set[tuple[str, str]]] = {}
+    retry_ids: set[tuple[str, str]] = set()
+    compaction_ids: set[tuple[str, str]] = set()
+    child_scopes: set[tuple[str, str]] = set()
+    subagent_call_ids: set[tuple[str, str]] = set()
+    memory_recall_ids: set[tuple[str, str]] = set()
+    memory_write_ids: set[tuple[str, str]] = set()
+    skill_disclosure_ids: set[tuple[str, str]] = set()
+    mcp_disclosure_ids: set[tuple[str, str]] = set()
+
+    for event in sorted(events, key=lambda item: item.seq):
+        source = event.source
+        native_run_id = str(source.native_run_id or event.run_id)
+        native_event_id = str(source.native_event_id or event.event_id)
+        identity = (native_run_id, native_event_id)
+
+        if isinstance(event, (RunCompleted, RunFailed, RunCanceled)):
+            terminal_by_scope.setdefault((event.run_id, event.scope_id), set()).add(identity)
+        native_type = str(source.metadata.get("native_event_type") or "")
+        if (
+            isinstance(event, ContextCompactionCompleted)
+            or native_type == "context.compaction.completed"
+        ):
+            compaction_ids.add(identity)
+        # Some provider-native status facts use the current scope as their
+        # parent for grouping.  That is not a child Agent and must not satisfy
+        # a subagent assertion.
+        if event.parent_scope_id and event.parent_scope_id != event.scope_id:
+            child_scopes.add((event.run_id, event.scope_id))
+
+        details = _status_details(event)
+        if native_type == "memory.recalled":
+            memory_recall_ids.add(identity)
+        elif native_type == "memory.write" and str(details.get("decision") or "") == "commit":
+            memory_write_ids.add(identity)
+        elif native_type == "skill.disclosed":
+            skill_disclosure_ids.add(identity)
+        elif native_type == "mcp.disclosed":
+            mcp_disclosure_ids.add(identity)
+        detail_kind = str(source.metadata.get("activity_kind") or details.get("kind") or "")
+        failure_action = str(source.metadata.get("failure_action") or "")
+        if (
+            native_type == "model.call.failed"
+            and (
+                failure_action == "retry_same_model"
+                or (
+                    not failure_action
+                    and getattr(event, "item_kind", "") != "status"
+                )
+            )
+        ) or detail_kind == "provider.retry":
+            retry_ids.add(identity)
+        if detail_kind in {"delegation.route", "subagent.event"}:
+            call_id = str(
+                source.metadata.get("activity_call_id")
+                or details.get("call_id")
+                or details.get("callId")
+                or ""
+            ).strip()
+            if call_id:
+                subagent_call_ids.add((native_run_id, call_id))
+
+    return {
+        "maxTerminalEventsPerScope": max(
+            (len(identities) for identities in terminal_by_scope.values()),
+            default=0,
+        ),
+        "retryCount": len(retry_ids),
+        "compactionCount": len(compaction_ids),
+        "subagentCount": (
+            len(subagent_call_ids) if subagent_call_ids else len(child_scopes)
+        ),
+        "memoryRecallCount": len(memory_recall_ids),
+        "memoryWriteCount": len(memory_write_ids),
+        "skillDisclosureCount": len(skill_disclosure_ids),
+        "mcpDisclosureCount": len(mcp_disclosure_ids),
+    }
+
+
+def project_file_checks(case: EvalCase, root: Path | None) -> list[FileCheckEvidence]:
+    """Check declared relative files without retaining their generated content."""
+
+    checks: list[FileCheckEvidence] = []
+    file_assertion_types = {
+        AssertionType.ARTIFACT_FILE_EXISTS,
+        AssertionType.ARTIFACT_FILE_EQUALS,
+        AssertionType.ARTIFACT_FILE_CONTAINS,
+    }
+    for assertion in case.assertions:
+        if assertion.type not in file_assertion_types:
+            continue
+        if isinstance(assertion.value, str):
+            relative_path = assertion.value
+            expected = None
+        else:
+            relative_path = str(assertion.value["path"])
+            expected = str(assertion.value["content"])
+        expected_digest = (
+            f"sha256:{hashlib.sha256(expected.encode('utf-8')).hexdigest()}"
+            if expected is not None
+            else None
+        )
+        base = {
+            "assertion_type": assertion.type.value,
+            "path": relative_path,
+            "expected_digest": expected_digest,
+        }
+        if root is None:
+            checks.append(FileCheckEvidence(**base, status="UNAVAILABLE"))
+            continue
+        try:
+            resolved_root = root.resolve(strict=True)
+            target = (resolved_root / relative_path).resolve(strict=False)
+            if not target.is_relative_to(resolved_root):
+                checks.append(FileCheckEvidence(**base, status="ERROR"))
+                continue
+            if not target.exists():
+                checks.append(FileCheckEvidence(**base, status="MISSING"))
+                continue
+            if not target.is_file():
+                checks.append(FileCheckEvidence(**base, status="NOT_FILE"))
+                continue
+            size = target.stat().st_size
+            if assertion.type is AssertionType.ARTIFACT_FILE_EXISTS:
+                checks.append(FileCheckEvidence(**base, status="PRESENT", size_bytes=size))
+                continue
+            if size > _MAX_VERIFICATION_FILE_BYTES:
+                checks.append(FileCheckEvidence(**base, status="TOO_LARGE", size_bytes=size))
+                continue
+            content = target.read_bytes()
+            actual_sha256 = f"sha256:{hashlib.sha256(content).hexdigest()}"
+            expected_bytes = (expected or "").encode("utf-8")
+            matched = (
+                content == expected_bytes
+                if assertion.type is AssertionType.ARTIFACT_FILE_EQUALS
+                else expected_bytes in content
+            )
+            checks.append(
+                FileCheckEvidence(
+                    **base,
+                    status="MATCH" if matched else "MISMATCH",
+                    actual_sha256=actual_sha256,
+                    size_bytes=len(content),
+                )
+            )
+        except OSError:
+            checks.append(FileCheckEvidence(**base, status="ERROR"))
+    return checks
+
+
+def _status_details(event: RuntimeEvent) -> dict[str, Any]:
+    """Read only structured status facts emitted by the canonical projector."""
+
+    payload = _status_payload(event)
+    details = payload.get("details")
+    return details if isinstance(details, dict) else {}
+
+
+def _status_payload(event: RuntimeEvent) -> dict[str, Any]:
+    """Parse one structured status payload without exposing free-form text."""
+
+    if getattr(event, "item_kind", "") != "status":
+        return {}
+    parts: tuple[Any, ...] = ()
+    if isinstance(event, ItemStarted) and event.initial:
+        parts = event.initial.parts
+    elif isinstance(event, ItemCompleted):
+        parts = event.snapshot.parts
+    elif isinstance(event, ItemUpdated):
+        parts = (event.update,)
+    for part in parts:
+        if not isinstance(part, TextContent):
+            continue
+        try:
+            payload = json.loads(part.text)
+        except (TypeError, ValueError):
+            continue
+        if not isinstance(payload, dict):
+            continue
+        event_name = payload.get("event")
+        details = payload.get("details")
+        if isinstance(event_name, str) and isinstance(details, dict):
+            return {"event": event_name, "details": details}
+    return {}
+
+
 def _event_payload(event: RuntimeEvent, policy: DataPolicy) -> dict[str, Any]:
     payload = dict(dump_runtime_event(event))
     if policy is DataPolicy.METADATA_ONLY:
@@ -255,4 +516,10 @@ def _redacted_payload(event_type: str, payload: dict[str, Any]) -> dict[str, Any
     return redact(payload)
 
 
-__all__ = ["EvidenceStore", "EvidenceStoreError", "project_tool_calls"]
+__all__ = [
+    "EvidenceStore",
+    "EvidenceStoreError",
+    "project_file_checks",
+    "project_runtime_facts",
+    "project_tool_calls",
+]