agentengine-sdk-python 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

ksadk/evaluation/evidence.py

@@ -0,0 +1,258 @@
+"""Policy-neutral RuntimeEvent projections used by evaluation adapters."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from ksadk.events.canonical import (
+    ItemCompleted,
+    ItemStarted,
+    RuntimeEvent,
+    dump_runtime_event,
+)
+from ksadk.events.content import ToolCallContent, ToolResultContent
+
+from .contracts import DataPolicy, ToolCallEvidence, TraceRef
+
+_SAFE_EVIDENCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$")
+
+
+class EvidenceStoreError(RuntimeError):
+    """Raised when evaluation evidence cannot be safely persisted or read."""
+
+
+class EvidenceStore:
+    """Persist queryable RuntimeEvent evidence below an evaluation report root."""
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root).expanduser().resolve()
+
+    def write_trace(
+        self,
+        run_id: str,
+        events: Iterable[RuntimeEvent],
+        *,
+        session_id: str,
+        policy: DataPolicy = DataPolicy.LOCAL_ONLY,
+    ) -> TraceRef:
+        event_list = sorted(events, key=lambda item: item.seq)
+        if not event_list:
+            raise EvidenceStoreError("RuntimeEvent evidence must not be empty")
+        if not session_id.strip():
+            raise EvidenceStoreError("RuntimeEvent evidence requires an explicit session_id")
+        invocation_ids = {event.run_id for event in event_list}
+        if len(invocation_ids) != 1:
+            raise EvidenceStoreError("RuntimeEvent evidence must describe one invocation")
+        invocation_id = next(iter(invocation_ids))
+        path = self._trace_path(run_id, session_id, invocation_id)
+        payload = {
+            "schemaVersion": "ksadk.eval.evidence/v2",
+            "runId": run_id,
+            "sessionId": session_id,
+            "invocationId": invocation_id,
+            "dataPolicy": policy.value,
+            "seqStart": event_list[0].seq,
+            "seqEnd": event_list[-1].seq,
+            "events": [_event_payload(event, policy) for event in event_list],
+        }
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = path.with_name(f".{path.name}.tmp")
+        try:
+            temporary.write_text(
+                json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
+                encoding="utf-8",
+            )
+            temporary.replace(path)
+        except OSError as exc:
+            temporary.unlink(missing_ok=True)
+            raise EvidenceStoreError("Unable to persist RuntimeEvent evidence") from exc
+        return TraceRef(
+            run_id=run_id,
+            session_id=session_id,
+            invocation_id=invocation_id,
+            seq_start=event_list[0].seq,
+            seq_end=event_list[-1].seq,
+        )
+
+    def read_trace(self, trace_ref: TraceRef) -> dict[str, Any]:
+        if not trace_ref.run_id or not trace_ref.session_id or not trace_ref.invocation_id:
+            raise EvidenceStoreError("TraceRef does not identify local evaluation evidence")
+        path = self._trace_path(
+            trace_ref.run_id,
+            trace_ref.session_id,
+            trace_ref.invocation_id,
+        )
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError) as exc:
+            raise EvidenceStoreError("RuntimeEvent evidence is missing or invalid") from exc
+        if (
+            payload.get("runId") != trace_ref.run_id
+            or payload.get("sessionId") != trace_ref.session_id
+            or payload.get("invocationId") != trace_ref.invocation_id
+        ):
+            raise EvidenceStoreError("RuntimeEvent evidence does not match TraceRef")
+        return payload
+
+    def _trace_path(self, run_id: str, session_id: str, invocation_id: str) -> Path:
+        for value in (run_id, session_id, invocation_id):
+            if not _SAFE_EVIDENCE_ID.fullmatch(value) or value in {".", ".."}:
+                raise EvidenceStoreError("Evidence identifiers must be path-safe")
+        identity = "\0".join((run_id, session_id, invocation_id)).encode("utf-8")
+        return self.root / "evidence" / f"{hashlib.sha256(identity).hexdigest()}.json"
+
+
+def project_tool_calls(events: Iterable[RuntimeEvent]) -> list[ToolCallEvidence]:
+    """Project tool lifecycle events without retaining arguments or results."""
+
+    calls: dict[str, ToolCallEvidence] = {}
+    order: list[str] = []
+    for event in sorted(events, key=lambda item: item.seq):
+        parts = ()
+        if isinstance(event, ItemStarted) and event.item_kind == "tool_call" and event.initial:
+            parts = event.initial.parts
+        elif isinstance(event, ItemCompleted) and event.item_kind == "tool_call":
+            parts = event.snapshot.parts
+        for part in parts:
+            if isinstance(part, ToolCallContent):
+                current = calls.get(part.call_id)
+                if current is None:
+                    order.append(part.call_id)
+                    current = ToolCallEvidence(
+                        call_id=part.call_id,
+                        name=part.name,
+                        status="INCOMPLETE",
+                        seq_start=event.seq,
+                    )
+                calls[part.call_id] = current
+            elif isinstance(part, ToolResultContent):
+                current = calls.get(part.call_id)
+                if current is None:
+                    # A provider may emit a terminal snapshot after reconnecting;
+                    # retain it as evidence instead of discarding the fact.
+                    order.append(part.call_id)
+                    current = ToolCallEvidence(
+                        call_id=part.call_id,
+                        name="unknown",
+                        status="INCOMPLETE",
+                    )
+                calls[part.call_id] = current.model_copy(
+                    update={
+                        "status": "ERROR" if part.is_error else "SUCCEEDED",
+                        "seq_end": event.seq,
+                    }
+                )
+    return [calls[call_id] for call_id in order]
+
+
+def _event_payload(event: RuntimeEvent, policy: DataPolicy) -> dict[str, Any]:
+    payload = dict(dump_runtime_event(event))
+    if policy is DataPolicy.METADATA_ONLY:
+        payload = _metadata_payload(event.event_type, payload)
+    elif policy is DataPolicy.REDACTED_TRACE:
+        payload = _redacted_payload(event.event_type, payload)
+    return {
+        "schemaVersion": event.schema_version,
+        "eventId": event.event_id,
+        "eventType": event.event_type,
+        "timestamp": event.timestamp,
+        "runId": event.run_id,
+        "seq": event.seq,
+        "event": payload,
+    }
+
+
+def _metadata_payload(event_type: str, payload: dict[str, Any]) -> dict[str, Any]:
+    allowed = {
+        "status",
+        "call_id",
+        "name",
+        "duration_ms",
+        "input_tokens",
+        "output_tokens",
+        "total_tokens",
+        "cached_tokens",
+        "reasoning_tokens",
+        "source",
+        "checkpoint_id",
+        "granularity",
+    }
+    return {key: value for key, value in payload.items() if key in allowed}
+
+
+def _redacted_payload(event_type: str, payload: dict[str, Any]) -> dict[str, Any]:
+    sensitive_keys = {
+        "text",
+        "summary",
+        "args",
+        "result",
+        "error",
+        "detail",
+        "artifact",
+        "data",
+        "content",
+        "prompt",
+        "input",
+        "output",
+        "message",
+        "messages",
+        "reasoning",
+        "headers",
+    }
+    safe_string_keys = {
+        "status",
+        "call_id",
+        "name",
+        "source",
+        "checkpoint_id",
+        "granularity",
+        "type",
+        "phase",
+        "role",
+        "finish_reason",
+    }
+
+    def sensitive(key: Any) -> bool:
+        normalized = str(key).strip().lower().replace("-", "_")
+        return normalized in sensitive_keys or any(
+            marker in normalized
+            for marker in ("secret", "password", "authorization", "credential", "api_key")
+        ) or normalized in {
+            "token",
+            "accesstoken",
+            "access_token",
+            "refreshtoken",
+            "refresh_token",
+            "authtoken",
+            "auth_token",
+            "bearer_token",
+            "id_token",
+            "api_token",
+        }
+
+    def redact_item(key: Any, value: Any) -> Any:
+        normalized = str(key).strip().lower().replace("-", "_")
+        if sensitive(key):
+            return "[REDACTED]"
+        if isinstance(value, str) and normalized not in safe_string_keys:
+            return "[REDACTED]"
+        return redact(value)
+
+    def redact(value: Any) -> Any:
+        if isinstance(value, dict):
+            return {key: redact_item(key, item) for key, item in value.items()}
+        if isinstance(value, list):
+            return [redact(item) for item in value]
+        if isinstance(value, tuple):
+            return [redact(item) for item in value]
+        return value
+
+    return redact(payload)
+
+
+__all__ = ["EvidenceStore", "EvidenceStoreError", "project_tool_calls"]

ksadk/evaluation/executor.py

@@ -0,0 +1,143 @@
+"""Single public evaluation handoff shared by CLI and Studio."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from uuid import uuid4
+
+from .adapters import EvaluationNotImplementedError, TargetAdapter
+from .contracts import (
+    CaseRun,
+    EvalRunReport,
+    EvalRunSpec,
+    EvalRunStatus,
+    EvaluationRequest,
+    MetricStatus,
+    TargetRunStatus,
+)
+from .evaluators import evaluate_case_async
+from .evidence import EvidenceStore
+from .storage import EvaluationStorage, EvaluationStorageError
+from .target import EvaluationExecutionError, EvaluationTarget
+
+__all__ = [
+    "EvaluationExecutionError",
+    "EvaluationNotImplementedError",
+    "TargetAdapter",
+    "execute_evaluation",
+]
+
+
+async def execute_evaluation(
+    request: EvaluationRequest,
+    *,
+    on_case_started: Callable[[str, int, int], None] | None = None,
+    adapter: TargetAdapter | None = None,
+    run_id: str | None = None,
+) -> EvalRunReport:
+    """Execute and persist one evaluation request."""
+
+    evidence_store = EvidenceStore(request.report_dir) if request.report_dir else None
+    target = EvaluationTarget(
+        request.target,
+        request.config,
+        evidence_store=evidence_store,
+        adapter=adapter,
+    )
+    snapshot = await target.snapshot()
+    spec = EvalRunSpec(
+        id=run_id or f"eval_{uuid4().hex}",
+        evalset=request.evalset,
+        target=snapshot,
+        config=request.config,
+        cloud_dataset=request.cloud_dataset,
+    )
+    case_runs: list[CaseRun] = []
+    try:
+        await _run_cases(
+            target,
+            spec,
+            case_runs=case_runs,
+            on_case_started=on_case_started,
+        )
+    except asyncio.CancelledError:
+        report = EvalRunReport(
+            spec=spec,
+            status=EvalRunStatus.CANCELLED,
+            case_runs=case_runs,
+        )
+        _persist_report(request, report)
+        raise
+    report = EvalRunReport(
+        spec=spec,
+        status=_report_status(case_runs),
+        case_runs=case_runs,
+    )
+    _persist_report(request, report)
+    return report
+
+
+async def _run_cases(
+    target: EvaluationTarget,
+    spec: EvalRunSpec,
+    *,
+    case_runs: list[CaseRun],
+    on_case_started: Callable[[str, int, int], None] | None,
+) -> None:
+    total_cases = len(spec.evalset.cases)
+    for index, case in enumerate(spec.evalset.cases, start=1):
+        _notify_case_started(on_case_started, case.id, index, total_cases)
+        target_run = await target.run_case(spec, case)
+        try:
+            metrics = await evaluate_case_async(
+                case, target_run, spec.config.evaluators, spec.config
+            )
+        except ValueError as exc:
+            raise EvaluationExecutionError(str(exc)) from exc
+        case_run = CaseRun(
+            case_id=case.id,
+            attempt=spec.attempt,
+            target_run=target_run,
+            metrics=metrics,
+        )
+        case_runs.append(case_run)
+        if spec.config.fail_fast and not case_run.passed:
+            break
+
+
+def _notify_case_started(
+    callback: Callable[[str, int, int], None] | None,
+    case_id: str,
+    index: int,
+    total_cases: int,
+) -> None:
+    if callback is None:
+        return
+    try:
+        callback(case_id, index, total_cases)
+    except Exception:
+        pass
+
+
+def _report_status(case_runs: list[CaseRun]) -> EvalRunStatus:
+    if any(
+        case.target_run.status is TargetRunStatus.ERROR
+        or any(metric.required and metric.status is MetricStatus.ERROR for metric in case.metrics)
+        for case in case_runs
+    ):
+        return EvalRunStatus.ERROR
+    if any(case.target_run.status is TargetRunStatus.CANCELLED for case in case_runs):
+        return EvalRunStatus.CANCELLED
+    if any(not case.passed for case in case_runs):
+        return EvalRunStatus.FAILED
+    return EvalRunStatus.PASSED
+
+
+def _persist_report(request: EvaluationRequest, report: EvalRunReport) -> None:
+    if not request.report_dir:
+        return
+    try:
+        EvaluationStorage(request.report_dir).write_report(report)
+    except EvaluationStorageError as exc:
+        raise EvaluationExecutionError("评测报告写入失败") from exc