agentengine-sdk-python 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

ksadk/evaluation/cloud_service.py

@@ -0,0 +1,199 @@
+"""Cloud EvalSet publication orchestration shared by CLI and Studio."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Protocol
+
+from pydantic import Field
+
+from .cloud_binding import CloudBinding, CloudBindingStore
+from .cloud_converter import (
+    CloudDatasetSnapshot,
+    evalset_from_dataset_snapshot,
+    evalset_to_dataset_snapshot,
+)
+from .contracts import CloudDatasetRef, DataPolicy, EvalSetVersion, EvaluationModel
+
+
+class CloudEvalSetPreviewError(ValueError):
+    """Raised before an EvalSet body is allowed to leave the local process."""
+
+
+class CloudEvalSetPublishResult(EvaluationModel):
+    """Provider acknowledgement for one immutable Dataset snapshot."""
+
+    dataset_id: str = Field(min_length=1)
+    dataset_version: int = Field(ge=1)
+    project_id: str | None = None
+    schema_hash: str = Field(min_length=64, max_length=64)
+    content_digest: str = Field(min_length=64, max_length=64)
+    row_count: int = Field(ge=0)
+
+
+class CloudEvalSetPullResult(EvaluationModel):
+    """One validated immutable cloud snapshot and its normalized local form."""
+
+    snapshot: CloudDatasetSnapshot
+    evalset: EvalSetVersion
+    cloud_dataset: CloudDatasetRef
+
+
+class CloudEvalSetCatalogItem(EvaluationModel):
+    """One immutable Dataset version exposed by the cloud catalog."""
+
+    dataset_id: str = Field(min_length=1)
+    name: str = Field(min_length=1)
+    project_id: str | None = None
+    version: int = Field(ge=1)
+    schema_hash: str = Field(min_length=64, max_length=64)
+    content_digest: str = Field(min_length=64, max_length=64)
+    row_count: int = Field(ge=0)
+
+
+class CloudDatasetClient(Protocol):
+    """Minimal provider contract required before a snapshot can be published."""
+
+    async def publish_snapshot(
+        self,
+        snapshot: CloudDatasetSnapshot,
+        *,
+        dataset_id: str | None,
+        base_version: int | None,
+        idempotency_key: str,
+    ) -> CloudEvalSetPublishResult: ...
+
+    async def read_snapshot(
+        self,
+        dataset_id: str,
+        version: int,
+        *,
+        project_id: str | None = None,
+    ) -> CloudDatasetSnapshot: ...
+
+    async def list_datasets(
+        self,
+        *,
+        project_id: str | None = None,
+    ) -> list[CloudEvalSetCatalogItem]: ...
+
+
+class CloudEvalSetService:
+    """Validate, publish, and bind EvalSets without changing runtime execution."""
+
+    def __init__(
+        self,
+        workspace_root: str | Path,
+        client: CloudDatasetClient,
+        *,
+        provider: str = "agent-eval/evalsmith",
+    ):
+        self.bindings = CloudBindingStore(workspace_root)
+        self.client = client
+        self.provider = provider
+
+    def preview(self, evalset: EvalSetVersion, *, data_policy: DataPolicy) -> CloudDatasetSnapshot:
+        """Return the exact outgoing snapshot or fail before any network request."""
+
+        if data_policy is DataPolicy.LOCAL_ONLY:
+            raise CloudEvalSetPreviewError("DataPolicy=local_only 禁止上传 EvalSet 正文")
+        if data_policy is DataPolicy.METADATA_ONLY:
+            raise CloudEvalSetPreviewError(
+                "DataPolicy=metadata_only 不能发布包含 Case 正文的 EvalSet"
+            )
+        if data_policy is not DataPolicy.FULL_TRACE:
+            raise CloudEvalSetPreviewError("当前端云评测集发布仅支持显式 DataPolicy=full_trace")
+        return evalset_to_dataset_snapshot(evalset)
+
+    async def publish(
+        self,
+        evalset: EvalSetVersion,
+        *,
+        evalset_path: str,
+        data_policy: DataPolicy,
+        dataset_id: str | None = None,
+        idempotency_key: str | None = None,
+    ) -> CloudEvalSetPublishResult:
+        """Publish a full snapshot to an existing Dataset and advance its binding."""
+
+        if idempotency_key is not None and not idempotency_key.strip():
+            raise ValueError("Idempotency-Key 不能为空")
+        snapshot = self.preview(evalset, data_policy=data_policy)
+        existing = self.bindings.read(evalset_path)
+        requested_dataset_id = str(dataset_id or "").strip() or None
+        target_dataset_id = requested_dataset_id or (existing.dataset_id if existing else None)
+        if target_dataset_id is None:
+            raise ValueError("datasetId is required for the first publish of an EvalSet")
+        resolved_idempotency_key = idempotency_key or self._idempotency_key(
+            target_dataset_id,
+            snapshot.content_digest,
+        )
+        result = await self.client.publish_snapshot(
+            snapshot,
+            dataset_id=target_dataset_id,
+            base_version=None,
+            idempotency_key=resolved_idempotency_key,
+        )
+        if result.dataset_id != target_dataset_id:
+            raise CloudEvalSetPreviewError("云端返回的 datasetId 与目标 Dataset 不一致")
+        if result.content_digest != snapshot.content_digest:
+            raise CloudEvalSetPreviewError("云端返回的 contentDigest 与本地预检结果不一致")
+        if result.schema_hash != snapshot.schema_hash:
+            raise CloudEvalSetPreviewError("云端返回的 schemaHash 与本地预检结果不一致")
+        if result.row_count != len(snapshot.rows):
+            raise CloudEvalSetPreviewError("云端返回的 RowCount 与本地预检结果不一致")
+        self.bindings.write(
+            CloudBinding(
+                evalset_path=evalset_path,
+                content_digest=snapshot.content_digest,
+                provider=self.provider,
+                project_id=result.project_id,
+                dataset_id=result.dataset_id,
+                dataset_version=result.dataset_version,
+                schema_hash=result.schema_hash,
+            )
+        )
+        return result
+
+    @staticmethod
+    def _idempotency_key(dataset_id: str, content_digest: str) -> str:
+        identity = f"{dataset_id}:{content_digest}".encode("utf-8")
+        return f"ksadk-evalset-{hashlib.sha256(identity).hexdigest()}"
+
+    async def pull(
+        self,
+        *,
+        dataset_id: str,
+        version: int,
+        project_id: str | None = None,
+    ) -> CloudEvalSetPullResult:
+        """Read and validate one immutable Dataset version before execution."""
+
+        if not dataset_id.strip() or version < 1:
+            raise ValueError("datasetId 和 version 必须有效")
+        snapshot = await self.client.read_snapshot(
+            dataset_id,
+            version,
+            project_id=project_id,
+        )
+        evalset = evalset_from_dataset_snapshot(snapshot)
+        if evalset.content_digest != snapshot.content_digest:
+            raise CloudEvalSetPreviewError("云端 snapshot 的 contentDigest 校验失败")
+        reference = CloudDatasetRef(
+            provider=self.provider,
+            project_id=project_id,
+            dataset_id=dataset_id,
+            version=version,
+            schema_hash=snapshot.schema_hash,
+            content_digest=snapshot.content_digest,
+            row_count=len(snapshot.rows),
+        )
+        return CloudEvalSetPullResult(
+            snapshot=snapshot,
+            evalset=evalset,
+            cloud_dataset=reference,
+        )
+
+    async def catalog(self, *, project_id: str | None = None) -> list[CloudEvalSetCatalogItem]:
+        return await self.client.list_datasets(project_id=project_id)

ksadk/evaluation/contracts.py

@@ -0,0 +1,478 @@
+"""Stable, serializable data objects shared by CLI and Studio evaluation."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+# ---------------------------------------------------------------------------
+# Base model & helpers
+# ---------------------------------------------------------------------------
+
+
+def _to_camel_case(value: str) -> str:
+    head, *tail = value.split("_")
+    return head + "".join(part.capitalize() for part in tail)
+
+
+def _content_digest(payload: dict[str, Any]) -> str:
+    import hashlib
+    import json
+
+    encoded = json.dumps(
+        payload,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    ).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+class EvaluationModel(BaseModel):
+    """Use camelCase on the wire while keeping Python fields snake_case."""
+
+    model_config = ConfigDict(
+        alias_generator=_to_camel_case,
+        populate_by_name=True,
+        extra="forbid",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+
+# Assertion kinds & data egress policy (configure how cases are asserted and
+# what data may leave the local process).
+class AssertionType(str, Enum):
+    """Deterministic assertion kinds supported by the first evaluation phase."""
+
+    RESPONSE_EQUALS = "response.equals"
+    RESPONSE_CONTAINS = "response.contains"
+    RESPONSE_NOT_CONTAINS = "response.notContains"
+    RESPONSE_JSON_SCHEMA = "response.jsonSchema"
+    RUNTIME_MAX_LATENCY_MS = "runtime.maxLatencyMs"
+    RUNTIME_MAX_INPUT_TOKENS = "runtime.maxInputTokens"
+    RUNTIME_MAX_OUTPUT_TOKENS = "runtime.maxOutputTokens"
+    RUNTIME_MAX_TOTAL_TOKENS = "runtime.maxTotalTokens"
+    TOOL_CALLED = "tool.called"
+    TOOL_NOT_CALLED = "tool.notCalled"
+    TOOL_SUCCEEDED = "tool.succeeded"
+    TOOL_SEQUENCE = "tool.sequence"
+
+
+class DataPolicy(str, Enum):
+    """Controls what evaluation data may leave the local process."""
+
+    LOCAL_ONLY = "local_only"
+    METADATA_ONLY = "metadata_only"
+    REDACTED_TRACE = "redacted_trace"
+    FULL_TRACE = "full_trace"
+
+
+# Lifecycle & outcome statuses (run-level, per-case target invocation, and
+# per-metric verdicts).
+class EvalRunStatus(str, Enum):
+    """Lifecycle status for an entire evaluation run."""
+
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    ERROR = "ERROR"
+    CANCELLED = "CANCELLED"
+
+
+class TargetRunStatus(str, Enum):
+    """Terminal status returned by one target invocation."""
+
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    ERROR = "ERROR"
+    CANCELLED = "CANCELLED"
+    UNAVAILABLE = "UNAVAILABLE"
+
+
+class MetricStatus(str, Enum):
+    """Outcome of a single deterministic metric."""
+
+    PASS = "PASS"
+    FAIL = "FAIL"
+    UNAVAILABLE = "UNAVAILABLE"
+    ERROR = "ERROR"
+
+
+# ---------------------------------------------------------------------------
+# Evaluation set & input cases
+# ---------------------------------------------------------------------------
+
+
+class EvalTurn(EvaluationModel):
+    """A single conversational turn within a multi-turn case."""
+
+    input: str = Field(min_length=1, max_length=32768)
+    expected_output: str | None = Field(default=None, max_length=32768)
+    expected_tools: list[dict[str, Any]] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class AssertionSpec(EvaluationModel):
+    """A single deterministic assertion with type-dependent value validation."""
+
+    type: AssertionType
+    value: Any
+    required: bool = True
+
+    @model_validator(mode="after")
+    def validate_value(self) -> "AssertionSpec":
+        if self.type is AssertionType.RESPONSE_JSON_SCHEMA:
+            if not isinstance(self.value, dict):
+                raise ValueError("response.jsonSchema 的 value 必须是对象")
+        elif self.type.value.startswith("runtime."):
+            if isinstance(self.value, bool) or not isinstance(self.value, (int, float)):
+                raise ValueError(f"{self.type} 的 value 必须是非负数字")
+            if self.value < 0:
+                raise ValueError(f"{self.type} 的 value 必须是非负数字")
+        elif self.type is AssertionType.TOOL_SEQUENCE:
+            if (
+                not isinstance(self.value, list)
+                or not self.value
+                or any(not isinstance(item, str) or not item.strip() for item in self.value)
+            ):
+                raise ValueError("tool.sequence 的 value 必须是非空工具名称数组")
+        elif not isinstance(self.value, str):
+            raise ValueError(f"{self.type} 的 value 必须是字符串")
+        return self
+
+
+class EvalCase(EvaluationModel):
+    """One evaluation case: ordered turns plus assertions to check."""
+
+    id: str = Field(min_length=1, max_length=128)
+    turns: list[EvalTurn] = Field(min_length=1)
+    assertions: list[AssertionSpec] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_single_input(cls, value: Any) -> Any:
+        if not isinstance(value, dict):
+            return value
+        data = dict(value)
+        if "input" in data and "turns" not in data:
+            turn = {"input": data.pop("input")}
+            for field in (
+                "expected_output",
+                "expectedOutput",
+                "reference_output",
+                "referenceOutput",
+                "expected_tools",
+                "expectedTools",
+            ):
+                if field in data:
+                    normalized_field = (
+                        "expected_output"
+                        if field in {"reference_output", "referenceOutput"}
+                        else field
+                    )
+                    turn[normalized_field] = data.pop(field)
+            data["turns"] = [turn]
+        return data
+
+    @property
+    def input(self) -> str:
+        """Return the final turn for legacy single-input consumers."""
+
+        return self.turns[-1].input
+
+
+class EvalSetVersion(EvaluationModel):
+    """A versioned, content-digested evaluation set of one or more cases."""
+
+    schema_version: Literal["ksadk.eval/v1"] = "ksadk.eval/v1"
+    name: str = Field(min_length=1, max_length=256)
+    cases: list[EvalCase] = Field(min_length=1)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    source_format: str = "native"
+    content_digest: str = ""
+
+    @model_validator(mode="after")
+    def validate_and_digest(self) -> "EvalSetVersion":
+        ids = [case.id for case in self.cases]
+        if len(ids) != len(set(ids)):
+            raise ValueError("评测集中的 case id 必须唯一")
+        expected = self.compute_digest()
+        if self.content_digest and self.content_digest != expected:
+            raise ValueError("contentDigest 与规范化评测集内容不一致")
+        self.content_digest = expected
+        return self
+
+    def compute_digest(self) -> str:
+        """Hash normalized cases, excluding transport format and the digest itself."""
+
+        payload = self.model_dump(
+            mode="json",
+            by_alias=False,
+            exclude={"content_digest", "source_format"},
+        )
+        return _content_digest(payload)
+
+
+class CloudDatasetRef(EvaluationModel):
+    """Immutable reference to the exact cloud Dataset snapshot used by a run."""
+
+    provider: str = Field(min_length=1, max_length=256)
+    project_id: str | None = Field(default=None, min_length=1, max_length=256)
+    dataset_id: str = Field(min_length=1, max_length=256)
+    version: int = Field(ge=1)
+    schema_hash: str = Field(min_length=64, max_length=64)
+    content_digest: str = Field(min_length=64, max_length=64)
+    row_count: int = Field(default=0, ge=0)
+
+
+# ---------------------------------------------------------------------------
+# Target identity & references
+# ---------------------------------------------------------------------------
+
+
+class TargetKind(str, Enum):
+    """Supported target categories; adapters own their implementation details."""
+
+    LOCAL_SOURCE = "local_source"
+    STUDIO_BUILD = "studio_build"
+    A2A = "a2a"
+    CODEX_WORKTREE = "codex_worktree"
+
+
+class TargetSnapshot(EvaluationModel):
+    """Immutable target identity recorded with an evaluation run."""
+
+    kind: TargetKind
+    entrypoint: str = Field(min_length=1, max_length=1024)
+    revision_digest: str = Field(min_length=1, max_length=128)
+    runtime: str = Field(default="", max_length=128)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class TargetRef(EvaluationModel):
+    """Unresolved target supplied by CLI/Studio before an adapter snapshots it."""
+
+    kind: TargetKind
+    locator: str = Field(min_length=1, max_length=2048)
+    entrypoint: str | None = Field(default=None, min_length=1, max_length=1024)
+    credential_ref: str | None = Field(default=None, min_length=1, max_length=512)
+    profile: str | None = Field(default=None, min_length=1, max_length=256)
+
+
+# ---------------------------------------------------------------------------
+# Execution config & request / spec
+# ---------------------------------------------------------------------------
+
+
+class EvaluationConfig(EvaluationModel):
+    """Execution limits and evaluator selection shared by CLI and Studio."""
+
+    timeout_seconds: int = Field(default=120, ge=1, le=3600)
+    fail_fast: bool = False
+    evaluators: list[str] = Field(default_factory=list)
+    data_policy: DataPolicy = DataPolicy.LOCAL_ONLY
+    judge_model: str | None = Field(default=None, min_length=1, max_length=256)
+    judge_api_base: str | None = Field(default=None, min_length=1, max_length=2048)
+    judge_api_key_env: str = Field(
+        default="KSADK_EVAL_JUDGE_API_KEY",
+        pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
+        max_length=128,
+    )
+
+
+class EvaluationRequest(EvaluationModel):
+    """Validated input passed into the one public evaluation executor."""
+
+    evalset: EvalSetVersion
+    target: TargetRef
+    config: EvaluationConfig = Field(default_factory=EvaluationConfig)
+    report_dir: str | None = Field(default=None, min_length=1, max_length=2048)
+    cloud_dataset: CloudDatasetRef | None = None
+
+
+class EvalRunSpec(EvaluationModel):
+    """Immutable execution plan after the target has been snapshotted."""
+
+    id: str = Field(min_length=1, max_length=128)
+    evalset: EvalSetVersion
+    target: TargetSnapshot
+    config: EvaluationConfig = Field(default_factory=EvaluationConfig)
+    environment_digest: str = Field(default="", max_length=128)
+    attempt: int = Field(default=1, ge=1)
+    cloud_dataset: CloudDatasetRef | None = None
+
+
+# ---------------------------------------------------------------------------
+# Run results: trace, usage, target run, metric results, case run, report
+# ---------------------------------------------------------------------------
+
+
+# Per-case adapter output: trace linkage, token usage, and the normalized
+# invocation result returned by a target adapter.
+class TraceRef(EvaluationModel):
+    """Queryable linkage to a recorded trace; at least one id required."""
+
+    run_id: str | None = None
+    trace_id: str | None = None
+    root_span_id: str | None = None
+    session_id: str | None = None
+    invocation_id: str | None = None
+    seq_start: int | None = Field(default=None, ge=1)
+    seq_end: int | None = Field(default=None, ge=1)
+    remote_task_id: str | None = None
+    seq_id: int | None = Field(default=None, ge=1)
+
+    @model_validator(mode="after")
+    def require_reference(self) -> "TraceRef":
+        if not any(
+            (
+                self.run_id,
+                self.trace_id,
+                self.session_id,
+                self.invocation_id,
+                self.remote_task_id,
+                self.seq_id,
+            )
+        ):
+            raise ValueError("TraceRef 至少需要一个可查询 ID")
+        return self
+
+
+class UsageSnapshot(EvaluationModel):
+    """Token usage for one invocation; zero unless the adapter reported it."""
+
+    input_tokens: int = Field(default=0, ge=0)
+    output_tokens: int = Field(default=0, ge=0)
+    total_tokens: int = Field(default=0, ge=0)
+    reported: bool = False
+
+
+class ToolCallEvidence(EvaluationModel):
+    """Non-sensitive projection of one runtime tool invocation."""
+
+    call_id: str = Field(min_length=1, max_length=256)
+    name: str = Field(min_length=1, max_length=256)
+    status: Literal["SUCCEEDED", "ERROR", "INCOMPLETE"]
+    seq_start: int | None = Field(default=None, ge=1)
+    seq_end: int | None = Field(default=None, ge=1)
+
+
+class TargetRun(EvaluationModel):
+    """Normalized result returned by a target adapter for one case."""
+
+    status: TargetRunStatus
+    output: str = ""
+    duration_ms: int | None = Field(default=None, ge=0)
+    usage: UsageSnapshot = Field(default_factory=UsageSnapshot)
+    error_code: str | None = None
+    error_message: str | None = None
+    trace_ref: TraceRef | None = None
+    trace_refs: list[TraceRef] = Field(default_factory=list)
+    tool_calls: list[ToolCallEvidence] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+# Evaluator output: per-metric verdicts, per-case aggregation, run-level
+# summary counts, and the canonical persisted report.
+class MetricResult(EvaluationModel):
+    """One deterministic evaluator result and its non-sensitive evidence."""
+
+    name: str = Field(min_length=1, max_length=128)
+    version: str = Field(default="v1", min_length=1, max_length=32)
+    status: MetricStatus
+    score: float | None = Field(default=None, ge=0, le=1)
+    required: bool = True
+    evidence: dict[str, Any] = Field(default_factory=dict)
+
+
+class CaseRun(EvaluationModel):
+    """Target result and metric results for one case attempt."""
+
+    case_id: str = Field(min_length=1, max_length=128)
+    attempt: int = Field(default=1, ge=1)
+    target_run: TargetRun
+    metrics: list[MetricResult] = Field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return self.target_run.status is TargetRunStatus.PASSED and all(
+            metric.status is MetricStatus.PASS for metric in self.metrics if metric.required
+        )
+
+
+class EvalRunSummary(EvaluationModel):
+    """Derived terminal counts; callers must not construct their own summary."""
+
+    total_cases: int = Field(default=0, ge=0)
+    passed_cases: int = Field(default=0, ge=0)
+    failed_cases: int = Field(default=0, ge=0)
+    unavailable_cases: int = Field(default=0, ge=0)
+    error_cases: int = Field(default=0, ge=0)
+    cancelled_cases: int = Field(default=0, ge=0)
+
+
+class EvalRunReport(EvaluationModel):
+    """Canonical persisted result consumed by CLI, Studio, and future sync."""
+
+    schema_version: Literal["ksadk.eval.report/v1"] = "ksadk.eval.report/v1"
+    spec: EvalRunSpec
+    status: EvalRunStatus
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    case_runs: list[CaseRun] = Field(default_factory=list)
+    summary: EvalRunSummary = Field(default_factory=EvalRunSummary)
+    report_digest: str = ""
+
+    @model_validator(mode="after")
+    def validate_case_ids(self) -> "EvalRunReport":
+        case_ids = [case.case_id for case in self.case_runs]
+        if len(case_ids) != len(set(case_ids)):
+            raise ValueError("一次评测中 case_id 不能重复")
+        self.summary = self._summarize_cases()
+        expected = self.compute_digest()
+        if self.report_digest and self.report_digest != expected:
+            legacy_payload = self.model_dump(
+                mode="json", by_alias=False, exclude={"report_digest"}
+            )
+            legacy_payload["spec"].pop("cloud_dataset", None)
+            if self.spec.cloud_dataset is not None or self.report_digest != _content_digest(
+                legacy_payload
+            ):
+                raise ValueError("reportDigest 与规范化报告内容不一致")
+        self.report_digest = expected
+        return self
+
+    def _summarize_cases(self) -> EvalRunSummary:
+        counts = EvalRunSummary(total_cases=len(self.case_runs))
+        for case_run in self.case_runs:
+            if case_run.target_run.status is TargetRunStatus.ERROR or any(
+                metric.required and metric.status is MetricStatus.ERROR
+                for metric in case_run.metrics
+            ):
+                counts.error_cases += 1
+            elif case_run.target_run.status is TargetRunStatus.CANCELLED:
+                counts.cancelled_cases += 1
+            elif case_run.target_run.status is TargetRunStatus.UNAVAILABLE or any(
+                metric.required and metric.status is MetricStatus.UNAVAILABLE
+                for metric in case_run.metrics
+            ):
+                counts.unavailable_cases += 1
+            elif case_run.passed:
+                counts.passed_cases += 1
+            else:
+                counts.failed_cases += 1
+        return counts
+
+    def compute_digest(self) -> str:
+        """Return a stable digest excluding the digest field itself."""
+
+        payload = self.model_dump(mode="json", by_alias=False, exclude={"report_digest"})
+        return _content_digest(payload)