sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/metric_views.py

@@ -0,0 +1,474 @@
+"""Small, aggregate-only views of the versioned metric catalog.
+
+This is a presentation layer, not a grader or publication authorization.
+Pass verified SDK scorecards; full evidence and scoring remain unchanged.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from dataclasses import dataclass, field
+from typing import Any, Mapping
+
+_PRIORITY_SOURCE = "docs/reports/sable-offline-evals-metric-priority.md"
+_PROPOSAL_SOURCE = "docs/multidomain-evaluation-proposal.md"
+
+
+@dataclass(frozen=True)
+class MetricCard:
+    metric_id: str
+    label: str
+    direction: str
+    unit: str
+    role: str
+    eligible_population: str
+    description: str
+    implementation: str
+    source: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return vars(self).copy()
+
+
+def _card(metric_id: str, label: str, direction: str, population: str,
+          description: str, *, unit: str = "ratio", role: str = "diagnostic",
+          planned: bool = False, source: str = _PRIORITY_SOURCE) -> MetricCard:
+    return MetricCard(metric_id, label, direction, unit, role, population, description,
+                      "planned" if planned else "available", source)
+
+
+_COVERAGE = _card("valid_response_coverage", "Valid response coverage", "higher",
+                  "all attempted cases", "Valid candidate responses divided by attempted cases.", role="reliability")
+_LATENCY_P50 = _card("runner_latency_p50_ms", "Runner latency p50", "lower",
+                     "attempted cases with captured runner spans",
+                     "Median runner wall time, including process and adapter overhead.", unit="ms", role="efficiency")
+_LATENCY_P95 = _card("runner_latency_p95_ms", "Runner latency p95", "lower",
+                     "attempted cases with captured runner spans",
+                     "95th percentile runner wall time; not isolated model inference time.", unit="ms", role="efficiency")
+
+_CATALOG: dict[str, tuple[MetricCard, ...]] = {
+    "rag.invoke": (
+        _card("three_way_accuracy", "Routing accuracy", "higher", "all invocation cases",
+              "Correct choice of CALL_RAG, ASK_BUYER, or ANSWER_FROM_CONTEXT.", role="primary",
+              source="docs/rag-invocation-v1.md"),
+        _card("required_lookup_recall", "Required lookup recall", "higher", "CALL_RAG gold cases",
+              "Required lookup opportunities on which the candidate calls RAG."),
+        _card("unnecessary_call_rate", "Unnecessary call rate", "lower", "non-CALL_RAG gold cases",
+              "Calls on opportunities labeled clarification or answer from context."),
+        _card("clarification_accuracy", "Clarification accuracy", "higher", "ASK_BUYER gold cases",
+              "Required clarification opportunities on which the candidate asks."),
+        _card("answer_from_context_accuracy", "Answer from context accuracy", "higher",
+              "ANSWER_FROM_CONTEXT gold cases", "Context-sufficient opportunities answered without retrieval."),
+        _COVERAGE, _LATENCY_P50, _LATENCY_P95,
+    ),
+    "rag.retrieve": (
+        _card("complete_evidence_success", "Complete evidence in context", "higher",
+              "all corpus-answerable retrieval questions",
+              "All required reference groups reach exact final context within the declared UTF-8 byte budget; provisional anchors are recovery proxies.", role="primary"),
+        _card("candidate_evidence_recall_at_50", "Candidate evidence recall@50", "higher",
+              "all corpus-answerable retrieval questions",
+              "Required reference groups recovered in the first-stage candidate pool of at most fifty."),
+        _card("final_evidence_group_recall", "Final evidence coverage", "higher",
+              "all corpus-answerable retrieval questions",
+              "Required reference groups represented in the exact context delivered by the candidate."),
+        _card("ndcg_at_5", "nDCG@5", "higher", "all corpus-answerable retrieval questions",
+              "Final reranked top-five ranking quality normalized against corpus-wide reference relevance, including genuine retrieval misses."),
+        _card("grounded_answer_success", "Grounded answer success", "higher",
+              "answerable and fallback opportunities",
+              "Requires a separately versioned answer and claim-support grader; not computed by retrieval capture.", planned=True),
+        _card("retrieval_latency_p95_ms", "Retrieval and assembly p95", "lower",
+              "attempted cases with SDK-observed adapter spans",
+              "Actual per-case adapter wall time; excludes index construction. Caller-supplied component timing is diagnostic only.", unit="ms", role="efficiency"),
+        _card("cost_per_evidence_complete_query", "Cost per evidence-complete query", "lower",
+              "evidence-complete queries with complete cost evidence",
+              "Unavailable without a versioned verified pricing and resource accounting policy; no inferred zero-cost local runs.", unit="USD/query", role="efficiency"),
+        _COVERAGE,
+    ),
+    "rag.rerank": (
+        _card("ndcg_at_5", "nDCG@5", "higher", "answerable, gold-present frozen pools",
+              "Top-five graded ranking quality normalized by the ideal ranking.", role="primary",
+              source="src/sphynx/metrics.py"),
+        _card("hit_at_5", "Hit@5", "higher", "answerable frozen pools",
+              "Whether a relevance-grade 2 or 3 passage appears in the first five.", source="src/sphynx/metrics.py"),
+        _card("mrr_at_5", "MRR@5", "higher", "answerable frozen pools",
+              "Reciprocal rank of the first relevance-grade 2 or 3 passage within five.", source="src/sphynx/metrics.py"),
+        _card("evidence_group_recall_at_5", "Evidence group recall@5", "higher", "pools with required evidence groups",
+              "Fraction of required evidence groups represented among the top five.", source="src/sphynx/metrics.py"),
+        _COVERAGE,
+        _card("candidate_failure_rate", "Candidate failure rate", "lower", "all attempted cases",
+              "Failed, timed out, or invalid candidate responses divided by attempted cases.", role="reliability"),
+        _LATENCY_P50, _LATENCY_P95,
+    ),
+}
+
+# Paired development grading; semantic credit requires capture-bound owner judgments.
+_ROBUSTNESS_SOURCE = "docs/question-robustness.md"
+_CATALOG["rag.question_robustness"] = (
+    _card("required_lookup_recall", "Required lookup recall", "higher",
+          "reviewed required-lookup opportunities, sliced by utterance variant",
+          "Correct RAG calls divided by required-lookup opportunities; direct, embedded, implicit and distractor variants remain separate.",
+          role="primary", source=_ROBUSTNESS_SOURCE),
+    _card("unnecessary_call_rate", "Unnecessary call rate", "lower",
+          "reviewed non-lookup controls",
+          "RAG calls on reviewed context-sufficient, non-request or clarification controls.",
+          source=_ROBUSTNESS_SOURCE),
+    _card("query_intent_preservation", "Query intent preservation", "higher",
+          "eligible emitted query sets on required-lookup cases",
+          "Owner-reviewed preservation of the information need across the full query set; unavailable without captured queries and versioned judgments.",
+          source=_ROBUSTNESS_SOURCE),
+    _card("required_constraint_coverage", "Required constraint coverage", "higher",
+          "eligible emitted required-lookup query sets with required constraints",
+          "Mean per-case fraction of required constraints retained, using owner judgments or versioned typed equivalence rules; lexical overlap is insufficient.",
+          source=_ROBUSTNESS_SOURCE),
+    _card("complete_evidence_success", "Complete evidence in context", "higher",
+          "all corpus-answerable required-lookup utterances",
+          "All required evidence groups reach exact final context; valid intentional skips score zero and provisional source anchors remain recovery proxies.",
+          role="primary", source=_ROBUSTNESS_SOURCE),
+    _card("paired_lookup_drop_pp", "Paired lookup drop", "lower",
+          "complete equivalent direct/variant pairs in the declared required-lookup families",
+          "Mean direct minus variant lookup success, in percentage points; negative drops are improvements and missing pairs make the primary comparison incomplete.",
+          unit="pp", source=_ROBUSTNESS_SOURCE),
+    _card("paired_evidence_drop_pp", "Paired evidence drop", "lower",
+          "complete equivalent direct/variant pairs in the declared corpus-answerable families",
+          "Mean direct minus variant complete-evidence success, in percentage points, with uncertainty clustered by semantic family.",
+          unit="pp", source=_ROBUSTNESS_SOURCE),
+    _card("valid_capture_coverage", "Valid capture coverage", "higher",
+          "all attempted robustness cases",
+          "Valid candidate outcomes divided by all attempted cases; validity does not imply correct routing, query meaning or evidence.",
+          role="reliability", source=_ROBUSTNESS_SOURCE),
+)
+
+_CATALOG["rag.rerank.public"] = (
+    _card("ndcg_at_5", "nDCG@5", "higher", "public pools with at least one positive qrel",
+          "Native graded gain normalized by the ideal frozen BM25 pool ranking.", role="primary", source="docs/public-reranking-v1.md"),
+    _card("hit_at_5", "Hit@5", "higher", "public pools with at least one positive qrel",
+          "At least one native relevance grade >0 passage in the first five.", source="docs/public-reranking-v1.md"),
+    _card("mrr_at_5", "MRR@5", "higher", "public pools with at least one positive qrel",
+          "Reciprocal rank of the first native positive qrel in the first five.", source="docs/public-reranking-v1.md"),
+    _card("pool_oracle_coverage", "BM25 pool oracle coverage", "higher", "all public questions",
+          "Fraction of questions whose frozen 15-passage pool includes any positive qrel.", source="docs/public-reranking-v1.md"),
+    _COVERAGE,
+    _card("candidate_failure_rate", "Candidate failure rate", "lower", "all attempted public questions",
+          "Failures and invalid responses divided by all attempts.", role="reliability"),
+    _LATENCY_P50, _LATENCY_P95,
+)
+
+# These public design cards are definitions only: no runner or grader is installed.
+_PLANNED: dict[str, tuple[tuple[str, str, str, str, str], ...]] = {
+    "rag.answer": (
+        ("selective_task_utility", "Selective task utility", "higher", "utility", "answerable and fallback opportunities"),
+        ("unsupported_critical_claim_rate", "Unsupported critical claims", "lower", "ratio", "material answer claims"),
+        ("claim_support_precision", "Claim support precision", "higher", "ratio", "answer claims"),
+        ("required_claim_recall", "Required claim recall", "higher", "ratio", "required supported claims"),
+        ("citation_resolution_accuracy", "Citation resolution", "higher", "ratio", "cited source anchors"),
+        ("fallback_action_accuracy", "Fallback action accuracy", "higher", "ratio", "insufficient or conflicting context"),
+    ),
+    "highlighting": (
+        ("span_precision", "Span precision", "higher", "ratio", "predicted spans"),
+        ("span_recall", "Span recall", "higher", "ratio", "reference spans"),
+        ("span_overlap", "Span overlap", "higher", "ratio", "aligned reference and predicted spans"),
+        ("boundary_correctness", "Boundary correctness", "higher", "ratio", "highlight boundaries"),
+        ("semantic_correctness", "Semantic correctness", "higher", "ratio", "eligible highlighting cases"),
+    ),
+    "transcription": (
+        ("semantic_task_success", "Semantic task success", "higher", "ratio", "audio task cases"),
+        ("word_error_rate", "Word error rate", "lower", "ratio", "reference words"),
+        ("domain_entity_accuracy", "Domain entity accuracy", "higher", "ratio", "domain entities in speech"),
+        ("diarization_error_rate", "Diarization error", "lower", "ratio", "reference speaker time"),
+        ("timestamp_error_ms", "Timestamp error", "lower", "ms", "aligned words or segments"),
+        ("real_time_factor", "Real time factor", "lower", "ratio", "audio duration"),
+    ),
+    "tts": (
+        ("intelligibility", "Intelligibility", "higher", "rating", "synthesized speech cases"),
+        ("naturalness", "Naturalness", "higher", "rating", "synthesized speech cases"),
+        ("speaker_fidelity", "Speaker fidelity", "higher", "rating", "reference speaker cases"),
+        ("real_time_factor", "Real time factor", "lower", "ratio", "synthesized audio duration"),
+        ("first_audio_latency_ms", "First audio latency", "lower", "ms", "speech requests"),
+    ),
+    "dsl": (
+        ("parse_validity", "Parse validity", "higher", "ratio", "submitted programs"),
+        ("type_validity", "Type validity", "higher", "ratio", "typed programs"),
+        ("execution_equivalence", "Execution equivalence", "higher", "ratio", "executable programs"),
+        ("safety_constraint_violation_rate", "Safety violations", "lower", "ratio", "program execution opportunities"),
+        ("verified_terminal_state_success", "Verified final state", "higher", "ratio", "DSL task cases"),
+    ),
+    "browser": (
+        ("verified_terminal_state_success", "Verified final state", "higher", "ratio", "resettable browser tasks"),
+        ("action_accuracy", "Action accuracy", "higher", "ratio", "eligible actions"),
+        ("forbidden_action_rate", "Forbidden actions", "lower", "ratio", "attempted actions"),
+        ("false_success_rate", "False success claims", "lower", "ratio", "claimed successful tasks"),
+        ("verification_coverage", "Verification coverage", "higher", "ratio", "required postconditions"),
+        ("interactive_element_recall", "Interactive element recall", "higher", "ratio", "eligible UI controls"),
+    ),
+    "latency": (
+        ("time_to_first_token_ms", "Time to first token", "lower", "ms", "generation requests"),
+        ("first_audio_latency_ms", "First audio latency", "lower", "ms", "speech requests"),
+        ("completion_latency_p95_ms", "Completion latency p95", "lower", "ms", "completed requests"),
+        ("deadline_miss_rate", "Deadline miss rate", "lower", "ratio", "requests with declared deadlines"),
+        ("verified_throughput", "Verified throughput", "higher", "requests/s", "fixed-concurrency trace replay"),
+        ("cost_per_success", "Cost per verified success", "lower", "currency/task", "verified successful tasks"),
+    ),
+}
+for _track, _rows in _PLANNED.items():
+    _source = (_PROPOSAL_SOURCE + "; " + _PRIORITY_SOURCE) if _track in ("highlighting", "tts", "dsl", "browser", "latency") else _PRIORITY_SOURCE
+    _CATALOG[_track] = tuple(
+        _card(metric_id, label, direction, population,
+              "Planned definition; the pack must version its rubric, denominator, and aggregation before scoring.",
+              unit=unit, planned=True, source=_source)
+        for metric_id, label, direction, unit, population in _rows
+    )
+
+_ALIASES = {
+    "rag.rerank.beir.v1": "rag.rerank.public", "rag.question_robustness.v1": "rag.question_robustness",
+    "rag.invoke.v1": "rag.invoke", "rag.invocation": "rag.invoke", "rag_invoke": "rag.invoke",
+    "rag.rerank.v1": "rag.rerank", "rag_rerank": "rag.rerank", "rag.rerank.sable.dev.v1": "rag.rerank",
+    "rag.retrieve.v1": "rag.retrieve", "rag_retrieve": "rag.retrieve",
+    "asr": "transcription", "browser_accuracy": "browser",
+}
+
+
+def _track_name(track: str) -> str:
+    name = _ALIASES.get(track, track)
+    if name not in _CATALOG:
+        raise ValueError(f"Unknown metric track {track!r}; choose from {', '.join(_CATALOG)}")
+    return name
+
+
+def list_metric_tracks() -> tuple[str, ...]:
+    """Return measured and planned public catalog tracks."""
+    return tuple(_CATALOG)
+
+
+def list_metrics(track: str, limit: int = 8) -> tuple[MetricCard, ...]:
+    """Return the first at most eight public cards in a deliberate display order."""
+    if type(limit) is not int or not 1 <= limit <= 8:
+        raise ValueError("limit must be an integer from 1 through 8")
+    return _CATALOG[_track_name(track)][:limit]
+
+
+@dataclass(frozen=True)
+class MetricSummary:
+    track: str
+    metrics: tuple[Mapping[str, Any], ...]
+    scope: Mapping[str, str | bool] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": "sphynx.metric.summary.v1", "track": self.track,
+                "scope": dict(self.scope), "metrics": [dict(item) for item in self.metrics]}
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, allow_nan=False, sort_keys=True)
+
+    def to_text(self) -> str:
+        """Return an aggregate table with coverage and evaluation qualifications."""
+        lines = [f"Metrics: {self.track}"]
+        lines.extend(f"{label}: {value}" for label, value in _scope_sections(self.scope))
+        lines.append("")
+        headers = ("Metric", "Value", "Unit", "Status", "Coverage", "Missing", "Undefined")
+        rows = [(
+            metric["label"], _format_metric_value(metric), metric["unit"], metric["status"],
+            _format_metric_coverage(metric),
+            _format_metric_count(metric, "missing_count", absent="n/a"),
+            _format_metric_count(metric, "undefined_count", absent="n/a"),
+        ) for metric in self.metrics]
+        widths = [max(minimum, len(header), *(len(str(row[index])) for row in rows))
+                  for index, (header, minimum) in enumerate(zip(headers, (30, 12, 4, 6, 10, 7, 9)))]
+        alignments = ("<", ">", "<", "<", ">", ">", ">")
+
+        def render_row(row: tuple[Any, ...]) -> str:
+            return "  ".join(f"{str(value):{alignment}{width}}"
+                             for value, alignment, width in zip(row, alignments, widths)).rstrip()
+
+        lines.extend((render_row(headers), "  ".join("-" * width for width in widths)))
+        lines.extend(render_row(row) for row in rows)
+        lines.extend(("", "Coverage = observed cases / eligible cases."))
+        timing = _runner_timing_scope(self.metrics)
+        if timing:
+            lines.append(timing)
+        return "\n".join(lines) + "\n"
+
+    def to_markdown(self) -> str:
+        lines = [f"**{label}:** {value}" for label, value in _scope_sections(self.scope)]
+        if lines:
+            lines.append("")
+        lines.extend(["| Metric | Value | Unit | Status | Eligible | Observed | Missing | Undefined |",
+                      "| --- | ---: | --- | --- | ---: | ---: | ---: | ---: |"])
+        for metric in self.metrics:
+            lines.append("| " + " | ".join(str(item) for item in (
+                metric["label"], _format_metric_value(metric), metric["unit"], metric["status"],
+                _format_metric_count(metric, "eligible_count"),
+                _format_metric_count(metric, "observed_count"),
+                _format_metric_count(metric, "missing_count"),
+                _format_metric_count(metric, "undefined_count"),
+            )) + " |")
+        timing = _runner_timing_scope(self.metrics)
+        if timing:
+            lines.extend(("", timing))
+        return "\n".join(lines) + "\n"
+
+
+def _format_metric_value(metric: Mapping[str, Any]) -> str:
+    """Use one display policy without turning absent or partial values into scores."""
+    value = metric["value"]
+    if value is None or metric["status"] != "measured":
+        return str(metric["status"])
+    if metric["metric_id"].split("_", 1)[0] in ("ndcg", "mrr"):
+        return f"{value:.4f}"
+    if metric["unit"] == "ratio":
+        return f"{value:.1%}"
+    if metric["unit"] == "ms":
+        return f"{value:.2f} ms"
+    return f"{value:.4f} {metric['unit']}"
+
+
+def _format_metric_count(metric: Mapping[str, Any], key: str, *, absent: str = "—") -> str:
+    value = metric[key]
+    return absent if value is None or metric["status"] == "withheld" else str(value)
+
+
+def _format_metric_coverage(metric: Mapping[str, Any]) -> str:
+    if metric["status"] == "withheld":
+        return "withheld"
+    if metric["observed_count"] is None and metric["eligible_count"] is None:
+        return "n/a"
+    observed = _format_metric_count(metric, "observed_count", absent="n/a")
+    eligible = _format_metric_count(metric, "eligible_count", absent="n/a")
+    return f"{observed}/{eligible}"
+
+
+def _scope_sections(scope: Mapping[str, str | bool]) -> list[tuple[str, str]]:
+    sections = []
+    for label, keys in (
+        ("Evaluation scope", ("trust_tier", "development_only", "origin", "label_status")),
+        ("Label basis", ("label_basis",)),
+        ("Metric semantics", ("metric_semantics",)),
+        ("Versions", ("pack_id", "metric_release", "grader_release")),
+        ("Question bank", ("bank_id", "bank_release", "population", "contains_source_excerpt_controls")),
+    ):
+        values = [f"{key}={scope[key]}" for key in keys if key in scope]
+        if values:
+            sections.append((label, "; ".join(values).replace("\n", " ").replace("\r", " ")))
+    return sections
+
+
+def _runner_timing_scope(metrics: tuple[Mapping[str, Any], ...]) -> str | None:
+    if any(metric["metric_id"].startswith("runner_latency_") for metric in metrics):
+        return ("Runner timing scope: wall time includes process and adapter overhead; "
+                "not isolated model inference time.")
+    return None
+
+
+_SCOPE_KEYS = ("trust_tier", "development_only", "origin", "label_status",
+               "label_basis", "metric_semantics", "pack_id", "metric_release", "grader_release",
+               "bank_id", "bank_release", "population", "contains_source_excerpt_controls")
+
+
+def _scope_metadata(data: Mapping[str, Any]) -> dict[str, str | bool]:
+    """Preserve aggregate qualifications, excluding IDs, digests, and evidence."""
+    scope: dict[str, str | bool] = {}
+    releases = data.get("releases")
+    for key in _SCOPE_KEYS:
+        value = data.get(key)
+        if value is None and key in ("metric_release", "grader_release") and isinstance(releases, Mapping):
+            value = releases.get(key)
+        if key in ("development_only", "contains_source_excerpt_controls"):
+            if type(value) is bool:
+                scope[key] = value
+        elif isinstance(value, str) and value:
+            scope[key] = value
+    return scope
+
+
+def _number(value: Any) -> float | None:
+    return float(value) if type(value) in (int, float) and math.isfinite(value) else None
+
+
+def _count(value: Any) -> int | None:
+    return value if type(value) is int and value >= 0 else None
+
+
+def _counts(data: Mapping[str, Any]) -> tuple[int | None, int | None, int | None]:
+    counts = data.get("counts")
+    if isinstance(counts, Mapping):
+        return (_count(counts.get("attempted_count")), _count(counts.get("success_count")),
+                _count(counts.get("failure_count")))
+    if "case_count" in data and _count(data.get("success_count")) is not None and _count(data.get("failure_count")) is not None:
+        return (_count(data.get("case_count")), _count(data.get("success_count")),
+                _count(data.get("failure_count")))
+    cases = data.get("cases")
+    if isinstance(cases, (tuple, list)) and all(isinstance(row, Mapping) and
+                                               isinstance(row.get("status"), str) for row in cases):
+        successes = sum(row["status"] == "success" for row in cases)
+        return len(cases), successes, len(cases) - successes
+    return None, None, None
+
+
+def summarize_metrics(scorecard: Any, track: str | None = None, limit: int = 8) -> MetricSummary:
+    """Select aggregate display metrics from a verified SDK scorecard.
+
+    This function neither verifies evidence nor grants permission to share a
+    private scorecard. Obtain it through report()/report_invocation() or
+    grade_private_rerank(). It emits no case IDs, passages, labels, slices,
+    candidate details, arbitrary metric fields, or unknown metric IDs.
+    """
+    data = scorecard.to_dict() if hasattr(scorecard, "to_dict") else scorecard
+    if not isinstance(data, Mapping):
+        raise TypeError("scorecard must be a mapping or SDK scorecard with to_dict()")
+    if track is None:
+        track = data.get("track") or data.get("pack_id")
+        if not isinstance(track, str):
+            raise ValueError("track is required when the scorecard has no recognized track or pack_id")
+    name = _track_name(track)
+    definitions = data.get("metric_definitions", ())
+    visibility = {row["metric_id"]: row.get("visibility", "public")
+                  for row in definitions if isinstance(row, Mapping) and "metric_id" in row}
+    aggregates = data.get("metrics", {})
+    if not isinstance(aggregates, Mapping):
+        aggregates = {}
+    attempted, successful, failed = _counts(data)
+    rows = []
+    for card in list_metrics(name, limit):
+        source_id = "runner_latency_ms" if card.metric_id.startswith("runner_latency_") else card.metric_id
+        item = aggregates.get(source_id)
+        if source_id == "runner_latency_ms" and name == "rag.invoke":
+            item = data.get("runner_latency_ms")
+        entry = {**card.to_dict(), "value": None, "status": "planned" if card.implementation == "planned" else "unavailable",
+                 "eligible_count": None, "observed_count": None, "missing_count": None,
+                 "undefined_count": None}
+        if card.implementation == "planned":
+            rows.append(entry)
+            continue
+        if visibility.get(source_id, "public") != "public" or (
+            isinstance(item, Mapping) and item.get("visibility", "public") != "public"
+        ):
+            entry["status"] = "withheld"
+            rows.append(entry)
+            continue
+        if card.metric_id in ("valid_response_coverage", "candidate_failure_rate"):
+            if attempted is not None and successful is not None and failed is not None and successful + failed == attempted:
+                entry.update(eligible_count=attempted, observed_count=attempted,
+                             missing_count=0, undefined_count=0)
+                entry["value"] = (successful if card.metric_id == "valid_response_coverage" else failed) / attempted if attempted else None
+                entry["status"] = "measured" if attempted else "undefined"
+        elif isinstance(item, Mapping):
+            if source_id == "runner_latency_ms" and name == "rag.invoke":
+                entry.update(eligible_count=attempted, observed_count=attempted,
+                             missing_count=0 if attempted is not None else None, undefined_count=0)
+                value = item.get("p50" if card.metric_id == "runner_latency_p50_ms" else "p95")
+            else:
+                for key in ("eligible_count", "observed_count", "missing_count", "undefined_count"):
+                    entry[key] = _count(item.get(key))
+                value = item.get("p95" if card.metric_id == "runner_latency_p95_ms" else "value")
+            if item.get("status") == "unavailable":
+                entry["status"] = "unavailable"
+            elif entry["eligible_count"] == 0:
+                entry["status"] = "undefined"
+            elif item.get("complete") is False or (entry["missing_count"] or 0) > 0:
+                entry["status"] = "incomplete"
+            else:
+                entry["value"] = _number(value)
+                if entry["value"] is not None:
+                    entry["status"] = "measured"
+        rows.append(entry)
+    return MetricSummary(name, tuple(rows), _scope_metadata(data))