sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/metrics.py

@@ -0,0 +1,282 @@
+"""Versioned illustrative frozen-pool RAG ranking metrics.
+
+The gain, discount, relevance threshold, and required-group rules follow the
+local ``rag-evals/src/metrics-v2.ts`` source pinned by T-001. This narrower
+Sphynx release deliberately publishes only four ranking metrics and runner
+wall latency. No answer-quality or abstention grade is inferred.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Sequence
+
+from .contracts import CaseReferences, ContractError, FrozenPoolCase, _finite_number
+from .rag import RagRerankPack, load_rag_rerank_pack
+
+METRIC_RELEASE = "rag.rerank.metric.v1"
+GRADER_RELEASE = "rag.rerank.grader.v1"
+_QUALITY_METRICS = ("ndcg_at_5", "hit_at_5", "mrr_at_5", "evidence_group_recall_at_5")
+_LATENCY_METRIC = "runner_latency_ms"
+
+
+@dataclass(frozen=True)
+class RankingMetricDefinition:
+    metric_id: str
+    release: str
+    unit: str
+    direction: str
+    primary: bool
+    eligible_population: str
+    aggregation: str
+    undefined_policy: str
+    missing_policy: str
+
+    def to_dict(self) -> dict[str, str | bool]:
+        return vars(self).copy()
+
+
+METRIC_DEFINITIONS = (
+    RankingMetricDefinition(
+        "ndcg_at_5", METRIC_RELEASE, "ratio", "higher", True,
+        "answerable cases in the fixed pack", "arithmetic mean across eligible cases",
+        "null for unsupported cases or zero ideal DCG", "null and incomplete aggregate when an eligible case fails or lacks a valid ranking",
+    ),
+    RankingMetricDefinition(
+        "hit_at_5", METRIC_RELEASE, "ratio", "higher", False,
+        "answerable cases in the fixed pack", "arithmetic mean across eligible cases",
+        "null for unsupported cases", "null and incomplete aggregate when an eligible case fails or lacks a valid ranking",
+    ),
+    RankingMetricDefinition(
+        "mrr_at_5", METRIC_RELEASE, "ratio", "higher", False,
+        "answerable cases in the fixed pack", "arithmetic mean across eligible cases",
+        "null for unsupported cases", "null and incomplete aggregate when an eligible case fails or lacks a valid ranking",
+    ),
+    RankingMetricDefinition(
+        "evidence_group_recall_at_5", METRIC_RELEASE, "ratio", "higher", False,
+        "answerable cases with required evidence groups", "arithmetic mean across eligible cases",
+        "null for unsupported cases or no required groups", "null and incomplete aggregate when an eligible case fails or lacks a valid ranking",
+    ),
+    RankingMetricDefinition(
+        _LATENCY_METRIC, METRIC_RELEASE, "ms", "lower", False,
+        "all attempted cases with a runner monotonic span", "per-case span; aggregate p50 and p95",
+        "null when no runner span exists", "null and incomplete aggregate when an attempted case lacks a runner span",
+    ),
+)
+
+
+def metric_definitions() -> tuple[RankingMetricDefinition, ...]:
+    return METRIC_DEFINITIONS
+
+
+@dataclass(frozen=True)
+class CaseMetricResult:
+    case_id: str
+    pool_digest: str
+    terminal_status: str
+    answerable: bool
+    values: tuple[tuple[str, float | None], ...]
+    undefined_reasons: tuple[tuple[str, str], ...]
+    metric_release: str = METRIC_RELEASE
+    grader_release: str = GRADER_RELEASE
+
+    def value_for(self, metric_id: str) -> float | None:
+        return dict(self.values)[metric_id]
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "case_id": self.case_id,
+            "pool_digest": self.pool_digest,
+            "terminal_status": self.terminal_status,
+            "answerable": self.answerable,
+            "metric_release": self.metric_release,
+            "grader_release": self.grader_release,
+            "values": dict(self.values),
+            "undefined_reasons": dict(self.undefined_reasons),
+        }
+
+
+@dataclass(frozen=True)
+class MetricAggregate:
+    metric_id: str
+    release: str
+    unit: str
+    direction: str
+    value: float | None
+    p95: float | None
+    eligible_count: int
+    observed_count: int
+    missing_count: int
+    undefined_count: int
+    complete: bool
+
+    def to_dict(self) -> dict[str, str | float | int | bool | None]:
+        return vars(self).copy()
+
+
+def _dcg(grades: Sequence[int], cutoff: int = 5) -> float:
+    return sum((2 ** grade - 1) / math.log2(index + 2) for index, grade in enumerate(grades[:cutoff]))
+
+
+def _percentile(values: Sequence[float], fraction: float) -> float:
+    ordered = sorted(values)
+    position = (len(ordered) - 1) * fraction
+    lower = math.floor(position)
+    upper = math.ceil(position)
+    return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
+
+
+def grade_case(
+    case: FrozenPoolCase,
+    references: CaseReferences,
+    ranked_ids: Sequence[str] | None,
+    latency_ms: float | None,
+    *,
+    terminal_status: str = "ok",
+    relevance_threshold: int = 2,
+    pack: RagRerankPack | None = None,
+) -> CaseMetricResult:
+    """Grade one attempt against the pinned pack, retaining explicit absence.
+
+    ``ranked_ids`` must cover the entire frozen pool exactly once for an ``ok``
+    attempt. Use :func:`sphynx.contracts.ranked_candidate_ids` to apply the
+    stable descending-score, original-pool-order tie rule first. Latency is
+    supplied from the runner's monotonic wall clock, not candidate output.
+    """
+
+    pack = pack or load_rag_rerank_pack()
+    original = next((item for item in pack.cases if item.case_id == case.case_id), None)
+    if original is None or original.pool_digest != case.pool_digest:
+        raise ContractError(f"Case {case.case_id!r} does not match the frozen pack pool and corpus identity")
+    if not isinstance(references, CaseReferences) or references.case_id != case.case_id:
+        raise ContractError("references must match the frozen case identity")
+    if references != pack.reference_for(case.case_id):
+        raise ContractError("references differ from the selected frozen pack labels")
+    expected_ids = {candidate.candidate_id for candidate in case.candidates}
+    by_id = {candidate.candidate_id: candidate for candidate in references.candidates}
+    if set(by_id) != expected_ids:
+        raise ContractError("reference candidate IDs do not match the frozen pool")
+    pinned_threshold = pack.relevance_threshold
+    if type(relevance_threshold) is not int or relevance_threshold != pinned_threshold:
+        raise ContractError(
+            f"{METRIC_RELEASE} requires the pack's relevance_threshold={pinned_threshold}; "
+            "another threshold requires a new metric and grader release"
+        )
+    if type(terminal_status) is not str or not terminal_status:
+        raise ContractError("terminal_status must be a nonempty string")
+    if latency_ms is not None:
+        latency_ms = _finite_number(latency_ms, "runner latency_ms")
+        if latency_ms < 0:
+            raise ContractError("runner latency_ms cannot be negative")
+    if terminal_status == "ok":
+        if ranked_ids is None or isinstance(ranked_ids, (str, bytes)):
+            raise ContractError("ok attempt requires a full ranked ID sequence")
+        ranked_ids = tuple(ranked_ids)
+        if len(ranked_ids) != len(expected_ids) or set(ranked_ids) != expected_ids:
+            raise ContractError("ranked IDs must cover the exact frozen pool once")
+    elif ranked_ids is not None:
+        raise ContractError("failed attempt cannot carry a quality ranking")
+    required_groups = tuple(group for group in references.evidence_groups if group.required)
+    answerable = bool(required_groups)
+    reasons: dict[str, str] = {}
+    values: dict[str, float | None] = {metric: None for metric in _QUALITY_METRICS}
+    if not answerable:
+        reasons.update({metric: "unsupported_query" for metric in _QUALITY_METRICS})
+    elif terminal_status != "ok":
+        reasons.update({metric: f"candidate_{terminal_status}" for metric in _QUALITY_METRICS})
+    else:
+        assert ranked_ids is not None
+        grades = [by_id[candidate_id].relevance_grade for candidate_id in ranked_ids]
+        ideal = sorted((reference.relevance_grade for reference in references.candidates), reverse=True)
+        ideal_dcg = _dcg(ideal)
+        if ideal_dcg == 0:
+            reasons["ndcg_at_5"] = "zero_ideal_dcg"
+        else:
+            values["ndcg_at_5"] = _dcg(grades) / ideal_dcg
+        first_relevant = next((index for index, grade in enumerate(grades[:5], start=1) if grade >= relevance_threshold), None)
+        values["hit_at_5"] = 1.0 if first_relevant is not None else 0.0
+        values["mrr_at_5"] = 1.0 / first_relevant if first_relevant is not None else 0.0
+        top_ids = set(ranked_ids[:5])
+        covered = sum(bool(top_ids.intersection(group.acceptable_candidate_ids)) for group in required_groups)
+        values["evidence_group_recall_at_5"] = covered / len(required_groups)
+    values[_LATENCY_METRIC] = latency_ms
+    if latency_ms is None:
+        reasons[_LATENCY_METRIC] = "runner_span_not_captured"
+    return CaseMetricResult(
+        case.case_id, case.pool_digest, terminal_status, answerable,
+        tuple(values.items()), tuple(reasons.items()),
+    )
+
+
+def aggregate_case_metrics(results: Sequence[CaseMetricResult], *, pack: RagRerankPack | None = None) -> dict[str, MetricAggregate]:
+    """Aggregate a complete selected cohort without masking missing cases.
+
+    The caller must supply one result for every fixed pack case. If any eligible
+    case has no value, the metric's aggregate is ``None`` and ``complete`` is
+    false. Unsupported cases are counted as undefined, not as failures/zeros.
+    """
+
+    pack = pack or load_rag_rerank_pack()
+    if len(results) != len(pack.cases):
+        raise ContractError("aggregation requires one result for every fixed pack case")
+    if any(not isinstance(result, CaseMetricResult) for result in results):
+        raise ContractError("aggregation requires CaseMetricResult values")
+    by_case = {result.case_id: result for result in results}
+    if len(by_case) != len(results) or set(by_case) != {case.case_id for case in pack.cases}:
+        raise ContractError("aggregate case IDs must equal the fixed pack case IDs exactly once")
+    for case in pack.cases:
+        result = by_case[case.case_id]
+        if result.pool_digest != case.pool_digest:
+            raise ContractError(f"aggregate pool digest mismatch for {case.case_id}")
+        if result.metric_release != pack.metric_release or result.grader_release != GRADER_RELEASE:
+            raise ContractError(f"aggregate release mismatch for {case.case_id}")
+        expected_answerable = any(group.required for group in pack.reference_for(case.case_id).evidence_groups)
+        if result.answerable is not expected_answerable:
+            raise ContractError(f"aggregate answerability mismatch for {case.case_id}")
+        if type(result.terminal_status) is not str or not result.terminal_status:
+            raise ContractError(f"aggregate terminal status is invalid for {case.case_id}")
+        value_map = dict(result.values)
+        if len(result.values) != len(METRIC_DEFINITIONS) or set(value_map) != {rule.metric_id for rule in METRIC_DEFINITIONS}:
+            raise ContractError(f"aggregate metric fields mismatch for {case.case_id}")
+        for metric_id in _QUALITY_METRICS:
+            value = value_map[metric_id]
+            if value is not None and (type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1):
+                raise ContractError(f"aggregate {metric_id} value is invalid for {case.case_id}")
+            if not expected_answerable or result.terminal_status != "ok":
+                if value is not None:
+                    raise ContractError(f"aggregate {metric_id} must be undefined or missing for {case.case_id}")
+            elif value is None:
+                raise ContractError(f"aggregate {metric_id} is missing from successful answerable case {case.case_id}")
+        latency = value_map[_LATENCY_METRIC]
+        if latency is not None and (type(latency) not in (int, float) or not math.isfinite(latency) or latency < 0):
+            raise ContractError(f"aggregate runner latency is invalid for {case.case_id}")
+    aggregate: dict[str, MetricAggregate] = {}
+    for definition in METRIC_DEFINITIONS:
+        values: list[float] = []
+        eligible_count = missing_count = undefined_count = 0
+        for case in pack.cases:
+            result = by_case[case.case_id]
+            eligible = definition.metric_id == _LATENCY_METRIC or any(
+                group.required for group in pack.reference_for(case.case_id).evidence_groups
+            )
+            if not eligible:
+                undefined_count += 1
+                continue
+            eligible_count += 1
+            value = result.value_for(definition.metric_id)
+            if value is None:
+                missing_count += 1
+            else:
+                values.append(value)
+        complete = missing_count == 0 and eligible_count > 0
+        if complete:
+            value = _percentile(values, 0.5) if definition.metric_id == _LATENCY_METRIC else sum(values) / len(values)
+            p95 = _percentile(values, 0.95) if definition.metric_id == _LATENCY_METRIC else None
+        else:
+            value = p95 = None
+        aggregate[definition.metric_id] = MetricAggregate(
+            definition.metric_id, definition.release, definition.unit, definition.direction,
+            value, p95, eligible_count, len(values), missing_count, undefined_count, complete,
+        )
+    return aggregate