sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/comparison_core.py

@@ -0,0 +1,149 @@
+"""Storage-independent paired arithmetic and validated RAG comparisons.
+
+This module does not read files, resolve references, authenticate users, or run
+candidates. Its caller must first verify the records against their evidence and
+installed releases. The local results adapter performs that verification before
+calling this function; a hosted adapter must provide its own equivalent gate.
+
+The metric-only helper assigns no trust tier or report schema. The full-record
+adapter retains the existing illustrative reranking comparison semantics. This
+module is not a grader or a newly trusted hosted result schema.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+
+class ComparisonInputError(ValueError):
+    """Validated comparison records were supplied with misaligned case rows."""
+
+
+def compare_validated_records(
+    a: Mapping[str, Any],
+    b: Mapping[str, Any],
+    rows_a: Sequence[Mapping[str, Any]],
+    rows_b: Sequence[Mapping[str, Any]],
+) -> dict[str, Any]:
+    """Return the existing comparison record, including paired B-minus-A deltas.
+
+    ``a`` and ``b`` have the verified ``metrics.json`` shape. Their ordered case
+    judgments are supplied separately, so storage and execution are irrelevant
+    to the calculation. Candidate identities may differ; the workload, grading,
+    execution, and metric identities must match. Incomplete eligible pairs never
+    become zero, and a complete mean is null if any eligible pair is missing.
+
+    Latency uses each case's runner wall span, including unsupported questions.
+    Its comparison is an arithmetic mean of paired differences, not a difference
+    of aggregate p50/p95 values and not an inference-only latency claim.
+    """
+
+    base: dict[str, Any] = {
+        "schema_version": "sphynx.comparison.v1",
+        "evaluation_a": a["evaluation_id"],
+        "evaluation_b": b["evaluation_id"],
+        "compatible": False,
+        "reason": None,
+        "trust_tier": "local_unverified",
+        "illustrative": True,
+        "metrics": {},
+    }
+    if a["status"] not in {"completed", "incomplete"} or b["status"] not in {"completed", "incomplete"}:
+        base["reason"] = f"non-gradable evaluation status: A={a['status']}, B={b['status']}"
+        return base
+    for field in ("pack_id", "track", "trust_tier", "releases", "identity", "metric_definitions"):
+        if a.get(field) != b.get(field):
+            if field == "identity":
+                for subfield in ("pack_id", "pack_release", "track", "cases", "execution_policy", "runner", "trust_tier"):
+                    if a[field].get(subfield) != b[field].get(subfield):
+                        base["reason"] = f"comparison identity mismatch: {subfield}"
+                        return base
+            base["reason"] = f"comparison identity mismatch: {field}"
+            return base
+
+    # A storage adapter must never accidentally pair reordered or truncated rows.
+    # This checks the adapter boundary; it does not replace evidence verification.
+    cases = a["identity"]["cases"]
+    if len(rows_a) != len(cases) or len(rows_b) != len(cases):
+        raise ComparisonInputError("judgment count does not match evaluated cases")
+    for row_a, row_b, case in zip(rows_a, rows_b, cases):
+        if any(row.get("case_id") != case.get("case_id") or row.get("pool_digest") != case.get("pool_digest")
+               for row in (row_a, row_b)):
+            raise ComparisonInputError("judgment order or case/pool identity mismatch")
+        if any(type(row.get("answerable")) is not bool for row in (row_a, row_b)) or row_a["answerable"] != row_b["answerable"]:
+            raise ComparisonInputError("judgment eligibility differs between paired cases")
+
+    base["compatible"] = True
+    base["candidate_a"] = a["candidate"]
+    base["candidate_b"] = b["candidate"]
+    base["releases"] = a["releases"]
+    base["metrics"] = paired_metric_deltas(a["metric_definitions"], rows_a, rows_b)
+    return base
+
+
+def paired_metric_deltas(
+    metric_definitions: Sequence[Mapping[str, Any]],
+    rows_a: Sequence[Mapping[str, Any]],
+    rows_b: Sequence[Mapping[str, Any]],
+) -> dict[str, Any]:
+    """Calculate metric records without assigning a report schema or trust tier.
+
+    The trusted caller must authenticate and verify the evidence, finite metric
+    values, grading releases, and comparison compatibility before calling. This
+    helper only checks paired row alignment and computes arithmetic; it cannot
+    make unverified records trustworthy. It performs no storage or execution.
+
+    Rows use the current per-case metric contract: case_id, pool_digest,
+    answerable, status, and values. The existing unsupported-query eligibility
+    rule applies to quality metrics; runner_latency_ms includes every case.
+    """
+
+    if len(rows_a) != len(rows_b):
+        raise ComparisonInputError("judgment counts differ between paired records")
+    for row_a, row_b in zip(rows_a, rows_b):
+        if any(row_a.get(field) != row_b.get(field) for field in ("case_id", "pool_digest")):
+            raise ComparisonInputError("judgment order or case/pool identity mismatch")
+        if any(type(row.get("answerable")) is not bool for row in (row_a, row_b)) or row_a["answerable"] != row_b["answerable"]:
+            raise ComparisonInputError("judgment eligibility differs between paired cases")
+    metrics: dict[str, Any] = {}
+    for definition in metric_definitions:
+        metric_id = definition["metric_id"]
+        per_case: list[dict[str, Any]] = []
+        deltas: list[float] = []
+        undefined_count = missing_count = 0
+        for row_a, row_b in zip(rows_a, rows_b):
+            value_a = row_a["values"][metric_id]
+            value_b = row_b["values"][metric_id]
+            undefined = not row_a["answerable"] and metric_id != "runner_latency_ms"
+            if undefined:
+                undefined_count += 1
+                delta = None
+                reason = "unsupported_query"
+            elif value_a is None or value_b is None:
+                missing_count += 1
+                delta = None
+                reason = f"missing value: A={row_a['status'] if value_a is None else 'observed'}, " \
+                         f"B={row_b['status'] if value_b is None else 'observed'}"
+            else:
+                delta = value_b - value_a
+                deltas.append(delta)
+                reason = None
+            per_case.append({
+                "case_id": row_a["case_id"], "value_a": value_a, "value_b": value_b,
+                "delta_b_minus_a": delta, "reason": reason,
+            })
+        metrics[metric_id] = {
+            "unit": definition["unit"],
+            "direction": definition["direction"],
+            "primary": definition["primary"],
+            "eligible_count": len(rows_a) - undefined_count,
+            "paired_count": len(deltas),
+            "missing_pair_count": missing_count,
+            "undefined_count": undefined_count,
+            "complete": missing_count == 0 and bool(deltas),
+            "mean_delta": sum(deltas) / len(deltas) if missing_count == 0 and deltas else None,
+            "observed_mean_delta": sum(deltas) / len(deltas) if deltas else None,
+            "per_case": per_case,
+        }
+    return metrics

sphynx/comparison_views.py

@@ -0,0 +1,223 @@
+"""Immutable permitted comparison views; native policy remains authoritative."""
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+import json
+import math
+from typing import Any
+
+from .evaluation_protocols import _envelope
+from .workflow_contracts import (
+    WorkflowContractError, _object, _optional_text, _text, _thaw, contract_digest,
+)
+
+VIEW_SCHEMA = "sphynx.comparison-view.v1"
+
+
+def _number(value: Any, name: str) -> None:
+    if value is not None and (type(value) not in (int, float) or not math.isfinite(value)):
+        raise WorkflowContractError(name + " must be a finite number or null")
+
+
+def _items(value, cls, name):
+    if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
+        raise WorkflowContractError(name + " must be a sequence")
+    values = tuple(value)
+    if any(not isinstance(item, cls) for item in values):
+        raise WorkflowContractError(name + " contains an invalid typed value")
+    return values
+
+
+def _decode(cls, value):
+    if not isinstance(value, Mapping):
+        raise WorkflowContractError("comparison item must be an object")
+    try:
+        return cls(**dict(value))
+    except TypeError as exc:
+        raise WorkflowContractError("invalid comparison item fields") from exc
+
+
+@dataclass(frozen=True)
+class ComparisonPair:
+    """A native observation key and its values; keys do not imply independence."""
+
+    key: Mapping[str, Any]
+    baseline: float | bool | None
+    candidate: float | bool | None
+    delta: float | None
+    availability: str
+    reason: str | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        object.__setattr__(self, "key", _object(self.key, "pair key"))
+        if not self.key:
+            raise WorkflowContractError("pair requires an explicit observation key")
+        for name in ("baseline", "candidate", "delta"):
+            if name == "delta" or type(getattr(self, name)) is not bool:
+                _number(getattr(self, name), name)
+        if self.availability not in ("available", "missing", "undefined", "unavailable"):
+            raise WorkflowContractError("invalid pair availability")
+        _optional_text(self.reason, "pair reason")
+        if self.availability == "available":
+            if any(getattr(self, name) is None for name in ("baseline", "candidate", "delta")):
+                raise WorkflowContractError("available pair requires both values and native delta")
+        elif self.delta is not None or self.reason is None:
+            raise WorkflowContractError("unavailable pair requires a reason and null delta")
+        object.__setattr__(self, "metadata", _object(self.metadata, "pair metadata"))
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"key": _thaw(self.key), "baseline": self.baseline, "candidate": self.candidate,
+                "delta": self.delta, "availability": self.availability, "reason": self.reason,
+                "metadata": _thaw(self.metadata)}
+
+
+@dataclass(frozen=True)
+class ComparisonMetric:
+    """Native metric meaning, complete/observed deltas and coverage."""
+
+    metric_id: str
+    unit: str
+    direction: str
+    delta_convention: str
+    status: str
+    population: Mapping[str, Any]
+    method: Mapping[str, Any]
+    delta: float | None = None
+    observed_delta: float | None = None
+    counts: Mapping[str, int] = field(default_factory=dict)
+    pairs: tuple[ComparisonPair, ...] = ()
+    uncertainty: Mapping[str, Any] | None = None
+    reason: str | None = None
+
+    def __post_init__(self):
+        for name in ("metric_id", "unit", "direction", "delta_convention"):
+            _text(getattr(self, name), name)
+        if self.status not in ("supported", "incomplete", "unavailable"):
+            raise WorkflowContractError("invalid comparison metric status")
+        for name in ("delta", "observed_delta"):
+            _number(getattr(self, name), name)
+        _optional_text(self.reason, "metric reason")
+        if (self.status == "supported") != (self.delta is not None):
+            raise WorkflowContractError("metric status and complete-population delta disagree")
+        if self.status == "unavailable" and (self.observed_delta is not None or self.reason is None):
+            raise WorkflowContractError("unavailable metric requires reason and null observed delta")
+        for name in ("population", "method", "counts"):
+            object.__setattr__(self, name, _object(getattr(self, name), name))
+        if not self.population or not self.method:
+            raise WorkflowContractError("metric requires explicit population and method")
+        if any(type(value) is not int or value < 0 for value in self.counts.values()):
+            raise WorkflowContractError("comparison counts must be nonnegative integers")
+        pairs = _items(self.pairs, ComparisonPair, "pairs")
+        if len({contract_digest(pair.key) for pair in pairs}) != len(pairs):
+            raise WorkflowContractError("duplicate comparison observation key")
+        object.__setattr__(self, "pairs", pairs)
+        if self.uncertainty is not None:
+            object.__setattr__(self, "uncertainty", _object(self.uncertainty, "uncertainty"))
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"metric_id": self.metric_id, "unit": self.unit, "direction": self.direction,
+                "delta_convention": self.delta_convention, "status": self.status,
+                "population": _thaw(self.population), "method": _thaw(self.method),
+                "delta": self.delta, "observed_delta": self.observed_delta,
+                "counts": _thaw(self.counts), "pairs": [pair.to_dict() for pair in self.pairs],
+                "uncertainty": None if self.uncertainty is None else _thaw(self.uncertainty),
+                "reason": self.reason}
+
+
+@dataclass(frozen=True)
+class ComparisonView:
+    """Readable native comparison projection, with no rank or acceptance verdict."""
+
+    baseline: Mapping[str, Any]
+    candidate: Mapping[str, Any]
+    backend: Mapping[str, Any]
+    status: str
+    native: Mapping[str, Any] = field(repr=False)
+    metrics: tuple[ComparisonMetric, ...] = ()
+    reason: str | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        for name in ("baseline", "candidate", "backend", "native", "metadata"):
+            object.__setattr__(self, name, _object(getattr(self, name), name))
+        for source in (self.baseline, self.candidate):
+            for name in ("grading_id", "status", "trust_tier"):
+                _text(source.get(name), "comparison source " + name)
+            _optional_text(source.get("run_id"), "comparison source run_id")
+        _text(self.backend.get("backend_id"), "comparison backend_id")
+        if self.status not in ("supported", "incompatible", "incomplete", "unavailable"):
+            raise WorkflowContractError("invalid comparison status")
+        _optional_text(self.reason, "comparison reason")
+        metrics = _items(self.metrics, ComparisonMetric, "metrics")
+        if len({metric.metric_id for metric in metrics}) != len(metrics):
+            raise WorkflowContractError("duplicate comparison metric")
+        if self.status in ("incompatible", "unavailable") and (metrics or self.reason is None):
+            raise WorkflowContractError("refused/unavailable view requires reason and no metrics")
+        if self.status == "supported" and any(metric.status == "incomplete" for metric in metrics):
+            raise WorkflowContractError("supported view cannot hide incomplete metrics")
+        object.__setattr__(self, "metrics", metrics)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": VIEW_SCHEMA, "baseline": _thaw(self.baseline),
+                "candidate": _thaw(self.candidate), "backend": _thaw(self.backend),
+                "status": self.status, "reason": self.reason,
+                "metrics": [metric.to_dict() for metric in self.metrics],
+                "metadata": _thaw(self.metadata), "native": _thaw(self.native)}
+
+    @classmethod
+    def from_dict(cls, value: Mapping[str, Any]) -> ComparisonView:
+        values = _envelope(value, VIEW_SCHEMA, cls)
+        raw_metrics = values.get("metrics", ())
+        if isinstance(raw_metrics, (str, bytes)) or not isinstance(raw_metrics, Sequence):
+            raise WorkflowContractError("metrics must be an array")
+        metrics = []
+        for raw in raw_metrics:
+            if not isinstance(raw, Mapping):
+                raise WorkflowContractError("metric must be an object")
+            item = dict(raw)
+            pairs = item.get("pairs", ())
+            if isinstance(pairs, (str, bytes)) or not isinstance(pairs, Sequence):
+                raise WorkflowContractError("pairs must be an array")
+            item["pairs"] = tuple(_decode(ComparisonPair, pair) for pair in pairs)
+            metrics.append(_decode(ComparisonMetric, item))
+        values["metrics"] = tuple(metrics)
+        return _decode(cls, values)
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, allow_nan=False, sort_keys=True)
+
+    def to_markdown(self) -> str:
+        def cell(value):
+            return str(value).replace("|", "\\|").replace("\n", " ").replace("\r", " ")
+        lines = ["# Evaluation comparison", "",
+                 "Baseline: " + cell(self.baseline["grading_id"]),
+                 "Candidate: " + cell(self.candidate["grading_id"]),
+                 "Status: " + self.status]
+        if self.reason:
+            lines.append("Reason: " + cell(self.reason))
+        if self.metrics:
+            lines += ["", "| Metric | Unit / direction | Complete delta | Observed delta | Coverage |",
+                      "| --- | --- | ---: | ---: | --- |"]
+            for metric in self.metrics:
+                lines.append("| " + " | ".join(cell(value) for value in (
+                    metric.metric_id, metric.unit + " / " + metric.direction,
+                    metric.delta if metric.delta is not None else "unavailable",
+                    metric.observed_delta if metric.observed_delta is not None else "unavailable",
+                    json.dumps(_thaw(metric.counts), sort_keys=True))) + " |")
+            lines.append("")
+            for metric in self.metrics:
+                lines.append(cell(metric.metric_id + ": " + metric.delta_convention + "; " +
+                                  json.dumps(_thaw(metric.method), sort_keys=True)))
+                lines.append("Status: " + metric.status)
+                if metric.reason:
+                    lines.append("Reason: " + cell(metric.reason))
+                lines.append("Uncertainty: " + (cell(json.dumps(_thaw(metric.uncertainty), sort_keys=True))
+                             if metric.uncertainty is not None else "not estimated by the native policy"))
+                reasons = sorted({pair.reason for pair in metric.pairs if pair.reason is not None})
+                if reasons:
+                    lines.append("Missing/undefined pair reasons: " + "; ".join(cell(reason) for reason in reasons))
+        lines += ["", "Native policy controls compatibility. No rank or acceptance verdict is inferred.",
+                  "Use to_dict() for populations, case/trial pairs, limitations and native detail."]
+        return "\n".join(lines) + "\n"