sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/_pr_client.py

@@ -0,0 +1,237 @@
+"""Durable PR operation handles over the existing authenticated transport."""
+from __future__ import annotations
+from collections.abc import Callable
+from typing import Any, TYPE_CHECKING
+from urllib.parse import quote
+import math
+import time
+
+from .client import HostedClientError, _retry_key
+from .pr_evaluations import PrEvaluationRequest, PrEvaluationStatus, PrEvaluationResult, _saved
+from ._analytical_values import identifier, require, MetricProjectionSummary, ReportProjectionSummary, ComparisonProjectionSummary
+from .analytical_views import ProjectionPage
+
+if TYPE_CHECKING:
+    from .client import Client
+    from .diagnostics import Diagnostic
+    from .progress import ProgressView
+    from .pr_evaluations import PrCaseChangePage
+
+_READ_RETRY={'hosted_transport_failed','hosted_request_failed','service_unavailable','store_unavailable',
+             'projection_unavailable','pr_evaluation_conflict','result_not_ready','evaluation_projection_pending'}
+_PARSE_ERRORS=(ValueError,TypeError,KeyError,AttributeError,RecursionError,UnicodeError)
+
+
+class PrEvaluationError(HostedClientError):
+    """Keep the original request/key or known handle after an uncertain response."""
+    def __init__(self,cause:HostedClientError,*,pr_evaluation_id:str|None=None,request:PrEvaluationRequest|None=None,
+                 idempotency_key:str|None=None,handle:PrEvaluation|None=None,submission:bool=False,operation_failed:bool=False):
+        self.pr_evaluation_id,self.request,self.idempotency_key=pr_evaluation_id,request,idempotency_key
+        self._submission=submission
+        self._operation_failed=operation_failed
+        message=('PR submission needs reconciliation; retain the original request and key' if submission
+                 else 'PR evaluation failed; read result() on the original handle to inspect retained coverage' if operation_failed
+                 else 'PR evaluation read did not complete; retain the original handle and inspect its status')
+        super().__init__(cause.code,message,handle=handle)
+    @property
+    def diagnostic(self)->Diagnostic:
+        from .diagnostics import Diagnostic
+        recovery={} if self.pr_evaluation_id is None else {'record_id':self.pr_evaluation_id}
+        if self._operation_failed:
+            return Diagnostic(self.code,'The PR operation failed; its permitted results remain readable.',
+                ('Read result() on the same handle to inspect coverage and errors before choosing a new operation.',),
+                retryability='fix_required',recovery=recovery)
+        if self._submission:
+            return Diagnostic(self.code,'The PR submission outcome needs reconciliation.',
+                ('Replay the original immutable request with the original idempotency key; retain any returned handle.',),
+                retryability='reconcile_before_retry',recovery=recovery)
+        if self.code in _READ_RETRY or self.code=='wait_timeout':
+            return Diagnostic(self.code,'Read the original PR evaluation handle again.',
+                ('Resume waiting or read result() on the same handle; use get_pr_evaluation() after restarting.',),
+                retryability='read_original_handle',recovery=recovery)
+        return Diagnostic(self.code,'The selected PR evaluation needs an access or input check.',
+            ('Inspect the retained handle; have the operator check current access and private profile admission.',),
+            retryability='fix_required',recovery=recovery)
+
+
+def _invalid()->HostedClientError:
+    return HostedClientError('invalid_server_response','the PR operation response could not be verified')
+
+
+def _path(identity:str)->str:return '/v1/pr-evaluations/'+_saved(identity,'preval')
+
+
+def _budget(deadline:float|None)->None:
+    if deadline is not None and time.monotonic()>=deadline:raise HostedClientError('wait_timeout','the read budget expired')
+
+
+def _bound(previous:PrEvaluationStatus,fresh:PrEvaluationStatus)->None:
+    require((fresh.id,fresh.request,fresh.template,fresh.expected_children,fresh.created_at,fresh.deadline_at)==
+            (previous.id,previous.request,previous.template,previous.expected_children,previous.created_at,previous.deadline_at))
+    def roster(status):
+        return {r.requirement_id:(r.suite_id,r.scope,r.mode,r.bank_id,r.release_id,r.cohort) for r in status.requirements}
+    require(roster(previous)==roster(fresh))
+    if previous.revisions is not None:require((previous.revisions,previous.plan_digest)==(fresh.revisions,fresh.plan_digest))
+    if previous.decision_id is not None:require(fresh.decision_id==previous.decision_id)
+    if previous.status in ('completed','failed'):require(previous.status==fresh.status)
+
+
+def _status(client:Client,identity:str,*,handle:PrEvaluation|None=None,deadline:float|None=None)->PrEvaluationStatus:
+    try:
+        try:
+            _budget(deadline)
+            options={} if deadline is None else {'deadline':deadline}
+            status=PrEvaluationStatus.from_dict(client._request('GET',_path(identity),**options))
+            require(status.id==identity)
+            if handle is not None:_bound(handle._status,status)
+            return status
+        except _PARSE_ERRORS:raise _invalid() from None
+    except HostedClientError as error:
+        raise PrEvaluationError(error,pr_evaluation_id=identity,handle=handle) from None
+
+
+def submit(client:Client,request:PrEvaluationRequest,*,idempotency_key:str|None=None)->PrEvaluation:
+    require(isinstance(request,PrEvaluationRequest),'submit a typed PrEvaluationRequest')
+    original=PrEvaluationRequest.from_dict(request.to_dict());key=_retry_key(idempotency_key)
+    body={'schema_version':'sphynx.pr-evaluation-submission.v1','request':original.to_dict(),'idempotency_key':key}
+    try:
+        raw=client._request('POST','/v1/pr-evaluations',body,retry_key=key)
+        try:
+            status=PrEvaluationStatus.from_dict(raw);require(status.request==original)
+        except _PARSE_ERRORS:raise _invalid() from None
+        return PrEvaluation(client,status,idempotency_key=key)
+    except HostedClientError as error:
+        raise PrEvaluationError(error,request=original,idempotency_key=key,submission=True) from None
+
+
+def get(client:Client,identity:str)->PrEvaluation:
+    _saved(identity,'preval');return PrEvaluation(client,_status(client,identity))
+
+
+class PrEvaluation:
+    """One durable private PR operation. Polling never submits candidate work."""
+    def __init__(self,client:Client,status:PrEvaluationStatus,*,idempotency_key:str|None=None):
+        require(isinstance(status,PrEvaluationStatus))
+        self._client,self._status,self.id,self.idempotency_key=client,status,status.id,idempotency_key
+        self._last_result:PrEvaluationResult|None=None
+    @property
+    def status(self)->str:return self._status.status
+    @property
+    def request(self)->PrEvaluationRequest:return self._status.request
+    @property
+    def observation(self)->PrEvaluationStatus:return self._status
+    @property
+    def progress(self)->ProgressView:
+        from .progress import _progress
+        return _progress('submission',self.status,{'submission_id':self.id})
+    def refresh(self)->PrEvaluation:
+        return self._refresh()
+    def _refresh(self,deadline:float|None=None)->PrEvaluation:
+        fresh=_status(self._client,self.id,handle=self,deadline=deadline)
+        self._status=fresh;return self
+    def result(self)->PrEvaluationResult:
+        """Read finished reports or terminal coverage without executing work."""
+        return self._result()
+    def _result(self,deadline:float|None=None)->PrEvaluationResult:
+        try:
+            options={} if deadline is None else {'deadline':deadline}
+            _budget(deadline)
+            raw=self._client._request('GET',_path(self.id)+'/result',**options)
+            try:
+                result=PrEvaluationResult.from_dict(raw);_bound(self._status,result.operation)
+            except _PARSE_ERRORS:raise _invalid() from None
+            self._status,self._last_result=result.operation,result
+            return result
+        except HostedClientError as error:raise PrEvaluationError(error,pr_evaluation_id=self.id,handle=self) from None
+    def projection(self,requirement_id:str)->ProjectionPage:
+        """Read one authorized existing v2 projection; it grants no acceptance."""
+        requirement=next((r for r in self._status.requirements if r.requirement_id==identifier(requirement_id)),None)
+        require(requirement is not None,'requirement_id must belong to the original operation')
+        assert requirement is not None
+        try:
+            raw=self._client._request('GET',_path(self.id)+'/requirements/'+quote(requirement_id,safe='')+'/projection')
+            try:
+                page=ProjectionPage.from_dict(raw)
+                require(page.selection is None and len(page.items)==1 and
+                        (page.bank_id,page.release_id,page.cohort_id)==(requirement.bank_id,requirement.release_id,requirement.cohort.identity_id))
+                if page.items[0].availability=='available':
+                    reports=[r.summary for r in page.rows if r.row_kind=='report'];metrics=[r.summary for r in page.rows if r.row_kind=='metric']
+                    comparisons=[r.summary for r in page.rows if r.row_kind=='comparison']
+                    require(len(reports)==1 and len(comparisons)<=1 and len(metrics)+len(comparisons)+1==len(page.rows) and isinstance(reports[0],ReportProjectionSummary))
+                    report=reports[0];assert isinstance(report,ReportProjectionSummary)
+                    require((report.scope,report.mode)==(requirement.scope,requirement.mode))
+                    typed=[m for m in metrics if isinstance(m,MetricProjectionSummary)];require(len(typed)==len(metrics))
+                    require(len({m.metric_id for m in typed})==len(typed) and all((m.run_id,m.grading_id)==(report.run_id,report.grading_id) for m in typed))
+                    require(all(isinstance(c,ComparisonProjectionSummary) and c.candidate_grading_id==report.grading_id for c in comparisons))
+                if self._last_result is not None:
+                    observed=next(r for r in self._last_result.requirements if r.status.requirement_id==requirement_id)
+                    if observed.report is not None:
+                        proof=observed.report.projection;require(page.team_id==observed.report.team_id and
+                            (page.items[0].event_id,page.items[0].source_digest)==(proof.event_id,proof.source_digest))
+                        if page.items[0].availability=='available':
+                            require(report==observed.report.report)
+                            require({m.metric_id:m for m in typed}=={m.metric_id:m for m in observed.report.metrics})
+                            if observed.comparison.regressions is not None:
+                                require(len(comparisons)==1 and isinstance(comparisons[0],ComparisonProjectionSummary)
+                                    and comparisons[0].artifact_digest==observed.comparison.regressions.comparison_digest)
+                return page
+            except _PARSE_ERRORS:raise _invalid() from None
+        except HostedClientError as error:raise PrEvaluationError(error,pr_evaluation_id=self.id,handle=self) from None
+    def case_changes(self,requirement_id:str,*,cursor:str|None=None,limit:int=100)->PrCaseChangePage:
+        """Read case scores from a committed comparison. This starts no work."""
+        from .pr_evaluations import PrCaseChangePage, _case_cursor
+        require(any(r.requirement_id==identifier(requirement_id) for r in self._status.requirements),
+                'requirement_id must belong to the original operation')
+        require(type(limit) is int and 1<=limit<=500,'limit must be an integer from 1 to 500')
+        query='?limit='+str(limit)
+        if cursor is not None:query+='&cursor='+quote(_case_cursor(cursor),safe='')
+        try:
+            raw=self._client._request('GET',_path(self.id)+'/requirements/'+quote(requirement_id,safe='')+'/case-changes'+query)
+            try:
+                page=PrCaseChangePage.from_dict(raw)
+                require((page.plan_id,page.requirement_id)==(self.id,requirement_id) and len(page.items)<=limit)
+                if cursor is None:require(page.offset==0)
+                if self._last_result is not None:
+                    observed=next(r for r in self._last_result.requirements if r.status.requirement_id==requirement_id)
+                    require(observed.report is not None and observed.comparison.regressions is not None)
+                    assert observed.report is not None and observed.comparison.regressions is not None
+                    require((page.source_digest,page.comparison_digest,page.total)==(observed.report.projection.source_digest,
+                        observed.comparison.regressions.comparison_digest,observed.comparison.regressions.case_change_count))
+                return page
+            except _PARSE_ERRORS:raise _invalid() from None
+        except HostedClientError as error:raise PrEvaluationError(error,pr_evaluation_id=self.id,handle=self) from None
+    def wait(self,*,timeout:float=900,poll_interval:float=2,on_progress:Callable[[str],None]|None=None)->PrEvaluationResult:
+        """Wait on this handle; socket/retry phases use the remaining read budget."""
+        return self._wait(timeout=timeout,poll_interval=poll_interval,on_progress=on_progress,reports_only=False)
+    def wait_for_reports(self,*,timeout:float=900,poll_interval:float=2,on_progress:Callable[[str],None]|None=None)->PrEvaluationResult:
+        """Read completed required reports while human review or policy is pending.
+
+        A returned report is not an acceptance decision. Use current_acceptance
+        and the bank review status in the returned result for that distinction.
+        """
+        return self._wait(timeout=timeout,poll_interval=poll_interval,on_progress=on_progress,reports_only=True)
+    def _wait(self,*,timeout:float,poll_interval:float,on_progress:Callable[[str],None]|None,reports_only:bool)->PrEvaluationResult:
+        for value in (timeout,poll_interval):
+            if type(value) not in (int,float) or not math.isfinite(value) or value<=0:raise ValueError('timeout and poll_interval must be finite positive seconds')
+        deadline=time.monotonic()+timeout;previous=None
+        while True:
+            if self.status!=previous:
+                if on_progress is not None:on_progress(self.status)
+                previous=self.status
+            if self.status=='failed':
+                code=self._status.error.code if self._status.error is not None else 'pr_evaluation_failed'
+                raise PrEvaluationError(HostedClientError(code,'PR evaluation failed; inspect result() for retained coverage'),pr_evaluation_id=self.id,handle=self,operation_failed=True)
+            ready=bool(self._status.requirements) and all(r.state in ('completed','incomplete') for r in self._status.requirements)
+            if self.status=='completed' or reports_only and ready:
+                try:return self._result(deadline)
+                except PrEvaluationError as error:
+                    if error.code not in _READ_RETRY:raise
+            remaining=deadline-time.monotonic()
+            if remaining<=0:raise PrEvaluationError(HostedClientError('wait_timeout','waiting expired'),pr_evaluation_id=self.id,handle=self)
+            time.sleep(min(poll_interval,remaining))
+            try:self._refresh(deadline)
+            except PrEvaluationError as error:
+                if error.code not in _READ_RETRY:raise
+                if time.monotonic()>=deadline:raise PrEvaluationError(HostedClientError('wait_timeout','waiting expired'),pr_evaluation_id=self.id,handle=self) from None
+    def to_dict(self)->dict[str,Any]:return self._status.to_dict()
+    def __repr__(self)->str:return f'PrEvaluation(id={self.id!r}, status={self.status!r})'

sphynx/_rag_diagnostics_calibration.py

@@ -0,0 +1,285 @@
+"""Offline judge audit and gated PPI arithmetic over supplied labels.
+
+This module does not run a model or check that a declared sampling design is
+true. Descriptive audit scores apply only to the supplied records. PPI output is
+an arithmetic demonstration, never an automatically validated performance claim
+or ARES parity result.
+
+``judge_audit`` rows contain ``id``, ``human_label`` (bool or None),
+``judge_label`` (bool or None) and ``judge_status`` (judged, abstained, missing).
+An omitted status means judged for a boolean label and missing otherwise.
+
+``ppi_mean`` uses a separate prediction cohort and labeled audit cohort. Rows
+contain ``id``, ``cluster_id``, numeric ``prediction`` in [0, 1], and the same
+frozen ``predictor_digest``. Audit rows also need a boolean ``human_label``.
+
+Required PPI declarations are ``predictor_digest`` (sha256:<64 hex digits>),
+``predictor_frozen``, ``audit_representative``, ``prediction_representative``,
+``same_target_population``, ``cohorts_independent``, ``cohorts_disjoint``,
+``iid_rows``, ``audit_held_out_from_predictor_tuning`` (all True),
+``target_population``, ``sampling_protocol``, ``predictor_model``,
+``predictor_revision``, ``prompt_version`` and ``metric_version`` (nonempty strings), and
+``provenance_kind`` (synthetic or independent_human_audit).
+
+Repeated or shared cluster IDs need a different variance estimator and are not
+accepted. Unknown labels, overlapping IDs and undeclared designs are not
+silently filtered. At least two complete observations in each cohort are needed
+for sample variances. Small synthetic samples still do not support inference.
+
+Method source: https://arxiv.org/html/2301.09633v3 (independent-samples PPI
+mean estimator). ARES: https://aclanthology.org/2024.naacl-long.20/ . This module
+does not reproduce ARES judge training or establish finite-sample coverage.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from statistics import NormalDist, fmean, variance
+from typing import Any, Mapping, Sequence
+
+
+class CalibrationError(ValueError):
+    """The supplied audit records do not follow the explicit label contract."""
+
+
+_DECLARATIONS = (
+    "predictor_frozen",
+    "audit_representative",
+    "prediction_representative",
+    "same_target_population",
+    "cohorts_independent",
+    "cohorts_disjoint",
+    "iid_rows",
+    "audit_held_out_from_predictor_tuning",
+)
+_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
+
+
+def _ratio(numerator: int, denominator: int, reason: str) -> dict[str, Any]:
+    return {
+        "value": numerator / denominator if denominator else None,
+        "numerator": numerator,
+        "denominator": denominator,
+        "status": "observed" if denominator else "undefined",
+        "reason": None if denominator else reason,
+    }
+
+
+def _rows(value: Sequence[Mapping[str, Any]]) -> list[Mapping[str, Any]]:
+    if not isinstance(value, (list, tuple)) or any(not isinstance(row, Mapping) for row in value):
+        raise CalibrationError("records_must_be_an_array_of_objects")
+    result = list(value)
+    ids = [row.get("id") for row in result]
+    if any(type(value) is not str or not value.strip() for value in ids):
+        raise CalibrationError("record_ids_must_be_nonempty_strings")
+    if len(set(ids)) != len(ids):
+        raise CalibrationError("duplicate_record_ids")
+    return result
+
+
+def judge_audit(
+    records: Sequence[Mapping[str, Any]], *, provenance_kind: str = "unspecified"
+) -> dict[str, Any]:
+    """Return descriptive confusion metrics with explicit missing populations.
+
+    Accuracy and disagreement use only decisive judge labels paired with known
+    human labels. Abstention and missing-judge rates use all supplied records.
+    Neither unknown human labels nor judge abstentions become negative labels.
+    """
+
+    rows = _rows(records)
+    tp = fp = tn = fn = 0
+    missing_human = missing_judge = abstained = decisive = 0
+    for row in rows:
+        human, judge = row.get("human_label"), row.get("judge_label")
+        if human is not None and type(human) is not bool:
+            raise CalibrationError("human_label_must_be_boolean_or_null")
+        if judge is not None and type(judge) is not bool:
+            raise CalibrationError("judge_label_must_be_boolean_or_null")
+        status = row.get("judge_status", "judged" if judge is not None else "missing")
+        if status not in {"judged", "abstained", "missing"}:
+            raise CalibrationError("unsupported_judge_status")
+        if (status == "judged") != (judge is not None):
+            raise CalibrationError("judge_status_and_label_disagree")
+        missing_human += human is None
+        missing_judge += status == "missing"
+        abstained += status == "abstained"
+        decisive += status == "judged"
+        if human is None or judge is None:
+            continue
+        tp += human and judge
+        fp += not human and judge
+        tn += not human and not judge
+        fn += human and not judge
+    paired = tp + fp + tn + fn
+    count = len(rows)
+    metrics = {
+        "accuracy": _ratio(tp + tn, paired, "no_paired_decisive_judgments"),
+        "precision": _ratio(tp, tp + fp, "no_predicted_positives"),
+        "recall": _ratio(tp, tp + fn, "no_human_positives"),
+        "specificity": _ratio(tn, tn + fp, "no_human_negatives"),
+        "disagreement": _ratio(fp + fn, paired, "no_paired_decisive_judgments"),
+        "judge_abstention_rate": _ratio(abstained, count, "empty_audit"),
+        "judge_missing_rate": _ratio(missing_judge, count, "empty_audit"),
+        "human_label_coverage": _ratio(count - missing_human, count, "empty_audit"),
+        "paired_decisive_coverage": _ratio(paired, count, "empty_audit"),
+    }
+    chance_agreement = None
+    kappa = None
+    kappa_reason = "no_paired_decisive_judgments"
+    if paired:
+        chance_agreement = ((tp + fp) * (tp + fn) + (tn + fn) * (tn + fp)) / paired**2
+        if chance_agreement < 1:
+            kappa = ((tp + tn) / paired - chance_agreement) / (1 - chance_agreement)
+            kappa_reason = None
+        else:
+            kappa_reason = "constant_labels_make_chance_agreement_one"
+    metrics["cohen_kappa"] = {
+        "value": kappa,
+        "denominator": paired,
+        "expected_agreement": chance_agreement,
+        "status": "observed" if kappa is not None else "undefined",
+        "reason": kappa_reason,
+    }
+    return {
+        "schema_version": "sphynx.rag-diagnostics.judge-audit.v1",
+        "scope": "supplied_records_descriptive_only",
+        "provenance_kind": provenance_kind,
+        "counts": {
+            "records": count,
+            "human_labeled": count - missing_human,
+            "missing_human_label": missing_human,
+            "judge_decisive": decisive,
+            "judge_abstained": abstained,
+            "missing_judge_label": missing_judge,
+            "paired_decisive": paired,
+        },
+        "confusion": {"true_positive": tp, "false_positive": fp, "true_negative": tn, "false_negative": fn},
+        "metrics": metrics,
+        "performance_inference_validated": False,
+        "ares_parity": False,
+        "notice": "Scores describe the supplied records. Unknown, abstained and missing labels are not negatives. Independent sampling and population transfer are not established.",
+    }
+
+
+def _unavailable(reason: str, predictions: int, audit: int) -> dict[str, Any]:
+    return {
+        "schema_version": "sphynx.rag-diagnostics.ppi-mean-arithmetic.v1",
+        "status": "unavailable",
+        "reason": reason,
+        "prediction_count": predictions,
+        "audit_count": audit,
+        "raw_point_estimate": None,
+        "raw_normal_interval": None,
+        "performance_confidence_interval": None,
+        "performance_inference_validated": False,
+        "ares_parity": False,
+    }
+
+
+def ppi_mean(
+    prediction_cohort: Sequence[Mapping[str, Any]],
+    audit_cohort: Sequence[Mapping[str, Any]],
+    *,
+    declarations: Mapping[str, Any] | None = None,
+    confidence: float = 0.95,
+) -> dict[str, Any]:
+    """Compute a raw rectified-mean example under declared disjoint IID design.
+
+    Point estimate = mean(prediction cohort) + mean(human - audit prediction).
+    Variance = s²(prediction cohort)/N + s²(audit residuals)/n.
+
+    The normal interval is intentionally unbounded and separately named. This
+    helper never clips the estimate or interval to [0, 1], never claims a finite
+    sample coverage guarantee, and never upgrades declarations into verified
+    independence. Synthetic output remains demonstration arithmetic.
+    """
+
+    try:
+        predictions, audit = _rows(prediction_cohort), _rows(audit_cohort)
+    except CalibrationError as error:
+        return _unavailable(str(error), len(prediction_cohort) if isinstance(prediction_cohort, (list, tuple)) else 0,
+                            len(audit_cohort) if isinstance(audit_cohort, (list, tuple)) else 0)
+    unavailable = lambda reason: _unavailable(reason, len(predictions), len(audit))
+    if not isinstance(declarations, Mapping):
+        return unavailable("sampling_and_provenance_declarations_required")
+    missing = [name for name in _DECLARATIONS if declarations.get(name) is not True]
+    if missing:
+        return unavailable("sampling_declarations_not_affirmed:" + ",".join(missing))
+    for name in ("target_population", "sampling_protocol", "predictor_model",
+                 "predictor_revision", "prompt_version", "metric_version"):
+        if type(declarations.get(name)) is not str or not declarations[name].strip():
+            return unavailable(name + "_required")
+    digest = declarations.get("predictor_digest")
+    if type(digest) is not str or _DIGEST.fullmatch(digest) is None:
+        return unavailable("frozen_predictor_digest_required")
+    if declarations.get("provenance_kind") not in {"synthetic", "independent_human_audit"}:
+        return unavailable("explicit_provenance_kind_required")
+    if type(confidence) not in (int, float) or not math.isfinite(confidence) or not 0 < confidence < 1:
+        return unavailable("invalid_confidence")
+    quantile_probability = 0.5 + confidence / 2
+    if not 0 < quantile_probability < 1:
+        return unavailable("normal_quantile_not_representable")
+    if len(predictions) < 2 or len(audit) < 2:
+        return unavailable("at_least_two_observations_per_cohort_required")
+    if {row["id"] for row in predictions} & {row["id"] for row in audit}:
+        return unavailable("cohorts_overlap")
+    clusters = []
+    for row in predictions + audit:
+        if row.get("predictor_digest") != digest:
+            return unavailable("predictor_identity_mismatch")
+        value = row.get("prediction")
+        if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1:
+            return unavailable("prediction_must_be_finite_probability")
+        cluster = row.get("cluster_id")
+        if type(cluster) is not str or not cluster.strip():
+            return unavailable("explicit_independent_cluster_ids_required")
+        clusters.append(cluster)
+    if len(set(clusters)) != len(clusters):
+        return unavailable("shared_or_repeated_clusters_need_different_variance")
+    if any(type(row.get("human_label")) is not bool for row in audit):
+        return unavailable("complete_boolean_human_audit_labels_required")
+    prediction_values = [float(row["prediction"]) for row in predictions]
+    residuals = [float(row["human_label"]) - float(row["prediction"]) for row in audit]
+    prediction_mean = fmean(prediction_values)
+    correction = fmean(residuals)
+    point = prediction_mean + correction
+    prediction_variance = variance(prediction_values) / len(predictions)
+    residual_variance = variance(residuals) / len(audit)
+    standard_error = math.sqrt(prediction_variance + residual_variance)
+    quantile = NormalDist().inv_cdf(quantile_probability)
+    synthetic = declarations["provenance_kind"] == "synthetic"
+    return {
+        "schema_version": "sphynx.rag-diagnostics.ppi-mean-arithmetic.v1",
+        "status": "arithmetic_only",
+        "reason": "synthetic_demonstration_no_performance_inference" if synthetic else "sampling_declarations_not_independently_verified",
+        "prediction_count": len(predictions),
+        "audit_count": len(audit),
+        "raw_prediction_mean": prediction_mean,
+        "raw_human_residual_correction": correction,
+        "raw_point_estimate": point,
+        "raw_normal_interval": {
+            "lower_unbounded": point - quantile * standard_error,
+            "upper_unbounded": point + quantile * standard_error,
+            "confidence": float(confidence),
+            "standard_error": standard_error,
+            "prediction_variance_component": prediction_variance,
+            "audit_residual_variance_component": residual_variance,
+            "method": "independent_disjoint_iid_normal_approximation",
+            "finite_sample_coverage_guarantee": False,
+        },
+        "performance_confidence_interval": None,
+        "performance_inference_validated": False,
+        "sampling_design_verified": False,
+        "synthetic_demonstration": synthetic,
+        "ares_parity": False,
+        "predictor_digest": digest,
+        "declared_predictor_identity": {
+            key: declarations[key]
+            for key in ("predictor_model", "predictor_revision", "prompt_version", "metric_version")
+        },
+        "declared_target_population": declarations["target_population"],
+        "declared_sampling_protocol": declarations["sampling_protocol"],
+        "notice": "Raw PPI mean arithmetic only. The normal approximation is not validated performance confidence. Synthetic or tiny samples do not establish population quality. Sampling, independent human review and frozen predictor provenance require separate verification.",
+    }

sphynx/_rag_diagnostics_catalog.py

@@ -0,0 +1,39 @@
+"""Readable metric definitions for the local review surface."""
+INFO={
+ 'retrieval.recall':('Document recall',r'R_k / R','Known positive qrels found in the first k results / all known positive qrels. The full-reference denominator stays fixed when selection drops documents.'),
+ 'retrieval.precision':('Document precision',r'R_k / k','Known positive qrels in the first k positions / k, including unfilled positions. Unjudged entries keep their original ranks.'),
+ 'retrieval.average_precision':('Average precision',r'\frac{1}{R}\sum_{r=1}^{k}P@r\,\mathbf{1}[r\text{ relevant}]','AP per case. Macro-averaging eligible case AP values produces MAP. Denominator is all known positives, not min(R,k).'),
+ 'retrieval.ndcg_linear':('nDCG · linear gain',r'\frac{\sum_{r=1}^{k}g_r/\log_2(r+1)}{\operatorname{IDCG}@k}','trec_eval-style linear relevance gain. This is a distinct metric from the existing Sphynx exponential-gain nDCG.'),
+ 'retrieval.reciprocal_rank':('Reciprocal rank',r'1/r_{\mathrm{first\ relevant}}','Zero when no known positive appears by k. Macro-averaging the per-case values produces MRR.'),
+ 'retrieval.unjudged_fraction':('Unjudged fraction',r'U_k / |L_{:k}|','Unknown judgments / returned items by k. An unknown contributes zero gain to the compatibility projection, but remains unknown, never a negative label.'),
+ 'evidence.atom_recall':('Required-atom recall',r'|A_{\mathrm{covered}}|/|A_{\mathrm{required}}|','Coverage of distinct required evidence atoms at each recorded stage. This is an evidence proxy, not context sufficiency.'),
+ 'evidence.relation_recall':('Required-relation recall',r'|E_{\mathrm{covered}}|/|E_{\mathrm{required}}|','Coverage of explicitly labeled required relations; independent from individual atom coverage.'),
+ 'evidence.complete_required_atoms':('All required atoms',r'\mathbf{1}[A_{\mathrm{required}}\subseteq A_{\mathrm{covered}}]','One only if every required atom is covered. It does not establish independently judged sufficiency.'),
+ 'context.sufficient':('Context sufficiency',r'\mathbf{1}[\text{reviewed context is sufficient}]','Independent question/context judgment for the exact saved stage. A plausible but stale answer can have sufficient context and still be wrong.'),
+ 'generation.answer_correctness':('Answer correctness',r'\mathbf{1}[\text{answer is correct}]','Independent explicit review of the answer. This is not computed from phrase overlap or context faithfulness.'),
+ 'generation.response_relevance':('Response relevance',r'\mathbf{1}[\text{answer addresses the question}]','Explicit reviewed relevance, not embedding similarity or keyword matching.'),
+ 'generation.required_fact_coverage':('Required-fact coverage',r'|A_{\mathrm{expressed\ correctly}}|/|A_{\mathrm{required}}|','Unique required atoms linked to correct reviewed answer claims. Omissions and unresolved claim truth remain visible.'),
+ 'generation.faithfulness':('Claim faithfulness',r'|C_{\mathrm{supported}}|/|C_{\mathrm{all}}|','Requires an exhaustive response claim audit and resolved context-support judgments. It says nothing by itself about source truth.'),
+ 'generation.unsupported_claim_fraction':('Unsupported-claim fraction',r'|C_{\mathrm{unsupported}}|/|C_{\mathrm{all}}|','All response claims must be audited. Unknown support is unavailable; it is not counted as a supported or unsupported claim.'),
+ 'generation.claim_correctness':('Claim correctness',r'|C_{\mathrm{correct}}|/|C_{\mathrm{all}}|','Claim truth reviewed independently of its support by retrieved material.'),
+ 'citation.reference_validity':('Citation ID validity',r'|\mathrm{valid\ references}|/|\mathrm{references}|','Checks known document identities. It does not establish citation entailment.'),
+ 'citation.necessity_precision':('Citation necessity precision',r'|\mathrm{supporting\ or\ necessary\ citations}|/|\mathrm{citations}|','ALCE-inspired: joint support must hold; a citation earns credit if it supports alone or its removal breaks joint support. All judgments are supplied labels.'),
+ 'citation.completeness':('Citation completeness',r'|C_{\mathrm{jointly\ cited\ and\ supported}}|/|C_{\mathrm{citation\ required}}|','An uncited citation-required claim scores zero. No citations makes precision undefined, not perfect. Not official ALCE segmentation/model parity.'),
+ 'outcome.abstention_correctness':('Abstention policy correctness',r'\mathbf{1}[\mathrm{actual}=\mathrm{expected}]','Expected abstention is an explicit independent label; source answerability is not substituted for it.'),
+ 'outcome.task_success':('Task success',r'\mathbf{1}[\text{correct relevant answer or expected abstention}]','Conjunctive endpoint gate. A runtime failure counts as failure; missing semantic truth is unavailable.'),
+ 'outcome.grounded_task_success':('Grounded task success',r'\mathbf{1}[\text{task success}\land\text{support}\land\text{required citations}]','Requires exhaustive faithfulness, required facts and citation requirements, or an explicitly appropriate abstention. No weighted overall RAG score.'),
+ 'operational.recorded_latency_ms':('Recorded latency',r'\mathrm{saved\ latency}_{\mathrm{ms}}','Saved synthetic/component telemetry, never the elapsed time used to replay grading. This local view reports case values and their macro mean; it does not benchmark fresh inference.'),
+ 'operational.recorded_cost_usd':('Recorded cost',r'\mathrm{saved\ USD}','Fixture/component-reported cost only. No pricing model or verified cost attestation is implemented.'),
+ 'operational.recorded_tokens':('Recorded tokens',r'\mathrm{saved\ token\ count}','Fixture/component-reported count, not tokenization performed by the prototype.'),
+ 'operational.failure':('Execution failure',r'\mathbf{1}[\mathrm{status}\ne\mathrm{ok}]','Explicit saved error/timeout status, including failed cases in the operational denominator.'),
+ 'operational.timeout':('Execution timeout',r'\mathbf{1}[\mathrm{status}=\mathrm{timeout}]','A timeout is an execution failure, not an empty successful response.'),
+}
+
+
+def describe(metric_id):
+    base=metric_id.split('@')[0]
+    if base.startswith('context.sufficient.'):
+        base='context.sufficient'
+    title,formula,meaning=INFO.get(base,(metric_id,'','See the recorded numerator, denominator and unavailable reason.'))
+    if '@' in metric_id:title+=' @'+metric_id.split('@')[1]
+    return {'title':title,'formula':formula,'meaning':meaning}

sphynx/_rag_diagnostics_common.py

@@ -0,0 +1,26 @@
+"""Shared wire records; no model/provider or Sphynx imports."""
+from __future__ import annotations
+
+import hashlib
+import json
+
+VERSION = 'sphynx.rag-diagnostics.metric.v1'
+STAGES = ('indexed', 'retrieved', 'reranked', 'selected', 'prompt')
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False).encode('utf-8')
+
+
+def digest(value):
+    return hashlib.sha256(canonical(value)).hexdigest()
+
+
+def metric(metric_id, case_id, stage, value=None, *, numerator=None,
+           denominator=None, status=None, reason=None, grader_type='deterministic',
+           evidence_refs=None, population='this case', direction='higher', unit='ratio'):
+    return {'metric_id': metric_id, 'metric_version': VERSION, 'case_id': case_id,
+            'stage': stage, 'value': value, 'numerator': numerator,
+            'denominator': denominator, 'status': status or ('available' if value is not None else 'unavailable_missing_input'),
+            'reason': reason, 'grader_type': grader_type, 'evidence_refs': evidence_refs or [],
+            'population': population, 'direction': direction, 'unit': unit}