sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/score_reliability.py

@@ -0,0 +1,393 @@
+"""Grade frozen score fields against separate labels; never fit or infer.
+
+Probability, rank utility and pool-relative diagnostics are distinct contracts.
+Local lineage is caller-declared and does not grant benchmark/reviewer authority.
+The mathematical helpers port the pinned nextgen/calibrator source functions.
+"""
+from __future__ import annotations
+from dataclasses import dataclass
+from collections.abc import Mapping
+from pathlib import Path
+import math
+from .contracts import ContractError,canonical_json
+from .integration_contracts import require_digest,value_digest
+from ._track_capture import strict_json,byte_digest
+from .rerank_family_contracts import SOURCE_PINS
+
+REFERENCE_SCHEMA="sphynx.score-reliability.references.v1"
+REPORT_SCHEMA="sphynx.score-reliability.result.v1"
+POLICIES=frozenset({"fully_labeled","known_judgments_only","positive_unlabeled_diagnostic"})
+TARGETS={"frozen_strong_relevance":("sable_0_to_3",2),"native_positive_qrel":("native_public_qrels",1)}
+
+
+def _require(condition,message):
+    if not condition:raise ContractError(message)
+
+
+def _number(value):
+    try:return type(value) in (int,float) and math.isfinite(value)
+    except OverflowError:return False
+
+
+def _plain(value):
+    if isinstance(value,Mapping):value={key:_plain(child) for key,child in value.items()}
+    elif isinstance(value,(tuple,list)):value=[_plain(child) for child in value]
+    return strict_json(canonical_json(value))
+
+
+@dataclass(frozen=True)
+class ReliabilityReference:
+    """Exact held label release, inputs/corpus/cohort and declared label policy."""
+    _value_bytes:bytes
+
+    def __post_init__(self):
+        raw=strict_json(self._value_bytes)
+        fields={"schema_version","input_digest","corpus_digest","label_release","label_revision","label_policy","grade_scale",
+            "probability_target","authority","lineage","cases"}
+        _require(type(raw) is dict and set(raw)==fields and raw["schema_version"]==REFERENCE_SCHEMA,"reliability reference fields differ")
+        for field in ("input_digest","corpus_digest"):require_digest(raw[field],field)
+        _require(type(raw["label_release"]) is str and raw["label_release"] and type(raw["label_revision"]) is str and raw["label_revision"],"label release/revision is required")
+        _require(type(raw["label_policy"]) is str and type(raw["probability_target"]) is str
+            and raw["label_policy"] in POLICIES and raw["probability_target"] in TARGETS
+            and raw["grade_scale"]==TARGETS[raw["probability_target"]][0],"label semantics and probability target differ")
+        _require(raw["authority"] in ("synthetic_fixture","caller_declared_unverified","native_public_qrels"),"local reference authority cannot claim verified review")
+        lineage=raw["lineage"]
+        _require(type(lineage) is dict and set(lineage)=={"parent_reference_digest","relationship","changed_case_count","evidence_digest"},"label lineage fields differ")
+        for field in ("parent_reference_digest","evidence_digest"):
+            if lineage[field] is not None:require_digest(lineage[field],field)
+        _require(lineage["relationship"] in ("root","label_revision","subset") and type(lineage["changed_case_count"]) is int
+            and lineage["changed_case_count"]>=0,"label lineage relationship/count differs")
+        _require((lineage["relationship"]=="root")== (lineage["parent_reference_digest"] is None)
+            and (lineage["changed_case_count"]==0 or lineage["evidence_digest"] is not None),"changed label lineage needs evidence")
+        _require(lineage["relationship"]!="root" or lineage["changed_case_count"]==0,"root label release cannot claim changed parent labels")
+        cases=raw["cases"];_require(type(cases) is list and 1<=len(cases)<=10000,"reliability reference cohort is empty or too large")
+        ids=set();total=0
+        for row in cases:
+            _require(type(row) is dict and set(row)=={"query_id","cluster_id","input_digest","candidate_ids","dense_ranks","grades","judgment_status"},"reference case fields differ")
+            _require(type(row["query_id"]) is str and row["query_id"] and row["query_id"] not in ids
+                and type(row["cluster_id"]) is str and row["cluster_id"],"query/cluster identity differs")
+            ids.add(row["query_id"]);require_digest(row["input_digest"],"case input_digest")
+            candidates=row["candidate_ids"]
+            _require(type(candidates) is list and 1<=len(candidates)<=100 and all(type(cid) is str and cid for cid in candidates)
+                and len(candidates)==len(set(candidates)),"candidate IDs differ")
+            n=len(candidates);total+=n
+            _require(type(row["dense_ranks"]) is list and all(type(rank) is int for rank in row["dense_ranks"])
+                and sorted(row["dense_ranks"])==list(range(1,n+1)),"dense rank identity differs")
+            _require(type(row["grades"]) is list and type(row["judgment_status"]) is list
+                and len(row["grades"])==n and len(row["judgment_status"])==n,"reference label alignment differs")
+            for grade,status in zip(row["grades"],row["judgment_status"]):
+                _require(status in ("judged","unknown") and ((status=="unknown" and grade is None)
+                    or (status=="judged" and type(grade) is int and 0<=grade<=3)),"unknown labels cannot imply negatives")
+        _require(total<=1000000 and lineage["changed_case_count"]<=len(cases),"reference cohort/count exceeds bounds")
+        object.__setattr__(self,"_value_bytes",canonical_json(raw).encode())
+
+    @property
+    def digest(self):return byte_digest(self._value_bytes)
+
+    def to_dict(self):return strict_json(self._value_bytes)
+
+
+def validate_reliability_reference(value):return ReliabilityReference(canonical_json(_plain(value)).encode())
+
+
+def load_reliability_reference(path,*,expected_sha256):
+    raw=Path(path).read_bytes();_require(byte_digest(raw)==expected_sha256,"reference artifact bytes differ")
+    return ReliabilityReference(raw)
+
+
+def _mean(values):return sum(values)/len(values) if values else 0.0
+
+
+def _percentile(values,fraction):
+    if not values:return None
+    ordered=sorted(values);position=fraction*(len(ordered)-1);lower=math.floor(position);upper=math.ceil(position)
+    return ordered[lower]+(ordered[upper]-ordered[lower])*(position-lower)
+
+
+def _bins(probabilities,labels,bin_count=10,grades=None):
+    bins=[]
+    for index in range(bin_count):
+        lower=index/bin_count;upper=(index+1)/bin_count
+        selected=[i for i,p in enumerate(probabilities) if p>=lower and (p<=upper if index==bin_count-1 else p<upper)]
+        row={"lower":lower,"upper":upper,"count":len(selected),
+            "mean_probability":_mean([probabilities[i] for i in selected]) if selected else None,
+            "positive_rate":_mean([labels[i] for i in selected]) if selected else None}
+        if grades is not None:row["relevance_grade_distribution"]={str(g):sum(grades[i]==g for i in selected) for g in sorted({grades[i] for i in selected if grades[i] is not None})}
+        bins.append(row)
+    return bins
+
+
+def _log_loss(probability,label):
+    p=min(1-1e-12,max(1e-12,probability))
+    return -(label*math.log(p)+(1-label)*math.log(1-p))
+
+
+def _probability_metrics(queries,interpretation):
+    """Source probabilityMetrics math over grader-created labeled rows only."""
+    all_rows=[row for query in queries for row in query["rows"]]
+    top=[row["probability"] for query in queries for row in sorted(query["rows"],key=lambda row:(-row["score"],row["dense_rank"],row["candidate_id"]))[:5]]
+    p=[row["probability"] for row in all_rows];y=[row["label"] for row in all_rows];widths=[row["width"] for row in all_rows if row["width"]>0]
+    saturated=lambda value:value<=.05 or value>=.95
+    return {"interpretation":interpretation,"candidate_count":len(p),"positive_count":sum(y),
+        "brier_score":_mean([(v-label)**2 for v,label in zip(p,y)]) if p else None,
+        "log_loss":_mean([_log_loss(v,label) for v,label in zip(p,y)]) if p else None,
+        "saturation_fraction":sum(saturated(v) for v in p)/len(p) if p else 0,
+        "top_five_saturation_fraction":sum(saturated(v) for v in top)/len(top) if top else 0,
+        "probability_quantiles":{"p05":_percentile(p,.05),"p50":_percentile(p,.5),"p95":_percentile(p,.95)},
+        "mean_venn_abers_interval_width":_mean(widths) if widths else None,"p95_venn_abers_interval_width":_percentile(widths,.95),
+        "reliability":_bins(p,y)}
+
+
+def _score_only_diagnostics(queries):
+    """Score distributions need no labels and retain the original ranked pool."""
+    rows=[row for query in queries for row in query["rows"]]
+    probabilities=[row["probability"] for row in rows]
+    top=[row["probability"] for query in queries for row in sorted(query["rows"],
+        key=lambda row:(-row["score"],row["dense_rank"],row["candidate_id"]))[:5]]
+    widths=[row["width"] for row in rows if row["width"]>0]
+    saturated=lambda value:value<=.05 or value>=.95
+    return {"population":"all_captured_candidates_original_top_five","candidate_count":len(rows),
+        "top_five_candidate_count":len(top),
+        "saturation_fraction":sum(saturated(p) for p in probabilities)/len(probabilities) if probabilities else 0,
+        "top_five_saturation_fraction":sum(saturated(p) for p in top)/len(top) if top else 0,
+        "probability_quantiles":{"p05":_percentile(probabilities,.05),"p50":_percentile(probabilities,.5),"p95":_percentile(probabilities,.95)},
+        "mean_venn_abers_interval_width":_mean(widths) if widths else None,"p95_venn_abers_interval_width":_percentile(widths,.95)}
+
+
+def _calibration_summary(probabilities,labels):
+    if not probabilities:return {"brier_score":None,"log_loss":None,"expected_calibration_error":None,"max_reliability_bin_error":None,"reliability":_bins([],[])}
+    bins=_bins(probabilities,labels)
+    errors=[(row["count"]/len(labels),abs(row["mean_probability"]-row["positive_rate"])) for row in bins if row["count"]]
+    return {"brier_score":_mean([(p-y)**2 for p,y in zip(probabilities,labels)]),
+        "log_loss":_mean([_log_loss(p,y) for p,y in zip(probabilities,labels)]),
+        "expected_calibration_error":sum(weight*error for weight,error in errors),
+        "max_reliability_bin_error":max(error for _,error in errors),"reliability":bins}
+
+
+def grade_score_reliability(score_queries,references,*,field="evidence_probability",bootstrap_samples=1000,seed=0):
+    """Grade one typed score field; held references never enter the scorer.
+
+    score_queries contain query_id/cluster_id/input_digest/scores (the family
+    score envelope). Any probability target mismatch fails closed. Diagnostic
+    fields can be analyzed but cannot yield an admitted probability result.
+    """
+    _require(isinstance(references,ReliabilityReference),"validated held references are required")
+    _require(type(field) is str and field,"a named typed score field is required")
+    _bootstrap_parameters(bootstrap_samples,seed)
+    ref=references.to_dict();queries=_plain(score_queries)
+    _require(type(queries) is list and len(queries)==len(ref["cases"]),"score/reference query cohort differs")
+    refs={row["query_id"]:row for row in ref["cases"]};seen=set();graded=[];full_pool=[];all_unknown=0;kind=None;method=None
+    for query in queries:
+        _require(type(query) is dict and set(query)=={"query_id","cluster_id","input_digest","scores"},"score query fields differ")
+        _require(type(query["query_id"]) is str,"score query ID must be text")
+        case=refs.get(query["query_id"])
+        _require(case is not None and query["query_id"] not in seen and query["cluster_id"]==case["cluster_id"]
+            and query["input_digest"]==case["input_digest"],"score query/cluster/input identity differs")
+        seen.add(query["query_id"]);scores=query["scores"]
+        required={"schema_version","method","method_version","candidate_ids","fields","diagnostics","reference_sources","inference_observed"}
+        _require(type(scores) is dict and set(scores)==required and scores["schema_version"]=="sphynx.rerank-family.scores.v1"
+            and scores["candidate_ids"]==case["candidate_ids"] and type(scores["fields"]) is dict
+            and type(scores["method"]) is str and scores["method"] and type(scores["method_version"]) is str
+            and type(scores["inference_observed"]) is bool and type(scores["diagnostics"]) is dict
+            and type(scores["reference_sources"]) is dict,"typed score envelope differs")
+        for source,digest in scores["reference_sources"].items():
+            _require(type(source) is str and source,"score source identities differ");require_digest(digest,"score source digest")
+        current=(scores["method"],scores["method_version"])
+        _require(method is None or current==method,"methods differ inside one score cohort");method=current
+        utility=scores["fields"].get("ranking_utility");selected=scores["fields"].get(field)
+        _require(type(utility) is dict and set(utility)=={"kind","values"} and utility["kind"]=="utility"
+            and type(selected) is dict and set(selected) in ({"kind","values"},{"kind","values","probability_target"}),"selected score fields differ")
+        _require(type(selected["kind"]) is str and selected["kind"] in ("probability","diagnostic","pool_relative","utility")
+            and (kind is None or kind==selected["kind"]),"score field kinds differ");kind=selected["kind"]
+        n=len(case["candidate_ids"])
+        for values in (utility["values"],selected["values"]):
+            _require(type(values) is list and len(values)==n and all(_number(v) for v in values),"score field alignment/numbers differ")
+        _require(all(0<=v<=1 for v in selected["values"]),"reliability diagnostics require a bounded 0..1 field, not arbitrary rank utility")
+        target=selected.get("probability_target")
+        _require(kind!="probability" or target==ref["probability_target"],"probability score target differs from the held label target")
+        _require(kind=="probability" or target is None,"diagnostic score field cannot claim a probability target")
+        width=scores["fields"].get("venn_abers_interval_width",{"values":[0.0]*n,"kind":"diagnostic"})
+        _require(type(width) is dict and set(width)=={"kind","values"} and width["kind"]=="diagnostic"
+            and type(width["values"]) is list and len(width["values"])==n and all(_number(v) and 0<=v<=1 for v in width["values"]),"interval width diagnostics differ")
+        full_pool.append({"rows":[{"candidate_id":case["candidate_ids"][i],"dense_rank":case["dense_ranks"][i],
+            "score":utility["values"][i],"probability":selected["values"][i],"width":width["values"][i]} for i in range(n)]})
+        rows=[];unknown=0
+        for index,grade in enumerate(case["grades"]):
+            absent=grade is None;unknown+=absent
+            if absent and ref["label_policy"]!="positive_unlabeled_diagnostic":continue
+            label=0 if absent else int(grade>=TARGETS[ref["probability_target"]][1])
+            rows.append({"candidate_id":case["candidate_ids"][index],"dense_rank":case["dense_ranks"][index],
+                "score":utility["values"][index],"probability":selected["values"][index],"width":width["values"][index],"label":label})
+        all_unknown+=unknown;graded.append({"query_id":case["query_id"],"cluster_id":case["cluster_id"],"rows":rows,"unknown_count":unknown})
+    policy=ref["label_policy"]
+    interpretation="diagnostic_unjudged_as_negative" if policy=="positive_unlabeled_diagnostic" else "formal_fully_labeled" if not all_unknown else "conditional_known_judgments"
+    source=_probability_metrics(graded,interpretation)
+    p=[row["probability"] for query in graded for row in query["rows"]];labels=[row["label"] for query in graded for row in query["rows"]]
+    micro=_calibration_summary(p,labels)
+    per_query=[{"query_id":query["query_id"],"candidate_count":len(query["rows"]),"unknown_count":query["unknown_count"],
+        **_calibration_summary([row["probability"] for row in query["rows"]],[row["label"] for row in query["rows"]])} for query in graded]
+    macro={key:_mean([row[key] for row in per_query if row[key] is not None]) if any(row[key] is not None for row in per_query) else None
+        for key in ("brier_score","log_loss","expected_calibration_error","max_reliability_bin_error")}
+    eligible=kind=="probability" and policy!="positive_unlabeled_diagnostic" and (policy!="fully_labeled" or not all_unknown) and bool(p)
+    blockers=[]
+    if kind!="probability":blockers.append("field_is_not_an_admitted_probability")
+    if policy=="positive_unlabeled_diagnostic":blockers.append("positive_unlabeled_metrics_are_diagnostic")
+    if policy=="fully_labeled" and all_unknown:blockers.append("required_judgments_missing")
+    if not p:blockers.append("no_judged_candidates")
+    report={"schema_version":REPORT_SCHEMA,"method":method[0],"method_version":method[1],"field":field,"field_kind":kind,
+        "probability_target":ref["probability_target"],"label_policy":policy,"reference_digest":references.digest,
+        "score_capture_digest":value_digest(queries),"input_digest":ref["input_digest"],"corpus_digest":ref["corpus_digest"],
+        "label_release":ref["label_release"],"label_revision":ref["label_revision"],"lineage":ref["lineage"],
+        "authority":ref["authority"],"semantic_authority_verified":False,"inference_executed":False,
+        "query_count":len(graded),"candidate_count":sum(len(row["candidate_ids"]) for row in ref["cases"]),
+        "judged_count":sum(len(row["candidate_ids"]) for row in ref["cases"])-all_unknown,"unknown_count":all_unknown,
+        "formal_probability_eligible":eligible,"status":"eligible_conditional" if eligible and all_unknown else "eligible" if eligible else "diagnostic_or_blocked",
+        "blockers":blockers,"formal_passage_micro":micro if eligible else None,"formal_query_macro":macro if eligible else None,
+        "diagnostics":{"source_probability_metrics":source,
+            "source_probability_metrics_population":"all_candidates_explicit_unjudged_as_negative" if policy=="positive_unlabeled_diagnostic"
+                else "judged_subset_reranked_top_five" if all_unknown else "all_fully_judged_candidates_original_top_five",
+            "full_pool_score_only":_score_only_diagnostics(full_pool),
+            "passage_micro":micro,"query_macro":macro,"per_query":per_query},
+        "bootstrap":None,"notice":"Frozen score analysis only. Smaller intervals or saturation alone do not establish calibration. Known-only scores apply to the judged subset.",
+        "source_pins":{key:SOURCE_PINS[key] for key in ("nextgen-evaluation.ts","calibrator-fit.ts","metrics.ts")}}
+    report["scoring_protocol"]={"version":"source-score-reliability.v1","reliability_bin_count":10,
+        "log_loss_epsilon":1e-12,"saturation_low":.05,"saturation_high":.95,"saturation_top_k":5,
+        "interval_width_population":"strictly_positive_widths","label_threshold":TARGETS[ref["probability_target"]][1]}
+    report["declared_scorer_sources"]=[query["scores"]["reference_sources"] for query in queries]
+    if eligible:
+        clusters={row["query_id"]:row["cluster_id"] for row in graded}
+        values=[{"cluster":clusters[row["query_id"]],"value":row["brier_score"]} for row in per_query if row["brier_score"] is not None]
+        report["bootstrap"]=_cluster_bootstrap(values,bootstrap_samples,seed)
+        report["bootstrap"].update(estimate="query_macro_brier",reference_digest=references.digest,
+            score_capture_digest=report["score_capture_digest"],conditional_on_judged_subset=bool(all_unknown))
+    return report
+
+
+def _bootstrap_parameters(samples,seed):
+    _require(type(samples) is int and 1<=samples<=10000 and type(seed) is int and 0<=seed<=0xffffffff,
+        "bootstrap configuration must use bounded repetitions and a UInt32 seed")
+
+
+def _lcg32(seed):
+    state=seed
+    while True:
+        state=(1664525*state+1013904223)&0xffffffff
+        yield state/4294967296
+
+
+def _cluster_bootstrap(rows,samples,seed):
+    _bootstrap_parameters(samples,seed)
+    clusters=sorted({row["cluster"] for row in rows})
+    _require(len(rows)*samples<=10000000,"cluster bootstrap exceeds bounded work")
+    if not rows:return {"sampling_unit":"cluster_id","rng":"nextgen-evaluation.lcg32.v1","seed":seed,
+        "repetitions":samples,"independent_cluster_count":0,"observed_query_count":0,"interval_95":None,"standard_deviation":None}
+    grouped={cluster:[row["value"] for row in rows if row["cluster"]==cluster] for cluster in clusters}
+    random=_lcg32(seed);means=[]
+    for _ in range(samples):
+        drawn=[clusters[int(next(random)*len(clusters))] for _ in clusters]
+        means.append(_mean([value for cluster in drawn for value in grouped[cluster]]))
+    center=_mean(means)
+    return {"sampling_unit":"cluster_id","rng":"nextgen-evaluation.lcg32.v1","seed":seed,"repetitions":samples,
+        "independent_cluster_count":len(clusters),"observed_query_count":len(rows),
+        "interval_95":{"lower":_percentile(means,.025),"upper":_percentile(means,.975)},
+        "standard_deviation":math.sqrt(_mean([(value-center)**2 for value in means]))}
+
+
+def compare_score_reliability(baseline_queries,candidate_queries,references,*,baseline_field="evidence_probability",
+        candidate_field="evidence_probability",bootstrap_samples=1000,seed=0):
+    """Paired query Brier improvement with matched labels/inputs and clusters."""
+    left=grade_score_reliability(baseline_queries,references,field=baseline_field,bootstrap_samples=bootstrap_samples,seed=seed)
+    right=grade_score_reliability(candidate_queries,references,field=candidate_field,bootstrap_samples=bootstrap_samples,seed=seed)
+    _require(left["formal_probability_eligible"] and right["formal_probability_eligible"],"diagnostic/missing probability scores cannot enter a formal comparison")
+    a={row["query_id"]:row for row in left["diagnostics"]["per_query"]};b={row["query_id"]:row for row in right["diagnostics"]["per_query"]}
+    clusters={row["query_id"]:row["cluster_id"] for row in references.to_dict()["cases"]}
+    deltas=[{"cluster":clusters[qid],"value":a[qid]["brier_score"]-b[qid]["brier_score"]} for qid in a
+        if a[qid]["brier_score"] is not None and b[qid]["brier_score"] is not None]
+    return {"schema_version":"sphynx.score-reliability.comparison.v1","reference_digest":references.digest,
+        "baseline_score_capture_digest":left["score_capture_digest"],"candidate_score_capture_digest":right["score_capture_digest"],
+        "label_policy":left["label_policy"],"aggregation":"query_macro","metric":"brier_score",
+        "direction":"baseline_minus_candidate_positive_means_lower_candidate_error",
+        "mean_improvement":_mean([row["value"] for row in deltas]) if deltas else None,
+        "bootstrap":_cluster_bootstrap(deltas,bootstrap_samples,seed),"inference_executed":False,
+        "conditional_on_judged_subset":bool(left["unknown_count"])}
+
+
+def _mulberry32(seed):
+    state=seed&0xffffffff
+    while True:
+        state=(state+0x6d2b79f5)&0xffffffff
+        value=((state^(state>>15))*(state|1))&0xffffffff
+        value^=(value+((value^(value>>7))*(value|61)))&0xffffffff
+        yield ((value^(value>>14))&0xffffffff)/4294967296
+
+
+def _sigmoid(value):
+    return 1/(1+math.exp(-value)) if value>=0 else math.exp(value)/(1+math.exp(value))
+
+
+def _apply_calibrator(calibrator,value):
+    method=calibrator["method"]
+    if method=="temperature":return _sigmoid(value/calibrator["temperature"])
+    if method=="platt":return _sigmoid(calibrator["a"]*value+calibrator["b"])
+    if method=="beta":
+        p=min(1-1e-12,max(1e-12,_sigmoid(value)))
+        return _sigmoid(calibrator["a"]*math.log(p)-calibrator["b"]*math.log(1-p)+calibrator["c"])
+    index=next((i for i,upper in enumerate(calibrator["upper_bounds"]) if value<=upper),len(calibrator["probabilities"])-1)
+    return calibrator["probabilities"][index]
+
+
+def evaluate_fitted_calibrator(observations,calibrator,*,bin_count=10,bootstrap_samples=1000,seed=0):
+    """Exact source evaluator math for a fixed calibrator. No fitting/selection.
+
+    Observations are grader-only held raw-logit/label records. This math helper
+    cannot assert admission, cohort independence, or fitted-artifact provenance.
+    """
+    observations=_plain(observations);calibrator=_plain(calibrator)
+    _require(type(observations) is list and 1<=len(observations)<=10000,"calibrator observations differ")
+    _require(type(bin_count) is int and 1<=bin_count<=100 and type(bootstrap_samples) is int
+        and 1<=bootstrap_samples<=10000 and len(observations)*bootstrap_samples<=10000000
+        and type(seed) is int and 0<=seed<=0xffffffff,"bootstrap/bin configuration exceeds bounds")
+    fields={"temperature":{"method","temperature"},"platt":{"method","a","b"},"beta":{"method","a","b","c"},
+        "isotonic":{"method","upper_bounds","probabilities"}}
+    _require(type(calibrator) is dict and type(calibrator.get("method")) is str
+        and calibrator["method"] in fields and set(calibrator)==fields[calibrator["method"]],"fixed calibrator fields differ")
+    method=calibrator["method"]
+    if method=="isotonic":
+        bounds=calibrator["upper_bounds"];values=calibrator["probabilities"]
+        _require(type(bounds) is list and type(values) is list and bounds and len(bounds)==len(values)
+            and all(_number(v) for v in bounds) and all(a<b for a,b in zip(bounds,bounds[1:]))
+            and all(_number(v) and 0<=v<=1 for v in values) and values==sorted(values),"fixed isotonic knots differ")
+    else:_require(all(_number(v) for key,v in calibrator.items() if key!="method")
+        and (method!="temperature" or calibrator["temperature"]>0),"fixed calibrator coefficients differ")
+    probabilities=[];labels=[];grades=[]
+    for row in observations:
+        _require(type(row) is dict and {"query_id","category","raw_logit","label"}<=set(row)
+            and set(row)<={"query_id","category","raw_logit","label","relevance_grade","uncertainty"}
+            and type(row["query_id"]) is str and row["query_id"] and type(row["category"]) is str and row["category"]
+            and _number(row["raw_logit"]) and type(row["label"]) is int and row["label"] in (0,1),"calibrator held observation differs")
+        grade=row.get("relevance_grade");_require(grade is None or type(grade) is int and 0<=grade<=3,"relevance grade differs")
+        probability=_apply_calibrator(calibrator,row["raw_logit"])
+        _require(_number(probability) and 0<=probability<=1,"fixed calibrator produced an invalid probability")
+        probabilities.append(probability);labels.append(row["label"]);grades.append(grade)
+    bins=_bins(probabilities,labels,bin_count,grades)
+    errors=[abs(row["mean_probability"]-row["positive_rate"]) for row in bins if row["count"]]
+    random=_mulberry32(seed);bootstrap=[]
+    for _ in range(bootstrap_samples):
+        bootstrap.append(_mean([(probabilities[i]-labels[i])**2 for i in (int(next(random)*len(labels)) for _ in labels)]))
+    bootstrap_mean=_mean(bootstrap)
+    return {"method":method,"brier_score":_mean([(p-y)**2 for p,y in zip(probabilities,labels)]),
+        "log_loss":_mean([_log_loss(p,y) for p,y in zip(probabilities,labels)]),
+        "expected_calibration_error":sum(row["count"]/len(labels)*abs(row["mean_probability"]-row["positive_rate"]) for row in bins if row["count"]),
+        "max_reliability_bin_error":max([0,*errors]),"bootstrap_brier_standard_deviation":math.sqrt(_mean([(value-bootstrap_mean)**2 for value in bootstrap])),
+        "reliability":[{key if key!="positive_rate" else "empirical_relevance":value for key,value in row.items()} for row in bins]}
+
+
+def report_fitted_calibrator(observations,calibrator,*,bin_count=10,bootstrap_samples=1000,seed=0):
+    """Add reproducibility identity to the exact source evaluator result."""
+    result=evaluate_fitted_calibrator(observations,calibrator,bin_count=bin_count,bootstrap_samples=bootstrap_samples,seed=seed)
+    return {"schema_version":"sphynx.fixed-calibrator-reliability.v1","result":result,
+        "observations_digest":value_digest(observations),"calibrator_digest":value_digest(calibrator),
+        "bin_count":bin_count,"bootstrap":{"sampling_unit":"passage_observation","rng":"calibrator-fit.mulberry32.v1",
+            "seed":seed,"repetitions":bootstrap_samples},"source_digest":SOURCE_PINS["calibrator-fit.ts"],
+        "fitting_executed":False,"model_inference_executed":False,"admission_verified":False,
+        "notice":"Grader-only source calculation on a fixed calibrator; no fitting, probability admission or cohort independence is established."}

sphynx/speech_benchmarks.py

@@ -0,0 +1,204 @@
+"""Unreleased local Kido benchmark extensions; not part of the frozen 0.5.0 wheel.
+
+Use python -m sphynx.speech_benchmarks. No provider, imported code or model runs.
+"""
+from __future__ import annotations
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+import uuid
+
+from ._speech_kido import SpeechEvidenceError,digest,encoded,json_value,require
+from . import _speech_benchmark_sources as sources
+from ._speech_benchmark_scoring import (grade_records,evaluate_pt_br_listening,evaluate_pronunciation,
+    evaluate_language_switching,evaluate_stt_turn_lifecycle,evaluate_voice_continuity,
+    evaluate_code_switching,evaluate_playout_interruptions)
+from .speech_evidence import _new_directory,_read,_write_files,_sha
+
+SUITE_ID='speech.kido-benchmarks.v1'
+BUNDLE_SCHEMA='sphynx.speech.kido-benchmarks.bundle.v1'
+REPORT_SCHEMA='sphynx.speech.kido-benchmarks.report.v1'
+INVOCATION_SCHEMA='sphynx.speech.kido-benchmarks.invocation.v1'
+RECEIPT_SCHEMA='sphynx.speech.kido-benchmarks.receipt.v1'
+GRADER='speech.kido-benchmarks.grader.v1'
+MODULES=('speech_benchmarks.py','_speech_benchmark_sources.py','_speech_benchmark_scoring.py','_speech_kido.py','_speech_scoring.py','speech_evidence.py','speech_contracts.py')
+LIMITS=dict(historical_only=True,population_inference=False,new_waveform_quality=False,perceived_latency=False,controlled_barge_in=False)
+
+
+def list_speech_benchmarks():
+    return dict(suite_id=SUITE_ID,invocation_schema=INVOCATION_SCHEMA,grader_version=GRADER,
+        protocols=['pt_br_listening','pronunciation','language_switching','stt_turn_lifecycle','voice_continuity','code_switching','playout_interruptions'],
+        source_commit=sources.SOURCE_COMMIT,execution='offline retained-evidence grading',
+        hosted_admitted=False,provider_calls=False,fresh_quality_ratings=False,
+        implementation_sha256={name:digest(Path(__file__).with_name(name).read_bytes()) for name in
+            MODULES})
+
+
+def export_kido_benchmarks_source(repository,output_dir):
+    """Export committed benchmark bytes without running Kido's scripts.
+
+    Git checkout line endings are never treated as canonical source. The
+    complete selected export must match the independently reviewed SHA256.
+    """
+    import hashlib,subprocess
+    root=Path(repository).absolute();inventory,_=sources.source_inventory(root)
+    paths=list(inventory['files']);batch=''.join(sources.SOURCE_COMMIT+':'+p+'\n' for p in paths).encode()
+    try:result=subprocess.run(['git','cat-file','--batch'],input=batch,cwd=root,capture_output=True,check=True,timeout=60)
+    except (subprocess.SubprocessError,OSError):raise SpeechEvidenceError('Unable to read the pinned Git benchmark source') from None
+    raw=result.stdout;cursor=0;payload={}
+    for path in paths:
+        end=raw.find(b'\n',cursor);require(end>=cursor,'Incomplete Git object header')
+        header=raw[cursor:end].split();require(len(header)==3 and header[1]==b'blob','Missing committed benchmark blob')
+        size=int(header[2]);require(0<=size<=16*1024*1024,'Git benchmark blob exceeds limit');cursor=end+1
+        data=raw[cursor:cursor+size];cursor+=size
+        require(len(data)==size and raw[cursor:cursor+1]==b'\n','Incomplete committed benchmark blob');cursor+=1
+        require(hashlib.sha1(b'blob '+str(size).encode()+b'\0'+data).hexdigest().encode()==header[0],'Git benchmark object identity mismatch')
+        payload[path]=data
+    require(cursor==len(raw) and sum(map(len,payload.values()))<=64*1024*1024,'Git export census or byte limit changed')
+    inventory['files']={p:dict(sha256=digest(b),size_bytes=len(b)) for p,b in payload.items()}
+    require(digest(encoded(inventory))==sources.SOURCE_LOCK_SHA256,'Git export is not the reviewed benchmark source')
+    dest=_new_directory(output_dir);_write_files(dest,payload)
+    return dict(path=str(dest),source_commit=sources.SOURCE_COMMIT,source_lock_sha256=sources.SOURCE_LOCK_SHA256,files=len(payload),source_executed=False)
+
+
+def import_kido_benchmarks(source,output_dir):
+    inventory,raw=sources.source_bytes(source);evidence=sources.normalize_sources(raw)
+    dest=_new_directory(output_dir)
+    files=_write_files(dest,{'source/'+p:b for p,b in raw.items()}|{'evidence.json':encoded(evidence)})
+    manifest=dict(schema_version=BUNDLE_SCHEMA,suite_id=SUITE_ID,source_commit=sources.SOURCE_COMMIT,
+        source_lock_sha256=digest(encoded(inventory)),files=files,observations=len(evidence['observations']),
+        candidate_executed=False,provider_called=False,hosted_admitted=False)
+    data=encoded(manifest);(dest/'manifest.json').write_bytes(data)
+    return dict(path=str(dest),suite_id=SUITE_ID,manifest_sha256=digest(data),observations=manifest['observations'])
+
+
+@dataclass(frozen=True)
+class SpeechBenchmarks:
+    path:Path
+    manifest_sha256:str
+
+
+def _verified(bundle):
+    raw=_read(bundle.path,'manifest.json');require(digest(raw)==bundle.manifest_sha256,'Benchmark manifest changed')
+    manifest=json_value(raw)
+    require(manifest.get('schema_version')==BUNDLE_SCHEMA and manifest.get('suite_id')==SUITE_ID and
+        manifest.get('source_commit')==sources.SOURCE_COMMIT and manifest.get('source_lock_sha256')==sources.SOURCE_LOCK_SHA256,
+        'Not the pinned Kido benchmark suite')
+    inventory,source=sources.source_bytes(bundle.path/'source')
+    expected={'source/'+p for p in source}|{'evidence.json'}
+    require(type(manifest.get('files')) is dict and set(manifest['files'])==expected,'Benchmark file census changed')
+    for p,receipt in manifest['files'].items():
+        data=_read(bundle.path,p);require(receipt==dict(sha256=digest(data),size_bytes=len(data)),'Benchmark artifact changed')
+    evidence=sources.normalize_sources(source)
+    require(_read(bundle.path,'evidence.json')==encoded(evidence) and manifest.get('observations')==len(evidence['observations']),
+        'Benchmark normalized evidence differs from source')
+    require(all(manifest.get(k) is False for k in ('candidate_executed','provider_called','hosted_admitted')),'Benchmark execution boundary changed')
+    return manifest,evidence
+
+
+def load_speech_benchmarks(path,*,expected_manifest_sha256):
+    result=SpeechBenchmarks(Path(path).absolute(),_sha(expected_manifest_sha256));_verified(result);return result
+
+
+def tts_benchmark_cases(bundle,*,benchmark,case_ids=None,language_hint=None):
+    """Return TTS inputs only, never provider mappings, ratings, gold or STT data.
+
+    Mixed-language source requests had no fixed language hint. A caller must
+    choose an explicit two-letter hint to adapt them to the current TTS runner.
+    """
+    import re
+    require(benchmark in ('pt-br-listening','pronunciation','code-switching'),'Unknown TTS benchmark')
+    _,evidence=_verified(bundle)
+    cases=[dict(r) for r in evidence['cases'] if r['id'].startswith(benchmark+'/')]
+    from .speech_contracts import identifier
+    identity_map={}
+    for row in cases:
+        source_id=row['id'];row['id']=identifier(source_id.replace('/',':',1))
+        require(row['id'] not in identity_map,'Projected TTS case identity collision')
+        identity_map[row['id']]=source_id
+    if case_ids is not None:
+        require(type(case_ids) is list and len(set(case_ids))==len(case_ids),'Invalid selected case IDs')
+        require(set(case_ids)<={r['id'] for r in cases},'Selected case absent from benchmark')
+        cases=[r for r in cases if r['id'] in case_ids]
+    require(cases,'Empty benchmark selection')
+    if benchmark=='code-switching':
+        require(isinstance(language_hint,str) and re.fullmatch(r'[a-z]{2}',language_hint),'Choose a language hint explicitly for a mixed-language TTS plan')
+        for row in cases:row['language']=language_hint
+    else:require(language_hint is None,'Do not override the native benchmark language')
+    return dict(suite_id=SUITE_ID,benchmark=benchmark,cases=cases,
+        case_identity_map={r['id']:identity_map[r['id']] for r in cases},
+        cohort_identity=dict(kind=identifier(SUITE_ID+':'+benchmark),sha256=digest(encoded(dict(bundle=bundle.manifest_sha256,cases=cases)))),
+        source_bundle_sha256=bundle.manifest_sha256,new_waveform_ratings_available=False)
+
+
+def grade_speech_benchmarks(bundle,output_dir):
+    _,evidence=_verified(bundle);definition=list_speech_benchmarks()
+    invocation=dict(schema_version=INVOCATION_SCHEMA,id='speech-benchmark-invocation-'+uuid.uuid4().hex,
+        suite_id=SUITE_ID,bundle_manifest_sha256=bundle.manifest_sha256,source_commit=sources.SOURCE_COMMIT,
+        source_lock_sha256=sources.SOURCE_LOCK_SHA256,grader_version=GRADER,definition=definition,
+        candidate_executed=False,provider_called=False,hosted_admitted=False)
+    report=dict(schema_version=REPORT_SCHEMA,suite_id=SUITE_ID,invocation_id=invocation['id'],
+        invocation_sha256=digest(encoded(invocation)),results=grade_records(evidence),
+        source_audit=evidence['audit'],limits=LIMITS)
+    dest=_new_directory(output_dir);files=_write_files(dest,{'invocation.json':encoded(invocation),
+        'report.json':encoded(report),'evidence.json':encoded(evidence)})
+    receipt=dict(schema_version=RECEIPT_SCHEMA,suite_id=SUITE_ID,invocation_id=invocation['id'],files=files)
+    data=encoded(receipt);(dest/'receipt.json').write_bytes(data)
+    return dict(path=str(dest),suite_id=SUITE_ID,invocation_id=invocation['id'],receipt_sha256=digest(data))
+
+
+def load_speech_benchmark_report(path,*,expected_receipt_sha256):
+    root=Path(path).absolute();data=_read(root,'receipt.json')
+    require(digest(data)==_sha(expected_receipt_sha256),'Benchmark receipt changed');receipt=json_value(data)
+    require(receipt.get('schema_version')==RECEIPT_SCHEMA and receipt.get('suite_id')==SUITE_ID and
+        set(receipt.get('files',{}))=={'report.json','invocation.json','evidence.json'},'Not a speech benchmark evaluation')
+    payload={}
+    for p,h in receipt['files'].items():
+        raw=_read(root,p);require(h==dict(sha256=digest(raw),size_bytes=len(raw)),'Benchmark evaluation artifact changed');payload[p]=json_value(raw)
+    report,invocation,evidence=(payload[p] for p in ('report.json','invocation.json','evidence.json'))
+    require(invocation.get('schema_version')==INVOCATION_SCHEMA and report.get('schema_version')==REPORT_SCHEMA and
+        invocation.get('suite_id')==report.get('suite_id')==SUITE_ID and
+        invocation.get('id')==report.get('invocation_id')==receipt.get('invocation_id') and
+        report.get('invocation_sha256')==digest(encoded(invocation)) and
+        invocation.get('source_commit')==sources.SOURCE_COMMIT and invocation.get('source_lock_sha256')==sources.SOURCE_LOCK_SHA256 and
+        invocation.get('grader_version')==GRADER and all(invocation.get(k) is False for k in ('candidate_executed','provider_called','hosted_admitted')),
+        'Benchmark report identities or execution boundary changed')
+    definition=invocation.get('definition',{})
+    require(definition.get('suite_id')==SUITE_ID and definition.get('invocation_schema')==INVOCATION_SCHEMA and
+        definition.get('grader_version')==GRADER and definition.get('source_commit')==sources.SOURCE_COMMIT and
+        all(definition.get(k) is False for k in ('provider_calls','fresh_quality_ratings','hosted_admitted')),
+        'Benchmark definition boundary changed')
+    require(type(definition.get('implementation_sha256')) is dict and set(definition['implementation_sha256'])==set(MODULES),
+        'Benchmark grader source identities missing')
+    for value in definition['implementation_sha256'].values():_sha(value)
+    require(report.get('limits')==LIMITS and report.get('results')==grade_records(evidence) and report.get('source_audit')==evidence['audit'],'Benchmark calculations or scope disagree with retained evidence')
+    return report
+
+
+def main(argv=None):
+    import json
+    p=argparse.ArgumentParser(description='Separate offline speech.kido-benchmarks.v1 invocation')
+    commands=p.add_subparsers(dest='command',required=True);commands.add_parser('describe')
+    export=commands.add_parser('export-source');export.add_argument('--repository',required=True);export.add_argument('--output',required=True)
+    i=commands.add_parser('import-kido');i.add_argument('--source',required=True);i.add_argument('--output',required=True)
+    for name in ('grade','tts-cases'):
+        c=commands.add_parser(name);c.add_argument('--bundle',required=True);c.add_argument('--expected-manifest-sha256',required=True)
+        if name=='grade':c.add_argument('--output',required=True)
+        else:
+            c.add_argument('--benchmark',choices=('pt-br-listening','pronunciation','code-switching'),required=True)
+            c.add_argument('--case-id',action='append');c.add_argument('--language-hint')
+    args=p.parse_args(argv)
+    try:
+        if args.command=='describe':result=list_speech_benchmarks()
+        elif args.command=='export-source':result=export_kido_benchmarks_source(args.repository,args.output)
+        elif args.command=='import-kido':result=import_kido_benchmarks(args.source,args.output)
+        else:
+            bundle=load_speech_benchmarks(args.bundle,expected_manifest_sha256=args.expected_manifest_sha256)
+            result=grade_speech_benchmarks(bundle,args.output) if args.command=='grade' else tts_benchmark_cases(bundle,
+                benchmark=args.benchmark,case_ids=args.case_id,language_hint=args.language_hint)
+    except (ValueError,OSError,KeyError,TypeError) as error:
+        p.exit(2,'Speech benchmark operation failed ('+type(error).__name__+'); check pinned source, IDs and output path.\n')
+    print(json.dumps(result,indent=2,ensure_ascii=True))
+
+
+if __name__=='__main__':main()