sys1bench 0.3.0__py3-none-any.whl

sys1bench/generators/support_tickets.py

@@ -0,0 +1,212 @@
+"""Support ticket generator (queue: choice, is_angry: noul, priority: score).
+
+Priority policy (stated in the question, not derivable from the ticket alone):
+    base urgency by template (0..2) + 1 if angry + 1 if customer tier is gold/enterprise, capped at 4.
+The customer tier appears in a `Context:` line. In the *unknowable* arm the tier line is removed, so a
+calibrated model must spread mass over two adjacent levels.
+"""
+
+from __future__ import annotations
+
+from ..schemas import Controls, LabelProvenance, Level, Option, Question, TaskItem
+from .base import BaseGenerator, canary_for, estimate_tokens, pad_to_tokens, register_generator
+
+QUEUES = [
+    ("billing", "Charges, refunds, invoices, duplicate payments, autopay problems"),
+    ("shipping", "Delivery delays, tracking, lost or damaged packages, wrong address"),
+    ("account", "Login, password, two-factor, email change, account lockout"),
+    ("technical", "App crashes, errors, features not working, connectivity"),
+    ("cancellation", "Ending a subscription or closing an account"),
+    ("sales", "Pre-purchase questions, upgrades, quotes, discounts"),
+    ("privacy", "Data export or deletion requests, consent, GDPR/CCPA"),
+    ("returns", "Returning a product, exchanges, return labels, refund status after return"),
+    ("fraud", "Unrecognised transactions, stolen card, suspicious login"),
+    ("feedback", "Compliments or suggestions with no action requested"),
+    ("accessibility", "Screen reader, captions, contrast or keyboard navigation problems"),
+    ("partnership", "Business development, reseller or affiliate enquiries"),
+]
+
+# (queue, base_urgency, template)
+TEMPLATES = [
+    ("billing", 1, "I was charged ${amt} twice on {day} for order #{order}. Only one charge should exist."),
+    ("billing", 0, "Can you send me the invoice for order #{order}? I need it for expenses."),
+    ("billing", 1, "My autopay failed on {day} and now there's a late fee of ${amt}. I had funds available."),
+    ("shipping", 1, "Order #{order} says delivered on {day} but nothing arrived. Tracking shows a photo of a door that isn't mine."),
+    ("shipping", 0, "What's the estimated delivery for order #{order}? It's been {days} days."),
+    ("shipping", 2, "Package #{order} arrived crushed and the contents are broken. I need this for an event on {day}."),
+    ("account", 2, "I'm locked out. Two-factor codes go to a phone number I no longer have. Order #{order} is pending."),
+    ("account", 0, "How do I change the email on my account? I want to switch to my work address."),
+    ("technical", 2, "The app crashes on launch since the update on {day}. I can't access anything. Error code {code}."),
+    ("technical", 1, "Export to CSV produces an empty file for reports over {days} days."),
+    ("cancellation", 1, "Please cancel my subscription before the renewal on {day}. Reference #{order}."),
+    ("cancellation", 0, "If I cancel, do I keep access until the end of the billing period?"),
+    ("sales", 0, "Do you offer a discount for annual billing on the team plan? We'd be about {days} seats."),
+    ("privacy", 1, "Please delete all my personal data under GDPR. Account reference #{order}."),
+    ("returns", 1, "I returned order #{order} on {day} and tracking shows it was received. When is the refund?"),
+    ("fraud", 2, "There's a ${amt} charge on {day} I did not make. My card ending {last4} may be compromised."),
+    ("feedback", 0, "Just wanted to say the new dashboard is much better. No action needed."),
+    ("accessibility", 1, "The checkout button has no label so my screen reader announces 'button'. Order #{order} stuck."),
+    ("partnership", 0, "We're a reseller in {region} and would like to discuss a partnership."),
+    ("technical", 1, "Notifications stopped arriving on {day}. Settings look correct. Device: {device}."),
+]
+
+# Surface variants applied to the customer text (labels unaffected): openers, register, sign-offs, light typos.
+OPENERS = ["", "", "Hi,", "Hello team,", "Hey,", "Good morning.", "To whom it may concern,", "Quick one:", "ok so", "Hi there —"]
+REGISTER = {
+    "formal": lambda t: t, "casual": lambda t: t.replace("I was", "i was").replace("Please", "pls").replace("please", "pls").replace("Thanks", "thx"),
+    "terse": lambda t: t.rstrip(".") + ".", "verbose": lambda t: t + " I have been a customer for a while and this has never happened before.",
+}
+SIGNOFFS = ["", "", "Regards, {name}", "— {name}", "Thanks, {name}", "Sent from my phone", "Best, {name}"]
+NAMES = ["Priya", "Jonas", "Amara", "Luis", "Mei", "Tomasz", "Fatima", "Noah"]
+ANGRY = [
+    "This is completely unacceptable and I am furious.",
+    "Honestly, this is ridiculous. Fix it today or I'm done.",
+    "I've had it with this company. Absolutely appalling service.",
+    "Unbelievable. Every single time. I want this escalated NOW.",
+]
+CALM = [
+    "Thanks in advance for your help.",
+    "Let me know if you need anything else from me.",
+    "Appreciate you looking into it.",
+    "No rush, but I'd like to get this sorted.",
+]
+TIERS = ["free", "standard", "gold", "enterprise"]
+DISTRACTORS = [
+    "P.S. I once had a delivery problem years ago with a different company; that got resolved.",
+    "Also I noticed the password reset page has a typo, not important.",
+    "By the way, a friend asked me whether you offer refunds, I told her I'd ask.",
+    'Log excerpt (from another tab): {"event":"cancel_click","source":"pricing_page","ts":"2026-09-01T10:00:00Z"}',
+]
+
+PRIORITY_LEVELS = [
+    Level(level=0, description="No action needed or informational."),
+    Level(level=1, description="Routine request; respond within 2 business days."),
+    Level(level=2, description="Customer blocked or money at stake; respond within 1 business day."),
+    Level(level=3, description="Urgent: blocked plus angry or high-value customer; respond within 4 hours."),
+    Level(level=4, description="Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."),
+]
+PRIORITY_INSTRUCTIONS = (
+    "Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue "
+    "(informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. "
+    "Add 1 if the customer tier is gold or enterprise. Cap at 4."
+)
+
+
+@register_generator("support_tickets")
+class SupportTicketGenerator(BaseGenerator):
+    version = "1.1.0"
+
+    def _queues(self) -> list[tuple[str, str]]:
+        k = self.knobs.cardinality or 6
+        k = max(2, min(k, len(QUEUES)))
+        used = {t[0] for t in TEMPLATES}
+        ordered = [q for q in QUEUES if q[0] in used] + [q for q in QUEUES if q[0] not in used]
+        return ordered[:k]
+
+    def generate(self) -> list[TaskItem]:
+        kb = self.knobs
+        rng = self.rng("items")          # content stream: identical across control arms
+        ctl = self.rng("controls")       # control stream: distractors, padding, label noise, unknowable, none-correct
+        queues = self._queues()
+        qkeys = [q[0] for q in queues]
+        templates = [t for t in TEMPLATES if t[0] in qkeys] or TEMPLATES
+        items: list[TaskItem] = []
+        for i in range(kb.n):
+            queue, base, tmpl = rng.choice(templates)
+            angry = rng.random() < 0.3
+            tier = rng.choice(TIERS)
+            unknowable = ctl.random() < kb.unknowable_frac
+            none_correct = ctl.random() < kb.none_correct_frac
+            distract = ctl.random() < kb.distractor_density
+            noise_draw = ctl.random() < kb.label_noise
+            body = tmpl.format(amt=rng.choice([19, 45, 89, 120, 249, 1200]), day=rng.choice(["Monday", "the 3rd", "yesterday", "Sept 14"]),
+                               order=rng.randrange(10000, 99999), days=rng.randrange(2, 30), code=f"E{rng.randrange(100,999)}",
+                               last4=rng.randrange(1000, 9999), region=rng.choice(["Germany", "Brazil", "Japan", "Canada"]),
+                               device=rng.choice(["Pixel 8", "iPhone 15", "Galaxy S24"]))
+            closing = rng.choice(ANGRY if angry else CALM)
+            # 1.1.0 surface diversity (content stream so arms stay paired)
+            opener = rng.choice(OPENERS)
+            reg = rng.choice(list(REGISTER))
+            body = REGISTER[reg](body)
+            signoff = rng.choice(SIGNOFFS).format(name=rng.choice(NAMES))
+            if rng.random() < 0.15:  # light typos in the body, never inside the order number
+                chars = list(body)
+                for _ in range(max(1, len(chars) // 60)):
+                    j = rng.randrange(len(chars))
+                    if chars[j].isalpha():
+                        chars[j] = rng.choice("abcdefghijklmnopqrstuvwxyz")
+                body = "".join(chars)
+            age, open_t = rng.randrange(1, 60), rng.randrange(0, 4)
+            ctx = f"Context: customer tier {tier}; account age {age} months; open tickets {open_t}."
+            if unknowable:
+                ctx = f"Context: account age {age} months; open tickets {open_t}."
+            text = " ".join(x for x in (opener, body, closing, signoff) if x)
+            state = f"Customer: \"{text}\"\n{ctx}"
+            if distract:
+                state += "\n" + ctl.choice(DISTRACTORS)
+            state = pad_to_tokens(ctl, state, kb.target_tokens)
+
+            priority = min(4, base + (1 if angry else 0) + (1 if tier in ("gold", "enterprise") else 0))
+            queue_truth = queue
+            options = [Option(key=k, description=d) for k, d in queues]
+            if none_correct:
+                # drop the true queue from the option list; truth becomes the abstain key
+                options = [o for o in options if o.key != queue] or options
+                queue_truth = "none_of_the_above"
+            noise = False
+            if kb.label_noise and noise_draw:
+                noise = True
+                queue_truth = ctl.choice([o.key for o in options if o.key != queue_truth] or [queue_truth])
+                priority = ctl.choice([p for p in range(5) if p != priority])
+                angry = not angry
+
+            tid = f"tickets_{kb.seed}_{i:05d}"
+            items.append(TaskItem(
+                task_id=tid, tier="G", domain="support_triage", language=kb.language, state=state,
+                state_tokens=estimate_tokens(state),
+                questions={
+                    "queue": Question(type="choice", instructions="Which queue should handle this ticket?", criteria=options,
+                                      ground_truth=queue_truth, framing_group="tickets.queue", allow_abstain=none_correct),
+                    "is_angry": Question(type="noul", instructions="Is the customer expressing anger or hostility?",
+                                         ground_truth=angry, framing_group="tickets.is_angry"),
+                    "priority": Question(type="score", instructions=PRIORITY_INSTRUCTIONS, criteria=PRIORITY_LEVELS,
+                                         ground_truth=priority, framing_group="tickets.priority"),
+                },
+                label_provenance=LabelProvenance(source="generator", generator=self.generator_id, seed=kb.seed, noise_injected=noise),
+                controls=Controls(unknowable=unknowable, none_correct=none_correct, distractor_density=kb.distractor_density),
+                cost_matrix=self.cost_matrices(),
+                canary=canary_for(tid),
+                metadata={"template_queue": queue, "base_urgency": base, "tier": tier, "angry": angry, "register": reg},
+            ))
+        return items
+
+    def regex_rules(self) -> dict:
+        return {
+            "tickets.queue": {
+                "billing": [r"charged", r"invoice", r"autopay", r"late fee"],
+                "shipping": [r"deliver", r"tracking", r"package", r"arrived"],
+                "account": [r"locked out", r"two-factor", r"change the email", r"password"],
+                "technical": [r"crash", r"error code", r"empty file", r"notifications stopped"],
+                "cancellation": [r"cancel"],
+                "sales": [r"discount", r"annual billing", r"seats"],
+                "privacy": [r"GDPR", r"delete all my personal data"],
+                "returns": [r"returned order", r"refund\?"],
+                "fraud": [r"did not make", r"compromised"],
+                "feedback": [r"no action needed", r"wanted to say"],
+                "accessibility": [r"screen reader"],
+                "partnership": [r"reseller", r"partnership"],
+            },
+            "tickets.is_angry": [r"unacceptable", r"furious", r"ridiculous", r"appalling", r"had it with", r"escalated"],
+            "tickets.priority": {"4": [r"fraud|did not make|compromised"], "3": [r"crash|locked out|crushed"],
+                                 "2": [r"charged .* twice|late fee|delivered .* nothing arrived|refund"],
+                                 "1": [r"cancel|GDPR|export|screen reader"], "0": [r"no action needed|discount|partnership|invoice"]},
+        }
+
+    def cost_matrices(self) -> dict[str, dict[str, dict[str, float]]]:
+        # asymmetric: routing fraud/billing wrongly is expensive; misrouting feedback is cheap
+        base = {q[0]: 1.0 for q in QUEUES}
+        expensive = {"fraud": 8.0, "billing": 3.0, "privacy": 4.0, "account": 3.0}
+        queue = {t: {p: (0.0 if p == t else expensive.get(t, base[t])) for p in base} for t in base}
+        priority = {str(t): {str(p): float(abs(t - p) * (2 if p < t else 1)) for p in range(5)} for t in range(5)}
+        angry = {"true": {"true": 0.0, "false": 3.0}, "false": {"true": 0.5, "false": 0.0}}
+        return {"queue": queue, "priority": priority, "is_angry": angry}

sys1bench/metrics/__init__.py

@@ -0,0 +1,9 @@
+from . import (  # noqa: F401
+    calibration,
+    consistency,
+    decision_value,
+    efficiency,
+    ordinal,
+    robustness,
+    selective,
+)

sys1bench/metrics/calibration.py

@@ -0,0 +1,164 @@
+"""Calibration kernels. All functions take confidence/probability arrays and 0/1 correctness or one-hot labels.
+
+Conventions: `probs` is (N, K); `y` is integer labels (N,); `conf` is max-probability (N,); `correct` is 0/1.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy.optimize import minimize_scalar
+
+
+def _bins(conf: np.ndarray, m: int, scheme: str) -> np.ndarray:
+    if scheme == "width":
+        edges = np.linspace(0.0, 1.0, m + 1)
+    elif scheme == "mass":
+        edges = np.quantile(conf, np.linspace(0, 1, m + 1))
+        edges[0], edges[-1] = 0.0, 1.0
+        edges = np.unique(edges)
+    else:
+        raise ValueError(scheme)
+    idx = np.clip(np.searchsorted(edges, conf, side="right") - 1, 0, len(edges) - 2)
+    return idx
+
+
+def ece(conf: np.ndarray, correct: np.ndarray, m: int = 15, scheme: str = "width") -> float:
+    conf, correct = np.asarray(conf, float), np.asarray(correct, float)
+    if len(conf) == 0:
+        return float("nan")
+    idx = _bins(conf, m, scheme)
+    total = 0.0
+    for b in np.unique(idx):
+        mask = idx == b
+        total += mask.mean() * abs(conf[mask].mean() - correct[mask].mean())
+    return float(total)
+
+
+def mce(conf: np.ndarray, correct: np.ndarray, m: int = 15, scheme: str = "width") -> float:
+    conf, correct = np.asarray(conf, float), np.asarray(correct, float)
+    idx = _bins(conf, m, scheme)
+    gaps = [abs(conf[idx == b].mean() - correct[idx == b].mean()) for b in np.unique(idx)]
+    return float(max(gaps)) if gaps else float("nan")
+
+
+def ece_noise_floor(conf: np.ndarray, m: int = 15, scheme: str = "width", resamples: int = 200,
+                    seed: int = 0) -> tuple[float, float]:
+    """Expected ECE (mean, sd) of a *perfectly calibrated* model with this confidence histogram:
+    resample correctness ~ Bernoulli(conf). Report measured ECE / floor, not ECE alone."""
+    rng = np.random.default_rng(seed)
+    conf = np.asarray(conf, float)
+    vals = [ece(conf, rng.random(len(conf)) < conf, m, scheme) for _ in range(resamples)]
+    return float(np.mean(vals)), float(np.std(vals))
+
+
+def smooth_ece(conf: np.ndarray, correct: np.ndarray, bandwidth: float = 0.05) -> float:
+    """Binning-free calibration error: kernel-smoothed |E[correct|conf] - conf| integrated over the
+    empirical confidence distribution (Gaussian kernel)."""
+    conf, correct = np.asarray(conf, float), np.asarray(correct, float)
+    if len(conf) == 0:
+        return float("nan")
+    d = (conf[:, None] - conf[None, :]) / bandwidth
+    w = np.exp(-0.5 * d**2)
+    smoothed_acc = (w * correct[None, :]).sum(1) / w.sum(1)
+    return float(np.mean(np.abs(smoothed_acc - conf)))
+
+
+def brier(probs: np.ndarray, y: np.ndarray) -> float:
+    probs = np.asarray(probs, float)
+    onehot = np.zeros_like(probs)
+    onehot[np.arange(len(y)), np.asarray(y, int)] = 1.0
+    return float(((probs - onehot) ** 2).sum(1).mean())
+
+
+def brier_decomposition(conf: np.ndarray, correct: np.ndarray, m: int = 15) -> dict[str, float]:
+    """Murphy decomposition on the top-label (confidence vs correctness) problem:
+    BS = reliability - resolution + uncertainty."""
+    conf, correct = np.asarray(conf, float), np.asarray(correct, float)
+    idx = _bins(conf, m, "width")
+    base = correct.mean()
+    rel = res = 0.0
+    for b in np.unique(idx):
+        mask = idx == b
+        rel += mask.mean() * (conf[mask].mean() - correct[mask].mean()) ** 2
+        res += mask.mean() * (correct[mask].mean() - base) ** 2
+    unc = base * (1 - base)
+    return {"reliability": float(rel), "resolution": float(res), "uncertainty": float(unc),
+            "brier_top": float(((conf - correct) ** 2).mean())}
+
+
+def nll(probs: np.ndarray, y: np.ndarray, clip: float | None = 1e-4) -> float:
+    p = np.asarray(probs, float)[np.arange(len(y)), np.asarray(y, int)]
+    if clip is not None:
+        p = np.clip(p, clip, 1.0)
+    with np.errstate(divide="ignore"):
+        return float(-np.log(p).mean())
+
+
+def zero_prob_on_truth(probs: np.ndarray, y: np.ndarray) -> int:
+    p = np.asarray(probs, float)[np.arange(len(y)), np.asarray(y, int)]
+    return int((p <= 0.0).sum())
+
+
+def quantisation_report(probs: np.ndarray) -> dict[str, float | None]:
+    flat = np.asarray(probs, float).ravel()
+    flat = flat[np.isfinite(flat)]
+    step = None
+    for s in (0.1, 0.05, 0.01, 0.005, 0.001, 1e-4, 1e-5):
+        if np.allclose(np.round(flat / s) * s, flat, atol=s * 1e-3):
+            step = s
+            break
+    return {"step": step, "frac_exact_zero": float((flat == 0).mean()), "frac_exact_one": float((flat == 1).mean()),
+            "distinct_values": float(len(np.unique(np.round(flat, 6))))}
+
+
+def fit_temperature(probs: np.ndarray, y: np.ndarray, eps: float = 1e-6) -> float:
+    """Single temperature minimising NLL on held-out data. T>1 means the model was over-confident."""
+    probs = np.clip(np.asarray(probs, float), eps, 1.0)
+    logits = np.log(probs)
+    y = np.asarray(y, int)
+
+    def loss(t: float) -> float:
+        z = logits / t
+        z = z - z.max(1, keepdims=True)
+        logp = z - np.log(np.exp(z).sum(1, keepdims=True))
+        return float(-logp[np.arange(len(y)), y].mean())
+
+    res = minimize_scalar(loss, bounds=(0.05, 20.0), method="bounded")
+    return float(res.x)
+
+
+def apply_temperature(probs: np.ndarray, t: float, eps: float = 1e-6) -> np.ndarray:
+    z = np.log(np.clip(np.asarray(probs, float), eps, 1.0)) / t
+    z = z - z.max(1, keepdims=True)
+    p = np.exp(z)
+    return p / p.sum(1, keepdims=True)
+
+
+def calibration_summary(probs: np.ndarray, y: np.ndarray, m: int = 15, floor_resamples: int = 200) -> dict:
+    probs, y = np.asarray(probs, float), np.asarray(y, int)
+    conf = probs.max(1)
+    correct = (probs.argmax(1) == y).astype(float)
+    floor_mean, floor_sd = ece_noise_floor(conf, m, "width", floor_resamples)
+    e_w = ece(conf, correct, m, "width")
+    out = {
+        "n": len(y),
+        "accuracy": float(correct.mean()),
+        "mean_confidence": float(conf.mean()),
+        "ece_width_15": e_w,
+        "ece_mass_15": ece(conf, correct, m, "mass"),
+        "ece_width_10": ece(conf, correct, 10, "width"),
+        "ece_width_25": ece(conf, correct, 25, "width"),
+        "ece_floor_mean": floor_mean,
+        "ece_floor_sd": floor_sd,
+        "ece_over_floor": (e_w / floor_mean) if floor_mean > 0 else float("nan"),
+        "mce_15": mce(conf, correct, m),
+        "smooth_ece": smooth_ece(conf, correct),
+        "brier": brier(probs, y),
+        "nll_clipped": nll(probs, y, 1e-4),
+        "nll_raw": nll(probs, y, None),
+        "zero_prob_on_truth": zero_prob_on_truth(probs, y),
+        "overconfidence": float((conf - correct).mean()),
+    }
+    out.update(brier_decomposition(conf, correct, m))
+    out.update({f"quant_{k}": v for k, v in quantisation_report(probs).items()})
+    return out

sys1bench/metrics/consistency.py

@@ -0,0 +1,71 @@
+"""Distribution-stability kernels: JSD, flip rates, complement consistency, interference matrices."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def jsd(p: np.ndarray, q: np.ndarray, eps: float = 1e-12) -> float:
+    p, q = np.asarray(p, float) + eps, np.asarray(q, float) + eps
+    p, q = p / p.sum(), q / q.sum()
+    m = 0.5 * (p + q)
+    return float(0.5 * (p * np.log2(p / m)).sum() + 0.5 * (q * np.log2(q / m)).sum())
+
+
+def pairwise_mean_jsd(dists: list[np.ndarray]) -> float:
+    if len(dists) < 2:
+        return float("nan")
+    vals = [jsd(dists[i], dists[j]) for i in range(len(dists)) for j in range(i + 1, len(dists))]
+    return float(np.mean(vals))
+
+
+def flip_rate(argmaxes: list[str]) -> float:
+    """Fraction of variants whose argmax differs from the canonical (first) variant."""
+    if len(argmaxes) < 2:
+        return float("nan")
+    return float(np.mean([a != argmaxes[0] for a in argmaxes[1:]]))
+
+
+def complement_consistency(p_yes_q: np.ndarray, p_yes_not_q: np.ndarray) -> dict[str, float]:
+    dev = np.asarray(p_yes_q, float) + np.asarray(p_yes_not_q, float) - 1.0
+    return {"complement_mad": float(np.abs(dev).mean()), "complement_bias": float(dev.mean())}
+
+
+def framing_summary(acc_by_framing: dict[str, float], per_item_jsd: list[float], per_item_flip: list[float]) -> dict:
+    friendly = {f: a for f, a in acc_by_framing.items() if not f.startswith("adv")}
+    adversarial = {f: a for f, a in acc_by_framing.items() if f.startswith("adv")}
+    accs = np.array(list(friendly.values()), float)
+    adv = np.array(list(adversarial.values()), float)
+    return {
+        "n_adversarial": int(len(adv)),
+        "accuracy_adversarial_min": float(adv.min()) if len(adv) else float("nan"),
+        "accuracy_adversarial_drop": float(np.median(accs) - adv.min()) if len(adv) and len(accs) else float("nan"),
+        "n_framings": len(accs),
+        "accuracy_median": float(np.median(accs)) if len(accs) else float("nan"),
+        "accuracy_min": float(accs.min()) if len(accs) else float("nan"),
+        "accuracy_max": float(accs.max()) if len(accs) else float("nan"),
+        "accuracy_range": float(accs.max() - accs.min()) if len(accs) else float("nan"),
+        "accuracy_std": float(accs.std(ddof=1)) if len(accs) > 1 else float("nan"),
+        "mean_pairwise_jsd": float(np.nanmean(per_item_jsd)) if per_item_jsd else float("nan"),
+        "argmax_flip_rate": float(np.nanmean(per_item_flip)) if per_item_flip else float("nan"),
+    }
+
+
+def interference_matrix(alone: dict[str, np.ndarray], batched: dict[tuple[str, str], np.ndarray]) -> dict[str, dict[str, float]]:
+    """alone[q] = distribution when q asked alone; batched[(q, other)] = distribution of q when asked with `other`.
+    Returns M[q][other] = JSD(alone[q], batched[(q, other)])."""
+    out: dict[str, dict[str, float]] = {}
+    for (q, other), d in batched.items():
+        out.setdefault(q, {})[other] = jsd(alone[q], d)
+    return out
+
+
+def positional_bias_chi2(argmax_indices: np.ndarray, k: int) -> dict[str, float]:
+    """Chi-square test that argmax index is uniform over positions under random permutations."""
+    from scipy.stats import chisquare
+
+    counts = np.bincount(np.asarray(argmax_indices, int), minlength=k)[:k]
+    if counts.sum() == 0:
+        return {"chi2": float("nan"), "p": float("nan"), "index0_share": float("nan")}
+    stat, p = chisquare(counts)
+    return {"chi2": float(stat), "p": float(p), "index0_share": float(counts[0] / counts.sum())}

sys1bench/metrics/decision_value.py

@@ -0,0 +1,43 @@
+"""Downstream decision value: expected cost of acting on the model's answers under a cost matrix,
+with and without an abstain/escalate option. This is the metric calibration is supposed to buy."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def expected_cost(probs: np.ndarray, truth_idx: np.ndarray, cost: np.ndarray, policy: str = "argmax",
+                  escalate_cost: float | None = None, threshold: float | None = None) -> dict[str, float]:
+    """cost[t, p] = cost of predicting p when truth is t. Policies:
+    - argmax: act on argmax.
+    - bayes: choose p minimising sum_t probs[t] * cost[t, p] (uses the probabilities).
+    - threshold: argmax if max prob >= threshold else escalate (pays escalate_cost, assumed correct).
+    """
+    probs, truth_idx, cost = np.asarray(probs, float), np.asarray(truth_idx, int), np.asarray(cost, float)
+    n = len(truth_idx)
+    if policy == "argmax":
+        pred = probs.argmax(1)
+        realised = cost[truth_idx, pred]
+        esc = np.zeros(n, bool)
+    elif policy == "bayes":
+        pred = (probs @ cost).argmin(1)
+        realised = cost[truth_idx, pred]
+        esc = np.zeros(n, bool)
+    elif policy == "threshold":
+        assert threshold is not None and escalate_cost is not None
+        pred = probs.argmax(1)
+        esc = probs.max(1) < threshold
+        realised = np.where(esc, escalate_cost, cost[truth_idx, pred])
+    else:
+        raise ValueError(policy)
+    oracle = cost[truth_idx, truth_idx]
+    return {"mean_cost": float(realised.mean()), "regret_vs_oracle": float((realised - oracle).mean()),
+            "escalation_rate": float(esc.mean()), "cost_per_10k": float(realised.mean() * 10_000)}
+
+
+def value_of_calibration(probs: np.ndarray, truth_idx: np.ndarray, cost: np.ndarray) -> float:
+    """Cost saved by acting on probabilities (Bayes) rather than argmax. Zero for symmetric costs;
+    positive when calibrated probabilities carry usable information about asymmetric risk."""
+    a = expected_cost(probs, truth_idx, cost, "argmax")["mean_cost"]
+    b = expected_cost(probs, truth_idx, cost, "bayes")["mean_cost"]
+    return float(a - b)

sys1bench/metrics/efficiency.py

@@ -0,0 +1,35 @@
+"""Latency, throughput, and cost. Hosted and local models are summarised separately by the report layer."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def latency_summary(ms: np.ndarray) -> dict[str, float]:
+    ms = np.asarray(ms, float)
+    ms = ms[np.isfinite(ms)]
+    if len(ms) == 0:
+        return {k: float("nan") for k in ("p50", "p90", "p95", "p99", "mean", "n")}
+    return {"p50": float(np.percentile(ms, 50)), "p90": float(np.percentile(ms, 90)),
+            "p95": float(np.percentile(ms, 95)), "p99": float(np.percentile(ms, 99)),
+            "mean": float(ms.mean()), "n": float(len(ms))}
+
+
+def throughput(n_decisions: int, n_questions: int, wall_seconds: float) -> dict[str, float]:
+    return {"decisions_per_s": n_decisions / wall_seconds if wall_seconds > 0 else float("nan"),
+            "questions_per_s": n_questions / wall_seconds if wall_seconds > 0 else float("nan")}
+
+
+def cost_summary(cost_per_call: np.ndarray, questions_per_call: np.ndarray | None = None,
+                 gpu_hour_usd: float | None = None, wall_seconds: float | None = None) -> dict[str, float]:
+    c = np.asarray(cost_per_call, float)
+    c = c[np.isfinite(c)]
+    out: dict[str, float] = {}
+    if len(c):
+        out["cost_per_100k_decisions"] = float(c.mean() * 100_000)
+        if questions_per_call is not None:
+            q = np.asarray(questions_per_call, float)
+            out["cost_per_100k_questions"] = float(c.sum() / max(q.sum(), 1) * 100_000)
+    if gpu_hour_usd is not None and wall_seconds is not None and len(c):
+        out["amortised_gpu_cost_per_100k_decisions"] = float(gpu_hour_usd * wall_seconds / 3600 / len(c) * 100_000)
+    return out

sys1bench/metrics/ordinal.py

@@ -0,0 +1,80 @@
+"""Ordinal metrics for the `score` primitive. Levels are integers; `probs` is (N, L) over levels."""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy.stats import spearmanr
+
+
+def expected_level(probs: np.ndarray, levels: np.ndarray) -> np.ndarray:
+    return np.asarray(probs, float) @ np.asarray(levels, float)
+
+
+def mae(pred: np.ndarray, true: np.ndarray) -> float:
+    return float(np.abs(np.asarray(pred, float) - np.asarray(true, float)).mean())
+
+
+def off_by_one_accuracy(pred: np.ndarray, true: np.ndarray) -> float:
+    return float((np.abs(np.asarray(pred, float) - np.asarray(true, float)) <= 1).mean())
+
+
+def quadratic_weighted_kappa(pred: np.ndarray, true: np.ndarray, levels: np.ndarray) -> float:
+    levels = list(np.asarray(levels))
+    k = len(levels)
+    idx = {lv: i for i, lv in enumerate(levels)}
+    o = np.zeros((k, k))
+    for p, t in zip(pred, true):
+        o[idx[int(t)], idx[int(p)]] += 1
+    n = o.sum()
+    if n == 0:
+        return float("nan")
+    w = np.array([[(i - j) ** 2 / (k - 1) ** 2 for j in range(k)] for i in range(k)])
+    e = np.outer(o.sum(1), o.sum(0)) / n
+    denom = (w * e).sum()
+    return float(1.0 - (w * o).sum() / denom) if denom > 0 else float("nan")
+
+
+def ranked_probability_score(probs: np.ndarray, true_idx: np.ndarray) -> float:
+    """Proper scoring rule for ordinal forecasts: mean squared difference of cumulative distributions,
+    normalised by L-1 so it lies in [0, 1]."""
+    probs = np.asarray(probs, float)
+    n, L = probs.shape
+    cdf = np.cumsum(probs, 1)
+    onehot = np.zeros_like(probs)
+    onehot[np.arange(n), np.asarray(true_idx, int)] = 1.0
+    ocdf = np.cumsum(onehot, 1)
+    return float(((cdf - ocdf) ** 2).sum(1).mean() / (L - 1))
+
+
+def monotonicity_rate(ladders: list[np.ndarray]) -> float:
+    """Each ladder is the expected level along a sequence of stimuli with increasing true severity.
+    Returns the fraction of ladders that are monotone non-decreasing."""
+    if not ladders:
+        return float("nan")
+    return float(np.mean([bool(np.all(np.diff(np.asarray(l, float)) >= -1e-9)) for l in ladders]))
+
+
+def ordinal_summary(probs: np.ndarray, true_levels: np.ndarray, levels: np.ndarray) -> dict:
+    probs, levels = np.asarray(probs, float), np.asarray(levels)
+    true_levels = np.asarray(true_levels)
+    idx = {lv: i for i, lv in enumerate(levels)}
+    true_idx = np.array([idx[int(t)] for t in true_levels])
+    argmax_lv = levels[probs.argmax(1)]
+    exp_lv = expected_level(probs, levels)
+    exp_round = levels[np.abs(exp_lv[:, None] - levels[None, :]).argmin(1)]
+    if len(set(true_levels.tolist())) > 1 and np.ptp(exp_lv) > 1e-12:
+        rho = spearmanr(exp_lv, true_levels).statistic
+    else:
+        rho = float("nan")
+    return {
+        "n": len(true_levels),
+        "exact_accuracy_argmax": float((argmax_lv == true_levels).mean()),
+        "exact_accuracy_expected": float((exp_round == true_levels).mean()),
+        "off_by_one_accuracy": off_by_one_accuracy(argmax_lv, true_levels),
+        "mae_argmax": mae(argmax_lv, true_levels),
+        "mae_expected": mae(exp_lv, true_levels),
+        "qwk_argmax": quadratic_weighted_kappa(argmax_lv, true_levels, levels),
+        "rps": ranked_probability_score(probs, true_idx),
+        "spearman_expected": float(rho) if rho == rho else float("nan"),
+        "argmax_vs_expected_disagreement": float((argmax_lv != exp_round).mean()),
+    }