decisio 0.1.1__py3-none-any.whl

decisio/__init__.py

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the decisio project
+"""decisio: a prefill-only decision server on vLLM.
+
+  decisio.serve         the HTTP server (`python -m decisio.serve.vllm_engine`), its routes, tasks and engines
+  decisio.readout       the letters readout and the per-task corrections (calibration, intent head)
+  decisio.vllm_plugin   the model classes vLLM loads through the `vllm.general_plugins` entry point
+  decisio.bench         the benchmark stage: JevBench v1.5 open-set scoring, Decision Index reports, latency
+
+Importing `decisio` pulls in nothing beyond the standard library.
+"""
+
+__version__ = "0.1.1"

decisio/bench/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the decisio project
+"""The benchmark stage: JevBench v1.5 open-set scoring, Decision Index reports and rows, the latency grid."""

decisio/bench/banking77_labels.json

@@ -0,0 +1,81 @@
+{
+ "banking77": [
+  "activate my card",
+  "age limit",
+  "apple pay or google pay",
+  "atm support",
+  "automatic top up",
+  "balance not updated after bank transfer",
+  "balance not updated after cheque or cash deposit",
+  "beneficiary not allowed",
+  "cancel transfer",
+  "card about to expire",
+  "card acceptance",
+  "card arrival",
+  "card delivery estimate",
+  "card linking",
+  "card not working",
+  "card payment fee charged",
+  "card payment not recognised",
+  "card payment wrong exchange rate",
+  "card swallowed",
+  "cash withdrawal charge",
+  "cash withdrawal not recognised",
+  "change pin",
+  "compromised card",
+  "contactless not working",
+  "country support",
+  "declined card payment",
+  "declined cash withdrawal",
+  "declined transfer",
+  "direct debit payment not recognised",
+  "disposable card limits",
+  "edit personal details",
+  "exchange charge",
+  "exchange rate",
+  "exchange via app",
+  "extra charge on statement",
+  "failed transfer",
+  "fiat currency support",
+  "get disposable virtual card",
+  "get physical card",
+  "getting spare card",
+  "getting virtual card",
+  "lost or stolen card",
+  "lost or stolen phone",
+  "order physical card",
+  "passcode forgotten",
+  "pending card payment",
+  "pending cash withdrawal",
+  "pending top up",
+  "pending transfer",
+  "pin blocked",
+  "receiving money",
+  "Refund not showing up",
+  "request refund",
+  "reverted card payment?",
+  "supported cards and currencies",
+  "terminate account",
+  "top up by bank transfer charge",
+  "top up by card charge",
+  "top up by cash or cheque",
+  "top up failed",
+  "top up limits",
+  "top up reverted",
+  "topping up by card",
+  "transaction charged twice",
+  "transfer fee charged",
+  "transfer into account",
+  "transfer not received by recipient",
+  "transfer timing",
+  "unable to verify identity",
+  "verify my identity",
+  "verify source of funds",
+  "verify top up",
+  "virtual card not working",
+  "visa or mastercard",
+  "why verify identity",
+  "wrong amount of cash received",
+  "wrong exchange rate for cash withdrawal"
+ ]
+}

decisio/bench/di_cal.py

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the decisio project
+"""Accuracy, ECE and Brier per Decision Index benchmark from the kit's own results.jsonl joined with its rows' gold
+(the kit scores accuracy and macro-F1; it reports no calibration). ECE: 10 equal-mass bins on the top probability.
+
+    python -m decisio.bench.di_cal <run dir with results.jsonl> --rows <rows.jsonl.gz> [--out di_cal.json]
+"""
+
+import argparse
+import ast
+import gzip
+import json
+from pathlib import Path
+
+import numpy as np
+
+
+def ece_equal_mass(conf, correct, bins=10):
+    """Expected calibration error over `bins` bins of equal item count, sorted by confidence."""
+    order = np.argsort(conf)
+    e = 0.0
+    for chunk in np.array_split(order, bins):
+        if len(chunk):
+            e += len(chunk) / len(conf) * abs(correct[chunk].mean() - conf[chunk].mean())
+    return float(e)
+
+
+def as_dict(x):
+    return x if isinstance(x, dict) else ast.literal_eval(x)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("run")
+    ap.add_argument("--rows", required=True)
+    ap.add_argument("--out", default=None)
+    a = ap.parse_args()
+    gold = {}
+    for line in gzip.open(a.rows, "rt"):
+        r = json.loads(line)
+        gold[r["id"]] = (r["family"], as_dict(r["expected"]))
+    per, missing, n = {}, 0, 0
+    for line in open(Path(a.run) / "results.jsonl"):
+        r = json.loads(line)
+        n += 1
+        key = (
+            r["group_id"]
+            if r["group_id"] in gold
+            else next((k for k in (r["run_id"].split(":", 2)[-1],) if k in gold), None)
+        )
+        if key is None or r.get("status") != "ok":
+            missing += 1
+            continue
+        fam, exp = gold[key]
+        for qn, g in exp.items():
+            ans = as_dict(r["response"])["answers"][qn]
+            p = np.array(list(ans["probabilities"].values()), dtype=np.float64)
+            keys = list(ans["probabilities"])
+            y = keys.index(str(g))
+            d = per.setdefault(fam, {"ok": [], "conf": [], "brier": [], "nll": []})
+            d["ok"].append(float(ans["choice"] == str(g)))
+            d["conf"].append(float(p.max()))
+            d["brier"].append(float(((p - np.eye(len(p))[y]) ** 2).sum()))
+            d["nll"].append(float(-np.log(max(p[y], 1e-300))))
+    out = {"rows": n, "not_joined_or_failed": missing, "benchmarks": {}}
+    for fam, d in sorted(per.items()):
+        ok, conf = np.array(d["ok"]), np.array(d["conf"])
+        out["benchmarks"][fam] = {
+            "n": len(ok),
+            "accuracy": float(ok.mean()),
+            "ece": ece_equal_mass(conf, ok),
+            "brier": float(np.mean(d["brier"])),
+            "nll": float(np.mean(d["nll"])),
+            "mean_confidence": float(conf.mean()),
+        }
+        b = out["benchmarks"][fam]
+        print(
+            f"{fam:14s} n {b['n']:5d} accuracy {b['accuracy']:.4f} ECE {b['ece']:.4f} Brier {b['brier']:.4f} "
+            f"NLL {b['nll']:.4f}"
+        )
+    json.dump(out, open(a.out or Path(a.run) / "di_cal.json", "w"), indent=1)
+    if missing:
+        raise SystemExit(f"{missing} of {n} rows not joined or failed")
+
+
+if __name__ == "__main__":
+    main()

decisio/bench/di_report.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the decisio project
+"""The Decision Index 0.2.1 values of the four benchmarks run here, computed by the kit's own index02 functions from the
+kit's own `benchmark-summary.json` (its `score` writes that file first; the index step after it needs all 38 benchmarks
+and stops on the ones not run, so the four values are computed here with the same functions, chance levels and
+coverage rule the board uses).
+
+    python -m decisio.bench.di_report <run dir with benchmark-summary.json and results.jsonl> --suite-dir <suite> \
+        [--out ...]
+GPQA Diamond is track-scored (the 0.1 panel's rule: unanswered groups score zero inside the metric), so the kit's
+`score_panel` runs over the results first, exactly as its index step does.
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+from decision_index.scoring import index02 as X
+from decision_index.scoring.index import score_panel
+from decision_index.scoring.report import load_results
+from decision_index.suite.io import Suite
+
+IDS = {4: "BANKING77", 5: "CLINC150+OOS", 25: "GPQA Diamond", 57: "MMLU-Pro"}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("run")
+    ap.add_argument("--suite-dir", required=True)
+    ap.add_argument("--out", default=None)
+    a = ap.parse_args()
+    s = json.load(open(Path(a.run) / "benchmark-summary.json"))
+    spec = X.spec("0.2.1")
+    by_id = {b["catalog_id"]: b for b in s["benchmarks"]}
+    scored = score_panel(Suite(Path(a.suite_dir), "0.2.1"), load_results(Path(a.run) / "results.jsonl"))
+    out = {
+        "edition": "0.2.1",
+        "engine": s.get("engine"),
+        "latency_ms": s.get("successful_request_latency_ms"),
+        "benchmarks": {},
+    }
+    for n, name in IDS.items():
+        b = by_id.get(n, {})
+        v = X.benchmark_value(n, spec, scored.get(n), b)
+        out["benchmarks"][name] = {
+            "catalog_id": n,
+            "metric": b.get("metric"),
+            "native_score": b.get("score"),
+            "requests": b.get("requests"),
+            "answered": b.get("answered"),
+            "errors": b.get("errors"),
+            "unsupported": b.get("unsupported"),
+            "median_ms": b.get("median_ms"),
+            "chance": X.chance_of(n, spec),
+            "raw": v.get("raw"),
+            "skill": v.get("skill"),
+            "coverage": v.get("coverage"),
+        }
+        r = out["benchmarks"][name]
+        print(
+            f"{name:14s} {str(r['metric']):9s} native {r['native_score']} answered {r['answered']}/{r['requests']} "
+            f"raw {r['raw']} skill {r['skill']} median {r['median_ms']} ms"
+        )
+    json.dump(out, open(a.out or Path(a.run) / "di_report.json", "w"), indent=1)
+
+
+if __name__ == "__main__":
+    main()

decisio/bench/di_rows.py

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the decisio project
+"""The rows file of the four Decision Index 0.2.1 benchmarks the runs in runs/ measured, taken from the kit's rebuilt
+suite directory: its selected rows of BANKING77, CLINC150+OOS and GPQA Diamond, then its added rows of MMLU-Pro, in
+the suite's order. Checks the counts per benchmark and, by default, the uncompressed sha256 the runs used.
+
+    python -m decisio.bench.di_rows <suite dir> <out rows.jsonl.gz> [--no-hash-check]
+"""
+
+import argparse
+import collections
+import gzip
+import hashlib
+import json
+from pathlib import Path
+
+FAMILIES = {"BANKING77": 3080, "CLINC150+OOS": 5500, "GPQA-Diamond": 198, "MMLU-Pro": 12032}
+RUNS_SHA256 = "9b8537423f834be743373e3b033ea367b7c4e196b61be1b8480cddab0338ce80"  # uncompressed, runs/ 2026-09-27, -30
+
+
+def rows(suite: Path) -> list[str]:
+    out = []
+    for name in ("selected-rows.jsonl.gz", "added-rows.jsonl.gz"):
+        with gzip.open(suite / name, "rt") as f:
+            out += [line for line in f if json.loads(line)["family"] in FAMILIES]
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("suite", type=Path)
+    ap.add_argument("out", type=Path)
+    ap.add_argument("--no-hash-check", action="store_true")
+    a = ap.parse_args()
+    got = rows(a.suite)
+    counts = collections.Counter(json.loads(line)["family"] for line in got)
+    if dict(counts) != FAMILIES:
+        raise SystemExit(f"row counts {dict(counts)} differ from the runs' {FAMILIES}")
+    digest = hashlib.sha256("".join(got).encode()).hexdigest()
+    if digest != RUNS_SHA256 and not a.no_hash_check:
+        raise SystemExit(f"rows sha256 {digest} differs from the runs' {RUNS_SHA256} (--no-hash-check to go on)")
+    with gzip.GzipFile(a.out, "wb", mtime=0) as f:
+        f.write("".join(got).encode())
+    print(f"{sum(counts.values())} rows {dict(counts)} sha256 {digest} -> {a.out}")
+
+
+if __name__ == "__main__":
+    main()

decisio/bench/jevbench_v15.py

@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the decisio project
+"""JevBench v1.5 open-set Intelligence, scored privately from the harness's own per-item results
+(github.com/fstandhartinger/jevbench, MIT; the frozen method: its docs/METHOD-v1.5.md sections 3.1-3.2, and the v1.5.0
+headline amendment A, equal request types).
+
+The v1.5 harness and scorer are not published (the public repository holds the v1.4.2.2 harness, the 231 published v1.2
+items, typed choice / noul / score, and the frozen v1.5 method), so this implements the method's open-set rules on the
+harness's results for the published items. It is a private reading, not a board number: the board's open set has 904
+items (601 published, of which the 370 added in v1.5 are not in the repository; 303 never published), and half of
+Intelligence is the sealed set, which only the evaluator runs.
+
+  Choice  CC = 100 (acc - c) / (1 - c), c the mean over the tier's items of 1 / options; invalid answers wrong
+  Noul    P(yes) <= 0.20 is No, >= 0.80 is Yes, anything between is an abstention, counted wrong;
+          CC = 100 (acc - 0.5) / 0.5
+  Score   CC = 100 (1 - mean nMAE / mean nMAE_chance); nMAE = |expected position - gold| / (K - 1); an item's
+          nMAE_chance is the mean of |l - gold| / (K - 1) over its K levels; an invalid answer takes the item's
+          largest possible error
+  Tiers   easy 0.10, standard 0.20, judge 0.30, hard 0.40, renormalised over the tiers that have items
+  Types   A (the v1.5.x headline): choice, noul, score a third each; B (the frozen method): 0.50 / 0.25 / 0.25
+  Per-tier values are not clipped.
+
+    python -m decisio.bench.jevbench_v15 --run easy=<tasks.jsonl>:<results.jsonl> --run standard=... --run hard=... \
+        --out v15.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+TIER_WEIGHTS = {"easy": 0.10, "standard": 0.20, "judge": 0.30, "hard": 0.40}
+TYPE_WEIGHTS = {
+    "A": {"choice": 1 / 3, "noul": 1 / 3, "score": 1 / 3},
+    "B": {"choice": 0.50, "noul": 0.25, "score": 0.25},
+}
+NOUL_NO, NOUL_YES = 0.20, 0.80
+PUBLIC_FILE_TIER = {"easy": "easy", "original": "standard", "hard": "hard"}  # the published v1.2 files' tiers
+
+
+def item_score(task: dict, rec: dict | None) -> dict:
+    """One item under the v1.5 rules: {"type", "unit"} plus the type's fields. `rec` is the harness's result row for
+    the task (None: not answered, scored as invalid)."""
+    qtype = task["question"]["type"]
+    probs = rec.get("probs") if rec and rec.get("valid") else None
+    if qtype == "choice":
+        k = len(task["labels"])
+        return {
+            "type": "choice",
+            "correct": bool(probs is not None and rec.get("predicted") == str(task["expected"])),
+            "chance": 1.0 / k,
+            "valid": probs is not None,
+        }
+    if qtype == "noul":
+        if probs is None:
+            return {"type": "noul", "correct": False, "abstained": False, "valid": False}
+        p = float(probs["yes"])
+        said = "yes" if p >= NOUL_YES else ("no" if p <= NOUL_NO else None)
+        return {
+            "type": "noul",
+            "correct": said == str(task["expected"]),
+            "abstained": said is None,
+            "valid": True,
+            "p_yes": p,
+        }
+    k, gold = len(task["labels"]), int(task["expected"])
+    chance = sum(abs(lv - gold) for lv in range(k)) / k / (k - 1)
+    if probs is None:
+        return {"type": "score", "nmae": max(gold, k - 1 - gold) / (k - 1), "nmae_chance": chance, "valid": False}
+    pred = sum(int(lv) * float(p) for lv, p in probs.items())
+    return {"type": "score", "nmae": abs(pred - gold) / (k - 1), "nmae_chance": chance, "valid": True}
+
+
+def cell_cc(qtype: str, items: list[dict]) -> float:
+    """Chance-corrected competence of one type x tier cell (not clipped)."""
+    n = len(items)
+    if qtype == "choice":
+        acc, c = sum(x["correct"] for x in items) / n, sum(x["chance"] for x in items) / n
+        return 100.0 * (acc - c) / (1.0 - c)
+    if qtype == "noul":
+        return 100.0 * (sum(x["correct"] for x in items) / n - 0.5) / 0.5
+    return 100.0 * (1.0 - sum(x["nmae"] for x in items) / sum(x["nmae_chance"] for x in items))
+
+
+def score_runs(runs: list[tuple[str, list[dict], list[dict]]]) -> dict:
+    """runs: [(tier, tasks, harness result rows)]. Returns the per-type per-tier competence, CC per type and I_open
+    under type weightings A and B, with the item counts."""
+    cells: dict[tuple[str, str], list[dict]] = {}
+    for tier, tasks, rows in runs:
+        if tier not in TIER_WEIGHTS:
+            raise ValueError(f"unknown tier {tier!r}")
+        by = {r["task_id"]: r for r in rows}
+        for t in tasks:
+            s = item_score(t, by.get(t["id"]))
+            cells.setdefault((s["type"], tier), []).append(s)
+    out = {
+        "per_type": {},
+        "n_items": sum(len(v) for v in cells.values()),
+        "invalid": sum(not x["valid"] for v in cells.values() for x in v),
+    }
+    for qtype in ("choice", "noul", "score"):
+        tiers = {tier: cell_cc(qtype, cells[(qtype, tier)]) for tier in TIER_WEIGHTS if (qtype, tier) in cells}
+        if not tiers:
+            continue
+        w = sum(TIER_WEIGHTS[t] for t in tiers)
+        rec = {
+            "cc": sum(TIER_WEIGHTS[t] * v for t, v in tiers.items()) / w,
+            "tiers": tiers,
+            "n": {t: len(cells[(qtype, t)]) for t in tiers},
+        }
+        if qtype == "noul":
+            its = [x for t in tiers for x in cells[(qtype, t)]]
+            rec["abstention_rate"] = sum(x["abstained"] for x in its) / len(its)
+        out["per_type"][qtype] = rec
+    for name, W in TYPE_WEIGHTS.items():
+        sup = {t: W[t] for t in out["per_type"]}
+        out[f"I_open_{name}"] = sum(sup[t] * out["per_type"][t]["cc"] for t in sup) / sum(sup.values())
+    out["tiers_missing"] = sorted(set(TIER_WEIGHTS) - {tier for _, tier in cells})
+    return out
+
+
+def load_jsonl(path):
+    return [json.loads(line) for line in open(path) if line.strip()]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--run", action="append", required=True, help="<tier>=<tasks.jsonl>:<results.jsonl>")
+    ap.add_argument("--out", required=True)
+    a = ap.parse_args()
+    runs = []
+    for spec in a.run:
+        tier, paths = spec.split("=", 1)
+        tasks, results = paths.split(":", 1)
+        runs.append((tier, load_jsonl(tasks), load_jsonl(results)))
+    rep = score_runs(runs)
+    rep["note"] = (
+        "private reading of JevBench v1.5 open-set Intelligence on the published v1.2 items; not a board "
+        "number (the board's open set has 904 items and half of Intelligence is sealed)"
+    )
+    Path(a.out).write_text(json.dumps(rep, indent=1))
+    for qtype, r in rep["per_type"].items():
+        print(
+            f"{qtype:6s} CC {r['cc']:6.2f}  "
+            + "  ".join(f"{t} {v:.1f} (n={r['n'][t]})" for t, v in r["tiers"].items())
+            + (f"  abstained {r['abstention_rate']:.3f}" if qtype == "noul" else "")
+        )
+    print(
+        f"I_open A (equal types) {rep['I_open_A']:.2f}   B (50/25/25) {rep['I_open_B']:.2f}   "
+        f"items {rep['n_items']}, "
+        f"invalid {rep['invalid']}, tiers without items {rep['tiers_missing']}"
+    )
+    print("JEVBENCH_V15_DONE")
+
+
+if __name__ == "__main__":
+    main()