refitgap 0.1.0__py3-none-any.whl

refitgap/identification.py

@@ -0,0 +1,101 @@
+"""The limits of rank summaries and archived counts, in exact fractions.
+
+identification_example(): fourteen records, three scorers with equal AUROC (6/11) and average
+precision (5/18), equal deficits (1/26), one all mismatch and one all attainable gap.
+f1_bounds_from_counts(): the range of best single-threshold F1 compatible with one confusion
+table. Both are in-sample statements, with no cross-fitting and no sampling error."""
+from fractions import Fraction
+
+import numpy as np
+
+# ------------------------------------------------------------------ rank arithmetic ---
+
+
+def f1_top(pos_ranks, k):
+    """F1 with the top k ranks predicted positive; 0 when k and the positives are both 0."""
+    tp = sum(1 for r in pos_ranks if r <= k)
+    den = k + len(pos_ranks)
+    return Fraction(2 * tp, den) if den else Fraction(0)
+
+
+def best_f1(pos_ranks, n):
+    """The best F1 over every single threshold (every top-k, k = 0..n) on these records."""
+    return max(f1_top(pos_ranks, k) for k in range(n + 1))
+
+
+def auroc(pos_ranks, n):
+    """AUROC (no ties; rank 1 is the top)."""
+    P = sorted(pos_ranks)
+    neg = n - len(P)
+    above = sum(sum(1 for q in range(r + 1, n + 1) if q not in P) for r in P)
+    return Fraction(above, len(P) * neg)
+
+
+def average_precision(pos_ranks):
+    """Average precision (no ties)."""
+    P = sorted(pos_ranks)
+    return sum(Fraction(i + 1, r) for i, r in enumerate(P)) / len(P)
+
+
+def rank_scores(n, selected, threshold=0.5):
+    """Scores 1..n ranks apart such that exactly the top `selected` ranks reach `threshold`
+    (rank r gets threshold + (selected + 1/2 - r) / (2(n + 1)); all in (0, 1) for threshold 1/2)."""
+    r = np.arange(1, n + 1, dtype=np.float64)
+    return threshold + (selected + 0.5 - r) / (2.0 * (n + 1))
+
+
+# ------------------------------------------------------------------ the example ---
+N_RECORDS = 14
+SCORERS = {                       # positive ranks (1 = top), records the carried rule selects
+    "reference": dict(positives=(4, 8, 9), selected=9),
+    "final A": dict(positives=(4, 8, 9), selected=10),     # same ranking, rule now takes one more record
+    "final B": dict(positives=(5, 6, 10), selected=10),    # different ranking
+}
+
+
+def identification_example():
+    """() -> dict of exact Fractions: per scorer the carried and best F1, AUROC and average
+    precision; per final scorer the deficit, attainable gap and change in mismatch; plus float
+    scores and a threshold of 1/2 that realise the example."""
+    out = dict(n_records=N_RECORDS, n_labels=1, threshold=0.5, scorers={}, decomposition={},
+               note=("in-sample: the best F1 is fitted and scored on the same fourteen records; no cross-fitting, "
+                     "no sampling uncertainty"))
+    for name, s in SCORERS.items():
+        P = s["positives"]
+        y = np.zeros(N_RECORDS, int)
+        y[[r - 1 for r in P]] = 1
+        out["scorers"][name] = dict(positives=P, selected=s["selected"], carried_f1=f1_top(P, s["selected"]),
+                                    best_f1=best_f1(P, N_RECORDS), auroc=auroc(P, N_RECORDS),
+                                    average_precision=average_precision(P),
+                                    scores=rank_scores(N_RECORDS, s["selected"]), labels=y)
+    ref = out["scorers"]["reference"]
+    for name in ("final A", "final B"):
+        f = out["scorers"][name]
+        D = ref["carried_f1"] - f["carried_f1"]
+        A = ref["best_f1"] - f["best_f1"]
+        out["decomposition"][name] = dict(deficit=D, attainable_gap=A, mismatch_change=D - A)
+    return out
+
+
+# ------------------------------------------------------------------ counts ---
+def _f1(tp, pred, pos):
+    den = pred + pos
+    return Fraction(2 * tp, den) if den else Fraction(0)
+
+
+def f1_bounds_from_counts(tp, fp, fn, tn, exact=False):
+    """(tp, fp, fn, tn) at one threshold -> (lower, upper) sharp bounds on the best F1 over all
+    thresholds and all rankings compatible with the table (Fractions with exact=True):
+        lower = max(2tp / (2tp + fp + fn), 2P / (2P + N)),  upper = max(2tp / (2tp + fn), 2P / (2P + fp))
+    with P = tp + fn, N = fp + tn. A bound on what the counts identify, not a confidence interval."""
+    vals = [int(v) for v in (tp, fp, fn, tn)]
+    if any(v < 0 for v in vals) or any(int(v) != v for v in (tp, fp, fn, tn)):
+        raise ValueError("counts must be non-negative integers")
+    tp, fp, fn, tn = vals
+    P, N = tp + fn, fp + tn
+    if P == 0:
+        lo = hi = Fraction(0)
+    else:
+        lo = max(_f1(tp, tp + fp, P), _f1(P, P + N, P))
+        hi = max(_f1(tp, tp, P), _f1(P, P + fp, P))
+    return (lo, hi) if exact else (float(lo), float(hi))

refitgap/inference.py

@@ -0,0 +1,319 @@
+"""Seed-paired contrasts: t intervals, TOST, sign-flip tests, Holm families and outcome classes.
+
+A contrast is a per-seed vector x (method minus comparator). family() Holm-adjusts the t and
+TOST p values across its members and assigns, at `margin` (default MARGIN = 0.005):
+    equivalent      Holm TOST p < ALPHA
+    material        Holm t p < ALPHA and |mean| >= margin
+    small non-zero  Holm t p < ALPHA and |mean| <  margin
+    unresolved      otherwise, or when the sign-flip p >= ALPHA overrules the two above
+Intervals are marginal (unadjusted). Sign-flip tests are exact up to SIGNFLIP_MAX_N seeds and
+Monte Carlo above (the result then says so). Non-finite values raise ValueError.
+"""
+import inspect
+import itertools
+
+import numpy as np
+from scipy import stats
+
+MARGIN, ALPHA = 0.005, 0.05
+BCA_N, BCA_SEED = 10_000, 20261029
+SIGNFLIP_MAX_N = 20                      # exact enumeration up to here (2^20 patterns)
+SIGNFLIP_MC_DRAWS = 100_000
+SIGNFLIP_MC_SEED = 20261114
+_SIGNFLIP_MC_CHUNK = 10_000
+
+# scipy renamed bootstrap's random_state= to rng=
+try:
+    _BOOT_RNG_KW = "rng" if "rng" in inspect.signature(stats.bootstrap).parameters else "random_state"
+except (TypeError, ValueError):
+    _BOOT_RNG_KW = "random_state"
+
+
+# ------------------------------------------------------------------ sign-flip tests ---
+def _signflip_p_enumerated(x):
+    x = np.asarray(x, float)
+    n, obs = len(x), abs(x.mean())
+    signs = np.array(list(itertools.product([-1.0, 1.0], repeat=n)))
+    means = np.abs((signs * x).mean(axis=1))
+    return float(np.mean(means >= obs - 1e-12))
+
+
+def _ni_signflip_p_enumerated(x, margin=MARGIN):
+    y = np.asarray(x, float) + margin
+    obs = y.mean()
+    if np.allclose(y, 0):
+        return 1.0
+    signs = np.array(list(itertools.product([-1.0, 1.0], repeat=len(y))))
+    return float(np.mean((signs * np.abs(y)).mean(1) >= obs - 1e-12)) if obs > 0 else 1.0
+
+
+def signflip_method(n):
+    """n seeds -> how the sign-flip p is computed, as text."""
+    if n <= SIGNFLIP_MAX_N:
+        return f"exact (all 2^{n} sign patterns)"
+    return (f"Monte Carlo ({SIGNFLIP_MC_DRAWS} random sign patterns from numpy default_rng({SIGNFLIP_MC_SEED}); "
+            f"p = (1 + count) / (1 + {SIGNFLIP_MC_DRAWS}), the observed pattern counted once)")
+
+
+def _mc_sign_chunks(n):
+    rng = np.random.default_rng(SIGNFLIP_MC_SEED)
+    left = SIGNFLIP_MC_DRAWS
+    while left > 0:
+        b = min(_SIGNFLIP_MC_CHUNK, left)
+        yield rng.integers(0, 2, size=(b, n)).astype(float) * 2.0 - 1.0
+        left -= b
+
+
+def signflip_p(x):
+    """Per-seed vector -> two-sided sign-flip p of the mean."""
+    x = np.asarray(x, float)
+    n = len(x)
+    if n <= SIGNFLIP_MAX_N:
+        return _signflip_p_enumerated(x)
+    obs = abs(x.mean())
+    count = sum(int(np.sum(np.abs((s * x).mean(axis=1)) >= obs - 1e-12)) for s in _mc_sign_chunks(n))
+    return float((1 + count) / (1 + SIGNFLIP_MC_DRAWS))
+
+
+def ni_signflip_p(x, margin=MARGIN):
+    """Per-seed vector -> one-sided sign-flip p of H0 'symmetric about -margin'."""
+    x = np.asarray(x, float)
+    n = len(x)
+    if n <= SIGNFLIP_MAX_N:
+        return _ni_signflip_p_enumerated(x, margin)
+    y = x + margin
+    obs = y.mean()
+    if np.allclose(y, 0) or not obs > 0:
+        return 1.0
+    ay = np.abs(y)
+    count = sum(int(np.sum((s * ay).mean(1) >= obs - 1e-12)) for s in _mc_sign_chunks(n))
+    return float((1 + count) / (1 + SIGNFLIP_MC_DRAWS))
+
+
+def ni_p(x, margin=MARGIN):
+    """Per-seed vector -> one-sided paired t p for non-inferiority (H0: mean <= -margin)."""
+    x = np.asarray(x, float); n = len(x); m = float(x.mean())
+    se = float(x.std(ddof=1) / np.sqrt(n)) if n > 1 else 0.0
+    if se == 0:
+        return 0.0 if m > -margin else 1.0
+    return float(1 - stats.t.cdf((m + margin) / se, n - 1))
+
+
+# ------------------------------------------------------------------ input checks ---
+def _check_finite(x, what):
+    x = np.asarray(x, float)
+    if x.ndim != 1:
+        raise ValueError(f"{what}: a per-seed vector (1-d) is expected, got shape {x.shape}")
+    bad = np.flatnonzero(~np.isfinite(x))
+    if bad.size:
+        raise ValueError(
+            f"{what}: non-finite value {float(x[bad[0]])!r} at position {int(bad[0])} ({bad.size} non-finite in "
+            "all); a failed run must be dropped (with the seed set adjusted openly) or re-run, not "
+            "stored as NaN or inf"
+        )
+    return x
+
+
+def _check_margin(margin):
+    m = float(margin)
+    if not np.isfinite(m) or m < 0:
+        raise ValueError(f"margin must be finite and non-negative, got {margin!r}")
+    return m
+
+
+# ------------------------------------------------------------------ one contrast ---
+def bca_ci(x):
+    """Per-seed vector -> BCa 95 % interval of the mean, or None (fewer than 3 values, constant,
+    or BCa undefined). Sensitivity only; it never decides a class."""
+    x = np.asarray(x, float)
+    if len(x) < 3 or np.ptp(x) == 0:
+        return None
+    try:
+        r = stats.bootstrap((x,), np.mean, method="BCa", n_resamples=BCA_N,
+                            **{_BOOT_RNG_KW: np.random.default_rng(BCA_SEED)})
+    except (ValueError, ArithmeticError):
+        return None
+    return [float(r.confidence_interval.low), float(r.confidence_interval.high)]
+
+
+def paired_stats(x, margin=MARGIN, name="x"):
+    """Per-seed vector -> dict: mean, sd, se, 95 % and 90 % t intervals, t p, TOST p at +-margin,
+    sign-flip p, BCa interval. All values equal: p_t is 1 for mean 0 and 0 otherwise."""
+    x = _check_finite(x, name)
+    margin = _check_margin(margin)
+    n = len(x)
+    m = float(x.mean())
+    if n < 2:
+        return dict(n=n, mean=m)
+    se = float(x.std(ddof=1) / np.sqrt(n))
+    t95 = stats.t.ppf(0.975, n - 1) * se
+    t90 = stats.t.ppf(0.95, n - 1) * se
+    p = float(stats.ttest_1samp(x, 0.0).pvalue) if se > 0 else 0.0
+    if se > 0:
+        p_lo = float(1 - stats.t.cdf((m + margin) / se, n - 1))
+        p_hi = float(stats.t.cdf((m - margin) / se, n - 1))
+    else:
+        p_lo = p_hi = 0.0 if abs(m) < margin else 1.0
+    p_tost = max(p_lo, p_hi)
+    st = dict(n=n, mean=m, se=se, sd=float(x.std(ddof=1)), ci95=[m - t95, m + t95], ci90=[m - t90, m + t90],
+              p_t=p, p_tost=p_tost, p_lo=p_lo, p_hi=p_hi, tost_pass=bool(p_tost < ALPHA),
+              p_signflip=signflip_p(x), values=x.tolist(), bca95=bca_ci(x))
+    if se == 0 and m == 0:
+        st["p_t"] = 1.0
+    if n > SIGNFLIP_MAX_N:
+        st["p_signflip_method"] = signflip_method(n)
+    return st
+
+
+def _holm_core(ps):
+    m = len(ps)
+    order = np.argsort(ps)
+    adj = np.empty(m)
+    running = 0.0
+    for rank, i in enumerate(order):
+        running = max(running, (m - rank) * ps[i])
+        adj[i] = min(1.0, running)
+    return adj.tolist()
+
+
+def holm(ps):
+    """p values in [0, 1] -> Holm step-down adjusted p values (a NaN raises)."""
+    ps = np.asarray(ps, float)
+    if ps.ndim != 1:
+        raise ValueError(f"holm: a 1-d sequence of p values is expected, got shape {ps.shape}")
+    bad = np.flatnonzero(~np.isfinite(ps) | (ps < 0) | (ps > 1))
+    if bad.size:
+        raise ValueError(f"holm: p value {float(ps[bad[0]])!r} at position {int(bad[0])} is not a number in [0, 1]")
+    return _holm_core(ps)
+
+
+def classify(st, margin=MARGIN):
+    """paired_stats dict with p_holm, p_tost_holm -> class (before the sign-flip override)."""
+    if st.get("n", 0) < 2:
+        return "unresolved"
+    if st["p_tost_holm"] < ALPHA:
+        return "equivalent"
+    if st["p_holm"] < ALPHA:
+        return "material" if abs(st["mean"]) >= margin else "small non-zero"
+    return "unresolved"
+
+
+def signflip_override(st, margin=MARGIN):
+    """Sets st['class']: classify(), turned unresolved when the sign-flip test disagrees."""
+    cls = classify(st, margin)
+    st["class_before_signflip"] = cls
+    st["signflip_disagrees"] = False
+    if cls in ("material", "small non-zero") and st["p_signflip"] >= ALPHA:
+        st["signflip_disagrees"] = True
+        cls = "unresolved"
+    st["class"] = cls
+    return st
+
+
+def classify_unadjusted(st, margin=MARGIN):
+    """paired_stats dict -> the same with its class as a family of one (no Holm)."""
+    if st.get("n", 0) < 2:
+        st["class"] = "unresolved"
+        return st
+    st["p_holm"], st["p_tost_holm"] = st["p_t"], st["p_tost"]
+    st["holm_family_size"] = 1
+    return signflip_override(st, margin)
+
+
+# ------------------------------------------------------------------ families ---
+def family(members, margin=MARGIN):
+    """{name: per-seed vector} -> {name: paired_stats dict with Holm p values and class}."""
+    margin = _check_margin(margin)
+    names = list(members)
+    sts = {k: paired_stats(members[k], margin, name=f"family member {k!r}") for k in names}
+    pt = holm([sts[k]["p_t"] for k in names])
+    pq = holm([sts[k]["p_tost"] for k in names])
+    for k, a, b in zip(names, pt, pq):
+        sts[k]["p_holm"], sts[k]["p_tost_holm"], sts[k]["holm_family_size"] = a, b, len(names)
+        signflip_override(sts[k], margin)
+    return sts
+
+
+def family_at(members, margin):
+    """Same as family(members, margin)."""
+    return family(members, margin)
+
+
+def family_ni(members, margin=MARGIN):
+    """family() plus, per member, non-inferiority at -margin ('ni': Holm one-sided t and
+    sign-flip both < ALPHA) and the mirror inferiority test ('inf': below -margin)."""
+    margin = _check_margin(margin)
+    names = list(members)
+    vals = {k: _check_finite(members[k], f"family member {k!r}") for k in names}
+    sts = family(vals, margin)
+    p_ni = [ni_p(vals[k], margin) for k in names]
+    for k, p, q in zip(names, p_ni, holm(p_ni)):
+        s = sts[k]
+        s["p_ni"], s["p_ni_holm"] = p, q
+        s["p_ni_signflip"] = ni_signflip_p(vals[k], margin)
+        s["ni_signflip_disagrees"] = bool(q < ALPHA and s["p_ni_signflip"] >= ALPHA)
+        s["ni"] = bool(q < ALPHA and not s["ni_signflip_disagrees"])
+        s["margin"] = margin
+    p_inf = [ni_p(-vals[k], -margin) for k in names]
+    for k, p, q in zip(names, p_inf, holm(p_inf)):
+        s = sts[k]
+        s["p_inf"], s["p_inf_holm"] = p, q
+        s["p_inf_signflip"] = ni_signflip_p(-vals[k], -margin)
+        s["inf"] = bool(q < ALPHA and s["p_inf_signflip"] < ALPHA)
+        if len(vals[k]) > SIGNFLIP_MAX_N:
+            s["p_ni_signflip_method"] = s["p_inf_signflip_method"] = signflip_method(len(vals[k]))
+    return sts
+
+
+def tost_power(sd, n=10, alphas=(0.0125, 0.05), reps=20_000, seed=20261029, margin=MARGIN):
+    """sd of a paired contrast -> {alpha: TOST power at a true difference of 0} (normal draws;
+    optimistic when sd comes from the same pairs)."""
+    if not sd or not np.isfinite(sd):
+        return {a: None for a in alphas}
+    x = np.random.default_rng(seed).normal(0.0, sd, (reps, n))
+    m, se = x.mean(1), x.std(1, ddof=1) / np.sqrt(n)
+    p = np.maximum(1 - stats.t.cdf((m + margin) / se, n - 1), stats.t.cdf((m - margin) / se, n - 1))
+    return {a: float(np.mean(p < a)) for a in alphas}
+
+
+# ------------------------------------------------------------------ dicts by seed ---
+def paired_vector(a, b, seeds=None, strict=True):
+    """{seed: value} twice -> (seeds, a - b in sorted seed order). Different seed sets raise
+    unless seeds= names the planned ones or strict=False takes the common ones; NaN raises."""
+    if seeds is not None:
+        seeds = sorted(int(s) for s in seeds)
+        miss = {"a": [s for s in seeds if s not in a], "b": [s for s in seeds if s not in b]}
+        if miss["a"] or miss["b"]:
+            raise ValueError(f"planned seeds missing: {miss}")
+    elif set(a) != set(b):
+        if strict:
+            raise ValueError(f"the two sides hold different seeds (only in a: {sorted(set(a) - set(b))}, "
+                             f"only in b: {sorted(set(b) - set(a))}); pass seeds= or strict=False")
+        seeds = sorted(set(a) & set(b))
+    else:
+        seeds = sorted(a)
+    if not seeds:
+        raise ValueError("no common seeds")
+    for side, d in (("a", a), ("b", b)):
+        bad = [s for s in seeds if not np.isfinite(float(d[s]))]
+        if bad:
+            raise ValueError(f"side {side}, seed {bad[0]}: non-finite value {float(d[bad[0]])!r}; a failed run must be "
+                             "dropped (seeds=, strict=False) or re-run, not stored as NaN or inf")
+    return seeds, np.array([a[s] - b[s] for s in seeds], float)
+
+
+def dropped_seeds(a, b, seeds=None):
+    """Seeds on one side only (or planned seeds missing from either)."""
+    if seeds is not None:
+        return sorted(s for s in seeds if s not in a or s not in b)
+    return sorted(set(a) ^ set(b))
+
+
+def paired_contrast(a, b, seeds=None, strict=True, margin=MARGIN):
+    """{seed: value} twice -> paired_stats of a - b with its unadjusted class, seeds used and
+    seeds dropped."""
+    used, x = paired_vector(a, b, seeds, strict)
+    st = classify_unadjusted(paired_stats(x, margin, name="paired contrast a - b"), margin)
+    st["seeds"] = used
+    st["dropped_seeds"] = [] if seeds is not None else dropped_seeds(a, b)
+    return st

refitgap/plots.py

@@ -0,0 +1,214 @@
+"""The three figures of a report, from analysis.analyse() tables: deficit decomposition
+(A and R per period), period profile (U, V_sep, V_shared) and method comparison (leads with
+intervals and the margin). PDF and SVG; needs matplotlib (extra "plots")."""
+import io
+from pathlib import Path
+
+import numpy as np
+
+A_COL = "#D95319"        # attainable term A (as in the paper)
+R_COL = "#0072BD"        # mismatch terms R, dR, C, M (as in the paper)
+INK = "#222222"
+MUTED = "#555555"
+GREY = "#7a7a7a"
+FIGURES = ("decomposition", "profile", "comparison")
+
+
+def _plt():
+    try:
+        import matplotlib
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+    except ImportError as e:                       # pragma: no cover
+        raise ImportError("figures need matplotlib: pip install 'refitgap[plots]'") from e
+    # fixed salt for the ids matplotlib writes into SVG, so a figure is the same byte for byte
+    plt.rcParams.update({"font.family": "sans-serif", "font.size": 8.5, "axes.labelsize": 8.5,
+                         "axes.titlesize": 9, "xtick.labelsize": 8, "ytick.labelsize": 8, "legend.fontsize": 7.5,
+                         "axes.edgecolor": MUTED, "axes.labelcolor": INK, "xtick.color": MUTED,
+                         "ytick.color": MUTED, "axes.spines.top": False, "axes.spines.right": False,
+                         "svg.fonttype": "none", "svg.hashsalt": "refitgap", "pdf.fonttype": 42,
+                         "lines.linewidth": 1.2})
+    return plt
+
+
+def _is_mean(p):
+    return str(p).startswith("mean")
+
+
+def _note(fig, text):
+    import textwrap
+    fig.text(0.01, -0.01, "\n".join(textwrap.wrap(text, 95)), ha="left", va="top", fontsize=7, color=MUTED)
+
+
+def _jitter(n, width=0.12):
+    return np.zeros(1) if n == 1 else np.linspace(-width, width, n)
+
+
+def figure_decomposition(result):
+    """Signed A and dR per period and their mean over the selected periods, against the
+    per-period reference. One panel per method that has a reference."""
+    plt = _plt()
+    rows = result["decomposition"]
+    methods = list(dict.fromkeys(r["method"] for r in rows))
+    if not methods:
+        fig, ax = plt.subplots(figsize=(4.2, 1.2))
+        ax.axis("off")
+        ax.text(0.5, 0.5, "No per-period reference scores were given,\nso B = A + dR is not computed.",
+                ha="center", va="center", color=MUTED)
+        return fig
+    fig, axes = plt.subplots(1, len(methods), figsize=(3.6 * len(methods) + 0.6, 2.9), squeeze=False)
+    conv = rows[0]["U_convention"]
+    for ax, m in zip(axes[0], methods):
+        mr = [r for r in rows if r["method"] == m]
+        cats = list(dict.fromkeys(r["period"] for r in mr))
+        xs = np.arange(len(cats), dtype=float)
+        seeds = sorted({r["seed"] for r in mr})
+        for term, col, mk, ls, off in (("A", A_COL, "s", "-", -0.17), ("dR", R_COL, "o", "--", 0.17)):
+            means = []
+            for i, c in enumerate(cats):
+                v = np.array([r[term] for r in mr if r["period"] == c])
+                if len(seeds) > 1:
+                    ax.plot(xs[i] + off + _jitter(len(v), 0.06), v, mk, ms=2.8, mfc="white", mec=col, mew=0.8,
+                            alpha=0.9, zorder=2)
+                means.append(v.mean())
+            per = [i for i, c in enumerate(cats) if not _is_mean(c)]
+            ax.plot(xs[per] + off, np.array(means)[per], ls=ls, color=col, lw=0.9, zorder=3)
+            ax.plot(xs + off, means, mk, ms=6, color=col, mec="white", mew=0.6, zorder=4,
+                    label="A, attainable gap" if term == "A" else "dR, change in mismatch")
+        ax.axhline(0, color=MUTED, lw=0.7, ls=":")
+        if any(_is_mean(c) for c in cats):
+            ax.axvline(len(cats) - 1.5, color=GREY, lw=0.5)
+        ax.set_xticks(xs)
+        ax.set_xticklabels(["mean" if _is_mean(c) else str(c) for c in cats])
+        ax.set_xlabel("period id")
+        ax.set_ylabel("reference minus final\nmacro-F1 difference (0 to 1 scale)")
+        ax.set_title(f"{m}: B = A + dR, rule '{rows[0]['rule']}', {conv}", loc="left")
+        n = len(seeds)
+    axes[0][0].legend(frameon=False, loc="best")
+    fig.tight_layout()
+    _note(fig, f"{'Small hollow markers: single runs. ' if n > 1 else ''}Filled markers: mean over {n} "
+               f"run{'s' if n > 1 else ''}. Positive where the final model is below the reference. Descriptive.")
+    return fig
+
+
+def figure_profile(result):
+    """U at the fixed rule, V_sep and V_shared per period for each method's final model."""
+    plt = _plt()
+    rule = result["settings"]["rule"]
+    rows = [r for r in result["levels"] if r["model"] == "final" and r["rule"] == rule and not _is_mean(r["period"])]
+    methods = list(dict.fromkeys(r["method"] for r in rows))
+    fig, axes = plt.subplots(1, len(methods), figsize=(3.6 * len(methods) + 0.6, 2.9), squeeze=False, sharey=True)
+    series = (("V_sep", "V_sep, own-period rules", INK, "o", "-"),
+              ("V_shared", "V_shared, one shared rule", GREY, "s", "--"),
+              ("U_full", f"U_full, fixed rule '{rule}'", R_COL, "^", ":"))
+    seen = []
+    for ax, m in zip(axes[0], methods):
+        mr = [r for r in rows if r["method"] == m]
+        per = list(dict.fromkeys(r["period"] for r in mr))
+        xs = np.arange(len(per), dtype=float)
+        nseed = len({r["seed"] for r in mr})
+        for key, lab, col, mk, ls in series:
+            seen.extend(r[key] for r in mr if r[key] is not None)
+            vals = [[r[key] for r in mr if r["period"] == p] for p in per]
+            if any(v is None for vv in vals for v in vv):
+                continue
+            if nseed > 1:
+                for i, vv in enumerate(vals):
+                    ax.plot(xs[i] + _jitter(len(vv), 0.08), vv, mk, ms=2.6, mfc="white", mec=col, mew=0.7, zorder=2)
+            ax.plot(xs, [np.mean(v) for v in vals], ls=ls, marker=mk, color=col, ms=5, label=lab, zorder=3)
+        ax.set_xticks(xs)
+        ax.set_xticklabels([str(p) for p in per])
+        ax.set_xlabel("period id")
+        ax.set_title(f"{m}: final model", loc="left")
+    # the axis spans the data (at least 0.1 wide, clipped to 0..1) so the gaps between the
+    # three levels stay visible; the note states the scale
+    lo, hi = float(min(seen)), float(max(seen))
+    pad = max(0.02, 0.15 * (hi - lo), (0.1 - (hi - lo)) / 2)
+    y0, y1 = max(0.0, lo - pad), min(1.0, hi + pad)
+    for ax in axes[0]:
+        ax.set_ylim(y0, y1)
+    axes[0][0].set_ylabel("macro-F1 (0 to 1 scale)")
+    axes[0][0].legend(frameon=False, loc="best", fontsize=7)
+    fig.tight_layout()
+    _note(fig, f"{'Small hollow markers: single runs. ' if nseed > 1 else ''}Lines: mean over {nseed} "
+               f"run{'s' if nseed > 1 else ''}. The axis spans the data, from {y0:.2f} to {y1:.2f} of the "
+               f"0 to 1 scale. Descriptive.")
+    return fig
+
+
+def figure_comparison(result):
+    """The paired lead at the fixed rule and after refit, one row per term and comparison,
+    with its marginal 95% t interval (two or more seeds) and the margin drawn."""
+    plt = _plt()
+    rows = result["contrasts"]
+    margin = result["settings"]["margin"]
+    if not rows:
+        fig, ax = plt.subplots(figsize=(4.2, 1.2))
+        ax.axis("off")
+        ax.text(0.5, 0.5, "No comparison: give a second method or per-period reference scores.", ha="center",
+                va="center", color=MUTED)
+        return fig
+    style = {"at the rule": (INK, "D"), "after own-period refit": (A_COL, "s"), "after one shared refit": (GREY, "o")}
+    fig, ax = plt.subplots(figsize=(5.4, 0.5 * len(rows) + 1.3))
+    ys = np.arange(len(rows))[::-1].astype(float)
+    for y, r in zip(ys, rows):
+        col, mk = style.get(r["term"], (INK, "D"))
+        if r["ci95_low"] is not None:
+            ax.plot([r["ci95_low"], r["ci95_high"]], [y, y], color=col, lw=1.3, zorder=2)
+        ax.plot(r["mean"], y, mk, color=col, ms=6.5, mec="white", mew=0.6, zorder=4)
+        vals = [float(v) for v in str(r.get("values") or "").split()]
+        if len(vals) > 1:
+            ax.plot(vals, y + 0.22 + _jitter(len(vals), 0.05), "s", ms=2.6, mfc="white", mec=col, mew=0.7, zorder=3)
+    ax.axvline(0, color=MUTED, lw=0.7, ls=":")
+    for s in (-margin, margin):
+        ax.axvline(s, color=MUTED, lw=0.7, ls="--")
+    ax.set_ylim(-0.6, len(rows) - 0.4)
+    lo, hi = ax.get_xlim()
+    ax.set_xlim(min(lo, -2 * margin), max(hi, 2 * margin))
+    ax.set_yticks(ys)
+    ax.set_yticklabels([f"{r['comparison']}\n{r['term']}" for r in rows], fontsize=7.5)
+    ax.set_xlabel("lead in macro-F1 (difference on the 0 to 1 scale)")
+    n = max(r["n"] for r in rows)
+    ax.set_title("paired leads, positive where the first named side is ahead", loc="left", fontsize=8)
+    fig.tight_layout()
+    note = ("Diamonds, squares and circles: means. Bars: marginal 95% t intervals over seeds. Small hollow squares: "
+            "single runs." if n > 1 else "One run: no interval is computed.")
+    _note(fig, note + f" Dashed lines: margin {margin:g} either side of zero (dotted). Descriptive unless a "
+                      "planned family is named.")
+    return fig
+
+
+def build(result):
+    """{name: matplotlib Figure} for the three default figures."""
+    return {"decomposition": figure_decomposition(result), "profile": figure_profile(result),
+            "comparison": figure_comparison(result)}
+
+
+def save_figures(result, outdir, formats=("pdf", "svg")):
+    """Write fig_<name>.<fmt> for the three figures; returns {name: {fmt: path}}."""
+    plt = _plt()
+    outdir = Path(outdir)
+    outdir.mkdir(parents=True, exist_ok=True)
+    paths = {}
+    for name, fig in build(result).items():
+        paths[name] = {}
+        for fmt in formats:
+            p = outdir / f"fig_{name}.{fmt}"
+            fig.savefig(p, format=fmt, bbox_inches="tight", metadata={"Date": None} if fmt == "svg" else
+                        {"CreationDate": None})
+            paths[name][fmt] = p
+        plt.close(fig)
+    return paths
+
+
+def svg_strings(result):
+    """{name: inline SVG markup} for the HTML report (no XML prolog, no script)."""
+    plt = _plt()
+    out = {}
+    for name, fig in build(result).items():
+        buf = io.StringIO()
+        fig.savefig(buf, format="svg", bbox_inches="tight", metadata={"Date": None})
+        plt.close(fig)
+        s = buf.getvalue()
+        out[name] = s[s.index("<svg"):]
+    return out

refitgap/records.py

@@ -0,0 +1,38 @@
+"""Record alignment for scores made elsewhere: evaluate() pairs rows by position, so put
+every model's rows in the labels' order first."""
+import numpy as np
+
+
+def align_to(reference_ids, ids, *arrays):
+    """(reference_ids, ids, *arrays) -> arrays with rows reordered to reference_ids. Duplicate
+    ids or a reference id missing from ids raise; extra ids are dropped."""
+    ref = list(reference_ids)
+    cur = list(ids)
+    if len(set(ref)) != len(ref):
+        raise ValueError("reference_ids contain duplicates")
+    if len(set(cur)) != len(cur):
+        raise ValueError("ids contain duplicates")
+    pos = {r: i for i, r in enumerate(cur)}
+    missing = [r for r in ref if r not in pos]
+    if missing:
+        raise ValueError(f"{len(missing)} reference ids have no row (e.g. {missing[0]!r})")
+    idx = np.array([pos[r] for r in ref], dtype=np.int64)
+    out = []
+    for a in arrays:
+        a = np.asarray(a)
+        if a.shape[0] != len(cur):
+            raise ValueError(f"an array has {a.shape[0]} rows for {len(cur)} ids")
+        out.append(a[idx])
+    return out[0] if len(out) == 1 else tuple(out)
+
+
+def drop_unlabelled(ids, Y, *arrays):
+    """Records whose label row has any missing entry (NaN) removed, from `ids`, `Y` and every
+    array in `arrays`. evaluate() refuses labels other than 0/1, so a missing label must be
+    removed before evaluation, and the same records removed for every model compared on that
+    period (align every model to the returned ids). Returns (ids, Y as int, *arrays)."""
+    Y = np.asarray(Y, dtype=float)
+    keep = ~np.isnan(Y).any(axis=1)
+    ids = [i for i, k in zip(ids, keep) if k]
+    rest = [np.asarray(a)[keep] for a in arrays]
+    return (ids, Y[keep].astype(int), *rest)