refitgap 0.1.0__py3-none-any.whl

refitgap/cli.py

@@ -0,0 +1,173 @@
+"""Command line: refitgap {evaluate, report, example} and --version.
+
+    refitgap evaluate --npz run_s0.npz run_s1.npz --seeds 0 1 --out OUT
+    refitgap report   (same inputs) --out OUT        # adds figures and report.html
+    refitgap example  [--out OUT]"""
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .analysis import analyse, run_from_arrays, run_from_npz, write_tables
+from .inference import MARGIN
+
+
+def _inputs(p):
+    g = p.add_argument_group("inputs (method a, and optionally method b paired by seed)")
+    g.add_argument("--npz", nargs="+", default=[], help="stored-run files (*_scores.npz) of method a")
+    g.add_argument("--npz-b", nargs="+", default=[], help="stored-run files of method b")
+    g.add_argument("--arrays", nargs="+", default=[], help="per-period arrays files (npz) of method a")
+    g.add_argument("--arrays-b", nargs="+", default=[], help="per-period arrays files of method b")
+    g.add_argument("--seeds", nargs="+", type=int, help="run seed of each file, in file order (stored-run files; "
+                                                         "arrays files may carry their own `seed`)")
+    g.add_argument("--names", nargs="+", default=["a", "b"], metavar="NAME",
+                   help="method names in the outputs: one for a single method, two for a paired comparison")
+    s = p.add_argument_group("settings")
+    s.add_argument("--periods", nargs="+", type=int, help="period ids to evaluate (stored-run files; default all)")
+    s.add_argument("--mean-over", nargs="+", type=int, help="period ids averaged in summaries and contrasts "
+                                                             "(default all evaluated)")
+    s.add_argument("--shared-periods", nargs="+", type=int, help="period ids of the shared rule (default all)")
+    s.add_argument("--shared-weights", nargs="+", type=float, help="weights of the shared periods (default uniform)")
+    s.add_argument("--splits", type=int, default=200, help="number of partitions (default 200, as in the paper)")
+    s.add_argument("--rule", default="inherited", help="fixed rule of the decomposition (default inherited)")
+    s.add_argument("--validation", action="store_true", help="stored-run files: add the rule fitted on the final "
+                                                             "model's validation scores as rule 'validation'")
+    s.add_argument("--u-convention", choices=["full", "half"], default="full",
+                   help="fixed-rule level used in the decomposition and contrasts: full = whole period (default; "
+                        "B = A + dR of the paper), half = held-out halves")
+    s.add_argument("--no-reference", action="store_true", help="ignore per-period reference scores")
+    s.add_argument("--margin", type=float, default=MARGIN,
+                   help="margin of the class rules (default 0.005): TOST, the class words and the non-inferiority "
+                        "and inferiority tests all use it, and the figures draw it")
+    s.add_argument("--family", help="name of a family the caller specified in advance; without it every row is "
+                                    "descriptive")
+    s.add_argument("--workers", type=int, default=1, help="processes for the evaluations (default 1)")
+    p.add_argument("--out", required=True, help="output directory")
+    p.add_argument("--quiet", action="store_true", help="print only the output paths")
+
+
+def load_runs(args):
+    if (args.npz or args.npz_b) and (args.arrays or args.arrays_b):
+        raise ValueError("use stored-run files or arrays files, not both")
+    ref = not args.no_reference
+    if args.npz:
+        if not args.seeds or len(args.seeds) != len(args.npz):
+            raise ValueError("--seeds needs one seed per stored-run file of method a (the same seeds pair method b)")
+        if args.npz_b and len(args.npz_b) != len(args.npz):
+            raise ValueError("--npz-b needs one file per seed, in the order of --seeds")
+        a = [run_from_npz(f, s, args.periods, args.validation, ref) for f, s in zip(args.npz, args.seeds)]
+        b = [run_from_npz(f, s, args.periods, args.validation, ref) for f, s in zip(args.npz_b, args.seeds)]
+        return a, b
+    if args.arrays:
+        if args.seeds and len(args.seeds) != len(args.arrays):
+            raise ValueError("--seeds needs one seed per arrays file of method a")
+        sa = args.seeds or [None] * len(args.arrays)
+        a = [run_from_arrays(f, s, ref) for f, s in zip(args.arrays, sa)]
+        if args.seeds and args.arrays_b and len(args.arrays_b) != len(args.seeds):
+            raise ValueError("--arrays-b needs one file per seed, in the order of --seeds")
+        sb = args.seeds or [None] * len(args.arrays_b)
+        b = [run_from_arrays(f, s, ref) for f, s in zip(args.arrays_b, sb)]
+        return a, b
+    raise ValueError("give --npz or --arrays")
+
+
+def run_analysis(args):
+    a, b = load_runs(args)
+    return analyse(a, b, rule=args.rule, mean_over=args.mean_over, splits=args.splits,
+                   shared_periods=args.shared_periods, shared_weights=args.shared_weights,
+                   u_convention=args.u_convention, margin=args.margin, family=args.family, names=_names(args),
+                   workers=args.workers)
+
+
+def summary_lines(result):
+    conv = result["settings"]["U_convention"]
+    out = []
+    means = [r for r in result["decomposition"] if str(r["period"]).startswith("mean")]
+    if means:
+        out.append(f"decomposition against the per-period reference, rule '{result['settings']['rule']}', "
+                   f"U_{conv} (descriptive):")
+        for r in means:
+            out.append(f"  {r['method']} seed {r['seed']} {r['period']}: B {r['B']:+.4f} = A {r['A']:+.4f} "
+                       f"+ dR {r['dR']:+.4f}")
+    if result["contrasts"]:
+        out.append("paired contrasts:")
+        for r in result["contrasts"]:
+            ci = (f" [{r['ci95_low']:+.4f}, {r['ci95_high']:+.4f}] (marginal 95% t)" if r["ci95_low"] is not None
+                  else " (one run, no interval)")
+            cls = f"  class {r['class']}" if r["class"] else ""
+            out.append(f"  {r['comparison']}, {r['term']}: {r['mean']:+.4f}{ci}  n {r['n']}  status {r['status']}{cls}")
+    out.extend(result["interpretation"])
+    return out
+
+
+def _write(result, args, report):
+    out = Path(args.out)
+    paths = write_tables(result, out)
+    if report:
+        from .report import write_report
+        try:
+            from .plots import save_figures, svg_strings
+            figs = save_figures(result, out)
+            svgs = svg_strings(result)
+        except ImportError as e:
+            print(f"note: {e}; the report is written without figures", file=sys.stderr)
+            figs, svgs = {}, {}
+        paths["report"] = write_report(result, out / "report.html", svgs)
+        for name, d in figs.items():
+            for fmt, p in d.items():
+                paths[f"fig_{name}.{fmt}"] = p
+    return paths
+
+
+def _names(args):
+    """One or two method names; a single name is padded so the paired path never sees it."""
+    n = list(args.names)
+    if len(n) > 2:
+        raise SystemExit("refitgap: --names takes one or two names")
+    if len(n) == 1:
+        n.append("b")
+    if len(n) == 2 and (args.npz_b or args.arrays_b) and n[0] == n[1]:
+        raise SystemExit("refitgap: the two method names must differ")
+    return tuple(n)
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="refitgap",
+                                description="Evaluation of thresholded forgetting from stored per-period scores.")
+    p.add_argument("--version", action="version", version=f"refitgap {__version__}")
+    sub = p.add_subparsers(dest="cmd")
+    pe = sub.add_parser("evaluate", help="levels, decomposition and contrasts as CSV and JSON")
+    _inputs(pe)
+    pr = sub.add_parser("report", help="evaluate, plus three figures (PDF, SVG) and a self-contained HTML report")
+    _inputs(pr)
+    px = sub.add_parser("example", help="the tiny example, worked by hand in API.txt")
+    px.add_argument("--out", help="also write its tables, figures and report here")
+    args = p.parse_args(argv)
+    if args.cmd is None:
+        p.print_help()
+        return 2
+    if args.cmd == "example":
+        from .tiny import lines, tiny_result
+        result = tiny_result()
+        for line in lines(result):
+            print(line)
+        if args.out:
+            args.quiet = True
+            for k, v in _write(result, args, report=True).items():
+                print(f"wrote {v}")
+        return 0
+    try:
+        result = run_analysis(args)
+    except (ValueError, KeyError) as e:
+        print(f"refitgap: error: {e}", file=sys.stderr)
+        return 2
+    if not args.quiet:
+        for line in summary_lines(result):
+            print(line)
+    for k, v in _write(result, args, report=args.cmd == "report").items():
+        print(f"wrote {v}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

refitgap/crossfit.py

@@ -0,0 +1,275 @@
+"""Cross-fitted levels of one model, and the decompositions built from them.
+
+Settings: sep (own-period rules, fitted per period), shared (one rule over `shared_periods`)
+and fixed (any threshold vector under `rules`, e.g. "inherited"). Partition b of period id j,
+seed s, is the b-th permutation of default_rng(7000 + 100*s + j); half 0 is the first n_j // 2
+records. Each rule is fitted on one half and scored on the other, both ways, and averaged.
+Every setting uses the same halves, so R = C + M and B = A + dR hold per partition."""
+import hashlib
+import numbers
+
+import numpy as np
+
+from .fitters import fit_exact, fit_grid, fit_shared_exact, macro_f1
+
+SPLIT_BASE = 7000                       # partition seed: 7000 + 100*seed + j
+SPLITS = 200                            # the paper's number of partitions
+IDENTITY_TOL = 1e-12
+
+
+MAX_PERIOD_ID = 100                     # period ids must lie in [0, MAX_PERIOD_ID); see check_partition_key
+
+
+def _is_int(x):
+    return isinstance(x, (numbers.Integral, np.integer)) and not isinstance(x, (bool, np.bool_))
+
+
+def check_partition_key(seed, j=None, base=SPLIT_BASE):
+    """(seed, period id) -> ValueError unless seed >= 0 and 0 <= id < 100, the range in
+    which base + 100*seed + j keys partitions uniquely."""
+    if not _is_int(seed) or seed < 0:
+        raise ValueError(f"seed must be a non-negative integer (it keys the partitions through "
+                         f"{base} + 100*seed + j), got {seed!r}")
+    if j is not None and (not _is_int(j) or not 0 <= j < MAX_PERIOD_ID):
+        raise ValueError(f"period id must be an integer in [0, {MAX_PERIOD_ID}), got {j!r}: the partition seed "
+                         f"{base} + 100*seed + j keys (seed, period id) pairs uniquely only for such ids "
+                         f"(period id 100 of seed s would repeat period id 0 of seed s + 1)")
+    if not _is_int(base) or base < 0:
+        raise ValueError(f"split base must be a non-negative integer, got {base!r}")
+
+
+def partitions(n, seed, j, splits=SPLITS, base=SPLIT_BASE):
+    """The partitions of period id j: `splits` successive permutations of range(n)
+    from default_rng(base + 100*seed + j). Half 0 is perm[:n // 2].
+    seed: a non-negative integer; j: an integer in [0, 100) (check_partition_key)."""
+    check_partition_key(seed, j, base)
+    rng = np.random.default_rng(base + 100 * seed + j)
+    return [rng.permutation(n) for _ in range(splits)]
+
+
+def fit_validation_rule(Pval, Yval):
+    """Validation scores and labels -> grid rule (19 points 0.05..0.95), as theta_1 was
+    fitted. Assumes probability-scale scores; for logits etc. pass your own rule under `rules`."""
+    return fit_grid(np.asarray(Pval, np.float64), np.asarray(Yval).astype(int)).astype(np.float64)
+
+
+def _check_inputs(Ps, Ys, period_ids):
+    if len(Ps) != len(Ys) or len(Ps) == 0:
+        raise ValueError("Ps and Ys must be non-empty lists of the same length (one entry per period)")
+    K = np.asarray(Ps[0]).shape[1]
+    for j, (P, Y) in enumerate(zip(Ps, Ys)):
+        P, Y = np.asarray(P), np.asarray(Y)
+        if P.ndim != 2 or P.shape != Y.shape or P.shape[1] != K:
+            raise ValueError(f"period {period_ids[j]}: scores and labels must be (n_j, K) with one K for all periods")
+        if P.shape[0] < 2:
+            raise ValueError(f"period {period_ids[j]}: needs at least 2 records to split in half")
+        if not np.all(np.isfinite(P)):
+            raise ValueError(f"period {period_ids[j]}: non-finite scores")
+        if not np.isin(Y, (0, 1)).all():
+            raise ValueError(f"period {period_ids[j]}: labels must be 0/1")
+    if len(set(period_ids)) != len(period_ids):
+        raise ValueError("period ids must be distinct")
+
+
+def _check_weights(w, n, what):
+    """Finite, non-negative, one per period, not all zero. Returned as a tuple of floats."""
+    w = np.asarray(w, float)
+    if w.shape != (n,) or not np.all(np.isfinite(w)) or np.any(w < 0) or w.sum() <= 0:
+        raise ValueError(f"{what}: one finite non-negative weight per period, not all zero")
+    return tuple(float(x) for x in w)
+
+
+def array_hash(a, dtype):
+    """sha16 of an array's shape and its bytes in `dtype` (C order). Equal hashes mean the same
+    values in the same record order, which is what pairs two evaluations' partitions."""
+    a = np.ascontiguousarray(a, dtype=dtype)
+    h = hashlib.sha256(repr(a.shape).encode())
+    h.update(a.tobytes())
+    return h.hexdigest()[:16]
+
+
+class Evaluation:
+    """Per-partition levels of one model, arrays of shape (splits, 2, periods): sep, shared,
+    U_half[name]; U_full[name] per period. Also rules, label hashes and settings. Made by evaluate()."""
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
+    def _cols(self, periods):
+        periods = self.period_ids if periods is None else list(periods)
+        missing = [p for p in periods if p not in self.period_ids]
+        if missing:
+            raise KeyError(f"period ids {missing} were not evaluated")
+        return periods, [self.period_ids.index(p) for p in periods]
+
+    def per_period(self):
+        """Per-period levels: mean over partitions of 0.5 * (direction 0 + direction 1)."""
+        hm = lambda X, c: float(np.mean(0.5 * (X[:, 0, c] + X[:, 1, c])))
+        out = {}
+        for c, j in enumerate(self.period_ids):
+            r = dict(V_sep=hm(self.sep, c),
+                     V_shared=(hm(self.shared, c) if self.single_model and j in self.shared_periods else None),
+                     U_half={k: hm(v, c) for k, v in self.U_half.items()},
+                     U_full={k: float(v[c]) for k, v in self.U_full.items()})
+            out[j] = r
+        return out
+
+    def summary(self, periods=None, weights=None):
+        """Periods and weights -> mean levels and R = C + M (period-weighted per partition, then
+        averaged). Weights default to uniform, or to shared_weights on the shared periods."""
+        periods, cols = self._cols(periods)
+        if weights is None and self.shared_weights is not None:
+            if periods != list(self.shared_periods):
+                raise ValueError("the shared rule was fitted with shared_weights; pass `weights` for periods "
+                                 f"{periods} (they differ from the shared periods {self.shared_periods})")
+            weights = self.shared_weights
+        if weights is None:
+            agg = lambda X: np.mean(0.5 * (X[:, 0, cols] + X[:, 1, cols]), axis=1)
+            aggf = lambda v: float(np.mean([v[c] for c in cols]))
+        else:
+            w = np.asarray(_check_weights(weights, len(cols), "weights"))
+            w = w / w.sum()
+            agg = lambda X: (0.5 * (X[:, 0, cols] + X[:, 1, cols])) @ w
+            aggf = lambda v: float(sum(wi * v[c] for wi, c in zip(w, cols)))
+        vsep_b = agg(self.sep)
+        out = dict(periods=periods, weights=(None if weights is None else w.tolist()), V_sep=float(vsep_b.mean()))
+        has_sh = self.single_model and all(p in self.shared_periods for p in periods)
+        if has_sh:
+            vsh_b = agg(self.shared)
+            out["V_shared"] = float(vsh_b.mean())
+            out["C"] = float(np.mean(vsep_b - vsh_b))
+        else:
+            out["V_shared"] = out["C"] = None
+        out["rules"] = {}
+        for name, Uh in self.U_half.items():
+            u_b = agg(Uh)
+            r = dict(U_half=float(u_b.mean()), U_full=aggf(self.U_full[name]),
+                     R=float(np.mean(vsep_b - u_b)))
+            r["R_full"] = out["V_sep"] - r["U_full"]
+            if has_sh:
+                r["M"] = float(np.mean(vsh_b - u_b))
+                r["identity_residual"] = abs(out["C"] + r["M"] - r["R"])
+                if r["identity_residual"] > IDENTITY_TOL:
+                    raise AssertionError(f"C + M != R for rule {name}: {r['identity_residual']:.3g}")
+            out["rules"][name] = r
+        return out
+
+
+def evaluate(Ps, Ys, seed, period_ids=None, rules=None, shared_periods=None, shared_weights=None,
+             splits=SPLITS, split_base=SPLIT_BASE, single_model=True):
+    """Per-period scores Ps and 0/1 labels Ys, (n_j, K) each -> Evaluation.
+
+    seed and period_ids (ints in [0, 100)) key the partitions: two models compared on the same
+    records need the same of both. rules: {name: (K,) thresholds}. shared_periods and
+    shared_weights: where the shared rule is fitted and how periods weigh (default all, uniform).
+    splits: partitions per period (200 in the paper). single_model=False for a per-period
+    reference (P_j_j on period j): no shared rule is fitted for it."""
+    N = len(Ps)
+    period_ids = list(range(N)) if period_ids is None else list(period_ids)
+    check_partition_key(seed, None, split_base)
+    for p in period_ids:
+        check_partition_key(seed, p, split_base)
+    period_ids = [int(p) for p in period_ids]
+    _check_inputs(Ps, Ys, period_ids)
+    Ps = [np.asarray(P, np.float64) for P in Ps]
+    Ys = [np.asarray(Y).astype(int) for Y in Ys]
+    rules = {} if rules is None else {k: np.asarray(v, np.float64) for k, v in rules.items()}
+    for k, v in rules.items():
+        if v.shape != (Ps[0].shape[1],):
+            raise ValueError(f"rule {k}: need one threshold per label")
+        if not np.all(np.isfinite(v)):
+            raise ValueError(f"rule {k}: non-finite threshold")
+    shared_periods = list(period_ids) if shared_periods is None else [int(p) for p in shared_periods]
+    if [p for p in shared_periods if p not in period_ids] or len(set(shared_periods)) != len(shared_periods):
+        raise ValueError("shared_periods: distinct ids of evaluated periods")
+    S = [period_ids.index(p) for p in shared_periods]
+    if shared_weights is not None:
+        shared_weights = _check_weights(shared_weights, len(S), "shared_weights")
+    perms = [partitions(len(Ys[c]), seed, period_ids[c], splits, split_base) for c in range(N)]
+    nan = np.full((splits, 2, N), np.nan)
+    sep, sh = nan.copy(), nan.copy()
+    U = {k: nan.copy() for k in rules}
+    for b in range(splits):
+        for d in (0, 1):
+            fit, ev = [], []
+            for c in range(N):
+                h = len(Ys[c]) // 2
+                A, B = perms[c][b][:h], perms[c][b][h:]
+                fit.append(A if d == 0 else B); ev.append(B if d == 0 else A)
+            for c in range(N):
+                Pe, Ye = Ps[c][ev[c]], Ys[c][ev[c]]
+                sep[b, d, c] = macro_f1(Pe, Ye, fit_exact(Ps[c][fit[c]], Ys[c][fit[c]]))
+                for k, th in rules.items():
+                    U[k][b, d, c] = macro_f1(Pe, Ye, th)
+            if not single_model:
+                continue                    # no shared rule across different models' scores
+            th_sh = fit_shared_exact([Ps[c][fit[c]] for c in S], [Ys[c][fit[c]] for c in S], shared_weights)
+            for c in S:
+                sh[b, d, c] = macro_f1(Ps[c][ev[c]], Ys[c][ev[c]], th_sh)
+    U_full = {k: np.array([macro_f1(Ps[c], Ys[c], th) for c in range(N)]) for k, th in rules.items()}
+    # pairing checks for decompose: label counts (readable) and a hash of each label matrix in
+    # record order (a permuted or different record set gives a different hash); the fixed rules
+    # are kept, with their hashes, so decompose can confirm both models were scored at one rule
+    sig = [(int(len(Y)), int(Y.sum())) for Y in Ys]
+    label_hash = [array_hash(Y, np.int8) for Y in Ys]
+    rule_hash = {k: array_hash(v, np.float64) for k, v in rules.items()}
+    return Evaluation(seed=seed, splits=splits, split_base=split_base, period_ids=period_ids,
+                      shared_periods=shared_periods, shared_weights=shared_weights, sep=sep, shared=sh,
+                      U_half=U, U_full=U_full, label_signature=sig, label_hash=label_hash,
+                      rules={k: v.copy() for k, v in rules.items()}, rule_hash=rule_hash,
+                      single_model=bool(single_model))
+
+
+def decompose(ref, final, rule, periods=None, weights=None, halves=False, same_rule=True):
+    """Two Evaluations on the same records -> dict with B = A + dR at `rule`, per period and mean.
+
+    B = U_ref - U_final (the deficit), A = V_sep,ref - V_sep,final, dR = R_final - R_ref with
+    R_x = V_sep,x - U_x; for two single models also dR = dC + dM. B and dR are also returned
+    as D and R (the earlier names). U on the full period unless
+    halves=True. Raises unless seed, period ids, splits, labels and the rule match;
+    same_rule=False allows a model-specific rule under one name."""
+    if ref.seed != final.seed or ref.period_ids != final.period_ids or ref.splits != final.splits \
+            or ref.split_base != final.split_base:
+        raise ValueError("ref and final must share seed, period ids and splits (same partitions)")
+    if ref.label_signature != final.label_signature or ref.label_hash != final.label_hash:
+        bad = [p for p, a, b in zip(ref.period_ids, ref.label_hash, final.label_hash) if a != b]
+        raise ValueError(f"ref and final were evaluated on different labels or record order (periods {bad})")
+    if rule not in ref.rules or rule not in final.rules:
+        raise KeyError(f"rule {rule!r} was not evaluated for both models")
+    matched = ref.rule_hash[rule] == final.rule_hash[rule]
+    if same_rule and not matched:
+        raise ValueError(f"rule {rule!r} is a different threshold vector in ref and final; "
+                         "pass same_rule=False only for a rule that is model-specific by design")
+    if weights is None and ref.shared_weights != final.shared_weights:
+        raise ValueError("ref and final were evaluated with different shared_weights "
+                         f"({ref.shared_weights} against {final.shared_weights}), so their default period "
+                         "weightings differ and D, A and R would mix two weightings; pass `weights` explicitly")
+    sr, sf = ref.summary(periods, weights), final.summary(periods, weights)
+    key = "U_half" if halves else "U_full"
+    ur, uf = sr["rules"][rule][key], sf["rules"][rule][key]
+    out = dict(periods=sr["periods"], rule=rule, same_rule=bool(matched), U=key, U_ref=ur, U_final=uf,
+               V_ref=sr["V_sep"], V_final=sf["V_sep"],
+               D=ur - uf, A=sr["V_sep"] - sf["V_sep"], R_ref=sr["V_sep"] - ur, R_final=sf["V_sep"] - uf)
+    out["R"] = out["R_final"] - out["R_ref"]
+    out["residual_D_A_R"] = abs(out["D"] - (out["A"] + out["R"]))
+    out["B"], out["dR"] = out["D"], out["R"]      # the paper's names; D and R kept for compatibility
+    if not (ref.single_model and final.single_model):
+        out["shared_terms"] = "omitted: the reference holds different models on different periods"
+    elif ref.shared_periods != final.shared_periods or ref.shared_weights != final.shared_weights:
+        out["shared_terms"] = "omitted: the two shared rules were fitted on different periods or weights"
+    elif sr["C"] is None or sf["C"] is None:
+        out["shared_terms"] = "omitted: the periods are not all in the shared set"
+    else:
+        out["shared_terms"] = "reported"
+        out["C_ref"], out["C_final"] = sr["C"], sf["C"]
+        out["M_ref"] = sr["V_shared"] - ur
+        out["M_final"] = sf["V_shared"] - uf
+        out["dC"] = out["C_final"] - out["C_ref"]
+        out["dM"] = out["M_final"] - out["M_ref"]
+        out["A_shared"] = sr["V_shared"] - sf["V_shared"]
+        out["residual_R_C_M"] = abs(out["R"] - (out["dC"] + out["dM"]))
+        out["residual_D_Ash_M"] = abs(out["D"] - (out["A_shared"] + out["dM"]))
+    for k in [k for k in out if k.startswith("residual")]:
+        if out[k] > IDENTITY_TOL:
+            raise AssertionError(f"{k} = {out[k]:.3g}")
+    return out

refitgap/example.py

@@ -0,0 +1,96 @@
+"""Synthetic two-learner data for the tests: four periods, four labels, six seeds per method.
+
+make_seed(s) builds one seed's labels and scores; main() prints a decomposition on them. The
+forgetful learner loses separation on early periods and its mismatch grows; the
+other loses less. The built-in example to run is `refitgap example` (src/refitgap/tiny.py)."""
+import numpy as np
+
+from . import crossfit, inference
+
+N_PERIODS, K, N_REC = 4, 4, 400
+SEEDS = range(6)
+SPLITS = 50          # the paper uses 200; 50 keeps the example quick
+
+
+def synth_period(rng, n, prev):
+    return (rng.random((n, K)) < prev).astype(int)
+
+
+def scores(rng, Y, sep, offset):
+    z = sep * Y + rng.normal(0, 1, Y.shape) + offset
+    return 1 / (1 + np.exp(-z))
+
+
+def make_seed(seed):
+    """Labels per period, the reference model of each period (s_j on period j), the final
+    model of two learners, and a validation split from period 0 that fits theta_1."""
+    rng = np.random.default_rng(seed)
+    prev = np.array([0.10, 0.20, 0.30, 0.15])
+    Ys = [synth_period(rng, N_REC, prev) for _ in range(N_PERIODS)]
+    Yval = synth_period(rng, 300, prev)
+    ref = [scores(rng, Ys[j], 2.5, -1.5) for j in range(N_PERIODS)]          # each period's own model
+    Pval_first = scores(rng, Yval, 2.5, -1.5)
+    # final models: separation lost on early periods, period-specific offset
+    sequential = [scores(rng, Ys[j], 2.5 - 0.30 * (N_PERIODS - 1 - j), -1.5 + 0.5 * (N_PERIODS - 1 - j))
+             for j in range(N_PERIODS)]
+    replay = [scores(rng, Ys[j], 2.5 - 0.10 * (N_PERIODS - 1 - j), -1.5 + 0.15 * (N_PERIODS - 1 - j))
+              for j in range(N_PERIODS)]
+    return Ys, Yval, Pval_first, ref, sequential, replay
+
+
+def main():
+    hist = [0, 1, 2]                      # historical periods; period 3 is the current one
+    per_seed = {"sequential": {}, "replay": {}}
+    alone = {"sequential": {}, "replay": {}}
+    method = {}
+    for s in SEEDS:
+        Ys, Yval, Pval_first, ref, sequential, replay = make_seed(s)
+        theta_1 = crossfit.fit_validation_rule(Pval_first, Yval)              # the inherited thresholds
+        rules = {"inherited": theta_1}
+        # the reference holds a different model per period: no shared-rule terms against it
+        ev_ref = crossfit.evaluate(ref, Ys, s, rules=rules, splits=SPLITS, single_model=False)
+        ev = {}
+        for name, Ps in (("sequential", sequential), ("replay", replay)):
+            ev[name] = crossfit.evaluate(Ps, Ys, s, rules=rules, splits=SPLITS)
+            per_seed[name][s] = crossfit.decompose(ev_ref, ev[name], "inherited", periods=hist)
+            alone[name][s] = ev[name].summary(periods=hist)
+        # method contrast: both are single final models, so R splits into dC + dM
+        method[s] = crossfit.decompose(ev["replay"], ev["sequential"], "inherited", periods=hist)
+        if s == 0:
+            d = per_seed["sequential"][0]
+            print("seed 0, sequential learner, historical periods 0-2, inherited thresholds theta_1:")
+            print(f"  against the per-period reference models: B = {d['B']:+.4f} = A {d['A']:+.4f} + dR {d['dR']:+.4f}"
+                  f"   (residual {d['residual_D_A_R']:.1e})")
+            sm = alone["sequential"][0]
+            r = sm["rules"]["inherited"]
+            print(f"  final model alone: V_sep {sm['V_sep']:.4f}  V_shared {sm['V_shared']:.4f}  U(theta_1) "
+                  f"{r['U_half']:.4f}, so R {r['R']:+.4f} = C {sm['C']:+.4f} + M {r['M']:+.4f}")
+            m = method[0]
+            print(
+                f"  against the replay learner's final model: B = {m['B']:+.4f} = A {m['A']:+.4f} + dR {m['dR']:+.4f};"
+                f"  dR = dC {m['dC']:+.4f} + dM {m['dM']:+.4f}"
+            )
+    print("\nper-seed decomposition, sequential learner (historical mean; B, A, dR against the reference models,"
+          " C and M of the final model alone):")
+    for s in SEEDS:
+        d, sm = per_seed["sequential"][s], alone["sequential"][s]
+        print(f"  seed {s}: B {d['B']:+.4f}  A {d['A']:+.4f}  dR {d['dR']:+.4f}  C {sm['C']:+.4f}  "
+              f"M {sm['rules']['inherited']['M']:+.4f}")
+    # paired contrasts, sequential minus replay, one family of three read together
+    members = {}
+    for term in ("B", "A", "dR"):
+        _, members[term] = inference.paired_vector({s: per_seed["sequential"][s][term] for s in SEEDS},
+                                                   {s: per_seed["replay"][s][term] for s in SEEDS},
+                                                   seeds=SEEDS)
+    fam = inference.family(members)
+    print("\nsequential minus replay, paired over 6 seeds (Holm over B, A, dR; margin 0.005):")
+    for term, st in fam.items():
+        print(f"  {term}: mean {st['mean']:+.4f}  95% CI (marginal) [{st['ci95'][0]:+.4f}, {st['ci95'][1]:+.4f}]"
+              f"  Holm p_t {st['p_holm']:.3g}  class {st['class']}")
+    c = inference.paired_contrast({s: alone["replay"][s]["V_shared"] for s in SEEDS},
+                                  {s: alone["sequential"][s]["V_shared"] for s in SEEDS}, seeds=SEEDS)
+    print(f"\nshared-rule level, replay minus sequential: {c['mean']:+.4f} [{c['ci95'][0]:+.4f}, {c['ci95'][1]:+.4f}]"
+          f"  class {c['class']} (unadjusted)")
+    c = inference.paired_contrast({s: method[s]["dC"] for s in SEEDS}, {s: 0.0 for s in SEEDS}, seeds=SEEDS)
+    print(f"dC, sequential final against replay final (cost of one rule for periods 0-2): {c['mean']:+.4f} "
+          f"[{c['ci95'][0]:+.4f}, {c['ci95'][1]:+.4f}]  class {c['class']} (unadjusted)")

refitgap/fitters.py

@@ -0,0 +1,114 @@
+"""Threshold fitters and thresholded F1.
+
+f1_vec, macro_f1 (per-label F1 and its mean), fit_exact (best cutpoint per label over every
+observed score), fit_shared_exact (one vector for several periods), fit_grid (19 points
+0.05..0.95, how theta_1 was fitted). Positive means score >= theta; F1 is 0 when nothing is
+predicted and nothing is positive. Fitters refuse non-finite scores and non-0/1 labels."""
+import numpy as np
+
+GRID = np.linspace(0.05, 0.95, 19)
+
+
+def _check_PY(P, Y, what):
+    if not np.all(np.isfinite(P)):
+        raise ValueError(f"{what}: non-finite scores")
+    Y = np.asarray(Y)
+    if Y.dtype != bool and not np.isin(Y, (0, 1)).all():
+        raise ValueError(f"{what}: labels must be 0/1")
+
+
+def _above(top):
+    """A cutpoint above every score: top + 1e-9 (the ported rule) wherever that is larger than
+    top, else the next float above top. top + 1e-9 == top only for |top| above about 1.7e7, so
+    for scores in [0, 1] (and up to that magnitude) the result is top + 1e-9, unchanged."""
+    top = np.asarray(top, np.float64)
+    c = top + 1e-9
+    return np.where(c > top, c, np.nextafter(top, np.inf))
+
+
+def f1_vec(P, Y, theta):
+    """Per-label F1 at thresholds theta, zero_division = 0 (as sklearn's f1_score with zero_division=0)."""
+    pred = P >= theta
+    Yb = Y.astype(bool)
+    tp = (pred & Yb).sum(0)
+    den = pred.sum(0) + Yb.sum(0)
+    return np.where(den > 0, 2.0 * tp / np.maximum(den, 1), 0.0)
+
+
+def macro_f1(P, Y, theta):
+    return float(f1_vec(P, Y, theta).mean())
+
+
+def fit_exact(P, Y):
+    """Per label, the cutpoint over all score-induced cutpoints that maximises F1 on (P, Y).
+    Ties go to the highest maximiser (exactly: each F1 value is one correctly rounded division
+    of integers, so equal F1 values are equal floats). A label with no positive gets a cutpoint
+    above every score (predicts nothing; see _above)."""
+    P = np.asarray(P, dtype=np.float64)
+    _check_PY(P, Y, "fit_exact")
+    Yb = np.asarray(Y).astype(bool)
+    n, K = P.shape
+    o = np.argsort(-P, axis=0, kind="mergesort")          # per column, as the 1-d sort in decomposition.py
+    ss = np.take_along_axis(P, o, 0)
+    ys = np.take_along_axis(Yb, o, 0)
+    npos = ys.sum(0)
+    f1 = 2 * np.cumsum(ys, 0) / (np.arange(1, n + 1)[:, None] + npos[None, :])
+    ok = np.ones((n, K), bool)
+    ok[:-1] = ss[:-1] > ss[1:]                             # only cut between distinct scores
+    idx = np.argmax(np.where(ok, f1, -1.0), 0)
+    out = ss[idx, np.arange(K)].astype(np.float64)
+    none = npos == 0
+    out[none] = _above(ss[0, none])
+    return out
+
+
+def fit_shared_exact(Ps, Ys, w=None):
+    """Per-period scores and labels (and weights w) -> one (K,) threshold vector maximising
+    sum_j w_j F1_jk(t) per label over all periods' cutpoints. Ties go to the higher cutpoint, up
+    to float rounding of the sum; a label with no positive anywhere gets a cutpoint above every
+    score. One period gives fit_exact."""
+    J = len(Ps)
+    K = Ys[0].shape[1]
+    w = np.full(J, 1.0 / J) if w is None else np.asarray(w, float)
+    for j in range(J):
+        _check_PY(np.asarray(Ps[j], np.float64), Ys[j], f"fit_shared_exact, period {j}")
+    out = np.empty(K)
+    for k in range(K):
+        cols = [np.asarray(P[:, k], np.float64) for P in Ps]
+        ys = [np.asarray(Y[:, k]).astype(bool) for Y in Ys]
+        cand = np.unique(np.concatenate(cols))[::-1]          # distinct cutpoints, descending
+        npos = [int(y.sum()) for y in ys]
+        if sum(npos) == 0:
+            out[k] = float(_above(cand[0]))
+            continue
+        obj = np.zeros(len(cand))
+        for j in range(J):
+            if npos[j] == 0:
+                continue
+            a = np.sort(cols[j])
+            p = np.sort(cols[j][ys[j]])
+            cnt = len(a) - np.searchsorted(a, cand, "left")   # predicted positives at score >= t
+            tp = npos[j] - np.searchsorted(p, cand, "left")
+            obj += w[j] * (2 * tp / (cnt + npos[j]))
+        out[k] = float(cand[int(np.argmax(obj))])
+    return out
+
+
+def shared_objective(Ps, Ys, theta, w=None):
+    """sum_j w_j macroF1_j(theta), the quantity fit_shared_exact maximises label by label."""
+    J = len(Ps)
+    w = np.full(J, 1.0 / J) if w is None else np.asarray(w, float)
+    return float(sum(w[j] * macro_f1(np.asarray(Ps[j], np.float64), np.asarray(Ys[j]), theta) for j in range(J)))
+
+
+def fit_grid(P, Y, grid=GRID):
+    """The coarse grid fitter (first index of the best grid value). This is how the
+    inherited theta_1 and the same-scorer validation rule were fitted in the paper."""
+    P = np.asarray(P, np.float64)
+    _check_PY(P, Y, "fit_grid")
+    Y = np.asarray(Y).astype(int)
+    th = np.empty(Y.shape[1])
+    for k in range(Y.shape[1]):
+        f = [macro_f1(P[:, [k]], Y[:, [k]], np.array([g])) for g in grid]
+        th[k] = grid[int(np.argmax(f))]
+    return th