refitgap 0.1.0__tar.gz

refitgap-0.1.0/CITATION.cff

@@ -0,0 +1,34 @@
+cff-version: 1.2.0
+message: "If you use refitgap, please cite the paper below and this software."
+title: "refitgap: score-based evaluation of thresholded forgetting in continual multi-label classification"
+type: software
+version: 0.1.0
+license: MIT
+repository-code: "https://github.com/ihshaish/refitgap"
+authors:
+  - family-names: Ihshaish
+    given-names: Hisham
+    email: hisham.ihshaish@uwe.ac.uk
+    orcid: "https://orcid.org/0000-0001-5530-4894"
+    affiliation: "School of Computing and Creative Technologies, University of the West of England, Bristol, UK; Department of Computer Science, Birzeit University, Birzeit, Palestine"
+keywords:
+  - continual learning
+  - multi-label classification
+  - backward transfer
+  - attainable gap
+  - evaluation
+preferred-citation:
+  type: article
+  title: "Decomposing Backward Transfer in Continual Multi-Label Text Classification"
+  status: submitted
+  notes: "Under review."
+  authors:
+    - family-names: Ihshaish
+      given-names: Hisham
+      affiliation: "University of the West of England; Birzeit University"
+    - family-names: Mayhew
+      given-names: Peter
+      affiliation: "University of the West of England; GE Aerospace"
+    - family-names: Aydin
+      given-names: Mehmet
+      affiliation: "University of the West of England"

refitgap-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hisham Ihshaish
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

refitgap-0.1.0/MANIFEST.in

@@ -0,0 +1,6 @@
+include README.txt LICENSE CITATION.cff
+recursive-include src/refitgap *.py *.txt
+include tests/*.py
+prune paper
+prune examples
+global-exclude __pycache__ *.py[cod] .DS_Store

refitgap-0.1.0/PKG-INFO

@@ -0,0 +1,128 @@
+Metadata-Version: 2.4
+Name: refitgap
+Version: 0.1.0
+Summary: Split a continual learner's thresholded forgetting into what the scores lost and what the carried rule lost, from stored scores.
+Author-email: Hisham Ihshaish <hisham.ihshaish@uwe.ac.uk>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ihshaish/refitgap
+Project-URL: Repository, https://github.com/ihshaish/refitgap
+Project-URL: Issues, https://github.com/ihshaish/refitgap/issues
+Keywords: continual learning,multi-label classification,backward transfer,thresholds,evaluation
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Description-Content-Type: text/plain
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Provides-Extra: plots
+Requires-Dist: matplotlib>=3.5; extra == "plots"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: scikit-learn>=1.3; extra == "test"
+Dynamic: license-file
+
+refitgap
+========
+
+Train a multi-label classifier period by period, score the final model on an old period, and
+its macro-F1 there drops. Part of the drop sits in the scores, which now separate the labels
+less well, and part in the thresholds, carried from period one and now in the wrong place.
+refitgap says how much of each, from stored scores alone:
+
+    deficit B = attainable gap A + change in mismatch dR
+
+The attainable gap compares the two models after each refits its thresholds on equal terms
+(split-half cross-fitting, same partitions); the change in mismatch is what each loses by
+keeping the carried rule. Same trick for two methods: replay's lead over sequential training
+at the inherited thresholds, after both refit, and the change. This is the code of
+"Decomposing Backward Transfer in Continual Multi-Label Text Classification" (Ihshaish,
+Mayhew and Aydin, under review), with its plans, stored results and scripts.
+
+Install
+-------
+
+    pip install refitgap                 # Python 3.10+, numpy and scipy, nothing else
+    pip install "refitgap[plots,test]"   # matplotlib for figures, pytest etc. for the tests
+
+(or pip install . from the clone). The entry points below read files in this repository, so
+clone it too. On the command line, refitgap evaluate | report | example runs the same
+evaluator on stored arrays (refitgap --help; API.txt, "Command line").
+
+Reproduce Figure 2
+------------------
+
+    python examples/figure2/reproduce_figure2.py                # a few minutes on 4 cores
+    python examples/figure2/reproduce_figure2.py --from-levels  # about a second
+
+All five paired comparisons of Figure 2 (supplement Table C.1) over periods 2 to 7: lead at
+the inherited thresholds, after both refit, and the change. Paired t intervals over six
+DistilBERT pairs (from stored per-pair levels: their 30 score files, about 60 MB, do not ship),
+percentile bootstrap over ten BiGRU pairs (from raw scores in paper/results/, or stored levels
+with --from-levels). Every lead, mean and bound must match examples/figure2/expected.json to
+1e-10.
+
+    python examples/two_scorer/table1.py
+
+Table 1: two scorers with equal AUROC (6/11), average precision (5/18) and deficit (1/26),
+one all change in mismatch, the other all attainable gap. Exact fractions, no luck involved!
+
+Compare your own models
+-----------------------
+
+    python examples/your_models/compare_your_models.py [DATA_DIR] --a mine --b baseline
+
+DATA_DIR holds three kinds of CSV (examples/your_models/data/ is a tiny synthetic set):
+
+    labels.csv                    period, record_id, one 0/1 column per label
+    scores_<method>_seed<k>.csv   period, record_id, one score column per label (the final
+                                  model of <method>, trained under seed k)
+    thresholds.csv                method, seed, the carried rule (one threshold per label)
+
+Out come a's leads over b at the carried rule and after both refit, and the change (refitted
+lead minus carried lead), per seed and with paired t intervals. In Python: refitgap.compare()
+and method_contrast(); API.txt has the lot.
+
+Interface
+---------
+
+In: aligned scores and labels, the carried rule, and a seed for the common partitions.
+- Scores: one (records, labels) array per period and model, probabilities in [0, 1]; logits go
+  through a sigmoid first, or declare score_scale="other" with thresholds on the same scale.
+  Labels: 0/1, same shape.
+- Alignment: rows match by record id, columns by label name. A shape mismatch, a duplicate or
+  missing id, a different label set, or different labels for the two models raises. Nothing
+  is truncated or dropped quietly.
+- Ties: a score equal to its threshold is a positive. A refitted threshold is an observed
+  score; of two equally good cutpoints the higher wins.
+- A label with no positive in a period has F1 0 there and stays in the macro mean.
+- Partitions: partition b of period id j under seed s is the b-th permutation of numpy
+  default_rng(7000 + 100*s + j); first half fits, second scores, then the reverse. 200 in the
+  paper. Both models of a pair share them, so give both the same seed and period ids. Period
+  ids are integers in [0, 100); the paper's periods 2 to 7 are ids 1 to 6.
+- Out: carried level U, refitted level V, deficit, attainable gap and change in mismatch, per
+  period and averaged; for two methods, leads per seed with means and 95% intervals. Positive
+  means the first-named model scores higher. Numbers only: refitgap recommends nothing, and
+  the attainable gap describes the scores, not what the representation kept.
+
+Repository layout
+-----------------
+
+    src/refitgap/   the evaluator; src/refitgap/API.txt is the reference
+    examples/       Table 1, Figure 2, your own models, walkthrough.ipynb (seconds, the lot)
+    paper/          plans, scripts, stored results, checks; README.txt maps tables to scripts,
+                    DATA.txt the corpora (public, not redistributed, bar a 3200-record ASRS sample)
+    paper/threshold_policy_survey.txt   how 34 sampled studies report their thresholds
+    tests/          python -m pytest (fast, a minute or two); -m slow adds the full Figure 2
+                    recomputation from raw scores (minutes)
+
+MIT licence. Please cite the paper and the software (CITATION.cff has both). Have fun with it!

refitgap-0.1.0/README.txt

@@ -0,0 +1,95 @@
+refitgap
+========
+
+Train a multi-label classifier period by period, score the final model on an old period, and
+its macro-F1 there drops. Part of the drop sits in the scores, which now separate the labels
+less well, and part in the thresholds, carried from period one and now in the wrong place.
+refitgap says how much of each, from stored scores alone:
+
+    deficit B = attainable gap A + change in mismatch dR
+
+The attainable gap compares the two models after each refits its thresholds on equal terms
+(split-half cross-fitting, same partitions); the change in mismatch is what each loses by
+keeping the carried rule. Same trick for two methods: replay's lead over sequential training
+at the inherited thresholds, after both refit, and the change. This is the code of
+"Decomposing Backward Transfer in Continual Multi-Label Text Classification" (Ihshaish,
+Mayhew and Aydin, under review), with its plans, stored results and scripts.
+
+Install
+-------
+
+    pip install refitgap                 # Python 3.10+, numpy and scipy, nothing else
+    pip install "refitgap[plots,test]"   # matplotlib for figures, pytest etc. for the tests
+
+(or pip install . from the clone). The entry points below read files in this repository, so
+clone it too. On the command line, refitgap evaluate | report | example runs the same
+evaluator on stored arrays (refitgap --help; API.txt, "Command line").
+
+Reproduce Figure 2
+------------------
+
+    python examples/figure2/reproduce_figure2.py                # a few minutes on 4 cores
+    python examples/figure2/reproduce_figure2.py --from-levels  # about a second
+
+All five paired comparisons of Figure 2 (supplement Table C.1) over periods 2 to 7: lead at
+the inherited thresholds, after both refit, and the change. Paired t intervals over six
+DistilBERT pairs (from stored per-pair levels: their 30 score files, about 60 MB, do not ship),
+percentile bootstrap over ten BiGRU pairs (from raw scores in paper/results/, or stored levels
+with --from-levels). Every lead, mean and bound must match examples/figure2/expected.json to
+1e-10.
+
+    python examples/two_scorer/table1.py
+
+Table 1: two scorers with equal AUROC (6/11), average precision (5/18) and deficit (1/26),
+one all change in mismatch, the other all attainable gap. Exact fractions, no luck involved!
+
+Compare your own models
+-----------------------
+
+    python examples/your_models/compare_your_models.py [DATA_DIR] --a mine --b baseline
+
+DATA_DIR holds three kinds of CSV (examples/your_models/data/ is a tiny synthetic set):
+
+    labels.csv                    period, record_id, one 0/1 column per label
+    scores_<method>_seed<k>.csv   period, record_id, one score column per label (the final
+                                  model of <method>, trained under seed k)
+    thresholds.csv                method, seed, the carried rule (one threshold per label)
+
+Out come a's leads over b at the carried rule and after both refit, and the change (refitted
+lead minus carried lead), per seed and with paired t intervals. In Python: refitgap.compare()
+and method_contrast(); API.txt has the lot.
+
+Interface
+---------
+
+In: aligned scores and labels, the carried rule, and a seed for the common partitions.
+- Scores: one (records, labels) array per period and model, probabilities in [0, 1]; logits go
+  through a sigmoid first, or declare score_scale="other" with thresholds on the same scale.
+  Labels: 0/1, same shape.
+- Alignment: rows match by record id, columns by label name. A shape mismatch, a duplicate or
+  missing id, a different label set, or different labels for the two models raises. Nothing
+  is truncated or dropped quietly.
+- Ties: a score equal to its threshold is a positive. A refitted threshold is an observed
+  score; of two equally good cutpoints the higher wins.
+- A label with no positive in a period has F1 0 there and stays in the macro mean.
+- Partitions: partition b of period id j under seed s is the b-th permutation of numpy
+  default_rng(7000 + 100*s + j); first half fits, second scores, then the reverse. 200 in the
+  paper. Both models of a pair share them, so give both the same seed and period ids. Period
+  ids are integers in [0, 100); the paper's periods 2 to 7 are ids 1 to 6.
+- Out: carried level U, refitted level V, deficit, attainable gap and change in mismatch, per
+  period and averaged; for two methods, leads per seed with means and 95% intervals. Positive
+  means the first-named model scores higher. Numbers only: refitgap recommends nothing, and
+  the attainable gap describes the scores, not what the representation kept.
+
+Repository layout
+-----------------
+
+    src/refitgap/   the evaluator; src/refitgap/API.txt is the reference
+    examples/       Table 1, Figure 2, your own models, walkthrough.ipynb (seconds, the lot)
+    paper/          plans, scripts, stored results, checks; README.txt maps tables to scripts,
+                    DATA.txt the corpora (public, not redistributed, bar a 3200-record ASRS sample)
+    paper/threshold_policy_survey.txt   how 34 sampled studies report their thresholds
+    tests/          python -m pytest (fast, a minute or two); -m slow adds the full Figure 2
+                    recomputation from raw scores (minutes)
+
+MIT licence. Please cite the paper and the software (CITATION.cff has both). Have fun with it!

refitgap-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "refitgap"
+dynamic = ["version"]
+description = "Split a continual learner's thresholded forgetting into what the scores lost and what the carried rule lost, from stored scores."
+readme = { file = "README.txt", content-type = "text/plain" }
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Hisham Ihshaish", email = "hisham.ihshaish@uwe.ac.uk" }]
+keywords = ["continual learning", "multi-label classification", "backward transfer", "thresholds", "evaluation"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+]
+dependencies = ["numpy>=1.24", "scipy>=1.10"]
+
+[project.optional-dependencies]
+plots = ["matplotlib>=3.5"]
+test = ["pytest>=7.0", "scikit-learn>=1.3"]
+
+[project.urls]
+Homepage = "https://github.com/ihshaish/refitgap"
+Repository = "https://github.com/ihshaish/refitgap"
+Issues = "https://github.com/ihshaish/refitgap/issues"
+
+[project.scripts]
+refitgap = "refitgap.cli:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+refitgap = ["*.txt"]
+
+[tool.setuptools.dynamic]
+version = { attr = "refitgap.__version__" }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "-p no:cacheprovider -m 'not slow'"
+markers = ["slow: recomputes stored paper numbers from score files (minutes to hours; run with -m slow)"]

refitgap-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

refitgap-0.1.0/src/refitgap/API.txt

@@ -0,0 +1,150 @@
+refitgap API
+============
+
+Stored per-period scores and labels in; cross-fitted macro-F1 levels, the split of a
+thresholded deficit into attainable gap and change in mismatch, and seed-paired contrasts
+out. numpy and scipy only; figures want matplotlib (pip install "refitgap[plots]").
+
+The estimator
+-------------
+
+For period id j and seed s, partition b is the b-th permutation of
+numpy.random.default_rng(7000 + 100*s + j); half 0 is the first n_j // 2 records. A refit
+picks, per label, the F1-maximising cutpoint on one half and is scored on the other, both
+ways round; a level is the mean over the partitions (200 in the paper). With U a model's
+macro-F1 at the carried rule theta_1 and V its refitted (attainable) level,
+
+    B = U_ref - U_final = (V_ref - V_final) + (R_final - R_ref) = A + dR,   R = V - U
+
+U is taken on the whole period by default (U_full), or on the held-out halves (U_half).
+For one rule shared across periods, R = C + M with C = V_sep - V_shared and
+M = V_shared - U, exact per partition. A score equal to its threshold is a positive; a label
+with no positive and no prediction has F1 0 and stays in the macro mean. Period ids are ints
+in [0, 100) and seeds non-negative ints, so no two (seed, period) pairs share partitions.
+
+Outcome rules
+-------------
+
+A contrast is a per-seed vector (method a minus method b, paired by seed). At margin 0.005:
+- equivalent: TOST p < 0.05 (Holm-adjusted within a family);
+- material: t p < 0.05 and |mean| >= margin;
+- small non-zero: t p < 0.05 and |mean| < margin;
+- unresolved: anything else, or a material / small non-zero call that the sign-flip test
+  (exact up to 20 seeds, Monte Carlo above) does not back;
+- non-inferior / inferior (family_ni): one-sided t and sign-flip tests at -margin.
+Intervals are marginal 95 % t intervals over seeds. Rows say "descriptive" unless you name
+a family fixed in advance (--family NAME); the class words then describe your numbers under
+the paper's rules, nothing grander.
+
+Functions
+---------
+
+compare(scores_ref, scores_final, labels, rule, *, seed, splits=200, record_ids=None,
+        label_ids=None, period_ids=None, labels_final=None, rule_name="carried", halves=False,
+        score_scale=None, fixed_split=None, ...)
+    Two scorers on the same labelled records, (n, K) each (lists for several periods), and
+    the carried rule `rule`, one threshold per label -> dict(carried, refitted, components,
+    per_period, metadata, decomposition, evaluations). rule_name is the name the vector goes by in the
+    returned Evaluations (evaluations["ref"], evaluations["final"]), which is what
+    method_contrast takes. labels_final: the labels exported with the final scorer, checked
+    cell by cell against `labels`. Mismatched shapes, ids or labels raise; scores or rule
+    outside [0, 1] raise unless score_scale="other".
+method_contrast(evals_a, evals_b, rule, *, periods=None, same_rule=True, names=("a", "b"), ...)
+    {seed: Evaluation} for two methods, and `rule`, the name of a rule held by every
+    Evaluation -> terms lead_at_carried_rule (G_U), lead_after_refit (G_V) and change
+    (G_V - G_U, the change after refitting, as in the paper), each with per-seed values,
+    mean, t interval and class. "placement" is G_U - G_V, the negative of change, kept for
+    compatibility. same_rule=False lets each method carry its own vector under that name.
+evaluate(Ps, Ys, seed, period_ids=None, rules=None, shared_periods=None, splits=200, ...)
+    One model's per-period scores and labels, and rules={name: (K,) vector} -> Evaluation
+    (levels per partition, both directions, every period); .summary() and .per_period()
+    give the means.
+decompose(ref, final, rule, periods=None, halves=False, same_rule=True)
+    Two Evaluations on the same records and a rule name -> B, A, dR (and dC, dM) per period
+    and mean. B and dR are also returned as D and R, their earlier names.
+
+    Two methods from Python, end to end (synthetic scores, two periods, five seeds):
+
+    import numpy as np, refitgap as rg
+    rng = np.random.default_rng(0); Y = [(rng.random((300, 3)) < 0.3).astype(int) for _ in range(2)]
+    ev = lambda sep: {s: rg.evaluate([1 / (1 + np.exp(1 - sep * y - rng.normal(size=y.shape))) for y in Y], Y, s,
+                                     rules={"carried": np.full(3, 0.5)}) for s in range(5)}
+    mc = rg.method_contrast(ev(2.5), ev(1.5), "carried", names=("mine", "baseline"))
+    print({t: round(mc["terms"][t]["mean"], 4) for t in ("lead_at_carried_rule", "lead_after_refit", "change")})
+
+    prints {'lead_at_carried_rule': 0.1484, 'lead_after_refit': 0.1903, 'change': 0.0419}.
+fit_exact(P, Y), fit_grid(P, Y), fit_shared_exact(Ps, Ys, w=None), fit_validation_rule(Pval, Yval)
+    Scores and labels -> (K,) thresholds: best cutpoint per label; the 19-point grid
+    0.05..0.95 (how theta_1 was fitted); one vector for several periods; the grid on a
+    validation split.
+f1_vec(P, Y, theta), macro_f1(P, Y, theta)
+    Per-label F1 and its mean at theta.
+partitions(n, seed, j, splits=200)
+    The index permutations behind every level.
+paired_stats(x), paired_contrast(a, b), family(members), family_ni(members), holm(ps)
+    Per-seed vectors (or {seed: value} dicts) -> mean, intervals, t / TOST / sign-flip p and
+    classes, as above.
+analyse(runs_a, runs_b=None, rule="inherited", mean_over=None, u_convention="full", ...)
+    Runs (make_run, run_from_npz, run_from_arrays) -> levels, decomposition and contrast
+    tables; write_tables(result, dir) writes levels.csv, decomposition.csv, contrasts.csv and
+    result.json; plots.save_figures and report.write_report draw from the same tables.
+make_run(final, labels, seed, rules, reference=None, ...), save_arrays(path, run)
+    A run from arrays; saved as an arrays file (scores_<j>, labels_<j>, optional
+    reference_<j>, rule_<name>, seed, label_names).
+load_run(path), evaluate_npz(path, seed, ...)
+    Read or evaluate a stored run file (P_<i>_<j>: model i on period j; Y_<j>; thresholds).
+align_to(reference_ids, ids, *arrays), drop_unlabelled(ids, Y, *arrays)
+    Reorder rows to a reference id order (missing or duplicate ids raise); drop records with
+    a missing (NaN) label.
+identification_example()
+    The fourteen-record case in exact fractions: equal AUROC (6/11) and average precision
+    (5/18), equal deficits (1/26), one all mismatch, one all attainable gap.
+f1_bounds_from_counts(tp, fp, fn, tn)
+    Sharp range of the best single-threshold F1 behind one confusion table
+    (40, 10, 10, 40 gives [0.8, 10/11]). A bound, not a confidence interval.
+
+Command line
+------------
+
+    refitgap evaluate --npz run_s0.npz run_s1.npz --seeds 0 1 [--npz-b ...] --out DIR
+    refitgap evaluate --arrays a_s0.npz ... [--arrays-b b_s0.npz ...] --out DIR
+    refitgap report   (same inputs) --out DIR      # adds three figures and report.html
+    refitgap example  [--out DIR]                  # the tiny example below
+    refitgap --version
+
+Settings: --periods, --mean-over, --shared-periods, --shared-weights, --splits (200),
+--rule (inherited), --validation, --u-convention full|half, --no-reference, --margin
+(0.005), --family NAME, --names, --workers. Two methods are paired by seed; stored label
+names must agree. levels.csv: V_sep, V_shared, C, U_half, U_full, R, M per method, seed,
+model, period and rule. decomposition.csv: B, A, dR per period against the per-period
+reference. contrasts.csv: paired leads with per-seed values, mean, interval and p.
+
+Tiny example
+------------
+
+Two periods of four records, two labels (a, b), seed 0, one partition, inherited thresholds 0.5.
+The reference of period 1 is the final model itself.
+
+    period  record  labels a,b  final a,b   reference a,b
+    0       r0      1,0         0.4,0.6     0.9,0.3
+    0       r1      1,0         0.4,0.2     0.9,0.3
+    0       r2      0,1         0.1,0.8     0.2,0.8
+    0       r3      0,1         0.1,0.3     0.2,0.8
+    1       r0, r2  1,0         0.9,0.3     same as final
+    1       r1, r3  0,1         0.2,0.8     same as final
+
+default_rng(7000).permutation(4) = [1, 2, 0, 3] and default_rng(7001) gives [3, 0, 1, 2],
+so period 0 splits {r1, r2} | {r0, r3} and period 1 {r3, r0} | {r1, r2}. The reference
+separates everything: V = U = 1. The final model on period 0 reaches V_sep = V_shared = 3/4
+and U = 1/4 at 0.5, so C = 0 and M = R = 1/2; on period 1 it is the reference, V_sep = U = 1
+and V_shared = 11/12. Period 0 thus splits B = 3/4 into A = 1/4 (all label b: positive r3 now
+scores below negative r0) and dR = 1/2 (all label a: still separated, but below 0.5). The
+tests check every printed value against these fractions to 1e-12.
+
+Limits
+------
+
+A, C, M and R describe what thresholded predictions from the stored scores can reach: a
+large A says the scores lost separation, not that the representation lost the information.
+Every level survives any strictly increasing per-label map of scores and rules. The package
+recommends nothing; retrain-or-recalibrate needs costs the scores do not carry.

refitgap-0.1.0/src/refitgap/__init__.py

@@ -0,0 +1,29 @@
+"""refitgap: thresholded forgetting, split into what the scores lost and what the thresholds lost.
+
+Stored per-period scores and labels in; cross-fitted levels, the split
+deficit = attainable gap + change in mismatch, and seed-paired contrasts out.
+compare() handles two scorers on the same records, method_contrast() the paired lead
+of one method over another. API.txt in this directory has the details. numpy and scipy
+only; figures need matplotlib (extra "plots"). Command line: refitgap.
+"""
+from .fitters import GRID, f1_vec, fit_exact, fit_grid, fit_shared_exact, macro_f1, shared_objective
+from .crossfit import (SPLITS, SPLIT_BASE, Evaluation, array_hash, decompose, evaluate, fit_validation_rule,
+                       partitions)
+from .inference import (ALPHA, MARGIN, classify, classify_unadjusted, family, family_ni, holm, paired_contrast,
+                        paired_stats, paired_vector, dropped_seeds)
+from .records import align_to, drop_unlabelled
+from .runs import evaluate_npz, load_run
+from .analysis import analyse, make_run, run_from_arrays, run_from_npz, save_arrays, write_tables
+from .twoscorer import compare, method_contrast
+from .identification import f1_bounds_from_counts, identification_example
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "GRID", "f1_vec", "fit_exact", "fit_grid", "fit_shared_exact", "macro_f1", "shared_objective", "SPLITS",
+    "SPLIT_BASE", "Evaluation", "array_hash", "decompose", "evaluate", "fit_validation_rule", "partitions",
+    "ALPHA", "MARGIN", "classify", "classify_unadjusted", "family", "family_ni", "holm", "paired_contrast",
+    "paired_stats", "paired_vector", "dropped_seeds", "align_to", "drop_unlabelled", "evaluate_npz",
+    "load_run", "analyse", "make_run", "run_from_arrays", "run_from_npz", "save_arrays", "write_tables",
+    "compare", "method_contrast", "f1_bounds_from_counts", "identification_example", "__version__",
+]

refitgap-0.1.0/src/refitgap/__main__.py

@@ -0,0 +1,6 @@
+"""python -m refitgap: the command line (see cli.py)."""
+import sys
+
+from .cli import main
+
+sys.exit(main())