refitgap 0.1.0__py3-none-any.whl

refitgap/report.py

@@ -0,0 +1,136 @@
+"""analysis.analyse() result -> one self-contained HTML page (inline SVG figures, tables,
+one sentence per comparison; no network, no script)."""
+import html
+from pathlib import Path
+
+from .analysis import CONTRAST_COLUMNS, DECOMP_COLUMNS, LEVEL_COLUMNS, VOCABULARY_NOTE
+
+CSS = """
+body{font-family:Georgia,'Times New Roman',serif;color:#222;background:#fff;max-width:72rem;margin:0 auto;
+padding:1rem 16px 3rem;line-height:1.45}
+h1{font-size:1.4rem;margin-bottom:.2rem}h2{font-size:1.1rem;margin-top:2rem;border-bottom:1px solid #ccc}
+table{border-collapse:collapse;font-family:Menlo,Consolas,monospace;font-size:.72rem;margin:.5rem 0}
+th,td{border-bottom:1px solid #e3e3e3;padding:2px 6px;text-align:right;white-space:nowrap}
+th{background:#f4f4f4;position:sticky;top:0}td.t,th.t{text-align:left}
+.scroll{overflow-x:auto;max-height:28rem;overflow-y:auto}
+.note{color:#555;font-size:.9rem}figure{margin:1rem 0}figure svg{max-width:100%;height:auto}
+figcaption{color:#555;font-size:.85rem}
+"""
+
+TEXT_COLS = {"method", "model", "period", "rule", "direction", "sample_unit", "interval_type", "family", "adjustment",
+             "status", "U_convention", "comparison", "term", "policy", "periods", "seeds", "values", "class"}
+
+
+def _fmt(v):
+    if v is None:
+        return ""
+    if isinstance(v, bool):
+        return "yes" if v else "no"
+    if isinstance(v, float):
+        return f"{v:.6g}" if abs(v) < 1e-3 and v != 0 else f"{v:.4f}"
+    return str(v)
+
+
+def _table(rows, cols):
+    if not rows:
+        return "<p class='note'>No rows.</p>"
+    head = "".join(f"<th class='{'t' if c in TEXT_COLS else ''}'>{html.escape(c)}</th>" for c in cols)
+    body = "".join("<tr>" + "".join(f"<td class='{'t' if c in TEXT_COLS else ''}'>{html.escape(_fmt(r.get(c)))}</td>"
+                                     for c in cols) + "</tr>" for r in rows)
+    return f"<div class='scroll'><table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>"
+
+
+def _kv(d):
+    rows = "".join(f"<tr><td class='t'>{html.escape(str(k))}</td><td class='t'>{html.escape(str(v))}</td></tr>"
+                   for k, v in d.items())
+    return f"<div class='scroll'><table><tbody>{rows}</tbody></table></div>"
+
+
+CAPTIONS = {
+    "decomposition": "Deficit decomposition against the per-period reference: A (orange squares, solid line) and "
+                     "dR, the change in mismatch (blue circles, dashed line), per period and their "
+                     "mean, macro-F1 on the 0 to 1 scale, positive where the final model is below the reference. "
+                     "B = A + dR holds in every row.",
+    "profile": "Period profile of each final model: V_sep under own-period rules (black circles, solid), V_shared "
+               "under one shared rule (grey squares, dashed) and U_full at the fixed rule (blue triangles, dotted), "
+               "macro-F1 on the 0 to 1 scale; the axis spans the data and its note gives the range.",
+    "comparison": "Paired leads at the fixed rule (black diamonds), after own-period refit (orange squares) and after "
+                  "one shared refit (grey circles), with marginal 95% t intervals over seeds where there are two or "
+                  "more. The dashed lines mark the margin, the dotted line zero.",
+}
+
+
+def render(result, svgs=None, title="refitgap report"):
+    """The HTML page as a string. svgs: {name: svg markup} (plots.svg_strings); None tries
+    to draw them and falls back to a note when matplotlib is unavailable."""
+    if svgs is None:
+        try:
+            from .plots import svg_strings
+            svgs = svg_strings(result)
+        except ImportError:
+            svgs = {}
+    st = result["settings"]
+    settings = {
+        "refitgap version": st.get("refitgap_version"),
+        "methods and inputs": "; ".join(f"{k}: {', '.join(v)}" for k, v in st["methods"].items()),
+        "seeds": " ".join(map(str, st["seeds"])),
+        "partitions (splits)": st["splits"], "partition rule": st["split_rule"] + f" (split_base {st['split_base']})",
+        "fixed rule of the decomposition": st["rule"],
+        "rule vectors": "; ".join(f"{src}: " + ", ".join(f"{k} = {[round(x, 6) for x in v]}" for k, v in r.items())
+                                  for src, r in st["rules"].items()),
+        "evaluated period ids": st["period_ids"], "periods averaged": st["mean_over"],
+        "shared periods": st["shared_periods"], "shared weights": st["shared_weights"] or "uniform",
+        "weights of the average": st["mean_weights"] or "uniform",
+        "U convention": ("U_full: the fixed-rule level on the whole period (decomposition B = A + dR as in Algorithm 1 "
+                         "of the paper)" if st["U_convention"] == "full" else
+                         "U_half: the fixed-rule level on the held-out halves (R = C + M exact per partition)"),
+        "margin": st["margin"], "family": st["family"] or "none (every row descriptive)",
+    }
+    parts = [f"<!doctype html><html lang='en'><head><meta charset='utf-8'>"
+             f"<meta name='viewport' content='width=device-width, initial-scale=1'><title>{html.escape(title)}</title>"
+             f"<style>{CSS}</style></head><body>",
+             f"<h1>{html.escape(title)}</h1>",
+             "<p class='note'>Evaluation of thresholded macro-F1 from stored per-period scores and labels. "
+             "Every quantity is a function of the scores, the labels, the rules, the seed and the period ids. The "
+             "terms describe what thresholded predictions built from these scores reach; they do not identify what "
+             "the model's internal features retain, and the report proposes no action.</p>",
+             "<h2>Inputs and validation</h2>",
+             _table(result["validation"], ["check", "result", "detail"]),
+             "<h2>Settings</h2>", _kv(settings),
+             "<h2>Interpretation</h2>"]
+    if result["interpretation"]:
+        parts.append("<ul>" + "".join(f"<li>{html.escape(s)}</li>" for s in result["interpretation"]) + "</ul>")
+    else:
+        parts.append("<p class='note'>No comparison was computed.</p>")
+    parts.append(f"<p class='note'>Rows are labelled descriptive unless a planned family was named. "
+                 f"{html.escape(VOCABULARY_NOTE)}</p>")
+    parts.append("<h2>Figures</h2>")
+    if svgs:
+        for name in ("decomposition", "profile", "comparison"):
+            if name in svgs:
+                parts.append(f"<figure>{svgs[name]}<figcaption>{html.escape(CAPTIONS[name])}</figcaption></figure>")
+    else:
+        parts.append("<p class='note'>Figures were not drawn: matplotlib is not installed "
+                     "(pip install 'refitgap[plots]'). The tables below hold every number.</p>")
+    parts += [
+        "<h2>Decomposition B = A + dR (reference minus final)</h2>",
+        _table(result["decomposition"], DECOMP_COLUMNS),
+        "<h2>Paired contrasts</h2>",
+        _table(result["contrasts"], CONTRAST_COLUMNS),
+        "<h2>Per-period levels</h2>",
+        _table(result["levels"], LEVEL_COLUMNS),
+        "<p class='note'>V_sep: own-period rules; V_shared: one shared rule; U_half, U_full: the fixed "
+        "rule on the held-out halves and on the whole period; C = V_sep - V_shared; M = V_shared - U_half; "
+        "R_half = V_sep - U_half = C + M; R_full = V_sep - U_full. In the decomposition, B = U_ref - U_final, "
+        "A = V_ref - V_final and dR = R_final - R_ref at the chosen U convention. Full-precision values are "
+        "in the CSV files.</p>",
+        "</body></html>",
+    ]
+    return "\n".join(parts)
+
+
+def write_report(result, path, svgs=None, title="refitgap report"):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(render(result, svgs, title), encoding="utf-8")
+    return path

refitgap/runs.py

@@ -0,0 +1,53 @@
+"""Stored run files (*_scores.npz): P_<i>_<j> is model i on period j's test records, Y_<j>
+the labels, `thresholds` theta_1, Pval_<i> / Yval the validation scores. Model N-1 is final."""
+import numpy as np
+
+
+def n_periods(z):
+    return max(int(k.split("_")[1]) for k in z.files if k.startswith("P_")) + 1
+
+
+def load_run(path, model=None, periods=None):
+    """npz path -> dict(Ps, Ys, period_ids, thresholds, Pval, Yval, N) for one model (default the
+    final; "diagonal" gives P_j_j on period j)."""
+    z = np.load(path)
+    N = n_periods(z)
+    periods = list(range(N)) if periods is None else [int(p) for p in periods]
+    if model == "diagonal":
+        Ps = [z[f"P_{j}_{j}"] for j in periods]
+        mi = None
+    else:
+        mi = N - 1 if model is None else int(model)
+        Ps = [z[f"P_{mi}_{j}"] for j in periods]
+    Ys = [z[f"Y_{j}"] for j in periods]
+    has_val = mi is not None and f"Pval_{mi}" in z.files and "Yval" in z.files
+    return dict(Ps=Ps, Ys=Ys, period_ids=periods, N=N,
+                thresholds=z["thresholds"].astype(np.float64) if "thresholds" in z.files else None,
+                Pval=z[f"Pval_{mi}"] if has_val else None, Yval=z["Yval"] if has_val else None)
+
+
+def evaluate_npz(path, seed, periods=None, model=None, splits=200, validation=False, scores_key="P",
+                 shared_periods=None):
+    """npz path and seed -> Evaluation of the final model (or `model`) at "inherited" and, with
+    validation=True, the grid rule on its validation scores. Picklable, for process pools."""
+    from .crossfit import evaluate, fit_validation_rule
+    if scores_key == "P":
+        r = load_run(path, model, periods)
+    else:
+        z = np.load(path)
+        N = n_periods(z)
+        mi = N - 1 if model is None else int(model)
+        periods = list(range(N)) if periods is None else [int(p) for p in periods]
+        r = dict(Ps=[z[f"{scores_key}_{mi}_{j}"] for j in periods], Ys=[z[f"Yx_{j}"] for j in periods],
+                 period_ids=periods, thresholds=z["thresholds"].astype(np.float64), Pval=None, Yval=None)
+    rules = {"inherited": r["thresholds"]}
+    if validation and r["Pval"] is not None:
+        rules["validation"] = fit_validation_rule(r["Pval"], r["Yval"])
+    return evaluate(r["Ps"], r["Ys"], seed, r["period_ids"], rules=rules, splits=splits,
+                    shared_periods=shared_periods, single_model=(model != "diagonal"))
+
+
+def _evaluate_task(task):
+    """Pool helper: task = (path, seed, periods, kwargs)."""
+    path, seed, periods, kw = task
+    return evaluate_npz(path, seed, periods, **kw)

refitgap/tiny.py

@@ -0,0 +1,68 @@
+"""The tiny example: eight records, two labels, two periods, two models, one partition.
+Every number it prints is worked by hand in API.txt.
+
+    refitgap example"""
+import numpy as np
+
+from .analysis import analyse, make_run
+
+SEED, SPLITS = 0, 1
+THETA = np.array([0.5, 0.5])
+LABEL_NAMES = ["label a", "label b"]
+# labels, columns (label a, label b), rows r0..r3 of each period
+Y0 = np.array([[1, 0], [1, 0], [0, 1], [0, 1]])
+Y1 = np.array([[1, 0], [0, 1], [1, 0], [0, 1]])
+# final model
+F0 = np.array([[0.4, 0.6], [0.4, 0.2], [0.1, 0.8], [0.1, 0.3]])
+F1 = np.array([[0.9, 0.3], [0.2, 0.8], [0.9, 0.3], [0.2, 0.8]])
+# reference model of period 0 (the reference of period 1 is the final model, F1)
+S0 = np.array([[0.9, 0.3], [0.9, 0.3], [0.2, 0.8], [0.2, 0.8]])
+
+# the values worked by hand in API.txt (fractions)
+HAND = {
+    "final": {0: dict(V_sep=3 / 4, V_shared=3 / 4, U_half=1 / 4, U_full=1 / 4, C=0.0, M=1 / 2, R_half=1 / 2),
+              1: dict(V_sep=1.0, V_shared=11 / 12, U_half=1.0, U_full=1.0, C=1 / 12, M=-1 / 12, R_half=0.0),
+              "mean": dict(V_sep=7 / 8, V_shared=5 / 6, U_half=5 / 8, U_full=5 / 8, C=1 / 24, M=5 / 24, R_half=1 / 4)},
+    "reference": {0: dict(V_sep=1.0, U_half=1.0, U_full=1.0), 1: dict(V_sep=1.0, U_half=1.0, U_full=1.0),
+                  "mean": dict(V_sep=1.0, U_half=1.0, U_full=1.0)},
+    "decomposition": {0: dict(B=3 / 4, A=1 / 4, dR=1 / 2), 1: dict(B=0.0, A=0.0, dR=0.0),
+                      "mean": dict(B=3 / 8, A=1 / 8, dR=1 / 4)},
+}
+
+
+def tiny_run():
+    return make_run([F0, F1], [Y0, Y1], SEED, {"inherited": THETA}, reference=[S0, F1], period_ids=[0, 1],
+                    source="tiny example", label_names=LABEL_NAMES)
+
+
+def tiny_result(u_convention="full"):
+    return analyse(
+        [tiny_run()], rule="inherited", splits=SPLITS, u_convention=u_convention, names=("tiny example", "b")
+    )
+
+
+def lines(result):
+    """The printed lines: one per quantity, 15 significant digits, so a reader can compare
+    them with the hand values and a test can parse them."""
+    out = [f"tiny example: 2 periods x 4 records, 2 labels, seed {SEED}, {SPLITS} partition, inherited thresholds 0.5",
+           "levels (macro-F1, 0 to 1; descriptive):"]
+    for r in result["levels"]:
+        p = "mean" if str(r["period"]).startswith("mean") else r["period"]
+        vals = [("V_sep", r["V_sep"]), ("V_shared", r["V_shared"]), ("U_half", r["U_half"]), ("U_full", r["U_full"]),
+                ("C", r["C"]), ("M", r["M"]), ("R_half", r["R_half"])]
+        out.append(f"  {r['model']} period {p}: " + "  ".join(f"{k} {v:.15g}" for k, v in vals if v is not None))
+    out.append("decomposition B = A + dR against the per-period reference, U_full (descriptive):")
+    for r in result["decomposition"]:
+        p = "mean" if str(r["period"]).startswith("mean") else r["period"]
+        out.append(f"  period {p}: B {r['B']:.15g}  A {r['A']:.15g}  dR {r['dR']:.15g}")
+    out.extend(result["interpretation"])
+    return out
+
+
+def main(argv=None):
+    for line in lines(tiny_result()):
+        print(line)
+
+
+if __name__ == "__main__":
+    main()