fairscope 0.3.0__py3-none-any.whl

fairscope/federated/audit.py

@@ -0,0 +1,217 @@
+"""One-call cross-node fairness audit. Composes fairscope.core; invents no statistics.
+
+Per node: DeLong AUC confidence interval -> Expected Calibration Error -> Brier/F1.
+Across nodes: max-min AUC gap and Bonferroni-corrected pairwise (unpaired) DeLong tests.
+An optional per-node recalibration step reports pre/post ECE. Mirrors the cross-node
+evaluation in the privacy-preserving federated-learning study.
+
+IMPORTANT: this module audits per-node PREDICTIONS only. It does not train models, perform
+secure aggregation, or provide any privacy guarantee.
+"""
+
+from __future__ import annotations
+
+from itertools import combinations
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import brier_score_loss, f1_score
+
+from ..core import (
+    bonferroni,
+    delong_auc_ci,
+    delong_unpaired_test,
+    expected_calibration_error,
+    isotonic_recalibrate,
+    temperature_scale,
+)
+
+
+class FederatedFairnessAudit:
+    """Audit per-node predictions for cross-node fairness.
+
+    Parameters
+    ----------
+    node_data : dict ``{node_name: (y_true, y_score)}`` where ``y_score`` is the
+        positive-class probability for that node's evaluation samples.
+    n_bins : int, ECE bin count.
+    alpha : float, significance level for DeLong CIs and the Bonferroni correction.
+    """
+
+    def __init__(self, node_data, *, n_bins=10, alpha=0.05):
+        self.node_data = {
+            k: (np.asarray(y), np.asarray(s, dtype=float)) for k, (y, s) in node_data.items()
+        }
+        self.n_bins = n_bins
+        self.alpha = alpha
+
+    def run(self) -> FederatedReport:
+        per_node = {}
+        for node, (y, s) in self.node_data.items():
+            if len(np.unique(y)) < 2:
+                raise ValueError(f"node {node!r} has a single class; AUC undefined")
+            ci = delong_auc_ci(y, s, alpha=self.alpha)
+            ece = expected_calibration_error(y, s, n_bins=self.n_bins)
+            per_node[node] = {
+                "ci": ci,
+                "ece": float(ece),
+                "n": int(len(y)),
+                "brier": float(brier_score_loss(y, s)),
+                "f1": float(f1_score(y, (s >= 0.5).astype(int), zero_division=0)),
+            }
+        nodes = sorted(per_node)
+        pairs, pvals = [], []
+        for a, b in combinations(nodes, 2):
+            ya, sa = self.node_data[a]
+            yb, sb = self.node_data[b]
+            res = delong_unpaired_test(ya, sa, yb, sb)
+            pairs.append((a, b))
+            pvals.append(res["p_value"])
+        corrected = bonferroni(np.array(pvals), alpha=self.alpha) if pvals else None
+        return FederatedReport(
+            per_node,
+            pairs,
+            pvals,
+            corrected,
+            node_data=self.node_data,
+            alpha=self.alpha,
+            n_bins=self.n_bins,
+        )
+
+
+class FederatedReport:
+    """Holds cross-node audit results; renders tables (and plots/PDF in later tasks)."""
+
+    def __init__(self, per_node, pairs, pvals, corrected, *, node_data, alpha, n_bins):
+        self.per_node = per_node
+        self.pairs = pairs
+        self.pvals = pvals
+        self.corrected = corrected
+        self._node_data = node_data
+        self.alpha = alpha
+        self.n_bins = n_bins
+
+    def to_dataframe(self) -> pd.DataFrame:
+        rows = []
+        for node, r in self.per_node.items():
+            ci = r["ci"]
+            rows.append(
+                {
+                    "node": node,
+                    "n": r["n"],
+                    "auc": ci["auc"],
+                    "ci_lower": ci["ci_lower"],
+                    "ci_upper": ci["ci_upper"],
+                    "ece": r["ece"],
+                    "brier": r["brier"],
+                    "f1": r["f1"],
+                }
+            )
+        return pd.DataFrame(rows)
+
+    def disparity(self) -> dict:
+        """Cross-node AUC disparity summary."""
+        aucs = {n: r["ci"]["auc"] for n, r in self.per_node.items()}
+        hi = max(aucs, key=aucs.get)
+        lo = min(aucs, key=aucs.get)
+        return {
+            "max_auc_gap": aucs[hi] - aucs[lo],
+            "best": hi,
+            "worst": lo,
+            "worst_pair": (lo, hi),
+        }
+
+    def recalibrate(self, method="temperature") -> dict:
+        """Recalibrate each node on its own (y, score) and report pre/post ECE.
+
+        ``method`` is 'temperature' (Guo et al., 2017) or 'isotonic' (Zadrozny &
+        Elkan, 2002) — both standard methods from ``fairscope.core``. Temperature
+        scaling operates on logits, so the per-node probabilities are converted to
+        logits first. Returns ``{node: {"ece_pre": float, "ece_post": float}}``.
+
+        NOTE: this fits and evaluates on the same per-node data (an in-sample
+        diagnostic). For a deployment estimate, recalibrate on a held-out split.
+        """
+        if method not in ("temperature", "isotonic"):
+            raise ValueError(f"unknown method: {method!r}; use 'temperature' or 'isotonic'")
+        out = {}
+        for node, (y, s) in self._node_data.items():
+            pre = expected_calibration_error(y, s, n_bins=self.n_bins)
+            if method == "temperature":
+                p = np.clip(s, 1e-7, 1 - 1e-7)
+                logits = np.log(p / (1 - p))
+                _, s_cal = temperature_scale(logits, y)
+            else:  # isotonic
+                _, s_cal = isotonic_recalibrate(s, y)
+            post = expected_calibration_error(y, s_cal, n_bins=self.n_bins)
+            out[node] = {"ece_pre": float(pre), "ece_post": float(post)}
+        return out
+
+    def plot_auc_forest(self):
+        """Forest plot of per-node AUC with DeLong 95% CIs. Returns a Figure."""
+        import matplotlib.pyplot as plt
+
+        df = self.to_dataframe()
+        ys = np.arange(len(df))
+        xerr = np.vstack([df.auc - df.ci_lower, df.ci_upper - df.auc])
+        fig, ax = plt.subplots(figsize=(6, 0.5 * len(df) + 1.5))
+        ax.errorbar(df.auc, ys, xerr=xerr, fmt="o", capsize=3)
+        ax.set_yticks(ys)
+        ax.set_yticklabels(df.node)
+        ax.set_xlabel("AUC (95% DeLong CI)")
+        ax.axvline(0.5, color="gray", linestyle="--", linewidth=1)
+        ax.set_title("Per-node discrimination")
+        fig.tight_layout()
+        return fig
+
+    def plot_calibration(self):
+        """Per-node reliability curves drawn with ``core.reliability_diagram``
+        (federated retains each node's (y, score), so these are true curves).
+        Returns a Figure."""
+        from ..core import reliability_diagram
+
+        ys, ss, labels = [], [], []
+        for node, (y, s) in self._node_data.items():
+            ys.append(np.asarray(y))
+            ss.append(np.asarray(s))
+            labels.append(np.full(len(y), node))
+        return reliability_diagram(
+            np.concatenate(ys),
+            np.concatenate(ss),
+            groups=np.concatenate(labels),
+            n_bins=self.n_bins,
+        )
+
+    def to_pdf(self, path):
+        """Write a multi-page PDF: summary, per-node AUC forest, per-node calibration.
+        Uses matplotlib only (no extra dependency)."""
+        import matplotlib.pyplot as plt
+        from matplotlib.backends.backend_pdf import PdfPages
+
+        with PdfPages(path) as pdf:
+            fig0, ax = plt.subplots(figsize=(8.5, 11))
+            ax.axis("off")
+            ax.text(0.02, 0.98, self.summary(), family="monospace", va="top", fontsize=8)
+            pdf.savefig(fig0)
+            plt.close(fig0)
+
+            forest = self.plot_auc_forest()
+            pdf.savefig(forest)
+            plt.close(forest)
+
+            cal = self.plot_calibration()
+            cal.axes[0].set_title("Per-node calibration")
+            pdf.savefig(cal)
+            plt.close(cal)
+
+    def summary(self) -> str:
+        lines = [self.to_dataframe().to_string(index=False)]
+        d = self.disparity()
+        lines.append(f"cross-node AUC gap: {d['best']} vs {d['worst']} = {d['max_auc_gap']:.3f}")
+        if self.corrected is not None:
+            for (a, b), padj, rej in zip(
+                self.pairs, self.corrected["adjusted"], self.corrected["reject"]
+            ):
+                if rej:
+                    lines.append(f"  {a} vs {b}: Bonferroni-adjusted p={padj:.4g} (significant)")
+        return "\n".join(lines)

fairscope/healthcare/__init__.py

@@ -0,0 +1,5 @@
+"""Clinical fairness auditing built on fairscope.core."""
+
+from .audit import HealthcareFairnessAudit, HealthcareReport
+
+__all__ = ["HealthcareFairnessAudit", "HealthcareReport"]

fairscope/healthcare/audit.py

@@ -0,0 +1,227 @@
+"""One-call clinical fairness audit. Composes fairscope.core; invents no statistics.
+
+Pipeline per protected attribute: per-subgroup DeLong AUC CIs -> per-subgroup ECE ->
+per-subgroup Brier/F1 -> Bonferroni-corrected pairwise (unpaired) DeLong tests across the
+attribute's subgroups. Mirrors the analysis in the diabetes paper (IEEE CIPHER 2026).
+"""
+
+from __future__ import annotations
+
+from itertools import combinations
+
+import numpy as np
+import pandas as pd
+
+from ..core import (
+    bonferroni,
+    delong_by_group,
+    delong_unpaired_test,
+    ece_by_group,
+    subgroup_metrics,
+)
+
+
+class HealthcareFairnessAudit:
+    """Audit a fitted classifier (or precomputed scores) for subgroup fairness.
+
+    Parameters
+    ----------
+    model : object with ``predict_proba`` (positive-class probability in column 1), or
+        ``None`` when using :meth:`from_scores`.
+    X_test, y_test : test features and binary labels.
+    protected_attr : dict ``{attribute_name: 1-D array of subgroup labels per sample}``.
+    """
+
+    def __init__(self, model, X_test, y_test, protected_attr, *, n_bins=10, alpha=0.05):
+        self.model = model
+        self.X_test = X_test
+        self.y_test = np.asarray(y_test)
+        self.protected_attr = protected_attr
+        self.n_bins = n_bins
+        self.alpha = alpha
+        self._scores = None
+
+    @classmethod
+    def from_scores(cls, y_true, y_score, protected_attr, *, n_bins=10, alpha=0.05):
+        """Build an audit from precomputed positive-class probabilities (no model)."""
+        obj = cls(None, None, y_true, protected_attr, n_bins=n_bins, alpha=alpha)
+        obj._scores = np.asarray(y_score, dtype=float)
+        return obj
+
+    def _get_scores(self):
+        if self._scores is not None:
+            return self._scores
+        if self.model is None or not hasattr(self.model, "predict_proba"):
+            raise ValueError(
+                "model must implement predict_proba(); or use "
+                "HealthcareFairnessAudit.from_scores(...)"
+            )
+        return np.asarray(self.model.predict_proba(self.X_test))[:, 1]
+
+    def run(self) -> HealthcareReport:
+        """Run the audit and return a :class:`HealthcareReport`.
+
+        Raises a clear, attribute-named ``ValueError`` if a subgroup is single-class
+        (AUC undefined) rather than letting a low-level error surface.
+        """
+        y = self.y_test
+        s = self._get_scores()
+        results = {}
+        for name, groups in self.protected_attr.items():
+            groups = np.asarray(groups)
+            try:
+                cis = delong_by_group(y, s, groups, alpha=self.alpha)
+                eces = ece_by_group(y, s, groups, n_bins=self.n_bins)
+                mets = subgroup_metrics(y, s, groups)
+            except ValueError as exc:
+                raise ValueError(f"protected attribute {name!r}: {exc}") from exc
+            vals = sorted(np.unique(groups).tolist())
+            pairs, pvals, deltas = [], [], []
+            for a, b in combinations(vals, 2):
+                ma, mb = groups == a, groups == b
+                res = delong_unpaired_test(y[ma], s[ma], y[mb], s[mb])
+                pairs.append((a, b))
+                pvals.append(res["p_value"])
+                deltas.append(res["delta"])
+            corrected = bonferroni(np.array(pvals), alpha=self.alpha) if pvals else None
+            results[name] = {
+                "groups": vals,
+                "ci": cis,
+                "ece": eces,
+                "metrics": mets,
+                "pairs": pairs,
+                "delta": deltas,
+                "p_value": pvals,
+                "p_adjusted": corrected["adjusted"].tolist() if corrected else [],
+                "reject": corrected["reject"].tolist() if corrected else [],
+            }
+        return HealthcareReport(
+            results, y, s, self.protected_attr, alpha=self.alpha, n_bins=self.n_bins
+        )
+
+    def shap_summary(self, max_samples=200):
+        """Mean absolute SHAP value per feature (optional). Requires
+        ``pip install fairscope[shap]`` and a model (not ``from_scores``). Returns a dict
+        ``{feature_index: mean_abs_shap}``."""
+        try:
+            import shap  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "SHAP summary requires the optional dependency: pip install fairscope[shap]"
+            ) from exc
+        return _shap_mean_abs(self.model, self.X_test, max_samples)  # pragma: no cover
+
+
+class HealthcareReport:
+    """Holds audit results and the raw (y, score, groups) needed to render reliability
+    curves; provides tables (here), and plots/PDF (see plotting methods)."""
+
+    def __init__(self, results, y_true, y_score, protected_attr, *, alpha=0.05, n_bins=10):
+        self.results = results
+        self.y_true = np.asarray(y_true)
+        self.y_score = np.asarray(y_score)
+        self.protected_attr = protected_attr
+        self.alpha = alpha
+        self.n_bins = n_bins
+
+    def to_dataframe(self) -> pd.DataFrame:
+        rows = []
+        for attr, r in self.results.items():
+            for g in r["groups"]:
+                ci = r["ci"][g]
+                m = r["metrics"][g]
+                rows.append(
+                    {
+                        "attribute": attr,
+                        "group": g,
+                        "n": m["n"],
+                        "auc": ci["auc"],
+                        "ci_lower": ci["ci_lower"],
+                        "ci_upper": ci["ci_upper"],
+                        "ece": r["ece"][g],
+                        "brier": m["brier"],
+                        "f1": m["f1"],
+                    }
+                )
+        return pd.DataFrame(rows)
+
+    def summary(self) -> str:
+        """Return a human-readable summary string (no print side effect)."""
+        df = self.to_dataframe()
+        lines = [df.to_string(index=False)]
+        for attr, r in self.results.items():
+            aucs = {g: r["ci"][g]["auc"] for g in r["groups"]}
+            hi, lo = max(aucs, key=aucs.get), min(aucs, key=aucs.get)
+            lines.append(
+                f"[{attr}] largest AUC gap: {hi} {aucs[hi]:.3f} vs "
+                f"{lo} {aucs[lo]:.3f} (delta={aucs[hi] - aucs[lo]:.3f})"
+            )
+            for (a, b), padj, rej in zip(r["pairs"], r["p_adjusted"], r["reject"]):
+                if rej:
+                    lines.append(f"  {a} vs {b}: Bonferroni-adjusted p={padj:.4g} (significant)")
+        return "\n".join(lines)
+
+    def plot_auc_forest(self, attribute=None):
+        """Forest plot of per-subgroup AUC with DeLong 95% CIs. Returns a Figure."""
+        import matplotlib.pyplot as plt
+
+        df = self.to_dataframe()
+        if attribute is not None:
+            df = df[df.attribute == attribute]
+        labels = [f"{a}:{g}" for a, g in zip(df.attribute, df.group)]
+        ys = np.arange(len(df))
+        xerr = np.vstack([df.auc - df.ci_lower, df.ci_upper - df.auc])
+        fig, ax = plt.subplots(figsize=(6, 0.5 * len(df) + 1.5))
+        ax.errorbar(df.auc, ys, xerr=xerr, fmt="o", capsize=3)
+        ax.set_yticks(ys)
+        ax.set_yticklabels(labels)
+        ax.set_xlabel("AUC (95% DeLong CI)")
+        ax.axvline(0.5, color="gray", linestyle="--", linewidth=1)
+        ax.set_title("Per-subgroup discrimination")
+        fig.tight_layout()
+        return fig
+
+    def plot_calibration(self, attribute=None):
+        """Reliability curves per subgroup for one attribute, drawn with
+        ``core.reliability_diagram``. Returns a Figure."""
+        from ..core import reliability_diagram
+
+        attr = attribute if attribute is not None else next(iter(self.protected_attr))
+        groups = np.asarray(self.protected_attr[attr])
+        return reliability_diagram(self.y_true, self.y_score, groups=groups, n_bins=self.n_bins)
+
+    def to_pdf(self, path):
+        """Write a multi-page PDF: summary, AUC forest, and one calibration page per
+        attribute. Uses matplotlib only (no extra dependency)."""
+        import matplotlib.pyplot as plt
+        from matplotlib.backends.backend_pdf import PdfPages
+
+        with PdfPages(path) as pdf:
+            fig0, ax = plt.subplots(figsize=(8.5, 11))
+            ax.axis("off")
+            ax.text(0.02, 0.98, self.summary(), family="monospace", va="top", fontsize=8)
+            pdf.savefig(fig0)
+            plt.close(fig0)
+
+            forest = self.plot_auc_forest()
+            pdf.savefig(forest)
+            plt.close(forest)
+
+            for attr in self.protected_attr:
+                cal = self.plot_calibration(attr)
+                cal.axes[0].set_title(f"Calibration: {attr}")
+                pdf.savefig(cal)
+                plt.close(cal)
+
+
+def _shap_mean_abs(model, X_test, max_samples):  # pragma: no cover - needs optional shap
+    import shap
+
+    if model is None:
+        raise ValueError("shap_summary requires a model (not from_scores).")
+    X = np.asarray(X_test)[:max_samples]
+    explainer = shap.Explainer(model.predict_proba, X)
+    values = explainer(X).values
+    vals = np.abs(values).mean(axis=0)
+    vals = vals[:, 1] if vals.ndim == 2 else vals
+    return {i: float(v) for i, v in enumerate(np.ravel(vals))}

fairscope/lending/__init__.py

@@ -0,0 +1,5 @@
+"""Mortgage-lending fairness auditing built on fairscope.core (+ optional econml)."""
+
+from .audit import LendingFairnessAudit, LendingReport
+
+__all__ = ["LendingFairnessAudit", "LendingReport"]

fairscope/lending/audit.py

@@ -0,0 +1,160 @@
+"""One-call lending fairness audit.
+
+Annual approval-gap analysis composes fairscope.core and is purely descriptive (no causal
+claim). Subgroup CATE estimation (Task 2) wraps econml under stated DML assumptions.
+Mirrors the analyses in P1 (Causal Forest DML, HMDA) and P2 (descriptive disparities).
+Ships no HMDA data and no model.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from ..core import disparate_impact
+
+
+class LendingFairnessAudit:
+    """Audit mortgage-approval outcomes for annual subgroup disparities.
+
+    Parameters
+    ----------
+    approved : 1-D binary array (1 = approved).
+    group : 1-D array of protected-group labels, aligned with ``approved``.
+    year : 1-D array of years, aligned with ``approved``.
+    reference : the group label to compare every other group against. Must be present
+        in each year.
+    alpha : significance level (reserved for the CATE step).
+    """
+
+    def __init__(self, *, approved, group, year, reference, alpha=0.05):
+        self.approved = np.asarray(approved)
+        self.group = np.asarray(group)
+        self.year = np.asarray(year)
+        self.reference = reference
+        self.alpha = alpha
+
+    @classmethod
+    def from_outcomes(cls, approved, group, year, *, reference, alpha=0.05):
+        """Build an audit from precomputed approval outcomes."""
+        return cls(approved=approved, group=group, year=year, reference=reference, alpha=alpha)
+
+    def estimate_cate(
+        self,
+        X,
+        *,
+        treatment=None,
+        outcome=None,
+        model_y=None,
+        model_t=None,
+        n_estimators=500,
+        random_state=0,
+    ):
+        """Per-subgroup conditional average treatment effect (CATE) of the protected
+        attribute on approval, via Causal Forest DML (``econml.dml.CausalForestDML``,
+        as in P1).
+
+        The CAUSAL CLAIM IS CONDITIONAL on the DML assumptions (unconfoundedness given
+        the supplied features ``X``, and overlap). This estimates an effect under those
+        assumptions; it does not, on its own, prove discrimination.
+
+        Parameters
+        ----------
+        X : array (n, k) of heterogeneity features.
+        treatment : binary array; defaults to ``group != reference``.
+        outcome : binary array; defaults to the ``approved`` outcomes.
+        model_y, model_t : nuisance estimators; default to random forests.
+
+        Returns
+        -------
+        dict
+            Keys ``ate``, ``effect``, and ``effect_interval``.
+
+        Notes
+        -----
+        Requires the optional dependency ``econml`` (``pip install fairscope[lending]``).
+        """
+        try:
+            import econml.dml  # noqa: F401
+        except ImportError as exc:  # optional dependency
+            raise ImportError(
+                "Subgroup CATE requires the optional dependency: " "pip install fairscope[lending]"
+            ) from exc
+        return _causal_forest_cate(  # pragma: no cover - exercised only with econml installed
+            self, X, treatment, outcome, model_y, model_t, n_estimators, random_state
+        )
+
+    def run(self) -> LendingReport:
+        rows = []
+        for yr in sorted(np.unique(self.year).tolist()):
+            m = self.year == yr
+            approved_y = self.approved[m]
+            group_y = self.group[m]
+            for g in sorted(np.unique(group_y).tolist()):
+                sel = group_y == g
+                rate = float(approved_y[sel].mean())
+                di = float(disparate_impact(approved_y, group_y, g, self.reference))
+                rows.append(
+                    {
+                        "year": yr,
+                        "group": g,
+                        "n": int(sel.sum()),
+                        "approval_rate": rate,
+                        "disparate_impact": di,
+                    }
+                )
+        return LendingReport(pd.DataFrame(rows), reference=self.reference, alpha=self.alpha)
+
+
+class LendingReport:
+    """Holds the annual approval-gap table (CATE results are returned separately)."""
+
+    def __init__(self, df, *, reference, alpha=0.05):
+        self._df = df
+        self.reference = reference
+        self.alpha = alpha
+
+    def to_dataframe(self) -> pd.DataFrame:
+        return self._df.copy()
+
+    def summary(self) -> str:
+        lines = [self._df.to_string(index=False)]
+        nonref = self._df[self._df.group != self.reference]
+        if not nonref.empty:
+            worst = nonref.nsmallest(1, "disparate_impact").iloc[0]
+            lines.append(
+                f"largest gap: {worst.group} in {int(worst.year)} "
+                f"DI={worst.disparate_impact:.3f} (approval {worst.approval_rate:.3f})"
+            )
+        return "\n".join(lines)
+
+
+def _causal_forest_cate(  # pragma: no cover - exercised only with econml installed
+    audit, X, treatment, outcome, model_y, model_t, n_estimators, random_state
+):
+    """Causal Forest DML CATE. Isolated so the optional-econml path is excluded from
+    coverage in CI (econml is not installed there); the importorskip tests exercise it
+    locally and for any contributor who installs ``fairscope[lending]``."""
+    from econml.dml import CausalForestDML
+    from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+
+    if treatment is None:
+        t = (audit.group != audit.reference).astype(int)
+    else:
+        t = np.asarray(treatment)
+    y = audit.approved if outcome is None else np.asarray(outcome)
+    x = np.asarray(X)
+    est = CausalForestDML(
+        model_y=model_y or RandomForestRegressor(random_state=random_state),
+        model_t=model_t or RandomForestClassifier(random_state=random_state),
+        discrete_treatment=True,
+        n_estimators=n_estimators,
+        random_state=random_state,
+    )
+    est.fit(y, t, X=x)
+    lo, hi = est.effect_interval(x, alpha=audit.alpha)
+    return {
+        "ate": float(est.ate(x)),
+        "effect": est.effect(x),
+        "effect_interval": (lo, hi),
+    }

fairscope/nlp/__init__.py

@@ -0,0 +1,29 @@
+"""Cross-Platform Fairness Evaluation (CPFE) for NLP, built on fairscope.core.
+
+Public API: the five-axis ``CPFEProtocol``/``CPFEReport``, the multiclass metric and
+significance primitives (axes 1-4), and the attribution-stability functions (axis 5).
+"""
+
+from .attribution import jaccard_topk, token_saliency
+from .cross_platform import CPFEProtocol, CPFEReport
+from .metrics import (
+    macro_auc,
+    macro_f1,
+    multiclass_ece,
+    per_class_disparate_impact,
+    per_class_equalized_odds,
+)
+from .significance import bootstrap_macro_auc_test
+
+__all__ = [
+    "CPFEProtocol",
+    "CPFEReport",
+    "macro_auc",
+    "macro_f1",
+    "multiclass_ece",
+    "per_class_disparate_impact",
+    "per_class_equalized_odds",
+    "bootstrap_macro_auc_test",
+    "jaccard_topk",
+    "token_saliency",
+]