wsidrift 0.1.0__tar.gz

wsidrift-0.1.0/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.4
+Name: wsidrift
+Version: 0.1.0
+Summary: Label-free validity monitor for computational pathology pipelines
+Author-email: Abhishek Thakur <a.thakur5690@gmail.com>
+License: MIT
+Keywords: digital-pathology,drift-detection,model-monitoring,whole-slide-image,computational-pathology,domain-shift
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: torch>=2.0.0
+Requires-Dist: torchvision>=0.15.0
+Requires-Dist: pytorch-lightning>=2.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: scipy>=1.10.0
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: openslide-python>=1.3.0
+Requires-Dist: opencv-python>=4.8.0
+Requires-Dist: Pillow>=10.0.0
+Requires-Dist: omegaconf>=2.3.0
+Requires-Dist: tqdm>=4.65.0
+
+# wsidrift
+
+A label-free, model-agnostic validity monitor for deployed computational pathology pipelines.
+
+Detects when a WSI processing pipeline has drifted from its validated reference — without requiring labels or retraining.
+
+## Architecture
+
+```
+Input WSIs ──► Feature Extractor ──► Tier 1: Input Divergence (MMD/Wasserstein)
+                                  └──► Tier 2: Embedding Monitor (Mahalanobis/Energy)
+                                              │
+                                    Conjunction Rule ──► Validity Flag
+```
+
+- **Tier 1** — covariate shift in stain/feature distributions (label-free, assumption-light)
+- **Tier 2** — model-response drift in encoder embedding space (closer to correctness)
+- **Flag** — raised only when *both* tiers move in the direction that historically predicted a large generalization gap (ΔG)
+
+## Status
+
+Private research repository. Part of a three-paper dissertation.
+
+## License
+MIT

wsidrift-0.1.0/README.md

@@ -0,0 +1,25 @@
+# wsidrift
+
+A label-free, model-agnostic validity monitor for deployed computational pathology pipelines.
+
+Detects when a WSI processing pipeline has drifted from its validated reference — without requiring labels or retraining.
+
+## Architecture
+
+```
+Input WSIs ──► Feature Extractor ──► Tier 1: Input Divergence (MMD/Wasserstein)
+                                  └──► Tier 2: Embedding Monitor (Mahalanobis/Energy)
+                                              │
+                                    Conjunction Rule ──► Validity Flag
+```
+
+- **Tier 1** — covariate shift in stain/feature distributions (label-free, assumption-light)
+- **Tier 2** — model-response drift in encoder embedding space (closer to correctness)
+- **Flag** — raised only when *both* tiers move in the direction that historically predicted a large generalization gap (ΔG)
+
+## Status
+
+Private research repository. Part of a three-paper dissertation.
+
+## License
+MIT

wsidrift-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wsidrift"
+version = "0.1.0"
+description = "Label-free validity monitor for computational pathology pipelines"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [{name = "Abhishek Thakur", email = "a.thakur5690@gmail.com"}]
+requires-python = ">=3.10"
+keywords = [
+    "digital-pathology", "drift-detection", "model-monitoring",
+    "whole-slide-image", "computational-pathology", "domain-shift"
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Healthcare Industry",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Scientific/Engineering :: Medical Science Apps.",
+]
+dependencies = [
+    "torch>=2.0.0",
+    "torchvision>=0.15.0",
+    "pytorch-lightning>=2.0.0",
+    "numpy>=1.24.0",
+    "scipy>=1.10.0",
+    "scikit-learn>=1.3.0",
+    "pandas>=2.0.0",
+    "openslide-python>=1.3.0",
+    "opencv-python>=4.8.0",
+    "Pillow>=10.0.0",
+    "omegaconf>=2.3.0",
+    "tqdm>=4.65.0",
+]
+
+[project.scripts]
+wsidrift = "wsidrift.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

wsidrift-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

wsidrift-0.1.0/src/wsidrift/__init__.py

@@ -0,0 +1,8 @@
+"""wsidrift: label-free validity monitor for computational pathology pipelines."""
+
+from .monitor.divergence import Tier1Monitor
+from .monitor.embedding import Tier2Monitor
+from .monitor.validity_score import ValidityMonitor
+
+__version__ = "0.1.0"
+__all__ = ["Tier1Monitor", "Tier2Monitor", "ValidityMonitor"]

wsidrift-0.1.0/src/wsidrift/adapt/__init__.py

@@ -0,0 +1,5 @@
+"""Domain adaptation modules for wsidrift."""
+from .grl import GradientReversalLayer, lambda_schedule
+from .dann import DANNModel
+
+__all__ = ["GradientReversalLayer", "lambda_schedule", "DANNModel"]

wsidrift-0.1.0/src/wsidrift/adapt/dann.py

@@ -0,0 +1,72 @@
+"""DANN: Domain-Adversarial Neural Network for stain-shift adaptation."""
+
+from __future__ import annotations
+
+import pytorch_lightning as pl
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torchvision.models import resnet50
+
+from .grl import GradientReversalLayer, lambda_schedule
+
+
+class DANNModel(pl.LightningModule):
+    """ResNet-50 backbone with task and domain classification heads.
+
+    Args:
+        n_classes:    Number of task classes.
+        n_domains:    Number of source domains (institutions / centers).
+        lr:           Learning rate.
+        gamma:        GRL lambda schedule steepness.
+    """
+
+    def __init__(
+        self,
+        n_classes: int = 2,
+        n_domains: int = 5,
+        lr: float = 1e-4,
+        gamma: float = 10.0,
+    ) -> None:
+        super().__init__()
+        self.save_hyperparameters()
+        backbone = resnet50(weights=None)
+        feat_dim = backbone.fc.in_features
+        backbone.fc = nn.Identity()
+        self.backbone = backbone
+        self.task_head = nn.Sequential(
+            nn.Linear(feat_dim, 256), nn.ReLU(), nn.Linear(256, n_classes)
+        )
+        self.domain_head = nn.Sequential(
+            nn.Linear(feat_dim, 256), nn.ReLU(), nn.Linear(256, n_domains)
+        )
+        self.grl = GradientReversalLayer()
+
+    def forward(self, x: torch.Tensor):
+        return self.task_head(self.backbone(x))
+
+    def get_embeddings(self, x: torch.Tensor) -> torch.Tensor:
+        return self.backbone(x)
+
+    def _step(self, batch, stage: str):
+        imgs, labels, meta = batch
+        p = self.global_step / max(self.trainer.max_steps, 1)
+        self.grl.lam = lambda_schedule(p, self.hparams.gamma)
+        feats = self.backbone(imgs)
+        task_loss = F.cross_entropy(self.task_head(feats), labels)
+        domain_labels = meta.get("domain_idx", torch.zeros_like(labels))
+        domain_loss = F.cross_entropy(self.domain_head(self.grl(feats)), domain_labels)
+        loss = task_loss + domain_loss
+        self.log(f"{stage}/task_loss", task_loss, prog_bar=True)
+        self.log(f"{stage}/domain_loss", domain_loss)
+        self.log(f"{stage}/loss", loss)
+        return loss
+
+    def training_step(self, batch, _):
+        return self._step(batch, "train")
+
+    def validation_step(self, batch, _):
+        self._step(batch, "val")
+
+    def configure_optimizers(self):
+        return torch.optim.Adam(self.parameters(), lr=self.hparams.lr)

wsidrift-0.1.0/src/wsidrift/adapt/grl.py

@@ -0,0 +1,39 @@
+"""Gradient Reversal Layer for domain-adversarial training."""
+
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+from torch.autograd import Function
+
+
+class _GRLFunction(Function):
+    @staticmethod
+    def forward(ctx, x, lam):
+        ctx.save_for_backward(torch.tensor(lam))
+        return x.clone()
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        (lam,) = ctx.saved_tensors
+        return -lam * grad_output, None
+
+
+class GradientReversalLayer(nn.Module):
+    """Apply gradient reversal with a scalar lambda."""
+
+    def __init__(self, lam: float = 1.0) -> None:
+        super().__init__()
+        self.lam = lam
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return _GRLFunction.apply(x, self.lam)
+
+
+def lambda_schedule(p: float, gamma: float = 10.0) -> float:
+    """Anneal lambda from 0 → 1 as training fraction p goes 0 → 1.
+
+    Schedule from Ganin et al. (JMLR 2016): 2/(1+exp(−γp))−1.
+    """
+    import math
+    return 2.0 / (1.0 + math.exp(-gamma * p)) - 1.0

wsidrift-0.1.0/src/wsidrift/cli.py

@@ -0,0 +1,55 @@
+"""CLI entrypoint for wsidrift."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="wsidrift",
+        description="Label-free validity monitor for computational pathology pipelines.",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    # wsidrift score
+    score_p = sub.add_parser("score", help="Score a deployment batch against a reference")
+    score_p.add_argument("--reference", required=True, help="Reference features .npy or .csv")
+    score_p.add_argument("--deployment", required=True, help="Deployment features .npy or .csv")
+    score_p.add_argument("--output", default=None, help="Save result JSON to this path")
+    score_p.add_argument("--mmd-threshold", type=float, default=0.05)
+    score_p.add_argument("--wass-threshold", type=float, default=0.10)
+
+    args = parser.parse_args(argv)
+
+    if args.command == "score":
+        import numpy as np
+        from wsidrift.monitor.divergence import Tier1Monitor
+        from wsidrift.monitor.validity_score import ValidityMonitor
+
+        X_ref = np.load(args.reference) if args.reference.endswith(".npy") else \
+                __import__("pandas").read_csv(args.reference).values
+        X_dep = np.load(args.deployment) if args.deployment.endswith(".npy") else \
+                __import__("pandas").read_csv(args.deployment).values
+
+        tier1 = Tier1Monitor(
+            mmd_threshold=args.mmd_threshold,
+            wass_threshold=args.wass_threshold,
+        ).fit(X_ref)
+        monitor = ValidityMonitor(tier1)
+        result = monitor.score(X_dep)
+        print(result.summary_line())
+
+        if args.output:
+            with open(args.output, "w") as f:
+                json.dump(result.to_dict(), f, indent=2)
+            print(f"Result saved to {args.output}")
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

wsidrift-0.1.0/src/wsidrift/cyclegan/__init__.py

@@ -0,0 +1,5 @@
+"""CycleGAN stain transfer stub for wsidrift E4/E7/E9 conditions."""
+from .generator import ResNetGenerator
+from .trainer import CycleGANTrainer
+
+__all__ = ["ResNetGenerator", "CycleGANTrainer"]

wsidrift-0.1.0/src/wsidrift/cyclegan/generator.py

@@ -0,0 +1,65 @@
+"""ResNet-based generator for CycleGAN stain transfer."""
+
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, dim: int) -> None:
+        super().__init__()
+        self.block = nn.Sequential(
+            nn.ReflectionPad2d(1),
+            nn.Conv2d(dim, dim, 3),
+            nn.InstanceNorm2d(dim),
+            nn.ReLU(inplace=True),
+            nn.ReflectionPad2d(1),
+            nn.Conv2d(dim, dim, 3),
+            nn.InstanceNorm2d(dim),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x + self.block(x)
+
+
+class ResNetGenerator(nn.Module):
+    """9-block ResNet generator for 256×256 image translation.
+
+    Args:
+        n_residual: Number of residual blocks (6 for 128px, 9 for 256px).
+        ngf:        Base number of generator filters.
+    """
+
+    def __init__(self, n_residual: int = 9, ngf: int = 64) -> None:
+        super().__init__()
+        layers: list[nn.Module] = [
+            nn.ReflectionPad2d(3),
+            nn.Conv2d(3, ngf, 7),
+            nn.InstanceNorm2d(ngf),
+            nn.ReLU(inplace=True),
+        ]
+        # downsampling
+        for mult in [1, 2]:
+            layers += [
+                nn.Conv2d(ngf * mult, ngf * mult * 2, 3, stride=2, padding=1),
+                nn.InstanceNorm2d(ngf * mult * 2),
+                nn.ReLU(inplace=True),
+            ]
+        # residual blocks
+        for _ in range(n_residual):
+            layers.append(ResidualBlock(ngf * 4))
+        # upsampling
+        for mult in [2, 1]:
+            layers += [
+                nn.ConvTranspose2d(
+                    ngf * mult * 2, ngf * mult, 3, stride=2, padding=1, output_padding=1
+                ),
+                nn.InstanceNorm2d(ngf * mult),
+                nn.ReLU(inplace=True),
+            ]
+        layers += [nn.ReflectionPad2d(3), nn.Conv2d(ngf, 3, 7), nn.Tanh()]
+        self.model = nn.Sequential(*layers)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.model(x)

wsidrift-0.1.0/src/wsidrift/cyclegan/trainer.py

@@ -0,0 +1,58 @@
+"""CycleGAN training harness (stub — full training loop in scripts/train_cyclegan.py)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+
+from .generator import ResNetGenerator
+
+
+@dataclass
+class CycleGANConfig:
+    source_dir: str = ""
+    target_dir: str = ""
+    checkpoint_dir: str = "checkpoints/cyclegan"
+    n_epochs: int = 200
+    batch_size: int = 4
+    lr: float = 2e-4
+    lambda_cycle: float = 10.0
+    lambda_identity: float = 5.0
+    n_residual: int = 9
+
+
+class CycleGANTrainer:
+    """Minimal wrapper to load/save CycleGAN generators and apply inference.
+
+    Full adversarial training is handled by scripts/train_cyclegan.py.
+    This class handles checkpoint management and inference for E4/E7/E9.
+    """
+
+    def __init__(self, config: CycleGANConfig | None = None) -> None:
+        self.config = config or CycleGANConfig()
+        self.G_AB = ResNetGenerator(n_residual=self.config.n_residual)
+        self.G_BA = ResNetGenerator(n_residual=self.config.n_residual)
+
+    def save(self, epoch: int) -> None:
+        outdir = Path(self.config.checkpoint_dir)
+        outdir.mkdir(parents=True, exist_ok=True)
+        torch.save(self.G_AB.state_dict(), outdir / f"G_AB_epoch{epoch:03d}.pt")
+        torch.save(self.G_BA.state_dict(), outdir / f"G_BA_epoch{epoch:03d}.pt")
+
+    def load(self, checkpoint_dir: str, epoch: int) -> None:
+        d = Path(checkpoint_dir)
+        self.G_AB.load_state_dict(torch.load(d / f"G_AB_epoch{epoch:03d}.pt", map_location="cpu"))
+        self.G_BA.load_state_dict(torch.load(d / f"G_BA_epoch{epoch:03d}.pt", map_location="cpu"))
+
+    @torch.no_grad()
+    def translate_AtoB(self, x: torch.Tensor) -> torch.Tensor:
+        self.G_AB.eval()
+        return self.G_AB(x)
+
+    @torch.no_grad()
+    def translate_BtoA(self, x: torch.Tensor) -> torch.Tensor:
+        self.G_BA.eval()
+        return self.G_BA(x)

wsidrift-0.1.0/src/wsidrift/data/__init__.py

@@ -0,0 +1 @@
+"""Dataset classes for TCGA and CAMELYON17."""

wsidrift-0.1.0/src/wsidrift/data/camelyon17_dataset.py

@@ -0,0 +1,77 @@
+"""CAMELYON17 patch dataset for wsidrift experiments."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms as T
+from PIL import Image
+
+
+_DEFAULT_TRANSFORM = T.Compose([
+    T.Resize(256),
+    T.CenterCrop(224),
+    T.ToTensor(),
+    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+# Map stages.csv slide-level labels to binary: negative=0, any metastasis=1
+_BINARY_MAP = {"negative": 0, "itc": 1, "micro": 1, "macro": 1}
+
+
+class CAMELYON17PatchDataset(Dataset):
+    """Patch-level dataset for CAMELYON17 loaded from a manifest CSV.
+
+    Expected manifest columns: path, slide_label, center, patient_id, split.
+    slide_label is the raw stages.csv value (negative/itc/micro/macro).
+
+    Args:
+        manifest_path: Path to manifest CSV.
+        split:         'train', 'val', 'test', or None.
+        center:        Filter to a specific center (0–4).
+        transform:     torchvision transform.
+    """
+
+    def __init__(
+        self,
+        manifest_path: str | Path,
+        split: str | None = None,
+        center: int | None = None,
+        transform=None,
+    ) -> None:
+        df = pd.read_csv(manifest_path)
+        if split is not None:
+            df = df[df["split"] == split]
+        if center is not None:
+            df = df[df["center"] == center]
+        df = df.reset_index(drop=True)
+        df["label"] = df["slide_label"].map(_BINARY_MAP).fillna(0).astype(int)
+        self.df = df
+        self.transform = transform or _DEFAULT_TRANSFORM
+
+    def __len__(self) -> int:
+        return len(self.df)
+
+    def __getitem__(self, idx: int):
+        row = self.df.iloc[idx]
+        img = Image.open(row["path"]).convert("RGB")
+        img = self.transform(img)
+        label = int(row["label"])
+        meta = {
+            "center": int(row["center"]),
+            "patient_id": str(row.get("patient_id", "")),
+            "slide_label": str(row["slide_label"]),
+        }
+        return img, label, meta
+
+    @property
+    def labels(self) -> np.ndarray:
+        return self.df["label"].to_numpy()
+
+    @property
+    def centers(self) -> list[int]:
+        return self.df["center"].tolist()

wsidrift-0.1.0/src/wsidrift/data/tcga_dataset.py

@@ -0,0 +1,71 @@
+"""TCGA patch dataset for wsidrift experiments."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms as T
+from PIL import Image
+
+
+_DEFAULT_TRANSFORM = T.Compose([
+    T.Resize(256),
+    T.CenterCrop(224),
+    T.ToTensor(),
+    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+
+class TCGAPatchDataset(Dataset):
+    """Patch-level dataset loaded from a manifest CSV.
+
+    Expected manifest columns: path, label, slide_id, tss, cohort, split.
+
+    Args:
+        manifest_path: Path to manifest CSV.
+        split:         One of 'train', 'val', 'test', or None (all).
+        cohort:        Filter to a specific cohort (e.g. 'BRCA').
+        transform:     torchvision transform; defaults to ImageNet normalization.
+    """
+
+    def __init__(
+        self,
+        manifest_path: str | Path,
+        split: str | None = None,
+        cohort: str | None = None,
+        transform=None,
+    ) -> None:
+        df = pd.read_csv(manifest_path)
+        if split is not None:
+            df = df[df["split"] == split]
+        if cohort is not None:
+            df = df[df["cohort"] == cohort]
+        self.df = df.reset_index(drop=True)
+        self.transform = transform or _DEFAULT_TRANSFORM
+
+    def __len__(self) -> int:
+        return len(self.df)
+
+    def __getitem__(self, idx: int):
+        row = self.df.iloc[idx]
+        img = Image.open(row["path"]).convert("RGB")
+        img = self.transform(img)
+        label = int(row["label"])
+        meta = {
+            "slide_id": row["slide_id"],
+            "tss": str(row.get("tss", "")),
+            "cohort": str(row.get("cohort", "")),
+        }
+        return img, label, meta
+
+    @property
+    def labels(self) -> np.ndarray:
+        return self.df["label"].to_numpy()
+
+    @property
+    def tss_codes(self) -> list[str]:
+        return self.df["tss"].tolist()

wsidrift-0.1.0/src/wsidrift/eval/__init__.py

@@ -0,0 +1,5 @@
+"""Evaluation utilities: ΔG, AUC, and statistical tests."""
+from .delta_gap import DeltaGap, DeltaGapResult
+from .stats import paired_ttest, wilcoxon_test, bonferroni_correct
+
+__all__ = ["DeltaGap", "DeltaGapResult", "paired_ttest", "wilcoxon_test", "bonferroni_correct"]

wsidrift-0.1.0/src/wsidrift/eval/delta_gap.py

@@ -0,0 +1,67 @@
+"""ΔG computation across E1-E9 conditions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from sklearn.metrics import roc_auc_score
+
+
+@dataclass
+class DeltaGapResult:
+    condition: str
+    cohort: str
+    auc_in_mean: float
+    auc_in_std: float
+    auc_cross_mean: float
+    auc_cross_std: float
+    delta_gap_mean: float
+    delta_gap_std: float
+
+    def __str__(self) -> str:
+        return (
+            f"{self.condition} [{self.cohort}]: "
+            f"ΔG = {self.delta_gap_mean:+.4f} ± {self.delta_gap_std:.4f}  "
+            f"(in={self.auc_in_mean:.4f}, cross={self.auc_cross_mean:.4f})"
+        )
+
+
+class DeltaGap:
+    """Compute ΔG = AUC_in − AUC_cross across multiple seeds.
+
+    Args:
+        condition: Condition name (e.g. 'E1').
+        cohort:    Cohort name (e.g. 'BRCA').
+    """
+
+    def __init__(self, condition: str, cohort: str) -> None:
+        self.condition = condition
+        self.cohort = cohort
+        self._in_auc: list[float] = []
+        self._cross_auc: list[float] = []
+
+    def add_seed(
+        self,
+        y_true_in: np.ndarray,
+        y_score_in: np.ndarray,
+        y_true_cross: np.ndarray,
+        y_score_cross: np.ndarray,
+    ) -> None:
+        self._in_auc.append(float(roc_auc_score(y_true_in, y_score_in)))
+        self._cross_auc.append(float(roc_auc_score(y_true_cross, y_score_cross)))
+
+    def compute(self) -> DeltaGapResult:
+        if not self._in_auc:
+            raise ValueError("No seeds added — call add_seed() first.")
+        dg = [i - c for i, c in zip(self._in_auc, self._cross_auc)]
+        return DeltaGapResult(
+            condition=self.condition,
+            cohort=self.cohort,
+            auc_in_mean=float(np.mean(self._in_auc)),
+            auc_in_std=float(np.std(self._in_auc)),
+            auc_cross_mean=float(np.mean(self._cross_auc)),
+            auc_cross_std=float(np.std(self._cross_auc)),
+            delta_gap_mean=float(np.mean(dg)),
+            delta_gap_std=float(np.std(dg)),
+        )