endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/utils/ai/data_loader_for_model_training.py

@@ -0,0 +1,262 @@
+# endoreg_db/utils/ai/data_loader_for_model_training.py
+
+from __future__ import annotations
+
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional, TypedDict
+
+from django.db import models
+
+from endoreg_db.models import (
+    AIDataSet,
+    Frame,
+    ImageClassificationAnnotation,
+    Label,
+    LabelSet,
+)
+
+
+class ImageMultilabelDataset(TypedDict):
+    """
+    In-memory representation of an image multi-label training dataset.
+
+    All lists are aligned by index:
+
+        image_paths[i]   -> path to image file for sample i
+        label_vectors[i] -> list[int|None] of length == len(labels)
+        label_masks[i]   -> list[int]       of length == len(labels)
+
+    Where:
+        - label_vectors[i][j] is:
+            1     -> positive annotation (value=True)
+            0     -> negative annotation (value=False)
+            None  -> UNKNOWN (no annotation for that (frame, label))
+
+        - label_masks[i][j] is:
+            1 -> this entry participates in the loss (0 or 1 is known)
+            0 -> IGNORE in the loss (value was None)
+    """
+
+    # type description of the returned dict.
+
+    image_paths: List[str]
+    label_vectors: List[List[Optional[int]]]
+    label_masks: List[List[int]]
+    labels: List[Label]
+    labelset: LabelSet
+
+    # New: keep track of which DB rows were used, and their legacy exam ids
+    frame_ids: List[int]  # Frame.pk for each sample
+    old_examination_ids: List[Optional[int]]  # may be None if not set
+
+
+def _infer_labelset_from_annotations(
+    annotations_qs: models.QuerySet[ImageClassificationAnnotation],
+) -> LabelSet:
+    """
+    Try to infer a unique LabelSet from the labels used in the annotations.
+
+    Strategy:
+        1. Collect all distinct label_ids from the annotations.
+        2. Fetch all Label objects + their label_sets.
+        3. Compute the intersection of all label_sets across all labels.
+        4. If there is exactly ONE common LabelSet, return it.
+           Otherwise, raise NotImplementedError for now.
+    """
+    label_ids = list(annotations_qs.values_list("label_id", flat=True).distinct())
+    if not label_ids:
+        raise ValueError("Cannot infer LabelSet: annotations queryset has no labels.")
+
+    labels_qs = Label.objects.filter(id__in=label_ids).prefetch_related("label_sets")
+    labelsets_for_each_label = []
+
+    for lbl in labels_qs:
+        # lbl.label_sets is the reverse of LabelSet.labels M2M
+        ls_ids = list(lbl.label_sets.values_list("id", flat=True))
+        if not ls_ids:
+            # This label is not part of any LabelSet -> ambiguous
+            raise NotImplementedError(
+                f"Label id={lbl.id}, name='{lbl.name}' is not part of any LabelSet. "
+                "Explicit LabelSet selection is required."
+            )
+        labelsets_for_each_label.append(set(ls_ids))
+
+    # Intersection of all labelset id sets
+    common_ids = set.intersection(*labelsets_for_each_label)
+    if not common_ids:
+        raise NotImplementedError(
+            "No common LabelSet across all labels in this AIDataSet. "
+            "Please specify a LabelSet explicitly."
+        )
+    if len(common_ids) > 1:
+        raise NotImplementedError(
+            "More than one common LabelSet found for the labels in this AIDataSet. "
+            "Please specify a LabelSet explicitly to disambiguate."
+        )
+
+    ls_id = next(iter(common_ids))
+    return LabelSet.objects.get(id=ls_id)
+
+
+def build_image_multilabel_dataset_from_db(
+    dataset: AIDataSet,
+    labelset: Optional[LabelSet] = None,
+) -> ImageMultilabelDataset:
+    """
+    Build an in-memory multilabel dataset for an IMAGE-based AIDataSet.
+
+    Steps:
+        1. Take all ImageClassificationAnnotation rows linked to this AIDataSet
+           (via dataset.image_annotations M2M).
+        2. Determine the LabelSet (either explicitly given or inferred).
+        3. For each used Frame, build:
+            - an image path
+            - a label vector (1, 0, or None for each label in LabelSet)
+            - a mask vector (1 where known, 0 where unknown)
+        4. Return a dict that can be wrapped in a torch/tf Dataset.
+
+    NOTE:
+        - This function does NOT write anything to the DB.
+        - It only reads DB rows and returns Python structures.
+    """
+    if dataset.dataset_type != AIDataSet.DATASET_TYPE_IMAGE:
+        raise ValueError(
+            f"build_image_multilabel_dataset_from_db expected dataset_type='image', "
+            f"got '{dataset.dataset_type}' for AIDataSet id={dataset.id}."
+        )
+
+    # Get the annotation relation dynamically (for future video/text types)
+    annotations_qs = dataset.get_annotations_queryset().select_related("frame", "label")
+
+    if annotations_qs.count() == 0:
+        raise ValueError(
+            f"AIDataSet id={dataset.id} has no annotations attached. "
+            "Make sure your import script populated image_annotations."
+        )
+
+    # Decide which LabelSet to use
+    if labelset is None:
+        labelset = _infer_labelset_from_annotations(annotations_qs)
+
+    # Fixed label order (= fixed column order for the label vectors)
+    labels_in_order: List[Label] = labelset.get_labels_in_order()
+    if not labels_in_order:
+        raise ValueError(
+            f"LabelSet id={labelset.id}, name='{labelset.name}' has no labels."
+        )
+
+    num_labels = len(labels_in_order)
+    label_index: Dict[int, int] = {
+        lbl.id: idx for idx, lbl in enumerate(labels_in_order)
+    }
+
+    # Group annotations by frame
+    anns_by_frame: Dict[int, List[ImageClassificationAnnotation]] = defaultdict(list)
+    frames_order: List[int] = []
+
+    for ann in annotations_qs:
+        frame_id = ann.frame_id
+        if frame_id not in anns_by_frame:
+            frames_order.append(frame_id)
+        anns_by_frame[frame_id].append(ann)
+
+    # Build vectors
+    image_paths: List[str] = []
+    label_vectors: List[List[Optional[int]]] = []
+    label_masks: List[List[int]] = []
+
+    # New: id tracking for splitting / logging
+    frame_ids: List[int] = []
+    old_examination_ids: List[Optional[int]] = []
+
+    # Cache frames to avoid repeated DB hits
+    frame_obj_by_id: Dict[int, Frame] = {}
+
+    for frame_id in frames_order:
+        frame_annotations = anns_by_frame[frame_id]
+
+        # Resolve frame object (from first annotation of this frame)
+        frame = frame_obj_by_id.get(frame_id)
+        if frame is None:
+            frame = frame_annotations[0].frame
+            frame_obj_by_id[frame_id] = frame
+
+            # New: remember DB ids for this sample
+        frame_ids.append(frame_id)
+        old_examination_ids.append(getattr(frame, "old_examination_id", None))
+
+        # Start with unknown for all labels
+        vec: List[Optional[int]] = [None] * num_labels
+
+        # Fill with 1/0 where we have annotations
+        for ann in frame_annotations:
+            idx = label_index.get(ann.label_id)
+            if idx is None:
+                # Label not part of this LabelSet: ignore
+                continue
+            vec[idx] = 1 if ann.value else 0
+
+        # Build mask: 1 where vec is known, 0 where unknown
+        mask: List[int] = [0 if v is None else 1 for v in vec]
+
+        # Resolve absolute image path from the Frame model
+        file_path: Path = frame.file_path
+        image_paths.append(str(file_path))
+        label_vectors.append(vec)
+        label_masks.append(mask)
+
+    return ImageMultilabelDataset(
+        image_paths=image_paths,
+        label_vectors=label_vectors,
+        label_masks=label_masks,
+        labels=labels_in_order,
+        labelset=labelset,
+        frame_ids=frame_ids,
+        old_examination_ids=old_examination_ids,
+    )
+
+
+def build_dataset_for_training(
+    dataset: AIDataSet,
+    labelset: Optional[LabelSet] = None,
+):
+    """
+    High-level entry point to build a training dataset from an AIDataSet row.
+
+    It inspects:
+        - dataset.dataset_type
+        - dataset.ai_model_type
+
+    and dispatches to the appropriate builder.
+
+    For now, we support:
+        - dataset_type = "image"
+        - ai_model_type = "image_multilabel_classification"
+
+    Later, you can extend this to:
+        - video segmentation
+        - text classification
+        etc.
+    """
+    # IMAGE MULTILABEL CASE
+    if (
+        dataset.dataset_type == AIDataSet.DATASET_TYPE_IMAGE
+        and dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL
+    ):
+        return build_image_multilabel_dataset_from_db(dataset, labelset=labelset)
+
+    # FUTURE EXTENSIONS (example structure, not yet implemented):
+    # if dataset.dataset_type == AIDataSet.DATASET_TYPE_VIDEO and \
+    #    dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_VIDEO_SEGMENTATION:
+    #     return build_video_segmentation_dataset_from_db(dataset, labelset=labelset)
+    #
+    # if dataset.dataset_type == AIDataSet.DATASET_TYPE_TEXT and \
+    #    dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_TEXT_CLASSIFICATION:
+    #     return build_text_classification_dataset_from_db(dataset, labelset=labelset)
+
+    raise NotImplementedError(
+        f"No dataset builder implemented for "
+        f"dataset_type='{dataset.dataset_type}', "
+        f"ai_model_type='{dataset.ai_model_type}'."
+    )

endoreg_db/utils/ai/get.py

@@ -1,5 +1,6 @@
 from endoreg_db.models import ModelMeta
 
+
 def get_latest_model_meta_by_model_name(model_name):
-    model_meta = ModelMeta.objects.filter(name=model_name).order_by('-version').first()
+    model_meta = ModelMeta.objects.filter(name=model_name).order_by("-version").first()
     return model_meta

endoreg_db/utils/ai/inference_dataset.py

@@ -4,6 +4,7 @@ from PIL import Image
 from torchvision import transforms
 from .preprocess import Cropper
 
+
 class InferenceDataset(Dataset):
     def __init__(self, paths, crops, config):
         self.paths = paths
@@ -12,13 +13,15 @@ class InferenceDataset(Dataset):
         self.config = config
 
         # Initialize the image transformations using torchvision
-        self.transforms = transforms.Compose([
-            # Convert PIL image to PyTorch tensor
-            transforms.ToTensor(),
-            # Normalize the image using the provided mean and std
-            transforms.Normalize(mean=self.config["mean"], std=self.config["std"])
-        ])
-        
+        self.transforms = transforms.Compose(
+            [
+                # Convert PIL image to PyTorch tensor
+                transforms.ToTensor(),
+                # Normalize the image using the provided mean and std
+                transforms.Normalize(mean=self.config["mean"], std=self.config["std"]),
+            ]
+        )
+
     def __len__(self):
         # Returns the total number of samples
         return len(self.paths)
@@ -27,7 +30,7 @@ class InferenceDataset(Dataset):
         # Open the image with Pillow
         with Image.open(self.paths[idx]) as pil_image:
             # Convert the image to RGB to ensure 3 channels
-            pil_image = pil_image.convert('RGB')
+            pil_image = pil_image.convert("RGB")
 
         # Get the corresponding crop for the current image
         crop = self.crops[idx]
@@ -36,17 +39,13 @@ class InferenceDataset(Dataset):
         cropped = self.cropper(
             np.array(pil_image),  # Convert PIL image to numpy array for cropping
             crop,
-            scale=[
-                self.config["size_x"],
-                self.config["size_y"]
-            ]    
+            scale=[self.config["size_x"], self.config["size_y"]],
         )
 
         # Convert cropped numpy array back to PIL image for torchvision transforms
-        cropped_pil = Image.fromarray(cropped.astype('uint8'), 'RGB')
-        
+        cropped_pil = Image.fromarray(cropped.astype("uint8"), "RGB")
+
         # Apply the transformations
         img = self.transforms(cropped_pil)
 
-
         return img

endoreg_db/utils/ai/model_training/config.py

@@ -0,0 +1,117 @@
+# endoreg_db/utils/ai/model_training/config.py
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from django.conf import settings
+
+
+# ---------------------------------------------------------------------
+# PATHS
+# ---------------------------------------------------------------------
+
+# Base project directory (e.g. /home/admin/dev/endoreg-db)
+BASE_DIR = Path(getattr(settings, "BASE_DIR", Path(__file__).resolve().parents[4]))
+
+# All training artifacts go here:
+#   /home/admin/dev/endoreg-db/data/model_training/
+TRAINING_ROOT = BASE_DIR / "data" / "model_training"
+CHECKPOINTS_DIR = TRAINING_ROOT / "checkpoints"
+RUNS_DIR = TRAINING_ROOT / "runs"
+
+for d in (TRAINING_ROOT, CHECKPOINTS_DIR, RUNS_DIR):
+    d.mkdir(parents=True, exist_ok=True)
+
+# Which LabelSet.version we train on (for label filtering)
+DEFAULT_LABELSET_VERSION_TO_TRAIN: int = 2
+
+
+# ---------------------------------------------------------------------
+# TRAINING CONFIG
+# ---------------------------------------------------------------------
+
+
+@dataclass
+class TrainingConfig:
+    """
+    Configuration for GastroNet multi-label training.
+
+    Most important knobs:
+    - dataset_id: which AIDataSet row to use from the database
+    - labelset_version_to_train: only labels belonging to LabelSet.version == this
+      are used for training (e.g. 2).
+    - treat_unlabeled_as_negative:
+        True  -> Option A: for v2 labels, if not annotated in a frame, we
+                 assume "absent" (0) and include it in the loss.
+        False -> keep "unknown" semantics (mask = 0, ignored in loss/metrics).
+
+    Learning rate schedule:
+    - lr_head / lr_backbone: base learning rates
+    - use_scheduler: if True, we use warm-up + cosine decay
+    - warmup_epochs: how many epochs to linearly increase LR from 0 → base LR
+    - min_lr: lowest LR reached at the end of cosine schedule
+    """
+
+    # --- WHAT TO TRAIN ON -------------------------------------------------
+    dataset_id: int
+
+    # Train only on labels belonging to ANY LabelSet with this version.
+    labelset_version_to_train: int = DEFAULT_LABELSET_VERSION_TO_TRAIN
+
+    # Path to GastroNet RN50 checkpoint (.pth); if None, backbone is random.
+    backbone_checkpoint: Optional[str] = None
+
+    # --- EPOCHS / BATCHING -----------------------------------------------
+    num_epochs: int = 5
+    batch_size: int = 32
+
+    # Split ratios (by colonoscopy exam groups, not by individual frames)
+    val_split: float = 0.2
+    test_split: float = 0.1
+
+    # --- LEARNING RATES --------------------------------------------------
+    # Base learning rates for classifier head and backbone.
+    lr_head: float = 1e-3  # usually larger (newly initialized layer)
+    lr_backbone: float = 1e-4  # smaller (pretrained GastroNet backbone)
+
+    # --- FOCAL LOSS HYPERPARAMETERS -------------------------------------
+    gamma_focal: float = 2.0  # how strongly to focus on hard examples
+    alpha_focal: float = 0.25  # weight for positives vs negatives
+
+    # --- DEVICE & SEED ---------------------------------------------------
+    device: str = "auto"  # "auto", "cpu", or "cuda"
+    random_seed: int = 42
+
+    # --- LABEL SEMANTICS -------------------------------------------------
+    # For the filtered labels (LabelSet.version == labelset_version_to_train):
+    # True  -> Option A: unlabeled => negative (0) and mask=1 (supervised)
+    # False -> keep unlabeled as unknown (mask=0, ignored)
+    treat_unlabeled_as_negative: bool = True
+
+    # --- LR SCHEDULER: WARM-UP + COSINE DECAY ----------------------------
+    # If True, we apply:
+    #   - linear warm-up for 'warmup_epochs'
+    #   - then CosineAnnealingLR for the remaining epochs
+    use_scheduler: bool = True
+
+    # Number of warm-up epochs (can be 0 for "no warm-up").
+    warmup_epochs: int = 3
+
+    # Minimum learning rate at the end of cosine decay for all param groups.
+    # (Both head and backbone decay towards this value.)
+    min_lr: float = 1e-6
+
+    # which CNN backbone / weights to use
+    # "gastro_rn50"          → current behavior (ResNet50 + GastroNet checkpoint)
+    # "resnet50_imagenet"    → ResNet50 with ImageNet weights
+    # "resnet50_random"      → ResNet50 with random initialization
+    # (later) "efficientnet_b0_imagenet", etc.
+    backbone_name: str = "gastro_rn50"
+
+    # whether to freeze backbone (feature extractor)
+    freeze_backbone: bool = True
+
+    # backbone_name: str = "gastro_rn50"

endoreg_db/utils/ai/model_training/dataset.py

@@ -0,0 +1,74 @@
+# endoreg_db/utils/ai/model_training/dataset.py
+
+from __future__ import annotations
+
+from typing import Optional, Sequence, Tuple, List
+
+import numpy as np
+from PIL import Image
+import torch
+from torch.utils.data import Dataset
+
+
+class EndoMultiLabelDataset(Dataset):
+    """
+    PyTorch dataset wrapping the output of build_dataset_for_training.
+
+    Each item is:
+        image_tensor: FloatTensor [3, H, W]
+        labels:       FloatTensor [num_labels]   (0 or 1; unknown -> 0 but masked)
+        mask:         FloatTensor [num_labels]   (1 known, 0 unknown)
+    """
+
+    def __init__(
+        self,
+        image_paths: Sequence[str],
+        label_vectors: Sequence[Sequence[Optional[int]]],
+        label_masks: Sequence[Sequence[int]],
+        image_size: int = 224,
+    ) -> None:
+        assert len(image_paths) == len(label_vectors) == len(label_masks), (
+            "image_paths, label_vectors, label_masks must have same length"
+        )
+
+        self.image_paths: List[str] = list(image_paths)
+
+        # Convert vectors with None → 0, but mask will ensure they are ignored
+        label_vec_list = []
+        mask_list = []
+        for vec, mask in zip(label_vectors, label_masks):
+            v = [0 if (x is None) else int(x) for x in vec]
+            m = [int(x) for x in mask]
+            label_vec_list.append(v)
+            mask_list.append(m)
+
+        self.labels = torch.tensor(label_vec_list, dtype=torch.float32)  # [N, C]
+        self.masks = torch.tensor(mask_list, dtype=torch.float32)  # [N, C]
+
+        self.num_labels = self.labels.shape[1]
+        self.image_size = image_size
+
+        # ImageNet-style normalization
+        self.mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
+        self.std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
+
+    def __len__(self) -> int:
+        return len(self.image_paths)
+
+    def _load_image(self, path: str) -> torch.Tensor:
+        """
+        Load image from disk, resize, convert to normalized tensor [3, H, W].
+        """
+        img = Image.open(path).convert("RGB")
+        img = img.resize((self.image_size, self.image_size))
+        arr = np.array(img, dtype=np.float32) / 255.0  # [H, W, C]
+        tensor = torch.from_numpy(arr).permute(2, 0, 1)  # [C, H, W]
+        tensor = (tensor - self.mean) / self.std
+        return tensor
+
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        path = self.image_paths[idx]
+        x = self._load_image(path)
+        y = self.labels[idx]
+        m = self.masks[idx]
+        return x, y, m

endoreg_db/utils/ai/model_training/losses.py

@@ -0,0 +1,68 @@
+# endoreg_db/utils/ai/model_training/losses.py
+
+from __future__ import annotations
+
+from typing import Optional
+
+import torch
+
+
+def compute_class_weights(
+    labels: torch.Tensor,
+    masks: torch.Tensor,
+    eps: float = 1e-6,
+) -> torch.Tensor:
+    """
+    Compute per-label weights based on positive counts.
+
+    labels: [N, C] in {0,1}
+    masks:  [N, C] in {0,1}, 1 = known, 0 = unknown
+
+    w_j = 1 / (pos_j + eps), normalized so that mean(w) ≈ 1.
+    """
+    known = masks > 0.5
+    pos_counts = (labels * known).sum(dim=0)  # [C]
+
+    raw_weights = 1.0 / (pos_counts + eps)
+    mean_w = raw_weights.mean().clamp(min=eps)
+    norm_weights = raw_weights / mean_w
+    return norm_weights  # [C]
+
+
+def focal_loss_with_mask(
+    logits: torch.Tensor,
+    targets: torch.Tensor,
+    masks: torch.Tensor,
+    class_weights: Optional[torch.Tensor] = None,
+    alpha: float = 0.25,
+    gamma: float = 2.0,
+    eps: float = 1e-6,
+) -> torch.Tensor:
+    """
+    Multi-label focal loss with:
+      - per-label class weights
+      - mask to ignore unknown labels.
+
+    logits: [B, C]  raw outputs
+    targets: [B, C] 0/1
+    masks: [B, C]   1 = known, 0 = unknown
+    class_weights: [C] or None
+    """
+    prob = torch.sigmoid(logits).clamp(eps, 1.0 - eps)  # [B, C]
+
+    # p_t: prob if y=1, (1-prob) if y=0
+    pt = prob * targets + (1.0 - prob) * (1.0 - targets)
+
+    alpha_factor = alpha * targets + (1.0 - alpha) * (1.0 - targets)
+    focal_factor = (1.0 - pt) ** gamma
+
+    loss = -alpha_factor * focal_factor * torch.log(pt)  # [B, C]
+
+    if class_weights is not None:
+        loss = loss * class_weights.view(1, -1)
+
+    # apply mask → ignore unknown labels
+    loss = loss * masks
+
+    denom = masks.sum().clamp(min=1.0)
+    return loss.sum() / denom

endoreg_db/utils/ai/model_training/metrics.py

@@ -0,0 +1,78 @@
+# endoreg_db/utils/ai/model_training/metrics.py
+from __future__ import annotations
+import torch
+
+
+def compute_metrics(logits, targets, masks, threshold=0.5):
+    """
+    Computes multi-label metrics:
+      - Global Precision/Recall/F1
+      - Per-label Precision/Recall/F1
+    """
+    probs = torch.sigmoid(logits)
+    preds = (probs >= threshold).int()
+    targets = targets.int()
+    masks = masks.int()
+
+    # Only evaluate where mask == 1
+    preds = preds * masks
+    targets = targets * masks
+
+    tp = (preds * targets).sum().item()
+    fp = (preds * (1 - targets)).sum().item()
+    fn = ((1 - preds) * targets).sum().item()
+    tn = ((1 - preds) * (1 - targets)).sum().item()
+
+    precision = tp / (tp + fp + 1e-6)
+    recall = tp / (tp + fn + 1e-6)
+    f1 = 2 * precision * recall / (precision + recall + 1e-6)
+    accuracy = (tp + tn) / (tp + tn + fp + fn + 1e-6)
+
+    # ------- PER-LABEL METRICS -------
+    per_label = []
+    num_labels = targets.shape[1]
+
+    for j in range(num_labels):
+        t = targets[:, j]
+        p = preds[:, j]
+        m = masks[:, j]
+
+        # consider only known labels
+        valid_idx = m == 1
+        if valid_idx.sum() == 0:
+            per_label.append(
+                {"precision": None, "recall": None, "f1": None, "support": 0}
+            )
+            continue
+
+        t = t[valid_idx]
+        p = p[valid_idx]
+
+        tp_j = ((p == 1) & (t == 1)).sum().item()
+        fp_j = ((p == 1) & (t == 0)).sum().item()
+        fn_j = ((p == 0) & (t == 1)).sum().item()
+
+        precision_j = tp_j / (tp_j + fp_j + 1e-6)
+        recall_j = tp_j / (tp_j + fn_j + 1e-6)
+        f1_j = 2 * precision_j * recall_j / (precision_j + recall_j + 1e-6)
+
+        per_label.append(
+            {
+                "precision": precision_j,
+                "recall": recall_j,
+                "f1": f1_j,
+                "support": t.sum().item(),
+            }
+        )
+
+    return {
+        "precision": precision,
+        "recall": recall,
+        "f1": f1,
+        "accuracy": accuracy,
+        "tp": tp,
+        "fp": fp,
+        "tn": tn,
+        "fn": fn,
+        "per_label": per_label,
+    }