kaiko-eva 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl

eva/vision/models/networks/decoders/segmentation/{conv2d.py → decoder2d.py}

@@ -1,19 +1,20 @@
 """Convolutional based semantic segmentation decoder."""
 
-from typing import List, Tuple
+from typing import List, Sequence, Tuple
 
 import torch
 from torch import nn
 from torch.nn import functional
 
-from eva.vision.models.networks.decoders import decoder
+from eva.vision.models.networks.decoders.segmentation import base
+from eva.vision.models.networks.decoders.segmentation.typings import DecoderInputs
 
 
-class ConvDecoder(decoder.Decoder):
-    """Convolutional segmentation decoder."""
+class Decoder2D(base.Decoder):
+    """Segmentation decoder for 2D applications."""
 
-    def __init__(self, layers: nn.Module) -> None:
-        """Initializes the convolutional based decoder head.
+    def __init__(self, layers: nn.Module, combine_features: bool = True) -> None:
+        """Initializes the based decoder head.
 
         Here the input nn layers will be directly applied to the
         features of shape (batch_size, hidden_size, n_patches_height,
@@ -21,13 +22,16 @@ class ConvDecoder(decoder.Decoder):
         Note the n_patches is also known as grid_size.
 
         Args:
-            layers: The convolutional layers to be used as the decoder head.
+            layers: The layers to be used as the decoder head.
+            combine_features: Whether to combine the features from different
+                feature levels into one tensor before applying the decoder head.
         """
         super().__init__()
 
         self._layers = layers
+        self._combine_features = combine_features
 
-    def _forward_features(self, features: List[torch.Tensor]) -> torch.Tensor:
+    def _forward_features(self, features: torch.Tensor | List[torch.Tensor]) -> torch.Tensor:
         """Forward function for multi-level feature maps to a single one.
 
         It will interpolate the features and concat them into a single tensor
@@ -46,6 +50,8 @@ class ConvDecoder(decoder.Decoder):
             A tensor of shape (batch_size, hidden_size, n_patches_height,
             n_patches_width) which is feature map of the decoder head.
         """
+        if isinstance(features, torch.Tensor):
+            features = [features]
         if not isinstance(features, list) or features[0].ndim != 4:
             raise ValueError(
                 "Input features should be a list of four (4) dimensional inputs of "
@@ -63,7 +69,9 @@ class ConvDecoder(decoder.Decoder):
         ]
         return torch.cat(upsampled_features, dim=1)
 
-    def _forward_head(self, patch_embeddings: torch.Tensor) -> torch.Tensor:
+    def _forward_head(
+        self, patch_embeddings: torch.Tensor | Sequence[torch.Tensor]
+    ) -> torch.Tensor:
         """Forward of the decoder head.
 
         Args:
@@ -75,12 +83,12 @@ class ConvDecoder(decoder.Decoder):
         """
         return self._layers(patch_embeddings)
 
-    def _cls_seg(
+    def _upscale(
         self,
         logits: torch.Tensor,
         image_size: Tuple[int, int],
     ) -> torch.Tensor:
-        """Classify each pixel of the image.
+        """Upscales the calculated logits to the target image size.
 
         Args:
             logits: The decoder outputs of shape (batch_size, n_classes,
@@ -93,22 +101,18 @@ class ConvDecoder(decoder.Decoder):
         """
         return functional.interpolate(logits, image_size, mode="bilinear")
 
-    def forward(
-        self,
-        features: List[torch.Tensor],
-        image_size: Tuple[int, int],
-    ) -> torch.Tensor:
+    def forward(self, decoder_inputs: DecoderInputs) -> torch.Tensor:
         """Maps the patch embeddings to a segmentation mask of the image size.
 
         Args:
-            features: List of multi-level image features of shape (batch_size,
-                hidden_size, n_patches_height, n_patches_width).
-            image_size: The target image size (height, width).
+            decoder_inputs: Inputs required by the decoder.
 
         Returns:
             Tensor containing scores for all of the classes with shape
             (batch_size, n_classes, image_height, image_width).
         """
-        patch_embeddings = self._forward_features(features)
-        logits = self._forward_head(patch_embeddings)
-        return self._cls_seg(logits, image_size)
+        features, image_size, _ = DecoderInputs(*decoder_inputs)
+        if self._combine_features:
+            features = self._forward_features(features)
+        logits = self._forward_head(features)
+        return self._upscale(logits, image_size)

eva/vision/models/networks/decoders/segmentation/linear.py

@@ -6,10 +6,10 @@ import torch
 from torch import nn
 from torch.nn import functional
 
-from eva.vision.models.networks.decoders import decoder
+from eva.vision.models.networks.decoders.segmentation import base
 
 
-class LinearDecoder(decoder.Decoder):
+class LinearDecoder(base.Decoder):
     """Linear decoder."""
 
     def __init__(self, layers: nn.Module) -> None:

eva/vision/models/networks/decoders/segmentation/semantic/__init__.py

@@ -0,0 +1,12 @@
+"""Semantic Segmentation decoder heads API."""
+
+from eva.vision.models.networks.decoders.segmentation.semantic.common import (
+    ConvDecoder1x1,
+    ConvDecoderMS,
+    SingleLinearDecoder,
+)
+from eva.vision.models.networks.decoders.segmentation.semantic.with_image import (
+    ConvDecoderWithImage,
+)
+
+__all__ = ["ConvDecoder1x1", "ConvDecoderMS", "SingleLinearDecoder", "ConvDecoderWithImage"]

eva/vision/models/networks/decoders/segmentation/{common.py → semantic/common.py}

@@ -7,10 +7,10 @@ output by an encoder into pixel-wise predictions for segmentation tasks.
 
 from torch import nn
 
-from eva.vision.models.networks.decoders.segmentation import conv2d, linear
+from eva.vision.models.networks.decoders.segmentation import decoder2d, linear
 
 
-class ConvDecoder1x1(conv2d.ConvDecoder):
+class ConvDecoder1x1(decoder2d.Decoder2D):
     """A convolutional decoder with a single 1x1 convolutional layer."""
 
     def __init__(self, in_features: int, num_classes: int) -> None:
@@ -29,7 +29,7 @@ class ConvDecoder1x1(conv2d.ConvDecoder):
         )
 
 
-class ConvDecoderMS(conv2d.ConvDecoder):
+class ConvDecoderMS(decoder2d.Decoder2D):
     """A multi-stage convolutional decoder with upsampling and convolutional layers.
 
     This decoder applies a series of upsampling and convolutional layers to transform

eva/vision/models/networks/decoders/segmentation/semantic/with_image.py

@@ -0,0 +1,94 @@
+"""Convolutional semantic segmentation decoders that use input image & feature maps as input."""
+
+from typing import List
+
+import torch
+from torch import nn
+from torchvision.transforms.functional import rgb_to_grayscale
+from typing_extensions import override
+
+from eva.vision.models.networks.decoders.segmentation import decoder2d
+from eva.vision.models.networks.decoders.segmentation.typings import DecoderInputs
+
+
+class ConvDecoderWithImage(decoder2d.Decoder2D):
+    """A convolutional that in addition to encoded features, also takes the input image as input.
+
+    In a first stage, the input features are upsampled and passed through a convolutional layer,
+    while in the second stage, the input image channels are concatenated with the upsampled features
+    and passed through additional convolutional blocks in order to combine the image prior
+    information with the encoded features. Lastly, a 1x1 conv operation reduces the number of
+    channels to the number of classes.
+    """
+
+    _default_hidden_dims = [64, 32, 32]
+
+    def __init__(
+        self,
+        in_features: int,
+        num_classes: int,
+        greyscale: bool = False,
+        hidden_dims: List[int] | None = None,
+    ) -> None:
+        """Initializes the decoder.
+
+        Args:
+            in_features: The hidden dimension size of the embeddings.
+            num_classes: Number of output classes as channels.
+            greyscale: Whether to convert input images to greyscale.
+            hidden_dims: List of hidden dimensions for the convolutional layers.
+        """
+        hidden_dims = hidden_dims or self._default_hidden_dims
+        if len(hidden_dims) != 3:
+            raise ValueError("Hidden dims must have 3 elements.")
+
+        super().__init__(
+            layers=nn.Sequential(
+                nn.Upsample(scale_factor=2),
+                Conv2dBnReLU(in_features, hidden_dims[0]),
+            )
+        )
+        self.greyscale = greyscale
+
+        additional_hidden_dims = 1 if greyscale else 3
+        self.image_block = nn.Sequential(
+            Conv2dBnReLU(hidden_dims[0] + additional_hidden_dims, hidden_dims[1]),
+            Conv2dBnReLU(hidden_dims[1], hidden_dims[2]),
+        )
+
+        self.classifier = nn.Conv2d(hidden_dims[2], num_classes, kernel_size=1)
+
+    @override
+    def forward(self, decoder_inputs: DecoderInputs) -> torch.Tensor:
+        if decoder_inputs.images is None:
+            raise ValueError("Input images are missing.")
+
+        logits = super().forward(decoder_inputs)
+        in_images = (
+            rgb_to_grayscale(decoder_inputs.images) if self.greyscale else decoder_inputs.images
+        )
+        logits = torch.cat([logits, in_images], dim=1)
+        logits = self.image_block(logits)
+
+        return self.classifier(logits)
+
+
+class Conv2dBnReLU(nn.Sequential):
+    """A single convolutional layer with batch normalization and ReLU activation."""
+
+    def __init__(
+        self, in_channels: int, out_channels: int, kernel_size: int = 3, padding: int = 1
+    ) -> None:
+        """Initializes the layer.
+
+        Args:
+            in_channels: Number of input channels.
+            out_channels: Number of output channels.
+            kernel_size: Size of the convolutional kernel.
+            padding: Padding size for the convolutional layer.
+        """
+        super().__init__(
+            nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, padding=padding),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(inplace=True),
+        )

eva/vision/models/networks/decoders/segmentation/typings.py

@@ -0,0 +1,18 @@
+"""Type-hints for segmentation decoders."""
+
+from typing import List, NamedTuple, Tuple
+
+import torch
+
+
+class DecoderInputs(NamedTuple):
+    """Input scheme for segmentation decoders."""
+
+    features: List[torch.Tensor]
+    """List of image features generated by the encoder from the original images."""
+
+    image_size: Tuple[int, int]
+    """Size of the original input images to be used for upsampling."""
+
+    images: torch.Tensor | None = None
+    """The original input images for which the encoder generated the encoded_images."""

eva/vision/utils/colormap.py

@@ -1,5 +1,7 @@
 """Color mapping constants."""
 
+from typing import List, Tuple
+
 COLORS = [
     (0, 0, 0),
     (255, 0, 0),  # Red
@@ -75,3 +77,21 @@ COLORS = [
 
 COLORMAP = dict(enumerate(COLORS)) | {255: (255, 255, 255)}
 """Class id to RGB color mapping."""
+
+
+def get_colors(num_colors: int) -> List[Tuple[int, int, int]]:
+    """Get a list of RGB colors.
+
+    If the number of colors is greater than the predefined colors, it will
+    repeat the colors until it reaches the requested number
+
+    Args:
+        num_colors: The number of colors to return.
+
+    Returns:
+        A list of RGB colors.
+    """
+    colors = COLORS
+    while len(colors) < num_colors:
+        colors = colors + COLORS[1:]
+    return colors

eva/vision/utils/io/__init__.py

@@ -2,7 +2,12 @@
 
 from eva.vision.utils.io.image import read_image, read_image_as_array, read_image_as_tensor
 from eva.vision.utils.io.mat import read_mat, save_mat
-from eva.vision.utils.io.nifti import fetch_nifti_shape, read_nifti, save_array_as_nifti
+from eva.vision.utils.io.nifti import (
+    fetch_nifti_axis_direction_code,
+    fetch_nifti_shape,
+    read_nifti,
+    save_array_as_nifti,
+)
 from eva.vision.utils.io.text import read_csv
 
 __all__ = [
@@ -10,6 +15,7 @@ __all__ = [
     "read_image_as_array",
     "read_image_as_tensor",
     "fetch_nifti_shape",
+    "fetch_nifti_axis_direction_code",
     "read_nifti",
     "save_array_as_nifti",
     "read_csv",

eva/vision/utils/io/nifti.py

@@ -5,6 +5,7 @@ from typing import Any, Tuple
 import nibabel as nib
 import numpy as np
 import numpy.typing as npt
+from nibabel import orientations
 
 from eva.vision.utils.io import _utils
 
@@ -28,13 +29,13 @@ def read_nifti(
         ValueError: If the input channel is invalid for the image.
     """
     _utils.check_file(path)
-    image_data = nib.load(path)  # type: ignore
+    image_data: nib.Nifti1Image = nib.load(path)  # type: ignore
     if slice_index is not None:
-        image_data = image_data.slicer[:, :, slice_index : slice_index + 1]  # type: ignore
+        image_data = image_data.slicer[:, :, slice_index : slice_index + 1]
 
-    image_array = image_data.get_fdata()  # type: ignore
+    image_array = image_data.get_fdata()
     if use_storage_dtype:
-        image_array = image_array.astype(image_data.get_data_dtype())  # type: ignore
+        image_array = image_array.astype(image_data.get_data_dtype())
 
     return image_array
 
@@ -73,3 +74,17 @@ def fetch_nifti_shape(path: str) -> Tuple[int]:
     _utils.check_file(path)
     image = nib.load(path)  # type: ignore
     return image.header.get_data_shape()  # type: ignore
+
+
+def fetch_nifti_axis_direction_code(path: str) -> str:
+    """Fetches the NIfTI axis direction code from a file.
+
+    Args:
+        path: The path to the NIfTI file.
+
+    Returns:
+        The axis direction codes as string (e.g. "LAS").
+    """
+    _utils.check_file(path)
+    image_data: nib.Nifti1Image = nib.load(path)  # type: ignore
+    return "".join(orientations.aff2axcodes(image_data.affine))

{kaiko_eva-0.1.1.dist-info → kaiko_eva-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaiko-eva
-Version: 0.1.1
+Version: 0.1.5
 Summary: Evaluation Framework for oncology foundation models.
 Keywords: machine-learning,evaluation-framework,oncology,foundation-models
 Author-Email: Ioannis Gatopoulos <ioannis@kaiko.ai>, =?utf-8?q?Nicolas_K=C3=A4nzig?= <nicolas@kaiko.ai>, Roman Moser <roman@kaiko.ai>
@@ -216,14 +216,14 @@ Project-URL: Homepage, https://kaiko-ai.github.io/eva/dev/
 Project-URL: Repository, https://github.com/kaiko-ai/eva
 Project-URL: Documentation, https://kaiko-ai.github.io/eva/dev/
 Requires-Python: >=3.10
-Requires-Dist: torch==2.3.0
-Requires-Dist: lightning>=2.2.2
-Requires-Dist: jsonargparse[omegaconf]==4.31.0
+Requires-Dist: torch>=2.3.0
+Requires-Dist: lightning>=2.2.0
+Requires-Dist: jsonargparse[omegaconf]>=4.30.0
 Requires-Dist: tensorboard>=2.16.2
 Requires-Dist: loguru>=0.7.2
-Requires-Dist: pandas>=2.2.0
+Requires-Dist: pandas>=2.0.0
 Requires-Dist: transformers>=4.38.2
-Requires-Dist: onnxruntime>=1.17.1
+Requires-Dist: onnxruntime>=1.15.1
 Requires-Dist: onnx>=1.16.0
 Requires-Dist: toolz>=0.12.1
 Requires-Dist: rich>=13.7.1
@@ -468,41 +468,10 @@ and [tutorials](https://kaiko-ai.github.io/eva/dev/user-guide/advanced/replicate
 
 ## Leaderboards
 
-In this section you will find model benchmarks which were generated with _`eva`_.
+The following table shows the FMs we have evaluated with _`eva`_. For more detailed information about the evaluation process, please refer to our [documentation](https://kaiko-ai.github.io/eva/main/leaderboards/).
 
-### Table I: WSI and microscopy image tasks
+![Pathology Leaderboard](./docs/images/leaderboard.svg)
 
-<br />
-
-<div align="center">
-
-| Model   | BACH  | CRC   | MHIST | PCam   | Camelyon16 | PANDA | CoNSeP | MoNuSAC |
-|---------|-------|-------|-------|--------|------------|-------|------------|-------|
-| ViT-S/16 _(random)_	<sup>[1]</sup> | 0.411|0.613|0.5|0.752|0.551|0.347|0.489|0.394|
-| ViT-S/16 _(ImageNet)_ <sup>[1]</sup> | 0.675|0.936|0.827|0.861|0.751|0.676|0.54|0.512|
-| DINO<sub>(p=16)</sub> <sup>[2]</sup> | 0.77|0.936|0.751|0.905|0.869|0.737|0.625|0.549|
-| Phikon <sup>[3]</sup>                | 0.715|0.942|0.766|0.925|0.879|0.784|0.68|0.554|
-| UNI <sup>[4]</sup>                   | 0.797|0.95|0.835|0.939|0.933|0.774|0.67|0.575|
-| ViT-S/16 _(kaiko.ai)_ <sup>[5]</sup> | 0.8|0.949|0.831|0.902|0.897|0.77|0.622|0.573|
-| ViT-S/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.825|0.948|0.826|0.887|0.879|0.741|0.677|0.617|
-| ViT-B/16 _(kaiko.ai)_	<sup>[5]</sup> | 0.846|0.959|0.839|0.906|0.891|0.753|0.647|0.572|
-| ViT-B/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.867|0.952|0.814|0.921|0.939|0.761|0.706|0.661|
-| ViT-L/14 _(kaiko.ai)_ <sup>[5]</sup> | 0.862|0.935|0.822|0.907|0.941|0.769|0.686|0.599|
-
-_Table I: Linear probing evaluation of FMs on patch-level downstream datasets.<br> We report balanced accuracy
- for classification tasks and generalized Dice score for semgetnation tasks, averaged over 5 runs. Results are 
- reported on the "test" split if available and otherwise on the "validation" split._
-
-</div>
-
-<br />
-
-_References_:
-1. _"Emerging properties in self-supervised vision transformers”_, [arXiv](https://arxiv.org/abs/2104.14294)
-2. _"Benchmarking self-supervised learning on diverse pathology datasets”_, [arXiv](https://arxiv.org/abs/2212.04690)
-3. _"Scaling self-supervised learning for histopathology with masked image modeling”_, [medRxiv](https://www.medrxiv.org/content/10.1101/2023.07.21.23292757v1)
-4. _"A General-Purpose Self-Supervised Model for Computational Pathology”_, [arXiv](https://arxiv.org/abs/2308.15474)
-5. _"Towards Training Large-Scale Pathology Foundation Models: from TCGA to Hospital Scale”_, [arXiv](https://arxiv.org/pdf/2404.15217)
 
 ## Contributing