kaiko-eva 0.0.0.dev6__py3-none-any.whl

eva/vision/data/datasets/_validators.py

@@ -0,0 +1,44 @@
+"""Dataset validation related functions."""
+
+from typing_extensions import List, Tuple
+
+from eva.vision.data.datasets import vision
+
+_SUFFIX_ERROR_MESSAGE = "Please verify that the data are properly downloaded and stored."
+"""Common suffix dataset verification error message."""
+
+
+def check_dataset_integrity(
+    dataset: vision.VisionDataset,
+    *,
+    length: int,
+    n_classes: int,
+    first_and_last_labels: Tuple[str, str],
+) -> None:
+    """Verifies the datasets integrity.
+
+    Raise:
+        ValuesError: If the input dataset's values do not
+            match the expected ones.
+    """
+    if len(dataset) != length:
+        raise ValueError(
+            f"Dataset's '{dataset.__class__.__qualname__}' length "
+            f"({len(dataset)}) does not match the expected one ({length}). "
+            f"{_SUFFIX_ERROR_MESSAGE}"
+        )
+
+    dataset_classes: List[str] = getattr(dataset, "classes", [])
+    if dataset_classes and len(dataset_classes) != n_classes:
+        raise ValueError(
+            f"Dataset's '{dataset.__class__.__qualname__}' number of classes "
+            f"({len(dataset_classes)}) does not match the expected one ({n_classes})."
+            f"{_SUFFIX_ERROR_MESSAGE}"
+        )
+
+    if dataset_classes and (dataset_classes[0], dataset_classes[-1]) != first_and_last_labels:
+        raise ValueError(
+            f"Dataset's '{dataset.__class__.__qualname__}' first and last labels "
+            f"({(dataset_classes[0], dataset_classes[-1])}) does not match the expected "
+            f"ones ({first_and_last_labels}). {_SUFFIX_ERROR_MESSAGE}"
+        )

eva/vision/data/datasets/classification/__init__.py

@@ -0,0 +1,15 @@
+"""Image classification datasets API."""
+
+from eva.vision.data.datasets.classification.bach import BACH
+from eva.vision.data.datasets.classification.crc import CRC
+from eva.vision.data.datasets.classification.mhist import MHIST
+from eva.vision.data.datasets.classification.patch_camelyon import PatchCamelyon
+from eva.vision.data.datasets.classification.total_segmentator import TotalSegmentatorClassification
+
+__all__ = [
+    "BACH",
+    "CRC",
+    "MHIST",
+    "PatchCamelyon",
+    "TotalSegmentatorClassification",
+]

eva/vision/data/datasets/classification/bach.py

@@ -0,0 +1,174 @@
+"""BACH dataset class."""
+
+import os
+from typing import Callable, Dict, List, Literal, Tuple
+
+import numpy as np
+from torchvision.datasets import folder, utils
+from typing_extensions import override
+
+from eva.vision.data.datasets import _utils, _validators, structs
+from eva.vision.data.datasets.classification import base
+from eva.vision.utils import io
+
+
+class BACH(base.ImageClassification):
+    """Dataset class for BACH images and corresponding targets."""
+
+    _train_index_ranges: List[Tuple[int, int]] = [
+        (0, 41),
+        (59, 60),
+        (90, 139),
+        (169, 240),
+        (258, 260),
+        (273, 345),
+        (368, 400),
+    ]
+    """Train range indices."""
+
+    _val_index_ranges: List[Tuple[int, int]] = [
+        (41, 59),
+        (60, 90),
+        (139, 169),
+        (240, 258),
+        (260, 273),
+        (345, 368),
+    ]
+    """Validation range indices."""
+
+    _resources: List[structs.DownloadResource] = [
+        structs.DownloadResource(
+            filename="ICIAR2018_BACH_Challenge.zip",
+            url="https://zenodo.org/records/3632035/files/ICIAR2018_BACH_Challenge.zip",
+        ),
+    ]
+    """Dataset resources."""
+
+    _license: str = "CC BY-NC-ND 4.0 (https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode)"
+    """Dataset license."""
+
+    def __init__(
+        self,
+        root: str,
+        split: Literal["train", "val"] | None = None,
+        download: bool = False,
+        image_transforms: Callable | None = None,
+        target_transforms: Callable | None = None,
+    ) -> None:
+        """Initialize the dataset.
+
+        The dataset is split into train and validation by taking into account
+        the patient IDs to avoid any data leakage.
+
+        Args:
+            root: Path to the root directory of the dataset. The dataset will
+                be downloaded and extracted here, if it does not already exist.
+            split: Dataset split to use. If `None`, the entire dataset is used.
+            download: Whether to download the data for the specified split.
+                Note that the download will be executed only by additionally
+                calling the :meth:`prepare_data` method and if the data does
+                not yet exist on disk.
+            image_transforms: A function/transform that takes in an image
+                and returns a transformed version.
+            target_transforms: A function/transform that takes in the target
+                and transforms it.
+        """
+        super().__init__(
+            image_transforms=image_transforms,
+            target_transforms=target_transforms,
+        )
+
+        self._root = root
+        self._split = split
+        self._download = download
+
+        self._samples: List[Tuple[str, int]] = []
+        self._indices: List[int] = []
+
+    @property
+    @override
+    def classes(self) -> List[str]:
+        return ["Benign", "InSitu", "Invasive", "Normal"]
+
+    @property
+    @override
+    def class_to_idx(self) -> Dict[str, int]:
+        return {"Benign": 0, "InSitu": 1, "Invasive": 2, "Normal": 3}
+
+    @property
+    def dataset_path(self) -> str:
+        """Returns the path of the image data of the dataset."""
+        return os.path.join(self._root, "ICIAR2018_BACH_Challenge", "Photos")
+
+    @override
+    def filename(self, index: int) -> str:
+        image_path, _ = self._samples[self._indices[index]]
+        return os.path.relpath(image_path, self.dataset_path)
+
+    @override
+    def prepare_data(self) -> None:
+        if self._download:
+            self._download_dataset()
+
+    @override
+    def configure(self) -> None:
+        self._samples = folder.make_dataset(
+            directory=self.dataset_path,
+            class_to_idx=self.class_to_idx,
+            extensions=(".tif"),
+        )
+        self._indices = self._make_indices()
+
+    @override
+    def validate(self) -> None:
+        _validators.check_dataset_integrity(
+            self,
+            length=268 if self._split == "train" else 132,
+            n_classes=4,
+            first_and_last_labels=("Benign", "Normal"),
+        )
+
+    @override
+    def load_image(self, index: int) -> np.ndarray:
+        image_path, _ = self._samples[self._indices[index]]
+        return io.read_image(image_path)
+
+    @override
+    def load_target(self, index: int) -> np.ndarray:
+        _, target = self._samples[self._indices[index]]
+        return np.asarray(target, dtype=np.int64)
+
+    @override
+    def __len__(self) -> int:
+        return len(self._indices)
+
+    def _download_dataset(self) -> None:
+        """Downloads the dataset."""
+        for resource in self._resources:
+            if os.path.isdir(self.dataset_path):
+                continue
+
+            self._print_license()
+            utils.download_and_extract_archive(
+                resource.url,
+                download_root=self._root,
+                filename=resource.filename,
+                remove_finished=True,
+            )
+
+    def _print_license(self) -> None:
+        """Prints the dataset license."""
+        print(f"Dataset license: {self._license}")
+
+    def _make_indices(self) -> List[int]:
+        """Builds the dataset indices for the specified split."""
+        split_index_ranges = {
+            "train": self._train_index_ranges,
+            "val": self._val_index_ranges,
+            None: [(0, 400)],
+        }
+        index_ranges = split_index_ranges.get(self._split)
+        if index_ranges is None:
+            raise ValueError("Invalid data split. Use 'train', 'val' or `None`.")
+
+        return _utils.ranges_to_indices(index_ranges)

eva/vision/data/datasets/classification/base.py

@@ -0,0 +1,103 @@
+"""Base for image classification datasets."""
+
+import abc
+from typing import Any, Callable, Dict, List, Tuple
+
+import numpy as np
+from typing_extensions import override
+
+from eva.vision.data.datasets import vision
+
+
+class ImageClassification(vision.VisionDataset[Tuple[np.ndarray, np.ndarray]], abc.ABC):
+    """Image classification abstract dataset."""
+
+    def __init__(
+        self,
+        image_transforms: Callable | None = None,
+        target_transforms: Callable | None = None,
+    ) -> None:
+        """Initializes the image classification dataset.
+
+        Args:
+            image_transforms: A function/transform that takes in an image
+                and returns a transformed version.
+            target_transforms: A function/transform that takes in the target
+                and transforms it.
+        """
+        super().__init__()
+
+        self._image_transforms = image_transforms
+        self._target_transforms = target_transforms
+
+    @property
+    def classes(self) -> List[str] | None:
+        """Returns the list with names of the dataset names."""
+
+    @property
+    def class_to_idx(self) -> Dict[str, int] | None:
+        """Returns a mapping of the class name to its target index."""
+
+    def load_metadata(self, index: int | None) -> Dict[str, Any] | List[Dict[str, Any]] | None:
+        """Returns the dataset metadata.
+
+        Args:
+            index: The index of the data sample to return the metadata of.
+                If `None`, it will return the metadata of the current dataset.
+
+        Returns:
+            The sample metadata.
+        """
+
+    @abc.abstractmethod
+    def load_image(self, index: int) -> np.ndarray:
+        """Returns the `index`'th image sample.
+
+        Args:
+            index: The index of the data sample to load.
+
+        Returns:
+            The image as a numpy array.
+        """
+
+    @abc.abstractmethod
+    def load_target(self, index: int) -> np.ndarray:
+        """Returns the `index`'th target sample.
+
+        Args:
+            index: The index of the data sample to load.
+
+        Returns:
+            The sample target as an array.
+        """
+
+    @abc.abstractmethod
+    @override
+    def __len__(self) -> int:
+        raise NotImplementedError
+
+    @override
+    def __getitem__(self, index: int) -> Tuple[np.ndarray, np.ndarray]:
+        image = self.load_image(index)
+        target = self.load_target(index)
+        return self._apply_transforms(image, target)
+
+    def _apply_transforms(
+        self, image: np.ndarray, target: np.ndarray
+    ) -> Tuple[np.ndarray, np.ndarray]:
+        """Applies the transforms to the provided data and returns them.
+
+        Args:
+            image: The desired image.
+            target: The target of the image.
+
+        Returns:
+            A tuple with the image and the target transformed.
+        """
+        if self._image_transforms is not None:
+            image = self._image_transforms(image)
+
+        if self._target_transforms is not None:
+            target = self._target_transforms(target)
+
+        return image, target

eva/vision/data/datasets/classification/crc.py

@@ -0,0 +1,176 @@
+"""CRC dataset class."""
+
+import os
+from typing import Callable, Dict, List, Literal, Tuple
+
+import numpy as np
+from torchvision.datasets import folder, utils
+from typing_extensions import override
+
+from eva.vision.data.datasets import _validators, structs
+from eva.vision.data.datasets.classification import base
+from eva.vision.utils import io
+
+
+class CRC(base.ImageClassification):
+    """Dataset class for CRC images and corresponding targets."""
+
+    _train_resource: structs.DownloadResource = structs.DownloadResource(
+        filename="NCT-CRC-HE-100K.zip",
+        url="https://zenodo.org/records/1214456/files/NCT-CRC-HE-100K.zip?download=1",
+        md5="md5:035777cf327776a71a05c95da6d6325f",
+    )
+    """Train resource."""
+
+    _val_resource: structs.DownloadResource = structs.DownloadResource(
+        filename="CRC-VAL-HE-7K.zip",
+        url="https://zenodo.org/records/1214456/files/CRC-VAL-HE-7K.zip?download=1",
+        md5="md5:2fd1651b4f94ebd818ebf90ad2b6ce06",
+    )
+    """Validation resource."""
+
+    _license: str = "CC BY 4.0 LEGAL CODE (https://creativecommons.org/licenses/by/4.0/legalcode)"
+    """Dataset license."""
+
+    def __init__(
+        self,
+        root: str,
+        split: Literal["train", "val"],
+        download: bool = False,
+        image_transforms: Callable | None = None,
+        target_transforms: Callable | None = None,
+    ) -> None:
+        """Initializes the dataset.
+
+        The dataset is split into a train (train) and validation (val) set:
+          - train: A set of 100,000 non-overlapping image patches from
+            hematoxylin & eosin (H&E) stained histological images of human
+            colorectal cancer (CRC) and normal tissue.
+          - val: A set of 7180 image patches from N=50 patients with colorectal
+            adenocarcinoma (no overlap with patients in NCT-CRC-HE-100K).
+
+        Args:
+            root: Path to the root directory of the dataset.
+            split: Dataset split to use.
+            download: Whether to download the data for the specified split.
+                Note that the download will be executed only by additionally
+                calling the :meth:`prepare_data` method and if the data does
+                not yet exist on disk.
+            image_transforms: A function/transform that takes in an image
+                and returns a transformed version.
+            target_transforms: A function/transform that takes in the target
+                and transforms it.
+        """
+        super().__init__(
+            image_transforms=image_transforms,
+            target_transforms=target_transforms,
+        )
+
+        self._root = root
+        self._split = split
+        self._download = download
+
+        self._samples: List[Tuple[str, int]] = []
+
+    @property
+    @override
+    def classes(self) -> List[str]:
+        return ["ADI", "BACK", "DEB", "LYM", "MUC", "MUS", "NORM", "STR", "TUM"]
+
+    @property
+    @override
+    def class_to_idx(self) -> Dict[str, int]:
+        return {
+            "ADI": 0,
+            "BACK": 1,
+            "DEB": 2,
+            "LYM": 3,
+            "MUC": 4,
+            "MUS": 5,
+            "NORM": 6,
+            "STR": 7,
+            "TUM": 8,
+        }
+
+    @override
+    def filename(self, index: int) -> str:
+        image_path, *_ = self._samples[index]
+        return os.path.relpath(image_path, self._dataset_dir)
+
+    @override
+    def prepare_data(self) -> None:
+        if self._download:
+            self._download_dataset()
+
+    @override
+    def configure(self) -> None:
+        self._samples = self._make_dataset()
+
+    @override
+    def validate(self) -> None:
+        expected_length = {
+            "train": 100000,
+            "val": 7180,
+            None: 107180,
+        }
+        _validators.check_dataset_integrity(
+            self,
+            length=expected_length.get(self._split, 0),
+            n_classes=9,
+            first_and_last_labels=("ADI", "TUM"),
+        )
+
+    @override
+    def load_image(self, index: int) -> np.ndarray:
+        image_path, _ = self._samples[index]
+        return io.read_image(image_path)
+
+    @override
+    def load_target(self, index: int) -> np.ndarray:
+        _, target = self._samples[index]
+        return np.asarray(target, dtype=np.int64)
+
+    @override
+    def __len__(self) -> int:
+        return len(self._samples)
+
+    @property
+    def _dataset_dir(self) -> str:
+        """Returns the full path of dataset directory."""
+        dataset_dirs = {
+            "train": os.path.join(self._root, "NCT-CRC-HE-100K"),
+            "val": os.path.join(self._root, "CRC-VAL-HE-7K"),
+        }
+        dataset_dir = dataset_dirs.get(self._split)
+        if dataset_dir is None:
+            raise ValueError("Invalid data split. Use 'train' or 'val'.")
+
+        return dataset_dir
+
+    def _make_dataset(self) -> List[Tuple[str, int]]:
+        """Builds the dataset for the specified split."""
+        dataset = folder.make_dataset(
+            directory=self._dataset_dir,
+            class_to_idx=self.class_to_idx,
+            extensions=(".tif"),
+        )
+        return dataset
+
+    def _download_dataset(self) -> None:
+        """Downloads the dataset resources."""
+        for resource in [self._train_resource, self._val_resource]:
+            resource_dir = resource.filename.rsplit(".", maxsplit=1)[0]
+            if os.path.isdir(os.path.join(self._root, resource_dir)):
+                continue
+
+            self._print_license()
+            utils.download_and_extract_archive(
+                resource.url,
+                download_root=self._root,
+                filename=resource.filename,
+                remove_finished=True,
+            )
+
+    def _print_license(self) -> None:
+        """Prints the dataset license."""
+        print(f"Dataset license: {self._license}")

eva/vision/data/datasets/classification/mhist.py

@@ -0,0 +1,106 @@
+"""MHIST dataset class."""
+
+import os
+from typing import Callable, Dict, List, Literal, Tuple
+
+import numpy as np
+from typing_extensions import override
+
+from eva.vision.data.datasets import _validators
+from eva.vision.data.datasets.classification import base
+from eva.vision.utils import io
+
+
+class MHIST(base.ImageClassification):
+    """MHIST dataset."""
+
+    def __init__(
+        self,
+        root: str,
+        split: Literal["train", "test"],
+        image_transforms: Callable | None = None,
+        target_transforms: Callable | None = None,
+    ) -> None:
+        """Initialize the dataset.
+
+        Args:
+            root: Path to the root directory of the dataset.
+            split: Dataset split to use.
+            image_transforms: A function/transform that takes in an image
+                and returns a transformed version.
+            target_transforms: A function/transform that takes in the target
+                and transforms it.
+        """
+        super().__init__(
+            image_transforms=image_transforms,
+            target_transforms=target_transforms,
+        )
+
+        self._root = root
+        self._split = split
+
+        self._samples: List[Tuple[str, str]] = []
+
+    @property
+    @override
+    def classes(self) -> List[str]:
+        return ["SSA", "HP"]
+
+    @property
+    @override
+    def class_to_idx(self) -> Dict[str, int]:
+        return {"SSA": 0, "HP": 1}
+
+    @override
+    def filename(self, index: int) -> str:
+        image_filename, _ = self._samples[index]
+        return image_filename
+
+    @override
+    def configure(self) -> None:
+        self._samples = self._make_dataset()
+
+    @override
+    def validate(self) -> None:
+        _validators.check_dataset_integrity(
+            self,
+            length=2175 if self._split == "train" else 977,
+            n_classes=2,
+            first_and_last_labels=("SSA", "HP"),
+        )
+
+    @override
+    def load_image(self, index: int) -> np.ndarray:
+        image_filename, _ = self._samples[index]
+        image_path = os.path.join(self._dataset_path, image_filename)
+        return io.read_image(image_path)
+
+    @override
+    def load_target(self, index: int) -> np.ndarray:
+        _, label = self._samples[index]
+        target = self.class_to_idx[label]
+        return np.asarray(target, dtype=np.int64)
+
+    @override
+    def __len__(self) -> int:
+        return len(self._samples)
+
+    def _make_dataset(self) -> List[Tuple[str, str]]:
+        """Generates and returns a list of samples of a form (image_filename, label)."""
+        data = io.read_csv(self._annotations_path)
+        samples = [
+            (sample["Image Name"], sample["Majority Vote Label"])
+            for sample in data
+            if sample["Partition"] == self._split
+        ]
+        return samples
+
+    @property
+    def _dataset_path(self) -> str:
+        """Returns the path of the image data of the dataset."""
+        return os.path.join(self._root, "images")
+
+    @property
+    def _annotations_path(self) -> str:
+        """Returns the path of the annotations file of the dataset."""
+        return os.path.join(self._root, "annotations.csv")