kaiko-eva 0.0.0.dev6__py3-none-any.whl → 0.0.0.dev8__py3-none-any.whl

eva/core/data/datamodules/call.py

@@ -19,7 +19,7 @@ def call_method_if_exists(objects: Iterable[Any], /, method: str) -> None:
 
 
 def _recursive_iter(objects: Iterable[Any], /) -> Iterable[datasets_lib.TorchDataset]:
-    """Iterates thought an iterable of objects and their respective iterable values.
+    """Iterates through an iterable of objects and their respective iterable values.
 
     Args:
         objects: The objects to iterate from.

eva/core/trainers/_logging.py

@@ -11,8 +11,9 @@ from loguru import logger
 def generate_session_id() -> str:
     """Generates and returns a unique string ID of an experiment.
 
-    The ID is composed of the run timestamp and a its config hash. If the
-    configuration hash is an empty string, it will use only the timestamp.
+    The ID is composed of the run timestamp and a hash based on th used
+    config. If the configuration hash is an empty string, it will use
+    only the timestamp.
     """
     timestamp = _generate_timestamp_hash()
     config_hash = _generate_config_hash()
@@ -34,8 +35,8 @@ def _generate_config_hash(max_hash_len: int = 8) -> str:
     config_path = _fetch_config_path()
     if config_path is None:
         logger.warning(
-            "No or multiple configuration file found from command line arguments. "
-            "No configuration hash code will created for this experiment."
+            "No or multiple configuration files found from command line arguments. "
+            "No configuration hash code will be created for this experiment."
         )
         return ""
 

eva/core/trainers/trainer.py

@@ -78,7 +78,7 @@ class Trainer(pl_trainer.Trainer):
         model: modules.ModelModule,
         datamodule: datamodules.DataModule,
     ) -> None:
-        """Runs a evaluation session out-of-place.
+        """Runs an evaluation session out-of-place.
 
         It performs an evaluation run (fit and evaluate) the model
         `self._n_run` times. Note that the input `base_model` would

eva/vision/data/datasets/_validators.py

@@ -1,5 +1,7 @@
 """Dataset validation related functions."""
 
+import os
+
 from typing_extensions import List, Tuple
 
 from eva.vision.data.datasets import vision
@@ -18,7 +20,7 @@ def check_dataset_integrity(
     """Verifies the datasets integrity.
 
     Raise:
-        ValuesError: If the input dataset's values do not
+        ValueError: If the input dataset's values do not
             match the expected ones.
     """
     if len(dataset) != length:
@@ -42,3 +44,16 @@ def check_dataset_integrity(
             f"({(dataset_classes[0], dataset_classes[-1])}) does not match the expected "
             f"ones ({first_and_last_labels}). {_SUFFIX_ERROR_MESSAGE}"
         )
+
+
+def check_dataset_exists(dataset_dir: str, download_available: bool) -> None:
+    """Verifies that the dataset folder exists.
+
+    Raise:
+        FileNotFoundError: If the dataset folder does not exist.
+    """
+    if not os.path.isdir(dataset_dir):
+        error_message = f"Dataset not found at '{dataset_dir}'."
+        if download_available:
+            error_message += " You can set `download=True` to download the dataset automatically."
+        raise FileNotFoundError(error_message)

eva/vision/data/datasets/classification/bach.py

@@ -96,24 +96,25 @@ class BACH(base.ImageClassification):
         return {"Benign": 0, "InSitu": 1, "Invasive": 2, "Normal": 3}
 
     @property
-    def dataset_path(self) -> str:
+    def _dataset_path(self) -> str:
         """Returns the path of the image data of the dataset."""
         return os.path.join(self._root, "ICIAR2018_BACH_Challenge", "Photos")
 
     @override
     def filename(self, index: int) -> str:
         image_path, _ = self._samples[self._indices[index]]
-        return os.path.relpath(image_path, self.dataset_path)
+        return os.path.relpath(image_path, self._dataset_path)
 
     @override
     def prepare_data(self) -> None:
         if self._download:
             self._download_dataset()
+        _validators.check_dataset_exists(self._root, True)
 
     @override
     def configure(self) -> None:
         self._samples = folder.make_dataset(
-            directory=self.dataset_path,
+            directory=self._dataset_path,
             class_to_idx=self.class_to_idx,
             extensions=(".tif"),
         )
@@ -145,7 +146,7 @@ class BACH(base.ImageClassification):
     def _download_dataset(self) -> None:
         """Downloads the dataset."""
         for resource in self._resources:
-            if os.path.isdir(self.dataset_path):
+            if os.path.isdir(self._dataset_path):
                 continue
 
             self._print_license()

eva/vision/data/datasets/classification/crc.py

@@ -95,12 +95,13 @@ class CRC(base.ImageClassification):
     @override
     def filename(self, index: int) -> str:
         image_path, *_ = self._samples[index]
-        return os.path.relpath(image_path, self._dataset_dir)
+        return os.path.relpath(image_path, self._dataset_path)
 
     @override
     def prepare_data(self) -> None:
         if self._download:
             self._download_dataset()
+        _validators.check_dataset_exists(self._root, True)
 
     @override
     def configure(self) -> None:
@@ -135,7 +136,7 @@ class CRC(base.ImageClassification):
         return len(self._samples)
 
     @property
-    def _dataset_dir(self) -> str:
+    def _dataset_path(self) -> str:
         """Returns the full path of dataset directory."""
         dataset_dirs = {
             "train": os.path.join(self._root, "NCT-CRC-HE-100K"),
@@ -150,7 +151,7 @@ class CRC(base.ImageClassification):
     def _make_dataset(self) -> List[Tuple[str, int]]:
         """Builds the dataset for the specified split."""
         dataset = folder.make_dataset(
-            directory=self._dataset_dir,
+            directory=self._dataset_path,
             class_to_idx=self.class_to_idx,
             extensions=(".tif"),
         )

eva/vision/data/datasets/classification/mhist.py

@@ -56,6 +56,10 @@ class MHIST(base.ImageClassification):
         image_filename, _ = self._samples[index]
         return image_filename
 
+    @override
+    def prepare_data(self) -> None:
+        _validators.check_dataset_exists(self._root, False)
+
     @override
     def configure(self) -> None:
         self._samples = self._make_dataset()

eva/vision/data/datasets/classification/patch_camelyon.py

@@ -114,6 +114,7 @@ class PatchCamelyon(base.ImageClassification):
     def prepare_data(self) -> None:
         if self._download:
             self._download_dataset()
+        _validators.check_dataset_exists(self._root, True)
 
     @override
     def validate(self) -> None:

eva/vision/data/datasets/classification/total_segmentator.py

@@ -108,6 +108,7 @@ class TotalSegmentatorClassification(base.ImageClassification):
     def prepare_data(self) -> None:
         if self._download:
             self._download_dataset()
+        _validators.check_dataset_exists(self._root, True)
 
     @override
     def configure(self) -> None:

{kaiko_eva-0.0.0.dev6.dist-info → kaiko_eva-0.0.0.dev8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaiko-eva
-Version: 0.0.0.dev6
+Version: 0.0.0.dev8
 Summary: Evaluation Framework for oncology foundation models.
 Keywords: machine-learning evaluation-framework oncology foundation-models
 Author-Email: Ioannis Gatopoulos <ioannis@kaiko.ai>, Nicolas Känzig <nicolas@kaiko.ai>, Roman Moser <roman@kaiko.ai>
@@ -240,7 +240,7 @@ Description-Content-Type: text/markdown
 
 <div align="center">
 
-<img src="./docs/images/eva-logo.png" width="400">
+<img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/eva-logo.png?raw=true" width="400">
 
 <br />
 
@@ -251,27 +251,21 @@ _Oncology FM Evaluation Framework by kaiko.ai_
 [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg?labelColor=gray)](https://github.com/kaiko-ai/eva#license)
 
 <p align="center">
-  <a href="#installation">Installation</a> •
-  <a href="#how-to-use">How To Use</a> •
+  <a href="https://github.com/kaiko-ai/eva#installation">Installation</a> •
+  <a href="https://github.com/kaiko-ai/eva#how-to-use">How To Use</a> •
   <a href="https://kaiko-ai.github.io/eva/">Documentation</a> •
   <a href="https://kaiko-ai.github.io/eva/dev/datasets/">Datasets</a> •
-  <a href="#benchmarks">Benchmarks</a> <br>
-  <a href="#contributing">Contribute</a> •
-  <a href="#acknowledgements">Acknowledgements</a>
+  <a href="https://github.com/kaiko-ai/eva#benchmarks">Benchmarks</a> <br>
+  <a href="https://github.com/kaiko-ai/eva#contributing">Contribute</a> •
+  <a href="https://github.com/kaiko-ai/eva#acknowledgements">Acknowledgements</a>
 </p>
 
 </div>
 
-### Introduction
+<br />
 
 _`eva`_ is an evaluation framework for oncology foundation models (FMs) by [kaiko.ai](https://kaiko.ai/). Check out the [documentation](https://kaiko-ai.github.io/eva/) for more information.
 
-<div align="center">
-
-<img src="./docs/images/eva-process.gif" width="800">
-
-</div>
-
 ### Highlights:
 - Easy and reliable benchmark of Oncology FMs
 - Automatic embedding inference and evaluation of a downstream task
@@ -294,7 +288,7 @@ pip install 'kaiko-eva[all]'
 
 To install the latest version of the `main` branch:
 ```sh
-pip install "kaiko-eva[vision] @ git+https://github.com/kaiko-ai/eva.git"
+pip install "kaiko-eva[all] @ git+https://github.com/kaiko-ai/eva.git"
 ```
 
 You can verify that the installation was successful by executing:
@@ -339,8 +333,8 @@ In this section you will find model benchmarks which were generated with _eva_.
 |--------------------------------------------------|-------|-------|-------|----------|-----------|
 | ViT-S/16 _(random)_	<sup>[1]</sup>               | 0.410 | 0.617 | 0.501 | 0.753    | 0.728     |
 | ViT-S/16 _(ImageNet)_ <sup>[1]</sup>             | 0.695 | 0.935 | 0.831 | 0.864    | 0.849     |
-| ViT-B/8 _(ImageNet)_ <sup>[1]</sup>              | 0.797 | 0.943 | 0.828 | 0.903    | 0.893     |
-| DINO<sub>(p=16)</sub> <sup>[2]</sup>             | 0.710 | 0.935 | 0.814 | 0.870    | 0.856     |
+| ViT-B/8 _(ImageNet)_ <sup>[1]</sup>              | 0.710 | 0.939 | 0.814 | 0.870    | 0.856     |
+| DINO<sub>(p=16)</sub> <sup>[2]</sup>             | 0.801 | 0.934 | 0.768 | 0.889    | 0.895     |
 | Phikon <sup>[3]</sup>                            | 0.725 | 0.935 | 0.777 | 0.912    | 0.915     |
 | ViT-S/16 _(kaiko.ai)_ <sup>[4]</sup>             | 0.797 | 0.943 | 0.828 | 0.903    | 0.893     |
 | ViT-S/8 _(kaiko.ai)_ <sup>[4]</sup>              | 0.834 | 0.946 | 0.832 | 0.897    | 0.887     |
@@ -349,7 +343,7 @@ In this section you will find model benchmarks which were generated with _eva_.
 | ViT-L/14 _(kaiko.ai)_ <sup>[4]</sup>             | 0.870 | 0.930 | 0.809 | 0.908    | 0.898     |
 
 _Table I: Linear probing evaluation of FMs on patch-level downstream datasets.<br> We report averaged balanced accuracy
-over 5 runs_, with an average standard deviation of ±0.003.
+over 5 runs, with an average standard deviation of ±0.003._
 
 </div>
 
@@ -359,7 +353,7 @@ _References_:
 1. _"Emerging properties in self-supervised vision transformers”_
 2. _"Benchmarking self-supervised learning on diverse pathology datasets”_
 3. _"Scaling self-supervised learning for histopathology with masked image modeling”_
-4. _"Towards training Large-Scale Medical Foundation Models: from TCGA to hospital-scale pathology FMs”_
+4. _"Towards Training Large-Scale Pathology Foundation Models: from TCGA to Hospital Scale”_
 
 ## Contributing
 
@@ -389,5 +383,5 @@ Our codebase is built using multiple opensource contributions
 
 ---
 <div align="center">
-  <img src="./docs/images/kaiko-logo.png" width="200">
+  <img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/kaiko-logo.png?raw=true" width="200">
 </div>

{kaiko_eva-0.0.0.dev6.dist-info → kaiko_eva-0.0.0.dev8.dist-info}/RECORD

@@ -15,7 +15,7 @@ eva/core/data/__init__.py,sha256=yG3BeOWhp1EjVYMFqx8M_TBWFDyfIwwksQGQmMdSPaI,340
 eva/core/data/dataloaders/__init__.py,sha256=fbNClVZ8J3QoGi4qiPq635ig1j9GdI7six3RhfwDbjY,110
 eva/core/data/dataloaders/dataloader.py,sha256=-mWFFLtem1Ijbi8XGveFSv5XzUU7SyKwiT5Ahikzghw,2368
 eva/core/data/datamodules/__init__.py,sha256=qZchYbgxo9lxYnGoqdk0C6MfS2IbF0WItO0kCdP9Mqc,229
-eva/core/data/datamodules/call.py,sha256=BYihrDdpAhMdRLhr1tA4I-veVe12bX3PNixGokbENQU,940
+eva/core/data/datamodules/call.py,sha256=jjj9w3UXYuQB-qyCcw1EZpRJW10OC1I3dvgvsuQWLck,940
 eva/core/data/datamodules/datamodule.py,sha256=dclC2YJAXUGEUpV9ZRWQS43-ksFIPgVeFudsyrj9kdc,3878
 eva/core/data/datamodules/schemas.py,sha256=EXnUPNd9Pj3RjnxJIzAcC2qp6TtBSvPDx28fV_ovWAA,1869
 eva/core/data/datasets/__init__.py,sha256=pMU-w6aQoRFgSwPB9GLXNilnsbwd6HqLlgw49e4rlj0,281
@@ -63,11 +63,11 @@ eva/core/models/networks/wrappers/from_function.py,sha256=fuh-UEe3eppTwuSA2gEgCm
 eva/core/models/networks/wrappers/huggingface.py,sha256=81j0pcEx3DW6gR-81Fz6tZkJPBZYiQ-g45igFvkqX1o,1289
 eva/core/models/networks/wrappers/onnx.py,sha256=LZEGOpg1VYrB3wXMAA5IMfiKNTkOXQ50agHjTvYnnsU,1718
 eva/core/trainers/__init__.py,sha256=jhsKJF7HAae7EOiG3gKIAHH_h3dZlTE2JRcCHJmOzJc,208
-eva/core/trainers/_logging.py,sha256=AsYw-6NigCqPWEFCYOpbgFe5-5WxcdqMNiVuDvpqsgk,2550
+eva/core/trainers/_logging.py,sha256=gi4FqPy2GuVmh0WZY6mYwF7zMPvnoFA050B0XdCP6PU,2571
 eva/core/trainers/_recorder.py,sha256=_Vfp7Njh_9qP-SWbBGYp8solnfFgIUi2Z9pGLXt52WY,5652
 eva/core/trainers/_utils.py,sha256=M3h8lVhUmkeSiEXpX9hRdMvThGFCnTP15gv-hd1CZkc,321
 eva/core/trainers/functional.py,sha256=pIeGXoO63Wh6n1mOYlBo5ACCteGuNV9pZhqxfN4RLSs,3775
-eva/core/trainers/trainer.py,sha256=8TVQb4HXZbvp5SXK1UnU2RIX9OcUbx6cbStRD5V3o3c,3260
+eva/core/trainers/trainer.py,sha256=j4rYWiG9COxBbZ6WIlpRPBE153XqlYK7eAWlRsZgljU,3261
 eva/core/utils/__init__.py,sha256=F1C69M9y7W8qh1J2k-X4frRHa7r1mPXewscC94fFYtk,58
 eva/core/utils/io/__init__.py,sha256=SAME0kuSvDE1DKFJwMBmnCkpDAy4ujXuRTSJsHNhwUI,112
 eva/core/utils/io/dataframe.py,sha256=CIHFowljH17waDkJ9YJVEVXAIcxMwoLjUgoBttiNk8w,509
@@ -77,14 +77,14 @@ eva/vision/__init__.py,sha256=Pp9FZomuqfbZdKSOmgSGfNYk8ISpXf7tLVBZfoFcybk,432
 eva/vision/data/__init__.py,sha256=aoKPmX8P2Q2k2W3nlq8vFU41FV6Sze-0SDuWtU-ETh4,111
 eva/vision/data/datasets/__init__.py,sha256=aV4qPqtlt0PnaGoxUW_xEwAr8b8ddkl_YE4_fAdavds,497
 eva/vision/data/datasets/_utils.py,sha256=5GAZEHn-VezxTXaW1jVZO5zvdVl1Vz8_5gV2qkoMu4s,1414
-eva/vision/data/datasets/_validators.py,sha256=SewZrhs1bVXqDmdUBYOfyXNgjvGZ50-5pzdbzkelWmg,1595
+eva/vision/data/datasets/_validators.py,sha256=uPbbUNnftb8mYzsKVrF-ZX_xinB2zQkuQLFYMprVjhY,2099
 eva/vision/data/datasets/classification/__init__.py,sha256=I9vTkETzGnTNNvyRB96ut1YHx9ARmZVO0-0l3ZLWEAs,520
-eva/vision/data/datasets/classification/bach.py,sha256=jJDQSJ4SJ2W-GxbUAvuYkiPTef-g5kdtYez2ewM89KM,5574
+eva/vision/data/datasets/classification/bach.py,sha256=_xuA4evV9jCI76bUKbzom4ECLKShCsd95S8PtvhRAH4,5637
 eva/vision/data/datasets/classification/base.py,sha256=zBqn8rQP59j1DEChf3rDXgyMtB_sbug8kPvgFCqZyl4,3060
-eva/vision/data/datasets/classification/crc.py,sha256=ZTSrClCWh-jE4Rpmy_uXaUbQ4zjt4vPkqYTflX6m2SQ,5813
-eva/vision/data/datasets/classification/mhist.py,sha256=kqZivJNje_wZaHt_i6uCXMU486FbArvs0_crX0ryiao,3146
-eva/vision/data/datasets/classification/patch_camelyon.py,sha256=28Lg7umKM3S1TKNCfbpOuQyLlLVZ92BnjMmQNEnSH8I,7194
-eva/vision/data/datasets/classification/total_segmentator.py,sha256=JcRHATXpkB0uMr3XzMp9uTBG-_cQNgPhbkXqGBUMuOE,8010
+eva/vision/data/datasets/classification/crc.py,sha256=7RR0PJWnhLMa3AUB_F2XMYawF5gnCNbGMv25ejOEeNA,5875
+eva/vision/data/datasets/classification/mhist.py,sha256=yoDHZ2vqa26YKVvJ9t6aidOVGazGIwUD6F3o0zNsxjM,3257
+eva/vision/data/datasets/classification/patch_camelyon.py,sha256=CH9sveoMppNWPQHm4qPTONRSGqX3O8P3OYwMB6mO678,7253
+eva/vision/data/datasets/classification/total_segmentator.py,sha256=OkbqS41ykdUX0wGf6jSja5WzeeRmevUnH5alfcEQhwg,8069
 eva/vision/data/datasets/segmentation/__init__.py,sha256=byQCBHicM6mQkljHPllUqRvoFaJxHtPMKcyjPmK6dUM,249
 eva/vision/data/datasets/segmentation/base.py,sha256=JogXJ3KiOaUybAcyvoqjR4yjlBfVTt2Rt8OOAz32Jrc,3630
 eva/vision/data/datasets/segmentation/total_segmentator.py,sha256=NUh-NlrsTcUsbe3qLd_d481mok970bNF7zIdpAS7eks,8075
@@ -104,8 +104,8 @@ eva/vision/utils/io/_utils.py,sha256=JzOt7Frj6ScF_aNjFtfHBn4ROnl6NhUZucmQhLc4Cww
 eva/vision/utils/io/image.py,sha256=2jzeVFMvIRhuTkIrQeLyu0y8GttLp6rWRjO9I2uw-I8,1489
 eva/vision/utils/io/nifti.py,sha256=ph9w8dNNSsJG2wI3NJNPTLyWdz2S0i9jD068nHXVVJs,1510
 eva/vision/utils/io/text.py,sha256=uECChKjeKi4KQ-NqdO7ywAFS_TOEp2DQ5QQcuG8cb-4,472
-kaiko_eva-0.0.0.dev6.dist-info/METADATA,sha256=653ZvK1f_JAdJqvhuItMUhdPj_207iutLC6-OGooT2I,21424
-kaiko_eva-0.0.0.dev6.dist-info/WHEEL,sha256=N2J68yzZqJh3mI_Wg92rwhw0rtJDFpZj9bwQIMJgaVg,90
-kaiko_eva-0.0.0.dev6.dist-info/entry_points.txt,sha256=oqtS2Yt5EBY4saLyCBC3Zev3huCORKTKWyPovX7QR8g,73
-kaiko_eva-0.0.0.dev6.dist-info/licenses/LICENSE,sha256=e6AEzr7j_R-PYr2qLO-JwLn8y70jbVD3U2mxbRmwcI4,11338
-kaiko_eva-0.0.0.dev6.dist-info/RECORD,,
+kaiko_eva-0.0.0.dev8.dist-info/METADATA,sha256=M4MrBZhhW_RFJcR3J8kjSgRN7lauwINudUoEO1y6AO8,21567
+kaiko_eva-0.0.0.dev8.dist-info/WHEEL,sha256=N2J68yzZqJh3mI_Wg92rwhw0rtJDFpZj9bwQIMJgaVg,90
+kaiko_eva-0.0.0.dev8.dist-info/entry_points.txt,sha256=oqtS2Yt5EBY4saLyCBC3Zev3huCORKTKWyPovX7QR8g,73
+kaiko_eva-0.0.0.dev8.dist-info/licenses/LICENSE,sha256=e6AEzr7j_R-PYr2qLO-JwLn8y70jbVD3U2mxbRmwcI4,11338
+kaiko_eva-0.0.0.dev8.dist-info/RECORD,,