kaiko-eva 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

eva/vision/utils/io/nifti.py

@@ -1,21 +1,22 @@
 """NIfTI I/O related functions."""
 
-from typing import Any
+from typing import Any, Tuple
 
 import nibabel as nib
+import numpy as np
 import numpy.typing as npt
 
 from eva.vision.utils.io import _utils
 
 
-def read_nifti_slice(
-    path: str, slice_index: int, *, use_storage_dtype: bool = True
+def read_nifti(
+    path: str, slice_index: int | None = None, *, use_storage_dtype: bool = True
 ) -> npt.NDArray[Any]:
-    """Reads and loads a NIfTI image from a file path as `uint8`.
+    """Reads and loads a NIfTI image from a file path.
 
     Args:
         path: The path to the NIfTI file.
-        slice_index: The image slice index to return.
+        slice_index: Whether to read only a slice from the file.
         use_storage_dtype: Whether to cast the raw image
             array to the inferred type.
 
@@ -28,21 +29,42 @@ def read_nifti_slice(
     """
     _utils.check_file(path)
     image_data = nib.load(path)  # type: ignore
-    image_slice = image_data.slicer[:, :, slice_index : slice_index + 1]  # type: ignore
-    image_array = image_slice.get_fdata()
+    if slice_index is not None:
+        image_data = image_data.slicer[:, :, slice_index : slice_index + 1]  # type: ignore
+
+    image_array = image_data.get_fdata()  # type: ignore
     if use_storage_dtype:
         image_array = image_array.astype(image_data.get_data_dtype())  # type: ignore
+
     return image_array
 
 
-def fetch_total_nifti_slices(path: str) -> int:
-    """Fetches the total slides of a NIfTI image file.
+def save_array_as_nifti(
+    array: npt.ArrayLike,
+    filename: str,
+    *,
+    dtype: npt.DTypeLike | None = np.int64,
+) -> None:
+    """Saved a numpy array as a NIfTI image file.
+
+    Args:
+        array: The image array to save.
+        filename: The name to save the image like.
+        dtype: The data type to save the image.
+    """
+    nifti_image = nib.Nifti1Image(array, affine=np.eye(4), dtype=dtype)  # type: ignore
+    nifti_image.header.get_xyzt_units()
+    nifti_image.to_filename(filename)
+
+
+def fetch_nifti_shape(path: str) -> Tuple[int]:
+    """Fetches the NIfTI image shape from a file.
 
     Args:
         path: The path to the NIfTI file.
 
     Returns:
-        The number of the total available slides.
+        The image shape.
 
     Raises:
         FileExistsError: If the path does not exist or it is unreachable.
@@ -50,5 +72,4 @@ def fetch_total_nifti_slices(path: str) -> int:
     """
     _utils.check_file(path)
     image = nib.load(path)  # type: ignore
-    image_shape = image.header.get_data_shape()  # type: ignore
-    return image_shape[-1]
+    return image.header.get_data_shape()  # type: ignore

eva/vision/utils/io/text.py

@@ -4,15 +4,22 @@ import csv
 from typing import Dict, List
 
 
-def read_csv(path: str) -> List[Dict[str, str]]:
+def read_csv(
+    path: str,
+    *,
+    delimiter: str = ",",
+    encoding: str = "utf-8",
+) -> List[Dict[str, str]]:
     """Reads a CSV file and returns its contents as a list of dictionaries.
 
     Args:
         path: The path to the CSV file.
+        delimiter: The character that separates fields in the CSV file.
+        encoding: The encoding of the CSV file.
 
     Returns:
         A list of dictionaries representing the data in the CSV file.
     """
-    with open(path, newline="") as file:
-        data = csv.DictReader(file, skipinitialspace=True)
+    with open(path, newline="", encoding=encoding) as file:
+        data = csv.DictReader(file, skipinitialspace=True, delimiter=delimiter)
         return list(data)

kaiko_eva-0.1.0.dist-info/METADATA

@@ -0,0 +1,553 @@
+Metadata-Version: 2.1
+Name: kaiko-eva
+Version: 0.1.0
+Summary: Evaluation Framework for oncology foundation models.
+Keywords: machine-learning,evaluation-framework,oncology,foundation-models
+Author-Email: Ioannis Gatopoulos <ioannis@kaiko.ai>, =?utf-8?q?Nicolas_K=C3=A4nzig?= <nicolas@kaiko.ai>, Roman Moser <roman@kaiko.ai>
+Maintainer-Email: Ioannis Gatopoulos <ioannis@kaiko.ai>, =?utf-8?q?Nicolas_K=C3=A4nzig?= <nicolas@kaiko.ai>, Roman Moser <roman@kaiko.ai>
+License:                                  Apache License
+                                    Version 2.0, January 2004
+                                 http://www.apache.org/licenses/
+         
+            TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+         
+            1. Definitions.
+         
+               "License" shall mean the terms and conditions for use, reproduction,
+               and distribution as defined by Sections 1 through 9 of this document.
+         
+               "Licensor" shall mean the copyright owner or entity authorized by
+               the copyright owner that is granting the License.
+         
+               "Legal Entity" shall mean the union of the acting entity and all
+               other entities that control, are controlled by, or are under common
+               control with that entity. For the purposes of this definition,
+               "control" means (i) the power, direct or indirect, to cause the
+               direction or management of such entity, whether by contract or
+               otherwise, or (ii) ownership of fifty percent (50%) or more of the
+               outstanding shares, or (iii) beneficial ownership of such entity.
+         
+               "You" (or "Your") shall mean an individual or Legal Entity
+               exercising permissions granted by this License.
+         
+               "Source" form shall mean the preferred form for making modifications,
+               including but not limited to software source code, documentation
+               source, and configuration files.
+         
+               "Object" form shall mean any form resulting from mechanical
+               transformation or translation of a Source form, including but
+               not limited to compiled object code, generated documentation,
+               and conversions to other media types.
+         
+               "Work" shall mean the work of authorship, whether in Source or
+               Object form, made available under the License, as indicated by a
+               copyright notice that is included in or attached to the work
+               (an example is provided in the Appendix below).
+         
+               "Derivative Works" shall mean any work, whether in Source or Object
+               form, that is based on (or derived from) the Work and for which the
+               editorial revisions, annotations, elaborations, or other modifications
+               represent, as a whole, an original work of authorship. For the purposes
+               of this License, Derivative Works shall not include works that remain
+               separable from, or merely link (or bind by name) to the interfaces of,
+               the Work and Derivative Works thereof.
+         
+               "Contribution" shall mean any work of authorship, including
+               the original version of the Work and any modifications or additions
+               to that Work or Derivative Works thereof, that is intentionally
+               submitted to Licensor for inclusion in the Work by the copyright owner
+               or by an individual or Legal Entity authorized to submit on behalf of
+               the copyright owner. For the purposes of this definition, "submitted"
+               means any form of electronic, verbal, or written communication sent
+               to the Licensor or its representatives, including but not limited to
+               communication on electronic mailing lists, source code control systems,
+               and issue tracking systems that are managed by, or on behalf of, the
+               Licensor for the purpose of discussing and improving the Work, but
+               excluding communication that is conspicuously marked or otherwise
+               designated in writing by the copyright owner as "Not a Contribution."
+         
+               "Contributor" shall mean Licensor and any individual or Legal Entity
+               on behalf of whom a Contribution has been received by Licensor and
+               subsequently incorporated within the Work.
+         
+            2. Grant of Copyright License. Subject to the terms and conditions of
+               this License, each Contributor hereby grants to You a perpetual,
+               worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+               copyright license to reproduce, prepare Derivative Works of,
+               publicly display, publicly perform, sublicense, and distribute the
+               Work and such Derivative Works in Source or Object form.
+         
+            3. Grant of Patent License. Subject to the terms and conditions of
+               this License, each Contributor hereby grants to You a perpetual,
+               worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+               (except as stated in this section) patent license to make, have made,
+               use, offer to sell, sell, import, and otherwise transfer the Work,
+               where such license applies only to those patent claims licensable
+               by such Contributor that are necessarily infringed by their
+               Contribution(s) alone or by combination of their Contribution(s)
+               with the Work to which such Contribution(s) was submitted. If You
+               institute patent litigation against any entity (including a
+               cross-claim or counterclaim in a lawsuit) alleging that the Work
+               or a Contribution incorporated within the Work constitutes direct
+               or contributory patent infringement, then any patent licenses
+               granted to You under this License for that Work shall terminate
+               as of the date such litigation is filed.
+         
+            4. Redistribution. You may reproduce and distribute copies of the
+               Work or Derivative Works thereof in any medium, with or without
+               modifications, and in Source or Object form, provided that You
+               meet the following conditions:
+         
+               (a) You must give any other recipients of the Work or
+                   Derivative Works a copy of this License; and
+         
+               (b) You must cause any modified files to carry prominent notices
+                   stating that You changed the files; and
+         
+               (c) You must retain, in the Source form of any Derivative Works
+                   that You distribute, all copyright, patent, trademark, and
+                   attribution notices from the Source form of the Work,
+                   excluding those notices that do not pertain to any part of
+                   the Derivative Works; and
+         
+               (d) If the Work includes a "NOTICE" text file as part of its
+                   distribution, then any Derivative Works that You distribute must
+                   include a readable copy of the attribution notices contained
+                   within such NOTICE file, excluding those notices that do not
+                   pertain to any part of the Derivative Works, in at least one
+                   of the following places: within a NOTICE text file distributed
+                   as part of the Derivative Works; within the Source form or
+                   documentation, if provided along with the Derivative Works; or,
+                   within a display generated by the Derivative Works, if and
+                   wherever such third-party notices normally appear. The contents
+                   of the NOTICE file are for informational purposes only and
+                   do not modify the License. You may add Your own attribution
+                   notices within Derivative Works that You distribute, alongside
+                   or as an addendum to the NOTICE text from the Work, provided
+                   that such additional attribution notices cannot be construed
+                   as modifying the License.
+         
+               You may add Your own copyright statement to Your modifications and
+               may provide additional or different license terms and conditions
+               for use, reproduction, or distribution of Your modifications, or
+               for any such Derivative Works as a whole, provided Your use,
+               reproduction, and distribution of the Work otherwise complies with
+               the conditions stated in this License.
+         
+            5. Submission of Contributions. Unless You explicitly state otherwise,
+               any Contribution intentionally submitted for inclusion in the Work
+               by You to the Licensor shall be under the terms and conditions of
+               this License, without any additional terms or conditions.
+               Notwithstanding the above, nothing herein shall supersede or modify
+               the terms of any separate license agreement you may have executed
+               with Licensor regarding such Contributions.
+         
+            6. Trademarks. This License does not grant permission to use the trade
+               names, trademarks, service marks, or product names of the Licensor,
+               except as required for reasonable and customary use in describing the
+               origin of the Work and reproducing the content of the NOTICE file.
+         
+            7. Disclaimer of Warranty. Unless required by applicable law or
+               agreed to in writing, Licensor provides the Work (and each
+               Contributor provides its Contributions) on an "AS IS" BASIS,
+               WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+               implied, including, without limitation, any warranties or conditions
+               of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+               PARTICULAR PURPOSE. You are solely responsible for determining the
+               appropriateness of using or redistributing the Work and assume any
+               risks associated with Your exercise of permissions under this License.
+         
+            8. Limitation of Liability. In no event and under no legal theory,
+               whether in tort (including negligence), contract, or otherwise,
+               unless required by applicable law (such as deliberate and grossly
+               negligent acts) or agreed to in writing, shall any Contributor be
+               liable to You for damages, including any direct, indirect, special,
+               incidental, or consequential damages of any character arising as a
+               result of this License or out of the use or inability to use the
+               Work (including but not limited to damages for loss of goodwill,
+               work stoppage, computer failure or malfunction, or any and all
+               other commercial damages or losses), even if such Contributor
+               has been advised of the possibility of such damages.
+         
+            9. Accepting Warranty or Additional Liability. While redistributing
+               the Work or Derivative Works thereof, You may choose to offer,
+               and charge a fee for, acceptance of support, warranty, indemnity,
+               or other liability obligations and/or rights consistent with this
+               License. However, in accepting such obligations, You may act only
+               on Your own behalf and on Your sole responsibility, not on behalf
+               of any other Contributor, and only if You agree to indemnify,
+               defend, and hold each Contributor harmless for any liability
+               incurred by, or claims asserted against, such Contributor by reason
+               of your accepting any such warranty or additional liability.
+         
+            END OF TERMS AND CONDITIONS
+         
+            APPENDIX: How to apply the Apache License to your work.
+         
+               To apply the Apache License to your work, attach the following
+               boilerplate notice, with the fields enclosed by brackets "[]"
+               replaced with your own identifying information. (Don't include
+               the brackets!)  The text should be enclosed in the appropriate
+               comment syntax for the file format. We also recommend that a
+               file or class name and description of purpose be included on the
+               same "printed page" as the copyright notice for easier
+               identification within third-party archives.
+         
+            Copyright 2024 kaiko.ai
+         
+            Licensed under the Apache License, Version 2.0 (the "License");
+            you may not use this file except in compliance with the License.
+            You may obtain a copy of the License at
+         
+                http://www.apache.org/licenses/LICENSE-2.0
+         
+            Unless required by applicable law or agreed to in writing, software
+            distributed under the License is distributed on an "AS IS" BASIS,
+            WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+            See the License for the specific language governing permissions and
+            limitations under the License.
+         
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Project-URL: Homepage, https://kaiko-ai.github.io/eva/dev/
+Project-URL: Repository, https://github.com/kaiko-ai/eva
+Project-URL: Documentation, https://kaiko-ai.github.io/eva/dev/
+Requires-Python: >=3.10
+Requires-Dist: torch==2.3.0
+Requires-Dist: lightning>=2.2.2
+Requires-Dist: jsonargparse[omegaconf]==4.31.0
+Requires-Dist: tensorboard>=2.16.2
+Requires-Dist: loguru>=0.7.2
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: transformers>=4.38.2
+Requires-Dist: onnxruntime>=1.17.1
+Requires-Dist: onnx>=1.16.0
+Requires-Dist: toolz>=0.12.1
+Requires-Dist: rich>=13.7.1
+Requires-Dist: torchmetrics>=1.4.2
+Provides-Extra: vision
+Requires-Dist: h5py>=3.10.0; extra == "vision"
+Requires-Dist: nibabel>=5.2.0; extra == "vision"
+Requires-Dist: opencv-python-headless>=4.9.0.80; extra == "vision"
+Requires-Dist: timm>=1.0.9; extra == "vision"
+Requires-Dist: torchvision>=0.17.0; extra == "vision"
+Requires-Dist: openslide-python>=1.3.1; extra == "vision"
+Requires-Dist: gdown>=5.2.0; extra == "vision"
+Requires-Dist: scikit-image>=0.24.0; extra == "vision"
+Requires-Dist: imagesize>=1.4.1; extra == "vision"
+Requires-Dist: scipy>=1.14.0; extra == "vision"
+Requires-Dist: monai>=1.3.2; extra == "vision"
+Provides-Extra: all
+Requires-Dist: h5py>=3.10.0; extra == "all"
+Requires-Dist: nibabel>=5.2.0; extra == "all"
+Requires-Dist: opencv-python-headless>=4.9.0.80; extra == "all"
+Requires-Dist: timm>=1.0.9; extra == "all"
+Requires-Dist: torchvision>=0.17.0; extra == "all"
+Requires-Dist: openslide-python>=1.3.1; extra == "all"
+Requires-Dist: gdown>=5.2.0; extra == "all"
+Requires-Dist: scikit-image>=0.24.0; extra == "all"
+Requires-Dist: imagesize>=1.4.1; extra == "all"
+Requires-Dist: scipy>=1.14.0; extra == "all"
+Requires-Dist: monai>=1.3.2; extra == "all"
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<br />
+
+<img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/eva-logo.png?raw=true" width="340">
+
+<br />
+<br />
+
+_Oncology FM Evaluation Framework by kaiko.ai_
+
+[![PyPI](https://img.shields.io/pypi/v/kaiko-eva.svg?logo=python)](https://pypi.python.org/pypi/kaiko-eva)
+[![docs](https://img.shields.io/badge/📚_docs-latest-green)](https://kaiko-ai.github.io/eva/latest)
+[![license](https://img.shields.io/badge/⚖️_License-Apache%202.0-blue.svg?labelColor=gray)](https://github.com/kaiko-ai/eva#license)<br>
+[![paper](http://img.shields.io/badge/OpenReview-MIDL_2024-B31B1B.svg)](https://openreview.net/forum?id=FNBQOPj18N&noteId=FNBQOPj18N)
+
+<p align="center">
+  <a href="https://github.com/kaiko-ai/eva#installation">Installation</a> •
+  <a href="https://github.com/kaiko-ai/eva#how-to-use">How To Use</a> •
+  <a href="https://github.com/kaiko-ai/eva#quick-start">Quick Start</a> •
+  <a href="https://kaiko-ai.github.io/eva/">Documentation</a> •
+  <a href="https://kaiko-ai.github.io/eva/dev/datasets/">Datasets</a> •
+  <a href="https://github.com/kaiko-ai/eva#benchmarks">Benchmarks</a> <br>
+  <a href="https://github.com/kaiko-ai/eva#contributing">Contribute</a> •
+  <a href="https://github.com/kaiko-ai/eva#acknowledgements">Acknowledgements</a>
+</p>
+
+</div>
+
+<br />
+
+_`eva`_ is an evaluation framework for oncology foundation models (FMs) by [kaiko.ai](https://kaiko.ai/).
+Check out the [documentation](https://kaiko-ai.github.io/eva/) for more information.
+
+### Highlights:
+- Easy and reliable benchmark of Oncology FMs
+- Supports path-level classification, slide-level classification and semantic segmentation downstream tasks
+- Automatic embedding inference and evaluation of a downstream task
+- Native support of popular medical [datasets](https://kaiko-ai.github.io/eva/dev/datasets/) and models
+- Produce statistics over multiple evaluation fits and multiple metrics
+
+## Installation
+
+Simple installation from PyPI:
+```sh
+# to install the core version only
+pip install kaiko-eva
+
+# to install the expanded `vision` version
+pip install 'kaiko-eva[vision]'
+
+# to install everything
+pip install 'kaiko-eva[all]'
+```
+
+To install the latest version of the `main` branch:
+```sh
+pip install "kaiko-eva[all] @ git+https://github.com/kaiko-ai/eva.git"
+```
+
+You can verify that the installation was successful by executing:
+```sh
+eva --version
+```
+
+## How To Use
+
+_`eva`_ can be used directly from the terminal as a CLI tool as follows:
+```sh
+eva {fit,predict,predict_fit} --config url/or/path/to/the/config.yaml 
+```
+
+_`eva`_ uses [jsonargparse](https://jsonargparse.readthedocs.io/en/v4.31.0/) to
+make it easily configurable by automatically generating command line interfaces (CLIs),
+which allows to call *any* Python object from the command line. Moreover, the configuration structure is always in sync with the code. Thus, _`eva`_ can be used either directly from Python or as a CLI tool (recommended).
+
+For more information, please refer to the [documentation](https://kaiko-ai.github.io/eva/dev/user-guide/tutorials/offline_vs_online/).
+
+<details>
+  <summary>Learn about Configs</summary>
+
+The following interfaces are identical:
+<table>
+<tr>
+<th>Python interface</th>
+<th>Configuration file</th>
+</tr>
+<tr>
+<td>
+<sub>
+
+```Python
+# main.py
+# execute with: `python main.py`
+
+from torch import nn
+
+from eva import core
+from eva.vision import datasets, transforms
+
+# initialize trainer
+trainer = core.Trainer(max_steps=100)
+
+# initialize model
+model = core.HeadModule(
+  backbone=nn.Flatten(),
+  head=nn.Linear(150528, 4),
+  criterion=nn.CrossEntropyLoss(),
+)
+
+# initialize data
+data = core.DataModule(
+  datasets=core.DatasetsSchema(
+    train=datasets.BACH(
+      root="data/bach",
+      split="train",
+      download=True,
+      transforms=transforms.ResizeAndCrop(),
+    ),
+  ),
+  dataloaders=core.DataloadersSchema(
+    train=core.DataLoader(batch_size=32),
+  ),
+)
+
+# perform fit
+pipeline = core.Interface()
+pipeline.fit(trainer, model=model, data=data)
+```
+</sub>
+<td>
+<sub>
+
+```yaml
+# main.yaml
+# execute with: `eva fit --config main.yaml`
+
+---
+trainer:
+  class_path: eva.Trainer
+  init_args:
+    max_steps: 100
+model:
+  class_path: eva.HeadModule
+  init_args:
+    backbone: torch.nn.Flatten
+    head:
+      class_path: torch.nn.Linear
+      init_args:
+        in_features: 150528
+        out_features: 4
+    criterion: torch.nn.CrossEntropyLoss
+data:
+  class_path: eva.DataModule
+  init_args:
+    datasets:
+      train:
+        class_path: eva.vision.datasets.BACH
+        init_args:
+          root: ./data/bach
+          split: train
+          download: true
+          transforms: eva.vision.transforms.ResizeAndCrop
+    dataloaders:
+      train:
+        batch_size: 32
+```
+</sub>
+</td>
+</tr>
+</table>
+
+The `.yaml` file defines the functionality of _`eva`_
+by parsing and translating its content to Python objects directly.
+Native supported configs can be found at the
+[configs](https://github.com/kaiko-ai/eva/tree/main/configs) directory
+of the repo, which can be both locally stored or remote.
+
+</details>
+
+## Quick Start
+
+We define two types of evaluations: **online** and **offline**.
+While online fit uses the backbone (FM) to perform forward passes
+during the fitting process, offline fit first generates embeddings
+with the backbone and then fits the model using these embeddings as
+input, resulting in a faster evaluation.
+
+Here are some examples to get you started:
+
+- Perform a downstream offline **classification** evaluation of `DINO ViT-S/16`
+on the `BACH` dataset with linear probing by first inferring the embeddings
+and then performing 5 sequential fits:
+  ```sh
+  export DOWNLOAD_DATA=true
+  eva predict_fit --config https://raw.githubusercontent.com/kaiko-ai/eva/main/configs/vision/dino_vit/offline/bach.yaml
+  ```
+
+- Perform a downstream online **segmentation** evaluation of `DINO ViT-S/16` on the
+`MoNuSAC` dataset with the `ConvDecoderMS` decoder:
+  ```sh
+  export DOWNLOAD_DATA=true
+  eva fit --config https://raw.githubusercontent.com/kaiko-ai/eva/main/configs/vision/dino_vit/online/monusac.yaml
+  ```
+
+For more examples, take a look at the [configs](https://github.com/kaiko-ai/eva/tree/main/configs)
+and [tutorials](https://kaiko-ai.github.io/eva/dev/user-guide/advanced/replicate_evaluations/).
+
+> [!NOTE]
+> All the datasets that support automatic download in the repo have by default the option to automatically download set to false.
+> For automatic download you have to manually set the environmental variable `DOWNLOAD_DATA=true` or in the configuration file `download=true`.
+
+## Leaderboards
+
+In this section you will find model benchmarks which were generated with _`eva`_.
+
+### Table I: WSI and microscopy image tasks
+
+<br />
+
+<div align="center">
+
+| Model   | BACH  | CRC   | MHIST | PCam   | Camelyon16 | PANDA | CoNSeP | MoNuSAC |
+|---------|-------|-------|-------|--------|------------|-------|------------|-------|
+| ViT-S/16 _(random)_	<sup>[1]</sup> | 0.411|0.613|0.5|0.752|0.551|0.347|0.489|0.394|
+| ViT-S/16 _(ImageNet)_ <sup>[1]</sup> | 0.675|0.936|0.827|0.861|0.751|0.676|0.54|0.512|
+| DINO<sub>(p=16)</sub> <sup>[2]</sup> | 0.77|0.936|0.751|0.905|0.869|0.737|0.625|0.549|
+| Phikon <sup>[3]</sup>                | 0.715|0.942|0.766|0.925|0.879|0.784|0.68|0.554|
+| UNI <sup>[4]</sup>                   | 0.797|0.95|0.835|0.939|0.933|0.774|0.67|0.575|
+| ViT-S/16 _(kaiko.ai)_ <sup>[5]</sup> | 0.8|0.949|0.831|0.902|0.897|0.77|0.622|0.573|
+| ViT-S/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.825|0.948|0.826|0.887|0.879|0.741|0.677|0.617|
+| ViT-B/16 _(kaiko.ai)_	<sup>[5]</sup> | 0.846|0.959|0.839|0.906|0.891|0.753|0.647|0.572|
+| ViT-B/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.867|0.952|0.814|0.921|0.939|0.761|0.706|0.661|
+| ViT-L/14 _(kaiko.ai)_ <sup>[5]</sup> | 0.862|0.935|0.822|0.907|0.941|0.769|0.686|0.599|
+
+_Table I: Linear probing evaluation of FMs on patch-level downstream datasets.<br> We report balanced accuracy
+ for classification tasks and generalized Dice score for semgetnation tasks, averaged over 5 runs. Results are 
+ reported on the "test" split if available and otherwise on the "validation" split._
+
+</div>
+
+<br />
+
+_References_:
+1. _"Emerging properties in self-supervised vision transformers”_, [arXiv](https://arxiv.org/abs/2104.14294)
+2. _"Benchmarking self-supervised learning on diverse pathology datasets”_, [arXiv](https://arxiv.org/abs/2212.04690)
+3. _"Scaling self-supervised learning for histopathology with masked image modeling”_, [medRxiv](https://www.medrxiv.org/content/10.1101/2023.07.21.23292757v1)
+4. _"A General-Purpose Self-Supervised Model for Computational Pathology”_, [arXiv](https://arxiv.org/abs/2308.15474)
+5. _"Towards Training Large-Scale Pathology Foundation Models: from TCGA to Hospital Scale”_, [arXiv](https://arxiv.org/pdf/2404.15217)
+
+## Contributing
+
+_`eva`_ is an open source project and welcomes contributions of all kinds. Please checkout the [developer](./docs/DEVELOPER_GUIDE.md)
+and [contributing guide](./docs/CONTRIBUTING.md) for help on how to do so.
+
+All contributors must follow the [code of conduct](./docs/CODE_OF_CONDUCT.md).
+
+
+## Acknowledgements
+
+Our codebase is built using multiple opensource contributions
+
+<div align="center">
+
+[![python](https://img.shields.io/badge/-Python-blue?logo=python&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![pytorch](https://img.shields.io/badge/PyTorch-ee4c2c?logo=pytorch&logoColor=white)](https://pytorch.org/get-started/locally/)
+[![lightning](https://img.shields.io/badge/-⚡️_Lightning-792ee5?logo=pytorchlightning&logoColor=white)](https://pytorchlightning.ai/)<br>
+[![black](https://img.shields.io/badge/Code%20Style-Black-black.svg?labelColor=gray)](https://black.readthedocs.io/en/stable/)
+[![isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Checked with pyright](https://microsoft.github.io/pyright/img/pyright_badge.svg)](https://microsoft.github.io/pyright/)<br>
+[![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm-project.org)
+[![Nox](https://img.shields.io/badge/%F0%9F%A6%8A-Nox-D85E00.svg)](https://github.com/wntrblm/nox)
+[![Built with Material for MkDocs](https://img.shields.io/badge/Material_for_MkDocs-526CFE?logo=MaterialForMkDocs&logoColor=white)](https://squidfunk.github.io/mkdocs-material/)
+
+</div>
+
+
+## Citation
+
+If you find this repository useful, please consider giving a star ⭐ and adding the following citation:
+
+```bibtex
+@inproceedings{kaiko.ai2024eva,
+    title={eva: Evaluation framework for pathology foundation models},
+    author={kaiko.ai and Ioannis Gatopoulos and Nicolas K{\"a}nzig and Roman Moser and Sebastian Ot{\'a}lora},
+    booktitle={Medical Imaging with Deep Learning},
+    year={2024},
+    url={https://openreview.net/forum?id=FNBQOPj18N}
+}
+```
+
+<br />
+
+<div align="center">
+  <img src="https://github.com/kaiko-ai/eva/blob/main/docs/images/kaiko-logo.png?raw=true" width="200">
+</div>