kaiko-eva 0.3.2__tar.gz → 0.4.0__tar.gz

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaiko-eva
-Version: 0.3.2
+Version: 0.4.0
 Summary: Evaluation Framework for oncology foundation models.
 Keywords: machine-learning,evaluation-framework,oncology,foundation-models
 Author-Email: Ioannis Gatopoulos <ioannis@kaiko.ai>, =?utf-8?q?Nicolas_K=C3=A4nzig?= <nicolas@kaiko.ai>, Roman Moser <roman@kaiko.ai>
@@ -245,6 +245,10 @@ Requires-Dist: einops>=0.8.1; extra == "vision"
 Provides-Extra: language
 Requires-Dist: datasets<4.0.0,>=2.19.0; extra == "language"
 Requires-Dist: litellm>=1.61.8; extra == "language"
+Requires-Dist: backoff>=2.2.1; extra == "language"
+Provides-Extra: multimodal
+Requires-Dist: litellm>=1.61.8; extra == "multimodal"
+Requires-Dist: backoff>=2.2.1; extra == "multimodal"
 Provides-Extra: all
 Requires-Dist: h5py>=3.10.0; extra == "all"
 Requires-Dist: nibabel>=4.0.1; extra == "all"
@@ -260,6 +264,7 @@ Requires-Dist: monai>=1.3.2; extra == "all"
 Requires-Dist: einops>=0.8.1; extra == "all"
 Requires-Dist: datasets<4.0.0,>=2.19.0; extra == "all"
 Requires-Dist: litellm>=1.61.8; extra == "all"
+Requires-Dist: backoff>=2.2.1; extra == "all"
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -298,7 +303,7 @@ Check out the [documentation](https://kaiko-ai.github.io/eva/) for more informat
 
 ### Highlights:
 - Easy and reliable benchmark of Oncology FMs
-- Supports patch-level classification, slide-level classification, semantic segmentation, and text classification downstream tasks
+- Supports patch-level classification, slide-level classification, semantic segmentation, and (visual) question answering tasks.
 - Automatic embedding inference and evaluation of a downstream task
 - Native support of popular medical [datasets](https://kaiko-ai.github.io/eva/dev/datasets/) and models
 - Produce statistics over multiple evaluation fits and multiple metrics
@@ -316,6 +321,9 @@ pip install 'kaiko-eva[vision]'
 # to install the expanded `language` version
 pip install 'kaiko-eva[language]'
 
+# to install the expanded `multimodal` version
+pip install 'kaiko-eva[multimodal]'
+
 # to install everything
 pip install 'kaiko-eva[all]'
 ```

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/README.md

@@ -34,7 +34,7 @@ Check out the [documentation](https://kaiko-ai.github.io/eva/) for more informat
 
 ### Highlights:
 - Easy and reliable benchmark of Oncology FMs
-- Supports patch-level classification, slide-level classification, semantic segmentation, and text classification downstream tasks
+- Supports patch-level classification, slide-level classification, semantic segmentation, and (visual) question answering tasks.
 - Automatic embedding inference and evaluation of a downstream task
 - Native support of popular medical [datasets](https://kaiko-ai.github.io/eva/dev/datasets/) and models
 - Produce statistics over multiple evaluation fits and multiple metrics
@@ -52,6 +52,9 @@ pip install 'kaiko-eva[vision]'
 # to install the expanded `language` version
 pip install 'kaiko-eva[language]'
 
+# to install the expanded `multimodal` version
+pip install 'kaiko-eva[multimodal]'
+
 # to install everything
 pip install 'kaiko-eva[all]'
 ```

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "kaiko-eva"
-version = "0.3.2"
+version = "0.4.0"
 description = "Evaluation Framework for oncology foundation models."
 keywords = [
     "machine-learning",
@@ -75,6 +75,11 @@ vision = [
 language = [
     "datasets<4.0.0,>=2.19.0",
     "litellm>=1.61.8",
+    "backoff>=2.2.1",
+]
+multimodal = [
+    "litellm>=1.61.8",
+    "backoff>=2.2.1",
 ]
 all = [
     "h5py>=3.10.0",
@@ -91,6 +96,7 @@ all = [
     "einops>=0.8.1",
     "datasets<4.0.0,>=2.19.0",
     "litellm>=1.61.8",
+    "backoff>=2.2.1",
 ]
 
 [project.scripts]

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/callbacks/config.py

@@ -51,6 +51,10 @@ class ConfigurationLogger(pl.Callback):
 
         save_as = os.path.join(log_dir, self._save_as)
         fs = cloud_io.get_filesystem(log_dir)
+
+        if not fs.exists(log_dir):
+            fs.makedirs(log_dir)
+
         with fs.open(save_as, "w") as output_file:
             yaml.dump(configuration, output_file, sort_keys=False)
 

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/cli/setup.py

@@ -59,7 +59,7 @@ def _initialize_logger() -> None:
         " :: <bold><level>{level}</level></bold>"
         " :: {message}",
         colorize=True,
-        level="INFO",
+        level=os.getenv("LOGURU_LEVEL", "INFO"),
     )
 
 

kaiko_eva-0.4.0/src/eva/core/data/dataloaders/__init__.py

@@ -0,0 +1,5 @@
+"""Dataloaders API."""
+
+from eva.core.data.dataloaders.dataloader import DataLoader
+
+__all__ = ["DataLoader"]

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/data/dataloaders/dataloader.py

@@ -77,7 +77,9 @@ class DataLoader:
             shuffle=self.shuffle,
             sampler=sampler or self.sampler,
             batch_sampler=self.batch_sampler,
-            num_workers=self.num_workers or multiprocessing.cpu_count(),
+            num_workers=(
+                multiprocessing.cpu_count() if self.num_workers is None else self.num_workers
+            ),
             collate_fn=self.collate_fn,
             pin_memory=self.pin_memory,
             drop_last=self.drop_last,

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/data/samplers/random.py

@@ -2,6 +2,7 @@
 
 from typing import Optional
 
+import torch
 from torch.utils import data
 from typing_extensions import override
 
@@ -10,30 +11,36 @@ from eva.core.data.samplers.sampler import SamplerWithDataSource
 
 
 class RandomSampler(data.RandomSampler, SamplerWithDataSource[int]):
-    """Samples elements randomly."""
+    """Samples elements randomly from a MapDataset."""
 
     data_source: datasets.MapDataset  # type: ignore
 
     def __init__(
-        self, replacement: bool = False, num_samples: Optional[int] = None, generator=None
+        self,
+        replacement: bool = False,
+        num_samples: Optional[int] = None,
+        seed: Optional[int] = None,
     ) -> None:
-        """Initializes the random sampler.
+        """Initialize the random sampler.
 
         Args:
-            data_source: dataset to sample from
-            replacement: samples are drawn on-demand with replacement if ``True``, default=``False``
-            num_samples: number of samples to draw, default=`len(dataset)`.
-            generator: Generator used in sampling.
+            replacement: Samples are drawn on-demand with replacement if ``True``, default=``False``
+            num_samples: Number of samples to draw, default=``len(dataset)``.
+            seed: Optional seed for the random number generator.
         """
         self.replacement = replacement
         self._num_samples = num_samples
-        self.generator = generator
+        self._generator = None
+
+        if seed is not None:
+            self._generator = torch.Generator()
+            self._generator.manual_seed(seed)
 
     @override
     def set_dataset(self, data_source: datasets.MapDataset) -> None:
         super().__init__(
             data_source,
             replacement=self.replacement,
-            num_samples=self.num_samples,
-            generator=self.generator,
+            num_samples=self._num_samples,
+            generator=self._generator,
         )

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/interface/interface.py

@@ -132,3 +132,24 @@ class Interface:
             n_runs=trainer.n_runs,
             verbose=trainer.n_runs > 1,
         )
+
+    def validate_test(
+        self,
+        trainer: eva_trainer.Trainer,
+        model: modules.ModelModule,
+        data: datamodules.DataModule,
+    ) -> None:
+        """Runs validation & test stages."""
+        if getattr(data.datasets, "val", None) is None:
+            raise ValueError("The provided data module does not contain a validation dataset.")
+        if getattr(data.datasets, "test", None) is None:
+            raise ValueError("The provided data module does not contain a test dataset.")
+
+        eva_trainer.run_evaluation_session(
+            base_trainer=trainer,
+            base_model=model,
+            datamodule=data,
+            stages=["validate", "test"],
+            n_runs=trainer.n_runs,
+            verbose=trainer.n_runs > 1,
+        )

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/loggers/log/__init__.py

@@ -2,5 +2,6 @@
 
 from eva.core.loggers.log.image import log_image
 from eva.core.loggers.log.parameters import log_parameters
+from eva.core.loggers.log.table import log_table
 
-__all__ = ["log_image", "log_parameters"]
+__all__ = ["log_image", "log_parameters", "log_table"]

kaiko_eva-0.4.0/src/eva/core/loggers/log/table.py

@@ -0,0 +1,73 @@
+"""Table log functionality."""
+
+import functools
+from typing import List
+
+import pandas as pd
+
+from eva.core.loggers import loggers
+from eva.core.loggers.log import utils
+
+
+@functools.singledispatch
+def log_table(
+    logger,
+    tag: str,
+    columns: List[str] | None = None,
+    data: List[List[str]] | None = None,
+    dataframe: pd.DataFrame | None = None,
+    step: int = 0,
+) -> None:
+    """Adds a a table to the logger.
+
+    The table can be defined either with `columns` and `data` or with `dataframe`.
+
+    Args:
+        logger: The logger to log the table to.
+        tag: The log tag.
+        columns: The column names of the table.
+        data: The data of the table as a list of lists.
+        dataframe: A pandas DataFrame to log.
+        step: The global step of the log.
+    """
+    utils.raise_not_supported(logger, "table")
+
+
+@log_table.register
+def _(
+    loggers: list,
+    tag: str,
+    columns: List[str] | None = None,
+    data: List[List[str]] | None = None,
+    dataframe: pd.DataFrame | None = None,
+    step: int = 0,
+) -> None:
+    """Adds a table to a list of supported loggers."""
+    for logger in loggers:
+        log_table(
+            logger,
+            tag=tag,
+            columns=columns,
+            data=data,
+            dataframe=dataframe,
+            step=step,
+        )
+
+
+@log_table.register
+def _(
+    logger: loggers.WandbLogger,
+    tag: str,
+    columns: List[str] | None = None,
+    data: List[List[str]] | None = None,
+    dataframe: pd.DataFrame | None = None,
+    step: int = 0,
+) -> None:
+    """Adds a table to a Wandb logger."""
+    logger.log_table(
+        key=tag,
+        columns=columns,
+        data=data,
+        dataframe=dataframe,
+        step=step,
+    )

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/models/modules/module.py

@@ -33,8 +33,8 @@ class ModelModule(pl.LightningModule):
         super().__init__()
 
         self._metrics = metrics or self.default_metrics
-        self._postprocess = postprocess or self.default_postprocess
 
+        self.postprocess = postprocess or self.default_postprocess
         self.metrics = metrics_lib.MetricModule.from_schema(self._metrics)
 
     @property
@@ -133,7 +133,7 @@ class ModelModule(pl.LightningModule):
         Returns:
             The updated outputs.
         """
-        self._postprocess(outputs)
+        self.postprocess(outputs)
         return memory.recursive_detach(outputs, to_cpu=self.metrics_device.type == "cpu")
 
     def _forward_and_log_metrics(

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/models/wrappers/base.py

@@ -25,7 +25,7 @@ class BaseModel(nn.Module, Generic[InputType, OutputType]):
 
         self._output_transforms = transforms
 
-        self._model: Callable[..., OutputType] | nn.Module
+        self.model: Callable[..., OutputType] | nn.Module
 
     @override
     def forward(self, tensor: InputType) -> OutputType:
@@ -43,7 +43,7 @@ class BaseModel(nn.Module, Generic[InputType, OutputType]):
         Args:
             tensor: The input tensor to the model.
         """
-        return self._model(tensor)
+        return self.model(tensor)
 
     def _apply_transforms(self, tensor: OutputType) -> OutputType:
         if self._output_transforms is not None:

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/models/wrappers/from_function.py

@@ -41,12 +41,12 @@ class ModelFromFunction(base.BaseModel[torch.Tensor, torch.Tensor]):
         self._arguments = arguments
         self._checkpoint_path = checkpoint_path
 
-        self.load_model()
+        self.model = self.load_model()
 
     @override
-    def load_model(self) -> None:
+    def load_model(self) -> nn.Module:
         class_path = jsonargparse.class_from_function(self._path, func_return=nn.Module)
         model = class_path(**self._arguments or {})
         if self._checkpoint_path is not None:
             _utils.load_model_weights(model, self._checkpoint_path)
-        self._model = model
+        return model

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/models/wrappers/from_torchhub.py

@@ -52,12 +52,12 @@ class TorchHubModel(base.BaseModel[torch.Tensor, torch.Tensor]):
         self._trust_repo = trust_repo
         self._model_kwargs = model_kwargs or {}
 
-        self.load_model()
+        self.model = self.load_model()
 
     @override
-    def load_model(self) -> None:
+    def load_model(self) -> nn.Module:
         """Builds and loads the torch.hub model."""
-        self._model: nn.Module = torch.hub.load(
+        model: nn.Module = torch.hub.load(
             repo_or_dir=self._repo_or_dir,
             model=self._model_name,
             trust_repo=self._trust_repo,
@@ -66,21 +66,23 @@ class TorchHubModel(base.BaseModel[torch.Tensor, torch.Tensor]):
         )  # type: ignore
 
         if self._checkpoint_path:
-            _utils.load_model_weights(self._model, self._checkpoint_path)
+            _utils.load_model_weights(model, self._checkpoint_path)
 
         TorchHubModel.__name__ = self._model_name
 
+        return model
+
     @override
     def model_forward(self, tensor: torch.Tensor) -> torch.Tensor | List[torch.Tensor]:
         if self._out_indices is not None:
-            if not hasattr(self._model, "get_intermediate_layers"):
+            if not hasattr(self.model, "get_intermediate_layers"):
                 raise ValueError(
                     "Only models with `get_intermediate_layers` are supported "
                     "when using `out_indices`."
                 )
 
             return list(
-                self._model.get_intermediate_layers(
+                self.model.get_intermediate_layers(  # type: ignore
                     tensor,
                     self._out_indices,
                     reshape=True,
@@ -89,4 +91,4 @@ class TorchHubModel(base.BaseModel[torch.Tensor, torch.Tensor]):
                 )
             )
 
-        return self._model(tensor)
+        return self.model(tensor)

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/models/wrappers/huggingface.py

@@ -4,6 +4,7 @@ from typing import Any, Callable, Dict
 
 import torch
 import transformers
+from torch import nn
 from typing_extensions import override
 
 from eva.core.models.wrappers import base
@@ -33,12 +34,10 @@ class HuggingFaceModel(base.BaseModel[torch.Tensor, torch.Tensor]):
         self._model_name_or_path = model_name_or_path
         self._model_kwargs = model_kwargs or {}
 
-        self.load_model()
+        self.model = self.load_model()
 
     @override
-    def load_model(self) -> None:
+    def load_model(self) -> nn.Module:
         # Use safetensors to avoid torch.load security vulnerability
         model_kwargs = {"use_safetensors": True, **self._model_kwargs}
-        self._model = transformers.AutoModel.from_pretrained(
-            self._model_name_or_path, **model_kwargs
-        )
+        return transformers.AutoModel.from_pretrained(self._model_name_or_path, **model_kwargs)

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/models/wrappers/onnx.py

@@ -30,21 +30,21 @@ class ONNXModel(base.BaseModel[torch.Tensor, torch.Tensor]):
         self._path = path
         self._device = device
 
-        self.load_model()
+        self.model = self.load_model()
 
     @override
     def load_model(self) -> Any:
         if self._device == "cuda" and not torch.cuda.is_available():
             raise ValueError("Device is set to 'cuda', but CUDA is not available.")
         provider = "CUDAExecutionProvider" if self._device == "cuda" else "CPUExecutionProvider"
-        self._model = ort.InferenceSession(self._path, providers=[provider])  # type: ignore
+        return ort.InferenceSession(self._path, providers=[provider])  # type: ignore
 
     @override
     def model_forward(self, tensor: torch.Tensor) -> torch.Tensor:
         # TODO: Use IO binding to avoid copying the tensor to CPU.
         # https://onnxruntime.ai/docs/api/python/api_summary.html#data-on-device
-        if not isinstance(self._model, ort.InferenceSession):
+        if not isinstance(self.model, ort.InferenceSession):
             raise ValueError("Model is not loaded.")
-        inputs = {self._model.get_inputs()[0].name: tensor.detach().cpu().numpy()}
-        outputs = self._model.run(None, inputs)[0]
+        inputs = {self.model.get_inputs()[0].name: tensor.detach().cpu().numpy()}
+        outputs = self.model.run(None, inputs)[0]
         return torch.from_numpy(outputs).float().to(tensor.device)

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/core/trainers/trainer.py

@@ -8,6 +8,7 @@ from lightning.pytorch import loggers as pl_loggers
 from lightning.pytorch import trainer as pl_trainer
 from lightning.pytorch.utilities import argparse
 from lightning_fabric.utilities import cloud_io
+from lightning_utilities.core.rank_zero import rank_zero_only
 from typing_extensions import override
 
 from eva.core import loggers as eva_loggers
@@ -66,6 +67,7 @@ class Trainer(pl_trainer.Trainer):
     def log_dir(self) -> str | None:
         return self.strategy.broadcast(self._log_dir)
 
+    @rank_zero_only
     def init_logger_run(self, run_id: int | None) -> None:
         """Setup the loggers & log directories when starting a new run.
 

{kaiko_eva-0.3.2 → kaiko_eva-0.4.0}/src/eva/language/__init__.py

@@ -1,6 +1,7 @@
 """eva language API."""
 
 try:
+    from eva.language import models
     from eva.language.data import datasets
 except ImportError as e:
     msg = (
@@ -10,4 +11,4 @@ except ImportError as e:
     )
     raise ImportError(str(e) + "\n\n" + msg) from e
 
-__all__ = ["datasets"]
+__all__ = ["models", "datasets"]

kaiko_eva-0.4.0/src/eva/language/callbacks/__init__.py

@@ -0,0 +1,5 @@
+"""Language callbacks API."""
+
+from eva.language.callbacks.writers import TextPredictionWriter
+
+__all__ = ["TextPredictionWriter"]

kaiko_eva-0.4.0/src/eva/language/callbacks/writers/__init__.py

@@ -0,0 +1,5 @@
+"""Language writers callbacks API."""
+
+from eva.language.callbacks.writers.prediction import TextPredictionWriter
+
+__all__ = ["TextPredictionWriter"]