kaiko-eva 0.0.0.dev6__py3-none-any.whl

eva/core/trainers/_logging.py

@@ -0,0 +1,81 @@
+"""Helper functions and utilities for trainer logging."""
+
+import hashlib
+import sys
+from datetime import datetime
+
+from lightning_fabric.utilities import cloud_io
+from loguru import logger
+
+
+def generate_session_id() -> str:
+    """Generates and returns a unique string ID of an experiment.
+
+    The ID is composed of the run timestamp and a its config hash. If the
+    configuration hash is an empty string, it will use only the timestamp.
+    """
+    timestamp = _generate_timestamp_hash()
+    config_hash = _generate_config_hash()
+    return f"{timestamp}_{config_hash}" if config_hash else timestamp
+
+
+def _generate_timestamp_hash() -> str:
+    """Generate a time-based hash id."""
+    timestamp = datetime.now()
+    return timestamp.strftime("%Y%m%d-%H%M%S%f")
+
+
+def _generate_config_hash(max_hash_len: int = 8) -> str:
+    """Generates a hash id based on a yaml configuration file.
+
+    Args:
+        max_hash_len: The maximum length of the produced hash id.
+    """
+    config_path = _fetch_config_path()
+    if config_path is None:
+        logger.warning(
+            "No or multiple configuration file found from command line arguments. "
+            "No configuration hash code will created for this experiment."
+        )
+        return ""
+
+    return _generate_hash_from_config(config_path, max_hash_len)
+
+
+def _fetch_config_path() -> str | None:
+    """Retrieves the configuration path from command line arguments.
+
+    It returns `None` if no or multiple configuration files found in
+    the system arguments.
+
+    Returns:
+        The path to the configuration file.
+    """
+    inputs = sys.argv
+    config_paths = [inputs[i + 1] for i, arg in enumerate(inputs) if arg == "--config"]
+    if len(config_paths) == 0 or len(config_paths) > 1:
+        # TODO combine the multiple configuration files
+        # and produced hash for the merged one.
+        return None
+
+    return config_paths[0]
+
+
+def _generate_hash_from_config(path: str, max_hash_len: int = 8) -> str:
+    """Return a hash from the contents of the configuration file.
+
+    Args:
+        path: Path to the configuration file.
+        max_hash_len: Maximum length of the returned hash.
+
+    Returns:
+        Hash of the configuration file content.
+    """
+    fs = cloud_io.get_filesystem(path)
+    with fs.open(path, "r") as stream:
+        config = stream.read()
+        if isinstance(config, str):
+            config = config.encode("utf-8")
+        config_sha256 = hashlib.sha256(config)
+        hash_id = config_sha256.hexdigest()
+    return hash_id[:max_hash_len]

eva/core/trainers/_recorder.py

@@ -0,0 +1,149 @@
+"""Multi-run summary recorder."""
+
+import collections
+import json
+import os
+import statistics
+import sys
+from typing import Any, Dict, List, Mapping
+
+from lightning.pytorch.utilities.types import _EVALUATE_OUTPUT
+from lightning_fabric.utilities import cloud_io
+from loguru import logger
+from omegaconf import OmegaConf
+from toolz import dicttoolz
+
+SESSION_METRICS = Mapping[str, List[float]]
+"""Session metrics type-hint."""
+
+
+class SessionRecorder:
+    """Multi-run (session) summary logger."""
+
+    def __init__(
+        self,
+        output_dir: str,
+        results_file: str = "results.json",
+        config_file: str = "config.yaml",
+    ) -> None:
+        """Initializes the recorder.
+
+        Args:
+            output_dir: The destination folder to save the results.
+            results_file: The name of the results json file.
+            config_file: The name of the yaml configuration file.
+        """
+        self._output_dir = output_dir
+        self._results_file = results_file
+        self._config_file = config_file
+
+        self._validation_metrics: List[SESSION_METRICS] = []
+        self._test_metrics: List[SESSION_METRICS] = []
+
+    @property
+    def filename(self) -> str:
+        """Returns the output filename."""
+        return os.path.join(self._output_dir, self._results_file)
+
+    @property
+    def config_path(self) -> str | None:
+        """Returns the path to the .yaml configuration file from sys args if available."""
+        if "--config" in sys.argv:
+            try:
+                config_path = sys.argv[sys.argv.index("--config") + 1]
+                if not config_path.endswith(".yaml"):
+                    logger.warning(f"Unexpected config file {config_path}, should be a .yaml file.")
+                else:
+                    return config_path
+            except IndexError as e:
+                logger.warning(f"Failed to fetch config_path from sys args {e}")
+
+    def update(
+        self,
+        validation_scores: _EVALUATE_OUTPUT,
+        test_scores: _EVALUATE_OUTPUT | None = None,
+    ) -> None:
+        """Updates the state of the tracked metrics in-place."""
+        self._update_validation_metrics(validation_scores)
+        self._update_test_metrics(test_scores)
+
+    def compute(self) -> Dict[str, List[Dict[str, Any]]]:
+        """Computes and returns the session statistics."""
+        validation_statistics = list(map(_calculate_statistics, self._validation_metrics))
+        test_statistics = list(map(_calculate_statistics, self._test_metrics))
+        return {"val": validation_statistics, "test": test_statistics}
+
+    def export(self) -> Dict[str, Any]:
+        """Exports the results."""
+        statistics = self.compute()
+        return {"metrics": statistics}
+
+    def save(self) -> None:
+        """Saves the recorded results."""
+        results = self.export()
+        _save_json(results, self.filename)
+        self._save_config()
+
+    def reset(self) -> None:
+        """Resets the state of the tracked metrics."""
+        self._validation_metrics = []
+        self._test_metrics = []
+
+    def _update_validation_metrics(self, metrics: _EVALUATE_OUTPUT) -> None:
+        """Updates the validation metrics in-place."""
+        self._validation_metrics = _update_session_metrics(self._validation_metrics, metrics)
+
+    def _update_test_metrics(self, metrics: _EVALUATE_OUTPUT | None) -> None:
+        """Updates the test metrics in-place."""
+        if metrics:
+            self._test_metrics = _update_session_metrics(self._test_metrics, metrics)
+
+    def _save_config(self) -> None:
+        """Saves the config yaml with resolved env placeholders to the output directory."""
+        if self.config_path:
+            config = OmegaConf.load(self.config_path)
+            fs = cloud_io.get_filesystem(self._output_dir, anon=False)
+            with fs.open(os.path.join(self._output_dir, self._config_file), "w") as file:
+                config_yaml = OmegaConf.to_yaml(config, resolve=True)
+                file.write(config_yaml)
+
+
+def _update_session_metrics(
+    session_metrics: List[SESSION_METRICS],
+    run_metrics: _EVALUATE_OUTPUT,
+) -> List[SESSION_METRICS]:
+    """Updates and returns the given metrics session with the new ones."""
+    session_metrics = session_metrics or _init_session_metrics(len(run_metrics))
+    for index, dataset_metrics in enumerate(run_metrics):
+        for name, value in dataset_metrics.items():
+            session_metrics[index][name].append(value)
+    return session_metrics
+
+
+def _init_session_metrics(n_datasets: int) -> List[SESSION_METRICS]:
+    """Returns the init session metrics."""
+    return [collections.defaultdict(list) for _ in range(n_datasets)]
+
+
+def _calculate_statistics(session_metrics: SESSION_METRICS) -> Dict[str, float | List[float]]:
+    """Calculate the metric statistics of a dataset session run."""
+
+    def _calculate_metric_statistics(values: List[float]) -> Dict[str, float | List[float]]:
+        """Calculates and returns the metric statistics."""
+        mean = statistics.mean(values)
+        stdev = statistics.stdev(values) if len(values) > 1 else 0
+        return {"mean": mean, "stdev": stdev, "values": values}
+
+    return dicttoolz.valmap(_calculate_metric_statistics, session_metrics)
+
+
+def _save_json(data: Dict[str, Any], save_as: str = "data.json"):
+    """Saves data to a json file."""
+    if not save_as.endswith(".json"):
+        raise ValueError()
+
+    output_dir = os.path.dirname(save_as)
+    fs = cloud_io.get_filesystem(output_dir, anon=False)
+    fs.makedirs(output_dir, exist_ok=True)
+    with fs.open(save_as, "w") as file:
+        json.dump(data, file, indent=4, sort_keys=True)

eva/core/trainers/_utils.py

@@ -0,0 +1,12 @@
+"""Training related utilities."""
+
+import copy
+from collections import abc
+from typing import Any
+
+
+def clone(*inputs: Any) -> Any:
+    """Deep copies a list of object and returns them."""
+    if not isinstance(inputs, abc.Iterable):
+        return copy.deepcopy(inputs)
+    return [copy.deepcopy(obj) for obj in inputs]

eva/core/trainers/functional.py

@@ -0,0 +1,113 @@
+"""Fit session related functions."""
+
+from typing import Tuple
+
+from lightning.pytorch.utilities.types import _EVALUATE_OUTPUT
+
+from eva.core.data import datamodules
+from eva.core.models import modules
+from eva.core.trainers import _recorder, _utils
+from eva.core.trainers import trainer as eva_trainer
+
+
+def run_evaluation_session(
+    base_trainer: eva_trainer.Trainer,
+    base_model: modules.ModelModule,
+    datamodule: datamodules.DataModule,
+    *,
+    n_runs: int = 1,
+) -> None:
+    """Runs a downstream evaluation session out-of-place.
+
+    It performs an evaluation run (fit and evaluate) on the model
+    multiple times. Note that as the input `base_trainer` and
+    `base_model` would be cloned, the input object would not
+    be modified.
+
+    Args:
+        base_trainer: The base trainer module to use.
+        base_model: The base model module to use.
+        datamodule: The data module.
+        n_runs: The amount of runs (fit and evaluate) to perform.
+    """
+    recorder = _recorder.SessionRecorder(output_dir=base_trainer.default_log_dir)
+    for run_index in range(n_runs):
+        validation_scores, test_scores = run_evaluation(
+            base_trainer, base_model, datamodule, run_id=f"run_{run_index}"
+        )
+        recorder.update(validation_scores, test_scores)
+    recorder.save()
+
+
+def run_evaluation(
+    base_trainer: eva_trainer.Trainer,
+    base_model: modules.ModelModule,
+    datamodule: datamodules.DataModule,
+    *,
+    run_id: str | None = None,
+) -> Tuple[_EVALUATE_OUTPUT, _EVALUATE_OUTPUT | None]:
+    """Fits and evaluates a model out-of-place.
+
+    Args:
+        base_trainer: The base trainer to use but not modify.
+        base_model: The model module to use but not modify.
+        datamodule: The data module.
+        run_id: The run id to be appended to the output log directory.
+            If `None`, it will use the log directory of the trainer as is.
+
+    Returns:
+        A tuple of with the validation and the test metrics (if exists).
+    """
+    trainer, model = _utils.clone(base_trainer, base_model)
+    trainer.setup_log_dirs(run_id or "")
+    return fit_and_validate(trainer, model, datamodule)
+
+
+def fit_and_validate(
+    trainer: eva_trainer.Trainer,
+    model: modules.ModelModule,
+    datamodule: datamodules.DataModule,
+) -> Tuple[_EVALUATE_OUTPUT, _EVALUATE_OUTPUT | None]:
+    """Fits and evaluates a model in-place.
+
+    If the test set is set in the datamodule, it will evaluate the model
+    on the test set as well.
+
+    Args:
+        trainer: The trainer module to use and update in-place.
+        model: The model module to use and update in-place.
+        datamodule: The data module.
+
+    Returns:
+        A tuple of with the validation and the test metrics (if exists).
+    """
+    trainer.fit(model, datamodule=datamodule)
+    validation_scores = trainer.validate(datamodule=datamodule)
+    test_scores = None if datamodule.datasets.test is None else trainer.test(datamodule=datamodule)
+    return validation_scores, test_scores
+
+
+def infer_model(
+    base_trainer: eva_trainer.Trainer,
+    base_model: modules.ModelModule,
+    datamodule: datamodules.DataModule,
+    *,
+    return_predictions: bool = False,
+) -> None:
+    """Performs model inference out-of-place.
+
+    Note that the input `base_model` and `base_trainer` would
+    not be modified.
+
+    Args:
+        base_trainer: The base trainer to use but not modify.
+        base_model: The model module to use but not modify.
+        datamodule: The data module.
+        return_predictions: Whether to return the model predictions.
+    """
+    trainer, model = _utils.clone(base_trainer, base_model)
+    return trainer.predict(
+        model=model,
+        datamodule=datamodule,
+        return_predictions=return_predictions,
+    )

eva/core/trainers/trainer.py

@@ -0,0 +1,97 @@
+"""Core trainer module."""
+
+import os
+from typing import Any
+
+from lightning.pytorch import loggers as pl_loggers
+from lightning.pytorch import trainer as pl_trainer
+from lightning.pytorch.utilities import argparse
+from typing_extensions import override
+
+from eva.core.data import datamodules
+from eva.core.models import modules
+from eva.core.trainers import _logging, functional
+
+
+class Trainer(pl_trainer.Trainer):
+    """Core trainer class.
+
+    This is an extended version of lightning's core trainer class.
+    """
+
+    @argparse._defaults_from_env_vars
+    def __init__(
+        self,
+        *args: Any,
+        default_root_dir: str = "logs",
+        n_runs: int = 1,
+        **kwargs: Any,
+    ) -> None:
+        """Initializes the trainer.
+
+        For the input arguments, refer to ::class::`lightning.pytorch.Trainer`.
+
+        Args:
+            args: Positional arguments of ::class::`lightning.pytorch.Trainer`.
+            default_root_dir: The default root directory to store the output logs.
+                Unlike in ::class::`lightning.pytorch.Trainer`, this path would be the
+                prioritized destination point.
+            n_runs: The amount of runs (fit and evaluate) to perform in an evaluation session.
+            kwargs: Kew-word arguments of ::class::`lightning.pytorch.Trainer`.
+        """
+        super().__init__(*args, default_root_dir=default_root_dir, **kwargs)
+
+        self._n_runs = n_runs
+
+        self._session_id: str = _logging.generate_session_id()
+        self._log_dir: str = self.default_log_dir
+
+        self.setup_log_dirs()
+
+    @property
+    def default_log_dir(self) -> str:
+        """Returns the default log directory."""
+        return os.path.join(self.default_root_dir, self._session_id)
+
+    @property
+    @override
+    def log_dir(self) -> str | None:
+        return self.strategy.broadcast(self._log_dir)
+
+    def setup_log_dirs(self, subdirectory: str = "") -> None:
+        """Setups the logging directory of the trainer and experimental loggers in-place.
+
+        Args:
+            subdirectory: Whether to append a subdirectory to the output log.
+        """
+        self._log_dir = os.path.join(self.default_root_dir, self._session_id, subdirectory)
+        os.fspath(self._log_dir)
+
+        for logger in self.loggers:
+            if isinstance(logger, (pl_loggers.CSVLogger, pl_loggers.TensorBoardLogger)):
+                logger._root_dir = self.default_root_dir
+                logger._name = self._session_id
+                logger._version = subdirectory
+
+    def run_evaluation_session(
+        self,
+        model: modules.ModelModule,
+        datamodule: datamodules.DataModule,
+    ) -> None:
+        """Runs a evaluation session out-of-place.
+
+        It performs an evaluation run (fit and evaluate) the model
+        `self._n_run` times. Note that the input `base_model` would
+        not be modified, so the weights of the input model will remain
+        as they are.
+
+        Args:
+            model: The base model module to evaluate.
+            datamodule: The data module.
+        """
+        functional.run_evaluation_session(
+            base_trainer=self,
+            base_model=model,
+            datamodule=datamodule,
+            n_runs=self._n_runs,
+        )

eva/core/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities and library level helper functionalities."""

eva/core/utils/io/__init__.py

@@ -0,0 +1,5 @@
+"""Core I/O utilities."""
+
+from eva.core.utils.io.dataframe import read_dataframe
+
+__all__ = ["read_dataframe"]

eva/core/utils/io/dataframe.py

@@ -0,0 +1,21 @@
+"""DataFrame related I/O operations."""
+
+import pandas as pd
+
+
+def read_dataframe(path: str) -> pd.DataFrame:
+    """Reads and loads a DataFrame file.
+
+    Args:
+        path: The path to the manifest file.
+
+    Returns:
+        The data of the file as a `DataFrame`.
+    """
+    if path.endswith(".csv"):
+        data = pd.read_csv(path)
+    elif path.endswith(".parquet"):
+        data = pd.read_parquet(path)
+    else:
+        raise ValueError(f"Failed to load manifest file at '{path}'.")
+    return data

eva/core/utils/multiprocessing.py

@@ -0,0 +1,44 @@
+"""Multiprocessing utilities."""
+
+import multiprocessing
+import sys
+import traceback
+from typing import Any
+
+
+class Process(multiprocessing.Process):
+    """Multiprocessing wrapper with logic to propagate exceptions to the parent process.
+
+    Source: https://stackoverflow.com/a/33599967/4992248
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize the process."""
+        multiprocessing.Process.__init__(self, *args, **kwargs)
+
+        self._parent_conn, self._child_conn = multiprocessing.Pipe()
+        self._exception = None
+
+    def run(self) -> None:
+        """Run the process."""
+        try:
+            multiprocessing.Process.run(self)
+            self._child_conn.send(None)
+        except Exception as e:
+            tb = traceback.format_exc()
+            self._child_conn.send((e, tb))
+
+    @property
+    def exception(self):
+        """Property that contains exception information from the process."""
+        if self._parent_conn.poll():
+            self._exception = self._parent_conn.recv()
+        return self._exception
+
+    def check_exceptions(self) -> None:
+        """Check for exception propagate it to the parent process."""
+        if not self.is_alive():
+            if self.exception:
+                error, traceback = self.exception
+                sys.stderr.write(traceback + "\n")
+                raise error

eva/core/utils/workers.py

@@ -0,0 +1,21 @@
+"""Processing workers utilities and helper functions."""
+
+import multiprocessing
+from typing import Any, Callable
+
+
+def main_worker_only(func: Callable) -> Any:
+    """Function decorator which will execute it only on main / worker process."""
+
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        """Wrapper function for the decorated method."""
+        if is_main_worker():
+            return func(*args, **kwargs)
+
+    return wrapper
+
+
+def is_main_worker() -> bool:
+    """Returns whether the main process / worker is currently used."""
+    process = multiprocessing.current_process()
+    return process.name == "MainProcess"

eva/vision/__init__.py

@@ -0,0 +1,14 @@
+"""eva vision API."""
+
+try:
+    from eva.vision import models, utils
+    from eva.vision.data import datasets, transforms
+except ImportError as e:
+    msg = (
+        "eva vision requirements are not installed.\n\n"
+        "Please pip install as follows:\n"
+        '  python -m pip install "eva[vision]" --upgrade'
+    )
+    raise ImportError(str(e) + "\n\n" + msg) from e
+
+__all__ = ["models", "utils", "datasets", "transforms"]

eva/vision/data/__init__.py

@@ -0,0 +1,5 @@
+"""Vision data API."""
+
+from eva.vision.data import datasets, transforms
+
+__all__ = ["datasets", "transforms"]

eva/vision/data/datasets/__init__.py

@@ -0,0 +1,22 @@
+"""Vision Datasets API."""
+
+from eva.vision.data.datasets.classification import (
+    BACH,
+    CRC,
+    MHIST,
+    PatchCamelyon,
+    TotalSegmentatorClassification,
+)
+from eva.vision.data.datasets.segmentation import ImageSegmentation, TotalSegmentator2D
+from eva.vision.data.datasets.vision import VisionDataset
+
+__all__ = [
+    "BACH",
+    "CRC",
+    "MHIST",
+    "ImageSegmentation",
+    "PatchCamelyon",
+    "TotalSegmentatorClassification",
+    "TotalSegmentator2D",
+    "VisionDataset",
+]

eva/vision/data/datasets/_utils.py

@@ -0,0 +1,50 @@
+"""Dataset related function and helper functions."""
+
+from typing import List, Tuple
+
+
+def indices_to_ranges(indices: List[int]) -> List[Tuple[int, int]]:
+    """Turns a list of indices to a list of ranges.
+
+    The produced range intervals are half-open inequalities: start <= x < end.
+
+    Args:
+        indices: The list of indices to produce the ranges from.
+
+    Return:
+        A list of half-open intervals.
+
+    Example:
+        >>> indices = [0, 1, 2, 4, 6, 7, 8]
+        >>> ranges = indices_to_ranges(indices)
+        >>> assert ranges == [(0, 3), (4, 5), (6, 9)]
+    """
+    ranges = []
+    start_index = 0
+    for i, current in enumerate(indices):
+        if i + 1 < len(indices) and current + 1 == indices[i + 1]:
+            continue
+
+        start = indices[start_index]
+        end = start if start_index == i else current
+        ranges.append((start, end + 1))
+        start_index = i + 1
+
+    return ranges
+
+
+def ranges_to_indices(ranges: List[Tuple[int, int]]) -> List[int]:
+    """Unpacks a list of ranges to individual indices.
+
+    Args:
+        ranges: The list of ranges to produce the indices from.
+
+    Return:
+        A list of the indices.
+
+    Example:
+        >>> ranges == [(0, 3), (4, 5), (6, 9)]
+        >>> indices = ranges_to_indices(ranges)
+        >>> assert indices == [0, 1, 2, 4, 6, 7, 8]
+    """
+    return [index for start, end in ranges for index in range(start, end)]