yaib 0.3.1__py3-none-any.whl

icu_benchmarks/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+"""Top-level package for YAIB."""
+
+__author__ = "Robin van de Water"
+__email__ = "robin.vandewater@hpi.de"
+__version__ = "0.1.0"

icu_benchmarks/contants.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class RunMode(str, Enum):
+    classification = "Classification"
+    imputation = "Imputation"
+    regression = "Regression"

icu_benchmarks/cross_validation.py

@@ -0,0 +1,120 @@
+import json
+from datetime import datetime
+import logging
+import gin
+from pathlib import Path
+from pytorch_lightning import seed_everything
+
+from icu_benchmarks.wandb_utils import wandb_log
+from icu_benchmarks.run_utils import aggregate_results
+from icu_benchmarks.data.split_process_data import preprocess_data
+from icu_benchmarks.models.train import train_common
+from icu_benchmarks.models.utils import JsonResultLoggingEncoder
+from icu_benchmarks.run_utils import log_full_line
+from icu_benchmarks.contants import RunMode
+
+
+@gin.configurable
+def execute_repeated_cv(
+    data_dir: Path,
+    log_dir: Path,
+    seed: int,
+    load_weights: bool = False,
+    source_dir: Path = None,
+    cv_repetitions: int = 5,
+    cv_repetitions_to_train: int = None,
+    cv_folds: int = 5,
+    cv_folds_to_train: int = None,
+    reproducible: bool = True,
+    debug: bool = False,
+    generate_cache: bool = False,
+    load_cache: bool = False,
+    test_on: str = "test",
+    mode: str = RunMode.classification,
+    pretrained_imputation_model: object = None,
+    cpu: bool = False,
+    verbose: bool = False,
+    wandb: bool = False,
+) -> float:
+    """Preprocesses data and trains a model for each fold.
+
+    Args:
+
+        data_dir: Path to the data directory.
+        log_dir: Path to the log directory.
+        seed: Random seed.
+        load_weights: Whether to load weights from source_dir.
+        source_dir: Path to the source directory.
+        cv_folds: Number of folds for cross validation.
+        cv_folds_to_train: Number of folds to use during training. If None, all folds are trained on.
+        cv_repetitions: Amount of cross validation repetitions.
+        cv_repetitions_to_train: Amount of training repetitions. If None, all repetitions are trained on.
+        reproducible: Whether to make torch reproducible.
+        debug: Whether to load less data and enable more logging.
+        generate_cache: Whether to generate and save cache.
+        load_cache: Whether to load previously cached data.
+        test_on: Dataset to test on. Can be "test" or "val" (e.g. for hyperparameter tuning).
+        mode: Run mode. Can be one of the values of RunMode
+        pretrained_imputation_model: Use a pretrained imputation model.
+        cpu: Whether to run on CPU.
+        verbose: Enable detailed logging.
+    Returns:
+        The average loss of all folds.
+    """
+    if not cv_repetitions_to_train:
+        cv_repetitions_to_train = cv_repetitions
+    if not cv_folds_to_train:
+        cv_folds_to_train = cv_folds
+    agg_loss = 0
+
+    seed_everything(seed, reproducible)
+    for repetition in range(cv_repetitions_to_train):
+        for fold_index in range(cv_folds_to_train):
+            start_time = datetime.now()
+            data = preprocess_data(
+                data_dir,
+                seed=seed,
+                debug=debug,
+                load_cache=load_cache,
+                generate_cache=generate_cache,
+                cv_repetitions=cv_repetitions,
+                repetition_index=repetition,
+                cv_folds=cv_folds,
+                fold_index=fold_index,
+                pretrained_imputation_model=pretrained_imputation_model,
+                runmode=mode,
+            )
+
+            repetition_fold_dir = log_dir / f"repetition_{repetition}" / f"fold_{fold_index}"
+            repetition_fold_dir.mkdir(parents=True, exist_ok=True)
+            preprocess_time = datetime.now() - start_time
+            start_time = datetime.now()
+            agg_loss += train_common(
+                data,
+                log_dir=repetition_fold_dir,
+                load_weights=load_weights,
+                source_dir=source_dir,
+                reproducible=reproducible,
+                test_on=test_on,
+                mode=mode,
+                cpu=cpu,
+                verbose=verbose,
+                use_wandb=wandb,
+            )
+            train_time = datetime.now() - start_time
+
+            log_full_line(
+                f"FINISHED FOLD {fold_index}| PREPROCESSING DURATION {preprocess_time}| TRAINING DURATION {train_time}",
+                level=logging.INFO,
+            )
+            durations = {"preprocessing_duration": preprocess_time, "train_duration": train_time}
+
+            with open(repetition_fold_dir / "durations.json", "w") as f:
+                json.dump(durations, f, cls=JsonResultLoggingEncoder)
+            if wandb:
+                wandb_log({"Iteration": repetition * cv_folds_to_train + fold_index})
+            if repetition * cv_folds_to_train + fold_index > 1:
+                aggregate_results(log_dir)
+        log_full_line(f"FINISHED CV REPETITION {repetition}", level=logging.INFO, char="=", num_newlines=3)
+
+    return agg_loss / (cv_repetitions_to_train * cv_folds_to_train)

icu_benchmarks/run.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+from datetime import datetime
+
+import gin
+import logging
+import sys
+from pathlib import Path
+import importlib.util
+
+import torch.cuda
+
+from icu_benchmarks.wandb_utils import update_wandb_config, apply_wandb_sweep, set_wandb_run_name
+from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters
+from scripts.plotting.utils import plot_aggregated_results
+from icu_benchmarks.cross_validation import execute_repeated_cv
+from icu_benchmarks.run_utils import (
+    build_parser,
+    create_run_dir,
+    aggregate_results,
+    log_full_line,
+    load_pretrained_imputation_model,
+    setup_logging,
+)
+from icu_benchmarks.contants import RunMode
+
+
+@gin.configurable("Run")
+def get_mode(mode: gin.REQUIRED):
+    # Check if enum is mode.
+    assert RunMode(mode)
+    return RunMode(mode)
+
+
+def main(my_args=tuple(sys.argv[1:])):
+    args, _ = build_parser().parse_known_args(my_args)
+
+    # Set arguments for wandb sweep
+    if args.wandb_sweep:
+        args = apply_wandb_sweep(args)
+
+    # Initialize loggers
+    log_format = "%(asctime)s - %(levelname)s - %(name)s : %(message)s"
+    date_format = "%Y-%m-%d %H:%M:%S"
+    verbose = args.verbose
+    setup_logging(date_format, log_format, verbose)
+
+    # Load weights if in evaluation mode
+    load_weights = args.command == "evaluate"
+    data_dir = Path(args.data_dir)
+
+    # Get arguments
+    name = args.name
+    task = args.task
+    model = args.model
+    reproducible = args.reproducible
+
+    # Set experiment name
+    if name is None:
+        name = data_dir.name
+    logging.info(f"Running experiment {name}.")
+
+    # Load task config
+    gin.parse_config_file(f"configs/tasks/{task}.gin")
+
+    mode = get_mode()
+
+    if args.wandb_sweep:
+        run_name = f"{mode}_{model}_{name}"
+        set_wandb_run_name(run_name)
+
+    logging.info(f"Task mode: {mode}.")
+    experiment = args.experiment
+
+    pretrained_imputation_model = load_pretrained_imputation_model(args.pretrained_imputation)
+
+    # Log imputation model to wandb
+    update_wandb_config(
+        {
+            "pretrained_imputation_model": pretrained_imputation_model.__class__.__name__
+            if pretrained_imputation_model is not None
+            else "None"
+        }
+    )
+    source_dir = None
+    log_dir_name = args.log_dir / name
+    log_dir = (
+        (log_dir_name / experiment)
+        if experiment
+        else (log_dir_name / (args.task_name if args.task_name is not None else args.task) / model)
+    )
+    if torch.cuda.is_available():
+        for name in range(0, torch.cuda.device_count()):
+            log_full_line(f"Available GPU {name}: {torch.cuda.get_device_name(name)}", level=logging.INFO)
+    else:
+        log_full_line(
+            "No GPUs available: please check your device and Torch,Cuda installation if unintended.", level=logging.WARNING
+        )
+
+    log_full_line(f"Logging to {log_dir}.", logging.INFO)
+
+    if args.preprocessor:
+        # Import custom supplied preprocessor
+        log_full_line(f"Importing custom preprocessor from {args.preprocessor}.", logging.INFO)
+        try:
+            spec = importlib.util.spec_from_file_location("CustomPreprocessor", args.preprocessor)
+            module = importlib.util.module_from_spec(spec)
+            sys.modules["preprocessor"] = module
+            spec.loader.exec_module(module)
+            gin.bind_parameter("preprocess.preprocessor", module.CustomPreprocessor)
+        except Exception as e:
+            logging.error(f"Could not import custom preprocessor from {args.preprocessor}: {e}")
+
+    if load_weights:
+        # Evaluate
+        log_dir /= f"from_{args.source_name}"
+        run_dir = create_run_dir(log_dir)
+        source_dir = args.source_dir
+        gin.parse_config_file(source_dir / "train_config.gin")
+    else:
+        # Train
+        checkpoint = log_dir / args.checkpoint if args.checkpoint else None
+        model_path = (
+            Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin"
+        )
+        gin_config_files = (
+            [Path(f"configs/experiments/{args.experiment}.gin")]
+            if args.experiment
+            else [model_path, Path(f"configs/tasks/{task}.gin")]
+        )
+        gin.parse_config_files_and_bindings(gin_config_files, args.hyperparams, finalize_config=False)
+        log_full_line(f"Data directory: {data_dir.resolve()}", level=logging.INFO)
+        run_dir = create_run_dir(log_dir)
+        choose_and_bind_hyperparameters(
+            args.tune,
+            data_dir,
+            run_dir,
+            args.seed,
+            run_mode=mode,
+            checkpoint=checkpoint,
+            debug=args.debug,
+            generate_cache=args.generate_cache,
+            load_cache=args.load_cache,
+            verbose=args.verbose,
+        )
+
+    log_full_line(f"Logging to {run_dir.resolve()}", level=logging.INFO)
+    log_full_line("STARTING TRAINING", level=logging.INFO, char="=", num_newlines=3)
+    start_time = datetime.now()
+    execute_repeated_cv(
+        data_dir,
+        run_dir,
+        args.seed,
+        load_weights=load_weights,
+        source_dir=source_dir,
+        reproducible=reproducible,
+        debug=args.debug,
+        verbose=args.verbose,
+        load_cache=args.load_cache,
+        generate_cache=args.generate_cache,
+        mode=mode,
+        pretrained_imputation_model=pretrained_imputation_model,
+        cpu=args.cpu,
+        wandb=args.wandb_sweep,
+    )
+
+    log_full_line("FINISHED TRAINING", level=logging.INFO, char="=", num_newlines=3)
+    execution_time = datetime.now() - start_time
+    log_full_line(f"DURATION: {execution_time}", level=logging.INFO, char="")
+    aggregate_results(run_dir, execution_time)
+    if args.plot:
+        plot_aggregated_results(run_dir, "aggregated_test_metrics.json")
+
+
+"""Main module."""
+if __name__ == "__main__":
+    main()

icu_benchmarks/run_utils.py

@@ -0,0 +1,254 @@
+import warnings
+from math import sqrt
+
+import torch
+import json
+from argparse import ArgumentParser, BooleanOptionalAction
+from datetime import datetime, timedelta
+import logging
+from pathlib import Path
+import scipy.stats as stats
+import shutil
+from statistics import mean, pstdev
+from icu_benchmarks.models.utils import JsonResultLoggingEncoder
+from icu_benchmarks.wandb_utils import wandb_log
+
+
+def build_parser() -> ArgumentParser:
+    """Builds an ArgumentParser for the command line.
+
+    Returns:
+        The configured ArgumentParser.
+    """
+    parser = ArgumentParser(description="Benchmark lib for processing and evaluation of deep learning models on ICU data")
+
+    parent_parser = ArgumentParser(add_help=False)
+    subparsers = parser.add_subparsers(title="Commands", dest="command", required=True)
+
+    # ARGUMENTS FOR ALL COMMANDS
+    general_args = parent_parser.add_argument_group("General arguments")
+    general_args.add_argument("-d", "--data-dir", required=True, type=Path, help="Path to the parquet data directory.")
+    general_args.add_argument("-t", "--task", default="BinaryClassification", required=True, help="Name of the task gin.")
+    general_args.add_argument("-n", "--name", required=False, help="Name of the (target) dataset.")
+    general_args.add_argument("-tn", "--task-name", required=False, help="Name of the task, used for naming experiments.")
+    general_args.add_argument("-m", "--model", default="LGBMClassifier", required=False, help="Name of the model gin.")
+    general_args.add_argument("-e", "--experiment", required=False, help="Name of the experiment gin.")
+    general_args.add_argument(
+        "-l", "--log-dir", required=False, default=Path("../yaib_logs/"), type=Path, help="Log directory with model weights."
+    )
+    general_args.add_argument(
+        "-s", "--seed", required=False, default=1234, type=int, help="Random seed for processing, tuning and training."
+    )
+    general_args.add_argument(
+        "-v",
+        "--verbose",
+        default=False,
+        required=False,
+        action=BooleanOptionalAction,
+        help="Whether to use verbose logging. Disable for clean logs.",
+    )
+    general_args.add_argument("--cpu", default=False, required=False, action=BooleanOptionalAction, help="Set to use CPU.")
+    general_args.add_argument(
+        "-db", "--debug", required=False, default=False, action=BooleanOptionalAction, help="Set to load less data."
+    )
+    general_args.add_argument(
+        "-lc",
+        "--load_cache",
+        required=False,
+        default=False,
+        action=BooleanOptionalAction,
+        help="Set to load generated data cache.",
+    )
+    general_args.add_argument(
+        "-gc",
+        "--generate_cache",
+        required=False,
+        default=False,
+        action=BooleanOptionalAction,
+        help="Set to generate data cache.",
+    )
+    general_args.add_argument("-p", "--preprocessor", required=False, type=Path, help="Load custom preprocessor from file.")
+    general_args.add_argument("-pl", "--plot", required=False, action=BooleanOptionalAction, help="Generate common plots.")
+    general_args.add_argument(
+        "-wd", "--wandb-sweep", required=False, action="store_true", help="Activates wandb hyper parameter sweep."
+    )
+    general_args.add_argument(
+        "-imp", "--pretrained-imputation", required=False, type=str, help="Path to pretrained imputation model."
+    )
+
+    # MODEL TRAINING ARGUMENTS
+    prep_and_train = subparsers.add_parser("train", help="Preprocess features and train model.", parents=[parent_parser])
+    prep_and_train.add_argument(
+        "--reproducible", required=False, default=True, action=BooleanOptionalAction, help="Make torch reproducible."
+    )
+    prep_and_train.add_argument("-hp", "--hyperparams", required=False, nargs="+", help="Hyperparameters for model.")
+    prep_and_train.add_argument("--tune", default=False, action=BooleanOptionalAction, help="Find best hyperparameters.")
+    prep_and_train.add_argument("--checkpoint", required=False, type=Path, help="Use previous checkpoint.")
+
+    # EVALUATION PARSER
+    evaluate = subparsers.add_parser("evaluate", help="Evaluate trained model on data.", parents=[parent_parser])
+    evaluate.add_argument("-sn", "--source-name", required=True, type=Path, help="Name of the source dataset.")
+    evaluate.add_argument("--source-dir", required=True, type=Path, help="Directory containing gin and model weights.")
+
+    return parser
+
+
+def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path:
+    """Creates a log directory with the current time as name.
+
+    Also creates a file in the log directory, if any parameters were randomly searched.
+    The filename contains the fixed hyperparameters to check against in future runs.
+
+    Args:
+        log_dir: Parent directory to create run directory in.
+        randomly_searched_params: String representing the randomly searched params.
+
+    Returns:
+        Path to the created run log directory.
+    """
+    log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))
+    log_dir_run.mkdir(parents=True)
+    if randomly_searched_params:
+        (log_dir_run / randomly_searched_params).touch()
+    return log_dir_run
+
+
+def aggregate_results(log_dir: Path, execution_time: timedelta = None):
+    """Aggregates results from all folds and writes to JSON file.
+
+    Args:
+        log_dir: Path to the log directory.
+        execution_time: Overall execution time.
+    """
+    aggregated = {}
+    for repetition in log_dir.iterdir():
+        if repetition.is_dir():
+            aggregated[repetition.name] = {}
+            for fold_iter in repetition.iterdir():
+                aggregated[repetition.name][fold_iter.name] = {}
+                if (fold_iter / "test_metrics.json").is_file():
+                    with open(fold_iter / "test_metrics.json", "r") as f:
+                        result = json.load(f)
+                        aggregated[repetition.name][fold_iter.name].update(result)
+                elif (fold_iter / "val_metrics.csv").is_file():
+                    with open(fold_iter / "val_metrics.csv", "r") as f:
+                        result = json.load(f)
+                        aggregated[repetition.name][fold_iter.name].update(result)
+                # Add durations to metrics
+                if (fold_iter / "durations.json").is_file():
+                    with open(fold_iter / "durations.json", "r") as f:
+                        result = json.load(f)
+                        aggregated[repetition.name][fold_iter.name].update(result)
+
+    # Aggregate results per metric
+    list_scores = {}
+    for repetition, folds in aggregated.items():
+        for fold, result in folds.items():
+            for metric, score in result.items():
+                if isinstance(score, (float, int)):
+                    list_scores[metric] = list_scores.setdefault(metric, [])
+                    list_scores[metric].append(score)
+
+    # Compute statistical metric over aggregated results
+    averaged_scores = {metric: (mean(list)) for metric, list in list_scores.items()}
+
+    # Calculate the population standard deviation over aggregated results over folds/iterations
+    # Divide by sqrt(n) to get standard deviation.
+    std_scores = {metric: (pstdev(list) / sqrt(len(list))) for metric, list in list_scores.items()}
+
+    confidence_interval = {
+        metric: (stats.t.interval(0.95, len(list) - 1, loc=mean(list), scale=stats.sem(list)))
+        for metric, list in list_scores.items()
+    }
+
+    accumulated_metrics = {
+        "avg": averaged_scores,
+        "std": std_scores,
+        "CI_0.95": confidence_interval,
+        "execution_time": execution_time.total_seconds() if execution_time is not None else 0.0,
+    }
+
+    with open(log_dir / "aggregated_test_metrics.json", "w") as f:
+        json.dump(aggregated, f, cls=JsonResultLoggingEncoder)
+
+    with open(log_dir / "accumulated_test_metrics.json", "w") as f:
+        json.dump(accumulated_metrics, f, cls=JsonResultLoggingEncoder)
+
+    logging.info(f"Accumulated results: {accumulated_metrics}")
+
+    wandb_log(json.loads(json.dumps(accumulated_metrics, cls=JsonResultLoggingEncoder)))
+
+
+def log_full_line(msg: str, level: int = logging.INFO, char: str = "-", num_newlines: int = 0):
+    """Logs a full line of a given character with a message centered.
+
+    Args:
+        msg: Message to log.
+        level: Logging level.
+        char: Character to use for the line.
+        num_newlines: Number of newlines to append.
+    """
+    terminal_size = shutil.get_terminal_size((80, 20))
+    reserved_chars = len(logging.getLevelName(level)) + 28
+    logging.log(
+        level,
+        "{0:{char}^{width}}{1}".format(msg, "\n" * num_newlines, char=char, width=terminal_size.columns - reserved_chars),
+    )
+
+
+def load_pretrained_imputation_model(use_pretrained_imputation):
+    """Loads a pretrained imputation model.
+
+    Args:
+        use_pretrained_imputation: Path to the pretrained imputation model.
+    """
+    if use_pretrained_imputation is not None and not Path(use_pretrained_imputation).exists():
+        logging.warning("The specified pretrained imputation model does not exist.")
+        use_pretrained_imputation = None
+
+    if use_pretrained_imputation is not None:
+        logging.info("Using pretrained imputation from" + str(use_pretrained_imputation))
+        pretrained_imputation_model_checkpoint = torch.load(use_pretrained_imputation, map_location=torch.device("cpu"))
+        if isinstance(pretrained_imputation_model_checkpoint, dict):
+            imputation_model_class = pretrained_imputation_model_checkpoint["class"]
+            pretrained_imputation_model = imputation_model_class(**pretrained_imputation_model_checkpoint["hyper_parameters"])
+            pretrained_imputation_model.set_trained_columns(pretrained_imputation_model_checkpoint["trained_columns"])
+            pretrained_imputation_model.load_state_dict(pretrained_imputation_model_checkpoint["state_dict"])
+        else:
+            pretrained_imputation_model = pretrained_imputation_model_checkpoint
+        pretrained_imputation_model = pretrained_imputation_model.to("cuda" if torch.cuda.is_available() else "cpu")
+        try:
+            logging.info(f"imputation model device: {next(pretrained_imputation_model.parameters()).device}")
+            pretrained_imputation_model.device = next(pretrained_imputation_model.parameters()).device
+        except Exception as e:
+            logging.debug(f"Could not set device of imputation model: {e}")
+    else:
+        pretrained_imputation_model = None
+
+    return pretrained_imputation_model
+
+
+def setup_logging(date_format, log_format, verbose):
+    """
+    Set up all loggers to use the same format and date format.
+
+    Args:
+        date_format: Format for the date.
+        log_format: Format for the log.
+        verbose: Whether to log debug messages.
+    """
+    logging.basicConfig(format=log_format, datefmt=date_format)
+    loggers = ["pytorch_lightning", "lightning_fabric"]
+    for logger in loggers:
+        logging.getLogger(logger).handlers[0].setFormatter(logging.Formatter(log_format, datefmt=date_format))
+
+    if not verbose:
+        logging.getLogger().setLevel(logging.INFO)
+        for logger in loggers:
+            logging.getLogger(logger).setLevel(logging.INFO)
+        warnings.filterwarnings("ignore")
+    else:
+        logging.getLogger().setLevel(logging.DEBUG)
+        for logger in loggers:
+            logging.getLogger(logger).setLevel(logging.DEBUG)
+        warnings.filterwarnings("default")

icu_benchmarks/wandb_utils.py

@@ -0,0 +1,61 @@
+from argparse import Namespace
+import logging
+import wandb
+
+
+def wandb_running() -> bool:
+    """Check if wandb is running."""
+    return wandb.run is not None
+
+
+def update_wandb_config(config: dict) -> None:
+    """updates wandb config if wandb is running
+
+    Args:
+        config (dict): config to set
+    """
+    logging.debug(f"Updating Wandb config: {config}")
+    if wandb_running():
+        wandb.config.update(config)
+
+
+def apply_wandb_sweep(args: Namespace) -> Namespace:
+    """applies the wandb sweep configuration to the namespace object
+
+    Args:
+        args (Namespace): parsed arguments
+
+    Returns:
+        Namespace: arguments with sweep configuration applied (some are applied via hyperparams)
+    """
+    wandb.init()
+    sweep_config = wandb.config
+    args.__dict__.update(sweep_config)
+    if args.hyperparams is None:
+        args.hyperparams = []
+    for key, value in sweep_config.items():
+        args.hyperparams.append(f"{key}=" + (("'" + value + "'") if isinstance(value, str) else str(value)))
+    logging.info(f"hyperparams after loading sweep config: {args.hyperparams}")
+    return args
+
+
+def wandb_log(log_dict):
+    """logs metrics to wandb
+
+    Args:
+        log_dict (dict): metric dict to log
+    """
+    if wandb_running():
+        wandb.log(log_dict)
+
+
+def set_wandb_run_name(run_name):
+    """stores the run name in wandb config
+
+    Args:
+        run_name (str): name of the current run
+    """
+    if wandb_running():
+        wandb.config.update({"run-name": run_name})
+        wandb.run.name = run_name
+        wandb.run.save()

yaib-0.3.1.dist-info/LICENSE

@@ -0,0 +1,25 @@
+
+
+MIT License
+
+Copyright (c) 2023, Robin van de Water, Hendrik Schmidt, Patrick Rockenschaub
+Copyright (c) 2021, ETH Zurich, Biomedical Informatics Group; ratschlab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

yaib-0.3.1.dist-info/METADATA

@@ -0,0 +1,310 @@
+Metadata-Version: 2.1
+Name: yaib
+Version: 0.3.1
+Summary: Yet Another ICU Benchmark is a holistic framework for the automation of the development of clinical prediction models on ICU data. Users can create custom datasets, cohorts, prediction tasks, endpoints, and models. 
+Home-page: https://github.com/rvandewater/YAIB
+Author: Robin van de Water
+Author-email: robin.vandewater@hpi.de
+License: MIT license
+Keywords: benchmark mimic-iii eicu hirid clinical-ml machine-learning benchmark time-series mimic-iv patient-monitoring amsterdamumcdb clinical-data ehr icu ricu pyicu
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: black (==23.3.0)
+Requires-Dist: coverage (==7.2.3)
+Requires-Dist: flake8 (==5.0.4)
+Requires-Dist: matplotlib (==3.7.1)
+Requires-Dist: gin-config (==0.5.0)
+Requires-Dist: pytorch-ignite (==0.4.11)
+Requires-Dist: torch (==2.0.1)
+Requires-Dist: pytorch-cuda (==11.8)
+Requires-Dist: lightgbm (==3.3.5)
+Requires-Dist: numpy (==1.24.3)
+Requires-Dist: pandas (==2.0.0)
+Requires-Dist: pyarrow (==11.0.0)
+Requires-Dist: pytest (==7.3.1)
+Requires-Dist: scikit-learn (==1.2.2)
+Requires-Dist: tensorboard (==2.12.2)
+Requires-Dist: tqdm (==4.64.1)
+Requires-Dist: pytorch-lightning (==2.0.3)
+Requires-Dist: wandb (==0.15.4)
+Requires-Dist: pip (==23.1)
+Requires-Dist: einops (==0.6.1)
+Requires-Dist: hydra-core (==1.3)
+Requires-Dist: recipies (==0.1.1)
+Provides-Extra: mps
+Requires-Dist: mkl (<2022) ; extra == 'mps'
+
+![YAIB logo](https://github.com/rvandewater/YAIB/blob/development/docs/figures/yaib_logo.png?raw=true)
+
+# 🧪 Yet Another ICU Benchmark
+
+[![CI](https://github.com/rvandewater/YAIB/actions/workflows/ci.yml/badge.svg?branch=development)](https://github.com/rvandewater/YAIB/actions/workflows/ci.yml)
+[![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+![Platform](https://img.shields.io/badge/platform-linux--64%20|%20win--64%20|%20osx--64-lightgrey)
+[![arXiv](https://img.shields.io/badge/arXiv-2306.05109-b31b1b.svg)](http://arxiv.org/abs/2306.05109)
+[![PyPI version shields.io](https://img.shields.io/pypi/v/yaib.svg)](https://pypi.python.org/pypi/yaib/)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+[//]: # (TODO: add coverage once we have some tests )
+
+Yet another ICU benchmark (YAIB) provides a framework for doing clinical machine learning experiments on Intensive Care Unit (
+ICU) EHR data.
+
+We support the following datasets out of the box:
+
+| **Dataset**                 | [MIMIC-III](https://physionet.org/content/mimiciii/) / [IV](https://physionet.org/content/mimiciv/) | [eICU-CRD](https://physionet.org/content/eicu-crd/) | [HiRID](https://physionet.org/content/hirid/1.1.1/) | [AUMCdb](https://doi.org/10.17026/dans-22u-f8vd) |
+|-----------------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------|--------------------------------------------------|
+| **Admissions**              | 40k / 73k                                                                                           | 200k                                                | 33k                                                 | 23k                                              |
+| **Version**                 | v1.4 / v2.2                                                                                         | v2.0                                                | v1.1.1                                              | v1.0.2                                           |  
+| **Frequency** (time-series) | 1 hour                                                                                              | 5 minutes                                           | 2 / 5 minutes                                       | up to 1 minute                                   |
+| **Originally published**    | 2015  / 2020                                                                                        | 2017                                                | 2020                                                | 2019                                             | 
+| **Origin**                  | USA                                                                                                 | USA                                                 | Switzerland                                         | Netherlands                                      |
+
+New datasets can also be added. We are currently working on a package to make this process as smooth as possible.
+The benchmark is designed for operating on preprocessed parquet files.
+<!-- We refer to  PyICU (in development)
+or [ricu package](https://github.com/eth-mds/ricu) for generating these parquet files for particular cohorts and endpoints. -->
+
+We provide five common tasks for clinical prediction by default:
+
+| No  | Task                      | Frequency                 | Type                  | 
+|-----|---------------------------|---------------------------|-----------------------|
+| 1   | ICU Mortality             | Once per Stay (after 24H) | Binary Classification |
+| 2   | Acute Kidney Injury (AKI) | Hourly (within 6H)        | Binary Classification |
+| 3   | Sepsis                    | Hourly (within 6H)        | Binary Classification |
+| 4   | Kidney Function(KF)       | Once per stay             | Regression            |
+| 5   | Length of Stay (LoS)      | Hourly (within 7D)        | Regression            |
+
+New tasks can be easily added.
+For the purposes of getting started right away, we include the eICU and MIMIC-III demo datasets in our repository.
+
+The following repositories may be relevant as well:
+
+- [YAIB-cohorts](https://github.com/rvandewater/YAIB-cohorts): Cohort generation for YAIB.
+- [YAIB-models](https://github.com/rvandewater/YAIB-models): Pretrained models for YAIB.
+- [ReciPys](https://github.com/rvandewater/ReciPys): Preprocessing package for YAIB pipelines.
+
+ For all YAIB related repositories, please see: https://github.com/stars/rvandewater/lists/yaib. 
+# 📄Paper
+
+To reproduce the benchmarks in our paper, we refer to: the [ML reproducibility document](PAPER.md).
+If you use this code in your research, please cite the following publication:
+
+```
+@article{vandewaterYetAnotherICUBenchmark2023,
+	title = {Yet Another ICU Benchmark: A Flexible Multi-Center Framework for Clinical ML},
+	shorttitle = {Yet Another ICU Benchmark},
+	url = {http://arxiv.org/abs/2306.05109},
+	language = {en},
+	urldate = {2023-06-09},
+	publisher = {arXiv},
+	author = {Robin van de Water and Hendrik Schmidt and Paul Elbers and Patrick Thoral and Bert Arnrich and Patrick Rockenschaub},
+	month = jun,
+	year = {2023},
+	note = {arXiv:2306.05109 [cs]},
+	keywords = {Computer Science - Machine Learning},
+}
+```
+
+This paper can also be found on arxiv [2306.05109](https://arxiv.org/abs/2306.05109)
+
+# 💿Installation
+YAIB is currently ideally installed from source, however we also offer it an early PyPi release.
+
+## Installation from source
+First, we clone this repository using git:
+````
+git clone https://github.com/rvandewater/YAIB.git
+````
+Please note the branch. The newest features and fixes are available at the development branch:
+````
+git checkout development
+````
+YAIB can be installed using a conda environment (preferred) or pip. Below are the three CLI commands to install YAIB 
+using **conda**.
+
+The first command will install an environment based on Python 3.10.
+
+```
+conda env update -f <environment.yml|environment_mps.yml>
+```
+
+> Use `environment.yml` on x86 hardware and `environment_mps.yml` on Macs with Metal Performance Shaders.
+
+We then activate the environment and install a package called `icu-benchmarks`, after which YAIB should be operational.
+
+```
+conda activate yaib
+pip install -e .
+```
+
+If you want to install the icu-benchmarks package with **pip**, execute the command below:
+
+```
+pip install torch numpy && pip install -e .
+```
+After installation, please check if your Pytorch version works with CUDA (in case available) to ensure the best performance. 
+YAIB will automatically list available processors at initialization in its log files.
+
+# 👩‍💻Usage
+
+Please refer to [our wiki](https://github.com/rvandewater/YAIB/wiki) for detailed information on how to use YAIB.
+
+## Quickstart 🚀 (demo data)
+
+In the folder `demo_data` we provide processed publicly available demo datasets from eICU and MIMIC with the necessary labels
+for `Mortality at 24h`,`Sepsis`, `Akute Kidney Injury`, `Kidney Function`, and `Length of Stay`.
+
+If you do not yet have access to the ICU datasets, you can run the following command to train models for the included demo
+cohorts:
+
+```
+wandb sweep --verbose experiments/demo_benchmark_classification.yml
+wandb sweep --verbose experiments/demo_benchmark_regression.yml
+```
+
+```train
+wandb agent <sweep_id>
+```
+
+> Tip: You can choose to run each of the configurations on a SLURM cluster instance by `wandb agent --count 1 <sweep_id>`
+
+> Note: You will need to have a wandb account and be logged in to run the above commands.
+
+## Getting the datasets
+
+HiRID, eICU, and MIMIC IV can be accessed through [PhysioNet](https://physionet.org/). A guide to this process can be
+found [here](https://eicu-crd.mit.edu/gettingstarted/access/).
+AUMCdb can be accessed through a separate access [procedure](https://github.com/AmsterdamUMC/AmsterdamUMCdb). We do not have
+involvement in the access procedure and can not answer to any requests for data access.
+
+## Cohort creation
+
+Since the datasets were created independently of each other, they do not share the same data structure or data identifiers. In
+order to make them interoperable, use the preprocessing utilities
+provided by the [ricu package](https://github.com/eth-mds/ricu).
+Ricu pre-defines a large number of clinical concepts and how to load them from a given dataset, providing a common interface to
+the data, that is used in this
+benchmark. Please refer to our [cohort definition](https://github.com/rvandewater/YAIB-cohorts) code for generating the cohorts
+using our python interface for ricu.
+After this, you can run the benchmark once you have gained access to the datasets.
+
+# 👟 Running YAIB
+
+## Preprocessing and Training
+
+The following command will run training and evaluation on the MIMIC demo dataset for (Binary) mortality prediction at 24h with
+the
+LGBMClassifier. Child samples are reduced due to the small amount of training data. We load available cache and, if available,
+load
+existing cache files.
+
+```
+icu-benchmarks train \
+    -d demo_data/mortality24/mimic_demo \
+    -n mimic_demo \
+    -t BinaryClassification \
+    -tn Mortality24 \
+    -m LGBMClassifier \
+    -hp LGBMClassifier.min_child_samples=10 \
+    --generate_cache
+    --load_cache \
+    --seed 2222 \
+    -s 2222 \
+    -l ../yaib_logs/ \
+    --tune
+```
+
+> For a list of available flags, run `icu-benchmarks train -h`.
+
+> Run with `PYTORCH_ENABLE_MPS_FALLBACK=1` on Macs with Metal Performance Shaders.
+
+[//]: # (> Please note that, for Windows based systems, paths need to be formatted differently, e.g: ` r"\..\data\mortality_seq\hirid"`.)
+> For Windows based systems, the next line character (\\)  needs to be replaced by (^) (Command Prompt) or (`) (Powershell)
+> respectively.
+
+
+Alternatively, the easiest method to train all the models in the paper is to run these commands from the directory root:
+
+```train
+wandb sweep --verbose experiments/benchmark_classification.yml
+wandb sweep --verbose experiments/benchmark_regression.yml
+```
+
+This will create two hyperparameter sweeps for WandB for the classification and regression tasks.
+This configuration will train all the models in the paper. You can then run the following command to train the models:
+
+```train
+wandb agent <sweep_id>
+```
+
+> Tip: You can choose to run each of the configurations on a SLURM cluster instance by `wandb agent --count 1 <sweep_id>`
+
+> Note: You will need to have a wandb account and be logged in to run the above commands.
+
+## Evaluate
+
+It is possible to evaluate a model trained on another dataset. In this case, the source dataset is the demo data from MIMIC and
+the target is the eICU demo:
+
+```
+icu-benchmarks evaluate \
+    -d demo_data/mortality24/eicu_demo \
+    -n eicu_demo \
+    -t BinaryClassification \
+    -tn Mortality24 \
+    -m LGBMClassifier \
+    --generate_cache \
+    --load_cache \
+    -s 2222 \
+    -l ../yaib_logs \
+    -sn mimic \
+    --source-dir ../yaib_logs/mimic_demo/Mortality24/LGBMClassifier/2022-12-12T15-24-46/fold_0
+```
+
+## Models
+
+We provide several existing machine learning models that are commonly used for multivariate time-series data.
+`pytorch` is used for the deep learning models, `lightgbm` for the boosted tree approaches, and `sklearn` for other classical
+machine learning models.
+The benchmark provides (among others) the following built-in models:
+
+- [Logistic Regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html?highlight=logistic+regression):
+  Standard regression approach.
+- [Elastic Net](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.ElasticNet.html): Linear regression with
+  combined L1 and L2 priors as regularizer.
+- [LightGBM](https://proceedings.neurips.cc/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf): Efficient gradient
+  boosting trees.
+- [Long Short-term Memory (LSTM)](https://ieeexplore.ieee.org/document/818041): The most commonly used type of Recurrent Neural
+  Networks for long sequences.
+- [Gated Recurrent Unit (GRU)](https://arxiv.org/abs/1406.1078) : A extension to LSTM which showed
+  improvements ([paper](https://arxiv.org/abs/1412.3555)).
+- [Temporal Convolutional Networks (TCN)](https://arxiv.org/pdf/1803.01271 ): 1D convolution approach to sequence data. By
+  using dilated convolution to extend the receptive field of the network it has shown great performance on long-term
+  dependencies.
+- [Transformers](https://papers.nips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf): The most common Attention
+  based approach.
+
+# 🛠️ Development
+
+To adapt YAIB to your own use case, you can use
+the [development information](https://github.com/rvandewater/YAIB/wiki/Contribution-and-development) page as a reference.
+We appreciate contributions to the project. Please read the [contribution guidelines](CONTRIBUTING.MD) before submitting a pull
+request.
+
+# Acknowledgements
+
+We do not own any of the datasets used in this benchmark. This project uses heavily adapted components of
+the [HiRID benchmark](https://github.com/ratschlab/HIRID-ICU-Benchmark/). We thank the authors for providing this codebase and
+encourage further development to benefit the scientific community. The demo datasets have been released under
+an [Open Data Commons Open Database License (ODbL)](https://opendatacommons.org/licenses/odbl/1-0/).
+
+# License
+
+This source code is released under the MIT license, included [here](LICENSE). We do not own any of the datasets used or
+included in this repository. 

yaib-0.3.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+icu_benchmarks/__init__.py,sha256=pBMRirXTJQo-EO2fGRq6eOp7aR7f5yqa6iYkH_QVObI,159
+icu_benchmarks/contants.py,sha256=_YB08CQmENCUnRWTIldaXzNKYeRlRo98qZi7qo6ZYck,155
+icu_benchmarks/cross_validation.py,sha256=_DXeu-V4qTEyb-V-4ci3hrpNH0BZfSnvMpbyJh8y5RY,4875
+icu_benchmarks/run.py,sha256=HrJC3Pj40Y9TDEaMCp13xgg4w5iou5SFcdzhkhmQpKI,6061
+icu_benchmarks/run_utils.py,sha256=d0LsHBepoFqBHFQNgVMbgHyGXwIjCq2_lp6chFRaEHU,11821
+icu_benchmarks/wandb_utils.py,sha256=zQOkXlI64TXn3ylbrxm9kbMc1TbNx2kVyNRNGmerBsU,1637
+yaib-0.3.1.dist-info/LICENSE,sha256=aQO49L5qimHt0UxPTlRLicU5cSVhdUnm7k1Ec98-r8E,1215
+yaib-0.3.1.dist-info/METADATA,sha256=9NpyKdl0p8qdgjxwcRU7DNaiHBRFTWquKaqk3Pe0KDw,15674
+yaib-0.3.1.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+yaib-0.3.1.dist-info/entry_points.txt,sha256=2FvHz3-zDtaqFJXungdMpL9sWlL8roqgPttNzmKr8kQ,59
+yaib-0.3.1.dist-info/top_level.txt,sha256=tMNfKvvlPACIbabj4GoWKS3Qd3sukUaiq7AcbqTXaiM,15
+yaib-0.3.1.dist-info/RECORD,,

yaib-0.3.1.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.38.4)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

yaib-0.3.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+icu-benchmarks = icu_benchmarks.run:main

yaib-0.3.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+icu_benchmarks