halib 0.2.30__py3-none-any.whl

halib/research/data/torchloader.py

@@ -0,0 +1,165 @@
+"""
+ * @author Hoang Van-Ha
+ * @email hoangvanhauit@gmail.com
+ * @create date 2024-03-27 15:40:22
+ * @modify date 2024-03-27 15:40:22
+ * @desc this module works as a utility tools for finding the best configuration for dataloader (num_workers, batch_size, pin_menory, etc.) that fits your hardware.
+"""
+from argparse import ArgumentParser
+
+import os
+import time
+import traceback
+
+from tqdm import tqdm
+from rich import inspect
+from typing import Union
+import itertools as it  # for cartesian product
+
+from torch.utils.data import DataLoader
+from torchvision import datasets, transforms
+
+from ...common.common import *
+from ...filetype import csvfile
+from ...filetype.yamlfile import load_yaml
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument("-cfg", "--cfg", type=str, help="cfg file for searching")
+    return parser.parse_args()
+
+
+def get_test_range(cfg: dict, search_item="num_workers"):
+    item_search_cfg = cfg["search_space"].get(search_item, None)
+    if item_search_cfg is None:
+        raise ValueError(f"search_item: {search_item} not found in cfg")
+    if isinstance(item_search_cfg, list):
+        return item_search_cfg
+    elif isinstance(item_search_cfg, dict):
+        if "mode" in item_search_cfg:
+            mode = item_search_cfg["mode"]
+            assert mode in ["range", "list"], f"mode: {mode} not supported"
+            value_in_mode = item_search_cfg.get(mode, None)
+            if value_in_mode is None:
+                raise ValueError(f"mode<{mode}>: data not found in <{search_item}>")
+            if mode == "range":
+                assert len(value_in_mode) == 3, f"range must have 3 values: start, stop, step"
+                start = value_in_mode[0]
+                stop = value_in_mode[1]
+                step = value_in_mode[2]
+                return list(range(start, stop, step))
+            elif mode == "list":
+                return item_search_cfg["list"]
+    else:
+        return [item_search_cfg]  # for int, float, str, bool, etc.
+
+
+def load_an_batch(loader_iter):
+    start = time.time()
+    next(loader_iter)
+    end = time.time()
+    return end - start
+
+
+def test_dataloader_with_cfg(origin_dataloader: DataLoader, cfg: Union[dict, str]):
+    try:
+        if isinstance(cfg, str):
+            cfg = load_yaml(cfg, to_dict=True)
+        dfmk = csvfile.DFCreator()
+        search_items = ["batch_size", "num_workers", "persistent_workers", "pin_memory"]
+        batch_limit = cfg["general"]["batch_limit"]
+        csv_cfg = cfg["general"]["to_csv"]
+        log_batch_info = cfg["general"]["log_batch_info"]
+
+        save_to_csv = csv_cfg["enabled"]
+        log_dir = csv_cfg["log_dir"]
+        filename = csv_cfg["filename"]
+        filename = f"{now_str()}_{filename}.csv"
+        outfile = os.path.join(log_dir, filename)
+
+        dfmk.create_table(
+            "cfg_search",
+            (search_items + ["avg_time_taken"]),
+        )
+        ls_range_test = []
+        for item in search_items:
+            range_test = get_test_range(cfg, search_item=item)
+            range_test = [(item, i) for i in range_test]
+            ls_range_test.append(range_test)
+
+        all_combinations = list(it.product(*ls_range_test))
+
+        rows = []
+        for cfg_idx, combine in enumerate(all_combinations):
+            console.rule(f"Testing cfg {cfg_idx+1}/{len(all_combinations)}")
+            inspect(combine)
+            batch_size = combine[search_items.index("batch_size")][1]
+            num_workers = combine[search_items.index("num_workers")][1]
+            persistent_workers = combine[search_items.index("persistent_workers")][1]
+            pin_memory = combine[search_items.index("pin_memory")][1]
+
+            test_dataloader = DataLoader(origin_dataloader.dataset, batch_size=batch_size, num_workers=num_workers, persistent_workers=persistent_workers, pin_memory=pin_memory, shuffle=True)
+            row = [
+                batch_size,
+                num_workers,
+                persistent_workers,
+                pin_memory,
+                0.0,
+            ]
+
+            # calculate the avg time taken to load the data for <batch_limit> batches
+            trainiter = iter(test_dataloader)
+            time_elapsed = 0
+            pprint('Start testing...')
+            for i in tqdm(range(batch_limit)):
+                single_batch_time = load_an_batch(trainiter)
+                if log_batch_info:
+                    pprint(f"Batch {i+1} took {single_batch_time:.4f} seconds to load")
+                time_elapsed += single_batch_time
+            row[-1] = time_elapsed / batch_limit
+            rows.append(row)
+        dfmk.insert_rows('cfg_search', rows)
+        dfmk.fill_table_from_row_pool('cfg_search')
+        with ConsoleLog("results"):
+            csvfile.fn_display_df(dfmk['cfg_search'])
+            if save_to_csv:
+                dfmk["cfg_search"].to_csv(outfile, index=False)
+                console.print(f"[red] Data saved to <{outfile}> [/red]")
+
+    except Exception as e:
+        traceback.print_exc()
+        print(e)
+        # get current directory of this python file
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        standar_cfg_path = os.path.join(current_dir, "torchloader_search.yaml")
+        pprint(
+            f"Make sure you get the  right <cfg.yaml> file. An example of <cfg.yaml> file can be found at this path: {standar_cfg_path}"
+        )
+        return
+
+def main():
+    args = parse_args()
+    cfg_yaml = args.cfg
+    cfg_dict = load_yaml(cfg_yaml, to_dict=True)
+
+    # Define transforms for data augmentation and normalization
+    transform = transforms.Compose(
+        [
+            transforms.RandomHorizontalFlip(),  # Randomly flip images horizontally
+            transforms.RandomRotation(10),  # Randomly rotate images by 10 degrees
+            transforms.ToTensor(),  # Convert images to PyTorch tensors
+            transforms.Normalize(
+                (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)
+            ),  # Normalize pixel values to [-1, 1]
+        ]
+    )
+    test_dataset = datasets.CIFAR10(
+        root="./data", train=False, download=True, transform=transform
+    )
+    batch_size = 64
+    train_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
+    test_dataloader_with_cfg(train_loader, cfg_dict)
+
+
+if __name__ == "__main__":
+    main()

halib/research/dataset.py

@@ -0,0 +1,208 @@
+# This script create a test version
+# of the watcam (wc) dataset
+# for testing the tflite model
+
+from argparse import ArgumentParser
+
+import os
+import click
+import shutil
+from tqdm import tqdm
+from rich import inspect
+from rich.pretty import pprint
+from torchvision.datasets import ImageFolder
+from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
+
+from ..common import console, seed_everything, ConsoleLog
+from ..system import filesys as fs
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument(
+        "-indir",
+        "--indir",
+        type=str,
+        help="orignal dataset path",
+    )
+    parser.add_argument(
+        "-outdir",
+        "--outdir",
+        type=str,
+        help="dataset out path",
+        default=".",  # default to current dir
+    )
+    parser.add_argument(
+        "-val_size",
+        "--val_size",
+        type=float,
+        help="validation size",  # no default value to force user to input
+        default=0.2,
+    )
+    # add using StratifiedShuffleSplit or ShuffleSplit
+    parser.add_argument(
+        "-seed",
+        "--seed",
+        type=int,
+        help="random seed",
+        default=42,
+    )
+    parser.add_argument(
+        "-inplace",
+        "--inplace",
+        action="store_true",
+        help="inplace operation, will overwrite the outdir if exists",
+    )
+
+    parser.add_argument(
+        "-stratified",
+        "--stratified",
+        action="store_true",
+        help="use StratifiedShuffleSplit instead of ShuffleSplit",
+    )
+    parser.add_argument(
+        "-no_train",
+        "--no_train",
+        action="store_true",
+        help="only create test set, no train set",
+    )
+    parser.add_argument(
+        "-reverse",
+        "--reverse",
+        action="store_true",
+        help="combine train and val set back to original dataset",
+    )
+    return parser.parse_args()
+
+
+def move_images(image_paths, target_set_dir):
+    for img_path in tqdm(image_paths):
+        # get folder name of the image
+        img_dir = os.path.dirname(img_path)
+        out_cls_dir = os.path.join(target_set_dir, os.path.basename(img_dir))
+        if not os.path.exists(out_cls_dir):
+            os.makedirs(out_cls_dir)
+        # move the image to the class folder
+        shutil.move(img_path, out_cls_dir)
+
+
+def split_dataset_cls(
+    indir, outdir, val_size, seed, inplace, stratified_split, no_train
+):
+    seed_everything(seed)
+    console.rule("Config confirm?")
+    pprint(locals())
+    click.confirm("Continue?", abort=True)
+    assert os.path.exists(indir), f"{indir} does not exist"
+
+    if not inplace:
+        assert (not inplace) and (
+            not os.path.exists(outdir)
+        ), f"{outdir} already exists; SKIP ...."
+
+    if inplace:
+        outdir = indir
+    if not os.path.exists(outdir):
+        os.makedirs(outdir)
+
+    console.rule(f"Creating train/val dataset")
+
+    sss = (
+        ShuffleSplit(n_splits=1, test_size=val_size)
+        if not stratified_split
+        else StratifiedShuffleSplit(n_splits=1, test_size=val_size)
+    )
+
+    pprint({"split strategy": sss, "indir": indir, "outdir": outdir})
+    dataset = ImageFolder(
+        root=indir,
+        transform=None,
+    )
+    train_dataset_indices = None
+    val_dataset_indices = None  # val here means test
+    for train_indices, val_indices in sss.split(dataset.samples, dataset.targets):
+        train_dataset_indices = train_indices
+        val_dataset_indices = val_indices
+
+    # get image paths for train/val split dataset
+    train_image_paths = [dataset.imgs[i][0] for i in train_dataset_indices]
+    val_image_paths = [dataset.imgs[i][0] for i in val_dataset_indices]
+
+    # start creating train/val folders then move images
+    out_train_dir = os.path.join(outdir, "train")
+    out_val_dir = os.path.join(outdir, "val")
+    if inplace:
+        assert os.path.exists(out_train_dir) == False, f"{out_train_dir} already exists"
+        assert os.path.exists(out_val_dir) == False, f"{out_val_dir} already exists"
+
+    os.makedirs(out_train_dir)
+    os.makedirs(out_val_dir)
+
+    if not no_train:
+        with ConsoleLog(f"Moving train images to {out_train_dir} "):
+            move_images(train_image_paths, out_train_dir)
+    else:
+        pprint("test only, skip moving train images")
+        # remove out_train_dir
+        shutil.rmtree(out_train_dir)
+
+    with ConsoleLog(f"Moving val images to {out_val_dir} "):
+        move_images(val_image_paths, out_val_dir)
+
+    if inplace:
+        pprint(f"remove all folders, except train and val")
+        for cls_dir in os.listdir(outdir):
+            if cls_dir not in ["train", "val"]:
+                shutil.rmtree(os.path.join(indir, cls_dir))
+
+
+def reverse_split_ds(indir):
+    console.rule(f"Reversing split dataset <{indir}>...")
+    ls_dirs = os.listdir(indir)
+    # make sure there are only two dirs 'train' and 'val'
+    assert len(ls_dirs) == 2, f"Found more than 2 dirs: {len(ls_dirs) } dirs"
+    assert "train" in ls_dirs, f"train dir not found in {indir}"
+    assert "val" in ls_dirs, f"val dir not found in {indir}"
+    train_dir = os.path.join(indir, "train")
+    val_dir = os.path.join(indir, "val")
+    all_train_files = fs.filter_files_by_extension(
+        train_dir, ["jpg", "jpeg", "png", "bmp", "gif", "tiff"]
+    )
+    all_val_files = fs.filter_files_by_extension(
+        val_dir, ["jpg", "jpeg", "png", "bmp", "gif", "tiff"]
+    )
+    # move all files from train to indir
+    with ConsoleLog(f"Moving train images to {indir} "):
+        move_images(all_train_files, indir)
+    with ConsoleLog(f"Moving val images to {indir} "):
+        move_images(all_val_files, indir)
+    with ConsoleLog(f"Removing train and val dirs"):
+        # remove train and val dirs
+        shutil.rmtree(train_dir)
+        shutil.rmtree(val_dir)
+
+
+def main():
+    args = parse_args()
+    indir = args.indir
+    outdir = args.outdir
+    if outdir == ".":
+        # get current folder of the indir
+        indir_parent_dir = os.path.dirname(os.path.normpath(indir))
+        indir_name = os.path.basename(indir)
+        outdir = os.path.join(indir_parent_dir, f"{indir_name}_split")
+    val_size = args.val_size
+    seed = args.seed
+    inplace = args.inplace
+    stratified_split = args.stratified
+    no_train = args.no_train
+    reverse = args.reverse
+    if not reverse:
+        split_dataset_cls(
+            indir, outdir, val_size, seed, inplace, stratified_split, no_train
+        )
+    else:
+        reverse_split_ds(indir)
+
+
+if __name__ == "__main__":
+    main()

halib/research/flop_csv.py

@@ -0,0 +1,34 @@
+from halib import *
+from flops import _calculate_flops_for_model
+
+from halib import *
+from argparse import ArgumentParser
+
+
+def main():
+    csv_file = "./results-imagenet.csv"
+    df = pd.read_csv(csv_file)
+    # make param_count column as float
+    # df['param_count'] = df['param_count'].astype(float)
+    df['param_count'] = pd.to_numeric(df['param_count'], errors='coerce').fillna(99999).astype(float)
+    df = df[df['param_count'] < 5.0]  # filter models with param_count < 20M
+
+    dict_ls = []
+
+    for index, row in tqdm(df.iterrows()):
+        console.rule(f"Row {index+1}/{len(df)}")
+        model = row['model']
+        num_class = 2
+        _, _, mflops = _calculate_flops_for_model(model, num_class)
+        dict_ls.append({'model': model, 'param_count': row['param_count'], 'mflops': mflops})
+
+    # Create a DataFrame from the list of dictionaries
+    result_df = pd.DataFrame(dict_ls)
+
+    final_df = pd.merge(df, result_df, on=['model', 'param_count'])
+    final_df.sort_values(by='mflops', inplace=True, ascending=True)
+    csvfile.fn_display_df(final_df)
+
+
+if __name__ == "__main__":
+    main()

halib/research/flops.py

@@ -0,0 +1,156 @@
+import os
+import sys
+import torch
+import timm
+from argparse import ArgumentParser
+from fvcore.nn import FlopCountAnalysis
+from halib import *
+from halib.filetype import csvfile
+from curriculum.utils.config import *
+from curriculum.utils.model_helper import *
+
+
+# ---------------------------------------------------------------------
+# Argument Parser
+# ---------------------------------------------------------------------
+def parse_args():
+    parser = ArgumentParser(description="Calculate FLOPs for TIMM or trained models")
+
+    # Option 1: Direct TIMM model
+    parser.add_argument(
+        "--model_name", type=str, help="TIMM model name (e.g., efficientnet_b0)"
+    )
+    parser.add_argument(
+        "--num_classes", type=int, default=1000, help="Number of output classes"
+    )
+
+    # Option 2: Experiment directory
+    parser.add_argument(
+        "--indir",
+        type=str,
+        default=None,
+        help="Directory containing trained experiment (with .yaml and .pth)",
+    )
+    parser.add_argument(
+        "-o", "--o", action="store_true", help="Open output CSV after saving"
+    )
+    return parser.parse_args()
+
+
+# ---------------------------------------------------------------------
+# Helper Functions
+# ---------------------------------------------------------------------
+def _get_list_of_proc_dirs(indir):
+    assert os.path.exists(indir), f"Input directory {indir} does not exist."
+    pth_files = [f for f in os.listdir(indir) if f.endswith(".pth")]
+    if len(pth_files) > 0:
+        return [indir]
+    return [
+        os.path.join(indir, f)
+        for f in os.listdir(indir)
+        if os.path.isdir(os.path.join(indir, f))
+    ]
+
+
+def _calculate_flops_for_model(model_name, num_classes):
+    """Calculate FLOPs for a plain TIMM model."""
+    try:
+        model = timm.create_model(model_name, pretrained=False, num_classes=num_classes)
+        input_size = timm.data.resolve_data_config(model.default_cfg)["input_size"]
+        dummy_input = torch.randn(1, *input_size)
+        model.eval() # ! set to eval mode to avoid some warnings or errors
+        flops = FlopCountAnalysis(model, dummy_input)
+        gflops = flops.total() / 1e9
+        mflops = flops.total() / 1e6
+        print(f"\nModel: **{model_name}**, Classes: {num_classes}")
+        print(f"Input size: {input_size}, FLOPs: **{gflops:.3f} GFLOPs**, **{mflops:.3f} MFLOPs**\n")
+        return model_name, gflops, mflops
+    except Exception as e:
+        print(f"[Error] Could not calculate FLOPs for {model_name}: {e}")
+        return model_name, -1, -1
+
+
+def _calculate_flops_for_experiment(exp_dir):
+    """Calculate FLOPs for a trained experiment directory."""
+    yaml_files = [f for f in os.listdir(exp_dir) if f.endswith(".yaml")]
+    pth_files = [f for f in os.listdir(exp_dir) if f.endswith(".pth")]
+
+    assert (
+        len(yaml_files) == 1
+    ), f"Expected 1 YAML file in {exp_dir}, found {len(yaml_files)}"
+    assert (
+        len(pth_files) == 1
+    ), f"Expected 1 PTH file in {exp_dir}, found {len(pth_files)}"
+
+    exp_cfg_yaml = os.path.join(exp_dir, yaml_files[0])
+    cfg = ExpConfig.from_yaml(exp_cfg_yaml)
+    ds_label_list = cfg.dataset.get_label_list()
+
+    try:
+        model = build_model(
+            cfg.model.name, num_classes=len(ds_label_list), pretrained=True
+        )
+        model_weights_path = os.path.join(exp_dir, pth_files[0])
+        model.load_state_dict(torch.load(model_weights_path, map_location="cpu"))
+        model.eval()
+
+        input_size = timm.data.resolve_data_config(model.default_cfg)["input_size"]
+        dummy_input = torch.randn(1, *input_size)
+        flops = FlopCountAnalysis(model, dummy_input)
+        gflops = flops.total() / 1e9
+        mflops = flops.total() / 1e6
+
+        return str(cfg), cfg.model.name, gflops, mflops
+    except Exception as e:
+        console.print(f"[red] Error processing {exp_dir}: {e}[/red]")
+        return str(cfg), cfg.model.name, -1, -1
+
+
+# ---------------------------------------------------------------------
+# Main Entry
+# ---------------------------------------------------------------------
+def main():
+    args = parse_args()
+
+    # Case 1: Direct TIMM model input
+    if args.model_name:
+        _calculate_flops_for_model(args.model_name, args.num_classes)
+        return
+
+    # Case 2: Experiment directory input
+    if args.indir is None:
+        print("[Error] Either --model_name or --indir must be specified.")
+        return
+
+    proc_dirs = _get_list_of_proc_dirs(args.indir)
+    pprint(proc_dirs)
+
+    dfmk = csvfile.DFCreator()
+    TABLE_NAME = "model_flops_results"
+    dfmk.create_table(TABLE_NAME, ["exp_name", "model_name", "gflops", "mflops"])
+
+    console.rule(f"Calculating FLOPs for models in {len(proc_dirs)} dir(s)...")
+    rows = []
+    for exp_dir in tqdm(proc_dirs):
+        dir_name = os.path.basename(exp_dir)
+        console.rule(f"{dir_name}")
+        exp_name, model_name, gflops, mflops = _calculate_flops_for_experiment(exp_dir)
+        rows.append([exp_name, model_name, gflops, mflops])
+
+    dfmk.insert_rows(TABLE_NAME, rows)
+    dfmk.fill_table_from_row_pool(TABLE_NAME)
+
+    outfile = f"zout/zreport/{now_str()}_model_flops_results.csv"
+    dfmk[TABLE_NAME].to_csv(outfile, sep=";", index=False)
+    csvfile.fn_display_df(dfmk[TABLE_NAME])
+
+    if args.o:
+        os.system(f"start {outfile}")
+
+
+# ---------------------------------------------------------------------
+# Script Entry
+# ---------------------------------------------------------------------
+if __name__ == "__main__":
+    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+    main()

halib/research/metrics.py

@@ -0,0 +1,137 @@
+# -------------------------------
+# Metrics Backend Interface
+# -------------------------------
+import inspect
+from typing import Dict, Union, List, Any
+from abc import ABC, abstractmethod
+
+class MetricsBackend(ABC):
+    """Interface for pluggable metrics computation backends."""
+
+    def __init__(self, metrics_info: Union[List[str], Dict[str, Any]]):
+        """
+        Initialize the backend with optional metrics_info.
+        `metrics_info` can be either:
+        - A list of metric names (strings). e.g., ["accuracy", "precision"]
+        - A dict mapping metric names with object that defines how to compute them. e.g: {"accuracy": torchmetrics.Accuracy(), "precision": torchmetrics.Precision()}
+
+        """
+        self.metric_info = metrics_info
+        self.validate_metrics_info(self.metric_info)
+
+    @property
+    def metric_names(self) -> List[str]:
+        """
+        Return a list of metric names.
+        If metric_info is a dict, return its keys; if it's a list, return it directly.
+        """
+        if isinstance(self.metric_info, dict):
+            return list(self.metric_info.keys())
+        elif isinstance(self.metric_info, list):
+            return self.metric_info
+        else:
+            raise TypeError("metric_info must be a list or a dict")
+
+    def validate_metrics_info(self, metrics_info):
+        if isinstance(metrics_info, list):
+            return metrics_info
+        elif isinstance(metrics_info, dict):
+            return {k: v for k, v in metrics_info.items() if isinstance(k, str)}
+        else:
+            raise TypeError(
+                "metrics_info must be a list of strings or a dict with string keys"
+            )
+
+    @abstractmethod
+    def compute_metrics(
+        self, metrics_info: Union[List[str], Dict[str, Any]],  metrics_data_dict: Dict[str, Any], *args, **kwargs
+    ) -> Dict[str, Any]:
+        pass
+
+    def prepare_metrics_backend_data(
+        self, raw_metric_data, *args, **kwargs
+    ):
+        """
+        Prepare the data for the metrics backend.
+        This method can be overridden by subclasses to customize data preparation.
+        """
+        return raw_metric_data
+
+    def calc_metrics(
+        self,  metrics_data_dict: Dict[str, Any], *args, **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Calculate metrics based on the provided metrics_info and data.
+        This method should be overridden by subclasses to implement specific metric calculations.
+        """
+        # prevalidate the metrics_data_dict
+        for metric in self.metric_names:
+            if metric not in metrics_data_dict:
+                raise ValueError(f"Metric '{metric}' not found in provided data.")
+        # Prepare the data for the backend
+        metrics_data_dict = self.prepare_metrics_backend_data(
+            metrics_data_dict, *args, **kwargs
+        )
+        # Call the abstract method to compute metrics
+        return self.compute_metrics(self.metric_info, metrics_data_dict, *args, **kwargs)
+
+class TorchMetricsBackend(MetricsBackend):
+    """TorchMetrics-based backend implementation."""
+
+    def __init__(self, metrics_info: Union[List[str], Dict[str, Any]]):
+        try:
+            import torch
+            from torchmetrics import Metric
+        except ImportError:
+            raise ImportError(
+                "TorchMetricsBackend requires torch and torchmetrics to be installed."
+            )
+        self.metric_info = metrics_info
+        self.torch = torch
+        self.Metric = Metric
+        self.validate_metrics_info(metrics_info)
+
+    def validate_metrics_info(self, metrics_info):
+        if not isinstance(metrics_info, dict):
+            raise TypeError(
+                "TorchMetricsBackend requires metrics_info as a dict {name: MetricInstance}"
+            )
+        for k, v in metrics_info.items():
+            if not isinstance(k, str):
+                raise TypeError(f"Key '{k}' is not a string")
+            if not isinstance(v, self.Metric):
+                raise TypeError(f"Value for key '{k}' must be a torchmetrics.Metric")
+        return metrics_info
+
+    def compute_metrics(self, metrics_info, metrics_data_dict, *args, **kwargs):
+        out_dict = {}
+        for metric, metric_instance in metrics_info.items():
+            if metric not in metrics_data_dict:
+                raise ValueError(f"Metric '{metric}' not found in provided data.")
+
+            metric_data = metrics_data_dict[metric]
+            sig = inspect.signature(metric_instance.update)
+            expected_args = list(sig.parameters.values())
+
+            if isinstance(metric_data, dict):
+                args = [metric_data[param.name] for param in expected_args]
+            elif isinstance(metric_data, (list, tuple)):
+                args = metric_data
+            else:
+                args = metric_data
+            if len(expected_args) == 1:
+                metric_instance.update(args)
+            else:
+                metric_instance.update(*args)
+
+            computed_value = metric_instance.compute()
+            if isinstance(computed_value, self.torch.Tensor):
+                computed_value = (
+                    computed_value.item()
+                    if computed_value.numel() == 1
+                    else computed_value.tolist()
+                )
+
+
+            out_dict[metric] = computed_value
+        return out_dict