fusion-bench 0.2.20__py3-none-any.whl → 0.2.21__py3-none-any.whl

fusion_bench/taskpool/dummy.py

@@ -4,13 +4,13 @@ This is the dummy task pool that is used for debugging purposes.
 
 from typing import Optional
 
+from lightning.pytorch.utilities import rank_zero_only
 from torch import nn
 
 from fusion_bench.models.separate_io import separate_save
 from fusion_bench.taskpool.base_pool import BaseTaskPool
 from fusion_bench.utils import timeit_context
 from fusion_bench.utils.parameters import count_parameters, print_parameters
-from lightning.pytorch.utilities import rank_zero_only
 
 
 def get_model_summary(model: nn.Module) -> dict:

fusion_bench/taskpool/lm_eval_harness/taskpool.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from typing import List, Literal, Optional, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Literal, Optional, Union
 
 import lightning.fabric
 import lm_eval
@@ -12,7 +12,6 @@ from fusion_bench import BaseTaskPool
 from fusion_bench.mixins import LightningFabricMixin
 from fusion_bench.utils.strenum import _version
 
-
 log = logging.getLogger(__name__)
 
 

fusion_bench/tasks/clip_classification/__init__.py

@@ -1,6 +1,6 @@
 import importlib
 import warnings
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Tuple
 
 from datasets import load_dataset
 
@@ -79,7 +79,9 @@ class CLIPTemplateFactory:
     }
 
     @staticmethod
-    def get_classnames_and_templates(dataset_name: str):
+    def get_classnames_and_templates(
+        dataset_name: str,
+    ) -> Tuple[List[str], List[Callable]]:
         """
         Retrieves class names and templates for the specified dataset.
 
@@ -169,7 +171,7 @@ class CLIPTemplateFactory:
         CLIPTemplateFactory._dataset_mapping[dataset_name] = dataset_info
 
     @staticmethod
-    def get_available_datasets():
+    def get_available_datasets() -> List[str]:
         """
         Get a list of all available dataset names.
 
@@ -179,5 +181,5 @@ class CLIPTemplateFactory:
         return list(CLIPTemplateFactory._dataset_mapping.keys())
 
 
-def get_classnames_and_templates(dataset_name: str):
+def get_classnames_and_templates(dataset_name: str) -> Tuple[List[str], List[Callable]]:
     return CLIPTemplateFactory.get_classnames_and_templates(dataset_name)

fusion_bench/utils/__init__.py

@@ -7,7 +7,12 @@ from .cache_utils import *
 from .devices import *
 from .dtype import parse_dtype
 from .fabric import seed_everything_by_time
-from .instantiate_utils import instantiate, is_instantiable
+from .instantiate_utils import (
+    instantiate,
+    is_instantiable,
+    set_print_function_call,
+    set_print_function_call_permeanent,
+)
 from .json import load_from_json, save_to_json
 from .lazy_state_dict import LazyStateDict
 from .misc import *

fusion_bench/utils/devices.py

@@ -1,4 +1,5 @@
 import gc
+import logging
 import os
 from typing import List, Optional, Union
 
@@ -12,7 +13,7 @@ from transformers.utils import (
 )
 
 __all__ = [
-    "cuda_empty_cache",
+    "clear_cuda_cache",
     "to_device",
     "num_devices",
     "get_device",
@@ -21,10 +22,19 @@ __all__ = [
     "get_device_capabilities",
 ]
 
+log = logging.getLogger(__name__)
 
-def cuda_empty_cache():
+
+def clear_cuda_cache():
+    """
+    Clears the CUDA memory cache to free up GPU memory.
+    Works only if CUDA is available.
+    """
     gc.collect()
-    torch.cuda.empty_cache()
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+    else:
+        log.warning("CUDA is not available. No cache to clear.")
 
 
 def to_device(obj, device: Optional[torch.device], **kwargs):
@@ -75,7 +85,7 @@ def num_devices(devices: Union[int, List[int], str]) -> int:
     Return the number of devices.
 
     Args:
-        devices: `devices` can be a single int to specify the number of devices, or a list of device ids, e.g. [0, 1, 2, 3]， or a str of device ids, e.g. "0,1,2,3" and "[0, 1, 2]".
+        devices: `devices` can be a single int to specify the number of devices, or a list of device ids, e.g. [0, 1, 2, 3], or a str of device ids, e.g. "0,1,2,3" and "[0, 1, 2]".
 
     Returns:
         The number of devices.

fusion_bench/utils/instantiate_utils.py

@@ -28,7 +28,7 @@ PRINT_FUNCTION_CALL_FUNC = print
 Function to be used for printing function calls.
 """
 
-CATCH_EXCEPTION = True
+CATCH_EXCEPTION = False
 
 
 @contextmanager
@@ -41,10 +41,12 @@ def set_print_function_call(value: bool):
     finally:
         PRINT_FUNCTION_CALL = old_value
 
+
 def set_print_function_call_permeanent(value: bool):
     global PRINT_FUNCTION_CALL
     PRINT_FUNCTION_CALL = value
 
+
 def is_instantiable(config: Union[DictConfig, Any]) -> bool:
     if OmegaConf.is_dict(config):
         return "_target_" in config

fusion_bench/utils/modelscope.py

@@ -2,27 +2,37 @@ import os
 from typing import Literal, Optional
 
 from datasets import load_dataset as datasets_load_dataset
+
 from fusion_bench.utils import validate_and_suggest_corrections
 
 try:
+    from modelscope import dataset_file_download as modelscope_dataset_file_download
+    from modelscope import model_file_download as modelscope_model_file_download
     from modelscope import snapshot_download as modelscope_snapshot_download
+
 except ImportError:
 
-    def modelscope_snapshot_download(*args, **kwargs):
+    def _raise_modelscope_not_installed_error(*args, **kwargs):
         raise ImportError(
             "ModelScope is not installed. Please install it using `pip install modelscope` to use ModelScope models."
         )
 
+    modelscope_snapshot_download = _raise_modelscope_not_installed_error
+    modelscope_model_file_download = _raise_modelscope_not_installed_error
+    modelscope_dataset_file_download = _raise_modelscope_not_installed_error
 
 try:
+    from huggingface_hub import hf_hub_download
     from huggingface_hub import snapshot_download as huggingface_snapshot_download
 except ImportError:
 
-    def huggingface_snapshot_download(*args, **kwargs):
+    def _raise_hugggingface_not_installed_error(*args, **kwargs):
         raise ImportError(
             "Hugging Face Hub is not installed. Please install it using `pip install huggingface_hub` to use Hugging Face models."
         )
 
+    huggingface_snapshot_download = _raise_hugggingface_not_installed_error
+    hf_hub_download = _raise_hugggingface_not_installed_error
 
 __all__ = [
     "load_dataset",
@@ -32,6 +42,12 @@ __all__ = [
 AVAILABLE_PLATFORMS = ["hf", "huggingface", "modelscope"]
 
 
+def _raise_unknown_platform_error(platform: str):
+    raise ValueError(
+        f"Unsupported platform: {platform}. Supported platforms are 'hf', 'huggingface', and 'modelscope'."
+    )
+
+
 def load_dataset(
     name: str,
     split: str = "train",
@@ -55,9 +71,7 @@ def load_dataset(
         dataset_dir = modelscope_snapshot_download(name, repo_type="dataset")
         return datasets_load_dataset(dataset_dir, split=split)
     else:
-        raise ValueError(
-            f"Unsupported platform: {platform}. Supported platforms are 'hf', 'huggingface', and 'modelscope'."
-        )
+        _raise_unknown_platform_error(platform)
 
 
 def resolve_repo_path(
@@ -138,9 +152,114 @@ def resolve_repo_path(
                 repo_id=repo_id, repo_type=repo_type, **kwargs
             )
         else:
-            raise ValueError(
-                f"Unsupported platform: {platform}. Supported platforms are 'hf', 'huggingface', and 'modelscope'."
-            )
+            _raise_unknown_platform_error(platform)
         return local_path
     except Exception as e:
         raise FileNotFoundError(f"Could not resolve checkpoint: {repo_id}. Error: {e}")
+
+
+def resolve_file_path(
+    repo_id: str,
+    filename: str,
+    repo_type: Literal["model", "dataset"] = "model",
+    platform: Literal["hf", "huggingface", "modelscope"] = "hf",
+    **kwargs,
+) -> str:
+    """
+    Resolve and download a specific file from a repository across multiple platforms.
+
+    This function downloads a specific file from repositories hosted on various platforms
+    including local paths, Hugging Face Hub, and ModelScope. It handles platform-specific
+    URL prefixes and automatically determines the appropriate download method.
+
+    Args:
+        repo_id (str): Repository identifier. Can be:
+            - Local directory path (file will be joined with this path if it exists)
+            - Hugging Face model/dataset ID (e.g., "bert-base-uncased")
+            - ModelScope model/dataset ID
+            - URL-prefixed ID (e.g., "hf://model-name", "modelscope://model-name").
+              The prefix will override the platform argument.
+        filename (str): The specific file to download from the repository.
+        repo_type (Literal["model", "dataset"], optional): Type of repository.
+            Defaults to "model". Used for ModelScope platform to determine the
+            correct download function.
+        platform (Literal["hf", "huggingface", "modelscope"], optional):
+            Platform to download from. Defaults to "hf". Options:
+            - "hf" or "huggingface": Hugging Face Hub
+            - "modelscope": ModelScope platform
+        **kwargs: Additional arguments passed to the underlying download functions
+            (e.g., cache_dir, force_download, use_auth_token).
+
+    Returns:
+        str: Local path to the downloaded file.
+
+    Raises:
+        ValueError: If an unsupported repo_type is specified for ModelScope platform.
+        ImportError: If required dependencies for the specified platform are not installed.
+        FileNotFoundError: If the file cannot be found or downloaded.
+
+    Examples:
+        >>> # Download config.json from a Hugging Face model
+        >>> resolve_file_path("bert-base-uncased", "config.json")
+        "/home/user/.cache/huggingface/hub/models--bert-base-uncased/.../config.json"
+
+        >>> # Download from ModelScope
+        >>> resolve_file_path(
+        ...     "damo/nlp_bert_backbone_base_std",
+        ...     "pytorch_model.bin",
+        ...     platform="modelscope"
+        ... )
+        "/home/user/.cache/modelscope/hub/.../pytorch_model.bin"
+
+        >>> # Local file path
+        >>> resolve_file_path("/path/to/local/model", "config.json")
+        "/path/to/local/model/config.json"
+
+        >>> # URL-prefixed repository
+        >>> resolve_file_path("hf://microsoft/DialoGPT-medium", "config.json")
+        "/home/user/.cache/huggingface/hub/.../config.json"
+
+        >>> # Download dataset file from ModelScope
+        >>> resolve_file_path(
+        ...     "DAMO_NLP/jd",
+        ...     "train.json",
+        ...     repo_type="dataset",
+        ...     platform="modelscope"
+        ... )
+        "/home/user/.cache/modelscope/datasets/.../train.json"
+    """
+    # If it's a HuggingFace Hub model id, download snapshot
+    if repo_id.startswith("hf://") or repo_id.startswith("huggingface://"):
+        repo_id = repo_id.replace("hf://", "").replace("huggingface://", "")
+        platform = "hf"
+    # If it's a ModelScope model id, download snapshot
+    elif repo_id.startswith("modelscope://"):
+        repo_id = repo_id.replace("modelscope://", "")
+        platform = "modelscope"
+
+    # If it's a local file or directory, return as is
+    if os.path.exists(repo_id):
+        return os.path.join(repo_id, filename)
+
+    if platform in ["hf", "huggingface"]:
+        return hf_hub_download(
+            repo_id=repo_id,
+            filename=filename,
+            repo_type=repo_type,
+            **kwargs,
+        )
+    elif platform == "modelscope":
+        if repo_type == "model":
+            return modelscope_model_file_download(
+                model_id=repo_id, file_path=filename, **kwargs
+            )
+        elif repo_type == "dataset":
+            return modelscope_dataset_file_download(
+                dataset_id=repo_id, file_path=filename, **kwargs
+            )
+        else:
+            raise ValueError(
+                f"Unsupported repo_type: {repo_type}. Supported types are 'model' and 'dataset'."
+            )
+    else:
+        _raise_unknown_platform_error(platform)

fusion_bench/utils/parameters.py

@@ -1,6 +1,6 @@
 import copy
 from collections import OrderedDict
-from typing import List, Mapping, Optional, Union
+from typing import Dict, List, Mapping, Optional, Union
 
 import torch
 from torch import nn
@@ -83,7 +83,7 @@ def vector_to_state_dict(
     vector: torch.Tensor,
     state_dict: Union[StateDictType, nn.Module],
     remove_keys: Optional[List[str]] = None,
-):
+) -> Dict[str, torch.Tensor]:
     """
     Convert a vector to a state dictionary.
 

fusion_bench/utils/rich_utils.py

@@ -189,6 +189,9 @@ if __name__ == "__main__":
 
 
 def setup_colorlogging(force=False, **config_kwargs):
+    """
+    Sets up color logging for the application.
+    """
     FORMAT = "%(message)s"
 
     logging.basicConfig(

fusion_bench/utils/state_dict_arithmetic.py

@@ -44,7 +44,7 @@ def state_dicts_check_keys(state_dicts: List[StateDictType]):
         assert keys == set(state_dict.keys()), "keys of state_dicts are not equal"
 
 
-def num_params_of_state_dict(state_dict: StateDictType):
+def num_params_of_state_dict(state_dict: StateDictType) -> int:
     """
     Returns the number of parameters in a state dict.
 
@@ -57,7 +57,7 @@ def num_params_of_state_dict(state_dict: StateDictType):
     return sum([state_dict[key].numel() for key in state_dict])
 
 
-def state_dict_flatten(state_dict: Dict[str, Tensor]):
+def state_dict_flatten(state_dict: Dict[str, Tensor]) -> Tensor:
     """
     Flattens a state dict.
 
@@ -73,7 +73,7 @@ def state_dict_flatten(state_dict: Dict[str, Tensor]):
     return torch.cat(flattened_state_dict)
 
 
-def state_dict_avg(state_dicts: List[StateDictType]):
+def state_dict_avg(state_dicts: List[StateDictType]) -> StateDictType:
     """
     Returns the average of a list of state dicts.
 
@@ -100,7 +100,7 @@ def state_dict_avg(state_dicts: List[StateDictType]):
 
 def state_dict_sub(
     a: StateDictType, b: StateDictType, strict: bool = True, device=None
-):
+) -> StateDictType:
     """
     Returns the difference between two state dicts `a-b`.
 
@@ -130,7 +130,7 @@ def state_dict_add(
     strict: bool = True,
     device=None,
     show_pbar: bool = False,
-):
+) -> StateDictType:
     """
     Returns the sum of two state dicts.
 
@@ -156,14 +156,14 @@ def state_dict_add(
     return ans
 
 
-def state_dict_add_scalar(a: StateDictType, scalar: Number):
+def state_dict_add_scalar(a: StateDictType, scalar: Number) -> StateDictType:
     ans = OrderedDict()
     for key in a:
         ans[key] = a[key] + scalar
     return ans
 
 
-def state_dict_mul(state_dict: StateDictType, scalar: float):
+def state_dict_mul(state_dict: StateDictType, scalar: float) -> StateDictType:
     """
     Returns the product of a state dict and a scalar.
 
@@ -180,7 +180,9 @@ def state_dict_mul(state_dict: StateDictType, scalar: float):
     return diff
 
 
-def state_dict_div(state_dict: StateDictType, scalar: float, show_pbar: bool = False):
+def state_dict_div(
+    state_dict: StateDictType, scalar: float, show_pbar: bool = False
+) -> StateDictType:
     """
     Returns the division of a state dict by a scalar.
 
@@ -197,16 +199,16 @@ def state_dict_div(state_dict: StateDictType, scalar: float, show_pbar: bool = F
     return diff
 
 
-def state_dict_power(state_dict: Dict[str, Tensor], p: float):
+def state_dict_power(state_dict: StateDictType, p: float) -> StateDictType:
     """
     Returns the power of a state dict.
 
     Args:
-        state_dict (Dict[str, Tensor]): The state dict to be powered.
+        state_dict (StateDictType): The state dict to be powered.
         p (float): The power to raise the state dict to.
 
     Returns:
-        Dict[str, Tensor]: The powered state dict.
+        StateDictType: The powered state dict.
     """
     powered_state_dict = {}
     for key in state_dict:
@@ -215,17 +217,17 @@ def state_dict_power(state_dict: Dict[str, Tensor], p: float):
 
 
 def state_dict_interpolation(
-    state_dicts: List[Dict[str, Tensor]], scalars: List[float]
-):
+    state_dicts: List[StateDictType], scalars: List[float]
+) -> StateDictType:
     """
     Interpolates between a list of state dicts using a list of scalars.
 
     Args:
-        state_dicts (List[Dict[str, Tensor]]): The list of state dicts to interpolate between.
+        state_dicts (List[StateDictType]): The list of state dicts to interpolate between.
         scalars (List[float]): The list of scalars to use for interpolation.
 
     Returns:
-        Dict: The interpolated state dict.
+        StateDictType: The interpolated state dict.
     """
     assert len(state_dicts) == len(
         scalars
@@ -243,15 +245,15 @@ def state_dict_interpolation(
     return interpolated_state_dict
 
 
-def state_dict_sum(state_dicts: List[StateDictType]):
+def state_dict_sum(state_dicts: List[StateDictType]) -> StateDictType:
     """
     Returns the sum of a list of state dicts.
 
     Args:
-        state_dicts (List[Dict[str, Tensor]]): The list of state dicts to sum.
+        state_dicts (List[StateDictType]): The list of state dicts to sum.
 
     Returns:
-        Dict: The sum of the state dicts.
+        StateDictType: The sum of the state dicts.
     """
     assert len(state_dicts) > 0, "The number of state_dicts must be greater than 0"
     assert all(
@@ -267,17 +269,17 @@ def state_dict_sum(state_dicts: List[StateDictType]):
 
 
 def state_dict_weighted_sum(
-    state_dicts: List[Dict[str, Tensor]], weights: List[float], device=None
-):
+    state_dicts: List[StateDictType], weights: List[float], device=None
+) -> StateDictType:
     """
     Returns the weighted sum of a list of state dicts.
 
     Args:
-        state_dicts (List[Dict[str, Tensor]]): The list of state dicts to interpolate between.
+        state_dicts (List[StateDictType]): The list of state dicts to interpolate between.
         weights (List[float]): The list of weights to use for the weighted sum.
 
     Returns:
-        Dict: The weighted sum of the state dicts.
+        StateDictType: The weighted sum of the state dicts.
     """
     assert len(state_dicts) == len(
         weights
@@ -302,7 +304,7 @@ def state_dict_weighted_sum(
     return weighted_sum_state_dict
 
 
-def state_dict_diff_abs(a: StateDictType, b: StateDictType):
+def state_dict_diff_abs(a: StateDictType, b: StateDictType) -> StateDictType:
     """
     Returns the per-layer abs of the difference between two state dicts.
 

{fusion_bench-0.2.20.dist-info → fusion_bench-0.2.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusion_bench
-Version: 0.2.20
+Version: 0.2.21
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 License: MIT License
@@ -45,13 +45,17 @@ Requires-Dist: rich
 Requires-Dist: scipy
 Requires-Dist: h5py
 Requires-Dist: pytest
+Requires-Dist: transformers!=4.49
+Requires-Dist: pillow!=11.2.1
 Provides-Extra: lm-eval-harness
 Requires-Dist: lm-eval; extra == "lm-eval-harness"
+Requires-Dist: immutabledict; extra == "lm-eval-harness"
+Requires-Dist: langdetect; extra == "lm-eval-harness"
 Dynamic: license-file
 
 <div align='center'>
 
-# FusionBench: A Comprehensive Benchmark/ToolKit of Deep Model Fusion
+# FusionBench: A Comprehensive Benchmark/Toolkit of Deep Model Fusion
 
 [![arXiv](https://img.shields.io/badge/arXiv-2406.03280-b31b1b.svg)](http://arxiv.org/abs/2406.03280)
 [![GitHub License](https://img.shields.io/github/license/tanganke/fusion_bench)](https://github.com/tanganke/fusion_bench/blob/main/LICENSE)
@@ -75,7 +79,7 @@ Projects based on FusionBench and news from the community (descending order of d
 <details>
   <summary>The-Hai Nguyen, Dang Huu-Tien, Takeshi Suzuki, and Le-Minh Nguyen. RegMean++: Enhancing Effectiveness and Generalization of Regression Mean for Model Merging. Aug, 2025. https://www.arxiv.org/abs/2508.03121</summary>
 
-Regression Mean (RegMean), an approach that formulates model merging as a linear regression problem, aims to find the optimal weights for each linear layer in the merge model by minimizing the discrepancy in predictions between the merge and candidate models. RegMean provides a precise closed-form solution for the merging problem; therefore, it offers explainability and computational efficiency. However, RegMean merges each linear layer independently, overlooking how the features and information in the earlier layers propagate through the layers and influence the final prediction in the merge model. In this paper, we introduce RegMean++, a simple yet effective alternative to RegMean, that explicitly incorporates both intra- and cross-layer dependencies between merge models' layers into RegMean's objective. By accounting for these dependencies, RegMean++ better captures the behaviors of the merge model. Extensive experiments demonstrate that RegMean++ consistently outperforms RegMean across diverse settings, including in-domain (ID) and out-of-domain (OOD) generalization, sequential merging, large-scale tasks, and robustness under several types of distribution shifts. Furthermore, RegMean++ achieves competitive or state-of-the-art performance compared to various recent advanced model merging methods. 
+Regression Mean (RegMean), an approach that formulates model merging as a linear regression problem, aims to find the optimal weights for each linear layer in the merge model by minimizing the discrepancy in predictions between the merge and candidate models. RegMean provides a precise closed-form solution for the merging problem; therefore, it offers explainability and computational efficiency. However, RegMean merges each linear layer independently, overlooking how the features and information in the earlier layers propagate through the layers and influence the final prediction in the merge model. In this paper, we introduce RegMean++, a simple yet effective alternative to RegMean, that explicitly incorporates both intra- and cross-layer dependencies between merge models' layers into RegMean's objective. By accounting for these dependencies, RegMean++ better captures the behaviors of the merge model. Extensive experiments demonstrate that RegMean++ consistently outperforms RegMean across diverse settings, including in-domain (ID) and out-of-domain (OOD) generalization, sequential merging, large-scale tasks, and robustness under several types of distribution shifts. Furthermore, RegMean++ achieves competitive or state-of-the-art performance compared to various recent advanced model merging methods.
 
   <img width="1000" alt="image" src="docs/algorithms/images/regmean_vs_regmean_plusplus.png">
 </details>
@@ -89,7 +93,7 @@ Model merging has emerged as a promising approach for multi-task learning (MTL),
 <details>
   <summary>Daniel Marczak, et al. No Task Left Behind: Isotropic Model Merging with Common and Task-Specific Subspaces. Feb 2025. https://arxiv.org/abs/2502.04959</summary>
 
-  Model merging integrates the weights of multiple task-specific models into a single multi-task model. Despite recent interest in the problem, a significant performance gap between the combined and single-task models remains. In this paper, we investigate the key characteristics of task matrices -- weight update matrices applied to a pre-trained model -- that enable effective merging. We show that alignment between singular components of task-specific and merged matrices strongly correlates with performance improvement over the pre-trained model. Based on this, we propose an isotropic merging framework that flattens the singular value spectrum of task matrices, enhances alignment, and reduces the performance gap. Additionally, we incorporate both common and task-specific subspaces to further improve alignment and performance. Our proposed approach achieves state-of-the-art performance across multiple scenarios, including various sets of tasks and model scales. This work advances the understanding of model merging dynamics, offering an effective methodology to merge models without requiring additional training. 
+  Model merging integrates the weights of multiple task-specific models into a single multi-task model. Despite recent interest in the problem, a significant performance gap between the combined and single-task models remains. In this paper, we investigate the key characteristics of task matrices -- weight update matrices applied to a pre-trained model -- that enable effective merging. We show that alignment between singular components of task-specific and merged matrices strongly correlates with performance improvement over the pre-trained model. Based on this, we propose an isotropic merging framework that flattens the singular value spectrum of task matrices, enhances alignment, and reduces the performance gap. Additionally, we incorporate both common and task-specific subspaces to further improve alignment and performance. Our proposed approach achieves state-of-the-art performance across multiple scenarios, including various sets of tasks and model scales. This work advances the understanding of model merging dynamics, offering an effective methodology to merge models without requiring additional training.
 </details>
 
 <details>
@@ -107,12 +111,12 @@ Merging multiple expert models offers a promising approach for performing multi-
 <details>
   <summary>Hongling Zheng, Li Shen, Anke Tang, Yong Luo et al. Learn From Model Beyond Fine-Tuning: A Survey. Nature Machine Intelligence. Jan, 2025. https://www.nature.com/articles/s42256-024-00961-0</summary>
 
-  > Foundation models (FM) have demonstrated remarkable performance across a wide range of tasks (especially in the fields of natural language processing and computer vision), primarily attributed to their ability to comprehend instructions and access extensive, high-quality data. This not only showcases their current effectiveness but also sets a promising trajectory towards the development of artificial general intelligence. Unfortunately, due to multiple constraints, the raw data of the model used for large model training are often inaccessible, so the use of end-to-end models for downstream tasks has become a new research trend, which we call Learn From Model (LFM) in this article. LFM focuses on the research, modification, and design of FM based on the model interface, so as to better understand the model structure and weights (in a black box environment), and to generalize the model to downstream tasks. The study of LFM techniques can be broadly categorized into five major areas: model tuning, model distillation, model reuse, meta learning and model editing. Each category encompasses a repertoire of methods and strategies that aim to enhance the capabilities and performance of FM. This paper gives a comprehensive review of the current methods based on FM from the perspective of LFM, in order to help readers better understand the current research status and ideas. To conclude, we summarize the survey by highlighting several critical areas for future exploration and addressing open issues that require further attention from the research community. The relevant papers we investigated in this article can be accessed at https://github.com/ruthless-man/Awesome-Learn-from-Model.
+  > Foundation models (FM) have demonstrated remarkable performance across a wide range of tasks (especially in the fields of natural language processing and computer vision), primarily attributed to their ability to comprehend instructions and access extensive, high-quality data. This not only showcases their current effectiveness but also sets a promising trajectory towards the development of artificial general intelligence. Unfortunately, due to multiple constraints, the raw data of the model used for large model training are often inaccessible, so the use of end-to-end models for downstream tasks has become a new research trend, which we call Learn From Model (LFM) in this article. LFM focuses on the research, modification, and design of FM based on the model interface, so as to better understand the model structure and weights (in a black box environment), and to generalize the model to downstream tasks. The study of LFM techniques can be broadly categorized into five major areas: model tuning, model distillation, model reuse, meta learning and model editing. Each category encompasses a repertoire of methods and strategies that aim to enhance the capabilities and performance of FM. This paper gives a comprehensive review of the current methods based on FM from the perspective of LFM, in order to help readers better understand the current research status and ideas. To conclude, we summarize the survey by highlighting several critical areas for future exploration and addressing open issues that require further attention from the research community. The relevant papers we investigated in this article can be accessed at <https://github.com/ruthless-man/Awesome-Learn-from-Model>.
 </details>
 
 <details>
   <summary>Li Shen, Anke Tang, Enneng Yang et al. Efficient and Effective Weight-Ensembling Mixture of Experts for Multi-Task Model Merging. Oct, 2024. https://github.com/EnnengYang/Efficient-WEMoE</summary>
- 
+
   <img width="1018" alt="image" src="https://github.com/user-attachments/assets/b7e1279e-87fc-4016-8867-1bff7700e271">
 
 </details>
@@ -138,7 +142,7 @@ Install from PyPI:
 pip install fusion-bench
 ```
 
-or install the latest version in development from github repository
+or install the latest version in development from the GitHub repository
 
 ```bash
 git clone https://github.com/tanganke/fusion_bench.git
@@ -155,7 +159,6 @@ pip install -e . # install the package in editable mode
 
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/10.5281/zenodo.10256836)
 
-
 ```bash
 pip install "fusion-bench[lm-eval-harness]"
 ```
@@ -205,8 +208,8 @@ The project is structured as follows:
 
 ## A Unified Command Line Interface
 
-The `fusion_bench` command-line interface is a powerful tool for researchers and practitioners in the field of model fusion. It provides a streamlined way to experiment with various fusion algorithms, model combinations, and evaluation tasks. 
-By leveraging Hydra's configuration management, fusion_bench offers flexibility in setting up experiments and reproducibility in results. 
+The `fusion_bench` command-line interface is a powerful tool for researchers and practitioners in the field of model fusion. It provides a streamlined way to experiment with various fusion algorithms, model combinations, and evaluation tasks.
+By leveraging Hydra's configuration management, fusion_bench offers flexibility in setting up experiments and reproducibility in results.
 The CLI's design allows for easy extension to new fusion methods, model types, and tasks, making it a versatile platform for advancing research in model fusion techniques.
 
 Read the [CLI documentation](https://tanganke.github.io/fusion_bench/cli/fusion_bench/) for more information.
@@ -245,7 +248,7 @@ class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
         )
 ```
 
-A corresponding configuration file should be created to specify the class and hyperparameters of the algorithm. 
+A corresponding configuration file should be created to specify the class and hyperparameters of the algorithm.
 Here we assume the configuration file is placed at `config/method/your_algorithm_config.yaml`.
 
 > [!NOTE]
@@ -280,7 +283,7 @@ Click on [<kbd>Use this template</kbd>](https://github.com/fusion-bench/fusion-b
 
 ### FusionBench Command Generator WebUI (for v0.1.x)
 
-FusionBench Command Generator is a user-friendly web interface for generating FusionBench commands based on configuration files. 
+FusionBench Command Generator is a user-friendly web interface for generating FusionBench commands based on configuration files.
 It provides an interactive way to select and customize FusionBench configurations, making it easier to run experiments with different settings.
 [Read more here](https://tanganke.github.io/fusion_bench/cli/fusion_bench_webui/).
 
@@ -291,18 +294,14 @@ It provides an interactive way to select and customize FusionBench configuration
 If you find this benchmark useful, please consider citing our work:
 
 ```bibtex
-@misc{tangFusionBenchComprehensiveBenchmark2024,
-  title = {{{FusionBench}}: {{A Comprehensive Benchmark}} of {{Deep Model Fusion}}},
-  shorttitle = {{{FusionBench}}},
-  author = {Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
-  year = {2024},
-  month = jun,
-  number = {arXiv:2406.03280},
-  eprint = {2406.03280},
-  publisher = {arXiv},
-  url = {http://arxiv.org/abs/2406.03280},
-  archiveprefix = {arxiv},
-  langid = {english},
-  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning}
+@article{tang2024fusionbench,
+  title={Fusionbench: A comprehensive benchmark of deep model fusion},
+  author={Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
+  journal={arXiv preprint arXiv:2406.03280},
+  year={2024}
 }
 ```
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=tanganke/fusion_bench&type=Date)](https://www.star-history.com/#tanganke/fusion_bench&Date)