fusion-bench 0.2.19__py3-none-any.whl → 0.2.21__py3-none-any.whl

fusion_bench/utils/state_dict_arithmetic.py

@@ -4,6 +4,7 @@ from typing import Callable, Dict, List, Literal, Union, cast
 
 import torch
 from torch import Tensor
+from tqdm.auto import tqdm
 
 from .parameters import check_parameters_all_equal
 from .type import BoolStateDictType, StateDictType
@@ -43,7 +44,7 @@ def state_dicts_check_keys(state_dicts: List[StateDictType]):
         assert keys == set(state_dict.keys()), "keys of state_dicts are not equal"
 
 
-def num_params_of_state_dict(state_dict: StateDictType):
+def num_params_of_state_dict(state_dict: StateDictType) -> int:
     """
     Returns the number of parameters in a state dict.
 
@@ -56,7 +57,7 @@ def num_params_of_state_dict(state_dict: StateDictType):
     return sum([state_dict[key].numel() for key in state_dict])
 
 
-def state_dict_flatten(state_dict: Dict[str, Tensor]):
+def state_dict_flatten(state_dict: Dict[str, Tensor]) -> Tensor:
     """
     Flattens a state dict.
 
@@ -72,7 +73,7 @@ def state_dict_flatten(state_dict: Dict[str, Tensor]):
     return torch.cat(flattened_state_dict)
 
 
-def state_dict_avg(state_dicts: List[StateDictType]):
+def state_dict_avg(state_dicts: List[StateDictType]) -> StateDictType:
     """
     Returns the average of a list of state dicts.
 
@@ -99,7 +100,7 @@ def state_dict_avg(state_dicts: List[StateDictType]):
 
 def state_dict_sub(
     a: StateDictType, b: StateDictType, strict: bool = True, device=None
-):
+) -> StateDictType:
     """
     Returns the difference between two state dicts `a-b`.
 
@@ -124,8 +125,12 @@ def state_dict_sub(
 
 
 def state_dict_add(
-    a: StateDictType, b: StateDictType, strict: bool = True, device=None
-):
+    a: StateDictType,
+    b: StateDictType,
+    strict: bool = True,
+    device=None,
+    show_pbar: bool = False,
+) -> StateDictType:
     """
     Returns the sum of two state dicts.
 
@@ -140,10 +145,10 @@ def state_dict_add(
     ans = {}
     if strict:
         check_parameters_all_equal([a, b])
-        for key in a:
+        for key in tqdm(tuple(a.keys())) if show_pbar else a:
             ans[key] = a[key] + b[key]
     else:
-        for key in a:
+        for key in tqdm(tuple(a.keys())) if show_pbar else a:
             if key in b:
                 ans[key] = a[key] + b[key]
     if device is not None:
@@ -151,14 +156,14 @@ def state_dict_add(
     return ans
 
 
-def state_dict_add_scalar(a: StateDictType, scalar: Number):
+def state_dict_add_scalar(a: StateDictType, scalar: Number) -> StateDictType:
     ans = OrderedDict()
     for key in a:
         ans[key] = a[key] + scalar
     return ans
 
 
-def state_dict_mul(state_dict: StateDictType, scalar: float):
+def state_dict_mul(state_dict: StateDictType, scalar: float) -> StateDictType:
     """
     Returns the product of a state dict and a scalar.
 
@@ -175,7 +180,9 @@ def state_dict_mul(state_dict: StateDictType, scalar: float):
     return diff
 
 
-def state_dict_div(state_dict: StateDictType, scalar: float):
+def state_dict_div(
+    state_dict: StateDictType, scalar: float, show_pbar: bool = False
+) -> StateDictType:
     """
     Returns the division of a state dict by a scalar.
 
@@ -187,21 +194,21 @@ def state_dict_div(state_dict: StateDictType, scalar: float):
         Dict: The division of the state dict by the scalar.
     """
     diff = OrderedDict()
-    for k in state_dict:
+    for k in tqdm(tuple(state_dict.keys())) if show_pbar else state_dict:
         diff[k] = state_dict[k] / scalar
     return diff
 
 
-def state_dict_power(state_dict: Dict[str, Tensor], p: float):
+def state_dict_power(state_dict: StateDictType, p: float) -> StateDictType:
     """
     Returns the power of a state dict.
 
     Args:
-        state_dict (Dict[str, Tensor]): The state dict to be powered.
+        state_dict (StateDictType): The state dict to be powered.
         p (float): The power to raise the state dict to.
 
     Returns:
-        Dict[str, Tensor]: The powered state dict.
+        StateDictType: The powered state dict.
     """
     powered_state_dict = {}
     for key in state_dict:
@@ -210,17 +217,17 @@ def state_dict_power(state_dict: Dict[str, Tensor], p: float):
 
 
 def state_dict_interpolation(
-    state_dicts: List[Dict[str, Tensor]], scalars: List[float]
-):
+    state_dicts: List[StateDictType], scalars: List[float]
+) -> StateDictType:
     """
     Interpolates between a list of state dicts using a list of scalars.
 
     Args:
-        state_dicts (List[Dict[str, Tensor]]): The list of state dicts to interpolate between.
+        state_dicts (List[StateDictType]): The list of state dicts to interpolate between.
         scalars (List[float]): The list of scalars to use for interpolation.
 
     Returns:
-        Dict: The interpolated state dict.
+        StateDictType: The interpolated state dict.
     """
     assert len(state_dicts) == len(
         scalars
@@ -238,15 +245,15 @@ def state_dict_interpolation(
     return interpolated_state_dict
 
 
-def state_dict_sum(state_dicts: List[StateDictType]):
+def state_dict_sum(state_dicts: List[StateDictType]) -> StateDictType:
     """
     Returns the sum of a list of state dicts.
 
     Args:
-        state_dicts (List[Dict[str, Tensor]]): The list of state dicts to sum.
+        state_dicts (List[StateDictType]): The list of state dicts to sum.
 
     Returns:
-        Dict: The sum of the state dicts.
+        StateDictType: The sum of the state dicts.
     """
     assert len(state_dicts) > 0, "The number of state_dicts must be greater than 0"
     assert all(
@@ -262,17 +269,17 @@ def state_dict_sum(state_dicts: List[StateDictType]):
 
 
 def state_dict_weighted_sum(
-    state_dicts: List[Dict[str, Tensor]], weights: List[float], device=None
-):
+    state_dicts: List[StateDictType], weights: List[float], device=None
+) -> StateDictType:
     """
     Returns the weighted sum of a list of state dicts.
 
     Args:
-        state_dicts (List[Dict[str, Tensor]]): The list of state dicts to interpolate between.
+        state_dicts (List[StateDictType]): The list of state dicts to interpolate between.
         weights (List[float]): The list of weights to use for the weighted sum.
 
     Returns:
-        Dict: The weighted sum of the state dicts.
+        StateDictType: The weighted sum of the state dicts.
     """
     assert len(state_dicts) == len(
         weights
@@ -297,7 +304,7 @@ def state_dict_weighted_sum(
     return weighted_sum_state_dict
 
 
-def state_dict_diff_abs(a: StateDictType, b: StateDictType):
+def state_dict_diff_abs(a: StateDictType, b: StateDictType) -> StateDictType:
     """
     Returns the per-layer abs of the difference between two state dicts.
 

{fusion_bench-0.2.19.dist-info → fusion_bench-0.2.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusion_bench
-Version: 0.2.19
+Version: 0.2.21
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 License: MIT License
@@ -45,13 +45,17 @@ Requires-Dist: rich
 Requires-Dist: scipy
 Requires-Dist: h5py
 Requires-Dist: pytest
+Requires-Dist: transformers!=4.49
+Requires-Dist: pillow!=11.2.1
 Provides-Extra: lm-eval-harness
 Requires-Dist: lm-eval; extra == "lm-eval-harness"
+Requires-Dist: immutabledict; extra == "lm-eval-harness"
+Requires-Dist: langdetect; extra == "lm-eval-harness"
 Dynamic: license-file
 
 <div align='center'>
 
-# FusionBench: A Comprehensive Benchmark/ToolKit of Deep Model Fusion
+# FusionBench: A Comprehensive Benchmark/Toolkit of Deep Model Fusion
 
 [![arXiv](https://img.shields.io/badge/arXiv-2406.03280-b31b1b.svg)](http://arxiv.org/abs/2406.03280)
 [![GitHub License](https://img.shields.io/github/license/tanganke/fusion_bench)](https://github.com/tanganke/fusion_bench/blob/main/LICENSE)
@@ -72,6 +76,14 @@ FusionBench is a benchmark suite designed to evaluate the performance of various
 
 Projects based on FusionBench and news from the community (descending order of date. If you have any work based on FusionBench, please feel free to let us know, we are willing to add it to the list. :partying_face:):
 
+<details>
+  <summary>The-Hai Nguyen, Dang Huu-Tien, Takeshi Suzuki, and Le-Minh Nguyen. RegMean++: Enhancing Effectiveness and Generalization of Regression Mean for Model Merging. Aug, 2025. https://www.arxiv.org/abs/2508.03121</summary>
+
+Regression Mean (RegMean), an approach that formulates model merging as a linear regression problem, aims to find the optimal weights for each linear layer in the merge model by minimizing the discrepancy in predictions between the merge and candidate models. RegMean provides a precise closed-form solution for the merging problem; therefore, it offers explainability and computational efficiency. However, RegMean merges each linear layer independently, overlooking how the features and information in the earlier layers propagate through the layers and influence the final prediction in the merge model. In this paper, we introduce RegMean++, a simple yet effective alternative to RegMean, that explicitly incorporates both intra- and cross-layer dependencies between merge models' layers into RegMean's objective. By accounting for these dependencies, RegMean++ better captures the behaviors of the merge model. Extensive experiments demonstrate that RegMean++ consistently outperforms RegMean across diverse settings, including in-domain (ID) and out-of-domain (OOD) generalization, sequential merging, large-scale tasks, and robustness under several types of distribution shifts. Furthermore, RegMean++ achieves competitive or state-of-the-art performance compared to various recent advanced model merging methods.
+
+  <img width="1000" alt="image" src="docs/algorithms/images/regmean_vs_regmean_plusplus.png">
+</details>
+
 <details>
   <summary>Hao Mark Chen, et al. FW-Merging: Scaling Model Merging with Frank-Wolfe Optimization. Mar 2025. https://arxiv.org/abs/2503.12649</summary>
 
@@ -81,7 +93,7 @@ Model merging has emerged as a promising approach for multi-task learning (MTL),
 <details>
   <summary>Daniel Marczak, et al. No Task Left Behind: Isotropic Model Merging with Common and Task-Specific Subspaces. Feb 2025. https://arxiv.org/abs/2502.04959</summary>
 
-  Model merging integrates the weights of multiple task-specific models into a single multi-task model. Despite recent interest in the problem, a significant performance gap between the combined and single-task models remains. In this paper, we investigate the key characteristics of task matrices -- weight update matrices applied to a pre-trained model -- that enable effective merging. We show that alignment between singular components of task-specific and merged matrices strongly correlates with performance improvement over the pre-trained model. Based on this, we propose an isotropic merging framework that flattens the singular value spectrum of task matrices, enhances alignment, and reduces the performance gap. Additionally, we incorporate both common and task-specific subspaces to further improve alignment and performance. Our proposed approach achieves state-of-the-art performance across multiple scenarios, including various sets of tasks and model scales. This work advances the understanding of model merging dynamics, offering an effective methodology to merge models without requiring additional training. 
+  Model merging integrates the weights of multiple task-specific models into a single multi-task model. Despite recent interest in the problem, a significant performance gap between the combined and single-task models remains. In this paper, we investigate the key characteristics of task matrices -- weight update matrices applied to a pre-trained model -- that enable effective merging. We show that alignment between singular components of task-specific and merged matrices strongly correlates with performance improvement over the pre-trained model. Based on this, we propose an isotropic merging framework that flattens the singular value spectrum of task matrices, enhances alignment, and reduces the performance gap. Additionally, we incorporate both common and task-specific subspaces to further improve alignment and performance. Our proposed approach achieves state-of-the-art performance across multiple scenarios, including various sets of tasks and model scales. This work advances the understanding of model merging dynamics, offering an effective methodology to merge models without requiring additional training.
 </details>
 
 <details>
@@ -99,12 +111,12 @@ Merging multiple expert models offers a promising approach for performing multi-
 <details>
   <summary>Hongling Zheng, Li Shen, Anke Tang, Yong Luo et al. Learn From Model Beyond Fine-Tuning: A Survey. Nature Machine Intelligence. Jan, 2025. https://www.nature.com/articles/s42256-024-00961-0</summary>
 
-  > Foundation models (FM) have demonstrated remarkable performance across a wide range of tasks (especially in the fields of natural language processing and computer vision), primarily attributed to their ability to comprehend instructions and access extensive, high-quality data. This not only showcases their current effectiveness but also sets a promising trajectory towards the development of artificial general intelligence. Unfortunately, due to multiple constraints, the raw data of the model used for large model training are often inaccessible, so the use of end-to-end models for downstream tasks has become a new research trend, which we call Learn From Model (LFM) in this article. LFM focuses on the research, modification, and design of FM based on the model interface, so as to better understand the model structure and weights (in a black box environment), and to generalize the model to downstream tasks. The study of LFM techniques can be broadly categorized into five major areas: model tuning, model distillation, model reuse, meta learning and model editing. Each category encompasses a repertoire of methods and strategies that aim to enhance the capabilities and performance of FM. This paper gives a comprehensive review of the current methods based on FM from the perspective of LFM, in order to help readers better understand the current research status and ideas. To conclude, we summarize the survey by highlighting several critical areas for future exploration and addressing open issues that require further attention from the research community. The relevant papers we investigated in this article can be accessed at https://github.com/ruthless-man/Awesome-Learn-from-Model.
+  > Foundation models (FM) have demonstrated remarkable performance across a wide range of tasks (especially in the fields of natural language processing and computer vision), primarily attributed to their ability to comprehend instructions and access extensive, high-quality data. This not only showcases their current effectiveness but also sets a promising trajectory towards the development of artificial general intelligence. Unfortunately, due to multiple constraints, the raw data of the model used for large model training are often inaccessible, so the use of end-to-end models for downstream tasks has become a new research trend, which we call Learn From Model (LFM) in this article. LFM focuses on the research, modification, and design of FM based on the model interface, so as to better understand the model structure and weights (in a black box environment), and to generalize the model to downstream tasks. The study of LFM techniques can be broadly categorized into five major areas: model tuning, model distillation, model reuse, meta learning and model editing. Each category encompasses a repertoire of methods and strategies that aim to enhance the capabilities and performance of FM. This paper gives a comprehensive review of the current methods based on FM from the perspective of LFM, in order to help readers better understand the current research status and ideas. To conclude, we summarize the survey by highlighting several critical areas for future exploration and addressing open issues that require further attention from the research community. The relevant papers we investigated in this article can be accessed at <https://github.com/ruthless-man/Awesome-Learn-from-Model>.
 </details>
 
 <details>
   <summary>Li Shen, Anke Tang, Enneng Yang et al. Efficient and Effective Weight-Ensembling Mixture of Experts for Multi-Task Model Merging. Oct, 2024. https://github.com/EnnengYang/Efficient-WEMoE</summary>
- 
+
   <img width="1018" alt="image" src="https://github.com/user-attachments/assets/b7e1279e-87fc-4016-8867-1bff7700e271">
 
 </details>
@@ -130,7 +142,7 @@ Install from PyPI:
 pip install fusion-bench
 ```
 
-or install the latest version in development from github repository
+or install the latest version in development from the GitHub repository
 
 ```bash
 git clone https://github.com/tanganke/fusion_bench.git
@@ -147,7 +159,6 @@ pip install -e . # install the package in editable mode
 
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/10.5281/zenodo.10256836)
 
-
 ```bash
 pip install "fusion-bench[lm-eval-harness]"
 ```
@@ -197,8 +208,8 @@ The project is structured as follows:
 
 ## A Unified Command Line Interface
 
-The `fusion_bench` command-line interface is a powerful tool for researchers and practitioners in the field of model fusion. It provides a streamlined way to experiment with various fusion algorithms, model combinations, and evaluation tasks. 
-By leveraging Hydra's configuration management, fusion_bench offers flexibility in setting up experiments and reproducibility in results. 
+The `fusion_bench` command-line interface is a powerful tool for researchers and practitioners in the field of model fusion. It provides a streamlined way to experiment with various fusion algorithms, model combinations, and evaluation tasks.
+By leveraging Hydra's configuration management, fusion_bench offers flexibility in setting up experiments and reproducibility in results.
 The CLI's design allows for easy extension to new fusion methods, model types, and tasks, making it a versatile platform for advancing research in model fusion techniques.
 
 Read the [CLI documentation](https://tanganke.github.io/fusion_bench/cli/fusion_bench/) for more information.
@@ -237,7 +248,7 @@ class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
         )
 ```
 
-A corresponding configuration file should be created to specify the class and hyperparameters of the algorithm. 
+A corresponding configuration file should be created to specify the class and hyperparameters of the algorithm.
 Here we assume the configuration file is placed at `config/method/your_algorithm_config.yaml`.
 
 > [!NOTE]
@@ -272,7 +283,7 @@ Click on [<kbd>Use this template</kbd>](https://github.com/fusion-bench/fusion-b
 
 ### FusionBench Command Generator WebUI (for v0.1.x)
 
-FusionBench Command Generator is a user-friendly web interface for generating FusionBench commands based on configuration files. 
+FusionBench Command Generator is a user-friendly web interface for generating FusionBench commands based on configuration files.
 It provides an interactive way to select and customize FusionBench configurations, making it easier to run experiments with different settings.
 [Read more here](https://tanganke.github.io/fusion_bench/cli/fusion_bench_webui/).
 
@@ -283,18 +294,14 @@ It provides an interactive way to select and customize FusionBench configuration
 If you find this benchmark useful, please consider citing our work:
 
 ```bibtex
-@misc{tangFusionBenchComprehensiveBenchmark2024,
-  title = {{{FusionBench}}: {{A Comprehensive Benchmark}} of {{Deep Model Fusion}}},
-  shorttitle = {{{FusionBench}}},
-  author = {Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
-  year = {2024},
-  month = jun,
-  number = {arXiv:2406.03280},
-  eprint = {2406.03280},
-  publisher = {arXiv},
-  url = {http://arxiv.org/abs/2406.03280},
-  archiveprefix = {arxiv},
-  langid = {english},
-  keywords = {Computer Science - Artificial Intelligence,Computer Science - Computation and Language,Computer Science - Machine Learning}
+@article{tang2024fusionbench,
+  title={Fusionbench: A comprehensive benchmark of deep model fusion},
+  author={Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
+  journal={arXiv preprint arXiv:2406.03280},
+  year={2024}
 }
 ```
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=tanganke/fusion_bench&type=Date)](https://www.star-history.com/#tanganke/fusion_bench&Date)