fusion-bench 0.2.13__py3-none-any.whl → 0.2.15__py3-none-any.whl

fusion_bench/models/smile_moe/utils/__init__.py

@@ -0,0 +1,24 @@
+from typing import List
+
+import torch
+from torch import Tensor
+
+from .svd_utils import svd
+
+__all__ = ["svd_utils", "_is_all_zeros"]
+
+
+def _is_all_zeros(tensor: Tensor | List[Tensor]) -> bool:
+    """
+    Check if a tensor or a list of tensors are all zeros.
+
+    Args:
+        tensor (Tensor | List[Tensor]): A tensor or a list of tensors.
+
+    Returns:
+        bool: True if all elements are zeros, False otherwise.
+    """
+    if isinstance(tensor, Tensor):
+        return torch.allclose(tensor, torch.zeros_like(tensor))
+    else:
+        return all(_is_all_zeros(t) for t in tensor)

fusion_bench/models/smile_moe/utils/svd_utils.py

@@ -0,0 +1,46 @@
+from typing import Optional, Tuple, Union
+
+import torch
+from torch import Tensor
+
+
+def _svd(w: Tensor, full_matrices: bool = True) -> Tuple[Tensor, Tensor, Tensor]:
+    """
+    Perform Singular Value Decomposition (SVD) on a tensor.
+
+    Args:
+        w (Tensor): The input tensor.
+        full_matrices (bool): Whether to compute the full-sized U and V matrices.
+
+    Returns:
+        Tuple[Tensor, Tensor, Tensor]: The U, S, and V matrices from SVD.
+    """
+    u, s, vh = torch.linalg.svd(
+        w, full_matrices=full_matrices, driver="gesvd" if w.is_cuda else None
+    )
+    v = vh.T
+    return u, s, v
+
+
+def svd(
+    w: Tensor,
+    full_matrices: bool = True,
+    accelerator: Optional[Union[torch.device, str]] = None,
+) -> Tuple[Tensor, Tensor, Tensor]:
+    """
+    Perform SVD on a tensor, optionally using a specified accelerator.
+
+    Args:
+        w (Tensor): The input tensor.
+        full_matrices (bool): Whether to compute the full-sized U and V matrices.
+        accelerator (Optional[Union[torch.device, str]]): The device to perform the computation on.
+
+    Returns:
+        Tuple[Tensor, Tensor, Tensor]: The U, S, and V matrices from SVD.
+    """
+    if accelerator is None:
+        return _svd(w, full_matrices=full_matrices)
+    original_device = w.device
+    w = w.to(accelerator)
+    u, s, v = _svd(w)
+    return u.to(original_device), s.to(original_device), v.to(original_device)

fusion_bench/taskpool/__init__.py

@@ -15,6 +15,7 @@ _import_structure = {
     "dummy": ["DummyTaskPool"],
     "gpt2_text_classification": ["GPT2TextClassificationTaskPool"],
     "llama": ["LlamaTestGenerationTaskPool"],
+    "lm_eval_harness": ["LMEvalHarnessTaskPool"],
     "nyuv2_taskpool": ["NYUv2TaskPool"],
     "openclip_vision": ["OpenCLIPVisionModelTaskPool"],
 }
@@ -30,6 +31,7 @@ if TYPE_CHECKING:
     from .dummy import DummyTaskPool
     from .gpt2_text_classification import GPT2TextClassificationTaskPool
     from .llama import LlamaTestGenerationTaskPool
+    from .lm_eval_harness import LMEvalHarnessTaskPool
     from .nyuv2_taskpool import NYUv2TaskPool
     from .openclip_vision import OpenCLIPVisionModelTaskPool
 

fusion_bench/taskpool/lm_eval_harness/__init__.py

@@ -0,0 +1,3 @@
+from .taskpool import LMEvalHarnessTaskPool
+
+__all__ = ["LMEvalHarnessTaskPool"]

fusion_bench/taskpool/lm_eval_harness/taskpool.py

@@ -0,0 +1,87 @@
+import logging
+import os
+from typing import List, Literal, Optional, Union, TYPE_CHECKING
+
+import lightning.fabric
+import lm_eval
+import lm_eval.models
+from lm_eval.__main__ import check_argument_types, cli_evaluate, setup_parser
+from omegaconf import DictConfig, ListConfig
+
+from fusion_bench import BaseTaskPool
+from fusion_bench.mixins import LightningFabricMixin
+from fusion_bench.utils.strenum import _version
+
+
+log = logging.getLogger(__name__)
+
+
+class LMEvalHarnessTaskPool(BaseTaskPool, LightningFabricMixin):
+    def __init__(
+        self,
+        tasks: Union[str, List[str]],
+        apply_chat_template: bool = False,
+        include_path: Optional[str] = None,
+        batch_size: int = 1,
+        metadata: Optional[DictConfig] = None,
+        verbosity: Optional[
+            Literal["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
+        ] = None,
+        output_path: Optional[str] = None,
+        log_samples: bool = False,
+        _usage_: Optional[str] = None,
+        _version_: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(_usage_=_usage_, _version_=_version_)
+        self.tasks = tasks
+        self.include_path = include_path
+        self.batch_size = batch_size
+        self.metadata = metadata
+        self.apply_chat_template = apply_chat_template
+        self.verbosity = verbosity
+        self.kwargs = kwargs
+        self.output_path = output_path
+        self.log_samples = log_samples
+
+    def evaluate(self, model, *command_line_args, **kwargs):
+        command_line_args = []
+        if self.include_path is not None:
+            command_line_args.extend(["--include_path", self.include_path])
+        if isinstance(self.tasks, (list, ListConfig)):
+            command_line_args.extend(["--tasks", ",".join(self.tasks)])
+        elif isinstance(self.tasks, str):
+            command_line_args.extend(["--tasks", self.tasks])
+        if self.apply_chat_template:
+            command_line_args.extend(
+                ["--apply_chat_template", str(self.apply_chat_template)]
+            )
+        if self.batch_size is not None:
+            command_line_args.extend(["--batch_size", str(self.batch_size)])
+        if self.verbosity is not None:
+            command_line_args.extend(["--verbosity", str(self.verbosity)])
+        if self.metadata is not None:
+            command_line_args.extend(["--metadata", str(self.metadata)])
+        if self.output_path is None:
+            command_line_args.extend(
+                [
+                    "--output_path",
+                    os.path.join(self.log_dir, "lm_eval_results"),
+                ]
+            )
+        else:
+            command_line_args.extend(["--output_path", self.output_path])
+        if self.log_samples:
+            command_line_args.extend(["--log_samples"])
+        for key, value in kwargs.items():
+            command_line_args.extend([f"--{key}", str(value)])
+
+        parser = setup_parser()
+        check_argument_types(parser)
+        args = parser.parse_args(args=command_line_args)
+        log.info("LM-Eval Harness arguments:\n%s", args)
+
+        if not lightning.fabric.is_wrapped(model):
+            model = self.fabric.setup(model)
+        args.model = lm_eval.models.huggingface.HFLM(pretrained=model)
+        cli_evaluate(args)

{fusion_bench-0.2.13.dist-info → fusion_bench-0.2.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusion_bench
-Version: 0.2.13
+Version: 0.2.15
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 License: MIT License
@@ -45,6 +45,8 @@ Requires-Dist: rich
 Requires-Dist: scipy
 Requires-Dist: h5py
 Requires-Dist: pytest
+Provides-Extra: lm-eval-harness
+Requires-Dist: lm-eval; extra == "lm-eval-harness"
 Dynamic: license-file
 
 <div align='center'>
@@ -61,7 +63,7 @@ Dynamic: license-file
 
 </div>
 
-> [!TIP]  
+> [!TIP]
 > Documentation is available at [tanganke.github.io/fusion_bench/](https://tanganke.github.io/fusion_bench/).
 
 ## Overview
@@ -122,7 +124,7 @@ Merging multiple expert models offers a promising approach for performing multi-
 
 ## Installation
 
-install from PyPI:
+Install from PyPI:
 
 ```bash
 pip install fusion-bench
@@ -137,6 +139,27 @@ cd fusion_bench
 pip install -e . # install the package in editable mode
 ```
 
+### Install with [Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness)
+
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/10.5281/zenodo.10256836)
+
+
+```bash
+pip install "fusion-bench[lm-eval-harness]"
+```
+
+or install from local directory
+
+```bash
+pip install -e ".[lm-eval-harness]"
+```
+
+This will install the latest version of fusion-bench and the dependencies required for LM-Eval Harness.
+Documentation for using LM-Eval Harness within FusionBench framework can be found at [this online documentation](https://tanganke.github.io/fusion_bench/taskpool/lm_eval_harness) or in the [`docs/taskpool/lm_eval_harness.md`](docs/taskpool/lm_eval_harness.md) markdown file.
+
+> [!TIP]
+> Documentation for merging large language models using FusionBench can be found at [this online documentation](https://tanganke.github.io/fusion_bench/modelpool/causal_lm) or in the [`docs/modelpool/causal_lm.md`](docs/modelpool/causal_lm.md) markdown file.
+
 ## Introduction to Deep Model Fusion
 
 Deep model fusion is a technique that merges, ensemble, or fuse multiple deep neural networks to obtain a unified model.

{fusion_bench-0.2.13.dist-info → fusion_bench-0.2.15.dist-info}/RECORD

@@ -49,7 +49,7 @@ fusion_bench/method/ensemble.py,sha256=rGxvJTeorfcBuE_e0XO-0-MAc9un7ZCC46ikKGuAc
 fusion_bench/method/model_recombination.py,sha256=2tviqmYSPOL0_Ktv8_gt_YzQ4tyCANHxXquUot_3Cgo,5360
 fusion_bench/method/simple_average.py,sha256=2ghcL1E-eLbIYDCHYCoR9WtiYSb1GvFAH163OTTTEEI,4481
 fusion_bench/method/ada_svd/__init__.py,sha256=4XzQbbvE9HI3NtEmEFvo8iC3ds_85vJXe7P7qJfL7kk,77
-fusion_bench/method/ada_svd/clip_vision.py,sha256=QrT6cSwgVEGxXEpVhkvKQVQaoRW5P9V52Y3_8NX0f-o,12556
+fusion_bench/method/ada_svd/clip_vision.py,sha256=XvXgIdlShAREMsubRgphyycGrhWqSnuVBo6S9bNYSd0,12581
 fusion_bench/method/adamerging/__init__.py,sha256=nt0saBT_3bqghk-pINQ-XCWm9UWwSZllu4R1sDuAJAA,376
 fusion_bench/method/adamerging/clip_layer_wise_adamerging.py,sha256=UUSldRPBxHVOfkMM7ZwqZay5Wjc6XQ3Vy9PgyqV_TZo,1311
 fusion_bench/method/adamerging/clip_task_wise_adamerging.py,sha256=rREVf8SKlQ9SiWdUWOYo91b1RW9PnNJxsW8MxHs_MUo,6095
@@ -157,8 +157,9 @@ fusion_bench/method/slerp/slerp.py,sha256=2_n10REnRoV5DuwCC0bDX8RM3MLL4Q_5rZiU0h
 fusion_bench/method/slerp/slerp_utils.py,sha256=vksRo6n7FqY7By9aqbwTL4XV3BjcU_GrUl_r85Kpfjc,3504
 fusion_bench/method/smile_upscaling/__init__.py,sha256=6ZpUSHUFVsT1U7V3TIDWBFqcHte7SjHW0wp6CAE8NVg,165
 fusion_bench/method/smile_upscaling/singular_projection_merging.py,sha256=0neZS9oZnl64wu1xb9ruGB7lbhYXyy4zj8l3E1QYRGQ,6670
-fusion_bench/method/smile_upscaling/smile_mistral_upscaling.py,sha256=C2WEzBcAyrgO1FpFgg3pdrPjCkeGK1PKr66XQN1WJGE,11945
-fusion_bench/method/smile_upscaling/smile_upscaling.py,sha256=UXmFJ2s3qb7-5hsMUbQdtkhCRx4Gh8UCiost09JyVqY,20299
+fusion_bench/method/smile_upscaling/smile_mistral_upscaling.py,sha256=T1M4hTRgIfDhy4zSccREPLajgMcdnohr0NtdWXtPzmA,8802
+fusion_bench/method/smile_upscaling/smile_qwen2_upscaling.py,sha256=gRo5TKhcFhclKtpI75ixc65YrhoW3kyVr_q-JrCN8n8,8713
+fusion_bench/method/smile_upscaling/smile_upscaling.py,sha256=epkurTJQdO2TlWf1v5qUSiF1Pi1Umz-uQDtHM_iX80c,8978
 fusion_bench/method/sparse_we_moe/__init__.py,sha256=V5VOpLwn6ZpsM09TmwFhhlJwMTBFXF7NE1-gW1MlAfc,133
 fusion_bench/method/sparse_we_moe/sparse_clip_we_moe.py,sha256=J8iVYks-SQ93dqh6FUQACvSmM364QHlVBYMKOCPbHrU,10288
 fusion_bench/method/sparse_we_moe/sparse_we_moe.py,sha256=6OYgj_D_4xTtqy_guA7whQu76LQ7gv-U2cIZkXe7bIg,10479
@@ -218,7 +219,7 @@ fusion_bench/modelpool/huggingface_automodel.py,sha256=OJ6EyYyjNv1_Bhjn-zli-e__B
 fusion_bench/modelpool/huggingface_gpt2_classification.py,sha256=j8nicVwtoLXY4RPE2dcepeEB3agBKkkH-xA3yMj1czw,2014
 fusion_bench/modelpool/nyuv2_modelpool.py,sha256=btuXmYxwfjI6MnGakhoOf53Iyb9fxYH20CavGTrTcnA,1375
 fusion_bench/modelpool/causal_lm/__init__.py,sha256=F432-aDIgAbUITj4GNZS9dgUKKhaDMCbTeHB-9MecaQ,99
-fusion_bench/modelpool/causal_lm/causal_lm.py,sha256=k0eOOcFbswVgBYhM9CEXvdCRU9zVC8Gw78QaiMWzeWo,4487
+fusion_bench/modelpool/causal_lm/causal_lm.py,sha256=fO8lF8YWwoe43sVVOqHW9Ike7x-924-I6QQgZqx9EgA,6505
 fusion_bench/modelpool/clip_vision/__init__.py,sha256=3b9gN2bWUsoA1EmpitnIMnIlX7nklxbkn4WJ0QJtS2c,43
 fusion_bench/modelpool/clip_vision/modelpool.py,sha256=JH1wLdWefvE242SYpXTnoSLkKX-YcadnidWd2bo8tWQ,5486
 fusion_bench/modelpool/openclip_vision/__init__.py,sha256=QDmAitKqUwRygN9QncdS_kGWZdfTKL4uUifC8xh9c10,47
@@ -231,7 +232,7 @@ fusion_bench/modelpool/seq_classification_lm/seq_classification_lm.py,sha256=sIK
 fusion_bench/models/__init__.py,sha256=TNOEH_2yAQP51m9mdWepNEf9VGUZgDthtgXbs4rhb4M,100
 fusion_bench/models/hf_clip.py,sha256=056UHeSjKKDYXg-o7CC2zsx4fC9R6IBkPGI8IFhWTNw,7291
 fusion_bench/models/parameter_dict.py,sha256=lkVaK6xInqHoQ3_N6zx8CNKH4dnf8AP8H9xAY6ds6lg,3515
-fusion_bench/models/rankone_moe.py,sha256=uwpAqk1cwxxprQ0hxuAwRuPvHDxxBKBDahd9vcaafXs,14248
+fusion_bench/models/rankone_moe.py,sha256=aY8IDM-ct7qKYH8ukBUsa_VDkDgGNtCqyNtNKlDTUTc,12046
 fusion_bench/models/separate_io.py,sha256=5AJlCxkHdVVffITnIRlF3ZIaKLRWDhJESVQN1lX-ZhU,3835
 fusion_bench/models/sparse_we_moe.py,sha256=b-yIeCsl2rz0i7BP9g_fqCEam7KUNjNX_J8oyZV6MJ8,16509
 fusion_bench/models/utils.py,sha256=AQFI2UZSItKfJpG8ex74FPjn_SjsADLhvpv1GYqu43U,2065
@@ -261,8 +262,12 @@ fusion_bench/models/modeling_losparse_llama/register.py,sha256=rRTvc5hK9GvTiEZGq
 fusion_bench/models/modeling_losparse_llama/utils.py,sha256=REQIGeJeNhlKxm2Y7EUumtzj4vdTQQIT1dSiB22_i0o,1886
 fusion_bench/models/modeling_smile_mistral/__init__.py,sha256=q9DmZhBYwTOUsaWOs--tMZ-9zYpAa8KYEJPoNT7IZj0,1171
 fusion_bench/models/modeling_smile_mistral/configuration_smile_mistral.py,sha256=yt1-JBlkJmlJw7dvB4_V8M0gy5ihD8isDxcmwyW85d4,633
-fusion_bench/models/modeling_smile_mistral/modeling_smile_mistral.py,sha256=FxTOryDgi8z5W13ry42xsru_nWx0tGOpZuL-ufPuxoA,39897
+fusion_bench/models/modeling_smile_mistral/modeling_smile_mistral.py,sha256=5cN1M_XhcFCoJb8yvO1KCwHD_UH__hJg_X2D9C85R34,33128
 fusion_bench/models/modeling_smile_mistral/register.py,sha256=7nSJC4FveUi78rp53Ps6TcPGedHZ79cikYM5GIfEZfw,400
+fusion_bench/models/modeling_smile_qwen2/__init__.py,sha256=eaf9PRQ-rZRHzL2odWpR6ha3sR0rNMylrf0ZvqGTvbU,241
+fusion_bench/models/modeling_smile_qwen2/configuration_smile_qwen2.py,sha256=aekcpLcUGo4e7GkOtaxKClpIU5byyY-LQNDb-sMeyNc,621
+fusion_bench/models/modeling_smile_qwen2/modeling_smile_qwen2.py,sha256=hfn2Tmu4h8t3xiKvUpbXpjcir-aywn7rvvsQK7vLwVA,37247
+fusion_bench/models/modeling_smile_qwen2/register.py,sha256=-6XqWWrUltFkJ76C6NCveJp4vPcqAoijJ3fItmDFsQs,391
 fusion_bench/models/nyuv2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fusion_bench/models/nyuv2/aspp.py,sha256=Nl-Kx9YmGp0BNpDedo9cYbynOwI4SUyILWN2VgiPDIc,2495
 fusion_bench/models/nyuv2/lightning_module.py,sha256=SLtC0yL6455uKeb-o07MR6v-xE4BTKm7B0E2ayQwEBU,5436
@@ -273,7 +278,10 @@ fusion_bench/models/open_clip/modeling.py,sha256=34wKcbxe5xb6fzAVdIz0QcsSXs-8FQF
 fusion_bench/models/open_clip/utils.py,sha256=YM_vGQSxIDoB2euHG54hhRGIcINJfR0NxNT5U42KRCw,10394
 fusion_bench/models/open_clip/variables_and_paths.py,sha256=_OBcKvZwSGvYSmgKtXOuekEJI-btW94Ia-BQ9n4isfY,1231
 fusion_bench/models/smile_moe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fusion_bench/models/smile_moe/linear.py,sha256=voFvx4Nnfgc6YReBcY9FUGG3WrxVRjyD3odX4jIS5Eg,8724
+fusion_bench/models/smile_moe/linear_from_hf_config.py,sha256=4vzYYjDHGOf1IO7gO0dzQC1xqcwEij9M7d4tVZm-7dY,11919
+fusion_bench/models/smile_moe/linear_from_module.py,sha256=Sv6rCj1FWr7wQhv2hO2waJIkoHly7HZCP4zbYn3Dk78,10815
+fusion_bench/models/smile_moe/utils/__init__.py,sha256=_FfU5HAQIwl9Ch8M8_VBxLRtvqk0lWr-k1WVfWg61jA,588
+fusion_bench/models/smile_moe/utils/svd_utils.py,sha256=A2u7lH5Bo2qhgwplHPAz56pdbHYUunk2PS6PSvTn19M,1407
 fusion_bench/models/surgery/__init__.py,sha256=tcUSi2m9GzGWfvRDQScIbdEbFBS_35gm9zGKN7VpE70,53
 fusion_bench/models/surgery/surgerymodelwrapper.py,sha256=F8jX88K5zVWC6HsfN-nGNkEiPwNrN11ydyQQ1EZHehM,5133
 fusion_bench/models/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -298,7 +306,7 @@ fusion_bench/scripts/nyuv2_mtl_train.py,sha256=W1C45R9NdF4O-UjCx1bUxRTdFE0-FlRpw
 fusion_bench/scripts/webui.py,sha256=ryA-2leSnHcYA88tTAYzJGDhiljbi0vl1Fibejzndlw,14398
 fusion_bench/scripts/clip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fusion_bench/scripts/clip/convert_checkpoint.py,sha256=zncgRAhInFpJDSHIm3GO4F6BzgsdAQVj3LLmV7g-JiQ,1221
-fusion_bench/taskpool/__init__.py,sha256=1AFQhYDmrev3HZqWgSPz_CfygVGOYP9D8nY_q7QS9nk,1207
+fusion_bench/taskpool/__init__.py,sha256=-ltXMsS3jeGxa9vnhOyrbITOUtfNjLwkGPfS2mKDOdY,1312
 fusion_bench/taskpool/base_pool.py,sha256=Cbe3ZgJ34DWSDZeZEjlgqR0b84aM1i68D9-vomaooo8,852
 fusion_bench/taskpool/dummy.py,sha256=Di9JZO3XyDYn6wAGukrJMTnkS_NaxGTeQYo_3j1JD3Y,1675
 fusion_bench/taskpool/gpt2_text_classification.py,sha256=PCNdc2SNGUFGxJ0snmwrnjTdSwmDt9fs7Pe0eDjdvaw,6091
@@ -313,6 +321,8 @@ fusion_bench/taskpool/clip_vision/utils/routing_analysis_utils.py,sha256=LY9wxWC
 fusion_bench/taskpool/llama/__init__.py,sha256=iB4ESMgnsl0m-z0YtRdPZiwGGv96-86R8pbSnkdet8Q,57
 fusion_bench/taskpool/llama/reward_model.py,sha256=ZpRSX4esBAuE0MdTjPHjqS2TnvGb6P8arOGxBeXnq6Y,5028
 fusion_bench/taskpool/llama/test_generation.py,sha256=kJ_5GruG12FsuJHDh_S7pbQgwEojTqhGpA_wVNH5KPc,6675
+fusion_bench/taskpool/lm_eval_harness/__init__.py,sha256=_usNxe4z9avClSWjwHMxoznnI_UQFMuo7uOEJhP8jMk,81
+fusion_bench/taskpool/lm_eval_harness/taskpool.py,sha256=2eLP4FqVxtIbYe66R1ksNGpt94QOFHmH8C9UgxlWnJ8,3280
 fusion_bench/taskpool/openclip_vision/__init__.py,sha256=02p77Mb1JE7trrv2HtIku5X667WY5LZ2zVuyL3uIcyo,59
 fusion_bench/taskpool/openclip_vision/openclip_taskpool.py,sha256=PtD_Y9CWzPI3WEil_RuXtCh8ImPKcSHtZTqfybmsGdg,6875
 fusion_bench/tasks/__init__.py,sha256=Z_ePIp4Xizkj78QopLg1dZkJAN_IF73MkbR_nkfHQ9Y,52
@@ -383,7 +393,7 @@ fusion_bench/utils/plot/token_notebook.py,sha256=bsntXf46Zz_RavTxNiB9c3-KvHw7LFw
 fusion_bench/utils/strenum/__init__.py,sha256=id9ORi1uXrDxhbmVxitJ1KDwLS4H3AAwFpaK5h1cQzw,8531
 fusion_bench/utils/strenum/_name_mangler.py,sha256=o11M5-bURW2RBvRTYXFQIPNeqLzburdoWLIqk8X3ydw,3397
 fusion_bench/utils/strenum/_version.py,sha256=6JQRo9LcvODbCOeVFYQb9HNJ_J9XiG_Zbn8ws2A3BV8,18466
-fusion_bench-0.2.13.dist-info/licenses/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
+fusion_bench-0.2.15.dist-info/licenses/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
 fusion_bench_config/README.md,sha256=Lc8YSBJ5oxf9KV5kKDivJ9LRyGuraGQPmBbgbdVA-j4,703
 fusion_bench_config/clip-vit-base-patch32_robustness_corrupted.yaml,sha256=7IxLQoLRz-sRWyV8Vqc5kQcmYE_9YQz2_77pmvAkum8,1207
 fusion_bench_config/fabric_model_fusion.yaml,sha256=5iPgaM8UOhuvBW2Hap_csst-eqlYRwb_lru8ngjrZ_g,948
@@ -555,7 +565,8 @@ fusion_bench_config/method/regmean/gpt2_regmean.yaml,sha256=n94aTboDdwSA7Tki8l_o
 fusion_bench_config/method/regmean/regmean.yaml,sha256=ZgVVLx-lHwVgjtjTl4VZUlthh8yyua87QvoJfmNHud4,101
 fusion_bench_config/method/slerp/slerp.yaml,sha256=xldDUULtfCdwzAkQUb0C8-TmbW7FqcAlIOsPX8p4n6w,116
 fusion_bench_config/method/smile_upscaling/singular_projection_merging.yaml,sha256=ZMn_ImRjjc2uozf7ocQIzbgvFDpBV7S-34KptbBXVGo,200
-fusion_bench_config/method/smile_upscaling/smile_mistral_upscaling.yaml,sha256=cah2cwaSfdwNc5AgY3soDrBHFyIOKRC5UdMPqUr-X7g,236
+fusion_bench_config/method/smile_upscaling/smile_mistral_upscaling.yaml,sha256=VFMrkbO69d0wCjTQCuKysYGVe6hEwNu792g1QkhU5Mk,383
+fusion_bench_config/method/smile_upscaling/smile_qwen2_upscaling.yaml,sha256=Rdcub7yFFn-jKXrlFoj8LQk1cRbJm2do91pV-YMSzTE,378
 fusion_bench_config/method/smile_upscaling/smile_upscaling.yaml,sha256=G88mabTTniDUtiUC9Vg3cj_sw6D05mE4_ZdyYI4Omjk,477
 fusion_bench_config/method/sparselo_pruning/llama_iterative_sparselo.yaml,sha256=1zfrT2FNmLyCivth4kzGR8Ai9jyQ87OXRbf4di4IE94,642
 fusion_bench_config/method/sparselo_pruning/llama_pcp_sparselo.yaml,sha256=UPnMt_GoMSHOeOx5Sv0oHPRoPhhvVRC5zdVA38OTwSg,636
@@ -715,8 +726,17 @@ fusion_bench_config/modelpool/CausalLMPool/llama_codealpaca.yaml,sha256=FuUsBrvk
 fusion_bench_config/modelpool/CausalLMPool/llama_for_causallm.yaml,sha256=H3UwSk4ChsGSrH49LuttxldFURW-4RVUtnIa0ClHKXo,802
 fusion_bench_config/modelpool/CausalLMPool/llama_metamathqa.yaml,sha256=vU0q06OUa1UM_Xvp2t27Rl3F6EDgYWPnDxeyzUH-QVI,589
 fusion_bench_config/modelpool/CausalLMPool/llama_ultrachat.yaml,sha256=MpgshGtmMXpUFRA1knjdGRVH4UgZbkkcTmCTrF3LlZk,573
+fusion_bench_config/modelpool/CausalLMPool/qwen2_math_1.5B_and_R1.yaml,sha256=Kbpam1Hds5URMP35dXGdVibH-vTmYPh3xHMkhj6Mgtg,648
 fusion_bench_config/modelpool/CausalLMPool/simle_mixtral_exp_v4.yaml,sha256=FynhZ1PRvyzsyzrHIuMpGgQGRMlu_xI7earm-CeIVeY,824
 fusion_bench_config/modelpool/CausalLMPool/single_llama_model.yaml,sha256=zQWfp7mYm6jQ8g41Eeh2d9vAbocZJ5btPX1ft9QpEZU,546
+fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.1-8B-Instruct.yaml,sha256=NDq_prH-b9Vw7lRjsyJIcbeF4MXVVdszxK1FPJxIJYs,453
+fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.1-8B.yaml,sha256=Mg_z2vnw7IkNPoMvhl_Ja6gT9tX942sqaNfjXQRzBvg,390
+fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.2-3B-Instruct.yaml,sha256=SfPEji6mWx9Dw48rE0B8MDrYv2NVLC-S98DK5xaU6So,453
+fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.2-3B.yaml,sha256=2vpOp9t8SUP2rkBw21mqwRYApkqXQiaYXcZm2oxLox4,390
+fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-2b-it.yaml,sha256=8up_cqEhabGeK6l6tMha9DJzsPoEIFN8bS_Kwv7LmCc,389
+fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-2b.yaml,sha256=SODG0kcnAP6yC0_J_SpSVMRV-v5qGV22gcWdiBaZo1I,368
+fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-9b-it.yaml,sha256=zwInWJS8yrhch4vOL1ypRKNWWpJKlhQsyY0Ln14CC-M,389
+fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-9b.yaml,sha256=ufmu4b3lyxn2XLDMVYxP-bKwYaGTjB5-JoYXLG8v8tY,368
 fusion_bench_config/modelpool/OpenCLIPVisionModelPool/README.md,sha256=DC0HF-isCHshipHTC0Rof6GvjTUa0i2DVQZKrklQQlU,2416
 fusion_bench_config/modelpool/OpenCLIPVisionModelPool/ViT-B-16_TA8.yaml,sha256=jbJqqciORJQknpSzh2zKiFm6VKDOsmaSk9XfPCVmHGg,1220
 fusion_bench_config/modelpool/OpenCLIPVisionModelPool/ViT-B-32_TA8.yaml,sha256=q2_E2R1wIOdxd-AF-wjXkPO64gJgD27YXsZ8FFLWUIo,1607
@@ -774,11 +794,12 @@ fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip-vit-single-task_sun397
 fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip-vit-single-task_svhn.yaml,sha256=2AqMiNCRRunLIrssHvFzu1lUzOaQn8uOHM9yjrQq-_A,109
 fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip_rankone_wemoe_clip-vit-classification_TA8.yaml,sha256=DNm1LRlQS9KbukEl6oEZzWLizyaOBcYZ2r7L8ZQtnJc,434
 fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip_sparse_wemoe_clip-vit-classification_TA8.yaml,sha256=EjN3Pu1F_7EuZrk-geyL4qohqJ5-F2UFjWjj2V57ju0,433
+fusion_bench_config/taskpool/LMEvalHarnessTaskPool/lm_eval.yaml,sha256=45kSz44pcjTDCL3dnEECRMnN0kIaoWKUFZMFy5JJIyw,416
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-B-16_TA8.yaml,sha256=GjpiiRownrBCpl-TNwWRW2PYePbF-Cl99jlLNPrK5T4,1017
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-B-32_TA8.yaml,sha256=WwiYMQKehtJixDPnu5o3vcWe4yJksXTWRqOzm3uVWXQ,1017
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-L-14_TA8.yaml,sha256=xGRt0J9joXTzWUew6DvoYprAWlPXhaVFw5AX4im5VQw,1017
-fusion_bench-0.2.13.dist-info/METADATA,sha256=RB7s0Jppl2mX5evPosbdXONlF__hlFeZf8dF4Qf36tQ,20085
-fusion_bench-0.2.13.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-fusion_bench-0.2.13.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
-fusion_bench-0.2.13.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
-fusion_bench-0.2.13.dist-info/RECORD,,
+fusion_bench-0.2.15.dist-info/METADATA,sha256=abOyRl-ejl7CvLRCaRP20vn7rdb5OF92GxS_S9qTK3Q,21171
+fusion_bench-0.2.15.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+fusion_bench-0.2.15.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
+fusion_bench-0.2.15.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
+fusion_bench-0.2.15.dist-info/RECORD,,

{fusion_bench-0.2.13.dist-info → fusion_bench-0.2.15.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fusion_bench_config/method/smile_upscaling/smile_mistral_upscaling.yaml

@@ -1,9 +1,12 @@
-name: smile_mistral_upscaling
+_target_: fusion_bench.method.smile_upscaling.smile_mistral_upscaling.SmileMistralUpscalingAlgorithm
+# device to put the models on
 device: cpu
+# device to perform SVD on
 accelerator: cuda
 # path to save/load the model
 model_path: null
-model_dtype: float16
+model_dtype: null
+# SmileMoE parameters
 num_experts_per_tok: 1
 rank_of_router: 8
 # if rank_of_expert < 0, dense expert is used.

fusion_bench_config/method/smile_upscaling/smile_qwen2_upscaling.yaml

@@ -0,0 +1,13 @@
+_target_: fusion_bench.method.smile_upscaling.smile_qwen2_upscaling.SmileQwen2UpscalingAlgorithm
+# device to put the models on
+device: cpu
+# device to perform SVD on
+accelerator: cuda
+# path to save/load the model
+model_path: null
+model_dtype: null
+# SmileMoE parameters
+num_experts_per_tok: 1
+rank_of_router: 8
+# if rank_of_expert < 0, dense expert is used.
+rank_of_expert: 64

fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.1-8B-Instruct.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: meta-llama/Llama-3.1-8B-Instruct
+  instruction: MergeBench/Llama-3.1-8B-Instruct_instruction
+  math: MergeBench/Llama-3.1-8B-Instruct_math
+  coding: MergeBench/Llama-3.1-8B-Instruct_coding
+  multilingual: MergeBench/Llama-3.1-8B-Instruct_multilingual
+  safety: MergeBench/Llama-3.1-8B-Instruct_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: meta-llama/Llama-3.1-8B-Instruct

fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.1-8B.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: meta-llama/Llama-3.1-8B
+  instruction: MergeBench/Llama-3.1-8B_instruction
+  math: MergeBench/Llama-3.1-8B_math
+  coding: MergeBench/Llama-3.1-8B_coding
+  multilingual: MergeBench/Llama-3.1-8B_multilingual
+  safety: MergeBench/Llama-3.1-8B_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: meta-llama/Llama-3.1-8B

fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.2-3B-Instruct.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: meta-llama/Llama-3.2-3B-Instruct
+  instruction: MergeBench/Llama-3.2-3B-Instruct_instruction
+  math: MergeBench/Llama-3.2-3B-Instruct_math
+  coding: MergeBench/Llama-3.2-3B-Instruct_coding
+  multilingual: MergeBench/Llama-3.2-3B-Instruct_multilingual
+  safety: MergeBench/Llama-3.2-3B-Instruct_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: meta-llama/Llama-3.2-3B-Instruct

fusion_bench_config/modelpool/CausalLMPool/mergebench/Llama-3.2-3B.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: meta-llama/Llama-3.2-3B
+  instruction: MergeBench/Llama-3.2-3B_instruction
+  math: MergeBench/Llama-3.2-3B_math
+  coding: MergeBench/Llama-3.2-3B_coding
+  multilingual: MergeBench/Llama-3.2-3B_multilingual
+  safety: MergeBench/Llama-3.2-3B_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: meta-llama/Llama-3.2-3B

fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-2b-it.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: google/gemma-2-2b-it
+  instruction: MergeBench/gemma-2-2b-it_instruction
+  math: MergeBench/gemma-2-2b-it_math
+  coding: MergeBench/gemma-2-2b-it_coding
+  multilingual: MergeBench/gemma-2-2b-it_multilingual
+  safety: MergeBench/gemma-2-2b-it_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: google/gemma-2-2b-it

fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-2b.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: google/gemma-2-2b
+  instruction: MergeBench/gemma-2-2b_instruction
+  math: MergeBench/gemma-2-2b_math
+  coding: MergeBench/gemma-2-2b_coding
+  multilingual: MergeBench/gemma-2-2b_multilingual
+  safety: MergeBench/gemma-2-2b_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: google/gemma-2-2b

fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-9b-it.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: google/gemma-2-9b-it
+  instruction: MergeBench/gemma-2-9b-it_instruction
+  math: MergeBench/gemma-2-9b-it_math
+  coding: MergeBench/gemma-2-9b-it_coding
+  multilingual: MergeBench/gemma-2-9b-it_multilingual
+  safety: MergeBench/gemma-2-9b-it_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: google/gemma-2-9b-it

fusion_bench_config/modelpool/CausalLMPool/mergebench/gemma-2-9b.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: google/gemma-2-9b
+  instruction: MergeBench/gemma-2-9b_instruction
+  math: MergeBench/gemma-2-9b_math
+  coding: MergeBench/gemma-2-9b_coding
+  multilingual: MergeBench/gemma-2-9b_multilingual
+  safety: MergeBench/gemma-2-9b_safety
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: google/gemma-2-9b

fusion_bench_config/modelpool/CausalLMPool/qwen2_math_1.5B_and_R1.yaml

@@ -0,0 +1,17 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+_recursive_: false
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: Qwen/Qwen2.5-1.5B
+  expert_1:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: Qwen/Qwen2.5-Math-1.5B
+  expert_2:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: Qwen/Qwen2.5-1.5B

fusion_bench_config/taskpool/LMEvalHarnessTaskPool/lm_eval.yaml

@@ -0,0 +1,12 @@
+_target_: fusion_bench.taskpool.LMEvalHarnessTaskPool
+
+tasks:
+  - truthfulqa
+batch_size: 1
+verbosity: null
+include_path: null
+apply_chat_template: false
+# if `output_path` is not given, the results will be saved to `log_dir/lm_eval_results`, where `log_dir` is the directory controlled by lightning Fabric.
+output_path: null
+# if `log_samples` is true, the samples will be saved to `output_path`.
+log_samples: false