fusion-bench 0.2.9__py3-none-any.whl → 0.2.10__py3-none-any.whl

fusion_bench/dataset/fer2013.py

@@ -7,6 +7,7 @@ def load_fer2013(path: str = "clip-benchmark/wds_fer2013", split: str = "train")
     dataset = dataset.rename_columns({"jpg": "image", "cls": "label"})
     return dataset
 
+
 if __name__ == "__main__":
     dataset = load_fer2013(split="test")
     print(dataset)

fusion_bench/method/__init__.py

@@ -54,6 +54,12 @@ _import_structure = {
     ],
     "ada_svd": ["AdaSVDMergingForCLIPVisionModel"],
     "task_singular_vector": ["TaskSingularVectorMerging"],
+    "isotropic_merging": [
+        "ISO_C_Merge",  # alias
+        "ISO_CTS_Merge",  # alias
+        "IsotropicMergingInCommonAndTaskSubspace",
+        "IsotropicMergingInCommonSubspace",
+    ],
     "opcm": ["OPCMForCLIP"],
     # plug-and-play model merging methods
     "concrete_subspace": [
@@ -120,6 +126,12 @@ if TYPE_CHECKING:
         WeightedEnsembleAlgorithm,
     )
     from .fisher_merging import FisherMergingForCLIPVisionModel
+    from .isotropic_merging import (
+        ISO_C_Merge,
+        ISO_CTS_Merge,
+        IsotropicMergingInCommonAndTaskSubspace,
+        IsotropicMergingInCommonSubspace,
+    )
     from .linear import (
         ExPOAlgorithm,
         ExPOAlgorithmForLlama,

fusion_bench/method/classification/continual_clip_finetune.py

@@ -26,8 +26,8 @@ from fusion_bench.models.hf_clip import HFCLIPClassifier
 from fusion_bench.models.linearized.linearized_model_utils import LinearizedModelWraper
 from fusion_bench.taskpool import CLIPVisionModelTaskPool
 from fusion_bench.utils.data import InfiniteDataLoader
-from fusion_bench.utils.json import load_from_json, save_to_json
 from fusion_bench.utils.fabric import seed_everything_by_time
+from fusion_bench.utils.json import load_from_json, save_to_json
 
 
 class ContinualImageClassificationFineTuningForCLIP(

fusion_bench/method/isotropic_merging/__init__.py

@@ -0,0 +1,15 @@
+"""
+This module contains the implementation of the Isotropic Merging in Common Subspace (ISO-C) algorithm and Isotropic Merging in Common and Task-Specific Subspaces (Iso-CTS) algorithm.
+Modified from the original implementation: https://github.com/danielm1405/iso-merging
+
+Reference:
+- Daniel Marczak, et al. No Task Left Behind: Isotropic Model Merging with Common and Task-Specific Subspaces. 2025. 
+    https://arxiv.org/abs/2502.04959
+"""
+
+from .iso import (
+    ISO_C_Merge,
+    ISO_CTS_Merge,
+    IsotropicMergingInCommonSubspace,
+    IsotropicMergingInCommonAndTaskSubspace,
+)

fusion_bench/method/isotropic_merging/iso.py

@@ -0,0 +1,114 @@
+from typing import List
+
+import torch
+
+from fusion_bench import BaseAlgorithm, BaseModelPool
+from fusion_bench.mixins import LightningFabricMixin
+from fusion_bench.utils.state_dict_arithmetic import (
+    state_dict_add,
+    state_dict_sub,
+    state_dict_mul,
+)
+
+from .iso_utils import iso_c, iso_cts, check_parameterNamesMatch
+
+
+class IsotropicMergingInCommonSubspace(BaseAlgorithm, LightningFabricMixin):
+    """
+    Isotropic Merging in Common Subspace (Iso-C)
+    """
+
+    def __init__(
+        self,
+        scaling_factor: float,
+        exclude_keys: List[str] = None,
+    ):
+        self.scaling_factor = scaling_factor
+        self.exclude_keys = exclude_keys
+        super().__init__()
+
+    def run(self, modelpool: BaseModelPool):
+        # load the pretrained model and the task vectors of all the finetuned models
+        with torch.no_grad():
+            pretrained_model = modelpool.load_pretrained_model()
+            task_vectors = []
+            for model_name in modelpool.model_names:
+                finetuned_model = modelpool.load_model(model_name)
+                task_vectors.append(
+                    state_dict_sub(
+                        finetuned_model.state_dict(), pretrained_model.state_dict()
+                    )
+                )
+                del finetuned_model  # free memory
+            check_parameterNamesMatch(task_vectors)
+
+        # compute the merged task vector
+        merged_tv = iso_c(
+            task_vectors,
+            accelerator=self.fabric.device,
+            exclude_keys=self.exclude_keys,
+        )
+
+        # merged_parameters = pretrained_parameters + scaling_factor * merged_task_vector
+        pretrained_model.load_state_dict(
+            state_dict_add(
+                pretrained_model.state_dict(),
+                state_dict_mul(merged_tv, self.scaling_factor),
+            )
+        )
+
+        return pretrained_model
+
+
+class IsotropicMergingInCommonAndTaskSubspace(BaseAlgorithm, LightningFabricMixin):
+    """
+    Isotropic Merging in Common and Task-Specific Subspaces (Iso-CTS)
+    """
+
+    def __init__(
+        self,
+        scaling_factor: float,
+        common_space_fraction: float,
+        exclude_keys: List[str] = None,
+    ):
+        self.common_space_fraction = common_space_fraction
+        self.scaling_factor = scaling_factor
+        self.exclude_keys = exclude_keys
+        super().__init__()
+
+    def run(self, modelpool: BaseModelPool):
+        # load the pretrained model and the task vectors of all the finetuned models
+        with torch.no_grad():
+            pretrained_model = modelpool.load_pretrained_model()
+            task_vectors = []
+            for model_name in modelpool.model_names:
+                finetuned_model = modelpool.load_model(model_name)
+                task_vectors.append(
+                    state_dict_sub(
+                        finetuned_model.state_dict(), pretrained_model.state_dict()
+                    )
+                )
+                del finetuned_model  # free memory
+            check_parameterNamesMatch(task_vectors)
+
+        # compute the merged task vector
+        merged_tv = iso_cts(
+            task_vectors,
+            common_space_fraction=self.common_space_fraction,
+            accelerator=self.fabric.device,
+            exclude_keys=self.exclude_keys,
+        )
+
+        # merged_parameters = pretrained_parameters + scaling_factor * merged_task_vector
+        pretrained_model.load_state_dict(
+            state_dict_add(
+                pretrained_model.state_dict(),
+                state_dict_mul(merged_tv, self.scaling_factor),
+            )
+        )
+
+        return pretrained_model
+
+
+ISO_C_Merge = IsotropicMergingInCommonSubspace  # alias
+ISO_CTS_Merge = IsotropicMergingInCommonAndTaskSubspace  # alias

fusion_bench/method/isotropic_merging/iso_utils.py

@@ -0,0 +1,176 @@
+import math
+from typing import List
+
+import torch
+
+from fusion_bench.utils import timeit_context
+from fusion_bench.utils.type import StateDictType
+
+
+def iso_c(
+    task_vectors: List[StateDictType],
+    accelerator="cuda",
+    exclude_keys: List[str] = None,
+) -> StateDictType:
+    exclude_keys = [] if exclude_keys is None else exclude_keys
+
+    with torch.no_grad(), timeit_context("ISO-C Merging"):
+        new_vector = {}
+        for key in task_vectors[0]:
+            print(f"Merging {key}...")
+            original_device = task_vectors[0][key].device
+            tvs = [
+                task_vector[key].to(device=accelerator, non_blocking=True)
+                for task_vector in task_vectors
+            ]
+            num_tvs = len(tvs)
+            new_vector[key] = sum(tvs) / num_tvs
+            del tvs  # free memory
+
+            if len(task_vectors[0][key].shape) == 2 and key not in exclude_keys:
+                # if the key is a 2D matrix, we need to merge the task vectors in the common space
+                new_vector[key] *= num_tvs
+                U, S, V = torch.linalg.svd(new_vector[key], full_matrices=False)
+                S_mean = torch.ones_like(S) * S.mean()
+
+                new_vector[key] = torch.linalg.multi_dot(
+                    (
+                        U,
+                        torch.diag(S_mean),
+                        V,
+                    )
+                )
+            new_vector[key] = new_vector[key].to(
+                device=original_device, non_blocking=True
+            )
+    return new_vector
+
+
+@torch.no_grad()
+def iso_cts(
+    task_vectors: List[StateDictType],
+    common_space_fraction: float,
+    accelerator: str = "cuda",
+    exclude_keys: List[str] = None,
+):
+    exclude_keys = [] if exclude_keys is None else exclude_keys
+    new_vector = {}
+
+    print("ISO-CTS Merging")
+    for key in task_vectors[0]:
+        shape_ = task_vectors[0][key].shape
+        original_device = task_vectors[0][key].device
+        is_2d_matrix = (len(shape_) == 2) and (key not in exclude_keys)
+        if not is_2d_matrix:
+            print(f"Combining by avg {key}...")
+            for i, task_vector in enumerate(task_vectors):
+                vec = task_vector[key].to(device=accelerator, non_blocking=True)
+                if i == 0:
+                    new_vector[key] = vec.clone()
+                else:
+                    new_vector[key] += (vec - new_vector[key]) / (i + 1)
+
+            # move the new vector to the original device
+            new_vector[key] = new_vector[key].to(
+                device=original_device, non_blocking=True
+            )
+            continue
+
+        print(f"Computing common space using sum for {key}...")
+        combined_w = sum(
+            [
+                task_vector[key].to(device=accelerator, non_blocking=True)
+                for task_vector in task_vectors
+            ]
+        )
+
+        ### Calculate the common space size (making sure that task specific space is equally divisible) ###
+        common_space_index_s = int(min(shape_) * common_space_fraction)
+        _task_specific_total_space_index_s = round(
+            (min(shape_) - common_space_index_s) / len(task_vectors)
+        ) * len(task_vectors)
+        common_space_index_s = min(shape_) - _task_specific_total_space_index_s
+
+        u, s, v = torch.linalg.svd(combined_w, full_matrices=False)
+        common_space_u = u[:, :common_space_index_s]
+        common_space_s = s[:common_space_index_s]
+        common_space_v = v[:common_space_index_s, :]
+        ###################################################################
+
+        ### Calculate task specific space ###
+        n_dims_per_task = int((min(shape_) - common_space_index_s) / len(task_vectors))
+        for i, task_vector in enumerate(task_vectors):
+            w = task_vector[key].to(device=accelerator)
+
+            # calculate the projection onto task specific space to remove the common space
+            w_ts = w - common_space_u @ common_space_u.T @ w
+            u_ts, s_ts, v_ts = torch.linalg.svd(w_ts, full_matrices=False)
+
+            if i == 0:
+                combined_space_u = torch.zeros_like(u_ts, device=accelerator)
+                combined_space_s = torch.zeros_like(s_ts, device=accelerator)
+                combined_space_v = torch.zeros_like(v_ts, device=accelerator)
+
+            combined_space_u[:, i * n_dims_per_task : (i + 1) * n_dims_per_task] = u_ts[
+                :, :n_dims_per_task
+            ]
+            combined_space_s[i * n_dims_per_task : (i + 1) * n_dims_per_task] = s_ts[
+                :n_dims_per_task
+            ]
+            combined_space_v[i * n_dims_per_task : (i + 1) * n_dims_per_task, :] = v_ts[
+                :n_dims_per_task, :
+            ]
+        ###################################################################
+
+        combined_space_u[
+            :,
+            len(task_vectors) * n_dims_per_task : len(task_vectors) * n_dims_per_task
+            + common_space_index_s,
+        ] = common_space_u
+        combined_space_s[
+            len(task_vectors) * n_dims_per_task : len(task_vectors) * n_dims_per_task
+            + common_space_index_s
+        ] = common_space_s
+        combined_space_v[
+            len(task_vectors) * n_dims_per_task : len(task_vectors) * n_dims_per_task
+            + common_space_index_s,
+            :,
+        ] = common_space_v
+
+        ### Orthogonalize combined_space_u and combined_space_v ###
+        u_combined_space_u, s_combined_space_u, v_combined_space_u = torch.linalg.svd(
+            combined_space_u, full_matrices=False
+        )
+        u_combined_space_v, s_combined_space_v, v_combined_space_v = torch.linalg.svd(
+            combined_space_v, full_matrices=False
+        )
+        combined_space_u = u_combined_space_u @ v_combined_space_u
+        combined_space_v = u_combined_space_v @ v_combined_space_v
+        ###################################################################
+
+        combined_space_s = torch.ones_like(combined_space_s) * combined_space_s.mean()
+
+        new_vector[key] = torch.linalg.multi_dot(
+            (
+                combined_space_u,
+                torch.diag(combined_space_s),
+                combined_space_v,
+            )
+        )
+        new_vector[key] = new_vector[key].to(device=original_device, non_blocking=True)
+
+    return new_vector
+
+
+def check_parameterNamesMatch(checkpoints):
+    parameter_names = set(checkpoints[0].keys())
+
+    if len(checkpoints) >= 2:
+        # raise ValueError("Number of models is less than 2.")
+        for checkpoint in checkpoints[1:]:
+            current_parameterNames = set(checkpoint.keys())
+            if current_parameterNames != parameter_names:
+                raise ValueError(
+                    "Differing parameter names in models. "
+                    f"The different parameters are {parameter_names.symmetric_difference(current_parameterNames)}"
+                )

fusion_bench/method/task_singular_vector/TSVM.py

@@ -9,15 +9,20 @@ fusion_bench \
 ```
 """
 
-from typing import List, Optional
+from typing import List, Optional, Union, Iterable
 
 import torch
 from torch import Tensor, nn
+from omegaconf import ListConfig
 
 from fusion_bench import BaseAlgorithm
 from fusion_bench.mixins import LightningFabricMixin
 from fusion_bench.utils import timeit_context
-from fusion_bench.utils.state_dict_arithmetic import state_dict_sub, state_dict_add
+from fusion_bench.utils.state_dict_arithmetic import (
+    state_dict_add,
+    state_dict_sub,
+    state_dict_mul,
+)
 from fusion_bench.utils.type import StateDictType
 
 from .utils import (
@@ -33,9 +38,11 @@ class TaskSingularVectorMerging(BaseAlgorithm, LightningFabricMixin):
 
     def __init__(
         self,
+        alpha: Union[float, Iterable[float]] = None,
         remove_keys: Optional[List[str]] = None,
         **kwargs,
     ):
+        self.alpha = alpha
         self.remove_keys = remove_keys if remove_keys is not None else []
         super().__init__(**kwargs)
 
@@ -50,6 +57,14 @@ class TaskSingularVectorMerging(BaseAlgorithm, LightningFabricMixin):
 
         with timeit_context("Flattening out Checkpoints"):
             task_vectors = [state_dict_sub(check, ptm_check) for check in ft_checks]
+            if isinstance(self.alpha, Iterable):
+                assert len(self.alpha) == len(
+                    task_vectors
+                ), "Alpha and task vectors must have the same length"
+                task_vectors = [
+                    state_dict_mul(state_dict=tv, scalar=alpha)
+                    for alpha, tv in zip(self.alpha, task_vectors)
+                ]
 
         new_merged_tv = TSVM_utils.compute_and_sum_svd_mem_reduction(
             task_vectors,
@@ -57,6 +72,11 @@ class TaskSingularVectorMerging(BaseAlgorithm, LightningFabricMixin):
             accelerator=self.fabric.device,
         )
 
+        # If alpha is a float, we need to scale the new merged task vector by alpha
+        if self.alpha is not None and isinstance(self.alpha, float):
+            print(f"Scaling new merged task vector by alpha: {self.alpha}")
+            new_merged_tv = state_dict_mul(state_dict=new_merged_tv, scalar=self.alpha)
+
         pretrained_model.load_state_dict(
             state_dict_add(new_merged_tv, pretrained_model.state_dict())
         )

{fusion_bench-0.2.9.dist-info → fusion_bench-0.2.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: fusion_bench
-Version: 0.2.9
+Version: 0.2.10
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 License: MIT License

{fusion_bench-0.2.9.dist-info → fusion_bench-0.2.10.dist-info}/RECORD

@@ -15,7 +15,7 @@ fusion_bench/constants/__init__.py,sha256=Pyc4dLbl6oNduOCdnpeXQ9LDyVoIrkdl9eZ_l2
 fusion_bench/constants/paths.py,sha256=DVZyQ9FLhkyUdw6ARpXUCAMf_B8hFyJ6UNI-oYly3pE,591
 fusion_bench/dataset/__init__.py,sha256=OJiYmcqz0Vm5O7mE4PB5QFJeL_KjrsseQTRsQATGTm4,1050
 fusion_bench/dataset/clip_dataset.py,sha256=XLpCOiXlLEP3DffAlBn4P2PpUenbEFl-Yk9MNy6nbbI,2790
-fusion_bench/dataset/fer2013.py,sha256=bAdujQSj1PcUVFlKJgqcHAuE9AWz7JE1fzZ6scFVvmc,403
+fusion_bench/dataset/fer2013.py,sha256=Lub_xVhHfqaiPprvOsDVspJNioh1FjSrkhn3gL_UXDA,404
 fusion_bench/dataset/gpt2_glue.py,sha256=Qq1ZkEIQsTjj8tImvkZDNlduocSYwlEfVrDReZqDWdw,8761
 fusion_bench/dataset/gsm8k.py,sha256=CmANZ0A89PfPwVu_myKhXk1D9IwypOpjH3iqDo1KxcQ,2233
 fusion_bench/dataset/image_dataset.py,sha256=MSZE_UESyRRQDwnkm2KpyIARUg9SWcwqnH4fDNstzS4,1870
@@ -41,7 +41,7 @@ fusion_bench/dataset/llama/stanford_shp.py,sha256=6ueXKnFXIBBobacU1h5WxGLZrSOtBk
 fusion_bench/dataset/llama/ultrachat.py,sha256=Go7WvrDAYnm184fdazHGRYLbSY6Xd7jrESyQeUJtOww,1736
 fusion_bench/dataset/llama/wikitext.py,sha256=9ZHR-nMfXRumd3o-PIj3n7B83YlVeqpGkZ2zJs2B-9Y,2883
 fusion_bench/dataset/llama/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fusion_bench/method/__init__.py,sha256=Hx_e9afWildQmZsHvInllReP1ACN2LfF8MjQqb3gkKc,6228
+fusion_bench/method/__init__.py,sha256=py1hn-gRqwzESwS5KvDapclyUvIW42r0nDsuINgFBB4,6601
 fusion_bench/method/base_algorithm.py,sha256=5dutGZfPqNhO8F8FOlo3UFR91TZu2Xj7O0pTB40JvWo,1135
 fusion_bench/method/dummy.py,sha256=hb1y6LR_geRZ5eRgGwt5zJUcHYorCeIbs5i76CvurUc,1031
 fusion_bench/method/ensemble.py,sha256=rGxvJTeorfcBuE_e0XO-0-MAc9un7ZCC46ikKGuAcN4,3077
@@ -65,7 +65,7 @@ fusion_bench/method/analysis/task_vector_cos_similarity.py,sha256=pL-XsWTo258yZT
 fusion_bench/method/analysis/task_vector_violin_plot.py,sha256=ie8hPl6QsVz9MQ6C2OEpzIBxQnmVKNf1FPc5bThmQGM,7606
 fusion_bench/method/classification/__init__.py,sha256=emB06UOMDHK5pfQ1WuvLG9Fm0aEEtZxSjpVw8fVE0fM,167
 fusion_bench/method/classification/clip_finetune.py,sha256=DlV1isp8vz6jwXNYQ6zbblAoUfnssL-WBpDeaXI5BVw,15727
-fusion_bench/method/classification/continual_clip_finetune.py,sha256=v_2tmkb92okNbKRiW7AYdGEA7y2wQUwEjD01aeI99v0,11530
+fusion_bench/method/classification/continual_clip_finetune.py,sha256=OLhZKS-6aCnafevZkZYcNMKTWDDj3DATB27eZl_i8EY,11530
 fusion_bench/method/concrete_subspace/__init__.py,sha256=yjadcpquHZbeZYsbfYhe2JlX46kObfiWJRsIoVcOEg4,223
 fusion_bench/method/concrete_subspace/clip_concrete_adamerging.py,sha256=90_0HkOIl0XQG89xMa0UiBhrwfV2YqfLxlS04AouR3o,24755
 fusion_bench/method/concrete_subspace/clip_concrete_task_arithmetic.py,sha256=Nx-3AiAeIt5zmcC21Ta2_-4cAQg9hOWvThurXNZzA-w,10580
@@ -85,6 +85,9 @@ fusion_bench/method/fisher_merging/__init__.py,sha256=KWsjrtxKkPYwcUA5rB_6UNIqve
 fusion_bench/method/fisher_merging/clip_fisher_merging.py,sha256=QCutGqjkfW3OWETPZsCChqLRAhvfJp4QKD9TGSpTyV0,7635
 fusion_bench/method/fisher_merging/fisher_merging.py,sha256=CPU-tJiDv9FCIBYl7Pn0zA5cdRB1Md5kWchRDlJgly0,20456
 fusion_bench/method/fisher_merging/gpt2_fisher_merging.py,sha256=LZmz41jZ5dSsAHxfOUpr3u2rlCgUPTDR7xMsIlQM-jc,7576
+fusion_bench/method/isotropic_merging/__init__.py,sha256=Bg12OiltvZLMmZm066quvtG0LOWSVqI5RggYeaMDGFA,585
+fusion_bench/method/isotropic_merging/iso.py,sha256=GILofZQiTcOnJRQ28RmzOjqkso5Xih9WuFuB2JDWA_M,3773
+fusion_bench/method/isotropic_merging/iso_utils.py,sha256=7L8PYUIJROwHJQmhFY-tdEhkLAnzVKXr-ae55FQ1QSo,6928
 fusion_bench/method/linear/__init__.py,sha256=ChfkoOEAb-rUKwpowFPel-a1hRfS8gCrbnWD-jlRbe4,283
 fusion_bench/method/linear/expo.py,sha256=LCHTWlsPm1Mjhrq0mfpWLVC7skkI9ZksGduy3TxULoU,3939
 fusion_bench/method/linear/linear_interpolation.py,sha256=IONw9BPiRJouY8bE9Abfyz7qVI_1B1n8KGZa0f7Pza8,2157
@@ -151,7 +154,7 @@ fusion_bench/method/tall_mask/utils.py,sha256=Wlp8WcPwR_lCaBIZ9rgG6ewLfSzz3G7kPk
 fusion_bench/method/task_arithmetic/__init__.py,sha256=pSx_NV5Ra_6UXpyYWCi6ANQoAnEtymZt_X1dDN9wT4Y,96
 fusion_bench/method/task_arithmetic/task_arithmetic.py,sha256=1D0uuNtqyA1VS35jh6AnEVsX72HnT02THyerck_lmso,5441
 fusion_bench/method/task_singular_vector/TSVC.py,sha256=yn4SrZNvtA6PoGYJmbmtNeDyDbGnRCgfZ7ZCg914AZU,410
-fusion_bench/method/task_singular_vector/TSVM.py,sha256=ANBGC1GM8c9oy_xlY-ZEyoWO9mnZh5aiF_rrvVH73l8,1925
+fusion_bench/method/task_singular_vector/TSVM.py,sha256=2MqeJazsZNBTKghrtZDqXE2XoO_BShK60n3SEMjV74k,2787
 fusion_bench/method/task_singular_vector/__init__.py,sha256=WMucyl9pu_Ev2kcdrfT4moqMMbzD7hHQVFME5Su5jMA,298
 fusion_bench/method/task_singular_vector/utils/TSVC_utils.py,sha256=FytKbal48EW6iGIA-2zV7QSVbYTVflXr4Mr56q0W75k,2286
 fusion_bench/method/task_singular_vector/utils/TSVM_utils.py,sha256=dsTMQ15zFJ1MPqDOt2TJ01O9Bwq_klyG9xL9hRD2aI0,27521
@@ -484,6 +487,8 @@ fusion_bench_config/method/ensemble/weighted_ensemble.yaml,sha256=U_wQXtogtgiqOT
 fusion_bench_config/method/fisher_merging/clip_fisher_merging.yaml,sha256=rl7kfVvdo2pG-DnglQUbjzkyBqnq1FpfoSDSjFtdLwk,633
 fusion_bench_config/method/fisher_merging/fisher_merging.yaml,sha256=B1wrv9mhaOID4KcAUEMZNxlvY3tR3Q3UGualFslvx-Y,475
 fusion_bench_config/method/fisher_merging/gpt2_fisher_merging.yaml,sha256=AE7XZqRDj4__J_ipEcjPs7qTB2J3xLQyFRlq1W4iHFE,563
+fusion_bench_config/method/isotropic_merging/iso_c.yaml,sha256=Lh_OtTaUJ08--h85fUr2asF85xLe1NMCK8fVAhHOzdQ,82
+fusion_bench_config/method/isotropic_merging/iso_cts.yaml,sha256=x5vZo__kO8njl4_gFdXnOt15X_qFLv6-diSWHOR4clw,111
 fusion_bench_config/method/linear/expo.yaml,sha256=St3NW6cKVRV3vCn8y0gxQ8k66VTdtsLTEWQTbO9wQ0Y,420
 fusion_bench_config/method/linear/linear_interpolation.yaml,sha256=IQgltk5REITSx8xLuLP11ByPbuMgy7dHz_BrxIgwOas,67
 fusion_bench_config/method/linear/llama_expo.yaml,sha256=SEsC-l5gugY0vlsQkTJqzVgWJnMjFzWuTz814UKbFeM,624
@@ -515,7 +520,7 @@ fusion_bench_config/method/sparselo_pruning/llama_iterative_sparselo.yaml,sha256
 fusion_bench_config/method/sparselo_pruning/llama_pcp_sparselo.yaml,sha256=w1OWb38nW08K_hvrRMsCwmRxHWLGQfSSXg5nTiYaP8E,635
 fusion_bench_config/method/sparselo_pruning/llama_sparselo.yaml,sha256=J6vYIwqzh95-B3ekDias3FnCrVr4sig4zxpWyvz8hZ0,613
 fusion_bench_config/method/surgery/adamerging_surgery.yaml,sha256=Ne9JlJFgsRYcygBNCOBSN1ygBcLkE6I-8yusfTxyg-Y,826
-fusion_bench_config/method/task_singular_vector/TaskSingularVectorMerging.yaml,sha256=Se2v7AwwGqulXEVktRRzznpba4nNrWegY2bOwvjrHG8,74
+fusion_bench_config/method/task_singular_vector/TaskSingularVectorMerging.yaml,sha256=CLONjN9TXQ0OQwZHaje0q3WJWxR3LD1b5q5KrWJfZIA,169
 fusion_bench_config/method/trust_region/clip_task_arithmetic.yaml,sha256=mK09Ohsvj0Q6suj5qJM4DyCzRy192QBt4wjHS6W29IY,197
 fusion_bench_config/method/wemoe/sparse_weight_ensembling_moe.yaml,sha256=jiAco7M1XO0aekHFZKLKlXL_jRoCA8bgGD44Z7iB208,1001
 fusion_bench_config/method/wemoe/weight_ensembling_moe.yaml,sha256=OEv5yhyUCe5lXeT2PyXC49yrHXEM7i8SZDw6IQRDtAE,620
@@ -719,9 +724,9 @@ fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip-vit-single-task_sun397
 fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip-vit-single-task_svhn.yaml,sha256=2AqMiNCRRunLIrssHvFzu1lUzOaQn8uOHM9yjrQq-_A,109
 fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip_rankone_wemoe_clip-vit-classification_TA8.yaml,sha256=iQMj2VpDTe_D8OfCo94w5Ud2MON-EGa0DzVr6UmphrA,436
 fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip_sparse_wemoe_clip-vit-classification_TA8.yaml,sha256=i5Bn8bLl2cgqvrgtIGmoovUfSMehk_m-6C2wwcx5JMU,435
-fusion_bench-0.2.9.dist-info/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
-fusion_bench-0.2.9.dist-info/METADATA,sha256=CvYwT5a-5o-Xbj9-HEgnl3gGAhNvss7Z1YggC8rYZcU,16779
-fusion_bench-0.2.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-fusion_bench-0.2.9.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
-fusion_bench-0.2.9.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
-fusion_bench-0.2.9.dist-info/RECORD,,
+fusion_bench-0.2.10.dist-info/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
+fusion_bench-0.2.10.dist-info/METADATA,sha256=kBYozBf6hgA-7ebsn7znqJdhCz4H0dJSv2jVIEkBvyA,16780
+fusion_bench-0.2.10.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+fusion_bench-0.2.10.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
+fusion_bench-0.2.10.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
+fusion_bench-0.2.10.dist-info/RECORD,,

fusion_bench_config/method/isotropic_merging/iso_c.yaml

@@ -0,0 +1,4 @@
+_target_: fusion_bench.method.ISO_C_Merge
+
+scaling_factor: 1.0
+exclude_keys: null

fusion_bench_config/method/isotropic_merging/iso_cts.yaml

@@ -0,0 +1,5 @@
+_target_: fusion_bench.method.ISO_CTS_Merge
+
+scaling_factor: 1.0
+common_space_fraction: 0.8
+exclude_keys: null

fusion_bench_config/method/task_singular_vector/TaskSingularVectorMerging.yaml

@@ -1,2 +1,8 @@
 _target_: fusion_bench.method.TaskSingularVectorMerging
 remove_keys: null
+
+# alpha is a float or a list of floats
+# example:
+# alpha: 1
+# alpha: [1, 0.5, 0.25]
+alpha: 1